pcxenc: Return more meaningful error codes
[libav.git] / libavcodec / pcxenc.c
1 /*
2 * PC Paintbrush PCX (.pcx) image encoder
3 * Copyright (c) 2009 Daniel Verkamp <daniel at drv.nu>
4 *
5 * This file is part of Libav.
6 *
7 * Libav is free software; you can redistribute it and/or
8 * modify it under the terms of the GNU Lesser General Public
9 * License as published by the Free Software Foundation; either
10 * version 2.1 of the License, or (at your option) any later version.
11 *
12 * Libav is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
15 * Lesser General Public License for more details.
16 *
17 * You should have received a copy of the GNU Lesser General Public
18 * License along with Libav; if not, write to the Free Software
19 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
20 */
21
22 /**
23 * @file
24 * PCX image encoder
25 * @author Daniel Verkamp
26 * @see http://www.qzx.com/pc-gpe/pcx.txt
27 */
28
29 #include "avcodec.h"
30 #include "bytestream.h"
31 #include "internal.h"
32
33 static const uint32_t monoblack_pal[16] = { 0x000000, 0xFFFFFF };
34
35 static av_cold int pcx_encode_init(AVCodecContext *avctx)
36 {
37 avctx->coded_frame = av_frame_alloc();
38 if (!avctx->coded_frame)
39 return AVERROR(ENOMEM);
40
41 avctx->coded_frame->pict_type = AV_PICTURE_TYPE_I;
42 avctx->coded_frame->key_frame = 1;
43
44 return 0;
45 }
46
47 static av_cold int pcx_encode_close(AVCodecContext *avctx)
48 {
49 av_frame_free(&avctx->coded_frame);
50 return 0;
51 }
52
53 /**
54 * PCX run-length encoder
55 * @param dst output buffer
56 * @param dst_size size of output buffer
57 * @param src input buffer
58 * @param src_plane_size size of one plane of input buffer in bytes
59 * @param nplanes number of planes in input buffer
60 * @return number of bytes written to dst or -1 on error
61 * @bug will not work for nplanes != 1 && bpp != 8
62 */
63 static int pcx_rle_encode( uint8_t *dst, int dst_size,
64 const uint8_t *src, int src_plane_size, int nplanes)
65 {
66 int p;
67 const uint8_t *dst_start = dst;
68
69 // check worst-case upper bound on dst_size
70 if (dst_size < 2LL * src_plane_size * nplanes || src_plane_size <= 0)
71 return AVERROR(EINVAL);
72
73 for (p = 0; p < nplanes; p++) {
74 int count = 1;
75 const uint8_t *src_plane = src + p;
76 const uint8_t *src_plane_end = src_plane + src_plane_size * nplanes;
77 uint8_t prev = *src_plane;
78 src_plane += nplanes;
79
80 for (; ; src_plane += nplanes) {
81 if (src_plane < src_plane_end && *src_plane == prev && count < 0x3F) {
82 // current byte is same as prev
83 ++count;
84 } else {
85 // output prev * count
86 if (count != 1 || prev >= 0xC0)
87 *dst++ = 0xC0 | count;
88 *dst++ = prev;
89
90 if (src_plane == src_plane_end)
91 break;
92
93 // start new run
94 count = 1;
95 prev = *src_plane;
96 }
97 }
98 }
99
100 return dst - dst_start;
101 }
102
103 static int pcx_encode_frame(AVCodecContext *avctx, AVPacket *pkt,
104 const AVFrame *frame, int *got_packet)
105 {
106 const uint8_t *buf_end;
107 uint8_t *buf;
108
109 int bpp, nplanes, i, y, line_bytes, written, ret, max_pkt_size;
110 const uint32_t *pal = NULL;
111 const uint8_t *src;
112
113 if (avctx->width > 65535 || avctx->height > 65535) {
114 av_log(avctx, AV_LOG_ERROR, "image dimensions do not fit in 16 bits\n");
115 return AVERROR(EINVAL);
116 }
117
118 switch (avctx->pix_fmt) {
119 case AV_PIX_FMT_RGB24:
120 bpp = 8;
121 nplanes = 3;
122 break;
123 case AV_PIX_FMT_RGB8:
124 case AV_PIX_FMT_BGR8:
125 case AV_PIX_FMT_RGB4_BYTE:
126 case AV_PIX_FMT_BGR4_BYTE:
127 case AV_PIX_FMT_GRAY8:
128 case AV_PIX_FMT_PAL8:
129 bpp = 8;
130 nplanes = 1;
131 pal = (uint32_t *)frame->data[1];
132 break;
133 case AV_PIX_FMT_MONOBLACK:
134 bpp = 1;
135 nplanes = 1;
136 pal = monoblack_pal;
137 break;
138 default:
139 av_log(avctx, AV_LOG_ERROR, "unsupported pixfmt\n");
140 return AVERROR(EINVAL);
141 }
142
143 line_bytes = (avctx->width * bpp + 7) >> 3;
144 line_bytes = (line_bytes + 1) & ~1;
145
146 max_pkt_size = 128 + avctx->height * 2 * line_bytes * nplanes + (pal ? 256*3 + 1 : 0);
147 if ((ret = ff_alloc_packet(pkt, max_pkt_size)) < 0) {
148 av_log(avctx, AV_LOG_ERROR, "Error getting output packet of size %d.\n", max_pkt_size);
149 return ret;
150 }
151 buf = pkt->data;
152 buf_end = pkt->data + pkt->size;
153
154 bytestream_put_byte(&buf, 10); // manufacturer
155 bytestream_put_byte(&buf, 5); // version
156 bytestream_put_byte(&buf, 1); // encoding
157 bytestream_put_byte(&buf, bpp); // bits per pixel per plane
158 bytestream_put_le16(&buf, 0); // x min
159 bytestream_put_le16(&buf, 0); // y min
160 bytestream_put_le16(&buf, avctx->width - 1); // x max
161 bytestream_put_le16(&buf, avctx->height - 1); // y max
162 bytestream_put_le16(&buf, 0); // horizontal DPI
163 bytestream_put_le16(&buf, 0); // vertical DPI
164 for (i = 0; i < 16; i++)
165 bytestream_put_be24(&buf, pal ? pal[i] : 0);// palette (<= 16 color only)
166 bytestream_put_byte(&buf, 0); // reserved
167 bytestream_put_byte(&buf, nplanes); // number of planes
168 bytestream_put_le16(&buf, line_bytes); // scanline plane size in bytes
169
170 while (buf - pkt->data < 128)
171 *buf++= 0;
172
173 src = frame->data[0];
174
175 for (y = 0; y < avctx->height; y++) {
176 if ((written = pcx_rle_encode(buf, buf_end - buf,
177 src, line_bytes, nplanes)) < 0) {
178 av_log(avctx, AV_LOG_ERROR, "buffer too small\n");
179 return AVERROR_BUG;
180 }
181 buf += written;
182 src += frame->linesize[0];
183 }
184
185 if (nplanes == 1 && bpp == 8) {
186 if (buf_end - buf < 257) {
187 av_log(avctx, AV_LOG_ERROR, "buffer too small\n");
188 return AVERROR_BUG;
189 }
190 bytestream_put_byte(&buf, 12);
191 for (i = 0; i < 256; i++) {
192 bytestream_put_be24(&buf, pal[i]);
193 }
194 }
195
196 pkt->size = buf - pkt->data;
197 pkt->flags |= AV_PKT_FLAG_KEY;
198 *got_packet = 1;
199
200 return 0;
201 }
202
203 AVCodec ff_pcx_encoder = {
204 .name = "pcx",
205 .long_name = NULL_IF_CONFIG_SMALL("PC Paintbrush PCX image"),
206 .type = AVMEDIA_TYPE_VIDEO,
207 .id = AV_CODEC_ID_PCX,
208 .init = pcx_encode_init,
209 .close = pcx_encode_close,
210 .encode2 = pcx_encode_frame,
211 .pix_fmts = (const enum AVPixelFormat[]){
212 AV_PIX_FMT_RGB24,
213 AV_PIX_FMT_RGB8, AV_PIX_FMT_BGR8, AV_PIX_FMT_RGB4_BYTE, AV_PIX_FMT_BGR4_BYTE,
214 AV_PIX_FMT_GRAY8, AV_PIX_FMT_PAL8,
215 AV_PIX_FMT_MONOBLACK,
216 AV_PIX_FMT_NONE
217 },
218 };