sgienc: Make sure to initialize skipped header portions
[libav.git] / libavcodec / sgienc.c
1 /*
2 * SGI image encoder
3 * Todd Kirby <doubleshot@pacbell.net>
4 *
5 * This file is part of Libav.
6 *
7 * Libav is free software; you can redistribute it and/or
8 * modify it under the terms of the GNU Lesser General Public
9 * License as published by the Free Software Foundation; either
10 * version 2.1 of the License, or (at your option) any later version.
11 *
12 * Libav is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
15 * Lesser General Public License for more details.
16 *
17 * You should have received a copy of the GNU Lesser General Public
18 * License along with Libav; if not, write to the Free Software
19 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
20 */
21
22 #include "libavutil/opt.h"
23
24 #include "avcodec.h"
25 #include "bytestream.h"
26 #include "internal.h"
27 #include "sgi.h"
28 #include "rle.h"
29
30 #define SGI_SINGLE_CHAN 2
31 #define SGI_MULTI_CHAN 3
32
33 typedef struct SgiContext {
34 AVClass *class;
35
36 int rle;
37 } SgiContext;
38
39 static av_cold int encode_init(AVCodecContext *avctx)
40 {
41 if (avctx->width > 65535 || avctx->height > 65535) {
42 av_log(avctx, AV_LOG_ERROR,
43 "Unsupported resolution %dx%d.\n", avctx->width, avctx->height);
44 return AVERROR_INVALIDDATA;
45 }
46
47 return 0;
48 }
49
50 static int sgi_rle_encode(PutByteContext *pbc, const uint8_t *src,
51 int w, int bpp)
52 {
53 int val, count, x, start = bytestream2_tell_p(pbc);
54 void (*bytestream2_put)(PutByteContext *, unsigned int);
55
56 if (bpp == 1)
57 bytestream2_put = bytestream2_put_byte;
58 else
59 bytestream2_put = bytestream2_put_be16;
60
61 for (x = 0; x < w; x += count) {
62 /* see if we can encode the next set of pixels with RLE */
63 count = ff_rle_count_pixels(src, w - x, bpp, 1);
64 if (count > 1) {
65 if (bytestream2_get_bytes_left_p(pbc) < bpp * 2)
66 return AVERROR_INVALIDDATA;
67
68 val = bpp == 1 ? *src : AV_RB16(src);
69 bytestream2_put(pbc, count);
70 bytestream2_put(pbc, val);
71 } else {
72 int i;
73 /* fall back on uncompressed */
74 count = ff_rle_count_pixels(src, w - x, bpp, 0);
75 if (bytestream2_get_bytes_left_p(pbc) < bpp * (count + 1))
76 return AVERROR_INVALIDDATA;
77
78 bytestream2_put(pbc, count + 0x80);
79 for (i = 0; i < count; i++) {
80 val = bpp == 1 ? src[i] : AV_RB16(src + i * bpp);
81 bytestream2_put(pbc, val);
82 }
83 }
84
85 src += count * bpp;
86 }
87
88 return bytestream2_tell_p(pbc) - start;
89 }
90
91 static int encode_frame(AVCodecContext *avctx, AVPacket *pkt,
92 const AVFrame *frame, int *got_packet)
93 {
94 SgiContext *s = avctx->priv_data;
95 const AVFrame * const p = frame;
96 PutByteContext pbc;
97 uint8_t *in_buf, *encode_buf;
98 int x, y, z, length, tablesize, ret, i;
99 unsigned int width, height, depth, dimension;
100 unsigned int bytes_per_channel, pixmax, put_be;
101
102 #if FF_API_CODED_FRAME
103 FF_DISABLE_DEPRECATION_WARNINGS
104 avctx->coded_frame->pict_type = AV_PICTURE_TYPE_I;
105 avctx->coded_frame->key_frame = 1;
106 FF_ENABLE_DEPRECATION_WARNINGS
107 #endif
108
109 #if FF_API_CODER_TYPE
110 FF_DISABLE_DEPRECATION_WARNINGS
111 if (avctx->coder_type == FF_CODER_TYPE_RAW)
112 s->rle = 0;
113 FF_ENABLE_DEPRECATION_WARNINGS
114 #endif
115
116 width = avctx->width;
117 height = avctx->height;
118 bytes_per_channel = 1;
119 pixmax = 0xFF;
120 put_be = HAVE_BIGENDIAN;
121
122 switch (avctx->pix_fmt) {
123 case AV_PIX_FMT_GRAY8:
124 dimension = SGI_SINGLE_CHAN;
125 depth = SGI_GRAYSCALE;
126 break;
127 case AV_PIX_FMT_RGB24:
128 dimension = SGI_MULTI_CHAN;
129 depth = SGI_RGB;
130 break;
131 case AV_PIX_FMT_RGBA:
132 dimension = SGI_MULTI_CHAN;
133 depth = SGI_RGBA;
134 break;
135 case AV_PIX_FMT_GRAY16LE:
136 put_be = !HAVE_BIGENDIAN;
137 case AV_PIX_FMT_GRAY16BE:
138 bytes_per_channel = 2;
139 pixmax = 0xFFFF;
140 dimension = SGI_SINGLE_CHAN;
141 depth = SGI_GRAYSCALE;
142 break;
143 case AV_PIX_FMT_RGB48LE:
144 put_be = !HAVE_BIGENDIAN;
145 case AV_PIX_FMT_RGB48BE:
146 bytes_per_channel = 2;
147 pixmax = 0xFFFF;
148 dimension = SGI_MULTI_CHAN;
149 depth = SGI_RGB;
150 break;
151 case AV_PIX_FMT_RGBA64LE:
152 put_be = !HAVE_BIGENDIAN;
153 case AV_PIX_FMT_RGBA64BE:
154 bytes_per_channel = 2;
155 pixmax = 0xFFFF;
156 dimension = SGI_MULTI_CHAN;
157 depth = SGI_RGBA;
158 break;
159 default:
160 return AVERROR_INVALIDDATA;
161 }
162
163 tablesize = depth * height * 4;
164 length = SGI_HEADER_SIZE;
165 if (!s->rle)
166 length += depth * height * width;
167 else // assume sgi_rle_encode() produces at most 2x size of input
168 length += tablesize * 2 + depth * height * (2 * width + 1);
169
170 if ((ret = ff_alloc_packet(pkt, bytes_per_channel * length)) < 0) {
171 av_log(avctx, AV_LOG_ERROR, "Error getting output packet of size %d.\n", length);
172 return ret;
173 }
174
175 bytestream2_init_writer(&pbc, pkt->data, pkt->size);
176
177 /* Encode header. */
178 bytestream2_put_be16(&pbc, SGI_MAGIC);
179 bytestream2_put_byte(&pbc, s->rle); /* RLE 1 - VERBATIM 0 */
180 bytestream2_put_byte(&pbc, bytes_per_channel);
181 bytestream2_put_be16(&pbc, dimension);
182 bytestream2_put_be16(&pbc, width);
183 bytestream2_put_be16(&pbc, height);
184 bytestream2_put_be16(&pbc, depth);
185
186 bytestream2_put_be32(&pbc, 0L); /* pixmin */
187 bytestream2_put_be32(&pbc, pixmax);
188 bytestream2_put_be32(&pbc, 0L); /* dummy */
189
190 /* name */
191 for (i = 0; i < 80; i++)
192 bytestream2_put_byte(&pbc, 0L);
193
194 /* colormap */
195 bytestream2_put_be32(&pbc, 0L);
196
197 /* The rest of the 512 byte header is unused. */
198 for (i = 0; i < 404; i++)
199 bytestream2_put_byte(&pbc, 0L);
200
201 if (s->rle) {
202 PutByteContext taboff_pcb, tablen_pcb;
203
204 /* Skip RLE offset table. */
205 bytestream2_init_writer(&taboff_pcb, pbc.buffer, tablesize);
206 bytestream2_skip_p(&pbc, tablesize);
207
208 /* Skip RLE length table. */
209 bytestream2_init_writer(&tablen_pcb, pbc.buffer, tablesize);
210 bytestream2_skip_p(&pbc, tablesize);
211
212 /* Make an intermediate consecutive buffer. */
213 if (!(encode_buf = av_malloc(width * bytes_per_channel)))
214 return AVERROR(ENOMEM);
215
216 for (z = 0; z < depth; z++) {
217 in_buf = p->data[0] + p->linesize[0] * (height - 1) + z * bytes_per_channel;
218
219 for (y = 0; y < height; y++) {
220 bytestream2_put_be32(&taboff_pcb, bytestream2_tell_p(&pbc));
221
222 for (x = 0; x < width * bytes_per_channel; x += bytes_per_channel)
223 encode_buf[x] = in_buf[depth * x];
224
225 length = sgi_rle_encode(&pbc, encode_buf, width,
226 bytes_per_channel);
227 if (length < 1) {
228 av_free(encode_buf);
229 return AVERROR_INVALIDDATA;
230 }
231
232 bytestream2_put_be32(&tablen_pcb, length);
233 in_buf -= p->linesize[0];
234 }
235 }
236
237 av_free(encode_buf);
238 } else {
239 for (z = 0; z < depth; z++) {
240 in_buf = p->data[0] + p->linesize[0] * (height - 1) + z * bytes_per_channel;
241
242 for (y = 0; y < height; y++) {
243 for (x = 0; x < width * depth; x += depth)
244 if (bytes_per_channel == 1)
245 bytestream2_put_byte(&pbc, in_buf[x]);
246 else
247 if (put_be)
248 bytestream2_put_be16(&pbc, ((uint16_t *)in_buf)[x]);
249 else
250 bytestream2_put_le16(&pbc, ((uint16_t *)in_buf)[x]);
251
252 in_buf -= p->linesize[0];
253 }
254 }
255 }
256
257 /* total length */
258 pkt->size = bytestream2_tell_p(&pbc);
259 pkt->flags |= AV_PKT_FLAG_KEY;
260 *got_packet = 1;
261
262 return 0;
263 }
264
265 #define OFFSET(x) offsetof(SgiContext, x)
266 #define VE AV_OPT_FLAG_VIDEO_PARAM | AV_OPT_FLAG_ENCODING_PARAM
267 static const AVOption options[] = {
268 { "rle", "Use run-length compression", OFFSET(rle), AV_OPT_TYPE_INT, { .i64 = 1 }, 0, 1, VE },
269
270 { NULL },
271 };
272
273 static const AVClass sgi_class = {
274 .class_name = "sgi",
275 .item_name = av_default_item_name,
276 .option = options,
277 .version = LIBAVUTIL_VERSION_INT,
278 };
279
280 AVCodec ff_sgi_encoder = {
281 .name = "sgi",
282 .long_name = NULL_IF_CONFIG_SMALL("SGI image"),
283 .type = AVMEDIA_TYPE_VIDEO,
284 .id = AV_CODEC_ID_SGI,
285 .priv_data_size = sizeof(SgiContext),
286 .priv_class = &sgi_class,
287 .init = encode_init,
288 .encode2 = encode_frame,
289 .pix_fmts = (const enum AVPixelFormat[]) {
290 AV_PIX_FMT_RGB24, AV_PIX_FMT_RGBA,
291 AV_PIX_FMT_RGB48LE, AV_PIX_FMT_RGB48BE,
292 AV_PIX_FMT_RGBA64LE, AV_PIX_FMT_RGBA64BE,
293 AV_PIX_FMT_GRAY16LE, AV_PIX_FMT_GRAY16BE, AV_PIX_FMT_GRAY8,
294 AV_PIX_FMT_NONE
295 },
296 };