kill warnings patch by (Måns Rullgård <mru inprovide com>)
[libav.git] / vhook / ppm.c
CommitLineData
bee0d9e5
CY
1/*
2 * PPM Video Hook
3 * Copyright (c) 2003 Charles Yates
4 *
5 * This library is free software; you can redistribute it and/or
6 * modify it under the terms of the GNU Lesser General Public
7 * License as published by the Free Software Foundation; either
8 * version 2 of the License, or (at your option) any later version.
9 *
10 * This library is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 * Lesser General Public License for more details.
14 *
15 * You should have received a copy of the GNU Lesser General Public
16 * License along with this library; if not, write to the Free Software
17 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
18 */
19
20#include <stdio.h>
21#include <unistd.h>
22#include <fcntl.h>
23#include <sys/types.h>
24#include <sys/wait.h>
25#include <ctype.h>
26#include "framehook.h"
27
28/** Bi-directional pipe structure.
29*/
30
31typedef struct rwpipe
32{
1e4ddde2
CY
33 int pid;
34 FILE *reader;
35 FILE *writer;
bee0d9e5
CY
36}
37rwpipe;
38
39/** Create a bidirectional pipe for the given command.
40*/
41
42rwpipe *rwpipe_open( int argc, char *argv[] )
43{
1e4ddde2 44 rwpipe *this = av_mallocz( sizeof( rwpipe ) );
bee0d9e5 45
1e4ddde2
CY
46 if ( this != NULL )
47 {
bee0d9e5
CY
48 int input[ 2 ];
49 int output[ 2 ];
50
51 pipe( input );
52 pipe( output );
53
54 this->pid = fork();
55
56 if ( this->pid == 0 )
57 {
1f3f9507
MN
58#define COMMAND_SIZE 10240
59 char *command = av_mallocz( COMMAND_SIZE );
1e4ddde2 60 int i;
bee0d9e5 61
1e4ddde2
CY
62 strcpy( command, "" );
63 for ( i = 0; i < argc; i ++ )
64 {
1f3f9507
MN
65 pstrcat( command, COMMAND_SIZE, argv[ i ] );
66 pstrcat( command, COMMAND_SIZE, " " );
1e4ddde2 67 }
bee0d9e5
CY
68
69 dup2( output[ 0 ], STDIN_FILENO );
70 dup2( input[ 1 ], STDOUT_FILENO );
71
72 close( input[ 0 ] );
73 close( input[ 1 ] );
74 close( output[ 0 ] );
75 close( output[ 1 ] );
76
77 execl("/bin/sh", "sh", "-c", command, NULL );
1e4ddde2 78 exit( 255 );
bee0d9e5
CY
79 }
80 else
81 {
82 close( input[ 1 ] );
83 close( output[ 0 ] );
84
85 this->reader = fdopen( input[ 0 ], "r" );
86 this->writer = fdopen( output[ 1 ], "w" );
87 }
1e4ddde2 88 }
bee0d9e5 89
1e4ddde2 90 return this;
bee0d9e5
CY
91}
92
93/** Read data from the pipe.
94*/
95
96FILE *rwpipe_reader( rwpipe *this )
97{
1e4ddde2
CY
98 if ( this != NULL )
99 return this->reader;
100 else
101 return NULL;
bee0d9e5
CY
102}
103
104/** Write data to the pipe.
105*/
106
107FILE *rwpipe_writer( rwpipe *this )
108{
1e4ddde2
CY
109 if ( this != NULL )
110 return this->writer;
111 else
112 return NULL;
113}
114
115/* Read a number from the pipe - assumes PNM style headers.
116*/
117
118int rwpipe_read_number( rwpipe *rw )
119{
120 int value = 0;
121 int c = 0;
122 FILE *in = rwpipe_reader( rw );
123
124 do
125 {
126 c = fgetc( in );
127
128 while( c != EOF && !isdigit( c ) && c != '#' )
129 c = fgetc( in );
130
131 if ( c == '#' )
132 while( c != EOF && c != '\n' )
133 c = fgetc( in );
134 }
135 while ( c != EOF && !isdigit( c ) );
136
137 while( c != EOF && isdigit( c ) )
138 {
139 value = value * 10 + ( c - '0' );
140 c = fgetc( in );
141 }
142
143 return value;
144}
145
146/** Read a PPM P6 header.
147*/
148
149int rwpipe_read_ppm_header( rwpipe *rw, int *width, int *height )
150{
151 char line[ 3 ];
152 FILE *in = rwpipe_reader( rw );
153 int max;
154
155 fgets( line, 3, in );
156 if ( !strncmp( line, "P6", 2 ) )
157 {
158 *width = rwpipe_read_number( rw );
159 *height = rwpipe_read_number( rw );
160 max = rwpipe_read_number( rw );
161 return max != 255 || *width <= 0 || *height <= 0;
162 }
163 return 1;
bee0d9e5
CY
164}
165
166/** Close the pipe and process.
167*/
168
169void rwpipe_close( rwpipe *this )
170{
1e4ddde2
CY
171 if ( this != NULL )
172 {
173 fclose( this->reader );
174 fclose( this->writer );
175 waitpid( this->pid, NULL, 0 );
176 av_free( this );
177 }
bee0d9e5
CY
178}
179
1e4ddde2
CY
180/** Context info for this vhook - stores the pipe and image buffers.
181*/
182
bee0d9e5
CY
183typedef struct
184{
1e4ddde2
CY
185 rwpipe *rw;
186 int size1;
187 char *buf1;
188 int size2;
189 char *buf2;
bee0d9e5
CY
190}
191ContextInfo;
192
1e4ddde2
CY
193/** Initialise the context info for this vhook.
194*/
bee0d9e5 195
1e4ddde2 196int Configure(void **ctxp, int argc, char *argv[])
bee0d9e5 197{
1e4ddde2
CY
198 if ( argc > 1 )
199 {
200 *ctxp = av_mallocz(sizeof(ContextInfo));
201 if ( ctxp != NULL && argc > 1 )
202 {
203 ContextInfo *info = (ContextInfo *)*ctxp;
204 info->rw = rwpipe_open( argc - 1, &argv[ 1 ] );
205 return 0;
206 }
207 }
208 return 1;
bee0d9e5
CY
209}
210
1e4ddde2
CY
211/** Process a frame.
212*/
bee0d9e5
CY
213
214void Process(void *ctx, AVPicture *picture, enum PixelFormat pix_fmt, int width, int height, int64_t pts)
215{
1e4ddde2 216 int err = 0;
bee0d9e5 217 ContextInfo *ci = (ContextInfo *) ctx;
bee0d9e5
CY
218 AVPicture picture1;
219 AVPicture picture2;
220 AVPicture *pict = picture;
1e4ddde2
CY
221 int out_width;
222 int out_height;
223 int i;
224 uint8_t *ptr = NULL;
225 FILE *in = rwpipe_reader( ci->rw );
226 FILE *out = rwpipe_writer( ci->rw );
227
228 /* Check that we have a pipe to talk to. */
229 if ( in == NULL || out == NULL )
230 err = 1;
231
232 /* Convert to RGB24 if necessary */
233 if ( !err && pix_fmt != PIX_FMT_RGB24 )
234 {
235 int size = avpicture_get_size(PIX_FMT_RGB24, width, height);
236
237 if ( size != ci->size1 )
238 {
239 av_free( ci->buf1 );
240 ci->buf1 = av_malloc(size);
241 ci->size1 = size;
242 err = ci->buf1 == NULL;
243 }
244
245 if ( !err )
246 {
247 avpicture_fill(&picture1, ci->buf1, PIX_FMT_RGB24, width, height);
248 if (img_convert(&picture1, PIX_FMT_RGB24, picture, pix_fmt, width, height) < 0)
249 err = 1;
250 pict = &picture1;
251 }
252 }
253
254 /* Write out the PPM */
255 if ( !err )
256 {
257 ptr = pict->data[ 0 ];
258 fprintf( out, "P6\n%d %d\n255\n", width, height );
259 for ( i = 0; !err && i < height; i ++ )
260 {
261 err = !fwrite( ptr, width * 3, 1, out );
262 ptr += pict->linesize[ 0 ];
bee0d9e5 263 }
1e4ddde2
CY
264 if ( !err )
265 err = fflush( out );
bee0d9e5
CY
266 }
267
1e4ddde2
CY
268 /* Read the PPM returned. */
269 if ( !err && !rwpipe_read_ppm_header( ci->rw, &out_width, &out_height ) )
270 {
bee0d9e5 271 int size = avpicture_get_size(PIX_FMT_RGB24, out_width, out_height);
1e4ddde2
CY
272
273 if ( size != ci->size2 )
274 {
275 av_free( ci->buf2 );
276 ci->buf2 = av_malloc(size);
277 ci->size2 = size;
278 err = ci->buf2 == NULL;
279 }
280
281 if ( !err )
282 {
283 avpicture_fill(&picture2, ci->buf2, PIX_FMT_RGB24, out_width, out_height);
284 ptr = picture2.data[ 0 ];
285 for ( i = 0; !err && i < out_height; i ++ )
286 {
287 err = !fread( ptr, out_width * 3, 1, in );
288 ptr += picture2.linesize[ 0 ];
289 }
bee0d9e5 290 }
1e4ddde2 291 }
bee0d9e5 292
1e4ddde2
CY
293 /* Convert the returned PPM back to the input format */
294 if ( !err )
295 {
296 /* Actually, this is wrong, since the out_width/out_height returned from the
297 * filter won't necessarily be the same as width and height - img_resample
298 * won't scale rgb24, so the only way out of this is to convert to something
299 * that img_resample does like [which may or may not be pix_fmt], rescale
300 * and finally convert to pix_fmt... slow, but would provide the most flexibility.
301 *
302 * Currently, we take the upper left width/height pixels from the filtered image,
303 * smaller images are going to be corrupted or cause a crash.
304 *
305 * Finally, what should we do in case of this call failing? Up to now, failures
306 * are gracefully ignored and the original image is returned - in this case, a
307 * failure may corrupt the input.
308 */
309 if (img_convert(picture, pix_fmt, &picture2, PIX_FMT_RGB24, width, height) < 0)
310 {
311 }
312 }
bee0d9e5
CY
313}
314
1e4ddde2
CY
315/** Clean up the effect.
316*/
317
bee0d9e5
CY
318void Release(void *ctx)
319{
320 ContextInfo *ci;
321 ci = (ContextInfo *) ctx;
322
323 if (ctx)
1e4ddde2
CY
324 {
325 rwpipe_close( ci->rw );
326 av_free( ci->buf1 );
327 av_free( ci->buf2 );
bee0d9e5 328 av_free(ctx);
1e4ddde2 329 }
bee0d9e5
CY
330}
331