ppm vhook follow up patch by (Charles Yates <charles dot yates at pandora dot be>)

mostly tab -> space cosmetics

Originally committed as revision 2313 to svn://svn.ffmpeg.org/ffmpeg/trunk

2 files changed, 203 insertions, 144 deletions

diff --git a/vhook/Makefile b/vhook/Makefile
index 1e4a38d0b6..cfe1ab660d 100644
--- a/vhook/Makefile
+++ b/vhook/Makefile
@@ -7,7 +7,7 @@ CFLAGS=-fPIC $(OPTFLAGS) -Wall -I.. -I$(SRC_PATH) -I$(SRC_PATH)/libavformat -I$(
 ifeq ($(CONFIG_DARWIN),yes)
 SHFLAGS+=-bundle -flat_namespace -undefined suppress
 endif
-HOOKS=null.so fish.so
+HOOKS=null.so fish.so ppm.so
 
 ifeq ($(HAVE_IMLIB2),yes)
     HOOKS += imlib2.so
diff --git a/vhook/ppm.c b/vhook/ppm.c
index 8a5e1887b3..022f6a106b 100644
--- a/vhook/ppm.c
+++ b/vhook/ppm.c
@@ -30,9 +30,9 @@
 
 typedef struct rwpipe
 {
-	int pid;
-	FILE *reader;
-	FILE *writer;
+    int pid;
+    FILE *reader;
+    FILE *writer;
 }
 rwpipe;
 
@@ -41,10 +41,10 @@ rwpipe;
 
 rwpipe *rwpipe_open( int argc, char *argv[] )
 {
-	rwpipe *this = av_mallocz( sizeof( rwpipe ) );
+    rwpipe *this = av_mallocz( sizeof( rwpipe ) );
 
-	if ( this != NULL )
-	{
+    if ( this != NULL )
+    {
         int input[ 2 ];
         int output[ 2 ];
 
@@ -55,15 +55,15 @@ rwpipe *rwpipe_open( int argc, char *argv[] )
 
         if ( this->pid == 0 )
         {
-			char *command = av_mallocz( 10240 );
-			int i;
+            char *command = av_mallocz( 10240 );
+            int i;
 
-			strcpy( command, "" );
-			for ( i = 0; i < argc; i ++ )
-			{
-				strcat( command, argv[ i ] );
-				strcat( command, " " );
-			}
+            strcpy( command, "" );
+            for ( i = 0; i < argc; i ++ )
+            {
+                strcat( command, argv[ i ] );
+                strcat( command, " " );
+            }
 
             dup2( output[ 0 ], STDIN_FILENO );
             dup2( input[ 1 ], STDOUT_FILENO );
@@ -74,7 +74,7 @@ rwpipe *rwpipe_open( int argc, char *argv[] )
             close( output[ 1 ] );
 
             execl("/bin/sh", "sh", "-c", command, NULL );
-			exit( 255 );
+            exit( 255 );
         }
         else
         {
@@ -84,9 +84,9 @@ rwpipe *rwpipe_open( int argc, char *argv[] )
             this->reader = fdopen( input[ 0 ], "r" );
             this->writer = fdopen( output[ 1 ], "w" );
         }
-	}
+    }
 
-	return this;
+    return this;
 }
 
 /** Read data from the pipe.
@@ -94,10 +94,10 @@ rwpipe *rwpipe_open( int argc, char *argv[] )
 
 FILE *rwpipe_reader( rwpipe *this )
 {
-	if ( this != NULL )
-		return this->reader;
-	else
-		return NULL;
+    if ( this != NULL )
+        return this->reader;
+    else
+        return NULL;
 }
 
 /** Write data to the pipe.
@@ -105,10 +105,61 @@ FILE *rwpipe_reader( rwpipe *this )
 
 FILE *rwpipe_writer( rwpipe *this )
 {
-	if ( this != NULL )
-		return this->writer;
-	else
-		return NULL;
+    if ( this != NULL )
+        return this->writer;
+    else
+        return NULL;
+}
+
+/* Read a number from the pipe - assumes PNM style headers.
+*/
+
+int rwpipe_read_number( rwpipe *rw )
+{
+    int value = 0;
+    int c = 0;
+    FILE *in = rwpipe_reader( rw );
+
+    do 
+    {
+        c = fgetc( in );
+
+        while( c != EOF && !isdigit( c ) && c != '#' )
+            c = fgetc( in );
+
+        if ( c == '#' )
+            while( c != EOF && c != '\n' )
+                c = fgetc( in );
+    }
+    while ( c != EOF && !isdigit( c ) );
+
+    while( c != EOF && isdigit( c ) )
+    {
+        value = value * 10 + ( c - '0' );
+        c = fgetc( in );
+    }
+
+    return value;
+}
+
+/** Read a PPM P6 header.
+*/
+
+int rwpipe_read_ppm_header( rwpipe *rw, int *width, int *height )
+{
+    char line[ 3 ];
+    FILE *in = rwpipe_reader( rw );
+    int max;
+
+    fgets( line, 3, in );
+    if ( !strncmp( line, "P6", 2 ) )
+    {
+        *width = rwpipe_read_number( rw );
+        *height = rwpipe_read_number( rw );
+        max = rwpipe_read_number( rw );
+        return max != 255 || *width <= 0 || *height <= 0;
+    }
+    return 1;
 }
 
 /** Close the pipe and process.
@@ -116,156 +167,164 @@ FILE *rwpipe_writer( rwpipe *this )
 
 void rwpipe_close( rwpipe *this )
 {
-	if ( this != NULL )
-	{
-		fclose( this->reader );
-		fclose( this->writer );
-		waitpid( this->pid, NULL, 0 );
-		av_free( this );
-	}
+    if ( this != NULL )
+    {
+        fclose( this->reader );
+        fclose( this->writer );
+        waitpid( this->pid, NULL, 0 );
+        av_free( this );
+    }
 }
 
+/** Context info for this vhook - stores the pipe and image buffers.
+*/
+
 typedef struct 
 {
-	rwpipe *rw;
+    rwpipe *rw;
+    int size1;
+    char *buf1;
+    int size2;
+    char *buf2;
 } 
 ContextInfo;
 
-int Configure(void **ctxp, int argc, char *argv[])
-{
-    *ctxp = av_mallocz(sizeof(ContextInfo));
-	if ( ctxp != NULL && argc > 1 )
-	{
-		ContextInfo *info = (ContextInfo *)*ctxp;
-		info->rw = rwpipe_open( argc - 1, &argv[ 1 ] );
-	}
-    return 0;
-}
+/** Initialise the context info for this vhook.
+*/
 
-int rwpipe_read_number( rwpipe *rw )
+int Configure(void **ctxp, int argc, char *argv[])
 {
-	int value = 0;
-	int c = 0;
-	FILE *in = rwpipe_reader( rw );
-
-	do 
-	{
-		c = fgetc( in );
-
-		while( c != EOF && !isdigit( c ) && c != '#' )
-			c = fgetc( in );
-
-		if ( c == '#' )
-			while( c != EOF && c != '\n' )
-				c = fgetc( in );
-	}
-	while ( c != EOF && !isdigit( c ) );
-
-	while( c != EOF && isdigit( c ) )
-	{
-		value = value * 10 + ( c - '0' );
-		c = fgetc( in );
-	}
-
-	return value;
+    if ( argc > 1 )
+    {
+        *ctxp = av_mallocz(sizeof(ContextInfo));
+        if ( ctxp != NULL && argc > 1 )
+        {
+            ContextInfo *info = (ContextInfo *)*ctxp;
+            info->rw = rwpipe_open( argc - 1, &argv[ 1 ] );
+            return 0;
+        }
+    }
+    return 1;
 }
 
-int rwpipe_read_ppm_header( rwpipe *rw, int *width, int *height )
-{
-	char line[ 3 ];
-	FILE *in = rwpipe_reader( rw );
-	int max;
-
-	fgets( line, 3, in );
-	if ( !strncmp( line, "P6", 2 ) )
-	{
-		*width = rwpipe_read_number( rw );
-		*height = rwpipe_read_number( rw );
-		max = rwpipe_read_number( rw );
-		return max != 255 || *width <= 0 || *height <= 0;
-	}
-	return 1;
-}
+/** Process a frame.
+*/
 
 void Process(void *ctx, AVPicture *picture, enum PixelFormat pix_fmt, int width, int height, int64_t pts)
 {
-	int err = 0;
+    int err = 0;
     ContextInfo *ci = (ContextInfo *) ctx;
-    char *buf1 = 0;
-    char *buf2 = 0;
     AVPicture picture1;
     AVPicture picture2;
     AVPicture *pict = picture;
-	int out_width;
-	int out_height;
-	int i;
-	uint8_t *ptr = NULL;
-	FILE *in = rwpipe_reader( ci->rw );
-	FILE *out = rwpipe_writer( ci->rw );
-
-	/* Convert to RGB24 if necessary */
-    if (pix_fmt != PIX_FMT_RGB24) {
-        int size;
-
-        size = avpicture_get_size(PIX_FMT_RGB24, width, height);
-        buf1 = av_malloc(size);
-
-        avpicture_fill(&picture1, buf1, PIX_FMT_RGB24, width, height);
-        if (img_convert(&picture1, PIX_FMT_RGB24, 
-                        picture, pix_fmt, width, height) < 0) {
-			err = 1;
+    int out_width;
+    int out_height;
+    int i;
+    uint8_t *ptr = NULL;
+    FILE *in = rwpipe_reader( ci->rw );
+    FILE *out = rwpipe_writer( ci->rw );
+
+    /* Check that we have a pipe to talk to. */
+    if ( in == NULL || out == NULL )
+        err = 1;
+
+    /* Convert to RGB24 if necessary */
+    if ( !err && pix_fmt != PIX_FMT_RGB24 ) 
+    {
+        int size = avpicture_get_size(PIX_FMT_RGB24, width, height);
+
+        if ( size != ci->size1 )
+        {
+            av_free( ci->buf1 );
+            ci->buf1 = av_malloc(size);
+            ci->size1 = size;
+            err = ci->buf1 == NULL;
+        }
+
+        if ( !err )
+        {
+            avpicture_fill(&picture1, ci->buf1, PIX_FMT_RGB24, width, height);
+            if (img_convert(&picture1, PIX_FMT_RGB24, picture, pix_fmt, width, height) < 0)
+                err = 1;
+            pict = &picture1;
+        }
+    }
+
+    /* Write out the PPM */
+    if ( !err )
+    {
+        ptr = pict->data[ 0 ];
+        fprintf( out, "P6\n%d %d\n255\n", width, height );
+        for ( i = 0; !err && i < height; i ++ )
+        {
+            err = !fwrite( ptr, width * 3, 1, out );
+            ptr += pict->linesize[ 0 ];
         }
-        pict = &picture1;
+        if ( !err )
+            err = fflush( out );
     }
 
-	/* Write out the PPM */
-	if ( !err )
-	{
-		ptr = pict->data[ 0 ];
-		fprintf( out, "P6\n%d %d\n255\n", width, height );
-		for ( i = 0; !err && i < height; i ++ )
-		{
-			err = !fwrite( ptr, width * 3, 1, out );
-			ptr += pict->linesize[ 0 ];
-		}
-		if ( !err )
-			err = fflush( out );
-	}
-
-	/* Read the PPM returned. */
-	if ( !err && !rwpipe_read_ppm_header( ci->rw, &out_width, &out_height ) )
-	{
+    /* Read the PPM returned. */
+    if ( !err && !rwpipe_read_ppm_header( ci->rw, &out_width, &out_height ) )
+    {
         int size = avpicture_get_size(PIX_FMT_RGB24, out_width, out_height);
-        buf2 = av_malloc(size);
-        avpicture_fill(&picture2, buf2, PIX_FMT_RGB24, out_width, out_height);
-		ptr = picture2.data[ 0 ];
-		for ( i = 0; !err && i < out_height; i ++ )
-		{
-			err = !fread( ptr, out_width * 3, 1, in );
-			ptr += picture2.linesize[ 0 ];
-		}
-	}
-
-	/* Convert the returned PPM back to the input format */
-	if ( !err )
-	{
-        if (img_convert(picture, pix_fmt, &picture2, PIX_FMT_RGB24, width, height) < 0) {
+
+        if ( size != ci->size2 )
+        {
+            av_free( ci->buf2 );
+            ci->buf2 = av_malloc(size);
+            ci->size2 = size;
+            err = ci->buf2 == NULL;
+        }
+
+        if ( !err )
+        {
+            avpicture_fill(&picture2, ci->buf2, PIX_FMT_RGB24, out_width, out_height);
+            ptr = picture2.data[ 0 ];
+            for ( i = 0; !err && i < out_height; i ++ )
+            {
+                err = !fread( ptr, out_width * 3, 1, in );
+                ptr += picture2.linesize[ 0 ];
+            }
         }
-	}
+    }
 
-    av_free(buf1);
-    av_free(buf2);
+    /* Convert the returned PPM back to the input format */
+    if ( !err )
+    {
+        /* Actually, this is wrong, since the out_width/out_height returned from the
+         * filter won't necessarily be the same as width and height - img_resample 
+         * won't scale rgb24, so the only way out of this is to convert to something 
+         * that img_resample does like [which may or may not be pix_fmt], rescale 
+         * and finally convert to pix_fmt... slow, but would provide the most flexibility.
+         *
+         * Currently, we take the upper left width/height pixels from the filtered image,
+         * smaller images are going to be corrupted or cause a crash.
+         *
+         * Finally, what should we do in case of this call failing? Up to now, failures
+         * are gracefully ignored and the original image is returned - in this case, a
+         * failure may corrupt the input.
+         */
+        if (img_convert(picture, pix_fmt, &picture2, PIX_FMT_RGB24, width, height) < 0) 
+        {
+        }
+    }
 }
 
+/** Clean up the effect.
+*/
+
 void Release(void *ctx)
 {
     ContextInfo *ci;
     ci = (ContextInfo *) ctx;
 
     if (ctx)
-	{
-		rwpipe_close( ci->rw );
+    {
+        rwpipe_close( ci->rw );
+        av_free( ci->buf1 );
+        av_free( ci->buf2 );
         av_free(ctx);
-	}
+    }
 }