Bug#699722: src:libav: x264 decoding crashes

[anton at khirnov.net]

Hi,
a Libav developer here.

This is really a bug/abuse of Libav API in xpra -- the file x264lib.c uses an
uninitialized AVFrame on stack. This has always been forbidden by Libav API, but
used to work until some changes a few months ago. Functions
avcodec_alloc_frame() and avcodec_free_frame() (in release 9 only, av_free() in
0.8 and earlier)  must be used to allocate and free an AVFrame.

Attached is a patch that fixes it.

-- 
Anton Khirnov
-------------- next part --------------
--- xpra/x264/x264lib.c	2013-01-01 08:26:41.000000000 +0100
+++ xpra/x264/x264lib1.c	2013-02-05 07:48:07.672379166 +0100
@@ -55,6 +55,7 @@
 	// Decoding
 	AVCodec *codec;
 	AVCodecContext *codec_ctx;
+	AVFrame *frame;
 	struct SwsContext *yuv2rgb;
 
 	// Encoding
@@ -251,6 +252,11 @@
 		fprintf(stderr, "could not open codec\n");
 		return 1;
 	}
+	ctx->frame = avcodec_alloc_frame();
+	if (!ctx->frame) {
+	    fprintf(stderr, "could not allocate an AVFrame for decoding\n");
+	    return 1;
+	}
 	return 0;
 }
 struct x264lib_ctx *init_decoder(int width, int height, int csc_fmt)
@@ -275,6 +281,7 @@
 		sws_freeContext(ctx->yuv2rgb);
 		ctx->yuv2rgb = NULL;
 	}
+	avcodec_free_frame(&ctx->frame);
 }
 void clean_decoder(struct x264lib_ctx *ctx)
 {
@@ -390,7 +397,7 @@
 	int got_picture;
 	int len;
 	int i;
-	AVFrame picture;
+	AVFrame *picture = ctx->frame;
 	AVPacket avpkt;
 
 	av_init_packet(&avpkt);
@@ -398,12 +405,12 @@
 	if (!ctx->codec_ctx || !ctx->codec)
 		return 1;
 
-	avcodec_get_frame_defaults(&picture);
+	avcodec_get_frame_defaults(picture);
 
 	avpkt.data = in;
 	avpkt.size = size;
 
-	len = avcodec_decode_video2(ctx->codec_ctx, &picture, &got_picture, &avpkt);
+	len = avcodec_decode_video2(ctx->codec_ctx, picture, &got_picture, &avpkt);
 	if (len < 0) {
 		fprintf(stderr, "Error while decoding frame\n");
 		memset(out, 0, sizeof(*out));
@@ -411,13 +418,13 @@
 	}
 
 	for (i = 0; i < 3; i++) {
-		(*out)[i] = picture.data[i];
-		*outsize += ctx->height * picture.linesize[i];
-		(*outstride)[i] = picture.linesize[i];
+		(*out)[i] = picture->data[i];
+		*outsize += ctx->height * picture->linesize[i];
+		(*outstride)[i] = picture->linesize[i];
 	}
 
     if (*outsize == 0) {
-        fprintf(stderr, "Decoded image, size %d %d %d, ptr %p %p %p\n", (*outstride)[0] * ctx->height, (*outstride)[1]*ctx->height, (*outstride)[2]*ctx->height, picture.data[0], picture.data[1], picture.data[2]);
+        fprintf(stderr, "Decoded image, size %d %d %d, ptr %p %p %p\n", (*outstride)[0] * ctx->height, (*outstride)[1]*ctx->height, (*outstride)[2]*ctx->height, picture->data[0], picture->data[1], picture->data[2]);
         return 3;
     }