lavf: allow BSFs to drop packets.

If pkt->size == 0 && pkt->side_data_elems == 0 after bsf->filter()
returns, the packet is considered dropped.

8 files changed, 31 insertions, 15 deletions

diff --git a/doc/APIchanges b/doc/APIchanges
index 2fc6a71942..2e72128fbc 100644
--- a/doc/APIchanges
+++ b/doc/APIchanges
@@ -15,6 +15,10 @@ libavutil:     2015-08-28
 
 API changes, most recent first:
 
+2016-03-11 - xxxxxxx - lavf/lavc 57.28.101
+  Add requirement to bitstream filtering API that returned packets with
+  size == 0 and side_data_elems == 0 are to be skipped by the caller.
+
 2016-XX-XX - xxxxxxx - lavf 57.28.100
   Add protocol blacklisting API
 
diff --git a/ffmpeg.c b/ffmpeg.c
index d1485888e8..25c9140a52 100644
--- a/ffmpeg.c
+++ b/ffmpeg.c
@@ -694,6 +694,8 @@ static void write_frame(AVFormatContext *s, AVPacket *pkt, OutputStream *ost)
         if (exit_on_error)
             exit_program(1);
     }
+    if (pkt->size == 0 && pkt->side_data_elems == 0)
+        return;
 
     if (!(s->oformat->flags & AVFMT_NOTIMESTAMPS)) {
         if (pkt->dts != AV_NOPTS_VALUE &&
diff --git a/libavcodec/avcodec.h b/libavcodec/avcodec.h
index d676c57c05..637984bb26 100644
--- a/libavcodec/avcodec.h
+++ b/libavcodec/avcodec.h
@@ -5253,7 +5253,8 @@ AVBitStreamFilterContext *av_bitstream_filter_init(const char *name);
  * If the return value is 0, the output buffer is not allocated and
  * should be considered identical to the input buffer, or in case
  * *poutbuf was set it points to the input buffer (not necessarily to
- * its starting address).
+ * its starting address). A special case is if *poutbuf was set to NULL and
+ * *poutbuf_size was set to 0, which indicates the packet should be dropped.
  */
 int av_bitstream_filter_filter(AVBitStreamFilterContext *bsfc,
                                AVCodecContext *avctx, const char *args,
diff --git a/libavcodec/version.h b/libavcodec/version.h
index bddf503009..8153047011 100644
--- a/libavcodec/version.h
+++ b/libavcodec/version.h
@@ -29,7 +29,7 @@
 
 #define LIBAVCODEC_VERSION_MAJOR  57
 #define LIBAVCODEC_VERSION_MINOR  28
-#define LIBAVCODEC_VERSION_MICRO 100
+#define LIBAVCODEC_VERSION_MICRO 101
 
 #define LIBAVCODEC_VERSION_INT  AV_VERSION_INT(LIBAVCODEC_VERSION_MAJOR, \
                                                LIBAVCODEC_VERSION_MINOR, \
diff --git a/libavformat/avformat.h b/libavformat/avformat.h
index a558f2d392..ccb8033f9a 100644
--- a/libavformat/avformat.h
+++ b/libavformat/avformat.h
@@ -2850,7 +2850,9 @@ int avformat_queue_attached_pictures(AVFormatContext *s);
  * Apply a list of bitstream filters to a packet.
  *
  * @param codec AVCodecContext, usually from an AVStream
- * @param pkt the packet to apply filters to
+ * @param pkt the packet to apply filters to. If, on success, the returned
+ *        packet has size == 0 and side_data_elems == 0, it indicates that
+ *        the packet should be dropped
  * @param bsfc a NULL-terminated list of filters to apply
  * @return  >=0 on success;
  *          AVERROR code on failure
diff --git a/libavformat/mux.c b/libavformat/mux.c
index eb0b9739c2..9ca5df4095 100644
--- a/libavformat/mux.c
+++ b/libavformat/mux.c
@@ -1025,6 +1025,19 @@ int av_interleaved_write_frame(AVFormatContext *s, AVPacket *pkt)
     if (pkt) {
         AVStream *st = s->streams[pkt->stream_index];
 
+        if (s->oformat->check_bitstream) {
+            if (!st->internal->bitstream_checked) {
+                if ((ret = s->oformat->check_bitstream(s, pkt)) < 0)
+                    goto fail;
+                else if (ret == 1)
+                    st->internal->bitstream_checked = 1;
+            }
+        }
+
+        av_apply_bitstream_filters(st->codec, pkt, st->internal->bsfc);
+        if (pkt->size == 0 && pkt->side_data_elems == 0)
+            return 0;
+
         if (s->debug & FF_FDEBUG_TS)
             av_log(s, AV_LOG_TRACE, "av_interleaved_write_frame size:%d dts:%s pts:%s\n",
                 pkt->size, av_ts2str(pkt->dts), av_ts2str(pkt->pts));
@@ -1038,17 +1051,6 @@ int av_interleaved_write_frame(AVFormatContext *s, AVPacket *pkt)
             ret = AVERROR(EINVAL);
             goto fail;
         }
-
-        if (s->oformat->check_bitstream) {
-            if (!st->internal->bitstream_checked) {
-                if ((ret = s->oformat->check_bitstream(s, pkt)) < 0)
-                    goto fail;
-                else if (ret == 1)
-                    st->internal->bitstream_checked = 1;
-            }
-        }
-
-        av_apply_bitstream_filters(st->codec, pkt, st->internal->bsfc);
     } else {
         av_log(s, AV_LOG_TRACE, "av_interleaved_write_frame FLUSH\n");
         flush = 1;
diff --git a/libavformat/utils.c b/libavformat/utils.c
index f4ae8b4f58..e0aea877ff 100644
--- a/libavformat/utils.c
+++ b/libavformat/utils.c
@@ -4710,6 +4710,11 @@ int av_apply_bitstream_filters(AVCodecContext *codec, AVPacket *pkt,
                                            &new_pkt.data, &new_pkt.size,
                                            pkt->data, pkt->size,
                                            pkt->flags & AV_PKT_FLAG_KEY);
+        if (a == 0 && new_pkt.size == 0 && new_pkt.side_data_elems == 0) {
+            av_packet_unref(pkt);
+            memset(pkt, 0, sizeof(*pkt));
+            return 0;
+        }
         if(a == 0 && new_pkt.data != pkt->data) {
             uint8_t *t = av_malloc(new_pkt.size + AV_INPUT_BUFFER_PADDING_SIZE); //the new should be a subset of the old so cannot overflow
             if (t) {
diff --git a/libavformat/version.h b/libavformat/version.h
index 7dcce2cabf..8088178b87 100644
--- a/libavformat/version.h
+++ b/libavformat/version.h
@@ -31,7 +31,7 @@
 
 #define LIBAVFORMAT_VERSION_MAJOR  57
 #define LIBAVFORMAT_VERSION_MINOR  28
-#define LIBAVFORMAT_VERSION_MICRO 100
+#define LIBAVFORMAT_VERSION_MICRO 101
 
 #define LIBAVFORMAT_VERSION_INT AV_VERSION_INT(LIBAVFORMAT_VERSION_MAJOR, \
                                                LIBAVFORMAT_VERSION_MINOR, \