aboutsummaryrefslogtreecommitdiffstats
path: root/libavfilter/vf_spp.c
diff options
context:
space:
mode:
authorMichael Niedermayer <michaelni@gmx.at>2014-12-20 20:39:13 +0100
committerMichael Niedermayer <michaelni@gmx.at>2014-12-20 20:39:13 +0100
commit1ac5abb1d062b6ca983d494068bb9fd30390a941 (patch)
tree898337122aee4ed2a3e9d432f3ffb02405b7ce01 /libavfilter/vf_spp.c
parent5318cf521f66cd58642c07e8036f6232d4c60ed2 (diff)
downloadffmpeg-1ac5abb1d062b6ca983d494068bb9fd30390a941.tar.gz
avfilter/vf_spp: fix overflows with depth > 8
Signed-off-by: Michael Niedermayer <michaelni@gmx.at>
Diffstat (limited to 'libavfilter/vf_spp.c')
-rw-r--r--libavfilter/vf_spp.c20
1 files changed, 11 insertions, 9 deletions
diff --git a/libavfilter/vf_spp.c b/libavfilter/vf_spp.c
index 67dc80000a..4c7158ed62 100644
--- a/libavfilter/vf_spp.c
+++ b/libavfilter/vf_spp.c
@@ -186,16 +186,17 @@ static void store_slice_c(uint8_t *dst, const uint16_t *src,
}
}
-static void store_slice16_c(uint16_t *dst, const uint16_t *src,
+static void store_slice16_c(uint16_t *dst, const int16_t *src,
int dst_linesize, int src_linesize,
int width, int height, int log2_scale,
- const uint8_t dither[8][8])
+ const uint8_t dither[8][8], int depth)
{
int y, x;
+ unsigned int mask = -1<<depth;
#define STORE16(pos) do { \
temp = ((src[x + y*src_linesize + pos] << log2_scale) + (d[pos]>>1)) >> 5; \
- if (temp & 0x400) \
+ if (temp & mask ) \
temp = ~(temp >> 31); \
dst[x + y*dst_linesize + pos] = temp; \
} while (0)
@@ -242,7 +243,7 @@ static inline int norm_qscale(int qscale, int type)
static void filter(SPPContext *p, uint8_t *dst, uint8_t *src,
int dst_linesize, int src_linesize, int width, int height,
- const uint8_t *qp_table, int qp_stride, int is_luma, int sample_bytes)
+ const uint8_t *qp_table, int qp_stride, int is_luma, int depth)
{
int x, y, i;
const int count = 1 << p->log2_count;
@@ -251,6 +252,7 @@ static void filter(SPPContext *p, uint8_t *dst, uint8_t *src,
int16_t *block = (int16_t *)block_align;
int16_t *block2 = (int16_t *)(block_align + 16);
uint16_t *psrc16 = (uint16_t*)p->src;
+ const int sample_bytes = (depth+7) / 8;
for (y = 0; y < height; y++) {
int index = 8 + 8*linesize + y*linesize;
@@ -305,7 +307,7 @@ static void filter(SPPContext *p, uint8_t *dst, uint8_t *src,
store_slice16_c((uint16_t*)(dst + (y - 8) * dst_linesize), p->temp + 8 + y*linesize,
dst_linesize/2, linesize, width,
FFMIN(8, height + 8 - y), MAX_LEVEL - p->log2_count,
- ldither);
+ ldither, depth);
}
}
}
@@ -366,7 +368,7 @@ static int filter_frame(AVFilterLink *inlink, AVFrame *in)
int qp_stride = 0;
const int8_t *qp_table = NULL;
const AVPixFmtDescriptor *desc = av_pix_fmt_desc_get(inlink->format);
- const int sample_bytes = desc->comp[0].depth_minus1 < 8 ? 1 : 2;
+ const int depth = desc->comp[0].depth_minus1 + 1;
/* if we are not in a constant user quantizer mode and we don't want to use
* the quantizers from the B-frames (B-frames often have a higher QP), we
@@ -426,11 +428,11 @@ static int filter_frame(AVFilterLink *inlink, AVFrame *in)
out->height = in->height;
}
- filter(spp, out->data[0], in->data[0], out->linesize[0], in->linesize[0], inlink->w, inlink->h, qp_table, qp_stride, 1, sample_bytes);
+ filter(spp, out->data[0], in->data[0], out->linesize[0], in->linesize[0], inlink->w, inlink->h, qp_table, qp_stride, 1, depth);
if (out->data[2]) {
- filter(spp, out->data[1], in->data[1], out->linesize[1], in->linesize[1], cw, ch, qp_table, qp_stride, 0, sample_bytes);
- filter(spp, out->data[2], in->data[2], out->linesize[2], in->linesize[2], cw, ch, qp_table, qp_stride, 0, sample_bytes);
+ filter(spp, out->data[1], in->data[1], out->linesize[1], in->linesize[1], cw, ch, qp_table, qp_stride, 0, depth);
+ filter(spp, out->data[2], in->data[2], out->linesize[2], in->linesize[2], cw, ch, qp_table, qp_stride, 0, depth);
}
emms_c();
}