j2k/jpeg2000: merge float DWT and related code

the float conditions are largely changed to check the DWT
instead of the bitexact flag, which previously was wrong
(DWT53 is always int)

Signed-off-by: Michael Niedermayer <michaelni@gmx.at>

4 files changed, 83 insertions, 37 deletions

diff --git a/libavcodec/j2k.c b/libavcodec/j2k.c
index 5a7949737b..d5be7c3394 100644
--- a/libavcodec/j2k.c
+++ b/libavcodec/j2k.c
@@ -184,7 +184,7 @@ int ff_j2k_init_component(Jpeg2000Component *comp,
     uint8_t log2_band_prec_width, log2_band_prec_height;
     int reslevelno, bandno, gbandno = 0, ret, i, j, csize = 1;
 
-    if (ret=ff_jpeg2000_dwt_init(&comp->dwt, comp->coord, codsty->nreslevels2decode-1, codsty->transform == FF_DWT53 ? FF_DWT53 : FF_DWT97_INT))
+    if (ret=ff_jpeg2000_dwt_init(&comp->dwt, comp->coord, codsty->nreslevels2decode-1, codsty->transform))
         return ret;
     for (i = 0; i < 2; i++)
         csize *= comp->coord[i][1] - comp->coord[i][0];
@@ -261,7 +261,7 @@ int ff_j2k_init_component(Jpeg2000Component *comp,
             case JPEG2000_QSTY_SI:
                 /*TODO: Compute formula to implement. */
                 numbps = cbps +
-                         lut_gain[codsty->transform][bandno + (reslevelno > 0)];
+                         lut_gain[codsty->transform == FF_DWT53][bandno + (reslevelno > 0)];
                 band->f_stepsize = SHL(2048 + qntsty->mant[gbandno],
                                             2 + numbps - qntsty->expn[gbandno]);
                 break;
diff --git a/libavcodec/j2kdec.c b/libavcodec/j2kdec.c
index 149e58967e..39983cbf48 100644
--- a/libavcodec/j2kdec.c
+++ b/libavcodec/j2kdec.c
@@ -275,7 +275,11 @@ static int get_cox(Jpeg2000DecoderContext *s, Jpeg2000CodingStyle *c)
     if (c->cblk_style != 0) { // cblk style
         av_log(s->avctx, AV_LOG_WARNING, "extra cblk styles %X\n", c->cblk_style);
     }
-    c->transform = bytestream2_get_byteu(&s->g); // transformation
+    c->transform = bytestream2_get_byteu(&s->g); // DWT transformation type
+    /* set integer 9/7 DWT in case of BITEXACT flag */
+    if ((s->avctx->flags & CODEC_FLAG_BITEXACT) && (c->transform == FF_DWT97))
+        c->transform = FF_DWT97_INT;
+
     if (c->csty & JPEG2000_CSTY_PREC) {
         int i;
         for (i = 0; i < c->nreslevels; i++) {
@@ -815,6 +819,20 @@ static int decode_cblk(Jpeg2000DecoderContext *s, Jpeg2000CodingStyle *codsty,
     return 0;
 }
 
+/* Float dequantization of a codeblock.*/
+static void dequantization_float(int x, int y, Jpeg2000Cblk *cblk,
+                                 Jpeg2000Component *comp,
+                                 Jpeg2000T1Context *t1, Jpeg2000Band *band)
+{
+    int i, j, idx;
+    float *datap = &comp->data[(comp->coord[0][1] - comp->coord[0][0]) * y + x];
+    for (j = 0; j < (cblk->coord[1][1] - cblk->coord[1][0]); ++j)
+        for (i = 0; i < (cblk->coord[0][1] - cblk->coord[0][0]); ++i) {
+            idx        = (comp->coord[0][1] - comp->coord[0][0]) * j + i;
+            datap[idx] = (float)(t1->data[j][i]) * band->f_stepsize;
+        }
+}
+
 /* Integer dequantization of a codeblock.*/
 static void dequantization_int(int x, int y, Jpeg2000Cblk *cblk,
                                Jpeg2000Component *comp,
@@ -941,7 +959,10 @@ static int decode_tile(Jpeg2000DecoderContext *s, Jpeg2000Tile *tile)
                         x = cblk->coord[0][0];
                         y = cblk->coord[1][0];
 
-                        dequantization_int(x, y, cblk, comp, &t1, band);
+                        if (codsty->transform == FF_DWT97)
+                            dequantization_float(x, y, cblk, comp, &t1, band);
+                        else
+                            dequantization_int(x, y, cblk, comp, &t1, band);
                    } /* end cblk */
                 } /*end prec */
             } /* end band */
@@ -957,6 +978,10 @@ static int decode_tile(Jpeg2000DecoderContext *s, Jpeg2000Tile *tile)
 
     if (s->precision <= 8) {
         for (compno = 0; compno < s->ncomponents; compno++) {
+            Jpeg2000Component *comp = tile->comp + compno;
+            float *datap = (float*)comp->data;
+            int32_t *i_datap = (int32_t *) comp->data;
+
             y = tile->comp[compno].coord[1][0] - s->image_offset_y;
             line = s->picture->data[0] + y * s->picture->linesize[0];
             for (; y < tile->comp[compno].coord[1][1] - s->image_offset_y; y += s->cdy[compno]) {
@@ -966,10 +991,16 @@ static int decode_tile(Jpeg2000DecoderContext *s, Jpeg2000Tile *tile)
                 dst = line + x * s->ncomponents + compno;
 
                 for (; x < tile->comp[compno].coord[0][1] - s->image_offset_x; x += s->cdx[compno]) {
-                    int val = *src[compno]++ << (8 - s->cbps[compno]);
-                    val += 1 << 7;
-                    val = av_clip(val, 0, (1 << 8) - 1);
-                    *dst = val;
+                     int val;
+                    /* DC level shift and clip see ISO 15444-1:2002 G.1.2 */
+                    if (tile->codsty->transform == FF_DWT97)
+                        val = lrintf(*datap) + (1 << (s->cbps[compno] - 1));
+                    else
+                        val = *i_datap + (1 << (s->cbps[compno] - 1));
+                    val = av_clip(val, 0, (1 << s->cbps[compno]) - 1);
+                    *dst = val << (8 - s->cbps[compno]);
+                    datap++;
+                    i_datap++;
                     dst += s->ncomponents;
                 }
                 line += s->picture->linesize[0];
@@ -977,23 +1008,33 @@ static int decode_tile(Jpeg2000DecoderContext *s, Jpeg2000Tile *tile)
         }
     } else {
         for (compno = 0; compno < s->ncomponents; compno++) {
-            y = tile->comp[compno].coord[1][0] - s->image_offset_y;
-            line = s->picture->data[0] + y * s->picture->linesize[0];
+            Jpeg2000Component *comp = tile->comp + compno;
+            float *datap = (float*)comp->data;
+            int32_t *i_datap = (int32_t *) comp->data;
+            uint16_t *linel;
+
+            y     = tile->comp[compno].coord[1][0] - s->image_offset_y;
+            linel = (uint16_t*)s->picture->data[0] + y * (s->picture->linesize[0] >> 1);
             for (; y < tile->comp[compno].coord[1][1] - s->image_offset_y; y += s->cdy[compno]) {
                 uint16_t *dst;
 
                 x = tile->comp[compno].coord[0][0] - s->image_offset_x;
-                dst = (uint16_t *)(line + (x * s->ncomponents + compno) * 2);
+                dst = linel + (x * s->ncomponents + compno);
                 for (; x < tile->comp[compno].coord[0][1] - s->image_offset_x; x += s-> cdx[compno]) {
-                    int32_t val;
-
-                    val = *src[compno]++ << (16 - s->cbps[compno]);
-                    val += 1 << 15;
-                    val = av_clip(val, 0, (1 << 16) - 1);
-                    *dst = val;
+                     int val;
+                    /* DC level shift and clip see ISO 15444-1:2002 G.1.2 */
+                    if (tile->codsty->transform == FF_DWT97)
+                        val = lrintf(*datap) + (1 << (s->cbps[compno] - 1));
+                    else
+                        val = *i_datap + (1 << (s->cbps[compno] - 1));
+                    val = av_clip(val, 0, (1 << s->cbps[compno]) - 1);
+                    /* align 12 bit values in little-endian mode */
+                    *dst = val << (16 - s->cbps[compno]);
+                    datap++;
+                    i_datap++;
                     dst += s->ncomponents;
                 }
-                line += s->picture->linesize[0];
+                linel += s->picture->linesize[0]>>1;
             }
         }
     }
diff --git a/libavcodec/j2kenc.c b/libavcodec/j2kenc.c
index 4b7228c187..d31b84a121 100644
--- a/libavcodec/j2kenc.c
+++ b/libavcodec/j2kenc.c
@@ -280,7 +280,7 @@ static int put_cod(Jpeg2000EncoderContext *s)
     bytestream_put_byte(&s->buf, codsty->log2_cblk_width-2); // cblk width
     bytestream_put_byte(&s->buf, codsty->log2_cblk_height-2); // cblk height
     bytestream_put_byte(&s->buf, 0); // cblk style
-    bytestream_put_byte(&s->buf, codsty->transform); // transformation
+    bytestream_put_byte(&s->buf, codsty->transform == FF_DWT53); // transformation
     return 0;
 }
 
@@ -432,7 +432,7 @@ static void init_quantization(Jpeg2000EncoderContext *s)
             for (bandno = 0; bandno < nbands; bandno++, gbandno++){
                 int expn, mant;
 
-                if (codsty->transform == FF_DWT97){
+                if (codsty->transform == FF_DWT97_INT){
                     int bandpos = bandno + (reslevelno>0),
                         ss = 81920000 / dwt_norms[0][bandpos][lev],
                         log = av_log2(ss);
@@ -802,7 +802,7 @@ static void truncpasses(Jpeg2000EncoderContext *s, Jpeg2000Tile *tile)
                         Jpeg2000Cblk *cblk = prec->cblk + cblkno;
 
                         cblk->ninclpasses = getcut(cblk, s->lambda,
-                                (int64_t)dwt_norms[codsty->transform][bandpos][lev] * (int64_t)band->i_stepsize >> 16);
+                                (int64_t)dwt_norms[codsty->transform == FF_DWT53][bandpos][lev] * (int64_t)band->i_stepsize >> 16);
                     }
                 }
             }
@@ -986,7 +986,7 @@ static av_cold int j2kenc_init(AVCodecContext *avctx)
     codsty->nreslevels       = 7;
     codsty->log2_cblk_width  = 4;
     codsty->log2_cblk_height = 4;
-    codsty->transform        = avctx->prediction_method;
+    codsty->transform        = avctx->prediction_method ? FF_DWT53 : FF_DWT97_INT;
 
     qntsty->nguardbits       = 1;
 
diff --git a/libavcodec/jpeg2000dec.c b/libavcodec/jpeg2000dec.c
index 59af444068..071524659b 100644
--- a/libavcodec/jpeg2000dec.c
+++ b/libavcodec/jpeg2000dec.c
@@ -1056,10 +1056,10 @@ static int jpeg2000_decode_tile(Jpeg2000DecoderContext *s, Jpeg2000Tile *tile,
                         x = cblk->coord[0][0];
                         y = cblk->coord[1][0];
 
-                        if (s->avctx->flags & CODEC_FLAG_BITEXACT)
-                            dequantization_int(x, y, cblk, comp, &t1, band);
-                        else
+                        if (codsty->transform == FF_DWT97)
                             dequantization_float(x, y, cblk, comp, &t1, band);
+                        else
+                            dequantization_int(x, y, cblk, comp, &t1, band);
                    } /* end cblk */
                 } /*end prec */
             } /* end band */
@@ -1079,7 +1079,8 @@ static int jpeg2000_decode_tile(Jpeg2000DecoderContext *s, Jpeg2000Tile *tile,
     if (s->precision <= 8) {
         for (compno = 0; compno < s->ncomponents; compno++) {
             Jpeg2000Component *comp = tile->comp + compno;
-            int32_t *datap = (int32_t *)comp->data;
+            float *datap = comp->data;
+            int32_t *i_datap = (int32_t *) comp->data;
             y    = tile->comp[compno].coord[1][0] - s->image_offset_y;
             line = picture->data[0] + y * picture->linesize[0];
             for (; y < tile->comp[compno].coord[1][1] - s->image_offset_y; y += s->cdy[compno]) {
@@ -1089,12 +1090,16 @@ static int jpeg2000_decode_tile(Jpeg2000DecoderContext *s, Jpeg2000Tile *tile,
                 dst = line + x * s->ncomponents + compno;
 
                 for (; x < tile->comp[compno].coord[0][1] - s->image_offset_x; x += s->cdx[compno]) {
-                    *datap += 1 << (s->cbps[compno] - 1);
-                    if (*datap < 0)
-                        *datap = 0;
-                    else if (*datap >= (1 << s->cbps[compno]))
-                        *datap = (1 << s->cbps[compno]) - 1;
-                    *dst = *datap++;
+                     int val;
+                    /* DC level shift and clip see ISO 15444-1:2002 G.1.2 */
+                    if (tile->codsty->transform == FF_DWT97)
+                        val = lrintf(*datap) + (1 << (s->cbps[compno] - 1));
+                    else
+                        val = *i_datap + (1 << (s->cbps[compno] - 1));
+                    val = av_clip(val, 0, (1 << s->cbps[compno]) - 1);
+                    *dst = val << (8 - s->cbps[compno]);
+                    datap++;
+                    i_datap++;
                     dst += s->ncomponents;
                 }
                 line += picture->linesize[0];
@@ -1114,15 +1119,15 @@ static int jpeg2000_decode_tile(Jpeg2000DecoderContext *s, Jpeg2000Tile *tile,
                 x   = tile->comp[compno].coord[0][0] - s->image_offset_x;
                 dst = linel + (x * s->ncomponents + compno);
                 for (; x < s->avctx->width; x += s->cdx[compno]) {
-                    int16_t val;
+                    int val;
                     /* DC level shift and clip see ISO 15444-1:2002 G.1.2 */
-                    if (s->avctx->flags & CODEC_FLAG_BITEXACT)
-                        val = *i_datap + (1 << (s->cbps[compno] - 1));
-                    else
+                    if (tile->codsty->transform == FF_DWT97)
                         val = lrintf(*datap) + (1 << (s->cbps[compno] - 1));
+                    else
+                        val = *i_datap + (1 << (s->cbps[compno] - 1));
                     val = av_clip(val, 0, (1 << s->cbps[compno]) - 1);
                     /* align 12 bit values in little-endian mode */
-                    *dst = val << 4;
+                    *dst = val << (16 - s->cbps[compno]);
                     datap++;
                     i_datap++;
                     dst += s->ncomponents;