j2kdec/jpeg2000dec: partially merge quantization code

The quantization code needs more work, not so much work
merging but more work investigating what is correct.

Signed-off-by: Michael Niedermayer <michaelni@gmx.at>

3 files changed, 25 insertions, 25 deletions

diff --git a/libavcodec/j2k.c b/libavcodec/j2k.c
index 2a86dcedd4..7da05a8701 100644
--- a/libavcodec/j2k.c
+++ b/libavcodec/j2k.c
@@ -278,15 +278,16 @@ int ff_j2k_init_component(Jpeg2000Component *comp,
                 gain      = cbps;
                 stepsize  = pow(2.0, gain - qntsty->expn[gbandno]);
                 stepsize *= (qntsty->mant[gbandno] / 2048.0 + 1.0);
-                /* FIXME: In openjepg code stespize = stepsize * 0.5. Why?
-                 * If not set output of entropic decoder is not correct. */
-//                 stepsize *= 0.5;
                 break;
             default:
                 stepsize = 0;
                 av_log(avctx, AV_LOG_ERROR, "Unknown quantization format\n");
                 break;
             }
+            /* FIXME: In openjepg code stespize = stepsize * 0.5. Why?
+             * If not set output of entropic decoder is not correct. */
+            if (!av_codec_is_encoder(avctx->codec))
+                stepsize *= 0.5;
             /* BITEXACT computing case --> convert to int */
 //             if (avctx->flags & CODEC_FLAG_BITEXACT)
             band->stepsize = stepsize * (1 << 16);
diff --git a/libavcodec/j2kdec.c b/libavcodec/j2kdec.c
index d2e003ce8f..42862b927a 100644
--- a/libavcodec/j2kdec.c
+++ b/libavcodec/j2kdec.c
@@ -26,6 +26,7 @@
  * @author Kamil Nowosad
  */
 
+#include "libavutil/avassert.h"
 #include "libavutil/common.h"
 #include "libavutil/opt.h"
 #include "avcodec.h"
@@ -814,6 +815,22 @@ static int decode_cblk(Jpeg2000DecoderContext *s, Jpeg2000CodingStyle *codsty,
     return 0;
 }
 
+/* Integer dequantization of a codeblock.*/
+static void dequantization_int(int x, int y, Jpeg2000Cblk *cblk,
+                               Jpeg2000Component *comp,
+                               Jpeg2000T1Context *t1, Jpeg2000Band *band)
+{
+    int i, j, idx;
+    int32_t *datap =
+        (int32_t *) &comp->data[(comp->coord[0][1] - comp->coord[0][0]) * y + x];
+    for (j = 0; j < (cblk->coord[1][1] - cblk->coord[1][0]); ++j)
+        for (i = 0; i < (cblk->coord[0][1] - cblk->coord[0][0]); ++i) {
+            idx        = (comp->coord[0][1] - comp->coord[0][0]) * j + i;
+            datap[idx] =
+                ((int32_t)(t1->data[j][i]) * ((int32_t)band->stepsize) + (1 << 15)) >> 16;
+        }
+}
+
 static void mct_decode(Jpeg2000DecoderContext *s, Jpeg2000Tile *tile)
 {
     int i, *src[3], i0, i1, i2, csize = 1;
@@ -890,25 +907,7 @@ static int decode_tile(Jpeg2000DecoderContext *s, Jpeg2000Tile *tile)
                         x = cblk->coord[0][0];
                         y = cblk->coord[1][0];
 
-                        if (codsty->transform == FF_DWT53) {
-                            for (j = 0; j < (cblk->coord[1][1] - cblk->coord[1][0]); ++j) {
-                                int *datap = &comp->data[(comp->coord[0][1] - comp->coord[0][0]) * (y+j) + x];
-                                int *ptr = t1.data[j];
-                                for (i = 0; i < (cblk->coord[0][1] - cblk->coord[0][0]); ++i) {
-                                    datap[i] = ptr[i] >> 1;
-                                }
-                            }
-                        } else{
-                            for (j = 0; j < (cblk->coord[1][1] - cblk->coord[1][0]); ++j) {
-                                int *datap = &comp->data[(comp->coord[0][1] - comp->coord[0][0]) * (y+j) + x];
-                                int *ptr = t1.data[j];
-                                for (i = 0; i < (cblk->coord[0][1] - cblk->coord[0][0]); ++i) {
-                                    int tmp = ((int64_t)ptr[i]) * ((int64_t)band->stepsize) >> 16, tmp2;
-                                    tmp2 = FFABS(tmp>>1) + (tmp&1);
-                                    datap[i] = tmp < 0 ? -tmp2 : tmp2;
-                                }
-                            }
-                        }
+                        dequantization_int(x, y, cblk, comp, &t1, band);
                    } /* end cblk */
                 } /*end prec */
             } /* end band */
diff --git a/libavcodec/jpeg2000.c b/libavcodec/jpeg2000.c
index 5b15efcc29..c5fc35dbd1 100644
--- a/libavcodec/jpeg2000.c
+++ b/libavcodec/jpeg2000.c
@@ -288,15 +288,15 @@ int ff_jpeg2000_init_component(Jpeg2000Component *comp,
                 gain            = cbps;
                 band->stepsize  = pow(2.0, gain - qntsty->expn[gbandno]);
                 band->stepsize *= (float)qntsty->mant[gbandno] / 2048.0 + 1.0;
-                /* FIXME: In openjepg code stespize = stepsize * 0.5. Why?
-                 * If not set output of entropic decoder is not correct. */
-                band->stepsize *= 0.5;
                 break;
             default:
                 band->stepsize = 0;
                 av_log(avctx, AV_LOG_ERROR, "Unknown quantization format\n");
                 break;
             }
+            /* FIXME: In openjepg code stespize = stepsize * 0.5. Why?
+                * If not set output of entropic decoder is not correct. */
+            band->stepsize *= 0.5;
             /* BITEXACT computing case --> convert to int */
             if (avctx->flags & CODEC_FLAG_BITEXACT)
                 band->stepsize = (int32_t)(band->stepsize * (1 << 16));