[libav.git] / libavcodec / i386 / mpegvideo_mmx_template.c

/*
 * MPEG video MMX templates
 *
 * Copyright (c) 2002 Michael Niedermayer <michaelni@gmx.at>
 *
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public
 * License as published by the Free Software Foundation; either
 * version 2 of the License, or (at your option) any later version.
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public
 * License along with this library; if not, write to the Free Software
 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
 */
#undef SPREADW
#undef PMAXW
#ifdef HAVE_MMX2
#define SPREADW(a) "pshufw $0, " #a ", " #a " \n\t"
#define PMAXW(a,b) "pmaxsw " #a ", " #b " \n\t"
#define PMAX(a,b) \
            "pshufw $0x0E," #a ", " #b "		\n\t"\
	    PMAXW(b, a)\
            "pshufw $0x01," #a ", " #b "		\n\t"\
	    PMAXW(b, a)
#else
#define SPREADW(a) \
	"punpcklwd " #a ", " #a " \n\t"\
	"punpcklwd " #a ", " #a " \n\t"
#define PMAXW(a,b) \
	"psubusw " #a ", " #b " \n\t"\
	"paddw " #a ", " #b " \n\t"
#define PMAX(a,b)  \
            "movq " #a ", " #b "		\n\t"\
            "psrlq $32, " #a "			\n\t"\
	    PMAXW(b, a)\
            "movq " #a ", " #b "		\n\t"\
            "psrlq $16, " #a "			\n\t"\
	    PMAXW(b, a)

#endif

static int RENAME(dct_quantize)(MpegEncContext *s,
                            DCTELEM *block, int n,
                            int qscale, int *overflow)
{
    long last_non_zero_p1;
    int level=0, q; //=0 is cuz gcc says uninitalized ...
    const uint16_t *qmat, *bias;
    __align8 int16_t temp_block[64];

    assert((7&(int)(&temp_block[0])) == 0); //did gcc align it correctly?

    //s->fdct (block);
    RENAMEl(ff_fdct) (block); //cant be anything else ...

    if(s->dct_error_sum)
        s->denoise_dct(s, block);

    if (s->mb_intra) {
        int dummy;
        if (n < 4)
            q = s->y_dc_scale;
        else
            q = s->c_dc_scale;
        /* note: block[0] is assumed to be positive */
        if (!s->h263_aic) {
#if 1
        asm volatile (
        	"mul %%ecx		\n\t"
        	: "=d" (level), "=a"(dummy)
        	: "a" ((block[0]>>2) + q), "c" (inverse[q<<1])
        );
#else
        asm volatile (
        	"xorl %%edx, %%edx	\n\t"
        	"divw %%cx		\n\t"
        	"movzwl %%ax, %%eax	\n\t"
        	: "=a" (level)
        	: "a" ((block[0]>>2) + q), "c" (q<<1)
        	: "%edx"
        );
#endif
        } else
            /* For AIC we skip quant/dequant of INTRADC */
            level = (block[0] + 4)>>3;

        block[0]=0; //avoid fake overflow
//        temp_block[0] = (block[0] + (q >> 1)) / q;
        last_non_zero_p1 = 1;
        bias = s->q_intra_matrix16[qscale][1];
        qmat = s->q_intra_matrix16[qscale][0];
    } else {
        last_non_zero_p1 = 0;
        bias = s->q_inter_matrix16[qscale][1];
        qmat = s->q_inter_matrix16[qscale][0];
    }

    if((s->out_format == FMT_H263 || s->out_format == FMT_H261) && s->mpeg_quant==0){

        asm volatile(
            "movd %%"REG_a", %%mm3		\n\t" // last_non_zero_p1
            SPREADW(%%mm3)
            "pxor %%mm7, %%mm7			\n\t" // 0
            "pxor %%mm4, %%mm4			\n\t" // 0
            "movq (%2), %%mm5			\n\t" // qmat[0]
            "pxor %%mm6, %%mm6			\n\t"
            "psubw (%3), %%mm6			\n\t" // -bias[0]
            "mov $-128, %%"REG_a"		\n\t"
            ".balign 16				\n\t"
            "1:					\n\t"
            "pxor %%mm1, %%mm1			\n\t" // 0
            "movq (%1, %%"REG_a"), %%mm0	\n\t" // block[i]
            "pcmpgtw %%mm0, %%mm1		\n\t" // block[i] <= 0 ? 0xFF : 0x00
            "pxor %%mm1, %%mm0			\n\t"
            "psubw %%mm1, %%mm0			\n\t" // ABS(block[i])
            "psubusw %%mm6, %%mm0		\n\t" // ABS(block[i]) + bias[0]
            "pmulhw %%mm5, %%mm0		\n\t" // (ABS(block[i])*qmat[0] - bias[0]*qmat[0])>>16
            "por %%mm0, %%mm4			\n\t"
            "pxor %%mm1, %%mm0			\n\t"
            "psubw %%mm1, %%mm0			\n\t" // out=((ABS(block[i])*qmat[0] - bias[0]*qmat[0])>>16)*sign(block[i])
            "movq %%mm0, (%5, %%"REG_a")	\n\t"
            "pcmpeqw %%mm7, %%mm0		\n\t" // out==0 ? 0xFF : 0x00
            "movq (%4, %%"REG_a"), %%mm1	\n\t"
            "movq %%mm7, (%1, %%"REG_a")	\n\t" // 0
            "pandn %%mm1, %%mm0			\n\t"
	    PMAXW(%%mm0, %%mm3)
            "add $8, %%"REG_a"			\n\t"
            " js 1b				\n\t"
	    PMAX(%%mm3, %%mm0)
            "movd %%mm3, %%"REG_a"		\n\t"
            "movzb %%al, %%"REG_a"		\n\t" // last_non_zero_p1
	    : "+a" (last_non_zero_p1)
            : "r" (block+64), "r" (qmat), "r" (bias),
              "r" (inv_zigzag_direct16+64), "r" (temp_block+64)
        );
        // note the asm is split cuz gcc doesnt like that many operands ...
        asm volatile(
            "movd %1, %%mm1			\n\t" // max_qcoeff
	    SPREADW(%%mm1)
            "psubusw %%mm1, %%mm4		\n\t"
            "packuswb %%mm4, %%mm4		\n\t"
            "movd %%mm4, %0			\n\t" // *overflow
        : "=g" (*overflow)
        : "g" (s->max_qcoeff)
        );
    }else{ // FMT_H263
        asm volatile(
            "movd %%"REG_a", %%mm3		\n\t" // last_non_zero_p1
            SPREADW(%%mm3)
            "pxor %%mm7, %%mm7			\n\t" // 0
            "pxor %%mm4, %%mm4			\n\t" // 0
            "mov $-128, %%"REG_a"		\n\t"
            ".balign 16				\n\t"
            "1:					\n\t"
            "pxor %%mm1, %%mm1			\n\t" // 0
            "movq (%1, %%"REG_a"), %%mm0	\n\t" // block[i]
            "pcmpgtw %%mm0, %%mm1		\n\t" // block[i] <= 0 ? 0xFF : 0x00
            "pxor %%mm1, %%mm0			\n\t"
            "psubw %%mm1, %%mm0			\n\t" // ABS(block[i])
            "movq (%3, %%"REG_a"), %%mm6	\n\t" // bias[0]
            "paddusw %%mm6, %%mm0		\n\t" // ABS(block[i]) + bias[0]
            "movq (%2, %%"REG_a"), %%mm5		\n\t" // qmat[i]
            "pmulhw %%mm5, %%mm0		\n\t" // (ABS(block[i])*qmat[0] + bias[0]*qmat[0])>>16
            "por %%mm0, %%mm4			\n\t"
            "pxor %%mm1, %%mm0			\n\t"
            "psubw %%mm1, %%mm0			\n\t" // out=((ABS(block[i])*qmat[0] - bias[0]*qmat[0])>>16)*sign(block[i])
            "movq %%mm0, (%5, %%"REG_a")	\n\t"
            "pcmpeqw %%mm7, %%mm0		\n\t" // out==0 ? 0xFF : 0x00
            "movq (%4, %%"REG_a"), %%mm1		\n\t"
            "movq %%mm7, (%1, %%"REG_a")		\n\t" // 0
            "pandn %%mm1, %%mm0			\n\t"
	    PMAXW(%%mm0, %%mm3)
            "add $8, %%"REG_a"			\n\t"
            " js 1b				\n\t"
	    PMAX(%%mm3, %%mm0)
            "movd %%mm3, %%"REG_a"		\n\t"
            "movzb %%al, %%"REG_a"		\n\t" // last_non_zero_p1
	    : "+a" (last_non_zero_p1)
            : "r" (block+64), "r" (qmat+64), "r" (bias+64),
              "r" (inv_zigzag_direct16+64), "r" (temp_block+64)
        );
        // note the asm is split cuz gcc doesnt like that many operands ...
        asm volatile(
            "movd %1, %%mm1			\n\t" // max_qcoeff
	    SPREADW(%%mm1)
            "psubusw %%mm1, %%mm4		\n\t"
            "packuswb %%mm4, %%mm4		\n\t"
            "movd %%mm4, %0			\n\t" // *overflow
        : "=g" (*overflow)
        : "g" (s->max_qcoeff)
        );
    }

    if(s->mb_intra) block[0]= level;
    else            block[0]= temp_block[0];

    if(s->dsp.idct_permutation_type == FF_SIMPLE_IDCT_PERM){
        if(last_non_zero_p1 <= 1) goto end;
        block[0x08] = temp_block[0x01]; block[0x10] = temp_block[0x08];
        block[0x20] = temp_block[0x10];
        if(last_non_zero_p1 <= 4) goto end;
        block[0x18] = temp_block[0x09]; block[0x04] = temp_block[0x02];
        block[0x09] = temp_block[0x03];
        if(last_non_zero_p1 <= 7) goto end;
        block[0x14] = temp_block[0x0A]; block[0x28] = temp_block[0x11];
        block[0x12] = temp_block[0x18]; block[0x02] = temp_block[0x20];
        if(last_non_zero_p1 <= 11) goto end;
        block[0x1A] = temp_block[0x19]; block[0x24] = temp_block[0x12];
        block[0x19] = temp_block[0x0B]; block[0x01] = temp_block[0x04];
        block[0x0C] = temp_block[0x05];
        if(last_non_zero_p1 <= 16) goto end;
        block[0x11] = temp_block[0x0C]; block[0x29] = temp_block[0x13];
        block[0x16] = temp_block[0x1A]; block[0x0A] = temp_block[0x21];
        block[0x30] = temp_block[0x28]; block[0x22] = temp_block[0x30];
        block[0x38] = temp_block[0x29]; block[0x06] = temp_block[0x22];
        if(last_non_zero_p1 <= 24) goto end;
        block[0x1B] = temp_block[0x1B]; block[0x21] = temp_block[0x14];
        block[0x1C] = temp_block[0x0D]; block[0x05] = temp_block[0x06];
        block[0x0D] = temp_block[0x07]; block[0x15] = temp_block[0x0E];
        block[0x2C] = temp_block[0x15]; block[0x13] = temp_block[0x1C];
        if(last_non_zero_p1 <= 32) goto end;
        block[0x0B] = temp_block[0x23]; block[0x34] = temp_block[0x2A];
        block[0x2A] = temp_block[0x31]; block[0x32] = temp_block[0x38];
        block[0x3A] = temp_block[0x39]; block[0x26] = temp_block[0x32];
        block[0x39] = temp_block[0x2B]; block[0x03] = temp_block[0x24];
        if(last_non_zero_p1 <= 40) goto end;
        block[0x1E] = temp_block[0x1D]; block[0x25] = temp_block[0x16];
        block[0x1D] = temp_block[0x0F]; block[0x2D] = temp_block[0x17];
        block[0x17] = temp_block[0x1E]; block[0x0E] = temp_block[0x25];
        block[0x31] = temp_block[0x2C]; block[0x2B] = temp_block[0x33];
        if(last_non_zero_p1 <= 48) goto end;
        block[0x36] = temp_block[0x3A]; block[0x3B] = temp_block[0x3B];
        block[0x23] = temp_block[0x34]; block[0x3C] = temp_block[0x2D];
        block[0x07] = temp_block[0x26]; block[0x1F] = temp_block[0x1F];
        block[0x0F] = temp_block[0x27]; block[0x35] = temp_block[0x2E];
        if(last_non_zero_p1 <= 56) goto end;
        block[0x2E] = temp_block[0x35]; block[0x33] = temp_block[0x3C];
        block[0x3E] = temp_block[0x3D]; block[0x27] = temp_block[0x36];
        block[0x3D] = temp_block[0x2F]; block[0x2F] = temp_block[0x37];
        block[0x37] = temp_block[0x3E]; block[0x3F] = temp_block[0x3F];
    }else if(s->dsp.idct_permutation_type == FF_LIBMPEG2_IDCT_PERM){
        if(last_non_zero_p1 <= 1) goto end;
        block[0x04] = temp_block[0x01];
        block[0x08] = temp_block[0x08]; block[0x10] = temp_block[0x10];
        if(last_non_zero_p1 <= 4) goto end;
        block[0x0C] = temp_block[0x09]; block[0x01] = temp_block[0x02];
        block[0x05] = temp_block[0x03];
        if(last_non_zero_p1 <= 7) goto end;
        block[0x09] = temp_block[0x0A]; block[0x14] = temp_block[0x11];
        block[0x18] = temp_block[0x18]; block[0x20] = temp_block[0x20];
        if(last_non_zero_p1 <= 11) goto end;
        block[0x1C] = temp_block[0x19];
        block[0x11] = temp_block[0x12]; block[0x0D] = temp_block[0x0B];
        block[0x02] = temp_block[0x04]; block[0x06] = temp_block[0x05];
        if(last_non_zero_p1 <= 16) goto end;
        block[0x0A] = temp_block[0x0C]; block[0x15] = temp_block[0x13];
        block[0x19] = temp_block[0x1A]; block[0x24] = temp_block[0x21];
        block[0x28] = temp_block[0x28]; block[0x30] = temp_block[0x30];
        block[0x2C] = temp_block[0x29]; block[0x21] = temp_block[0x22];
        if(last_non_zero_p1 <= 24) goto end;
        block[0x1D] = temp_block[0x1B]; block[0x12] = temp_block[0x14];
        block[0x0E] = temp_block[0x0D]; block[0x03] = temp_block[0x06];
        block[0x07] = temp_block[0x07]; block[0x0B] = temp_block[0x0E];
        block[0x16] = temp_block[0x15]; block[0x1A] = temp_block[0x1C];
        if(last_non_zero_p1 <= 32) goto end;
        block[0x25] = temp_block[0x23]; block[0x29] = temp_block[0x2A];
        block[0x34] = temp_block[0x31]; block[0x38] = temp_block[0x38];
        block[0x3C] = temp_block[0x39]; block[0x31] = temp_block[0x32];
        block[0x2D] = temp_block[0x2B]; block[0x22] = temp_block[0x24];
        if(last_non_zero_p1 <= 40) goto end;
        block[0x1E] = temp_block[0x1D]; block[0x13] = temp_block[0x16];
        block[0x0F] = temp_block[0x0F]; block[0x17] = temp_block[0x17];
        block[0x1B] = temp_block[0x1E]; block[0x26] = temp_block[0x25];
        block[0x2A] = temp_block[0x2C]; block[0x35] = temp_block[0x33];
        if(last_non_zero_p1 <= 48) goto end;
        block[0x39] = temp_block[0x3A]; block[0x3D] = temp_block[0x3B];
        block[0x32] = temp_block[0x34]; block[0x2E] = temp_block[0x2D];
            block[0x23] = temp_block[0x26]; block[0x1F] = temp_block[0x1F];
        block[0x27] = temp_block[0x27]; block[0x2B] = temp_block[0x2E];
        if(last_non_zero_p1 <= 56) goto end;
        block[0x36] = temp_block[0x35]; block[0x3A] = temp_block[0x3C];
        block[0x3E] = temp_block[0x3D]; block[0x33] = temp_block[0x36];
        block[0x2F] = temp_block[0x2F]; block[0x37] = temp_block[0x37];
        block[0x3B] = temp_block[0x3E]; block[0x3F] = temp_block[0x3F];
    }else{
        if(last_non_zero_p1 <= 1) goto end;
        block[0x01] = temp_block[0x01];
        block[0x08] = temp_block[0x08]; block[0x10] = temp_block[0x10];
        if(last_non_zero_p1 <= 4) goto end;
        block[0x09] = temp_block[0x09]; block[0x02] = temp_block[0x02];
        block[0x03] = temp_block[0x03];
        if(last_non_zero_p1 <= 7) goto end;
        block[0x0A] = temp_block[0x0A]; block[0x11] = temp_block[0x11];
        block[0x18] = temp_block[0x18]; block[0x20] = temp_block[0x20];
        if(last_non_zero_p1 <= 11) goto end;
        block[0x19] = temp_block[0x19];
        block[0x12] = temp_block[0x12]; block[0x0B] = temp_block[0x0B];
        block[0x04] = temp_block[0x04]; block[0x05] = temp_block[0x05];
        if(last_non_zero_p1 <= 16) goto end;
        block[0x0C] = temp_block[0x0C]; block[0x13] = temp_block[0x13];
        block[0x1A] = temp_block[0x1A]; block[0x21] = temp_block[0x21];
        block[0x28] = temp_block[0x28]; block[0x30] = temp_block[0x30];
        block[0x29] = temp_block[0x29]; block[0x22] = temp_block[0x22];
        if(last_non_zero_p1 <= 24) goto end;
        block[0x1B] = temp_block[0x1B]; block[0x14] = temp_block[0x14];
        block[0x0D] = temp_block[0x0D]; block[0x06] = temp_block[0x06];
        block[0x07] = temp_block[0x07]; block[0x0E] = temp_block[0x0E];
        block[0x15] = temp_block[0x15]; block[0x1C] = temp_block[0x1C];
        if(last_non_zero_p1 <= 32) goto end;
        block[0x23] = temp_block[0x23]; block[0x2A] = temp_block[0x2A];
        block[0x31] = temp_block[0x31]; block[0x38] = temp_block[0x38];
        block[0x39] = temp_block[0x39]; block[0x32] = temp_block[0x32];
        block[0x2B] = temp_block[0x2B]; block[0x24] = temp_block[0x24];
        if(last_non_zero_p1 <= 40) goto end;
        block[0x1D] = temp_block[0x1D]; block[0x16] = temp_block[0x16];
        block[0x0F] = temp_block[0x0F]; block[0x17] = temp_block[0x17];
        block[0x1E] = temp_block[0x1E]; block[0x25] = temp_block[0x25];
        block[0x2C] = temp_block[0x2C]; block[0x33] = temp_block[0x33];
        if(last_non_zero_p1 <= 48) goto end;
        block[0x3A] = temp_block[0x3A]; block[0x3B] = temp_block[0x3B];
        block[0x34] = temp_block[0x34]; block[0x2D] = temp_block[0x2D];
        block[0x26] = temp_block[0x26]; block[0x1F] = temp_block[0x1F];
        block[0x27] = temp_block[0x27]; block[0x2E] = temp_block[0x2E];
        if(last_non_zero_p1 <= 56) goto end;
        block[0x35] = temp_block[0x35]; block[0x3C] = temp_block[0x3C];
        block[0x3D] = temp_block[0x3D]; block[0x36] = temp_block[0x36];
        block[0x2F] = temp_block[0x2F]; block[0x37] = temp_block[0x37];
        block[0x3E] = temp_block[0x3E]; block[0x3F] = temp_block[0x3F];
    }
    end:
/*
    for(i=0; i<last_non_zero_p1; i++)
    {
       int j= zigzag_direct_noperm[i];
       block[block_permute_op(j)]= temp_block[j];
    }
*/

    return last_non_zero_p1 - 1;
}
Commit	Line	Data
2f349de2	1	/*
ff4ec49e FB	2	* MPEG video MMX templates
	3	*
	4	* Copyright (c) 2002 Michael Niedermayer <michaelni@gmx.at>
	5	*
	6	* This library is free software; you can redistribute it and/or
	7	* modify it under the terms of the GNU Lesser General Public
	8	* License as published by the Free Software Foundation; either
	9	* version 2 of the License, or (at your option) any later version.
	10	*
	11	* This library is distributed in the hope that it will be useful,
	12	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	13	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	14	* Lesser General Public License for more details.
	15	*
	16	* You should have received a copy of the GNU Lesser General Public
	17	* License along with this library; if not, write to the Free Software
	18	* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
	19	*/
2f349de2 MN	20	#undef SPREADW
	21	#undef PMAXW
	22	#ifdef HAVE_MMX2
	23	#define SPREADW(a) "pshufw $0, " #a ", " #a " \n\t"
	24	#define PMAXW(a,b) "pmaxsw " #a ", " #b " \n\t"
12e96681 MN	25	#define PMAX(a,b) \
	26	"pshufw $0x0E," #a ", " #b " \n\t"\
	27	PMAXW(b, a)\
	28	"pshufw $0x01," #a ", " #b " \n\t"\
	29	PMAXW(b, a)
2f349de2 MN	30	#else
	31	#define SPREADW(a) \
	32	"punpcklwd " #a ", " #a " \n\t"\
	33	"punpcklwd " #a ", " #a " \n\t"
	34	#define PMAXW(a,b) \
	35	"psubusw " #a ", " #b " \n\t"\
	36	"paddw " #a ", " #b " \n\t"
12e96681 MN	37	#define PMAX(a,b) \
	38	"movq " #a ", " #b " \n\t"\
	39	"psrlq $32, " #a " \n\t"\
	40	PMAXW(b, a)\
	41	"movq " #a ", " #b " \n\t"\
	42	"psrlq $16, " #a " \n\t"\
	43	PMAXW(b, a)
	44
2f349de2 MN	45	#endif
	46
	47	static int RENAME(dct_quantize)(MpegEncContext *s,
	48	DCTELEM *block, int n,
d7e9533a	49	int qscale, int *overflow)
2f349de2	50	{
053dea12 AJ	51	long last_non_zero_p1;
053dea12 AJ	52	int level=0, q; //=0 is cuz gcc says uninitalized ...
0c1a9eda	53	const uint16_t qmat, bias;
f8188626	54	__align8 int16_t temp_block[64];
115329f1	55
f8188626	56	assert((7&(int)(&temp_block[0])) == 0); //did gcc align it correctly?
2f349de2	57
28db7fce	58	//s->fdct (block);
cf3bf5bb	59	RENAMEl(ff_fdct) (block); //cant be anything else ...
d7e9533a	60
821cb11f	61	if(s->dct_error_sum)
783df5f3	62	s->denoise_dct(s, block);
821cb11f	63
2f349de2 MN	64	if (s->mb_intra) {
	65	int dummy;
	66	if (n < 4)
1a684b3d	67	q = s->y_dc_scale;
2f349de2	68	else
1a684b3d	69	q = s->c_dc_scale;
2f349de2	70	/* note: block[0] is assumed to be positive */
ef5b1b5a	71	if (!s->h263_aic) {
2f349de2	72	#if 1
d7e9533a	73	asm volatile (
61b3b639	74	"mul %%ecx \n\t"
d7e9533a	75	: "=d" (level), "=a"(dummy)
1a684b3d	76	: "a" ((block[0]>>2) + q), "c" (inverse[q<<1])
d7e9533a	77	);
2f349de2	78	#else
d7e9533a MN	79	asm volatile (
	80	"xorl %%edx, %%edx \n\t"
	81	"divw %%cx \n\t"
	82	"movzwl %%ax, %%eax \n\t"
	83	: "=a" (level)
1a684b3d	84	: "a" ((block[0]>>2) + q), "c" (q<<1)
d7e9533a MN	85	: "%edx"
d7e9533a MN	86	);
2f349de2	87	#endif
ef5b1b5a J	88	} else
ef5b1b5a J	89	/* For AIC we skip quant/dequant of INTRADC */
1a684b3d	90	level = (block[0] + 4)>>3;
115329f1	91
d7e9533a	92	block[0]=0; //avoid fake overflow
2f349de2	93	// temp_block[0] = (block[0] + (q >> 1)) / q;
2f349de2	94	last_non_zero_p1 = 1;
642ccefb MN	95	bias = s->q_intra_matrix16[qscale][1];
642ccefb MN	96	qmat = s->q_intra_matrix16[qscale][0];
2f349de2	97	} else {
2f349de2	98	last_non_zero_p1 = 0;
642ccefb MN	99	bias = s->q_inter_matrix16[qscale][1];
642ccefb MN	100	qmat = s->q_inter_matrix16[qscale][0];
2f349de2 MN	101	}
2f349de2 MN	102
1c3990db	103	if((s->out_format == FMT_H263 \|\| s->out_format == FMT_H261) && s->mpeg_quant==0){
115329f1	104
d7e9533a	105	asm volatile(
053dea12	106	"movd %%"REG_a", %%mm3 \n\t" // last_non_zero_p1
d7e9533a MN	107	SPREADW(%%mm3)
	108	"pxor %%mm7, %%mm7 \n\t" // 0
	109	"pxor %%mm4, %%mm4 \n\t" // 0
d04fdff1	110	"movq (%2), %%mm5 \n\t" // qmat[0]
d7e9533a	111	"pxor %%mm6, %%mm6 \n\t"
d04fdff1	112	"psubw (%3), %%mm6 \n\t" // -bias[0]
053dea12	113	"mov $-128, %%"REG_a" \n\t"
d7e9533a MN	114	".balign 16 \n\t"
	115	"1: \n\t"
	116	"pxor %%mm1, %%mm1 \n\t" // 0
053dea12	117	"movq (%1, %%"REG_a"), %%mm0 \n\t" // block[i]
d7e9533a	118	"pcmpgtw %%mm0, %%mm1 \n\t" // block[i] <= 0 ? 0xFF : 0x00
115329f1	119	"pxor %%mm1, %%mm0 \n\t"
d7e9533a MN	120	"psubw %%mm1, %%mm0 \n\t" // ABS(block[i])
	121	"psubusw %%mm6, %%mm0 \n\t" // ABS(block[i]) + bias[0]
	122	"pmulhw %%mm5, %%mm0 \n\t" // (ABS(block[i])qmat[0] - bias[0]qmat[0])>>16
115329f1 DB	123	"por %%mm0, %%mm4 \n\t"
115329f1 DB	124	"pxor %%mm1, %%mm0 \n\t"
d7e9533a	125	"psubw %%mm1, %%mm0 \n\t" // out=((ABS(block[i])qmat[0] - bias[0]qmat[0])>>16)*sign(block[i])
053dea12	126	"movq %%mm0, (%5, %%"REG_a") \n\t"
d7e9533a	127	"pcmpeqw %%mm7, %%mm0 \n\t" // out==0 ? 0xFF : 0x00
115329f1	128	"movq (%4, %%"REG_a"), %%mm1 \n\t"
053dea12	129	"movq %%mm7, (%1, %%"REG_a") \n\t" // 0
d7e9533a MN	130	"pandn %%mm1, %%mm0 \n\t"
d7e9533a MN	131	PMAXW(%%mm0, %%mm3)
053dea12	132	"add $8, %%"REG_a" \n\t"
d7e9533a	133	" js 1b \n\t"
12e96681	134	PMAX(%%mm3, %%mm0)
053dea12 AJ	135	"movd %%mm3, %%"REG_a" \n\t"
053dea12 AJ	136	"movzb %%al, %%"REG_a" \n\t" // last_non_zero_p1
d04fdff1 ZK	137	: "+a" (last_non_zero_p1)
	138	: "r" (block+64), "r" (qmat), "r" (bias),
	139	"r" (inv_zigzag_direct16+64), "r" (temp_block+64)
	140	);
	141	// note the asm is split cuz gcc doesnt like that many operands ...
	142	asm volatile(
	143	"movd %1, %%mm1 \n\t" // max_qcoeff
d7e9533a	144	SPREADW(%%mm1)
115329f1	145	"psubusw %%mm1, %%mm4 \n\t"
d7e9533a	146	"packuswb %%mm4, %%mm4 \n\t"
d04fdff1 ZK	147	"movd %%mm4, %0 \n\t" // *overflow
	148	: "=g" (*overflow)
	149	: "g" (s->max_qcoeff)
	150	);
d7e9533a MN	151	}else{ // FMT_H263
d7e9533a MN	152	asm volatile(
053dea12	153	"movd %%"REG_a", %%mm3 \n\t" // last_non_zero_p1
d7e9533a MN	154	SPREADW(%%mm3)
	155	"pxor %%mm7, %%mm7 \n\t" // 0
	156	"pxor %%mm4, %%mm4 \n\t" // 0
053dea12	157	"mov $-128, %%"REG_a" \n\t"
d7e9533a MN	158	".balign 16 \n\t"
	159	"1: \n\t"
	160	"pxor %%mm1, %%mm1 \n\t" // 0
053dea12	161	"movq (%1, %%"REG_a"), %%mm0 \n\t" // block[i]
d7e9533a	162	"pcmpgtw %%mm0, %%mm1 \n\t" // block[i] <= 0 ? 0xFF : 0x00
115329f1	163	"pxor %%mm1, %%mm0 \n\t"
d7e9533a	164	"psubw %%mm1, %%mm0 \n\t" // ABS(block[i])
053dea12	165	"movq (%3, %%"REG_a"), %%mm6 \n\t" // bias[0]
d7e9533a	166	"paddusw %%mm6, %%mm0 \n\t" // ABS(block[i]) + bias[0]
053dea12	167	"movq (%2, %%"REG_a"), %%mm5 \n\t" // qmat[i]
d7e9533a	168	"pmulhw %%mm5, %%mm0 \n\t" // (ABS(block[i])qmat[0] + bias[0]qmat[0])>>16
115329f1 DB	169	"por %%mm0, %%mm4 \n\t"
115329f1 DB	170	"pxor %%mm1, %%mm0 \n\t"
d7e9533a	171	"psubw %%mm1, %%mm0 \n\t" // out=((ABS(block[i])qmat[0] - bias[0]qmat[0])>>16)*sign(block[i])
053dea12	172	"movq %%mm0, (%5, %%"REG_a") \n\t"
d7e9533a	173	"pcmpeqw %%mm7, %%mm0 \n\t" // out==0 ? 0xFF : 0x00
115329f1	174	"movq (%4, %%"REG_a"), %%mm1 \n\t"
053dea12	175	"movq %%mm7, (%1, %%"REG_a") \n\t" // 0
d7e9533a MN	176	"pandn %%mm1, %%mm0 \n\t"
d7e9533a MN	177	PMAXW(%%mm0, %%mm3)
053dea12	178	"add $8, %%"REG_a" \n\t"
d7e9533a	179	" js 1b \n\t"
12e96681	180	PMAX(%%mm3, %%mm0)
053dea12 AJ	181	"movd %%mm3, %%"REG_a" \n\t"
053dea12 AJ	182	"movzb %%al, %%"REG_a" \n\t" // last_non_zero_p1
d04fdff1 ZK	183	: "+a" (last_non_zero_p1)
d04fdff1 ZK	184	: "r" (block+64), "r" (qmat+64), "r" (bias+64),
d7e9533a MN	185	"r" (inv_zigzag_direct16+64), "r" (temp_block+64)
	186	);
	187	// note the asm is split cuz gcc doesnt like that many operands ...
	188	asm volatile(
	189	"movd %1, %%mm1 \n\t" // max_qcoeff
	190	SPREADW(%%mm1)
115329f1	191	"psubusw %%mm1, %%mm4 \n\t"
d7e9533a MN	192	"packuswb %%mm4, %%mm4 \n\t"
d7e9533a MN	193	"movd %%mm4, %0 \n\t" // *overflow
d04fdff1 ZK	194	: "=g" (*overflow)
d04fdff1 ZK	195	: "g" (s->max_qcoeff)
d7e9533a MN	196	);
	197	}
	198
2ad1516a MN	199	if(s->mb_intra) block[0]= level;
	200	else block[0]= temp_block[0];
	201
b0368839	202	if(s->dsp.idct_permutation_type == FF_SIMPLE_IDCT_PERM){
2ad1516a	203	if(last_non_zero_p1 <= 1) goto end;
115329f1 DB	204	block[0x08] = temp_block[0x01]; block[0x10] = temp_block[0x08];
115329f1 DB	205	block[0x20] = temp_block[0x10];
2ad1516a	206	if(last_non_zero_p1 <= 4) goto end;
115329f1 DB	207	block[0x18] = temp_block[0x09]; block[0x04] = temp_block[0x02];
115329f1 DB	208	block[0x09] = temp_block[0x03];
2ad1516a	209	if(last_non_zero_p1 <= 7) goto end;
115329f1 DB	210	block[0x14] = temp_block[0x0A]; block[0x28] = temp_block[0x11];
115329f1 DB	211	block[0x12] = temp_block[0x18]; block[0x02] = temp_block[0x20];
2ad1516a	212	if(last_non_zero_p1 <= 11) goto end;
115329f1 DB	213	block[0x1A] = temp_block[0x19]; block[0x24] = temp_block[0x12];
	214	block[0x19] = temp_block[0x0B]; block[0x01] = temp_block[0x04];
	215	block[0x0C] = temp_block[0x05];
2ad1516a	216	if(last_non_zero_p1 <= 16) goto end;
115329f1 DB	217	block[0x11] = temp_block[0x0C]; block[0x29] = temp_block[0x13];
	218	block[0x16] = temp_block[0x1A]; block[0x0A] = temp_block[0x21];
	219	block[0x30] = temp_block[0x28]; block[0x22] = temp_block[0x30];
	220	block[0x38] = temp_block[0x29]; block[0x06] = temp_block[0x22];
2ad1516a	221	if(last_non_zero_p1 <= 24) goto end;
115329f1 DB	222	block[0x1B] = temp_block[0x1B]; block[0x21] = temp_block[0x14];
	223	block[0x1C] = temp_block[0x0D]; block[0x05] = temp_block[0x06];
	224	block[0x0D] = temp_block[0x07]; block[0x15] = temp_block[0x0E];
	225	block[0x2C] = temp_block[0x15]; block[0x13] = temp_block[0x1C];
2ad1516a	226	if(last_non_zero_p1 <= 32) goto end;
115329f1 DB	227	block[0x0B] = temp_block[0x23]; block[0x34] = temp_block[0x2A];
	228	block[0x2A] = temp_block[0x31]; block[0x32] = temp_block[0x38];
	229	block[0x3A] = temp_block[0x39]; block[0x26] = temp_block[0x32];
	230	block[0x39] = temp_block[0x2B]; block[0x03] = temp_block[0x24];
2ad1516a	231	if(last_non_zero_p1 <= 40) goto end;
115329f1 DB	232	block[0x1E] = temp_block[0x1D]; block[0x25] = temp_block[0x16];
	233	block[0x1D] = temp_block[0x0F]; block[0x2D] = temp_block[0x17];
	234	block[0x17] = temp_block[0x1E]; block[0x0E] = temp_block[0x25];
	235	block[0x31] = temp_block[0x2C]; block[0x2B] = temp_block[0x33];
2ad1516a	236	if(last_non_zero_p1 <= 48) goto end;
115329f1 DB	237	block[0x36] = temp_block[0x3A]; block[0x3B] = temp_block[0x3B];
	238	block[0x23] = temp_block[0x34]; block[0x3C] = temp_block[0x2D];
	239	block[0x07] = temp_block[0x26]; block[0x1F] = temp_block[0x1F];
	240	block[0x0F] = temp_block[0x27]; block[0x35] = temp_block[0x2E];
2ad1516a	241	if(last_non_zero_p1 <= 56) goto end;
115329f1 DB	242	block[0x2E] = temp_block[0x35]; block[0x33] = temp_block[0x3C];
	243	block[0x3E] = temp_block[0x3D]; block[0x27] = temp_block[0x36];
	244	block[0x3D] = temp_block[0x2F]; block[0x2F] = temp_block[0x37];
2ad1516a	245	block[0x37] = temp_block[0x3E]; block[0x3F] = temp_block[0x3F];
b0368839	246	}else if(s->dsp.idct_permutation_type == FF_LIBMPEG2_IDCT_PERM){
2ad1516a	247	if(last_non_zero_p1 <= 1) goto end;
115329f1 DB	248	block[0x04] = temp_block[0x01];
115329f1 DB	249	block[0x08] = temp_block[0x08]; block[0x10] = temp_block[0x10];
2ad1516a	250	if(last_non_zero_p1 <= 4) goto end;
115329f1 DB	251	block[0x0C] = temp_block[0x09]; block[0x01] = temp_block[0x02];
115329f1 DB	252	block[0x05] = temp_block[0x03];
2ad1516a	253	if(last_non_zero_p1 <= 7) goto end;
115329f1 DB	254	block[0x09] = temp_block[0x0A]; block[0x14] = temp_block[0x11];
115329f1 DB	255	block[0x18] = temp_block[0x18]; block[0x20] = temp_block[0x20];
2ad1516a	256	if(last_non_zero_p1 <= 11) goto end;
115329f1 DB	257	block[0x1C] = temp_block[0x19];
	258	block[0x11] = temp_block[0x12]; block[0x0D] = temp_block[0x0B];
	259	block[0x02] = temp_block[0x04]; block[0x06] = temp_block[0x05];
2ad1516a	260	if(last_non_zero_p1 <= 16) goto end;
115329f1 DB	261	block[0x0A] = temp_block[0x0C]; block[0x15] = temp_block[0x13];
	262	block[0x19] = temp_block[0x1A]; block[0x24] = temp_block[0x21];
	263	block[0x28] = temp_block[0x28]; block[0x30] = temp_block[0x30];
	264	block[0x2C] = temp_block[0x29]; block[0x21] = temp_block[0x22];
2ad1516a	265	if(last_non_zero_p1 <= 24) goto end;
115329f1 DB	266	block[0x1D] = temp_block[0x1B]; block[0x12] = temp_block[0x14];
	267	block[0x0E] = temp_block[0x0D]; block[0x03] = temp_block[0x06];
	268	block[0x07] = temp_block[0x07]; block[0x0B] = temp_block[0x0E];
	269	block[0x16] = temp_block[0x15]; block[0x1A] = temp_block[0x1C];
2ad1516a	270	if(last_non_zero_p1 <= 32) goto end;
115329f1 DB	271	block[0x25] = temp_block[0x23]; block[0x29] = temp_block[0x2A];
	272	block[0x34] = temp_block[0x31]; block[0x38] = temp_block[0x38];
	273	block[0x3C] = temp_block[0x39]; block[0x31] = temp_block[0x32];
	274	block[0x2D] = temp_block[0x2B]; block[0x22] = temp_block[0x24];
2ad1516a	275	if(last_non_zero_p1 <= 40) goto end;
115329f1 DB	276	block[0x1E] = temp_block[0x1D]; block[0x13] = temp_block[0x16];
	277	block[0x0F] = temp_block[0x0F]; block[0x17] = temp_block[0x17];
	278	block[0x1B] = temp_block[0x1E]; block[0x26] = temp_block[0x25];
	279	block[0x2A] = temp_block[0x2C]; block[0x35] = temp_block[0x33];
2ad1516a	280	if(last_non_zero_p1 <= 48) goto end;
115329f1 DB	281	block[0x39] = temp_block[0x3A]; block[0x3D] = temp_block[0x3B];
	282	block[0x32] = temp_block[0x34]; block[0x2E] = temp_block[0x2D];
	283	block[0x23] = temp_block[0x26]; block[0x1F] = temp_block[0x1F];
	284	block[0x27] = temp_block[0x27]; block[0x2B] = temp_block[0x2E];
2ad1516a	285	if(last_non_zero_p1 <= 56) goto end;
115329f1 DB	286	block[0x36] = temp_block[0x35]; block[0x3A] = temp_block[0x3C];
	287	block[0x3E] = temp_block[0x3D]; block[0x33] = temp_block[0x36];
	288	block[0x2F] = temp_block[0x2F]; block[0x37] = temp_block[0x37];
2ad1516a MN	289	block[0x3B] = temp_block[0x3E]; block[0x3F] = temp_block[0x3F];
	290	}else{
	291	if(last_non_zero_p1 <= 1) goto end;
115329f1 DB	292	block[0x01] = temp_block[0x01];
115329f1 DB	293	block[0x08] = temp_block[0x08]; block[0x10] = temp_block[0x10];
2ad1516a	294	if(last_non_zero_p1 <= 4) goto end;
115329f1 DB	295	block[0x09] = temp_block[0x09]; block[0x02] = temp_block[0x02];
115329f1 DB	296	block[0x03] = temp_block[0x03];
2ad1516a	297	if(last_non_zero_p1 <= 7) goto end;
115329f1 DB	298	block[0x0A] = temp_block[0x0A]; block[0x11] = temp_block[0x11];
115329f1 DB	299	block[0x18] = temp_block[0x18]; block[0x20] = temp_block[0x20];
2ad1516a	300	if(last_non_zero_p1 <= 11) goto end;
115329f1 DB	301	block[0x19] = temp_block[0x19];
	302	block[0x12] = temp_block[0x12]; block[0x0B] = temp_block[0x0B];
	303	block[0x04] = temp_block[0x04]; block[0x05] = temp_block[0x05];
2ad1516a	304	if(last_non_zero_p1 <= 16) goto end;
115329f1 DB	305	block[0x0C] = temp_block[0x0C]; block[0x13] = temp_block[0x13];
	306	block[0x1A] = temp_block[0x1A]; block[0x21] = temp_block[0x21];
	307	block[0x28] = temp_block[0x28]; block[0x30] = temp_block[0x30];
	308	block[0x29] = temp_block[0x29]; block[0x22] = temp_block[0x22];
2ad1516a	309	if(last_non_zero_p1 <= 24) goto end;
115329f1 DB	310	block[0x1B] = temp_block[0x1B]; block[0x14] = temp_block[0x14];
	311	block[0x0D] = temp_block[0x0D]; block[0x06] = temp_block[0x06];
	312	block[0x07] = temp_block[0x07]; block[0x0E] = temp_block[0x0E];
	313	block[0x15] = temp_block[0x15]; block[0x1C] = temp_block[0x1C];
2ad1516a	314	if(last_non_zero_p1 <= 32) goto end;
115329f1 DB	315	block[0x23] = temp_block[0x23]; block[0x2A] = temp_block[0x2A];
	316	block[0x31] = temp_block[0x31]; block[0x38] = temp_block[0x38];
	317	block[0x39] = temp_block[0x39]; block[0x32] = temp_block[0x32];
	318	block[0x2B] = temp_block[0x2B]; block[0x24] = temp_block[0x24];
2ad1516a	319	if(last_non_zero_p1 <= 40) goto end;
115329f1 DB	320	block[0x1D] = temp_block[0x1D]; block[0x16] = temp_block[0x16];
	321	block[0x0F] = temp_block[0x0F]; block[0x17] = temp_block[0x17];
	322	block[0x1E] = temp_block[0x1E]; block[0x25] = temp_block[0x25];
	323	block[0x2C] = temp_block[0x2C]; block[0x33] = temp_block[0x33];
2ad1516a	324	if(last_non_zero_p1 <= 48) goto end;
115329f1 DB	325	block[0x3A] = temp_block[0x3A]; block[0x3B] = temp_block[0x3B];
	326	block[0x34] = temp_block[0x34]; block[0x2D] = temp_block[0x2D];
	327	block[0x26] = temp_block[0x26]; block[0x1F] = temp_block[0x1F];
	328	block[0x27] = temp_block[0x27]; block[0x2E] = temp_block[0x2E];
2ad1516a	329	if(last_non_zero_p1 <= 56) goto end;
115329f1 DB	330	block[0x35] = temp_block[0x35]; block[0x3C] = temp_block[0x3C];
	331	block[0x3D] = temp_block[0x3D]; block[0x36] = temp_block[0x36];
	332	block[0x2F] = temp_block[0x2F]; block[0x37] = temp_block[0x37];
2ad1516a MN	333	block[0x3E] = temp_block[0x3E]; block[0x3F] = temp_block[0x3F];
	334	}
	335	end:
2f349de2 MN	336	/*
	337	for(i=0; i<last_non_zero_p1; i++)
	338	{
	339	int j= zigzag_direct_noperm[i];
	340	block[block_permute_op(j)]= temp_block[j];
	341	}
	342	*/
d7e9533a	343
2f349de2 MN	344	return last_non_zero_p1 - 1;
2f349de2 MN	345	}