[libav.git] / libavcodec / ppc / idct_altivec.c

/*
 * Copyright (c) 2001 Michel Lespinasse
 *
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public
 * License as published by the Free Software Foundation; either
 * version 2 of the License, or (at your option) any later version.
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public
 * License along with this library; if not, write to the Free Software
 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
 *
 */

/*
 * NOTE: This code is based on GPL code from the libmpeg2 project.  The
 * author, Michel Lespinasses, has given explicit permission to release
 * under LGPL as part of ffmpeg.
 *
 */

/*
 * FFMpeg integration by Dieter Shirley
 *
 * This file is a direct copy of the altivec idct module from the libmpeg2
 * project.  I've deleted all of the libmpeg2 specific code, renamed the functions and
 * re-ordered the function parameters.  The only change to the IDCT function
 * itself was to factor out the partial transposition, and to perform a full
 * transpose at the end of the function.
 */


#include <stdlib.h>                                      /* malloc(), free() */
#include <string.h>
#include "../dsputil.h"
#include "dsputil_altivec.h"

#define vector_s16_t vector signed short
#define vector_u16_t vector unsigned short
#define vector_s8_t vector signed char
#define vector_u8_t vector unsigned char
#define vector_s32_t vector signed int
#define vector_u32_t vector unsigned int

#define IDCT_HALF					\
    /* 1st stage */					\
    t1 = vec_mradds (a1, vx7, vx1 );			\
    t8 = vec_mradds (a1, vx1, vec_subs (zero, vx7));	\
    t7 = vec_mradds (a2, vx5, vx3);			\
    t3 = vec_mradds (ma2, vx3, vx5);			\
							\
    /* 2nd stage */					\
    t5 = vec_adds (vx0, vx4);				\
    t0 = vec_subs (vx0, vx4);				\
    t2 = vec_mradds (a0, vx6, vx2);			\
    t4 = vec_mradds (a0, vx2, vec_subs (zero, vx6));	\
    t6 = vec_adds (t8, t3);				\
    t3 = vec_subs (t8, t3);				\
    t8 = vec_subs (t1, t7);				\
    t1 = vec_adds (t1, t7);				\
							\
    /* 3rd stage */					\
    t7 = vec_adds (t5, t2);				\
    t2 = vec_subs (t5, t2);				\
    t5 = vec_adds (t0, t4);				\
    t0 = vec_subs (t0, t4);				\
    t4 = vec_subs (t8, t3);				\
    t3 = vec_adds (t8, t3);				\
							\
    /* 4th stage */					\
    vy0 = vec_adds (t7, t1);				\
    vy7 = vec_subs (t7, t1);				\
    vy1 = vec_mradds (c4, t3, t5);			\
    vy6 = vec_mradds (mc4, t3, t5);			\
    vy2 = vec_mradds (c4, t4, t0);			\
    vy5 = vec_mradds (mc4, t4, t0);			\
    vy3 = vec_adds (t2, t6);				\
    vy4 = vec_subs (t2, t6);

	
#define IDCT								\
    vector_s16_t vx0, vx1, vx2, vx3, vx4, vx5, vx6, vx7;		\
    vector_s16_t vy0, vy1, vy2, vy3, vy4, vy5, vy6, vy7;		\
    vector_s16_t a0, a1, a2, ma2, c4, mc4, zero, bias;			\
    vector_s16_t t0, t1, t2, t3, t4, t5, t6, t7, t8;			\
    vector_u16_t shift;							\
									\
    c4 = vec_splat (constants[0], 0);					\
    a0 = vec_splat (constants[0], 1);					\
    a1 = vec_splat (constants[0], 2);					\
    a2 = vec_splat (constants[0], 3);					\
    mc4 = vec_splat (constants[0], 4);					\
    ma2 = vec_splat (constants[0], 5);					\
    bias = (vector_s16_t)vec_splat ((vector_s32_t)constants[0], 3);	\
									\
    zero = vec_splat_s16 (0);						\
    shift = vec_splat_u16 (4);						\
									\
    vx0 = vec_mradds (vec_sl (block[0], shift), constants[1], zero);	\
    vx1 = vec_mradds (vec_sl (block[1], shift), constants[2], zero);	\
    vx2 = vec_mradds (vec_sl (block[2], shift), constants[3], zero);	\
    vx3 = vec_mradds (vec_sl (block[3], shift), constants[4], zero);	\
    vx4 = vec_mradds (vec_sl (block[4], shift), constants[1], zero);	\
    vx5 = vec_mradds (vec_sl (block[5], shift), constants[4], zero);	\
    vx6 = vec_mradds (vec_sl (block[6], shift), constants[3], zero);	\
    vx7 = vec_mradds (vec_sl (block[7], shift), constants[2], zero);	\
									\
    IDCT_HALF								\
									\
    vx0 = vec_mergeh (vy0, vy4);					\
    vx1 = vec_mergel (vy0, vy4);					\
    vx2 = vec_mergeh (vy1, vy5);					\
    vx3 = vec_mergel (vy1, vy5);					\
    vx4 = vec_mergeh (vy2, vy6);					\
    vx5 = vec_mergel (vy2, vy6);					\
    vx6 = vec_mergeh (vy3, vy7);					\
    vx7 = vec_mergel (vy3, vy7);					\
									\
    vy0 = vec_mergeh (vx0, vx4);					\
    vy1 = vec_mergel (vx0, vx4);					\
    vy2 = vec_mergeh (vx1, vx5);					\
    vy3 = vec_mergel (vx1, vx5);					\
    vy4 = vec_mergeh (vx2, vx6);					\
    vy5 = vec_mergel (vx2, vx6);					\
    vy6 = vec_mergeh (vx3, vx7);					\
    vy7 = vec_mergel (vx3, vx7);					\
									\
    vx0 = vec_adds (vec_mergeh (vy0, vy4), bias);			\
    vx1 = vec_mergel (vy0, vy4);					\
    vx2 = vec_mergeh (vy1, vy5);					\
    vx3 = vec_mergel (vy1, vy5);					\
    vx4 = vec_mergeh (vy2, vy6);					\
    vx5 = vec_mergel (vy2, vy6);					\
    vx6 = vec_mergeh (vy3, vy7);					\
    vx7 = vec_mergel (vy3, vy7);					\
									\
    IDCT_HALF								\
									\
    shift = vec_splat_u16 (6);						\
    vx0 = vec_sra (vy0, shift);						\
    vx1 = vec_sra (vy1, shift);						\
    vx2 = vec_sra (vy2, shift);						\
    vx3 = vec_sra (vy3, shift);						\
    vx4 = vec_sra (vy4, shift);						\
    vx5 = vec_sra (vy5, shift);						\
    vx6 = vec_sra (vy6, shift);						\
    vx7 = vec_sra (vy7, shift);


#ifdef CONFIG_DARWIN
static const vector_s16_t constants[5] = {
    (vector_s16_t)(23170, 13573, 6518, 21895, -23170, -21895, 32, 31),
    (vector_s16_t)(16384, 22725, 21407, 19266, 16384, 19266, 21407, 22725),
    (vector_s16_t)(22725, 31521, 29692, 26722, 22725, 26722, 29692, 31521),
    (vector_s16_t)(21407, 29692, 27969, 25172, 21407, 25172, 27969, 29692),
    (vector_s16_t)(19266, 26722, 25172, 22654, 19266, 22654, 25172, 26722)
};
#else
// broken gcc
static const vector_s16_t constants[5] = {
    (vector_s16_t){23170, 13573, 6518, 21895, -23170, -21895, 32, 31},
    (vector_s16_t){16384, 22725, 21407, 19266, 16384, 19266, 21407, 22725},
    (vector_s16_t){22725, 31521, 29692, 26722, 22725, 26722, 29692, 31521},
    (vector_s16_t){21407, 29692, 27969, 25172, 21407, 25172, 27969, 29692},
    (vector_s16_t){19266, 26722, 25172, 22654, 19266, 22654, 25172, 26722}
};
#endif

void idct_put_altivec(uint8_t* dest, int stride, vector_s16_t* block)
{
POWERPC_TBL_DECLARE(altivec_idct_put_num, 1);
#ifdef ALTIVEC_USE_REFERENCE_C_CODE
POWERPC_TBL_START_COUNT(altivec_idct_put_num, 1);
    void simple_idct_put(UINT8 *dest, int line_size, INT16 *block);
    simple_idct_put(dest, stride, (INT16*)block);
POWERPC_TBL_STOP_COUNT(altivec_idct_put_num, 1);
#else /* ALTIVEC_USE_REFERENCE_C_CODE */
    vector_u8_t tmp;

POWERPC_TBL_START_COUNT(altivec_idct_put_num, 1);

    IDCT

#define COPY(dest,src)						\
    tmp = vec_packsu (src, src);				\
    vec_ste ((vector_u32_t)tmp, 0, (unsigned int *)dest);	\
    vec_ste ((vector_u32_t)tmp, 4, (unsigned int *)dest);

    COPY (dest, vx0)	dest += stride;
    COPY (dest, vx1)	dest += stride;
    COPY (dest, vx2)	dest += stride;
    COPY (dest, vx3)	dest += stride;
    COPY (dest, vx4)	dest += stride;
    COPY (dest, vx5)	dest += stride;
    COPY (dest, vx6)	dest += stride;
    COPY (dest, vx7)

POWERPC_TBL_STOP_COUNT(altivec_idct_put_num, 1);
#endif /* ALTIVEC_USE_REFERENCE_C_CODE */
}

void idct_add_altivec(uint8_t* dest, int stride, vector_s16_t* block)
{
POWERPC_TBL_DECLARE(altivec_idct_add_num, 1);
#ifdef ALTIVEC_USE_REFERENCE_C_CODE
POWERPC_TBL_START_COUNT(altivec_idct_add_num, 1);
    void simple_idct_add(UINT8 *dest, int line_size, INT16 *block);
    simple_idct_add(dest, stride, (INT16*)block);
POWERPC_TBL_STOP_COUNT(altivec_idct_add_num, 1);
#else /* ALTIVEC_USE_REFERENCE_C_CODE */
    vector_u8_t tmp;
    vector_s16_t tmp2, tmp3;
    vector_u8_t perm0;
    vector_u8_t perm1;
    vector_u8_t p0, p1, p;

POWERPC_TBL_START_COUNT(altivec_idct_add_num, 1);

    IDCT

    p0 = vec_lvsl (0, dest);
    p1 = vec_lvsl (stride, dest);
    p = vec_splat_u8 (-1);
    perm0 = vec_mergeh (p, p0);
    perm1 = vec_mergeh (p, p1);

#define ADD(dest,src,perm)						\
    /* *(uint64_t *)&tmp = *(uint64_t *)dest; */			\
    tmp = vec_ld (0, dest);						\
    tmp2 = (vector_s16_t)vec_perm (tmp, (vector_u8_t)zero, perm);	\
    tmp3 = vec_adds (tmp2, src);					\
    tmp = vec_packsu (tmp3, tmp3);					\
    vec_ste ((vector_u32_t)tmp, 0, (unsigned int *)dest);		\
    vec_ste ((vector_u32_t)tmp, 4, (unsigned int *)dest);

    ADD (dest, vx0, perm0)	dest += stride;
    ADD (dest, vx1, perm1)	dest += stride;
    ADD (dest, vx2, perm0)	dest += stride;
    ADD (dest, vx3, perm1)	dest += stride;
    ADD (dest, vx4, perm0)	dest += stride;
    ADD (dest, vx5, perm1)	dest += stride;
    ADD (dest, vx6, perm0)	dest += stride;
    ADD (dest, vx7, perm1)

POWERPC_TBL_STOP_COUNT(altivec_idct_add_num, 1);
#endif /* ALTIVEC_USE_REFERENCE_C_CODE */
}
Commit	Line	Data
05c4072b MN	1	/*
	2	* Copyright (c) 2001 Michel Lespinasse
	3	*
	4	* This library is free software; you can redistribute it and/or
	5	* modify it under the terms of the GNU Lesser General Public
	6	* License as published by the Free Software Foundation; either
	7	* version 2 of the License, or (at your option) any later version.
	8	*
	9	* This library is distributed in the hope that it will be useful,
	10	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	11	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	12	* Lesser General Public License for more details.
	13	*
	14	* You should have received a copy of the GNU Lesser General Public
	15	* License along with this library; if not, write to the Free Software
	16	* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
	17	*
	18	*/
	19
	20	/*
	21	* NOTE: This code is based on GPL code from the libmpeg2 project. The
	22	* author, Michel Lespinasses, has given explicit permission to release
	23	* under LGPL as part of ffmpeg.
	24	*
	25	*/
	26
	27	/*
	28	* FFMpeg integration by Dieter Shirley
	29	*
	30	* This file is a direct copy of the altivec idct module from the libmpeg2
	31	* project. I've deleted all of the libmpeg2 specific code, renamed the functions and
	32	* re-ordered the function parameters. The only change to the IDCT function
	33	* itself was to factor out the partial transposition, and to perform a full
	34	* transpose at the end of the function.
	35	*/
	36
	37
	38	#include <stdlib.h> /* malloc(), free() */
	39	#include <string.h>
	40	#include "../dsputil.h"
db40a39a	41	#include "dsputil_altivec.h"
05c4072b MN	42
	43	#define vector_s16_t vector signed short
	44	#define vector_u16_t vector unsigned short
	45	#define vector_s8_t vector signed char
	46	#define vector_u8_t vector unsigned char
	47	#define vector_s32_t vector signed int
	48	#define vector_u32_t vector unsigned int
	49
	50	#define IDCT_HALF \
	51	/* 1st stage */ \
	52	t1 = vec_mradds (a1, vx7, vx1 ); \
	53	t8 = vec_mradds (a1, vx1, vec_subs (zero, vx7)); \
	54	t7 = vec_mradds (a2, vx5, vx3); \
	55	t3 = vec_mradds (ma2, vx3, vx5); \
	56	\
	57	/* 2nd stage */ \
	58	t5 = vec_adds (vx0, vx4); \
	59	t0 = vec_subs (vx0, vx4); \
	60	t2 = vec_mradds (a0, vx6, vx2); \
	61	t4 = vec_mradds (a0, vx2, vec_subs (zero, vx6)); \
	62	t6 = vec_adds (t8, t3); \
	63	t3 = vec_subs (t8, t3); \
	64	t8 = vec_subs (t1, t7); \
	65	t1 = vec_adds (t1, t7); \
	66	\
	67	/* 3rd stage */ \
	68	t7 = vec_adds (t5, t2); \
	69	t2 = vec_subs (t5, t2); \
	70	t5 = vec_adds (t0, t4); \
	71	t0 = vec_subs (t0, t4); \
	72	t4 = vec_subs (t8, t3); \
	73	t3 = vec_adds (t8, t3); \
	74	\
	75	/* 4th stage */ \
	76	vy0 = vec_adds (t7, t1); \
	77	vy7 = vec_subs (t7, t1); \
	78	vy1 = vec_mradds (c4, t3, t5); \
	79	vy6 = vec_mradds (mc4, t3, t5); \
	80	vy2 = vec_mradds (c4, t4, t0); \
	81	vy5 = vec_mradds (mc4, t4, t0); \
	82	vy3 = vec_adds (t2, t6); \
	83	vy4 = vec_subs (t2, t6);
	84
	85
	86	#define IDCT \
	87	vector_s16_t vx0, vx1, vx2, vx3, vx4, vx5, vx6, vx7; \
	88	vector_s16_t vy0, vy1, vy2, vy3, vy4, vy5, vy6, vy7; \
	89	vector_s16_t a0, a1, a2, ma2, c4, mc4, zero, bias; \
	90	vector_s16_t t0, t1, t2, t3, t4, t5, t6, t7, t8; \
	91	vector_u16_t shift; \
	92	\
	93	c4 = vec_splat (constants[0], 0); \
	94	a0 = vec_splat (constants[0], 1); \
	95	a1 = vec_splat (constants[0], 2); \
	96	a2 = vec_splat (constants[0], 3); \
	97	mc4 = vec_splat (constants[0], 4); \
	98	ma2 = vec_splat (constants[0], 5); \
	99	bias = (vector_s16_t)vec_splat ((vector_s32_t)constants[0], 3); \
	100	\
	101	zero = vec_splat_s16 (0); \
	102	shift = vec_splat_u16 (4); \
	103	\
	104	vx0 = vec_mradds (vec_sl (block[0], shift), constants[1], zero); \
	105	vx1 = vec_mradds (vec_sl (block[1], shift), constants[2], zero); \
106	vx2 = vec_mradds (vec_sl (block[2], shift), constants[3], zero); \
107	vx3 = vec_mradds (vec_sl (block[3], shift), constants[4], zero); \
108	vx4 = vec_mradds (vec_sl (block[4], shift), constants[1], zero); \
109	vx5 = vec_mradds (vec_sl (block[5], shift), constants[4], zero); \
110	vx6 = vec_mradds (vec_sl (block[6], shift), constants[3], zero); \
111	vx7 = vec_mradds (vec_sl (block[7], shift), constants[2], zero); \
112	\
113	IDCT_HALF \
114	\
115	vx0 = vec_mergeh (vy0, vy4); \
116	vx1 = vec_mergel (vy0, vy4); \
117	vx2 = vec_mergeh (vy1, vy5); \
118	vx3 = vec_mergel (vy1, vy5); \
119	vx4 = vec_mergeh (vy2, vy6); \
120	vx5 = vec_mergel (vy2, vy6); \
121	vx6 = vec_mergeh (vy3, vy7); \
122	vx7 = vec_mergel (vy3, vy7); \
123	\
124	vy0 = vec_mergeh (vx0, vx4); \
125	vy1 = vec_mergel (vx0, vx4); \
126	vy2 = vec_mergeh (vx1, vx5); \
127	vy3 = vec_mergel (vx1, vx5); \
128	vy4 = vec_mergeh (vx2, vx6); \
129	vy5 = vec_mergel (vx2, vx6); \
130	vy6 = vec_mergeh (vx3, vx7); \
131	vy7 = vec_mergel (vx3, vx7); \
132	\
133	vx0 = vec_adds (vec_mergeh (vy0, vy4), bias); \
134	vx1 = vec_mergel (vy0, vy4); \
135	vx2 = vec_mergeh (vy1, vy5); \
136	vx3 = vec_mergel (vy1, vy5); \
137	vx4 = vec_mergeh (vy2, vy6); \
138	vx5 = vec_mergel (vy2, vy6); \
139	vx6 = vec_mergeh (vy3, vy7); \
140	vx7 = vec_mergel (vy3, vy7); \
141	\
142	IDCT_HALF \
143	\
144	shift = vec_splat_u16 (6); \
145	vx0 = vec_sra (vy0, shift); \
146	vx1 = vec_sra (vy1, shift); \
147	vx2 = vec_sra (vy2, shift); \
148	vx3 = vec_sra (vy3, shift); \
149	vx4 = vec_sra (vy4, shift); \
150	vx5 = vec_sra (vy5, shift); \
151	vx6 = vec_sra (vy6, shift); \
152	vx7 = vec_sra (vy7, shift);
153
3b991c54 RD	154
3b991c54 RD	155	#ifdef CONFIG_DARWIN
05c4072b MN	156	static const vector_s16_t constants[5] = {
	157	(vector_s16_t)(23170, 13573, 6518, 21895, -23170, -21895, 32, 31),
	158	(vector_s16_t)(16384, 22725, 21407, 19266, 16384, 19266, 21407, 22725),
	159	(vector_s16_t)(22725, 31521, 29692, 26722, 22725, 26722, 29692, 31521),
	160	(vector_s16_t)(21407, 29692, 27969, 25172, 21407, 25172, 27969, 29692),
	161	(vector_s16_t)(19266, 26722, 25172, 22654, 19266, 22654, 25172, 26722)
	162	};
3b991c54 RD	163	#else
	164	// broken gcc
	165	static const vector_s16_t constants[5] = {
	166	(vector_s16_t){23170, 13573, 6518, 21895, -23170, -21895, 32, 31},
	167	(vector_s16_t){16384, 22725, 21407, 19266, 16384, 19266, 21407, 22725},
	168	(vector_s16_t){22725, 31521, 29692, 26722, 22725, 26722, 29692, 31521},
	169	(vector_s16_t){21407, 29692, 27969, 25172, 21407, 25172, 27969, 29692},
	170	(vector_s16_t){19266, 26722, 25172, 22654, 19266, 22654, 25172, 26722}
	171	};
	172	#endif
05c4072b MN	173
	174	void idct_put_altivec(uint8_t* dest, int stride, vector_s16_t* block)
	175	{
35e5fb06	176	POWERPC_TBL_DECLARE(altivec_idct_put_num, 1);
db40a39a	177	#ifdef ALTIVEC_USE_REFERENCE_C_CODE
35e5fb06	178	POWERPC_TBL_START_COUNT(altivec_idct_put_num, 1);
db40a39a MN	179	void simple_idct_put(UINT8 dest, int line_size, INT16 block);
db40a39a MN	180	simple_idct_put(dest, stride, (INT16*)block);
35e5fb06	181	POWERPC_TBL_STOP_COUNT(altivec_idct_put_num, 1);
db40a39a	182	#else /* ALTIVEC_USE_REFERENCE_C_CODE */
05c4072b MN	183	vector_u8_t tmp;
05c4072b MN	184
35e5fb06	185	POWERPC_TBL_START_COUNT(altivec_idct_put_num, 1);
db40a39a	186
05c4072b MN	187	IDCT
	188
	189	#define COPY(dest,src) \
	190	tmp = vec_packsu (src, src); \
	191	vec_ste ((vector_u32_t)tmp, 0, (unsigned int *)dest); \
	192	vec_ste ((vector_u32_t)tmp, 4, (unsigned int *)dest);
	193
	194	COPY (dest, vx0) dest += stride;
	195	COPY (dest, vx1) dest += stride;
	196	COPY (dest, vx2) dest += stride;
	197	COPY (dest, vx3) dest += stride;
	198	COPY (dest, vx4) dest += stride;
	199	COPY (dest, vx5) dest += stride;
	200	COPY (dest, vx6) dest += stride;
	201	COPY (dest, vx7)
db40a39a	202
35e5fb06	203	POWERPC_TBL_STOP_COUNT(altivec_idct_put_num, 1);
db40a39a	204	#endif /* ALTIVEC_USE_REFERENCE_C_CODE */
05c4072b MN	205	}
	206
	207	void idct_add_altivec(uint8_t* dest, int stride, vector_s16_t* block)
	208	{
35e5fb06	209	POWERPC_TBL_DECLARE(altivec_idct_add_num, 1);
db40a39a	210	#ifdef ALTIVEC_USE_REFERENCE_C_CODE
35e5fb06	211	POWERPC_TBL_START_COUNT(altivec_idct_add_num, 1);
db40a39a MN	212	void simple_idct_add(UINT8 dest, int line_size, INT16 block);
db40a39a MN	213	simple_idct_add(dest, stride, (INT16*)block);
35e5fb06	214	POWERPC_TBL_STOP_COUNT(altivec_idct_add_num, 1);
db40a39a	215	#else /* ALTIVEC_USE_REFERENCE_C_CODE */
05c4072b MN	216	vector_u8_t tmp;
	217	vector_s16_t tmp2, tmp3;
	218	vector_u8_t perm0;
	219	vector_u8_t perm1;
	220	vector_u8_t p0, p1, p;
	221
35e5fb06	222	POWERPC_TBL_START_COUNT(altivec_idct_add_num, 1);
db40a39a	223
05c4072b MN	224	IDCT
	225
	226	p0 = vec_lvsl (0, dest);
	227	p1 = vec_lvsl (stride, dest);
	228	p = vec_splat_u8 (-1);
	229	perm0 = vec_mergeh (p, p0);
	230	perm1 = vec_mergeh (p, p1);
	231
	232	#define ADD(dest,src,perm) \
	233	/* (uint64_t )&tmp = (uint64_t )dest; */ \
	234	tmp = vec_ld (0, dest); \
	235	tmp2 = (vector_s16_t)vec_perm (tmp, (vector_u8_t)zero, perm); \
	236	tmp3 = vec_adds (tmp2, src); \
	237	tmp = vec_packsu (tmp3, tmp3); \
	238	vec_ste ((vector_u32_t)tmp, 0, (unsigned int *)dest); \
	239	vec_ste ((vector_u32_t)tmp, 4, (unsigned int *)dest);
	240
	241	ADD (dest, vx0, perm0) dest += stride;
	242	ADD (dest, vx1, perm1) dest += stride;
	243	ADD (dest, vx2, perm0) dest += stride;
	244	ADD (dest, vx3, perm1) dest += stride;
	245	ADD (dest, vx4, perm0) dest += stride;
	246	ADD (dest, vx5, perm1) dest += stride;
	247	ADD (dest, vx6, perm0) dest += stride;
	248	ADD (dest, vx7, perm1)
db40a39a	249
35e5fb06	250	POWERPC_TBL_STOP_COUNT(altivec_idct_add_num, 1);
db40a39a	251	#endif /* ALTIVEC_USE_REFERENCE_C_CODE */
05c4072b MN	252	}
05c4072b MN	253