/*********************************************************************/
/* Copyright 2005-2010 The University of Texas at Austin.           */
/* All rights reserved.                                              */
/*                                                                   */
/* Redistribution and use in source and binary forms, with or        */
/* without modification, are permitted provided that the following   */
/* conditions are met:                                               */
/*                                                                   */
/*   1. Redistributions of source code must retain the above         */
/*      copyright notice, this list of conditions and the following  */
/*      disclaimer.                                                  */
/*                                                                   */
/*   2. Redistributions in binary form must reproduce the above      */
/*      copyright notice, this list of conditions and the following  */
/*      disclaimer in the documentation and/or other materials       */
/*      provided with the distribution.                              */
/*                                                                   */
/*    THIS  SOFTWARE IS PROVIDED  BY THE  UNIVERSITY OF  TEXAS AT    */
/*    AUSTIN  ``AS IS''  AND ANY  EXPRESS OR  IMPLIED WARRANTIES,    */
/*    INCLUDING, BUT  NOT LIMITED  TO, THE IMPLIED  WARRANTIES OF    */
/*    MERCHANTABILITY  AND FITNESS FOR  A PARTICULAR  PURPOSE ARE    */
/*    DISCLAIMED.  IN  NO EVENT SHALL THE UNIVERSITY  OF TEXAS AT    */
/*    AUSTIN OR CONTRIBUTORS BE  LIABLE FOR ANY DIRECT, INDIRECT,    */
/*    INCIDENTAL,  SPECIAL, EXEMPLARY,  OR  CONSEQUENTIAL DAMAGES    */
/*    (INCLUDING, BUT  NOT LIMITED TO,  PROCUREMENT OF SUBSTITUTE    */
/*    GOODS  OR  SERVICES; LOSS  OF  USE,  DATA,  OR PROFITS;  OR    */
/*    BUSINESS INTERRUPTION) HOWEVER CAUSED  AND ON ANY THEORY OF    */
/*    LIABILITY, WHETHER  IN CONTRACT, STRICT  LIABILITY, OR TORT    */
/*    (INCLUDING NEGLIGENCE OR OTHERWISE)  ARISING IN ANY WAY OUT    */
/*    OF  THE  USE OF  THIS  SOFTWARE,  EVEN  IF ADVISED  OF  THE    */
/*    POSSIBILITY OF SUCH DAMAGE.                                    */
/*                                                                   */
/* The views and conclusions contained in the software and           */
/* documentation are those of the authors and should not be          */
/* interpreted as representing official policies, either expressed   */
/* or implied, of The University of Texas at Austin.                 */
/*********************************************************************/

#define ASSEMBLER
#include "common.h"

#define APREFETCHSIZE 24
#define APREFETCH_CATEGORY 0

#define M	%i0
#define N	%i1
#define K	%i2
#define A	%i5
#define B	%i3
#define C	%i4

#define LDC	%o0
#define AO	%o1
#define BO	%o2
#define I	%o3
#define J	%o4
#define L	%o5

#define C1	%l0
#define C2	%l1
#define C3	%l2
#define C4	%l3

#define OFFSET	%l4
#define	KK	%l5
#define TEMP1	%l6
#define TEMP2	%l7
#define AORIG	%o7

#ifdef DOUBLE
#define c01	%f0
#define c02	%f2
#define c03	%f4
#define c04	%f6
#define c05	%f8
#define c06	%f10
#define c07	%f12
#define c08	%f14
#define c09	%f16
#define c10	%f18
#define c11	%f20
#define c12	%f22
#define c13	%f24
#define c14	%f26
#define c15	%f28
#define c16	%f30

#define a1	%f32
#define a2	%f34
#define a3	%f36
#define a4	%f38
#define a5	%f40

#define b1	%f42
#define b2	%f44
#define b3	%f46
#define b4	%f48
#define b5	%f50
#define b6	%f52
#define b7	%f54
#define b8	%f56
#define b9	%f58

#define cc01	0
#define cc02	2
#define cc03	4
#define cc04	6
#define cc05	8
#define cc06	10
#define cc07	12
#define cc08	14
#define cc09	16
#define cc10	18
#define cc11	20
#define cc12	22
#define cc13	24
#define cc14	26
#define cc15	28
#define cc16	30

#define aa1	 1
#define aa2	 3
#define aa3	 5
#define aa4	 7
#define aa5	 9

#define bb1	11
#define bb2	13
#define bb3	15
#define bb4	17
#define bb5	19
#define bb6	21
#define bb7	23
#define bb8	25
#define bb9	27
#else
#define c01	%f0
#define c02	%f1
#define c03	%f2
#define c04	%f3
#define c05	%f4
#define c06	%f5
#define c07	%f6
#define c08	%f7
#define c09	%f8
#define c10	%f9
#define c11	%f10
#define c12	%f11
#define c13	%f12
#define c14	%f13
#define c15	%f14
#define c16	%f15

#define a1	%f16
#define a2	%f17
#define a3	%f18
#define a4	%f19
#define a5	%f20

#define b1	%f21
#define b2	%f22
#define b3	%f23
#define b4	%f24
#define b5	%f25
#define b6	%f26
#define b7	%f27
#define b8	%f28
#define b9	%f29

#define cc01	0
#define cc02	1
#define cc03	2
#define cc04	3
#define cc05	4
#define cc06	5
#define cc07	6
#define cc08	7
#define cc09	8
#define cc10	9
#define cc11	10
#define cc12	11
#define cc13	12
#define cc14	13
#define cc15	14
#define cc16	15

#define aa1	16
#define aa2	17
#define aa3	18
#define aa4	19
#define aa5	20

#define bb1	21
#define bb2	22
#define bb3	23
#define bb4	24
#define bb5	25
#define bb6	26
#define bb7	27
#define bb8	28
#define bb9	29

#endif

#ifndef CONJ
#define FMADD1	FMADD
#define FMADD2	FMADD
#define FMADD3	FMADD
#define FMADD4	FNMSUB
#else
#if defined(LN) || defined(LT)
#define FMADD1	FMADD
#define FMADD2	FNMSUB
#define FMADD3	FMADD
#define FMADD4	FMADD
#endif
#if defined(RN) || defined(RT)
#define FMADD1	FMADD
#define FMADD2	FMADD
#define FMADD3	FNMSUB
#define FMADD4	FMADD
#endif
#endif

        .register %g2, #scratch
        .register %g3, #scratch

	PROLOGUE
	SAVESP

#ifndef __64BIT__
#ifdef DOUBLE
	ld	[%sp + STACK_START + 32], A
	ld	[%sp + STACK_START + 36], B
	ld	[%sp + STACK_START + 40], C
	ld	[%sp + STACK_START + 44], LDC
	ld	[%sp + STACK_START + 48], OFFSET
#else
	ld	[%sp + STACK_START + 28], B
	ld	[%sp + STACK_START + 32], C
	ld	[%sp + STACK_START + 36], LDC
	ld	[%sp + STACK_START + 40], OFFSET
#endif
#else
	ldx	[%sp + STACK_START + 56], B
	ldx	[%sp + STACK_START + 64], C
	ldx	[%sp + STACK_START + 72], LDC
	ldx	[%sp + STACK_START + 80], OFFSET
#endif

	cmp	M, 0
	ble,pn	%icc, .LL999
	nop

	sll	LDC, ZBASE_SHIFT, LDC

#ifdef LN
	smul	M, K, TEMP1
	sll	TEMP1, ZBASE_SHIFT, TEMP1
	add	A, TEMP1, A

	sll	M, ZBASE_SHIFT, TEMP1
	add	C, TEMP1, C
#endif

#ifdef RN
	neg	OFFSET, KK
#endif

#ifdef RT
	smul	N, K, TEMP1
	sll	TEMP1, ZBASE_SHIFT, TEMP1
	add	B, TEMP1, B

	smul	N, LDC, TEMP1
	add	C, TEMP1, C

	sub	N, OFFSET, KK
#endif

	and	N, 1, J
	cmp	J, 0
	ble,pn	%icc, .LL20
	nop

#ifdef RT
	sll	K, ZBASE_SHIFT, TEMP1
	sub	B, TEMP1, B
#endif

#ifndef RT
	mov	C,  C1
	add	C,  LDC, C
#else
	sub	C,  LDC, C1
	sub	C,  LDC, C
#endif

#ifdef LN
	add	M, OFFSET, KK
#endif

#ifdef LT
	mov	OFFSET, KK
#endif

#if defined(LN) || defined(RT)
	mov	A, AORIG
#else
	mov	A, AO
#endif

	mov	M, I
	.align 4

.LL32:
#if defined(LT) || defined(RN)
	mov	B, BO
#else
#ifdef LN
	sll	K,  ZBASE_SHIFT, TEMP1
	sub	AORIG, TEMP1, AORIG
#endif

	sll	KK, ZBASE_SHIFT + 0, TEMP1

	add	AORIG, TEMP1, AO
	add	B,     TEMP1, BO
#endif

	LDF	[AO +  0 * SIZE], a1
	LDF	[AO +  1 * SIZE], a2
	LDF	[AO +  2 * SIZE], a3
	LDF	[AO +  3 * SIZE], a4

	LDF	[BO +  0 * SIZE], b1
	LDF	[BO +  1 * SIZE], b2
	LDF	[BO +  2 * SIZE], b3
	FCLR	(cc01)
	LDF	[BO +  3 * SIZE], b4
	FCLR	(cc02)

	LDF	[BO +  4 * SIZE], b5
	FCLR	(cc03)
	LDF	[BO +  5 * SIZE], b6
	FCLR	(cc04)
	LDF	[BO +  6 * SIZE], b7
	FCLR	(cc05)
	LDF	[BO +  7 * SIZE], b8
	FCLR	(cc06)

	prefetch [C1 + 2 * SIZE], 3
	FCLR	(cc07)

#if defined(LT) || defined(RN)
	sra	KK, 2, L
#else
	sub	K, KK, L
	sra	L,  2, L
#endif
	cmp	L,  0
	ble,pn	%icc, .LL35
	FCLR	(cc08)
	.align 4

.LL33:
	FMADD1	(aa1, bb1, cc01, cc01)
	prefetch [AO + (APREFETCHSIZE +  0) * SIZE], APREFETCH_CATEGORY
	FMADD2	(aa2, bb1, cc02, cc02)
	LDF	[BO +  8 * SIZE], b1

	FMADD3	(aa1, bb2, cc03, cc03)
	LDF	[AO +  4 * SIZE], a1
	FMADD4	(aa2, bb2, cc04, cc04)
	LDF	[AO +  5 * SIZE], a2

	FMADD1	(aa3, bb3, cc01, cc01)
	LDF	[BO +  9 * SIZE], b2
	FMADD2	(aa4, bb3, cc02, cc02)
	LDF	[BO + 10 * SIZE], b3

	FMADD3	(aa3, bb4, cc03, cc03)
	LDF	[AO +  6 * SIZE], a3
	FMADD4	(aa4, bb4, cc04, cc04)
	LDF	[AO +  7 * SIZE], a4

	FMADD1	(aa1, bb5, cc01, cc01)
	LDF	[BO + 11 * SIZE], b4
	FMADD2	(aa2, bb5, cc02, cc02)
	LDF	[BO + 12 * SIZE], b5

	FMADD3	(aa1, bb6, cc03, cc03)
	LDF	[AO +  8 * SIZE], a1
	FMADD4	(aa2, bb6, cc04, cc04)
	LDF	[AO +  9 * SIZE], a2

	FMADD1	(aa3, bb7, cc01, cc01)
	LDF	[BO + 13 * SIZE], b6

	FMADD2	(aa4, bb7, cc02, cc02)
	LDF	[BO + 14 * SIZE], b7

	FMADD3	(aa3, bb8, cc03, cc03)
	LDF	[AO + 10 * SIZE], a3
	FMADD4	(aa4, bb8, cc04, cc04)
	LDF	[AO + 11 * SIZE], a4

	add	AO,  8 * SIZE, AO
	add	L, -1, L
	add	BO,  8 * SIZE, BO
	cmp	L, 0

	bg,pt	%icc, .LL33
	LDF	[BO +  7 * SIZE], b8
	.align 4

.LL35:
#if defined(LT) || defined(RN)
	and	KK, 3, L
#else
	sub	K, KK, L
	and	L,  3, L
#endif
	cmp	L,  0
	ble,a,pn %icc, .LL38
	nop
	.align 4

.LL37:
	FMADD1	(aa1, bb1, cc01, cc01)
	add	L, -1, L
	FMADD2	(aa2, bb1, cc02, cc02)
	LDF	[BO + 2 * SIZE], b1

	FMADD3	(aa1, bb2, cc03, cc03)
	LDF	[AO + 2 * SIZE], a1
	FMADD4	(aa2, bb2, cc04, cc04)
	LDF	[AO + 3 * SIZE], a2

	add	AO, 2 * SIZE, AO
	cmp	L, 0
	add	BO, 2 * SIZE, BO
	bg,pt	%icc, .LL37
	LDF	[BO + 1 * SIZE], b2
	.align 4

.LL38:
	FADD	c01, c04, c01
	FADD	c02, c03, c02

#if defined(LN) || defined(RT)
	sub	KK, 1, TEMP1

	sll	TEMP1, ZBASE_SHIFT, TEMP1

	add	AORIG, TEMP1, AO
	add	B,     TEMP1, BO
#endif

#if defined(LN) || defined(LT)
	LDF	[BO +  0 * SIZE], a1
	LDF	[BO +  1 * SIZE], a2
#else
	LDF	[AO +  0 * SIZE], a1
	LDF	[AO +  1 * SIZE], a2
#endif

	FSUB	a1, c01, c01
	FSUB	a2, c02, c02

#if defined(LN) || defined(LT)
	LDF	[AO +  0 * SIZE], a1
	LDF	[AO +  1 * SIZE], a2
#else
	LDF	[BO +  0 * SIZE], a1
	LDF	[BO +  1 * SIZE], a2
#endif

	FMUL	a1, c01, b1
	FMUL	a2, c01, b2

#ifndef CONJ
	FNMSUB	(aa2, cc02, bb1, cc01)
	FMADD	(aa1, cc02, bb2, cc02)
#else
	FMADD	(aa2, cc02, bb1, cc01)
	FMSUB	(aa1, cc02, bb2, cc02)
#endif

#ifdef LN
	add	C1, -2 * SIZE, C1
#endif

#if defined(LN) || defined(LT)
	STF	c01, [BO +  0 * SIZE]
	STF	c02, [BO +  1 * SIZE]
#else
	STF	c01, [AO +  0 * SIZE]
	STF	c02, [AO +  1 * SIZE]
#endif

	STF	c01, [C1 + 0 * SIZE]
	STF	c02, [C1 + 1 * SIZE]

#ifndef LN
	add	C1, 2 * SIZE, C1
#endif

#ifdef RT
	sll	K, ZBASE_SHIFT, TEMP1
	add	AORIG, TEMP1, AORIG
#endif

#if defined(LT) || defined(RN)
	sub	K, KK, TEMP1
	sll	TEMP1, ZBASE_SHIFT, TEMP1
	add	AO, TEMP1, AO
	add	BO, TEMP1, BO
#endif

#ifdef LT
	add	KK, 1, KK
#endif

#ifdef LN
	sub	KK, 1, KK
#endif

	add	I, -1, I
	cmp	I, 0
	bg,pt	%icc, .LL32
	nop

#ifdef LN
	sll	K, ZBASE_SHIFT, TEMP1
	add	B, TEMP1, B
#endif

#if defined(LT) || defined(RN)
	mov	BO, B
#endif

#ifdef RN
	add	KK, 1, KK
#endif

#ifdef RT
	sub	KK, 1, KK
#endif
	.align 4

.LL20:
	and	N, 2, J
	cmp	J, 0
	ble,pn	%icc, .LL30
	nop

#ifdef RT
	sll	K, ZBASE_SHIFT + 1, TEMP1
	sub	B, TEMP1, B
#endif

#ifndef RT
	mov	C,  C1
	add	C,  LDC, C2
	add	C2, LDC, C
#else
	sub	C,  LDC, C2
	sub	C2, LDC, C1
	sub	C2, LDC, C
#endif

#ifdef LN
	add	M, OFFSET, KK
#endif

#ifdef LT
	mov	OFFSET, KK
#endif

#if defined(LN) || defined(RT)
	mov	A, AORIG
#else
	mov	A, AO
#endif

	mov	M, I
	.align 4

.LL22:
#if defined(LT) || defined(RN)
	mov	B, BO
#else
#ifdef LN
	sll	K,  ZBASE_SHIFT, TEMP1
	sub	AORIG, TEMP1, AORIG
#endif

	sll	KK, ZBASE_SHIFT + 0, TEMP1
	sll	KK, ZBASE_SHIFT + 1, TEMP2

	add	AORIG, TEMP1, AO
	add	B,     TEMP2, BO
#endif

	LDF	[AO +  0 * SIZE], a1
	LDF	[AO +  1 * SIZE], a2

	LDF	[BO +  0 * SIZE], b1
	LDF	[BO +  1 * SIZE], b2
	LDF	[BO +  2 * SIZE], b3
	LDF	[BO +  3 * SIZE], b4
	LDF	[BO +  4 * SIZE], b5
	FCLR	(cc01)

	LDF	[BO +  5 * SIZE], b6
	FCLR	(cc02)
	LDF	[BO +  6 * SIZE], b7
	FCLR	(cc03)
	LDF	[BO +  7 * SIZE], b8
	FCLR	(cc04)
	LDF	[BO +  8 * SIZE], b9
	FCLR	(cc05)

	prefetch [C1 + 2 * SIZE], 3
	FCLR	(cc06)
	prefetch [C2 + 2 * SIZE], 3
	FCLR	(cc07)

#if defined(LT) || defined(RN)
	sra	KK, 2, L
#else
	sub	K, KK, L
	sra	L,  2, L
#endif
	cmp	L,  0
	ble,pn	%icc, .LL25
	FCLR	(cc08)
	.align 4

.LL23:
	FMADD1	(aa1, bb1, cc01, cc01)
	LDF	[AO +  2 * SIZE], a3
	FMADD2	(aa2, bb1, cc02, cc02)
	LDF	[AO +  3 * SIZE], a4

	FMADD3	(aa1, bb2, cc03, cc03)
	LDF	[BO + 16 * SIZE], b1
	FMADD4	(aa2, bb2, cc04, cc04)
	LDF	[BO +  9 * SIZE], b2

	FMADD1	(aa1, bb3, cc05, cc05)
	prefetch [AO + (APREFETCHSIZE +  0) * SIZE], APREFETCH_CATEGORY
	FMADD2	(aa2, bb3, cc06, cc06)
	add	L, -1, L

	FMADD3	(aa1, bb4, cc07, cc07)
	LDF	[BO + 10 * SIZE], b3
	FMADD4	(aa2, bb4, cc08, cc08)
	LDF	[BO + 11 * SIZE], b4

	FMADD1	(aa3, bb5, cc01, cc01)
	LDF	[AO +  4 * SIZE], a1
	FMADD2	(aa4, bb5, cc02, cc02)
	LDF	[AO +  5 * SIZE], a2

	FMADD3	(aa3, bb6, cc03, cc03)
	LDF	[BO + 12 * SIZE], b5
	FMADD4	(aa4, bb6, cc04, cc04)
	LDF	[BO + 13 * SIZE], b6

	FMADD1	(aa3, bb7, cc05, cc05)
	cmp	L, 0
	FMADD2	(aa4, bb7, cc06, cc06)
	add	AO,  8 * SIZE, AO

	FMADD3	(aa3, bb8, cc07, cc07)
	LDF	[BO + 14 * SIZE], b7
	FMADD4	(aa4, bb8, cc08, cc08)
	LDF	[BO + 15 * SIZE], b8

	FMADD1	(aa1, bb9, cc01, cc01)
	LDF	[AO -  2 * SIZE], a3
	FMADD2	(aa2, bb9, cc02, cc02)
	LDF	[AO -  1 * SIZE], a4

	FMADD3	(aa1, bb2, cc03, cc03)
	LDF	[BO + 24 * SIZE], b9
	FMADD4	(aa2, bb2, cc04, cc04)
	LDF	[BO + 17 * SIZE], b2

	FMADD1	(aa1, bb3, cc05, cc05)
	add	BO, 16 * SIZE, BO
	FMADD2	(aa2, bb3, cc06, cc06)
	nop

	FMADD3	(aa1, bb4, cc07, cc07)
	LDF	[BO +  2 * SIZE], b3
	FMADD4	(aa2, bb4, cc08, cc08)
	LDF	[BO +  3 * SIZE], b4

	FMADD1	(aa3, bb5, cc01, cc01)
	LDF	[AO +  0 * SIZE], a1
	FMADD2	(aa4, bb5, cc02, cc02)
	LDF	[AO +  1 * SIZE], a2
	FMADD3	(aa3, bb6, cc03, cc03)
	LDF	[BO +  4 * SIZE], b5
	FMADD4	(aa4, bb6, cc04, cc04)
	LDF	[BO +  5 * SIZE], b6

	FMADD1	(aa3, bb7, cc05, cc05)
	nop
	FMADD2	(aa4, bb7, cc06, cc06)
	LDF	[BO +  6 * SIZE], b7

	FMADD3	(aa3, bb8, cc07, cc07)
	FMADD4	(aa4, bb8, cc08, cc08)
	bg,pt	%icc, .LL23
	LDF	[BO +  7 * SIZE], b8
	.align 4

.LL25:
#if defined(LT) || defined(RN)
	and	KK, 3, L
#else
	sub	K, KK, L
	and	L,  3, L
#endif
	cmp	L,  0
	ble,a,pn %icc, .LL28
	nop
	.align 4

.LL27:
	FMADD1	(aa1, bb1, cc01, cc01)
	add	L, -1, L
	FMADD2	(aa2, bb1, cc02, cc02)
	LDF	[BO + 4 * SIZE], b1

	FMADD3	(aa1, bb2, cc03, cc03)
	add	AO, 2 * SIZE, AO
	FMADD4	(aa2, bb2, cc04, cc04)
	LDF	[BO + 5 * SIZE], b2

	FMADD1	(aa1, bb3, cc05, cc05)
	cmp	L, 0
	FMADD2	(aa2, bb3, cc06, cc06)
	LDF	[BO + 6 * SIZE], b3

	FMADD3	(aa1, bb4, cc07, cc07)
	LDF	[AO + 0 * SIZE], a1
	FMADD4	(aa2, bb4, cc08, cc08)
	LDF	[AO + 1 * SIZE], a2

	LDF	[BO + 7 * SIZE], b4
	bg,pt	%icc, .LL27
	add	BO, 4 * SIZE, BO
	.align 4

.LL28:
	FADD	c01, c04, c01
	FADD	c02, c03, c02
	FADD	c05, c08, c05
	FADD	c06, c07, c06

#if defined(LN) || defined(RT)
#ifdef LN
	sub	KK, 1, TEMP1
#else
	sub	KK, 2, TEMP1
#endif
	sll	TEMP1, ZBASE_SHIFT + 0, TEMP2
	sll	TEMP1, ZBASE_SHIFT + 1, TEMP1

	add	AORIG, TEMP2, AO
	add	B,     TEMP1, BO
#endif

#if defined(LN) || defined(LT)
	LDF	[BO +  0 * SIZE], a1
	LDF	[BO +  1 * SIZE], a2
	LDF	[BO +  2 * SIZE], a3
	LDF	[BO +  3 * SIZE], a4
#else
	LDF	[AO +  0 * SIZE], a1
	LDF	[AO +  1 * SIZE], a2
	LDF	[AO +  2 * SIZE], a3
	LDF	[AO +  3 * SIZE], a4
#endif

	FSUB	a1, c01, c01
	FSUB	a2, c02, c02
	FSUB	a3, c05, c05
	FSUB	a4, c06, c06

#if defined(LN) || defined(LT)
	LDF	[AO +  0 * SIZE], a1
	LDF	[AO +  1 * SIZE], a2

	FMUL	a1, c01, b1
	FMUL	a2, c01, b2
	FMUL	a1, c05, b3
	FMUL	a2, c05, b4

#ifndef CONJ
	FNMSUB	(aa2, cc02, bb1, cc01)
	FMADD	(aa1, cc02, bb2, cc02)
	FNMSUB	(aa2, cc06, bb3, cc05)
	FMADD	(aa1, cc06, bb4, cc06)
#else
	FMADD	(aa2, cc02, bb1, cc01)
	FMSUB	(aa1, cc02, bb2, cc02)
	FMADD	(aa2, cc06, bb3, cc05)
	FMSUB	(aa1, cc06, bb4, cc06)
#endif
#endif

#ifdef RN
	LDF	[BO +  0 * SIZE], b1
	LDF	[BO +  1 * SIZE], b2
	LDF	[BO +  2 * SIZE], b3
	LDF	[BO +  3 * SIZE], b4

	FMUL	b1, c01, a1
	FMUL	b2, c01, a2

#ifndef CONJ
	FNMSUB	(bb2, cc02, aa1, cc01)
	FMADD	(bb1, cc02, aa2, cc02)
#else
	FMADD	(bb2, cc02, aa1, cc01)
	FMSUB	(bb1, cc02, aa2, cc02)
#endif

	FNMSUB	(bb3, cc01, cc05, cc05)
	FNMSUB	(bb3, cc02, cc06, cc06)

#ifndef CONJ
	FMADD	(bb4, cc02, cc05, cc05)
	FNMSUB	(bb4, cc01, cc06, cc06)
#else
	FNMSUB	(bb4, cc02, cc05, cc05)
	FMADD	(bb4, cc01, cc06, cc06)
#endif

	LDF	[BO +  6 * SIZE], b1
	LDF	[BO +  7 * SIZE], b2

	FMUL	b1, c05, a1
	FMUL	b2, c05, a2

#ifndef CONJ
	FNMSUB	(bb2, cc06, aa1, cc05)
	FMADD	(bb1, cc06, aa2, cc06)
#else
	FMADD	(bb2, cc06, aa1, cc05)
	FMSUB	(bb1, cc06, aa2, cc06)
#endif
#endif

#ifdef RT
	LDF	[BO +  6 * SIZE], b1
	LDF	[BO +  7 * SIZE], b2
	LDF	[BO +  4 * SIZE], b3
	LDF	[BO +  5 * SIZE], b4

	FMUL	b1, c05, a1
	FMUL	b2, c05, a2

#ifndef CONJ
	FNMSUB	(bb2, cc06, aa1, cc05)
	FMADD	(bb1, cc06, aa2, cc06)
#else
	FMADD	(bb2, cc06, aa1, cc05)
	FMSUB	(bb1, cc06, aa2, cc06)
#endif

	FNMSUB	(bb3, cc05, cc01, cc01)
	FNMSUB	(bb3, cc06, cc02, cc02)

#ifndef CONJ
	FMADD	(bb4, cc06, cc01, cc01)
	FNMSUB	(bb4, cc05, cc02, cc02)
#else
	FNMSUB	(bb4, cc06, cc01, cc01)
	FMADD	(bb4, cc05, cc02, cc02)
#endif

	LDF	[BO +  0 * SIZE], b1
	LDF	[BO +  1 * SIZE], b2

	FMUL	b1, c01, a1
	FMUL	b2, c01, a2

#ifndef CONJ
	FNMSUB	(bb2, cc02, aa1, cc01)
	FMADD	(bb1, cc02, aa2, cc02)
#else
	FMADD	(bb2, cc02, aa1, cc01)
	FMSUB	(bb1, cc02, aa2, cc02)
#endif
#endif

#ifdef LN
	add	C1, -2 * SIZE, C1
	add	C2, -2 * SIZE, C2
#endif

#if defined(LN) || defined(LT)
	STF	c01, [BO +  0 * SIZE]
	STF	c02, [BO +  1 * SIZE]
	STF	c05, [BO +  2 * SIZE]
	STF	c06, [BO +  3 * SIZE]
#else
	STF	c01, [AO +  0 * SIZE]
	STF	c02, [AO +  1 * SIZE]
	STF	c05, [AO +  2 * SIZE]
	STF	c06, [AO +  3 * SIZE]
#endif

	STF	c01, [C1 + 0 * SIZE]
	STF	c02, [C1 + 1 * SIZE]
	STF	c05, [C2 + 0 * SIZE]
	STF	c06, [C2 + 1 * SIZE]

#ifndef LN
	add	C1, 2 * SIZE, C1
	add	C2, 2 * SIZE, C2
#endif

#ifdef RT
	sll	K, ZBASE_SHIFT, TEMP1
	add	AORIG, TEMP1, AORIG
#endif

#if defined(LT) || defined(RN)
	sub	K, KK, TEMP1
	sll	TEMP1, ZBASE_SHIFT + 0, TEMP2
	sll	TEMP1, ZBASE_SHIFT + 1, TEMP1
	add	AO, TEMP2, AO
	add	BO, TEMP1, BO
#endif

#ifdef LT
	add	KK, 1, KK
#endif

#ifdef LN
	sub	KK, 1, KK
#endif

	add	I, -1, I
	cmp	I, 0
	bg,pt	%icc, .LL22
	nop

#ifdef LN
	sll	K, ZBASE_SHIFT + 1, TEMP1
	add	B, TEMP1, B
#endif

#if defined(LT) || defined(RN)
	mov	BO, B
#endif

#ifdef RN
	add	KK, 2, KK
#endif

#ifdef RT
	sub	KK, 2, KK
#endif
	.align 4

.LL30:
	sra	N, 2, J
	cmp	J, 0
	ble,pn	%icc, .LL999
	nop
	.align 4

.LL11:
#ifdef RT
	sll	K, ZBASE_SHIFT + 2, TEMP1
	sub	B, TEMP1, B
#endif

#ifndef RT
	mov	C,  C1
	add	C,  LDC, C2
	add	C2, LDC, C3
	add	C3, LDC, C4
	add	C4, LDC, C
#else
	sub	C,  LDC, C4
	sub	C4, LDC, C3
	sub	C3, LDC, C2
	sub	C2, LDC, C1
	sub	C2, LDC, C
#endif

#ifdef LN
	add	M, OFFSET, KK
#endif

#ifdef LT
	mov	OFFSET, KK
#endif

#if defined(LN) || defined(RT)
	mov	A, AORIG
#else
	mov	A, AO
#endif

	mov	M, I
	.align 4

.LL12:
#if defined(LT) || defined(RN)
	mov	B, BO
#else
#ifdef LN
	sll	K,  ZBASE_SHIFT, TEMP1
	sub	AORIG, TEMP1, AORIG
#endif

	sll	KK, ZBASE_SHIFT + 0, TEMP1
	sll	KK, ZBASE_SHIFT + 2, TEMP2

	add	AORIG, TEMP1, AO
	add	B,     TEMP2, BO
#endif

	LDF	[AO +  0 * SIZE], a1
	FCLR	(cc01)
	LDF	[AO +  1 * SIZE], a2
	FCLR	(cc05)
	LDF	[AO +  8 * SIZE], a5
	FCLR	(cc09)
	LDF	[BO +  0 * SIZE], b1
	FCLR	(cc13)

	LDF	[BO +  1 * SIZE], b2
	FCLR	(cc02)
	LDF	[BO +  2 * SIZE], b3
	FCLR	(cc06)
	LDF	[BO +  3 * SIZE], b4
	FCLR	(cc10)
	LDF	[BO +  4 * SIZE], b5
	FCLR	(cc14)

	LDF	[BO +  5 * SIZE], b6
	FCLR	(cc03)
	LDF	[BO +  6 * SIZE], b7
	FCLR	(cc07)
	LDF	[BO +  7 * SIZE], b8
	FCLR	(cc11)
	LDF	[BO +  8 * SIZE], b9
	FCLR	(cc15)

	prefetch [C1 + 1 * SIZE], 3
	FCLR	(cc04)
	prefetch [C2 + 2 * SIZE], 3
	FCLR	(cc08)
	prefetch [C3 + 1 * SIZE], 3
	FCLR	(cc12)
	prefetch [C4 + 2 * SIZE], 3
	FCLR	(cc16)

#if defined(LT) || defined(RN)
	sra	KK, 3, L
#else
	sub	K, KK, L
	sra	L,  3, L
#endif
	cmp	L,  0
	ble,pn	%icc, .LL15
	nop
	.align 4

.LL13:
	FMADD1	(aa1, bb1, cc01, cc01)
	FMADD2	(aa2, bb1, cc02, cc02)
	FMADD3	(aa1, bb2, cc03, cc03)
	FMADD4	(aa2, bb2, cc04, cc04)

	FMADD1	(aa1, bb3, cc05, cc05)
	LDF	[BO + 16 * SIZE], b1
	FMADD2	(aa2, bb3, cc06, cc06)
	LDF	[BO +  9 * SIZE], b2

	FMADD3	(aa1, bb4, cc07, cc07)
	LDF	[BO + 10 * SIZE], b3
	FMADD4	(aa2, bb4, cc08, cc08)
	LDF	[BO + 11 * SIZE], b4

	FMADD1	(aa1, bb5, cc09, cc09)
	LDF	[AO +  2 * SIZE], a3
	FMADD2	(aa2, bb5, cc10, cc10)
	LDF	[AO +  3 * SIZE], a4

	FMADD3	(aa1, bb6, cc11, cc11)
	prefetch [AO + (APREFETCHSIZE +  0) * SIZE], APREFETCH_CATEGORY
	FMADD4	(aa2, bb6, cc12, cc12)
	nop

	FMADD1	(aa1, bb7, cc13, cc13)
	LDF	[BO + 12 * SIZE], b5
	FMADD2	(aa2, bb7, cc14, cc14)
	LDF	[BO + 13 * SIZE], b6

	FMADD3	(aa1, bb8, cc15, cc15)
	LDF	[BO + 14 * SIZE], b7
	FMADD4	(aa2, bb8, cc16, cc16)
	LDF	[BO + 15 * SIZE], b8

	FMADD1	(aa3, bb9, cc01, cc01)
	FMADD2	(aa4, bb9, cc02, cc02)
	FMADD3	(aa3, bb2, cc03, cc03)
	FMADD4	(aa4, bb2, cc04, cc04)

	FMADD1	(aa3, bb3, cc05, cc05)
	LDF	[BO + 24 * SIZE], b9
	FMADD2	(aa4, bb3, cc06, cc06)
	LDF	[BO + 17 * SIZE], b2

	FMADD3	(aa3, bb4, cc07, cc07)
	LDF	[BO + 18 * SIZE], b3
	FMADD4	(aa4, bb4, cc08, cc08)
	LDF	[BO + 19 * SIZE], b4

	FMADD1	(aa3, bb5, cc09, cc09)
	LDF	[AO +  4 * SIZE], a1
	FMADD2	(aa4, bb5, cc10, cc10)
	LDF	[AO +  5 * SIZE], a2

	FMADD3	(aa3, bb6, cc11, cc11)
	add	L, -1, L
	FMADD4	(aa4, bb6, cc12, cc12)
	nop

	FMADD1	(aa3, bb7, cc13, cc13)
	LDF	[BO + 20 * SIZE], b5
	FMADD2	(aa4, bb7, cc14, cc14)
	LDF	[BO + 21 * SIZE], b6

	FMADD3	(aa3, bb8, cc15, cc15)
	LDF	[BO + 22 * SIZE], b7
	FMADD4	(aa4, bb8, cc16, cc16)
	LDF	[BO + 23 * SIZE], b8

	FMADD1	(aa1, bb1, cc01, cc01)
	FMADD2	(aa2, bb1, cc02, cc02)
	FMADD3	(aa1, bb2, cc03, cc03)
	FMADD4	(aa2, bb2, cc04, cc04)

	FMADD1	(aa1, bb3, cc05, cc05)
	LDF	[BO + 32 * SIZE], b1
	FMADD2	(aa2, bb3, cc06, cc06)
	LDF	[BO + 25 * SIZE], b2

	FMADD3	(aa1, bb4, cc07, cc07)
	LDF	[BO + 26 * SIZE], b3
	FMADD4	(aa2, bb4, cc08, cc08)
	LDF	[BO + 27 * SIZE], b4

	FMADD1	(aa1, bb5, cc09, cc09)
	LDF	[AO +  6 * SIZE], a3
	FMADD2	(aa2, bb5, cc10, cc10)
	LDF	[AO +  7 * SIZE], a4

	FMADD3	(aa1, bb6, cc11, cc11)
	nop
	FMADD4	(aa2, bb6, cc12, cc12)
	nop

	FMADD1	(aa1, bb7, cc13, cc13)
	LDF	[BO + 28 * SIZE], b5
	FMADD2	(aa2, bb7, cc14, cc14)
	LDF	[BO + 29 * SIZE], b6

	FMADD3	(aa1, bb8, cc15, cc15)
	LDF	[BO + 30 * SIZE], b7
	FMADD4	(aa2, bb8, cc16, cc16)
	LDF	[BO + 31 * SIZE], b8

	FMADD1	(aa3, bb9, cc01, cc01)
	FMADD2	(aa4, bb9, cc02, cc02)
	FMADD3	(aa3, bb2, cc03, cc03)
	FMADD4	(aa4, bb2, cc04, cc04)

	FMADD1	(aa3, bb3, cc05, cc05)
	LDF	[BO + 40 * SIZE], b9
	FMADD2	(aa4, bb3, cc06, cc06)
	LDF	[BO + 33 * SIZE], b2

	FMADD3	(aa3, bb4, cc07, cc07)
	LDF	[BO + 34 * SIZE], b3
	FMADD4	(aa4, bb4, cc08, cc08)
	LDF	[BO + 35 * SIZE], b4

	FMADD1	(aa3, bb5, cc09, cc09)
	LDF	[AO + 16 * SIZE], a1  /****/
	FMADD2	(aa4, bb5, cc10, cc10)
	LDF	[AO +  9 * SIZE], a2

	FMADD3	(aa3, bb6, cc11, cc11)
	nop
	FMADD4	(aa4, bb6, cc12, cc12)
	nop

	FMADD1	(aa3, bb7, cc13, cc13)
	LDF	[BO + 36 * SIZE], b5
	FMADD2	(aa4, bb7, cc14, cc14)
	LDF	[BO + 37 * SIZE], b6

	FMADD3	(aa3, bb8, cc15, cc15)
	LDF	[BO + 38 * SIZE], b7
	FMADD4	(aa4, bb8, cc16, cc16)
	LDF	[BO + 39 * SIZE], b8

	FMADD1	(aa5, bb1, cc01, cc01)
	FMADD2	(aa2, bb1, cc02, cc02)
	FMADD3	(aa5, bb2, cc03, cc03)
	FMADD4	(aa2, bb2, cc04, cc04)

	FMADD1	(aa5, bb3, cc05, cc05)
	LDF	[BO + 48 * SIZE], b1
	FMADD2	(aa2, bb3, cc06, cc06)
	LDF	[BO + 41 * SIZE], b2

	FMADD3	(aa5, bb4, cc07, cc07)
	LDF	[BO + 42 * SIZE], b3
	FMADD4	(aa2, bb4, cc08, cc08)
	LDF	[BO + 43 * SIZE], b4

	FMADD1	(aa5, bb5, cc09, cc09)
	LDF	[AO + 10 * SIZE], a3
	FMADD2	(aa2, bb5, cc10, cc10)
	LDF	[AO + 11 * SIZE], a4

	FMADD3	(aa5, bb6, cc11, cc11)
	prefetch [AO + (APREFETCHSIZE +  8) * SIZE], APREFETCH_CATEGORY
	FMADD4	(aa2, bb6, cc12, cc12)
	nop

	FMADD1	(aa5, bb7, cc13, cc13)
	LDF	[BO + 44 * SIZE], b5
	FMADD2	(aa2, bb7, cc14, cc14)
	LDF	[BO + 45 * SIZE], b6

	FMADD3	(aa5, bb8, cc15, cc15)
	LDF	[BO + 46 * SIZE], b7
	FMADD4	(aa2, bb8, cc16, cc16)
	LDF	[BO + 47 * SIZE], b8

	FMADD1	(aa3, bb9, cc01, cc01)
	FMADD2	(aa4, bb9, cc02, cc02)
	FMADD3	(aa3, bb2, cc03, cc03)
	FMADD4	(aa4, bb2, cc04, cc04)

	FMADD1	(aa3, bb3, cc05, cc05)
	LDF	[BO + 56 * SIZE], b9
	FMADD2	(aa4, bb3, cc06, cc06)
	LDF	[BO + 49 * SIZE], b2

	FMADD3	(aa3, bb4, cc07, cc07)
	LDF	[BO + 50 * SIZE], b3
	FMADD4	(aa4, bb4, cc08, cc08)
	LDF	[BO + 51 * SIZE], b4

	FMADD1	(aa3, bb5, cc09, cc09)
	LDF	[AO + 12 * SIZE], a5
	FMADD2	(aa4, bb5, cc10, cc10)
	LDF	[AO + 13 * SIZE], a2

	FMADD3	(aa3, bb6, cc11, cc11)
	cmp	L, 0
	FMADD4	(aa4, bb6, cc12, cc12)
	nop

	FMADD1	(aa3, bb7, cc13, cc13)
	LDF	[BO + 52 * SIZE], b5
	FMADD2	(aa4, bb7, cc14, cc14)
	LDF	[BO + 53 * SIZE], b6

	FMADD3	(aa3, bb8, cc15, cc15)
	LDF	[BO + 54 * SIZE], b7
	FMADD4	(aa4, bb8, cc16, cc16)
	LDF	[BO + 55 * SIZE], b8

	FMADD1	(aa5, bb1, cc01, cc01)
	FMADD2	(aa2, bb1, cc02, cc02)
	FMADD3	(aa5, bb2, cc03, cc03)
	FMADD4	(aa2, bb2, cc04, cc04)

	FMADD1	(aa5, bb3, cc05, cc05)
	LDF	[BO + 64 * SIZE], b1
	FMADD2	(aa2, bb3, cc06, cc06)
	LDF	[BO + 57 * SIZE], b2

	FMADD3	(aa5, bb4, cc07, cc07)
	LDF	[BO + 58 * SIZE], b3
	FMADD4	(aa2, bb4, cc08, cc08)
	LDF	[BO + 59 * SIZE], b4

	FMADD1	(aa5, bb5, cc09, cc09)
	LDF	[AO + 14 * SIZE], a3
	FMADD2	(aa2, bb5, cc10, cc10)
	LDF	[AO + 15 * SIZE], a4

	FMADD3	(aa5, bb6, cc11, cc11)
	add	BO, 64 * SIZE, BO
	FMADD4	(aa2, bb6, cc12, cc12)
	add	AO, 16 * SIZE, AO

	FMADD1	(aa5, bb7, cc13, cc13)
	LDF	[BO -  4 * SIZE], b5
	FMADD2	(aa2, bb7, cc14, cc14)
	LDF	[BO -  3 * SIZE], b6

	FMADD3	(aa5, bb8, cc15, cc15)
	LDF	[BO -  2 * SIZE], b7
	FMADD4	(aa2, bb8, cc16, cc16)
	LDF	[BO -  1 * SIZE], b8

	FMADD1	(aa3, bb9, cc01, cc01)
	FMADD2	(aa4, bb9, cc02, cc02)
	FMADD3	(aa3, bb2, cc03, cc03)
	FMADD4	(aa4, bb2, cc04, cc04)

	FMADD1	(aa3, bb3, cc05, cc05)
	LDF	[BO +  8 * SIZE], b9
	FMADD2	(aa4, bb3, cc06, cc06)
	LDF	[BO +  1 * SIZE], b2

	FMADD3	(aa3, bb4, cc07, cc07)
	LDF	[BO +  2 * SIZE], b3
	FMADD4	(aa4, bb4, cc08, cc08)
	LDF	[BO +  3 * SIZE], b4

	FMADD1	(aa3, bb5, cc09, cc09)
	LDF	[AO +  8 * SIZE], a5  /****/
	FMADD2	(aa4, bb5, cc10, cc10)
	LDF	[AO +  1 * SIZE], a2

	FMADD3	(aa3, bb6, cc11, cc11)
	FMADD4	(aa4, bb6, cc12, cc12)

	FMADD1	(aa3, bb7, cc13, cc13)
	LDF	[BO +  4 * SIZE], b5
	FMADD2	(aa4, bb7, cc14, cc14)
	LDF	[BO +  5 * SIZE], b6

	FMADD3	(aa3, bb8, cc15, cc15)
	LDF	[BO +  6 * SIZE], b7
	FMADD4	(aa4, bb8, cc16, cc16)
	ble,pn	%icc, .LL15
	LDF	[BO +  7 * SIZE], b8

	FMADD1	(aa1, bb1, cc01, cc01)
	FMADD2	(aa2, bb1, cc02, cc02)
	FMADD3	(aa1, bb2, cc03, cc03)
	FMADD4	(aa2, bb2, cc04, cc04)

	FMADD1	(aa1, bb3, cc05, cc05)
	LDF	[BO + 16 * SIZE], b1
	FMADD2	(aa2, bb3, cc06, cc06)
	LDF	[BO +  9 * SIZE], b2

	FMADD3	(aa1, bb4, cc07, cc07)
	LDF	[BO + 10 * SIZE], b3
	FMADD4	(aa2, bb4, cc08, cc08)
	LDF	[BO + 11 * SIZE], b4

	FMADD1	(aa1, bb5, cc09, cc09)
	LDF	[AO +  2 * SIZE], a3
	FMADD2	(aa2, bb5, cc10, cc10)
	LDF	[AO +  3 * SIZE], a4

	FMADD3	(aa1, bb6, cc11, cc11)
	prefetch [AO + (APREFETCHSIZE +  0) * SIZE], APREFETCH_CATEGORY
	FMADD4	(aa2, bb6, cc12, cc12)
	nop

	FMADD1	(aa1, bb7, cc13, cc13)
	LDF	[BO + 12 * SIZE], b5
	FMADD2	(aa2, bb7, cc14, cc14)
	LDF	[BO + 13 * SIZE], b6

	FMADD3	(aa1, bb8, cc15, cc15)
	LDF	[BO + 14 * SIZE], b7
	FMADD4	(aa2, bb8, cc16, cc16)
	LDF	[BO + 15 * SIZE], b8

	FMADD1	(aa3, bb9, cc01, cc01)
	FMADD2	(aa4, bb9, cc02, cc02)
	FMADD3	(aa3, bb2, cc03, cc03)
	FMADD4	(aa4, bb2, cc04, cc04)

	FMADD1	(aa3, bb3, cc05, cc05)
	LDF	[BO + 24 * SIZE], b9
	FMADD2	(aa4, bb3, cc06, cc06)
	LDF	[BO + 17 * SIZE], b2

	FMADD3	(aa3, bb4, cc07, cc07)
	LDF	[BO + 18 * SIZE], b3
	FMADD4	(aa4, bb4, cc08, cc08)
	LDF	[BO + 19 * SIZE], b4

	FMADD1	(aa3, bb5, cc09, cc09)
	LDF	[AO +  4 * SIZE], a1
	FMADD2	(aa4, bb5, cc10, cc10)
	LDF	[AO +  5 * SIZE], a2

	FMADD3	(aa3, bb6, cc11, cc11)
	add	L, -1, L
	FMADD4	(aa4, bb6, cc12, cc12)
	nop

	FMADD1	(aa3, bb7, cc13, cc13)
	LDF	[BO + 20 * SIZE], b5
	FMADD2	(aa4, bb7, cc14, cc14)
	LDF	[BO + 21 * SIZE], b6

	FMADD3	(aa3, bb8, cc15, cc15)
	LDF	[BO + 22 * SIZE], b7
	FMADD4	(aa4, bb8, cc16, cc16)
	LDF	[BO + 23 * SIZE], b8

	FMADD1	(aa1, bb1, cc01, cc01)
	FMADD2	(aa2, bb1, cc02, cc02)
	FMADD3	(aa1, bb2, cc03, cc03)
	FMADD4	(aa2, bb2, cc04, cc04)

	FMADD1	(aa1, bb3, cc05, cc05)
	LDF	[BO + 32 * SIZE], b1
	FMADD2	(aa2, bb3, cc06, cc06)
	LDF	[BO + 25 * SIZE], b2

	FMADD3	(aa1, bb4, cc07, cc07)
	LDF	[BO + 26 * SIZE], b3
	FMADD4	(aa2, bb4, cc08, cc08)
	LDF	[BO + 27 * SIZE], b4

	FMADD1	(aa1, bb5, cc09, cc09)
	LDF	[AO +  6 * SIZE], a3
	FMADD2	(aa2, bb5, cc10, cc10)
	LDF	[AO +  7 * SIZE], a4

	FMADD3	(aa1, bb6, cc11, cc11)
	nop
	FMADD4	(aa2, bb6, cc12, cc12)
	nop

	FMADD1	(aa1, bb7, cc13, cc13)
	LDF	[BO + 28 * SIZE], b5
	FMADD2	(aa2, bb7, cc14, cc14)
	LDF	[BO + 29 * SIZE], b6

	FMADD3	(aa1, bb8, cc15, cc15)
	LDF	[BO + 30 * SIZE], b7
	FMADD4	(aa2, bb8, cc16, cc16)
	LDF	[BO + 31 * SIZE], b8

	FMADD1	(aa3, bb9, cc01, cc01)
	FMADD2	(aa4, bb9, cc02, cc02)
	FMADD3	(aa3, bb2, cc03, cc03)
	FMADD4	(aa4, bb2, cc04, cc04)

	FMADD1	(aa3, bb3, cc05, cc05)
	LDF	[BO + 40 * SIZE], b9
	FMADD2	(aa4, bb3, cc06, cc06)
	LDF	[BO + 33 * SIZE], b2

	FMADD3	(aa3, bb4, cc07, cc07)
	LDF	[BO + 34 * SIZE], b3
	FMADD4	(aa4, bb4, cc08, cc08)
	LDF	[BO + 35 * SIZE], b4

	FMADD1	(aa3, bb5, cc09, cc09)
	LDF	[AO + 16 * SIZE], a1  /****/
	FMADD2	(aa4, bb5, cc10, cc10)
	LDF	[AO +  9 * SIZE], a2

	FMADD3	(aa3, bb6, cc11, cc11)
	nop
	FMADD4	(aa4, bb6, cc12, cc12)
	nop

	FMADD1	(aa3, bb7, cc13, cc13)
	LDF	[BO + 36 * SIZE], b5
	FMADD2	(aa4, bb7, cc14, cc14)
	LDF	[BO + 37 * SIZE], b6

	FMADD3	(aa3, bb8, cc15, cc15)
	LDF	[BO + 38 * SIZE], b7
	FMADD4	(aa4, bb8, cc16, cc16)
	LDF	[BO + 39 * SIZE], b8

	FMADD1	(aa5, bb1, cc01, cc01)
	FMADD2	(aa2, bb1, cc02, cc02)
	FMADD3	(aa5, bb2, cc03, cc03)
	FMADD4	(aa2, bb2, cc04, cc04)

	FMADD1	(aa5, bb3, cc05, cc05)
	LDF	[BO + 48 * SIZE], b1
	FMADD2	(aa2, bb3, cc06, cc06)
	LDF	[BO + 41 * SIZE], b2

	FMADD3	(aa5, bb4, cc07, cc07)
	LDF	[BO + 42 * SIZE], b3
	FMADD4	(aa2, bb4, cc08, cc08)
	LDF	[BO + 43 * SIZE], b4

	FMADD1	(aa5, bb5, cc09, cc09)
	LDF	[AO + 10 * SIZE], a3
	FMADD2	(aa2, bb5, cc10, cc10)
	LDF	[AO + 11 * SIZE], a4

	FMADD3	(aa5, bb6, cc11, cc11)
	prefetch [AO + (APREFETCHSIZE +  8) * SIZE], APREFETCH_CATEGORY
	FMADD4	(aa2, bb6, cc12, cc12)
	nop

	FMADD1	(aa5, bb7, cc13, cc13)
	LDF	[BO + 44 * SIZE], b5
	FMADD2	(aa2, bb7, cc14, cc14)
	LDF	[BO + 45 * SIZE], b6

	FMADD3	(aa5, bb8, cc15, cc15)
	LDF	[BO + 46 * SIZE], b7
	FMADD4	(aa2, bb8, cc16, cc16)
	LDF	[BO + 47 * SIZE], b8

	FMADD1	(aa3, bb9, cc01, cc01)
	FMADD2	(aa4, bb9, cc02, cc02)
	FMADD3	(aa3, bb2, cc03, cc03)
	FMADD4	(aa4, bb2, cc04, cc04)

	FMADD1	(aa3, bb3, cc05, cc05)
	LDF	[BO + 56 * SIZE], b9
	FMADD2	(aa4, bb3, cc06, cc06)
	LDF	[BO + 49 * SIZE], b2

	FMADD3	(aa3, bb4, cc07, cc07)
	LDF	[BO + 50 * SIZE], b3
	FMADD4	(aa4, bb4, cc08, cc08)
	LDF	[BO + 51 * SIZE], b4

	FMADD1	(aa3, bb5, cc09, cc09)
	LDF	[AO + 12 * SIZE], a5
	FMADD2	(aa4, bb5, cc10, cc10)
	LDF	[AO + 13 * SIZE], a2

	FMADD3	(aa3, bb6, cc11, cc11)
	cmp	L, 0
	FMADD4	(aa4, bb6, cc12, cc12)
	nop

	FMADD1	(aa3, bb7, cc13, cc13)
	LDF	[BO + 52 * SIZE], b5
	FMADD2	(aa4, bb7, cc14, cc14)
	LDF	[BO + 53 * SIZE], b6

	FMADD3	(aa3, bb8, cc15, cc15)
	LDF	[BO + 54 * SIZE], b7
	FMADD4	(aa4, bb8, cc16, cc16)
	LDF	[BO + 55 * SIZE], b8

	FMADD1	(aa5, bb1, cc01, cc01)
	FMADD2	(aa2, bb1, cc02, cc02)
	FMADD3	(aa5, bb2, cc03, cc03)
	FMADD4	(aa2, bb2, cc04, cc04)

	FMADD1	(aa5, bb3, cc05, cc05)
	LDF	[BO + 64 * SIZE], b1
	FMADD2	(aa2, bb3, cc06, cc06)
	LDF	[BO + 57 * SIZE], b2

	FMADD3	(aa5, bb4, cc07, cc07)
	LDF	[BO + 58 * SIZE], b3
	FMADD4	(aa2, bb4, cc08, cc08)
	LDF	[BO + 59 * SIZE], b4

	FMADD1	(aa5, bb5, cc09, cc09)
	LDF	[AO + 14 * SIZE], a3
	FMADD2	(aa2, bb5, cc10, cc10)
	LDF	[AO + 15 * SIZE], a4

	FMADD3	(aa5, bb6, cc11, cc11)
	add	BO, 64 * SIZE, BO
	FMADD4	(aa2, bb6, cc12, cc12)
	add	AO, 16 * SIZE, AO

	FMADD1	(aa5, bb7, cc13, cc13)
	LDF	[BO -  4 * SIZE], b5
	FMADD2	(aa2, bb7, cc14, cc14)
	LDF	[BO -  3 * SIZE], b6

	FMADD3	(aa5, bb8, cc15, cc15)
	LDF	[BO -  2 * SIZE], b7
	FMADD4	(aa2, bb8, cc16, cc16)
	LDF	[BO -  1 * SIZE], b8

	FMADD1	(aa3, bb9, cc01, cc01)
	FMADD2	(aa4, bb9, cc02, cc02)
	FMADD3	(aa3, bb2, cc03, cc03)
	FMADD4	(aa4, bb2, cc04, cc04)

	FMADD1	(aa3, bb3, cc05, cc05)
	LDF	[BO +  8 * SIZE], b9
	FMADD2	(aa4, bb3, cc06, cc06)
	LDF	[BO +  1 * SIZE], b2

	FMADD3	(aa3, bb4, cc07, cc07)
	LDF	[BO +  2 * SIZE], b3
	FMADD4	(aa4, bb4, cc08, cc08)
	LDF	[BO +  3 * SIZE], b4

	FMADD1	(aa3, bb5, cc09, cc09)
	LDF	[AO +  8 * SIZE], a5  /****/
	FMADD2	(aa4, bb5, cc10, cc10)
	LDF	[AO +  1 * SIZE], a2

	FMADD3	(aa3, bb6, cc11, cc11)
	FMADD4	(aa4, bb6, cc12, cc12)

	FMADD1	(aa3, bb7, cc13, cc13)
	LDF	[BO +  4 * SIZE], b5
	FMADD2	(aa4, bb7, cc14, cc14)
	LDF	[BO +  5 * SIZE], b6

	FMADD3	(aa3, bb8, cc15, cc15)
	LDF	[BO +  6 * SIZE], b7
	FMADD4	(aa4, bb8, cc16, cc16)
	bg,pt	%icc, .LL13
	LDF	[BO +  7 * SIZE], b8
	.align 4

.LL15:
#if defined(LT) || defined(RN)
	and	KK, 7, L
#else
	sub	K, KK, L
	and	L,  7, L
#endif
	cmp	L,  0
	ble,a,pn %icc, .LL18
	nop
	.align 4

.LL17:
	FMADD1	(aa1, bb1, cc01, cc01)
	add	L, -1, L
	FMADD2	(aa2, bb1, cc02, cc02)
	nop

	FMADD3	(aa1, bb2, cc03, cc03)
	LDF	[BO +  8 * SIZE], b1
	FMADD4	(aa2, bb2, cc04, cc04)
	LDF	[BO +  9 * SIZE], b2

	FMADD1	(aa1, bb3, cc05, cc05)
	cmp	L, 0
	FMADD2	(aa2, bb3, cc06, cc06)
	nop

	FMADD3	(aa1, bb4, cc07, cc07)
	LDF	[BO + 10 * SIZE], b3
	FMADD4	(aa2, bb4, cc08, cc08)
	LDF	[BO + 11 * SIZE], b4

	FMADD1	(aa1, bb5, cc09, cc09)
	nop
	FMADD2	(aa2, bb5, cc10, cc10)
	nop

	FMADD3	(aa1, bb6, cc11, cc11)
	LDF	[BO + 12 * SIZE], b5
	FMADD4	(aa2, bb6, cc12, cc12)
	LDF	[BO + 13 * SIZE], b6

	FMADD1	(aa1, bb7, cc13, cc13)
	add	AO, 2 * SIZE, AO
	FMADD2	(aa2, bb7, cc14, cc14)
	add	BO, 8 * SIZE, BO

	FMADD3	(aa1, bb8, cc15, cc15)
	LDF	[AO +  0 * SIZE], a1
	FMADD4	(aa2, bb8, cc16, cc16)
	LDF	[AO +  1 * SIZE], a2

	LDF	[BO +  6 * SIZE], b7
	bg,pt	%icc, .LL17
	LDF	[BO +  7 * SIZE], b8
	nop
	.align 4

.LL18:
	FADD	c01, c04, c01
	FADD	c02, c03, c02
	FADD	c05, c08, c05
	FADD	c06, c07, c06

	FADD	c09, c12, c09
	FADD	c10, c11, c10
	FADD	c13, c16, c13
	FADD	c14, c15, c14

#if defined(LN) || defined(RT)
#ifdef LN
	sub	KK, 1, TEMP1
#else
	sub	KK, 4, TEMP1
#endif
	sll	TEMP1, ZBASE_SHIFT + 0, TEMP2
	sll	TEMP1, ZBASE_SHIFT + 2, TEMP1

	add	AORIG, TEMP2, AO
	add	B,     TEMP1, BO
#endif

#if defined(LN) || defined(LT)
	LDF	[BO +  0 * SIZE], a1
	LDF	[BO +  1 * SIZE], a2
	LDF	[BO +  2 * SIZE], a3
	LDF	[BO +  3 * SIZE], a4

	LDF	[BO +  4 * SIZE], b1
	LDF	[BO +  5 * SIZE], b2
	LDF	[BO +  6 * SIZE], b3
	LDF	[BO +  7 * SIZE], b4
#else
	LDF	[AO +  0 * SIZE], a1
	LDF	[AO +  1 * SIZE], a2
	LDF	[AO +  2 * SIZE], a3
	LDF	[AO +  3 * SIZE], a4

	LDF	[AO +  4 * SIZE], b1
	LDF	[AO +  5 * SIZE], b2
	LDF	[AO +  6 * SIZE], b3
	LDF	[AO +  7 * SIZE], b4
#endif

	FSUB	a1, c01, c01
	FSUB	a2, c02, c02
	FSUB	a3, c05, c05
	FSUB	a4, c06, c06

	FSUB	b1, c09, c09
	FSUB	b2, c10, c10
	FSUB	b3, c13, c13
	FSUB	b4, c14, c14

#if defined(LN) || defined(LT)
	LDF	[AO +  0 * SIZE], a1
	LDF	[AO +  1 * SIZE], a2

	FMUL	a1, c01, b1
	FMUL	a2, c01, b2
	FMUL	a1, c05, b3
	FMUL	a2, c05, b4
	FMUL	a1, c09, b5
	FMUL	a2, c09, b6
	FMUL	a1, c13, b7
	FMUL	a2, c13, b8

#ifndef CONJ
	FNMSUB	(aa2, cc02, bb1, cc01)
	FMADD	(aa1, cc02, bb2, cc02)
	FNMSUB	(aa2, cc06, bb3, cc05)
	FMADD	(aa1, cc06, bb4, cc06)
	FNMSUB	(aa2, cc10, bb5, cc09)
	FMADD	(aa1, cc10, bb6, cc10)
	FNMSUB	(aa2, cc14, bb7, cc13)
	FMADD	(aa1, cc14, bb8, cc14)
#else
	FMADD	(aa2, cc02, bb1, cc01)
	FMSUB	(aa1, cc02, bb2, cc02)
	FMADD	(aa2, cc06, bb3, cc05)
	FMSUB	(aa1, cc06, bb4, cc06)
	FMADD	(aa2, cc10, bb5, cc09)
	FMSUB	(aa1, cc10, bb6, cc10)
	FMADD	(aa2, cc14, bb7, cc13)
	FMSUB	(aa1, cc14, bb8, cc14)
#endif
#endif

#ifdef RN
	LDF	[BO +  0 * SIZE], b1
	LDF	[BO +  1 * SIZE], b2
	LDF	[BO +  2 * SIZE], b3
	LDF	[BO +  3 * SIZE], b4
	LDF	[BO +  4 * SIZE], b5
	LDF	[BO +  5 * SIZE], b6
	LDF	[BO +  6 * SIZE], b7
	LDF	[BO +  7 * SIZE], b8

	FMUL	b1, c01, a1
	FMUL	b2, c01, a2

#ifndef CONJ
	FNMSUB	(bb2, cc02, aa1, cc01)
	FMADD	(bb1, cc02, aa2, cc02)
#else
	FMADD	(bb2, cc02, aa1, cc01)
	FMSUB	(bb1, cc02, aa2, cc02)
#endif

	FNMSUB	(bb3, cc01, cc05, cc05)
	FNMSUB	(bb3, cc02, cc06, cc06)
	FNMSUB	(bb5, cc01, cc09, cc09)
	FNMSUB	(bb5, cc02, cc10, cc10)
	FNMSUB	(bb7, cc01, cc13, cc13)
	FNMSUB	(bb7, cc02, cc14, cc14)

#ifndef CONJ
	FMADD	(bb4, cc02, cc05, cc05)
	FNMSUB	(bb4, cc01, cc06, cc06)
	FMADD	(bb6, cc02, cc09, cc09)
	FNMSUB	(bb6, cc01, cc10, cc10)
	FMADD	(bb8, cc02, cc13, cc13)
	FNMSUB	(bb8, cc01, cc14, cc14)
#else
	FNMSUB	(bb4, cc02, cc05, cc05)
	FMADD	(bb4, cc01, cc06, cc06)
	FNMSUB	(bb6, cc02, cc09, cc09)
	FMADD	(bb6, cc01, cc10, cc10)
	FNMSUB	(bb8, cc02, cc13, cc13)
	FMADD	(bb8, cc01, cc14, cc14)
#endif

	LDF	[BO + 10 * SIZE], b1
	LDF	[BO + 11 * SIZE], b2
	LDF	[BO + 12 * SIZE], b3
	LDF	[BO + 13 * SIZE], b4
	LDF	[BO + 14 * SIZE], b5
	LDF	[BO + 15 * SIZE], b6

	FMUL	b1, c05, a1
	FMUL	b2, c05, a2

#ifndef CONJ
	FNMSUB	(bb2, cc06, aa1, cc05)
	FMADD	(bb1, cc06, aa2, cc06)
#else
	FMADD	(bb2, cc06, aa1, cc05)
	FMSUB	(bb1, cc06, aa2, cc06)
#endif

	FNMSUB	(bb3, cc05, cc09, cc09)
	FNMSUB	(bb3, cc06, cc10, cc10)
	FNMSUB	(bb5, cc05, cc13, cc13)
	FNMSUB	(bb5, cc06, cc14, cc14)

#ifndef CONJ
	FMADD	(bb4, cc06, cc09, cc09)
	FNMSUB	(bb4, cc05, cc10, cc10)
	FMADD	(bb6, cc06, cc13, cc13)
	FNMSUB	(bb6, cc05, cc14, cc14)
#else
	FNMSUB	(bb4, cc06, cc09, cc09)
	FMADD	(bb4, cc05, cc10, cc10)
	FNMSUB	(bb6, cc06, cc13, cc13)
	FMADD	(bb6, cc05, cc14, cc14)
#endif

	LDF	[BO + 20 * SIZE], b1
	LDF	[BO + 21 * SIZE], b2
	LDF	[BO + 22 * SIZE], b3
	LDF	[BO + 23 * SIZE], b4

	FMUL	b1, c09, a1
	FMUL	b2, c09, a2

#ifndef CONJ
	FNMSUB	(bb2, cc10, aa1, cc09)
	FMADD	(bb1, cc10, aa2, cc10)
#else
	FMADD	(bb2, cc10, aa1, cc09)
	FMSUB	(bb1, cc10, aa2, cc10)
#endif

	FNMSUB	(bb3, cc09, cc13, cc13)
	FNMSUB	(bb3, cc10, cc14, cc14)

#ifndef CONJ
	FMADD	(bb4, cc10, cc13, cc13)
	FNMSUB	(bb4, cc09, cc14, cc14)
#else
	FNMSUB	(bb4, cc10, cc13, cc13)
	FMADD	(bb4, cc09, cc14, cc14)
#endif

	LDF	[BO + 30 * SIZE], b1
	LDF	[BO + 31 * SIZE], b2

	FMUL	b1, c13, a1
	FMUL	b2, c13, a2

#ifndef CONJ
	FNMSUB	(bb2, cc14, aa1, cc13)
	FMADD	(bb1, cc14, aa2, cc14)
#else
	FMADD	(bb2, cc14, aa1, cc13)
	FMSUB	(bb1, cc14, aa2, cc14)
#endif
#endif

#ifdef RT
	LDF	[BO + 30 * SIZE], b1
	LDF	[BO + 31 * SIZE], b2
	LDF	[BO + 28 * SIZE], b3
	LDF	[BO + 29 * SIZE], b4
	LDF	[BO + 26 * SIZE], b5
	LDF	[BO + 27 * SIZE], b6
	LDF	[BO + 24 * SIZE], b7
	LDF	[BO + 25 * SIZE], b8

	FMUL	b1, c13, a1
	FMUL	b2, c13, a2

#ifndef CONJ
	FNMSUB	(bb2, cc14, aa1, cc13)
	FMADD	(bb1, cc14, aa2, cc14)
#else
	FMADD	(bb2, cc14, aa1, cc13)
	FMSUB	(bb1, cc14, aa2, cc14)
#endif

	FNMSUB	(bb3, cc13, cc09, cc09)
	FNMSUB	(bb3, cc14, cc10, cc10)
	FNMSUB	(bb5, cc13, cc05, cc05)
	FNMSUB	(bb5, cc14, cc06, cc06)
	FNMSUB	(bb7, cc13, cc01, cc01)
	FNMSUB	(bb7, cc14, cc02, cc02)

#ifndef CONJ
	FMADD	(bb4, cc14, cc09, cc09)
	FNMSUB	(bb4, cc13, cc10, cc10)
	FMADD	(bb6, cc14, cc05, cc05)
	FNMSUB	(bb6, cc13, cc06, cc06)
	FMADD	(bb8, cc14, cc01, cc01)
	FNMSUB	(bb8, cc13, cc02, cc02)
#else
	FNMSUB	(bb4, cc14, cc09, cc09)
	FMADD	(bb4, cc13, cc10, cc10)
	FNMSUB	(bb6, cc14, cc05, cc05)
	FMADD	(bb6, cc13, cc06, cc06)
	FNMSUB	(bb8, cc14, cc01, cc01)
	FMADD	(bb8, cc13, cc02, cc02)
#endif

	LDF	[BO + 20 * SIZE], b1
	LDF	[BO + 21 * SIZE], b2
	LDF	[BO + 18 * SIZE], b3
	LDF	[BO + 19 * SIZE], b4
	LDF	[BO + 16 * SIZE], b5
	LDF	[BO + 17 * SIZE], b6

	FMUL	b1, c09, a1
	FMUL	b2, c09, a2

#ifndef CONJ
	FNMSUB	(bb2, cc10, aa1, cc09)
	FMADD	(bb1, cc10, aa2, cc10)
#else
	FMADD	(bb2, cc10, aa1, cc09)
	FMSUB	(bb1, cc10, aa2, cc10)
#endif

	FNMSUB	(bb3, cc09, cc05, cc05)
	FNMSUB	(bb3, cc10, cc06, cc06)
	FNMSUB	(bb5, cc09, cc01, cc01)
	FNMSUB	(bb5, cc10, cc02, cc02)

#ifndef CONJ
	FMADD	(bb4, cc10, cc05, cc05)
	FNMSUB	(bb4, cc09, cc06, cc06)
	FMADD	(bb6, cc10, cc01, cc01)
	FNMSUB	(bb6, cc09, cc02, cc02)
#else
	FNMSUB	(bb4, cc10, cc05, cc05)
	FMADD	(bb4, cc09, cc06, cc06)
	FNMSUB	(bb6, cc10, cc01, cc01)
	FMADD	(bb6, cc09, cc02, cc02)
#endif

	LDF	[BO + 10 * SIZE], b1
	LDF	[BO + 11 * SIZE], b2
	LDF	[BO +  8 * SIZE], b3
	LDF	[BO +  9 * SIZE], b4

	FMUL	b1, c05, a1
	FMUL	b2, c05, a2

#ifndef CONJ
	FNMSUB	(bb2, cc06, aa1, cc05)
	FMADD	(bb1, cc06, aa2, cc06)
#else
	FMADD	(bb2, cc06, aa1, cc05)
	FMSUB	(bb1, cc06, aa2, cc06)
#endif

	FNMSUB	(bb3, cc05, cc01, cc01)
	FNMSUB	(bb3, cc06, cc02, cc02)

#ifndef CONJ
	FMADD	(bb4, cc06, cc01, cc01)
	FNMSUB	(bb4, cc05, cc02, cc02)
#else
	FNMSUB	(bb4, cc06, cc01, cc01)
	FMADD	(bb4, cc05, cc02, cc02)
#endif

	LDF	[BO +  0 * SIZE], b1
	LDF	[BO +  1 * SIZE], b2

	FMUL	b1, c01, a1
	FMUL	b2, c01, a2

#ifndef CONJ
	FNMSUB	(bb2, cc02, aa1, cc01)
	FMADD	(bb1, cc02, aa2, cc02)
#else
	FMADD	(bb2, cc02, aa1, cc01)
	FMSUB	(bb1, cc02, aa2, cc02)
#endif
#endif

#ifdef LN
	add	C1, -2 * SIZE, C1
	add	C2, -2 * SIZE, C2
	add	C3, -2 * SIZE, C3
	add	C4, -2 * SIZE, C4
#endif

#if defined(LN) || defined(LT)
	STF	c01, [BO +  0 * SIZE]
	STF	c02, [BO +  1 * SIZE]
	STF	c05, [BO +  2 * SIZE]
	STF	c06, [BO +  3 * SIZE]

	STF	c09, [BO +  4 * SIZE]
	STF	c10, [BO +  5 * SIZE]
	STF	c13, [BO +  6 * SIZE]
	STF	c14, [BO +  7 * SIZE]
#else
	STF	c01, [AO +  0 * SIZE]
	STF	c02, [AO +  1 * SIZE]
	STF	c05, [AO +  2 * SIZE]
	STF	c06, [AO +  3 * SIZE]

	STF	c09, [AO +  4 * SIZE]
	STF	c10, [AO +  5 * SIZE]
	STF	c13, [AO +  6 * SIZE]
	STF	c14, [AO +  7 * SIZE]
#endif

	STF	c01, [C1 + 0 * SIZE]
	STF	c02, [C1 + 1 * SIZE]
	STF	c05, [C2 + 0 * SIZE]
	STF	c06, [C2 + 1 * SIZE]

	STF	c09, [C3 + 0 * SIZE]
	STF	c10, [C3 + 1 * SIZE]
	STF	c13, [C4 + 0 * SIZE]
	STF	c14, [C4 + 1 * SIZE]

#ifndef LN
	add	C1, 2 * SIZE, C1
	add	C2, 2 * SIZE, C2
	add	C3, 2 * SIZE, C3
	add	C4, 2 * SIZE, C4
#endif

#ifdef RT
	sll	K, ZBASE_SHIFT, TEMP1
	add	AORIG, TEMP1, AORIG
#endif

#if defined(LT) || defined(RN)
	sub	K, KK, TEMP1
	sll	TEMP1, ZBASE_SHIFT + 0, TEMP2
	sll	TEMP1, ZBASE_SHIFT + 2, TEMP1
	add	AO, TEMP2, AO
	add	BO, TEMP1, BO
#endif

#ifdef LT
	add	KK, 1, KK
#endif

#ifdef LN
	sub	KK, 1, KK
#endif

	add	I, -1, I
	cmp	I, 0
	bg,pt	%icc, .LL12
	nop

#ifdef LN
	sll	K, ZBASE_SHIFT + 2, TEMP1
	add	B, TEMP1, B
#endif

#if defined(LT) || defined(RN)
	mov	BO, B
#endif

#ifdef RN
	add	KK, 4, KK
#endif

#ifdef RT
	sub	KK, 4, KK
#endif

	add	J, -1, J
	cmp	J, 0
	bg,pt	%icc, .LL11
	nop
	.align 4

.LL999:
	return	%i7 + 8
	clr	%o0

	EPILOGUE