poly1305-ia64.S
// ====================================================================
// Written by Andy Polyakov, @dot-asm, initially for use in the OpenSSL
// project.
// ====================================================================
//
// Poly1305 for Itanium.
//
// January 2019
//
// Performance was reported to be ~2.1 cycles per byte on Itanium 2.
// With exception for processors in 95xx family, which have higher
// floating-point instructions' latencies and deliver ~2.6 cpb.
// Comparison to compiler-generated code is not exactly fair, because
// of different radixes. But just for reference, it was observed to be
// >3x faster. Originally it was argued that floating-point base 2^32
// implementation would be optimal. Upon closer look estimate for below
// integer base 2^64 implementation turned to be approximately same on
// Itanium 2. But floating-point code would be larger, and have higher
// overhead, which would negatively affect small-block performance...
#if defined(_HPUX_SOURCE)
# if !defined(_LP64)
# define ADDP addp4
# else
# define ADDP add
# endif
# define RUM rum
# define SUM sum
#else
# define ADDP add
# define RUM nop
# define SUM nop
#endif
.text
.explicit
.global poly1305_init#
.proc poly1305_init#
.align 64
poly1305_init:
.prologue
.save ar.pfs,r2
{ .mmi; alloc r2=ar.pfs,2,0,0,0
cmp.eq p6,p7=0,r33 } // key == NULL?
{ .mmi; ADDP r9=8,r32
ADDP r10=16,r32
ADDP r32=0,r32 };;
.body
{ .mmi; st8 [r32]=r0,24 // ctx->h0 = 0
st8 [r9]=r0 // ctx->h1 = 0
(p7) ADDP r8=0,r33 }
{ .mib; st8 [r10]=r0 // ctx->h2 = 0
(p6) mov r8=0
(p6) br.ret.spnt b0 };;
{ .mmi; ADDP r9=1,r33
ADDP r10=2,r33
ADDP r11=3,r33 };;
{ .mmi; ld1 r16=[r8],4 // load key, little-endian
ld1 r17=[r9],4 }
{ .mmi; ld1 r18=[r10],4
ld1 r19=[r11],4 };;
{ .mmi; ld1 r20=[r8],4
ld1 r21=[r9],4 }
{ .mmi; ld1 r22=[r10],4
ld1 r23=[r11],4
and r19=15,r19 };;
{ .mmi; ld1 r24=[r8],4
ld1 r25=[r9],4
and r20=-4,r20 }
{ .mmi; ld1 r26=[r10],4
ld1 r27=[r11],4
and r23=15,r23 };;
{ .mmi; ld1 r28=[r8],4
ld1 r29=[r9],4
and r24=-4,r24 }
{ .mmi; ld1 r30=[r10],4
ld1 r31=[r11],4
and r27=15,r27 };;
{ .mii; and r28=-4,r28
dep r16=r17,r16,8,8
dep r18=r19,r18,8,8 };;
{ .mii; and r31=15,r31
dep r16=r18,r16,16,16
dep r20=r21,r20,8,8 };;
{ .mii; dep r16=r20,r16,32,16
dep r22=r23,r22,8,8 };;
{ .mii; dep r16=r22,r16,48,16
dep r24=r25,r24,8,8 };;
{ .mii; dep r26=r27,r26,8,8
dep r28=r29,r28,8,8 };;
{ .mii; dep r24=r26,r24,16,16
dep r30=r31,r30,8,8 };;
{ .mii; st8 [r32]=r16,8 // ctx->r0
dep r24=r28,r24,32,16;;
dep r24=r30,r24,48,16 };;
{ .mii; st8 [r32]=r24,8 // ctx->r1
shr.u r25=r24,2;;
add r25=r25,r24 };;
{ .mib; st8 [r32]=r25 // ctx->s1
mov r8=0
br.ret.sptk b0 };;
.endp poly1305_init#
h0=r17; h1=r18; h2=r19;
i0=r20; i1=r21;
HF0=f8; HF1=f9; HF2=f10;
RF0=f11; RF1=f12; SF1=f13;
.global poly1305_blocks#
.proc poly1305_blocks#
.align 64
poly1305_blocks:
.prologue
.save ar.pfs,r2
{ .mii; alloc r2=ar.pfs,4,1,0,0
.save ar.lc,r3
mov r3=ar.lc
.save pr,r36
mov r36=pr }
.body
{ .mmi; ADDP r8=0,r32
ADDP r9=8,r32
and r29=7,r33 };;
{ .mmi; ld8 h0=[r8],16
ld8 h1=[r9],16
and r33=-8,r33 };;
{ .mmi; ld8 h2=[r8],16
ldf8 RF0=[r9],16
shr.u r34=r34,4 };;
{ .mmi; ldf8 RF1=[r8],-32
ldf8 SF1=[r9],-32
cmp.ltu p16,p17=1,r34 };;
{ .mmi;
(p16) add r34=-2,r34
(p17) mov r34=0
ADDP r10=0,r33 }
{ .mii; ADDP r11=8,r33
(p16) mov ar.ec=2
(p17) mov ar.ec=1 };;
{ .mib; RUM 1<<1 // go little-endian
mov ar.lc=r34
brp.loop.imp .Loop,.Lcend-16 }
{ .mmi; cmp.eq p8,p7=0,r29
cmp.eq p9,p0=1,r29
cmp.eq p10,p0=2,r29 }
{ .mmi; cmp.eq p11,p0=3,r29
cmp.eq p12,p0=4,r29
cmp.eq p13,p0=5,r29 }
{ .mmi; cmp.eq p14,p0=6,r29
cmp.eq p15,p0=7,r29
add r16=16,r10 };;
{ .mmb;
(p8) ld8 i0=[r10],16 // aligned input
(p8) ld8 i1=[r11],16
(p8) br.cond.sptk .Loop };;
// align first block
.pred.rel "mutex",p8,p9,p10,p11,p12,p13,p14,p15
{ .mmi; (p7) ld8 r14=[r10],24
(p7) ld8 r15=[r11],24 }
{ .mii; (p7) ld8 r16=[r16]
nop.i 0;;
(p15) shrp i0=r15,r14,56 }
{ .mii; (p15) shrp i1=r16,r15,56
(p14) shrp i0=r15,r14,48 }
{ .mii; (p14) shrp i1=r16,r15,48
(p13) shrp i0=r15,r14,40 }
{ .mii; (p13) shrp i1=r16,r15,40
(p12) shrp i0=r15,r14,32 }
{ .mii; (p12) shrp i1=r16,r15,32
(p11) shrp i0=r15,r14,24 }
{ .mii; (p11) shrp i1=r16,r15,24
(p10) shrp i0=r15,r14,16 }
{ .mii; (p10) shrp i1=r16,r15,16
(p9) shrp i0=r15,r14,8 }
{ .mii; (p9) shrp i1=r16,r15,8
mov r14=r16 };;
.Loop:
.pred.rel "mutex",p8,p9,p10,p11,p12,p13,p14,p15
{ .mmi; add h0=h0,i0
add h1=h1,i1
add h2=h2,r35 };;
{ .mmi; setf.sig HF0=h0
cmp.ltu p6,p0=h0,i0
cmp.ltu p7,p0=h1,i1 };;
{ .mmi; (p6) add h1=1,h1;;
setf.sig HF1=h1
(p6) cmp.eq.or p7,p0=0,h1 };;
{ .mmi; (p7) add h2=1,h2;;
setf.sig HF2=h2 };;
{ .mfi; (p16) ld8 r15=[r10],16
xmpy.lu f32=HF0,RF0 }
{ .mfi; (p16) ld8 r16=[r11],16
xmpy.hu f33=HF0,RF0 }
{ .mfi; xmpy.lu f36=HF0,RF1 }
{ .mfi; xmpy.hu f37=HF0,RF1 };;
{ .mfi; xmpy.lu f34=HF1,SF1
(p15) shrp i0=r15,r14,56 }
{ .mfi; xmpy.hu f35=HF1,SF1 }
{ .mfi; xmpy.lu f38=HF1,RF0
(p15) shrp i1=r16,r15,56 }
{ .mfi; xmpy.hu f39=HF1,RF0 }
{ .mfi; xmpy.lu f40=HF2,SF1
(p14) shrp i0=r15,r14,48 }
{ .mfi; xmpy.lu f41=HF2,RF0 };;
{ .mmi; getf.sig r22=f32
getf.sig r23=f33
(p14) shrp i1=r16,r15,48 }
{ .mmi; getf.sig r24=f34
getf.sig r25=f35
(p13) shrp i0=r15,r14,40 }
{ .mmi; getf.sig r26=f36
getf.sig r27=f37
(p13) shrp i1=r16,r15,40 }
{ .mmi; getf.sig r28=f38
getf.sig r29=f39
(p12) shrp i0=r15,r14,32 }
{ .mmi; getf.sig r30=f40
getf.sig r31=f41 };;
{ .mmi; add h0=r22,r24
add r23=r23,r25
(p12) shrp i1=r16,r15,32 }
{ .mmi; add h1=r26,r28
add r27=r27,r29
(p11) shrp i0=r15,r14,24 };;
{ .mmi; cmp.ltu p6,p0=h0,r24
cmp.ltu p7,p0=h1,r28
add r23=r23,r30 };;
{ .mmi; (p6) add r23=1,r23
(p7) add r27=1,r27
(p11) shrp i1=r16,r15,24 };;
{ .mmi; add h1=h1,r23;;
cmp.ltu p6,p7=h1,r23
(p10) shrp i0=r15,r14,16 };;
{ .mmi; (p6) add h2=r31,r27,1
(p7) add h2=r31,r27
(p10) shrp i1=r16,r15,16 };;
{ .mmi; (p8) mov i0=r15
and r22=-4,h2
shr.u r23=h2,2 };;
{ .mmi; add r22=r22,r23
and h2=3,h2
(p9) shrp i0=r15,r14,8 };;
{ .mmi; add h0=h0,r22;;
cmp.ltu p6,p0=h0,r22
(p9) shrp i1=r16,r15,8 };;
{ .mmi; (p8) mov i1=r16
(p6) cmp.eq.unc p7,p0=-1,h1
(p6) add h1=1,h1 };;
{ .mmb; (p7) add h2=1,h2
mov r14=r16
br.ctop.sptk .Loop };;
.Lcend:
{ .mii; SUM 1<<1 // back to big-endian
mov ar.lc=r3 };;
{ .mmi; st8 [r8]=h0,16
st8 [r9]=h1
mov pr=r36,0x1ffff };;
{ .mmb; st8 [r8]=h2
rum 1<<5
br.ret.sptk b0 };;
.endp poly1305_blocks#
.global poly1305_emit#
.proc poly1305_emit#
.align 64
poly1305_emit:
.prologue
.save ar.pfs,r2
{ .mmi; alloc r2=ar.pfs,3,0,0,0
ADDP r8=0,r32
ADDP r9=8,r32 };;
.body
{ .mmi; ld8 r16=[r8],16 // load hash
ld8 r17=[r9]
ADDP r10=0,r34 };;
{ .mmi; ld8 r18=[r8]
ld4 r24=[r10],8 // load nonce
ADDP r11=4,r34 };;
{ .mmi; ld4 r25=[r11],8
ld4 r26=[r10]
add r20=5,r16 };;
{ .mmi; ld4 r27=[r11]
cmp.ltu p6,p7=r20,r16
shl r25=r25,32 };;
{ .mmi;
(p6) add r21=1,r17
(p7) add r21=0,r17
(p6) cmp.eq.or.andcm p6,p7=-1,r17 };;
{ .mmi;
(p6) add r22=1,r18
(p7) add r22=0,r18
shl r27=r27,32 };;
{ .mmi; or r24=r24,r25
or r26=r26,r27
cmp.leu p6,p7=4,r22 };;
{ .mmi;
(p6) add r16=r20,r24
(p7) add r16=r16,r24
(p6) add r17=r21,r26 };;
{ .mii;
(p7) add r17=r17,r26
cmp.ltu p6,p7=r16,r24;;
(p6) add r17=1,r17 };;
{ .mmi; ADDP r8=0,r33
ADDP r9=4,r33
shr.u r20=r16,32 }
{ .mmi; ADDP r10=8,r33
ADDP r11=12,r33
shr.u r21=r17,32 };;
{ .mmi; st1 [r8]=r16,1 // write mac, little-endian
st1 [r9]=r20,1
shr.u r16=r16,8 }
{ .mii; st1 [r10]=r17,1
shr.u r20=r20,8
shr.u r17=r17,8 }
{ .mmi; st1 [r11]=r21,1
shr.u r21=r21,8 };;
{ .mmi; st1 [r8]=r16,1
st1 [r9]=r20,1
shr.u r16=r16,8 }
{ .mii; st1 [r10]=r17,1
shr.u r20=r20,8
shr.u r17=r17,8 }
{ .mmi; st1 [r11]=r21,1
shr.u r21=r21,8 };;
{ .mmi; st1 [r8]=r16,1
st1 [r9]=r20,1
shr.u r16=r16,8 }
{ .mii; st1 [r10]=r17,1
shr.u r20=r20,8
shr.u r17=r17,8 }
{ .mmi; st1 [r11]=r21,1
shr.u r21=r21,8 };;
{ .mmi; st1 [r8]=r16
st1 [r9]=r20 }
{ .mmb; st1 [r10]=r17
st1 [r11]=r21
br.ret.sptk b0 };;
.endp poly1305_emit#
stringz "Poly1305 for IA64, CRYPTOGAMS by \@dot-asm"
