chacha-armv8.pl@ 108351

最後變更在這個檔案從108351是 104078,由 vboxsync 提交於 12 月前
openssl-3.1.5: Applied and adjusted our OpenSSL changes to 3.1.4. bugref:10638
屬性 svn:executable 設為 ``*
檔案大小: 32.0 KB

行
1	#! /usr/bin/env perl
2	# Copyright 2016-2022 The OpenSSL Project Authors. All Rights Reserved.
3	#
4	# Licensed under the Apache License 2.0 (the "License"). You may not use
5	# this file except in compliance with the License. You can obtain a copy
6	# in the file LICENSE in the source distribution or at
7	# https://www.openssl.org/source/license.html
8
9	#
10	# ====================================================================
11	# Written by Andy Polyakov <[email protected]> for the OpenSSL
12	# project. The module is, however, dual licensed under OpenSSL and
13	# CRYPTOGAMS licenses depending on where you obtain it. For further
14	# details see http://www.openssl.org/~appro/cryptogams/.
15	# ====================================================================
16	#
17	# June 2015
18	#
19	# ChaCha20 for ARMv8.
20	#
21	# April 2019
22	#
23	# Replace 3xNEON+1xIALU code path with 4+1. 4+1 is actually fastest
24	# option on most(*), but not all, processors, yet 6+2 is retained.
25	# This is because penalties are considered tolerable in comparison to
26	# improvement on processors where 6+2 helps. Most notably +37% on
27	# ThunderX2. It's server-oriented processor which will have to serve
28	# as many requests as possible. While others are mostly clients, when
29	# performance doesn't have to be absolute top-notch, just fast enough,
30	# as majority of time is spent "entertaining" relatively slow human.
31	#
32	# Performance in cycles per byte out of large buffer.
33	#
34	# IALU/gcc-4.9 4xNEON+1xIALU 6xNEON+2xIALU
35	#
36	# Apple A7 5.50/+49% 2.72 1.60
37	# Cortex-A53 8.40/+80% 4.06 4.45(*)
38	# Cortex-A57 8.06/+43% 4.15 4.40(*)
39	# Denver 4.50/+82% 2.30 2.70(*)
40	# X-Gene 9.50/+46% 8.20 8.90(*)
41	# Mongoose 8.00/+44% 2.74 3.12(*)
42	# Kryo 8.17/+50% 4.47 4.65(*)
43	# ThunderX2 7.22/+48% 5.64 4.10
44	#
45	# (*) slower than 4+1:-(
46
47	# $output is the last argument if it looks like a file (it has an extension)
48	# $flavour is the first argument if it doesn't look like a file
49	$output = $#ARGV >= 0 && $ARGV[$#ARGV] =~ m\|\.\w+$\| ? pop : undef;
50	$flavour = $#ARGV >= 0 && $ARGV[0] !~ m\|\.\| ? shift : undef;
51
52	$0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
53	( $xlate="${dir}arm-xlate.pl" and -f $xlate ) or
54	( $xlate="${dir}../../perlasm/arm-xlate.pl" and -f $xlate) or
55	die "can't locate arm-xlate.pl";
56
57	open OUT,"\| \"$^X\" $xlate $flavour \"$output\""
58	or die "can't call $xlate: $!";
59	STDOUT=OUT;
60
61	sub AUTOLOAD() # thunk [simplified] x86-style perlasm
62	{ my $opcode = $AUTOLOAD; $opcode =~ s/.*:://; $opcode =~ s/_/\./;
63	my $arg = pop;
64	$arg = "#$arg" if ($arg*1 eq $arg);
65	$code .= "\t$opcode\t".join(',',@_,$arg)."\n";
66	}
67
68	my ($out,$inp,$len,$key,$ctr) = map("x$_",(0..4));
69
70	my @x=map("x$_",(5..17,19..21));
71	my @d=map("x$_",(22..28,30));
72
73	sub ROUND {
74	my ($a0,$b0,$c0,$d0)=@_;
75	my ($a1,$b1,$c1,$d1)=map(($_&~3)+(($_+1)&3),($a0,$b0,$c0,$d0));
76	my ($a2,$b2,$c2,$d2)=map(($_&~3)+(($_+1)&3),($a1,$b1,$c1,$d1));
77	my ($a3,$b3,$c3,$d3)=map(($_&~3)+(($_+1)&3),($a2,$b2,$c2,$d2));
78
79	(
80	"&add_32 (@x[$a0],@x[$a0],@x[$b0])",
81	"&add_32 (@x[$a1],@x[$a1],@x[$b1])",
82	"&add_32 (@x[$a2],@x[$a2],@x[$b2])",
83	"&add_32 (@x[$a3],@x[$a3],@x[$b3])",
84	"&eor_32 (@x[$d0],@x[$d0],@x[$a0])",
85	"&eor_32 (@x[$d1],@x[$d1],@x[$a1])",
86	"&eor_32 (@x[$d2],@x[$d2],@x[$a2])",
87	"&eor_32 (@x[$d3],@x[$d3],@x[$a3])",
88	"&ror_32 (@x[$d0],@x[$d0],16)",
89	"&ror_32 (@x[$d1],@x[$d1],16)",
90	"&ror_32 (@x[$d2],@x[$d2],16)",
91	"&ror_32 (@x[$d3],@x[$d3],16)",
92
93	"&add_32 (@x[$c0],@x[$c0],@x[$d0])",
94	"&add_32 (@x[$c1],@x[$c1],@x[$d1])",
95	"&add_32 (@x[$c2],@x[$c2],@x[$d2])",
96	"&add_32 (@x[$c3],@x[$c3],@x[$d3])",
97	"&eor_32 (@x[$b0],@x[$b0],@x[$c0])",
98	"&eor_32 (@x[$b1],@x[$b1],@x[$c1])",
99	"&eor_32 (@x[$b2],@x[$b2],@x[$c2])",
100	"&eor_32 (@x[$b3],@x[$b3],@x[$c3])",
101	"&ror_32 (@x[$b0],@x[$b0],20)",
102	"&ror_32 (@x[$b1],@x[$b1],20)",
103	"&ror_32 (@x[$b2],@x[$b2],20)",
104	"&ror_32 (@x[$b3],@x[$b3],20)",
105
106	"&add_32 (@x[$a0],@x[$a0],@x[$b0])",
107	"&add_32 (@x[$a1],@x[$a1],@x[$b1])",
108	"&add_32 (@x[$a2],@x[$a2],@x[$b2])",
109	"&add_32 (@x[$a3],@x[$a3],@x[$b3])",
110	"&eor_32 (@x[$d0],@x[$d0],@x[$a0])",
111	"&eor_32 (@x[$d1],@x[$d1],@x[$a1])",
112	"&eor_32 (@x[$d2],@x[$d2],@x[$a2])",
113	"&eor_32 (@x[$d3],@x[$d3],@x[$a3])",
114	"&ror_32 (@x[$d0],@x[$d0],24)",
115	"&ror_32 (@x[$d1],@x[$d1],24)",
116	"&ror_32 (@x[$d2],@x[$d2],24)",
117	"&ror_32 (@x[$d3],@x[$d3],24)",
118
119	"&add_32 (@x[$c0],@x[$c0],@x[$d0])",
120	"&add_32 (@x[$c1],@x[$c1],@x[$d1])",
121	"&add_32 (@x[$c2],@x[$c2],@x[$d2])",
122	"&add_32 (@x[$c3],@x[$c3],@x[$d3])",
123	"&eor_32 (@x[$b0],@x[$b0],@x[$c0])",
124	"&eor_32 (@x[$b1],@x[$b1],@x[$c1])",
125	"&eor_32 (@x[$b2],@x[$b2],@x[$c2])",
126	"&eor_32 (@x[$b3],@x[$b3],@x[$c3])",
127	"&ror_32 (@x[$b0],@x[$b0],25)",
128	"&ror_32 (@x[$b1],@x[$b1],25)",
129	"&ror_32 (@x[$b2],@x[$b2],25)",
130	"&ror_32 (@x[$b3],@x[$b3],25)"
131	);
132	}
133
134	$code.=<<___;
135	#include "arm_arch.h"
136	#ifndef __KERNEL__
137	.extern OPENSSL_armcap_P
138	.hidden OPENSSL_armcap_P
139
140	.extern ChaCha20_ctr32_sve
141	#endif
142
143	.text
144
145	.align 5
146	.Lsigma:
147	.quad 0x3320646e61707865,0x6b20657479622d32 // endian-neutral
148	.Lone:
149	.long 1,2,3,4
150	.Lrot24:
151	.long 0x02010003,0x06050407,0x0a09080b,0x0e0d0c0f
152	.asciz "ChaCha20 for ARMv8, CRYPTOGAMS by \@dot-asm"
153
154	.globl ChaCha20_ctr32_dflt
155	.type ChaCha20_ctr32_dflt,%function
156	.align 5
157	ChaCha20_ctr32_dflt:
158	AARCH64_SIGN_LINK_REGISTER
159	cmp $len,#192
160	b.lo .Lshort
161	#ifndef __KERNEL__
162	adrp x17,OPENSSL_armcap_P
163	ldr w17,[x17,#:lo12:OPENSSL_armcap_P]
164	.Lcheck_neon:
165	tst w17,#ARMV7_NEON
166	b.ne .LChaCha20_neon
167	#endif
168
169	.Lshort:
170	stp x29,x30,[sp,#-96]!
171	add x29,sp,#0
172
173	adr @x[0],.Lsigma
174	stp x19,x20,[sp,#16]
175	stp x21,x22,[sp,#32]
176	stp x23,x24,[sp,#48]
177	stp x25,x26,[sp,#64]
178	stp x27,x28,[sp,#80]
179	sub sp,sp,#64
180
181	ldp @d[0],@d[1],[@x[0]] // load sigma
182	ldp @d[2],@d[3],[$key] // load key
183	ldp @d[4],@d[5],[$key,#16]
184	ldp @d[6],@d[7],[$ctr] // load counter
185	#ifdef __AARCH64EB__
186	ror @d[2],@d[2],#32
187	ror @d[3],@d[3],#32
188	ror @d[4],@d[4],#32
189	ror @d[5],@d[5],#32
190	ror @d[6],@d[6],#32
191	ror @d[7],@d[7],#32
192	#endif
193
194	.Loop_outer:
195	mov.32 @x[0],@d[0] // unpack key block
196	lsr @x[1],@d[0],#32
197	mov.32 @x[2],@d[1]
198	lsr @x[3],@d[1],#32
199	mov.32 @x[4],@d[2]
200	lsr @x[5],@d[2],#32
201	mov.32 @x[6],@d[3]
202	lsr @x[7],@d[3],#32
203	mov.32 @x[8],@d[4]
204	lsr @x[9],@d[4],#32
205	mov.32 @x[10],@d[5]
206	lsr @x[11],@d[5],#32
207	mov.32 @x[12],@d[6]
208	lsr @x[13],@d[6],#32
209	mov.32 @x[14],@d[7]
210	lsr @x[15],@d[7],#32
211
212	mov $ctr,#10
213	subs $len,$len,#64
214	.Loop:
215	sub $ctr,$ctr,#1
216	___
217	foreach (&ROUND(0, 4, 8,12)) { eval; }
218	foreach (&ROUND(0, 5,10,15)) { eval; }
219	$code.=<<___;
220	cbnz $ctr,.Loop
221
222	add.32 @x[0],@x[0],@d[0] // accumulate key block
223	add @x[1],@x[1],@d[0],lsr#32
224	add.32 @x[2],@x[2],@d[1]
225	add @x[3],@x[3],@d[1],lsr#32
226	add.32 @x[4],@x[4],@d[2]
227	add @x[5],@x[5],@d[2],lsr#32
228	add.32 @x[6],@x[6],@d[3]
229	add @x[7],@x[7],@d[3],lsr#32
230	add.32 @x[8],@x[8],@d[4]
231	add @x[9],@x[9],@d[4],lsr#32
232	add.32 @x[10],@x[10],@d[5]
233	add @x[11],@x[11],@d[5],lsr#32
234	add.32 @x[12],@x[12],@d[6]
235	add @x[13],@x[13],@d[6],lsr#32
236	add.32 @x[14],@x[14],@d[7]
237	add @x[15],@x[15],@d[7],lsr#32
238
239	b.lo .Ltail
240
241	add @x[0],@x[0],@x[1],lsl#32 // pack
242	add @x[2],@x[2],@x[3],lsl#32
243	ldp @x[1],@x[3],[$inp,#0] // load input
244	add @x[4],@x[4],@x[5],lsl#32
245	add @x[6],@x[6],@x[7],lsl#32
246	ldp @x[5],@x[7],[$inp,#16]
247	add @x[8],@x[8],@x[9],lsl#32
248	add @x[10],@x[10],@x[11],lsl#32
249	ldp @x[9],@x[11],[$inp,#32]
250	add @x[12],@x[12],@x[13],lsl#32
251	add @x[14],@x[14],@x[15],lsl#32
252	ldp @x[13],@x[15],[$inp,#48]
253	add $inp,$inp,#64
254	#ifdef __AARCH64EB__
255	rev @x[0],@x[0]
256	rev @x[2],@x[2]
257	rev @x[4],@x[4]
258	rev @x[6],@x[6]
259	rev @x[8],@x[8]
260	rev @x[10],@x[10]
261	rev @x[12],@x[12]
262	rev @x[14],@x[14]
263	#endif
264	eor @x[0],@x[0],@x[1]
265	eor @x[2],@x[2],@x[3]
266	eor @x[4],@x[4],@x[5]
267	eor @x[6],@x[6],@x[7]
268	eor @x[8],@x[8],@x[9]
269	eor @x[10],@x[10],@x[11]
270	eor @x[12],@x[12],@x[13]
271	eor @x[14],@x[14],@x[15]
272
273	stp @x[0],@x[2],[$out,#0] // store output
274	add @d[6],@d[6],#1 // increment counter
275	stp @x[4],@x[6],[$out,#16]
276	stp @x[8],@x[10],[$out,#32]
277	stp @x[12],@x[14],[$out,#48]
278	add $out,$out,#64
279
280	b.hi .Loop_outer
281
282	ldp x19,x20,[x29,#16]
283	add sp,sp,#64
284	ldp x21,x22,[x29,#32]
285	ldp x23,x24,[x29,#48]
286	ldp x25,x26,[x29,#64]
287	ldp x27,x28,[x29,#80]
288	ldp x29,x30,[sp],#96
289	.Labort:
290	AARCH64_VALIDATE_LINK_REGISTER
291	ret
292
293	.align 4
294	.Ltail:
295	add $len,$len,#64
296	.Less_than_64:
297	sub $out,$out,#1
298	add $inp,$inp,$len
299	add $out,$out,$len
300	add $ctr,sp,$len
301	neg $len,$len
302
303	add @x[0],@x[0],@x[1],lsl#32 // pack
304	add @x[2],@x[2],@x[3],lsl#32
305	add @x[4],@x[4],@x[5],lsl#32
306	add @x[6],@x[6],@x[7],lsl#32
307	add @x[8],@x[8],@x[9],lsl#32
308	add @x[10],@x[10],@x[11],lsl#32
309	add @x[12],@x[12],@x[13],lsl#32
310	add @x[14],@x[14],@x[15],lsl#32
311	#ifdef __AARCH64EB__
312	rev @x[0],@x[0]
313	rev @x[2],@x[2]
314	rev @x[4],@x[4]
315	rev @x[6],@x[6]
316	rev @x[8],@x[8]
317	rev @x[10],@x[10]
318	rev @x[12],@x[12]
319	rev @x[14],@x[14]
320	#endif
321	stp @x[0],@x[2],[sp,#0]
322	stp @x[4],@x[6],[sp,#16]
323	stp @x[8],@x[10],[sp,#32]
324	stp @x[12],@x[14],[sp,#48]
325
326	.Loop_tail:
327	ldrb w10,[$inp,$len]
328	ldrb w11,[$ctr,$len]
329	add $len,$len,#1
330	eor w10,w10,w11
331	strb w10,[$out,$len]
332	cbnz $len,.Loop_tail
333
334	stp xzr,xzr,[sp,#0]
335	stp xzr,xzr,[sp,#16]
336	stp xzr,xzr,[sp,#32]
337	stp xzr,xzr,[sp,#48]
338
339	ldp x19,x20,[x29,#16]
340	add sp,sp,#64
341	ldp x21,x22,[x29,#32]
342	ldp x23,x24,[x29,#48]
343	ldp x25,x26,[x29,#64]
344	ldp x27,x28,[x29,#80]
345	ldp x29,x30,[sp],#96
346	AARCH64_VALIDATE_LINK_REGISTER
347	ret
348	.size ChaCha20_ctr32_dflt,.-ChaCha20_ctr32_dflt
349
350	.globl ChaCha20_ctr32
351	.type ChaCha20_ctr32,%function
352	.align 5
353	ChaCha20_ctr32:
354	AARCH64_SIGN_LINK_REGISTER
355	cbz $len,.Labort
356	cmp $len,#192
357	b.lo .Lshort
358	#ifndef __KERNEL__
359	adrp x17,OPENSSL_armcap_P
360	ldr w17,[x17,#:lo12:OPENSSL_armcap_P]
361	tst w17,#ARMV8_SVE
362	b.eq .Lcheck_neon
363	stp x29,x30,[sp,#-16]!
364	sub sp,sp,#16
365	// SVE handling will inevitably increment the counter
366	// Neon/Scalar code that follows to process tail data needs to
367	// use new counter, unfortunately the input counter buffer
368	// pointed to by ctr is meant to be read-only per API contract
369	// we have to copy the buffer to stack to be writable by SVE
370	ldp x5,x6,[$ctr]
371	stp x5,x6,[sp]
372	mov $ctr,sp
373	bl ChaCha20_ctr32_sve
374	cbz $len,1f
375	bl ChaCha20_ctr32_dflt
376	1:
377	add sp,sp,#16
378	ldp x29,x30,[sp],#16
379	AARCH64_VALIDATE_LINK_REGISTER
380	ret
381	#endif
382	b .Lshort
383	.size ChaCha20_ctr32,.-ChaCha20_ctr32
384	___
385
386	{{{
387	my @K = map("v$_.4s",(0..3));
388	my ($xt0,$xt1,$xt2,$xt3, $CTR,$ROT24) = map("v$_.4s",(4..9));
389	my @X = map("v$_.4s",(16,20,24,28, 17,21,25,29, 18,22,26,30, 19,23,27,31));
390	my ($xa0,$xa1,$xa2,$xa3, $xb0,$xb1,$xb2,$xb3,
391	$xc0,$xc1,$xc2,$xc3, $xd0,$xd1,$xd2,$xd3) = @X;
392
393	sub NEON_lane_ROUND {
394	my ($a0,$b0,$c0,$d0)=@_;
395	my ($a1,$b1,$c1,$d1)=map(($_&~3)+(($_+1)&3),($a0,$b0,$c0,$d0));
396	my ($a2,$b2,$c2,$d2)=map(($_&~3)+(($_+1)&3),($a1,$b1,$c1,$d1));
397	my ($a3,$b3,$c3,$d3)=map(($_&~3)+(($_+1)&3),($a2,$b2,$c2,$d2));
398	my @x=map("'$_'",@X);
399
400	(
401	"&add (@x[$a0],@x[$a0],@x[$b0])", # Q1
402	"&add (@x[$a1],@x[$a1],@x[$b1])", # Q2
403	"&add (@x[$a2],@x[$a2],@x[$b2])", # Q3
404	"&add (@x[$a3],@x[$a3],@x[$b3])", # Q4
405	"&eor (@x[$d0],@x[$d0],@x[$a0])",
406	"&eor (@x[$d1],@x[$d1],@x[$a1])",
407	"&eor (@x[$d2],@x[$d2],@x[$a2])",
408	"&eor (@x[$d3],@x[$d3],@x[$a3])",
409	"&rev32_16 (@x[$d0],@x[$d0])",
410	"&rev32_16 (@x[$d1],@x[$d1])",
411	"&rev32_16 (@x[$d2],@x[$d2])",
412	"&rev32_16 (@x[$d3],@x[$d3])",
413
414	"&add (@x[$c0],@x[$c0],@x[$d0])",
415	"&add (@x[$c1],@x[$c1],@x[$d1])",
416	"&add (@x[$c2],@x[$c2],@x[$d2])",
417	"&add (@x[$c3],@x[$c3],@x[$d3])",
418	"&eor ('$xt0',@x[$b0],@x[$c0])",
419	"&eor ('$xt1',@x[$b1],@x[$c1])",
420	"&eor ('$xt2',@x[$b2],@x[$c2])",
421	"&eor ('$xt3',@x[$b3],@x[$c3])",
422	"&ushr (@x[$b0],'$xt0',20)",
423	"&ushr (@x[$b1],'$xt1',20)",
424	"&ushr (@x[$b2],'$xt2',20)",
425	"&ushr (@x[$b3],'$xt3',20)",
426	"&sli (@x[$b0],'$xt0',12)",
427	"&sli (@x[$b1],'$xt1',12)",
428	"&sli (@x[$b2],'$xt2',12)",
429	"&sli (@x[$b3],'$xt3',12)",
430
431	"&add (@x[$a0],@x[$a0],@x[$b0])",
432	"&add (@x[$a1],@x[$a1],@x[$b1])",
433	"&add (@x[$a2],@x[$a2],@x[$b2])",
434	"&add (@x[$a3],@x[$a3],@x[$b3])",
435	"&eor ('$xt0',@x[$d0],@x[$a0])",
436	"&eor ('$xt1',@x[$d1],@x[$a1])",
437	"&eor ('$xt2',@x[$d2],@x[$a2])",
438	"&eor ('$xt3',@x[$d3],@x[$a3])",
439	"&tbl (@x[$d0],'{$xt0}','$ROT24')",
440	"&tbl (@x[$d1],'{$xt1}','$ROT24')",
441	"&tbl (@x[$d2],'{$xt2}','$ROT24')",
442	"&tbl (@x[$d3],'{$xt3}','$ROT24')",
443
444	"&add (@x[$c0],@x[$c0],@x[$d0])",
445	"&add (@x[$c1],@x[$c1],@x[$d1])",
446	"&add (@x[$c2],@x[$c2],@x[$d2])",
447	"&add (@x[$c3],@x[$c3],@x[$d3])",
448	"&eor ('$xt0',@x[$b0],@x[$c0])",
449	"&eor ('$xt1',@x[$b1],@x[$c1])",
450	"&eor ('$xt2',@x[$b2],@x[$c2])",
451	"&eor ('$xt3',@x[$b3],@x[$c3])",
452	"&ushr (@x[$b0],'$xt0',25)",
453	"&ushr (@x[$b1],'$xt1',25)",
454	"&ushr (@x[$b2],'$xt2',25)",
455	"&ushr (@x[$b3],'$xt3',25)",
456	"&sli (@x[$b0],'$xt0',7)",
457	"&sli (@x[$b1],'$xt1',7)",
458	"&sli (@x[$b2],'$xt2',7)",
459	"&sli (@x[$b3],'$xt3',7)"
460	);
461	}
462
463	$code.=<<___;
464
465	#ifdef __KERNEL__
466	.globl ChaCha20_neon
467	#endif
468	.type ChaCha20_neon,%function
469	.align 5
470	ChaCha20_neon:
471	AARCH64_SIGN_LINK_REGISTER
472	.LChaCha20_neon:
473	stp x29,x30,[sp,#-96]!
474	add x29,sp,#0
475
476	adr @x[0],.Lsigma
477	stp x19,x20,[sp,#16]
478	stp x21,x22,[sp,#32]
479	stp x23,x24,[sp,#48]
480	stp x25,x26,[sp,#64]
481	stp x27,x28,[sp,#80]
482	cmp $len,#512
483	b.hs .L512_or_more_neon
484
485	sub sp,sp,#64
486
487	ldp @d[0],@d[1],[@x[0]] // load sigma
488	ld1 {@K[0]},[@x[0]],#16
489	ldp @d[2],@d[3],[$key] // load key
490	ldp @d[4],@d[5],[$key,#16]
491	ld1 {@K[1],@K[2]},[$key]
492	ldp @d[6],@d[7],[$ctr] // load counter
493	ld1 {@K[3]},[$ctr]
494	stp d8,d9,[sp] // meet ABI requirements
495	ld1 {$CTR,$ROT24},[@x[0]]
496	#ifdef __AARCH64EB__
497	rev64 @K[0],@K[0]
498	ror @d[2],@d[2],#32
499	ror @d[3],@d[3],#32
500	ror @d[4],@d[4],#32
501	ror @d[5],@d[5],#32
502	ror @d[6],@d[6],#32
503	ror @d[7],@d[7],#32
504	#endif
505
506	.Loop_outer_neon:
507	dup $xa0,@{K[0]}[0] // unpack key block
508	mov.32 @x[0],@d[0]
509	dup $xa1,@{K[0]}[1]
510	lsr @x[1],@d[0],#32
511	dup $xa2,@{K[0]}[2]
512	mov.32 @x[2],@d[1]
513	dup $xa3,@{K[0]}[3]
514	lsr @x[3],@d[1],#32
515	dup $xb0,@{K[1]}[0]
516	mov.32 @x[4],@d[2]
517	dup $xb1,@{K[1]}[1]
518	lsr @x[5],@d[2],#32
519	dup $xb2,@{K[1]}[2]
520	mov.32 @x[6],@d[3]
521	dup $xb3,@{K[1]}[3]
522	lsr @x[7],@d[3],#32
523	dup $xd0,@{K[3]}[0]
524	mov.32 @x[8],@d[4]
525	dup $xd1,@{K[3]}[1]
526	lsr @x[9],@d[4],#32
527	dup $xd2,@{K[3]}[2]
528	mov.32 @x[10],@d[5]
529	dup $xd3,@{K[3]}[3]
530	lsr @x[11],@d[5],#32
531	add $xd0,$xd0,$CTR
532	mov.32 @x[12],@d[6]
533	dup $xc0,@{K[2]}[0]
534	lsr @x[13],@d[6],#32
535	dup $xc1,@{K[2]}[1]
536	mov.32 @x[14],@d[7]
537	dup $xc2,@{K[2]}[2]
538	lsr @x[15],@d[7],#32
539	dup $xc3,@{K[2]}[3]
540
541	mov $ctr,#10
542	subs $len,$len,#320
543	.Loop_neon:
544	sub $ctr,$ctr,#1
545	___
546	my @plus_one=&ROUND(0,4,8,12);
547	foreach (&NEON_lane_ROUND(0,4,8,12)) { eval; eval(shift(@plus_one)); }
548
549	@plus_one=&ROUND(0,5,10,15);
550	foreach (&NEON_lane_ROUND(0,5,10,15)) { eval; eval(shift(@plus_one)); }
551	$code.=<<___;
552	cbnz $ctr,.Loop_neon
553
554	add $xd0,$xd0,$CTR
555
556	zip1 $xt0,$xa0,$xa1 // transpose data
557	zip1 $xt1,$xa2,$xa3
558	zip2 $xt2,$xa0,$xa1
559	zip2 $xt3,$xa2,$xa3
560	zip1.64 $xa0,$xt0,$xt1
561	zip2.64 $xa1,$xt0,$xt1
562	zip1.64 $xa2,$xt2,$xt3
563	zip2.64 $xa3,$xt2,$xt3
564
565	zip1 $xt0,$xb0,$xb1
566	zip1 $xt1,$xb2,$xb3
567	zip2 $xt2,$xb0,$xb1
568	zip2 $xt3,$xb2,$xb3
569	zip1.64 $xb0,$xt0,$xt1
570	zip2.64 $xb1,$xt0,$xt1
571	zip1.64 $xb2,$xt2,$xt3
572	zip2.64 $xb3,$xt2,$xt3
573
574	zip1 $xt0,$xc0,$xc1
575	add.32 @x[0],@x[0],@d[0] // accumulate key block
576	zip1 $xt1,$xc2,$xc3
577	add @x[1],@x[1],@d[0],lsr#32
578	zip2 $xt2,$xc0,$xc1
579	add.32 @x[2],@x[2],@d[1]
580	zip2 $xt3,$xc2,$xc3
581	add @x[3],@x[3],@d[1],lsr#32
582	zip1.64 $xc0,$xt0,$xt1
583	add.32 @x[4],@x[4],@d[2]
584	zip2.64 $xc1,$xt0,$xt1
585	add @x[5],@x[5],@d[2],lsr#32
586	zip1.64 $xc2,$xt2,$xt3
587	add.32 @x[6],@x[6],@d[3]
588	zip2.64 $xc3,$xt2,$xt3
589	add @x[7],@x[7],@d[3],lsr#32
590
591	zip1 $xt0,$xd0,$xd1
592	add.32 @x[8],@x[8],@d[4]
593	zip1 $xt1,$xd2,$xd3
594	add @x[9],@x[9],@d[4],lsr#32
595	zip2 $xt2,$xd0,$xd1
596	add.32 @x[10],@x[10],@d[5]
597	zip2 $xt3,$xd2,$xd3
598	add @x[11],@x[11],@d[5],lsr#32
599	zip1.64 $xd0,$xt0,$xt1
600	add.32 @x[12],@x[12],@d[6]
601	zip2.64 $xd1,$xt0,$xt1
602	add @x[13],@x[13],@d[6],lsr#32
603	zip1.64 $xd2,$xt2,$xt3
604	add.32 @x[14],@x[14],@d[7]
605	zip2.64 $xd3,$xt2,$xt3
606	add @x[15],@x[15],@d[7],lsr#32
607
608	b.lo .Ltail_neon
609
610	add @x[0],@x[0],@x[1],lsl#32 // pack
611	add @x[2],@x[2],@x[3],lsl#32
612	ldp @x[1],@x[3],[$inp,#0] // load input
613	add $xa0,$xa0,@K[0] // accumulate key block
614	add @x[4],@x[4],@x[5],lsl#32
615	add @x[6],@x[6],@x[7],lsl#32
616	ldp @x[5],@x[7],[$inp,#16]
617	add $xb0,$xb0,@K[1]
618	add @x[8],@x[8],@x[9],lsl#32
619	add @x[10],@x[10],@x[11],lsl#32
620	ldp @x[9],@x[11],[$inp,#32]
621	add $xc0,$xc0,@K[2]
622	add @x[12],@x[12],@x[13],lsl#32
623	add @x[14],@x[14],@x[15],lsl#32
624	ldp @x[13],@x[15],[$inp,#48]
625	add $xd0,$xd0,@K[3]
626	add $inp,$inp,#64
627	#ifdef __AARCH64EB__
628	rev @x[0],@x[0]
629	rev @x[2],@x[2]
630	rev @x[4],@x[4]
631	rev @x[6],@x[6]
632	rev @x[8],@x[8]
633	rev @x[10],@x[10]
634	rev @x[12],@x[12]
635	rev @x[14],@x[14]
636	#endif
637	ld1.8 {$xt0-$xt3},[$inp],#64
638	eor @x[0],@x[0],@x[1]
639	add $xa1,$xa1,@K[0]
640	eor @x[2],@x[2],@x[3]
641	add $xb1,$xb1,@K[1]
642	eor @x[4],@x[4],@x[5]
643	add $xc1,$xc1,@K[2]
644	eor @x[6],@x[6],@x[7]
645	add $xd1,$xd1,@K[3]
646	eor @x[8],@x[8],@x[9]
647	eor $xa0,$xa0,$xt0
648	movi $xt0,#5
649	eor @x[10],@x[10],@x[11]
650	eor $xb0,$xb0,$xt1
651	eor @x[12],@x[12],@x[13]
652	eor $xc0,$xc0,$xt2
653	eor @x[14],@x[14],@x[15]
654	eor $xd0,$xd0,$xt3
655	add $CTR,$CTR,$xt0 // += 5
656	ld1.8 {$xt0-$xt3},[$inp],#64
657
658	stp @x[0],@x[2],[$out,#0] // store output
659	add @d[6],@d[6],#5 // increment counter
660	stp @x[4],@x[6],[$out,#16]
661	stp @x[8],@x[10],[$out,#32]
662	stp @x[12],@x[14],[$out,#48]
663	add $out,$out,#64
664
665	st1.8 {$xa0-$xd0},[$out],#64
666	add $xa2,$xa2,@K[0]
667	add $xb2,$xb2,@K[1]
668	add $xc2,$xc2,@K[2]
669	add $xd2,$xd2,@K[3]
670	ld1.8 {$xa0-$xd0},[$inp],#64
671
672	eor $xa1,$xa1,$xt0
673	eor $xb1,$xb1,$xt1
674	eor $xc1,$xc1,$xt2
675	eor $xd1,$xd1,$xt3
676	st1.8 {$xa1-$xd1},[$out],#64
677	add $xa3,$xa3,@K[0]
678	add $xb3,$xb3,@K[1]
679	add $xc3,$xc3,@K[2]
680	add $xd3,$xd3,@K[3]
681	ld1.8 {$xa1-$xd1},[$inp],#64
682
683	eor $xa2,$xa2,$xa0
684	eor $xb2,$xb2,$xb0
685	eor $xc2,$xc2,$xc0
686	eor $xd2,$xd2,$xd0
687	st1.8 {$xa2-$xd2},[$out],#64
688
689	eor $xa3,$xa3,$xa1
690	eor $xb3,$xb3,$xb1
691	eor $xc3,$xc3,$xc1
692	eor $xd3,$xd3,$xd1
693	st1.8 {$xa3-$xd3},[$out],#64
694
695	b.hi .Loop_outer_neon
696
697	ldp d8,d9,[sp] // meet ABI requirements
698
699	ldp x19,x20,[x29,#16]
700	add sp,sp,#64
701	ldp x21,x22,[x29,#32]
702	ldp x23,x24,[x29,#48]
703	ldp x25,x26,[x29,#64]
704	ldp x27,x28,[x29,#80]
705	ldp x29,x30,[sp],#96
706	AARCH64_VALIDATE_LINK_REGISTER
707	ret
708
709	.align 4
710	.Ltail_neon:
711	add $len,$len,#320
712	ldp d8,d9,[sp] // meet ABI requirements
713	cmp $len,#64
714	b.lo .Less_than_64
715
716	add @x[0],@x[0],@x[1],lsl#32 // pack
717	add @x[2],@x[2],@x[3],lsl#32
718	ldp @x[1],@x[3],[$inp,#0] // load input
719	add @x[4],@x[4],@x[5],lsl#32
720	add @x[6],@x[6],@x[7],lsl#32
721	ldp @x[5],@x[7],[$inp,#16]
722	add @x[8],@x[8],@x[9],lsl#32
723	add @x[10],@x[10],@x[11],lsl#32
724	ldp @x[9],@x[11],[$inp,#32]
725	add @x[12],@x[12],@x[13],lsl#32
726	add @x[14],@x[14],@x[15],lsl#32
727	ldp @x[13],@x[15],[$inp,#48]
728	add $inp,$inp,#64
729	#ifdef __AARCH64EB__
730	rev @x[0],@x[0]
731	rev @x[2],@x[2]
732	rev @x[4],@x[4]
733	rev @x[6],@x[6]
734	rev @x[8],@x[8]
735	rev @x[10],@x[10]
736	rev @x[12],@x[12]
737	rev @x[14],@x[14]
738	#endif
739	eor @x[0],@x[0],@x[1]
740	eor @x[2],@x[2],@x[3]
741	eor @x[4],@x[4],@x[5]
742	eor @x[6],@x[6],@x[7]
743	eor @x[8],@x[8],@x[9]
744	eor @x[10],@x[10],@x[11]
745	eor @x[12],@x[12],@x[13]
746	eor @x[14],@x[14],@x[15]
747
748	stp @x[0],@x[2],[$out,#0] // store output
749	add $xa0,$xa0,@K[0] // accumulate key block
750	stp @x[4],@x[6],[$out,#16]
751	add $xb0,$xb0,@K[1]
752	stp @x[8],@x[10],[$out,#32]
753	add $xc0,$xc0,@K[2]
754	stp @x[12],@x[14],[$out,#48]
755	add $xd0,$xd0,@K[3]
756	add $out,$out,#64
757	b.eq .Ldone_neon
758	sub $len,$len,#64
759	cmp $len,#64
760	b.lo .Last_neon
761
762	ld1.8 {$xt0-$xt3},[$inp],#64
763	eor $xa0,$xa0,$xt0
764	eor $xb0,$xb0,$xt1
765	eor $xc0,$xc0,$xt2
766	eor $xd0,$xd0,$xt3
767	st1.8 {$xa0-$xd0},[$out],#64
768	b.eq .Ldone_neon
769
770	add $xa0,$xa1,@K[0]
771	add $xb0,$xb1,@K[1]
772	sub $len,$len,#64
773	add $xc0,$xc1,@K[2]
774	cmp $len,#64
775	add $xd0,$xd1,@K[3]
776	b.lo .Last_neon
777
778	ld1.8 {$xt0-$xt3},[$inp],#64
779	eor $xa1,$xa0,$xt0
780	eor $xb1,$xb0,$xt1
781	eor $xc1,$xc0,$xt2
782	eor $xd1,$xd0,$xt3
783	st1.8 {$xa1-$xd1},[$out],#64
784	b.eq .Ldone_neon
785
786	add $xa0,$xa2,@K[0]
787	add $xb0,$xb2,@K[1]
788	sub $len,$len,#64
789	add $xc0,$xc2,@K[2]
790	cmp $len,#64
791	add $xd0,$xd2,@K[3]
792	b.lo .Last_neon
793
794	ld1.8 {$xt0-$xt3},[$inp],#64
795	eor $xa2,$xa0,$xt0
796	eor $xb2,$xb0,$xt1
797	eor $xc2,$xc0,$xt2
798	eor $xd2,$xd0,$xt3
799	st1.8 {$xa2-$xd2},[$out],#64
800	b.eq .Ldone_neon
801
802	add $xa0,$xa3,@K[0]
803	add $xb0,$xb3,@K[1]
804	add $xc0,$xc3,@K[2]
805	add $xd0,$xd3,@K[3]
806	sub $len,$len,#64
807
808	.Last_neon:
809	st1.8 {$xa0-$xd0},[sp]
810
811	sub $out,$out,#1
812	add $inp,$inp,$len
813	add $out,$out,$len
814	add $ctr,sp,$len
815	neg $len,$len
816
817	.Loop_tail_neon:
818	ldrb w10,[$inp,$len]
819	ldrb w11,[$ctr,$len]
820	add $len,$len,#1
821	eor w10,w10,w11
822	strb w10,[$out,$len]
823	cbnz $len,.Loop_tail_neon
824
825	stp xzr,xzr,[sp,#0]
826	stp xzr,xzr,[sp,#16]
827	stp xzr,xzr,[sp,#32]
828	stp xzr,xzr,[sp,#48]
829
830	.Ldone_neon:
831	ldp x19,x20,[x29,#16]
832	add sp,sp,#64
833	ldp x21,x22,[x29,#32]
834	ldp x23,x24,[x29,#48]
835	ldp x25,x26,[x29,#64]
836	ldp x27,x28,[x29,#80]
837	ldp x29,x30,[sp],#96
838	AARCH64_VALIDATE_LINK_REGISTER
839	ret
840	.size ChaCha20_neon,.-ChaCha20_neon
841	___
842	{
843	my @K = map("v$_.4s",(0..6));
844	my ($T0,$T1,$T2,$T3,$T4,$T5)=@K;
845	my ($A0,$B0,$C0,$D0,$A1,$B1,$C1,$D1,$A2,$B2,$C2,$D2,
846	$A3,$B3,$C3,$D3,$A4,$B4,$C4,$D4,$A5,$B5,$C5,$D5) = map("v$_.4s",(8..31));
847	my $rot24 = @K[6];
848	my $ONE = "v7.4s";
849
850	sub NEONROUND {
851	my $odd = pop;
852	my ($a,$b,$c,$d,$t)=@_;
853
854	(
855	"&add ('$a','$a','$b')",
856	"&eor ('$d','$d','$a')",
857	"&rev32_16 ('$d','$d')", # vrot ($d,16)
858
859	"&add ('$c','$c','$d')",
860	"&eor ('$t','$b','$c')",
861	"&ushr ('$b','$t',20)",
862	"&sli ('$b','$t',12)",
863
864	"&add ('$a','$a','$b')",
865	"&eor ('$d','$d','$a')",
866	"&tbl ('$d','{$d}','$rot24')",
867
868	"&add ('$c','$c','$d')",
869	"&eor ('$t','$b','$c')",
870	"&ushr ('$b','$t',25)",
871	"&sli ('$b','$t',7)",
872
873	"&ext ('$c','$c','$c',8)",
874	"&ext ('$d','$d','$d',$odd?4:12)",
875	"&ext ('$b','$b','$b',$odd?12:4)"
876	);
877	}
878
879	$code.=<<___;
880	.type ChaCha20_512_neon,%function
881	.align 5
882	ChaCha20_512_neon:
883	AARCH64_SIGN_LINK_REGISTER
884	stp x29,x30,[sp,#-96]!
885	add x29,sp,#0
886
887	adr @x[0],.Lsigma
888	stp x19,x20,[sp,#16]
889	stp x21,x22,[sp,#32]
890	stp x23,x24,[sp,#48]
891	stp x25,x26,[sp,#64]
892	stp x27,x28,[sp,#80]
893
894	.L512_or_more_neon:
895	sub sp,sp,#128+64
896
897	eor $ONE,$ONE,$ONE
898	ldp @d[0],@d[1],[@x[0]] // load sigma
899	ld1 {@K[0]},[@x[0]],#16
900	ldp @d[2],@d[3],[$key] // load key
901	ldp @d[4],@d[5],[$key,#16]
902	ld1 {@K[1],@K[2]},[$key]
903	ldp @d[6],@d[7],[$ctr] // load counter
904	ld1 {@K[3]},[$ctr]
905	ld1 {$ONE}[0],[@x[0]]
906	add $key,@x[0],#16 // .Lrot24
907	#ifdef __AARCH64EB__
908	rev64 @K[0],@K[0]
909	ror @d[2],@d[2],#32
910	ror @d[3],@d[3],#32
911	ror @d[4],@d[4],#32
912	ror @d[5],@d[5],#32
913	ror @d[6],@d[6],#32
914	ror @d[7],@d[7],#32
915	#endif
916	add @K[3],@K[3],$ONE // += 1
917	stp @K[0],@K[1],[sp,#0] // off-load key block, invariant part
918	add @K[3],@K[3],$ONE // not typo
919	str @K[2],[sp,#32]
920	add @K[4],@K[3],$ONE
921	add @K[5],@K[4],$ONE
922	add @K[6],@K[5],$ONE
923	shl $ONE,$ONE,#2 // 1 -> 4
924
925	stp d8,d9,[sp,#128+0] // meet ABI requirements
926	stp d10,d11,[sp,#128+16]
927	stp d12,d13,[sp,#128+32]
928	stp d14,d15,[sp,#128+48]
929
930	sub $len,$len,#512 // not typo
931
932	.Loop_outer_512_neon:
933	mov $A0,@K[0]
934	mov $A1,@K[0]
935	mov $A2,@K[0]
936	mov $A3,@K[0]
937	mov $A4,@K[0]
938	mov $A5,@K[0]
939	mov $B0,@K[1]
940	mov.32 @x[0],@d[0] // unpack key block
941	mov $B1,@K[1]
942	lsr @x[1],@d[0],#32
943	mov $B2,@K[1]
944	mov.32 @x[2],@d[1]
945	mov $B3,@K[1]
946	lsr @x[3],@d[1],#32
947	mov $B4,@K[1]
948	mov.32 @x[4],@d[2]
949	mov $B5,@K[1]
950	lsr @x[5],@d[2],#32
951	mov $D0,@K[3]
952	mov.32 @x[6],@d[3]
953	mov $D1,@K[4]
954	lsr @x[7],@d[3],#32
955	mov $D2,@K[5]
956	mov.32 @x[8],@d[4]
957	mov $D3,@K[6]
958	lsr @x[9],@d[4],#32
959	mov $C0,@K[2]
960	mov.32 @x[10],@d[5]
961	mov $C1,@K[2]
962	lsr @x[11],@d[5],#32
963	add $D4,$D0,$ONE // +4
964	mov.32 @x[12],@d[6]
965	add $D5,$D1,$ONE // +4
966	lsr @x[13],@d[6],#32
967	mov $C2,@K[2]
968	mov.32 @x[14],@d[7]
969	mov $C3,@K[2]
970	lsr @x[15],@d[7],#32
971	mov $C4,@K[2]
972	stp @K[3],@K[4],[sp,#48] // off-load key block, variable part
973	mov $C5,@K[2]
974	stp @K[5],@K[6],[sp,#80]
975
976	mov $ctr,#5
977	ld1 {$rot24},[$key]
978	subs $len,$len,#512
979	.Loop_upper_neon:
980	sub $ctr,$ctr,#1
981	___
982	my @thread0=&NEONROUND($A0,$B0,$C0,$D0,$T0,0);
983	my @thread1=&NEONROUND($A1,$B1,$C1,$D1,$T1,0);
984	my @thread2=&NEONROUND($A2,$B2,$C2,$D2,$T2,0);
985	my @thread3=&NEONROUND($A3,$B3,$C3,$D3,$T3,0);
986	my @thread4=&NEONROUND($A4,$B4,$C4,$D4,$T4,0);
987	my @thread5=&NEONROUND($A5,$B5,$C5,$D5,$T5,0);
988	my @thread67=(&ROUND(0,4,8,12),&ROUND(0,5,10,15));
989	my $diff = ($#thread0+1)*6 - $#thread67 - 1;
990	my $i = 0;
991
992	foreach (@thread0) {
993	eval; eval(shift(@thread67));
994	eval(shift(@thread1)); eval(shift(@thread67));
995	eval(shift(@thread2)); eval(shift(@thread67));
996	eval(shift(@thread3)); eval(shift(@thread67));
997	eval(shift(@thread4)); eval(shift(@thread67));
998	eval(shift(@thread5)); eval(shift(@thread67));
999	}
1000
1001	@thread0=&NEONROUND($A0,$B0,$C0,$D0,$T0,1);
1002	@thread1=&NEONROUND($A1,$B1,$C1,$D1,$T1,1);
1003	@thread2=&NEONROUND($A2,$B2,$C2,$D2,$T2,1);
1004	@thread3=&NEONROUND($A3,$B3,$C3,$D3,$T3,1);
1005	@thread4=&NEONROUND($A4,$B4,$C4,$D4,$T4,1);
1006	@thread5=&NEONROUND($A5,$B5,$C5,$D5,$T5,1);
1007	@thread67=(&ROUND(0,4,8,12),&ROUND(0,5,10,15));
1008
1009	foreach (@thread0) {
1010	eval; eval(shift(@thread67));
1011	eval(shift(@thread1)); eval(shift(@thread67));
1012	eval(shift(@thread2)); eval(shift(@thread67));
1013	eval(shift(@thread3)); eval(shift(@thread67));
1014	eval(shift(@thread4)); eval(shift(@thread67));
1015	eval(shift(@thread5)); eval(shift(@thread67));
1016	}
1017	$code.=<<___;
1018	cbnz $ctr,.Loop_upper_neon
1019
1020	add.32 @x[0],@x[0],@d[0] // accumulate key block
1021	add @x[1],@x[1],@d[0],lsr#32
1022	add.32 @x[2],@x[2],@d[1]
1023	add @x[3],@x[3],@d[1],lsr#32
1024	add.32 @x[4],@x[4],@d[2]
1025	add @x[5],@x[5],@d[2],lsr#32
1026	add.32 @x[6],@x[6],@d[3]
1027	add @x[7],@x[7],@d[3],lsr#32
1028	add.32 @x[8],@x[8],@d[4]
1029	add @x[9],@x[9],@d[4],lsr#32
1030	add.32 @x[10],@x[10],@d[5]
1031	add @x[11],@x[11],@d[5],lsr#32
1032	add.32 @x[12],@x[12],@d[6]
1033	add @x[13],@x[13],@d[6],lsr#32
1034	add.32 @x[14],@x[14],@d[7]
1035	add @x[15],@x[15],@d[7],lsr#32
1036
1037	add @x[0],@x[0],@x[1],lsl#32 // pack
1038	add @x[2],@x[2],@x[3],lsl#32
1039	ldp @x[1],@x[3],[$inp,#0] // load input
1040	add @x[4],@x[4],@x[5],lsl#32
1041	add @x[6],@x[6],@x[7],lsl#32
1042	ldp @x[5],@x[7],[$inp,#16]
1043	add @x[8],@x[8],@x[9],lsl#32
1044	add @x[10],@x[10],@x[11],lsl#32
1045	ldp @x[9],@x[11],[$inp,#32]
1046	add @x[12],@x[12],@x[13],lsl#32
1047	add @x[14],@x[14],@x[15],lsl#32
1048	ldp @x[13],@x[15],[$inp,#48]
1049	add $inp,$inp,#64
1050	#ifdef __AARCH64EB__
1051	rev @x[0],@x[0]
1052	rev @x[2],@x[2]
1053	rev @x[4],@x[4]
1054	rev @x[6],@x[6]
1055	rev @x[8],@x[8]
1056	rev @x[10],@x[10]
1057	rev @x[12],@x[12]
1058	rev @x[14],@x[14]
1059	#endif
1060	eor @x[0],@x[0],@x[1]
1061	eor @x[2],@x[2],@x[3]
1062	eor @x[4],@x[4],@x[5]
1063	eor @x[6],@x[6],@x[7]
1064	eor @x[8],@x[8],@x[9]
1065	eor @x[10],@x[10],@x[11]
1066	eor @x[12],@x[12],@x[13]
1067	eor @x[14],@x[14],@x[15]
1068
1069	stp @x[0],@x[2],[$out,#0] // store output
1070	add @d[6],@d[6],#1 // increment counter
1071	mov.32 @x[0],@d[0] // unpack key block
1072	lsr @x[1],@d[0],#32
1073	stp @x[4],@x[6],[$out,#16]
1074	mov.32 @x[2],@d[1]
1075	lsr @x[3],@d[1],#32
1076	stp @x[8],@x[10],[$out,#32]
1077	mov.32 @x[4],@d[2]
1078	lsr @x[5],@d[2],#32
1079	stp @x[12],@x[14],[$out,#48]
1080	add $out,$out,#64
1081	mov.32 @x[6],@d[3]
1082	lsr @x[7],@d[3],#32
1083	mov.32 @x[8],@d[4]
1084	lsr @x[9],@d[4],#32
1085	mov.32 @x[10],@d[5]
1086	lsr @x[11],@d[5],#32
1087	mov.32 @x[12],@d[6]
1088	lsr @x[13],@d[6],#32
1089	mov.32 @x[14],@d[7]
1090	lsr @x[15],@d[7],#32
1091
1092	mov $ctr,#5
1093	.Loop_lower_neon:
1094	sub $ctr,$ctr,#1
1095	___
1096	@thread0=&NEONROUND($A0,$B0,$C0,$D0,$T0,0);
1097	@thread1=&NEONROUND($A1,$B1,$C1,$D1,$T1,0);
1098	@thread2=&NEONROUND($A2,$B2,$C2,$D2,$T2,0);
1099	@thread3=&NEONROUND($A3,$B3,$C3,$D3,$T3,0);
1100	@thread4=&NEONROUND($A4,$B4,$C4,$D4,$T4,0);
1101	@thread5=&NEONROUND($A5,$B5,$C5,$D5,$T5,0);
1102	@thread67=(&ROUND(0,4,8,12),&ROUND(0,5,10,15));
1103
1104	foreach (@thread0) {
1105	eval; eval(shift(@thread67));
1106	eval(shift(@thread1)); eval(shift(@thread67));
1107	eval(shift(@thread2)); eval(shift(@thread67));
1108	eval(shift(@thread3)); eval(shift(@thread67));
1109	eval(shift(@thread4)); eval(shift(@thread67));
1110	eval(shift(@thread5)); eval(shift(@thread67));
1111	}
1112
1113	@thread0=&NEONROUND($A0,$B0,$C0,$D0,$T0,1);
1114	@thread1=&NEONROUND($A1,$B1,$C1,$D1,$T1,1);
1115	@thread2=&NEONROUND($A2,$B2,$C2,$D2,$T2,1);
1116	@thread3=&NEONROUND($A3,$B3,$C3,$D3,$T3,1);
1117	@thread4=&NEONROUND($A4,$B4,$C4,$D4,$T4,1);
1118	@thread5=&NEONROUND($A5,$B5,$C5,$D5,$T5,1);
1119	@thread67=(&ROUND(0,4,8,12),&ROUND(0,5,10,15));
1120
1121	foreach (@thread0) {
1122	eval; eval(shift(@thread67));
1123	eval(shift(@thread1)); eval(shift(@thread67));
1124	eval(shift(@thread2)); eval(shift(@thread67));
1125	eval(shift(@thread3)); eval(shift(@thread67));
1126	eval(shift(@thread4)); eval(shift(@thread67));
1127	eval(shift(@thread5)); eval(shift(@thread67));
1128	}
1129	$code.=<<___;
1130	cbnz $ctr,.Loop_lower_neon
1131
1132	add.32 @x[0],@x[0],@d[0] // accumulate key block
1133	ldp @K[0],@K[1],[sp,#0]
1134	add @x[1],@x[1],@d[0],lsr#32
1135	ldp @K[2],@K[3],[sp,#32]
1136	add.32 @x[2],@x[2],@d[1]
1137	ldp @K[4],@K[5],[sp,#64]
1138	add @x[3],@x[3],@d[1],lsr#32
1139	ldr @K[6],[sp,#96]
1140	add $A0,$A0,@K[0]
1141	add.32 @x[4],@x[4],@d[2]
1142	add $A1,$A1,@K[0]
1143	add @x[5],@x[5],@d[2],lsr#32
1144	add $A2,$A2,@K[0]
1145	add.32 @x[6],@x[6],@d[3]
1146	add $A3,$A3,@K[0]
1147	add @x[7],@x[7],@d[3],lsr#32
1148	add $A4,$A4,@K[0]
1149	add.32 @x[8],@x[8],@d[4]
1150	add $A5,$A5,@K[0]
1151	add @x[9],@x[9],@d[4],lsr#32
1152	add $C0,$C0,@K[2]
1153	add.32 @x[10],@x[10],@d[5]
1154	add $C1,$C1,@K[2]
1155	add @x[11],@x[11],@d[5],lsr#32
1156	add $C2,$C2,@K[2]
1157	add.32 @x[12],@x[12],@d[6]
1158	add $C3,$C3,@K[2]
1159	add @x[13],@x[13],@d[6],lsr#32
1160	add $C4,$C4,@K[2]
1161	add.32 @x[14],@x[14],@d[7]
1162	add $C5,$C5,@K[2]
1163	add @x[15],@x[15],@d[7],lsr#32
1164	add $D4,$D4,$ONE // +4
1165	add @x[0],@x[0],@x[1],lsl#32 // pack
1166	add $D5,$D5,$ONE // +4
1167	add @x[2],@x[2],@x[3],lsl#32
1168	add $D0,$D0,@K[3]
1169	ldp @x[1],@x[3],[$inp,#0] // load input
1170	add $D1,$D1,@K[4]
1171	add @x[4],@x[4],@x[5],lsl#32
1172	add $D2,$D2,@K[5]
1173	add @x[6],@x[6],@x[7],lsl#32
1174	add $D3,$D3,@K[6]
1175	ldp @x[5],@x[7],[$inp,#16]
1176	add $D4,$D4,@K[3]
1177	add @x[8],@x[8],@x[9],lsl#32
1178	add $D5,$D5,@K[4]
1179	add @x[10],@x[10],@x[11],lsl#32
1180	add $B0,$B0,@K[1]
1181	ldp @x[9],@x[11],[$inp,#32]
1182	add $B1,$B1,@K[1]
1183	add @x[12],@x[12],@x[13],lsl#32
1184	add $B2,$B2,@K[1]
1185	add @x[14],@x[14],@x[15],lsl#32
1186	add $B3,$B3,@K[1]
1187	ldp @x[13],@x[15],[$inp,#48]
1188	add $B4,$B4,@K[1]
1189	add $inp,$inp,#64
1190	add $B5,$B5,@K[1]
1191
1192	#ifdef __AARCH64EB__
1193	rev @x[0],@x[0]
1194	rev @x[2],@x[2]
1195	rev @x[4],@x[4]
1196	rev @x[6],@x[6]
1197	rev @x[8],@x[8]
1198	rev @x[10],@x[10]
1199	rev @x[12],@x[12]
1200	rev @x[14],@x[14]
1201	#endif
1202	ld1.8 {$T0-$T3},[$inp],#64
1203	eor @x[0],@x[0],@x[1]
1204	eor @x[2],@x[2],@x[3]
1205	eor @x[4],@x[4],@x[5]
1206	eor @x[6],@x[6],@x[7]
1207	eor @x[8],@x[8],@x[9]
1208	eor $A0,$A0,$T0
1209	eor @x[10],@x[10],@x[11]
1210	eor $B0,$B0,$T1
1211	eor @x[12],@x[12],@x[13]
1212	eor $C0,$C0,$T2
1213	eor @x[14],@x[14],@x[15]
1214	eor $D0,$D0,$T3
1215	ld1.8 {$T0-$T3},[$inp],#64
1216
1217	stp @x[0],@x[2],[$out,#0] // store output
1218	add @d[6],@d[6],#7 // increment counter
1219	stp @x[4],@x[6],[$out,#16]
1220	stp @x[8],@x[10],[$out,#32]
1221	stp @x[12],@x[14],[$out,#48]
1222	add $out,$out,#64
1223	st1.8 {$A0-$D0},[$out],#64
1224
1225	ld1.8 {$A0-$D0},[$inp],#64
1226	eor $A1,$A1,$T0
1227	eor $B1,$B1,$T1
1228	eor $C1,$C1,$T2
1229	eor $D1,$D1,$T3
1230	st1.8 {$A1-$D1},[$out],#64
1231
1232	ld1.8 {$A1-$D1},[$inp],#64
1233	eor $A2,$A2,$A0
1234	ldp @K[0],@K[1],[sp,#0]
1235	eor $B2,$B2,$B0
1236	ldp @K[2],@K[3],[sp,#32]
1237	eor $C2,$C2,$C0
1238	eor $D2,$D2,$D0
1239	st1.8 {$A2-$D2},[$out],#64
1240
1241	ld1.8 {$A2-$D2},[$inp],#64
1242	eor $A3,$A3,$A1
1243	eor $B3,$B3,$B1
1244	eor $C3,$C3,$C1
1245	eor $D3,$D3,$D1
1246	st1.8 {$A3-$D3},[$out],#64
1247
1248	ld1.8 {$A3-$D3},[$inp],#64
1249	eor $A4,$A4,$A2
1250	eor $B4,$B4,$B2
1251	eor $C4,$C4,$C2
1252	eor $D4,$D4,$D2
1253	st1.8 {$A4-$D4},[$out],#64
1254
1255	shl $A0,$ONE,#1 // 4 -> 8
1256	eor $A5,$A5,$A3
1257	eor $B5,$B5,$B3
1258	eor $C5,$C5,$C3
1259	eor $D5,$D5,$D3
1260	st1.8 {$A5-$D5},[$out],#64
1261
1262	add @K[3],@K[3],$A0 // += 8
1263	add @K[4],@K[4],$A0
1264	add @K[5],@K[5],$A0
1265	add @K[6],@K[6],$A0
1266
1267	b.hs .Loop_outer_512_neon
1268
1269	adds $len,$len,#512
1270	ushr $ONE,$ONE,#1 // 4 -> 2
1271
1272	ldp d10,d11,[sp,#128+16] // meet ABI requirements
1273	ldp d12,d13,[sp,#128+32]
1274	ldp d14,d15,[sp,#128+48]
1275
1276	stp @K[0],@K[0],[sp,#0] // wipe off-load area
1277	stp @K[0],@K[0],[sp,#32]
1278	stp @K[0],@K[0],[sp,#64]
1279
1280	b.eq .Ldone_512_neon
1281
1282	sub $key,$key,#16 // .Lone
1283	cmp $len,#192
1284	add sp,sp,#128
1285	sub @K[3],@K[3],$ONE // -= 2
1286	ld1 {$CTR,$ROT24},[$key]
1287	b.hs .Loop_outer_neon
1288
1289	ldp d8,d9,[sp,#0] // meet ABI requirements
1290	eor @K[1],@K[1],@K[1]
1291	eor @K[2],@K[2],@K[2]
1292	eor @K[3],@K[3],@K[3]
1293	eor @K[4],@K[4],@K[4]
1294	eor @K[5],@K[5],@K[5]
1295	eor @K[6],@K[6],@K[6]
1296	b .Loop_outer
1297
1298	.Ldone_512_neon:
1299	ldp d8,d9,[sp,#128+0] // meet ABI requirements
1300	ldp x19,x20,[x29,#16]
1301	add sp,sp,#128+64
1302	ldp x21,x22,[x29,#32]
1303	ldp x23,x24,[x29,#48]
1304	ldp x25,x26,[x29,#64]
1305	ldp x27,x28,[x29,#80]
1306	ldp x29,x30,[sp],#96
1307	AARCH64_VALIDATE_LINK_REGISTER
1308	ret
1309	.size ChaCha20_512_neon,.-ChaCha20_512_neon
1310	___
1311	}
1312	}}}
1313
1314	foreach (split("\n",$code)) {
1315	s/\`([^\`]*)\`/eval $1/geo;
1316
1317	(s/\b([a-z]+)\.32\b/$1/ and (s/x([0-9]+)/w$1/g or 1)) or
1318	(m/\b(eor\|ext\|mov\|tbl)\b/ and (s/\.4s/\.16b/g or 1)) or
1319	(s/\b((?:ld\|st)1)\.8\b/$1/ and (s/\.4s/\.16b/g or 1)) or
1320	(m/\b(ld\|st)[rp]\b/ and (s/v([0-9]+)\.4s/q$1/g or 1)) or
1321	(m/\b(dup\|ld1)\b/ and (s/\.4(s}?\[[0-3]\])/.$1/g or 1)) or
1322	(s/\b(zip[12])\.64\b/$1/ and (s/\.4s/\.2d/g or 1)) or
1323	(s/\brev32\.16\b/rev32/ and (s/\.4s/\.8h/g or 1));
1324
1325	#s/\bq([0-9]+)#(lo\|hi)/sprintf "d%d",2*$1+($2 eq "hi")/geo;
1326
1327	print $_,"\n";
1328	}
1329	close STDOUT or die "error closing STDOUT: $!"; # flush

注意: 瀏覽 TracBrowser 來幫助您使用儲存庫瀏覽器

source: vbox/trunk/src/libs/openssl-3.1.7/crypto/chacha/asm/chacha-armv8.pl@ 108351

以其他格式下載: