am 02d138cf: am e9ada863: external/boringssl: bump revision.

* commit '02d138cf':
  external/boringssl: bump revision.

BORINGSSL_REVISION

-4cc1e838aef70be6194c2c77e1de9015b9b7410a
+5aa8a8643851e309b48a1b5a5d91d2fd183eae52

linux-aarch64/crypto/aes/aesv8-armx.S

@@ -6,7 +6,7 @@
 .arch	armv8-a+crypto
 #endif
 .align	5
-rcon:
+.Lrcon:
 .long	0x01,0x01,0x01,0x01
 .long	0x0c0f0e0d,0x0c0f0e0d,0x0c0f0e0d,0x0c0f0e0d	// rotate-n-splat
 .long	0x1b,0x1b,0x1b,0x1b
@@ -31,7 +31,7 @@ aes_v8_set_encrypt_key:
 	tst	w1,#0x3f
 	b.ne	.Lenc_key_abort
 
-	adr	x3,rcon
+	adr	x3,.Lrcon
 	cmp	w1,#192
 
 	eor	v0.16b,v0.16b,v0.16b
@@ -55,7 +55,7 @@ aes_v8_set_encrypt_key:
 	ext	v5.16b,v0.16b,v5.16b,#12
 	eor	v3.16b,v3.16b,v5.16b
 	ext	v5.16b,v0.16b,v5.16b,#12
-	 eor	v6.16b,v6.16b,v1.16b
+	eor	v6.16b,v6.16b,v1.16b
 	eor	v3.16b,v3.16b,v5.16b
 	shl	v1.16b,v1.16b,#1
 	eor	v3.16b,v3.16b,v6.16b
@@ -72,7 +72,7 @@ aes_v8_set_encrypt_key:
 	ext	v5.16b,v0.16b,v5.16b,#12
 	eor	v3.16b,v3.16b,v5.16b
 	ext	v5.16b,v0.16b,v5.16b,#12
-	 eor	v6.16b,v6.16b,v1.16b
+	eor	v6.16b,v6.16b,v1.16b
 	eor	v3.16b,v3.16b,v5.16b
 	shl	v1.16b,v1.16b,#1
 	eor	v3.16b,v3.16b,v6.16b
@@ -86,7 +86,7 @@ aes_v8_set_encrypt_key:
 	ext	v5.16b,v0.16b,v5.16b,#12
 	eor	v3.16b,v3.16b,v5.16b
 	ext	v5.16b,v0.16b,v5.16b,#12
-	 eor	v6.16b,v6.16b,v1.16b
+	eor	v6.16b,v6.16b,v1.16b
 	eor	v3.16b,v3.16b,v5.16b
 	eor	v3.16b,v3.16b,v6.16b
 	st1	{v3.4s},[x2]
@@ -117,7 +117,7 @@ aes_v8_set_encrypt_key:
 
 	dup	v5.4s,v3.s[3]
 	eor	v5.16b,v5.16b,v4.16b
-	 eor	v6.16b,v6.16b,v1.16b
+	eor	v6.16b,v6.16b,v1.16b
 	ext	v4.16b,v0.16b,v4.16b,#12
 	shl	v1.16b,v1.16b,#1
 	eor	v4.16b,v4.16b,v5.16b
@@ -148,7 +148,7 @@ aes_v8_set_encrypt_key:
 	ext	v5.16b,v0.16b,v5.16b,#12
 	eor	v3.16b,v3.16b,v5.16b
 	ext	v5.16b,v0.16b,v5.16b,#12
-	 eor	v6.16b,v6.16b,v1.16b
+	eor	v6.16b,v6.16b,v1.16b
 	eor	v3.16b,v3.16b,v5.16b
 	shl	v1.16b,v1.16b,#1
 	eor	v3.16b,v3.16b,v6.16b
@@ -229,17 +229,17 @@ aes_v8_encrypt:
 
 .Loop_enc:
 	aese	v2.16b,v0.16b
-	ld1	{v0.4s},[x2],#16
 	aesmc	v2.16b,v2.16b
+	ld1	{v0.4s},[x2],#16
 	subs	w3,w3,#2
 	aese	v2.16b,v1.16b
-	ld1	{v1.4s},[x2],#16
 	aesmc	v2.16b,v2.16b
+	ld1	{v1.4s},[x2],#16
 	b.gt	.Loop_enc
 
 	aese	v2.16b,v0.16b
-	ld1	{v0.4s},[x2]
 	aesmc	v2.16b,v2.16b
+	ld1	{v0.4s},[x2]
 	aese	v2.16b,v1.16b
 	eor	v2.16b,v2.16b,v0.16b
 
@@ -258,17 +258,17 @@ aes_v8_decrypt:
 
 .Loop_dec:
 	aesd	v2.16b,v0.16b
-	ld1	{v0.4s},[x2],#16
 	aesimc	v2.16b,v2.16b
+	ld1	{v0.4s},[x2],#16
 	subs	w3,w3,#2
 	aesd	v2.16b,v1.16b
-	ld1	{v1.4s},[x2],#16
 	aesimc	v2.16b,v2.16b
+	ld1	{v1.4s},[x2],#16
 	b.gt	.Loop_dec
 
 	aesd	v2.16b,v0.16b
-	ld1	{v0.4s},[x2]
 	aesimc	v2.16b,v2.16b
+	ld1	{v0.4s},[x2]
 	aesd	v2.16b,v1.16b
 	eor	v2.16b,v2.16b,v0.16b
 
@@ -292,13 +292,13 @@ aes_v8_cbc_encrypt:
 	ld1	{v6.16b},[x4]
 	ld1	{v0.16b},[x0],x8
 
-	ld1	{v16.4s-v17.4s},[x3]		// load key schedule...
+	ld1	{v16.4s,v17.4s},[x3]		// load key schedule...
 	sub	w5,w5,#6
 	add	x7,x3,x5,lsl#4	// pointer to last 7 round keys
 	sub	w5,w5,#2
-	ld1	{v18.4s-v19.4s},[x7],#32
-	ld1	{v20.4s-v21.4s},[x7],#32
-	ld1	{v22.4s-v23.4s},[x7],#32
+	ld1	{v18.4s,v19.4s},[x7],#32
+	ld1	{v20.4s,v21.4s},[x7],#32
+	ld1	{v22.4s,v23.4s},[x7],#32
 	ld1	{v7.4s},[x7]
 
 	add	x7,x3,#32
@@ -310,76 +310,99 @@ aes_v8_cbc_encrypt:
 	eor	v5.16b,v16.16b,v7.16b
 	b.eq	.Lcbc_enc128
 
+	ld1	{v2.4s,v3.4s},[x7]
+	add	x7,x3,#16
+	add	x6,x3,#16*4
+	add	x12,x3,#16*5
+	aese	v0.16b,v16.16b
+	aesmc	v0.16b,v0.16b
+	add	x14,x3,#16*6
+	add	x3,x3,#16*7
+	b	.Lenter_cbc_enc
+
+.align	4
 .Loop_cbc_enc:
 	aese	v0.16b,v16.16b
-	ld1	{v16.4s},[x7],#16
 	aesmc	v0.16b,v0.16b
-	subs	w6,w6,#2
+	st1	{v6.16b},[x1],#16
+.Lenter_cbc_enc:
 	aese	v0.16b,v17.16b
-	ld1	{v17.4s},[x7],#16
 	aesmc	v0.16b,v0.16b
-	b.gt	.Loop_cbc_enc
+	aese	v0.16b,v2.16b
+	aesmc	v0.16b,v0.16b
+	ld1	{v16.4s},[x6]
+	cmp	w5,#4
+	aese	v0.16b,v3.16b
+	aesmc	v0.16b,v0.16b
+	ld1	{v17.4s},[x12]
+	b.eq	.Lcbc_enc192
 
 	aese	v0.16b,v16.16b
 	aesmc	v0.16b,v0.16b
-	 subs	x2,x2,#16
+	ld1	{v16.4s},[x14]
 	aese	v0.16b,v17.16b
 	aesmc	v0.16b,v0.16b
-	 csel	x8,xzr,x8,eq
+	ld1	{v17.4s},[x3]
+	nop
+
+.Lcbc_enc192:
+	aese	v0.16b,v16.16b
+	aesmc	v0.16b,v0.16b
+	subs	x2,x2,#16
+	aese	v0.16b,v17.16b
+	aesmc	v0.16b,v0.16b
+	csel	x8,xzr,x8,eq
 	aese	v0.16b,v18.16b
 	aesmc	v0.16b,v0.16b
-	 add	x7,x3,#16
 	aese	v0.16b,v19.16b
 	aesmc	v0.16b,v0.16b
-	 ld1	{v16.16b},[x0],x8
+	ld1	{v16.16b},[x0],x8
 	aese	v0.16b,v20.16b
 	aesmc	v0.16b,v0.16b
-	 eor	v16.16b,v16.16b,v5.16b
+	eor	v16.16b,v16.16b,v5.16b
 	aese	v0.16b,v21.16b
 	aesmc	v0.16b,v0.16b
-	 ld1 {v17.4s},[x7],#16	// re-pre-load rndkey[1]
+	ld1	{v17.4s},[x7]		// re-pre-load rndkey[1]
 	aese	v0.16b,v22.16b
 	aesmc	v0.16b,v0.16b
 	aese	v0.16b,v23.16b
-
-	 mov	w6,w5
 	eor	v6.16b,v0.16b,v7.16b
-	st1	{v6.16b},[x1],#16
 	b.hs	.Loop_cbc_enc
 
+	st1	{v6.16b},[x1],#16
 	b	.Lcbc_done
 
 .align	5
 .Lcbc_enc128:
-	ld1	{v2.4s-v3.4s},[x7]
+	ld1	{v2.4s,v3.4s},[x7]
 	aese	v0.16b,v16.16b
 	aesmc	v0.16b,v0.16b
 	b	.Lenter_cbc_enc128
 .Loop_cbc_enc128:
 	aese	v0.16b,v16.16b
 	aesmc	v0.16b,v0.16b
-	 st1	{v6.16b},[x1],#16
+	st1	{v6.16b},[x1],#16
 .Lenter_cbc_enc128:
 	aese	v0.16b,v17.16b
 	aesmc	v0.16b,v0.16b
-	 subs	x2,x2,#16
+	subs	x2,x2,#16
 	aese	v0.16b,v2.16b
 	aesmc	v0.16b,v0.16b
-	 csel	x8,xzr,x8,eq
+	csel	x8,xzr,x8,eq
 	aese	v0.16b,v3.16b
 	aesmc	v0.16b,v0.16b
 	aese	v0.16b,v18.16b
 	aesmc	v0.16b,v0.16b
 	aese	v0.16b,v19.16b
 	aesmc	v0.16b,v0.16b
-	 ld1	{v16.16b},[x0],x8
+	ld1	{v16.16b},[x0],x8
 	aese	v0.16b,v20.16b
 	aesmc	v0.16b,v0.16b
 	aese	v0.16b,v21.16b
 	aesmc	v0.16b,v0.16b
 	aese	v0.16b,v22.16b
 	aesmc	v0.16b,v0.16b
-	 eor	v16.16b,v16.16b,v5.16b
+	eor	v16.16b,v16.16b,v5.16b
 	aese	v0.16b,v23.16b
 	eor	v6.16b,v0.16b,v7.16b
 	b.hs	.Loop_cbc_enc128
@@ -404,81 +427,80 @@ aes_v8_cbc_encrypt:
 
 .Loop3x_cbc_dec:
 	aesd	v0.16b,v16.16b
-	aesd	v1.16b,v16.16b
-	aesd	v18.16b,v16.16b
-	ld1	{v16.4s},[x7],#16
 	aesimc	v0.16b,v0.16b
+	aesd	v1.16b,v16.16b
 	aesimc	v1.16b,v1.16b
+	aesd	v18.16b,v16.16b
 	aesimc	v18.16b,v18.16b
+	ld1	{v16.4s},[x7],#16
 	subs	w6,w6,#2
 	aesd	v0.16b,v17.16b
-	aesd	v1.16b,v17.16b
-	aesd	v18.16b,v17.16b
-	ld1	{v17.4s},[x7],#16
 	aesimc	v0.16b,v0.16b
+	aesd	v1.16b,v17.16b
 	aesimc	v1.16b,v1.16b
+	aesd	v18.16b,v17.16b
 	aesimc	v18.16b,v18.16b
+	ld1	{v17.4s},[x7],#16
 	b.gt	.Loop3x_cbc_dec
 
 	aesd	v0.16b,v16.16b
-	aesd	v1.16b,v16.16b
-	aesd	v18.16b,v16.16b
-	 eor	v4.16b,v6.16b,v7.16b
 	aesimc	v0.16b,v0.16b
+	aesd	v1.16b,v16.16b
 	aesimc	v1.16b,v1.16b
+	aesd	v18.16b,v16.16b
 	aesimc	v18.16b,v18.16b
-	 eor	v5.16b,v2.16b,v7.16b
+	eor	v4.16b,v6.16b,v7.16b
+	subs	x2,x2,#0x30
+	eor	v5.16b,v2.16b,v7.16b
+	csel	x6,x2,x6,lo			// x6, w6, is zero at this point
 	aesd	v0.16b,v17.16b
-	aesd	v1.16b,v17.16b
-	aesd	v18.16b,v17.16b
-	 eor	v17.16b,v3.16b,v7.16b
-	 subs	x2,x2,#0x30
 	aesimc	v0.16b,v0.16b
+	aesd	v1.16b,v17.16b
 	aesimc	v1.16b,v1.16b
+	aesd	v18.16b,v17.16b
 	aesimc	v18.16b,v18.16b
-	 orr	v6.16b,v19.16b,v19.16b
-	 csel	x6,x2,x6,lo			// x6, w6, is zero at this point
-	aesd	v0.16b,v20.16b
-	aesd	v1.16b,v20.16b
-	aesd	v18.16b,v20.16b
-	 add	x0,x0,x6		// x0 is adjusted in such way that
+	eor	v17.16b,v3.16b,v7.16b
+	add	x0,x0,x6		// x0 is adjusted in such way that
 					// at exit from the loop v1.16b-v18.16b
 					// are loaded with last "words"
+	orr	v6.16b,v19.16b,v19.16b
+	mov	x7,x3
+	aesd	v0.16b,v20.16b
 	aesimc	v0.16b,v0.16b
+	aesd	v1.16b,v20.16b
 	aesimc	v1.16b,v1.16b
+	aesd	v18.16b,v20.16b
 	aesimc	v18.16b,v18.16b
-	 mov	x7,x3
+	ld1	{v2.16b},[x0],#16
 	aesd	v0.16b,v21.16b
-	aesd	v1.16b,v21.16b
-	aesd	v18.16b,v21.16b
-	 ld1	{v2.16b},[x0],#16
 	aesimc	v0.16b,v0.16b
+	aesd	v1.16b,v21.16b
 	aesimc	v1.16b,v1.16b
+	aesd	v18.16b,v21.16b
 	aesimc	v18.16b,v18.16b
-	 ld1	{v3.16b},[x0],#16
+	ld1	{v3.16b},[x0],#16
 	aesd	v0.16b,v22.16b
-	aesd	v1.16b,v22.16b
-	aesd	v18.16b,v22.16b
-	 ld1	{v19.16b},[x0],#16
 	aesimc	v0.16b,v0.16b
+	aesd	v1.16b,v22.16b
 	aesimc	v1.16b,v1.16b
+	aesd	v18.16b,v22.16b
 	aesimc	v18.16b,v18.16b
-	 ld1 {v16.4s},[x7],#16	// re-pre-load rndkey[0]
+	ld1	{v19.16b},[x0],#16
 	aesd	v0.16b,v23.16b
 	aesd	v1.16b,v23.16b
 	aesd	v18.16b,v23.16b
-
-	 add	w6,w5,#2
+	ld1	{v16.4s},[x7],#16	// re-pre-load rndkey[0]
+	add	w6,w5,#2
 	eor	v4.16b,v4.16b,v0.16b
 	eor	v5.16b,v5.16b,v1.16b
 	eor	v18.16b,v18.16b,v17.16b
-	 ld1 {v17.4s},[x7],#16	// re-pre-load rndkey[1]
-	 orr	v0.16b,v2.16b,v2.16b
+	ld1	{v17.4s},[x7],#16	// re-pre-load rndkey[1]
 	st1	{v4.16b},[x1],#16
-	 orr	v1.16b,v3.16b,v3.16b
+	orr	v0.16b,v2.16b,v2.16b
 	st1	{v5.16b},[x1],#16
+	orr	v1.16b,v3.16b,v3.16b
 	st1	{v18.16b},[x1],#16
-	 orr	v18.16b,v19.16b,v19.16b
+	orr	v18.16b,v19.16b,v19.16b
 	b.hs	.Loop3x_cbc_dec
 
 	cmn	x2,#0x30
@@ -487,54 +509,54 @@ aes_v8_cbc_encrypt:
 
 .Lcbc_dec_tail:
 	aesd	v1.16b,v16.16b
-	aesd	v18.16b,v16.16b
-	ld1	{v16.4s},[x7],#16
 	aesimc	v1.16b,v1.16b
+	aesd	v18.16b,v16.16b
 	aesimc	v18.16b,v18.16b
+	ld1	{v16.4s},[x7],#16
 	subs	w6,w6,#2
 	aesd	v1.16b,v17.16b
-	aesd	v18.16b,v17.16b
-	ld1	{v17.4s},[x7],#16
 	aesimc	v1.16b,v1.16b
+	aesd	v18.16b,v17.16b
 	aesimc	v18.16b,v18.16b
+	ld1	{v17.4s},[x7],#16
 	b.gt	.Lcbc_dec_tail
 
 	aesd	v1.16b,v16.16b
-	aesd	v18.16b,v16.16b
 	aesimc	v1.16b,v1.16b
+	aesd	v18.16b,v16.16b
 	aesimc	v18.16b,v18.16b
 	aesd	v1.16b,v17.16b
-	aesd	v18.16b,v17.16b
 	aesimc	v1.16b,v1.16b
+	aesd	v18.16b,v17.16b
 	aesimc	v18.16b,v18.16b
 	aesd	v1.16b,v20.16b
-	aesd	v18.16b,v20.16b
 	aesimc	v1.16b,v1.16b
+	aesd	v18.16b,v20.16b
 	aesimc	v18.16b,v18.16b
-	 cmn	x2,#0x20
+	cmn	x2,#0x20
 	aesd	v1.16b,v21.16b
-	aesd	v18.16b,v21.16b
 	aesimc	v1.16b,v1.16b
+	aesd	v18.16b,v21.16b
 	aesimc	v18.16b,v18.16b
-	 eor	v5.16b,v6.16b,v7.16b
+	eor	v5.16b,v6.16b,v7.16b
 	aesd	v1.16b,v22.16b
-	aesd	v18.16b,v22.16b
 	aesimc	v1.16b,v1.16b
+	aesd	v18.16b,v22.16b
 	aesimc	v18.16b,v18.16b
-	 eor	v17.16b,v3.16b,v7.16b
+	eor	v17.16b,v3.16b,v7.16b
 	aesd	v1.16b,v23.16b
 	aesd	v18.16b,v23.16b
 	b.eq	.Lcbc_dec_one
 	eor	v5.16b,v5.16b,v1.16b
 	eor	v17.16b,v17.16b,v18.16b
-	 orr	v6.16b,v19.16b,v19.16b
+	orr	v6.16b,v19.16b,v19.16b
 	st1	{v5.16b},[x1],#16
 	st1	{v17.16b},[x1],#16
 	b	.Lcbc_done
 
 .Lcbc_dec_one:
 	eor	v5.16b,v5.16b,v18.16b
-	 orr	v6.16b,v19.16b,v19.16b
+	orr	v6.16b,v19.16b,v19.16b
 	st1	{v5.16b},[x1],#16
 
 .Lcbc_done:
@@ -547,181 +569,181 @@ aes_v8_cbc_encrypt:
 .type	aes_v8_ctr32_encrypt_blocks,%function
 .align	5
 aes_v8_ctr32_encrypt_blocks:
-	stp		x29,x30,[sp,#-16]!
-	add		x29,sp,#0
-	ldr		w5,[x3,#240]
-
-	ldr		w8, [x4, #12]
-	ld1		{v0.4s},[x4]
-
-	ld1		{v16.4s-v17.4s},[x3]		// load key schedule...
-	sub		w5,w5,#4
-	mov		x12,#16
-	cmp		x2,#2
-	add		x7,x3,x5,lsl#4	// pointer to last 5 round keys
-	sub		w5,w5,#2
-	ld1		{v20.4s-v21.4s},[x7],#32
-	ld1		{v22.4s-v23.4s},[x7],#32
-	ld1		{v7.4s},[x7]
-	add		x7,x3,#32
-	mov		w6,w5
+	stp	x29,x30,[sp,#-16]!
+	add	x29,sp,#0
+	ldr	w5,[x3,#240]
+
+	ldr	w8, [x4, #12]
+	ld1	{v0.4s},[x4]
+
+	ld1	{v16.4s,v17.4s},[x3]		// load key schedule...
+	sub	w5,w5,#4
+	mov	x12,#16
+	cmp	x2,#2
+	add	x7,x3,x5,lsl#4	// pointer to last 5 round keys
+	sub	w5,w5,#2
+	ld1	{v20.4s,v21.4s},[x7],#32
+	ld1	{v22.4s,v23.4s},[x7],#32
+	ld1	{v7.4s},[x7]
+	add	x7,x3,#32
+	mov	w6,w5
 	csel	x12,xzr,x12,lo
 #ifndef __ARMEB__
-	rev		w8, w8
+	rev	w8, w8
 #endif
-	orr		v1.16b,v0.16b,v0.16b
-	add		w10, w8, #1
-	orr		v18.16b,v0.16b,v0.16b
-	add		w8, w8, #2
-	orr		v6.16b,v0.16b,v0.16b
-	rev		w10, w10
-	mov		v1.s[3],w10
-	b.ls		.Lctr32_tail
-	rev		w12, w8
-	sub		x2,x2,#3		// bias
-	mov		v18.s[3],w12
-	b		.Loop3x_ctr32
+	orr	v1.16b,v0.16b,v0.16b
+	add	w10, w8, #1
+	orr	v18.16b,v0.16b,v0.16b
+	add	w8, w8, #2
+	orr	v6.16b,v0.16b,v0.16b
+	rev	w10, w10
+	mov	v1.s[3],w10
+	b.ls	.Lctr32_tail
+	rev	w12, w8
+	sub	x2,x2,#3		// bias
+	mov	v18.s[3],w12
+	b	.Loop3x_ctr32
 
 .align	4
 .Loop3x_ctr32:
-	aese		v0.16b,v16.16b
-	aese		v1.16b,v16.16b
-	aese		v18.16b,v16.16b
-	ld1		{v16.4s},[x7],#16
-	aesmc		v0.16b,v0.16b
-	aesmc		v1.16b,v1.16b
-	aesmc		v18.16b,v18.16b
-	subs		w6,w6,#2
-	aese		v0.16b,v17.16b
-	aese		v1.16b,v17.16b
-	aese		v18.16b,v17.16b
-	ld1		{v17.4s},[x7],#16
-	aesmc		v0.16b,v0.16b
-	aesmc		v1.16b,v1.16b
-	aesmc		v18.16b,v18.16b
-	b.gt		.Loop3x_ctr32
-
-	aese		v0.16b,v16.16b
-	aese		v1.16b,v16.16b
-	aese		v18.16b,v16.16b
-	 mov		x7,x3
-	aesmc		v4.16b,v0.16b
-	 ld1		{v2.16b},[x0],#16
-	aesmc		v5.16b,v1.16b
-	aesmc		v18.16b,v18.16b
-	 orr		v0.16b,v6.16b,v6.16b
-	aese		v4.16b,v17.16b
-	 ld1		{v3.16b},[x0],#16
-	aese		v5.16b,v17.16b
-	aese		v18.16b,v17.16b
-	 orr		v1.16b,v6.16b,v6.16b
-	aesmc		v4.16b,v4.16b
-	 ld1		{v19.16b},[x0],#16
-	aesmc		v5.16b,v5.16b
-	aesmc		v17.16b,v18.16b
-	 orr		v18.16b,v6.16b,v6.16b
-	 add		w9,w8,#1
-	aese		v4.16b,v20.16b
-	aese		v5.16b,v20.16b
-	aese		v17.16b,v20.16b
-	 eor		v2.16b,v2.16b,v7.16b
-	 add		w10,w8,#2
-	aesmc		v4.16b,v4.16b
-	aesmc		v5.16b,v5.16b
-	aesmc		v17.16b,v17.16b
-	 eor		v3.16b,v3.16b,v7.16b
-	 add		w8,w8,#3
-	aese		v4.16b,v21.16b
-	aese		v5.16b,v21.16b
-	aese		v17.16b,v21.16b
-	 eor		v19.16b,v19.16b,v7.16b
-	 rev		w9,w9
-	aesmc		v4.16b,v4.16b
-	 ld1	 {v16.4s},[x7],#16	// re-pre-load rndkey[0]
-	aesmc		v5.16b,v5.16b
-	aesmc		v17.16b,v17.16b
-	 mov	v0.s[3], w9
-	 rev		w10,w10
-	aese		v4.16b,v22.16b
-	aese		v5.16b,v22.16b
-	aese		v17.16b,v22.16b
-	 mov	v1.s[3], w10
-	 rev		w12,w8
-	aesmc		v4.16b,v4.16b
-	aesmc		v5.16b,v5.16b
-	aesmc		v17.16b,v17.16b
-	 mov	v18.s[3], w12
-	 subs		x2,x2,#3
-	aese		v4.16b,v23.16b
-	aese		v5.16b,v23.16b
-	aese		v17.16b,v23.16b
-
-	 mov		w6,w5
-	eor		v2.16b,v2.16b,v4.16b
-	eor		v3.16b,v3.16b,v5.16b
-	eor		v19.16b,v19.16b,v17.16b
-	 ld1	 {v17.4s},[x7],#16	// re-pre-load rndkey[1]
-	st1		{v2.16b},[x1],#16
-	st1		{v3.16b},[x1],#16
-	st1		{v19.16b},[x1],#16
-	b.hs		.Loop3x_ctr32
-
-	adds		x2,x2,#3
-	b.eq		.Lctr32_done
-	cmp		x2,#1
-	mov		x12,#16
+	aese	v0.16b,v16.16b
+	aesmc	v0.16b,v0.16b
+	aese	v1.16b,v16.16b
+	aesmc	v1.16b,v1.16b
+	aese	v18.16b,v16.16b
+	aesmc	v18.16b,v18.16b
+	ld1	{v16.4s},[x7],#16
+	subs	w6,w6,#2
+	aese	v0.16b,v17.16b
+	aesmc	v0.16b,v0.16b
+	aese	v1.16b,v17.16b
+	aesmc	v1.16b,v1.16b
+	aese	v18.16b,v17.16b
+	aesmc	v18.16b,v18.16b
+	ld1	{v17.4s},[x7],#16
+	b.gt	.Loop3x_ctr32
+
+	aese	v0.16b,v16.16b
+	aesmc	v4.16b,v0.16b
+	aese	v1.16b,v16.16b
+	aesmc	v5.16b,v1.16b
+	ld1	{v2.16b},[x0],#16
+	orr	v0.16b,v6.16b,v6.16b
+	aese	v18.16b,v16.16b
+	aesmc	v18.16b,v18.16b
+	ld1	{v3.16b},[x0],#16
+	orr	v1.16b,v6.16b,v6.16b
+	aese	v4.16b,v17.16b
+	aesmc	v4.16b,v4.16b
+	aese	v5.16b,v17.16b
+	aesmc	v5.16b,v5.16b
+	ld1	{v19.16b},[x0],#16
+	mov	x7,x3
+	aese	v18.16b,v17.16b
+	aesmc	v17.16b,v18.16b
+	orr	v18.16b,v6.16b,v6.16b
+	add	w9,w8,#1
+	aese	v4.16b,v20.16b
+	aesmc	v4.16b,v4.16b
+	aese	v5.16b,v20.16b
+	aesmc	v5.16b,v5.16b
+	eor	v2.16b,v2.16b,v7.16b
+	add	w10,w8,#2
+	aese	v17.16b,v20.16b
+	aesmc	v17.16b,v17.16b
+	eor	v3.16b,v3.16b,v7.16b
+	add	w8,w8,#3
+	aese	v4.16b,v21.16b
+	aesmc	v4.16b,v4.16b
+	aese	v5.16b,v21.16b
+	aesmc	v5.16b,v5.16b
+	eor	v19.16b,v19.16b,v7.16b
+	rev	w9,w9
+	aese	v17.16b,v21.16b
+	aesmc	v17.16b,v17.16b
+	mov	v0.s[3], w9
+	rev	w10,w10
+	aese	v4.16b,v22.16b
+	aesmc	v4.16b,v4.16b
+	aese	v5.16b,v22.16b
+	aesmc	v5.16b,v5.16b
+	mov	v1.s[3], w10
+	rev	w12,w8
+	aese	v17.16b,v22.16b
+	aesmc	v17.16b,v17.16b
+	mov	v18.s[3], w12
+	subs	x2,x2,#3
+	aese	v4.16b,v23.16b
+	aese	v5.16b,v23.16b
+	aese	v17.16b,v23.16b
+
+	eor	v2.16b,v2.16b,v4.16b
+	ld1	{v16.4s},[x7],#16	// re-pre-load rndkey[0]
+	st1	{v2.16b},[x1],#16
+	eor	v3.16b,v3.16b,v5.16b
+	mov	w6,w5
+	st1	{v3.16b},[x1],#16
+	eor	v19.16b,v19.16b,v17.16b
+	ld1	{v17.4s},[x7],#16	// re-pre-load rndkey[1]
+	st1	{v19.16b},[x1],#16
+	b.hs	.Loop3x_ctr32
+
+	adds	x2,x2,#3
+	b.eq	.Lctr32_done
+	cmp	x2,#1
+	mov	x12,#16
 	csel	x12,xzr,x12,eq
 
 .Lctr32_tail:
-	aese		v0.16b,v16.16b
-	aese		v1.16b,v16.16b
-	ld1		{v16.4s},[x7],#16
-	aesmc		v0.16b,v0.16b
-	aesmc		v1.16b,v1.16b
-	subs		w6,w6,#2
-	aese		v0.16b,v17.16b
-	aese		v1.16b,v17.16b
-	ld1		{v17.4s},[x7],#16
-	aesmc		v0.16b,v0.16b
-	aesmc		v1.16b,v1.16b
-	b.gt		.Lctr32_tail
-
-	aese		v0.16b,v16.16b
-	aese		v1.16b,v16.16b
-	aesmc		v0.16b,v0.16b
-	aesmc		v1.16b,v1.16b
-	aese		v0.16b,v17.16b
-	aese		v1.16b,v17.16b
-	aesmc		v0.16b,v0.16b
-	aesmc		v1.16b,v1.16b
-	 ld1		{v2.16b},[x0],x12
-	aese		v0.16b,v20.16b
-	aese		v1.16b,v20.16b
-	 ld1		{v3.16b},[x0]
-	aesmc		v0.16b,v0.16b
-	aesmc		v1.16b,v1.16b
-	aese		v0.16b,v21.16b
-	aese		v1.16b,v21.16b
-	aesmc		v0.16b,v0.16b
-	aesmc		v1.16b,v1.16b
-	aese		v0.16b,v22.16b
-	aese		v1.16b,v22.16b
-	 eor		v2.16b,v2.16b,v7.16b
-	aesmc		v0.16b,v0.16b
-	aesmc		v1.16b,v1.16b
-	 eor		v3.16b,v3.16b,v7.16b
-	aese		v0.16b,v23.16b
-	aese		v1.16b,v23.16b
-
-	cmp		x2,#1
-	eor		v2.16b,v2.16b,v0.16b
-	eor		v3.16b,v3.16b,v1.16b
-	st1		{v2.16b},[x1],#16
-	b.eq		.Lctr32_done
-	st1		{v3.16b},[x1]
+	aese	v0.16b,v16.16b
+	aesmc	v0.16b,v0.16b
+	aese	v1.16b,v16.16b
+	aesmc	v1.16b,v1.16b
+	ld1	{v16.4s},[x7],#16
+	subs	w6,w6,#2
+	aese	v0.16b,v17.16b
+	aesmc	v0.16b,v0.16b
+	aese	v1.16b,v17.16b
+	aesmc	v1.16b,v1.16b
+	ld1	{v17.4s},[x7],#16
+	b.gt	.Lctr32_tail
+
+	aese	v0.16b,v16.16b
+	aesmc	v0.16b,v0.16b
+	aese	v1.16b,v16.16b
+	aesmc	v1.16b,v1.16b
+	aese	v0.16b,v17.16b
+	aesmc	v0.16b,v0.16b
+	aese	v1.16b,v17.16b
+	aesmc	v1.16b,v1.16b
+	ld1	{v2.16b},[x0],x12
+	aese	v0.16b,v20.16b
+	aesmc	v0.16b,v0.16b
+	aese	v1.16b,v20.16b
+	aesmc	v1.16b,v1.16b
+	ld1	{v3.16b},[x0]
+	aese	v0.16b,v21.16b
+	aesmc	v0.16b,v0.16b
+	aese	v1.16b,v21.16b
+	aesmc	v1.16b,v1.16b
+	eor	v2.16b,v2.16b,v7.16b
+	aese	v0.16b,v22.16b
+	aesmc	v0.16b,v0.16b
+	aese	v1.16b,v22.16b
+	aesmc	v1.16b,v1.16b
+	eor	v3.16b,v3.16b,v7.16b
+	aese	v0.16b,v23.16b
+	aese	v1.16b,v23.16b
+
+	cmp	x2,#1
+	eor	v2.16b,v2.16b,v0.16b
+	eor	v3.16b,v3.16b,v1.16b
+	st1	{v2.16b},[x1],#16
+	b.eq	.Lctr32_done
+	st1	{v3.16b},[x1]
 
 .Lctr32_done:
-	ldr		x29,[sp],#16
+	ldr	x29,[sp],#16
 	ret
 .size	aes_v8_ctr32_encrypt_blocks,.-aes_v8_ctr32_encrypt_blocks
 #endif

linux-aarch64/crypto/modes/ghashv8-armx.S

@@ -4,114 +4,227 @@
 #if !defined(__clang__)
 .arch	armv8-a+crypto
 #endif
-.global	gcm_init_v8
+.globl	gcm_init_v8
 .type	gcm_init_v8,%function
 .align	4
 gcm_init_v8:
-	ld1		{v17.2d},[x1]		//load H
-	movi		v16.16b,#0xe1
-	ext		v3.16b,v17.16b,v17.16b,#8
-	shl	v16.2d,v16.2d,#57
-	ushr	v18.2d,v16.2d,#63
-	ext		v16.16b,v18.16b,v16.16b,#8		//t0=0xc2....01
-	dup		v17.4s,v17.s[1]
-	ushr	v19.2d,v3.2d,#63
+	ld1	{v17.2d},[x1]		//load input H
+	movi	v19.16b,#0xe1
+	shl	v19.2d,v19.2d,#57		//0xc2.0
+	ext	v3.16b,v17.16b,v17.16b,#8
+	ushr	v18.2d,v19.2d,#63
+	dup	v17.4s,v17.s[1]
+	ext	v16.16b,v18.16b,v19.16b,#8		//t0=0xc2....01
+	ushr	v18.2d,v3.2d,#63
 	sshr	v17.4s,v17.4s,#31		//broadcast carry bit
-	and		v19.16b,v19.16b,v16.16b
+	and	v18.16b,v18.16b,v16.16b
 	shl	v3.2d,v3.2d,#1
-	ext		v19.16b,v19.16b,v19.16b,#8
-	and		v16.16b,v16.16b,v17.16b
-	orr		v3.16b,v3.16b,v19.16b		//H<<<=1
-	eor		v3.16b,v3.16b,v16.16b		//twisted H
-	st1		{v3.2d},[x0]
+	ext	v18.16b,v18.16b,v18.16b,#8
+	and	v16.16b,v16.16b,v17.16b
+	orr	v3.16b,v3.16b,v18.16b		//H<<<=1
+	eor	v20.16b,v3.16b,v16.16b		//twisted H
+	st1	{v20.2d},[x0],#16		//store Htable[0]
+
+	//calculate H^2
+	ext	v16.16b,v20.16b,v20.16b,#8		//Karatsuba pre-processing
+	pmull	v0.1q,v20.1d,v20.1d
+	eor	v16.16b,v16.16b,v20.16b
+	pmull2	v2.1q,v20.2d,v20.2d
+	pmull	v1.1q,v16.1d,v16.1d
+
+	ext	v17.16b,v0.16b,v2.16b,#8		//Karatsuba post-processing
+	eor	v18.16b,v0.16b,v2.16b
+	eor	v1.16b,v1.16b,v17.16b
+	eor	v1.16b,v1.16b,v18.16b
+	pmull	v18.1q,v0.1d,v19.1d		//1st phase
+
+	ins	v2.d[0],v1.d[1]
+	ins	v1.d[1],v0.d[0]
+	eor	v0.16b,v1.16b,v18.16b
+
+	ext	v18.16b,v0.16b,v0.16b,#8		//2nd phase
+	pmull	v0.1q,v0.1d,v19.1d
+	eor	v18.16b,v18.16b,v2.16b
+	eor	v22.16b,v0.16b,v18.16b
+
+	ext	v17.16b,v22.16b,v22.16b,#8		//Karatsuba pre-processing
+	eor	v17.16b,v17.16b,v22.16b
+	ext	v21.16b,v16.16b,v17.16b,#8		//pack Karatsuba pre-processed
+	st1	{v21.2d,v22.2d},[x0]		//store Htable[1..2]
 
 	ret
 .size	gcm_init_v8,.-gcm_init_v8
-
-.global	gcm_gmult_v8
+.globl	gcm_gmult_v8
 .type	gcm_gmult_v8,%function
 .align	4
 gcm_gmult_v8:
-	ld1		{v17.2d},[x0]		//load Xi
-	movi		v19.16b,#0xe1
-	ld1		{v20.2d},[x1]		//load twisted H
+	ld1	{v17.2d},[x0]		//load Xi
+	movi	v19.16b,#0xe1
+	ld1	{v20.2d,v21.2d},[x1]	//load twisted H, ...
 	shl	v19.2d,v19.2d,#57
 #ifndef __ARMEB__
 	rev64	v17.16b,v17.16b
 #endif
-	ext		v21.16b,v20.16b,v20.16b,#8
-	mov		x3,#0
-	ext		v3.16b,v17.16b,v17.16b,#8
-	mov		x12,#0
-	eor		v21.16b,v21.16b,v20.16b		//Karatsuba pre-processing
-	mov		x2,x0
-	b		.Lgmult_v8
-.size	gcm_gmult_v8,.-gcm_gmult_v8
+	ext	v3.16b,v17.16b,v17.16b,#8
+
+	pmull	v0.1q,v20.1d,v3.1d		//H.loXi.lo
+	eor	v17.16b,v17.16b,v3.16b		//Karatsuba pre-processing
+	pmull2	v2.1q,v20.2d,v3.2d		//H.hiXi.hi
+	pmull	v1.1q,v21.1d,v17.1d		//(H.lo+H.hi)(Xi.lo+Xi.hi)
+
+	ext	v17.16b,v0.16b,v2.16b,#8		//Karatsuba post-processing
+	eor	v18.16b,v0.16b,v2.16b
+	eor	v1.16b,v1.16b,v17.16b
+	eor	v1.16b,v1.16b,v18.16b
+	pmull	v18.1q,v0.1d,v19.1d		//1st phase of reduction
+
+	ins	v2.d[0],v1.d[1]
+	ins	v1.d[1],v0.d[0]
+	eor	v0.16b,v1.16b,v18.16b
+
+	ext	v18.16b,v0.16b,v0.16b,#8		//2nd phase of reduction
+	pmull	v0.1q,v0.1d,v19.1d
+	eor	v18.16b,v18.16b,v2.16b
+	eor	v0.16b,v0.16b,v18.16b
 
-.global	gcm_ghash_v8
+#ifndef __ARMEB__
+	rev64	v0.16b,v0.16b
+#endif
+	ext	v0.16b,v0.16b,v0.16b,#8
+	st1	{v0.2d},[x0]		//write out Xi
+
+	ret
+.size	gcm_gmult_v8,.-gcm_gmult_v8
+.globl	gcm_ghash_v8
 .type	gcm_ghash_v8,%function
 .align	4
 gcm_ghash_v8:
-	ld1		{v0.2d},[x0]		//load [rotated] Xi
-	subs		x3,x3,#16
-	movi		v19.16b,#0xe1
-	mov		x12,#16
-	ld1		{v20.2d},[x1]		//load twisted H
-	csel	x12,xzr,x12,eq
-	ext		v0.16b,v0.16b,v0.16b,#8
-	shl	v19.2d,v19.2d,#57
-	ld1		{v17.2d},[x2],x12	//load [rotated] inp
-	ext		v21.16b,v20.16b,v20.16b,#8
+	ld1	{v0.2d},[x0]		//load [rotated] Xi
+						//"[rotated]" means that
+						//loaded value would have
+						//to be rotated in order to
+						//make it appear as in
+						//alorithm specification
+	subs	x3,x3,#32		//see if x3 is 32 or larger
+	mov	x12,#16		//x12 is used as post-
+						//increment for input pointer;
+						//as loop is modulo-scheduled
+						//x12 is zeroed just in time
+						//to preclude oversteping
+						//inp[len], which means that
+						//last block[s] are actually
+						//loaded twice, but last
+						//copy is not processed
+	ld1	{v20.2d,v21.2d},[x1],#32	//load twisted H, ..., H^2
+	movi	v19.16b,#0xe1
+	ld1	{v22.2d},[x1]
+	csel	x12,xzr,x12,eq			//is it time to zero x12?
+	ext	v0.16b,v0.16b,v0.16b,#8		//rotate Xi
+	ld1	{v16.2d},[x2],#16	//load [rotated] I[0]
+	shl	v19.2d,v19.2d,#57		//compose 0xc2.0 constant
 #ifndef __ARMEB__
+	rev64	v16.16b,v16.16b
 	rev64	v0.16b,v0.16b
+#endif
+	ext	v3.16b,v16.16b,v16.16b,#8		//rotate I[0]
+	b.lo	.Lodd_tail_v8		//x3 was less than 32
+	ld1	{v17.2d},[x2],x12	//load [rotated] I[1]
+#ifndef __ARMEB__
 	rev64	v17.16b,v17.16b
 #endif
-	eor		v21.16b,v21.16b,v20.16b		//Karatsuba pre-processing
-	ext		v3.16b,v17.16b,v17.16b,#8
-	b		.Loop_v8
+	ext	v7.16b,v17.16b,v17.16b,#8
+	eor	v3.16b,v3.16b,v0.16b		//I[i]^=Xi
+	pmull	v4.1q,v20.1d,v7.1d		//HIi+1
+	eor	v17.16b,v17.16b,v7.16b		//Karatsuba pre-processing
+	pmull2	v6.1q,v20.2d,v7.2d
+	b	.Loop_mod2x_v8
 
 .align	4
-.Loop_v8:
-	ext		v18.16b,v0.16b,v0.16b,#8
-	eor		v3.16b,v3.16b,v0.16b		//inp^=Xi
-	eor		v17.16b,v17.16b,v18.16b		//v17.16b is rotated inp^Xi
+.Loop_mod2x_v8:
+	ext	v18.16b,v3.16b,v3.16b,#8
+	subs	x3,x3,#32		//is there more data?
+	pmull	v0.1q,v22.1d,v3.1d		//H^2.loXi.lo
+	csel	x12,xzr,x12,lo			//is it time to zero x12?
+
+	pmull	v5.1q,v21.1d,v17.1d
+	eor	v18.16b,v18.16b,v3.16b		//Karatsuba pre-processing
+	pmull2	v2.1q,v22.2d,v3.2d		//H^2.hiXi.hi
+	eor	v0.16b,v0.16b,v4.16b		//accumulate
+	pmull2	v1.1q,v21.2d,v18.2d		//(H^2.lo+H^2.hi)(Xi.lo+Xi.hi)
+	ld1	{v16.2d},[x2],x12	//load [rotated] I[i+2]
+
+	eor	v2.16b,v2.16b,v6.16b
+	csel	x12,xzr,x12,eq			//is it time to zero x12?
+	eor	v1.16b,v1.16b,v5.16b
+
+	ext	v17.16b,v0.16b,v2.16b,#8		//Karatsuba post-processing
+	eor	v18.16b,v0.16b,v2.16b
+	eor	v1.16b,v1.16b,v17.16b
+	ld1	{v17.2d},[x2],x12	//load [rotated] I[i+3]
+#ifndef __ARMEB__
+	rev64	v16.16b,v16.16b
+#endif
+	eor	v1.16b,v1.16b,v18.16b
+	pmull	v18.1q,v0.1d,v19.1d		//1st phase of reduction
+
+#ifndef __ARMEB__
+	rev64	v17.16b,v17.16b
+#endif
+	ins	v2.d[0],v1.d[1]
+	ins	v1.d[1],v0.d[0]
+	ext	v7.16b,v17.16b,v17.16b,#8
+	ext	v3.16b,v16.16b,v16.16b,#8
+	eor	v0.16b,v1.16b,v18.16b
+	pmull	v4.1q,v20.1d,v7.1d		//HIi+1
+	eor	v3.16b,v3.16b,v2.16b		//accumulate v3.16b early
+
+	ext	v18.16b,v0.16b,v0.16b,#8		//2nd phase of reduction
+	pmull	v0.1q,v0.1d,v19.1d
+	eor	v3.16b,v3.16b,v18.16b
+	eor	v17.16b,v17.16b,v7.16b		//Karatsuba pre-processing
+	eor	v3.16b,v3.16b,v0.16b
+	pmull2	v6.1q,v20.2d,v7.2d
+	b.hs	.Loop_mod2x_v8		//there was at least 32 more bytes
+
+	eor	v2.16b,v2.16b,v18.16b
+	ext	v3.16b,v16.16b,v16.16b,#8		//re-construct v3.16b
+	adds	x3,x3,#32		//re-construct x3
+	eor	v0.16b,v0.16b,v2.16b		//re-construct v0.16b
+	b.eq	.Ldone_v8		//is x3 zero?
+.Lodd_tail_v8:
+	ext	v18.16b,v0.16b,v0.16b,#8
+	eor	v3.16b,v3.16b,v0.16b		//inp^=Xi
+	eor	v17.16b,v16.16b,v18.16b		//v17.16b is rotated inp^Xi
 
-.Lgmult_v8:
 	pmull	v0.1q,v20.1d,v3.1d		//H.loXi.lo
-	eor		v17.16b,v17.16b,v3.16b		//Karatsuba pre-processing
+	eor	v17.16b,v17.16b,v3.16b		//Karatsuba pre-processing
 	pmull2	v2.1q,v20.2d,v3.2d		//H.hiXi.hi
-	subs		x3,x3,#16
 	pmull	v1.1q,v21.1d,v17.1d		//(H.lo+H.hi)(Xi.lo+Xi.hi)
-	csel	x12,xzr,x12,eq
 
-	ext		v17.16b,v0.16b,v2.16b,#8		//Karatsuba post-processing
-	eor		v18.16b,v0.16b,v2.16b
-	eor		v1.16b,v1.16b,v17.16b
-	 ld1	{v17.2d},[x2],x12	//load [rotated] inp
-	eor		v1.16b,v1.16b,v18.16b
-	pmull	v18.1q,v0.1d,v19.1d		//1st phase
+	ext	v17.16b,v0.16b,v2.16b,#8		//Karatsuba post-processing
+	eor	v18.16b,v0.16b,v2.16b
+	eor	v1.16b,v1.16b,v17.16b
+	eor	v1.16b,v1.16b,v18.16b
+	pmull	v18.1q,v0.1d,v19.1d		//1st phase of reduction
 
 	ins	v2.d[0],v1.d[1]
 	ins	v1.d[1],v0.d[0]
-#ifndef __ARMEB__
-	 rev64	v17.16b,v17.16b
-#endif
-	eor		v0.16b,v1.16b,v18.16b
-	 ext		v3.16b,v17.16b,v17.16b,#8
+	eor	v0.16b,v1.16b,v18.16b
 
-	ext		v18.16b,v0.16b,v0.16b,#8		//2nd phase
+	ext	v18.16b,v0.16b,v0.16b,#8		//2nd phase of reduction
 	pmull	v0.1q,v0.1d,v19.1d
-	eor		v18.16b,v18.16b,v2.16b
-	eor		v0.16b,v0.16b,v18.16b
-	b.hs		.Loop_v8
+	eor	v18.16b,v18.16b,v2.16b
+	eor	v0.16b,v0.16b,v18.16b
 
+.Ldone_v8:
 #ifndef __ARMEB__
 	rev64	v0.16b,v0.16b
 #endif
-	ext		v0.16b,v0.16b,v0.16b,#8
-	st1		{v0.2d},[x0]		//write out Xi
+	ext	v0.16b,v0.16b,v0.16b,#8
+	st1	{v0.2d},[x0]		//write out Xi
 
 	ret
 .size	gcm_ghash_v8,.-gcm_ghash_v8
-.asciz  "GHASH for ARMv8, CRYPTOGAMS by <appro@openssl.org>"
-.align  2
+.byte	71,72,65,83,72,32,102,111,114,32,65,82,77,118,56,44,32,67,82,89,80,84,79,71,65,77,83,32,98,121,32,60,97,112,112,114,111,64,111,112,101,110,115,115,108,46,111,114,103,62,0
+.align	2
+.align	2