From aea47606333cfd3e7a09cab3e42e488c79a416af Mon Sep 17 00:00:00 2001
From: Adam Langley <agl@chromium.org>
Date: Tue, 5 Nov 2013 13:10:11 -0500
Subject: [PATCH 52/52] Optional NEON support on ARM.

This patch causes ARM to build both the NEON and generic versions of
ChaCha20 and Poly1305. The NEON code can be enabled at run-time by
calling CRYPTO_set_NEON_capable(1).
---
 .gitignore                     |   1 +
 Configure                      |   2 +-
 apps/speed.c                   |   5 +
 crypto/chacha/chacha_enc.c     |  18 +
 crypto/chacha/chacha_vec.c     |   7 +
 crypto/chacha/chacha_vec_arm.s | 846 +++++++++++++++++++++++++++++++++++++++++
 crypto/cryptlib.c              |  14 +
 crypto/crypto.h                |   8 +
 crypto/poly1305/poly1305.c     |  35 ++
 crypto/poly1305/poly1305_arm.c |   9 +-
 10 files changed, 941 insertions(+), 4 deletions(-)
 create mode 100644 crypto/chacha/chacha_vec_arm.s

diff --git a/Configure b/Configure
index 1b95384..18b7af0 100755
--- a/Configure
+++ b/Configure
@@ -136,7 +136,7 @@ my $alpha_asm="alphacpuid.o:bn_asm.o alpha-mont.o:::::sha1-alpha.o:::::::ghash-a
 my $mips32_asm=":bn-mips.o::aes_cbc.o aes-mips.o:::sha1-mips.o sha256-mips.o::::::::::";
 my $mips64_asm=":bn-mips.o mips-mont.o::aes_cbc.o aes-mips.o:::sha1-mips.o sha256-mips.o sha512-mips.o::::::::::";
 my $s390x_asm="s390xcap.o s390xcpuid.o:bn-s390x.o s390x-mont.o s390x-gf2m.o::aes-s390x.o aes-ctr.o aes-xts.o:::sha1-s390x.o sha256-s390x.o sha512-s390x.o::rc4-s390x.o:::::::ghash-s390x.o:";
-my $armv4_asm="armcap.o armv4cpuid.o:bn_asm.o armv4-mont.o armv4-gf2m.o::aes_cbc.o aes-armv4.o:::sha1-armv4-large.o sha256-armv4.o sha512-armv4.o:::::::ghash-armv4.o::chacha_vec.o:poly1305_arm.o poly1305_arm_asm.o:void";
+my $armv4_asm="armcap.o armv4cpuid.o:bn_asm.o armv4-mont.o armv4-gf2m.o::aes_cbc.o aes-armv4.o:::sha1-armv4-large.o sha256-armv4.o sha512-armv4.o:::::::ghash-armv4.o::chacha_vec_arm.o chacha_enc.o:poly1305.o poly1305_arm.o poly1305_arm_asm.o:void";
 my $parisc11_asm="pariscid.o:bn_asm.o parisc-mont.o::aes_core.o aes_cbc.o aes-parisc.o:::sha1-parisc.o sha256-parisc.o sha512-parisc.o::rc4-parisc.o:::::ghash-parisc.o::::32";
 my $parisc20_asm="pariscid.o:pa-risc2W.o parisc-mont.o::aes_core.o aes_cbc.o aes-parisc.o:::sha1-parisc.o sha256-parisc.o sha512-parisc.o::rc4-parisc.o:::::ghash-parisc.o::::64";
 my $ppc32_asm="ppccpuid.o ppccap.o:bn-ppc.o ppc-mont.o ppc64-mont.o::aes_core.o aes_cbc.o aes-ppc.o:::sha1-ppc.o sha256-ppc.o::::::::::";
diff --git a/crypto/chacha/chacha_enc.c b/crypto/chacha/chacha_enc.c
index 54d1ca3..e4b648f 100644
--- a/crypto/chacha/chacha_enc.c
+++ b/crypto/chacha/chacha_enc.c
@@ -61,6 +61,7 @@
 
 #if !defined(OPENSSL_NO_CHACHA)
 
+#include <openssl/crypto.h>
 #include <openssl/chacha.h>
 
 /* sigma contains the ChaCha constants, which happen to be an ASCII string. */
@@ -87,6 +88,15 @@ static const char sigma[16] = "expand 32-byte k";
 
 typedef unsigned int uint32_t;
 
+#if __arm__
+/* Defined in chacha_vec.c */
+void CRYPTO_chacha_20_neon(unsigned char *out,
+		           const unsigned char *in, size_t in_len,
+		           const unsigned char key[32],
+		           const unsigned char nonce[8],
+		           size_t counter);
+#endif
+
 /* chacha_core performs |num_rounds| rounds of ChaCha20 on the input words in
  * |input| and writes the 64 output bytes to |output|. */
 static void chacha_core(unsigned char output[64], const uint32_t input[16],
@@ -124,6 +134,16 @@ void CRYPTO_chacha_20(unsigned char *out,
 	unsigned char buf[64];
 	size_t todo, i;
 
+#if __arm__
+	if (CRYPTO_is_NEON_capable() &&
+	    ((intptr_t)in & 15) == 0 &&
+	    ((intptr_t)out & 15) == 0)
+		{
+		CRYPTO_chacha_20_neon(out, in, in_len, key, nonce, counter);
+		return;
+		}
+#endif
+
 	input[0] = U8TO32_LITTLE(sigma + 0);
 	input[1] = U8TO32_LITTLE(sigma + 4);
 	input[2] = U8TO32_LITTLE(sigma + 8);
diff --git a/crypto/chacha/chacha_vec.c b/crypto/chacha/chacha_vec.c
index 33b2238..1226c39 100644
--- a/crypto/chacha/chacha_vec.c
+++ b/crypto/chacha/chacha_vec.c
@@ -154,7 +154,14 @@ typedef unsigned vec __attribute__ ((vector_size (16)));
 	STORE(op + d + 8, LOAD(in + d + 8) ^ REVV_BE(v2));      \
 	STORE(op + d +12, LOAD(in + d +12) ^ REVV_BE(v3));
 
+#if __ARM_NEON__
+/* For ARM, we can't depend on NEON support, so this function is compiled with
+ * a different name, along with the generic code, and can be enabled at
+ * run-time. */
+void CRYPTO_chacha_20_neon(
+#else
 void CRYPTO_chacha_20(
+#endif
 	unsigned char *out,
 	const unsigned char *in,
 	size_t inlen,
diff --git a/crypto/chacha/chacha_vec_arm.S b/crypto/chacha/chacha_vec_arm.S
new file mode 100644
index 0000000..24a5050
--- /dev/null
+++ b/crypto/chacha/chacha_vec_arm.S
@@ -0,0 +1,863 @@
+# This file contains a pre-compiled version of chacha_vec.c for ARM. This is
+# needed to support switching on NEON code at runtime. If the whole of OpenSSL
+# were to be compiled with the needed flags to build chacha_vec.c, then it
+# wouldn't be possible to run on non-NEON systems.
+#
+# This file was generated by:
+#
+#     /opt/gcc-linaro-arm-linux-gnueabihf-4.7-2012.10-20121022_linux/bin/arm-linux-gnueabihf-gcc -O3 -mcpu=cortex-a8 -mfpu=neon -S chacha_vec.c -I ../../include -fpic -o chacha_vec_arm.S
+#
+# And then EABI attribute 28 was set to zero to allow linking with soft-float
+# code.
+
+	.syntax unified
+	.cpu cortex-a8
+	.eabi_attribute 27, 3
+	.eabi_attribute 28, 0
+	.fpu neon
+	.eabi_attribute 20, 1
+	.eabi_attribute 21, 1
+	.eabi_attribute 23, 3
+	.eabi_attribute 24, 1
+	.eabi_attribute 25, 1
+	.eabi_attribute 26, 2
+	.eabi_attribute 30, 2
+	.eabi_attribute 34, 1
+	.eabi_attribute 18, 4
+	.thumb
+	.file	"chacha_vec.c"
+	.text
+	.align	2
+	.global	CRYPTO_chacha_20_neon
+	.thumb
+	.thumb_func
+	.type	CRYPTO_chacha_20_neon, %function
+CRYPTO_chacha_20_neon:
+	@ args = 8, pretend = 0, frame = 304
+	@ frame_needed = 1, uses_anonymous_args = 0
+	@ link register save eliminated.
+	push	{r4, r5, r6, r7, r8, r9, sl, fp}
+	fstmfdd	sp!, {d8, d9, d10, d11, d12, d13, d14, d15}
+	sub	sp, sp, #304
+	add	r7, sp, #0
+	movw	ip, #43691
+	movt	ip, 43690
+	str	r2, [r7, #196]
+	sub	sp, sp, #96
+	ldr	r4, [r7, #196]
+	ldr	r6, [r7, #400]
+	ldr	r2, .L38+16
+	umull	r4, ip, ip, r4
+	ldr	r6, [r6, #0]
+	ldr	r8, [r7, #400]
+.LPIC24:
+	add	r2, pc
+	add	r4, sp, #15
+	str	r3, [r7, #244]
+	str	r6, [r7, #176]
+	bic	r4, r4, #15
+	str	r0, [r7, #188]
+	str	r4, [r7, #200]
+	lsrs	ip, ip, #7
+	str	r1, [r7, #184]
+	ldmia	r2, {r0, r1, r2, r3}
+	ldr	r4, [r8, #4]
+	ldr	r5, [r7, #244]
+	vld1.64	{d24-d25}, [r5:64]
+	vldr	d26, [r5, #16]
+	vldr	d27, [r5, #24]
+	ldr	r9, [r7, #200]
+	ldr	r8, [r7, #404]
+	ldr	r5, [r7, #176]
+	add	r6, r9, #64
+	str	r4, [r7, #300]
+	mov	r4, #0
+	str	r8, [r7, #288]
+	str	r5, [r7, #296]
+	str	r4, [r7, #292]
+	stmia	r6, {r0, r1, r2, r3}
+	vldr	d22, [r9, #64]
+	vldr	d23, [r9, #72]
+	vldr	d20, [r7, #288]
+	vldr	d21, [r7, #296]
+	str	ip, [r7, #192]
+	beq	.L20
+	lsl	r6, ip, #1
+	ldr	r1, [r9, #68]
+	add	r3, r6, ip
+	str	r6, [r7, #180]
+	ldr	r2, [r9, #72]
+	add	r8, r8, #2
+	ldr	r5, [r9, #76]
+	vldr	d18, .L38
+	vldr	d19, .L38+8
+	str	r4, [r7, #240]
+	ldr	r6, [r7, #184]
+	ldr	r4, [r7, #188]
+	str	r0, [r7, #224]
+	str	r1, [r7, #220]
+	str	r8, [r7, #208]
+	str	r2, [r7, #216]
+	str	r3, [r7, #204]
+	str	r5, [r7, #212]
+	str	r6, [r7, #252]
+	str	r4, [r7, #248]
+.L4:
+	ldr	r2, [r7, #244]
+	add	r9, r7, #216
+	ldr	r3, [r7, #244]
+	vadd.i32	q8, q10, q9
+	ldr	r6, [r7, #208]
+	vmov	q15, q13  @ v4si
+	ldr	r5, [r7, #240]
+	vmov	q3, q12  @ v4si
+	ldr	r4, [r7, #244]
+	vmov	q2, q11  @ v4si
+	adds	r5, r5, r6
+	ldr	r2, [r2, #8]
+	ldr	r6, [r7, #400]
+	vmov	q5, q10  @ v4si
+	ldr	r3, [r3, #12]
+	vmov	q1, q13  @ v4si
+	ldr	r0, [r7, #244]
+	vmov	q0, q12  @ v4si
+	ldr	r1, [r7, #244]
+	vmov	q4, q11  @ v4si
+	ldmia	r9, {r9, sl, fp}
+	str	r5, [r7, #228]
+	ldr	r5, [r4, #24]
+	ldr	r0, [r0, #0]
+	ldr	r1, [r1, #4]
+	str	r2, [r7, #264]
+	str	r3, [r7, #236]
+	ldr	r2, [r6, #4]
+	ldr	r3, [r4, #28]
+	str	r5, [r7, #280]
+	ldr	r5, [r6, #0]
+	movs	r6, #0
+	ldr	ip, [r7, #228]
+	ldr	r8, [r7, #212]
+	str	r0, [r7, #232]
+	str	r1, [r7, #268]
+	ldr	r0, [r4, #16]
+	ldr	r1, [r4, #20]
+	movs	r4, #10
+	str	r2, [r7, #24]
+	str	r3, [r7, #284]
+	str	r4, [r7, #256]
+	ldr	r2, [r7, #264]
+	str	r9, [r7, #276]
+	mov	r9, r6
+	ldr	r6, [r7, #280]
+	str	r8, [r7, #260]
+	mov	r8, sl
+	str	r1, [r7, #272]
+	mov	sl, ip
+	str	r6, [r7, #264]
+	mov	r6, r5
+	ldr	r3, [r7, #236]
+	mov	r5, r0
+	ldr	ip, [r7, #24]
+	ldr	r1, [r7, #268]
+	ldr	r0, [r7, #232]
+	b	.L39
+.L40:
+	.align	3
+.L38:
+	.word	1
+	.word	0
+	.word	0
+	.word	0
+	.word	.LANCHOR0-(.LPIC24+4)
+.L39:
+.L3:
+	vadd.i32	q4, q4, q0
+	add	r8, r8, r1
+	vadd.i32	q2, q2, q3
+	str	r8, [r7, #268]
+	veor	q5, q5, q4
+	ldr	r8, [r7, #276]
+	veor	q8, q8, q2
+	add	fp, fp, r0
+	str	fp, [r7, #280]
+	add	r8, r8, r2
+	vrev32.16	q5, q5
+	str	r8, [r7, #276]
+	vrev32.16	q8, q8
+	vadd.i32	q1, q1, q5
+	vadd.i32	q15, q15, q8
+	ldr	r8, [r7, #280]
+	veor	q0, q1, q0
+	ldr	r4, [r7, #260]
+	veor	q3, q15, q3
+	eor	sl, sl, r8
+	ldr	r8, [r7, #276]
+	add	fp, r4, r3
+	vshl.i32	q7, q0, #12
+	ldr	r4, [r7, #268]
+	vshl.i32	q6, q3, #12
+	eor	r6, r6, r8
+	eor	r9, r9, r4
+	ldr	r4, [r7, #272]
+	vsri.32	q7, q0, #20
+	ror	r8, r6, #16
+	ldr	r6, [r7, #264]
+	eor	ip, ip, fp
+	vsri.32	q6, q3, #20
+	ror	sl, sl, #16
+	ror	r9, r9, #16
+	add	r5, r5, sl
+	vadd.i32	q4, q4, q7
+	str	r5, [r7, #236]
+	vadd.i32	q2, q2, q6
+	add	r5, r4, r9
+	add	r4, r6, r8
+	ldr	r6, [r7, #284]
+	ror	ip, ip, #16
+	veor	q5, q4, q5
+	veor	q8, q2, q8
+	add	r6, r6, ip
+	str	r6, [r7, #264]
+	eors	r1, r1, r5
+	ldr	r6, [r7, #236]
+	vshl.i32	q3, q5, #8
+	vshl.i32	q14, q8, #8
+	eors	r2, r2, r4
+	eors	r0, r0, r6
+	ldr	r6, [r7, #264]
+	vsri.32	q3, q5, #24
+	ror	r1, r1, #20
+	eors	r3, r3, r6
+	ldr	r6, [r7, #280]
+	ror	r0, r0, #20
+	vsri.32	q14, q8, #24
+	adds	r6, r0, r6
+	str	r6, [r7, #284]
+	ldr	r6, [r7, #268]
+	vadd.i32	q1, q1, q3
+	vadd.i32	q15, q15, q14
+	ror	r2, r2, #20
+	adds	r6, r1, r6
+	str	r6, [r7, #260]
+	ldr	r6, [r7, #276]
+	veor	q6, q15, q6
+	veor	q7, q1, q7
+	ror	r3, r3, #20
+	adds	r6, r2, r6
+	str	r6, [r7, #280]
+	ldr	r6, [r7, #284]
+	vshl.i32	q0, q6, #7
+	vshl.i32	q5, q7, #7
+	add	fp, r3, fp
+	eor	sl, r6, sl
+	ldr	r6, [r7, #260]
+	eor	ip, fp, ip
+	vsri.32	q0, q6, #25
+	eor	r9, r6, r9
+	ldr	r6, [r7, #280]
+	ror	sl, sl, #24
+	vsri.32	q5, q7, #25
+	eor	r8, r6, r8
+	ldr	r6, [r7, #236]
+	ror	r9, r9, #24
+	ror	ip, ip, #24
+	add	r6, sl, r6
+	str	r6, [r7, #276]
+	ldr	r6, [r7, #264]
+	add	r5, r9, r5
+	str	r5, [r7, #272]
+	vext.32	q5, q5, q5, #1
+	add	r5, ip, r6
+	ldr	r6, [r7, #276]
+	vext.32	q0, q0, q0, #1
+	vadd.i32	q4, q4, q5
+	eors	r0, r0, r6
+	ldr	r6, [r7, #272]
+	vadd.i32	q2, q2, q0
+	vext.32	q3, q3, q3, #3
+	ror	r8, r8, #24
+	eors	r1, r1, r6
+	vext.32	q14, q14, q14, #3
+	add	r4, r8, r4
+	ldr	r6, [r7, #284]
+	veor	q3, q4, q3
+	veor	q14, q2, q14
+	eors	r2, r2, r4
+	ror	r1, r1, #25
+	vext.32	q1, q1, q1, #2
+	adds	r6, r1, r6
+	str	r6, [r7, #284]
+	vext.32	q15, q15, q15, #2
+	ldr	r6, [r7, #260]
+	eors	r3, r3, r5
+	ror	r2, r2, #25
+	vrev32.16	q8, q14
+	adds	r6, r2, r6
+	vrev32.16	q3, q3
+	str	r6, [r7, #268]
+	vadd.i32	q1, q1, q3
+	ldr	r6, [r7, #280]
+	vadd.i32	q15, q15, q8
+	ror	r3, r3, #25
+	veor	q5, q1, q5
+	adds	r6, r3, r6
+	veor	q0, q15, q0
+	str	r6, [r7, #264]
+	ldr	r6, [r7, #268]
+	ror	r0, r0, #25
+	add	fp, r0, fp
+	vshl.i32	q6, q5, #12
+	eor	sl, r6, sl
+	ldr	r6, [r7, #284]
+	vshl.i32	q14, q0, #12
+	eor	r8, fp, r8
+	eor	ip, r6, ip
+	ldr	r6, [r7, #264]
+	vsri.32	q6, q5, #20
+	ror	sl, sl, #16
+	eor	r9, r6, r9
+	ror	r6, r8, #16
+	vsri.32	q14, q0, #20
+	ldr	r8, [r7, #272]
+	ror	ip, ip, #16
+	add	r5, sl, r5
+	add	r8, r6, r8
+	add	r4, ip, r4
+	str	r4, [r7, #236]
+	eor	r0, r8, r0
+	str	r5, [r7, #280]
+	vadd.i32	q4, q4, q6
+	ldr	r5, [r7, #236]
+	vadd.i32	q2, q2, q14
+	ldr	r4, [r7, #276]
+	ror	r0, r0, #20
+	veor	q3, q4, q3
+	eors	r1, r1, r5
+	veor	q0, q2, q8
+	str	r8, [r7, #272]
+	str	r0, [r7, #24]
+	add	fp, r0, fp
+	ldr	r8, [r7, #280]
+	ror	r9, r9, #16
+	ldr	r0, [r7, #284]
+	add	r4, r9, r4
+	str	fp, [r7, #260]
+	ror	r1, r1, #20
+	add	fp, r1, r0
+	eor	r2, r8, r2
+	ldr	r0, [r7, #260]
+	eors	r3, r3, r4
+	vshl.i32	q5, q3, #8
+	str	r4, [r7, #232]
+	vshl.i32	q8, q0, #8
+	ldr	r4, [r7, #268]
+	ldr	r5, [r7, #264]
+	ror	r2, r2, #20
+	ror	r3, r3, #20
+	eors	r6, r6, r0
+	adds	r5, r3, r5
+	add	r8, r2, r4
+	vsri.32	q5, q3, #24
+	ldr	r4, [r7, #272]
+	eor	r9, r5, r9
+	eor	ip, fp, ip
+	vsri.32	q8, q0, #24
+	eor	sl, r8, sl
+	ror	r6, r6, #24
+	ldr	r0, [r7, #280]
+	str	r5, [r7, #276]
+	adds	r4, r6, r4
+	ldr	r5, [r7, #236]
+	vadd.i32	q1, q1, q5
+	str	r4, [r7, #272]
+	vadd.i32	q15, q15, q8
+	ldr	r4, [r7, #232]
+	ror	ip, ip, #24
+	ror	sl, sl, #24
+	ror	r9, r9, #24
+	add	r5, ip, r5
+	add	r0, sl, r0
+	str	r5, [r7, #264]
+	add	r5, r9, r4
+	str	r0, [r7, #284]
+	veor	q6, q1, q6
+	ldr	r4, [r7, #24]
+	veor	q14, q15, q14
+	ldr	r0, [r7, #272]
+	eors	r3, r3, r5
+	vshl.i32	q0, q6, #7
+	vext.32	q1, q1, q1, #2
+	eors	r0, r0, r4
+	ldr	r4, [r7, #284]
+	str	r0, [r7, #280]
+	vshl.i32	q3, q14, #7
+	eors	r2, r2, r4
+	ldr	r4, [r7, #280]
+	ldr	r0, [r7, #264]
+	vsri.32	q0, q6, #25
+	ror	r2, r2, #25
+	ror	r3, r3, #25
+	eors	r1, r1, r0
+	vsri.32	q3, q14, #25
+	ror	r0, r4, #25
+	ldr	r4, [r7, #256]
+	ror	r1, r1, #25
+	vext.32	q5, q5, q5, #1
+	subs	r4, r4, #1
+	str	r4, [r7, #256]
+	vext.32	q15, q15, q15, #2
+	vext.32	q8, q8, q8, #1
+	vext.32	q0, q0, q0, #3
+	vext.32	q3, q3, q3, #3
+	bne	.L3
+	ldr	r4, [r7, #264]
+	vadd.i32	q14, q10, q9
+	str	r2, [r7, #264]
+	vadd.i32	q10, q10, q5
+	ldr	r2, [r7, #252]
+	vld1.64	{d12-d13}, [r2:64]
+	ldr	r2, [r7, #220]
+	vadd.i32	q4, q11, q4
+	str	ip, [r7, #24]
+	mov	ip, sl
+	mov	sl, r8
+	ldr	r8, [r7, #260]
+	add	sl, sl, r2
+	ldr	r2, [r7, #212]
+	str	r4, [r7, #280]
+	vadd.i32	q0, q12, q0
+	ldr	r4, [r7, #224]
+	add	r8, r8, r2
+	ldr	r2, [r7, #240]
+	vadd.i32	q1, q13, q1
+	str	r0, [r7, #232]
+	add	fp, fp, r4
+	mov	r0, r5
+	ldr	r4, [r7, #216]
+	mov	r5, r6
+	mov	r6, r9
+	ldr	r9, [r7, #276]
+	adds	r2, r2, #3
+	str	r2, [r7, #240]
+	vadd.i32	q2, q11, q2
+	ldr	r2, [r7, #252]
+	add	r9, r9, r4
+	vadd.i32	q3, q12, q3
+	ldr	r4, [r7, #228]
+	vadd.i32	q15, q13, q15
+	str	r1, [r7, #268]
+	vadd.i32	q8, q14, q8
+	str	r3, [r7, #236]
+	veor	q4, q4, q6
+	ldr	r3, [r7, #284]
+	ldr	r1, [r7, #272]
+	add	ip, r4, ip
+	ldr	r4, [r7, #248]
+	vst1.64	{d8-d9}, [r4:64]
+	vldr	d8, [r2, #16]
+	vldr	d9, [r2, #24]
+	veor	q0, q0, q4
+	vstr	d0, [r4, #16]
+	vstr	d1, [r4, #24]
+	vldr	d0, [r2, #32]
+	vldr	d1, [r2, #40]
+	veor	q1, q1, q0
+	vstr	d2, [r4, #32]
+	vstr	d3, [r4, #40]
+	vldr	d2, [r2, #48]
+	vldr	d3, [r2, #56]
+	veor	q10, q10, q1
+	vstr	d20, [r4, #48]
+	vstr	d21, [r4, #56]
+	vldr	d8, [r2, #64]
+	vldr	d9, [r2, #72]
+	veor	q2, q2, q4
+	vstr	d4, [r4, #64]
+	vstr	d5, [r4, #72]
+	vldr	d10, [r2, #80]
+	vldr	d11, [r2, #88]
+	veor	q3, q3, q5
+	vstr	d6, [r4, #80]
+	vstr	d7, [r4, #88]
+	vldr	d12, [r2, #96]
+	vldr	d13, [r2, #104]
+	veor	q15, q15, q6
+	vstr	d30, [r4, #96]
+	vstr	d31, [r4, #104]
+	vldr	d20, [r2, #112]
+	vldr	d21, [r2, #120]
+	veor	q8, q8, q10
+	vstr	d16, [r4, #112]
+	vstr	d17, [r4, #120]
+	ldr	r4, [r2, #128]
+	ldr	r2, [r7, #248]
+	vadd.i32	q10, q14, q9
+	eor	r4, fp, r4
+	vadd.i32	q10, q10, q9
+	str	r4, [r2, #128]
+	ldr	r4, [r7, #252]
+	ldr	r2, [r4, #132]
+	eor	r2, sl, r2
+	ldr	sl, [r7, #248]
+	str	r2, [sl, #132]
+	ldr	r2, [r4, #136]
+	eor	r2, r9, r2
+	str	r2, [sl, #136]
+	ldr	r2, [r4, #140]
+	eor	r2, r8, r2
+	str	r2, [sl, #140]
+	ldr	r2, [r7, #244]
+	ldr	r4, [r4, #144]
+	ldr	r2, [r2, #0]
+	str	r4, [r7, #44]
+	ldr	r4, [r7, #232]
+	add	r8, r4, r2
+	ldr	r2, [r7, #44]
+	ldr	r4, [r7, #244]
+	eor	r8, r8, r2
+	ldr	r2, [r7, #252]
+	str	r8, [sl, #144]
+	ldr	r4, [r4, #4]
+	ldr	r2, [r2, #148]
+	str	r2, [r7, #40]
+	ldr	r2, [r7, #268]
+	add	r8, r2, r4
+	ldr	r4, [r7, #40]
+	ldr	r2, [r7, #244]
+	eor	r8, r8, r4
+	ldr	r4, [r7, #252]
+	str	r8, [sl, #148]
+	ldr	r2, [r2, #8]
+	ldr	r4, [r4, #152]
+	str	r4, [r7, #36]
+	ldr	r4, [r7, #264]
+	add	r8, r4, r2
+	ldr	r2, [r7, #36]
+	eor	r8, r8, r2
+	str	r8, [sl, #152]
+	ldr	r2, [r7, #252]
+	ldr	r4, [r7, #244]
+	ldr	r2, [r2, #156]
+	ldr	r4, [r4, #12]
+	str	r2, [r7, #32]
+	ldr	r2, [r7, #236]
+	add	r8, r2, r4
+	ldr	r4, [r7, #32]
+	ldr	r2, [r7, #252]
+	eor	r8, r8, r4
+	str	r8, [sl, #156]
+	ldr	r8, [r7, #244]
+	ldr	r2, [r2, #160]
+	ldr	r4, [r8, #16]
+	adds	r0, r0, r4
+	ldr	r4, [r7, #252]
+	eors	r0, r0, r2
+	str	r0, [sl, #160]
+	ldr	r0, [r8, #20]
+	ldr	r2, [r4, #164]
+	adds	r1, r1, r0
+	ldr	r0, [r7, #280]
+	eors	r1, r1, r2
+	str	r1, [sl, #164]
+	ldr	r2, [r8, #24]
+	ldr	r1, [r4, #168]
+	adds	r2, r0, r2
+	eors	r2, r2, r1
+	str	r2, [sl, #168]
+	ldr	r1, [r8, #28]
+	ldr	r2, [r4, #172]
+	adds	r3, r3, r1
+	eors	r3, r3, r2
+	str	r3, [sl, #172]
+	ldr	r3, [r4, #176]
+	eor	r3, ip, r3
+	str	r3, [sl, #176]
+	ldr	r3, [r4, #180]
+	ldr	r4, [r7, #400]
+	eors	r6, r6, r3
+	str	r6, [sl, #180]
+	ldr	r6, [r7, #252]
+	ldr	r2, [r4, #0]
+	ldr	r3, [r6, #184]
+	adds	r5, r5, r2
+	eors	r5, r5, r3
+	str	r5, [sl, #184]
+	ldr	r2, [r6, #188]
+	adds	r6, r6, #192
+	ldr	r3, [r4, #4]
+	str	r6, [r7, #252]
+	ldr	r0, [r7, #24]
+	ldr	r1, [r7, #240]
+	adds	r4, r0, r3
+	eors	r4, r4, r2
+	ldr	r2, [r7, #204]
+	str	r4, [sl, #188]
+	add	sl, sl, #192
+	cmp	r1, r2
+	str	sl, [r7, #248]
+	bne	.L4
+	ldr	r4, [r7, #192]
+	ldr	r3, [r7, #180]
+	ldr	r6, [r7, #188]
+	adds	r5, r3, r4
+	ldr	r8, [r7, #184]
+	lsls	r5, r5, #6
+	adds	r4, r6, r5
+	add	r5, r8, r5
+.L2:
+	ldr	r9, [r7, #196]
+	movw	r3, #43691
+	movt	r3, 43690
+	ldr	sl, [r7, #196]
+	umull	r9, r3, r3, r9
+	lsrs	r3, r3, #7
+	add	r3, r3, r3, lsl #1
+	sub	r3, sl, r3, lsl #6
+	lsrs	r6, r3, #6
+	beq	.L5
+	add	r1, r5, #16
+	add	r2, r4, #16
+	mov	r0, r6
+	vldr	d30, .L41
+	vldr	d31, .L41+8
+.L6:
+	vmov	q8, q10  @ v4si
+	movs	r3, #10
+	vmov	q1, q13  @ v4si
+	vmov	q14, q12  @ v4si
+	vmov	q3, q11  @ v4si
+.L7:
+	vadd.i32	q3, q3, q14
+	subs	r3, r3, #1
+	veor	q2, q8, q3
+	vrev32.16	q2, q2
+	vadd.i32	q8, q1, q2
+	veor	q9, q8, q14
+	vshl.i32	q14, q9, #12
+	vsri.32	q14, q9, #20
+	vadd.i32	q3, q3, q14
+	veor	q2, q3, q2
+	vshl.i32	q9, q2, #8
+	vsri.32	q9, q2, #24
+	vadd.i32	q8, q8, q9
+	vext.32	q9, q9, q9, #3
+	veor	q14, q8, q14
+	vext.32	q1, q8, q8, #2
+	vshl.i32	q8, q14, #7
+	vsri.32	q8, q14, #25
+	vext.32	q8, q8, q8, #1
+	vadd.i32	q3, q3, q8
+	veor	q2, q3, q9
+	vrev32.16	q2, q2
+	vadd.i32	q9, q1, q2
+	veor	q8, q9, q8
+	vshl.i32	q14, q8, #12
+	vsri.32	q14, q8, #20
+	vadd.i32	q3, q3, q14
+	veor	q2, q3, q2
+	vshl.i32	q8, q2, #8
+	vsri.32	q8, q2, #24
+	vadd.i32	q9, q9, q8
+	vext.32	q8, q8, q8, #1
+	veor	q14, q9, q14
+	vext.32	q1, q9, q9, #2
+	vshl.i32	q9, q14, #7
+	vsri.32	q9, q14, #25
+	vext.32	q14, q9, q9, #3
+	bne	.L7
+	vadd.i32	q8, q10, q8
+	subs	r0, r0, #1
+	vadd.i32	q3, q11, q3
+	vldr	d0, [r1, #-16]
+	vldr	d1, [r1, #-8]
+	vadd.i32	q14, q12, q14
+	vadd.i32	q1, q13, q1
+	veor	q3, q3, q0
+	vstr	d6, [r2, #-16]
+	vstr	d7, [r2, #-8]
+	vadd.i32	q10, q10, q15
+	vld1.64	{d8-d9}, [r1:64]
+	veor	q14, q14, q4
+	vst1.64	{d28-d29}, [r2:64]
+	vldr	d10, [r1, #16]
+	vldr	d11, [r1, #24]
+	veor	q1, q1, q5
+	vstr	d2, [r2, #16]
+	vstr	d3, [r2, #24]
+	vldr	d18, [r1, #32]
+	vldr	d19, [r1, #40]
+	add	r1, r1, #64
+	veor	q8, q8, q9
+	vstr	d16, [r2, #32]
+	vstr	d17, [r2, #40]
+	add	r2, r2, #64
+	bne	.L6
+	lsls	r6, r6, #6
+	adds	r4, r4, r6
+	adds	r5, r5, r6
+.L5:
+	ldr	r6, [r7, #196]
+	ands	ip, r6, #63
+	beq	.L1
+	vmov	q8, q10  @ v4si
+	movs	r3, #10
+	vmov	q14, q13  @ v4si
+	vmov	q9, q12  @ v4si
+	vmov	q15, q11  @ v4si
+.L10:
+	vadd.i32	q15, q15, q9
+	subs	r3, r3, #1
+	veor	q8, q8, q15
+	vrev32.16	q8, q8
+	vadd.i32	q3, q14, q8
+	veor	q9, q3, q9
+	vshl.i32	q14, q9, #12
+	vsri.32	q14, q9, #20
+	vadd.i32	q15, q15, q14
+	veor	q9, q15, q8
+	vshl.i32	q8, q9, #8
+	vsri.32	q8, q9, #24
+	vadd.i32	q9, q3, q8
+	vext.32	q8, q8, q8, #3
+	veor	q2, q9, q14
+	vext.32	q14, q9, q9, #2
+	vshl.i32	q9, q2, #7
+	vsri.32	q9, q2, #25
+	vext.32	q9, q9, q9, #1
+	vadd.i32	q15, q15, q9
+	veor	q3, q15, q8
+	vrev32.16	q3, q3
+	vadd.i32	q14, q14, q3
+	veor	q8, q14, q9
+	vshl.i32	q9, q8, #12
+	vsri.32	q9, q8, #20
+	vadd.i32	q15, q15, q9
+	veor	q3, q15, q3
+	vshl.i32	q8, q3, #8
+	vsri.32	q8, q3, #24
+	vadd.i32	q14, q14, q8
+	vext.32	q8, q8, q8, #1
+	veor	q3, q14, q9
+	vext.32	q14, q14, q14, #2
+	vshl.i32	q9, q3, #7
+	vsri.32	q9, q3, #25
+	vext.32	q9, q9, q9, #3
+	bne	.L10
+	cmp	ip, #15
+	vadd.i32	q11, q11, q15
+	bhi	.L37
+	ldr	r9, [r7, #200]
+	vst1.64	{d22-d23}, [r9:128]
+.L14:
+	ldr	sl, [r7, #196]
+	and	r3, sl, #48
+	cmp	ip, r3
+	bls	.L1
+	adds	r0, r5, r3
+	adds	r1, r4, r3
+	add	r2, r0, #16
+	add	r6, r1, #16
+	cmp	r1, r2
+	it	cc
+	cmpcc	r0, r6
+	rsb	r9, r3, ip
+	ite	cc
+	movcc	r2, #0
+	movcs	r2, #1
+	cmp	r9, #15
+	ite	ls
+	movls	r2, #0
+	andhi	r2, r2, #1
+	lsr	r8, r9, #4
+	eor	r2, r2, #1
+	cmp	r8, #0
+	it	eq
+	orreq	r2, r2, #1
+	lsl	sl, r8, #4
+	cbnz	r2, .L35
+	ldr	fp, [r7, #200]
+	add	r6, fp, r3
+.L17:
+	vld1.8	{q8}, [r0]!
+	adds	r2, r2, #1
+	cmp	r8, r2
+	vld1.8	{q9}, [r6]!
+	veor	q8, q9, q8
+	vst1.8	{q8}, [r1]!
+	bhi	.L17
+	cmp	r9, sl
+	add	r3, r3, sl
+	beq	.L1
+.L35:
+	ldr	r0, [r7, #200]
+.L25:
+	ldrb	r2, [r5, r3]	@ zero_extendqisi2
+	ldrb	r1, [r3, r0]	@ zero_extendqisi2
+	eors	r2, r2, r1
+	strb	r2, [r4, r3]
+	adds	r3, r3, #1
+	cmp	ip, r3
+	bhi	.L25
+.L1:
+	add	r7, r7, #304
+	mov	sp, r7
+	fldmfdd	sp!, {d8, d9, d10, d11, d12, d13, d14, d15}
+	pop	{r4, r5, r6, r7, r8, r9, sl, fp}
+	bx	lr
+.L37:
+	cmp	ip, #31
+	vld1.64	{d0-d1}, [r5:64]
+	vadd.i32	q9, q12, q9
+	veor	q11, q11, q0
+	vst1.64	{d22-d23}, [r4:64]
+	bls	.L12
+	cmp	ip, #47
+	vldr	d2, [r5, #16]
+	vldr	d3, [r5, #24]
+	vadd.i32	q13, q13, q14
+	veor	q9, q9, q1
+	vstr	d18, [r4, #16]
+	vstr	d19, [r4, #24]
+	bls	.L13
+	vadd.i32	q8, q8, q10
+	vldr	d0, [r5, #32]
+	vldr	d1, [r5, #40]
+	ldr	r6, [r7, #200]
+	vstr	d16, [r6, #48]
+	vstr	d17, [r6, #56]
+	veor	q8, q13, q0
+	vstr	d16, [r4, #32]
+	vstr	d17, [r4, #40]
+	b	.L14
+.L12:
+	ldr	r8, [r7, #200]
+	vstr	d18, [r8, #16]
+	vstr	d19, [r8, #24]
+	b	.L14
+.L20:
+	ldr	r5, [r7, #184]
+	ldr	r4, [r7, #188]
+	b	.L2
+.L13:
+	ldr	r6, [r7, #200]
+	vstr	d26, [r6, #32]
+	vstr	d27, [r6, #40]
+	b	.L14
+.L42:
+	.align	3
+.L41:
+	.word	1
+	.word	0
+	.word	0
+	.word	0
+	.size	CRYPTO_chacha_20_neon, .-CRYPTO_chacha_20_neon
+	.section	.rodata
+	.align	3
+.LANCHOR0 = . + 0
+.LC0:
+	.word	1634760805
+	.word	857760878
+	.word	2036477234
+	.word	1797285236
+	.ident	"GCC: (crosstool-NG linaro-1.13.1-4.7-2012.10-20121022 - Linaro GCC 2012.10) 4.7.3 20121001 (prerelease)"
+	.section	.note.GNU-stack,"",%progbits
diff --git a/crypto/cryptlib.c b/crypto/cryptlib.c
index 7bef015..3b6ab1d 100644
--- a/crypto/cryptlib.c
+++ b/crypto/cryptlib.c
@@ -661,6 +661,20 @@ const char *CRYPTO_get_lock_name(int type)
 		return(sk_OPENSSL_STRING_value(app_locks,type-CRYPTO_NUM_LOCKS));
 	}
 
+#if __arm__
+static int global_arm_neon_enabled = 0;
+
+void CRYPTO_set_NEON_capable(int on)
+	{
+	global_arm_neon_enabled = on != 0;
+	}
+
+int CRYPTO_is_NEON_capable()
+	{
+	return global_arm_neon_enabled;
+	}
+#endif
+
 #if	defined(__i386)   || defined(__i386__)   || defined(_M_IX86) || \
 	defined(__INTEL__) || \
 	defined(__x86_64) || defined(__x86_64__) || defined(_M_AMD64) || defined(_M_X64)
diff --git a/crypto/crypto.h b/crypto/crypto.h
index e11ac73..db339c3 100644
--- a/crypto/crypto.h
+++ b/crypto/crypto.h
@@ -414,6 +414,14 @@ void CRYPTO_cleanup_all_ex_data(void);
 
 int CRYPTO_get_new_lockid(char *name);
 
+/* CRYPTO_set_NEON_capable enables any NEON (ARM vector) dependent code. This
+ * code should be called before any non-init functions. */
+void CRYPTO_set_NEON_capable(int on);
+
+/* CRYPTO_is_NEON_capable returns the last value given to
+ * CRYPTO_set_NEON_capable, or else zero if it has never been called. */
+int CRYPTO_is_NEON_capable();
+
 int CRYPTO_num_locks(void); /* return CRYPTO_NUM_LOCKS (shared libs!) */
 void CRYPTO_lock(int mode, int type,const char *file,int line);
 void CRYPTO_set_locking_callback(void (*func)(int mode,int type,
diff --git a/crypto/poly1305/poly1305.c b/crypto/poly1305/poly1305.c
index 2e5621d..00d53bf 100644
--- a/crypto/poly1305/poly1305.c
+++ b/crypto/poly1305/poly1305.c
@@ -90,6 +90,17 @@ static void U32TO8_LE(unsigned char *m, uint32_t v)
 	}
 #endif
 
+#if __arm__
+void CRYPTO_poly1305_init_neon(poly1305_state* state,
+			       const unsigned char key[32]);
+
+void CRYPTO_poly1305_update_neon(poly1305_state* state,
+				 const unsigned char *in,
+				 size_t in_len);
+
+void CRYPTO_poly1305_finish_neon(poly1305_state* state, unsigned char mac[16]);
+#endif
+
 static uint64_t
 mul32x32_64(uint32_t a, uint32_t b)
 	{
@@ -207,6 +218,14 @@ void CRYPTO_poly1305_init(poly1305_state *statep, const unsigned char key[32])
 	struct poly1305_state_st *state = (struct poly1305_state_st*) statep;
 	uint32_t t0,t1,t2,t3;
 
+#if __arm__
+	if (CRYPTO_is_NEON_capable())
+		{
+		CRYPTO_poly1305_init_neon(statep, key);
+		return;
+		}
+#endif
+
 	t0 = U8TO32_LE(key+0);
 	t1 = U8TO32_LE(key+4);
 	t2 = U8TO32_LE(key+8);
@@ -241,6 +260,14 @@ void CRYPTO_poly1305_update(poly1305_state *statep, const unsigned char *in,
 	unsigned int i;
 	struct poly1305_state_st *state = (struct poly1305_state_st*) statep;
 
+#if __arm__
+	if (CRYPTO_is_NEON_capable())
+		{
+		CRYPTO_poly1305_update_neon(statep, in, in_len);
+		return;
+		}
+#endif
+
 	if (state->buf_used)
 		{
 		unsigned int todo = 16 - state->buf_used;
@@ -282,6 +309,14 @@ void CRYPTO_poly1305_finish(poly1305_state *statep, unsigned char mac[16])
 	uint32_t g0,g1,g2,g3,g4;
 	uint32_t b, nb;
 
+#if __arm__
+	if (CRYPTO_is_NEON_capable())
+		{
+		CRYPTO_poly1305_finish_neon(statep, mac);
+		return;
+		}
+#endif
+
 	if (state->buf_used)
 		poly1305_update(state, state->buf, state->buf_used);
 
diff --git a/crypto/poly1305/poly1305_arm.c b/crypto/poly1305/poly1305_arm.c
index adcef35..34e339d 100644
--- a/crypto/poly1305/poly1305_arm.c
+++ b/crypto/poly1305/poly1305_arm.c
@@ -51,6 +51,7 @@
  * SUPERCOP by D. J. Bernstein and Peter Schwabe. */
 
 #include <stdint.h>
+#include <string.h>
 
 #include <openssl/poly1305.h>
 
@@ -202,7 +203,8 @@ struct poly1305_state_st {
 	unsigned char key[16];
 };
 
-void CRYPTO_poly1305_init(poly1305_state *state, const unsigned char key[32])
+void CRYPTO_poly1305_init_neon(poly1305_state *state,
+			       const unsigned char key[32])
 	{
 	struct poly1305_state_st *st = (struct poly1305_state_st*) (state);
 	fe1305x2 *const r = (fe1305x2 *) (st->data + (15 & (-(int) st->data)));
@@ -227,7 +229,8 @@ void CRYPTO_poly1305_init(poly1305_state *state, const unsigned char key[32])
 	st->buf_used = 0;
 	}
 
-void CRYPTO_poly1305_update(poly1305_state *state, const unsigned char *in, size_t in_len)
+void CRYPTO_poly1305_update_neon(poly1305_state *state, const unsigned char *in,
+				 size_t in_len)
 	{
 	struct poly1305_state_st *st = (struct poly1305_state_st*) (state);
 	fe1305x2 *const r = (fe1305x2 *) (st->data + (15 & (-(int) st->data)));
@@ -285,7 +288,7 @@ void CRYPTO_poly1305_update(poly1305_state *state, const unsigned char *in, size
 		}
 	}
 
-void CRYPTO_poly1305_finish(poly1305_state* state, unsigned char mac[16])
+void CRYPTO_poly1305_finish_neon(poly1305_state* state, unsigned char mac[16])
 	{
 	struct poly1305_state_st *st = (struct poly1305_state_st*) (state);
 	fe1305x2 *const r = (fe1305x2 *) (st->data + (15 & (-(int) st->data)));
-- 
1.8.4.1