| // SPDX-License-Identifier: GPL-2.0 |
| /* |
| * ARM64 NEON-accelerated implementation of Speck128-XTS and Speck64-XTS |
| * |
| * Copyright (c) 2018 Google, Inc |
| * |
| * Author: Eric Biggers <ebiggers@google.com> |
| */ |
| |
| #include <linux/linkage.h> |
| |
| .text |
| |
| // arguments |
| ROUND_KEYS .req x0 // const {u64,u32} *round_keys |
| NROUNDS .req w1 // int nrounds |
| NROUNDS_X .req x1 |
| DST .req x2 // void *dst |
| SRC .req x3 // const void *src |
| NBYTES .req w4 // unsigned int nbytes |
| TWEAK .req x5 // void *tweak |
| |
| // registers which hold the data being encrypted/decrypted |
| // (underscores avoid a naming collision with ARM64 registers x0-x3) |
| X_0 .req v0 |
| Y_0 .req v1 |
| X_1 .req v2 |
| Y_1 .req v3 |
| X_2 .req v4 |
| Y_2 .req v5 |
| X_3 .req v6 |
| Y_3 .req v7 |
| |
| // the round key, duplicated in all lanes |
| ROUND_KEY .req v8 |
| |
| // index vector for tbl-based 8-bit rotates |
| ROTATE_TABLE .req v9 |
| ROTATE_TABLE_Q .req q9 |
| |
| // temporary registers |
| TMP0 .req v10 |
| TMP1 .req v11 |
| TMP2 .req v12 |
| TMP3 .req v13 |
| |
| // multiplication table for updating XTS tweaks |
| GFMUL_TABLE .req v14 |
| GFMUL_TABLE_Q .req q14 |
| |
| // next XTS tweak value(s) |
| TWEAKV_NEXT .req v15 |
| |
| // XTS tweaks for the blocks currently being encrypted/decrypted |
| TWEAKV0 .req v16 |
| TWEAKV1 .req v17 |
| TWEAKV2 .req v18 |
| TWEAKV3 .req v19 |
| TWEAKV4 .req v20 |
| TWEAKV5 .req v21 |
| TWEAKV6 .req v22 |
| TWEAKV7 .req v23 |
| |
| .align 4 |
| .Lror64_8_table: |
| .octa 0x080f0e0d0c0b0a090007060504030201 |
| .Lror32_8_table: |
| .octa 0x0c0f0e0d080b0a090407060500030201 |
| .Lrol64_8_table: |
| .octa 0x0e0d0c0b0a09080f0605040302010007 |
| .Lrol32_8_table: |
| .octa 0x0e0d0c0f0a09080b0605040702010003 |
| .Lgf128mul_table: |
| .octa 0x00000000000000870000000000000001 |
| .Lgf64mul_table: |
| .octa 0x0000000000000000000000002d361b00 |
| |
| /* |
| * _speck_round_128bytes() - Speck encryption round on 128 bytes at a time |
| * |
| * Do one Speck encryption round on the 128 bytes (8 blocks for Speck128, 16 for |
| * Speck64) stored in X0-X3 and Y0-Y3, using the round key stored in all lanes |
| * of ROUND_KEY. 'n' is the lane size: 64 for Speck128, or 32 for Speck64. |
| * 'lanes' is the lane specifier: "2d" for Speck128 or "4s" for Speck64. |
| */ |
| .macro _speck_round_128bytes n, lanes |
| |
| // x = ror(x, 8) |
| tbl X_0.16b, {X_0.16b}, ROTATE_TABLE.16b |
| tbl X_1.16b, {X_1.16b}, ROTATE_TABLE.16b |
| tbl X_2.16b, {X_2.16b}, ROTATE_TABLE.16b |
| tbl X_3.16b, {X_3.16b}, ROTATE_TABLE.16b |
| |
| // x += y |
| add X_0.\lanes, X_0.\lanes, Y_0.\lanes |
| add X_1.\lanes, X_1.\lanes, Y_1.\lanes |
| add X_2.\lanes, X_2.\lanes, Y_2.\lanes |
| add X_3.\lanes, X_3.\lanes, Y_3.\lanes |
| |
| // x ^= k |
| eor X_0.16b, X_0.16b, ROUND_KEY.16b |
| eor X_1.16b, X_1.16b, ROUND_KEY.16b |
| eor X_2.16b, X_2.16b, ROUND_KEY.16b |
| eor X_3.16b, X_3.16b, ROUND_KEY.16b |
| |
| // y = rol(y, 3) |
| shl TMP0.\lanes, Y_0.\lanes, #3 |
| shl TMP1.\lanes, Y_1.\lanes, #3 |
| shl TMP2.\lanes, Y_2.\lanes, #3 |
| shl TMP3.\lanes, Y_3.\lanes, #3 |
| sri TMP0.\lanes, Y_0.\lanes, #(\n - 3) |
| sri TMP1.\lanes, Y_1.\lanes, #(\n - 3) |
| sri TMP2.\lanes, Y_2.\lanes, #(\n - 3) |
| sri TMP3.\lanes, Y_3.\lanes, #(\n - 3) |
| |
| // y ^= x |
| eor Y_0.16b, TMP0.16b, X_0.16b |
| eor Y_1.16b, TMP1.16b, X_1.16b |
| eor Y_2.16b, TMP2.16b, X_2.16b |
| eor Y_3.16b, TMP3.16b, X_3.16b |
| .endm |
| |
| /* |
| * _speck_unround_128bytes() - Speck decryption round on 128 bytes at a time |
| * |
| * This is the inverse of _speck_round_128bytes(). |
| */ |
| .macro _speck_unround_128bytes n, lanes |
| |
| // y ^= x |
| eor TMP0.16b, Y_0.16b, X_0.16b |
| eor TMP1.16b, Y_1.16b, X_1.16b |
| eor TMP2.16b, Y_2.16b, X_2.16b |
| eor TMP3.16b, Y_3.16b, X_3.16b |
| |
| // y = ror(y, 3) |
| ushr Y_0.\lanes, TMP0.\lanes, #3 |
| ushr Y_1.\lanes, TMP1.\lanes, #3 |
| ushr Y_2.\lanes, TMP2.\lanes, #3 |
| ushr Y_3.\lanes, TMP3.\lanes, #3 |
| sli Y_0.\lanes, TMP0.\lanes, #(\n - 3) |
| sli Y_1.\lanes, TMP1.\lanes, #(\n - 3) |
| sli Y_2.\lanes, TMP2.\lanes, #(\n - 3) |
| sli Y_3.\lanes, TMP3.\lanes, #(\n - 3) |
| |
| // x ^= k |
| eor X_0.16b, X_0.16b, ROUND_KEY.16b |
| eor X_1.16b, X_1.16b, ROUND_KEY.16b |
| eor X_2.16b, X_2.16b, ROUND_KEY.16b |
| eor X_3.16b, X_3.16b, ROUND_KEY.16b |
| |
| // x -= y |
| sub X_0.\lanes, X_0.\lanes, Y_0.\lanes |
| sub X_1.\lanes, X_1.\lanes, Y_1.\lanes |
| sub X_2.\lanes, X_2.\lanes, Y_2.\lanes |
| sub X_3.\lanes, X_3.\lanes, Y_3.\lanes |
| |
| // x = rol(x, 8) |
| tbl X_0.16b, {X_0.16b}, ROTATE_TABLE.16b |
| tbl X_1.16b, {X_1.16b}, ROTATE_TABLE.16b |
| tbl X_2.16b, {X_2.16b}, ROTATE_TABLE.16b |
| tbl X_3.16b, {X_3.16b}, ROTATE_TABLE.16b |
| .endm |
| |
| .macro _next_xts_tweak next, cur, tmp, n |
| .if \n == 64 |
| /* |
| * Calculate the next tweak by multiplying the current one by x, |
| * modulo p(x) = x^128 + x^7 + x^2 + x + 1. |
| */ |
| sshr \tmp\().2d, \cur\().2d, #63 |
| and \tmp\().16b, \tmp\().16b, GFMUL_TABLE.16b |
| shl \next\().2d, \cur\().2d, #1 |
| ext \tmp\().16b, \tmp\().16b, \tmp\().16b, #8 |
| eor \next\().16b, \next\().16b, \tmp\().16b |
| .else |
| /* |
| * Calculate the next two tweaks by multiplying the current ones by x^2, |
| * modulo p(x) = x^64 + x^4 + x^3 + x + 1. |
| */ |
| ushr \tmp\().2d, \cur\().2d, #62 |
| shl \next\().2d, \cur\().2d, #2 |
| tbl \tmp\().16b, {GFMUL_TABLE.16b}, \tmp\().16b |
| eor \next\().16b, \next\().16b, \tmp\().16b |
| .endif |
| .endm |
| |
| /* |
| * _speck_xts_crypt() - Speck-XTS encryption/decryption |
| * |
| * Encrypt or decrypt NBYTES bytes of data from the SRC buffer to the DST buffer |
| * using Speck-XTS, specifically the variant with a block size of '2n' and round |
| * count given by NROUNDS. The expanded round keys are given in ROUND_KEYS, and |
| * the current XTS tweak value is given in TWEAK. It's assumed that NBYTES is a |
| * nonzero multiple of 128. |
| */ |
| .macro _speck_xts_crypt n, lanes, decrypting |
| |
| /* |
| * If decrypting, modify the ROUND_KEYS parameter to point to the last |
| * round key rather than the first, since for decryption the round keys |
| * are used in reverse order. |
| */ |
| .if \decrypting |
| mov NROUNDS, NROUNDS /* zero the high 32 bits */ |
| .if \n == 64 |
| add ROUND_KEYS, ROUND_KEYS, NROUNDS_X, lsl #3 |
| sub ROUND_KEYS, ROUND_KEYS, #8 |
| .else |
| add ROUND_KEYS, ROUND_KEYS, NROUNDS_X, lsl #2 |
| sub ROUND_KEYS, ROUND_KEYS, #4 |
| .endif |
| .endif |
| |
| // Load the index vector for tbl-based 8-bit rotates |
| .if \decrypting |
| ldr ROTATE_TABLE_Q, .Lrol\n\()_8_table |
| .else |
| ldr ROTATE_TABLE_Q, .Lror\n\()_8_table |
| .endif |
| |
| // One-time XTS preparation |
| .if \n == 64 |
| // Load first tweak |
| ld1 {TWEAKV0.16b}, [TWEAK] |
| |
| // Load GF(2^128) multiplication table |
| ldr GFMUL_TABLE_Q, .Lgf128mul_table |
| .else |
| // Load first tweak |
| ld1 {TWEAKV0.8b}, [TWEAK] |
| |
| // Load GF(2^64) multiplication table |
| ldr GFMUL_TABLE_Q, .Lgf64mul_table |
| |
| // Calculate second tweak, packing it together with the first |
| ushr TMP0.2d, TWEAKV0.2d, #63 |
| shl TMP1.2d, TWEAKV0.2d, #1 |
| tbl TMP0.8b, {GFMUL_TABLE.16b}, TMP0.8b |
| eor TMP0.8b, TMP0.8b, TMP1.8b |
| mov TWEAKV0.d[1], TMP0.d[0] |
| .endif |
| |
| .Lnext_128bytes_\@: |
| |
| // Calculate XTS tweaks for next 128 bytes |
| _next_xts_tweak TWEAKV1, TWEAKV0, TMP0, \n |
| _next_xts_tweak TWEAKV2, TWEAKV1, TMP0, \n |
| _next_xts_tweak TWEAKV3, TWEAKV2, TMP0, \n |
| _next_xts_tweak TWEAKV4, TWEAKV3, TMP0, \n |
| _next_xts_tweak TWEAKV5, TWEAKV4, TMP0, \n |
| _next_xts_tweak TWEAKV6, TWEAKV5, TMP0, \n |
| _next_xts_tweak TWEAKV7, TWEAKV6, TMP0, \n |
| _next_xts_tweak TWEAKV_NEXT, TWEAKV7, TMP0, \n |
| |
| // Load the next source blocks into {X,Y}[0-3] |
| ld1 {X_0.16b-Y_1.16b}, [SRC], #64 |
| ld1 {X_2.16b-Y_3.16b}, [SRC], #64 |
| |
| // XOR the source blocks with their XTS tweaks |
| eor TMP0.16b, X_0.16b, TWEAKV0.16b |
| eor Y_0.16b, Y_0.16b, TWEAKV1.16b |
| eor TMP1.16b, X_1.16b, TWEAKV2.16b |
| eor Y_1.16b, Y_1.16b, TWEAKV3.16b |
| eor TMP2.16b, X_2.16b, TWEAKV4.16b |
| eor Y_2.16b, Y_2.16b, TWEAKV5.16b |
| eor TMP3.16b, X_3.16b, TWEAKV6.16b |
| eor Y_3.16b, Y_3.16b, TWEAKV7.16b |
| |
| /* |
| * De-interleave the 'x' and 'y' elements of each block, i.e. make it so |
| * that the X[0-3] registers contain only the second halves of blocks, |
| * and the Y[0-3] registers contain only the first halves of blocks. |
| * (Speck uses the order (y, x) rather than the more intuitive (x, y).) |
| */ |
| uzp2 X_0.\lanes, TMP0.\lanes, Y_0.\lanes |
| uzp1 Y_0.\lanes, TMP0.\lanes, Y_0.\lanes |
| uzp2 X_1.\lanes, TMP1.\lanes, Y_1.\lanes |
| uzp1 Y_1.\lanes, TMP1.\lanes, Y_1.\lanes |
| uzp2 X_2.\lanes, TMP2.\lanes, Y_2.\lanes |
| uzp1 Y_2.\lanes, TMP2.\lanes, Y_2.\lanes |
| uzp2 X_3.\lanes, TMP3.\lanes, Y_3.\lanes |
| uzp1 Y_3.\lanes, TMP3.\lanes, Y_3.\lanes |
| |
| // Do the cipher rounds |
| mov x6, ROUND_KEYS |
| mov w7, NROUNDS |
| .Lnext_round_\@: |
| .if \decrypting |
| ld1r {ROUND_KEY.\lanes}, [x6] |
| sub x6, x6, #( \n / 8 ) |
| _speck_unround_128bytes \n, \lanes |
| .else |
| ld1r {ROUND_KEY.\lanes}, [x6], #( \n / 8 ) |
| _speck_round_128bytes \n, \lanes |
| .endif |
| subs w7, w7, #1 |
| bne .Lnext_round_\@ |
| |
| // Re-interleave the 'x' and 'y' elements of each block |
| zip1 TMP0.\lanes, Y_0.\lanes, X_0.\lanes |
| zip2 Y_0.\lanes, Y_0.\lanes, X_0.\lanes |
| zip1 TMP1.\lanes, Y_1.\lanes, X_1.\lanes |
| zip2 Y_1.\lanes, Y_1.\lanes, X_1.\lanes |
| zip1 TMP2.\lanes, Y_2.\lanes, X_2.\lanes |
| zip2 Y_2.\lanes, Y_2.\lanes, X_2.\lanes |
| zip1 TMP3.\lanes, Y_3.\lanes, X_3.\lanes |
| zip2 Y_3.\lanes, Y_3.\lanes, X_3.\lanes |
| |
| // XOR the encrypted/decrypted blocks with the tweaks calculated earlier |
| eor X_0.16b, TMP0.16b, TWEAKV0.16b |
| eor Y_0.16b, Y_0.16b, TWEAKV1.16b |
| eor X_1.16b, TMP1.16b, TWEAKV2.16b |
| eor Y_1.16b, Y_1.16b, TWEAKV3.16b |
| eor X_2.16b, TMP2.16b, TWEAKV4.16b |
| eor Y_2.16b, Y_2.16b, TWEAKV5.16b |
| eor X_3.16b, TMP3.16b, TWEAKV6.16b |
| eor Y_3.16b, Y_3.16b, TWEAKV7.16b |
| mov TWEAKV0.16b, TWEAKV_NEXT.16b |
| |
| // Store the ciphertext in the destination buffer |
| st1 {X_0.16b-Y_1.16b}, [DST], #64 |
| st1 {X_2.16b-Y_3.16b}, [DST], #64 |
| |
| // Continue if there are more 128-byte chunks remaining |
| subs NBYTES, NBYTES, #128 |
| bne .Lnext_128bytes_\@ |
| |
| // Store the next tweak and return |
| .if \n == 64 |
| st1 {TWEAKV_NEXT.16b}, [TWEAK] |
| .else |
| st1 {TWEAKV_NEXT.8b}, [TWEAK] |
| .endif |
| ret |
| .endm |
| |
| ENTRY(speck128_xts_encrypt_neon) |
| _speck_xts_crypt n=64, lanes=2d, decrypting=0 |
| ENDPROC(speck128_xts_encrypt_neon) |
| |
| ENTRY(speck128_xts_decrypt_neon) |
| _speck_xts_crypt n=64, lanes=2d, decrypting=1 |
| ENDPROC(speck128_xts_decrypt_neon) |
| |
| ENTRY(speck64_xts_encrypt_neon) |
| _speck_xts_crypt n=32, lanes=4s, decrypting=0 |
| ENDPROC(speck64_xts_encrypt_neon) |
| |
| ENTRY(speck64_xts_decrypt_neon) |
| _speck_xts_crypt n=32, lanes=4s, decrypting=1 |
| ENDPROC(speck64_xts_decrypt_neon) |
