Adding memcpy tuned for Cortex-A15.
The strategy for large block sizes is LDRD and STRD with offset addressing,
where the main loop copies 64 bytes in every iteration, (i.e., 8 calls to
LDRD and STRD pairs), interleaving load and stores (i.e., the pairs of LDRD
and STRD of the same data are consecutive instructions), and the writeback
of an updated address is a separate instruction, which allows us to write
back the accumulated update once per iteration.
This strategy is implemented in memcpy.S. In some configurations, a plain
version of memcpy (included from memcpy-stub.c) is used instead of the
optimized one.
Validation:
* Correctness: checked memcpy using a test harness for block sizes
ranging between 1 to 128, and source and destination buffers alignment
ranging in { 0,1,2,3,4,8,12 } bytes each.
* Performance: benchmarking on Cortex-A15 FPGA indicates that this strategy
is better for A15 than the strategy used by glibc and even slightly better
than using NEON. Benchmarking on Cortex-A9 bare metal and Linux shows
that the proposed strategy is reasonable: not as fast as the version of
memcpy from glibc (which is the best open source strategy for A9), but
comparable with csl and bionic.
* Integration with GCC: no regression for arm-none-eabi --with-cpu
cortex-a15 and cortex-a9.
Change-Id: Ied56354d8992c62ae3e02d582a2bd55585d814b9
Signed-off-by: Vassilis Laganakos <vasileios.laganakos@arm.com>
diff --git a/libc/arch-arm/bionic/memcpy.a15.S b/libc/arch-arm/bionic/memcpy.a15.S
new file mode 100644
index 0000000..d1bfb7c
--- /dev/null
+++ b/libc/arch-arm/bionic/memcpy.a15.S
@@ -0,0 +1,423 @@
+/*
+ * Copyright (c) 2013 ARM Ltd
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ * 3. The name of the company may not be used to endorse or promote
+ * products derived from this software without specific prior written
+ * permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY ARM LTD ``AS IS'' AND ANY EXPRESS OR IMPLIED
+ * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+ * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ * IN NO EVENT SHALL ARM LTD BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
+ * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+ * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+ * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#if (defined (__OPTIMIZE_SIZE__) || defined (PREFER_SIZE_OVER_SPEED) || \
+ (!(defined (__ARM_ARCH_7A__))))
+
+ /* Do nothing here. See memcpy-stub.c in the same directory. */
+
+#else
+ /* Prototype: void *memcpy (void *dst, const void *src, size_t count). */
+
+ /* Use the version of memcpy implemented using LDRD and STRD.
+ This version is tuned for Cortex-A15.
+ This might not be the best for other ARMv7-A CPUs,
+ but there is no predefine to distinguish between
+ different CPUs in the same architecture,
+ and this version is better than the plain memcpy provided in newlib.
+
+ Therefore, we use this version for all ARMv7-A CPUS. */
+
+ /* To make the same code compile for both ARM and Thumb instruction
+ sets, switch to unified syntax at the beginning of this function.
+ However, by using the same code, we may be missing optimization
+ opportunities. For instance, in LDRD/STRD instructions, the first
+ destination register must be even and the second consecutive in
+ ARM state, but not in Thumb state. */
+
+ .syntax unified
+
+#if defined (__thumb__)
+ .thumb
+ .thumb_func
+#endif
+
+ .global memcpy
+ .type memcpy, %function
+memcpy:
+
+ /* Assumes that n >= 0, and dst, src are valid pointers.
+ If there is at least 8 bytes to copy, use LDRD/STRD.
+ If src and dst are misaligned with different offsets,
+ first copy byte by byte until dst is aligned,
+ and then copy using LDRD/STRD and shift if needed.
+ When less than 8 left, copy a word and then byte by byte. */
+
+ /* Save registers (r0 holds the return value):
+ optimized push {r0, r4, r5, lr}.
+ To try and improve performance, stack layout changed,
+ i.e., not keeping the stack looking like users expect
+ (highest numbered register at highest address). */
+ push {r0, lr}
+ strd r4, r5, [sp, #-8]!
+
+ /* TODO: Add debug frame directives.
+ We don't need exception unwind directives, because the code below
+ does not throw any exceptions and does not call any other functions.
+ Generally, newlib functions like this lack debug information for
+ assembler source. */
+
+ /* Get copying of tiny blocks out of the way first. */
+ /* Is there at least 4 bytes to copy? */
+ subs r2, r2, #4
+ blt copy_less_than_4 /* If n < 4. */
+
+ /* Check word alignment. */
+ ands ip, r0, #3 /* ip = last 2 bits of dst. */
+ bne dst_not_word_aligned /* If dst is not word-aligned. */
+
+ /* Get here if dst is word-aligned. */
+ ands ip, r1, #3 /* ip = last 2 bits of src. */
+ bne src_not_word_aligned /* If src is not word-aligned. */
+word_aligned:
+ /* Get here if source and dst both are word-aligned.
+ The number of bytes remaining to copy is r2+4. */
+
+ /* Is there is at least 64 bytes to copy? */
+ subs r2, r2, #60
+ blt copy_less_than_64 /* If r2 + 4 < 64. */
+
+ /* First, align the destination buffer to 8-bytes,
+ to make sure double loads and stores don't cross cache line boundary,
+ as they are then more expensive even if the data is in the cache
+ (require two load/store issue cycles instead of one).
+ If only one of the buffers is not 8-bytes aligned,
+ then it's more important to align dst than src,
+ because there is more penalty for stores
+ than loads that cross cacheline boundary.
+ This check and realignment are only worth doing
+ if there is a lot to copy. */
+
+ /* Get here if dst is word aligned,
+ i.e., the 2 least significant bits are 0.
+ If dst is not 2w aligned (i.e., the 3rd bit is not set in dst),
+ then copy 1 word (4 bytes). */
+ ands r3, r0, #4
+ beq 11f /* If dst already two-word aligned. */
+ ldr r3, [r1], #4
+ str r3, [r0], #4
+ subs r2, r2, #4
+ blt copy_less_than_64
+
+11:
+ /* TODO: Align to cacheline (useful for PLD optimization). */
+
+ /* Every loop iteration copies 64 bytes. */
+1:
+ .irp offset, #0, #8, #16, #24, #32, #40, #48, #56
+ ldrd r4, r5, [r1, \offset]
+ strd r4, r5, [r0, \offset]
+ .endr
+
+ add r0, r0, #64
+ add r1, r1, #64
+ subs r2, r2, #64
+ bge 1b /* If there is more to copy. */
+
+copy_less_than_64:
+
+ /* Get here if less than 64 bytes to copy, -64 <= r2 < 0.
+ Restore the count if there is more than 7 bytes to copy. */
+ adds r2, r2, #56
+ blt copy_less_than_8
+
+ /* Copy 8 bytes at a time. */
+2:
+ ldrd r4, r5, [r1], #8
+ strd r4, r5, [r0], #8
+ subs r2, r2, #8
+ bge 2b /* If there is more to copy. */
+
+copy_less_than_8:
+
+ /* Get here if less than 8 bytes to copy, -8 <= r2 < 0.
+ Check if there is more to copy. */
+ cmn r2, #8
+ beq return /* If r2 + 8 == 0. */
+
+ /* Restore the count if there is more than 3 bytes to copy. */
+ adds r2, r2, #4
+ blt copy_less_than_4
+
+ /* Copy 4 bytes. */
+ ldr r3, [r1], #4
+ str r3, [r0], #4
+
+copy_less_than_4:
+ /* Get here if less than 4 bytes to copy, -4 <= r2 < 0. */
+
+ /* Restore the count, check if there is more to copy. */
+ adds r2, r2, #4
+ beq return /* If r2 == 0. */
+
+ /* Get here with r2 is in {1,2,3}={01,10,11}. */
+ /* Logical shift left r2, insert 0s, update flags. */
+ lsls r2, r2, #31
+
+ /* Copy byte by byte.
+ Condition ne means the last bit of r2 is 0.
+ Condition cs means the second to last bit of r2 is set,
+ i.e., r2 is 1 or 3. */
+ itt ne
+ ldrbne r3, [r1], #1
+ strbne r3, [r0], #1
+
+ itttt cs
+ ldrbcs r4, [r1], #1
+ ldrbcs r5, [r1]
+ strbcs r4, [r0], #1
+ strbcs r5, [r0]
+
+return:
+ /* Restore registers: optimized pop {r0, r4, r5, pc} */
+ ldrd r4, r5, [sp], #8
+ pop {r0, pc} /* This is the only return point of memcpy. */
+
+#ifndef __ARM_FEATURE_UNALIGNED
+
+ /* The following assembly macro implements misaligned copy in software.
+ Assumes that dst is word aligned, src is at offset "pull" bits from
+ word, push = 32 - pull, and the number of bytes that remain to copy
+ is r2 + 4, r2 >= 0. */
+
+ /* In the code below, r2 is the number of bytes that remain to be
+ written. The number of bytes read is always larger, because we have
+ partial words in the shift queue. */
+
+ .macro miscopy pull push shiftleft shiftright
+
+ /* Align src to the previous word boundary. */
+ bic r1, r1, #3
+
+ /* Initialize the shift queue. */
+ ldr r5, [r1], #4 /* Load a word from source. */
+
+ subs r2, r2, #4
+ blt 6f /* Go to misaligned copy of less than 8 bytes. */
+
+ /* Get here if there is more than 8 bytes to copy.
+ The number of bytes to copy is r2+8, r2 >= 0. */
+
+ /* Save registers: push { r6, r7 }.
+ We need additional registers for LDRD and STRD, because in ARM state
+ the first destination register must be even and the second
+ consecutive. */
+ strd r6, r7, [sp, #-8]!
+
+ subs r2, r2, #56
+ blt 4f /* Go to misaligned copy of less than 64 bytes. */
+
+3:
+ /* Get here if there is more than 64 bytes to copy.
+ The number of bytes to copy is r2+64, r2 >= 0. */
+
+ /* Copy 64 bytes in every iteration.
+ Use a partial word from the shift queue. */
+ .irp offset, #0, #8, #16, #24, #32, #40, #48, #56
+ mov r6, r5, \shiftleft #\pull
+ ldrd r4, r5, [r1, \offset]
+ orr r6, r6, r4, \shiftright #\push
+ mov r7, r4, \shiftleft #\pull
+ orr r7, r7, r5, \shiftright #\push
+ strd r6, r7, [r0, \offset]
+ .endr
+
+ add r1, r1, #64
+ add r0, r0, #64
+ subs r2, r2, #64
+ bge 3b
+
+4:
+ /* Get here if there is less than 64 bytes to copy (-64 <= r2 < 0)
+ and they are misaligned. */
+
+ /* Restore the count if there is more than 7 bytes to copy. */
+ adds r2, r2, #56
+
+ /* If less than 8 bytes to copy,
+ restore registers saved for this loop: optimized poplt { r6, r7 }. */
+ itt lt
+ ldrdlt r6, r7, [sp], #8
+ blt 6f /* Go to misaligned copy of less than 8 bytes. */
+
+5:
+ /* Copy 8 bytes at a time.
+ Use a partial word from the shift queue. */
+ mov r6, r5, \shiftleft #\pull
+ ldrd r4, r5, [r1], #8
+ orr r6, r6, r4, \shiftright #\push
+ mov r7, r4, \shiftleft #\pull
+ orr r7, r7, r5, \shiftright #\push
+ strd r6, r7, [r0], #8
+
+ subs r2, r2, #8
+ bge 5b /* If there is more to copy. */
+
+ /* Restore registers saved for this loop: optimized pop { r6, r7 }. */
+ ldrd r6, r7, [sp], #8
+
+6:
+ /* Get here if there less than 8 bytes to copy (-8 <= r2 < 0)
+ and they are misaligned. */
+
+ /* Check if there is more to copy. */
+ cmn r2, #8
+ beq return
+
+ /* Check if there is less than 4 bytes to copy. */
+ cmn r2, #4
+
+ itt lt
+ /* Restore src offset from word-align. */
+ sublt r1, r1, #(\push / 8)
+ blt copy_less_than_4
+
+ /* Use a partial word from the shift queue. */
+ mov r3, r5, \shiftleft #\pull
+ /* Load a word from src, but without writeback
+ (this word is not fully written to dst). */
+ ldr r5, [r1]
+
+ /* Restore src offset from word-align. */
+ add r1, r1, #(\pull / 8)
+
+ /* Shift bytes to create one dst word and store it. */
+ orr r3, r3, r5, \shiftright #\push
+ str r3, [r0], #4
+
+ /* Use single byte copying of the remaining bytes. */
+ b copy_less_than_4
+
+ .endm
+
+#endif /* not __ARM_FEATURE_UNALIGNED */
+
+dst_not_word_aligned:
+
+ /* Get here when dst is not aligned and ip has the last 2 bits of dst,
+ i.e., ip is the offset of dst from word.
+ The number of bytes that remains to copy is r2 + 4,
+ i.e., there are at least 4 bytes to copy.
+ Write a partial word (0 to 3 bytes), such that dst becomes
+ word-aligned. */
+
+ /* If dst is at ip bytes offset from a word (with 0 < ip < 4),
+ then there are (4 - ip) bytes to fill up to align dst to the next
+ word. */
+ rsb ip, ip, #4 /* ip = #4 - ip. */
+ cmp ip, #2
+
+ /* Copy byte by byte with conditionals. */
+ itt gt
+ ldrbgt r3, [r1], #1
+ strbgt r3, [r0], #1
+
+ itt ge
+ ldrbge r4, [r1], #1
+ strbge r4, [r0], #1
+
+ ldrb lr, [r1], #1
+ strb lr, [r0], #1
+
+ /* Update the count.
+ ip holds the number of bytes we have just copied. */
+ subs r2, r2, ip /* r2 = r2 - ip. */
+ blt copy_less_than_4 /* If r2 < ip. */
+
+ /* Get here if there are more than 4 bytes to copy.
+ Check if src is aligned. If beforehand src and dst were not word
+ aligned but congruent (same offset), then now they are both
+ word-aligned, and we can copy the rest efficiently (without
+ shifting). */
+ ands ip, r1, #3 /* ip = last 2 bits of src. */
+ beq word_aligned /* If r1 is word-aligned. */
+
+src_not_word_aligned:
+ /* Get here when src is not word-aligned, but dst is word-aligned.
+ The number of bytes that remains to copy is r2+4. */
+
+#ifdef __ARM_FEATURE_UNALIGNED
+ /* Copy word by word using LDR when alignment can be done in hardware,
+ i.e., SCTLR.A is set, supporting unaligned access in LDR and STR. */
+ subs r2, r2, #60
+ blt 8f
+
+7:
+ /* Copy 64 bytes in every loop iteration. */
+ .irp offset, #0, #4, #8, #12, #16, #20, #24, #28, #32, #36, #40, #44, #48, #52, #56, #60
+ ldr r3, [r1, \offset]
+ str r3, [r0, \offset]
+ .endr
+
+ add r0, r0, #64
+ add r1, r1, #64
+ subs r2, r2, #64
+ bge 7b
+
+8:
+ /* Get here if less than 64 bytes to copy, -64 <= r2 < 0.
+ Check if there is more than 3 bytes to copy. */
+ adds r2, r2, #60
+ blt copy_less_than_4
+
+9:
+ /* Get here if there is less than 64 but at least 4 bytes to copy,
+ where the number of bytes to copy is r2+4. */
+ ldr r3, [r1], #4
+ str r3, [r0], #4
+ subs r2, r2, #4
+ bge 9b
+
+ b copy_less_than_4
+
+#else /* not __ARM_FEATURE_UNALIGNED */
+
+ /* ip has last 2 bits of src,
+ i.e., ip is the offset of src from word, and ip > 0.
+ Compute shifts needed to copy from src to dst. */
+ cmp ip, #2
+ beq miscopy_16_16 /* If ip == 2. */
+ bge miscopy_24_8 /* If ip == 3. */
+
+ /* Get here if ip == 1. */
+
+ /* Endian independent macros for shifting bytes within registers. */
+
+#ifndef __ARMEB__
+miscopy_8_24: miscopy pull=8 push=24 shiftleft=lsr shiftright=lsl
+miscopy_16_16: miscopy pull=16 push=16 shiftleft=lsr shiftright=lsl
+miscopy_24_8: miscopy pull=24 push=8 shiftleft=lsr shiftright=lsl
+#else /* not __ARMEB__ */
+miscopy_8_24: miscopy pull=8 push=24 shiftleft=lsl shiftright=lsr
+miscopy_16_16: miscopy pull=16 push=16 shiftleft=lsl shiftright=lsr
+miscopy_24_8: miscopy pull=24 push=8 shiftleft=lsl shiftright=lsr
+#endif /* not __ARMEB__ */
+
+#endif /* not __ARM_FEATURE_UNALIGNED */
+
+#endif /* memcpy */
