added support for doubles with sse2
(cherry picked from commit a52039328c08555b1143dbc9100b8f6df44f2b90)
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 47cecb5..11dad3b 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -7,6 +7,7 @@
# architecture/optimization options
option(USE_SIMD "use SIMD (SSE/AVX/NEON/ALTIVEC) CPU features? - " ON)
+option(DISABLE_SIMD_AVX "disable AVX CPU features? - " OFF)
option(USE_SIMD_NEON "force using NEON on ARM? (requires USE_SIMD)" OFF)
option(USE_SCALAR_VECT "use 4-element vector scalar operations (if no other SIMD)" ON)
@@ -133,10 +134,17 @@
endif()
if (USE_SIMD AND USE_TYPE_DOUBLE)
if(WIN32)
- set_property(SOURCE pffft_double.c PROPERTY COMPILE_FLAGS "/arch:AVX")
+ if(DISABLE_SIMD_AVX)
+ set_property(SOURCE pffft_double.c PROPERTY COMPILE_FLAGS "/arch:SSE2")
+ else()
+ set_property(SOURCE pffft_double.c PROPERTY COMPILE_FLAGS "/arch:AVX")
+ endif()
else()
set_property(SOURCE pffft_double.c PROPERTY COMPILE_FLAGS "-march=native")
endif()
+ if(DISABLE_SIMD_AVX)
+ target_compile_definitions(PFFFT PRIVATE PFFFT_AVX_DISABLE=1)
+ endif()
endif()
target_link_libraries( PFFFT ${MATHLIB} )
set_property(TARGET PFFFT APPEND PROPERTY INTERFACE_INCLUDE_DIRECTORIES
diff --git a/simd/pf_avx_double.h b/simd/pf_avx_double.h
index fe0efa8..251f0b9 100644
--- a/simd/pf_avx_double.h
+++ b/simd/pf_avx_double.h
@@ -46,7 +46,7 @@
/*
AVX support macros
*/
-#if !defined(SIMD_SZ) && !defined(PFFFT_SIMD_DISABLE) && defined(__AVX__)
+#if !defined(SIMD_SZ) && !defined(PFFFT_SIMD_DISABLE) && !defined(PFFFT_AVX_DISABLE) && defined(__AVX__)
#pragma message( __FILE__ ": AVX macros are defined" )
#include <immintrin.h>
diff --git a/simd/pf_double.h b/simd/pf_double.h
index c6c73ab..1025827 100644
--- a/simd/pf_double.h
+++ b/simd/pf_double.h
@@ -60,6 +60,7 @@
typedef double vsfscalar;
#include "pf_avx_double.h"
+#include "pf_sse2_double.h"
#include "pf_neon_double.h"
#ifndef SIMD_SZ
diff --git a/simd/pf_sse2_double.h b/simd/pf_sse2_double.h
new file mode 100644
index 0000000..1c1739d
--- /dev/null
+++ b/simd/pf_sse2_double.h
@@ -0,0 +1,272 @@
+/*
+ Copyright (c) 2020 Dario Mambro ( dario.mambro@gmail.com )
+*/
+
+/* Copyright (c) 2013 Julien Pommier ( pommier@modartt.com )
+
+ Redistribution and use of the Software in source and binary forms,
+ with or without modification, is permitted provided that the
+ following conditions are met:
+
+ - Neither the names of NCAR's Computational and Information Systems
+ Laboratory, the University Corporation for Atmospheric Research,
+ nor the names of its sponsors or contributors may be used to
+ endorse or promote products derived from this Software without
+ specific prior written permission.
+
+ - Redistributions of source code must retain the above copyright
+ notices, this list of conditions, and the disclaimer below.
+
+ - Redistributions in binary form must reproduce the above copyright
+ notice, this list of conditions, and the disclaimer below in the
+ documentation and/or other materials provided with the
+ distribution.
+
+ THIS SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ EXPRESS OR IMPLIED, INCLUDING, BUT NOT LIMITED TO THE WARRANTIES OF
+ MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ NONINFRINGEMENT. IN NO EVENT SHALL THE CONTRIBUTORS OR COPYRIGHT
+ HOLDERS BE LIABLE FOR ANY CLAIM, INDIRECT, INCIDENTAL, SPECIAL,
+ EXEMPLARY, OR CONSEQUENTIAL DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS WITH THE
+ SOFTWARE.
+*/
+
+#ifndef PF_NEON_DBL_H
+#define PF_NEON_DBL_H
+
+/*
+ SSE2 64bit support macros
+*/
+#if !defined(SIMD_SZ) && !defined(PFFFT_SIMD_DISABLE) && (( __SSE2__ ) || defined ( __x86_64__ ))
+#pragma message __FILE__ ": SSE2 double macros are defined"
+
+#include <emmintrin.h>
+
+typedef struct {
+ __m128d d128[2];
+} __m256d;
+
+typedef __m256d v4sf;
+
+# define SIMD_SZ 4
+
+typedef union v4sf_union {
+ v4sf v;
+ double f[SIMD_SZ];
+} v4sf_union;
+
+
+#if defined(__GNUC__) || defined(__clang__)
+
+#pragma push_macro("FORCE_INLINE")
+#define FORCE_INLINE static inline __attribute__((always_inline))
+
+#elif defined (_MSC_VER)
+#define FORCE_INLINE static __forceinline
+
+#else
+#error "Macro name collisions may happens with unknown compiler"
+#ifdef FORCE_INLINE
+#undef FORCE_INLINE
+#endif
+#define FORCE_INLINE static inline
+#endif
+
+FORCE_INLINE __m256d _mm256_setzero_pd(void)
+{
+ __m256d ret;
+ ret.d128[0] = ret.d128[1] = _mm_setzero_pd();
+ return ret;
+}
+
+FORCE_INLINE __m256d _mm256_mul_pd(__m256d a, __m256d b)
+{
+ __m256d ret;
+ ret.d128[0] = _mm_mul_pd(a.d128[0], b.d128[0]);
+ ret.d128[1] = _mm_mul_pd(a.d128[1], b.d128[1]);
+ return ret;
+}
+
+FORCE_INLINE __m256d _mm256_add_pd(__m256d a, __m256d b)
+{
+ __m256d ret;
+ ret.d128[0] = _mm_add_pd(a.d128[0], b.d128[0]);
+ ret.d128[1] = _mm_add_pd(a.d128[1], b.d128[1]);
+ return ret;
+}
+
+FORCE_INLINE __m256d _mm256_sub_pd(__m256d a, __m256d b)
+{
+ __m256d ret;
+ ret.d128[0] = _mm_sub_pd(a.d128[0], b.d128[0]);
+ ret.d128[1] = _mm_sub_pd(a.d128[1], b.d128[1]);
+ return ret;
+}
+
+FORCE_INLINE __m256d _mm256_set1_pd(double a)
+{
+ __m256d ret;
+ ret.d128[0] = ret.d128[1] = _mm_set1_pd(a);
+ return ret;
+}
+
+FORCE_INLINE __m256d _mm256_load_pd (double const * mem_addr)
+{
+ __m256d res;
+ res.d128[0] = _mm_load_pd((const double *)mem_addr);
+ res.d128[1] = _mm_load_pd((const double *)mem_addr + 2);
+ return res;
+}
+FORCE_INLINE __m256d _mm256_loadu_pd (double const * mem_addr)
+{
+ __m256d res;
+ res.d128[0] = _mm_loadu_pd((const double *)mem_addr);
+ res.d128[1] = _mm_loadu_pd((const double *)mem_addr + 2);
+ return res;
+}
+
+
+# define VARCH "SSE2"
+# define VREQUIRES_ALIGN 1
+# define VZERO() _mm256_setzero_pd()
+# define VMUL(a,b) _mm256_mul_pd(a,b)
+# define VADD(a,b) _mm256_add_pd(a,b)
+# define VMADD(a,b,c) _mm256_add_pd(_mm256_mul_pd(a,b), c)
+# define VSUB(a,b) _mm256_sub_pd(a,b)
+# define LD_PS1(p) _mm256_set1_pd(p)
+# define VLOAD_UNALIGNED(ptr) _mm256_loadu_pd(ptr)
+# define VLOAD_ALIGNED(ptr) _mm256_load_pd(ptr)
+
+
+FORCE_INLINE __m128d _mm256_castpd256_pd128(__m256d a)
+{
+ return a.d128[0];
+}
+
+FORCE_INLINE __m128d _mm256_extractf128_pd (__m256d a, const int imm8)
+{
+ assert(imm8 >= 0 && imm8 <= 1);
+ return a.d128[imm8];
+}
+FORCE_INLINE __m256d _mm256_insertf128_pd_1(__m256d a, __m128d b)
+{
+ __m256d res;
+ res.d128[0] = a.d128[0];
+ res.d128[1] = b;
+ return res;
+}
+FORCE_INLINE __m256d _mm256_castpd128_pd256(__m128d a)
+{
+ __m256d res;
+ res.d128[0] = a;
+ return res;
+}
+
+FORCE_INLINE __m256d _mm256_shuffle_pd_00(__m256d a, __m256d b)
+{
+ __m256d res;
+ res.d128[0] = _mm_shuffle_pd(a.d128[0],b.d128[0],0);
+ res.d128[1] = _mm_shuffle_pd(a.d128[1],b.d128[1],0);
+ return res;
+}
+
+FORCE_INLINE __m256d _mm256_shuffle_pd_11(__m256d a, __m256d b)
+{
+ __m256d res;
+ res.d128[0] = _mm_shuffle_pd(a.d128[0],b.d128[0], 3);
+ res.d128[1] = _mm_shuffle_pd(a.d128[1],b.d128[1], 3);
+ return res;
+}
+
+FORCE_INLINE __m256d _mm256_permute2f128_pd_0x20(__m256d a, __m256d b) {
+ __m256d res;
+ res.d128[0] = a.d128[0];
+ res.d128[1] = b.d128[0];
+ return res;
+}
+
+
+FORCE_INLINE __m256d _mm256_permute2f128_pd_0x31(__m256d a, __m256d b)
+{
+ __m256d res;
+ res.d128[0] = a.d128[1];
+ res.d128[1] = b.d128[1];
+ return res;
+}
+
+FORCE_INLINE __m256d _mm256_reverse(__m256d x)
+{
+ __m256d res;
+ res.d128[0] = _mm_shuffle_pd(x.d128[1],x.d128[1],1);
+ res.d128[1] = _mm_shuffle_pd(x.d128[0],x.d128[0],1);
+ return res;
+}
+
+/* INTERLEAVE2 (in1, in2, out1, out2) pseudo code:
+out1 = [ in1[0], in2[0], in1[1], in2[1] ]
+out2 = [ in1[2], in2[2], in1[3], in2[3] ]
+*/
+# define INTERLEAVE2(in1, in2, out1, out2) { \
+ __m128d low1__ = _mm256_castpd256_pd128(in1); \
+ __m128d low2__ = _mm256_castpd256_pd128(in2); \
+ __m128d high1__ = _mm256_extractf128_pd(in1, 1); \
+ __m128d high2__ = _mm256_extractf128_pd(in2, 1); \
+ __m256d tmp__ = _mm256_insertf128_pd_1( \
+ _mm256_castpd128_pd256(_mm_shuffle_pd(low1__, low2__, 0)), \
+ _mm_shuffle_pd(low1__, low2__, 3)); \
+ out2 = _mm256_insertf128_pd_1( \
+ _mm256_castpd128_pd256(_mm_shuffle_pd(high1__, high2__, 0)), \
+ _mm_shuffle_pd(high1__, high2__, 3)); \
+ out1 = tmp__; \
+}
+
+/*UNINTERLEAVE2(in1, in2, out1, out2) pseudo code:
+out1 = [ in1[0], in1[2], in2[0], in2[2] ]
+out2 = [ in1[1], in1[3], in2[1], in2[3] ]
+*/
+# define UNINTERLEAVE2(in1, in2, out1, out2) { \
+ __m128d low1__ = _mm256_castpd256_pd128(in1); \
+ __m128d low2__ = _mm256_castpd256_pd128(in2); \
+ __m128d high1__ = _mm256_extractf128_pd(in1, 1); \
+ __m128d high2__ = _mm256_extractf128_pd(in2, 1); \
+ __m256d tmp__ = _mm256_insertf128_pd_1( \
+ _mm256_castpd128_pd256(_mm_shuffle_pd(low1__, high1__, 0)), \
+ _mm_shuffle_pd(low2__, high2__, 0)); \
+ out2 = _mm256_insertf128_pd_1( \
+ _mm256_castpd128_pd256(_mm_shuffle_pd(low1__, high1__, 3)), \
+ _mm_shuffle_pd(low2__, high2__, 3)); \
+ out1 = tmp__; \
+}
+
+# define VTRANSPOSE4(row0, row1, row2, row3) { \
+ __m256d tmp3, tmp2, tmp1, tmp0; \
+ \
+ tmp0 = _mm256_shuffle_pd_00((row0),(row1)); \
+ tmp2 = _mm256_shuffle_pd_11((row0),(row1)); \
+ tmp1 = _mm256_shuffle_pd_00((row2),(row3)); \
+ tmp3 = _mm256_shuffle_pd_11((row2),(row3)); \
+ \
+ (row0) = _mm256_permute2f128_pd_0x20(tmp0, tmp1); \
+ (row1) = _mm256_permute2f128_pd_0x20(tmp2, tmp3); \
+ (row2) = _mm256_permute2f128_pd_0x31(tmp0, tmp1); \
+ (row3) = _mm256_permute2f128_pd_0x31(tmp2, tmp3); \
+ }
+
+/*VSWAPHL(a, b) pseudo code:
+return [ b[0], b[1], a[2], a[3] ]
+*/
+# define VSWAPHL(a,b) \
+ _mm256_insertf128_pd_1(_mm256_castpd128_pd256(_mm256_castpd256_pd128(b)), _mm256_extractf128_pd(a, 1))
+
+/* reverse/flip all floats */
+# define VREV_S(a) _mm256_reverse(a)
+
+/* reverse/flip complex floats */
+# define VREV_C(a) _mm256_insertf128_pd_1(_mm256_castpd128_pd256(_mm256_extractf128_pd(a, 1)), _mm256_castpd256_pd128(a))
+
+# define VALIGNED(ptr) ((((uintptr_t)(ptr)) & 0x1F) == 0)
+
+#endif
+#endif
