10 #ifndef EIGEN_COMPLEX_AVX_H 11 #define EIGEN_COMPLEX_AVX_H 20 EIGEN_STRONG_INLINE Packet4cf() {}
21 EIGEN_STRONG_INLINE
explicit Packet4cf(
const __m256& a) : v(a) {}
25 template<>
struct packet_traits<
std::complex<float> > : default_packet_traits
27 typedef Packet4cf type;
28 typedef Packet2cf half;
48 template<>
struct unpacket_traits<Packet4cf> {
typedef std::complex<float> type;
enum {size=4, alignment=
Aligned32};
typedef Packet2cf half; };
50 template<> EIGEN_STRONG_INLINE Packet4cf padd<Packet4cf>(
const Packet4cf& a,
const Packet4cf& b) {
return Packet4cf(_mm256_add_ps(a.v,b.v)); }
51 template<> EIGEN_STRONG_INLINE Packet4cf psub<Packet4cf>(
const Packet4cf& a,
const Packet4cf& b) {
return Packet4cf(_mm256_sub_ps(a.v,b.v)); }
52 template<> EIGEN_STRONG_INLINE Packet4cf pnegate(
const Packet4cf& a)
54 return Packet4cf(pnegate(a.v));
56 template<> EIGEN_STRONG_INLINE Packet4cf pconj(
const Packet4cf& a)
58 const __m256 mask = _mm256_castsi256_ps(_mm256_setr_epi32(0x00000000,0x80000000,0x00000000,0x80000000,0x00000000,0x80000000,0x00000000,0x80000000));
59 return Packet4cf(_mm256_xor_ps(a.v,mask));
62 template<> EIGEN_STRONG_INLINE Packet4cf pmul<Packet4cf>(
const Packet4cf& a,
const Packet4cf& b)
64 __m256 tmp1 = _mm256_mul_ps(_mm256_moveldup_ps(a.v), b.v);
65 __m256 tmp2 = _mm256_mul_ps(_mm256_movehdup_ps(a.v), _mm256_permute_ps(b.v, _MM_SHUFFLE(2,3,0,1)));
66 __m256 result = _mm256_addsub_ps(tmp1, tmp2);
67 return Packet4cf(result);
70 template<> EIGEN_STRONG_INLINE Packet4cf pand <Packet4cf>(
const Packet4cf& a,
const Packet4cf& b) {
return Packet4cf(_mm256_and_ps(a.v,b.v)); }
71 template<> EIGEN_STRONG_INLINE Packet4cf por <Packet4cf>(
const Packet4cf& a,
const Packet4cf& b) {
return Packet4cf(_mm256_or_ps(a.v,b.v)); }
72 template<> EIGEN_STRONG_INLINE Packet4cf pxor <Packet4cf>(
const Packet4cf& a,
const Packet4cf& b) {
return Packet4cf(_mm256_xor_ps(a.v,b.v)); }
73 template<> EIGEN_STRONG_INLINE Packet4cf pandnot<Packet4cf>(
const Packet4cf& a,
const Packet4cf& b) {
return Packet4cf(_mm256_andnot_ps(a.v,b.v)); }
75 template<> EIGEN_STRONG_INLINE Packet4cf pload <Packet4cf>(
const std::complex<float>* from) { EIGEN_DEBUG_ALIGNED_LOAD
return Packet4cf(pload<Packet8f>(&numext::real_ref(*from))); }
76 template<> EIGEN_STRONG_INLINE Packet4cf ploadu<Packet4cf>(
const std::complex<float>* from) { EIGEN_DEBUG_UNALIGNED_LOAD
return Packet4cf(ploadu<Packet8f>(&numext::real_ref(*from))); }
79 template<> EIGEN_STRONG_INLINE Packet4cf pset1<Packet4cf>(
const std::complex<float>& from)
81 return Packet4cf(_mm256_castpd_ps(_mm256_broadcast_sd((
const double*)(
const void*)&from)));
84 template<> EIGEN_STRONG_INLINE Packet4cf ploaddup<Packet4cf>(
const std::complex<float>* from)
87 Packet2cf a = ploaddup<Packet2cf>(from);
88 Packet2cf b = ploaddup<Packet2cf>(from+1);
89 return Packet4cf(_mm256_insertf128_ps(_mm256_castps128_ps256(a.v), b.v, 1));
92 template<> EIGEN_STRONG_INLINE
void pstore <std::complex<float> >(std::complex<float>* to,
const Packet4cf& from) { EIGEN_DEBUG_ALIGNED_STORE pstore(&numext::real_ref(*to), from.v); }
93 template<> EIGEN_STRONG_INLINE
void pstoreu<std::complex<float> >(std::complex<float>* to,
const Packet4cf& from) { EIGEN_DEBUG_UNALIGNED_STORE pstoreu(&numext::real_ref(*to), from.v); }
95 template<> EIGEN_DEVICE_FUNC
inline Packet4cf pgather<std::complex<float>, Packet4cf>(
const std::complex<float>* from, Index stride)
97 return Packet4cf(_mm256_set_ps(std::imag(from[3*stride]), std::real(from[3*stride]),
98 std::imag(from[2*stride]), std::real(from[2*stride]),
99 std::imag(from[1*stride]), std::real(from[1*stride]),
100 std::imag(from[0*stride]), std::real(from[0*stride])));
103 template<> EIGEN_DEVICE_FUNC
inline void pscatter<std::complex<float>, Packet4cf>(std::complex<float>* to,
const Packet4cf& from, Index stride)
105 __m128 low = _mm256_extractf128_ps(from.v, 0);
106 to[stride*0] = std::complex<float>(_mm_cvtss_f32(_mm_shuffle_ps(low, low, 0)),
107 _mm_cvtss_f32(_mm_shuffle_ps(low, low, 1)));
108 to[stride*1] = std::complex<float>(_mm_cvtss_f32(_mm_shuffle_ps(low, low, 2)),
109 _mm_cvtss_f32(_mm_shuffle_ps(low, low, 3)));
111 __m128 high = _mm256_extractf128_ps(from.v, 1);
112 to[stride*2] = std::complex<float>(_mm_cvtss_f32(_mm_shuffle_ps(high, high, 0)),
113 _mm_cvtss_f32(_mm_shuffle_ps(high, high, 1)));
114 to[stride*3] = std::complex<float>(_mm_cvtss_f32(_mm_shuffle_ps(high, high, 2)),
115 _mm_cvtss_f32(_mm_shuffle_ps(high, high, 3)));
119 template<> EIGEN_STRONG_INLINE std::complex<float> pfirst<Packet4cf>(
const Packet4cf& a)
121 return pfirst(Packet2cf(_mm256_castps256_ps128(a.v)));
124 template<> EIGEN_STRONG_INLINE Packet4cf preverse(
const Packet4cf& a) {
125 __m128 low = _mm256_extractf128_ps(a.v, 0);
126 __m128 high = _mm256_extractf128_ps(a.v, 1);
127 __m128d lowd = _mm_castps_pd(low);
128 __m128d highd = _mm_castps_pd(high);
129 low = _mm_castpd_ps(_mm_shuffle_pd(lowd,lowd,0x1));
130 high = _mm_castpd_ps(_mm_shuffle_pd(highd,highd,0x1));
131 __m256 result = _mm256_setzero_ps();
132 result = _mm256_insertf128_ps(result, low, 1);
133 result = _mm256_insertf128_ps(result, high, 0);
134 return Packet4cf(result);
137 template<> EIGEN_STRONG_INLINE std::complex<float> predux<Packet4cf>(
const Packet4cf& a)
139 return predux(padd(Packet2cf(_mm256_extractf128_ps(a.v,0)),
140 Packet2cf(_mm256_extractf128_ps(a.v,1))));
143 template<> EIGEN_STRONG_INLINE Packet4cf preduxp<Packet4cf>(
const Packet4cf* vecs)
145 Packet8f t0 = _mm256_shuffle_ps(vecs[0].v, vecs[0].v, _MM_SHUFFLE(3, 1, 2 ,0));
146 Packet8f t1 = _mm256_shuffle_ps(vecs[1].v, vecs[1].v, _MM_SHUFFLE(3, 1, 2 ,0));
147 t0 = _mm256_hadd_ps(t0,t1);
148 Packet8f t2 = _mm256_shuffle_ps(vecs[2].v, vecs[2].v, _MM_SHUFFLE(3, 1, 2 ,0));
149 Packet8f t3 = _mm256_shuffle_ps(vecs[3].v, vecs[3].v, _MM_SHUFFLE(3, 1, 2 ,0));
150 t2 = _mm256_hadd_ps(t2,t3);
152 t1 = _mm256_permute2f128_ps(t0,t2, 0 + (2<<4));
153 t3 = _mm256_permute2f128_ps(t0,t2, 1 + (3<<4));
155 return Packet4cf(_mm256_add_ps(t1,t3));
158 template<> EIGEN_STRONG_INLINE std::complex<float> predux_mul<Packet4cf>(
const Packet4cf& a)
160 return predux_mul(pmul(Packet2cf(_mm256_extractf128_ps(a.v, 0)),
161 Packet2cf(_mm256_extractf128_ps(a.v, 1))));
165 struct palign_impl<Offset,Packet4cf>
167 static EIGEN_STRONG_INLINE
void run(Packet4cf& first,
const Packet4cf& second)
169 if (Offset==0)
return;
170 palign_impl<Offset*2,Packet8f>::run(first.v, second.v);
174 template<>
struct conj_helper<Packet4cf, Packet4cf, false,true>
176 EIGEN_STRONG_INLINE Packet4cf pmadd(
const Packet4cf& x,
const Packet4cf& y,
const Packet4cf& c)
const 177 {
return padd(pmul(x,y),c); }
179 EIGEN_STRONG_INLINE Packet4cf pmul(
const Packet4cf& a,
const Packet4cf& b)
const 181 return internal::pmul(a, pconj(b));
185 template<>
struct conj_helper<Packet4cf, Packet4cf, true,false>
187 EIGEN_STRONG_INLINE Packet4cf pmadd(
const Packet4cf& x,
const Packet4cf& y,
const Packet4cf& c)
const 188 {
return padd(pmul(x,y),c); }
190 EIGEN_STRONG_INLINE Packet4cf pmul(
const Packet4cf& a,
const Packet4cf& b)
const 192 return internal::pmul(pconj(a), b);
196 template<>
struct conj_helper<Packet4cf, Packet4cf, true,true>
198 EIGEN_STRONG_INLINE Packet4cf pmadd(
const Packet4cf& x,
const Packet4cf& y,
const Packet4cf& c)
const 199 {
return padd(pmul(x,y),c); }
201 EIGEN_STRONG_INLINE Packet4cf pmul(
const Packet4cf& a,
const Packet4cf& b)
const 203 return pconj(internal::pmul(a, b));
207 template<>
struct conj_helper<Packet8f, Packet4cf, false,false>
209 EIGEN_STRONG_INLINE Packet4cf pmadd(
const Packet8f& x,
const Packet4cf& y,
const Packet4cf& c)
const 210 {
return padd(c, pmul(x,y)); }
212 EIGEN_STRONG_INLINE Packet4cf pmul(
const Packet8f& x,
const Packet4cf& y)
const 213 {
return Packet4cf(Eigen::internal::pmul(x, y.v)); }
216 template<>
struct conj_helper<Packet4cf, Packet8f, false,false>
218 EIGEN_STRONG_INLINE Packet4cf pmadd(
const Packet4cf& x,
const Packet8f& y,
const Packet4cf& c)
const 219 {
return padd(c, pmul(x,y)); }
221 EIGEN_STRONG_INLINE Packet4cf pmul(
const Packet4cf& x,
const Packet8f& y)
const 222 {
return Packet4cf(Eigen::internal::pmul(x.v, y)); }
225 template<> EIGEN_STRONG_INLINE Packet4cf pdiv<Packet4cf>(
const Packet4cf& a,
const Packet4cf& b)
227 Packet4cf num = pmul(a, pconj(b));
228 __m256 tmp = _mm256_mul_ps(b.v, b.v);
229 __m256 tmp2 = _mm256_shuffle_ps(tmp,tmp,0xB1);
230 __m256 denom = _mm256_add_ps(tmp, tmp2);
231 return Packet4cf(_mm256_div_ps(num.v, denom));
234 template<> EIGEN_STRONG_INLINE Packet4cf pcplxflip<Packet4cf>(
const Packet4cf& x)
236 return Packet4cf(_mm256_shuffle_ps(x.v, x.v, _MM_SHUFFLE(2, 3, 0 ,1)));
242 EIGEN_STRONG_INLINE Packet2cd() {}
243 EIGEN_STRONG_INLINE
explicit Packet2cd(
const __m256d& a) : v(a) {}
247 template<>
struct packet_traits<
std::complex<double> > : default_packet_traits
249 typedef Packet2cd type;
250 typedef Packet1cd half;
270 template<>
struct unpacket_traits<Packet2cd> {
typedef std::complex<double> type;
enum {size=2, alignment=
Aligned32};
typedef Packet1cd half; };
272 template<> EIGEN_STRONG_INLINE Packet2cd padd<Packet2cd>(
const Packet2cd& a,
const Packet2cd& b) {
return Packet2cd(_mm256_add_pd(a.v,b.v)); }
273 template<> EIGEN_STRONG_INLINE Packet2cd psub<Packet2cd>(
const Packet2cd& a,
const Packet2cd& b) {
return Packet2cd(_mm256_sub_pd(a.v,b.v)); }
274 template<> EIGEN_STRONG_INLINE Packet2cd pnegate(
const Packet2cd& a) {
return Packet2cd(pnegate(a.v)); }
275 template<> EIGEN_STRONG_INLINE Packet2cd pconj(
const Packet2cd& a)
277 const __m256d mask = _mm256_castsi256_pd(_mm256_set_epi32(0x80000000,0x0,0x0,0x0,0x80000000,0x0,0x0,0x0));
278 return Packet2cd(_mm256_xor_pd(a.v,mask));
281 template<> EIGEN_STRONG_INLINE Packet2cd pmul<Packet2cd>(
const Packet2cd& a,
const Packet2cd& b)
283 __m256d tmp1 = _mm256_shuffle_pd(a.v,a.v,0x0);
284 __m256d even = _mm256_mul_pd(tmp1, b.v);
285 __m256d tmp2 = _mm256_shuffle_pd(a.v,a.v,0xF);
286 __m256d tmp3 = _mm256_shuffle_pd(b.v,b.v,0x5);
287 __m256d odd = _mm256_mul_pd(tmp2, tmp3);
288 return Packet2cd(_mm256_addsub_pd(even, odd));
291 template<> EIGEN_STRONG_INLINE Packet2cd pand <Packet2cd>(
const Packet2cd& a,
const Packet2cd& b) {
return Packet2cd(_mm256_and_pd(a.v,b.v)); }
292 template<> EIGEN_STRONG_INLINE Packet2cd por <Packet2cd>(
const Packet2cd& a,
const Packet2cd& b) {
return Packet2cd(_mm256_or_pd(a.v,b.v)); }
293 template<> EIGEN_STRONG_INLINE Packet2cd pxor <Packet2cd>(
const Packet2cd& a,
const Packet2cd& b) {
return Packet2cd(_mm256_xor_pd(a.v,b.v)); }
294 template<> EIGEN_STRONG_INLINE Packet2cd pandnot<Packet2cd>(
const Packet2cd& a,
const Packet2cd& b) {
return Packet2cd(_mm256_andnot_pd(a.v,b.v)); }
296 template<> EIGEN_STRONG_INLINE Packet2cd pload <Packet2cd>(
const std::complex<double>* from)
297 { EIGEN_DEBUG_ALIGNED_LOAD
return Packet2cd(pload<Packet4d>((
const double*)from)); }
298 template<> EIGEN_STRONG_INLINE Packet2cd ploadu<Packet2cd>(
const std::complex<double>* from)
299 { EIGEN_DEBUG_UNALIGNED_LOAD
return Packet2cd(ploadu<Packet4d>((
const double*)from)); }
301 template<> EIGEN_STRONG_INLINE Packet2cd pset1<Packet2cd>(
const std::complex<double>& from)
305 return Packet2cd(_mm256_broadcast_pd((
const __m128d*)(
const void*)&from));
308 template<> EIGEN_STRONG_INLINE Packet2cd ploaddup<Packet2cd>(
const std::complex<double>* from) {
return pset1<Packet2cd>(*from); }
310 template<> EIGEN_STRONG_INLINE
void pstore <std::complex<double> >(std::complex<double> * to,
const Packet2cd& from) { EIGEN_DEBUG_ALIGNED_STORE pstore((
double*)to, from.v); }
311 template<> EIGEN_STRONG_INLINE
void pstoreu<std::complex<double> >(std::complex<double> * to,
const Packet2cd& from) { EIGEN_DEBUG_UNALIGNED_STORE pstoreu((
double*)to, from.v); }
313 template<> EIGEN_DEVICE_FUNC
inline Packet2cd pgather<std::complex<double>, Packet2cd>(
const std::complex<double>* from, Index stride)
315 return Packet2cd(_mm256_set_pd(std::imag(from[1*stride]), std::real(from[1*stride]),
316 std::imag(from[0*stride]), std::real(from[0*stride])));
319 template<> EIGEN_DEVICE_FUNC
inline void pscatter<std::complex<double>, Packet2cd>(std::complex<double>* to,
const Packet2cd& from, Index stride)
321 __m128d low = _mm256_extractf128_pd(from.v, 0);
322 to[stride*0] = std::complex<double>(_mm_cvtsd_f64(low), _mm_cvtsd_f64(_mm_shuffle_pd(low, low, 1)));
323 __m128d high = _mm256_extractf128_pd(from.v, 1);
324 to[stride*1] = std::complex<double>(_mm_cvtsd_f64(high), _mm_cvtsd_f64(_mm_shuffle_pd(high, high, 1)));
327 template<> EIGEN_STRONG_INLINE std::complex<double> pfirst<Packet2cd>(
const Packet2cd& a)
329 __m128d low = _mm256_extractf128_pd(a.v, 0);
330 EIGEN_ALIGN16
double res[2];
331 _mm_store_pd(res, low);
332 return std::complex<double>(res[0],res[1]);
335 template<> EIGEN_STRONG_INLINE Packet2cd preverse(
const Packet2cd& a) {
336 __m256d result = _mm256_permute2f128_pd(a.v, a.v, 1);
337 return Packet2cd(result);
340 template<> EIGEN_STRONG_INLINE std::complex<double> predux<Packet2cd>(
const Packet2cd& a)
342 return predux(padd(Packet1cd(_mm256_extractf128_pd(a.v,0)),
343 Packet1cd(_mm256_extractf128_pd(a.v,1))));
346 template<> EIGEN_STRONG_INLINE Packet2cd preduxp<Packet2cd>(
const Packet2cd* vecs)
348 Packet4d t0 = _mm256_permute2f128_pd(vecs[0].v,vecs[1].v, 0 + (2<<4));
349 Packet4d t1 = _mm256_permute2f128_pd(vecs[0].v,vecs[1].v, 1 + (3<<4));
351 return Packet2cd(_mm256_add_pd(t0,t1));
354 template<> EIGEN_STRONG_INLINE std::complex<double> predux_mul<Packet2cd>(
const Packet2cd& a)
356 return predux(pmul(Packet1cd(_mm256_extractf128_pd(a.v,0)),
357 Packet1cd(_mm256_extractf128_pd(a.v,1))));
361 struct palign_impl<Offset,Packet2cd>
363 static EIGEN_STRONG_INLINE
void run(Packet2cd& first,
const Packet2cd& second)
365 if (Offset==0)
return;
366 palign_impl<Offset*2,Packet4d>::run(first.v, second.v);
370 template<>
struct conj_helper<Packet2cd, Packet2cd, false,true>
372 EIGEN_STRONG_INLINE Packet2cd pmadd(
const Packet2cd& x,
const Packet2cd& y,
const Packet2cd& c)
const 373 {
return padd(pmul(x,y),c); }
375 EIGEN_STRONG_INLINE Packet2cd pmul(
const Packet2cd& a,
const Packet2cd& b)
const 377 return internal::pmul(a, pconj(b));
381 template<>
struct conj_helper<Packet2cd, Packet2cd, true,false>
383 EIGEN_STRONG_INLINE Packet2cd pmadd(
const Packet2cd& x,
const Packet2cd& y,
const Packet2cd& c)
const 384 {
return padd(pmul(x,y),c); }
386 EIGEN_STRONG_INLINE Packet2cd pmul(
const Packet2cd& a,
const Packet2cd& b)
const 388 return internal::pmul(pconj(a), b);
392 template<>
struct conj_helper<Packet2cd, Packet2cd, true,true>
394 EIGEN_STRONG_INLINE Packet2cd pmadd(
const Packet2cd& x,
const Packet2cd& y,
const Packet2cd& c)
const 395 {
return padd(pmul(x,y),c); }
397 EIGEN_STRONG_INLINE Packet2cd pmul(
const Packet2cd& a,
const Packet2cd& b)
const 399 return pconj(internal::pmul(a, b));
403 template<>
struct conj_helper<Packet4d, Packet2cd, false,false>
405 EIGEN_STRONG_INLINE Packet2cd pmadd(
const Packet4d& x,
const Packet2cd& y,
const Packet2cd& c)
const 406 {
return padd(c, pmul(x,y)); }
408 EIGEN_STRONG_INLINE Packet2cd pmul(
const Packet4d& x,
const Packet2cd& y)
const 409 {
return Packet2cd(Eigen::internal::pmul(x, y.v)); }
412 template<>
struct conj_helper<Packet2cd, Packet4d, false,false>
414 EIGEN_STRONG_INLINE Packet2cd pmadd(
const Packet2cd& x,
const Packet4d& y,
const Packet2cd& c)
const 415 {
return padd(c, pmul(x,y)); }
417 EIGEN_STRONG_INLINE Packet2cd pmul(
const Packet2cd& x,
const Packet4d& y)
const 418 {
return Packet2cd(Eigen::internal::pmul(x.v, y)); }
421 template<> EIGEN_STRONG_INLINE Packet2cd pdiv<Packet2cd>(
const Packet2cd& a,
const Packet2cd& b)
423 Packet2cd num = pmul(a, pconj(b));
424 __m256d tmp = _mm256_mul_pd(b.v, b.v);
425 __m256d denom = _mm256_hadd_pd(tmp, tmp);
426 return Packet2cd(_mm256_div_pd(num.v, denom));
429 template<> EIGEN_STRONG_INLINE Packet2cd pcplxflip<Packet2cd>(
const Packet2cd& x)
431 return Packet2cd(_mm256_shuffle_pd(x.v, x.v, 0x5));
434 EIGEN_DEVICE_FUNC
inline void 435 ptranspose(PacketBlock<Packet4cf,4>& kernel) {
436 __m256d P0 = _mm256_castps_pd(kernel.packet[0].v);
437 __m256d P1 = _mm256_castps_pd(kernel.packet[1].v);
438 __m256d P2 = _mm256_castps_pd(kernel.packet[2].v);
439 __m256d P3 = _mm256_castps_pd(kernel.packet[3].v);
441 __m256d T0 = _mm256_shuffle_pd(P0, P1, 15);
442 __m256d T1 = _mm256_shuffle_pd(P0, P1, 0);
443 __m256d T2 = _mm256_shuffle_pd(P2, P3, 15);
444 __m256d T3 = _mm256_shuffle_pd(P2, P3, 0);
446 kernel.packet[1].v = _mm256_castpd_ps(_mm256_permute2f128_pd(T0, T2, 32));
447 kernel.packet[3].v = _mm256_castpd_ps(_mm256_permute2f128_pd(T0, T2, 49));
448 kernel.packet[0].v = _mm256_castpd_ps(_mm256_permute2f128_pd(T1, T3, 32));
449 kernel.packet[2].v = _mm256_castpd_ps(_mm256_permute2f128_pd(T1, T3, 49));
452 EIGEN_DEVICE_FUNC
inline void 453 ptranspose(PacketBlock<Packet2cd,2>& kernel) {
454 __m256d tmp = _mm256_permute2f128_pd(kernel.packet[0].v, kernel.packet[1].v, 0+(2<<4));
455 kernel.packet[1].v = _mm256_permute2f128_pd(kernel.packet[0].v, kernel.packet[1].v, 1+(3<<4));
456 kernel.packet[0].v = tmp;
463 #endif // EIGEN_COMPLEX_AVX_H
Definition: StdDeque.h:58
Definition: Eigen_Colamd.h:54
Definition: Constants.h:231