libstdc++
simd.h
1 // Definition of the public simd interfaces -*- C++ -*-
2 
3 // Copyright (C) 2020-2021 Free Software Foundation, Inc.
4 //
5 // This file is part of the GNU ISO C++ Library. This library is free
6 // software; you can redistribute it and/or modify it under the
7 // terms of the GNU General Public License as published by the
8 // Free Software Foundation; either version 3, or (at your option)
9 // any later version.
10 
11 // This library is distributed in the hope that it will be useful,
12 // but WITHOUT ANY WARRANTY; without even the implied warranty of
13 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 // GNU General Public License for more details.
15 
16 // Under Section 7 of GPL version 3, you are granted additional
17 // permissions described in the GCC Runtime Library Exception, version
18 // 3.1, as published by the Free Software Foundation.
19 
20 // You should have received a copy of the GNU General Public License and
21 // a copy of the GCC Runtime Library Exception along with this program;
22 // see the files COPYING3 and COPYING.RUNTIME respectively. If not, see
23 // <http://www.gnu.org/licenses/>.
24 
25 #ifndef _GLIBCXX_EXPERIMENTAL_SIMD_H
26 #define _GLIBCXX_EXPERIMENTAL_SIMD_H
27 
28 #if __cplusplus >= 201703L
29 
30 #include "simd_detail.h"
31 #include "numeric_traits.h"
32 #include <bit>
33 #include <bitset>
34 #ifdef _GLIBCXX_DEBUG_UB
35 #include <cstdio> // for stderr
36 #endif
37 #include <cstring>
38 #include <functional>
39 #include <iosfwd>
40 #include <utility>
41 
42 #if _GLIBCXX_SIMD_X86INTRIN
43 #include <x86intrin.h>
44 #elif _GLIBCXX_SIMD_HAVE_NEON
45 #include <arm_neon.h>
46 #endif
47 
48 /** @ingroup ts_simd
49  * @{
50  */
51 /* There are several closely related types, with the following naming
52  * convention:
53  * _Tp: vectorizable (arithmetic) type (or any type)
54  * _TV: __vector_type_t<_Tp, _Np>
55  * _TW: _SimdWrapper<_Tp, _Np>
56  * _TI: __intrinsic_type_t<_Tp, _Np>
57  * _TVT: _VectorTraits<_TV> or _VectorTraits<_TW>
58  * If one additional type is needed use _U instead of _T.
59  * Otherwise use _T\d, _TV\d, _TW\d, TI\d, _TVT\d.
60  *
61  * More naming conventions:
62  * _Ap or _Abi: An ABI tag from the simd_abi namespace
63  * _Ip: often used for integer types with sizeof(_Ip) == sizeof(_Tp),
64  * _IV, _IW as for _TV, _TW
65  * _Np: number of elements (not bytes)
66  * _Bytes: number of bytes
67  *
68  * Variable names:
69  * __k: mask object (vector- or bitmask)
70  */
71 _GLIBCXX_SIMD_BEGIN_NAMESPACE
72 
73 #if !_GLIBCXX_SIMD_X86INTRIN
74 using __m128 [[__gnu__::__vector_size__(16)]] = float;
75 using __m128d [[__gnu__::__vector_size__(16)]] = double;
76 using __m128i [[__gnu__::__vector_size__(16)]] = long long;
77 using __m256 [[__gnu__::__vector_size__(32)]] = float;
78 using __m256d [[__gnu__::__vector_size__(32)]] = double;
79 using __m256i [[__gnu__::__vector_size__(32)]] = long long;
80 using __m512 [[__gnu__::__vector_size__(64)]] = float;
81 using __m512d [[__gnu__::__vector_size__(64)]] = double;
82 using __m512i [[__gnu__::__vector_size__(64)]] = long long;
83 #endif
84 
85 namespace simd_abi {
86 // simd_abi forward declarations {{{
87 // implementation details:
88 struct _Scalar;
89 
90 template <int _Np>
91  struct _Fixed;
92 
93 // There are two major ABIs that appear on different architectures.
94 // Both have non-boolean values packed into an N Byte register
95 // -> #elements = N / sizeof(T)
96 // Masks differ:
97 // 1. Use value vector registers for masks (all 0 or all 1)
98 // 2. Use bitmasks (mask registers) with one bit per value in the corresponding
99 // value vector
100 //
101 // Both can be partially used, masking off the rest when doing horizontal
102 // operations or operations that can trap (e.g. FP_INVALID or integer division
103 // by 0). This is encoded as the number of used bytes.
104 template <int _UsedBytes>
105  struct _VecBuiltin;
106 
107 template <int _UsedBytes>
108  struct _VecBltnBtmsk;
109 
110 template <typename _Tp, int _Np>
111  using _VecN = _VecBuiltin<sizeof(_Tp) * _Np>;
112 
113 template <int _UsedBytes = 16>
114  using _Sse = _VecBuiltin<_UsedBytes>;
115 
116 template <int _UsedBytes = 32>
117  using _Avx = _VecBuiltin<_UsedBytes>;
118 
119 template <int _UsedBytes = 64>
120  using _Avx512 = _VecBltnBtmsk<_UsedBytes>;
121 
122 template <int _UsedBytes = 16>
123  using _Neon = _VecBuiltin<_UsedBytes>;
124 
125 // implementation-defined:
126 using __sse = _Sse<>;
127 using __avx = _Avx<>;
128 using __avx512 = _Avx512<>;
129 using __neon = _Neon<>;
130 using __neon128 = _Neon<16>;
131 using __neon64 = _Neon<8>;
132 
133 // standard:
134 template <typename _Tp, size_t _Np, typename...>
135  struct deduce;
136 
137 template <int _Np>
138  using fixed_size = _Fixed<_Np>;
139 
140 using scalar = _Scalar;
141 
142 // }}}
143 } // namespace simd_abi
144 // forward declarations is_simd(_mask), simd(_mask), simd_size {{{
145 template <typename _Tp>
146  struct is_simd;
147 
148 template <typename _Tp>
149  struct is_simd_mask;
150 
151 template <typename _Tp, typename _Abi>
152  class simd;
153 
154 template <typename _Tp, typename _Abi>
155  class simd_mask;
156 
157 template <typename _Tp, typename _Abi>
158  struct simd_size;
159 
160 // }}}
161 // load/store flags {{{
162 struct element_aligned_tag
163 {
164  template <typename _Tp, typename _Up = typename _Tp::value_type>
165  static constexpr size_t _S_alignment = alignof(_Up);
166 
167  template <typename _Tp, typename _Up>
168  _GLIBCXX_SIMD_INTRINSIC static constexpr _Up*
169  _S_apply(_Up* __ptr)
170  { return __ptr; }
171 };
172 
173 struct vector_aligned_tag
174 {
175  template <typename _Tp, typename _Up = typename _Tp::value_type>
176  static constexpr size_t _S_alignment
177  = std::__bit_ceil(sizeof(_Up) * _Tp::size());
178 
179  template <typename _Tp, typename _Up>
180  _GLIBCXX_SIMD_INTRINSIC static constexpr _Up*
181  _S_apply(_Up* __ptr)
182  {
183  return static_cast<_Up*>(
184  __builtin_assume_aligned(__ptr, _S_alignment<_Tp, _Up>));
185  }
186 };
187 
188 template <size_t _Np> struct overaligned_tag
189 {
190  template <typename _Tp, typename _Up = typename _Tp::value_type>
191  static constexpr size_t _S_alignment = _Np;
192 
193  template <typename _Tp, typename _Up>
194  _GLIBCXX_SIMD_INTRINSIC static constexpr _Up*
195  _S_apply(_Up* __ptr)
196  { return static_cast<_Up*>(__builtin_assume_aligned(__ptr, _Np)); }
197 };
198 
199 inline constexpr element_aligned_tag element_aligned = {};
200 
201 inline constexpr vector_aligned_tag vector_aligned = {};
202 
203 template <size_t _Np>
204  inline constexpr overaligned_tag<_Np> overaligned = {};
205 
206 // }}}
207 template <size_t _Xp>
208  using _SizeConstant = integral_constant<size_t, _Xp>;
209 
210 namespace __detail
211 {
212  struct _Minimum
213  {
214  template <typename _Tp>
215  _GLIBCXX_SIMD_INTRINSIC constexpr
216  _Tp
217  operator()(_Tp __a, _Tp __b) const
218  {
219  using std::min;
220  return min(__a, __b);
221  }
222  };
223 
224  struct _Maximum
225  {
226  template <typename _Tp>
227  _GLIBCXX_SIMD_INTRINSIC constexpr
228  _Tp
229  operator()(_Tp __a, _Tp __b) const
230  {
231  using std::max;
232  return max(__a, __b);
233  }
234  };
235 } // namespace __detail
236 
237 // unrolled/pack execution helpers
238 // __execute_n_times{{{
239 template <typename _Fp, size_t... _I>
240  _GLIBCXX_SIMD_INTRINSIC constexpr void
241  __execute_on_index_sequence(_Fp&& __f, index_sequence<_I...>)
242  { ((void)__f(_SizeConstant<_I>()), ...); }
243 
244 template <typename _Fp>
245  _GLIBCXX_SIMD_INTRINSIC constexpr void
246  __execute_on_index_sequence(_Fp&&, index_sequence<>)
247  { }
248 
249 template <size_t _Np, typename _Fp>
250  _GLIBCXX_SIMD_INTRINSIC constexpr void
251  __execute_n_times(_Fp&& __f)
252  {
253  __execute_on_index_sequence(static_cast<_Fp&&>(__f),
254  make_index_sequence<_Np>{});
255  }
256 
257 // }}}
258 // __generate_from_n_evaluations{{{
259 template <typename _R, typename _Fp, size_t... _I>
260  _GLIBCXX_SIMD_INTRINSIC constexpr _R
261  __execute_on_index_sequence_with_return(_Fp&& __f, index_sequence<_I...>)
262  { return _R{__f(_SizeConstant<_I>())...}; }
263 
264 template <size_t _Np, typename _R, typename _Fp>
265  _GLIBCXX_SIMD_INTRINSIC constexpr _R
266  __generate_from_n_evaluations(_Fp&& __f)
267  {
268  return __execute_on_index_sequence_with_return<_R>(
269  static_cast<_Fp&&>(__f), make_index_sequence<_Np>{});
270  }
271 
272 // }}}
273 // __call_with_n_evaluations{{{
274 template <size_t... _I, typename _F0, typename _FArgs>
275  _GLIBCXX_SIMD_INTRINSIC constexpr auto
276  __call_with_n_evaluations(index_sequence<_I...>, _F0&& __f0, _FArgs&& __fargs)
277  { return __f0(__fargs(_SizeConstant<_I>())...); }
278 
279 template <size_t _Np, typename _F0, typename _FArgs>
280  _GLIBCXX_SIMD_INTRINSIC constexpr auto
281  __call_with_n_evaluations(_F0&& __f0, _FArgs&& __fargs)
282  {
283  return __call_with_n_evaluations(make_index_sequence<_Np>{},
284  static_cast<_F0&&>(__f0),
285  static_cast<_FArgs&&>(__fargs));
286  }
287 
288 // }}}
289 // __call_with_subscripts{{{
290 template <size_t _First = 0, size_t... _It, typename _Tp, typename _Fp>
291  _GLIBCXX_SIMD_INTRINSIC constexpr auto
292  __call_with_subscripts(_Tp&& __x, index_sequence<_It...>, _Fp&& __fun)
293  { return __fun(__x[_First + _It]...); }
294 
295 template <size_t _Np, size_t _First = 0, typename _Tp, typename _Fp>
296  _GLIBCXX_SIMD_INTRINSIC constexpr auto
297  __call_with_subscripts(_Tp&& __x, _Fp&& __fun)
298  {
299  return __call_with_subscripts<_First>(static_cast<_Tp&&>(__x),
300  make_index_sequence<_Np>(),
301  static_cast<_Fp&&>(__fun));
302  }
303 
304 // }}}
305 
306 // vvv ---- type traits ---- vvv
307 // integer type aliases{{{
308 using _UChar = unsigned char;
309 using _SChar = signed char;
310 using _UShort = unsigned short;
311 using _UInt = unsigned int;
312 using _ULong = unsigned long;
313 using _ULLong = unsigned long long;
314 using _LLong = long long;
315 
316 //}}}
317 // __first_of_pack{{{
318 template <typename _T0, typename...>
319  struct __first_of_pack
320  { using type = _T0; };
321 
322 template <typename... _Ts>
323  using __first_of_pack_t = typename __first_of_pack<_Ts...>::type;
324 
325 //}}}
326 // __value_type_or_identity_t {{{
327 template <typename _Tp>
328  typename _Tp::value_type
329  __value_type_or_identity_impl(int);
330 
331 template <typename _Tp>
332  _Tp
333  __value_type_or_identity_impl(float);
334 
335 template <typename _Tp>
336  using __value_type_or_identity_t
337  = decltype(__value_type_or_identity_impl<_Tp>(int()));
338 
339 // }}}
340 // __is_vectorizable {{{
341 template <typename _Tp>
342  struct __is_vectorizable : public is_arithmetic<_Tp> {};
343 
344 template <>
345  struct __is_vectorizable<bool> : public false_type {};
346 
347 template <typename _Tp>
348  inline constexpr bool __is_vectorizable_v = __is_vectorizable<_Tp>::value;
349 
350 // Deduces to a vectorizable type
351 template <typename _Tp, typename = enable_if_t<__is_vectorizable_v<_Tp>>>
352  using _Vectorizable = _Tp;
353 
354 // }}}
355 // _LoadStorePtr / __is_possible_loadstore_conversion {{{
356 template <typename _Ptr, typename _ValueType>
357  struct __is_possible_loadstore_conversion
358  : conjunction<__is_vectorizable<_Ptr>, __is_vectorizable<_ValueType>> {};
359 
360 template <>
361  struct __is_possible_loadstore_conversion<bool, bool> : true_type {};
362 
363 // Deduces to a type allowed for load/store with the given value type.
364 template <typename _Ptr, typename _ValueType,
365  typename = enable_if_t<
366  __is_possible_loadstore_conversion<_Ptr, _ValueType>::value>>
367  using _LoadStorePtr = _Ptr;
368 
369 // }}}
370 // __is_bitmask{{{
371 template <typename _Tp, typename = void_t<>>
372  struct __is_bitmask : false_type {};
373 
374 template <typename _Tp>
375  inline constexpr bool __is_bitmask_v = __is_bitmask<_Tp>::value;
376 
377 // the __mmaskXX case:
378 template <typename _Tp>
379  struct __is_bitmask<_Tp,
380  void_t<decltype(declval<unsigned&>() = declval<_Tp>() & 1u)>>
381  : true_type {};
382 
383 // }}}
384 // __int_for_sizeof{{{
385 #pragma GCC diagnostic push
386 #pragma GCC diagnostic ignored "-Wpedantic"
387 template <size_t _Bytes>
388  constexpr auto
389  __int_for_sizeof()
390  {
391  if constexpr (_Bytes == sizeof(int))
392  return int();
393  #ifdef __clang__
394  else if constexpr (_Bytes == sizeof(char))
395  return char();
396  #else
397  else if constexpr (_Bytes == sizeof(_SChar))
398  return _SChar();
399  #endif
400  else if constexpr (_Bytes == sizeof(short))
401  return short();
402  #ifndef __clang__
403  else if constexpr (_Bytes == sizeof(long))
404  return long();
405  #endif
406  else if constexpr (_Bytes == sizeof(_LLong))
407  return _LLong();
408  #ifdef __SIZEOF_INT128__
409  else if constexpr (_Bytes == sizeof(__int128))
410  return __int128();
411  #endif // __SIZEOF_INT128__
412  else if constexpr (_Bytes % sizeof(int) == 0)
413  {
414  constexpr size_t _Np = _Bytes / sizeof(int);
415  struct _Ip
416  {
417  int _M_data[_Np];
418 
419  _GLIBCXX_SIMD_INTRINSIC constexpr _Ip
420  operator&(_Ip __rhs) const
421  {
422  return __generate_from_n_evaluations<_Np, _Ip>(
423  [&](auto __i) { return __rhs._M_data[__i] & _M_data[__i]; });
424  }
425 
426  _GLIBCXX_SIMD_INTRINSIC constexpr _Ip
427  operator|(_Ip __rhs) const
428  {
429  return __generate_from_n_evaluations<_Np, _Ip>(
430  [&](auto __i) { return __rhs._M_data[__i] | _M_data[__i]; });
431  }
432 
433  _GLIBCXX_SIMD_INTRINSIC constexpr _Ip
434  operator^(_Ip __rhs) const
435  {
436  return __generate_from_n_evaluations<_Np, _Ip>(
437  [&](auto __i) { return __rhs._M_data[__i] ^ _M_data[__i]; });
438  }
439 
440  _GLIBCXX_SIMD_INTRINSIC constexpr _Ip
441  operator~() const
442  {
443  return __generate_from_n_evaluations<_Np, _Ip>(
444  [&](auto __i) { return ~_M_data[__i]; });
445  }
446  };
447  return _Ip{};
448  }
449  else
450  static_assert(_Bytes != _Bytes, "this should be unreachable");
451  }
452 #pragma GCC diagnostic pop
453 
454 template <typename _Tp>
455  using __int_for_sizeof_t = decltype(__int_for_sizeof<sizeof(_Tp)>());
456 
457 template <size_t _Np>
458  using __int_with_sizeof_t = decltype(__int_for_sizeof<_Np>());
459 
460 // }}}
461 // __is_fixed_size_abi{{{
462 template <typename _Tp>
463  struct __is_fixed_size_abi : false_type {};
464 
465 template <int _Np>
466  struct __is_fixed_size_abi<simd_abi::fixed_size<_Np>> : true_type {};
467 
468 template <typename _Tp>
469  inline constexpr bool __is_fixed_size_abi_v = __is_fixed_size_abi<_Tp>::value;
470 
471 // }}}
472 // constexpr feature detection{{{
473 constexpr inline bool __have_mmx = _GLIBCXX_SIMD_HAVE_MMX;
474 constexpr inline bool __have_sse = _GLIBCXX_SIMD_HAVE_SSE;
475 constexpr inline bool __have_sse2 = _GLIBCXX_SIMD_HAVE_SSE2;
476 constexpr inline bool __have_sse3 = _GLIBCXX_SIMD_HAVE_SSE3;
477 constexpr inline bool __have_ssse3 = _GLIBCXX_SIMD_HAVE_SSSE3;
478 constexpr inline bool __have_sse4_1 = _GLIBCXX_SIMD_HAVE_SSE4_1;
479 constexpr inline bool __have_sse4_2 = _GLIBCXX_SIMD_HAVE_SSE4_2;
480 constexpr inline bool __have_xop = _GLIBCXX_SIMD_HAVE_XOP;
481 constexpr inline bool __have_avx = _GLIBCXX_SIMD_HAVE_AVX;
482 constexpr inline bool __have_avx2 = _GLIBCXX_SIMD_HAVE_AVX2;
483 constexpr inline bool __have_bmi = _GLIBCXX_SIMD_HAVE_BMI1;
484 constexpr inline bool __have_bmi2 = _GLIBCXX_SIMD_HAVE_BMI2;
485 constexpr inline bool __have_lzcnt = _GLIBCXX_SIMD_HAVE_LZCNT;
486 constexpr inline bool __have_sse4a = _GLIBCXX_SIMD_HAVE_SSE4A;
487 constexpr inline bool __have_fma = _GLIBCXX_SIMD_HAVE_FMA;
488 constexpr inline bool __have_fma4 = _GLIBCXX_SIMD_HAVE_FMA4;
489 constexpr inline bool __have_f16c = _GLIBCXX_SIMD_HAVE_F16C;
490 constexpr inline bool __have_popcnt = _GLIBCXX_SIMD_HAVE_POPCNT;
491 constexpr inline bool __have_avx512f = _GLIBCXX_SIMD_HAVE_AVX512F;
492 constexpr inline bool __have_avx512dq = _GLIBCXX_SIMD_HAVE_AVX512DQ;
493 constexpr inline bool __have_avx512vl = _GLIBCXX_SIMD_HAVE_AVX512VL;
494 constexpr inline bool __have_avx512bw = _GLIBCXX_SIMD_HAVE_AVX512BW;
495 constexpr inline bool __have_avx512dq_vl = __have_avx512dq && __have_avx512vl;
496 constexpr inline bool __have_avx512bw_vl = __have_avx512bw && __have_avx512vl;
497 
498 constexpr inline bool __have_neon = _GLIBCXX_SIMD_HAVE_NEON;
499 constexpr inline bool __have_neon_a32 = _GLIBCXX_SIMD_HAVE_NEON_A32;
500 constexpr inline bool __have_neon_a64 = _GLIBCXX_SIMD_HAVE_NEON_A64;
501 constexpr inline bool __support_neon_float =
502 #if defined __GCC_IEC_559
503  __GCC_IEC_559 == 0;
504 #elif defined __FAST_MATH__
505  true;
506 #else
507  false;
508 #endif
509 
510 #ifdef _ARCH_PWR10
511 constexpr inline bool __have_power10vec = true;
512 #else
513 constexpr inline bool __have_power10vec = false;
514 #endif
515 #ifdef __POWER9_VECTOR__
516 constexpr inline bool __have_power9vec = true;
517 #else
518 constexpr inline bool __have_power9vec = false;
519 #endif
520 #if defined __POWER8_VECTOR__
521 constexpr inline bool __have_power8vec = true;
522 #else
523 constexpr inline bool __have_power8vec = __have_power9vec;
524 #endif
525 #if defined __VSX__
526 constexpr inline bool __have_power_vsx = true;
527 #else
528 constexpr inline bool __have_power_vsx = __have_power8vec;
529 #endif
530 #if defined __ALTIVEC__
531 constexpr inline bool __have_power_vmx = true;
532 #else
533 constexpr inline bool __have_power_vmx = __have_power_vsx;
534 #endif
535 
536 // }}}
537 // __is_scalar_abi {{{
538 template <typename _Abi>
539  constexpr bool
540  __is_scalar_abi()
541  { return is_same_v<simd_abi::scalar, _Abi>; }
542 
543 // }}}
544 // __abi_bytes_v {{{
545 template <template <int> class _Abi, int _Bytes>
546  constexpr int
547  __abi_bytes_impl(_Abi<_Bytes>*)
548  { return _Bytes; }
549 
550 template <typename _Tp>
551  constexpr int
552  __abi_bytes_impl(_Tp*)
553  { return -1; }
554 
555 template <typename _Abi>
556  inline constexpr int __abi_bytes_v
557  = __abi_bytes_impl(static_cast<_Abi*>(nullptr));
558 
559 // }}}
560 // __is_builtin_bitmask_abi {{{
561 template <typename _Abi>
562  constexpr bool
563  __is_builtin_bitmask_abi()
564  { return is_same_v<simd_abi::_VecBltnBtmsk<__abi_bytes_v<_Abi>>, _Abi>; }
565 
566 // }}}
567 // __is_sse_abi {{{
568 template <typename _Abi>
569  constexpr bool
570  __is_sse_abi()
571  {
572  constexpr auto _Bytes = __abi_bytes_v<_Abi>;
573  return _Bytes <= 16 && is_same_v<simd_abi::_VecBuiltin<_Bytes>, _Abi>;
574  }
575 
576 // }}}
577 // __is_avx_abi {{{
578 template <typename _Abi>
579  constexpr bool
580  __is_avx_abi()
581  {
582  constexpr auto _Bytes = __abi_bytes_v<_Abi>;
583  return _Bytes > 16 && _Bytes <= 32
584  && is_same_v<simd_abi::_VecBuiltin<_Bytes>, _Abi>;
585  }
586 
587 // }}}
588 // __is_avx512_abi {{{
589 template <typename _Abi>
590  constexpr bool
591  __is_avx512_abi()
592  {
593  constexpr auto _Bytes = __abi_bytes_v<_Abi>;
594  return _Bytes <= 64 && is_same_v<simd_abi::_Avx512<_Bytes>, _Abi>;
595  }
596 
597 // }}}
598 // __is_neon_abi {{{
599 template <typename _Abi>
600  constexpr bool
601  __is_neon_abi()
602  {
603  constexpr auto _Bytes = __abi_bytes_v<_Abi>;
604  return _Bytes <= 16 && is_same_v<simd_abi::_VecBuiltin<_Bytes>, _Abi>;
605  }
606 
607 // }}}
608 // __make_dependent_t {{{
609 template <typename, typename _Up>
610  struct __make_dependent
611  { using type = _Up; };
612 
613 template <typename _Tp, typename _Up>
614  using __make_dependent_t = typename __make_dependent<_Tp, _Up>::type;
615 
616 // }}}
617 // ^^^ ---- type traits ---- ^^^
618 
619 // __invoke_ub{{{
620 template <typename... _Args>
621  [[noreturn]] _GLIBCXX_SIMD_ALWAYS_INLINE void
622  __invoke_ub([[maybe_unused]] const char* __msg,
623  [[maybe_unused]] const _Args&... __args)
624  {
625 #ifdef _GLIBCXX_DEBUG_UB
626  __builtin_fprintf(stderr, __msg, __args...);
627  __builtin_trap();
628 #else
629  __builtin_unreachable();
630 #endif
631  }
632 
633 // }}}
634 // __assert_unreachable{{{
635 template <typename _Tp>
636  struct __assert_unreachable
637  { static_assert(!is_same_v<_Tp, _Tp>, "this should be unreachable"); };
638 
639 // }}}
640 // __size_or_zero_v {{{
641 template <typename _Tp, typename _Ap, size_t _Np = simd_size<_Tp, _Ap>::value>
642  constexpr size_t
643  __size_or_zero_dispatch(int)
644  { return _Np; }
645 
646 template <typename _Tp, typename _Ap>
647  constexpr size_t
648  __size_or_zero_dispatch(float)
649  { return 0; }
650 
651 template <typename _Tp, typename _Ap>
652  inline constexpr size_t __size_or_zero_v
653  = __size_or_zero_dispatch<_Tp, _Ap>(0);
654 
655 // }}}
656 // __div_roundup {{{
657 inline constexpr size_t
658 __div_roundup(size_t __a, size_t __b)
659 { return (__a + __b - 1) / __b; }
660 
661 // }}}
662 // _ExactBool{{{
663 class _ExactBool
664 {
665  const bool _M_data;
666 
667 public:
668  _GLIBCXX_SIMD_INTRINSIC constexpr _ExactBool(bool __b) : _M_data(__b) {}
669 
670  _ExactBool(int) = delete;
671 
672  _GLIBCXX_SIMD_INTRINSIC constexpr operator bool() const { return _M_data; }
673 };
674 
675 // }}}
676 // __may_alias{{{
677 /**@internal
678  * Helper __may_alias<_Tp> that turns _Tp into the type to be used for an
679  * aliasing pointer. This adds the __may_alias attribute to _Tp (with compilers
680  * that support it).
681  */
682 template <typename _Tp>
683  using __may_alias [[__gnu__::__may_alias__]] = _Tp;
684 
685 // }}}
686 // _UnsupportedBase {{{
687 // simd and simd_mask base for unsupported <_Tp, _Abi>
688 struct _UnsupportedBase
689 {
690  _UnsupportedBase() = delete;
691  _UnsupportedBase(const _UnsupportedBase&) = delete;
692  _UnsupportedBase& operator=(const _UnsupportedBase&) = delete;
693  ~_UnsupportedBase() = delete;
694 };
695 
696 // }}}
697 // _InvalidTraits {{{
698 /**
699  * @internal
700  * Defines the implementation of __a given <_Tp, _Abi>.
701  *
702  * Implementations must ensure that only valid <_Tp, _Abi> instantiations are
703  * possible. Static assertions in the type definition do not suffice. It is
704  * important that SFINAE works.
705  */
706 struct _InvalidTraits
707 {
708  using _IsValid = false_type;
709  using _SimdBase = _UnsupportedBase;
710  using _MaskBase = _UnsupportedBase;
711 
712  static constexpr size_t _S_full_size = 0;
713  static constexpr bool _S_is_partial = false;
714 
715  static constexpr size_t _S_simd_align = 1;
716  struct _SimdImpl;
717  struct _SimdMember {};
718  struct _SimdCastType;
719 
720  static constexpr size_t _S_mask_align = 1;
721  struct _MaskImpl;
722  struct _MaskMember {};
723  struct _MaskCastType;
724 };
725 
726 // }}}
727 // _SimdTraits {{{
728 template <typename _Tp, typename _Abi, typename = void_t<>>
729  struct _SimdTraits : _InvalidTraits {};
730 
731 // }}}
732 // __private_init, __bitset_init{{{
733 /**
734  * @internal
735  * Tag used for private init constructor of simd and simd_mask
736  */
737 inline constexpr struct _PrivateInit {} __private_init = {};
738 
739 inline constexpr struct _BitsetInit {} __bitset_init = {};
740 
741 // }}}
742 // __is_narrowing_conversion<_From, _To>{{{
743 template <typename _From, typename _To, bool = is_arithmetic_v<_From>,
744  bool = is_arithmetic_v<_To>>
745  struct __is_narrowing_conversion;
746 
747 // ignore "signed/unsigned mismatch" in the following trait.
748 // The implicit conversions will do the right thing here.
749 template <typename _From, typename _To>
750  struct __is_narrowing_conversion<_From, _To, true, true>
751  : public __bool_constant<(
752  __digits_v<_From> > __digits_v<_To>
753  || __finite_max_v<_From> > __finite_max_v<_To>
754  || __finite_min_v<_From> < __finite_min_v<_To>
755  || (is_signed_v<_From> && is_unsigned_v<_To>))> {};
756 
757 template <typename _Tp>
758  struct __is_narrowing_conversion<_Tp, bool, true, true>
759  : public true_type {};
760 
761 template <>
762  struct __is_narrowing_conversion<bool, bool, true, true>
763  : public false_type {};
764 
765 template <typename _Tp>
766  struct __is_narrowing_conversion<_Tp, _Tp, true, true>
767  : public false_type {};
768 
769 template <typename _From, typename _To>
770  struct __is_narrowing_conversion<_From, _To, false, true>
771  : public negation<is_convertible<_From, _To>> {};
772 
773 // }}}
774 // __converts_to_higher_integer_rank{{{
775 template <typename _From, typename _To, bool = (sizeof(_From) < sizeof(_To))>
776  struct __converts_to_higher_integer_rank : public true_type {};
777 
778 // this may fail for char -> short if sizeof(char) == sizeof(short)
779 template <typename _From, typename _To>
780  struct __converts_to_higher_integer_rank<_From, _To, false>
781  : public is_same<decltype(declval<_From>() + declval<_To>()), _To> {};
782 
783 // }}}
784 // __data(simd/simd_mask) {{{
785 template <typename _Tp, typename _Ap>
786  _GLIBCXX_SIMD_INTRINSIC constexpr const auto&
787  __data(const simd<_Tp, _Ap>& __x);
788 
789 template <typename _Tp, typename _Ap>
790  _GLIBCXX_SIMD_INTRINSIC constexpr auto&
791  __data(simd<_Tp, _Ap>& __x);
792 
793 template <typename _Tp, typename _Ap>
794  _GLIBCXX_SIMD_INTRINSIC constexpr const auto&
795  __data(const simd_mask<_Tp, _Ap>& __x);
796 
797 template <typename _Tp, typename _Ap>
798  _GLIBCXX_SIMD_INTRINSIC constexpr auto&
799  __data(simd_mask<_Tp, _Ap>& __x);
800 
801 // }}}
802 // _SimdConverter {{{
803 template <typename _FromT, typename _FromA, typename _ToT, typename _ToA,
804  typename = void>
805  struct _SimdConverter;
806 
807 template <typename _Tp, typename _Ap>
808  struct _SimdConverter<_Tp, _Ap, _Tp, _Ap, void>
809  {
810  template <typename _Up>
811  _GLIBCXX_SIMD_INTRINSIC const _Up&
812  operator()(const _Up& __x)
813  { return __x; }
814  };
815 
816 // }}}
817 // __to_value_type_or_member_type {{{
818 template <typename _V>
819  _GLIBCXX_SIMD_INTRINSIC constexpr auto
820  __to_value_type_or_member_type(const _V& __x) -> decltype(__data(__x))
821  { return __data(__x); }
822 
823 template <typename _V>
824  _GLIBCXX_SIMD_INTRINSIC constexpr const typename _V::value_type&
825  __to_value_type_or_member_type(const typename _V::value_type& __x)
826  { return __x; }
827 
828 // }}}
829 // __bool_storage_member_type{{{
830 template <size_t _Size>
831  struct __bool_storage_member_type;
832 
833 template <size_t _Size>
834  using __bool_storage_member_type_t =
835  typename __bool_storage_member_type<_Size>::type;
836 
837 // }}}
838 // _SimdTuple {{{
839 // why not tuple?
840 // 1. tuple gives no guarantee about the storage order, but I require
841 // storage
842 // equivalent to array<_Tp, _Np>
843 // 2. direct access to the element type (first template argument)
844 // 3. enforces equal element type, only different _Abi types are allowed
845 template <typename _Tp, typename... _Abis>
846  struct _SimdTuple;
847 
848 //}}}
849 // __fixed_size_storage_t {{{
850 template <typename _Tp, int _Np>
851  struct __fixed_size_storage;
852 
853 template <typename _Tp, int _Np>
854  using __fixed_size_storage_t = typename __fixed_size_storage<_Tp, _Np>::type;
855 
856 // }}}
857 // _SimdWrapper fwd decl{{{
858 template <typename _Tp, size_t _Size, typename = void_t<>>
859  struct _SimdWrapper;
860 
861 template <typename _Tp>
862  using _SimdWrapper8 = _SimdWrapper<_Tp, 8 / sizeof(_Tp)>;
863 template <typename _Tp>
864  using _SimdWrapper16 = _SimdWrapper<_Tp, 16 / sizeof(_Tp)>;
865 template <typename _Tp>
866  using _SimdWrapper32 = _SimdWrapper<_Tp, 32 / sizeof(_Tp)>;
867 template <typename _Tp>
868  using _SimdWrapper64 = _SimdWrapper<_Tp, 64 / sizeof(_Tp)>;
869 
870 // }}}
871 // __is_simd_wrapper {{{
872 template <typename _Tp>
873  struct __is_simd_wrapper : false_type {};
874 
875 template <typename _Tp, size_t _Np>
876  struct __is_simd_wrapper<_SimdWrapper<_Tp, _Np>> : true_type {};
877 
878 template <typename _Tp>
879  inline constexpr bool __is_simd_wrapper_v = __is_simd_wrapper<_Tp>::value;
880 
881 // }}}
882 // _BitOps {{{
883 struct _BitOps
884 {
885  // _S_bit_iteration {{{
886  template <typename _Tp, typename _Fp>
887  static void
888  _S_bit_iteration(_Tp __mask, _Fp&& __f)
889  {
890  static_assert(sizeof(_ULLong) >= sizeof(_Tp));
891  conditional_t<sizeof(_Tp) <= sizeof(_UInt), _UInt, _ULLong> __k;
892  if constexpr (is_convertible_v<_Tp, decltype(__k)>)
893  __k = __mask;
894  else
895  __k = __mask.to_ullong();
896  while(__k)
897  {
898  __f(std::__countr_zero(__k));
899  __k &= (__k - 1);
900  }
901  }
902 
903  //}}}
904 };
905 
906 //}}}
907 // __increment, __decrement {{{
908 template <typename _Tp = void>
909  struct __increment
910  { constexpr _Tp operator()(_Tp __a) const { return ++__a; } };
911 
912 template <>
913  struct __increment<void>
914  {
915  template <typename _Tp>
916  constexpr _Tp
917  operator()(_Tp __a) const
918  { return ++__a; }
919  };
920 
921 template <typename _Tp = void>
922  struct __decrement
923  { constexpr _Tp operator()(_Tp __a) const { return --__a; } };
924 
925 template <>
926  struct __decrement<void>
927  {
928  template <typename _Tp>
929  constexpr _Tp
930  operator()(_Tp __a) const
931  { return --__a; }
932  };
933 
934 // }}}
935 // _ValuePreserving(OrInt) {{{
936 template <typename _From, typename _To,
937  typename = enable_if_t<negation<
938  __is_narrowing_conversion<__remove_cvref_t<_From>, _To>>::value>>
939  using _ValuePreserving = _From;
940 
941 template <typename _From, typename _To,
942  typename _DecayedFrom = __remove_cvref_t<_From>,
943  typename = enable_if_t<conjunction<
944  is_convertible<_From, _To>,
945  disjunction<
946  is_same<_DecayedFrom, _To>, is_same<_DecayedFrom, int>,
947  conjunction<is_same<_DecayedFrom, _UInt>, is_unsigned<_To>>,
948  negation<__is_narrowing_conversion<_DecayedFrom, _To>>>>::value>>
949  using _ValuePreservingOrInt = _From;
950 
951 // }}}
952 // __intrinsic_type {{{
953 template <typename _Tp, size_t _Bytes, typename = void_t<>>
954  struct __intrinsic_type;
955 
956 template <typename _Tp, size_t _Size>
957  using __intrinsic_type_t =
958  typename __intrinsic_type<_Tp, _Size * sizeof(_Tp)>::type;
959 
960 template <typename _Tp>
961  using __intrinsic_type2_t = typename __intrinsic_type<_Tp, 2>::type;
962 template <typename _Tp>
963  using __intrinsic_type4_t = typename __intrinsic_type<_Tp, 4>::type;
964 template <typename _Tp>
965  using __intrinsic_type8_t = typename __intrinsic_type<_Tp, 8>::type;
966 template <typename _Tp>
967  using __intrinsic_type16_t = typename __intrinsic_type<_Tp, 16>::type;
968 template <typename _Tp>
969  using __intrinsic_type32_t = typename __intrinsic_type<_Tp, 32>::type;
970 template <typename _Tp>
971  using __intrinsic_type64_t = typename __intrinsic_type<_Tp, 64>::type;
972 
973 // }}}
974 // _BitMask {{{
975 template <size_t _Np, bool _Sanitized = false>
976  struct _BitMask;
977 
978 template <size_t _Np, bool _Sanitized>
979  struct __is_bitmask<_BitMask<_Np, _Sanitized>, void> : true_type {};
980 
981 template <size_t _Np>
982  using _SanitizedBitMask = _BitMask<_Np, true>;
983 
984 template <size_t _Np, bool _Sanitized>
985  struct _BitMask
986  {
987  static_assert(_Np > 0);
988 
989  static constexpr size_t _NBytes = __div_roundup(_Np, __CHAR_BIT__);
990 
991  using _Tp = conditional_t<_Np == 1, bool,
992  make_unsigned_t<__int_with_sizeof_t<std::min(
993  sizeof(_ULLong), std::__bit_ceil(_NBytes))>>>;
994 
995  static constexpr int _S_array_size = __div_roundup(_NBytes, sizeof(_Tp));
996 
997  _Tp _M_bits[_S_array_size];
998 
999  static constexpr int _S_unused_bits
1000  = _Np == 1 ? 0 : _S_array_size * sizeof(_Tp) * __CHAR_BIT__ - _Np;
1001 
1002  static constexpr _Tp _S_bitmask = +_Tp(~_Tp()) >> _S_unused_bits;
1003 
1004  constexpr _BitMask() noexcept = default;
1005 
1006  constexpr _BitMask(unsigned long long __x) noexcept
1007  : _M_bits{static_cast<_Tp>(__x)} {}
1008 
1009  _BitMask(bitset<_Np> __x) noexcept : _BitMask(__x.to_ullong()) {}
1010 
1011  constexpr _BitMask(const _BitMask&) noexcept = default;
1012 
1013  template <bool _RhsSanitized, typename = enable_if_t<_RhsSanitized == false
1014  && _Sanitized == true>>
1015  constexpr _BitMask(const _BitMask<_Np, _RhsSanitized>& __rhs) noexcept
1016  : _BitMask(__rhs._M_sanitized()) {}
1017 
1018  constexpr operator _SimdWrapper<bool, _Np>() const noexcept
1019  {
1020  static_assert(_S_array_size == 1);
1021  return _M_bits[0];
1022  }
1023 
1024  // precondition: is sanitized
1025  constexpr _Tp
1026  _M_to_bits() const noexcept
1027  {
1028  static_assert(_S_array_size == 1);
1029  return _M_bits[0];
1030  }
1031 
1032  // precondition: is sanitized
1033  constexpr unsigned long long
1034  to_ullong() const noexcept
1035  {
1036  static_assert(_S_array_size == 1);
1037  return _M_bits[0];
1038  }
1039 
1040  // precondition: is sanitized
1041  constexpr unsigned long
1042  to_ulong() const noexcept
1043  {
1044  static_assert(_S_array_size == 1);
1045  return _M_bits[0];
1046  }
1047 
1048  constexpr bitset<_Np>
1049  _M_to_bitset() const noexcept
1050  {
1051  static_assert(_S_array_size == 1);
1052  return _M_bits[0];
1053  }
1054 
1055  constexpr decltype(auto)
1056  _M_sanitized() const noexcept
1057  {
1058  if constexpr (_Sanitized)
1059  return *this;
1060  else if constexpr (_Np == 1)
1061  return _SanitizedBitMask<_Np>(_M_bits[0]);
1062  else
1063  {
1064  _SanitizedBitMask<_Np> __r = {};
1065  for (int __i = 0; __i < _S_array_size; ++__i)
1066  __r._M_bits[__i] = _M_bits[__i];
1067  if constexpr (_S_unused_bits > 0)
1068  __r._M_bits[_S_array_size - 1] &= _S_bitmask;
1069  return __r;
1070  }
1071  }
1072 
1073  template <size_t _Mp, bool _LSanitized>
1074  constexpr _BitMask<_Np + _Mp, _Sanitized>
1075  _M_prepend(_BitMask<_Mp, _LSanitized> __lsb) const noexcept
1076  {
1077  constexpr size_t _RN = _Np + _Mp;
1078  using _Rp = _BitMask<_RN, _Sanitized>;
1079  if constexpr (_Rp::_S_array_size == 1)
1080  {
1081  _Rp __r{{_M_bits[0]}};
1082  __r._M_bits[0] <<= _Mp;
1083  __r._M_bits[0] |= __lsb._M_sanitized()._M_bits[0];
1084  return __r;
1085  }
1086  else
1087  __assert_unreachable<_Rp>();
1088  }
1089 
1090  // Return a new _BitMask with size _NewSize while dropping _DropLsb least
1091  // significant bits. If the operation implicitly produces a sanitized bitmask,
1092  // the result type will have _Sanitized set.
1093  template <size_t _DropLsb, size_t _NewSize = _Np - _DropLsb>
1094  constexpr auto
1095  _M_extract() const noexcept
1096  {
1097  static_assert(_Np > _DropLsb);
1098  static_assert(_DropLsb + _NewSize <= sizeof(_ULLong) * __CHAR_BIT__,
1099  "not implemented for bitmasks larger than one ullong");
1100  if constexpr (_NewSize == 1)
1101  // must sanitize because the return _Tp is bool
1102  return _SanitizedBitMask<1>(_M_bits[0] & (_Tp(1) << _DropLsb));
1103  else
1104  return _BitMask<_NewSize,
1105  ((_NewSize + _DropLsb == sizeof(_Tp) * __CHAR_BIT__
1106  && _NewSize + _DropLsb <= _Np)
1107  || ((_Sanitized || _Np == sizeof(_Tp) * __CHAR_BIT__)
1108  && _NewSize + _DropLsb >= _Np))>(_M_bits[0]
1109  >> _DropLsb);
1110  }
1111 
1112  // True if all bits are set. Implicitly sanitizes if _Sanitized == false.
1113  constexpr bool
1114  all() const noexcept
1115  {
1116  if constexpr (_Np == 1)
1117  return _M_bits[0];
1118  else if constexpr (!_Sanitized)
1119  return _M_sanitized().all();
1120  else
1121  {
1122  constexpr _Tp __allbits = ~_Tp();
1123  for (int __i = 0; __i < _S_array_size - 1; ++__i)
1124  if (_M_bits[__i] != __allbits)
1125  return false;
1126  return _M_bits[_S_array_size - 1] == _S_bitmask;
1127  }
1128  }
1129 
1130  // True if at least one bit is set. Implicitly sanitizes if _Sanitized ==
1131  // false.
1132  constexpr bool
1133  any() const noexcept
1134  {
1135  if constexpr (_Np == 1)
1136  return _M_bits[0];
1137  else if constexpr (!_Sanitized)
1138  return _M_sanitized().any();
1139  else
1140  {
1141  for (int __i = 0; __i < _S_array_size - 1; ++__i)
1142  if (_M_bits[__i] != 0)
1143  return true;
1144  return _M_bits[_S_array_size - 1] != 0;
1145  }
1146  }
1147 
1148  // True if no bit is set. Implicitly sanitizes if _Sanitized == false.
1149  constexpr bool
1150  none() const noexcept
1151  {
1152  if constexpr (_Np == 1)
1153  return !_M_bits[0];
1154  else if constexpr (!_Sanitized)
1155  return _M_sanitized().none();
1156  else
1157  {
1158  for (int __i = 0; __i < _S_array_size - 1; ++__i)
1159  if (_M_bits[__i] != 0)
1160  return false;
1161  return _M_bits[_S_array_size - 1] == 0;
1162  }
1163  }
1164 
1165  // Returns the number of set bits. Implicitly sanitizes if _Sanitized ==
1166  // false.
1167  constexpr int
1168  count() const noexcept
1169  {
1170  if constexpr (_Np == 1)
1171  return _M_bits[0];
1172  else if constexpr (!_Sanitized)
1173  return _M_sanitized().none();
1174  else
1175  {
1176  int __result = __builtin_popcountll(_M_bits[0]);
1177  for (int __i = 1; __i < _S_array_size; ++__i)
1178  __result += __builtin_popcountll(_M_bits[__i]);
1179  return __result;
1180  }
1181  }
1182 
1183  // Returns the bit at offset __i as bool.
1184  constexpr bool
1185  operator[](size_t __i) const noexcept
1186  {
1187  if constexpr (_Np == 1)
1188  return _M_bits[0];
1189  else if constexpr (_S_array_size == 1)
1190  return (_M_bits[0] >> __i) & 1;
1191  else
1192  {
1193  const size_t __j = __i / (sizeof(_Tp) * __CHAR_BIT__);
1194  const size_t __shift = __i % (sizeof(_Tp) * __CHAR_BIT__);
1195  return (_M_bits[__j] >> __shift) & 1;
1196  }
1197  }
1198 
1199  template <size_t __i>
1200  constexpr bool
1201  operator[](_SizeConstant<__i>) const noexcept
1202  {
1203  static_assert(__i < _Np);
1204  constexpr size_t __j = __i / (sizeof(_Tp) * __CHAR_BIT__);
1205  constexpr size_t __shift = __i % (sizeof(_Tp) * __CHAR_BIT__);
1206  return static_cast<bool>(_M_bits[__j] & (_Tp(1) << __shift));
1207  }
1208 
1209  // Set the bit at offset __i to __x.
1210  constexpr void
1211  set(size_t __i, bool __x) noexcept
1212  {
1213  if constexpr (_Np == 1)
1214  _M_bits[0] = __x;
1215  else if constexpr (_S_array_size == 1)
1216  {
1217  _M_bits[0] &= ~_Tp(_Tp(1) << __i);
1218  _M_bits[0] |= _Tp(_Tp(__x) << __i);
1219  }
1220  else
1221  {
1222  const size_t __j = __i / (sizeof(_Tp) * __CHAR_BIT__);
1223  const size_t __shift = __i % (sizeof(_Tp) * __CHAR_BIT__);
1224  _M_bits[__j] &= ~_Tp(_Tp(1) << __shift);
1225  _M_bits[__j] |= _Tp(_Tp(__x) << __shift);
1226  }
1227  }
1228 
1229  template <size_t __i>
1230  constexpr void
1231  set(_SizeConstant<__i>, bool __x) noexcept
1232  {
1233  static_assert(__i < _Np);
1234  if constexpr (_Np == 1)
1235  _M_bits[0] = __x;
1236  else
1237  {
1238  constexpr size_t __j = __i / (sizeof(_Tp) * __CHAR_BIT__);
1239  constexpr size_t __shift = __i % (sizeof(_Tp) * __CHAR_BIT__);
1240  constexpr _Tp __mask = ~_Tp(_Tp(1) << __shift);
1241  _M_bits[__j] &= __mask;
1242  _M_bits[__j] |= _Tp(_Tp(__x) << __shift);
1243  }
1244  }
1245 
1246  // Inverts all bits. Sanitized input leads to sanitized output.
1247  constexpr _BitMask
1248  operator~() const noexcept
1249  {
1250  if constexpr (_Np == 1)
1251  return !_M_bits[0];
1252  else
1253  {
1254  _BitMask __result{};
1255  for (int __i = 0; __i < _S_array_size - 1; ++__i)
1256  __result._M_bits[__i] = ~_M_bits[__i];
1257  if constexpr (_Sanitized)
1258  __result._M_bits[_S_array_size - 1]
1259  = _M_bits[_S_array_size - 1] ^ _S_bitmask;
1260  else
1261  __result._M_bits[_S_array_size - 1] = ~_M_bits[_S_array_size - 1];
1262  return __result;
1263  }
1264  }
1265 
1266  constexpr _BitMask&
1267  operator^=(const _BitMask& __b) & noexcept
1268  {
1269  __execute_n_times<_S_array_size>(
1270  [&](auto __i) { _M_bits[__i] ^= __b._M_bits[__i]; });
1271  return *this;
1272  }
1273 
1274  constexpr _BitMask&
1275  operator|=(const _BitMask& __b) & noexcept
1276  {
1277  __execute_n_times<_S_array_size>(
1278  [&](auto __i) { _M_bits[__i] |= __b._M_bits[__i]; });
1279  return *this;
1280  }
1281 
1282  constexpr _BitMask&
1283  operator&=(const _BitMask& __b) & noexcept
1284  {
1285  __execute_n_times<_S_array_size>(
1286  [&](auto __i) { _M_bits[__i] &= __b._M_bits[__i]; });
1287  return *this;
1288  }
1289 
1290  friend constexpr _BitMask
1291  operator^(const _BitMask& __a, const _BitMask& __b) noexcept
1292  {
1293  _BitMask __r = __a;
1294  __r ^= __b;
1295  return __r;
1296  }
1297 
1298  friend constexpr _BitMask
1299  operator|(const _BitMask& __a, const _BitMask& __b) noexcept
1300  {
1301  _BitMask __r = __a;
1302  __r |= __b;
1303  return __r;
1304  }
1305 
1306  friend constexpr _BitMask
1307  operator&(const _BitMask& __a, const _BitMask& __b) noexcept
1308  {
1309  _BitMask __r = __a;
1310  __r &= __b;
1311  return __r;
1312  }
1313 
1314  _GLIBCXX_SIMD_INTRINSIC
1315  constexpr bool
1316  _M_is_constprop() const
1317  {
1318  if constexpr (_S_array_size == 0)
1319  return __builtin_constant_p(_M_bits[0]);
1320  else
1321  {
1322  for (int __i = 0; __i < _S_array_size; ++__i)
1323  if (!__builtin_constant_p(_M_bits[__i]))
1324  return false;
1325  return true;
1326  }
1327  }
1328  };
1329 
1330 // }}}
1331 
1332 // vvv ---- builtin vector types [[gnu::vector_size(N)]] and operations ---- vvv
1333 // __min_vector_size {{{
1334 template <typename _Tp = void>
1335  static inline constexpr int __min_vector_size = 2 * sizeof(_Tp);
1336 
1337 #if _GLIBCXX_SIMD_HAVE_NEON
1338 template <>
1339  inline constexpr int __min_vector_size<void> = 8;
1340 #else
1341 template <>
1342  inline constexpr int __min_vector_size<void> = 16;
1343 #endif
1344 
1345 // }}}
1346 // __vector_type {{{
1347 template <typename _Tp, size_t _Np, typename = void>
1348  struct __vector_type_n {};
1349 
1350 // substition failure for 0-element case
1351 template <typename _Tp>
1352  struct __vector_type_n<_Tp, 0, void> {};
1353 
1354 // special case 1-element to be _Tp itself
1355 template <typename _Tp>
1356  struct __vector_type_n<_Tp, 1, enable_if_t<__is_vectorizable_v<_Tp>>>
1357  { using type = _Tp; };
1358 
1359 // else, use GNU-style builtin vector types
1360 template <typename _Tp, size_t _Np>
1361  struct __vector_type_n<_Tp, _Np,
1362  enable_if_t<__is_vectorizable_v<_Tp> && _Np >= 2>>
1363  {
1364  static constexpr size_t _S_Np2 = std::__bit_ceil(_Np * sizeof(_Tp));
1365 
1366  static constexpr size_t _S_Bytes =
1367 #ifdef __i386__
1368  // Using [[gnu::vector_size(8)]] would wreak havoc on the FPU because
1369  // those objects are passed via MMX registers and nothing ever calls EMMS.
1370  _S_Np2 == 8 ? 16 :
1371 #endif
1372  _S_Np2 < __min_vector_size<_Tp> ? __min_vector_size<_Tp>
1373  : _S_Np2;
1374 
1375  using type [[__gnu__::__vector_size__(_S_Bytes)]] = _Tp;
1376  };
1377 
1378 template <typename _Tp, size_t _Bytes, size_t = _Bytes % sizeof(_Tp)>
1379  struct __vector_type;
1380 
1381 template <typename _Tp, size_t _Bytes>
1382  struct __vector_type<_Tp, _Bytes, 0>
1383  : __vector_type_n<_Tp, _Bytes / sizeof(_Tp)> {};
1384 
1385 template <typename _Tp, size_t _Size>
1386  using __vector_type_t = typename __vector_type_n<_Tp, _Size>::type;
1387 
1388 template <typename _Tp>
1389  using __vector_type2_t = typename __vector_type<_Tp, 2>::type;
1390 template <typename _Tp>
1391  using __vector_type4_t = typename __vector_type<_Tp, 4>::type;
1392 template <typename _Tp>
1393  using __vector_type8_t = typename __vector_type<_Tp, 8>::type;
1394 template <typename _Tp>
1395  using __vector_type16_t = typename __vector_type<_Tp, 16>::type;
1396 template <typename _Tp>
1397  using __vector_type32_t = typename __vector_type<_Tp, 32>::type;
1398 template <typename _Tp>
1399  using __vector_type64_t = typename __vector_type<_Tp, 64>::type;
1400 
1401 // }}}
1402 // __is_vector_type {{{
1403 template <typename _Tp, typename = void_t<>>
1404  struct __is_vector_type : false_type {};
1405 
1406 template <typename _Tp>
1407  struct __is_vector_type<
1408  _Tp, void_t<typename __vector_type<
1409  remove_reference_t<decltype(declval<_Tp>()[0])>, sizeof(_Tp)>::type>>
1410  : is_same<_Tp, typename __vector_type<
1411  remove_reference_t<decltype(declval<_Tp>()[0])>,
1412  sizeof(_Tp)>::type> {};
1413 
1414 template <typename _Tp>
1415  inline constexpr bool __is_vector_type_v = __is_vector_type<_Tp>::value;
1416 
1417 // }}}
1418 // __is_intrinsic_type {{{
1419 #if _GLIBCXX_SIMD_HAVE_SSE_ABI
1420 template <typename _Tp>
1421  using __is_intrinsic_type = __is_vector_type<_Tp>;
1422 #else // not SSE (x86)
1423 template <typename _Tp, typename = void_t<>>
1424  struct __is_intrinsic_type : false_type {};
1425 
1426 template <typename _Tp>
1427  struct __is_intrinsic_type<
1428  _Tp, void_t<typename __intrinsic_type<
1429  remove_reference_t<decltype(declval<_Tp>()[0])>, sizeof(_Tp)>::type>>
1430  : is_same<_Tp, typename __intrinsic_type<
1431  remove_reference_t<decltype(declval<_Tp>()[0])>,
1432  sizeof(_Tp)>::type> {};
1433 #endif
1434 
1435 template <typename _Tp>
1436  inline constexpr bool __is_intrinsic_type_v = __is_intrinsic_type<_Tp>::value;
1437 
1438 // }}}
1439 // _VectorTraits{{{
1440 template <typename _Tp, typename = void_t<>>
1441  struct _VectorTraitsImpl;
1442 
1443 template <typename _Tp>
1444  struct _VectorTraitsImpl<_Tp, enable_if_t<__is_vector_type_v<_Tp>
1445  || __is_intrinsic_type_v<_Tp>>>
1446  {
1447  using type = _Tp;
1448  using value_type = remove_reference_t<decltype(declval<_Tp>()[0])>;
1449  static constexpr int _S_full_size = sizeof(_Tp) / sizeof(value_type);
1450  using _Wrapper = _SimdWrapper<value_type, _S_full_size>;
1451  template <typename _Up, int _W = _S_full_size>
1452  static constexpr bool _S_is
1453  = is_same_v<value_type, _Up> && _W == _S_full_size;
1454  };
1455 
1456 template <typename _Tp, size_t _Np>
1457  struct _VectorTraitsImpl<_SimdWrapper<_Tp, _Np>,
1458  void_t<__vector_type_t<_Tp, _Np>>>
1459  {
1460  using type = __vector_type_t<_Tp, _Np>;
1461  using value_type = _Tp;
1462  static constexpr int _S_full_size = sizeof(type) / sizeof(value_type);
1463  using _Wrapper = _SimdWrapper<_Tp, _Np>;
1464  static constexpr bool _S_is_partial = (_Np == _S_full_size);
1465  static constexpr int _S_partial_width = _Np;
1466  template <typename _Up, int _W = _S_full_size>
1467  static constexpr bool _S_is
1468  = is_same_v<value_type, _Up>&& _W == _S_full_size;
1469  };
1470 
1471 template <typename _Tp, typename = typename _VectorTraitsImpl<_Tp>::type>
1472  using _VectorTraits = _VectorTraitsImpl<_Tp>;
1473 
1474 // }}}
1475 // __as_vector{{{
1476 template <typename _V>
1477  _GLIBCXX_SIMD_INTRINSIC constexpr auto
1478  __as_vector(_V __x)
1479  {
1480  if constexpr (__is_vector_type_v<_V>)
1481  return __x;
1482  else if constexpr (is_simd<_V>::value || is_simd_mask<_V>::value)
1483  return __data(__x)._M_data;
1484  else if constexpr (__is_vectorizable_v<_V>)
1485  return __vector_type_t<_V, 2>{__x};
1486  else
1487  return __x._M_data;
1488  }
1489 
1490 // }}}
1491 // __as_wrapper{{{
1492 template <size_t _Np = 0, typename _V>
1493  _GLIBCXX_SIMD_INTRINSIC constexpr auto
1494  __as_wrapper(_V __x)
1495  {
1496  if constexpr (__is_vector_type_v<_V>)
1497  return _SimdWrapper<typename _VectorTraits<_V>::value_type,
1498  (_Np > 0 ? _Np : _VectorTraits<_V>::_S_full_size)>(__x);
1499  else if constexpr (is_simd<_V>::value || is_simd_mask<_V>::value)
1500  {
1501  static_assert(_V::size() == _Np);
1502  return __data(__x);
1503  }
1504  else
1505  {
1506  static_assert(_V::_S_size == _Np);
1507  return __x;
1508  }
1509  }
1510 
1511 // }}}
1512 // __intrin_bitcast{{{
1513 template <typename _To, typename _From>
1514  _GLIBCXX_SIMD_INTRINSIC constexpr _To
1515  __intrin_bitcast(_From __v)
1516  {
1517  static_assert((__is_vector_type_v<_From> || __is_intrinsic_type_v<_From>)
1518  && (__is_vector_type_v<_To> || __is_intrinsic_type_v<_To>));
1519  if constexpr (sizeof(_To) == sizeof(_From))
1520  return reinterpret_cast<_To>(__v);
1521  else if constexpr (sizeof(_From) > sizeof(_To))
1522  if constexpr (sizeof(_To) >= 16)
1523  return reinterpret_cast<const __may_alias<_To>&>(__v);
1524  else
1525  {
1526  _To __r;
1527  __builtin_memcpy(&__r, &__v, sizeof(_To));
1528  return __r;
1529  }
1530 #if _GLIBCXX_SIMD_X86INTRIN && !defined __clang__
1531  else if constexpr (__have_avx && sizeof(_From) == 16 && sizeof(_To) == 32)
1532  return reinterpret_cast<_To>(__builtin_ia32_ps256_ps(
1533  reinterpret_cast<__vector_type_t<float, 4>>(__v)));
1534  else if constexpr (__have_avx512f && sizeof(_From) == 16
1535  && sizeof(_To) == 64)
1536  return reinterpret_cast<_To>(__builtin_ia32_ps512_ps(
1537  reinterpret_cast<__vector_type_t<float, 4>>(__v)));
1538  else if constexpr (__have_avx512f && sizeof(_From) == 32
1539  && sizeof(_To) == 64)
1540  return reinterpret_cast<_To>(__builtin_ia32_ps512_256ps(
1541  reinterpret_cast<__vector_type_t<float, 8>>(__v)));
1542 #endif // _GLIBCXX_SIMD_X86INTRIN
1543  else if constexpr (sizeof(__v) <= 8)
1544  return reinterpret_cast<_To>(
1545  __vector_type_t<__int_for_sizeof_t<_From>, sizeof(_To) / sizeof(_From)>{
1546  reinterpret_cast<__int_for_sizeof_t<_From>>(__v)});
1547  else
1548  {
1549  static_assert(sizeof(_To) > sizeof(_From));
1550  _To __r = {};
1551  __builtin_memcpy(&__r, &__v, sizeof(_From));
1552  return __r;
1553  }
1554  }
1555 
1556 // }}}
1557 // __vector_bitcast{{{
1558 template <typename _To, size_t _NN = 0, typename _From,
1559  typename _FromVT = _VectorTraits<_From>,
1560  size_t _Np = _NN == 0 ? sizeof(_From) / sizeof(_To) : _NN>
1561  _GLIBCXX_SIMD_INTRINSIC constexpr __vector_type_t<_To, _Np>
1562  __vector_bitcast(_From __x)
1563  {
1564  using _R = __vector_type_t<_To, _Np>;
1565  return __intrin_bitcast<_R>(__x);
1566  }
1567 
1568 template <typename _To, size_t _NN = 0, typename _Tp, size_t _Nx,
1569  size_t _Np
1570  = _NN == 0 ? sizeof(_SimdWrapper<_Tp, _Nx>) / sizeof(_To) : _NN>
1571  _GLIBCXX_SIMD_INTRINSIC constexpr __vector_type_t<_To, _Np>
1572  __vector_bitcast(const _SimdWrapper<_Tp, _Nx>& __x)
1573  {
1574  static_assert(_Np > 1);
1575  return __intrin_bitcast<__vector_type_t<_To, _Np>>(__x._M_data);
1576  }
1577 
1578 // }}}
1579 // __convert_x86 declarations {{{
1580 #ifdef _GLIBCXX_SIMD_WORKAROUND_PR85048
1581 template <typename _To, typename _Tp, typename _TVT = _VectorTraits<_Tp>>
1582  _To __convert_x86(_Tp);
1583 
1584 template <typename _To, typename _Tp, typename _TVT = _VectorTraits<_Tp>>
1585  _To __convert_x86(_Tp, _Tp);
1586 
1587 template <typename _To, typename _Tp, typename _TVT = _VectorTraits<_Tp>>
1588  _To __convert_x86(_Tp, _Tp, _Tp, _Tp);
1589 
1590 template <typename _To, typename _Tp, typename _TVT = _VectorTraits<_Tp>>
1591  _To __convert_x86(_Tp, _Tp, _Tp, _Tp, _Tp, _Tp, _Tp, _Tp);
1592 
1593 template <typename _To, typename _Tp, typename _TVT = _VectorTraits<_Tp>>
1594  _To __convert_x86(_Tp, _Tp, _Tp, _Tp, _Tp, _Tp, _Tp, _Tp, _Tp, _Tp, _Tp, _Tp,
1595  _Tp, _Tp, _Tp, _Tp);
1596 #endif // _GLIBCXX_SIMD_WORKAROUND_PR85048
1597 
1598 //}}}
1599 // __bit_cast {{{
1600 template <typename _To, typename _From>
1601  _GLIBCXX_SIMD_INTRINSIC constexpr _To
1602  __bit_cast(const _From __x)
1603  {
1604  // TODO: implement with / replace by __builtin_bit_cast ASAP
1605  static_assert(sizeof(_To) == sizeof(_From));
1606  constexpr bool __to_is_vectorizable
1607  = is_arithmetic_v<_To> || is_enum_v<_To>;
1608  constexpr bool __from_is_vectorizable
1609  = is_arithmetic_v<_From> || is_enum_v<_From>;
1610  if constexpr (__is_vector_type_v<_To> && __is_vector_type_v<_From>)
1611  return reinterpret_cast<_To>(__x);
1612  else if constexpr (__is_vector_type_v<_To> && __from_is_vectorizable)
1613  {
1614  using _FV [[gnu::vector_size(sizeof(_From))]] = _From;
1615  return reinterpret_cast<_To>(_FV{__x});
1616  }
1617  else if constexpr (__to_is_vectorizable && __from_is_vectorizable)
1618  {
1619  using _TV [[gnu::vector_size(sizeof(_To))]] = _To;
1620  using _FV [[gnu::vector_size(sizeof(_From))]] = _From;
1621  return reinterpret_cast<_TV>(_FV{__x})[0];
1622  }
1623  else if constexpr (__to_is_vectorizable && __is_vector_type_v<_From>)
1624  {
1625  using _TV [[gnu::vector_size(sizeof(_To))]] = _To;
1626  return reinterpret_cast<_TV>(__x)[0];
1627  }
1628  else
1629  {
1630  _To __r;
1631  __builtin_memcpy(reinterpret_cast<char*>(&__r),
1632  reinterpret_cast<const char*>(&__x), sizeof(_To));
1633  return __r;
1634  }
1635  }
1636 
1637 // }}}
1638 // __to_intrin {{{
1639 template <typename _Tp, typename _TVT = _VectorTraits<_Tp>,
1640  typename _R
1641  = __intrinsic_type_t<typename _TVT::value_type, _TVT::_S_full_size>>
1642  _GLIBCXX_SIMD_INTRINSIC constexpr _R
1643  __to_intrin(_Tp __x)
1644  {
1645  static_assert(sizeof(__x) <= sizeof(_R),
1646  "__to_intrin may never drop values off the end");
1647  if constexpr (sizeof(__x) == sizeof(_R))
1648  return reinterpret_cast<_R>(__as_vector(__x));
1649  else
1650  {
1651  using _Up = __int_for_sizeof_t<_Tp>;
1652  return reinterpret_cast<_R>(
1653  __vector_type_t<_Up, sizeof(_R) / sizeof(_Up)>{__bit_cast<_Up>(__x)});
1654  }
1655  }
1656 
1657 // }}}
1658 // __make_vector{{{
1659 template <typename _Tp, typename... _Args>
1660  _GLIBCXX_SIMD_INTRINSIC constexpr __vector_type_t<_Tp, sizeof...(_Args)>
1661  __make_vector(const _Args&... __args)
1662  {
1663  return __vector_type_t<_Tp, sizeof...(_Args)>{static_cast<_Tp>(__args)...};
1664  }
1665 
1666 // }}}
1667 // __vector_broadcast{{{
1668 template <size_t _Np, typename _Tp>
1669  _GLIBCXX_SIMD_INTRINSIC constexpr __vector_type_t<_Tp, _Np>
1670  __vector_broadcast(_Tp __x)
1671  {
1672  return __call_with_n_evaluations<_Np>(
1673  [](auto... __xx) { return __vector_type_t<_Tp, _Np>{__xx...}; },
1674  [&__x](int) { return __x; });
1675  }
1676 
1677 // }}}
1678 // __generate_vector{{{
1679  template <typename _Tp, size_t _Np, typename _Gp, size_t... _I>
1680  _GLIBCXX_SIMD_INTRINSIC constexpr __vector_type_t<_Tp, _Np>
1681  __generate_vector_impl(_Gp&& __gen, index_sequence<_I...>)
1682  {
1683  return __vector_type_t<_Tp, _Np>{
1684  static_cast<_Tp>(__gen(_SizeConstant<_I>()))...};
1685  }
1686 
1687 template <typename _V, typename _VVT = _VectorTraits<_V>, typename _Gp>
1688  _GLIBCXX_SIMD_INTRINSIC constexpr _V
1689  __generate_vector(_Gp&& __gen)
1690  {
1691  if constexpr (__is_vector_type_v<_V>)
1692  return __generate_vector_impl<typename _VVT::value_type,
1693  _VVT::_S_full_size>(
1694  static_cast<_Gp&&>(__gen), make_index_sequence<_VVT::_S_full_size>());
1695  else
1696  return __generate_vector_impl<typename _VVT::value_type,
1697  _VVT::_S_partial_width>(
1698  static_cast<_Gp&&>(__gen),
1699  make_index_sequence<_VVT::_S_partial_width>());
1700  }
1701 
1702 template <typename _Tp, size_t _Np, typename _Gp>
1703  _GLIBCXX_SIMD_INTRINSIC constexpr __vector_type_t<_Tp, _Np>
1704  __generate_vector(_Gp&& __gen)
1705  {
1706  return __generate_vector_impl<_Tp, _Np>(static_cast<_Gp&&>(__gen),
1707  make_index_sequence<_Np>());
1708  }
1709 
1710 // }}}
1711 // __xor{{{
1712 template <typename _TW>
1713  _GLIBCXX_SIMD_INTRINSIC constexpr _TW
1714  __xor(_TW __a, _TW __b) noexcept
1715  {
1716  if constexpr (__is_vector_type_v<_TW> || __is_simd_wrapper_v<_TW>)
1717  {
1718  using _Tp = typename conditional_t<__is_simd_wrapper_v<_TW>, _TW,
1719  _VectorTraitsImpl<_TW>>::value_type;
1720  if constexpr (is_floating_point_v<_Tp>)
1721  {
1722  using _Ip = make_unsigned_t<__int_for_sizeof_t<_Tp>>;
1723  return __vector_bitcast<_Tp>(__vector_bitcast<_Ip>(__a)
1724  ^ __vector_bitcast<_Ip>(__b));
1725  }
1726  else if constexpr (__is_vector_type_v<_TW>)
1727  return __a ^ __b;
1728  else
1729  return __a._M_data ^ __b._M_data;
1730  }
1731  else
1732  return __a ^ __b;
1733  }
1734 
1735 // }}}
1736 // __or{{{
1737 template <typename _TW>
1738  _GLIBCXX_SIMD_INTRINSIC constexpr _TW
1739  __or(_TW __a, _TW __b) noexcept
1740  {
1741  if constexpr (__is_vector_type_v<_TW> || __is_simd_wrapper_v<_TW>)
1742  {
1743  using _Tp = typename conditional_t<__is_simd_wrapper_v<_TW>, _TW,
1744  _VectorTraitsImpl<_TW>>::value_type;
1745  if constexpr (is_floating_point_v<_Tp>)
1746  {
1747  using _Ip = make_unsigned_t<__int_for_sizeof_t<_Tp>>;
1748  return __vector_bitcast<_Tp>(__vector_bitcast<_Ip>(__a)
1749  | __vector_bitcast<_Ip>(__b));
1750  }
1751  else if constexpr (__is_vector_type_v<_TW>)
1752  return __a | __b;
1753  else
1754  return __a._M_data | __b._M_data;
1755  }
1756  else
1757  return __a | __b;
1758  }
1759 
1760 // }}}
1761 // __and{{{
1762 template <typename _TW>
1763  _GLIBCXX_SIMD_INTRINSIC constexpr _TW
1764  __and(_TW __a, _TW __b) noexcept
1765  {
1766  if constexpr (__is_vector_type_v<_TW> || __is_simd_wrapper_v<_TW>)
1767  {
1768  using _Tp = typename conditional_t<__is_simd_wrapper_v<_TW>, _TW,
1769  _VectorTraitsImpl<_TW>>::value_type;
1770  if constexpr (is_floating_point_v<_Tp>)
1771  {
1772  using _Ip = make_unsigned_t<__int_for_sizeof_t<_Tp>>;
1773  return __vector_bitcast<_Tp>(__vector_bitcast<_Ip>(__a)
1774  & __vector_bitcast<_Ip>(__b));
1775  }
1776  else if constexpr (__is_vector_type_v<_TW>)
1777  return __a & __b;
1778  else
1779  return __a._M_data & __b._M_data;
1780  }
1781  else
1782  return __a & __b;
1783  }
1784 
1785 // }}}
1786 // __andnot{{{
1787 #if _GLIBCXX_SIMD_X86INTRIN && !defined __clang__
1788 static constexpr struct
1789 {
1790  _GLIBCXX_SIMD_INTRINSIC __v4sf
1791  operator()(__v4sf __a, __v4sf __b) const noexcept
1792  { return __builtin_ia32_andnps(__a, __b); }
1793 
1794  _GLIBCXX_SIMD_INTRINSIC __v2df
1795  operator()(__v2df __a, __v2df __b) const noexcept
1796  { return __builtin_ia32_andnpd(__a, __b); }
1797 
1798  _GLIBCXX_SIMD_INTRINSIC __v2di
1799  operator()(__v2di __a, __v2di __b) const noexcept
1800  { return __builtin_ia32_pandn128(__a, __b); }
1801 
1802  _GLIBCXX_SIMD_INTRINSIC __v8sf
1803  operator()(__v8sf __a, __v8sf __b) const noexcept
1804  { return __builtin_ia32_andnps256(__a, __b); }
1805 
1806  _GLIBCXX_SIMD_INTRINSIC __v4df
1807  operator()(__v4df __a, __v4df __b) const noexcept
1808  { return __builtin_ia32_andnpd256(__a, __b); }
1809 
1810  _GLIBCXX_SIMD_INTRINSIC __v4di
1811  operator()(__v4di __a, __v4di __b) const noexcept
1812  {
1813  if constexpr (__have_avx2)
1814  return __builtin_ia32_andnotsi256(__a, __b);
1815  else
1816  return reinterpret_cast<__v4di>(
1817  __builtin_ia32_andnpd256(reinterpret_cast<__v4df>(__a),
1818  reinterpret_cast<__v4df>(__b)));
1819  }
1820 
1821  _GLIBCXX_SIMD_INTRINSIC __v16sf
1822  operator()(__v16sf __a, __v16sf __b) const noexcept
1823  {
1824  if constexpr (__have_avx512dq)
1825  return _mm512_andnot_ps(__a, __b);
1826  else
1827  return reinterpret_cast<__v16sf>(
1828  _mm512_andnot_si512(reinterpret_cast<__v8di>(__a),
1829  reinterpret_cast<__v8di>(__b)));
1830  }
1831 
1832  _GLIBCXX_SIMD_INTRINSIC __v8df
1833  operator()(__v8df __a, __v8df __b) const noexcept
1834  {
1835  if constexpr (__have_avx512dq)
1836  return _mm512_andnot_pd(__a, __b);
1837  else
1838  return reinterpret_cast<__v8df>(
1839  _mm512_andnot_si512(reinterpret_cast<__v8di>(__a),
1840  reinterpret_cast<__v8di>(__b)));
1841  }
1842 
1843  _GLIBCXX_SIMD_INTRINSIC __v8di
1844  operator()(__v8di __a, __v8di __b) const noexcept
1845  { return _mm512_andnot_si512(__a, __b); }
1846 } _S_x86_andnot;
1847 #endif // _GLIBCXX_SIMD_X86INTRIN && !__clang__
1848 
1849 template <typename _TW>
1850  _GLIBCXX_SIMD_INTRINSIC constexpr _TW
1851  __andnot(_TW __a, _TW __b) noexcept
1852  {
1853  if constexpr (__is_vector_type_v<_TW> || __is_simd_wrapper_v<_TW>)
1854  {
1855  using _TVT = conditional_t<__is_simd_wrapper_v<_TW>, _TW,
1856  _VectorTraitsImpl<_TW>>;
1857  using _Tp = typename _TVT::value_type;
1858 #if _GLIBCXX_SIMD_X86INTRIN && !defined __clang__
1859  if constexpr (sizeof(_TW) >= 16)
1860  {
1861  const auto __ai = __to_intrin(__a);
1862  const auto __bi = __to_intrin(__b);
1863  if (!__builtin_is_constant_evaluated()
1864  && !(__builtin_constant_p(__ai) && __builtin_constant_p(__bi)))
1865  {
1866  const auto __r = _S_x86_andnot(__ai, __bi);
1867  if constexpr (is_convertible_v<decltype(__r), _TW>)
1868  return __r;
1869  else
1870  return reinterpret_cast<typename _TVT::type>(__r);
1871  }
1872  }
1873 #endif // _GLIBCXX_SIMD_X86INTRIN
1874  using _Ip = make_unsigned_t<__int_for_sizeof_t<_Tp>>;
1875  return __vector_bitcast<_Tp>(~__vector_bitcast<_Ip>(__a)
1876  & __vector_bitcast<_Ip>(__b));
1877  }
1878  else
1879  return ~__a & __b;
1880  }
1881 
1882 // }}}
1883 // __not{{{
1884 template <typename _Tp, typename _TVT = _VectorTraits<_Tp>>
1885  _GLIBCXX_SIMD_INTRINSIC constexpr _Tp
1886  __not(_Tp __a) noexcept
1887  {
1888  if constexpr (is_floating_point_v<typename _TVT::value_type>)
1889  return reinterpret_cast<typename _TVT::type>(
1890  ~__vector_bitcast<unsigned>(__a));
1891  else
1892  return ~__a;
1893  }
1894 
1895 // }}}
1896 // __concat{{{
1897 template <typename _Tp, typename _TVT = _VectorTraits<_Tp>,
1898  typename _R = __vector_type_t<typename _TVT::value_type,
1899  _TVT::_S_full_size * 2>>
1900  constexpr _R
1901  __concat(_Tp a_, _Tp b_)
1902  {
1903 #ifdef _GLIBCXX_SIMD_WORKAROUND_XXX_1
1904  using _W
1905  = conditional_t<is_floating_point_v<typename _TVT::value_type>, double,
1906  conditional_t<(sizeof(_Tp) >= 2 * sizeof(long long)),
1907  long long, typename _TVT::value_type>>;
1908  constexpr int input_width = sizeof(_Tp) / sizeof(_W);
1909  const auto __a = __vector_bitcast<_W>(a_);
1910  const auto __b = __vector_bitcast<_W>(b_);
1911  using _Up = __vector_type_t<_W, sizeof(_R) / sizeof(_W)>;
1912 #else
1913  constexpr int input_width = _TVT::_S_full_size;
1914  const _Tp& __a = a_;
1915  const _Tp& __b = b_;
1916  using _Up = _R;
1917 #endif
1918  if constexpr (input_width == 2)
1919  return reinterpret_cast<_R>(_Up{__a[0], __a[1], __b[0], __b[1]});
1920  else if constexpr (input_width == 4)
1921  return reinterpret_cast<_R>(
1922  _Up{__a[0], __a[1], __a[2], __a[3], __b[0], __b[1], __b[2], __b[3]});
1923  else if constexpr (input_width == 8)
1924  return reinterpret_cast<_R>(
1925  _Up{__a[0], __a[1], __a[2], __a[3], __a[4], __a[5], __a[6], __a[7],
1926  __b[0], __b[1], __b[2], __b[3], __b[4], __b[5], __b[6], __b[7]});
1927  else if constexpr (input_width == 16)
1928  return reinterpret_cast<_R>(
1929  _Up{__a[0], __a[1], __a[2], __a[3], __a[4], __a[5], __a[6],
1930  __a[7], __a[8], __a[9], __a[10], __a[11], __a[12], __a[13],
1931  __a[14], __a[15], __b[0], __b[1], __b[2], __b[3], __b[4],
1932  __b[5], __b[6], __b[7], __b[8], __b[9], __b[10], __b[11],
1933  __b[12], __b[13], __b[14], __b[15]});
1934  else if constexpr (input_width == 32)
1935  return reinterpret_cast<_R>(
1936  _Up{__a[0], __a[1], __a[2], __a[3], __a[4], __a[5], __a[6],
1937  __a[7], __a[8], __a[9], __a[10], __a[11], __a[12], __a[13],
1938  __a[14], __a[15], __a[16], __a[17], __a[18], __a[19], __a[20],
1939  __a[21], __a[22], __a[23], __a[24], __a[25], __a[26], __a[27],
1940  __a[28], __a[29], __a[30], __a[31], __b[0], __b[1], __b[2],
1941  __b[3], __b[4], __b[5], __b[6], __b[7], __b[8], __b[9],
1942  __b[10], __b[11], __b[12], __b[13], __b[14], __b[15], __b[16],
1943  __b[17], __b[18], __b[19], __b[20], __b[21], __b[22], __b[23],
1944  __b[24], __b[25], __b[26], __b[27], __b[28], __b[29], __b[30],
1945  __b[31]});
1946  }
1947 
1948 // }}}
1949 // __zero_extend {{{
1950 template <typename _Tp, typename _TVT = _VectorTraits<_Tp>>
1951  struct _ZeroExtendProxy
1952  {
1953  using value_type = typename _TVT::value_type;
1954  static constexpr size_t _Np = _TVT::_S_full_size;
1955  const _Tp __x;
1956 
1957  template <typename _To, typename _ToVT = _VectorTraits<_To>,
1958  typename
1959  = enable_if_t<is_same_v<typename _ToVT::value_type, value_type>>>
1960  _GLIBCXX_SIMD_INTRINSIC operator _To() const
1961  {
1962  constexpr size_t _ToN = _ToVT::_S_full_size;
1963  if constexpr (_ToN == _Np)
1964  return __x;
1965  else if constexpr (_ToN == 2 * _Np)
1966  {
1967 #ifdef _GLIBCXX_SIMD_WORKAROUND_XXX_3
1968  if constexpr (__have_avx && _TVT::template _S_is<float, 4>)
1969  return __vector_bitcast<value_type>(
1970  _mm256_insertf128_ps(__m256(), __x, 0));
1971  else if constexpr (__have_avx && _TVT::template _S_is<double, 2>)
1972  return __vector_bitcast<value_type>(
1973  _mm256_insertf128_pd(__m256d(), __x, 0));
1974  else if constexpr (__have_avx2 && _Np * sizeof(value_type) == 16)
1975  return __vector_bitcast<value_type>(
1976  _mm256_insertf128_si256(__m256i(), __to_intrin(__x), 0));
1977  else if constexpr (__have_avx512f && _TVT::template _S_is<float, 8>)
1978  {
1979  if constexpr (__have_avx512dq)
1980  return __vector_bitcast<value_type>(
1981  _mm512_insertf32x8(__m512(), __x, 0));
1982  else
1983  return reinterpret_cast<__m512>(
1984  _mm512_insertf64x4(__m512d(),
1985  reinterpret_cast<__m256d>(__x), 0));
1986  }
1987  else if constexpr (__have_avx512f
1988  && _TVT::template _S_is<double, 4>)
1989  return __vector_bitcast<value_type>(
1990  _mm512_insertf64x4(__m512d(), __x, 0));
1991  else if constexpr (__have_avx512f && _Np * sizeof(value_type) == 32)
1992  return __vector_bitcast<value_type>(
1993  _mm512_inserti64x4(__m512i(), __to_intrin(__x), 0));
1994 #endif
1995  return __concat(__x, _Tp());
1996  }
1997  else if constexpr (_ToN == 4 * _Np)
1998  {
1999 #ifdef _GLIBCXX_SIMD_WORKAROUND_XXX_3
2000  if constexpr (__have_avx512dq && _TVT::template _S_is<double, 2>)
2001  {
2002  return __vector_bitcast<value_type>(
2003  _mm512_insertf64x2(__m512d(), __x, 0));
2004  }
2005  else if constexpr (__have_avx512f
2006  && is_floating_point_v<value_type>)
2007  {
2008  return __vector_bitcast<value_type>(
2009  _mm512_insertf32x4(__m512(), reinterpret_cast<__m128>(__x),
2010  0));
2011  }
2012  else if constexpr (__have_avx512f && _Np * sizeof(value_type) == 16)
2013  {
2014  return __vector_bitcast<value_type>(
2015  _mm512_inserti32x4(__m512i(), __to_intrin(__x), 0));
2016  }
2017 #endif
2018  return __concat(__concat(__x, _Tp()),
2019  __vector_type_t<value_type, _Np * 2>());
2020  }
2021  else if constexpr (_ToN == 8 * _Np)
2022  return __concat(operator __vector_type_t<value_type, _Np * 4>(),
2023  __vector_type_t<value_type, _Np * 4>());
2024  else if constexpr (_ToN == 16 * _Np)
2025  return __concat(operator __vector_type_t<value_type, _Np * 8>(),
2026  __vector_type_t<value_type, _Np * 8>());
2027  else
2028  __assert_unreachable<_Tp>();
2029  }
2030  };
2031 
2032 template <typename _Tp, typename _TVT = _VectorTraits<_Tp>>
2033  _GLIBCXX_SIMD_INTRINSIC _ZeroExtendProxy<_Tp, _TVT>
2034  __zero_extend(_Tp __x)
2035  { return {__x}; }
2036 
2037 // }}}
2038 // __extract<_Np, By>{{{
2039 template <int _Offset,
2040  int _SplitBy,
2041  typename _Tp,
2042  typename _TVT = _VectorTraits<_Tp>,
2043  typename _R = __vector_type_t<typename _TVT::value_type,
2044  _TVT::_S_full_size / _SplitBy>>
2045  _GLIBCXX_SIMD_INTRINSIC constexpr _R
2046  __extract(_Tp __in)
2047  {
2048  using value_type = typename _TVT::value_type;
2049 #if _GLIBCXX_SIMD_X86INTRIN // {{{
2050  if constexpr (sizeof(_Tp) == 64 && _SplitBy == 4 && _Offset > 0)
2051  {
2052  if constexpr (__have_avx512dq && is_same_v<double, value_type>)
2053  return _mm512_extractf64x2_pd(__to_intrin(__in), _Offset);
2054  else if constexpr (is_floating_point_v<value_type>)
2055  return __vector_bitcast<value_type>(
2056  _mm512_extractf32x4_ps(__intrin_bitcast<__m512>(__in), _Offset));
2057  else
2058  return reinterpret_cast<_R>(
2059  _mm512_extracti32x4_epi32(__intrin_bitcast<__m512i>(__in),
2060  _Offset));
2061  }
2062  else
2063 #endif // _GLIBCXX_SIMD_X86INTRIN }}}
2064  {
2065 #ifdef _GLIBCXX_SIMD_WORKAROUND_XXX_1
2066  using _W = conditional_t<
2067  is_floating_point_v<value_type>, double,
2068  conditional_t<(sizeof(_R) >= 16), long long, value_type>>;
2069  static_assert(sizeof(_R) % sizeof(_W) == 0);
2070  constexpr int __return_width = sizeof(_R) / sizeof(_W);
2071  using _Up = __vector_type_t<_W, __return_width>;
2072  const auto __x = __vector_bitcast<_W>(__in);
2073 #else
2074  constexpr int __return_width = _TVT::_S_full_size / _SplitBy;
2075  using _Up = _R;
2076  const __vector_type_t<value_type, _TVT::_S_full_size>& __x
2077  = __in; // only needed for _Tp = _SimdWrapper<value_type, _Np>
2078 #endif
2079  constexpr int _O = _Offset * __return_width;
2080  return __call_with_subscripts<__return_width, _O>(
2081  __x, [](auto... __entries) {
2082  return reinterpret_cast<_R>(_Up{__entries...});
2083  });
2084  }
2085  }
2086 
2087 // }}}
2088 // __lo/__hi64[z]{{{
2089 template <typename _Tp,
2090  typename _R
2091  = __vector_type8_t<typename _VectorTraits<_Tp>::value_type>>
2092  _GLIBCXX_SIMD_INTRINSIC constexpr _R
2093  __lo64(_Tp __x)
2094  {
2095  _R __r{};
2096  __builtin_memcpy(&__r, &__x, 8);
2097  return __r;
2098  }
2099 
2100 template <typename _Tp,
2101  typename _R
2102  = __vector_type8_t<typename _VectorTraits<_Tp>::value_type>>
2103  _GLIBCXX_SIMD_INTRINSIC constexpr _R
2104  __hi64(_Tp __x)
2105  {
2106  static_assert(sizeof(_Tp) == 16, "use __hi64z if you meant it");
2107  _R __r{};
2108  __builtin_memcpy(&__r, reinterpret_cast<const char*>(&__x) + 8, 8);
2109  return __r;
2110  }
2111 
2112 template <typename _Tp,
2113  typename _R
2114  = __vector_type8_t<typename _VectorTraits<_Tp>::value_type>>
2115  _GLIBCXX_SIMD_INTRINSIC constexpr _R
2116  __hi64z([[maybe_unused]] _Tp __x)
2117  {
2118  _R __r{};
2119  if constexpr (sizeof(_Tp) == 16)
2120  __builtin_memcpy(&__r, reinterpret_cast<const char*>(&__x) + 8, 8);
2121  return __r;
2122  }
2123 
2124 // }}}
2125 // __lo/__hi128{{{
2126 template <typename _Tp>
2127  _GLIBCXX_SIMD_INTRINSIC constexpr auto
2128  __lo128(_Tp __x)
2129  { return __extract<0, sizeof(_Tp) / 16>(__x); }
2130 
2131 template <typename _Tp>
2132  _GLIBCXX_SIMD_INTRINSIC constexpr auto
2133  __hi128(_Tp __x)
2134  {
2135  static_assert(sizeof(__x) == 32);
2136  return __extract<1, 2>(__x);
2137  }
2138 
2139 // }}}
2140 // __lo/__hi256{{{
2141 template <typename _Tp>
2142  _GLIBCXX_SIMD_INTRINSIC constexpr auto
2143  __lo256(_Tp __x)
2144  {
2145  static_assert(sizeof(__x) == 64);
2146  return __extract<0, 2>(__x);
2147  }
2148 
2149 template <typename _Tp>
2150  _GLIBCXX_SIMD_INTRINSIC constexpr auto
2151  __hi256(_Tp __x)
2152  {
2153  static_assert(sizeof(__x) == 64);
2154  return __extract<1, 2>(__x);
2155  }
2156 
2157 // }}}
2158 // __auto_bitcast{{{
2159 template <typename _Tp>
2160  struct _AutoCast
2161  {
2162  static_assert(__is_vector_type_v<_Tp>);
2163 
2164  const _Tp __x;
2165 
2166  template <typename _Up, typename _UVT = _VectorTraits<_Up>>
2167  _GLIBCXX_SIMD_INTRINSIC constexpr operator _Up() const
2168  { return __intrin_bitcast<typename _UVT::type>(__x); }
2169  };
2170 
2171 template <typename _Tp>
2172  _GLIBCXX_SIMD_INTRINSIC constexpr _AutoCast<_Tp>
2173  __auto_bitcast(const _Tp& __x)
2174  { return {__x}; }
2175 
2176 template <typename _Tp, size_t _Np>
2177  _GLIBCXX_SIMD_INTRINSIC constexpr
2178  _AutoCast<typename _SimdWrapper<_Tp, _Np>::_BuiltinType>
2179  __auto_bitcast(const _SimdWrapper<_Tp, _Np>& __x)
2180  { return {__x._M_data}; }
2181 
2182 // }}}
2183 // ^^^ ---- builtin vector types [[gnu::vector_size(N)]] and operations ---- ^^^
2184 
2185 #if _GLIBCXX_SIMD_HAVE_SSE_ABI
2186 // __bool_storage_member_type{{{
2187 #if _GLIBCXX_SIMD_HAVE_AVX512F && _GLIBCXX_SIMD_X86INTRIN
2188 template <size_t _Size>
2189  struct __bool_storage_member_type
2190  {
2191  static_assert((_Size & (_Size - 1)) != 0,
2192  "This trait may only be used for non-power-of-2 sizes. "
2193  "Power-of-2 sizes must be specialized.");
2194  using type =
2195  typename __bool_storage_member_type<std::__bit_ceil(_Size)>::type;
2196  };
2197 
2198 template <>
2199  struct __bool_storage_member_type<1> { using type = bool; };
2200 
2201 template <>
2202  struct __bool_storage_member_type<2> { using type = __mmask8; };
2203 
2204 template <>
2205  struct __bool_storage_member_type<4> { using type = __mmask8; };
2206 
2207 template <>
2208  struct __bool_storage_member_type<8> { using type = __mmask8; };
2209 
2210 template <>
2211  struct __bool_storage_member_type<16> { using type = __mmask16; };
2212 
2213 template <>
2214  struct __bool_storage_member_type<32> { using type = __mmask32; };
2215 
2216 template <>
2217  struct __bool_storage_member_type<64> { using type = __mmask64; };
2218 #endif // _GLIBCXX_SIMD_HAVE_AVX512F
2219 
2220 // }}}
2221 // __intrinsic_type (x86){{{
2222 // the following excludes bool via __is_vectorizable
2223 #if _GLIBCXX_SIMD_HAVE_SSE
2224 template <typename _Tp, size_t _Bytes>
2225  struct __intrinsic_type<_Tp, _Bytes,
2226  enable_if_t<__is_vectorizable_v<_Tp> && _Bytes <= 64>>
2227  {
2228  static_assert(!is_same_v<_Tp, long double>,
2229  "no __intrinsic_type support for long double on x86");
2230 
2231  static constexpr size_t _S_VBytes = _Bytes <= 16 ? 16
2232  : _Bytes <= 32 ? 32
2233  : 64;
2234 
2235  using type [[__gnu__::__vector_size__(_S_VBytes)]]
2236  = conditional_t<is_integral_v<_Tp>, long long int, _Tp>;
2237  };
2238 #endif // _GLIBCXX_SIMD_HAVE_SSE
2239 
2240 // }}}
2241 #endif // _GLIBCXX_SIMD_HAVE_SSE_ABI
2242 // __intrinsic_type (ARM){{{
2243 #if _GLIBCXX_SIMD_HAVE_NEON
2244 template <>
2245  struct __intrinsic_type<float, 8, void>
2246  { using type = float32x2_t; };
2247 
2248 template <>
2249  struct __intrinsic_type<float, 16, void>
2250  { using type = float32x4_t; };
2251 
2252 #if _GLIBCXX_SIMD_HAVE_NEON_A64
2253 template <>
2254  struct __intrinsic_type<double, 8, void>
2255  { using type = float64x1_t; };
2256 
2257 template <>
2258  struct __intrinsic_type<double, 16, void>
2259  { using type = float64x2_t; };
2260 #endif
2261 
2262 #define _GLIBCXX_SIMD_ARM_INTRIN(_Bits, _Np) \
2263 template <> \
2264  struct __intrinsic_type<__int_with_sizeof_t<_Bits / 8>, \
2265  _Np * _Bits / 8, void> \
2266  { using type = int##_Bits##x##_Np##_t; }; \
2267 template <> \
2268  struct __intrinsic_type<make_unsigned_t<__int_with_sizeof_t<_Bits / 8>>, \
2269  _Np * _Bits / 8, void> \
2270  { using type = uint##_Bits##x##_Np##_t; }
2271 _GLIBCXX_SIMD_ARM_INTRIN(8, 8);
2272 _GLIBCXX_SIMD_ARM_INTRIN(8, 16);
2273 _GLIBCXX_SIMD_ARM_INTRIN(16, 4);
2274 _GLIBCXX_SIMD_ARM_INTRIN(16, 8);
2275 _GLIBCXX_SIMD_ARM_INTRIN(32, 2);
2276 _GLIBCXX_SIMD_ARM_INTRIN(32, 4);
2277 _GLIBCXX_SIMD_ARM_INTRIN(64, 1);
2278 _GLIBCXX_SIMD_ARM_INTRIN(64, 2);
2279 #undef _GLIBCXX_SIMD_ARM_INTRIN
2280 
2281 template <typename _Tp, size_t _Bytes>
2282  struct __intrinsic_type<_Tp, _Bytes,
2283  enable_if_t<__is_vectorizable_v<_Tp> && _Bytes <= 16>>
2284  {
2285  static constexpr int _SVecBytes = _Bytes <= 8 ? 8 : 16;
2286  using _Ip = __int_for_sizeof_t<_Tp>;
2287  using _Up = conditional_t<
2288  is_floating_point_v<_Tp>, _Tp,
2289  conditional_t<is_unsigned_v<_Tp>, make_unsigned_t<_Ip>, _Ip>>;
2290  static_assert(!is_same_v<_Tp, _Up> || _SVecBytes != _Bytes,
2291  "should use explicit specialization above");
2292  using type = typename __intrinsic_type<_Up, _SVecBytes>::type;
2293  };
2294 #endif // _GLIBCXX_SIMD_HAVE_NEON
2295 
2296 // }}}
2297 // __intrinsic_type (PPC){{{
2298 #ifdef __ALTIVEC__
2299 template <typename _Tp>
2300  struct __intrinsic_type_impl;
2301 
2302 #define _GLIBCXX_SIMD_PPC_INTRIN(_Tp) \
2303  template <> \
2304  struct __intrinsic_type_impl<_Tp> { using type = __vector _Tp; }
2305 _GLIBCXX_SIMD_PPC_INTRIN(float);
2306 #ifdef __VSX__
2307 _GLIBCXX_SIMD_PPC_INTRIN(double);
2308 #endif
2309 _GLIBCXX_SIMD_PPC_INTRIN(signed char);
2310 _GLIBCXX_SIMD_PPC_INTRIN(unsigned char);
2311 _GLIBCXX_SIMD_PPC_INTRIN(signed short);
2312 _GLIBCXX_SIMD_PPC_INTRIN(unsigned short);
2313 _GLIBCXX_SIMD_PPC_INTRIN(signed int);
2314 _GLIBCXX_SIMD_PPC_INTRIN(unsigned int);
2315 #if defined __VSX__ || __SIZEOF_LONG__ == 4
2316 _GLIBCXX_SIMD_PPC_INTRIN(signed long);
2317 _GLIBCXX_SIMD_PPC_INTRIN(unsigned long);
2318 #endif
2319 #ifdef __VSX__
2320 _GLIBCXX_SIMD_PPC_INTRIN(signed long long);
2321 _GLIBCXX_SIMD_PPC_INTRIN(unsigned long long);
2322 #endif
2323 #undef _GLIBCXX_SIMD_PPC_INTRIN
2324 
2325 template <typename _Tp, size_t _Bytes>
2326  struct __intrinsic_type<_Tp, _Bytes,
2327  enable_if_t<__is_vectorizable_v<_Tp> && _Bytes <= 16>>
2328  {
2329  static constexpr bool _S_is_ldouble = is_same_v<_Tp, long double>;
2330  // allow _Tp == long double with -mlong-double-64
2331  static_assert(!(_S_is_ldouble && sizeof(long double) > sizeof(double)),
2332  "no __intrinsic_type support for 128-bit floating point on PowerPC");
2333 #ifndef __VSX__
2334  static_assert(!(is_same_v<_Tp, double>
2335  || (_S_is_ldouble && sizeof(long double) == sizeof(double))),
2336  "no __intrinsic_type support for 64-bit floating point on PowerPC w/o VSX");
2337 #endif
2338  using type =
2339  typename __intrinsic_type_impl<
2340  conditional_t<is_floating_point_v<_Tp>,
2341  conditional_t<_S_is_ldouble, double, _Tp>,
2342  __int_for_sizeof_t<_Tp>>>::type;
2343  };
2344 #endif // __ALTIVEC__
2345 
2346 // }}}
2347 // _SimdWrapper<bool>{{{1
2348 template <size_t _Width>
2349  struct _SimdWrapper<bool, _Width,
2350  void_t<typename __bool_storage_member_type<_Width>::type>>
2351  {
2352  using _BuiltinType = typename __bool_storage_member_type<_Width>::type;
2353  using value_type = bool;
2354 
2355  static constexpr size_t _S_full_size = sizeof(_BuiltinType) * __CHAR_BIT__;
2356 
2357  _GLIBCXX_SIMD_INTRINSIC constexpr _SimdWrapper<bool, _S_full_size>
2358  __as_full_vector() const { return _M_data; }
2359 
2360  _GLIBCXX_SIMD_INTRINSIC constexpr _SimdWrapper() = default;
2361  _GLIBCXX_SIMD_INTRINSIC constexpr _SimdWrapper(_BuiltinType __k)
2362  : _M_data(__k) {};
2363 
2364  _GLIBCXX_SIMD_INTRINSIC operator const _BuiltinType&() const
2365  { return _M_data; }
2366 
2367  _GLIBCXX_SIMD_INTRINSIC operator _BuiltinType&()
2368  { return _M_data; }
2369 
2370  _GLIBCXX_SIMD_INTRINSIC _BuiltinType __intrin() const
2371  { return _M_data; }
2372 
2373  _GLIBCXX_SIMD_INTRINSIC constexpr value_type operator[](size_t __i) const
2374  { return _M_data & (_BuiltinType(1) << __i); }
2375 
2376  template <size_t __i>
2377  _GLIBCXX_SIMD_INTRINSIC constexpr value_type
2378  operator[](_SizeConstant<__i>) const
2379  { return _M_data & (_BuiltinType(1) << __i); }
2380 
2381  _GLIBCXX_SIMD_INTRINSIC constexpr void _M_set(size_t __i, value_type __x)
2382  {
2383  if (__x)
2384  _M_data |= (_BuiltinType(1) << __i);
2385  else
2386  _M_data &= ~(_BuiltinType(1) << __i);
2387  }
2388 
2389  _GLIBCXX_SIMD_INTRINSIC
2390  constexpr bool _M_is_constprop() const
2391  { return __builtin_constant_p(_M_data); }
2392 
2393  _GLIBCXX_SIMD_INTRINSIC constexpr bool _M_is_constprop_none_of() const
2394  {
2395  if (__builtin_constant_p(_M_data))
2396  {
2397  constexpr int __nbits = sizeof(_BuiltinType) * __CHAR_BIT__;
2398  constexpr _BuiltinType __active_mask
2399  = ~_BuiltinType() >> (__nbits - _Width);
2400  return (_M_data & __active_mask) == 0;
2401  }
2402  return false;
2403  }
2404 
2405  _GLIBCXX_SIMD_INTRINSIC constexpr bool _M_is_constprop_all_of() const
2406  {
2407  if (__builtin_constant_p(_M_data))
2408  {
2409  constexpr int __nbits = sizeof(_BuiltinType) * __CHAR_BIT__;
2410  constexpr _BuiltinType __active_mask
2411  = ~_BuiltinType() >> (__nbits - _Width);
2412  return (_M_data & __active_mask) == __active_mask;
2413  }
2414  return false;
2415  }
2416 
2417  _BuiltinType _M_data;
2418  };
2419 
2420 // _SimdWrapperBase{{{1
2421 template <bool _MustZeroInitPadding, typename _BuiltinType>
2422  struct _SimdWrapperBase;
2423 
2424 template <typename _BuiltinType>
2425  struct _SimdWrapperBase<false, _BuiltinType> // no padding or no SNaNs
2426  {
2427  _GLIBCXX_SIMD_INTRINSIC constexpr _SimdWrapperBase() = default;
2428  _GLIBCXX_SIMD_INTRINSIC constexpr _SimdWrapperBase(_BuiltinType __init)
2429  : _M_data(__init)
2430  {}
2431 
2432  _BuiltinType _M_data;
2433  };
2434 
2435 template <typename _BuiltinType>
2436  struct _SimdWrapperBase<true, _BuiltinType> // with padding that needs to
2437  // never become SNaN
2438  {
2439  _GLIBCXX_SIMD_INTRINSIC constexpr _SimdWrapperBase() : _M_data() {}
2440  _GLIBCXX_SIMD_INTRINSIC constexpr _SimdWrapperBase(_BuiltinType __init)
2441  : _M_data(__init)
2442  {}
2443 
2444  _BuiltinType _M_data;
2445  };
2446 
2447 // }}}
2448 // _SimdWrapper{{{
2449 template <typename _Tp, size_t _Width>
2450  struct _SimdWrapper<
2451  _Tp, _Width,
2452  void_t<__vector_type_t<_Tp, _Width>, __intrinsic_type_t<_Tp, _Width>>>
2453  : _SimdWrapperBase<__has_iec559_behavior<__signaling_NaN, _Tp>::value
2454  && sizeof(_Tp) * _Width
2455  == sizeof(__vector_type_t<_Tp, _Width>),
2456  __vector_type_t<_Tp, _Width>>
2457  {
2458  using _Base
2459  = _SimdWrapperBase<__has_iec559_behavior<__signaling_NaN, _Tp>::value
2460  && sizeof(_Tp) * _Width
2461  == sizeof(__vector_type_t<_Tp, _Width>),
2462  __vector_type_t<_Tp, _Width>>;
2463 
2464  static_assert(__is_vectorizable_v<_Tp>);
2465  static_assert(_Width >= 2); // 1 doesn't make sense, use _Tp directly then
2466 
2467  using _BuiltinType = __vector_type_t<_Tp, _Width>;
2468  using value_type = _Tp;
2469 
2470  static inline constexpr size_t _S_full_size
2471  = sizeof(_BuiltinType) / sizeof(value_type);
2472  static inline constexpr int _S_size = _Width;
2473  static inline constexpr bool _S_is_partial = _S_full_size != _S_size;
2474 
2475  using _Base::_M_data;
2476 
2477  _GLIBCXX_SIMD_INTRINSIC constexpr _SimdWrapper<_Tp, _S_full_size>
2478  __as_full_vector() const
2479  { return _M_data; }
2480 
2481  _GLIBCXX_SIMD_INTRINSIC constexpr _SimdWrapper(initializer_list<_Tp> __init)
2482  : _Base(__generate_from_n_evaluations<_Width, _BuiltinType>(
2483  [&](auto __i) { return __init.begin()[__i.value]; })) {}
2484 
2485  _GLIBCXX_SIMD_INTRINSIC constexpr _SimdWrapper() = default;
2486  _GLIBCXX_SIMD_INTRINSIC constexpr _SimdWrapper(const _SimdWrapper&)
2487  = default;
2488  _GLIBCXX_SIMD_INTRINSIC constexpr _SimdWrapper(_SimdWrapper&&) = default;
2489 
2490  _GLIBCXX_SIMD_INTRINSIC constexpr _SimdWrapper&
2491  operator=(const _SimdWrapper&) = default;
2492  _GLIBCXX_SIMD_INTRINSIC constexpr _SimdWrapper&
2493  operator=(_SimdWrapper&&) = default;
2494 
2495  template <typename _V, typename = enable_if_t<disjunction_v<
2496  is_same<_V, __vector_type_t<_Tp, _Width>>,
2497  is_same<_V, __intrinsic_type_t<_Tp, _Width>>>>>
2498  _GLIBCXX_SIMD_INTRINSIC constexpr _SimdWrapper(_V __x)
2499  // __vector_bitcast can convert e.g. __m128 to __vector(2) float
2500  : _Base(__vector_bitcast<_Tp, _Width>(__x)) {}
2501 
2502  template <typename... _As,
2503  typename = enable_if_t<((is_same_v<simd_abi::scalar, _As> && ...)
2504  && sizeof...(_As) <= _Width)>>
2505  _GLIBCXX_SIMD_INTRINSIC constexpr
2506  operator _SimdTuple<_Tp, _As...>() const
2507  {
2508  const auto& dd = _M_data; // workaround for GCC7 ICE
2509  return __generate_from_n_evaluations<sizeof...(_As),
2510  _SimdTuple<_Tp, _As...>>([&](
2511  auto __i) constexpr { return dd[int(__i)]; });
2512  }
2513 
2514  _GLIBCXX_SIMD_INTRINSIC constexpr operator const _BuiltinType&() const
2515  { return _M_data; }
2516 
2517  _GLIBCXX_SIMD_INTRINSIC constexpr operator _BuiltinType&()
2518  { return _M_data; }
2519 
2520  _GLIBCXX_SIMD_INTRINSIC constexpr _Tp operator[](size_t __i) const
2521  { return _M_data[__i]; }
2522 
2523  template <size_t __i>
2524  _GLIBCXX_SIMD_INTRINSIC constexpr _Tp operator[](_SizeConstant<__i>) const
2525  { return _M_data[__i]; }
2526 
2527  _GLIBCXX_SIMD_INTRINSIC constexpr void _M_set(size_t __i, _Tp __x)
2528  { _M_data[__i] = __x; }
2529 
2530  _GLIBCXX_SIMD_INTRINSIC
2531  constexpr bool _M_is_constprop() const
2532  { return __builtin_constant_p(_M_data); }
2533 
2534  _GLIBCXX_SIMD_INTRINSIC constexpr bool _M_is_constprop_none_of() const
2535  {
2536  if (__builtin_constant_p(_M_data))
2537  {
2538  bool __r = true;
2539  if constexpr (is_floating_point_v<_Tp>)
2540  {
2541  using _Ip = __int_for_sizeof_t<_Tp>;
2542  const auto __intdata = __vector_bitcast<_Ip>(_M_data);
2543  __execute_n_times<_Width>(
2544  [&](auto __i) { __r &= __intdata[__i.value] == _Ip(); });
2545  }
2546  else
2547  __execute_n_times<_Width>(
2548  [&](auto __i) { __r &= _M_data[__i.value] == _Tp(); });
2549  return __r;
2550  }
2551  return false;
2552  }
2553 
2554  _GLIBCXX_SIMD_INTRINSIC constexpr bool _M_is_constprop_all_of() const
2555  {
2556  if (__builtin_constant_p(_M_data))
2557  {
2558  bool __r = true;
2559  if constexpr (is_floating_point_v<_Tp>)
2560  {
2561  using _Ip = __int_for_sizeof_t<_Tp>;
2562  const auto __intdata = __vector_bitcast<_Ip>(_M_data);
2563  __execute_n_times<_Width>(
2564  [&](auto __i) { __r &= __intdata[__i.value] == ~_Ip(); });
2565  }
2566  else
2567  __execute_n_times<_Width>(
2568  [&](auto __i) { __r &= _M_data[__i.value] == ~_Tp(); });
2569  return __r;
2570  }
2571  return false;
2572  }
2573  };
2574 
2575 // }}}
2576 
2577 // __vectorized_sizeof {{{
2578 template <typename _Tp>
2579  constexpr size_t
2580  __vectorized_sizeof()
2581  {
2582  if constexpr (!__is_vectorizable_v<_Tp>)
2583  return 0;
2584 
2585  if constexpr (sizeof(_Tp) <= 8)
2586  {
2587  // X86:
2588  if constexpr (__have_avx512bw)
2589  return 64;
2590  if constexpr (__have_avx512f && sizeof(_Tp) >= 4)
2591  return 64;
2592  if constexpr (__have_avx2)
2593  return 32;
2594  if constexpr (__have_avx && is_floating_point_v<_Tp>)
2595  return 32;
2596  if constexpr (__have_sse2)
2597  return 16;
2598  if constexpr (__have_sse && is_same_v<_Tp, float>)
2599  return 16;
2600  /* The following is too much trouble because of mixed MMX and x87 code.
2601  * While nothing here explicitly calls MMX instructions of registers,
2602  * they are still emitted but no EMMS cleanup is done.
2603  if constexpr (__have_mmx && sizeof(_Tp) <= 4 && is_integral_v<_Tp>)
2604  return 8;
2605  */
2606 
2607  // PowerPC:
2608  if constexpr (__have_power8vec
2609  || (__have_power_vmx && (sizeof(_Tp) < 8))
2610  || (__have_power_vsx && is_floating_point_v<_Tp>) )
2611  return 16;
2612 
2613  // ARM:
2614  if constexpr (__have_neon_a64
2615  || (__have_neon_a32 && !is_same_v<_Tp, double>) )
2616  return 16;
2617  if constexpr (__have_neon
2618  && sizeof(_Tp) < 8
2619  // Only allow fp if the user allows non-ICE559 fp (e.g.
2620  // via -ffast-math). ARMv7 NEON fp is not conforming to
2621  // IEC559.
2622  && (__support_neon_float || !is_floating_point_v<_Tp>))
2623  return 16;
2624  }
2625 
2626  return sizeof(_Tp);
2627  }
2628 
2629 // }}}
2630 namespace simd_abi {
2631 // most of simd_abi is defined in simd_detail.h
2632 template <typename _Tp>
2633  inline constexpr int max_fixed_size
2634  = (__have_avx512bw && sizeof(_Tp) == 1) ? 64 : 32;
2635 
2636 // compatible {{{
2637 #if defined __x86_64__ || defined __aarch64__
2638 template <typename _Tp>
2639  using compatible = conditional_t<(sizeof(_Tp) <= 8), _VecBuiltin<16>, scalar>;
2640 #elif defined __ARM_NEON
2641 // FIXME: not sure, probably needs to be scalar (or dependent on the hard-float
2642 // ABI?)
2643 template <typename _Tp>
2644  using compatible
2645  = conditional_t<(sizeof(_Tp) < 8
2646  && (__support_neon_float || !is_floating_point_v<_Tp>)),
2647  _VecBuiltin<16>, scalar>;
2648 #else
2649 template <typename>
2650  using compatible = scalar;
2651 #endif
2652 
2653 // }}}
2654 // native {{{
2655 template <typename _Tp>
2656  constexpr auto
2657  __determine_native_abi()
2658  {
2659  constexpr size_t __bytes = __vectorized_sizeof<_Tp>();
2660  if constexpr (__bytes == sizeof(_Tp))
2661  return static_cast<scalar*>(nullptr);
2662  else if constexpr (__have_avx512vl || (__have_avx512f && __bytes == 64))
2663  return static_cast<_VecBltnBtmsk<__bytes>*>(nullptr);
2664  else
2665  return static_cast<_VecBuiltin<__bytes>*>(nullptr);
2666  }
2667 
2668 template <typename _Tp, typename = enable_if_t<__is_vectorizable_v<_Tp>>>
2669  using native = remove_pointer_t<decltype(__determine_native_abi<_Tp>())>;
2670 
2671 // }}}
2672 // __default_abi {{{
2673 #if defined _GLIBCXX_SIMD_DEFAULT_ABI
2674 template <typename _Tp>
2675  using __default_abi = _GLIBCXX_SIMD_DEFAULT_ABI<_Tp>;
2676 #else
2677 template <typename _Tp>
2678  using __default_abi = compatible<_Tp>;
2679 #endif
2680 
2681 // }}}
2682 } // namespace simd_abi
2683 
2684 // traits {{{1
2685 // is_abi_tag {{{2
2686 template <typename _Tp, typename = void_t<>>
2687  struct is_abi_tag : false_type {};
2688 
2689 template <typename _Tp>
2690  struct is_abi_tag<_Tp, void_t<typename _Tp::_IsValidAbiTag>>
2691  : public _Tp::_IsValidAbiTag {};
2692 
2693 template <typename _Tp>
2694  inline constexpr bool is_abi_tag_v = is_abi_tag<_Tp>::value;
2695 
2696 // is_simd(_mask) {{{2
2697 template <typename _Tp>
2698  struct is_simd : public false_type {};
2699 
2700 template <typename _Tp>
2701  inline constexpr bool is_simd_v = is_simd<_Tp>::value;
2702 
2703 template <typename _Tp>
2704  struct is_simd_mask : public false_type {};
2705 
2706 template <typename _Tp>
2707 inline constexpr bool is_simd_mask_v = is_simd_mask<_Tp>::value;
2708 
2709 // simd_size {{{2
2710 template <typename _Tp, typename _Abi, typename = void>
2711  struct __simd_size_impl {};
2712 
2713 template <typename _Tp, typename _Abi>
2714  struct __simd_size_impl<
2715  _Tp, _Abi,
2716  enable_if_t<conjunction_v<__is_vectorizable<_Tp>, is_abi_tag<_Abi>>>>
2717  : _SizeConstant<_Abi::template _S_size<_Tp>> {};
2718 
2719 template <typename _Tp, typename _Abi = simd_abi::__default_abi<_Tp>>
2720  struct simd_size : __simd_size_impl<_Tp, _Abi> {};
2721 
2722 template <typename _Tp, typename _Abi = simd_abi::__default_abi<_Tp>>
2723  inline constexpr size_t simd_size_v = simd_size<_Tp, _Abi>::value;
2724 
2725 // simd_abi::deduce {{{2
2726 template <typename _Tp, size_t _Np, typename = void>
2727  struct __deduce_impl;
2728 
2729 namespace simd_abi {
2730 /**
2731  * @tparam _Tp The requested `value_type` for the elements.
2732  * @tparam _Np The requested number of elements.
2733  * @tparam _Abis This parameter is ignored, since this implementation cannot
2734  * make any use of it. Either __a good native ABI is matched and used as `type`
2735  * alias, or the `fixed_size<_Np>` ABI is used, which internally is built from
2736  * the best matching native ABIs.
2737  */
2738 template <typename _Tp, size_t _Np, typename...>
2739  struct deduce : __deduce_impl<_Tp, _Np> {};
2740 
2741 template <typename _Tp, size_t _Np, typename... _Abis>
2742  using deduce_t = typename deduce<_Tp, _Np, _Abis...>::type;
2743 } // namespace simd_abi
2744 
2745 // }}}2
2746 // rebind_simd {{{2
2747 template <typename _Tp, typename _V, typename = void>
2748  struct rebind_simd;
2749 
2750 template <typename _Tp, typename _Up, typename _Abi>
2751  struct rebind_simd<
2752  _Tp, simd<_Up, _Abi>,
2753  void_t<simd_abi::deduce_t<_Tp, simd_size_v<_Up, _Abi>, _Abi>>>
2754  {
2755  using type
2756  = simd<_Tp, simd_abi::deduce_t<_Tp, simd_size_v<_Up, _Abi>, _Abi>>;
2757  };
2758 
2759 template <typename _Tp, typename _Up, typename _Abi>
2760  struct rebind_simd<
2761  _Tp, simd_mask<_Up, _Abi>,
2762  void_t<simd_abi::deduce_t<_Tp, simd_size_v<_Up, _Abi>, _Abi>>>
2763  {
2764  using type
2765  = simd_mask<_Tp, simd_abi::deduce_t<_Tp, simd_size_v<_Up, _Abi>, _Abi>>;
2766  };
2767 
2768 template <typename _Tp, typename _V>
2769  using rebind_simd_t = typename rebind_simd<_Tp, _V>::type;
2770 
2771 // resize_simd {{{2
2772 template <int _Np, typename _V, typename = void>
2773  struct resize_simd;
2774 
2775 template <int _Np, typename _Tp, typename _Abi>
2776  struct resize_simd<_Np, simd<_Tp, _Abi>,
2777  void_t<simd_abi::deduce_t<_Tp, _Np, _Abi>>>
2778  { using type = simd<_Tp, simd_abi::deduce_t<_Tp, _Np, _Abi>>; };
2779 
2780 template <int _Np, typename _Tp, typename _Abi>
2781  struct resize_simd<_Np, simd_mask<_Tp, _Abi>,
2782  void_t<simd_abi::deduce_t<_Tp, _Np, _Abi>>>
2783  { using type = simd_mask<_Tp, simd_abi::deduce_t<_Tp, _Np, _Abi>>; };
2784 
2785 template <int _Np, typename _V>
2786  using resize_simd_t = typename resize_simd<_Np, _V>::type;
2787 
2788 // }}}2
2789 // memory_alignment {{{2
2790 template <typename _Tp, typename _Up = typename _Tp::value_type>
2791  struct memory_alignment
2792  : public _SizeConstant<vector_aligned_tag::_S_alignment<_Tp, _Up>> {};
2793 
2794 template <typename _Tp, typename _Up = typename _Tp::value_type>
2795  inline constexpr size_t memory_alignment_v = memory_alignment<_Tp, _Up>::value;
2796 
2797 // class template simd [simd] {{{1
2798 template <typename _Tp, typename _Abi = simd_abi::__default_abi<_Tp>>
2799  class simd;
2800 
2801 template <typename _Tp, typename _Abi>
2802  struct is_simd<simd<_Tp, _Abi>> : public true_type {};
2803 
2804 template <typename _Tp>
2805  using native_simd = simd<_Tp, simd_abi::native<_Tp>>;
2806 
2807 template <typename _Tp, int _Np>
2808  using fixed_size_simd = simd<_Tp, simd_abi::fixed_size<_Np>>;
2809 
2810 template <typename _Tp, size_t _Np>
2811  using __deduced_simd = simd<_Tp, simd_abi::deduce_t<_Tp, _Np>>;
2812 
2813 // class template simd_mask [simd_mask] {{{1
2814 template <typename _Tp, typename _Abi = simd_abi::__default_abi<_Tp>>
2815  class simd_mask;
2816 
2817 template <typename _Tp, typename _Abi>
2818  struct is_simd_mask<simd_mask<_Tp, _Abi>> : public true_type {};
2819 
2820 template <typename _Tp>
2821  using native_simd_mask = simd_mask<_Tp, simd_abi::native<_Tp>>;
2822 
2823 template <typename _Tp, int _Np>
2824  using fixed_size_simd_mask = simd_mask<_Tp, simd_abi::fixed_size<_Np>>;
2825 
2826 template <typename _Tp, size_t _Np>
2827  using __deduced_simd_mask = simd_mask<_Tp, simd_abi::deduce_t<_Tp, _Np>>;
2828 
2829 // casts [simd.casts] {{{1
2830 // static_simd_cast {{{2
2831 template <typename _Tp, typename _Up, typename _Ap, bool = is_simd_v<_Tp>,
2832  typename = void>
2833  struct __static_simd_cast_return_type;
2834 
2835 template <typename _Tp, typename _A0, typename _Up, typename _Ap>
2836  struct __static_simd_cast_return_type<simd_mask<_Tp, _A0>, _Up, _Ap, false,
2837  void>
2838  : __static_simd_cast_return_type<simd<_Tp, _A0>, _Up, _Ap> {};
2839 
2840 template <typename _Tp, typename _Up, typename _Ap>
2841  struct __static_simd_cast_return_type<
2842  _Tp, _Up, _Ap, true, enable_if_t<_Tp::size() == simd_size_v<_Up, _Ap>>>
2843  { using type = _Tp; };
2844 
2845 template <typename _Tp, typename _Ap>
2846  struct __static_simd_cast_return_type<_Tp, _Tp, _Ap, false,
2847 #ifdef _GLIBCXX_SIMD_FIX_P2TS_ISSUE66
2848  enable_if_t<__is_vectorizable_v<_Tp>>
2849 #else
2850  void
2851 #endif
2852  >
2853  { using type = simd<_Tp, _Ap>; };
2854 
2855 template <typename _Tp, typename = void>
2856  struct __safe_make_signed { using type = _Tp;};
2857 
2858 template <typename _Tp>
2859  struct __safe_make_signed<_Tp, enable_if_t<is_integral_v<_Tp>>>
2860  {
2861  // the extra make_unsigned_t is because of PR85951
2862  using type = make_signed_t<make_unsigned_t<_Tp>>;
2863  };
2864 
2865 template <typename _Tp>
2866  using safe_make_signed_t = typename __safe_make_signed<_Tp>::type;
2867 
2868 template <typename _Tp, typename _Up, typename _Ap>
2869  struct __static_simd_cast_return_type<_Tp, _Up, _Ap, false,
2870 #ifdef _GLIBCXX_SIMD_FIX_P2TS_ISSUE66
2871  enable_if_t<__is_vectorizable_v<_Tp>>
2872 #else
2873  void
2874 #endif
2875  >
2876  {
2877  using type = conditional_t<
2878  (is_integral_v<_Up> && is_integral_v<_Tp> &&
2879 #ifndef _GLIBCXX_SIMD_FIX_P2TS_ISSUE65
2880  is_signed_v<_Up> != is_signed_v<_Tp> &&
2881 #endif
2882  is_same_v<safe_make_signed_t<_Up>, safe_make_signed_t<_Tp>>),
2883  simd<_Tp, _Ap>, fixed_size_simd<_Tp, simd_size_v<_Up, _Ap>>>;
2884  };
2885 
2886 template <typename _Tp, typename _Up, typename _Ap,
2887  typename _R
2888  = typename __static_simd_cast_return_type<_Tp, _Up, _Ap>::type>
2889  _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR _R
2890  static_simd_cast(const simd<_Up, _Ap>& __x)
2891  {
2892  if constexpr (is_same<_R, simd<_Up, _Ap>>::value)
2893  return __x;
2894  else
2895  {
2896  _SimdConverter<_Up, _Ap, typename _R::value_type, typename _R::abi_type>
2897  __c;
2898  return _R(__private_init, __c(__data(__x)));
2899  }
2900  }
2901 
2902 namespace __proposed {
2903 template <typename _Tp, typename _Up, typename _Ap,
2904  typename _R
2905  = typename __static_simd_cast_return_type<_Tp, _Up, _Ap>::type>
2906  _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR typename _R::mask_type
2907  static_simd_cast(const simd_mask<_Up, _Ap>& __x)
2908  {
2909  using _RM = typename _R::mask_type;
2910  return {__private_init, _RM::abi_type::_MaskImpl::template _S_convert<
2911  typename _RM::simd_type::value_type>(__x)};
2912  }
2913 } // namespace __proposed
2914 
2915 // simd_cast {{{2
2916 template <typename _Tp, typename _Up, typename _Ap,
2917  typename _To = __value_type_or_identity_t<_Tp>>
2918  _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR auto
2919  simd_cast(const simd<_ValuePreserving<_Up, _To>, _Ap>& __x)
2920  -> decltype(static_simd_cast<_Tp>(__x))
2921  { return static_simd_cast<_Tp>(__x); }
2922 
2923 namespace __proposed {
2924 template <typename _Tp, typename _Up, typename _Ap,
2925  typename _To = __value_type_or_identity_t<_Tp>>
2926  _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR auto
2927  simd_cast(const simd_mask<_ValuePreserving<_Up, _To>, _Ap>& __x)
2928  -> decltype(static_simd_cast<_Tp>(__x))
2929  { return static_simd_cast<_Tp>(__x); }
2930 } // namespace __proposed
2931 
2932 // }}}2
2933 // resizing_simd_cast {{{
2934 namespace __proposed {
2935 /* Proposed spec:
2936 
2937 template <class T, class U, class Abi>
2938 T resizing_simd_cast(const simd<U, Abi>& x)
2939 
2940 p1 Constraints:
2941  - is_simd_v<T> is true and
2942  - T::value_type is the same type as U
2943 
2944 p2 Returns:
2945  A simd object with the i^th element initialized to x[i] for all i in the
2946  range of [0, min(T::size(), simd_size_v<U, Abi>)). If T::size() is larger
2947  than simd_size_v<U, Abi>, the remaining elements are value-initialized.
2948 
2949 template <class T, class U, class Abi>
2950 T resizing_simd_cast(const simd_mask<U, Abi>& x)
2951 
2952 p1 Constraints: is_simd_mask_v<T> is true
2953 
2954 p2 Returns:
2955  A simd_mask object with the i^th element initialized to x[i] for all i in
2956 the range of [0, min(T::size(), simd_size_v<U, Abi>)). If T::size() is larger
2957  than simd_size_v<U, Abi>, the remaining elements are initialized to false.
2958 
2959  */
2960 
2961 template <typename _Tp, typename _Up, typename _Ap>
2962  _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR enable_if_t<
2963  conjunction_v<is_simd<_Tp>, is_same<typename _Tp::value_type, _Up>>, _Tp>
2964  resizing_simd_cast(const simd<_Up, _Ap>& __x)
2965  {
2966  if constexpr (is_same_v<typename _Tp::abi_type, _Ap>)
2967  return __x;
2968  else if constexpr (simd_size_v<_Up, _Ap> == 1)
2969  {
2970  _Tp __r{};
2971  __r[0] = __x[0];
2972  return __r;
2973  }
2974  else if constexpr (_Tp::size() == 1)
2975  return __x[0];
2976  else if constexpr (sizeof(_Tp) == sizeof(__x)
2977  && !__is_fixed_size_abi_v<_Ap>)
2978  return {__private_init,
2979  __vector_bitcast<typename _Tp::value_type, _Tp::size()>(
2980  _Ap::_S_masked(__data(__x))._M_data)};
2981  else
2982  {
2983  _Tp __r{};
2984  __builtin_memcpy(&__data(__r), &__data(__x),
2985  sizeof(_Up)
2986  * std::min(_Tp::size(), simd_size_v<_Up, _Ap>));
2987  return __r;
2988  }
2989  }
2990 
2991 template <typename _Tp, typename _Up, typename _Ap>
2992  _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR
2993  enable_if_t<is_simd_mask_v<_Tp>, _Tp>
2994  resizing_simd_cast(const simd_mask<_Up, _Ap>& __x)
2995  {
2996  return {__private_init, _Tp::abi_type::_MaskImpl::template _S_convert<
2997  typename _Tp::simd_type::value_type>(__x)};
2998  }
2999 } // namespace __proposed
3000 
3001 // }}}
3002 // to_fixed_size {{{2
3003 template <typename _Tp, int _Np>
3004  _GLIBCXX_SIMD_INTRINSIC fixed_size_simd<_Tp, _Np>
3005  to_fixed_size(const fixed_size_simd<_Tp, _Np>& __x)
3006  { return __x; }
3007 
3008 template <typename _Tp, int _Np>
3009  _GLIBCXX_SIMD_INTRINSIC fixed_size_simd_mask<_Tp, _Np>
3010  to_fixed_size(const fixed_size_simd_mask<_Tp, _Np>& __x)
3011  { return __x; }
3012 
3013 template <typename _Tp, typename _Ap>
3014  _GLIBCXX_SIMD_INTRINSIC auto
3015  to_fixed_size(const simd<_Tp, _Ap>& __x)
3016  {
3017  return simd<_Tp, simd_abi::fixed_size<simd_size_v<_Tp, _Ap>>>([&__x](
3018  auto __i) constexpr { return __x[__i]; });
3019  }
3020 
3021 template <typename _Tp, typename _Ap>
3022  _GLIBCXX_SIMD_INTRINSIC auto
3023  to_fixed_size(const simd_mask<_Tp, _Ap>& __x)
3024  {
3025  constexpr int _Np = simd_mask<_Tp, _Ap>::size();
3026  fixed_size_simd_mask<_Tp, _Np> __r;
3027  __execute_n_times<_Np>([&](auto __i) constexpr { __r[__i] = __x[__i]; });
3028  return __r;
3029  }
3030 
3031 // to_native {{{2
3032 template <typename _Tp, int _Np>
3033  _GLIBCXX_SIMD_INTRINSIC
3034  enable_if_t<(_Np == native_simd<_Tp>::size()), native_simd<_Tp>>
3035  to_native(const fixed_size_simd<_Tp, _Np>& __x)
3036  {
3037  alignas(memory_alignment_v<native_simd<_Tp>>) _Tp __mem[_Np];
3038  __x.copy_to(__mem, vector_aligned);
3039  return {__mem, vector_aligned};
3040  }
3041 
3042 template <typename _Tp, size_t _Np>
3043  _GLIBCXX_SIMD_INTRINSIC
3044  enable_if_t<(_Np == native_simd_mask<_Tp>::size()), native_simd_mask<_Tp>>
3045  to_native(const fixed_size_simd_mask<_Tp, _Np>& __x)
3046  {
3047  return native_simd_mask<_Tp>([&](auto __i) constexpr { return __x[__i]; });
3048  }
3049 
3050 // to_compatible {{{2
3051 template <typename _Tp, size_t _Np>
3052  _GLIBCXX_SIMD_INTRINSIC enable_if_t<(_Np == simd<_Tp>::size()), simd<_Tp>>
3053  to_compatible(const simd<_Tp, simd_abi::fixed_size<_Np>>& __x)
3054  {
3055  alignas(memory_alignment_v<simd<_Tp>>) _Tp __mem[_Np];
3056  __x.copy_to(__mem, vector_aligned);
3057  return {__mem, vector_aligned};
3058  }
3059 
3060 template <typename _Tp, size_t _Np>
3061  _GLIBCXX_SIMD_INTRINSIC
3062  enable_if_t<(_Np == simd_mask<_Tp>::size()), simd_mask<_Tp>>
3063  to_compatible(const simd_mask<_Tp, simd_abi::fixed_size<_Np>>& __x)
3064  { return simd_mask<_Tp>([&](auto __i) constexpr { return __x[__i]; }); }
3065 
3066 // masked assignment [simd_mask.where] {{{1
3067 
3068 // where_expression {{{1
3069 // const_where_expression<M, T> {{{2
3070 template <typename _M, typename _Tp>
3071  class const_where_expression
3072  {
3073  using _V = _Tp;
3074  static_assert(is_same_v<_V, __remove_cvref_t<_Tp>>);
3075 
3076  struct _Wrapper { using value_type = _V; };
3077 
3078  protected:
3079  using _Impl = typename _V::_Impl;
3080 
3081  using value_type =
3082  typename conditional_t<is_arithmetic_v<_V>, _Wrapper, _V>::value_type;
3083 
3084  _GLIBCXX_SIMD_INTRINSIC friend const _M&
3085  __get_mask(const const_where_expression& __x)
3086  { return __x._M_k; }
3087 
3088  _GLIBCXX_SIMD_INTRINSIC friend const _Tp&
3089  __get_lvalue(const const_where_expression& __x)
3090  { return __x._M_value; }
3091 
3092  const _M& _M_k;
3093  _Tp& _M_value;
3094 
3095  public:
3096  const_where_expression(const const_where_expression&) = delete;
3097  const_where_expression& operator=(const const_where_expression&) = delete;
3098 
3099  _GLIBCXX_SIMD_INTRINSIC const_where_expression(const _M& __kk, const _Tp& dd)
3100  : _M_k(__kk), _M_value(const_cast<_Tp&>(dd)) {}
3101 
3102  _GLIBCXX_SIMD_INTRINSIC _V
3103  operator-() const&&
3104  {
3105  return {__private_init,
3106  _Impl::template _S_masked_unary<negate>(__data(_M_k),
3107  __data(_M_value))};
3108  }
3109 
3110  template <typename _Up, typename _Flags>
3111  [[nodiscard]] _GLIBCXX_SIMD_INTRINSIC _V
3112  copy_from(const _LoadStorePtr<_Up, value_type>* __mem, _Flags) const&&
3113  {
3114  return {__private_init,
3115  _Impl::_S_masked_load(__data(_M_value), __data(_M_k),
3116  _Flags::template _S_apply<_V>(__mem))};
3117  }
3118 
3119  template <typename _Up, typename _Flags>
3120  _GLIBCXX_SIMD_INTRINSIC void
3121  copy_to(_LoadStorePtr<_Up, value_type>* __mem, _Flags) const&&
3122  {
3123  _Impl::_S_masked_store(__data(_M_value),
3124  _Flags::template _S_apply<_V>(__mem),
3125  __data(_M_k));
3126  }
3127  };
3128 
3129 // const_where_expression<bool, T> {{{2
3130 template <typename _Tp>
3131  class const_where_expression<bool, _Tp>
3132  {
3133  using _M = bool;
3134  using _V = _Tp;
3135 
3136  static_assert(is_same_v<_V, __remove_cvref_t<_Tp>>);
3137 
3138  struct _Wrapper { using value_type = _V; };
3139 
3140  protected:
3141  using value_type =
3142  typename conditional_t<is_arithmetic_v<_V>, _Wrapper, _V>::value_type;
3143 
3144  _GLIBCXX_SIMD_INTRINSIC friend const _M&
3145  __get_mask(const const_where_expression& __x)
3146  { return __x._M_k; }
3147 
3148  _GLIBCXX_SIMD_INTRINSIC friend const _Tp&
3149  __get_lvalue(const const_where_expression& __x)
3150  { return __x._M_value; }
3151 
3152  const bool _M_k;
3153  _Tp& _M_value;
3154 
3155  public:
3156  const_where_expression(const const_where_expression&) = delete;
3157  const_where_expression& operator=(const const_where_expression&) = delete;
3158 
3159  _GLIBCXX_SIMD_INTRINSIC const_where_expression(const bool __kk, const _Tp& dd)
3160  : _M_k(__kk), _M_value(const_cast<_Tp&>(dd)) {}
3161 
3162  _GLIBCXX_SIMD_INTRINSIC _V operator-() const&&
3163  { return _M_k ? -_M_value : _M_value; }
3164 
3165  template <typename _Up, typename _Flags>
3166  [[nodiscard]] _GLIBCXX_SIMD_INTRINSIC _V
3167  copy_from(const _LoadStorePtr<_Up, value_type>* __mem, _Flags) const&&
3168  { return _M_k ? static_cast<_V>(__mem[0]) : _M_value; }
3169 
3170  template <typename _Up, typename _Flags>
3171  _GLIBCXX_SIMD_INTRINSIC void
3172  copy_to(_LoadStorePtr<_Up, value_type>* __mem, _Flags) const&&
3173  {
3174  if (_M_k)
3175  __mem[0] = _M_value;
3176  }
3177  };
3178 
3179 // where_expression<M, T> {{{2
3180 template <typename _M, typename _Tp>
3181  class where_expression : public const_where_expression<_M, _Tp>
3182  {
3183  using _Impl = typename const_where_expression<_M, _Tp>::_Impl;
3184 
3185  static_assert(!is_const<_Tp>::value,
3186  "where_expression may only be instantiated with __a non-const "
3187  "_Tp parameter");
3188 
3189  using typename const_where_expression<_M, _Tp>::value_type;
3190  using const_where_expression<_M, _Tp>::_M_k;
3191  using const_where_expression<_M, _Tp>::_M_value;
3192 
3193  static_assert(
3194  is_same<typename _M::abi_type, typename _Tp::abi_type>::value, "");
3195  static_assert(_M::size() == _Tp::size(), "");
3196 
3197  _GLIBCXX_SIMD_INTRINSIC friend _Tp& __get_lvalue(where_expression& __x)
3198  { return __x._M_value; }
3199 
3200  public:
3201  where_expression(const where_expression&) = delete;
3202  where_expression& operator=(const where_expression&) = delete;
3203 
3204  _GLIBCXX_SIMD_INTRINSIC where_expression(const _M& __kk, _Tp& dd)
3205  : const_where_expression<_M, _Tp>(__kk, dd) {}
3206 
3207  template <typename _Up>
3208  _GLIBCXX_SIMD_INTRINSIC void operator=(_Up&& __x) &&
3209  {
3210  _Impl::_S_masked_assign(__data(_M_k), __data(_M_value),
3211  __to_value_type_or_member_type<_Tp>(
3212  static_cast<_Up&&>(__x)));
3213  }
3214 
3215 #define _GLIBCXX_SIMD_OP_(__op, __name) \
3216  template <typename _Up> \
3217  _GLIBCXX_SIMD_INTRINSIC void operator __op##=(_Up&& __x)&& \
3218  { \
3219  _Impl::template _S_masked_cassign( \
3220  __data(_M_k), __data(_M_value), \
3221  __to_value_type_or_member_type<_Tp>(static_cast<_Up&&>(__x)), \
3222  [](auto __impl, auto __lhs, auto __rhs) constexpr { \
3223  return __impl.__name(__lhs, __rhs); \
3224  }); \
3225  } \
3226  static_assert(true)
3227  _GLIBCXX_SIMD_OP_(+, _S_plus);
3228  _GLIBCXX_SIMD_OP_(-, _S_minus);
3229  _GLIBCXX_SIMD_OP_(*, _S_multiplies);
3230  _GLIBCXX_SIMD_OP_(/, _S_divides);
3231  _GLIBCXX_SIMD_OP_(%, _S_modulus);
3232  _GLIBCXX_SIMD_OP_(&, _S_bit_and);
3233  _GLIBCXX_SIMD_OP_(|, _S_bit_or);
3234  _GLIBCXX_SIMD_OP_(^, _S_bit_xor);
3235  _GLIBCXX_SIMD_OP_(<<, _S_shift_left);
3236  _GLIBCXX_SIMD_OP_(>>, _S_shift_right);
3237 #undef _GLIBCXX_SIMD_OP_
3238 
3239  _GLIBCXX_SIMD_INTRINSIC void operator++() &&
3240  {
3241  __data(_M_value)
3242  = _Impl::template _S_masked_unary<__increment>(__data(_M_k),
3243  __data(_M_value));
3244  }
3245 
3246  _GLIBCXX_SIMD_INTRINSIC void operator++(int) &&
3247  {
3248  __data(_M_value)
3249  = _Impl::template _S_masked_unary<__increment>(__data(_M_k),
3250  __data(_M_value));
3251  }
3252 
3253  _GLIBCXX_SIMD_INTRINSIC void operator--() &&
3254  {
3255  __data(_M_value)
3256  = _Impl::template _S_masked_unary<__decrement>(__data(_M_k),
3257  __data(_M_value));
3258  }
3259 
3260  _GLIBCXX_SIMD_INTRINSIC void operator--(int) &&
3261  {
3262  __data(_M_value)
3263  = _Impl::template _S_masked_unary<__decrement>(__data(_M_k),
3264  __data(_M_value));
3265  }
3266 
3267  // intentionally hides const_where_expression::copy_from
3268  template <typename _Up, typename _Flags>
3269  _GLIBCXX_SIMD_INTRINSIC void
3270  copy_from(const _LoadStorePtr<_Up, value_type>* __mem, _Flags) &&
3271  {
3272  __data(_M_value)
3273  = _Impl::_S_masked_load(__data(_M_value), __data(_M_k),
3274  _Flags::template _S_apply<_Tp>(__mem));
3275  }
3276  };
3277 
3278 // where_expression<bool, T> {{{2
3279 template <typename _Tp>
3280  class where_expression<bool, _Tp> : public const_where_expression<bool, _Tp>
3281  {
3282  using _M = bool;
3283  using typename const_where_expression<_M, _Tp>::value_type;
3284  using const_where_expression<_M, _Tp>::_M_k;
3285  using const_where_expression<_M, _Tp>::_M_value;
3286 
3287  public:
3288  where_expression(const where_expression&) = delete;
3289  where_expression& operator=(const where_expression&) = delete;
3290 
3291  _GLIBCXX_SIMD_INTRINSIC where_expression(const _M& __kk, _Tp& dd)
3292  : const_where_expression<_M, _Tp>(__kk, dd) {}
3293 
3294 #define _GLIBCXX_SIMD_OP_(__op) \
3295  template <typename _Up> \
3296  _GLIBCXX_SIMD_INTRINSIC void operator __op(_Up&& __x)&& \
3297  { if (_M_k) _M_value __op static_cast<_Up&&>(__x); }
3298 
3299  _GLIBCXX_SIMD_OP_(=)
3300  _GLIBCXX_SIMD_OP_(+=)
3301  _GLIBCXX_SIMD_OP_(-=)
3302  _GLIBCXX_SIMD_OP_(*=)
3303  _GLIBCXX_SIMD_OP_(/=)
3304  _GLIBCXX_SIMD_OP_(%=)
3305  _GLIBCXX_SIMD_OP_(&=)
3306  _GLIBCXX_SIMD_OP_(|=)
3307  _GLIBCXX_SIMD_OP_(^=)
3308  _GLIBCXX_SIMD_OP_(<<=)
3309  _GLIBCXX_SIMD_OP_(>>=)
3310  #undef _GLIBCXX_SIMD_OP_
3311 
3312  _GLIBCXX_SIMD_INTRINSIC void operator++() &&
3313  { if (_M_k) ++_M_value; }
3314 
3315  _GLIBCXX_SIMD_INTRINSIC void operator++(int) &&
3316  { if (_M_k) ++_M_value; }
3317 
3318  _GLIBCXX_SIMD_INTRINSIC void operator--() &&
3319  { if (_M_k) --_M_value; }
3320 
3321  _GLIBCXX_SIMD_INTRINSIC void operator--(int) &&
3322  { if (_M_k) --_M_value; }
3323 
3324  // intentionally hides const_where_expression::copy_from
3325  template <typename _Up, typename _Flags>
3326  _GLIBCXX_SIMD_INTRINSIC void
3327  copy_from(const _LoadStorePtr<_Up, value_type>* __mem, _Flags) &&
3328  { if (_M_k) _M_value = __mem[0]; }
3329  };
3330 
3331 // where {{{1
3332 template <typename _Tp, typename _Ap>
3333  _GLIBCXX_SIMD_INTRINSIC where_expression<simd_mask<_Tp, _Ap>, simd<_Tp, _Ap>>
3334  where(const typename simd<_Tp, _Ap>::mask_type& __k, simd<_Tp, _Ap>& __value)
3335  { return {__k, __value}; }
3336 
3337 template <typename _Tp, typename _Ap>
3338  _GLIBCXX_SIMD_INTRINSIC
3339  const_where_expression<simd_mask<_Tp, _Ap>, simd<_Tp, _Ap>>
3340  where(const typename simd<_Tp, _Ap>::mask_type& __k,
3341  const simd<_Tp, _Ap>& __value)
3342  { return {__k, __value}; }
3343 
3344 template <typename _Tp, typename _Ap>
3345  _GLIBCXX_SIMD_INTRINSIC
3346  where_expression<simd_mask<_Tp, _Ap>, simd_mask<_Tp, _Ap>>
3347  where(const remove_const_t<simd_mask<_Tp, _Ap>>& __k,
3348  simd_mask<_Tp, _Ap>& __value)
3349  { return {__k, __value}; }
3350 
3351 template <typename _Tp, typename _Ap>
3352  _GLIBCXX_SIMD_INTRINSIC
3353  const_where_expression<simd_mask<_Tp, _Ap>, simd_mask<_Tp, _Ap>>
3354  where(const remove_const_t<simd_mask<_Tp, _Ap>>& __k,
3355  const simd_mask<_Tp, _Ap>& __value)
3356  { return {__k, __value}; }
3357 
3358 template <typename _Tp>
3359  _GLIBCXX_SIMD_INTRINSIC where_expression<bool, _Tp>
3360  where(_ExactBool __k, _Tp& __value)
3361  { return {__k, __value}; }
3362 
3363 template <typename _Tp>
3364  _GLIBCXX_SIMD_INTRINSIC const_where_expression<bool, _Tp>
3365  where(_ExactBool __k, const _Tp& __value)
3366  { return {__k, __value}; }
3367 
3368  template <typename _Tp, typename _Ap>
3369  void where(bool __k, simd<_Tp, _Ap>& __value) = delete;
3370 
3371  template <typename _Tp, typename _Ap>
3372  void where(bool __k, const simd<_Tp, _Ap>& __value) = delete;
3373 
3374 // proposed mask iterations {{{1
3375 namespace __proposed {
3376 template <size_t _Np>
3377  class where_range
3378  {
3379  const bitset<_Np> __bits;
3380 
3381  public:
3382  where_range(bitset<_Np> __b) : __bits(__b) {}
3383 
3384  class iterator
3385  {
3386  size_t __mask;
3387  size_t __bit;
3388 
3389  _GLIBCXX_SIMD_INTRINSIC void __next_bit()
3390  { __bit = __builtin_ctzl(__mask); }
3391 
3392  _GLIBCXX_SIMD_INTRINSIC void __reset_lsb()
3393  {
3394  // 01100100 - 1 = 01100011
3395  __mask &= (__mask - 1);
3396  // __asm__("btr %1,%0" : "+r"(__mask) : "r"(__bit));
3397  }
3398 
3399  public:
3400  iterator(decltype(__mask) __m) : __mask(__m) { __next_bit(); }
3401  iterator(const iterator&) = default;
3402  iterator(iterator&&) = default;
3403 
3404  _GLIBCXX_SIMD_ALWAYS_INLINE size_t operator->() const
3405  { return __bit; }
3406 
3407  _GLIBCXX_SIMD_ALWAYS_INLINE size_t operator*() const
3408  { return __bit; }
3409 
3410  _GLIBCXX_SIMD_ALWAYS_INLINE iterator& operator++()
3411  {
3412  __reset_lsb();
3413  __next_bit();
3414  return *this;
3415  }
3416 
3417  _GLIBCXX_SIMD_ALWAYS_INLINE iterator operator++(int)
3418  {
3419  iterator __tmp = *this;
3420  __reset_lsb();
3421  __next_bit();
3422  return __tmp;
3423  }
3424 
3425  _GLIBCXX_SIMD_ALWAYS_INLINE bool operator==(const iterator& __rhs) const
3426  { return __mask == __rhs.__mask; }
3427 
3428  _GLIBCXX_SIMD_ALWAYS_INLINE bool operator!=(const iterator& __rhs) const
3429  { return __mask != __rhs.__mask; }
3430  };
3431 
3432  iterator begin() const
3433  { return __bits.to_ullong(); }
3434 
3435  iterator end() const
3436  { return 0; }
3437  };
3438 
3439 template <typename _Tp, typename _Ap>
3440  where_range<simd_size_v<_Tp, _Ap>>
3441  where(const simd_mask<_Tp, _Ap>& __k)
3442  { return __k.__to_bitset(); }
3443 
3444 } // namespace __proposed
3445 
3446 // }}}1
3447 // reductions [simd.reductions] {{{1
3448 template <typename _Tp, typename _Abi, typename _BinaryOperation = plus<>>
3449  _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR _Tp
3450  reduce(const simd<_Tp, _Abi>& __v,
3451  _BinaryOperation __binary_op = _BinaryOperation())
3452  { return _Abi::_SimdImpl::_S_reduce(__v, __binary_op); }
3453 
3454 template <typename _M, typename _V, typename _BinaryOperation = plus<>>
3455  _GLIBCXX_SIMD_INTRINSIC typename _V::value_type
3456  reduce(const const_where_expression<_M, _V>& __x,
3457  typename _V::value_type __identity_element,
3458  _BinaryOperation __binary_op)
3459  {
3460  if (__builtin_expect(none_of(__get_mask(__x)), false))
3461  return __identity_element;
3462 
3463  _V __tmp = __identity_element;
3464  _V::_Impl::_S_masked_assign(__data(__get_mask(__x)), __data(__tmp),
3465  __data(__get_lvalue(__x)));
3466  return reduce(__tmp, __binary_op);
3467  }
3468 
3469 template <typename _M, typename _V>
3470  _GLIBCXX_SIMD_INTRINSIC typename _V::value_type
3471  reduce(const const_where_expression<_M, _V>& __x, plus<> __binary_op = {})
3472  { return reduce(__x, 0, __binary_op); }
3473 
3474 template <typename _M, typename _V>
3475  _GLIBCXX_SIMD_INTRINSIC typename _V::value_type
3476  reduce(const const_where_expression<_M, _V>& __x, multiplies<> __binary_op)
3477  { return reduce(__x, 1, __binary_op); }
3478 
3479 template <typename _M, typename _V>
3480  _GLIBCXX_SIMD_INTRINSIC typename _V::value_type
3481  reduce(const const_where_expression<_M, _V>& __x, bit_and<> __binary_op)
3482  { return reduce(__x, ~typename _V::value_type(), __binary_op); }
3483 
3484 template <typename _M, typename _V>
3485  _GLIBCXX_SIMD_INTRINSIC typename _V::value_type
3486  reduce(const const_where_expression<_M, _V>& __x, bit_or<> __binary_op)
3487  { return reduce(__x, 0, __binary_op); }
3488 
3489 template <typename _M, typename _V>
3490  _GLIBCXX_SIMD_INTRINSIC typename _V::value_type
3491  reduce(const const_where_expression<_M, _V>& __x, bit_xor<> __binary_op)
3492  { return reduce(__x, 0, __binary_op); }
3493 
3494 template <typename _Tp, typename _Abi>
3495  _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR _Tp
3496  hmin(const simd<_Tp, _Abi>& __v) noexcept
3497  {
3498  return _Abi::_SimdImpl::_S_reduce(__v, __detail::_Minimum());
3499  }
3500 
3501 template <typename _Tp, typename _Abi>
3502  _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR _Tp
3503  hmax(const simd<_Tp, _Abi>& __v) noexcept
3504  {
3505  return _Abi::_SimdImpl::_S_reduce(__v, __detail::_Maximum());
3506  }
3507 
3508 template <typename _M, typename _V>
3509  _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR
3510  typename _V::value_type
3511  hmin(const const_where_expression<_M, _V>& __x) noexcept
3512  {
3513  using _Tp = typename _V::value_type;
3514  constexpr _Tp __id_elem =
3515 #ifdef __FINITE_MATH_ONLY__
3516  __finite_max_v<_Tp>;
3517 #else
3518  __value_or<__infinity, _Tp>(__finite_max_v<_Tp>);
3519 #endif
3520  _V __tmp = __id_elem;
3521  _V::_Impl::_S_masked_assign(__data(__get_mask(__x)), __data(__tmp),
3522  __data(__get_lvalue(__x)));
3523  return _V::abi_type::_SimdImpl::_S_reduce(__tmp, __detail::_Minimum());
3524  }
3525 
3526 template <typename _M, typename _V>
3527  _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR
3528  typename _V::value_type
3529  hmax(const const_where_expression<_M, _V>& __x) noexcept
3530  {
3531  using _Tp = typename _V::value_type;
3532  constexpr _Tp __id_elem =
3533 #ifdef __FINITE_MATH_ONLY__
3534  __finite_min_v<_Tp>;
3535 #else
3536  [] {
3537  if constexpr (__value_exists_v<__infinity, _Tp>)
3538  return -__infinity_v<_Tp>;
3539  else
3540  return __finite_min_v<_Tp>;
3541  }();
3542 #endif
3543  _V __tmp = __id_elem;
3544  _V::_Impl::_S_masked_assign(__data(__get_mask(__x)), __data(__tmp),
3545  __data(__get_lvalue(__x)));
3546  return _V::abi_type::_SimdImpl::_S_reduce(__tmp, __detail::_Maximum());
3547  }
3548 
3549 // }}}1
3550 // algorithms [simd.alg] {{{
3551 template <typename _Tp, typename _Ap>
3552  _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR simd<_Tp, _Ap>
3553  min(const simd<_Tp, _Ap>& __a, const simd<_Tp, _Ap>& __b)
3554  { return {__private_init, _Ap::_SimdImpl::_S_min(__data(__a), __data(__b))}; }
3555 
3556 template <typename _Tp, typename _Ap>
3557  _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR simd<_Tp, _Ap>
3558  max(const simd<_Tp, _Ap>& __a, const simd<_Tp, _Ap>& __b)
3559  { return {__private_init, _Ap::_SimdImpl::_S_max(__data(__a), __data(__b))}; }
3560 
3561 template <typename _Tp, typename _Ap>
3562  _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR
3563  pair<simd<_Tp, _Ap>, simd<_Tp, _Ap>>
3564  minmax(const simd<_Tp, _Ap>& __a, const simd<_Tp, _Ap>& __b)
3565  {
3566  const auto pair_of_members
3567  = _Ap::_SimdImpl::_S_minmax(__data(__a), __data(__b));
3568  return {simd<_Tp, _Ap>(__private_init, pair_of_members.first),
3569  simd<_Tp, _Ap>(__private_init, pair_of_members.second)};
3570  }
3571 
3572 template <typename _Tp, typename _Ap>
3573  _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR simd<_Tp, _Ap>
3574  clamp(const simd<_Tp, _Ap>& __v, const simd<_Tp, _Ap>& __lo,
3575  const simd<_Tp, _Ap>& __hi)
3576  {
3577  using _Impl = typename _Ap::_SimdImpl;
3578  return {__private_init,
3579  _Impl::_S_min(__data(__hi),
3580  _Impl::_S_max(__data(__lo), __data(__v)))};
3581  }
3582 
3583 // }}}
3584 
3585 template <size_t... _Sizes, typename _Tp, typename _Ap,
3586  typename = enable_if_t<((_Sizes + ...) == simd<_Tp, _Ap>::size())>>
3587  inline tuple<simd<_Tp, simd_abi::deduce_t<_Tp, _Sizes>>...>
3588  split(const simd<_Tp, _Ap>&);
3589 
3590 // __extract_part {{{
3591 template <int _Index, int _Total, int _Combine = 1, typename _Tp, size_t _Np>
3592  _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_CONST
3593  _SimdWrapper<_Tp, _Np / _Total * _Combine>
3594  __extract_part(const _SimdWrapper<_Tp, _Np> __x);
3595 
3596 template <int Index, int Parts, int _Combine = 1, typename _Tp, typename _A0,
3597  typename... _As>
3598  _GLIBCXX_SIMD_INTRINSIC auto
3599  __extract_part(const _SimdTuple<_Tp, _A0, _As...>& __x);
3600 
3601 // }}}
3602 // _SizeList {{{
3603 template <size_t _V0, size_t... _Values>
3604  struct _SizeList
3605  {
3606  template <size_t _I>
3607  static constexpr size_t _S_at(_SizeConstant<_I> = {})
3608  {
3609  if constexpr (_I == 0)
3610  return _V0;
3611  else
3612  return _SizeList<_Values...>::template _S_at<_I - 1>();
3613  }
3614 
3615  template <size_t _I>
3616  static constexpr auto _S_before(_SizeConstant<_I> = {})
3617  {
3618  if constexpr (_I == 0)
3619  return _SizeConstant<0>();
3620  else
3621  return _SizeConstant<
3622  _V0 + _SizeList<_Values...>::template _S_before<_I - 1>()>();
3623  }
3624 
3625  template <size_t _Np>
3626  static constexpr auto _S_pop_front(_SizeConstant<_Np> = {})
3627  {
3628  if constexpr (_Np == 0)
3629  return _SizeList();
3630  else
3631  return _SizeList<_Values...>::template _S_pop_front<_Np - 1>();
3632  }
3633  };
3634 
3635 // }}}
3636 // __extract_center {{{
3637 template <typename _Tp, size_t _Np>
3638  _GLIBCXX_SIMD_INTRINSIC _SimdWrapper<_Tp, _Np / 2>
3639  __extract_center(_SimdWrapper<_Tp, _Np> __x)
3640  {
3641  static_assert(_Np >= 4);
3642  static_assert(_Np % 4 == 0); // x0 - x1 - x2 - x3 -> return {x1, x2}
3643 #if _GLIBCXX_SIMD_X86INTRIN // {{{
3644  if constexpr (__have_avx512f && sizeof(_Tp) * _Np == 64)
3645  {
3646  const auto __intrin = __to_intrin(__x);
3647  if constexpr (is_integral_v<_Tp>)
3648  return __vector_bitcast<_Tp>(_mm512_castsi512_si256(
3649  _mm512_shuffle_i32x4(__intrin, __intrin,
3650  1 + 2 * 0x4 + 2 * 0x10 + 3 * 0x40)));
3651  else if constexpr (sizeof(_Tp) == 4)
3652  return __vector_bitcast<_Tp>(_mm512_castps512_ps256(
3653  _mm512_shuffle_f32x4(__intrin, __intrin,
3654  1 + 2 * 0x4 + 2 * 0x10 + 3 * 0x40)));
3655  else if constexpr (sizeof(_Tp) == 8)
3656  return __vector_bitcast<_Tp>(_mm512_castpd512_pd256(
3657  _mm512_shuffle_f64x2(__intrin, __intrin,
3658  1 + 2 * 0x4 + 2 * 0x10 + 3 * 0x40)));
3659  else
3660  __assert_unreachable<_Tp>();
3661  }
3662  else if constexpr (sizeof(_Tp) * _Np == 32 && is_floating_point_v<_Tp>)
3663  return __vector_bitcast<_Tp>(
3664  _mm_shuffle_pd(__lo128(__vector_bitcast<double>(__x)),
3665  __hi128(__vector_bitcast<double>(__x)), 1));
3666  else if constexpr (sizeof(__x) == 32 && sizeof(_Tp) * _Np <= 32)
3667  return __vector_bitcast<_Tp>(
3668  _mm_alignr_epi8(__hi128(__vector_bitcast<_LLong>(__x)),
3669  __lo128(__vector_bitcast<_LLong>(__x)),
3670  sizeof(_Tp) * _Np / 4));
3671  else
3672 #endif // _GLIBCXX_SIMD_X86INTRIN }}}
3673  {
3674  __vector_type_t<_Tp, _Np / 2> __r;
3675  __builtin_memcpy(&__r,
3676  reinterpret_cast<const char*>(&__x)
3677  + sizeof(_Tp) * _Np / 4,
3678  sizeof(_Tp) * _Np / 2);
3679  return __r;
3680  }
3681  }
3682 
3683 template <typename _Tp, typename _A0, typename... _As>
3684  _GLIBCXX_SIMD_INTRINSIC
3685  _SimdWrapper<_Tp, _SimdTuple<_Tp, _A0, _As...>::_S_size() / 2>
3686  __extract_center(const _SimdTuple<_Tp, _A0, _As...>& __x)
3687  {
3688  if constexpr (sizeof...(_As) == 0)
3689  return __extract_center(__x.first);
3690  else
3691  return __extract_part<1, 4, 2>(__x);
3692  }
3693 
3694 // }}}
3695 // __split_wrapper {{{
3696 template <size_t... _Sizes, typename _Tp, typename... _As>
3697  auto
3698  __split_wrapper(_SizeList<_Sizes...>, const _SimdTuple<_Tp, _As...>& __x)
3699  {
3700  return split<_Sizes...>(
3701  fixed_size_simd<_Tp, _SimdTuple<_Tp, _As...>::_S_size()>(__private_init,
3702  __x));
3703  }
3704 
3705 // }}}
3706 
3707 // split<simd>(simd) {{{
3708 template <typename _V, typename _Ap,
3709  size_t Parts = simd_size_v<typename _V::value_type, _Ap> / _V::size()>
3710  enable_if_t<simd_size_v<typename _V::value_type, _Ap> == Parts * _V::size()
3711  && is_simd_v<_V>, array<_V, Parts>>
3712  split(const simd<typename _V::value_type, _Ap>& __x)
3713  {
3714  using _Tp = typename _V::value_type;
3715  if constexpr (Parts == 1)
3716  {
3717  return {simd_cast<_V>(__x)};
3718  }
3719  else if (__x._M_is_constprop())
3720  {
3721  return __generate_from_n_evaluations<Parts, array<_V, Parts>>([&](
3722  auto __i) constexpr {
3723  return _V([&](auto __j) constexpr {
3724  return __x[__i * _V::size() + __j];
3725  });
3726  });
3727  }
3728  else if constexpr (
3729  __is_fixed_size_abi_v<_Ap>
3730  && (is_same_v<typename _V::abi_type, simd_abi::scalar>
3731  || (__is_fixed_size_abi_v<typename _V::abi_type>
3732  && sizeof(_V) == sizeof(_Tp) * _V::size() // _V doesn't have padding
3733  )))
3734  {
3735  // fixed_size -> fixed_size (w/o padding) or scalar
3736 #ifdef _GLIBCXX_SIMD_USE_ALIASING_LOADS
3737  const __may_alias<_Tp>* const __element_ptr
3738  = reinterpret_cast<const __may_alias<_Tp>*>(&__data(__x));
3739  return __generate_from_n_evaluations<Parts, array<_V, Parts>>([&](
3740  auto __i) constexpr {
3741  return _V(__element_ptr + __i * _V::size(), vector_aligned);
3742  });
3743 #else
3744  const auto& __xx = __data(__x);
3745  return __generate_from_n_evaluations<Parts, array<_V, Parts>>([&](
3746  auto __i) constexpr {
3747  [[maybe_unused]] constexpr size_t __offset
3748  = decltype(__i)::value * _V::size();
3749  return _V([&](auto __j) constexpr {
3750  constexpr _SizeConstant<__j + __offset> __k;
3751  return __xx[__k];
3752  });
3753  });
3754 #endif
3755  }
3756  else if constexpr (is_same_v<typename _V::abi_type, simd_abi::scalar>)
3757  {
3758  // normally memcpy should work here as well
3759  return __generate_from_n_evaluations<Parts, array<_V, Parts>>([&](
3760  auto __i) constexpr { return __x[__i]; });
3761  }
3762  else
3763  {
3764  return __generate_from_n_evaluations<Parts, array<_V, Parts>>([&](
3765  auto __i) constexpr {
3766  if constexpr (__is_fixed_size_abi_v<typename _V::abi_type>)
3767  return _V([&](auto __j) constexpr {
3768  return __x[__i * _V::size() + __j];
3769  });
3770  else
3771  return _V(__private_init,
3772  __extract_part<decltype(__i)::value, Parts>(__data(__x)));
3773  });
3774  }
3775  }
3776 
3777 // }}}
3778 // split<simd_mask>(simd_mask) {{{
3779 template <typename _V, typename _Ap,
3780  size_t _Parts
3781  = simd_size_v<typename _V::simd_type::value_type, _Ap> / _V::size()>
3782  enable_if_t<is_simd_mask_v<_V> && simd_size_v<typename
3783  _V::simd_type::value_type, _Ap> == _Parts * _V::size(), array<_V, _Parts>>
3784  split(const simd_mask<typename _V::simd_type::value_type, _Ap>& __x)
3785  {
3786  if constexpr (is_same_v<_Ap, typename _V::abi_type>)
3787  return {__x};
3788  else if constexpr (_Parts == 1)
3789  return {__proposed::static_simd_cast<_V>(__x)};
3790  else if constexpr (_Parts == 2 && __is_sse_abi<typename _V::abi_type>()
3791  && __is_avx_abi<_Ap>())
3792  return {_V(__private_init, __lo128(__data(__x))),
3793  _V(__private_init, __hi128(__data(__x)))};
3794  else if constexpr (_V::size() <= __CHAR_BIT__ * sizeof(_ULLong))
3795  {
3796  const bitset __bits = __x.__to_bitset();
3797  return __generate_from_n_evaluations<_Parts, array<_V, _Parts>>([&](
3798  auto __i) constexpr {
3799  constexpr size_t __offset = __i * _V::size();
3800  return _V(__bitset_init, (__bits >> __offset).to_ullong());
3801  });
3802  }
3803  else
3804  {
3805  return __generate_from_n_evaluations<_Parts, array<_V, _Parts>>([&](
3806  auto __i) constexpr {
3807  constexpr size_t __offset = __i * _V::size();
3808  return _V(
3809  __private_init, [&](auto __j) constexpr {
3810  return __x[__j + __offset];
3811  });
3812  });
3813  }
3814  }
3815 
3816 // }}}
3817 // split<_Sizes...>(simd) {{{
3818 template <size_t... _Sizes, typename _Tp, typename _Ap, typename>
3819  _GLIBCXX_SIMD_ALWAYS_INLINE
3820  tuple<simd<_Tp, simd_abi::deduce_t<_Tp, _Sizes>>...>
3821  split(const simd<_Tp, _Ap>& __x)
3822  {
3823  using _SL = _SizeList<_Sizes...>;
3824  using _Tuple = tuple<__deduced_simd<_Tp, _Sizes>...>;
3825  constexpr size_t _Np = simd_size_v<_Tp, _Ap>;
3826  constexpr size_t _N0 = _SL::template _S_at<0>();
3827  using _V = __deduced_simd<_Tp, _N0>;
3828 
3829  if (__x._M_is_constprop())
3830  return __generate_from_n_evaluations<sizeof...(_Sizes), _Tuple>([&](
3831  auto __i) constexpr {
3832  using _Vi = __deduced_simd<_Tp, _SL::_S_at(__i)>;
3833  constexpr size_t __offset = _SL::_S_before(__i);
3834  return _Vi([&](auto __j) constexpr { return __x[__offset + __j]; });
3835  });
3836  else if constexpr (_Np == _N0)
3837  {
3838  static_assert(sizeof...(_Sizes) == 1);
3839  return {simd_cast<_V>(__x)};
3840  }
3841  else if constexpr // split from fixed_size, such that __x::first.size == _N0
3842  (__is_fixed_size_abi_v<
3843  _Ap> && __fixed_size_storage_t<_Tp, _Np>::_S_first_size == _N0)
3844  {
3845  static_assert(
3846  !__is_fixed_size_abi_v<typename _V::abi_type>,
3847  "How can <_Tp, _Np> be __a single _SimdTuple entry but __a "
3848  "fixed_size_simd "
3849  "when deduced?");
3850  // extract first and recurse (__split_wrapper is needed to deduce a new
3851  // _Sizes pack)
3852  return tuple_cat(make_tuple(_V(__private_init, __data(__x).first)),
3853  __split_wrapper(_SL::template _S_pop_front<1>(),
3854  __data(__x).second));
3855  }
3856  else if constexpr ((!is_same_v<simd_abi::scalar,
3857  simd_abi::deduce_t<_Tp, _Sizes>> && ...)
3858  && (!__is_fixed_size_abi_v<
3859  simd_abi::deduce_t<_Tp, _Sizes>> && ...))
3860  {
3861  if constexpr (((_Sizes * 2 == _Np) && ...))
3862  return {{__private_init, __extract_part<0, 2>(__data(__x))},
3863  {__private_init, __extract_part<1, 2>(__data(__x))}};
3864  else if constexpr (is_same_v<_SizeList<_Sizes...>,
3865  _SizeList<_Np / 3, _Np / 3, _Np / 3>>)
3866  return {{__private_init, __extract_part<0, 3>(__data(__x))},
3867  {__private_init, __extract_part<1, 3>(__data(__x))},
3868  {__private_init, __extract_part<2, 3>(__data(__x))}};
3869  else if constexpr (is_same_v<_SizeList<_Sizes...>,
3870  _SizeList<2 * _Np / 3, _Np / 3>>)
3871  return {{__private_init, __extract_part<0, 3, 2>(__data(__x))},
3872  {__private_init, __extract_part<2, 3>(__data(__x))}};
3873  else if constexpr (is_same_v<_SizeList<_Sizes...>,
3874  _SizeList<_Np / 3, 2 * _Np / 3>>)
3875  return {{__private_init, __extract_part<0, 3>(__data(__x))},
3876  {__private_init, __extract_part<1, 3, 2>(__data(__x))}};
3877  else if constexpr (is_same_v<_SizeList<_Sizes...>,
3878  _SizeList<_Np / 2, _Np / 4, _Np / 4>>)
3879  return {{__private_init, __extract_part<0, 2>(__data(__x))},
3880  {__private_init, __extract_part<2, 4>(__data(__x))},
3881  {__private_init, __extract_part<3, 4>(__data(__x))}};
3882  else if constexpr (is_same_v<_SizeList<_Sizes...>,
3883  _SizeList<_Np / 4, _Np / 4, _Np / 2>>)
3884  return {{__private_init, __extract_part<0, 4>(__data(__x))},
3885  {__private_init, __extract_part<1, 4>(__data(__x))},
3886  {__private_init, __extract_part<1, 2>(__data(__x))}};
3887  else if constexpr (is_same_v<_SizeList<_Sizes...>,
3888  _SizeList<_Np / 4, _Np / 2, _Np / 4>>)
3889  return {{__private_init, __extract_part<0, 4>(__data(__x))},
3890  {__private_init, __extract_center(__data(__x))},
3891  {__private_init, __extract_part<3, 4>(__data(__x))}};
3892  else if constexpr (((_Sizes * 4 == _Np) && ...))
3893  return {{__private_init, __extract_part<0, 4>(__data(__x))},
3894  {__private_init, __extract_part<1, 4>(__data(__x))},
3895  {__private_init, __extract_part<2, 4>(__data(__x))},
3896  {__private_init, __extract_part<3, 4>(__data(__x))}};
3897  // else fall through
3898  }
3899 #ifdef _GLIBCXX_SIMD_USE_ALIASING_LOADS
3900  const __may_alias<_Tp>* const __element_ptr
3901  = reinterpret_cast<const __may_alias<_Tp>*>(&__x);
3902  return __generate_from_n_evaluations<sizeof...(_Sizes), _Tuple>([&](
3903  auto __i) constexpr {
3904  using _Vi = __deduced_simd<_Tp, _SL::_S_at(__i)>;
3905  constexpr size_t __offset = _SL::_S_before(__i);
3906  constexpr size_t __base_align = alignof(simd<_Tp, _Ap>);
3907  constexpr size_t __a
3908  = __base_align - ((__offset * sizeof(_Tp)) % __base_align);
3909  constexpr size_t __b = ((__a - 1) & __a) ^ __a;
3910  constexpr size_t __alignment = __b == 0 ? __a : __b;
3911  return _Vi(__element_ptr + __offset, overaligned<__alignment>);
3912  });
3913 #else
3914  return __generate_from_n_evaluations<sizeof...(_Sizes), _Tuple>([&](
3915  auto __i) constexpr {
3916  using _Vi = __deduced_simd<_Tp, _SL::_S_at(__i)>;
3917  const auto& __xx = __data(__x);
3918  using _Offset = decltype(_SL::_S_before(__i));
3919  return _Vi([&](auto __j) constexpr {
3920  constexpr _SizeConstant<_Offset::value + __j> __k;
3921  return __xx[__k];
3922  });
3923  });
3924 #endif
3925  }
3926 
3927 // }}}
3928 
3929 // __subscript_in_pack {{{
3930 template <size_t _I, typename _Tp, typename _Ap, typename... _As>
3931  _GLIBCXX_SIMD_INTRINSIC constexpr _Tp
3932  __subscript_in_pack(const simd<_Tp, _Ap>& __x, const simd<_Tp, _As>&... __xs)
3933  {
3934  if constexpr (_I < simd_size_v<_Tp, _Ap>)
3935  return __x[_I];
3936  else
3937  return __subscript_in_pack<_I - simd_size_v<_Tp, _Ap>>(__xs...);
3938  }
3939 
3940 // }}}
3941 // __store_pack_of_simd {{{
3942 template <typename _Tp, typename _A0, typename... _As>
3943  _GLIBCXX_SIMD_INTRINSIC void
3944  __store_pack_of_simd(char* __mem, const simd<_Tp, _A0>& __x0,
3945  const simd<_Tp, _As>&... __xs)
3946  {
3947  constexpr size_t __n_bytes = sizeof(_Tp) * simd_size_v<_Tp, _A0>;
3948  __builtin_memcpy(__mem, &__data(__x0), __n_bytes);
3949  if constexpr (sizeof...(__xs) > 0)
3950  __store_pack_of_simd(__mem + __n_bytes, __xs...);
3951  }
3952 
3953 // }}}
3954 // concat(simd...) {{{
3955 template <typename _Tp, typename... _As>
3956  inline _GLIBCXX_SIMD_CONSTEXPR
3957  simd<_Tp, simd_abi::deduce_t<_Tp, (simd_size_v<_Tp, _As> + ...)>>
3958  concat(const simd<_Tp, _As>&... __xs)
3959  {
3960  using _Rp = __deduced_simd<_Tp, (simd_size_v<_Tp, _As> + ...)>;
3961  if constexpr (sizeof...(__xs) == 1)
3962  return simd_cast<_Rp>(__xs...);
3963  else if ((... && __xs._M_is_constprop()))
3964  return simd<_Tp,
3965  simd_abi::deduce_t<_Tp, (simd_size_v<_Tp, _As> + ...)>>([&](
3966  auto __i) constexpr { return __subscript_in_pack<__i>(__xs...); });
3967  else
3968  {
3969  _Rp __r{};
3970  __store_pack_of_simd(reinterpret_cast<char*>(&__data(__r)), __xs...);
3971  return __r;
3972  }
3973  }
3974 
3975 // }}}
3976 // concat(array<simd>) {{{
3977 template <typename _Tp, typename _Abi, size_t _Np>
3978  _GLIBCXX_SIMD_ALWAYS_INLINE
3979  _GLIBCXX_SIMD_CONSTEXPR __deduced_simd<_Tp, simd_size_v<_Tp, _Abi> * _Np>
3980  concat(const array<simd<_Tp, _Abi>, _Np>& __x)
3981  {
3982  return __call_with_subscripts<_Np>(__x, [](const auto&... __xs) {
3983  return concat(__xs...);
3984  });
3985  }
3986 
3987 // }}}
3988 
3989 /// @cond undocumented
3990 // _SmartReference {{{
3991 template <typename _Up, typename _Accessor = _Up,
3992  typename _ValueType = typename _Up::value_type>
3993  class _SmartReference
3994  {
3995  friend _Accessor;
3996  int _M_index;
3997  _Up& _M_obj;
3998 
3999  _GLIBCXX_SIMD_INTRINSIC constexpr _ValueType _M_read() const noexcept
4000  {
4001  if constexpr (is_arithmetic_v<_Up>)
4002  return _M_obj;
4003  else
4004  return _M_obj[_M_index];
4005  }
4006 
4007  template <typename _Tp>
4008  _GLIBCXX_SIMD_INTRINSIC constexpr void _M_write(_Tp&& __x) const
4009  { _Accessor::_S_set(_M_obj, _M_index, static_cast<_Tp&&>(__x)); }
4010 
4011  public:
4012  _GLIBCXX_SIMD_INTRINSIC constexpr
4013  _SmartReference(_Up& __o, int __i) noexcept
4014  : _M_index(__i), _M_obj(__o) {}
4015 
4016  using value_type = _ValueType;
4017 
4018  _GLIBCXX_SIMD_INTRINSIC _SmartReference(const _SmartReference&) = delete;
4019 
4020  _GLIBCXX_SIMD_INTRINSIC constexpr operator value_type() const noexcept
4021  { return _M_read(); }
4022 
4023  template <typename _Tp,
4024  typename
4025  = _ValuePreservingOrInt<__remove_cvref_t<_Tp>, value_type>>
4026  _GLIBCXX_SIMD_INTRINSIC constexpr _SmartReference operator=(_Tp&& __x) &&
4027  {
4028  _M_write(static_cast<_Tp&&>(__x));
4029  return {_M_obj, _M_index};
4030  }
4031 
4032 #define _GLIBCXX_SIMD_OP_(__op) \
4033  template <typename _Tp, \
4034  typename _TT \
4035  = decltype(declval<value_type>() __op declval<_Tp>()), \
4036  typename = _ValuePreservingOrInt<__remove_cvref_t<_Tp>, _TT>, \
4037  typename = _ValuePreservingOrInt<_TT, value_type>> \
4038  _GLIBCXX_SIMD_INTRINSIC constexpr _SmartReference \
4039  operator __op##=(_Tp&& __x) && \
4040  { \
4041  const value_type& __lhs = _M_read(); \
4042  _M_write(__lhs __op __x); \
4043  return {_M_obj, _M_index}; \
4044  }
4045  _GLIBCXX_SIMD_ALL_ARITHMETICS(_GLIBCXX_SIMD_OP_);
4046  _GLIBCXX_SIMD_ALL_SHIFTS(_GLIBCXX_SIMD_OP_);
4047  _GLIBCXX_SIMD_ALL_BINARY(_GLIBCXX_SIMD_OP_);
4048 #undef _GLIBCXX_SIMD_OP_
4049 
4050  template <typename _Tp = void,
4051  typename
4052  = decltype(++declval<conditional_t<true, value_type, _Tp>&>())>
4053  _GLIBCXX_SIMD_INTRINSIC constexpr _SmartReference operator++() &&
4054  {
4055  value_type __x = _M_read();
4056  _M_write(++__x);
4057  return {_M_obj, _M_index};
4058  }
4059 
4060  template <typename _Tp = void,
4061  typename
4062  = decltype(declval<conditional_t<true, value_type, _Tp>&>()++)>
4063  _GLIBCXX_SIMD_INTRINSIC constexpr value_type operator++(int) &&
4064  {
4065  const value_type __r = _M_read();
4066  value_type __x = __r;
4067  _M_write(++__x);
4068  return __r;
4069  }
4070 
4071  template <typename _Tp = void,
4072  typename
4073  = decltype(--declval<conditional_t<true, value_type, _Tp>&>())>
4074  _GLIBCXX_SIMD_INTRINSIC constexpr _SmartReference operator--() &&
4075  {
4076  value_type __x = _M_read();
4077  _M_write(--__x);
4078  return {_M_obj, _M_index};
4079  }
4080 
4081  template <typename _Tp = void,
4082  typename
4083  = decltype(declval<conditional_t<true, value_type, _Tp>&>()--)>
4084  _GLIBCXX_SIMD_INTRINSIC constexpr value_type operator--(int) &&
4085  {
4086  const value_type __r = _M_read();
4087  value_type __x = __r;
4088  _M_write(--__x);
4089  return __r;
4090  }
4091 
4092  _GLIBCXX_SIMD_INTRINSIC friend void
4093  swap(_SmartReference&& __a, _SmartReference&& __b) noexcept(
4094  conjunction<
4095  is_nothrow_constructible<value_type, _SmartReference&&>,
4096  is_nothrow_assignable<_SmartReference&&, value_type&&>>::value)
4097  {
4098  value_type __tmp = static_cast<_SmartReference&&>(__a);
4099  static_cast<_SmartReference&&>(__a) = static_cast<value_type>(__b);
4100  static_cast<_SmartReference&&>(__b) = std::move(__tmp);
4101  }
4102 
4103  _GLIBCXX_SIMD_INTRINSIC friend void
4104  swap(value_type& __a, _SmartReference&& __b) noexcept(
4105  conjunction<
4106  is_nothrow_constructible<value_type, value_type&&>,
4107  is_nothrow_assignable<value_type&, value_type&&>,
4108  is_nothrow_assignable<_SmartReference&&, value_type&&>>::value)
4109  {
4110  value_type __tmp(std::move(__a));
4111  __a = static_cast<value_type>(__b);
4112  static_cast<_SmartReference&&>(__b) = std::move(__tmp);
4113  }
4114 
4115  _GLIBCXX_SIMD_INTRINSIC friend void
4116  swap(_SmartReference&& __a, value_type& __b) noexcept(
4117  conjunction<
4118  is_nothrow_constructible<value_type, _SmartReference&&>,
4119  is_nothrow_assignable<value_type&, value_type&&>,
4120  is_nothrow_assignable<_SmartReference&&, value_type&&>>::value)
4121  {
4122  value_type __tmp(__a);
4123  static_cast<_SmartReference&&>(__a) = std::move(__b);
4124  __b = std::move(__tmp);
4125  }
4126  };
4127 
4128 // }}}
4129 // __scalar_abi_wrapper {{{
4130 template <int _Bytes>
4131  struct __scalar_abi_wrapper
4132  {
4133  template <typename _Tp> static constexpr size_t _S_full_size = 1;
4134  template <typename _Tp> static constexpr size_t _S_size = 1;
4135  template <typename _Tp> static constexpr size_t _S_is_partial = false;
4136 
4137  template <typename _Tp, typename _Abi = simd_abi::scalar>
4138  static constexpr bool _S_is_valid_v
4139  = _Abi::template _IsValid<_Tp>::value && sizeof(_Tp) == _Bytes;
4140  };
4141 
4142 // }}}
4143 // __decay_abi metafunction {{{
4144 template <typename _Tp>
4145  struct __decay_abi { using type = _Tp; };
4146 
4147 template <int _Bytes>
4148  struct __decay_abi<__scalar_abi_wrapper<_Bytes>>
4149  { using type = simd_abi::scalar; };
4150 
4151 // }}}
4152 // __find_next_valid_abi metafunction {{{1
4153 // Given an ABI tag A<N>, find an N2 < N such that A<N2>::_S_is_valid_v<_Tp> ==
4154 // true, N2 is a power-of-2, and A<N2>::_S_is_partial<_Tp> is false. Break
4155 // recursion at 2 elements in the resulting ABI tag. In this case
4156 // type::_S_is_valid_v<_Tp> may be false.
4157 template <template <int> class _Abi, int _Bytes, typename _Tp>
4158  struct __find_next_valid_abi
4159  {
4160  static constexpr auto _S_choose()
4161  {
4162  constexpr int _NextBytes = std::__bit_ceil(_Bytes) / 2;
4163  using _NextAbi = _Abi<_NextBytes>;
4164  if constexpr (_NextBytes < sizeof(_Tp) * 2) // break recursion
4165  return _Abi<_Bytes>();
4166  else if constexpr (_NextAbi::template _S_is_partial<_Tp> == false
4167  && _NextAbi::template _S_is_valid_v<_Tp>)
4168  return _NextAbi();
4169  else
4170  return __find_next_valid_abi<_Abi, _NextBytes, _Tp>::_S_choose();
4171  }
4172 
4173  using type = decltype(_S_choose());
4174  };
4175 
4176 template <int _Bytes, typename _Tp>
4177  struct __find_next_valid_abi<__scalar_abi_wrapper, _Bytes, _Tp>
4178  { using type = simd_abi::scalar; };
4179 
4180 // _AbiList {{{1
4181 template <template <int> class...>
4182  struct _AbiList
4183  {
4184  template <typename, int> static constexpr bool _S_has_valid_abi = false;
4185  template <typename, int> using _FirstValidAbi = void;
4186  template <typename, int> using _BestAbi = void;
4187  };
4188 
4189 template <template <int> class _A0, template <int> class... _Rest>
4190  struct _AbiList<_A0, _Rest...>
4191  {
4192  template <typename _Tp, int _Np>
4193  static constexpr bool _S_has_valid_abi
4194  = _A0<sizeof(_Tp) * _Np>::template _S_is_valid_v<
4195  _Tp> || _AbiList<_Rest...>::template _S_has_valid_abi<_Tp, _Np>;
4196 
4197  template <typename _Tp, int _Np>
4198  using _FirstValidAbi = conditional_t<
4199  _A0<sizeof(_Tp) * _Np>::template _S_is_valid_v<_Tp>,
4200  typename __decay_abi<_A0<sizeof(_Tp) * _Np>>::type,
4201  typename _AbiList<_Rest...>::template _FirstValidAbi<_Tp, _Np>>;
4202 
4203  template <typename _Tp, int _Np>
4204  static constexpr auto _S_determine_best_abi()
4205  {
4206  static_assert(_Np >= 1);
4207  constexpr int _Bytes = sizeof(_Tp) * _Np;
4208  if constexpr (_Np == 1)
4209  return __make_dependent_t<_Tp, simd_abi::scalar>{};
4210  else
4211  {
4212  constexpr int __fullsize = _A0<_Bytes>::template _S_full_size<_Tp>;
4213  // _A0<_Bytes> is good if:
4214  // 1. The ABI tag is valid for _Tp
4215  // 2. The storage overhead is no more than padding to fill the next
4216  // power-of-2 number of bytes
4217  if constexpr (_A0<_Bytes>::template _S_is_valid_v<
4218  _Tp> && __fullsize / 2 < _Np)
4219  return typename __decay_abi<_A0<_Bytes>>::type{};
4220  else
4221  {
4222  using _Bp =
4223  typename __find_next_valid_abi<_A0, _Bytes, _Tp>::type;
4224  if constexpr (_Bp::template _S_is_valid_v<
4225  _Tp> && _Bp::template _S_size<_Tp> <= _Np)
4226  return _Bp{};
4227  else
4228  return
4229  typename _AbiList<_Rest...>::template _BestAbi<_Tp, _Np>{};
4230  }
4231  }
4232  }
4233 
4234  template <typename _Tp, int _Np>
4235  using _BestAbi = decltype(_S_determine_best_abi<_Tp, _Np>());
4236  };
4237 
4238 // }}}1
4239 
4240 // the following lists all native ABIs, which makes them accessible to
4241 // simd_abi::deduce and select_best_vector_type_t (for fixed_size). Order
4242 // matters: Whatever comes first has higher priority.
4243 using _AllNativeAbis = _AbiList<simd_abi::_VecBltnBtmsk, simd_abi::_VecBuiltin,
4244  __scalar_abi_wrapper>;
4245 
4246 // valid _SimdTraits specialization {{{1
4247 template <typename _Tp, typename _Abi>
4248  struct _SimdTraits<_Tp, _Abi, void_t<typename _Abi::template _IsValid<_Tp>>>
4249  : _Abi::template __traits<_Tp> {};
4250 
4251 // __deduce_impl specializations {{{1
4252 // try all native ABIs (including scalar) first
4253 template <typename _Tp, size_t _Np>
4254  struct __deduce_impl<
4255  _Tp, _Np, enable_if_t<_AllNativeAbis::template _S_has_valid_abi<_Tp, _Np>>>
4256  { using type = _AllNativeAbis::_FirstValidAbi<_Tp, _Np>; };
4257 
4258 // fall back to fixed_size only if scalar and native ABIs don't match
4259 template <typename _Tp, size_t _Np, typename = void>
4260  struct __deduce_fixed_size_fallback {};
4261 
4262 template <typename _Tp, size_t _Np>
4263  struct __deduce_fixed_size_fallback<_Tp, _Np,
4264  enable_if_t<simd_abi::fixed_size<_Np>::template _S_is_valid_v<_Tp>>>
4265  { using type = simd_abi::fixed_size<_Np>; };
4266 
4267 template <typename _Tp, size_t _Np, typename>
4268  struct __deduce_impl : public __deduce_fixed_size_fallback<_Tp, _Np> {};
4269 
4270 //}}}1
4271 /// @endcond
4272 
4273 // simd_mask {{{
4274 template <typename _Tp, typename _Abi>
4275  class simd_mask : public _SimdTraits<_Tp, _Abi>::_MaskBase
4276  {
4277  // types, tags, and friends {{{
4278  using _Traits = _SimdTraits<_Tp, _Abi>;
4279  using _MemberType = typename _Traits::_MaskMember;
4280 
4281  // We map all masks with equal element sizeof to a single integer type, the
4282  // one given by __int_for_sizeof_t<_Tp>. This is the approach
4283  // [[gnu::vector_size(N)]] types take as well and it reduces the number of
4284  // template specializations in the implementation classes.
4285  using _Ip = __int_for_sizeof_t<_Tp>;
4286  static constexpr _Ip* _S_type_tag = nullptr;
4287 
4288  friend typename _Traits::_MaskBase;
4289  friend class simd<_Tp, _Abi>; // to construct masks on return
4290  friend typename _Traits::_SimdImpl; // to construct masks on return and
4291  // inspect data on masked operations
4292  public:
4293  using _Impl = typename _Traits::_MaskImpl;
4294  friend _Impl;
4295 
4296  // }}}
4297  // member types {{{
4298  using value_type = bool;
4299  using reference = _SmartReference<_MemberType, _Impl, value_type>;
4300  using simd_type = simd<_Tp, _Abi>;
4301  using abi_type = _Abi;
4302 
4303  // }}}
4304  static constexpr size_t size() // {{{
4305  { return __size_or_zero_v<_Tp, _Abi>; }
4306 
4307  // }}}
4308  // constructors & assignment {{{
4309  simd_mask() = default;
4310  simd_mask(const simd_mask&) = default;
4311  simd_mask(simd_mask&&) = default;
4312  simd_mask& operator=(const simd_mask&) = default;
4313  simd_mask& operator=(simd_mask&&) = default;
4314 
4315  // }}}
4316  // access to internal representation (optional feature) {{{
4317  _GLIBCXX_SIMD_ALWAYS_INLINE explicit
4318  simd_mask(typename _Traits::_MaskCastType __init)
4319  : _M_data{__init} {}
4320  // conversions to internal type is done in _MaskBase
4321 
4322  // }}}
4323  // bitset interface (extension to be proposed) {{{
4324  // TS_FEEDBACK:
4325  // Conversion of simd_mask to and from bitset makes it much easier to
4326  // interface with other facilities. I suggest adding `static
4327  // simd_mask::from_bitset` and `simd_mask::to_bitset`.
4328  _GLIBCXX_SIMD_ALWAYS_INLINE static simd_mask
4329  __from_bitset(bitset<size()> bs)
4330  { return {__bitset_init, bs}; }
4331 
4332  _GLIBCXX_SIMD_ALWAYS_INLINE bitset<size()>
4333  __to_bitset() const
4334  { return _Impl::_S_to_bits(_M_data)._M_to_bitset(); }
4335 
4336  // }}}
4337  // explicit broadcast constructor {{{
4338  _GLIBCXX_SIMD_ALWAYS_INLINE explicit _GLIBCXX_SIMD_CONSTEXPR
4339  simd_mask(value_type __x)
4340  : _M_data(_Impl::template _S_broadcast<_Ip>(__x)) {}
4341 
4342  // }}}
4343  // implicit type conversion constructor {{{
4344  #ifdef _GLIBCXX_SIMD_ENABLE_IMPLICIT_MASK_CAST
4345  // proposed improvement
4346  template <typename _Up, typename _A2,
4347  typename = enable_if_t<simd_size_v<_Up, _A2> == size()>>
4348  _GLIBCXX_SIMD_ALWAYS_INLINE explicit(sizeof(_MemberType)
4349  != sizeof(typename _SimdTraits<_Up, _A2>::_MaskMember))
4350  simd_mask(const simd_mask<_Up, _A2>& __x)
4351  : simd_mask(__proposed::static_simd_cast<simd_mask>(__x)) {}
4352  #else
4353  // conforming to ISO/IEC 19570:2018
4354  template <typename _Up, typename = enable_if_t<conjunction<
4355  is_same<abi_type, simd_abi::fixed_size<size()>>,
4356  is_same<_Up, _Up>>::value>>
4357  _GLIBCXX_SIMD_ALWAYS_INLINE
4358  simd_mask(const simd_mask<_Up, simd_abi::fixed_size<size()>>& __x)
4359  : _M_data(_Impl::_S_from_bitmask(__data(__x), _S_type_tag)) {}
4360  #endif
4361 
4362  // }}}
4363  // load constructor {{{
4364  template <typename _Flags>
4365  _GLIBCXX_SIMD_ALWAYS_INLINE
4366  simd_mask(const value_type* __mem, _Flags)
4367  : _M_data(_Impl::template _S_load<_Ip>(
4368  _Flags::template _S_apply<simd_mask>(__mem))) {}
4369 
4370  template <typename _Flags>
4371  _GLIBCXX_SIMD_ALWAYS_INLINE
4372  simd_mask(const value_type* __mem, simd_mask __k, _Flags)
4373  : _M_data{}
4374  {
4375  _M_data
4376  = _Impl::_S_masked_load(_M_data, __k._M_data,
4377  _Flags::template _S_apply<simd_mask>(__mem));
4378  }
4379 
4380  // }}}
4381  // loads [simd_mask.load] {{{
4382  template <typename _Flags>
4383  _GLIBCXX_SIMD_ALWAYS_INLINE void
4384  copy_from(const value_type* __mem, _Flags)
4385  {
4386  _M_data = _Impl::template _S_load<_Ip>(
4387  _Flags::template _S_apply<simd_mask>(__mem));
4388  }
4389 
4390  // }}}
4391  // stores [simd_mask.store] {{{
4392  template <typename _Flags>
4393  _GLIBCXX_SIMD_ALWAYS_INLINE void
4394  copy_to(value_type* __mem, _Flags) const
4395  { _Impl::_S_store(_M_data, _Flags::template _S_apply<simd_mask>(__mem)); }
4396 
4397  // }}}
4398  // scalar access {{{
4399  _GLIBCXX_SIMD_ALWAYS_INLINE reference
4400  operator[](size_t __i)
4401  {
4402  if (__i >= size())
4403  __invoke_ub("Subscript %d is out of range [0, %d]", __i, size() - 1);
4404  return {_M_data, int(__i)};
4405  }
4406 
4407  _GLIBCXX_SIMD_ALWAYS_INLINE value_type
4408  operator[](size_t __i) const
4409  {
4410  if (__i >= size())
4411  __invoke_ub("Subscript %d is out of range [0, %d]", __i, size() - 1);
4412  if constexpr (__is_scalar_abi<_Abi>())
4413  return _M_data;
4414  else
4415  return static_cast<bool>(_M_data[__i]);
4416  }
4417 
4418  // }}}
4419  // negation {{{
4420  _GLIBCXX_SIMD_ALWAYS_INLINE simd_mask
4421  operator!() const
4422  { return {__private_init, _Impl::_S_bit_not(_M_data)}; }
4423 
4424  // }}}
4425  // simd_mask binary operators [simd_mask.binary] {{{
4426  #ifdef _GLIBCXX_SIMD_ENABLE_IMPLICIT_MASK_CAST
4427  // simd_mask<int> && simd_mask<uint> needs disambiguation
4428  template <typename _Up, typename _A2,
4429  typename
4430  = enable_if_t<is_convertible_v<simd_mask<_Up, _A2>, simd_mask>>>
4431  _GLIBCXX_SIMD_ALWAYS_INLINE friend simd_mask
4432  operator&&(const simd_mask& __x, const simd_mask<_Up, _A2>& __y)
4433  {
4434  return {__private_init,
4435  _Impl::_S_logical_and(__x._M_data, simd_mask(__y)._M_data)};
4436  }
4437 
4438  template <typename _Up, typename _A2,
4439  typename
4440  = enable_if_t<is_convertible_v<simd_mask<_Up, _A2>, simd_mask>>>
4441  _GLIBCXX_SIMD_ALWAYS_INLINE friend simd_mask
4442  operator||(const simd_mask& __x, const simd_mask<_Up, _A2>& __y)
4443  {
4444  return {__private_init,
4445  _Impl::_S_logical_or(__x._M_data, simd_mask(__y)._M_data)};
4446  }
4447  #endif // _GLIBCXX_SIMD_ENABLE_IMPLICIT_MASK_CAST
4448 
4449  _GLIBCXX_SIMD_ALWAYS_INLINE friend simd_mask
4450  operator&&(const simd_mask& __x, const simd_mask& __y)
4451  {
4452  return {__private_init, _Impl::_S_logical_and(__x._M_data, __y._M_data)};
4453  }
4454 
4455  _GLIBCXX_SIMD_ALWAYS_INLINE friend simd_mask
4456  operator||(const simd_mask& __x, const simd_mask& __y)
4457  {
4458  return {__private_init, _Impl::_S_logical_or(__x._M_data, __y._M_data)};
4459  }
4460 
4461  _GLIBCXX_SIMD_ALWAYS_INLINE friend simd_mask
4462  operator&(const simd_mask& __x, const simd_mask& __y)
4463  { return {__private_init, _Impl::_S_bit_and(__x._M_data, __y._M_data)}; }
4464 
4465  _GLIBCXX_SIMD_ALWAYS_INLINE friend simd_mask
4466  operator|(const simd_mask& __x, const simd_mask& __y)
4467  { return {__private_init, _Impl::_S_bit_or(__x._M_data, __y._M_data)}; }
4468 
4469  _GLIBCXX_SIMD_ALWAYS_INLINE friend simd_mask
4470  operator^(const simd_mask& __x, const simd_mask& __y)
4471  { return {__private_init, _Impl::_S_bit_xor(__x._M_data, __y._M_data)}; }
4472 
4473  _GLIBCXX_SIMD_ALWAYS_INLINE friend simd_mask&
4474  operator&=(simd_mask& __x, const simd_mask& __y)
4475  {
4476  __x._M_data = _Impl::_S_bit_and(__x._M_data, __y._M_data);
4477  return __x;
4478  }
4479 
4480  _GLIBCXX_SIMD_ALWAYS_INLINE friend simd_mask&
4481  operator|=(simd_mask& __x, const simd_mask& __y)
4482  {
4483  __x._M_data = _Impl::_S_bit_or(__x._M_data, __y._M_data);
4484  return __x;
4485  }
4486 
4487  _GLIBCXX_SIMD_ALWAYS_INLINE friend simd_mask&
4488  operator^=(simd_mask& __x, const simd_mask& __y)
4489  {
4490  __x._M_data = _Impl::_S_bit_xor(__x._M_data, __y._M_data);
4491  return __x;
4492  }
4493 
4494  // }}}
4495  // simd_mask compares [simd_mask.comparison] {{{
4496  _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend simd_mask
4497  operator==(const simd_mask& __x, const simd_mask& __y)
4498  { return !operator!=(__x, __y); }
4499 
4500  _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend simd_mask
4501  operator!=(const simd_mask& __x, const simd_mask& __y)
4502  { return {__private_init, _Impl::_S_bit_xor(__x._M_data, __y._M_data)}; }
4503 
4504  // }}}
4505  // private_init ctor {{{
4506  _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR
4507  simd_mask(_PrivateInit, typename _Traits::_MaskMember __init)
4508  : _M_data(__init) {}
4509 
4510  // }}}
4511  // private_init generator ctor {{{
4512  template <typename _Fp, typename = decltype(bool(declval<_Fp>()(size_t())))>
4513  _GLIBCXX_SIMD_INTRINSIC constexpr
4514  simd_mask(_PrivateInit, _Fp&& __gen)
4515  : _M_data()
4516  {
4517  __execute_n_times<size()>([&](auto __i) constexpr {
4518  _Impl::_S_set(_M_data, __i, __gen(__i));
4519  });
4520  }
4521 
4522  // }}}
4523  // bitset_init ctor {{{
4524  _GLIBCXX_SIMD_INTRINSIC simd_mask(_BitsetInit, bitset<size()> __init)
4525  : _M_data(
4526  _Impl::_S_from_bitmask(_SanitizedBitMask<size()>(__init), _S_type_tag))
4527  {}
4528 
4529  // }}}
4530  // __cvt {{{
4531  // TS_FEEDBACK:
4532  // The conversion operator this implements should be a ctor on simd_mask.
4533  // Once you call .__cvt() on a simd_mask it converts conveniently.
4534  // A useful variation: add `explicit(sizeof(_Tp) != sizeof(_Up))`
4535  struct _CvtProxy
4536  {
4537  template <typename _Up, typename _A2,
4538  typename
4539  = enable_if_t<simd_size_v<_Up, _A2> == simd_size_v<_Tp, _Abi>>>
4540  operator simd_mask<_Up, _A2>() &&
4541  {
4542  using namespace std::experimental::__proposed;
4543  return static_simd_cast<simd_mask<_Up, _A2>>(_M_data);
4544  }
4545 
4546  const simd_mask<_Tp, _Abi>& _M_data;
4547  };
4548 
4549  _GLIBCXX_SIMD_INTRINSIC _CvtProxy
4550  __cvt() const
4551  { return {*this}; }
4552 
4553  // }}}
4554  // operator?: overloads (suggested extension) {{{
4555  #ifdef __GXX_CONDITIONAL_IS_OVERLOADABLE__
4556  _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend simd_mask
4557  operator?:(const simd_mask& __k, const simd_mask& __where_true,
4558  const simd_mask& __where_false)
4559  {
4560  auto __ret = __where_false;
4561  _Impl::_S_masked_assign(__k._M_data, __ret._M_data, __where_true._M_data);
4562  return __ret;
4563  }
4564 
4565  template <typename _U1, typename _U2,
4566  typename _Rp = simd<common_type_t<_U1, _U2>, _Abi>,
4567  typename = enable_if_t<conjunction_v<
4568  is_convertible<_U1, _Rp>, is_convertible<_U2, _Rp>,
4569  is_convertible<simd_mask, typename _Rp::mask_type>>>>
4570  _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend _Rp
4571  operator?:(const simd_mask& __k, const _U1& __where_true,
4572  const _U2& __where_false)
4573  {
4574  _Rp __ret = __where_false;
4575  _Rp::_Impl::_S_masked_assign(
4576  __data(static_cast<typename _Rp::mask_type>(__k)), __data(__ret),
4577  __data(static_cast<_Rp>(__where_true)));
4578  return __ret;
4579  }
4580 
4581  #ifdef _GLIBCXX_SIMD_ENABLE_IMPLICIT_MASK_CAST
4582  template <typename _Kp, typename _Ak, typename _Up, typename _Au,
4583  typename = enable_if_t<
4584  conjunction_v<is_convertible<simd_mask<_Kp, _Ak>, simd_mask>,
4585  is_convertible<simd_mask<_Up, _Au>, simd_mask>>>>
4586  _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend simd_mask
4587  operator?:(const simd_mask<_Kp, _Ak>& __k, const simd_mask& __where_true,
4588  const simd_mask<_Up, _Au>& __where_false)
4589  {
4590  simd_mask __ret = __where_false;
4591  _Impl::_S_masked_assign(simd_mask(__k)._M_data, __ret._M_data,
4592  __where_true._M_data);
4593  return __ret;
4594  }
4595  #endif // _GLIBCXX_SIMD_ENABLE_IMPLICIT_MASK_CAST
4596  #endif // __GXX_CONDITIONAL_IS_OVERLOADABLE__
4597 
4598  // }}}
4599  // _M_is_constprop {{{
4600  _GLIBCXX_SIMD_INTRINSIC constexpr bool
4601  _M_is_constprop() const
4602  {
4603  if constexpr (__is_scalar_abi<_Abi>())
4604  return __builtin_constant_p(_M_data);
4605  else
4606  return _M_data._M_is_constprop();
4607  }
4608 
4609  // }}}
4610 
4611  private:
4612  friend const auto& __data<_Tp, abi_type>(const simd_mask&);
4613  friend auto& __data<_Tp, abi_type>(simd_mask&);
4614  alignas(_Traits::_S_mask_align) _MemberType _M_data;
4615  };
4616 
4617 // }}}
4618 
4619 /// @cond undocumented
4620 // __data(simd_mask) {{{
4621 template <typename _Tp, typename _Ap>
4622  _GLIBCXX_SIMD_INTRINSIC constexpr const auto&
4623  __data(const simd_mask<_Tp, _Ap>& __x)
4624  { return __x._M_data; }
4625 
4626 template <typename _Tp, typename _Ap>
4627  _GLIBCXX_SIMD_INTRINSIC constexpr auto&
4628  __data(simd_mask<_Tp, _Ap>& __x)
4629  { return __x._M_data; }
4630 
4631 // }}}
4632 /// @endcond
4633 
4634 // simd_mask reductions [simd_mask.reductions] {{{
4635 template <typename _Tp, typename _Abi>
4636  _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR bool
4637  all_of(const simd_mask<_Tp, _Abi>& __k) noexcept
4638  {
4639  if (__builtin_is_constant_evaluated() || __k._M_is_constprop())
4640  {
4641  for (size_t __i = 0; __i < simd_size_v<_Tp, _Abi>; ++__i)
4642  if (!__k[__i])
4643  return false;
4644  return true;
4645  }
4646  else
4647  return _Abi::_MaskImpl::_S_all_of(__k);
4648  }
4649 
4650 template <typename _Tp, typename _Abi>
4651  _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR bool
4652  any_of(const simd_mask<_Tp, _Abi>& __k) noexcept
4653  {
4654  if (__builtin_is_constant_evaluated() || __k._M_is_constprop())
4655  {
4656  for (size_t __i = 0; __i < simd_size_v<_Tp, _Abi>; ++__i)
4657  if (__k[__i])
4658  return true;
4659  return false;
4660  }
4661  else
4662  return _Abi::_MaskImpl::_S_any_of(__k);
4663  }
4664 
4665 template <typename _Tp, typename _Abi>
4666  _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR bool
4667  none_of(const simd_mask<_Tp, _Abi>& __k) noexcept
4668  {
4669  if (__builtin_is_constant_evaluated() || __k._M_is_constprop())
4670  {
4671  for (size_t __i = 0; __i < simd_size_v<_Tp, _Abi>; ++__i)
4672  if (__k[__i])
4673  return false;
4674  return true;
4675  }
4676  else
4677  return _Abi::_MaskImpl::_S_none_of(__k);
4678  }
4679 
4680 template <typename _Tp, typename _Abi>
4681  _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR bool
4682  some_of(const simd_mask<_Tp, _Abi>& __k) noexcept
4683  {
4684  if (__builtin_is_constant_evaluated() || __k._M_is_constprop())
4685  {
4686  for (size_t __i = 1; __i < simd_size_v<_Tp, _Abi>; ++__i)
4687  if (__k[__i] != __k[__i - 1])
4688  return true;
4689  return false;
4690  }
4691  else
4692  return _Abi::_MaskImpl::_S_some_of(__k);
4693  }
4694 
4695 template <typename _Tp, typename _Abi>
4696  _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR int
4697  popcount(const simd_mask<_Tp, _Abi>& __k) noexcept
4698  {
4699  if (__builtin_is_constant_evaluated() || __k._M_is_constprop())
4700  {
4701  const int __r = __call_with_subscripts<simd_size_v<_Tp, _Abi>>(
4702  __k, [](auto... __elements) { return ((__elements != 0) + ...); });
4703  if (__builtin_is_constant_evaluated() || __builtin_constant_p(__r))
4704  return __r;
4705  }
4706  return _Abi::_MaskImpl::_S_popcount(__k);
4707  }
4708 
4709 template <typename _Tp, typename _Abi>
4710  _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR int
4711  find_first_set(const simd_mask<_Tp, _Abi>& __k)
4712  {
4713  if (__builtin_is_constant_evaluated() || __k._M_is_constprop())
4714  {
4715  constexpr size_t _Np = simd_size_v<_Tp, _Abi>;
4716  const size_t _Idx = __call_with_n_evaluations<_Np>(
4717  [](auto... __indexes) { return std::min({__indexes...}); },
4718  [&](auto __i) { return __k[__i] ? +__i : _Np; });
4719  if (_Idx >= _Np)
4720  __invoke_ub("find_first_set(empty mask) is UB");
4721  if (__builtin_constant_p(_Idx))
4722  return _Idx;
4723  }
4724  return _Abi::_MaskImpl::_S_find_first_set(__k);
4725  }
4726 
4727 template <typename _Tp, typename _Abi>
4728  _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR int
4729  find_last_set(const simd_mask<_Tp, _Abi>& __k)
4730  {
4731  if (__builtin_is_constant_evaluated() || __k._M_is_constprop())
4732  {
4733  constexpr size_t _Np = simd_size_v<_Tp, _Abi>;
4734  const int _Idx = __call_with_n_evaluations<_Np>(
4735  [](auto... __indexes) { return std::max({__indexes...}); },
4736  [&](auto __i) { return __k[__i] ? int(__i) : -1; });
4737  if (_Idx < 0)
4738  __invoke_ub("find_first_set(empty mask) is UB");
4739  if (__builtin_constant_p(_Idx))
4740  return _Idx;
4741  }
4742  return _Abi::_MaskImpl::_S_find_last_set(__k);
4743  }
4744 
4745 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR bool
4746 all_of(_ExactBool __x) noexcept
4747 { return __x; }
4748 
4749 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR bool
4750 any_of(_ExactBool __x) noexcept
4751 { return __x; }
4752 
4753 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR bool
4754 none_of(_ExactBool __x) noexcept
4755 { return !__x; }
4756 
4757 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR bool
4758 some_of(_ExactBool) noexcept
4759 { return false; }
4760 
4761 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR int
4762 popcount(_ExactBool __x) noexcept
4763 { return __x; }
4764 
4765 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR int
4766 find_first_set(_ExactBool)
4767 { return 0; }
4768 
4769 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR int
4770 find_last_set(_ExactBool)
4771 { return 0; }
4772 
4773 // }}}
4774 
4775 /// @cond undocumented
4776 // _SimdIntOperators{{{1
4777 template <typename _V, typename _Impl, bool>
4778  class _SimdIntOperators {};
4779 
4780 template <typename _V, typename _Impl>
4781  class _SimdIntOperators<_V, _Impl, true>
4782  {
4783  _GLIBCXX_SIMD_INTRINSIC const _V& __derived() const
4784  { return *static_cast<const _V*>(this); }
4785 
4786  template <typename _Tp>
4787  _GLIBCXX_SIMD_INTRINSIC static _GLIBCXX_SIMD_CONSTEXPR _V
4788  _S_make_derived(_Tp&& __d)
4789  { return {__private_init, static_cast<_Tp&&>(__d)}; }
4790 
4791  public:
4792  _GLIBCXX_SIMD_CONSTEXPR friend _V& operator%=(_V& __lhs, const _V& __x)
4793  { return __lhs = __lhs % __x; }
4794 
4795  _GLIBCXX_SIMD_CONSTEXPR friend _V& operator&=(_V& __lhs, const _V& __x)
4796  { return __lhs = __lhs & __x; }
4797 
4798  _GLIBCXX_SIMD_CONSTEXPR friend _V& operator|=(_V& __lhs, const _V& __x)
4799  { return __lhs = __lhs | __x; }
4800 
4801  _GLIBCXX_SIMD_CONSTEXPR friend _V& operator^=(_V& __lhs, const _V& __x)
4802  { return __lhs = __lhs ^ __x; }
4803 
4804  _GLIBCXX_SIMD_CONSTEXPR friend _V& operator<<=(_V& __lhs, const _V& __x)
4805  { return __lhs = __lhs << __x; }
4806 
4807  _GLIBCXX_SIMD_CONSTEXPR friend _V& operator>>=(_V& __lhs, const _V& __x)
4808  { return __lhs = __lhs >> __x; }
4809 
4810  _GLIBCXX_SIMD_CONSTEXPR friend _V& operator<<=(_V& __lhs, int __x)
4811  { return __lhs = __lhs << __x; }
4812 
4813  _GLIBCXX_SIMD_CONSTEXPR friend _V& operator>>=(_V& __lhs, int __x)
4814  { return __lhs = __lhs >> __x; }
4815 
4816  _GLIBCXX_SIMD_CONSTEXPR friend _V operator%(const _V& __x, const _V& __y)
4817  {
4818  return _SimdIntOperators::_S_make_derived(
4819  _Impl::_S_modulus(__data(__x), __data(__y)));
4820  }
4821 
4822  _GLIBCXX_SIMD_CONSTEXPR friend _V operator&(const _V& __x, const _V& __y)
4823  {
4824  return _SimdIntOperators::_S_make_derived(
4825  _Impl::_S_bit_and(__data(__x), __data(__y)));
4826  }
4827 
4828  _GLIBCXX_SIMD_CONSTEXPR friend _V operator|(const _V& __x, const _V& __y)
4829  {
4830  return _SimdIntOperators::_S_make_derived(
4831  _Impl::_S_bit_or(__data(__x), __data(__y)));
4832  }
4833 
4834  _GLIBCXX_SIMD_CONSTEXPR friend _V operator^(const _V& __x, const _V& __y)
4835  {
4836  return _SimdIntOperators::_S_make_derived(
4837  _Impl::_S_bit_xor(__data(__x), __data(__y)));
4838  }
4839 
4840  _GLIBCXX_SIMD_CONSTEXPR friend _V operator<<(const _V& __x, const _V& __y)
4841  {
4842  return _SimdIntOperators::_S_make_derived(
4843  _Impl::_S_bit_shift_left(__data(__x), __data(__y)));
4844  }
4845 
4846  _GLIBCXX_SIMD_CONSTEXPR friend _V operator>>(const _V& __x, const _V& __y)
4847  {
4848  return _SimdIntOperators::_S_make_derived(
4849  _Impl::_S_bit_shift_right(__data(__x), __data(__y)));
4850  }
4851 
4852  template <typename _VV = _V>
4853  _GLIBCXX_SIMD_CONSTEXPR friend _V operator<<(const _V& __x, int __y)
4854  {
4855  using _Tp = typename _VV::value_type;
4856  if (__y < 0)
4857  __invoke_ub("The behavior is undefined if the right operand of a "
4858  "shift operation is negative. [expr.shift]\nA shift by "
4859  "%d was requested",
4860  __y);
4861  if (size_t(__y) >= sizeof(declval<_Tp>() << __y) * __CHAR_BIT__)
4862  __invoke_ub(
4863  "The behavior is undefined if the right operand of a "
4864  "shift operation is greater than or equal to the width of the "
4865  "promoted left operand. [expr.shift]\nA shift by %d was requested",
4866  __y);
4867  return _SimdIntOperators::_S_make_derived(
4868  _Impl::_S_bit_shift_left(__data(__x), __y));
4869  }
4870 
4871  template <typename _VV = _V>
4872  _GLIBCXX_SIMD_CONSTEXPR friend _V operator>>(const _V& __x, int __y)
4873  {
4874  using _Tp = typename _VV::value_type;
4875  if (__y < 0)
4876  __invoke_ub(
4877  "The behavior is undefined if the right operand of a shift "
4878  "operation is negative. [expr.shift]\nA shift by %d was requested",
4879  __y);
4880  if (size_t(__y) >= sizeof(declval<_Tp>() << __y) * __CHAR_BIT__)
4881  __invoke_ub(
4882  "The behavior is undefined if the right operand of a shift "
4883  "operation is greater than or equal to the width of the promoted "
4884  "left operand. [expr.shift]\nA shift by %d was requested",
4885  __y);
4886  return _SimdIntOperators::_S_make_derived(
4887  _Impl::_S_bit_shift_right(__data(__x), __y));
4888  }
4889 
4890  // unary operators (for integral _Tp)
4891  _GLIBCXX_SIMD_CONSTEXPR _V operator~() const
4892  { return {__private_init, _Impl::_S_complement(__derived()._M_data)}; }
4893  };
4894 
4895 //}}}1
4896 /// @endcond
4897 
4898 // simd {{{
4899 template <typename _Tp, typename _Abi>
4900  class simd : public _SimdIntOperators<
4901  simd<_Tp, _Abi>, typename _SimdTraits<_Tp, _Abi>::_SimdImpl,
4902  conjunction<is_integral<_Tp>,
4903  typename _SimdTraits<_Tp, _Abi>::_IsValid>::value>,
4904  public _SimdTraits<_Tp, _Abi>::_SimdBase
4905  {
4906  using _Traits = _SimdTraits<_Tp, _Abi>;
4907  using _MemberType = typename _Traits::_SimdMember;
4908  using _CastType = typename _Traits::_SimdCastType;
4909  static constexpr _Tp* _S_type_tag = nullptr;
4910  friend typename _Traits::_SimdBase;
4911 
4912  public:
4913  using _Impl = typename _Traits::_SimdImpl;
4914  friend _Impl;
4915  friend _SimdIntOperators<simd, _Impl, true>;
4916 
4917  using value_type = _Tp;
4918  using reference = _SmartReference<_MemberType, _Impl, value_type>;
4919  using mask_type = simd_mask<_Tp, _Abi>;
4920  using abi_type = _Abi;
4921 
4922  static constexpr size_t size()
4923  { return __size_or_zero_v<_Tp, _Abi>; }
4924 
4925  _GLIBCXX_SIMD_CONSTEXPR simd() = default;
4926  _GLIBCXX_SIMD_CONSTEXPR simd(const simd&) = default;
4927  _GLIBCXX_SIMD_CONSTEXPR simd(simd&&) noexcept = default;
4928  _GLIBCXX_SIMD_CONSTEXPR simd& operator=(const simd&) = default;
4929  _GLIBCXX_SIMD_CONSTEXPR simd& operator=(simd&&) noexcept = default;
4930 
4931  // implicit broadcast constructor
4932  template <typename _Up,
4933  typename = enable_if_t<!is_same_v<__remove_cvref_t<_Up>, bool>>>
4934  _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
4935  simd(_ValuePreservingOrInt<_Up, value_type>&& __x)
4936  : _M_data(
4937  _Impl::_S_broadcast(static_cast<value_type>(static_cast<_Up&&>(__x))))
4938  {}
4939 
4940  // implicit type conversion constructor (convert from fixed_size to
4941  // fixed_size)
4942  template <typename _Up>
4943  _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
4944  simd(const simd<_Up, simd_abi::fixed_size<size()>>& __x,
4945  enable_if_t<
4946  conjunction<
4947  is_same<simd_abi::fixed_size<size()>, abi_type>,
4948  negation<__is_narrowing_conversion<_Up, value_type>>,
4949  __converts_to_higher_integer_rank<_Up, value_type>>::value,
4950  void*> = nullptr)
4951  : simd{static_cast<array<_Up, size()>>(__x).data(), vector_aligned} {}
4952 
4953  // explicit type conversion constructor
4954 #ifdef _GLIBCXX_SIMD_ENABLE_STATIC_CAST
4955  template <typename _Up, typename _A2,
4956  typename = decltype(static_simd_cast<simd>(
4957  declval<const simd<_Up, _A2>&>()))>
4958  _GLIBCXX_SIMD_ALWAYS_INLINE explicit _GLIBCXX_SIMD_CONSTEXPR
4959  simd(const simd<_Up, _A2>& __x)
4960  : simd(static_simd_cast<simd>(__x)) {}
4961 #endif // _GLIBCXX_SIMD_ENABLE_STATIC_CAST
4962 
4963  // generator constructor
4964  template <typename _Fp>
4965  _GLIBCXX_SIMD_ALWAYS_INLINE explicit _GLIBCXX_SIMD_CONSTEXPR
4966  simd(_Fp&& __gen, _ValuePreservingOrInt<decltype(declval<_Fp>()(
4967  declval<_SizeConstant<0>&>())),
4968  value_type>* = nullptr)
4969  : _M_data(_Impl::_S_generator(static_cast<_Fp&&>(__gen), _S_type_tag)) {}
4970 
4971  // load constructor
4972  template <typename _Up, typename _Flags>
4973  _GLIBCXX_SIMD_ALWAYS_INLINE
4974  simd(const _Up* __mem, _Flags)
4975  : _M_data(
4976  _Impl::_S_load(_Flags::template _S_apply<simd>(__mem), _S_type_tag))
4977  {}
4978 
4979  // loads [simd.load]
4980  template <typename _Up, typename _Flags>
4981  _GLIBCXX_SIMD_ALWAYS_INLINE void
4982  copy_from(const _Vectorizable<_Up>* __mem, _Flags)
4983  {
4984  _M_data = static_cast<decltype(_M_data)>(
4985  _Impl::_S_load(_Flags::template _S_apply<simd>(__mem), _S_type_tag));
4986  }
4987 
4988  // stores [simd.store]
4989  template <typename _Up, typename _Flags>
4990  _GLIBCXX_SIMD_ALWAYS_INLINE void
4991  copy_to(_Vectorizable<_Up>* __mem, _Flags) const
4992  {
4993  _Impl::_S_store(_M_data, _Flags::template _S_apply<simd>(__mem),
4994  _S_type_tag);
4995  }
4996 
4997  // scalar access
4998  _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR reference
4999  operator[](size_t __i)
5000  { return {_M_data, int(__i)}; }
5001 
5002  _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR value_type
5003  operator[]([[maybe_unused]] size_t __i) const
5004  {
5005  if constexpr (__is_scalar_abi<_Abi>())
5006  {
5007  _GLIBCXX_DEBUG_ASSERT(__i == 0);
5008  return _M_data;
5009  }
5010  else
5011  return _M_data[__i];
5012  }
5013 
5014  // increment and decrement:
5015  _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR simd&
5016  operator++()
5017  {
5018  _Impl::_S_increment(_M_data);
5019  return *this;
5020  }
5021 
5022  _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR simd
5023  operator++(int)
5024  {
5025  simd __r = *this;
5026  _Impl::_S_increment(_M_data);
5027  return __r;
5028  }
5029 
5030  _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR simd&
5031  operator--()
5032  {
5033  _Impl::_S_decrement(_M_data);
5034  return *this;
5035  }
5036 
5037  _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR simd
5038  operator--(int)
5039  {
5040  simd __r = *this;
5041  _Impl::_S_decrement(_M_data);
5042  return __r;
5043  }
5044 
5045  // unary operators (for any _Tp)
5046  _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR mask_type
5047  operator!() const
5048  { return {__private_init, _Impl::_S_negate(_M_data)}; }
5049 
5050  _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR simd
5051  operator+() const
5052  { return *this; }
5053 
5054  _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR simd
5055  operator-() const
5056  { return {__private_init, _Impl::_S_unary_minus(_M_data)}; }
5057 
5058  // access to internal representation (suggested extension)
5059  _GLIBCXX_SIMD_ALWAYS_INLINE explicit _GLIBCXX_SIMD_CONSTEXPR
5060  simd(_CastType __init) : _M_data(__init) {}
5061 
5062  // compound assignment [simd.cassign]
5063  _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend simd&
5064  operator+=(simd& __lhs, const simd& __x)
5065  { return __lhs = __lhs + __x; }
5066 
5067  _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend simd&
5068  operator-=(simd& __lhs, const simd& __x)
5069  { return __lhs = __lhs - __x; }
5070 
5071  _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend simd&
5072  operator*=(simd& __lhs, const simd& __x)
5073  { return __lhs = __lhs * __x; }
5074 
5075  _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend simd&
5076  operator/=(simd& __lhs, const simd& __x)
5077  { return __lhs = __lhs / __x; }
5078 
5079  // binary operators [simd.binary]
5080  _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend simd
5081  operator+(const simd& __x, const simd& __y)
5082  { return {__private_init, _Impl::_S_plus(__x._M_data, __y._M_data)}; }
5083 
5084  _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend simd
5085  operator-(const simd& __x, const simd& __y)
5086  { return {__private_init, _Impl::_S_minus(__x._M_data, __y._M_data)}; }
5087 
5088  _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend simd
5089  operator*(const simd& __x, const simd& __y)
5090  { return {__private_init, _Impl::_S_multiplies(__x._M_data, __y._M_data)}; }
5091 
5092  _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend simd
5093  operator/(const simd& __x, const simd& __y)
5094  { return {__private_init, _Impl::_S_divides(__x._M_data, __y._M_data)}; }
5095 
5096  // compares [simd.comparison]
5097  _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend mask_type
5098  operator==(const simd& __x, const simd& __y)
5099  { return simd::_S_make_mask(_Impl::_S_equal_to(__x._M_data, __y._M_data)); }
5100 
5101  _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend mask_type
5102  operator!=(const simd& __x, const simd& __y)
5103  {
5104  return simd::_S_make_mask(
5105  _Impl::_S_not_equal_to(__x._M_data, __y._M_data));
5106  }
5107 
5108  _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend mask_type
5109  operator<(const simd& __x, const simd& __y)
5110  { return simd::_S_make_mask(_Impl::_S_less(__x._M_data, __y._M_data)); }
5111 
5112  _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend mask_type
5113  operator<=(const simd& __x, const simd& __y)
5114  {
5115  return simd::_S_make_mask(_Impl::_S_less_equal(__x._M_data, __y._M_data));
5116  }
5117 
5118  _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend mask_type
5119  operator>(const simd& __x, const simd& __y)
5120  { return simd::_S_make_mask(_Impl::_S_less(__y._M_data, __x._M_data)); }
5121 
5122  _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend mask_type
5123  operator>=(const simd& __x, const simd& __y)
5124  {
5125  return simd::_S_make_mask(_Impl::_S_less_equal(__y._M_data, __x._M_data));
5126  }
5127 
5128  // operator?: overloads (suggested extension) {{{
5129 #ifdef __GXX_CONDITIONAL_IS_OVERLOADABLE__
5130  _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend simd
5131  operator?:(const mask_type& __k, const simd& __where_true,
5132  const simd& __where_false)
5133  {
5134  auto __ret = __where_false;
5135  _Impl::_S_masked_assign(__data(__k), __data(__ret), __data(__where_true));
5136  return __ret;
5137  }
5138 
5139 #endif // __GXX_CONDITIONAL_IS_OVERLOADABLE__
5140  // }}}
5141 
5142  // "private" because of the first arguments's namespace
5143  _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR
5144  simd(_PrivateInit, const _MemberType& __init)
5145  : _M_data(__init) {}
5146 
5147  // "private" because of the first arguments's namespace
5148  _GLIBCXX_SIMD_INTRINSIC
5149  simd(_BitsetInit, bitset<size()> __init) : _M_data()
5150  { where(mask_type(__bitset_init, __init), *this) = ~*this; }
5151 
5152  _GLIBCXX_SIMD_INTRINSIC constexpr bool
5153  _M_is_constprop() const
5154  {
5155  if constexpr (__is_scalar_abi<_Abi>())
5156  return __builtin_constant_p(_M_data);
5157  else
5158  return _M_data._M_is_constprop();
5159  }
5160 
5161  private:
5162  _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR static mask_type
5163  _S_make_mask(typename mask_type::_MemberType __k)
5164  { return {__private_init, __k}; }
5165 
5166  friend const auto& __data<value_type, abi_type>(const simd&);
5167  friend auto& __data<value_type, abi_type>(simd&);
5168  alignas(_Traits::_S_simd_align) _MemberType _M_data;
5169  };
5170 
5171 // }}}
5172 /// @cond undocumented
5173 // __data {{{
5174 template <typename _Tp, typename _Ap>
5175  _GLIBCXX_SIMD_INTRINSIC constexpr const auto&
5176  __data(const simd<_Tp, _Ap>& __x)
5177  { return __x._M_data; }
5178 
5179 template <typename _Tp, typename _Ap>
5180  _GLIBCXX_SIMD_INTRINSIC constexpr auto&
5181  __data(simd<_Tp, _Ap>& __x)
5182  { return __x._M_data; }
5183 
5184 // }}}
5185 namespace __float_bitwise_operators { //{{{
5186 template <typename _Tp, typename _Ap>
5187  _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR simd<_Tp, _Ap>
5188  operator^(const simd<_Tp, _Ap>& __a, const simd<_Tp, _Ap>& __b)
5189  {
5190  return {__private_init,
5191  _Ap::_SimdImpl::_S_bit_xor(__data(__a), __data(__b))};
5192  }
5193 
5194 template <typename _Tp, typename _Ap>
5195  _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR simd<_Tp, _Ap>
5196  operator|(const simd<_Tp, _Ap>& __a, const simd<_Tp, _Ap>& __b)
5197  {
5198  return {__private_init,
5199  _Ap::_SimdImpl::_S_bit_or(__data(__a), __data(__b))};
5200  }
5201 
5202 template <typename _Tp, typename _Ap>
5203  _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR simd<_Tp, _Ap>
5204  operator&(const simd<_Tp, _Ap>& __a, const simd<_Tp, _Ap>& __b)
5205  {
5206  return {__private_init,
5207  _Ap::_SimdImpl::_S_bit_and(__data(__a), __data(__b))};
5208  }
5209 } // namespace __float_bitwise_operators }}}
5210 /// @endcond
5211 
5212 /// @}
5213 _GLIBCXX_SIMD_END_NAMESPACE
5214 
5215 #endif // __cplusplus >= 201703L
5216 #endif // _GLIBCXX_EXPERIMENTAL_SIMD_H
5217 
5218 // vim: foldmethod=marker foldmarker={{{,}}}
constexpr duration< __common_rep_t< _Rep1, __disable_if_is_duration< _Rep2 > >, _Period > operator%(const duration< _Rep1, _Period > &__d, const _Rep2 &__s)
Definition: chrono:729
constexpr time_point< _Clock, typename common_type< duration< _Rep1, _Period1 >, _Dur2 >::type > operator+(const duration< _Rep1, _Period1 > &__lhs, const time_point< _Clock, _Dur2 > &__rhs)
Adjust a time point forwards by the given duration.
Definition: chrono:1016
constexpr duration< __common_rep_t< _Rep2, _Rep1 >, _Period > operator*(const _Rep1 &__s, const duration< _Rep2, _Period > &__d)
Definition: chrono:700
constexpr common_type< duration< _Rep1, _Period1 >, duration< _Rep2, _Period2 > >::type operator-(const duration< _Rep1, _Period1 > &__lhs, const duration< _Rep2, _Period2 > &__rhs)
The difference between two durations.
Definition: chrono:660
constexpr duration< __common_rep_t< _Rep1, __disable_if_is_duration< _Rep2 > >, _Period > operator/(const duration< _Rep1, _Period > &__d, const _Rep2 &__s)
Definition: chrono:706
typename remove_reference< _Tp >::type remove_reference_t
Alias template for remove_reference.
Definition: type_traits:1645
typename make_unsigned< _Tp >::type make_unsigned_t
Alias template for make_unsigned.
Definition: type_traits:1980
void void_t
A metafunction that always yields void, used for detecting valid types.
Definition: type_traits:2607
integral_constant< bool, true > true_type
The type used as a compile-time boolean with true value.
Definition: type_traits:83
typename conditional< _Cond, _Iftrue, _Iffalse >::type conditional_t
Alias template for conditional.
Definition: type_traits:2589
typename remove_pointer< _Tp >::type remove_pointer_t
Alias template for remove_pointer.
Definition: type_traits:2055
integral_constant< bool, false > false_type
The type used as a compile-time boolean with false value.
Definition: type_traits:86
typename remove_const< _Tp >::type remove_const_t
Alias template for remove_const.
Definition: type_traits:1576
typename enable_if< _Cond, _Tp >::type enable_if_t
Alias template for enable_if.
Definition: type_traits:2585
constexpr auto tuple_cat(_Tpls &&... __tpls) -> typename __tuple_cat_result< _Tpls... >::__type
tuple_cat
Definition: tuple:1732
auto declval() noexcept -> decltype(__declval< _Tp >(0))
Definition: type_traits:2364
constexpr std::remove_reference< _Tp >::type && move(_Tp &&__t) noexcept
Convert a value to an rvalue.
Definition: move.h:104
void swap(any &__x, any &__y) noexcept
Exchange the states of two any objects.
Definition: any:428
_Tp * end(valarray< _Tp > &__va) noexcept
Return an iterator pointing to one past the last element of the valarray.
Definition: valarray:1237
_Tp * begin(valarray< _Tp > &__va) noexcept
Return an iterator pointing to the first element of the valarray.
Definition: valarray:1215
constexpr const _Tp & clamp(const _Tp &, const _Tp &, const _Tp &)
Returns the value clamped between lo and hi.
Definition: stl_algo.h:3656
constexpr const _Tp & max(const _Tp &, const _Tp &)
This does what you think it does.
Definition: stl_algobase.h:254
constexpr pair< const _Tp &, const _Tp & > minmax(const _Tp &, const _Tp &)
Determines min and max at once as an ordered pair.
Definition: stl_algo.h:3301
constexpr const _Tp & min(const _Tp &, const _Tp &)
This does what you think it does.
Definition: stl_algobase.h:230
constexpr _Tp reduce(_InputIterator __first, _InputIterator __last, _Tp __init, _BinaryOperation __binary_op)
Calculate reduction of values in a range.
Definition: numeric:278
std::basic_istream< _CharT, _Traits > & operator>>(std::basic_istream< _CharT, _Traits > &__is, bitset< _Nb > &__x)
Global I/O operators for bitsets.
Definition: bitset:1472
bitset< _Nb > operator^(const bitset< _Nb > &__x, const bitset< _Nb > &__y) noexcept
Global bitwise operations on bitsets.
Definition: bitset:1453
std::basic_ostream< _CharT, _Traits > & operator<<(std::basic_ostream< _CharT, _Traits > &__os, const bitset< _Nb > &__x)
Global I/O operators for bitsets.
Definition: bitset:1540
bitset< _Nb > operator|(const bitset< _Nb > &__x, const bitset< _Nb > &__y) noexcept
Global bitwise operations on bitsets.
Definition: bitset:1444
bitset< _Nb > operator&(const bitset< _Nb > &__x, const bitset< _Nb > &__y) noexcept
Global bitwise operations on bitsets.
Definition: bitset:1435
constexpr auto size(const _Container &__cont) noexcept(noexcept(__cont.size())) -> decltype(__cont.size())
Return the size of a container.
Definition: range_access.h:245
constexpr auto data(_Container &__cont) noexcept(noexcept(__cont.data())) -> decltype(__cont.data())
Return the data pointer of a container.
Definition: range_access.h:290