19 #ifndef HWY_HIGHWAY_INCLUDED
20 #define HWY_HIGHWAY_INCLUDED
41 #define HWY_FULL1(T) hwy::HWY_NAMESPACE::ScalableTag<T>
42 #define HWY_FULL2(T, LMUL) \
43 hwy::HWY_NAMESPACE::ScalableTag<T, CeilLog2(HWY_MAX(0, LMUL))>
44 #define HWY_3TH_ARG(arg1, arg2, arg3, ...) arg3
46 #define HWY_FULL_RECOMPOSER(args_with_paren) HWY_3TH_ARG args_with_paren
48 #define HWY_CHOOSE_FULL(...) \
49 HWY_FULL_RECOMPOSER((__VA_ARGS__, HWY_FULL2, HWY_FULL1, ))
50 #define HWY_FULL(...) HWY_CHOOSE_FULL(__VA_ARGS__())(__VA_ARGS__)
53 #define HWY_CAPPED(T, MAX_N) \
54 hwy::HWY_NAMESPACE::CappedTag<T, HWY_MIN(MAX_N, HWY_LANES(T))>
70 #if HWY_STATIC_TARGET == HWY_SCALAR
71 #define HWY_STATIC_DISPATCH(FUNC_NAME) N_SCALAR::FUNC_NAME
72 #elif HWY_STATIC_TARGET == HWY_RVV
73 #define HWY_STATIC_DISPATCH(FUNC_NAME) N_RVV::FUNC_NAME
74 #elif HWY_STATIC_TARGET == HWY_WASM2
75 #define HWY_STATIC_DISPATCH(FUNC_NAME) N_WASM2::FUNC_NAME
76 #elif HWY_STATIC_TARGET == HWY_WASM
77 #define HWY_STATIC_DISPATCH(FUNC_NAME) N_WASM::FUNC_NAME
78 #elif HWY_STATIC_TARGET == HWY_NEON
79 #define HWY_STATIC_DISPATCH(FUNC_NAME) N_NEON::FUNC_NAME
80 #elif HWY_STATIC_TARGET == HWY_SVE
81 #define HWY_STATIC_DISPATCH(FUNC_NAME) N_SVE::FUNC_NAME
82 #elif HWY_STATIC_TARGET == HWY_SVE2
83 #define HWY_STATIC_DISPATCH(FUNC_NAME) N_SVE2::FUNC_NAME
84 #elif HWY_STATIC_TARGET == HWY_PPC8
85 #define HWY_STATIC_DISPATCH(FUNC_NAME) N_PPC8::FUNC_NAME
86 #elif HWY_STATIC_TARGET == HWY_SSSE3
87 #define HWY_STATIC_DISPATCH(FUNC_NAME) N_SSSE3::FUNC_NAME
88 #elif HWY_STATIC_TARGET == HWY_SSE4
89 #define HWY_STATIC_DISPATCH(FUNC_NAME) N_SSE4::FUNC_NAME
90 #elif HWY_STATIC_TARGET == HWY_AVX2
91 #define HWY_STATIC_DISPATCH(FUNC_NAME) N_AVX2::FUNC_NAME
92 #elif HWY_STATIC_TARGET == HWY_AVX3
93 #define HWY_STATIC_DISPATCH(FUNC_NAME) N_AVX3::FUNC_NAME
94 #elif HWY_STATIC_TARGET == HWY_AVX3_DL
95 #define HWY_STATIC_DISPATCH(FUNC_NAME) N_AVX3_DL::FUNC_NAME
100 template <
typename RetType,
typename... Args>
112 template <FunctionType* const table[]>
117 return (table[chosen_target.
GetIndex()])(args...);
123 template <
typename RetType,
typename... Args>
130 #if HWY_TARGETS & HWY_SCALAR
131 #define HWY_CHOOSE_SCALAR(FUNC_NAME) &N_SCALAR::FUNC_NAME
136 #define HWY_CHOOSE_SCALAR(FUNC_NAME) &HWY_STATIC_DISPATCH(FUNC_NAME)
139 #if HWY_TARGETS & HWY_WASM2
140 #define HWY_CHOOSE_WASM2(FUNC_NAME) &N_WASM2::FUNC_NAME
142 #define HWY_CHOOSE_WASM2(FUNC_NAME) nullptr
145 #if HWY_TARGETS & HWY_WASM
146 #define HWY_CHOOSE_WASM(FUNC_NAME) &N_WASM::FUNC_NAME
148 #define HWY_CHOOSE_WASM(FUNC_NAME) nullptr
151 #if HWY_TARGETS & HWY_RVV
152 #define HWY_CHOOSE_RVV(FUNC_NAME) &N_RVV::FUNC_NAME
154 #define HWY_CHOOSE_RVV(FUNC_NAME) nullptr
157 #if HWY_TARGETS & HWY_NEON
158 #define HWY_CHOOSE_NEON(FUNC_NAME) &N_NEON::FUNC_NAME
160 #define HWY_CHOOSE_NEON(FUNC_NAME) nullptr
163 #if HWY_TARGETS & HWY_SVE
164 #define HWY_CHOOSE_SVE(FUNC_NAME) &N_SVE::FUNC_NAME
166 #define HWY_CHOOSE_SVE(FUNC_NAME) nullptr
169 #if HWY_TARGETS & HWY_SVE2
170 #define HWY_CHOOSE_SVE2(FUNC_NAME) &N_SVE2::FUNC_NAME
172 #define HWY_CHOOSE_SVE2(FUNC_NAME) nullptr
175 #if HWY_TARGETS & HWY_PPC8
176 #define HWY_CHOOSE_PCC8(FUNC_NAME) &N_PPC8::FUNC_NAME
178 #define HWY_CHOOSE_PPC8(FUNC_NAME) nullptr
181 #if HWY_TARGETS & HWY_SSSE3
182 #define HWY_CHOOSE_SSSE3(FUNC_NAME) &N_SSSE3::FUNC_NAME
184 #define HWY_CHOOSE_SSSE3(FUNC_NAME) nullptr
187 #if HWY_TARGETS & HWY_SSE4
188 #define HWY_CHOOSE_SSE4(FUNC_NAME) &N_SSE4::FUNC_NAME
190 #define HWY_CHOOSE_SSE4(FUNC_NAME) nullptr
193 #if HWY_TARGETS & HWY_AVX2
194 #define HWY_CHOOSE_AVX2(FUNC_NAME) &N_AVX2::FUNC_NAME
196 #define HWY_CHOOSE_AVX2(FUNC_NAME) nullptr
199 #if HWY_TARGETS & HWY_AVX3
200 #define HWY_CHOOSE_AVX3(FUNC_NAME) &N_AVX3::FUNC_NAME
202 #define HWY_CHOOSE_AVX3(FUNC_NAME) nullptr
205 #if HWY_TARGETS & HWY_AVX3_DL
206 #define HWY_CHOOSE_AVX3_DL(FUNC_NAME) &N_AVX3_DL::FUNC_NAME
208 #define HWY_CHOOSE_AVX3_DL(FUNC_NAME) nullptr
211 #define HWY_DISPATCH_TABLE(FUNC_NAME) \
212 HWY_CONCAT(FUNC_NAME, HighwayDispatchTable)
243 #if HWY_IDE || ((HWY_TARGETS & (HWY_TARGETS - 1)) == 0)
249 #define HWY_EXPORT(FUNC_NAME) \
250 HWY_MAYBE_UNUSED static decltype(&HWY_STATIC_DISPATCH(FUNC_NAME)) \
251 const HWY_DISPATCH_TABLE(FUNC_NAME)[1] = { \
252 &HWY_STATIC_DISPATCH(FUNC_NAME)}
253 #define HWY_DYNAMIC_DISPATCH(FUNC_NAME) HWY_STATIC_DISPATCH(FUNC_NAME)
259 #define HWY_EXPORT(FUNC_NAME) \
260 static decltype(&HWY_STATIC_DISPATCH(FUNC_NAME)) \
261 const HWY_DISPATCH_TABLE(FUNC_NAME)[HWY_MAX_DYNAMIC_TARGETS + 2] = { \
264 &decltype(hwy::FunctionCacheFactory(&HWY_STATIC_DISPATCH( \
265 FUNC_NAME)))::ChooseAndCall<HWY_DISPATCH_TABLE(FUNC_NAME)>, \
266 HWY_CHOOSE_TARGET_LIST(FUNC_NAME), \
267 HWY_CHOOSE_SCALAR(FUNC_NAME), \
269 #define HWY_DYNAMIC_DISPATCH(FUNC_NAME) \
270 (*(HWY_DISPATCH_TABLE(FUNC_NAME)[hwy::GetChosenTarget().GetIndex()]))
275 #define HWY_CAP_INTEGER64 HWY_HAVE_INTEGER64
276 #define HWY_CAP_FLOAT16 HWY_HAVE_FLOAT16
277 #define HWY_CAP_FLOAT64 HWY_HAVE_FLOAT64
288 #if defined(HWY_HIGHWAY_PER_TARGET) == defined(HWY_TARGET_TOGGLE)
289 #ifdef HWY_HIGHWAY_PER_TARGET
290 #undef HWY_HIGHWAY_PER_TARGET
292 #define HWY_HIGHWAY_PER_TARGET
296 #if HWY_TARGET == HWY_SSSE3 || HWY_TARGET == HWY_SSE4
298 #elif HWY_TARGET == HWY_AVX2
300 #elif HWY_TARGET == HWY_AVX3 || HWY_TARGET == HWY_AVX3_DL
302 #elif HWY_TARGET == HWY_PPC8
303 #error "PPC is not yet supported"
304 #elif HWY_TARGET == HWY_NEON
306 #elif HWY_TARGET == HWY_SVE || HWY_TARGET == HWY_SVE2
308 #elif HWY_TARGET == HWY_WASM2
310 #elif HWY_TARGET == HWY_WASM
312 #elif HWY_TARGET == HWY_RVV
314 #elif HWY_TARGET == HWY_SCALAR
317 #pragma message("HWY_TARGET does not match any known target")
Definition: aligned_allocator.h:27
FunctionCache< RetType, Args... > FunctionCacheFactory(RetType(*)(Args...))
Definition: highway.h:124
HWY_DLLEXPORT ChosenTarget & GetChosenTarget()
Definition: targets.h:230
HWY_DLLEXPORT void Update()
size_t HWY_INLINE GetIndex() const
Definition: targets.h:248
Definition: highway.h:101
RetType() FunctionType(Args...)
Definition: highway.h:103
static RetType ChooseAndCall(Args... args)
Definition: highway.h:113