diff --git a/cmake/dispatch-levels.cmake b/cmake/dispatch-levels.cmake new file mode 100644 index 000000000..3da39f8fa --- /dev/null +++ b/cmake/dispatch-levels.cmake @@ -0,0 +1,60 @@ +# Copyright 2026 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +## +## This is a cmake helper to split a spec provided as `a|b|c` +## into three fields `[a, b, c]` +## + +include_guard(GLOBAL) + +# Scoped this file via PUSH/POP so it does not affect the includer +cmake_policy(PUSH) +cmake_policy(SET CMP0007 NEW) + +# Splits `spec` on '|' into `n` fields +function(svs_split_fields spec n) + string(REPLACE "|" ";" fields "${spec}") + list(LENGTH fields n_fields) + if(NOT n_fields EQUAL n) + message(FATAL_ERROR "Malformed spec '${spec}': expected ${n} '|'-separated fields, got ${n_fields}.") + endif() + math(EXPR last_index "${n} - 1") + foreach(i RANGE 0 ${last_index}) + list(GET fields ${i} value) + list(GET ARGN ${i} out_var) + set(${out_var} "${value}" PARENT_SCOPE) + endforeach() +endfunction() + +# e.g. svs_parse_isa_level("AVX2|haswell|avx2" level arch infix) sets +# level=AVX2, arch=haswell, infix=avx2 +function(svs_parse_isa_level spec out_level out_arch out_infix) + svs_split_fields("${spec}" 3 level arch infix) + set(${out_level} "${level}" PARENT_SCOPE) + set(${out_arch} "${arch}" PARENT_SCOPE) + set(${out_infix} "${infix}" PARENT_SCOPE) +endfunction() + +# e.g. svs_parse_tu_spec("avx2.cpp|AVX2|haswell|avx2" src level arch infix) sets +# src=avx2.cpp, level=AVX2, arch=haswell, infix=avx2 +function(svs_parse_tu_spec spec out_src out_level out_arch out_infix) + svs_split_fields("${spec}" 4 src level arch infix) + set(${out_src} "${src}" PARENT_SCOPE) + set(${out_level} "${level}" PARENT_SCOPE) + set(${out_arch} "${arch}" PARENT_SCOPE) + set(${out_infix} "${infix}" PARENT_SCOPE) +endfunction() + +cmake_policy(POP) diff --git a/cmake/dispatch-surface.cmake b/cmake/dispatch-surface.cmake new file mode 100644 index 000000000..0027784e7 --- /dev/null +++ b/cmake/dispatch-surface.cmake @@ -0,0 +1,82 @@ +# Copyright 2026 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +##### +##### The dispatch surface for the x86 distance kernels. +##### +##### This file is the single place the extent list and the ISA levels are +##### written down. Everything derived from them is generated: +##### +##### - svs/core/distance/dispatch_surface.h, which drives every `extern +##### template` and explicit instantiation, and `supported_dim_list` +##### - the object library each ISA level's translation unit is compiled into, +##### and the instruction budget it is compiled at +##### +##### Edit this file. The header is not in the source tree at all: it is written +##### into the build directory on every configure and installed from there. +##### +##### A build may point somewhere else with -DSVS_DISPATCH_SURFACE_FILE=, +##### in which case only that build tree describes the overridden surface. +##### +##### Bookkeeping +##### +##### To add or remove a fixed extent: +##### Edit `SVS_SUPPORTED_DIMS`. The generated header, every `extern +##### template`, and `supported_dim_list` follow automatically. +##### +##### To add an ISA level: +##### 1. Add a row to `SVS_ISA_LEVELS`. +##### 2. Add a `SVS_TYPE_PAIRS_` list in +##### include/svs/multi-arch/x86/preprocessor.h, listing the element-type +##### pairs that level has kernels for. +##### 3. Add the level's translation unit at +##### include/svs/multi-arch/x86/.cpp. +##### The object library and its compile flags follow from the row here. A +##### level without a type-pair list is a compile error, not an empty +##### instantiation set. +##### +##### To add or remove an element-type pair for a level: +##### Edit that level's `SVS_TYPE_PAIRS_` list in +##### include/svs/multi-arch/x86/preprocessor.h. Not configured here: a type +##### pair exists because an implementation exists for it. +##### +##### To change a level's instruction budget: +##### Edit the middle field of its row in `SVS_ISA_LEVELS`, observing the +##### constraint recorded there. +##### + + +## +## List of supported dimensions; kernels will be available for these dimensions +## If used with dimensions not in this list, compute will be dispatched to +## svs::Dynamic, which is automatically added to the list of supported dims. +## + +set(SVS_SUPPORTED_DIMS 64 96 100 128 160 200 512 768) + +## +## ISA mapping +## +## All compute kernels will be compiled for the architectures below +## NONE is not added; it only serves as a fallback and it will compile with +## the flags set by the consumer. +## Format is "ENUM|arch|infix", where +## - ENUM is the AVX_AVAILABILITY value from distance_core.h +## - arch is the -march flag passed to the compiler +## - infix is a **unique** string used for object and library names + +set(SVS_ISA_LEVELS + "AVX2|haswell|avx2" + "AVX512|cascadelake|avx512" +) diff --git a/cmake/generate-dispatch-surface.cmake b/cmake/generate-dispatch-surface.cmake new file mode 100644 index 000000000..2ddf483ab --- /dev/null +++ b/cmake/generate-dispatch-surface.cmake @@ -0,0 +1,226 @@ +# Copyright 2026 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +##### +##### Derives the dispatch surface declared in cmake/dispatch-surface.cmake. +##### +##### Defines svs_generate_dispatch_surface(), which produces: +##### /generated/include/svs/core/distance/dispatch_surface.h (build tree only) +##### SVS_DISPATCH_TU_SPECS -- "|||", one per ISA level +##### +##### cmake/multi-arch.cmake calls the function and receives both. +##### + +include_guard(GLOBAL) + +include("${CMAKE_CURRENT_LIST_DIR}/dispatch-levels.cmake") + +set(SVS_DISPATCH_GEN_CMAKE_DIR "${CMAKE_CURRENT_LIST_DIR}") + +set(SVS_DEFAULT_DISPATCH_SURFACE_FILE "${CMAKE_CURRENT_LIST_DIR}/dispatch-surface.cmake") +set(SVS_DISPATCH_SURFACE_FILE "${SVS_DEFAULT_DISPATCH_SURFACE_FILE}" + CACHE FILEPATH + "Declaration of the ahead-of-time distance-kernel dispatch surface" +) + +##### +##### Read the declaration +##### + +function(svs_dispatch_read_declaration surface_file x86_src_dir + out_supported_dims out_dim_list out_dim_count out_isa_levels) + set(SVS_DISPATCH_SURFACE_FILE "${surface_file}") + set(SVS_X86_SRC_DIR "${x86_src_dir}") + include("${SVS_DISPATCH_GEN_CMAKE_DIR}/validate-dispatch-surface.cmake") + set(${out_supported_dims} "${SVS_SUPPORTED_DIMS}" PARENT_SCOPE) + set(${out_dim_list} "${SVS_DIM_LIST}" PARENT_SCOPE) + set(${out_dim_count} "${SVS_DIM_COUNT}" PARENT_SCOPE) + set(${out_isa_levels} "${SVS_ISA_LEVELS}" PARENT_SCOPE) +endfunction() + +##### +##### Build the macro bodies +##### + +function(svs_dispatch_build_expansions dim_list isa_levels + out_dim_loop out_target_loop out_level_loop out_level_defines) + set(dim_loop "\\\n") + foreach(dim IN LISTS dim_list) + string(APPEND dim_loop " M(${dim}) \\\n") + endforeach() + string(APPEND dim_loop " /* end */") + + set(target_loop "\\\n") + set(level_loop "\\\n") + set(level_defines "") + foreach(level_spec IN LISTS isa_levels) + svs_parse_isa_level("${level_spec}" level _ _) + string(APPEND level_loop " M(${level}) \\\n") + string(APPEND level_defines "#define SVS_ISA_LEVEL_${level} 1\n") + foreach(dim IN LISTS dim_list) + string(APPEND target_loop " M(${dim}, ${level}) \\\n") + endforeach() + endforeach() + string(APPEND target_loop " /* end */") + string(APPEND level_loop " /* end */") + string(STRIP "${level_defines}" level_defines) + + set(${out_dim_loop} "${dim_loop}" PARENT_SCOPE) + set(${out_target_loop} "${target_loop}" PARENT_SCOPE) + set(${out_level_loop} "${level_loop}" PARENT_SCOPE) + set(${out_level_defines} "${level_defines}" PARENT_SCOPE) +endfunction() + +##### +##### Derive the translation-unit specs +##### + +function(svs_dispatch_derive_tu_specs isa_levels x86_src_dir + out_tu_specs out_levels out_level_report) + set(tu_specs) + set(levels) + set(level_report) + foreach(level_spec IN LISTS isa_levels) + svs_parse_isa_level("${level_spec}" level arch infix) + list(APPEND tu_specs "${x86_src_dir}/${infix}.cpp|${level}|${arch}|${infix}") + list(APPEND levels "${level}") + list(APPEND level_report "AVX_AVAILABILITY::${level} -march=${arch} ${infix}.cpp") + endforeach() + set(${out_tu_specs} "${tu_specs}" PARENT_SCOPE) + set(${out_levels} "${levels}" PARENT_SCOPE) + set(${out_level_report} "${level_report}" PARENT_SCOPE) +endfunction() + +##### +##### Emit the header +##### + +# Writes the build-tree header from the template and the expansion strings above. +function(svs_dispatch_emit_header template_file dim_count dim_loop target_loop + level_loop level_defines include_dir out_header) + set(SVS_GEN_DIM_COUNT "${dim_count}") + set(SVS_GEN_DIM_LOOP "${dim_loop}") + set(SVS_GEN_TARGET_LOOP "${target_loop}") + set(SVS_GEN_LEVEL_LOOP "${level_loop}") + set(SVS_GEN_LEVEL_DEFINES "${level_defines}") + set(header "${include_dir}/svs/core/distance/dispatch_surface.h") + configure_file("${template_file}" "${header}" @ONLY) + set(${out_header} "${header}" PARENT_SCOPE) +endfunction() + +##### +##### Emit the manifest +##### + +function(svs_dispatch_emit_manifest supported_dims levels manifest_file) + string(REPLACE ";" " " levels_text "${levels}") + string(REPLACE ";" " " extents_text "${supported_dims}") + file(GENERATE + OUTPUT "${manifest_file}" + CONTENT "set(SVS_MANIFEST_FIXED_EXTENTS ${extents_text}) +set(SVS_MANIFEST_LEVELS ${levels_text}) +" + ) +endfunction() + +##### +##### Orchestrate +##### + +# Reads the declaration, writes the build-tree header and the ctest manifest, and +# reports the surface. Returns the TU specs and the generated header path, which is +# how cmake/multi-arch.cmake receives them; nothing here escapes via a bare +# file-scope set(). +function(svs_generate_dispatch_surface out_tu_specs out_header) + set(x86_src_dir "${PROJECT_SOURCE_DIR}/include/svs/multi-arch/x86") + + svs_dispatch_read_declaration( + "${SVS_DISPATCH_SURFACE_FILE}" "${x86_src_dir}" + supported_dims dim_list dim_count isa_levels + ) + + # Re-run configure when the declaration changes, so the generated header and + # the translation units cannot go stale. + set_property( + DIRECTORY APPEND PROPERTY CMAKE_CONFIGURE_DEPENDS + "${SVS_DISPATCH_SURFACE_FILE}" + ) + + svs_dispatch_build_expansions( + "${dim_list}" "${isa_levels}" + dim_loop target_loop level_loop level_defines + ) + svs_dispatch_derive_tu_specs( + "${isa_levels}" "${x86_src_dir}" + tu_specs levels level_report + ) + + # The build always compiles against the build-tree copy, and it is placed + # ahead of the source include directory so that it wins. + set(generated_include_dir "${CMAKE_CURRENT_BINARY_DIR}/generated/include") + svs_dispatch_emit_header( + "${SVS_DISPATCH_GEN_CMAKE_DIR}/templates/dispatch_surface.h.in" + "${dim_count}" "${dim_loop}" "${target_loop}" "${level_loop}" "${level_defines}" + "${generated_include_dir}" + header + ) + target_include_directories( + ${SVS_LIB} BEFORE INTERFACE $ + ) + + svs_dispatch_emit_manifest( + "${supported_dims}" "${levels}" + "${CMAKE_BINARY_DIR}/dispatch_surface.manifest.cmake" + ) + + ##### + ##### Report the surface + ##### + + list(LENGTH isa_levels level_count) + string(REPLACE ";" " " dims_display "${supported_dims}") + message(STATUS "Dispatch surface: ${dim_count} extents x ${level_count} ISA levels") + message(STATUS " extents: ${dims_display} svs::Dynamic") + foreach(entry IN LISTS level_report) + message(STATUS " level: ${entry}") + endforeach() + + # Every enumerator without a translation unit is still reachable -- the entry + # points fall back to it -- so its kernels are built by each consumer instead. + set(enum_header "${PROJECT_SOURCE_DIR}/include/svs/core/distance/distance_core.h") + if(EXISTS "${enum_header}") + file(READ "${enum_header}" enum_text) + if(enum_text MATCHES "enum class AVX_AVAILABILITY[ \t\r\n]*{([^}]*)}") + string(REPLACE "," ";" enumerators "${CMAKE_MATCH_1}") + set(undeclared) + foreach(enumerator IN LISTS enumerators) + string(STRIP "${enumerator}" enumerator) + if(enumerator AND NOT enumerator IN_LIST levels) + list(APPEND undeclared "${enumerator}") + endif() + endforeach() + if(undeclared) + string(REPLACE ";" ", " undeclared_display "${undeclared}") + message(STATUS " not in the surface: ${undeclared_display}") + message(STATUS + " dispatched to, but compiled by no translation unit, so " + "every consumer instantiates those kernels itself, at its own -march" + ) + endif() + endif() + endif() + + set(${out_tu_specs} "${tu_specs}" PARENT_SCOPE) + set(${out_header} "${header}" PARENT_SCOPE) +endfunction() diff --git a/cmake/multi-arch.cmake b/cmake/multi-arch.cmake index aeb81e693..3b1386c9d 100644 --- a/cmake/multi-arch.cmake +++ b/cmake/multi-arch.cmake @@ -12,25 +12,31 @@ # See the License for the specific language governing permissions and # limitations under the License. -set(SVS_X86_SRC_DIR "${PROJECT_SOURCE_DIR}/include/svs/multi-arch/x86") -set(SVS_X86 - "${SVS_X86_SRC_DIR}/avx2.cpp,avx2,haswell" - "${SVS_X86_SRC_DIR}/avx512.cpp,avx512,cascadelake" -) +# Writes the generated dispatch-surface header and populates +# SVS_DISPATCH_TU_SPECS -- "|||", one entry per ISA +# level, e.g. "avx2.cpp|AVX2|haswell|avx2" +# The extent list and the levels themselves are declared in +# cmake/dispatch-surface.cmake. +include("${CMAKE_CURRENT_LIST_DIR}/dispatch-levels.cmake") +include("${CMAKE_CURRENT_LIST_DIR}/generate-dispatch-surface.cmake") +svs_generate_dispatch_surface(SVS_DISPATCH_TU_SPECS SVS_GENERATED_DISPATCH_HEADER) set(SVS_X86_OBJECT_FILES) -foreach(x86_info IN LISTS SVS_X86) - string(REPLACE "," ";" x86_info "${x86_info}") - list(GET x86_info 0 src) - list(GET x86_info 1 avx) - list(GET x86_info 2 arch) - set(lib_name "svs_x86_${avx}") +foreach(tu_spec IN LISTS SVS_DISPATCH_TU_SPECS) + svs_parse_tu_spec("${tu_spec}" src level arch infix) + + # Define unique lib_name and obj_name for this TU SPEC with provided infix. + # Requires unique infixes in cmake/dispatch-surface.cmake. + + set(lib_name "svs_x86_${infix}") add_library(${lib_name} INTERFACE) target_compile_options(${lib_name} INTERFACE -march=${arch} -mtune=${arch}) - set(obj_name ${arch}_obj) + set(obj_name ${infix}_obj) add_library(${obj_name} OBJECT ${src}) - target_link_libraries(${obj_name} PRIVATE ${SVS_LIB} svs::compile_options fmt::fmt ${lib_name}) + target_link_libraries( + ${obj_name} PRIVATE ${SVS_LIB} svs::compile_options fmt::fmt ${lib_name} + ) list(APPEND SVS_X86_OBJECT_FILES $) endforeach() diff --git a/cmake/options.cmake b/cmake/options.cmake index 75be25326..b02b6785c 100644 --- a/cmake/options.cmake +++ b/cmake/options.cmake @@ -184,7 +184,7 @@ target_compile_options( ) -if(CMAKE_BUILD_TYPE STREQUAL Release OR CMAKE_BUILD_TYPE STREQUAL RelWithDebInfo) +if(CMAKE_BUILD_TYPE STREQUAL Release OR CMAKE_BUILD_TYPE STREQUAL RelWithDebugInfo) target_compile_options(svs_compile_options INTERFACE -O3) endif() diff --git a/cmake/templates/dispatch_surface.h.in b/cmake/templates/dispatch_surface.h.in new file mode 100644 index 000000000..8c021d30a --- /dev/null +++ b/cmake/templates/dispatch_surface.h.in @@ -0,0 +1,37 @@ +/* + * Copyright 2026 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// GENERATED FILE -- DO NOT EDIT. Regenerated on every CMake configure + +#pragma once + +// Number of extents with a fixed-extent kernel, including svs::Dynamic. +#define SVS_SUPPORTED_DIM_COUNT @SVS_GEN_DIM_COUNT@ + +// Invokes M(extent) once per extent. +#define SVS_FOR_EACH_SUPPORTED_DIM(M) @SVS_GEN_DIM_LOOP@ + +// Invokes M(extent, isa_level) once per (extent, ISA level) pair -- that is, +// once per kernel the library compiles ahead of time, modulo type pairs. +#define SVS_FOR_EACH_DISPATCH_TARGET(M) @SVS_GEN_TARGET_LOOP@ + +// Invokes M(isa_level) once per ISA level, weakest first. AVX_AVAILABILITY +// enumerators without a translation unit are absent: this is the surface. +#define SVS_FOR_EACH_ISA_LEVEL(M) @SVS_GEN_LEVEL_LOOP@ + +// One per ISA level in the surface, and only those: a level absent from it has no +// instantiations. Tested with `defined` so a -Wundef build stays quiet. +@SVS_GEN_LEVEL_DEFINES@ diff --git a/cmake/validate-dispatch-surface.cmake b/cmake/validate-dispatch-surface.cmake new file mode 100644 index 000000000..a23a0b244 --- /dev/null +++ b/cmake/validate-dispatch-surface.cmake @@ -0,0 +1,182 @@ +# Copyright 2026 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +##### +##### Reads and checks a dispatch-surface declaration. +##### +##### Inputs: +##### SVS_DISPATCH_SURFACE_FILE -- the declaration to read +##### SVS_X86_SRC_DIR -- where per-level translation units live +##### +##### Outputs: +##### SVS_SUPPORTED_DIMS, SVS_ISA_LEVELS -- verbatim from the declaration +##### SVS_DIM_LIST -- extents, with svs::Dynamic appended +##### SVS_DIM_COUNT -- length of SVS_DIM_LIST +##### SVS_FIXED_DIM_COUNT -- length of SVS_SUPPORTED_DIMS +##### + +# Scoped this file via PUSH/POP so it does not affect the includer +cmake_policy(PUSH) +cmake_policy(SET CMP0007 NEW) +cmake_policy(SET CMP0057 NEW) + +include("${CMAKE_CURRENT_LIST_DIR}/dispatch-levels.cmake") + +if(NOT SVS_DISPATCH_SURFACE_FILE) + message(FATAL_ERROR "SVS_DISPATCH_SURFACE_FILE is not set.") +endif() +if(NOT EXISTS "${SVS_DISPATCH_SURFACE_FILE}") + message(FATAL_ERROR + "SVS_DISPATCH_SURFACE_FILE does not exist: ${SVS_DISPATCH_SURFACE_FILE}" + ) +endif() +if(NOT SVS_X86_SRC_DIR) + message(FATAL_ERROR "SVS_X86_SRC_DIR is not set.") +endif() + +# Set the variables and include the surface file to populate them +set(SVS_SUPPORTED_DIMS) +set(SVS_ISA_LEVELS) +include("${SVS_DISPATCH_SURFACE_FILE}") + + +# Sanity checks DIMS +if(NOT SVS_SUPPORTED_DIMS) + message(FATAL_ERROR + "SVS_SUPPORTED_DIMS is empty in ${SVS_DISPATCH_SURFACE_FILE}. At least " + "one fixed extent is required." + ) +endif() + +# Only numerical values +foreach(dim IN LISTS SVS_SUPPORTED_DIMS) + if(NOT dim MATCHES "^[1-9][0-9]*$") + message(FATAL_ERROR + "SVS_SUPPORTED_DIMS contains '${dim}', which is not a positive " + "integer. svs::Dynamic is required and is appended automatically, " + "so it must not be listed." + ) + endif() +endforeach() + +# Remove duplicates +set(svs_dims_deduped ${SVS_SUPPORTED_DIMS}) +list(REMOVE_DUPLICATES svs_dims_deduped) +list(LENGTH SVS_SUPPORTED_DIMS SVS_FIXED_DIM_COUNT) +list(LENGTH svs_dims_deduped svs_dims_unique) +if(NOT SVS_FIXED_DIM_COUNT EQUAL svs_dims_unique) + message(FATAL_ERROR + "SVS_SUPPORTED_DIMS contains duplicate extents. Every extent must " + "appear exactly once." + ) +endif() + +# Append svs::Dynamic +set(SVS_DIM_LIST ${SVS_SUPPORTED_DIMS} "svs::Dynamic") +list(LENGTH SVS_DIM_LIST SVS_DIM_COUNT) + + +# Sanity checks ISA +if(NOT SVS_ISA_LEVELS) + message(FATAL_ERROR "SVS_ISA_LEVELS is empty in ${SVS_DISPATCH_SURFACE_FILE}.") +endif() + +set(svs_distance_core_header + "${CMAKE_CURRENT_LIST_DIR}/../include/svs/core/distance/distance_core.h" +) +if(NOT EXISTS "${svs_distance_core_header}") + message(FATAL_ERROR + "Cannot find ${svs_distance_core_header} to read AVX_AVAILABILITY from." + ) +endif() +file(READ "${svs_distance_core_header}" svs_distance_core_text) +if(NOT svs_distance_core_text MATCHES "enum class AVX_AVAILABILITY[ \t\r\n]*{([^}]*)}") + message(FATAL_ERROR + "Cannot find 'enum class AVX_AVAILABILITY { ... }' in " + "${svs_distance_core_header}." + ) +endif() +string(REPLACE "," ";" svs_legal_levels "${CMAKE_MATCH_1}") +list(TRANSFORM svs_legal_levels STRIP) +# NONE means "no level is present" and is only there as fallback +list(REMOVE_ITEM svs_legal_levels "NONE") + +set(svs_seen_levels) +set(svs_seen_infixes) +set(svs_seen_archs) +foreach(level_spec IN LISTS SVS_ISA_LEVELS) + svs_parse_isa_level("${level_spec}" level arch infix) + foreach(field level arch infix) + if(NOT ${field}) + message(FATAL_ERROR + "Malformed SVS_ISA_LEVELS entry '${level_spec}': ${field} is empty." + ) + endif() + endforeach() + if(level STREQUAL "NONE") + message(FATAL_ERROR + "ISA level 'NONE' in SVS_ISA_LEVELS is not declarable: it means no " + "level is present, so it has no translation unit and no object " + "library for a row to name." + ) + endif() + if(NOT level IN_LIST svs_legal_levels) + string(REPLACE ";" ", " svs_legal_levels_display "${svs_legal_levels}") + message(FATAL_ERROR + "Unknown ISA level '${level}' in SVS_ISA_LEVELS: not an enumerator " + "of svs::distance::AVX_AVAILABILITY in ${svs_distance_core_header} " + "(legal levels: ${svs_legal_levels_display})." + ) + endif() + if(level IN_LIST svs_seen_levels) + message(FATAL_ERROR "Duplicate ISA level '${level}' in SVS_ISA_LEVELS.") + endif() + if(infix IN_LIST svs_seen_infixes) + message(FATAL_ERROR + "Duplicate TU infix '${infix}' in SVS_ISA_LEVELS; infixes name " + "generated files and must be unique." + ) + endif() + if(arch IN_LIST svs_seen_archs) + message(FATAL_ERROR + "Duplicate -march '${arch}' in SVS_ISA_LEVELS; each level's -march " + "is its instruction budget, so two levels sharing one budget compile " + "the weaker level with instructions its runtime predicate does not " + "guarantee, and hosts routed to it fault." + ) + endif() + if(NOT EXISTS "${SVS_X86_SRC_DIR}/${infix}.cpp") + message(FATAL_ERROR + "ISA level '${level}' has no translation unit: expected " + "${SVS_X86_SRC_DIR}/${infix}.cpp. Adding a level to SVS_ISA_LEVELS " + "requires creating that file." + ) + endif() + list(APPEND svs_seen_levels ${level}) + list(APPEND svs_seen_infixes ${infix}) + list(APPEND svs_seen_archs ${arch}) +endforeach() + +# distance_core.h #errors on x86_64 unless both are present, so a surface +# omitting either is not a smaller build: it cannot compile. +foreach(svs_mandatory_level AVX2 AVX512) + if(NOT svs_mandatory_level IN_LIST svs_seen_levels) + message(FATAL_ERROR + "SVS_ISA_LEVELS omits mandatory level '${svs_mandatory_level}'; " + "distance_core.h requires both AVX2 and AVX512 to be present." + ) + endif() +endforeach() + +cmake_policy(POP) diff --git a/include/svs/core/distance/cosine.h b/include/svs/core/distance/cosine.h index 9f4924997..fcb9d429d 100644 --- a/include/svs/core/distance/cosine.h +++ b/include/svs/core/distance/cosine.h @@ -500,26 +500,11 @@ struct CosineSimilarityImpl { #if defined(__x86_64__) #include "svs/multi-arch/x86/preprocessor.h" -// TODO: connect with dim_supported_list -DISTANCE_CS_EXTERN_TEMPLATE(64, AVX_AVAILABILITY::AVX512); -DISTANCE_CS_EXTERN_TEMPLATE(96, AVX_AVAILABILITY::AVX512); -DISTANCE_CS_EXTERN_TEMPLATE(100, AVX_AVAILABILITY::AVX512); -DISTANCE_CS_EXTERN_TEMPLATE(128, AVX_AVAILABILITY::AVX512); -DISTANCE_CS_EXTERN_TEMPLATE(160, AVX_AVAILABILITY::AVX512); -DISTANCE_CS_EXTERN_TEMPLATE(200, AVX_AVAILABILITY::AVX512); -DISTANCE_CS_EXTERN_TEMPLATE(512, AVX_AVAILABILITY::AVX512); -DISTANCE_CS_EXTERN_TEMPLATE(768, AVX_AVAILABILITY::AVX512); -DISTANCE_CS_EXTERN_TEMPLATE(Dynamic, AVX_AVAILABILITY::AVX512); - -DISTANCE_CS_EXTERN_TEMPLATE(64, AVX_AVAILABILITY::AVX2); -DISTANCE_CS_EXTERN_TEMPLATE(96, AVX_AVAILABILITY::AVX2); -DISTANCE_CS_EXTERN_TEMPLATE(100, AVX_AVAILABILITY::AVX2); -DISTANCE_CS_EXTERN_TEMPLATE(128, AVX_AVAILABILITY::AVX2); -DISTANCE_CS_EXTERN_TEMPLATE(160, AVX_AVAILABILITY::AVX2); -DISTANCE_CS_EXTERN_TEMPLATE(200, AVX_AVAILABILITY::AVX2); -DISTANCE_CS_EXTERN_TEMPLATE(512, AVX_AVAILABILITY::AVX2); -DISTANCE_CS_EXTERN_TEMPLATE(768, AVX_AVAILABILITY::AVX2); -DISTANCE_CS_EXTERN_TEMPLATE(Dynamic, AVX_AVAILABILITY::AVX2); + +// Declare cosine similarity kernels for all supported combinations +#define SVS_CS_EXTERN(DIM, LEVEL) SVS_INSTANTIATE_CS(extern template, DIM, LEVEL) +SVS_FOR_EACH_DISPATCH_TARGET(SVS_CS_EXTERN) +#undef SVS_CS_EXTERN #endif } // namespace svs::distance diff --git a/include/svs/core/distance/euclidean.h b/include/svs/core/distance/euclidean.h index b038a6fcc..0529636da 100644 --- a/include/svs/core/distance/euclidean.h +++ b/include/svs/core/distance/euclidean.h @@ -438,26 +438,11 @@ template struct L2Impl { #include "svs/multi-arch/x86/preprocessor.h" -// TODO: connect with dim_supported_list -DISTANCE_L2_EXTERN_TEMPLATE(64, AVX_AVAILABILITY::AVX512); -DISTANCE_L2_EXTERN_TEMPLATE(96, AVX_AVAILABILITY::AVX512); -DISTANCE_L2_EXTERN_TEMPLATE(100, AVX_AVAILABILITY::AVX512); -DISTANCE_L2_EXTERN_TEMPLATE(128, AVX_AVAILABILITY::AVX512); -DISTANCE_L2_EXTERN_TEMPLATE(160, AVX_AVAILABILITY::AVX512); -DISTANCE_L2_EXTERN_TEMPLATE(200, AVX_AVAILABILITY::AVX512); -DISTANCE_L2_EXTERN_TEMPLATE(512, AVX_AVAILABILITY::AVX512); -DISTANCE_L2_EXTERN_TEMPLATE(768, AVX_AVAILABILITY::AVX512); -DISTANCE_L2_EXTERN_TEMPLATE(Dynamic, AVX_AVAILABILITY::AVX512); - -DISTANCE_L2_EXTERN_TEMPLATE(64, AVX_AVAILABILITY::AVX2); -DISTANCE_L2_EXTERN_TEMPLATE(96, AVX_AVAILABILITY::AVX2); -DISTANCE_L2_EXTERN_TEMPLATE(100, AVX_AVAILABILITY::AVX2); -DISTANCE_L2_EXTERN_TEMPLATE(200, AVX_AVAILABILITY::AVX2); -DISTANCE_L2_EXTERN_TEMPLATE(128, AVX_AVAILABILITY::AVX2); -DISTANCE_L2_EXTERN_TEMPLATE(200, AVX_AVAILABILITY::AVX2); -DISTANCE_L2_EXTERN_TEMPLATE(512, AVX_AVAILABILITY::AVX2); -DISTANCE_L2_EXTERN_TEMPLATE(768, AVX_AVAILABILITY::AVX2); -DISTANCE_L2_EXTERN_TEMPLATE(Dynamic, AVX_AVAILABILITY::AVX2); +// Declare L2 kernels for all supported combinations +#define SVS_L2_EXTERN(DIM, LEVEL) SVS_INSTANTIATE_L2(extern template, DIM, LEVEL) +SVS_FOR_EACH_DISPATCH_TARGET(SVS_L2_EXTERN) +#undef SVS_L2_EXTERN + #endif } // namespace svs::distance diff --git a/include/svs/core/distance/inner_product.h b/include/svs/core/distance/inner_product.h index 0f7837a53..edf83c256 100644 --- a/include/svs/core/distance/inner_product.h +++ b/include/svs/core/distance/inner_product.h @@ -388,26 +388,12 @@ template struct IPImpl { #if defined(__x86_64__) #include "svs/multi-arch/x86/preprocessor.h" -// TODO: connect with dim_supported_list -DISTANCE_IP_EXTERN_TEMPLATE(64, AVX_AVAILABILITY::AVX512); -DISTANCE_IP_EXTERN_TEMPLATE(96, AVX_AVAILABILITY::AVX512); -DISTANCE_IP_EXTERN_TEMPLATE(100, AVX_AVAILABILITY::AVX512); -DISTANCE_IP_EXTERN_TEMPLATE(128, AVX_AVAILABILITY::AVX512); -DISTANCE_IP_EXTERN_TEMPLATE(160, AVX_AVAILABILITY::AVX512); -DISTANCE_IP_EXTERN_TEMPLATE(200, AVX_AVAILABILITY::AVX512); -DISTANCE_IP_EXTERN_TEMPLATE(512, AVX_AVAILABILITY::AVX512); -DISTANCE_IP_EXTERN_TEMPLATE(768, AVX_AVAILABILITY::AVX512); -DISTANCE_IP_EXTERN_TEMPLATE(Dynamic, AVX_AVAILABILITY::AVX512); - -DISTANCE_IP_EXTERN_TEMPLATE(64, AVX_AVAILABILITY::AVX2); -DISTANCE_IP_EXTERN_TEMPLATE(96, AVX_AVAILABILITY::AVX2); -DISTANCE_IP_EXTERN_TEMPLATE(100, AVX_AVAILABILITY::AVX2); -DISTANCE_IP_EXTERN_TEMPLATE(128, AVX_AVAILABILITY::AVX2); -DISTANCE_IP_EXTERN_TEMPLATE(160, AVX_AVAILABILITY::AVX2); -DISTANCE_IP_EXTERN_TEMPLATE(200, AVX_AVAILABILITY::AVX2); -DISTANCE_IP_EXTERN_TEMPLATE(512, AVX_AVAILABILITY::AVX2); -DISTANCE_IP_EXTERN_TEMPLATE(768, AVX_AVAILABILITY::AVX2); -DISTANCE_IP_EXTERN_TEMPLATE(Dynamic, AVX_AVAILABILITY::AVX2); + +// Declare MIP kernels for all supported combinations +#define SVS_IP_EXTERN(DIM, LEVEL) SVS_INSTANTIATE_IP(extern template, DIM, LEVEL) +SVS_FOR_EACH_DISPATCH_TARGET(SVS_IP_EXTERN) +#undef SVS_IP_EXTERN + #endif } // namespace svs::distance diff --git a/include/svs/multi-arch/x86/avx2.cpp b/include/svs/multi-arch/x86/avx2.cpp index bff53ae10..210c7cbfd 100644 --- a/include/svs/multi-arch/x86/avx2.cpp +++ b/include/svs/multi-arch/x86/avx2.cpp @@ -16,41 +16,16 @@ #if defined(__x86_64__) #include "svs/core/distance/cosine.h" +#include "svs/core/distance/dispatch_surface.h" #include "svs/core/distance/euclidean.h" #include "svs/core/distance/inner_product.h" +#include "svs/multi-arch/x86/preprocessor.h" namespace svs::distance { -// TODO: connect with dim_supported_list -DISTANCE_L2_INSTANTIATE_TEMPLATE(64, AVX_AVAILABILITY::AVX2); -DISTANCE_L2_INSTANTIATE_TEMPLATE(96, AVX_AVAILABILITY::AVX2); -DISTANCE_L2_INSTANTIATE_TEMPLATE(100, AVX_AVAILABILITY::AVX2); -DISTANCE_L2_INSTANTIATE_TEMPLATE(128, AVX_AVAILABILITY::AVX2); -DISTANCE_L2_INSTANTIATE_TEMPLATE(160, AVX_AVAILABILITY::AVX2); -DISTANCE_L2_INSTANTIATE_TEMPLATE(200, AVX_AVAILABILITY::AVX2); -DISTANCE_L2_INSTANTIATE_TEMPLATE(512, AVX_AVAILABILITY::AVX2); -DISTANCE_L2_INSTANTIATE_TEMPLATE(768, AVX_AVAILABILITY::AVX2); -DISTANCE_L2_INSTANTIATE_TEMPLATE(Dynamic, AVX_AVAILABILITY::AVX2); - -DISTANCE_IP_INSTANTIATE_TEMPLATE(64, AVX_AVAILABILITY::AVX2); -DISTANCE_IP_INSTANTIATE_TEMPLATE(96, AVX_AVAILABILITY::AVX2); -DISTANCE_IP_INSTANTIATE_TEMPLATE(100, AVX_AVAILABILITY::AVX2); -DISTANCE_IP_INSTANTIATE_TEMPLATE(128, AVX_AVAILABILITY::AVX2); -DISTANCE_IP_INSTANTIATE_TEMPLATE(160, AVX_AVAILABILITY::AVX2); -DISTANCE_IP_INSTANTIATE_TEMPLATE(200, AVX_AVAILABILITY::AVX2); -DISTANCE_IP_INSTANTIATE_TEMPLATE(512, AVX_AVAILABILITY::AVX2); -DISTANCE_IP_INSTANTIATE_TEMPLATE(768, AVX_AVAILABILITY::AVX2); -DISTANCE_IP_INSTANTIATE_TEMPLATE(Dynamic, AVX_AVAILABILITY::AVX2); - -DISTANCE_CS_INSTANTIATE_TEMPLATE(64, AVX_AVAILABILITY::AVX2); -DISTANCE_CS_INSTANTIATE_TEMPLATE(96, AVX_AVAILABILITY::AVX2); -DISTANCE_CS_INSTANTIATE_TEMPLATE(100, AVX_AVAILABILITY::AVX2); -DISTANCE_CS_INSTANTIATE_TEMPLATE(128, AVX_AVAILABILITY::AVX2); -DISTANCE_CS_INSTANTIATE_TEMPLATE(160, AVX_AVAILABILITY::AVX2); -DISTANCE_CS_INSTANTIATE_TEMPLATE(200, AVX_AVAILABILITY::AVX2); -DISTANCE_CS_INSTANTIATE_TEMPLATE(512, AVX_AVAILABILITY::AVX2); -DISTANCE_CS_INSTANTIATE_TEMPLATE(768, AVX_AVAILABILITY::AVX2); -DISTANCE_CS_INSTANTIATE_TEMPLATE(Dynamic, AVX_AVAILABILITY::AVX2); +#define SVS_DEFINE_FOR_DIM(DIM) SVS_INSTANTIATE_DISTANCES(template, DIM, AVX2) +SVS_FOR_EACH_SUPPORTED_DIM(SVS_DEFINE_FOR_DIM) +#undef SVS_DEFINE_FOR_DIM } // namespace svs::distance diff --git a/include/svs/multi-arch/x86/avx512.cpp b/include/svs/multi-arch/x86/avx512.cpp index bee150d75..fa472289e 100644 --- a/include/svs/multi-arch/x86/avx512.cpp +++ b/include/svs/multi-arch/x86/avx512.cpp @@ -16,41 +16,16 @@ #if defined(__x86_64__) #include "svs/core/distance/cosine.h" +#include "svs/core/distance/dispatch_surface.h" #include "svs/core/distance/euclidean.h" #include "svs/core/distance/inner_product.h" +#include "svs/multi-arch/x86/preprocessor.h" namespace svs::distance { -// TODO: connect with dim_supported_list -DISTANCE_L2_INSTANTIATE_TEMPLATE(64, AVX_AVAILABILITY::AVX512); -DISTANCE_L2_INSTANTIATE_TEMPLATE(96, AVX_AVAILABILITY::AVX512); -DISTANCE_L2_INSTANTIATE_TEMPLATE(100, AVX_AVAILABILITY::AVX512); -DISTANCE_L2_INSTANTIATE_TEMPLATE(128, AVX_AVAILABILITY::AVX512); -DISTANCE_L2_INSTANTIATE_TEMPLATE(160, AVX_AVAILABILITY::AVX512); -DISTANCE_L2_INSTANTIATE_TEMPLATE(200, AVX_AVAILABILITY::AVX512); -DISTANCE_L2_INSTANTIATE_TEMPLATE(512, AVX_AVAILABILITY::AVX512); -DISTANCE_L2_INSTANTIATE_TEMPLATE(768, AVX_AVAILABILITY::AVX512); -DISTANCE_L2_INSTANTIATE_TEMPLATE(Dynamic, AVX_AVAILABILITY::AVX512); - -DISTANCE_IP_INSTANTIATE_TEMPLATE(64, AVX_AVAILABILITY::AVX512); -DISTANCE_IP_INSTANTIATE_TEMPLATE(96, AVX_AVAILABILITY::AVX512); -DISTANCE_IP_INSTANTIATE_TEMPLATE(100, AVX_AVAILABILITY::AVX512); -DISTANCE_IP_INSTANTIATE_TEMPLATE(128, AVX_AVAILABILITY::AVX512); -DISTANCE_IP_INSTANTIATE_TEMPLATE(160, AVX_AVAILABILITY::AVX512); -DISTANCE_IP_INSTANTIATE_TEMPLATE(200, AVX_AVAILABILITY::AVX512); -DISTANCE_IP_INSTANTIATE_TEMPLATE(512, AVX_AVAILABILITY::AVX512); -DISTANCE_IP_INSTANTIATE_TEMPLATE(768, AVX_AVAILABILITY::AVX512); -DISTANCE_IP_INSTANTIATE_TEMPLATE(Dynamic, AVX_AVAILABILITY::AVX512); - -DISTANCE_CS_INSTANTIATE_TEMPLATE(64, AVX_AVAILABILITY::AVX512); -DISTANCE_CS_INSTANTIATE_TEMPLATE(96, AVX_AVAILABILITY::AVX512); -DISTANCE_CS_INSTANTIATE_TEMPLATE(100, AVX_AVAILABILITY::AVX512); -DISTANCE_CS_INSTANTIATE_TEMPLATE(128, AVX_AVAILABILITY::AVX512); -DISTANCE_CS_INSTANTIATE_TEMPLATE(160, AVX_AVAILABILITY::AVX512); -DISTANCE_CS_INSTANTIATE_TEMPLATE(200, AVX_AVAILABILITY::AVX512); -DISTANCE_CS_INSTANTIATE_TEMPLATE(512, AVX_AVAILABILITY::AVX512); -DISTANCE_CS_INSTANTIATE_TEMPLATE(768, AVX_AVAILABILITY::AVX512); -DISTANCE_CS_INSTANTIATE_TEMPLATE(Dynamic, AVX_AVAILABILITY::AVX512); +#define SVS_DEFINE_FOR_DIM(DIM) SVS_INSTANTIATE_DISTANCES(template, DIM, AVX512) +SVS_FOR_EACH_SUPPORTED_DIM(SVS_DEFINE_FOR_DIM) +#undef SVS_DEFINE_FOR_DIM } // namespace svs::distance diff --git a/include/svs/multi-arch/x86/preprocessor.h b/include/svs/multi-arch/x86/preprocessor.h index 4e0cb941d..1f4ce4e16 100644 --- a/include/svs/multi-arch/x86/preprocessor.h +++ b/include/svs/multi-arch/x86/preprocessor.h @@ -16,74 +16,62 @@ #pragma once -#define DISTANCE_L2_TEMPLATE_HELPER(SPEC, N, AVX) \ - SPEC struct L2Impl; \ - SPEC struct L2Impl; \ - SPEC struct L2Impl; \ - SPEC struct L2Impl; \ - SPEC struct L2Impl; \ - SPEC struct L2Impl; \ - SPEC struct L2Impl; \ - SPEC struct L2Impl; \ - SPEC struct L2Impl; \ - SPEC struct L2Impl; \ - SPEC struct L2Impl; \ - SPEC struct L2Impl; \ - SPEC struct L2Impl; \ - SPEC struct L2Impl; \ - SPEC struct L2Impl; \ - SPEC struct L2Impl; +// This header is auto-generated by cmake +// Check +#include "svs/core/distance/dispatch_surface.h" -#define DISTANCE_L2_INSTANTIATE_TEMPLATE(N, AVX) \ - DISTANCE_L2_TEMPLATE_HELPER(template, N, AVX); +// Invoke a provided macro for all SVS-supported datatype combinations +// Used to initialize compute kernels (inner product, l2, cosine) ahead of time +// for multi-arch dispatching +// Important: If a project uses SVS and calls compute kernels on datatype +// combinations not in this list, they will be compiled with their project +// and won't support multi-arch dispatching +#define SVS_FOR_EACH_TYPE_PAIR(M, ...) \ + M(float, float, __VA_ARGS__) \ + M(float, int8_t, __VA_ARGS__) \ + M(float, uint8_t, __VA_ARGS__) \ + M(float, svs::float16::Float16, __VA_ARGS__) \ + M(int8_t, float, __VA_ARGS__) \ + M(int8_t, int8_t, __VA_ARGS__) \ + M(int8_t, uint8_t, __VA_ARGS__) \ + M(int8_t, svs::float16::Float16, __VA_ARGS__) \ + M(uint8_t, float, __VA_ARGS__) \ + M(uint8_t, int8_t, __VA_ARGS__) \ + M(uint8_t, uint8_t, __VA_ARGS__) \ + M(uint8_t, svs::float16::Float16, __VA_ARGS__) \ + M(svs::float16::Float16, float, __VA_ARGS__) \ + M(svs::float16::Float16, int8_t, __VA_ARGS__) \ + M(svs::float16::Float16, uint8_t, __VA_ARGS__) \ + M(svs::float16::Float16, svs::float16::Float16, __VA_ARGS__) -#define DISTANCE_L2_EXTERN_TEMPLATE(N, AVX) \ - DISTANCE_L2_TEMPLATE_HELPER(extern template, N, AVX); +// Alias macro for each ISA level to support targeted combinations per ISA +#define SVS_TYPE_PAIRS_NONE SVS_FOR_EACH_TYPE_PAIR +#define SVS_TYPE_PAIRS_AVX2 SVS_FOR_EACH_TYPE_PAIR +#define SVS_TYPE_PAIRS_AVX512 SVS_FOR_EACH_TYPE_PAIR -#define DISTANCE_IP_TEMPLATE_HELPER(SPEC, N, AVX) \ - SPEC struct IPImpl; \ - SPEC struct IPImpl; \ - SPEC struct IPImpl; \ - SPEC struct IPImpl; \ - SPEC struct IPImpl; \ - SPEC struct IPImpl; \ - SPEC struct IPImpl; \ - SPEC struct IPImpl; \ - SPEC struct IPImpl; \ - SPEC struct IPImpl; \ - SPEC struct IPImpl; \ - SPEC struct IPImpl; \ - SPEC struct IPImpl; \ - SPEC struct IPImpl; \ - SPEC struct IPImpl; \ - SPEC struct IPImpl; +// Unpack LEVEL from macro arguments +#define SVS_TYPE_PAIRS_FOR_(LEVEL) SVS_TYPE_PAIRS_##LEVEL +#define SVS_TYPE_PAIRS_FOR(LEVEL, M, ...) SVS_TYPE_PAIRS_FOR_(LEVEL)(M, __VA_ARGS__) -#define DISTANCE_IP_INSTANTIATE_TEMPLATE(N, AVX) \ - DISTANCE_IP_TEMPLATE_HELPER(template, N, AVX); +// SPEC wrapper for definition and declaration +#define SVS_DECLARE_ONE_L2(Ea, Eb, SPEC, N, LEVEL) \ + SPEC struct L2Impl; +#define SVS_DECLARE_ONE_IP(Ea, Eb, SPEC, N, LEVEL) \ + SPEC struct IPImpl; +#define SVS_DECLARE_ONE_CS(Ea, Eb, SPEC, N, LEVEL) \ + SPEC struct CosineSimilarityImpl; -#define DISTANCE_IP_EXTERN_TEMPLATE(N, AVX) \ - DISTANCE_IP_TEMPLATE_HELPER(extern template, N, AVX); +// SPEC is `template` for a definition or `extern template` for a declaration. +#define SVS_INSTANTIATE_L2(SPEC, N, LEVEL) \ + SVS_TYPE_PAIRS_FOR(LEVEL, SVS_DECLARE_ONE_L2, SPEC, N, LEVEL) +#define SVS_INSTANTIATE_IP(SPEC, N, LEVEL) \ + SVS_TYPE_PAIRS_FOR(LEVEL, SVS_DECLARE_ONE_IP, SPEC, N, LEVEL) +#define SVS_INSTANTIATE_CS(SPEC, N, LEVEL) \ + SVS_TYPE_PAIRS_FOR(LEVEL, SVS_DECLARE_ONE_CS, SPEC, N, LEVEL) -#define DISTANCE_CS_TEMPLATE_HELPER(SPEC, N, AVX) \ - SPEC struct CosineSimilarityImpl; \ - SPEC struct CosineSimilarityImpl; \ - SPEC struct CosineSimilarityImpl; \ - SPEC struct CosineSimilarityImpl; \ - SPEC struct CosineSimilarityImpl; \ - SPEC struct CosineSimilarityImpl; \ - SPEC struct CosineSimilarityImpl; \ - SPEC struct CosineSimilarityImpl; \ - SPEC struct CosineSimilarityImpl; \ - SPEC struct CosineSimilarityImpl; \ - SPEC struct CosineSimilarityImpl; \ - SPEC struct CosineSimilarityImpl; \ - SPEC struct CosineSimilarityImpl; \ - SPEC struct CosineSimilarityImpl; \ - SPEC struct CosineSimilarityImpl; \ - SPEC struct CosineSimilarityImpl; - -#define DISTANCE_CS_INSTANTIATE_TEMPLATE(N, AVX) \ - DISTANCE_CS_TEMPLATE_HELPER(template, N, AVX); - -#define DISTANCE_CS_EXTERN_TEMPLATE(N, AVX) \ - DISTANCE_CS_TEMPLATE_HELPER(extern template, N, AVX); +// All distances are available for the same combinations +// We therefore use one macro to do all three in one go +#define SVS_INSTANTIATE_DISTANCES(SPEC, N, LEVEL) \ + SVS_INSTANTIATE_L2(SPEC, N, LEVEL) \ + SVS_INSTANTIATE_IP(SPEC, N, LEVEL) \ + SVS_INSTANTIATE_CS(SPEC, N, LEVEL)