Thanks to visit codestin.com
Credit goes to github.com

Skip to content

Commit a56789a

Browse files
agibsoncccagibsonccc
authored
Add files for pr_2_2_cmake_type_system (#10246)
1 parent 30f4b55 commit a56789a

15 files changed

+7313
-0
lines changed

‎libnd4j/cmake/CombinationCaching.cmake‎

Lines changed: 179 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,179 @@
1+
# CombinationCaching.cmake
2+
# Smart type combination reuse and caching system
3+
4+
# Main function to handle combination reuse
5+
function(use_cached_type_combinations)
6+
# Check if combinations already exist in memory
7+
if(DEFINED COMBINATIONS_2 AND DEFINED COMBINATIONS_3)
8+
list(LENGTH COMBINATIONS_2 combo2_count)
9+
list(LENGTH COMBINATIONS_3 combo3_count)
10+
if(combo2_count GREATER 0 AND combo3_count GREATER 0)
11+
message(STATUS "♻️ Reusing existing combinations: 2-type=${combo2_count}, 3-type=${combo3_count}")
12+
return()
13+
endif()
14+
endif()
15+
16+
# Check for cached combinations file
17+
set(COMBO_CACHE_FILE "${CMAKE_BINARY_DIR}/type_combinations.cache")
18+
if(EXISTS "${COMBO_CACHE_FILE}")
19+
message(STATUS "📁 Loading combinations from cache file...")
20+
include("${COMBO_CACHE_FILE}")
21+
22+
# Verify cache loaded successfully
23+
if(DEFINED COMBINATIONS_2 AND DEFINED COMBINATIONS_3)
24+
list(LENGTH COMBINATIONS_2 combo2_count)
25+
list(LENGTH COMBINATIONS_3 combo3_count)
26+
message(STATUS "✅ Loaded from cache: 2-type=${combo2_count}, 3-type=${combo3_count}")
27+
return()
28+
else()
29+
message(STATUS "⚠️ Cache file corrupted, regenerating...")
30+
endif()
31+
endif()
32+
33+
# Only generate if we absolutely have to
34+
message(STATUS "🔄 Generating type combinations (first time only)...")
35+
36+
# Ensure type system is initialized
37+
if(NOT DEFINED SD_COMMON_TYPES_COUNT OR SD_COMMON_TYPES_COUNT EQUAL 0)
38+
validate_and_process_types_failfast()
39+
build_indexed_type_lists()
40+
endif()
41+
42+
initialize_dynamic_combinations()
43+
44+
# Verify combinations were created
45+
if(DEFINED COMBINATIONS_2 AND DEFINED COMBINATIONS_3)
46+
list(LENGTH COMBINATIONS_2 combo2_count)
47+
list(LENGTH COMBINATIONS_3 combo3_count)
48+
message(STATUS "✅ Generated combinations: 2-type=${combo2_count}, 3-type=${combo3_count}")
49+
50+
# Cache for future use
51+
save_combinations_to_cache("${COMBO_CACHE_FILE}")
52+
else()
53+
message(FATAL_ERROR "❌ Failed to generate type combinations!")
54+
endif()
55+
endfunction()
56+
57+
# Save combinations to cache file
58+
function(save_combinations_to_cache cache_file)
59+
message(STATUS "💾 Saving combinations to cache...")
60+
61+
set(cache_content "# Generated type combinations cache\n")
62+
string(APPEND cache_content "# Generated on: ${CMAKE_CURRENT_TIMESTAMP}\n\n")
63+
64+
# Save 2-type combinations
65+
string(APPEND cache_content "set(COMBINATIONS_2 \"")
66+
foreach(combo ${COMBINATIONS_2})
67+
string(APPEND cache_content "${combo};")
68+
endforeach()
69+
string(APPEND cache_content "\" CACHE INTERNAL \"2-type combinations\")\n\n")
70+
71+
# Save 3-type combinations
72+
string(APPEND cache_content "set(COMBINATIONS_3 \"")
73+
foreach(combo ${COMBINATIONS_3})
74+
string(APPEND cache_content "${combo};")
75+
endforeach()
76+
string(APPEND cache_content "\" CACHE INTERNAL \"3-type combinations\")\n\n")
77+
78+
# Save metadata
79+
string(APPEND cache_content "set(COMBINATIONS_CACHE_TIMESTAMP \"${CMAKE_CURRENT_TIMESTAMP}\" CACHE INTERNAL \"Cache timestamp\")\n")
80+
string(APPEND cache_content "set(COMBINATIONS_CACHE_VALID TRUE CACHE INTERNAL \"Cache validity flag\")\n")
81+
82+
file(WRITE "${cache_file}" "${cache_content}")
83+
message(STATUS "💾 Combinations cached to: ${cache_file}")
84+
endfunction()
85+
86+
# Generate reusable combination strings once
87+
function(generate_combination_strings_once)
88+
if(DEFINED GLOBAL_COMB2_STRING AND DEFINED GLOBAL_COMB3_STRING)
89+
message(STATUS "♻️ Reusing existing combination strings")
90+
return()
91+
endif()
92+
93+
message(STATUS "🔧 Generating combination strings...")
94+
95+
# Generate 2-type combination string
96+
set(COMB2_STRING "")
97+
foreach(combination ${COMBINATIONS_2})
98+
string(REPLACE "," ";" combo_parts "${combination}")
99+
list(GET combo_parts 0 t1)
100+
list(GET combo_parts 1 t2)
101+
string(APPEND COMB2_STRING "INSTANTIATE_2(${t1}, ${t2})")
102+
endforeach()
103+
104+
# Generate 3-type combination string
105+
set(COMB3_STRING "")
106+
foreach(combination ${COMBINATIONS_3})
107+
string(REPLACE "," ";" combo_parts "${combination}")
108+
list(GET combo_parts 0 t1)
109+
list(GET combo_parts 1 t2)
110+
list(GET combo_parts 2 t3)
111+
string(APPEND COMB3_STRING "INSTANTIATE_3(${t1}, ${t2}, ${t3}); ")
112+
endforeach()
113+
114+
# Cache globally for reuse
115+
set(GLOBAL_COMB2_STRING "${COMB2_STRING}" CACHE INTERNAL "Global 2-type combination string")
116+
set(GLOBAL_COMB3_STRING "${COMB3_STRING}" CACHE INTERNAL "Global 3-type combination string")
117+
118+
list(LENGTH COMBINATIONS_2 combo2_count)
119+
list(LENGTH COMBINATIONS_3 combo3_count)
120+
message(STATUS "✅ Generated strings: 2-type=${combo2_count} instantiations, 3-type=${combo3_count} instantiations")
121+
endfunction()
122+
123+
# Clear cache if needed (for development)
124+
function(clear_combination_cache)
125+
set(COMBO_CACHE_FILE "${CMAKE_BINARY_DIR}/type_combinations.cache")
126+
if(EXISTS "${COMBO_CACHE_FILE}")
127+
file(REMOVE "${COMBO_CACHE_FILE}")
128+
message(STATUS "🗑️ Cleared combination cache")
129+
endif()
130+
131+
# Clear memory cache
132+
unset(COMBINATIONS_2 CACHE)
133+
unset(COMBINATIONS_3 CACHE)
134+
unset(GLOBAL_COMB2_STRING CACHE)
135+
unset(GLOBAL_COMB3_STRING CACHE)
136+
unset(COMBINATIONS_CACHE_VALID CACHE)
137+
endfunction()
138+
139+
# Validate cache integrity
140+
function(validate_combination_cache)
141+
if(NOT DEFINED COMBINATIONS_CACHE_VALID OR NOT COMBINATIONS_CACHE_VALID)
142+
message(STATUS "⚠️ Combination cache is invalid, will regenerate")
143+
clear_combination_cache()
144+
return()
145+
endif()
146+
147+
# Check if combinations are reasonable
148+
if(DEFINED COMBINATIONS_2 AND DEFINED COMBINATIONS_3)
149+
list(LENGTH COMBINATIONS_2 combo2_count)
150+
list(LENGTH COMBINATIONS_3 combo3_count)
151+
152+
if(combo2_count LESS 10 OR combo3_count LESS 10)
153+
message(STATUS "⚠️ Combination counts seem too low, will regenerate")
154+
clear_combination_cache()
155+
return()
156+
endif()
157+
158+
message(STATUS "✅ Cache validation passed")
159+
else()
160+
message(STATUS "⚠️ Combinations not found in cache, will regenerate")
161+
clear_combination_cache()
162+
endif()
163+
endfunction()
164+
165+
# Main setup function
166+
function(setup_smart_combination_reuse)
167+
message(STATUS "=== SETTING UP SMART COMBINATION REUSE ===")
168+
169+
# Validate existing cache
170+
validate_combination_cache()
171+
172+
# Use cached combinations without regeneration
173+
use_cached_type_combinations()
174+
175+
# Create combination strings once, reuse everywhere
176+
generate_combination_strings_once()
177+
178+
message(STATUS "✅ Smart combination reuse setup complete")
179+
endfunction()

‎libnd4j/cmake/DuplicateInstantiationDetection.cmake‎

Lines changed: 64 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,64 @@
1+
# GCC and Clang flags for C++ template duplicate instantiation issues
2+
cmake_minimum_required(VERSION 3.15)
3+
4+
if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
5+
# For C++ template duplicate instantiation errors
6+
if(NOT SD_CUDA)
7+
add_compile_options(-fpermissive)
8+
add_compile_options(-ftemplate-depth=1024)
9+
add_compile_options(-fno-gnu-unique)
10+
message(STATUS "Added -fpermissive: Allows duplicate template instantiations")
11+
message(STATUS "Added template-related flags for C++ duplicate handling")
12+
else()
13+
# Also set CXX flags for host compilation
14+
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fpermissive -Wno-error")
15+
message(STATUS "Added CUDA-specific compiler flags for duplicate template instantiations")
16+
message(STATUS "Added --disable-warnings to suppress NVCC errors")
17+
endif()
18+
elseif(CMAKE_CXX_COMPILER_ID STREQUAL "Clang")
19+
# For Clang template duplicate instantiation handling
20+
if(NOT SD_CUDA)
21+
# Template instantiation depth
22+
add_compile_options(-ftemplate-depth=1024)
23+
24+
# Check if this is an Android build
25+
if(ANDROID OR CMAKE_SYSTEM_NAME STREQUAL "Android")
26+
# Android NDK specific handling - be more permissive like GCC
27+
add_compile_options(-Wno-error)
28+
add_compile_options(-Wno-duplicate-decl-specifier)
29+
add_compile_options(-Wno-unused-command-line-argument)
30+
add_compile_options(-ffunction-sections)
31+
add_compile_options(-fdata-sections)
32+
# Android linker doesn't support ICF well, use basic folding
33+
set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wl,--gc-sections")
34+
set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -Wl,--gc-sections")
35+
message(STATUS "Added Android-specific Clang flags for duplicate template handling")
36+
else()
37+
# Desktop Clang with full template folding support
38+
add_compile_options(-ffunction-sections)
39+
add_compile_options(-fdata-sections)
40+
add_compile_options(-fmerge-all-constants)
41+
add_compile_options(-fno-unique-section-names)
42+
# Use LLD linker for better template folding support
43+
add_compile_options(-fuse-ld=lld)
44+
# Linker flags for identical code folding (ICF)
45+
set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wl,--icf=all -Wl,--gc-sections")
46+
set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -Wl,--icf=all -Wl,--gc-sections")
47+
message(STATUS "Added desktop Clang template folding with LLD linker")
48+
endif()
49+
else()
50+
# CUDA with Clang
51+
if(ANDROID OR CMAKE_SYSTEM_NAME STREQUAL "Android")
52+
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -ftemplate-depth=1024 -Wno-error -Wno-duplicate-decl-specifier")
53+
set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wl,--gc-sections")
54+
set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -Wl,--gc-sections")
55+
message(STATUS "Added Android CUDA-specific Clang flags")
56+
else()
57+
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -ftemplate-depth=1024 -ffunction-sections -fdata-sections")
58+
set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} -Xcompiler=-ffunction-sections -Xcompiler=-fdata-sections")
59+
set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fuse-ld=lld -Wl,--icf=all")
60+
set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -fuse-ld=lld -Wl,--icf=all")
61+
message(STATUS "Added desktop CUDA-specific Clang template folding")
62+
endif()
63+
endif()
64+
endif()

0 commit comments

Comments
 (0)