diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml index 7574747..f5bcc64 100644 --- a/.github/workflows/build-and-test.yml +++ b/.github/workflows/build-and-test.yml @@ -74,15 +74,18 @@ jobs: strategy: fail-fast: false matrix: - os: [ubuntu-latest, windows-latest] - compiler: [g++, clang++, cl] - exclude: - - os: ubuntu-latest - compiler: cl - - os: windows-latest - compiler: g++ - - os: windows-latest - compiler: clang++ + # TODO(zpzim): Windows is disabled for now because it is flaky. + #os: [ubuntu-latest, windows-latest] + #compiler: [g++, clang++, cl] + os: [ubuntu-latest] + compiler: [g++, clang++] + #exclude: + # - os: ubuntu-latest + # compiler: cl + # - os: windows-latest + # compiler: g++ + # - os: windows-latest + # compiler: clang++ runs-on: ${{ matrix.os }} @@ -205,7 +208,7 @@ jobs: set -e cd $GITHUB_WORKSPACE python3 setup.py sdist - python3 -m pip install dist/* + python3 -m pip install -v dist/* python3 -m pip install numpy pandas tqdm - name: pyscamp Test diff --git a/.github/workflows/python-publish-test.yml b/.github/workflows/python-publish-test.yml new file mode 100644 index 0000000..bbd32f9 --- /dev/null +++ b/.github/workflows/python-publish-test.yml @@ -0,0 +1,32 @@ +name: Upload Test Python Package + +on: + push: + branches: [ master ] + workflow_dispatch: + +jobs: + deploy: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + with: + submodules: 'true' + fetch-depth: 0 + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: '3.x' + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install build + - name: Build package + run: python -m build --sdist + - name: Publish package + uses: pypa/gh-action-pypi-publish@27b31702a0e7fc50959f5ad993c78deac1bdfc29 + with: + user: __token__ + password: ${{ secrets.TEST_PYPI_API_TOKEN }} + repository_url: https://test.pypi.org/legacy/ + skip_existing: True diff --git a/CMakeLists.txt b/CMakeLists.txt index 39c272c..256c5b1 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -102,17 +102,20 @@ if (SCAMP_ENABLE_BINARY_DISTRIBUTION) message(STATUS "Building REDISTRIBUTABLE SCAMP Binary.") endif() -# Use clang tidy if available -find_program( - CLANG_TIDY_EXE - NAMES clang-tidy - DOC "Path to clang-tidy executable (v5+)" - ) -if(NOT CLANG_TIDY_EXE) - message(STATUS "clang-tidy not found.") +if (SCAMP_USE_CLANG_TIDY) + find_program( + CLANG_TIDY_EXE + NAMES clang-tidy + DOC "Path to clang-tidy executable (v5+)" + ) + if(NOT CLANG_TIDY_EXE) + message(STATUS "clang-tidy not found.") + else() + message(STATUS "clang-tidy found: ${CLANG_TIDY_EXE}") + set(DO_CLANG_TIDY "${CLANG_TIDY_EXE}" "-checks=*,-cppcoreguidelines-pro-bounds-pointer-arithmetic,-fuchsia-default-arguments,-hicpp-vararg,-cppcoreguidelines-pro-type-vararg, -cppcoreguidelines-pro-bounds-constant-array-index,-google-runtime-int") + endif() else() - message(STATUS "clang-tidy found: ${CLANG_TIDY_EXE}") - set(DO_CLANG_TIDY "${CLANG_TIDY_EXE}" "-checks=*,-cppcoreguidelines-pro-bounds-pointer-arithmetic,-fuchsia-default-arguments,-hicpp-vararg,-cppcoreguidelines-pro-type-vararg, -cppcoreguidelines-pro-bounds-constant-array-index,-google-runtime-int") + message(STATUS "clang-tidy is not enabled. To use clang-tidy checks pass -DSCAMP_USE_CLANG_TIDY=ON to cmake") endif() # Use clang format if available @@ -151,7 +154,7 @@ if (CMAKE_CUDA_COMPILER) set(CMAKE_CUDA_STANDARD 14) include_directories(${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES}) set(CUDA_SEPARABLE_COMPILATION ON) - set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} -Xcompiler=-fPIC -lineinfo") + set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} -lineinfo") if (CUDA_VERSION VERSION_GREATER_EQUAL "11.0") set(CUDA_GENCODE_FLAGS "${CUDA_GENCODE_FLAGS} -gencode arch=compute_80,code=sm_80") endif() @@ -184,6 +187,7 @@ CHECK_CXX_COMPILER_FLAG("/fp:contract" COMPILER_OPT_FPCONTRACT_SUPPORTED) CHECK_CXX_COMPILER_FLAG("/fp:precise" COMPILER_OPT_FPPRECISE_SUPPORTED) CHECK_CXX_COMPILER_FLAG("-fPIC" COMPILER_OPT_PIC_SUPPORTED) CHECK_CXX_COMPILER_FLAG("-Wall" COMPILER_OPT_WARN_ALL_SUPPORTED) +CHECK_CXX_COMPILER_FLAG("/W4" COMPILER_OPT_W4_SUPPORTED) CHECK_CXX_COMPILER_FLAG("-Wno-sign-compare" COMPILER_OPT_NO_WARN_SIGN_COMPARE_SUPPORTED) set(SCAMP_ALL_BUILD_FLAGS "") @@ -191,6 +195,7 @@ set(SCAMP_OPTIMIZED_BUILD_FLAGS "") if (COMPILER_OPT_PIC_SUPPORTED) set(SCAMP_ALL_BUILD_FLAGS "${SCAMP_ALL_BUILD_FLAGS} -fPIC") + set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} -Xcompiler=-fPIC") set(CMAKE_POSITION_INDEPENDENT_CODE ON) endif() @@ -228,10 +233,14 @@ if (COMPILER_OPT_FPPRECISE_SUPPORTED) set(SCAMP_OPTIMIZED_BUILD_FLAGS "${SCAMP_OPTIMIZED_BUILD_FLAGS} /fp:precise") endif() -if (COMPILER_OPT_WARN_ALL_SUPPORTED) +if (COMPILER_OPT_WARN_ALL_SUPPORTED AND NOT MSVC) set(SCAMP_ALL_BUILD_FLAGS "${SCAMP_ALL_BUILD_FLAGS} -Wall") endif() +if (COMPILER_OPT_W4_SUPPORTED AND MSVC) + set(SCAMP_ALL_BUILD_FLAGS "${SCAMP_ALL_BUILD_FLAGS} /W4") +endif() + if (COMPILER_OPT_NO_WARN_SIGN_COMPARE_SUPPORTED) set(SCAMP_ALL_BUILD_FLAGS "${SCAMP_ALL_BUILD_FLAGS} -Wno-sign-compare") endif() @@ -289,46 +298,32 @@ if (BUILD_CLIENT_SERVER) add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/src/distributed) endif() -if (BUILD_TESTING) - set(STORED_BUILD_TESTING ${BUILD_TESTING}) -endif() -set(BUILD_TESTING OFF) -# Cpu Kernels depend on Eigen and cpu_features -add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/third_party/eigen) -add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/third_party/cpu_features) -if (STORED_BUILD_TESTING) - set(BUILD_TESTING ${STORED_BUILD_TESTING}) -else() - unset(BUILD_TESTING) -endif() - # Add core functional libraries. add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/src/core) # Python module if (BUILD_PYTHON_MODULE) add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/src/python) -endif() - -if (BUILD_BENCHMARKS) - add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/src/benchmark) -endif() +else() + if (BUILD_BENCHMARKS) + add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/src/benchmark) + endif() + if(NOT TARGET gflags) + add_subdirectory(third_party/gflags) + include_directories(third_party/gflags/include/) + endif() -if(NOT TARGET gflags) - add_subdirectory(third_party/gflags) - include_directories(third_party/gflags/include/) + # C++/CUDA executable + add_executable(SCAMP ${CMAKE_CURRENT_SOURCE_DIR}/src/main.cpp) + if (CMAKE_CUDA_COMPILER) + target_compile_definitions(SCAMP PUBLIC -D_HAS_CUDA_) + target_link_libraries(SCAMP ${CUDA_cudart_LIBRARY} gflags common scamp_interface scamp_utils) + else() + target_link_libraries(SCAMP gflags common scamp_interface scamp_utils) + endif() endif() -# C++/CUDA executable -add_executable(SCAMP ${CMAKE_CURRENT_SOURCE_DIR}/src/main.cpp) -if (CMAKE_CUDA_COMPILER) - target_compile_definitions(SCAMP PUBLIC -D_HAS_CUDA_) - target_link_libraries(SCAMP ${CUDA_cudart_LIBRARY} gflags common scamp_interface scamp_utils) -else() - target_link_libraries(SCAMP gflags common scamp_interface scamp_utils) -endif() - # clang-format target function(prepend var prefix) set(listVar "") diff --git a/MANIFEST.in b/MANIFEST.in index d8d36dd..3fe88fd 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,8 +1,9 @@ -graft src +graft src/common +graft src/core +graft src/python graft test graft cmake graft third_party/cpu_features -graft third_party/gflags graft third_party/pybind11 graft third_party/eigen prune .github diff --git a/setup.py b/setup.py index 70c4ac5..bdc69e2 100644 --- a/setup.py +++ b/setup.py @@ -42,7 +42,8 @@ def build_extension(self, ext): env['CXXFLAGS'] = '{} -DVERSION_INFO=\\"{}\\"'.format(env.get('CXXFLAGS', ''), self.distribution.get_version()) extdir = os.path.abspath(os.path.dirname(self.get_ext_fullpath(ext.name))) - cmake_args = ['-DPYTHON_EXECUTABLE=' + sys.executable] + python_exe = os.environ.get('PYSCAMP_PYTHON_EXECUTABLE_PATH', sys.executable) + cmake_args = ['-DPYTHON_EXECUTABLE=' + python_exe] cmake_args += ['-DCMAKE_LIBRARY_OUTPUT_DIRECTORY=' + extdir] force_cuda = os.environ.get("FORCE_CUDA", "") @@ -54,6 +55,8 @@ def build_extension(self, ext): # Default to release build. build_type = os.environ.get("PYSCAMP_BUILD_TYPE", "Release") + # We need to set CMAKE_BUILD_TYPE here in case we aren't using a multi-config generator (e.g. Ninja) + env['CMAKE_BUILD_TYPE'] = build_type # Pile all .so in one place and use $ORIGIN as RPATH cmake_args += ["-DCMAKE_BUILD_WITH_INSTALL_RPATH=TRUE"] @@ -81,12 +84,15 @@ def build_extension(self, ext): if not os.path.exists(self.build_temp): os.makedirs(self.build_temp) - subprocess.check_call(['cmake', ext.sourcedir] + cmake_args, cwd=self.build_temp, env=env) - subprocess.check_call(['cmake', - '--build', '.', - '--target', ext.name, - '--config', build_type, - '--parallel', '4'], cwd=self.build_temp) + configure_cmd = ['cmake', ext.sourcedir] + cmake_args + print("Configuring SCAMP") + print(' '.join(configure_cmd)) + subprocess.check_call(configure_cmd, cwd=self.build_temp, env=env) + + build_cmd = ['cmake', '--build', '.', '--target', ext.name, '--config', build_type, '--parallel', '4'] + print("Building SCAMP") + print(' '.join(build_cmd)) + subprocess.check_call(build_cmd, cwd=self.build_temp) setup( name='pyscamp', diff --git a/src/core/cpu_kernel/CMakeLists.txt b/src/core/cpu_kernel/CMakeLists.txt index 6f9ee76..f6c370d 100644 --- a/src/core/cpu_kernel/CMakeLists.txt +++ b/src/core/cpu_kernel/CMakeLists.txt @@ -1,11 +1,18 @@ include(SCAMPMacros) +set(BUILD_TESTING OFF) + if (SCAMP_ENABLE_BINARY_DISTRIBUTION) + add_subdirectory(${CMAKE_SOURCE_DIR}/third_party/cpu_features ${CMAKE_CURRENT_BINARY_DIR}/cpu_features) add_definitions("-D_SCAMP_DISTRIBUTABLE_") add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/avx) add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/avx2) endif() + +# Cpu Kernels depend on Eigen. +add_subdirectory(${CMAKE_SOURCE_DIR}/third_party/eigen ${CMAKE_CURRENT_BINARY_DIR}/eigen) + add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/baseline) add_library(cpu_kernels ${CMAKE_CURRENT_SOURCE_DIR}/kernel_dispatcher.cpp) @@ -13,7 +20,7 @@ add_library(cpu_kernels ${CMAKE_CURRENT_SOURCE_DIR}/kernel_dispatcher.cpp) if (SCAMP_ENABLE_BINARY_DISTRIBUTION) target_link_libraries(cpu_kernels dispatch_baseline dispatch_avx dispatch_avx2 CpuFeature::cpu_features) else() - target_link_libraries(cpu_kernels dispatch_baseline CpuFeature::cpu_features) + target_link_libraries(cpu_kernels dispatch_baseline) endif() mark_clang_tidy() diff --git a/src/core/cpu_kernel/kernel_dispatcher.cpp b/src/core/cpu_kernel/kernel_dispatcher.cpp index 69c08a0..9eb0b1e 100644 --- a/src/core/cpu_kernel/kernel_dispatcher.cpp +++ b/src/core/cpu_kernel/kernel_dispatcher.cpp @@ -5,13 +5,15 @@ #include "avx2/dispatch_avx2.h" #endif #include "baseline/dispatch_baseline.h" -#include "cpu_features_macros.h" +#if defined(_SCAMP_DISTRIBUTABLE_) +#include "cpu_features_macros.h" #if defined(CPU_FEATURES_ARCH_X86) #include "cpuinfo_x86.h" static const cpu_features::X86Features features = cpu_features::GetX86Info().features; #endif +#endif namespace SCAMP { diff --git a/test/run_intel_emulation_tests.sh b/test/run_intel_emulation_tests.sh index 00043ef..1e425b8 100755 --- a/test/run_intel_emulation_tests.sh +++ b/test/run_intel_emulation_tests.sh @@ -15,7 +15,7 @@ do echo $CMD eval $CMD ret=$? - if [ $ret -eq 0 ] ; then + if [[ $ret -eq 0 ]]; then echo "PASS" PASSED="${PASSED} $PROFILE/$ARCH" else @@ -25,7 +25,7 @@ do done done -if [ $FAILED ]; then +if [[ -n "$FAILED" ]]; then echo "Detected Failed Architectures: ${FAILED}" exit 1 fi diff --git a/third_party/pybind11 b/third_party/pybind11 index f7b4996..914c06f 160000 --- a/third_party/pybind11 +++ b/third_party/pybind11 @@ -1 +1 @@ -Subproject commit f7b499615e14d70ab098a20deb0cdb3889998a1a +Subproject commit 914c06fb252b6cc3727d0eedab6736e88a3fcb01