From 0fcf379cbbb00e97ed853c99833bd440e07d7364 Mon Sep 17 00:00:00 2001 From: Jean-Luc Fattebert Date: Fri, 10 Dec 2021 11:27:52 -0500 Subject: [PATCH 01/14] Tune for spock with LLVM13 --- drivers/loopCALPHADConcSolverBinary.cc | 2 +- drivers/loopCALPHADConcSolverTernary.cc | 2 +- scripts/build_spock_llvm13.sh | 32 +++++++++++++++++++ src/CALPHADFreeEnergyFunctionsBinary.cc | 12 +++---- src/CALPHADFreeEnergyFunctionsBinary3Ph2Sl.cc | 10 +++--- ...PHADFreeEnergyFunctionsBinaryThreePhase.cc | 10 +++--- src/CALPHADFreeEnergyFunctionsTernary.cc | 14 ++++---- src/KKSFreeEnergyFunctionDiluteBinary.cc | 8 ++--- src/KKSFreeEnergyFunctionDiluteBinary.h | 25 +++++++++++---- src/PhysicalConstants.h | 12 ++----- src/xlogx.cc | 30 +++++++---------- tests/testCALPHADConcSolverBinary3Ph2Sl.cc | 4 +-- .../testCALPHADConcSolverBinaryThreePhase.cc | 4 +-- 13 files changed, 98 insertions(+), 67 deletions(-) create mode 100755 scripts/build_spock_llvm13.sh diff --git a/drivers/loopCALPHADConcSolverBinary.cc b/drivers/loopCALPHADConcSolverBinary.cc index 90f4dca..fdfc3b5 100644 --- a/drivers/loopCALPHADConcSolverBinary.cc +++ b/drivers/loopCALPHADConcSolverBinary.cc @@ -93,7 +93,7 @@ int main(int argc, char* argv[]) // for(int i=0;i<4;i++)std::cout<<"Lmix_A["< -#ifdef HAVE_OPENMP_OFFLOAD -#pragma omp declare target -#endif -static const double smallx = 1.0e-5; -static const double inv_smallx = 1. / smallx; -static const double log_smallx = log(smallx); -static const double smallx_log_smallx = smallx * log_smallx; -#ifdef HAVE_OPENMP_OFFLOAD -#pragma omp end declare target -#endif +#define SMALLX 1.0e-5 +#define INVSMALLX 1.e5 +#define LOGSMALLX -11.512925464970229 +#define SMALLXLOGSMALLX -11.512925464970229e-5 namespace Thermo4PFM { @@ -21,39 +15,39 @@ namespace Thermo4PFM #endif double xlogx(const double x) { - if (x > smallx) + if (x > SMALLX) { return x * log(x); } else { - return smallx_log_smallx + (x - smallx) * log_smallx - + 0.5 * (x * x * inv_smallx - smallx); + return SMALLXLOGSMALLX + (x - SMALLX) * LOGSMALLX + + 0.5 * (x * x * INVSMALLX - SMALLX); } } double xlogx_deriv(const double x) { - if (x > smallx) + if (x > SMALLX) { return log(x) + 1.0; } else { - return log_smallx + x * inv_smallx; + return LOGSMALLX + x * INVSMALLX; } } template DataType xlogx_deriv2(const DataType x) { - if (x > smallx) + if (x > SMALLX) { return 1. / x; } else { - return inv_smallx; + return INVSMALLX; } } diff --git a/tests/testCALPHADConcSolverBinary3Ph2Sl.cc b/tests/testCALPHADConcSolverBinary3Ph2Sl.cc index f76858f..af94bd3 100644 --- a/tests/testCALPHADConcSolverBinary3Ph2Sl.cc +++ b/tests/testCALPHADConcSolverBinary3Ph2Sl.cc @@ -73,7 +73,7 @@ TEST_CASE("CALPHAD conc solver binary 3 phase, 2 sublattice KKS, " // Inputs to the solver const double RTinv - = 1.0 / (Thermo4PFM::gas_constant_R_JpKpmol * temperature); + = 1.0 / (GASCONSTANT_R_JPKPMOL * temperature); double hphi0 = interp_func(conc_interp_func_type, 0.5); double hphi1 = interp_func(conc_interp_func_type, 0.4); @@ -170,7 +170,7 @@ TEST_CASE("CALPHAD conc solver binary 3 phase, 2 sublattice KKS, " // Inputs to the solver const double RTinv - = 1.0 / (Thermo4PFM::gas_constant_R_JpKpmol * temperature); + = 1.0 / (GASCONSTANT_R_JPKPMOL * temperature); // NOTE: The sum of hphi should equal one, which is not necessarily true for // phi diff --git a/tests/testCALPHADConcSolverBinaryThreePhase.cc b/tests/testCALPHADConcSolverBinaryThreePhase.cc index f830c90..d341ab7 100644 --- a/tests/testCALPHADConcSolverBinaryThreePhase.cc +++ b/tests/testCALPHADConcSolverBinaryThreePhase.cc @@ -68,7 +68,7 @@ TEST_CASE("CALPHAD conc solver binary three phase KKS, two-phase consistancy", // Inputs to the solver const double RTinv - = 1.0 / (Thermo4PFM::gas_constant_R_JpKpmol * temperature); + = 1.0 / (GASCONSTANT_R_JPKPMOL * temperature); double hphi0 = interp_func(conc_interp_func_type, 1.0 - phi); double hphi1 = interp_func(conc_interp_func_type, phi); @@ -203,7 +203,7 @@ TEST_CASE("CALPHAD conc solver binary three phase KKS, three-phase convergence", // Inputs to the solver const double RTinv - = 1.0 / (Thermo4PFM::gas_constant_R_JpKpmol * temperature); + = 1.0 / (GASCONSTANT_R_JPKPMOL * temperature); double hphi0 = interp_func(conc_interp_func_type, 0.5); double hphi1 = interp_func(conc_interp_func_type, 0.4); From 45acb0efd4277a520ddc9ad8fc5f65f70681904d Mon Sep 17 00:00:00 2001 From: Jean-Luc Fattebert Date: Fri, 17 Dec 2021 14:49:40 -0500 Subject: [PATCH 02/14] Add build script for LLVM14 --- scripts/build_spock_llvm14.sh | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) create mode 100755 scripts/build_spock_llvm14.sh diff --git a/scripts/build_spock_llvm14.sh b/scripts/build_spock_llvm14.sh new file mode 100755 index 0000000..8ec64b6 --- /dev/null +++ b/scripts/build_spock_llvm14.sh @@ -0,0 +1,32 @@ +#!/bin/csh -f +#setenv LLVM_ROOT /sw/spock/ums/eiw/llvm/14.0.0-20210819 +setenv LLVM_ROOT /sw/spock/ums/eiw/llvm/14.0.0-20211207 +setenv LD_LIBRARY_PATH ${LLVM_ROOT}/lib:/opt/rocm-4.2.0/lib:/opt/rocm-4.2.0/lib64:$LD_LIBRARY_PATH +setenv PATH ${LLVM_ROOT}/bin:$PATH + +module load boost +module load cmake +module load cray-python + +which cc +which clang +which clang++ + +set ROOT = `pwd` + +set BUILD_DIR = ${ROOT}/build +mkdir -p ${BUILD_DIR} +cd ${BUILD_DIR} + +# call cmake +cmake -DCMAKE_CXX_COMPILER=clang++ \ + -DCMAKE_BUILD_TYPE=Release \ + -DWITH_OPENMP_OFFLOAD=ON \ + -DCMAKE_CXX_FLAGS="-O2 -fopenmp -fopenmp-targets=amdgcn-amd-amdhsa -std=c++11 -D__NO_MATH_INLINES -U__SSE2_MATH__ -U__SSE_MATH__" \ + -DWITH_OPENMP_OFFLOAD=ON \ + -DMPIEXEC_EXECUTABLE="srun" \ + -DMPIEXEC_NUMPROCS_FLAG="-n" \ + -DCMAKE_EXE_LINKER_FLAGS="-lm -fopenmp" \ + .. + +make From 326f6823c896997963e43665f1b5b1ea37f4884f Mon Sep 17 00:00:00 2001 From: Jean-Luc Fattebert Date: Wed, 22 Dec 2021 11:31:18 -0500 Subject: [PATCH 03/14] Fix build script for LLVM14 --- scripts/build_spock_llvm14.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/build_spock_llvm14.sh b/scripts/build_spock_llvm14.sh index 8ec64b6..d957936 100755 --- a/scripts/build_spock_llvm14.sh +++ b/scripts/build_spock_llvm14.sh @@ -1,5 +1,4 @@ #!/bin/csh -f -#setenv LLVM_ROOT /sw/spock/ums/eiw/llvm/14.0.0-20210819 setenv LLVM_ROOT /sw/spock/ums/eiw/llvm/14.0.0-20211207 setenv LD_LIBRARY_PATH ${LLVM_ROOT}/lib:/opt/rocm-4.2.0/lib:/opt/rocm-4.2.0/lib64:$LD_LIBRARY_PATH setenv PATH ${LLVM_ROOT}/bin:$PATH @@ -20,12 +19,13 @@ cd ${BUILD_DIR} # call cmake cmake -DCMAKE_CXX_COMPILER=clang++ \ - -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_BUILD_TYPE=Debug \ -DWITH_OPENMP_OFFLOAD=ON \ -DCMAKE_CXX_FLAGS="-O2 -fopenmp -fopenmp-targets=amdgcn-amd-amdhsa -std=c++11 -D__NO_MATH_INLINES -U__SSE2_MATH__ -U__SSE_MATH__" \ -DWITH_OPENMP_OFFLOAD=ON \ -DMPIEXEC_EXECUTABLE="srun" \ -DMPIEXEC_NUMPROCS_FLAG="-n" \ + -DMPIEXEC_NUMPROCS="1" \ -DCMAKE_EXE_LINKER_FLAGS="-lm -fopenmp" \ .. From c2a841b5526075e2225d179f4e329e5cc557ed50 Mon Sep 17 00:00:00 2001 From: Jean-Luc Fattebert Date: Wed, 22 Dec 2021 11:50:42 -0500 Subject: [PATCH 04/14] Link with list of .o instead of .a --- drivers/CMakeLists.txt | 22 ++++++++++++---------- src/CMakeLists.txt | 1 + tests/CMakeLists.txt | 29 +++++++++++++++-------------- 3 files changed, 28 insertions(+), 24 deletions(-) diff --git a/drivers/CMakeLists.txt b/drivers/CMakeLists.txt index 784a7c4..660801b 100644 --- a/drivers/CMakeLists.txt +++ b/drivers/CMakeLists.txt @@ -1,25 +1,27 @@ find_package(PythonInterp 3.2 REQUIRED) +message(STATUS "SOURCES=${SOURCES}") include_directories( ${CMAKE_SOURCE_DIR}/src ) add_executable(loopxlogx - ${CMAKE_SOURCE_DIR}/drivers/loopxlogx.cc ) + ${CMAKE_SOURCE_DIR}/drivers/loopxlogx.cc ${SOURCES}) add_executable(loopCALPHADSpeciesPhaseGibbsEnergy - ${CMAKE_SOURCE_DIR}/drivers/loopCALPHADSpeciesPhaseGibbsEnergy.cc ) + ${CMAKE_SOURCE_DIR}/drivers/loopCALPHADSpeciesPhaseGibbsEnergy.cc ${SOURCES}) add_executable(loopDeterminant ${CMAKE_SOURCE_DIR}/drivers/loopDeterminant.cc ${CMAKE_SOURCE_DIR}/src/Determinant.cc ) add_executable(loopCALPHADConcSolverBinary - ${CMAKE_SOURCE_DIR}/drivers/loopCALPHADConcSolverBinary.cc ) + ${CMAKE_SOURCE_DIR}/drivers/loopCALPHADConcSolverBinary.cc ${SOURCES}) add_executable(loopCALPHADConcSolverTernary - ${CMAKE_SOURCE_DIR}/drivers/loopCALPHADConcSolverTernary.cc ) + ${CMAKE_SOURCE_DIR}/drivers/loopCALPHADConcSolverTernary.cc + ${SOURCES} ) add_executable(loopCALPHADbinaryKKS - ${CMAKE_SOURCE_DIR}/drivers/loopCALPHADbinaryKKS.cc ) + ${CMAKE_SOURCE_DIR}/drivers/loopCALPHADbinaryKKS.cc ${SOURCES}) add_executable(plotEnergyVsComposition ${CMAKE_SOURCE_DIR}/drivers/plotEnergyVsComposition.cc ) @@ -31,11 +33,11 @@ add_executable(computeTieLineTernary target_link_libraries(plotEnergyVsComposition thermo4pfm) target_link_libraries(computeKKSTernary thermo4pfm) target_link_libraries(computeTieLineTernary thermo4pfm) -target_link_libraries(loopxlogx thermo4pfm) -target_link_libraries(loopCALPHADSpeciesPhaseGibbsEnergy thermo4pfm) -target_link_libraries(loopCALPHADConcSolverBinary thermo4pfm) -target_link_libraries(loopCALPHADConcSolverTernary thermo4pfm) -target_link_libraries(loopCALPHADbinaryKKS thermo4pfm) +#target_link_libraries(loopxlogx thermo4pfm) +#target_link_libraries(loopCALPHADSpeciesPhaseGibbsEnergy thermo4pfm) +#target_link_libraries(loopCALPHADConcSolverBinary thermo4pfm) +#target_link_libraries(loopCALPHADConcSolverTernary thermo4pfm) +#target_link_libraries(loopCALPHADbinaryKKS thermo4pfm) target_link_libraries(loopxlogx m) if (${OPENMP_CXX_FOUND}) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 245afe6..1b2801a 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -23,6 +23,7 @@ set(SOURCES ${CMAKE_SOURCE_DIR}/src/InterpolationType.cc ${CMAKE_SOURCE_DIR}/src/KKSdiluteBinaryConcSolver.cc ${CMAKE_SOURCE_DIR}/src/KKSFreeEnergyFunctionDiluteBinary.cc ${CMAKE_SOURCE_DIR}/src/xlogx.cc) +set(SOURCES ${SOURCES} PARENT_SCOPE) if(${SHARED_LIB}) add_library(thermo4pfm SHARED ${SOURCES}) diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 09f0e8e..b8e31cb 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -1,4 +1,5 @@ find_package(PythonInterp 3.2 REQUIRED) +message(STATUS "SOURCES=${SOURCES}") include_directories( ${CMAKE_SOURCE_DIR}/src ) @@ -13,7 +14,7 @@ add_executable(testInterpolationFunctions add_executable(testCALPHADFunctions ${TEST_DIR}/testCALPHADFunctions.cc ) add_executable(testCALPHADSpeciesPhaseGibbsEnergyExpansion - ${TEST_DIR}/testCALPHADSpeciesPhaseGibbsEnergyExpansion.cc ) + ${TEST_DIR}/testCALPHADSpeciesPhaseGibbsEnergyExpansion.cc ${SOURCES}) add_executable(testCALPHADGibbsEnergy ${TEST_DIR}/testCALPHADGibbsEnergy.cc ) add_executable(testCALPHADbinaryKKS @@ -44,22 +45,22 @@ add_executable(testCALPHADternaryTieLine ${TEST_DIR}/testCALPHADternaryTieLine.cc ) add_executable(testLoopCALPHADSpeciesPhaseGibbsEnergy - ${TEST_DIR}/testLoopCALPHADSpeciesPhaseGibbsEnergy.cc ) + ${TEST_DIR}/testLoopCALPHADSpeciesPhaseGibbsEnergy.cc ${SOURCES}) add_executable(testLoopCALPHADbinaryEquilibrium - ${TEST_DIR}/testLoopCALPHADbinaryEquilibrium.cc ) + ${TEST_DIR}/testLoopCALPHADbinaryEquilibrium.cc ${SOURCES}) add_executable(testLoopCALPHADbinaryKKS - ${TEST_DIR}/testLoopCALPHADbinaryKKS.cc ) + ${TEST_DIR}/testLoopCALPHADbinaryKKS.cc ${SOURCES}) add_executable(testLoopCALPHADternaryKKS - ${TEST_DIR}/testLoopCALPHADternaryKKS.cc ) + ${TEST_DIR}/testLoopCALPHADternaryKKS.cc ${SOURCES}) add_executable(testLoopCALPHADternaryTieLine - ${TEST_DIR}/testLoopCALPHADternaryTieLine.cc ) + ${TEST_DIR}/testLoopCALPHADternaryTieLine.cc ${SOURCES}) add_executable(testLoopKKSdiluteBinaryKKS - ${TEST_DIR}/testLoopKKSdiluteBinaryKKS.cc ) + ${TEST_DIR}/testLoopKKSdiluteBinaryKKS.cc ${SOURCES}) target_link_libraries(testAlCuDatabases thermo4pfm) target_link_libraries(testInterpolationFunctions thermo4pfm) target_link_libraries(testCALPHADFunctions thermo4pfm) -target_link_libraries(testCALPHADSpeciesPhaseGibbsEnergyExpansion thermo4pfm) +#target_link_libraries(testCALPHADSpeciesPhaseGibbsEnergyExpansion thermo4pfm) target_link_libraries(testCALPHADGibbsEnergy thermo4pfm) target_link_libraries(testCALPHADbinaryKKS thermo4pfm) target_link_libraries(testCALPHADbinaryKKSThreePhase thermo4pfm) @@ -73,14 +74,14 @@ target_link_libraries(testCALPHADternaryTieLine thermo4pfm) target_link_libraries(testKKSdiluteBinaryKKS thermo4pfm) target_link_libraries(testKKSdiluteBinaryEquilibrium thermo4pfm) -target_link_libraries(testLoopCALPHADSpeciesPhaseGibbsEnergy thermo4pfm) -target_link_libraries(testLoopCALPHADbinaryKKS thermo4pfm) -target_link_libraries(testLoopCALPHADbinaryEquilibrium thermo4pfm) -target_link_libraries(testLoopCALPHADternaryKKS thermo4pfm) -target_link_libraries(testLoopCALPHADternaryTieLine thermo4pfm) +#target_link_libraries(testLoopCALPHADSpeciesPhaseGibbsEnergy thermo4pfm) +#target_link_libraries(testLoopCALPHADbinaryKKS thermo4pfm) +#target_link_libraries(testLoopCALPHADbinaryEquilibrium thermo4pfm) +#target_link_libraries(testLoopCALPHADternaryKKS thermo4pfm) +#target_link_libraries(testLoopCALPHADternaryTieLine thermo4pfm) target_link_libraries(testCALPHADJacobianBinary thermo4pfm) target_link_libraries(testCALPHADJacobianTernary thermo4pfm) -target_link_libraries(testLoopKKSdiluteBinaryKKS thermo4pfm) +#target_link_libraries(testLoopKKSdiluteBinaryKKS thermo4pfm) if (${OPENMP_CXX_FOUND}) target_link_libraries(testCALPHADSpeciesPhaseGibbsEnergyExpansion OpenMP::OpenMP_CXX) From d6cd24300663014dd266e126291156691f02ad70 Mon Sep 17 00:00:00 2001 From: Jean-Luc Fattebert Date: Wed, 22 Dec 2021 11:51:21 -0500 Subject: [PATCH 05/14] Limit loop size to avoid possible resources issues --- drivers/loopCALPHADConcSolverTernary.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/loopCALPHADConcSolverTernary.cc b/drivers/loopCALPHADConcSolverTernary.cc index 5af7079..cc190e3 100644 --- a/drivers/loopCALPHADConcSolverTernary.cc +++ b/drivers/loopCALPHADConcSolverTernary.cc @@ -22,7 +22,7 @@ typedef std::chrono::high_resolution_clock Clock; int main(int argc, char* argv[]) { - const int N = 10000000; + const int N = 1000000; #ifdef _OPENMP std::cout << "Compiled by an OpenMP-compliant implementation.\n"; From eb03c40c0432105afbddf1772c7a78ccec5e144a Mon Sep 17 00:00:00 2001 From: Jean-Luc Fattebert Date: Wed, 22 Dec 2021 15:06:51 -0500 Subject: [PATCH 06/14] Add script for Summit --- scripts/build_summit_llvm14.sh | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) create mode 100755 scripts/build_summit_llvm14.sh diff --git a/scripts/build_summit_llvm14.sh b/scripts/build_summit_llvm14.sh new file mode 100755 index 0000000..b3f3d5e --- /dev/null +++ b/scripts/build_summit_llvm14.sh @@ -0,0 +1,32 @@ +#!/bin/csh -f +module use /sw/summit/modulefiles/ums/stf010/Core/ +module load llvm/14 + +module load boost +module load cmake +module load python +module load cuda + +which cc +which clang +which clang++ + +set ROOT = `pwd` + +set BUILD_DIR = ${ROOT}/build +mkdir -p ${BUILD_DIR} +cd ${BUILD_DIR} + +# call cmake +cmake -DCMAKE_CXX_COMPILER=clang++ \ + -DCMAKE_BUILD_TYPE=Debug \ + -DWITH_OPENMP_OFFLOAD=ON \ + -DCMAKE_CXX_FLAGS="-fopenmp -fopenmp-targets=nvptx64 -Rpass=openmp-opt -Rpass-missed=openmp-opt" \ + -DWITH_OPENMP_OFFLOAD=ON \ + -DMPIEXEC_EXECUTABLE="srun" \ + -DMPIEXEC_NUMPROCS_FLAG="-n" \ + -DMPIEXEC_NUMPROCS="1" \ + -DCMAKE_EXE_LINKER_FLAGS="-lm -fopenmp" \ + .. + +make From 7e8e1bc248077315ef165ef5ca36e8b14b15263e Mon Sep 17 00:00:00 2001 From: Jean-Luc Fattebert Date: Wed, 22 Dec 2021 15:17:55 -0500 Subject: [PATCH 07/14] Fix summit LLVM build script --- scripts/build_summit_llvm14.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/build_summit_llvm14.sh b/scripts/build_summit_llvm14.sh index b3f3d5e..af02975 100755 --- a/scripts/build_summit_llvm14.sh +++ b/scripts/build_summit_llvm14.sh @@ -23,9 +23,10 @@ cmake -DCMAKE_CXX_COMPILER=clang++ \ -DWITH_OPENMP_OFFLOAD=ON \ -DCMAKE_CXX_FLAGS="-fopenmp -fopenmp-targets=nvptx64 -Rpass=openmp-opt -Rpass-missed=openmp-opt" \ -DWITH_OPENMP_OFFLOAD=ON \ - -DMPIEXEC_EXECUTABLE="srun" \ + -DMPIEXEC_EXECUTABLE="/sw/summit/xalt/1.2.1/bin/jsrun" \ -DMPIEXEC_NUMPROCS_FLAG="-n" \ -DMPIEXEC_NUMPROCS="1" \ + -DMPIEXEC_PREFLAGS="-a1;-c4;-bpacked:2;-g1" \ -DCMAKE_EXE_LINKER_FLAGS="-lm -fopenmp" \ .. From c3044fc4735619d08a1e085c182621c5df6f1842 Mon Sep 17 00:00:00 2001 From: Jean-Luc Fattebert Date: Wed, 22 Dec 2021 15:39:49 -0500 Subject: [PATCH 08/14] Enable both loops to run in loopCALPHADConcSolverTernary --- drivers/loopCALPHADConcSolverTernary.cc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/drivers/loopCALPHADConcSolverTernary.cc b/drivers/loopCALPHADConcSolverTernary.cc index cc190e3..aae6183 100644 --- a/drivers/loopCALPHADConcSolverTernary.cc +++ b/drivers/loopCALPHADConcSolverTernary.cc @@ -192,7 +192,7 @@ int main(int argc, char* argv[]) const double deviation = 1.e-4; -#ifndef HAVE_OPENMP_OFFLOAD +//#ifndef HAVE_OPENMP_OFFLOAD double* xhost = new double[4 * N]; for (int i = 0; i < 4 * N; i++) @@ -242,7 +242,7 @@ int main(int argc, char* argv[]) delete[] nits; } delete[] xhost; -#else +//#else double* xdev = new double[4 * N]; short* nits = new short[N]; @@ -331,5 +331,5 @@ int main(int argc, char* argv[]) delete[] xdev; delete[] nits; -#endif +//#endif } From a11bf3f24d94001791ee9676bc870683fab20f7e Mon Sep 17 00:00:00 2001 From: Jean-Luc Fattebert Date: Wed, 29 Dec 2021 13:30:28 -0500 Subject: [PATCH 09/14] Complete recent rebase --- src/KKSFreeEnergyFunctionDiluteBinary.h | 2 +- tests/testCALPHADConcSolverTernary.cc | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/KKSFreeEnergyFunctionDiluteBinary.h b/src/KKSFreeEnergyFunctionDiluteBinary.h index 63308a3..44bbe85 100644 --- a/src/KKSFreeEnergyFunctionDiluteBinary.h +++ b/src/KKSFreeEnergyFunctionDiluteBinary.h @@ -63,7 +63,7 @@ class KKSFreeEnergyFunctionDiluteBinary double computeFB(const double temperature) const; - char fenergy_diag_filename_[11]; + char* fenergy_diag_filename_; /// /// model parameters independent of T, c, ... diff --git a/tests/testCALPHADConcSolverTernary.cc b/tests/testCALPHADConcSolverTernary.cc index d9578ab..9641b5a 100644 --- a/tests/testCALPHADConcSolverTernary.cc +++ b/tests/testCALPHADConcSolverTernary.cc @@ -172,7 +172,7 @@ TEST_CASE("CALPHAD ternary solver", "[ternary solver]") L_ABC_S[i] = LmixABCPhaseA[i][0] + temperature * LmixABCPhaseA[i][1]; const double RTinv - = 1.0 / (Thermo4PFM::gas_constant_R_JpKpmol * temperature); + = 1.0 / (GASCONSTANT_R_JPKPMOL * temperature); double sol[4] = { 0.33, 0.38, 0.32, 0.33 }; double hphi = 0.5; From b34ddba851110f5e4440002f9594b8521fc09bfb Mon Sep 17 00:00:00 2001 From: Jean-Luc Fattebert Date: Wed, 29 Dec 2021 16:06:54 -0500 Subject: [PATCH 10/14] Build with RelWithDebInfo on Summit --- scripts/build_summit_llvm14.sh | 2 +- tests/CMakeLists.txt | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/build_summit_llvm14.sh b/scripts/build_summit_llvm14.sh index af02975..7017223 100755 --- a/scripts/build_summit_llvm14.sh +++ b/scripts/build_summit_llvm14.sh @@ -19,7 +19,7 @@ cd ${BUILD_DIR} # call cmake cmake -DCMAKE_CXX_COMPILER=clang++ \ - -DCMAKE_BUILD_TYPE=Debug \ + -DCMAKE_BUILD_TYPE=RelWithDebInfo \ -DWITH_OPENMP_OFFLOAD=ON \ -DCMAKE_CXX_FLAGS="-fopenmp -fopenmp-targets=nvptx64 -Rpass=openmp-opt -Rpass-missed=openmp-opt" \ -DWITH_OPENMP_OFFLOAD=ON \ diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index b8e31cb..bc2307a 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -42,7 +42,7 @@ add_executable(testCALPHADternaryKKS add_executable(testCALPHADConcSolverTernary ${TEST_DIR}/testCALPHADConcSolverTernary.cc ) add_executable(testCALPHADternaryTieLine - ${TEST_DIR}/testCALPHADternaryTieLine.cc ) + ${TEST_DIR}/testCALPHADternaryTieLine.cc ${SOURCES}) add_executable(testLoopCALPHADSpeciesPhaseGibbsEnergy ${TEST_DIR}/testLoopCALPHADSpeciesPhaseGibbsEnergy.cc ${SOURCES}) @@ -70,7 +70,7 @@ target_link_libraries(testCALPHADConcSolverBinaryThreePhase thermo4pfm) target_link_libraries(testCALPHADConcSolverBinary3Ph2Sl thermo4pfm) target_link_libraries(testCALPHADternaryKKS thermo4pfm) target_link_libraries(testCALPHADConcSolverTernary thermo4pfm) -target_link_libraries(testCALPHADternaryTieLine thermo4pfm) +#target_link_libraries(testCALPHADternaryTieLine thermo4pfm) target_link_libraries(testKKSdiluteBinaryKKS thermo4pfm) target_link_libraries(testKKSdiluteBinaryEquilibrium thermo4pfm) From 40774eace23b442da72ff76c15be18f801ff3cba Mon Sep 17 00:00:00 2001 From: Jean-Luc Fattebert Date: Thu, 30 Dec 2021 10:58:14 -0500 Subject: [PATCH 11/14] Bug fix --- src/KKSFreeEnergyFunctionDiluteBinary.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/KKSFreeEnergyFunctionDiluteBinary.cc b/src/KKSFreeEnergyFunctionDiluteBinary.cc index 57aff48..ea0ba7a 100644 --- a/src/KKSFreeEnergyFunctionDiluteBinary.cc +++ b/src/KKSFreeEnergyFunctionDiluteBinary.cc @@ -22,7 +22,7 @@ KKSFreeEnergyFunctionDiluteBinary::KKSFreeEnergyFunctionDiluteBinary( alpha_(1.) { std::string fenergy_diag_filename("energy.vtk"); - //fenergy_diag_filename_ = new char[fenergy_diag_filename.length() + 1]; + fenergy_diag_filename_ = new char[fenergy_diag_filename.length() + 1]; strcpy(fenergy_diag_filename_, fenergy_diag_filename.c_str()); readParameters(conc_db); From 8c7f4f6b3fcb392fe0026eb28d997073ed4e6d2e Mon Sep 17 00:00:00 2001 From: Jean-Luc Fattebert Date: Fri, 4 Mar 2022 12:12:26 -0500 Subject: [PATCH 12/14] Change compiler flags --- scripts/build_summit_llvm14.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/build_summit_llvm14.sh b/scripts/build_summit_llvm14.sh index 7017223..1939ba2 100755 --- a/scripts/build_summit_llvm14.sh +++ b/scripts/build_summit_llvm14.sh @@ -19,8 +19,9 @@ cd ${BUILD_DIR} # call cmake cmake -DCMAKE_CXX_COMPILER=clang++ \ - -DCMAKE_BUILD_TYPE=RelWithDebInfo \ + -DCMAKE_BUILD_TYPE=Release \ -DWITH_OPENMP_OFFLOAD=ON \ + -DCMAKE_CXX_FLAGS_RELEASE="-foffload-lto -fopenmp-new-driver -DNDEBUG" \ -DCMAKE_CXX_FLAGS="-fopenmp -fopenmp-targets=nvptx64 -Rpass=openmp-opt -Rpass-missed=openmp-opt" \ -DWITH_OPENMP_OFFLOAD=ON \ -DMPIEXEC_EXECUTABLE="/sw/summit/xalt/1.2.1/bin/jsrun" \ From fd0d84c1e664bb96ecbbdb9b1397ca5bc0581f37 Mon Sep 17 00:00:00 2001 From: Jean-Luc Fattebert Date: Fri, 4 Mar 2022 12:13:05 -0500 Subject: [PATCH 13/14] Turn off test testLoopCALPHADternaryTieLine --- tests/CMakeLists.txt | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index bc2307a..e51138a 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -52,8 +52,8 @@ add_executable(testLoopCALPHADbinaryKKS ${TEST_DIR}/testLoopCALPHADbinaryKKS.cc ${SOURCES}) add_executable(testLoopCALPHADternaryKKS ${TEST_DIR}/testLoopCALPHADternaryKKS.cc ${SOURCES}) -add_executable(testLoopCALPHADternaryTieLine - ${TEST_DIR}/testLoopCALPHADternaryTieLine.cc ${SOURCES}) +#add_executable(testLoopCALPHADternaryTieLine +# ${TEST_DIR}/testLoopCALPHADternaryTieLine.cc ${SOURCES}) add_executable(testLoopKKSdiluteBinaryKKS ${TEST_DIR}/testLoopKKSdiluteBinaryKKS.cc ${SOURCES}) @@ -88,7 +88,7 @@ if (${OPENMP_CXX_FOUND}) target_link_libraries(testLoopCALPHADSpeciesPhaseGibbsEnergy OpenMP::OpenMP_CXX) target_link_libraries(testLoopCALPHADbinaryEquilibrium OpenMP::OpenMP_CXX) target_link_libraries(testLoopCALPHADbinaryKKS OpenMP::OpenMP_CXX) - target_link_libraries(testLoopCALPHADternaryTieLine OpenMP::OpenMP_CXX) + #target_link_libraries(testLoopCALPHADternaryTieLine OpenMP::OpenMP_CXX) target_link_libraries(testLoopCALPHADternaryKKS OpenMP::OpenMP_CXX) target_link_libraries(testLoopKKSdiluteBinaryKKS OpenMP::OpenMP_CXX) endif() @@ -207,11 +207,11 @@ add_test(NAME testLoopCALPHADternaryKKS ${MPIEXEC_NUMPROCS} ${MPIEXEC_PREFLAGS} ${CMAKE_CURRENT_BINARY_DIR}/testLoopCALPHADternaryKKS WORKING_DIRECTORY ${TEST_DIR}) -add_test(NAME testLoopCALPHADternaryTieLine - COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROCS_FLAG} - ${MPIEXEC_NUMPROCS} ${MPIEXEC_PREFLAGS} - ${CMAKE_CURRENT_BINARY_DIR}/testLoopCALPHADternaryTieLine - WORKING_DIRECTORY ${TEST_DIR}) +#add_test(NAME testLoopCALPHADternaryTieLine +# COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROCS_FLAG} +# ${MPIEXEC_NUMPROCS} ${MPIEXEC_PREFLAGS} +# ${CMAKE_CURRENT_BINARY_DIR}/testLoopCALPHADternaryTieLine +# WORKING_DIRECTORY ${TEST_DIR}) add_test(NAME testLoopKKSdiluteBinaryKKS COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROCS_FLAG} ${MPIEXEC_NUMPROCS} ${MPIEXEC_PREFLAGS} From 6dc9e521c52f3da053bc09a2a81ee74909e332a0 Mon Sep 17 00:00:00 2001 From: Jean-Luc Fattebert Date: Fri, 4 Mar 2022 12:13:43 -0500 Subject: [PATCH 14/14] Minor adaptations in loopCALPHADConcSolverTernary --- drivers/loopCALPHADConcSolverTernary.cc | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/drivers/loopCALPHADConcSolverTernary.cc b/drivers/loopCALPHADConcSolverTernary.cc index aae6183..33800d0 100644 --- a/drivers/loopCALPHADConcSolverTernary.cc +++ b/drivers/loopCALPHADConcSolverTernary.cc @@ -22,7 +22,7 @@ typedef std::chrono::high_resolution_clock Clock; int main(int argc, char* argv[]) { - const int N = 1000000; + const int N = 1000; #ifdef _OPENMP std::cout << "Compiled by an OpenMP-compliant implementation.\n"; @@ -192,7 +192,7 @@ int main(int argc, char* argv[]) const double deviation = 1.e-4; -//#ifndef HAVE_OPENMP_OFFLOAD +#ifndef HAVE_OPENMP_OFFLOAD double* xhost = new double[4 * N]; for (int i = 0; i < 4 * N; i++) @@ -242,9 +242,10 @@ int main(int argc, char* argv[]) delete[] nits; } delete[] xhost; -//#else +#else double* xdev = new double[4 * N]; short* nits = new short[N]; + for (int i = 0; i < N; i++)nits[i]=-1; // Device solve { @@ -253,9 +254,10 @@ int main(int argc, char* argv[]) xdev[i] = -1; } - // warm-up GPU with a dummy allocation - double dummy[N]; -#pragma omp target enter data map(alloc : dummy[:N]) + // warm-up GPU with an empty target region +#pragma omp target +{ +} auto t1 = Clock::now(); @@ -276,10 +278,10 @@ int main(int argc, char* argv[]) double hphi = 0.5 + (i % 100) * deviation; double c0 = 0.33; double c1 = 0.33; - class Thermo4PFM::CALPHADConcSolverTernary solver; + Thermo4PFM::CALPHADConcSolverTernary solver; solver.setup(c0, c1, hphi, RTinv, L_AB_L, L_AC_L, L_BC_L, L_AB_S, L_AC_S, L_BC_S, L_ABC_L, L_ABC_S, fA, fB, fC); - nits[i] = solver.ComputeConcentration(&xdev[4 * i], 1.e-8, 50); + nits[i] = solver.ComputeConcentration(&xdev[4 * i], 1.e-8, 10); } } @@ -291,8 +293,6 @@ int main(int argc, char* argv[]) std::cout << "Device time/us/solve: " << (double)usec / (double)N << std::endl; -#pragma omp target exit data map(delete : dummy[:N]) - // print out some results std::cout << std::setprecision(12); int n = N > 20 ? 20 : N; @@ -331,5 +331,5 @@ int main(int argc, char* argv[]) delete[] xdev; delete[] nits; -//#endif +#endif }