Merge pull request #2546 from zma2/sync

 Sync examples with the GPU Optimization Guide repo

Author: jimmytwei

Publications/GPU-Opt-Guide/CMakeLists.txt

@@ -1,4 +1,4 @@
-cmake_minimum_required(VERSION 3.21)
+cmake_minimum_required(VERSION 3.21 FATAL_ERROR)
 option(BUILD_FORTRAN_EXAMPLES "Whether to build fortran examples" ON)
 set(CMAKE_C_COMPILER icx)
 set(CMAKE_CXX_COMPILER icpx)
@@ -13,23 +13,24 @@ enable_testing()
 
 project(GPUOptGuide
   LANGUAGES ${_languages}
-  DESCRIPTION "Code examples from Intel GPU Optimization guide")
+  DESCRIPTION "Examples from oneAPI GPU Optimization Guide")
 
 set(CMAKE_FIND_ROOT_PATH_MODE_LIBRARY BOTH)
 
 find_package(IntelSYCL REQUIRED)
 
+if (BUILD_FOTRAN_EXAMPLES)
+  check_language(Fortran)
+  if(CMAKE_Fortran_COMPILER)
+    enable_language(Fortran)
+  else()
+    message(FATAL_ERROR "No Fortran support detected, but Fortran tests were requested. Install oneAPI HPC Toolkit.")
+  endif()
+endif()
+
 set(MKL_THREADING tbb_thread)
 set(MKL_INTERFACE "ilp64")
 set(DPCPP_COMPILER ON)
-
-set(MKL_VERSION_2024 FALSE)
-find_package(MKL QUIET)
-if(MKL_FOUND)
-    if(MKL_VERSION VERSION_GREATER_EQUAL "2024.0.0")
-        set(MKL_VERSION_2024 TRUE)
-    endif()
-endif()
 find_package(MKL REQUIRED)
 
 string(CONCAT WARNING_CXX_FLAGS_STR
@@ -44,116 +45,162 @@ string(CONCAT WARNING_CXX_FLAGS_STR
 string(REPLACE " " ";" COMMON_CXX_FLAGS "${WARNING_CXX_FLAGS_STR}")
 
 function(add_example_with_mkl name)
-  set(_sources ${name}.cpp)
-  add_executable(${name} ${_sources})
-  add_sycl_to_target(TARGET ${name} SOURCES ${_sources})
-  target_compile_options(${name} PRIVATE ${COMMON_CXX_FLAGS})
-  if (MKL_VERSION_2024)
-    target_link_libraries(${name} PUBLIC MKL::MKL_SYCL)
+  cmake_parse_arguments(FUNC_SRC "" "" "SOURCES" ${ARGN})
+  if (FUNC_SRC_SOURCES)
+    set(_src ${FUNC_SRC_SOURCES})
   else()
-    target_link_libraries(${name} PUBLIC MKL::MKL_DPCPP)
+    set(_src ${name}.cpp)
   endif()
+  add_executable(${name} ${_src})
+  add_sycl_to_target(TARGET ${name} SOURCES ${_src})
+  target_compile_options(${name} PRIVATE ${COMMON_CXX_FLAGS})
+  target_compile_options(${name} PRIVATE -fiopenmp -fopenmp-targets=spir64 -qmkl)
+  target_link_libraries(${name} PRIVATE MKL::MKL_DPCPP)
+  target_link_options(${name} PRIVATE -fiopenmp -fopenmp-targets=spir64 -qmkl -lOpenCL)
   add_test(NAME ${name} COMMAND ${name} ${ARGN})
 endfunction(add_example_with_mkl)
 
 function(add_fortran_example_with_mkl name)
   if(CMAKE_Fortran_COMPILER)
-    set(_sources ${name}.f)
-    add_executable(${name} ${_sources})
-    add_sycl_to_target(TARGET ${name} SOURCES ${_sources})
+    set(_src ${name}.f)
+    add_executable(${name} ${_src})
+    add_sycl_to_target(TARGET ${name} SOURCES ${_src})
     target_compile_options(${name} PRIVATE -warn all)
-    target_compile_options(${name} PRIVATE -fpp -free -DMKL_ILP64 -i8)
+    target_compile_options(${name} PRIVATE -fiopenmp -fopenmp-targets=spir64 -qmkl -fpp -free)
     set_target_properties(${name} PROPERTIES LINKER_LANGUAGE Fortran)
-    if (MKL_VERSION_2024)
-      target_link_libraries(${name} PUBLIC MKL::MKL_SYCL)
-    else()
-      target_link_libraries(${name} PUBLIC MKL::MKL_DPCPP)
-    endif()
+    target_link_libraries(${name} PUBLIC MKL::MKL_DPCPP)
+    target_link_options(${name} PRIVATE -fiopenmp -fopenmp-targets=spir64 -qmkl -lOpenCL)
     add_test(NAME ${name} COMMAND ${name} ${ARGN})
   endif()
 endfunction(add_fortran_example_with_mkl)
 
+function(add_fortran_example_with_mkl_i8 name)
+  if(CMAKE_Fortran_COMPILER)
+    set(_src ${name}.f)
+    add_executable(${name} ${_src})
+    add_sycl_to_target(TARGET ${name} SOURCES ${_src})
+    target_compile_options(${name} PRIVATE -warn all)
+    target_compile_options(${name} PRIVATE -fiopenmp -fopenmp-targets=spir64 -qmkl -fpp -free -DMKL_ILP64 -i8)
+    set_target_properties(${name} PROPERTIES LINKER_LANGUAGE Fortran)
+    target_link_libraries(${name} PUBLIC MKL::MKL_DPCPP)
+    target_link_options(${name} PRIVATE -fiopenmp -fopenmp-targets=spir64 -qmkl -lOpenCL)
+    add_test(NAME ${name} COMMAND ${name} ${ARGN})
+  endif()
+endfunction(add_fortran_example_with_mkl_i8)
+
 function(add_example name)
-  set(_sources ${name}.cpp)
-  add_executable(${name} ${_sources})
-  add_sycl_to_target(TARGET ${name} SOURCES ${_sources})
+  cmake_parse_arguments(FUNC_SRC "" "" "SOURCES" ${ARGN})
+  if (FUNC_SRC_SOURCES)
+    set(_src ${FUNC_SRC_SOURCES})
+  else()
+    set(_src ${name}.cpp)
+  endif()
+  add_executable(${name} ${_src})
+  add_sycl_to_target(TARGET ${name} SOURCES ${_src})
   target_compile_options(${name} PRIVATE ${COMMON_CXX_FLAGS})
   target_link_options(${name} PRIVATE -fsycl-device-code-split=per_kernel)
   add_test(NAME ${name} COMMAND ${name} ${ARGN})
 endfunction(add_example)
 
 function(add_openmp_example name)
-  set(_sources ${name}.cpp)
-  add_executable(${name} ${_sources})
+  set(_src ${name}.cpp)
+  add_executable(${name} ${_src})
+  add_sycl_to_target(TARGET ${name} SOURCES ${_src})
   target_compile_options(${name} PRIVATE ${COMMON_CXX_FLAGS})
   add_test(NAME ${name} COMMAND ${name} ${ARGN})
 endfunction(add_openmp_example)
 
 function(add_fortran_example name)
   if(CMAKE_Fortran_COMPILER)
-    set(_sources ${name}.f90)
-    add_executable(${name} ${_sources})
-    add_sycl_to_target(TARGET ${name} SOURCES ${_sources})
+    set(_src ${name}.f90)
+    add_executable(${name} ${_src})
+    add_sycl_to_target(TARGET ${name} SOURCES ${_src})
     target_compile_options(${name} PRIVATE -warn all)
+    target_compile_options(${name} PRIVATE -fiopenmp -fopenmp-targets=spir64)
     set_target_properties(${name} PROPERTIES LINKER_LANGUAGE Fortran)
+    target_link_options(${name} PRIVATE -fiopenmp -fopenmp-targets=spir64)
     add_test(NAME ${name} COMMAND ${name} ${ARGN})
   endif()
 endfunction(add_fortran_example)
 
 function(add_fixed_fortran_example name)
   if(CMAKE_Fortran_COMPILER)
-    set(_sources ${name}.f)
-    add_executable(${name} ${_sources})
+    set(_src ${name}.f)
+    add_executable(${name} ${_src})
+    add_sycl_to_target(TARGET ${name} SOURCES ${_src})
     target_compile_options(${name} PRIVATE -warn all)
+    target_compile_options(${name} PRIVATE -fiopenmp -fopenmp-targets=spir64)
     set_target_properties(${name} PROPERTIES LINKER_LANGUAGE Fortran)
     add_test(NAME ${name} COMMAND ${name} ${ARGN})
   endif()
 endfunction(add_fixed_fortran_example)
 
 function(add_mpi_example name)
   if(MPI_FOUND)
-    set(_sources ${name}.cpp)
-    add_executable(${name} ${_sources})
-    add_sycl_to_target(TARGET ${name} SOURCES ${_sources})
+    set(_src ${name}.cpp)
+    add_executable(${name} ${_src})
+    add_sycl_to_target(TARGET ${name} SOURCES ${_src})
+    target_compile_options(${name} PRIVATE -O3 -fiopenmp -fopenmp-targets=spir64)
+    target_link_options(${name} PRIVATE -O3 -fiopenmp -fopenmp-targets=spir64)
     target_link_libraries(${name} PRIVATE MPI::MPI_CXX)
     add_test(NAME ${name} COMMAND ${name} ${ARGN})
   endif()
 endfunction(add_mpi_example)
 
+function(add_example_with_mkl_mpi name)
+  if(MPI_FOUND)
+    set(_src ${name}.cpp)
+    add_executable(${name} ${_src})
+    add_sycl_to_target(TARGET ${name} SOURCES ${_src})
+    target_compile_options(${name} PRIVATE ${COMMON_CXX_FLAGS})
+    if(NOT MKL_ROOT)
+      set(MKL_ROOT $ENV{MKLROOT} CACHE PATH "Folder contains MKL")
+    endif(NOT MKL_ROOT)
+    target_compile_options(${name} PRIVATE -fiopenmp -fopenmp-targets=spir64 -fsycl -DMKL_LP64  -I"${MKLROOT}/include")
+    target_link_options(${name} PRIVATE  -fiopenmp -fopenmp-targets=spir64 -fsycl  -L${MKLROOT}/lib -lmkl_sycl_blas -lmkl_intel_ilp64 -lmkl_tbb_thread -lmkl_core -lsycl -lpthread -lm -ldl)
+    target_link_libraries(${name} PRIVATE MPI::MPI_CXX)
+    add_test(NAME ${name} COMMAND ${name} ${ARGN})
+  endif()
+endfunction(add_example_with_mkl_mpi)
+
 include_directories(${CMAKE_CURRENT_SOURCE_DIR})
 
 add_subdirectory(atomics)
-add_subdirectory(matrix)
+add_subdirectory(buffer-accessors)
+add_subdirectory(buffers)
+add_subdirectory(composite-explicit-scaling)
+add_subdirectory(composite-implicit-scaling)
+add_subdirectory(conditionals)
 add_subdirectory(exec-model)
-add_subdirectory(explicit-scaling)
+add_subdirectory(flat)
+add_subdirectory(fp-computations)
+add_subdirectory(grf-mode-selection)
+add_subdirectory(host-device-memory)
 add_subdirectory(io-kernel)
 add_subdirectory(jitting)
+add_subdirectory(joint-matrix)
 add_subdirectory(kernels)
-add_subdirectory(memory-movement)
-add_subdirectory(restrict)
-add_subdirectory(slm)
-add_subdirectory(usm)
-add_subdirectory(sub-group)
-add_subdirectory(buffers)
-add_subdirectory(buffer-accessors)
-add_subdirectory(reduction)
-add_subdirectory(conditionals)
+add_subdirectory(libraries-fcorr)
 add_subdirectory(libraries-kernel)
 add_subdirectory(libraries-stdlib)
-add_subdirectory(libraries-fcorr)
-add_subdirectory(multiple-queue-submission)
+add_subdirectory(local-global-sync)
+add_subdirectory(matrix)
+add_subdirectory(memory-movement)
+add_subdirectory(MPI)
 add_subdirectory(multiple-devices)
 add_subdirectory(multiple-kernel-execution)
-add_subdirectory(work-group-size)
-add_subdirectory(registers)
+add_subdirectory(multiple-queue-submission)
+add_subdirectory(onemkl-scaling)
 add_subdirectory(OpenMP)
 add_subdirectory(optimize-data-transfers)
-add_subdirectory(MPI)
-add_subdirectory(grf-mode-selection)
-add_subdirectory(fp-computations)
-add_subdirectory(host-device-memory)
-add_subdirectory(joint-matrix)
-add_subdirectory(local-global-sync)
-#add_subdirectory(memory-sharing-with-media)
+add_subdirectory(overlap-data-transfers)
+add_subdirectory(porting-registers)
+add_subdirectory(prefetch)
+add_subdirectory(reduction)
 add_subdirectory(redundant-queues)
-add_subdirectory(implicit-scaling)
+add_subdirectory(registers)
+add_subdirectory(restrict)
+add_subdirectory(slm)
+add_subdirectory(sub-group)
+add_subdirectory(usm)
+add_subdirectory(work-group-size)

Publications/GPU-Opt-Guide/MPI/01_omp_mpich/CMakeLists.txt

@@ -1,3 +1 @@
 add_mpi_example(omp_mpich)
-target_compile_options(omp_mpich PRIVATE -fiopenmp)
-target_link_options(omp_mpich PRIVATE -fiopenmp)

Publications/GPU-Opt-Guide/MPI/01_omp_mpich/sample.json

@@ -7,7 +7,7 @@
   "description": "oneAPI GPU Optimization Guide Examples",
   "toolchain": [
     "dpcpp",
-    "ifort"
+    "ifx"
   ],
   "languages": [
     {

Publications/GPU-Opt-Guide/MPI/02_omp_mpi_onemkl_dgemm/CMakeLists.txt

@@ -0,0 +1 @@
+add_example_with_mkl_mpi(dgemm 8192 8192 8192)

Publications/GPU-Opt-Guide/MPI/02_omp_mpi_onemkl_dgemm/Makefile

@@ -0,0 +1,17 @@
+#OMP_AFFINITIZATION = 0 to ensure affinitization through MPI environment variables, and = 1 to use OpenMP to affinitize the MPI rank
+OMP_AFFINITIZATION=0
+
+CC=mpicxx
+INCLUDE=-I$(MKLROOT)/include
+LIB="$(MKLROOT)/lib"/libmkl_sycl.a -Wl,--start-group "$(MKLROOT)/lib"/libmkl_intel_lp64.a "$(MKLROOT)/lib"/libmkl_intel_thread.a "$(MKLROOT)/lib"/libmkl_core.a -Wl,--end-group -lsycl -lOpenCL -liomp5 -lpthread -ldl -lm -lstdc++
+CFLAGS=-cxx=icpx -fiopenmp -fopenmp-targets=spir64 -fsycl -DOMP_AFFINITIZATION=$(OMP_AFFINITIZATION)
+CFLAGS2=-cxx=icpx -fsycl-device-code-split=per_kernel -fiopenmp -fopenmp-targets=spir64 -fsycl
+
+dgemm: dgemm.o Makefile
+	$(CC) $(CFLAGS2) dgemm.o $(LIB) -o dgemm
+
+dgemm.o: dgemm.cpp Makefile
+	$(CC) $(CFLAGS) $(INCLUDE) -c dgemm.cpp -o dgemm.o
+
+clean:
+	rm -rf ./dgemm ./dgemm.o