[SYCL] Support standard library functions for AOT compilation (#2012)

Fallback libraries are now compiled into object files, so they can be
linked at compile time.

Signed-off-by: Andrew Savonichev <andrew.savonichev@intel.com>

Author: Andrew Savonichev

libdevice/cmake/modules/SYCLLibdevice.cmake

@@ -6,6 +6,13 @@ endif()
 
 set(clang $<TARGET_FILE:clang>)
 
+string(CONCAT sycl_targets_opt
+  "-fsycl-targets="
+  "spir64_x86_64-unknown-unknown-sycldevice,"
+  "spir64_gen-unknown-unknown-sycldevice,"
+  "spir64_fpga-unknown-unknown-sycldevice,"
+  "spir64-unknown-unknown-sycldevice")
+
 set(compile_opts
   # suppress an error about SYCL_EXTERNAL being used for
   # a function with a raw pointer parameter.
@@ -22,62 +29,62 @@ if (WIN32)
   set(devicelib-obj-file ${binary_dir}/libsycl-msvc.o)
   add_custom_command(OUTPUT ${devicelib-obj-file}
                      COMMAND ${clang} -fsycl -c
-                             ${compile_opts}
+                             ${compile_opts} ${sycl_targets_opt}
                              ${CMAKE_CURRENT_SOURCE_DIR}/msvc_wrapper.cpp
                              -o ${devicelib-obj-file}
                      MAIN_DEPENDENCY msvc_wrapper.cpp
-                     DEPENDS wrapper.h device.h spirv_vars.h clang
+                     DEPENDS wrapper.h device.h spirv_vars.h clang clang-offload-bundler
                      VERBATIM)
 else()
   set(devicelib-obj-file ${binary_dir}/libsycl-glibc.o)
   add_custom_command(OUTPUT ${devicelib-obj-file}
                      COMMAND ${clang} -fsycl -c
-                             ${compile_opts}
+                             ${compile_opts} ${sycl_targets_opt}
                              ${CMAKE_CURRENT_SOURCE_DIR}/glibc_wrapper.cpp
                              -o ${devicelib-obj-file}
                      MAIN_DEPENDENCY glibc_wrapper.cpp
-                     DEPENDS wrapper.h device.h spirv_vars.h clang
+                     DEPENDS wrapper.h device.h spirv_vars.h clang clang-offload-bundler
                      VERBATIM)
 endif()
 
 set(devicelib-obj-complex ${binary_dir}/libsycl-complex.o)
 add_custom_command(OUTPUT ${devicelib-obj-complex}
                    COMMAND ${clang} -fsycl -c
-                           ${compile_opts}
+                           ${compile_opts} ${sycl_targets_opt}
                            ${CMAKE_CURRENT_SOURCE_DIR}/complex_wrapper.cpp
                            -o ${devicelib-obj-complex}
                    MAIN_DEPENDENCY complex_wrapper.cpp
-                   DEPENDS device_complex.h device.h clang
+                   DEPENDS device_complex.h device.h clang clang-offload-bundler
                    VERBATIM)
 
 set(devicelib-obj-complex-fp64 ${binary_dir}/libsycl-complex-fp64.o)
 add_custom_command(OUTPUT ${devicelib-obj-complex-fp64}
                    COMMAND ${clang} -fsycl -c
-                           ${compile_opts}
+                           ${compile_opts} ${sycl_targets_opt}
                            ${CMAKE_CURRENT_SOURCE_DIR}/complex_wrapper_fp64.cpp
                            -o ${devicelib-obj-complex-fp64}
                    MAIN_DEPENDENCY complex_wrapper_fp64.cpp
-                   DEPENDS device_complex.h device.h clang
+                   DEPENDS device_complex.h device.h clang clang-offload-bundler
                    VERBATIM)
 
 set(devicelib-obj-cmath ${binary_dir}/libsycl-cmath.o)
 add_custom_command(OUTPUT ${devicelib-obj-cmath}
                    COMMAND ${clang} -fsycl -c
-                           ${compile_opts}
+                           ${compile_opts} ${sycl_targets_opt}
                            ${CMAKE_CURRENT_SOURCE_DIR}/cmath_wrapper.cpp
                            -o ${devicelib-obj-cmath}
                    MAIN_DEPENDENCY cmath_wrapper.cpp
-                   DEPENDS device_math.h device.h clang
+                   DEPENDS device_math.h device.h clang clang-offload-bundler
                    VERBATIM)
 
 set(devicelib-obj-cmath-fp64 ${binary_dir}/libsycl-cmath-fp64.o)
 add_custom_command(OUTPUT ${devicelib-obj-cmath-fp64}
                    COMMAND ${clang} -fsycl -c
-                           ${compile_opts}
+                           ${compile_opts} ${sycl_targets_opt}
                            ${CMAKE_CURRENT_SOURCE_DIR}/cmath_wrapper_fp64.cpp
                            -o ${devicelib-obj-cmath-fp64}
                    MAIN_DEPENDENCY cmath_wrapper_fp64.cpp
-                   DEPENDS device_math.h device.h clang
+                   DEPENDS device_math.h device.h clang clang-offload-bundler
                    VERBATIM)
 
 add_custom_command(OUTPUT ${binary_dir}/libsycl-fallback-cassert.spv
@@ -89,6 +96,15 @@ add_custom_command(OUTPUT ${binary_dir}/libsycl-fallback-cassert.spv
                    DEPENDS wrapper.h device.h clang spirv_vars.h llvm-spirv
                    VERBATIM)
 
+add_custom_command(OUTPUT ${binary_dir}/libsycl-fallback-cassert.o
+                   COMMAND ${clang} -fsycl -c
+                           ${compile_opts} ${sycl_targets_opt}
+                           ${CMAKE_CURRENT_SOURCE_DIR}/fallback-cassert.cpp
+                           -o ${binary_dir}/libsycl-fallback-cassert.o
+                   MAIN_DEPENDENCY fallback-cassert.cpp
+                   DEPENDS wrapper.h device.h clang spirv_vars.h clang-offload-bundler
+                   VERBATIM)
+
 add_custom_command(OUTPUT ${binary_dir}/libsycl-fallback-complex.spv
                    COMMAND ${clang} -S -fsycl-device-only -fno-sycl-use-bitcode
                            ${compile_opts}
@@ -98,6 +114,15 @@ add_custom_command(OUTPUT ${binary_dir}/libsycl-fallback-complex.spv
                    DEPENDS device_math.h device_complex.h device.h clang llvm-spirv
                    VERBATIM)
 
+add_custom_command(OUTPUT ${binary_dir}/libsycl-fallback-complex.o
+                   COMMAND ${clang} -fsycl -c
+                           ${compile_opts} ${sycl_targets_opt}
+                           ${CMAKE_CURRENT_SOURCE_DIR}/fallback-complex.cpp
+                           -o ${binary_dir}/libsycl-fallback-complex.o
+                   MAIN_DEPENDENCY fallback-complex.cpp
+                   DEPENDS device_math.h device_complex.h device.h clang clang-offload-bundler
+                   VERBATIM)
+
 add_custom_command(OUTPUT ${binary_dir}/libsycl-fallback-complex-fp64.spv
                    COMMAND ${clang} -S -fsycl-device-only -fno-sycl-use-bitcode
                            ${compile_opts}
@@ -107,6 +132,15 @@ add_custom_command(OUTPUT ${binary_dir}/libsycl-fallback-complex-fp64.spv
                    DEPENDS device_math.h device_complex.h device.h clang llvm-spirv
                    VERBATIM)
 
+add_custom_command(OUTPUT ${binary_dir}/libsycl-fallback-complex-fp64.o
+                   COMMAND ${clang} -fsycl -c
+                           ${compile_opts} ${sycl_targets_opt}
+                           ${CMAKE_CURRENT_SOURCE_DIR}/fallback-complex-fp64.cpp
+                           -o ${binary_dir}/libsycl-fallback-complex-fp64.o
+                   MAIN_DEPENDENCY fallback-complex-fp64.cpp
+                   DEPENDS device_math.h device_complex.h device.h clang clang-offload-bundler
+                   VERBATIM)
+
 add_custom_command(OUTPUT ${binary_dir}/libsycl-fallback-cmath.spv
                    COMMAND ${clang} -S -fsycl-device-only -fno-sycl-use-bitcode
                            ${compile_opts}
@@ -116,6 +150,15 @@ add_custom_command(OUTPUT ${binary_dir}/libsycl-fallback-cmath.spv
                    DEPENDS device_math.h device.h clang llvm-spirv
                    VERBATIM)
 
+add_custom_command(OUTPUT ${binary_dir}/libsycl-fallback-cmath.o
+                   COMMAND ${clang} -fsycl -c
+                           ${compile_opts} ${sycl_targets_opt}
+                           ${CMAKE_CURRENT_SOURCE_DIR}/fallback-cmath.cpp
+                           -o ${binary_dir}/libsycl-fallback-cmath.o
+                   MAIN_DEPENDENCY fallback-cmath.cpp
+                   DEPENDS device_math.h device.h clang clang-offload-bundler
+                   VERBATIM)
+
 add_custom_command(OUTPUT ${binary_dir}/libsycl-fallback-cmath-fp64.spv
                    COMMAND ${clang} -S -fsycl-device-only -fno-sycl-use-bitcode
                            ${compile_opts}
@@ -125,6 +168,15 @@ add_custom_command(OUTPUT ${binary_dir}/libsycl-fallback-cmath-fp64.spv
                    DEPENDS device_math.h device.h clang llvm-spirv
                    VERBATIM)
 
+add_custom_command(OUTPUT ${binary_dir}/libsycl-fallback-cmath-fp64.o
+                   COMMAND ${clang} -fsycl -c
+                           ${compile_opts} ${sycl_targets_opt}
+                           ${CMAKE_CURRENT_SOURCE_DIR}/fallback-cmath-fp64.cpp
+                           -o ${binary_dir}/libsycl-fallback-cmath-fp64.o
+                   MAIN_DEPENDENCY fallback-cmath-fp64.cpp
+                   DEPENDS device_math.h device.h clang clang-offload-bundler
+                   VERBATIM)
+
 add_custom_target(libsycldevice-obj DEPENDS
   ${devicelib-obj-file}
   ${devicelib-obj-complex}
@@ -138,8 +190,18 @@ add_custom_target(libsycldevice-spv DEPENDS
   ${binary_dir}/libsycl-fallback-complex-fp64.spv
   ${binary_dir}/libsycl-fallback-cmath.spv
   ${binary_dir}/libsycl-fallback-cmath-fp64.spv
+  )
+add_custom_target(libsycldevice-fallback-obj DEPENDS
+  ${binary_dir}/libsycl-fallback-cassert.o
+  ${binary_dir}/libsycl-fallback-complex.o
+  ${binary_dir}/libsycl-fallback-complex-fp64.o
+  ${binary_dir}/libsycl-fallback-cmath.o
+  ${binary_dir}/libsycl-fallback-cmath-fp64.o
 )
-add_custom_target(libsycldevice DEPENDS libsycldevice-obj libsycldevice-spv)
+add_custom_target(libsycldevice DEPENDS
+  libsycldevice-obj
+  libsycldevice-fallback-obj
+  libsycldevice-spv)
 
 # Place device libraries near the libsycl.so library in an install
 # directory as well
@@ -151,13 +213,18 @@ endif()
 
 install(FILES ${devicelib-obj-file}
               ${binary_dir}/libsycl-fallback-cassert.spv
+              ${binary_dir}/libsycl-fallback-cassert.o
               ${devicelib-obj-complex}
               ${binary_dir}/libsycl-fallback-complex.spv
+	      ${binary_dir}/libsycl-fallback-complex.o
               ${devicelib-obj-complex-fp64}
               ${binary_dir}/libsycl-fallback-complex-fp64.spv
+	      ${binary_dir}/libsycl-fallback-complex-fp64.o
               ${devicelib-obj-cmath}
               ${binary_dir}/libsycl-fallback-cmath.spv
+              ${binary_dir}/libsycl-fallback-cmath.o
               ${devicelib-obj-cmath-fp64}
               ${binary_dir}/libsycl-fallback-cmath-fp64.spv
+	      ${binary_dir}/libsycl-fallback-cmath-fp64.o
         DESTINATION ${install_dest}
         COMPONENT libsycldevice)

sycl/doc/extensions/C-CXX-StandardLibrary/C-CXX-StandardLibrary.rst

@@ -22,6 +22,16 @@ or, in case of Windows:
    clang++ -fsycl -c main.cpp -o main.obj
    clang++ -fsycl main.obj %SYCL_INSTALL%/lib/libsycl-msvc.o -o a.exe
 
+For Ahead-Of-Time compilation (AOT), fallback libraries (object files)
+must be linked as well:
+
+.. code:
+   clang++ -fsycl -c  -fsycl-targets=spir64_x86_64-unknown-unknown-sycldevice \
+       main.cpp -o main.o
+   clang++ -fsycl -fsycl-targets=spir64_x86_64-unknown-unknown-sycldevice \
+       main.o $(SYCL_INSTALL)/lib/libsycl-glibc.o                         \
+              $(SYCL_INSTALL)/lib/libsycl-fallback-cassert.o -o a.out
+
 List of supported functions from C standard library:
   - assert macro          (from <assert.h> or <cassert>)
   - logf, log             (from <math.h> or <cmath>)
@@ -270,6 +280,7 @@ Every set of functions is implemented in a separate fallback
 library. For example, a fallback for `cl_intel_devicelib_cassert`
 extension is provided as `libsycl-fallback-cassert.spv`
 
-NOTE that AOT compilation is not yet supported. Driver will have to
-check for extension support and link the corresponding SPIR-V fallback
-implementation, but this is not implemented yet.
+For AOT compilation, fallback libraries are provided as object files
+(e.g. `libsycl-fallback-cassert.o`) which contain device code in LLVM
+IR format. Device code in these object files is equivalent to device
+code in the `*.spv` files.

sycl/test/devicelib/assert-aot.cpp

@@ -0,0 +1,5 @@
+// REQUIRES: opencl-aot, cpu, linux
+
+// RUN: %clangxx -fsycl -fsycl-targets=spir64_x86_64-unknown-unknown-sycldevice %S/assert.cpp %sycl_libs_dir/libsycl-glibc.o %sycl_libs_dir/libsycl-fallback-cassert.o -o %t.aot.out
+// RUN: %CPU_RUN_PLACEHOLDER %t.aot.out >%t.aot.msg
+// RUN: FileCheck %S/assert.cpp --input-file %t.aot.msg --check-prefixes=CHECK-MESSAGE

sycl/test/devicelib/cmath-aot.cpp

@@ -0,0 +1,14 @@
+// REQUIRES: opencl-aot, cpu
+// UNSUPPORTED: windows
+
+// RUN: %clangxx -fsycl -fsycl-targets=spir64_x86_64-unknown-unknown-sycldevice %S/cmath_test.cpp %sycl_libs_dir/libsycl-cmath.o %sycl_libs_dir/libsycl-fallback-cmath.o -o %t.cmath.out
+// RUN: %CPU_RUN_PLACEHOLDER %t.cmath.out
+
+// RUN: %clangxx -fsycl -fsycl-targets=spir64_x86_64-unknown-unknown-sycldevice %S/cmath_fp64_test.cpp %sycl_libs_dir/libsycl-cmath-fp64.o %sycl_libs_dir/libsycl-fallback-cmath-fp64.o -o %t.cmath.fp64.out
+// RUN: %CPU_RUN_PLACEHOLDER %t.cmath.fp64.out
+
+// RUN: %clangxx -fsycl -fsycl-targets=spir64_x86_64-unknown-unknown-sycldevice %S/std_complex_math_test.cpp %sycl_libs_dir/libsycl-complex.o %sycl_libs_dir/libsycl-cmath.o %sycl_libs_dir/libsycl-fallback-complex.o %sycl_libs_dir/libsycl-fallback-cmath.o -o %t.complex.out
+// RUN: %CPU_RUN_PLACEHOLDER %t.complex.out
+
+// RUN: %clangxx -fsycl -fsycl-targets=spir64_x86_64-unknown-unknown-sycldevice %S/std_complex_math_fp64_test.cpp %sycl_libs_dir/libsycl-complex-fp64.o %sycl_libs_dir/libsycl-cmath-fp64.o %sycl_libs_dir/libsycl-fallback-complex-fp64.o %sycl_libs_dir/libsycl-fallback-cmath-fp64.o -o %t.complex.fp64.out
+// RUN: %CPU_RUN_PLACEHOLDER %t.complex.fp64.out