//------------------------------------------------------------------------------ // GB_jitifyer.c: CPU / CUDA jitifyer //------------------------------------------------------------------------------ // SuiteSparse:GraphBLAS, Timothy A. Davis, (c) 2017-2023, All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 //------------------------------------------------------------------------------ #include "GB.h" #include "GB_stringify.h" #include "GB_config.h" #include "GB_zstd.h" #include "GB_JITpackage.h" #include "GB_file.h" typedef GB_JIT_KERNEL_USER_OP_PROTO ((*GB_user_op_f)) ; typedef GB_JIT_KERNEL_USER_TYPE_PROTO ((*GB_user_type_f)) ; //------------------------------------------------------------------------------ // static objects: hash table, strings, and status //------------------------------------------------------------------------------ // The hash table is static and shared by all threads of the user application. // It is only visible inside this file. It starts out empty (NULL). Its size // is either zero (at the beginning), or a power of two (of size // GB_JITIFIER_INITIAL_SIZE or more). // The strings are used to create filenames and JIT compilation commands. #define GB_JITIFIER_INITIAL_SIZE (32*1024) static GB_jit_entry *GB_jit_table = NULL ; static int64_t GB_jit_table_size = 0 ; // always a power of 2 static uint64_t GB_jit_table_bits = 0 ; // hash mask (0xFFFF if size is 2^16) static int64_t GB_jit_table_populated = 0 ; static size_t GB_jit_table_allocated = 0 ; static bool GB_jit_use_cmake = #if defined (_MSC_VER) true ; // MSVC requires cmake #else false ; // otherwise, default is to skip cmake and compile directly #endif // path to user cache folder: static char *GB_jit_cache_path = NULL ; static size_t GB_jit_cache_path_allocated = 0 ; // path to error log file: static char *GB_jit_error_log = NULL ; static size_t GB_jit_error_log_allocated = 0 ; // name of the C compiler: static char *GB_jit_C_compiler = NULL ; static size_t GB_jit_C_compiler_allocated = 0 ; // flags for the C compiler: static char *GB_jit_C_flags = NULL ; static size_t GB_jit_C_flags_allocated = 0 ; // link flags for the C compiler: static char *GB_jit_C_link_flags = NULL ; static size_t GB_jit_C_link_flags_allocated = 0 ; // libraries to link against when using the direct compile/link: static char *GB_jit_C_libraries = NULL ; static size_t GB_jit_C_libraries_allocated = 0 ; // libraries to link against when using cmake: static char *GB_jit_C_cmake_libs = NULL ; static size_t GB_jit_C_cmake_libs_allocated = 0 ; // preface to add to each CPU JIT kernel: static char *GB_jit_C_preface = NULL ; static size_t GB_jit_C_preface_allocated = 0 ; // preface to add to each CUDA JIT kernel: static char *GB_jit_CUDA_preface = NULL ; static size_t GB_jit_CUDA_preface_allocated = 0 ; // temporary workspace for filenames and system commands: static char *GB_jit_temp = NULL ; static size_t GB_jit_temp_allocated = 0 ; // compile with -DJITINIT=4 (for example) to set the initial JIT C control #ifdef JITINIT #define GB_JIT_C_CONTROL_INIT JITINIT #else // default initial state #define GB_JIT_C_CONTROL_INIT GxB_JIT_ON #endif static GxB_JIT_Control GB_jit_control = GB_JIT_C_CONTROL_INIT ; //------------------------------------------------------------------------------ // check_table: check if the hash table is OK //------------------------------------------------------------------------------ #ifdef GB_DEBUG static void check_table (void) { int64_t populated = 0 ; if (GB_jit_table != NULL) { for (uint64_t k = 0 ; k < GB_jit_table_size ; k++) { GB_jit_entry *e = &(GB_jit_table [k]) ; if (e->dl_function != NULL) { uint64_t hash = e->hash ; uint64_t k2 = (hash & GB_jit_table_bits) ; populated++ ; } } } ASSERT (populated == GB_jit_table_populated) ; } #define ASSERT_TABLE_OK check_table ( ) ; #else #define ASSERT_TABLE_OK #endif //------------------------------------------------------------------------------ // malloc/free macros //------------------------------------------------------------------------------ // The JIT must use persistent malloc/free methods when GraphBLAS is used in // MATLAB. Outside of MATLAB, these are the same as malloc/free passed to // GxB_init (or ANSI C malloc/free if using GrB_init). Inside MATLAB, // GB_Global_persistent_malloc uses the same malloc/free given to GxB_init, but // then calls mexMakeMemoryPersistent to ensure the memory is not freed when a // mexFunction returns to the MATLAB m-file caller. #define OK(method) \ { \ GrB_Info myinfo = (method) ; \ if (myinfo != GrB_SUCCESS) \ { \ return (myinfo) ; \ } \ } #ifdef GB_MEMDUMP #define GB_MALLOC_PERSISTENT(X,siz) \ { \ X = GB_Global_persistent_malloc (siz) ; \ printf ("persistent malloc (%4d): %p size %lu\n", /* MEMDUMP */ \ __LINE__, X, siz) ; \ } #define GB_FREE_PERSISTENT(X) \ { \ if (X != NULL) \ { \ printf ("persistent free (%4d): %p\n", /* MEMDUMP */ \ __LINE__, X) ; \ } \ GB_Global_persistent_free ((void **) &(X)) ; \ } #else #define GB_MALLOC_PERSISTENT(X,siz) \ { \ X = GB_Global_persistent_malloc (siz) ; \ } #define GB_FREE_PERSISTENT(X) \ { \ GB_Global_persistent_free ((void **) &X) ; \ } #endif #define GB_FREE_STUFF(X) \ { \ GB_FREE_PERSISTENT (X) ; \ X ## _allocated = 0 ; \ } #define GB_MALLOC_STUFF(X,len) \ { \ GB_MALLOC_PERSISTENT (X, (len) + 2) ; \ if (X == NULL) \ { \ return (GrB_OUT_OF_MEMORY) ; \ } \ X ## _allocated = (len) + 2 ; \ } #define GB_COPY_STUFF(X,src) \ { \ size_t len = strlen (src) ; \ GB_MALLOC_STUFF (X, len) ; \ strncpy (X, src, X ## _allocated) ; \ } //------------------------------------------------------------------------------ // GB_jitifyer_finalize: free the JIT table and all the strings //------------------------------------------------------------------------------ void GB_jitifyer_finalize (void) { GB_jitifyer_table_free (true) ; GB_FREE_STUFF (GB_jit_cache_path) ; GB_FREE_STUFF (GB_jit_error_log) ; GB_FREE_STUFF (GB_jit_C_compiler) ; GB_FREE_STUFF (GB_jit_C_flags) ; GB_FREE_STUFF (GB_jit_C_link_flags) ; GB_FREE_STUFF (GB_jit_C_libraries) ; GB_FREE_STUFF (GB_jit_C_cmake_libs) ; GB_FREE_STUFF (GB_jit_C_preface) ; GB_FREE_STUFF (GB_jit_CUDA_preface) ; GB_FREE_STUFF (GB_jit_temp) ; } //------------------------------------------------------------------------------ // GB_jitifyer_init: initialize the JIT folders, flags, etc //------------------------------------------------------------------------------ // Returns GrB_SUCCESS or GrB_OUT_OF_MEMORY. If any other error occurs (such // as being unable to access the cache folder), the JIT is disabled, but // GrB_SUCCESS is returned. This is because GrB_init calls this method, and // GraphBLAS can continue without the JIT. GrB_Info GB_jitifyer_init (void) { //-------------------------------------------------------------------------- // initialize the JIT control //-------------------------------------------------------------------------- int control = (int) GB_JIT_C_CONTROL_INIT ; control = GB_IMAX (control, (int) GxB_JIT_OFF) ; #ifndef NJIT // The full JIT is available. control = GB_IMIN (control, (int) GxB_JIT_ON) ; #else // The JIT is restricted; only OFF, PAUSE, and RUN settings can be // used. No JIT kernels can be loaded or compiled. Only PreJIT kernels // can be used. control = GB_IMIN (control, (int) GxB_JIT_RUN) ; #endif GB_jit_control = (GxB_JIT_Control) control ; GB_jitifyer_finalize ( ) ; //-------------------------------------------------------------------------- // find the GB_jit_cache_path //-------------------------------------------------------------------------- char *cache_path = getenv ("GRAPHBLAS_CACHE_PATH") ; if (cache_path != NULL) { // use the environment variable GRAPHBLAS_CACHE_PATH as-is GB_COPY_STUFF (GB_jit_cache_path, cache_path) ; } else { // Linux, Mac, Unix: look for HOME char *home = getenv ("HOME") ; char *dot = "." ; if (home == NULL) { // Windows: look for LOCALAPPDATA home = getenv ("LOCALAPPDATA") ; dot = "" ; } if (home != NULL) { // found home; create the cache path size_t len = strlen (home) + 60 ; GB_MALLOC_STUFF (GB_jit_cache_path, len) ; snprintf (GB_jit_cache_path, GB_jit_cache_path_allocated, "%s/%sSuiteSparse/GrB%d.%d.%d" #if defined ( GBMATLAB ) && defined ( __APPLE__ ) "_matlab" #endif , home, dot, GxB_IMPLEMENTATION_MAJOR, GxB_IMPLEMENTATION_MINOR, GxB_IMPLEMENTATION_SUB) ; } } if (GB_jit_cache_path == NULL) { // cannot determine the JIT cache. Disable loading and compiling, but // continue with the rest of the initializations. The PreJIT could // still be used. GBURBLE ("(jit init: unable to access cache path, jit disabled) ") ; GB_jit_control = GxB_JIT_RUN ; GB_FREE_STUFF (GB_jit_cache_path) ; GB_COPY_STUFF (GB_jit_cache_path, "") ; } // replace backslash with forward slash for (char *p = GB_jit_cache_path ; *p != '\0' ; p++) { if ((*p) == '\\') (*p) = '/' ; } //-------------------------------------------------------------------------- // initialize the remaining strings //-------------------------------------------------------------------------- GB_COPY_STUFF (GB_jit_error_log, "") ; GB_COPY_STUFF (GB_jit_C_compiler, GB_C_COMPILER) ; GB_COPY_STUFF (GB_jit_C_flags, GB_C_FLAGS) ; GB_COPY_STUFF (GB_jit_C_link_flags, GB_C_LINK_FLAGS) ; GB_COPY_STUFF (GB_jit_C_libraries, GB_C_LIBRARIES) ; GB_COPY_STUFF (GB_jit_C_cmake_libs, GB_CMAKE_LIBRARIES) ; GB_COPY_STUFF (GB_jit_C_preface, "") ; GB_COPY_STUFF (GB_jit_CUDA_preface, "") ; OK (GB_jitifyer_alloc_space ( )) ; //-------------------------------------------------------------------------- // establish the cache path and src path, and make sure they exist //-------------------------------------------------------------------------- OK (GB_jitifyer_establish_paths (GrB_SUCCESS)) ; //-------------------------------------------------------------------------- // remove "-arch arm64" if compiling JIT kernels for MATLAB //-------------------------------------------------------------------------- // When the x86-based version of gcc-12 is configured to compile the MATLAB // GraphBLAS library on an Apple-Silicon-based Mac, cmake gives it the flag // "-arch arm64". MATLAB does not support that architecture directly, // using Rosetta 2 instead. gcc-12 also does not support "-arch arm64", so // it ignores it (which is the right thing to do), but it generates a // warning. This spurious warning message appears every time a JIT kernel // is compiled while inside MATLAB. As a result, "-arch arm64" is removed // from the initial C flags, if compiling for MATLAB. #ifdef GBMATLAB { #define ARCH_ARM64 "-arch arm64" char *dst = strstr (GB_jit_C_flags, ARCH_ARM64) ; if (dst != NULL) { // found it; now remove it from the C flags char *src = dst + strlen (ARCH_ARM64) ; while (((*dst++) = (*src++))) ; } } #endif //-------------------------------------------------------------------------- // hash all PreJIT kernels //-------------------------------------------------------------------------- void **Kernels = NULL ; void **Queries = NULL ; char **Names = NULL ; int32_t nkernels = 0 ; GB_prejit (&nkernels, &Kernels, &Queries, &Names) ; for (int k = 0 ; k < nkernels ; k++) { //---------------------------------------------------------------------- // get the name and function pointer of the PreJIT kernel //---------------------------------------------------------------------- void *dl_function = Kernels [k] ; GB_jit_query_func dl_query = (GB_jit_query_func) Queries [k] ; ASSERT (dl_function != NULL && dl_query != NULL && Names [k] != NULL) ; char kernel_name [GB_KLEN+1] ; strncpy (kernel_name, Names [k], GB_KLEN) ; kernel_name [GB_KLEN] = '\0' ; //---------------------------------------------------------------------- // parse the kernel name //---------------------------------------------------------------------- char *name_space = NULL ; char *kname = NULL ; uint64_t scode = 0 ; char *suffix = NULL ; GrB_Info info = GB_demacrofy_name (kernel_name, &name_space, &kname, &scode, &suffix) ; if (info != GrB_SUCCESS || !GB_STRING_MATCH (name_space, "GB_jit")) { // PreJIT error: kernel_name is invalid; ignore this kernel continue ; } //---------------------------------------------------------------------- // find the kcode of the kname //---------------------------------------------------------------------- GB_jit_encoding encoding_struct ; GB_jit_encoding *encoding = &encoding_struct ; memset (encoding, 0, sizeof (GB_jit_encoding)) ; #define IS(kernel) GB_STRING_MATCH (kname, kernel) GB_jit_kcode c = 0 ; if (IS ("add" )) c = GB_JIT_KERNEL_ADD ; else if (IS ("apply_bind1st")) c = GB_JIT_KERNEL_APPLYBIND1 ; else if (IS ("apply_bind2nd")) c = GB_JIT_KERNEL_APPLYBIND2 ; else if (IS ("apply_unop" )) c = GB_JIT_KERNEL_APPLYUNOP ; else if (IS ("AxB_dot2" )) c = GB_JIT_KERNEL_AXB_DOT2 ; else if (IS ("AxB_dot2n" )) c = GB_JIT_KERNEL_AXB_DOT2N ; else if (IS ("AxB_dot3" )) c = GB_JIT_KERNEL_AXB_DOT3 ; else if (IS ("AxB_dot4" )) c = GB_JIT_KERNEL_AXB_DOT4 ; else if (IS ("AxB_saxbit" )) c = GB_JIT_KERNEL_AXB_SAXBIT ; else if (IS ("AxB_saxpy3" )) c = GB_JIT_KERNEL_AXB_SAXPY3 ; else if (IS ("AxB_saxpy4" )) c = GB_JIT_KERNEL_AXB_SAXPY4 ; else if (IS ("AxB_saxpy5" )) c = GB_JIT_KERNEL_AXB_SAXPY5 ; else if (IS ("build" )) c = GB_JIT_KERNEL_BUILD ; else if (IS ("colscale" )) c = GB_JIT_KERNEL_COLSCALE ; else if (IS ("concat_bitmap")) c = GB_JIT_KERNEL_CONCAT_BITMAP ; else if (IS ("concat_full" )) c = GB_JIT_KERNEL_CONCAT_FULL ; else if (IS ("concat_sparse")) c = GB_JIT_KERNEL_CONCAT_SPARSE ; else if (IS ("convert_s2b" )) c = GB_JIT_KERNEL_CONVERTS2B ; else if (IS ("emult_02" )) c = GB_JIT_KERNEL_EMULT2 ; else if (IS ("emult_03" )) c = GB_JIT_KERNEL_EMULT3 ; else if (IS ("emult_04" )) c = GB_JIT_KERNEL_EMULT4 ; else if (IS ("emult_08" )) c = GB_JIT_KERNEL_EMULT8 ; else if (IS ("emult_bitmap" )) c = GB_JIT_KERNEL_EMULT_BITMAP ; else if (IS ("ewise_fulla" )) c = GB_JIT_KERNEL_EWISEFA ; else if (IS ("ewise_fulln" )) c = GB_JIT_KERNEL_EWISEFN ; else if (IS ("reduce" )) c = GB_JIT_KERNEL_REDUCE ; else if (IS ("rowscale" )) c = GB_JIT_KERNEL_ROWSCALE ; else if (IS ("select_bitmap")) c = GB_JIT_KERNEL_SELECT_BITMAP ; else if (IS ("select_phase1")) c = GB_JIT_KERNEL_SELECT1 ; else if (IS ("select_phase2")) c = GB_JIT_KERNEL_SELECT2 ; else if (IS ("split_bitmap" )) c = GB_JIT_KERNEL_SPLIT_BITMAP ; else if (IS ("split_full" )) c = GB_JIT_KERNEL_SPLIT_FULL ; else if (IS ("split_sparse" )) c = GB_JIT_KERNEL_SPLIT_SPARSE ; else if (IS ("subassign_05d")) c = GB_JIT_KERNEL_SUBASSIGN_05d ; else if (IS ("subassign_06d")) c = GB_JIT_KERNEL_SUBASSIGN_06d ; else if (IS ("subassign_22" )) c = GB_JIT_KERNEL_SUBASSIGN_22 ; else if (IS ("subassign_23" )) c = GB_JIT_KERNEL_SUBASSIGN_23 ; else if (IS ("subassign_25" )) c = GB_JIT_KERNEL_SUBASSIGN_25 ; else if (IS ("trans_bind1st")) c = GB_JIT_KERNEL_TRANSBIND1 ; else if (IS ("trans_bind2nd")) c = GB_JIT_KERNEL_TRANSBIND2 ; else if (IS ("trans_unop" )) c = GB_JIT_KERNEL_TRANSUNOP ; else if (IS ("union" )) c = GB_JIT_KERNEL_UNION ; else if (IS ("user_op" )) c = GB_JIT_KERNEL_USEROP ; else if (IS ("user_type" )) c = GB_JIT_KERNEL_USERTYPE ; else if (IS ("cuda_reduce" )) c = GB_JIT_CUDA_KERNEL_REDUCE ; else { // PreJIT error: kernel_name is invalid; ignore this kernel continue ; } #undef IS encoding->kcode = c ; encoding->code = scode ; encoding->suffix_len = (suffix == NULL) ? 0 : ((int32_t) strlen (suffix)) ; //---------------------------------------------------------------------- // get the hash of this PreJIT kernel //---------------------------------------------------------------------- // Query the kernel for its hash and version number. The hash is // needed now so the PreJIT kernel can be added to the hash table. // The type/op definitions and monoid id/term values for user-defined // types/ops/ monoids are ignored, because the user-defined objects // have not yet been created during this use of GraphBLAS (this method // is called by GrB_init). These definitions are checked the first // time the kernel is run. uint64_t hash = 0 ; char *ignored [5] ; int version [3] ; (void) dl_query (&hash, version, ignored, NULL, NULL, 0, 0) ; if (hash == 0 || hash == UINT64_MAX || (version [0] != GxB_IMPLEMENTATION_MAJOR) || (version [1] != GxB_IMPLEMENTATION_MINOR) || (version [2] != GxB_IMPLEMENTATION_SUB)) { // PreJIT error: the kernel is stale; ignore it continue ; } //---------------------------------------------------------------------- // make sure this kernel is not a duplicate //---------------------------------------------------------------------- int64_t k1 = -1, kk = -1 ; if (GB_jitifyer_lookup (hash, encoding, suffix, &k1, &kk) != NULL) { // PreJIT error: the kernel is a duplicate; ignore it continue ; } //---------------------------------------------------------------------- // insert the PreJIT kernel in the hash table //---------------------------------------------------------------------- if (!GB_jitifyer_insert (hash, encoding, suffix, NULL, dl_function, k)) { // PreJIT error: out of memory GB_jit_control = GxB_JIT_PAUSE ; return (GrB_OUT_OF_MEMORY) ; } } //-------------------------------------------------------------------------- // uncompress all the source files into the user source folder //-------------------------------------------------------------------------- return (GB_jitifyer_extract_JITpackage (GrB_SUCCESS)) ; } //------------------------------------------------------------------------------ // GB_jitifyer_path_256: establish a folder and its 256 subfolders //------------------------------------------------------------------------------ bool GB_jitifyer_path_256 (char *folder) { snprintf (GB_jit_temp, GB_jit_temp_allocated, "%s/%s", GB_jit_cache_path, folder) ; bool ok = GB_file_mkdir (GB_jit_temp) ; for (uint32_t bucket = 0 ; bucket <= 0xFF ; bucket++) { snprintf (GB_jit_temp, GB_jit_temp_allocated, "%s/%s/%02x", GB_jit_cache_path, folder, bucket) ; ok = ok && GB_file_mkdir (GB_jit_temp) ; } return (ok) ; } //------------------------------------------------------------------------------ // GB_jitifyer_establish_paths: make sure cache and its folders exist //------------------------------------------------------------------------------ // Returns GrB_SUCCESS if succesful, or GrB_OUT_OF_MEMORY if out of memory. If // the paths cannot be established, the JIT is disabled, and the // error_condition is returned. GrB_init uses this to return GrB_SUCCESS, // since GraphBLAS can continue without the JIT. GxB_set returns // GrB_INVALID_VALUE to indicate that the cache path is not valid. // If the JIT is disabled at compile time, the directories are not created and // GrB_SUCCESS is returned (except if an out of memory condition occurs). GrB_Info GB_jitifyer_establish_paths (GrB_Info error_condition) { //-------------------------------------------------------------------------- // construct the src and lock folders //-------------------------------------------------------------------------- bool ok = GB_file_mkdir (GB_jit_cache_path) ; // construct the c, lib, and lock paths and their 256 subfolders ok = ok && GB_jitifyer_path_256 ("c") ; ok = ok && GB_jitifyer_path_256 ("lib") ; ok = ok && GB_jitifyer_path_256 ("lock") ; // construct the src path snprintf (GB_jit_temp, GB_jit_temp_allocated, "%s/src", GB_jit_cache_path) ; ok = ok && GB_file_mkdir (GB_jit_temp) ; // construct the tmp path snprintf (GB_jit_temp, GB_jit_temp_allocated, "%s/tmp", GB_jit_cache_path) ; ok = ok && GB_file_mkdir (GB_jit_temp) ; //-------------------------------------------------------------------------- // make sure the cache and source paths exist //-------------------------------------------------------------------------- if (!ok) { // JIT is disabled, or cannot determine the JIT cache path. // Disable loading and compiling, but continue with the rest of the // initializations. The PreJIT could still be used. GBURBLE ("(jit: unable to access cache path, jit disabled) ") ; GB_jit_control = GxB_JIT_RUN ; GB_FREE_STUFF (GB_jit_cache_path) ; GB_COPY_STUFF (GB_jit_cache_path, "") ; } return (ok ? GrB_SUCCESS : error_condition) ; } //------------------------------------------------------------------------------ // GB_jitifyer_extract_JITpackage: extract the GraphBLAS source //------------------------------------------------------------------------------ // Returns GrB_SUCCESS if successful, GrB_OUT_OF_MEMORY if out of memory, or // error_condition if the files cannot be written to the cache folder for any // reason. If the JIT is disabled at compile time, this method does nothing. GrB_Info GB_jitifyer_extract_JITpackage (GrB_Info error_condition) { #ifndef NJIT //-------------------------------------------------------------------------- // lock the lock/00/src_lock file //-------------------------------------------------------------------------- snprintf (GB_jit_temp, GB_jit_temp_allocated, "%s/lock/00/src_lock", GB_jit_cache_path) ; FILE *fp_lock = NULL ; int fd_lock = -1 ; if (!GB_file_open_and_lock (GB_jit_temp, &fp_lock, &fd_lock)) { // failure; disable the JIT GBURBLE ("(jit: unable to write to source cache, jit disabled) ") ; GB_jit_control = GxB_JIT_RUN ; return (error_condition) ; } //-------------------------------------------------------------------------- // check the version number in src/GraphBLAS.h //-------------------------------------------------------------------------- snprintf (GB_jit_temp, GB_jit_temp_allocated, "%s/src/GraphBLAS.h", GB_jit_cache_path) ; FILE *fp_graphblas = fopen (GB_jit_temp, "r") ; if (fp_graphblas != NULL) { int v1 = -1, v2 = -1, v3 = -1 ; int r = fscanf (fp_graphblas, "// SuiteSparse:GraphBLAS %d.%d.%d", &v1, &v2, &v3) ; fclose (fp_graphblas) ; if (r == 3 && v1 == GxB_IMPLEMENTATION_MAJOR && v2 == GxB_IMPLEMENTATION_MINOR && v3 == GxB_IMPLEMENTATION_SUB) { // looks fine; assume the rest of the source is fine GB_file_unlock_and_close (&fp_lock, &fd_lock) ; return (GrB_SUCCESS) ; } } //-------------------------------------------------------------------------- // allocate workspace for the largest uncompressed file //-------------------------------------------------------------------------- size_t dst_size = 0 ; for (int k = 0 ; k < GB_JITpackage_nfiles ; k++) { size_t uncompressed_size = GB_JITpackage_index [k].uncompressed_size ; dst_size = GB_IMAX (dst_size, uncompressed_size) ; } uint8_t *dst ; GB_MALLOC_PERSISTENT (dst, (dst_size+2) * sizeof(uint8_t)) ; if (dst == NULL) { // JITPackage error: out of memory; disable the JIT GB_jit_control = GxB_JIT_RUN ; return (GrB_OUT_OF_MEMORY) ; } //-------------------------------------------------------------------------- // uncompress each file into the src folder //-------------------------------------------------------------------------- bool ok = true ; for (int k = 0 ; k < GB_JITpackage_nfiles ; k++) { // uncompress the blob uint8_t *src = GB_JITpackage_index [k].blob ; size_t src_size = GB_JITpackage_index [k].compressed_size ; size_t u = ZSTD_decompress (dst, dst_size, src, src_size) ; if (u != GB_JITpackage_index [k].uncompressed_size) { // JITPackage error: blob is invalid ok = false ; break ; } // construct the filename snprintf (GB_jit_temp, GB_jit_temp_allocated, "%s/src/%s", GB_jit_cache_path, GB_JITpackage_index [k].filename) ; // open the file FILE *fp_src = fopen (GB_jit_temp, "w") ; if (fp_src == NULL) { // JITPackage error: file cannot be created ok = false ; break ; } // write the uncompressed blob to the file size_t nwritten = fwrite (dst, sizeof (uint8_t), u, fp_src) ; fclose (fp_src) ; if (nwritten != u) { // JITPackage error: file is invalid ok = false ; break ; } } //-------------------------------------------------------------------------- // free workspace //-------------------------------------------------------------------------- GB_FREE_PERSISTENT (dst) ; //-------------------------------------------------------------------------- // unlock and close the lock/GB_src_lock file //-------------------------------------------------------------------------- GB_file_unlock_and_close (&fp_lock, &fd_lock) ; if (!ok) { // JITPackage error: disable the JIT GBURBLE ("(jit: unable to write to source cache, jit disabled) ") ; GB_jit_control = GxB_JIT_RUN ; return (error_condition) ; } #endif #pragma omp flush return (GrB_SUCCESS) ; } //------------------------------------------------------------------------------ // GB_jitifyer_get_control: get the JIT control //------------------------------------------------------------------------------ GxB_JIT_Control GB_jitifyer_get_control (void) { GxB_JIT_Control control ; #pragma omp critical (GB_jitifyer_worker) { control = GB_jit_control ; } return (control) ; } //------------------------------------------------------------------------------ // GB_jitifyer_set_control: set the JIT control //------------------------------------------------------------------------------ void GB_jitifyer_set_control (int control) { #pragma omp critical (GB_jitifyer_worker) { control = GB_IMAX (control, (int) GxB_JIT_OFF) ; #ifndef NJIT // The full JIT is available. control = GB_IMIN (control, (int) GxB_JIT_ON) ; #else // The JIT is restricted; only OFF, PAUSE, and RUN settings can be // used. No JIT kernels can be loaded or compiled. control = GB_IMIN (control, (int) GxB_JIT_RUN) ; #endif GB_jit_control = (GxB_JIT_Control) control ; if (GB_jit_control == GxB_JIT_OFF) { // free all loaded JIT kernels but do not free the JIT hash table, // and do not free the PreJIT kernels GB_jitifyer_table_free (false) ; } } } //------------------------------------------------------------------------------ // GB_jitifyer_alloc_space: allocate temporary workspace for the JIT //------------------------------------------------------------------------------ // Returns GrB_SUCCESS or GrB_OUT_OF_MEMORY. GrB_Info GB_jitifyer_alloc_space (void) { //-------------------------------------------------------------------------- // check inputs //-------------------------------------------------------------------------- if (GB_jit_C_flags == NULL || GB_jit_C_link_flags == NULL || GB_jit_C_libraries == NULL || GB_jit_C_cmake_libs == NULL || GB_jit_C_compiler == NULL || GB_jit_cache_path == NULL) { // JIT error: out of memory return (GrB_OUT_OF_MEMORY) ; } //-------------------------------------------------------------------------- // free the old GB_jit_temp and allocate it at the proper size //-------------------------------------------------------------------------- GB_FREE_STUFF (GB_jit_temp) ; size_t len = 2 * GB_jit_C_compiler_allocated + 2 * GB_jit_C_flags_allocated + GB_jit_C_link_flags_allocated + strlen (GB_OMP_INC) + 5 * GB_jit_cache_path_allocated + 7 * GB_KLEN + GB_jit_C_libraries_allocated + GB_jit_C_cmake_libs_allocated + GB_jit_error_log_allocated + 300 ; GB_MALLOC_STUFF (GB_jit_temp, len) ; return (GrB_SUCCESS) ; } //------------------------------------------------------------------------------ // GB_jitifyer_get_cache_path: return the current cache path //------------------------------------------------------------------------------ const char *GB_jitifyer_get_cache_path (void) { const char *s ; #pragma omp critical (GB_jitifyer_worker) { s = GB_jit_cache_path ; } return (s) ; } //------------------------------------------------------------------------------ // GB_jitifyer_set_cache_path: set a new cache path //------------------------------------------------------------------------------ // This method is only used by GxB_set. It returns GrB_SUCCESS if successful, // GrB_OUT_OF_MEMORY if out of memory, GrB_NULL_POINTER if the requested path // is a NULL string, or GrB_INVALID_VALUE if any file I/O error occurs. The // latter indicates that the requested path is not valid. GrB_Info GB_jitifyer_set_cache_path (const char *new_cache_path) { //-------------------------------------------------------------------------- // check inputs //-------------------------------------------------------------------------- if (new_cache_path == NULL) { return (GrB_NULL_POINTER) ; } //-------------------------------------------------------------------------- // set the cache path in a critical section //-------------------------------------------------------------------------- GrB_Info info ; #pragma omp critical (GB_jitifyer_worker) { info = GB_jitifyer_set_cache_path_worker (new_cache_path) ; } return (info) ; } //------------------------------------------------------------------------------ // GB_jitifyer_set_cache_path_worker: set cache path in a critical section //------------------------------------------------------------------------------ GrB_Info GB_jitifyer_set_cache_path_worker (const char *new_cache_path) { // free the old the cache path GB_FREE_STUFF (GB_jit_cache_path) ; // allocate the new GB_jit_cache_path GB_COPY_STUFF (GB_jit_cache_path, new_cache_path) ; // allocate workspace OK (GB_jitifyer_alloc_space ( )) ; // set the src path and make sure cache and src paths are accessible OK (GB_jitifyer_establish_paths (GrB_INVALID_VALUE)) ; // uncompress all the source files into the user source folder return (GB_jitifyer_extract_JITpackage (GrB_INVALID_VALUE)) ; } //------------------------------------------------------------------------------ // GB_jitifyer_get_error_log: return the current log file //------------------------------------------------------------------------------ const char *GB_jitifyer_get_error_log (void) { const char *s ; #pragma omp critical (GB_jitifyer_worker) { s = GB_jit_error_log ; } return (s) ; } //------------------------------------------------------------------------------ // GB_jitifyer_set_error_log: set a new log file //------------------------------------------------------------------------------ // If the new_error_log is NULL or the empty string, stderr is not redirected to // a log file. GrB_Info GB_jitifyer_set_error_log (const char *new_error_log) { //-------------------------------------------------------------------------- // set the log file in a critical section //-------------------------------------------------------------------------- GrB_Info info ; #pragma omp critical (GB_jitifyer_worker) { info = GB_jitifyer_set_error_log_worker ((new_error_log == NULL) ? "" : new_error_log) ; } return (info) ; } //------------------------------------------------------------------------------ // GB_jitifyer_set_error_log_worker: set log file in a critical section //------------------------------------------------------------------------------ GrB_Info GB_jitifyer_set_error_log_worker (const char *new_error_log) { // free the old log file GB_FREE_STUFF (GB_jit_error_log) ; // allocate the new GB_jit_error_log GB_COPY_STUFF (GB_jit_error_log, new_error_log) ; return (GB_jitifyer_alloc_space ( )) ; } //------------------------------------------------------------------------------ // GB_jitifyer_get_C_compiler: return the current C compiler //------------------------------------------------------------------------------ const char *GB_jitifyer_get_C_compiler (void) { const char *s ; #pragma omp critical (GB_jitifyer_worker) { s = GB_jit_C_compiler ; } return (s) ; } //------------------------------------------------------------------------------ // GB_jitifyer_set_C_compiler: set a new C compiler //------------------------------------------------------------------------------ GrB_Info GB_jitifyer_set_C_compiler (const char *new_C_compiler) { //-------------------------------------------------------------------------- // check inputs //-------------------------------------------------------------------------- if (new_C_compiler == NULL) { return (GrB_NULL_POINTER) ; } //-------------------------------------------------------------------------- // set the C compiler in a critical section //-------------------------------------------------------------------------- GrB_Info info ; #pragma omp critical (GB_jitifyer_worker) { info = GB_jitifyer_set_C_compiler_worker (new_C_compiler) ; } return (info) ; } //------------------------------------------------------------------------------ // GB_jitifyer_set_C_compiler_worker: set C compiler in a critical section //------------------------------------------------------------------------------ GrB_Info GB_jitifyer_set_C_compiler_worker (const char *new_C_compiler) { // free the old C compiler string GB_FREE_STUFF (GB_jit_C_compiler) ; // allocate the new GB_jit_C_compiler GB_COPY_STUFF (GB_jit_C_compiler, new_C_compiler) ; // allocate workspace return (GB_jitifyer_alloc_space ( )) ; } //------------------------------------------------------------------------------ // GB_jitifyer_get_C_flags: return the current C flags //------------------------------------------------------------------------------ const char *GB_jitifyer_get_C_flags (void) { const char *s ; #pragma omp critical (GB_jitifyer_worker) { s = GB_jit_C_flags ; } return (s) ; } //------------------------------------------------------------------------------ // GB_jitifyer_set_C_flags: set new C flags //------------------------------------------------------------------------------ GrB_Info GB_jitifyer_set_C_flags (const char *new_C_flags) { //-------------------------------------------------------------------------- // check inputs //-------------------------------------------------------------------------- if (new_C_flags == NULL) { return (GrB_NULL_POINTER) ; } //-------------------------------------------------------------------------- // set the C flags in a critical section //-------------------------------------------------------------------------- GrB_Info info ; #pragma omp critical (GB_jitifyer_worker) { info = GB_jitifyer_set_C_flags_worker (new_C_flags) ; } return (info) ; } //------------------------------------------------------------------------------ // GB_jitifyer_set_C_flags_worker: set C flags in a critical section //------------------------------------------------------------------------------ GrB_Info GB_jitifyer_set_C_flags_worker (const char *new_C_flags) { // free the old C flag string GB_FREE_STUFF (GB_jit_C_flags) ; // allocate the new GB_jit_C_flags GB_COPY_STUFF (GB_jit_C_flags, new_C_flags) ; // allocate workspace return (GB_jitifyer_alloc_space ( )) ; } //------------------------------------------------------------------------------ // GB_jitifyer_get_C_link_flags: return the current C link flags //------------------------------------------------------------------------------ const char *GB_jitifyer_get_C_link_flags (void) { const char *s ; #pragma omp critical (GB_jitifyer_worker) { s = GB_jit_C_link_flags ; } return (s) ; } //------------------------------------------------------------------------------ // GB_jitifyer_set_C_link_flags: set new C link flags //------------------------------------------------------------------------------ GrB_Info GB_jitifyer_set_C_link_flags (const char *new_C_link_flags) { //-------------------------------------------------------------------------- // check inputs //-------------------------------------------------------------------------- if (new_C_link_flags == NULL) { return (GrB_NULL_POINTER) ; } //-------------------------------------------------------------------------- // set the C link flags in a critical section //-------------------------------------------------------------------------- GrB_Info info ; #pragma omp critical (GB_jitifyer_worker) { info = GB_jitifyer_set_C_link_flags_worker (new_C_link_flags) ; } return (info) ; } //------------------------------------------------------------------------------ // GB_jitifyer_set_C_link_flags_worker: set C link flags in a critical section //------------------------------------------------------------------------------ GrB_Info GB_jitifyer_set_C_link_flags_worker (const char *new_C_link_flags) { // free the old C link flags string GB_FREE_STUFF (GB_jit_C_link_flags) ; // allocate the new GB_jit_C_link_flags GB_COPY_STUFF (GB_jit_C_link_flags, new_C_link_flags) ; // allocate workspace return (GB_jitifyer_alloc_space ( )) ; } //------------------------------------------------------------------------------ // GB_jitifyer_get_C_libraries: return the current C libraries //------------------------------------------------------------------------------ const char *GB_jitifyer_get_C_libraries (void) { const char *s ; #pragma omp critical (GB_jitifyer_worker) { s = GB_jit_C_libraries ; } return (s) ; } //------------------------------------------------------------------------------ // GB_jitifyer_set_C_libraries: set new C libraries //------------------------------------------------------------------------------ GrB_Info GB_jitifyer_set_C_libraries (const char *new_C_libraries) { //-------------------------------------------------------------------------- // check inputs //-------------------------------------------------------------------------- if (new_C_libraries == NULL) { return (GrB_NULL_POINTER) ; } //-------------------------------------------------------------------------- // set the C libraries in a critical section //-------------------------------------------------------------------------- GrB_Info info ; #pragma omp critical (GB_jitifyer_worker) { info = GB_jitifyer_set_C_libraries_worker (new_C_libraries) ; } return (info) ; } //------------------------------------------------------------------------------ // GB_jitifyer_set_C_libraries_worker: set C libraries in a critical section //------------------------------------------------------------------------------ GrB_Info GB_jitifyer_set_C_libraries_worker (const char *new_C_libraries) { // free the old C libraries string GB_FREE_STUFF (GB_jit_C_libraries) ; // allocate the new GB_jit_C_libraries GB_COPY_STUFF (GB_jit_C_libraries, new_C_libraries) ; // allocate workspace return (GB_jitifyer_alloc_space ( )) ; } //------------------------------------------------------------------------------ // GB_jitifyer_get_use_cmake: return true/false if cmake is in use //------------------------------------------------------------------------------ bool GB_jitifyer_get_use_cmake (void) { bool use_cmake ; #pragma omp critical (GB_jitifyer_worker) { use_cmake = GB_jit_use_cmake ; } return (use_cmake) ; } //------------------------------------------------------------------------------ // GB_jitifyer_set_use_cmake: set controls true/false to use cmake //------------------------------------------------------------------------------ void GB_jitifyer_set_use_cmake (bool use_cmake) { #pragma omp critical (GB_jitifyer_worker) { #if defined (_MSC_VER) // Windows requires cmake GB_jit_use_cmake = true ; #elif defined (__MINGW32__) // MINGW requires direct compile GB_jit_use_cmake = false ; #else // all other platforms have the option to use cmake or a direct compile GB_jit_use_cmake = use_cmake ; #endif } } //------------------------------------------------------------------------------ // GB_jitifyer_get_C_cmake_libs: return the current cmake libs //------------------------------------------------------------------------------ const char *GB_jitifyer_get_C_cmake_libs (void) { const char *s ; #pragma omp critical (GB_jitifyer_worker) { s = GB_jit_C_cmake_libs ; } return (s) ; } //------------------------------------------------------------------------------ // GB_jitifyer_set_C_cmake_libs: set new cmake libs //------------------------------------------------------------------------------ GrB_Info GB_jitifyer_set_C_cmake_libs (const char *new_cmake_libs) { //-------------------------------------------------------------------------- // check inputs //-------------------------------------------------------------------------- if (new_cmake_libs == NULL) { return (GrB_NULL_POINTER) ; } //-------------------------------------------------------------------------- // set the cmake libs in a critical section //-------------------------------------------------------------------------- GrB_Info info ; #pragma omp critical (GB_jitifyer_worker) { info = GB_jitifyer_set_C_cmake_libs_worker (new_cmake_libs) ; } return (info) ; } //------------------------------------------------------------------------------ // GB_jitifyer_set_C_cmake_libs_worker: set cmake libs in a critical section //------------------------------------------------------------------------------ GrB_Info GB_jitifyer_set_C_cmake_libs_worker (const char *new_cmake_libs) { // free the old C_cmake_libs string GB_FREE_STUFF (GB_jit_C_cmake_libs) ; // allocate the new GB_jit_C_cmake_libs GB_COPY_STUFF (GB_jit_C_cmake_libs, new_cmake_libs) ; // allocate workspace return (GB_jitifyer_alloc_space ( )) ; } //------------------------------------------------------------------------------ // GB_jitifyer_get_C_preface: return the current C preface //------------------------------------------------------------------------------ const char *GB_jitifyer_get_C_preface (void) { const char *s ; #pragma omp critical (GB_jitifyer_worker) { s = GB_jit_C_preface ; } return (s) ; } //------------------------------------------------------------------------------ // GB_jitifyer_set_C_preface: set new C preface //------------------------------------------------------------------------------ GrB_Info GB_jitifyer_set_C_preface (const char *new_C_preface) { //-------------------------------------------------------------------------- // check inputs //-------------------------------------------------------------------------- if (new_C_preface == NULL) { return (GrB_NULL_POINTER) ; } //-------------------------------------------------------------------------- // set the C preface in a critical section //-------------------------------------------------------------------------- GrB_Info info ; #pragma omp critical (GB_jitifyer_worker) { info = GB_jitifyer_set_C_preface_worker (new_C_preface) ; } return (info) ; } //------------------------------------------------------------------------------ // GB_jitifyer_set_C_preface_worker: set C preface in a critical section //------------------------------------------------------------------------------ GrB_Info GB_jitifyer_set_C_preface_worker (const char *new_C_preface) { // free the old strings that depend on the C preface GB_FREE_STUFF (GB_jit_C_preface) ; // allocate the new GB_jit_C_preface GB_COPY_STUFF (GB_jit_C_preface, new_C_preface) ; return (GrB_SUCCESS) ; } //------------------------------------------------------------------------------ // GB_jitifyer_get_CUDA_preface: return the current C preface //------------------------------------------------------------------------------ const char *GB_jitifyer_get_CUDA_preface (void) { const char *s ; #pragma omp critical (GB_jitifyer_worker) { s = GB_jit_CUDA_preface ; } return (s) ; } //------------------------------------------------------------------------------ // GB_jitifyer_set_CUDA_preface: set new C preface //------------------------------------------------------------------------------ GrB_Info GB_jitifyer_set_CUDA_preface (const char *new_CUDA_preface) { //-------------------------------------------------------------------------- // check inputs //-------------------------------------------------------------------------- if (new_CUDA_preface == NULL) { return (GrB_NULL_POINTER) ; } //-------------------------------------------------------------------------- // set the C preface in a critical section //-------------------------------------------------------------------------- GrB_Info info ; #pragma omp critical (GB_jitifyer_worker) { info = GB_jitifyer_set_CUDA_preface_worker (new_CUDA_preface) ; } return (info) ; } //------------------------------------------------------------------------------ // GB_jitifyer_set_CUDA_preface_worker: set C preface in a critical section //------------------------------------------------------------------------------ GrB_Info GB_jitifyer_set_CUDA_preface_worker (const char *new_CUDA_preface) { // free the old strings that depend on the C preface GB_FREE_STUFF (GB_jit_CUDA_preface) ; // allocate the new GB_jit_CUDA_preface GB_COPY_STUFF (GB_jit_CUDA_preface, new_CUDA_preface) ; return (GrB_SUCCESS) ; } //------------------------------------------------------------------------------ // GB_jitifyer_query: check if the type/op/monoid definitions match //------------------------------------------------------------------------------ // Returns true if type/op/monoid/etc definitions match, false otherwise. bool GB_jitifyer_query ( GB_jit_query_func dl_query, uint64_t hash, // hash code for the kernel // operator and type definitions GrB_Semiring semiring, GrB_Monoid monoid, GB_Operator op, GrB_Type type1, GrB_Type type2, GrB_Type type3 ) { //-------------------------------------------------------------------------- // get the terms to query //-------------------------------------------------------------------------- int version [3] ; char *library_defn [5] ; size_t zsize = 0 ; size_t tsize = 0 ; void *id = NULL ; void *term = NULL ; GB_Operator op1 = NULL, op2 = NULL ; if (semiring != NULL) { monoid = semiring->add ; op1 = (GB_Operator) monoid->op ; op2 = (GB_Operator) semiring->multiply ; } else if (monoid != NULL) { op1 = (GB_Operator) monoid->op ; } else { // op may be NULL, if this is a user_type kernel op1 = op ; } if (monoid != NULL && monoid->hash != 0) { // compare the user-defined identity and terminal values zsize = monoid->op->ztype->size ; tsize = (monoid->terminal == NULL) ? 0 : zsize ; id = monoid->identity ; term = monoid->terminal ; } //-------------------------------------------------------------------------- // query the JIT kernel for its definitions //-------------------------------------------------------------------------- uint64_t hash2 = 0 ; bool ok = dl_query (&hash2, version, library_defn, id, term, zsize, tsize) ; ok = ok && (version [0] == GxB_IMPLEMENTATION_MAJOR) && (version [1] == GxB_IMPLEMENTATION_MINOR) && (version [2] == GxB_IMPLEMENTATION_SUB) && (hash == hash2) ; //-------------------------------------------------------------------------- // compare current definitions with the ones in the JIT kernel //-------------------------------------------------------------------------- char *defn [5] ; defn [0] = (op1 == NULL) ? NULL : op1->defn ; defn [1] = (op2 == NULL) ? NULL : op2->defn ; defn [2] = (type1 == NULL) ? NULL : type1->defn ; defn [3] = (type2 == NULL) ? NULL : type2->defn ; defn [4] = (type3 == NULL) ? NULL : type3->defn ; for (int k = 0 ; k < 5 ; k++) { if ((defn [k] != NULL) != (library_defn [k] != NULL)) { // one is not NULL but the other is NULL ok = false ; } else if (defn [k] != NULL) { // both definitions are present // ensure the definition hasn't changed ok = ok && (strcmp (defn [k], library_defn [k]) == 0) ; } else { // both definitions are NULL, so they match } } return (ok) ; } //------------------------------------------------------------------------------ // GB_jitifyer_load: load a JIT kernel, compiling it if needed //------------------------------------------------------------------------------ // Returns GrB_SUCCESS if kernel is found (already loaded, or just now loaded, // or just now compiled and loaded). // Returns GrB_NO_VALUE if the kernel is not found and cannot be loaded or // compiled. This tells the caller that a generic method must be used. GrB_Info GB_jitifyer_load ( // output: void **dl_function, // pointer to JIT kernel // input: GB_jit_family family, // kernel family const char *kname, // kname for the kernel_name uint64_t hash, // hash code for the kernel GB_jit_encoding *encoding, // encoding of the problem const char *suffix, // suffix for the kernel_name (NULL if none) // operator and type definitions GrB_Semiring semiring, GrB_Monoid monoid, GB_Operator op, GrB_Type type1, GrB_Type type2, GrB_Type type3 ) { //-------------------------------------------------------------------------- // check for quick return //-------------------------------------------------------------------------- #ifdef GBMATLAB if (GB_Global_hack_get (3) != 0) { // the JIT can be disabled for testing, to test error handling GBURBLE ("(jit: test error handling) ") ; return (GrB_NOT_IMPLEMENTED) ; } #endif GrB_Info info ; if (hash == UINT64_MAX) { // The kernel may not be compiled; it does not have a valid definition. GBURBLE ("(jit: undefined) ") ; return (GrB_NO_VALUE) ; } if ((GB_jit_control == GxB_JIT_OFF) || (GB_jit_control == GxB_JIT_PAUSE)) { // The JIT control has disabled all JIT kernels. Punt to generic. return (GrB_NO_VALUE) ; } //-------------------------------------------------------------------------- // handle the GxB_JIT_RUN case: critical section not required //-------------------------------------------------------------------------- if ((GB_jit_control == GxB_JIT_RUN) && (family != GB_jit_user_op_family) && (family != GB_jit_user_type_family)) { //---------------------------------------------------------------------- // look up the kernel in the hash table //---------------------------------------------------------------------- int64_t k1 = -1, kk = -1 ; (*dl_function) = GB_jitifyer_lookup (hash, encoding, suffix, &k1, &kk) ; if (k1 >= 0) { // an unchecked PreJIT kernel; check it inside critical section } else if ((*dl_function) != NULL) { // found the kernel in the hash table return (GrB_SUCCESS) ; } else { // No kernels may be loaded or compiled, but existing kernels // already loaded may be run (handled above if dl_function was // found). This kernel was not loaded, so punt to generic. return (GrB_NO_VALUE) ; } } //-------------------------------------------------------------------------- // do the rest inside a critical section //-------------------------------------------------------------------------- #pragma omp critical (GB_jitifyer_worker) { info = GB_jitifyer_worker (dl_function, family, kname, hash, encoding, suffix, semiring, monoid, op, type1, type2, type3) ; } return (info) ; } //------------------------------------------------------------------------------ // GB_jitifyer_worker: do the work for GB_jitifyer_load in a critical section //------------------------------------------------------------------------------ GrB_Info GB_jitifyer_worker ( // output: void **dl_function, // pointer to JIT kernel // input: GB_jit_family family, // kernel family const char *kname, // kname for the kernel_name uint64_t hash, // hash code for the kernel GB_jit_encoding *encoding, // encoding of the problem const char *suffix, // suffix for the kernel_name (NULL if none) // operator and type definitions GrB_Semiring semiring, GrB_Monoid monoid, GB_Operator op, GrB_Type type1, GrB_Type type2, GrB_Type type3 ) { //-------------------------------------------------------------------------- // look up the kernel in the hash table //-------------------------------------------------------------------------- int64_t k1 = -1, kk = -1 ; (*dl_function) = GB_jitifyer_lookup (hash, encoding, suffix, &k1, &kk) ; if ((*dl_function) != NULL) { // found the kernel in the hash table GB_jit_entry *e = &(GB_jit_table [kk]) ; if (k1 >= 0) { // unchecked PreJIT kernel; check it now void **Kernels = NULL ; void **Queries = NULL ; char **Names = NULL ; int32_t nkernels = 0 ; GB_prejit (&nkernels, &Kernels, &Queries, &Names) ; GB_jit_query_func dl_query = (GB_jit_query_func) Queries [k1] ; bool ok = GB_jitifyer_query (dl_query, hash, semiring, monoid, op, type1, type2, type3) ; if (ok) { // PreJIT kernel is fine; flag it as checked by flipping // its prejit_index. GBURBLE ("(prejit: ok) ") ; e->prejit_index = GB_FLIP (k1) ; return (GrB_SUCCESS) ; } else { // remove the PreJIT kernel from the hash table; do not return. // Instead, keep going and compile a JIT kernel. GBURBLE ("(prejit: disabled) ") ; GB_jitifyer_entry_free (e) ; } } else if (family == GB_jit_user_op_family) { // user-defined operator; check it now GB_user_op_f GB_user_op = (GB_user_op_f) (*dl_function) ; void *ignore ; char *defn ; GB_user_op (&ignore, &defn) ; if (strcmp (defn, op->defn) == 0) { return (GrB_SUCCESS) ; } else { // the op has changed; need to re-JIT the kernel; do not return. // Instead, keep going and compile a JIT kernel. GBURBLE ("(jit: op changed) ") ; GB_jitifyer_entry_free (e) ; } } else if (family == GB_jit_user_type_family) { // user-defined type; check it now GB_user_type_f GB_user_type = (GB_user_type_f) (*dl_function) ; size_t ignore ; char *defn ; GB_user_type (&ignore, &defn) ; if (strcmp (defn, type1->defn) == 0) { return (GrB_SUCCESS) ; } else { // type has changed; need to re-JIT the kernel; do not return. // Instead, keep going and compile a JIT kernel. GBURBLE ("(jit: type changed) ") ; GB_jitifyer_entry_free (e) ; } } else { // JIT kernel, or checked PreJIT kernel return (GrB_SUCCESS) ; } } //-------------------------------------------------------------------------- // quick return if not in the hash table and load/compile is disabled //-------------------------------------------------------------------------- #ifndef NJIT if (GB_jit_control <= GxB_JIT_RUN) #endif { // No kernels may be loaded or compiled, but existing kernels already // loaded may be run (handled above if dl_function was found). This // kernel was not loaded, so punt to generic. return (GrB_NO_VALUE) ; } //-------------------------------------------------------------------------- // construct the kernel name //-------------------------------------------------------------------------- #ifndef NJIT GB_Operator op1 = NULL ; GB_Operator op2 = NULL ; int scode_digits = 0 ; switch (family) { case GB_jit_apply_family : op1 = op ; scode_digits = 10 ; break ; case GB_jit_assign_family : op1 = op ; scode_digits = 12 ; break ; case GB_jit_build_family : op1 = op ; scode_digits = 7 ; break ; case GB_jit_ewise_family : op1 = op ; scode_digits = 13 ; break ; case GB_jit_mxm_family : monoid = semiring->add ; op1 = (GB_Operator) semiring->add->op ; op2 = (GB_Operator) semiring->multiply ; scode_digits = 16 ; break ; case GB_jit_reduce_family : op1 = (GB_Operator) monoid->op ; scode_digits = 7 ; break ; case GB_jit_select_family : op1 = op ; scode_digits = 10 ; break ; case GB_jit_user_type_family : scode_digits = 1 ; break ; case GB_jit_user_op_family : scode_digits = 1 ; op1 = op ; break ; default: ; } char kernel_name [GB_KLEN] ; GB_macrofy_name (kernel_name, "GB_jit", kname, scode_digits, encoding->code, suffix) ; //-------------------------------------------------------------------------- // lock the kernel //-------------------------------------------------------------------------- // FIXME: add kernel_name to the lock filename. If the lock fails, // sleep for 1 second and try again repeatedly, with a timeout limit of // (say) 60 seconds. uint32_t bucket = hash & 0xFF ; snprintf (GB_jit_temp, GB_jit_temp_allocated, "%s/lock/%02x/%016" PRIx64 "_lock", GB_jit_cache_path, bucket, hash) ; FILE *fp_klock = NULL ; int fd_klock = -1 ; if (!GB_file_open_and_lock (GB_jit_temp, &fp_klock, &fd_klock)) { // JIT error: unable to lock the kernel // disable the JIT to avoid repeated load errors GB_jit_control = GxB_JIT_RUN ; return (GrB_NO_VALUE) ; } //-------------------------------------------------------------------------- // load the kernel, compiling it if needed //-------------------------------------------------------------------------- GrB_Info info = GB_jitifyer_load_worker (dl_function, kernel_name, family, kname, hash, encoding, suffix, semiring, monoid, op, op1, op2, type1, type2, type3) ; //-------------------------------------------------------------------------- // unlock the kernel //-------------------------------------------------------------------------- GB_file_unlock_and_close (&fp_klock, &fd_klock) ; return (info) ; #endif } //------------------------------------------------------------------------------ // GB_jitifyer_load_worker: load/compile a kernel //------------------------------------------------------------------------------ // This work is done inside a critical section for this process, and inside a // file lock/unlock section (fp_klock) to guard against access from other // processes. GrB_Info GB_jitifyer_load_worker ( // output: void **dl_function, // pointer to JIT kernel // input: char *kernel_name, // kernel file name (excluding the path) GB_jit_family family, // kernel family const char *kname, // kname for the kernel_name uint64_t hash, // hash code for the kernel GB_jit_encoding *encoding, // encoding of the problem const char *suffix, // suffix for the kernel_name (NULL if none) // operator and type definitions GrB_Semiring semiring, GrB_Monoid monoid, GB_Operator op, GB_Operator op1, GB_Operator op2, GrB_Type type1, GrB_Type type2, GrB_Type type3 ) { #ifndef NJIT //-------------------------------------------------------------------------- // try to load the lib*.so from the user's library folder //-------------------------------------------------------------------------- uint32_t bucket = hash & 0xFF ; snprintf (GB_jit_temp, GB_jit_temp_allocated, "%s/lib/%02x/%s%s%s", GB_jit_cache_path, bucket, GB_LIB_PREFIX, kernel_name, GB_LIB_SUFFIX) ; void *dl_handle = GB_file_dlopen (GB_jit_temp) ; //-------------------------------------------------------------------------- // check if the kernel was found, but needs to be compiled anyway //-------------------------------------------------------------------------- if (dl_handle != NULL) { // library is loaded but make sure the defn match GB_jit_query_func dl_query = (GB_jit_query_func) GB_file_dlsym (dl_handle, "GB_jit_query") ; bool ok = (dl_query != NULL) ; if (ok) { ok = GB_jitifyer_query (dl_query, hash, semiring, monoid, op, type1, type2, type3) ; } if (!ok) { // library is loaded but needs to change, so close it GB_file_dlclose (dl_handle) ; dl_handle = NULL ; // remove the library itself so it doesn't cause the error again remove (GB_jit_temp) ; GBURBLE ("(jit: loaded but must recompile) ") ; } } //-------------------------------------------------------------------------- // create and compile source file, if needed //-------------------------------------------------------------------------- if (dl_handle == NULL) { //---------------------------------------------------------------------- // quick return if the JIT is not permitted to compile new kernels //---------------------------------------------------------------------- if (GB_jit_control < GxB_JIT_ON) { // No new kernels may be compiled, so punt to generic. GBURBLE ("(jit: not compiled) ") ; return (GrB_NO_VALUE) ; } //---------------------------------------------------------------------- // create the source, compile it, and load it //---------------------------------------------------------------------- GBURBLE ("(jit: compile and load) ") ; GB_jit_kcode kcode = encoding->kcode ; const char *kernel_filetype = (kcode < GB_JIT_CUDA_KERNEL) ? "c" : "cu" ; // create (or recreate) the kernel source, compile it, and load it snprintf (GB_jit_temp, GB_jit_temp_allocated, "%s/c/%02x/%s.%s", GB_jit_cache_path, bucket, kernel_name, kernel_filetype) ; FILE *fp = fopen (GB_jit_temp, "w") ; if (fp != NULL) { // create the preface GB_macrofy_preface (fp, kernel_name, GB_jit_C_preface, GB_jit_CUDA_preface, kcode) ; // macrofy the kernel operators, types, and matrix formats GB_macrofy_family (fp, family, encoding->code, semiring, monoid, op, type1, type2, type3) ; // #include the kernel, renaming it for the PreJIT fprintf (fp, "#ifndef GB_JIT_RUNTIME\n" "#define GB_jit_kernel %s\n" "#define GB_jit_query %s_query\n" "#endif\n" "#include \"GB_jit_kernel_%s.%s\"\n", kernel_name, kernel_name, kname, kernel_filetype) ; // macrofy the query function bool builtin = (encoding->suffix_len == 0) ; GB_macrofy_query (fp, builtin, monoid, op1, op2, type1, type2, type3, hash, kcode) ; fclose (fp) ; } // if the source file was not created above, the compilation will // gracefully fail. // compile the kernel to get the lib*.so file if (kcode >= GB_JIT_CUDA_KERNEL) { // use NVCC to directly compile the CUDA kernel GB_jitifyer_nvcc_compile (kernel_name, bucket) ; } else if (GB_jit_use_cmake) { // use cmake to compile the CPU kernel GB_jitifyer_cmake_compile (kernel_name, hash) ; } else { // use the compiler to directly compile the CPU kernel GB_jitifyer_direct_compile (kernel_name, bucket) ; } // load the kernel from the lib*.so file snprintf (GB_jit_temp, GB_jit_temp_allocated, "%s/lib/%02x/%s%s%s", GB_jit_cache_path, bucket, GB_LIB_PREFIX, kernel_name, GB_LIB_SUFFIX) ; dl_handle = GB_file_dlopen (GB_jit_temp) ; //---------------------------------------------------------------------- // handle any error conditions //---------------------------------------------------------------------- if (dl_handle == NULL) { // unable to create the kernel source or open lib*.so file GBURBLE ("(jit: compiler error; compilation disabled) ") ; // disable the JIT to avoid repeated compilation errors GB_jit_control = GxB_JIT_LOAD ; // remove the compiled library remove (GB_jit_temp) ; return (GrB_NO_VALUE) ; } } else { GBURBLE ("(jit: load) ") ; } //-------------------------------------------------------------------------- // get the GB_jit_kernel function pointer //-------------------------------------------------------------------------- (*dl_function) = GB_file_dlsym (dl_handle, "GB_jit_kernel") ; if ((*dl_function) == NULL) { // JIT error: dlsym unable to find GB_jit_kernel: punt to generic GBURBLE ("(jit: load error; JIT loading disabled) ") ; GB_file_dlclose (dl_handle) ; dl_handle = NULL ; // disable the JIT to avoid repeated loading errors GB_jit_control = GxB_JIT_RUN ; // remove the compiled library remove (GB_jit_temp) ; return (GrB_NO_VALUE) ; } // insert the new kernel into the hash table if (!GB_jitifyer_insert (hash, encoding, suffix, dl_handle, (*dl_function), -1)) { // JIT error: unable to add kernel to hash table: punt to generic GB_file_dlclose (dl_handle) ; dl_handle = NULL ; // disable the JIT to avoid repeated errors GB_jit_control = GxB_JIT_PAUSE ; // remove the compiled library remove (GB_jit_temp) ; return (GrB_NO_VALUE) ; } return (GrB_SUCCESS) ; #else (*dl_function) = NULL ; return (GrB_INVALID_VALUE) ; #endif } //------------------------------------------------------------------------------ // GB_jitifyer_lookup: find a jit entry in the hash table //------------------------------------------------------------------------------ void *GB_jitifyer_lookup // return dl_function pointer, or NULL if not found ( // input: uint64_t hash, // hash = GB_jitifyer_hash_encoding (encoding) ; GB_jit_encoding *encoding, const char *suffix, // output int64_t *k1, // location of unchecked kernel in PreJIT table int64_t *kk // location of hash entry in hash table ) { (*k1) = -1 ; if (GB_jit_table == NULL) { // no table yet so it isn't present return (NULL) ; } uint32_t suffix_len = encoding->suffix_len ; bool builtin = (bool) (suffix_len == 0) ; // look up the entry in the hash table for (uint64_t k = hash ; ; k++) { k = k & GB_jit_table_bits ; GB_jit_entry *e = &(GB_jit_table [k]) ; if (e->dl_function == NULL) { // found an empty entry, so the entry is not in the table return (NULL) ; } else if (e->hash == hash && e->encoding.code == encoding->code && e->encoding.kcode == encoding->kcode && e->encoding.suffix_len == suffix_len && (builtin || (memcmp (e->suffix, suffix, suffix_len) == 0))) { // found the right entry: return the corresponding dl_function int64_t my_k1 ; GB_ATOMIC_READ my_k1 = e->prejit_index ; // >= 0: unchecked JIT kernel (*k1) = my_k1 ; (*kk) = k ; return (e->dl_function) ; } // otherwise, keep looking } } //------------------------------------------------------------------------------ // GB_jitifyer_insert: insert a jit entry in the hash table //------------------------------------------------------------------------------ bool GB_jitifyer_insert // return true if successful, false if failure ( // input: uint64_t hash, // hash for the problem GB_jit_encoding *encoding, // primary encoding const char *suffix, // suffix for user-defined types/operators void *dl_handle, // library handle from GB_file_dlopen; // NULL for PreJIT void *dl_function, // function handle from GB_file_dlsym int32_t prejit_index // index into PreJIT table; =>0 if unchecked. ) { size_t siz = 0 ; ASSERT_TABLE_OK ; //-------------------------------------------------------------------------- // ensure the hash table is large enough //-------------------------------------------------------------------------- if (GB_jit_table == NULL) { //---------------------------------------------------------------------- // allocate the initial hash table //---------------------------------------------------------------------- siz = GB_JITIFIER_INITIAL_SIZE * sizeof (struct GB_jit_entry_struct) ; GB_MALLOC_PERSISTENT (GB_jit_table, siz) ; if (GB_jit_table == NULL) { // JIT error: out of memory return (false) ; } memset (GB_jit_table, 0, siz) ; GB_jit_table_size = GB_JITIFIER_INITIAL_SIZE ; GB_jit_table_bits = GB_JITIFIER_INITIAL_SIZE - 1 ; GB_jit_table_allocated = siz ; } else if (4 * GB_jit_table_populated >= GB_jit_table_size) { //---------------------------------------------------------------------- // expand the existing hash table by a factor of 4 and rehash //---------------------------------------------------------------------- ASSERT_TABLE_OK ; // create a new table that is four times the size int64_t new_size = 4 * GB_jit_table_size ; int64_t new_bits = new_size - 1 ; siz = new_size * sizeof (struct GB_jit_entry_struct) ; GB_jit_entry *new_table ; GB_MALLOC_PERSISTENT (new_table, siz) ; if (new_table == NULL) { // JIT error: out of memory; leave the existing table as-is return (false) ; } // rehash into the new table memset (new_table, 0, siz) ; for (uint64_t k = 0 ; k < GB_jit_table_size ; k++) { if (GB_jit_table [k].dl_function != NULL) { // rehash the entry to the larger hash table uint64_t hash = GB_jit_table [k].hash ; for (uint64_t knew = hash ; ; knew++) { knew = knew & new_bits ; GB_jit_entry *e = &(new_table [knew]) ; if (e->dl_function == NULL) { // found an empty slot in the new table new_table [knew] = GB_jit_table [k] ; break ; } } } } // free the old table GB_FREE_STUFF (GB_jit_table) ; // use the new table GB_jit_table = new_table ; GB_jit_table_size = new_size ; GB_jit_table_bits = new_bits ; GB_jit_table_allocated = siz ; ASSERT_TABLE_OK ; } //-------------------------------------------------------------------------- // insert the jit entry in the hash table //-------------------------------------------------------------------------- uint64_t suffix_len = (uint64_t) (encoding->suffix_len) ; bool builtin = (bool) (suffix_len == 0) ; ASSERT_TABLE_OK ; for (uint64_t k = hash ; ; k++) { k = k & GB_jit_table_bits ; GB_jit_entry *e = &(GB_jit_table [k]) ; if (e->dl_function == NULL) { // found an empty slot e->suffix = NULL ; if (!builtin) { // allocate the suffix if the kernel is not builtin GB_MALLOC_PERSISTENT (e->suffix, suffix_len+2) ; if (e->suffix == NULL) { // JIT error: out of memory return (false) ; } strncpy (e->suffix, suffix, suffix_len+1) ; } e->hash = hash ; memcpy (&(e->encoding), encoding, sizeof (GB_jit_encoding)) ; e->dl_handle = dl_handle ; // NULL for PreJIT e->dl_function = dl_function ; GB_jit_table_populated++ ; e->prejit_index = prejit_index ; // -1 for JIT kernels ASSERT_TABLE_OK ; return (true) ; } // otherwise, keep looking } } //------------------------------------------------------------------------------ // GB_jitifyer_entry_free: free a single JIT hash table entry //------------------------------------------------------------------------------ void GB_jitifyer_entry_free (GB_jit_entry *e) { e->dl_function = NULL ; GB_jit_table_populated-- ; GB_FREE_PERSISTENT (e->suffix) ; // unload the dl library if (e->dl_handle != NULL) { GB_file_dlclose (e->dl_handle) ; e->dl_handle = NULL ; } ASSERT_TABLE_OK ; } //------------------------------------------------------------------------------ // GB_jitifyer_table_free: free the hash and clear all loaded kernels //------------------------------------------------------------------------------ // Clears all runtime JIT kernels from the hash table. PreJIT kernels and JIT // kernels containing user-defined operators are not freed if freall is true // (only done by GrB_finalize), but they are flagged as unchecked. This allows // the application to call GxB_set to set the JIT control to OFF then ON again, // to indicate that a user-defined type or operator has been changed, and that // all JIT kernels must cleared and all PreJIT kernels checked again before // using them. // After calling this function, the JIT is still enabled. GB_jitifyer_insert // will reallocate the table if it is NULL. void GB_jitifyer_table_free (bool freeall) { if (GB_jit_table != NULL) { for (uint64_t k = 0 ; k < GB_jit_table_size ; k++) { GB_jit_entry *e = &(GB_jit_table [k]) ; if (e->dl_function != NULL) { // found an entry if (e->dl_handle == NULL) { // flag the PreJIT kernel as unchecked e->prejit_index = GB_UNFLIP (e->prejit_index) ; } // free it if permitted if (freeall || (e->dl_handle != NULL && e->encoding.kcode != GB_JIT_KERNEL_USEROP)) { // free the entry GB_jitifyer_entry_free (e) ; } } } } ASSERT (GB_IMPLIES (freeall, GB_jit_table_populated == 0)) ; if (GB_jit_table_populated == 0) { // the JIT table is now empty, so free it GB_FREE_STUFF (GB_jit_table) ; GB_jit_table_size = 0 ; GB_jit_table_bits = 0 ; } } //------------------------------------------------------------------------------ // GB_jitifyer_command: run a command in a child process //------------------------------------------------------------------------------ // No error condition or status is returned. // If burble is on, stdout is left alone, so the stdout of the command is sent // to the stdout of the parent process. If burble is off, stdout is sent to // /dev/null (nul on Windows). If there is no error log file, stderr is not // redirected; otherwise, it is redirected to that file. The redirects are // handled by modifying the command string in the caller, so they do not have // to be handled here. // NOTE: this call to system(...) *cannot* be sanitized; CodeQL flags calls to // GB_jitifyer_command as security issues, but this is intentional. The JIT // allows the end user to create arbitary user-defined types and operators, // which GraphBLAS then injects into C source code of a JIT kernel created at // run time. The end user can also specify an arbitrary compiler to compile // JIT kernels. This code injection is intentional, and required for the JIT. // If a security-hardened GraphBLAS library is required, then GraphBLAS must be // compiled with -DNJIT to disable the JIT entirely. User-defined JIT kernels // will not be created at run time, and thus user-defined types and operators // will be slow, however. This option does not disable any PreJIT kernels, so // if fast user-defined kernels are required, they can be used with the PreJIT // mechanism; see the GraphBLAS User Guide for details. static void GB_jitifyer_command (char *command) { int result = system (command) ; } //------------------------------------------------------------------------------ // GB_jitifyer_cmake_compile: compile a kernel with cmake //------------------------------------------------------------------------------ // This method does not return any error/success code. If the compilation // fails for any reason, the subsequent load of the compiled kernel will fail. // This method works on most platforms (not yet on MINGW, so it is not used // there). For Windows using MSVC, this method is always used. // FUTURE: get this method to work in MINGW. #define GB_BLD_DIR "%s/tmp/%016" PRIx64 void GB_jitifyer_cmake_compile (char *kernel_name, uint64_t hash) { #ifndef NJIT uint32_t bucket = hash & 0xFF ; GBURBLE ("(jit: %s)\n", "cmake") ; char *burble_stdout = GB_Global_burble_get ( ) ? "" : GB_DEV_NULL ; char *err_redirect = (strlen (GB_jit_error_log) > 0) ? " 2>> " : "" ; #if defined (__MINGW32__) #define GB_SH_C "sh -c " #else #define GB_SH_C #endif // remove any prior build folder for this kernel, and all its contents snprintf (GB_jit_temp, GB_jit_temp_allocated, GB_SH_C "cmake -E remove_directory \"" GB_BLD_DIR "\" %s %s %s", GB_jit_cache_path, hash, // build path burble_stdout, err_redirect, GB_jit_error_log) ; GB_jitifyer_command (GB_jit_temp) ; // OK: see security comment above // create the build folder for this kernel snprintf (GB_jit_temp, GB_jit_temp_allocated, GB_BLD_DIR, GB_jit_cache_path, hash) ; if (!GB_file_mkdir (GB_jit_temp)) return ; // create the CMakeLists.txt file in the build folder for this kernel snprintf (GB_jit_temp, GB_jit_temp_allocated, GB_BLD_DIR "/CMakeLists.txt", GB_jit_cache_path, hash) ; FILE *fp = fopen (GB_jit_temp, "w") ; if (fp == NULL) return ; fprintf (fp, "cmake_minimum_required ( VERSION 3.13 )\n" // end user needs cmake 3.13 "project ( GBjit LANGUAGES C )\n" "include_directories ( \"%s/src\"%s)\n" "add_compile_definitions ( GB_JIT_RUNTIME )\n", GB_jit_cache_path, // include directories: cache/src ((strlen (GB_OMP_INC_DIRS) == 0) ? " " : " \"" GB_OMP_INC_DIRS "\" ")) ; // print the C flags, but escape any double quote characters fprintf (fp, "set ( CMAKE_C_FLAGS \"") ; for (char *p = GB_jit_C_flags ; *p != '\0' ; p++) { if (*p == '"') fprintf (fp, "\\") ; fprintf (fp, "%c", *p) ; } fprintf (fp, "\" )\n") ; fprintf (fp, "add_library ( %s SHARED \"%s/c/%02x/%s.c\" )\n", kernel_name, // target name for add_library command GB_jit_cache_path, bucket, kernel_name) ; // source file for add_library if (strlen (GB_jit_C_cmake_libs) > 0) { fprintf (fp, "target_link_libraries ( %s PUBLIC %s )\n", kernel_name, // target name of the library GB_jit_C_cmake_libs) ; // libraries to link against } fprintf (fp, "set_target_properties ( %s PROPERTIES\n" " C_STANDARD 11 C_STANDARD_REQUIRED ON )\n" "install ( TARGETS %s\n" " LIBRARY DESTINATION \"%s/lib/%02x\"\n" " ARCHIVE DESTINATION \"%s/lib/%02x\"\n" " RUNTIME DESTINATION \"%s/lib/%02x\" )\n", kernel_name, kernel_name, GB_jit_cache_path, bucket, GB_jit_cache_path, bucket, GB_jit_cache_path, bucket) ; fclose (fp) ; // generate the build system for this kernel snprintf (GB_jit_temp, GB_jit_temp_allocated, GB_SH_C "cmake -S \"" GB_BLD_DIR "\" -B \"" GB_BLD_DIR "\"" " -DCMAKE_C_COMPILER=\"%s\" %s %s %s", GB_jit_cache_path, hash, // -S source path GB_jit_cache_path, hash, // -B build path GB_jit_C_compiler, // C compiler to use burble_stdout, err_redirect, GB_jit_error_log) ; GB_jitifyer_command (GB_jit_temp) ; // OK: see security comment above // compile the library for this kernel snprintf (GB_jit_temp, GB_jit_temp_allocated, GB_SH_C "cmake --build \"" GB_BLD_DIR "\" --config Release %s %s %s", // can add "--verbose" here too GB_jit_cache_path, hash, // build path burble_stdout, err_redirect, GB_jit_error_log) ; GB_jitifyer_command (GB_jit_temp) ; // OK: see security comment above // install the library snprintf (GB_jit_temp, GB_jit_temp_allocated, GB_SH_C "cmake --install \"" GB_BLD_DIR "\" %s %s %s", GB_jit_cache_path, hash, // build path burble_stdout, err_redirect, GB_jit_error_log) ; GB_jitifyer_command (GB_jit_temp) ; // OK: see security comment above // remove the build folder and all its contents snprintf (GB_jit_temp, GB_jit_temp_allocated, GB_SH_C "cmake -E remove_directory \"" GB_BLD_DIR "\" %s %s %s", GB_jit_cache_path, hash, // build path burble_stdout, err_redirect, GB_jit_error_log) ; GB_jitifyer_command (GB_jit_temp) ; // OK: see security comment above #endif } //------------------------------------------------------------------------------ // GB_jitifyer_nvcc_compile: compile a CUDA kernel with NVRTC //------------------------------------------------------------------------------ // Compiles a CUDA JIT kernel in a *.cu file, containing host code that // launches one or more device kernels. // The input file has the form: // // %s/c/%02x/%s or [cache_path]/c/[bucket]/[kernel_name].cu // // and the libary file is linked as // // %s/lib/%02x/lib%s.so or [cache_path]/lib/[bucket]/lib[kernel_name].so // // All other temporary files (including *.o object files) are removed. void GB_jitifyer_nvcc_compile (char *kernel_name, uint32_t bucket) { #if defined ( GRAPHBLAS_HAS_CUDA ) && !defined ( NJIT ) char *burble_stdout = GB_Global_burble_get ( ) ? "" : GB_DEV_NULL ; char *err_redirect = (strlen (GB_jit_error_log) > 0) ? " 2>> " : "" ; GBURBLE ("(jit compiling cuda with nvcc: %s/c/%02x/%s.cu) ", GB_jit_cache_path, bucket, kernel_name) ; snprintf (GB_jit_temp, GB_jit_temp_allocated, // compile: "sh -c \"" // execute with POSIX shell "nvcc " // compiler command "-forward-unknown-to-host-compiler " "-DGB_JIT_RUNTIME=1 " // nvcc flags "-I/usr/local/cuda/include -std=c++17 -arch=sm_60 -fPIC " "-I%s/src " // include source directory "-o %s/c/%02x/%s%s " // *.o output file "-c %s/c/%02x/%s.cu " // *.cu input file "%s " // burble stdout "%s %s ; " // error log file // link: "nvcc " // compiler "-DGB_JIT_RUNTIME=1 " // nvcc flags "-I/usr/local/cuda/include -std=c++17 -arch=sm_60 " " -shared " "-o %s/lib/%02x/%s%s%s " // lib*.so output file "%s/c/%02x/%s%s " // *.o input file " -cudart shared " // "%s " // libraries to link with (any?) "%s " // burble stdout "%s %s\"", // error log file // compile: GB_jit_cache_path, // include source directory (cache/src) GB_jit_cache_path, bucket, kernel_name, GB_OBJ_SUFFIX, // *.o output file GB_jit_cache_path, bucket, kernel_name, // *.cu input file burble_stdout, // burble stdout err_redirect, GB_jit_error_log, // error log file // link: GB_jit_cache_path, bucket, GB_LIB_PREFIX, kernel_name, GB_LIB_SUFFIX, // lib*.so file GB_jit_cache_path, bucket, kernel_name, GB_OBJ_SUFFIX, // *.o input file // GB_jit_C_libraries // libraries to link with burble_stdout, // burble stdout err_redirect, GB_jit_error_log) ; // error log file // compile the library and return result GBURBLE ("\n(jit: %s) ", GB_jit_temp) ; GB_jitifyer_command (GB_jit_temp) ; // OK: see security comment above // remove the *.o file snprintf (GB_jit_temp, GB_jit_temp_allocated, "%s/c/%02x/%s%s", GB_jit_cache_path, bucket, kernel_name, GB_OBJ_SUFFIX) ; remove (GB_jit_temp) ; #endif } //------------------------------------------------------------------------------ // GB_jitifyer_direct_compile: compile a kernel with just the compiler //------------------------------------------------------------------------------ // This method does not return any error/success code. If the compilation // fails for any reason, the subsequent load of the compiled kernel will fail. // This method does not work on Windows with MSVC. It works for Linux, Mac, // or Windows with MINGW (for which it is currently the only option). // FUTURE: get this method to work in MSVC, since it's much faster than using // cmake on Windows. void GB_jitifyer_direct_compile (char *kernel_name, uint32_t bucket) { #ifndef NJIT char *burble_stdout = GB_Global_burble_get ( ) ? "" : GB_DEV_NULL ; char *err_redirect = (strlen (GB_jit_error_log) > 0) ? " 2>> " : "" ; snprintf (GB_jit_temp, GB_jit_temp_allocated, // compile: "sh -c \"" // execute with POSIX shell "%s " // compiler command "-DGB_JIT_RUNTIME=1 %s " // C flags "-I%s/src " // include source directory "%s " // openmp include directories "-o %s/c/%02x/%s%s " // *.o output file "-c %s/c/%02x/%s.c " // *.c input file "%s " // burble stdout "%s %s ; " // error log file // link: "%s " // C compiler "%s " // C flags "%s " // C link flags "-o %s/lib/%02x/%s%s%s " // lib*.so output file "%s/c/%02x/%s%s " // *.o input file "%s " // libraries to link with "%s " // burble stdout "%s %s\"", // error log file // compile: GB_jit_C_compiler, // C compiler GB_jit_C_flags, // C flags GB_jit_cache_path, // include source directory (cache/src) GB_OMP_INC, // openmp include GB_jit_cache_path, bucket, kernel_name, GB_OBJ_SUFFIX, // *.o output file GB_jit_cache_path, bucket, kernel_name, // *.c input file burble_stdout, // burble stdout err_redirect, GB_jit_error_log, // error log file // link: GB_jit_C_compiler, // C compiler GB_jit_C_flags, // C flags GB_jit_C_link_flags, // C link flags GB_jit_cache_path, bucket, GB_LIB_PREFIX, kernel_name, GB_LIB_SUFFIX, // lib*.so file GB_jit_cache_path, bucket, kernel_name, GB_OBJ_SUFFIX, // *.o input file GB_jit_C_libraries, // libraries to link with burble_stdout, // burble stdout err_redirect, GB_jit_error_log) ; // error log file // compile the library and return result GBURBLE ("(jit: %s) ", GB_jit_temp) ; GB_jitifyer_command (GB_jit_temp) ; // OK: see security comment above // remove the *.o file snprintf (GB_jit_temp, GB_jit_temp_allocated, "%s/c/%02x/%s%s", GB_jit_cache_path, bucket, kernel_name, GB_OBJ_SUFFIX) ; remove (GB_jit_temp) ; #endif } //------------------------------------------------------------------------------ // GB_jitifyer_hash: compute the hash //------------------------------------------------------------------------------ // xxHash uses switch statements with no default case. #if GB_COMPILER_GCC #pragma GCC diagnostic ignored "-Wswitch-default" #endif #define XXH_INLINE_ALL #define XXH_NO_STREAM #include "xxhash.h" // A hash value of zero is unique, and is used for all builtin operators and // types to indicate that its hash value is not required. // A hash value of UINT64_MAX is also special: it denotes an object that cannot // be JIT'd. // So in the nearly impossible case that XXH3_64bits returns a hash value that // happens to be zero or UINT64_MAX, it is reset to GB_MAGIC instead. uint64_t GB_jitifyer_hash_encoding ( GB_jit_encoding *encoding ) { uint64_t hash ; hash = XXH3_64bits ((const void *) encoding, sizeof (GB_jit_encoding)) ; return ((hash == 0 || hash == UINT64_MAX) ? GB_MAGIC : hash) ; } uint64_t GB_jitifyer_hash ( const void *bytes, // any string of bytes size_t nbytes, // # of bytes to hash bool jitable // true if the object can be JIT'd ) { if (!jitable) return (UINT64_MAX) ; if (bytes == NULL || nbytes == 0) return (0) ; uint64_t hash ; hash = XXH3_64bits (bytes, nbytes) ; return ((hash == 0 || hash == UINT64_MAX) ? GB_MAGIC : hash) ; }