#define CL_TARGET_OPENCL_VERSION GGML_OPENCL_TARGET_VERSION #define CL_USE_DEPRECATED_OPENCL_1_2_APIS // suppress warnings in CL headers for GCC and Clang #pragma GCC diagnostic ignored "-Woverlength-strings" #ifdef __clang__ #pragma GCC diagnostic ignored "-Wgnu-anonymous-struct" #endif #include "ggml-opencl.h" #include "ggml-backend.h" #include "ggml-impl.h" #include "ggml-backend-impl.h" #include "ggml.h" #include "cl-program-cache.h" #ifdef GGML_OPENCL_USE_ADRENO_BIN_KERNELS #include "libdl.h" #ifdef _WIN32 #define KERNEL_LIB_NAME "adreno-opencl-kernels.dll" #else #define KERNEL_LIB_NAME "libadreno-opencl-kernels.so" #endif // _WIN32 #endif // GGML_OPENCL_USE_ADRENO_BIN_KERNELS typedef const void * (*get_adreno_bin_kernel_func_t)( const char * name, const char * gpu_name, const char * compiler_ver, size_t * out_size ); #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #undef MIN #undef MAX #define MIN(a, b) ((a) < (b) ? (a) : (b)) #define MAX(a, b) ((a) > (b) ? (a) : (b)) #define CEIL_DIV(M, N) (((M) + (N)-1) / (N)) #define UNUSED(x) (void)(x) #define CL_CHECK(err) \ do { \ cl_int err_ = (err); \ if (err_ != CL_SUCCESS) { \ GGML_LOG_ERROR("ggml_opencl: %s error %d at %s:%d\n", \ #err, err_, __FILE__, __LINE__); \ GGML_ASSERT(0); \ } \ } while (0) //------------------------------------------------------------------------------ // OpenCL //------------------------------------------------------------------------------ bool ggml_cl_compute_forward(ggml_backend_t backend, struct ggml_tensor * tensor); static bool ggml_cl_is_q4_0_soa(const ggml_tensor * tensor); static bool ggml_cl_is_q8_0_soa(const ggml_tensor * tensor); static void ggml_cl_mul_mat(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst); // See https://gmplib.org/~tege/divcnst-pldi94.pdf figure 4.1. // Precompute mp (m' in the paper) and L such that division // can be computed using a multiply (high 32b of 64b result) // and a shift: // // n/d = (mulhi(n, mp) + n) >> L; struct fastdiv_vals { uint32_t mp; uint32_t L; uint32_t d; uint32_t pad; }; static_assert(sizeof(fastdiv_vals) == 16, "fastdiv_vals size incorrect"); static fastdiv_vals init_fastdiv_values(uint64_t d_64) { GGML_ASSERT(d_64 != 0); GGML_ASSERT(d_64 <= std::numeric_limits::max()); uint32_t d = (uint32_t)d_64; // compute L = ceil(log2(d)); uint32_t L = 0; while (L < 32 && (uint32_t{ 1 } << L) < d) { L++; } uint32_t mp = (uint32_t) ((uint64_t{ 1 } << 32) * ((uint64_t{ 1 } << L) - d) / d + 1); // pack divisor as well to reduce error surface return { mp, L, d, 0 }; } enum GPU_FAMILY { ADRENO, INTEL, UNKNOWN, }; enum ADRENO_GPU_GEN { ADRENO_UNKNOWN, A6X, A7X, A8X, X1E, X2E, }; enum ADRENO_CL_COMPILER_TYPE { E031, E17, DX, }; struct ggml_cl_version { cl_uint major = 0; cl_uint minor = 0; }; struct ggml_cl_compiler_version { ADRENO_CL_COMPILER_TYPE type; int major = -1; int minor = -1; int patch = -1; bool same(ADRENO_CL_COMPILER_TYPE t, int x, int y, int z) const { return major == x && minor == y && patch == z && type == t; } bool newer_than(ADRENO_CL_COMPILER_TYPE t, int x, int y, int z) const { return major*10000 + minor*100 + patch > x*10000 + y*100 + z && type == t; } bool newer_than_or_same(ADRENO_CL_COMPILER_TYPE t, int x, int y, int z) const { return same(t, x, y, z) || newer_than(t, x, y, z); } }; static size_t align_to(size_t value, size_t to_alignment) { GGML_ASSERT(to_alignment && "Invalid alignment (must be non-zero)"); GGML_ASSERT((to_alignment & (to_alignment - 1)) == 0 && "to_alignment must be power-of-two"); return ((value + to_alignment - 1) / to_alignment) * to_alignment; } // Parses a version string of form "XX.YY ". On an error returns ggml_cl_version with all zeroes. static ggml_cl_version parse_cl_version(std::string_view str) { size_t major_str_begin = 0; size_t major_str_end = str.find(".", major_str_begin); if (major_str_end == std::string::npos) { return {}; } size_t minor_str_begin = major_str_end + 1; size_t minor_str_end = str.find(" ", minor_str_begin); if (minor_str_end == std::string::npos) { return {}; } cl_uint version_major; if (std::from_chars(str.data() + major_str_begin, str.data() + major_str_end, version_major).ec != std::errc{}) { return {}; } cl_uint version_minor; if (std::from_chars(str.data() + minor_str_begin, str.data() + minor_str_end, version_minor).ec != std::errc{}) { return {}; } return { version_major, version_minor }; } // Returns OpenCL platform's version. On an error returns ggml_cl_version with all zeroes. static ggml_cl_version get_opencl_platform_version(cl_platform_id platform) { size_t param_size; CL_CHECK(clGetPlatformInfo(platform, CL_PLATFORM_VERSION, 0, nullptr, ¶m_size)); std::unique_ptr param_storage(new char[param_size]); CL_CHECK(clGetPlatformInfo(platform, CL_PLATFORM_VERSION, param_size, param_storage.get(), nullptr)); auto param_value = std::string_view(param_storage.get(), param_size); const std::string version_prefix = "OpenCL "; // Suffix: "XX.YY " if (param_value.find(version_prefix) != 0) { return {}; } param_value.remove_prefix(version_prefix.length()); return parse_cl_version(param_value); } // Returns the DEVICE's OpenCL version. On an error returns ggml_cl_version with all zeroes. static ggml_cl_version get_opencl_device_version(cl_device_id device) { size_t param_size; if (clGetDeviceInfo(device, CL_DEVICE_VERSION, 0, nullptr, ¶m_size) != CL_SUCCESS || !param_size) { return {}; } std::unique_ptr param_storage(new char[param_size]); if (clGetDeviceInfo(device, CL_DEVICE_VERSION, param_size, param_storage.get(), nullptr) != CL_SUCCESS) { return {}; } auto param_value = std::string_view(param_storage.get(), param_size); const std::string version_prefix = "OpenCL "; // "OpenCL . " if (param_value.find(version_prefix) != 0) { return {}; } param_value.remove_prefix(version_prefix.length()); return parse_cl_version(param_value); } // Return a version to use in OpenCL C compilation. On an error returns ggml_cl_version with all zeroes. static ggml_cl_version get_opencl_c_version(ggml_cl_version platform_version, cl_device_id device) { size_t param_size; #if CL_TARGET_OPENCL_VERSION >= 300 // CL_DEVICE_OPENCL_C_ALL_VERSIONS is an OpenCL 3.0 *device* query, so gating it on the // *platform* version is not enough: a 3.0 platform can expose 2.0 devices, where the // query returns CL_INVALID_VALUE and the old CL_CHECK aborted during backend init. // Gate on the device version, and treat a failure as "fall back to the legacy query" // rather than fatal -- a device may advertise 3.0 and still refuse the property. const ggml_cl_version device_version = get_opencl_device_version(device); if (platform_version.major >= 3 && device_version.major >= 3) { cl_int err = clGetDeviceInfo(device, CL_DEVICE_OPENCL_C_ALL_VERSIONS, 0, nullptr, ¶m_size); if (err == CL_SUCCESS && param_size) { std::unique_ptr versions(new cl_name_version[param_size]); err = clGetDeviceInfo(device, CL_DEVICE_OPENCL_C_ALL_VERSIONS, param_size, versions.get(), nullptr); if (err == CL_SUCCESS) { unsigned versions_count = param_size / sizeof(cl_name_version); cl_version version_max = 0; for (unsigned i = 0; i < versions_count; i++) { version_max = std::max(versions[i].version, version_max); } return { CL_VERSION_MAJOR(version_max), CL_VERSION_MINOR(version_max) }; } } // fall through to CL_DEVICE_OPENCL_C_VERSION below } #else GGML_UNUSED(platform_version); #endif // CL_TARGET_OPENCL_VERSION >= 300 if (clGetDeviceInfo(device, CL_DEVICE_OPENCL_C_VERSION, 0, nullptr, ¶m_size) != CL_SUCCESS || !param_size) { return {}; } std::unique_ptr param_storage(new char[param_size]); if (clGetDeviceInfo(device, CL_DEVICE_OPENCL_C_VERSION, param_size, param_storage.get(), nullptr) != CL_SUCCESS) { return {}; } auto param_value = std::string_view(param_storage.get(), param_size); const std::string version_prefix = "OpenCL C "; // Suffix: "XX.YY " if (param_value.find(version_prefix) != 0) { return {}; } param_value.remove_prefix(version_prefix.length()); return parse_cl_version(param_value); } static ADRENO_GPU_GEN get_adreno_gpu_gen(const char *device_name) { if (strstr(device_name, "610") || strstr(device_name, "612") || strstr(device_name, "613") || strstr(device_name, "615") || strstr(device_name, "616") || strstr(device_name, "618") || strstr(device_name, "619") || strstr(device_name, "620") || strstr(device_name, "630") || strstr(device_name, "640") || strstr(device_name, "642") || strstr(device_name, "643") || strstr(device_name, "644") || strstr(device_name, "650") || strstr(device_name, "660") || strstr(device_name, "663") || strstr(device_name, "680") || strstr(device_name, "685") || strstr(device_name, "690")) { return ADRENO_GPU_GEN::A6X; } if (strstr(device_name, "730") || strstr(device_name, "740") || strstr(device_name, "750")) { return ADRENO_GPU_GEN::A7X; } if (strstr(device_name, "810") || strstr(device_name, "830") || strstr(device_name, "840") || strstr(device_name, "850")) { return ADRENO_GPU_GEN::A8X; } if (strstr(device_name, "X1")) { return ADRENO_GPU_GEN::X1E; } if (strstr(device_name, "X2")) { return ADRENO_GPU_GEN::X2E; } return ADRENO_GPU_GEN::ADRENO_UNKNOWN; } static ggml_cl_compiler_version get_adreno_cl_compiler_version(const char *driver_version) { std::string driver_ver_str(driver_version); ADRENO_CL_COMPILER_TYPE type = ADRENO_CL_COMPILER_TYPE::E031; size_t compiler_ver_pos = driver_ver_str.find("E031"); size_t compiler_ver_len = 13; size_t compiler_major_offset = 5; size_t compiler_minor_offset = 8; size_t compiler_patch_offset = 11; if (compiler_ver_pos == std::string::npos) { compiler_ver_pos = driver_ver_str.find("E17"); if (compiler_ver_pos != std::string::npos) { type = ADRENO_CL_COMPILER_TYPE::E17; compiler_ver_len = 12; compiler_major_offset = 4; compiler_minor_offset = 7; compiler_patch_offset = 10; } } if (compiler_ver_pos == std::string::npos) { compiler_ver_pos = driver_ver_str.find("DX"); if (compiler_ver_pos == std::string::npos) { return {}; } type = ADRENO_CL_COMPILER_TYPE::DX; compiler_ver_len = 11; compiler_major_offset = 3; compiler_minor_offset = 6; compiler_patch_offset = 9; } std::string compiler_ver_str = driver_ver_str.substr(compiler_ver_pos, compiler_ver_len); int major = std::atoi(compiler_ver_str.substr(compiler_major_offset, 2).c_str()); int minor = std::atoi(compiler_ver_str.substr(compiler_minor_offset, 2).c_str()); int patch = std::atoi(compiler_ver_str.substr(compiler_patch_offset, 2).c_str()); return { type, major, minor, patch }; } // cl buffer wrapper struct ggml_cl_buffer { cl_mem buffer; size_t size; ggml_cl_buffer() : buffer(nullptr), size(0) {} ~ggml_cl_buffer() { if (buffer) { CL_CHECK(clReleaseMemObject(buffer)); } } void allocate(cl_context context, size_t new_size) { if (new_size > size) { size = new_size; if (buffer) { CL_CHECK(clReleaseMemObject(buffer)); } cl_int err; CL_CHECK((buffer = clCreateBuffer(context, CL_MEM_READ_WRITE, size, NULL, &err), err)); } } }; // Profiling struct ProfilingInfo { std::string op_name; std::string kernel_name; cl_kernel kernel; cl_event evt; cl_ulong cmd_queued; cl_ulong cmd_submit; cl_ulong cmd_start; cl_ulong cmd_end; cl_ulong overhead_start; cl_ulong overhead_end; // For the times below, see spec for clGetEventProfilingInfo // The time kernel spent in cmd queue - SUBMIT - QUEUED cl_ulong cmd_queued_duration_ns; // The time kernel spent for submission - START - SUBMIT cl_ulong cmd_submit_duration_ns; // Kernel execution time in nanoseconds - END - START cl_ulong cmd_duration_ns; // The time for the kernel to complete - COMPLETE - END cl_ulong cmd_complete_duration_ns; // Total time to finish the kernel - COMPLETE - QUEUED cl_ulong cmd_total_duration_ns; // Global and local work sizes. size_t global_size[3]; size_t local_size[3]; // Op output size. size_t output_size[4]; }; static void populateProfilingInfo( ProfilingInfo& info, cl_event evt, cl_kernel kernel, cl_uint work_dim, size_t global_size[3], size_t local_size[3], const ggml_tensor * tensor) { info.op_name = tensor->name; info.kernel = kernel; info.evt = evt; // 0 means not specified, e.g., 2D workgroup, or NULL for driver to choose info.local_size[0] = 0; info.local_size[1] = 0; info.local_size[2] = 0; info.global_size[0] = 0; info.global_size[1] = 0; info.global_size[2] = 0; if (local_size) { for (cl_uint i = 0; i < work_dim; ++i) { info.local_size[i] = local_size[i]; } } for (cl_uint i = 0; i < work_dim; ++i) { info.global_size[i] = global_size[i]; } info.output_size[0] = tensor->ne[0]; info.output_size[1] = tensor->ne[1]; info.output_size[2] = tensor->ne[2]; info.output_size[3] = tensor->ne[3]; } struct ggml_backend_opencl_context; #ifdef GGML_OPENCL_USE_ADRENO_KERNELS static void ggml_cl_adreno_xmem_attn_release_scratch(ggml_backend_opencl_context * backend_ctx); #endif // backend device context struct ggml_backend_opencl_device_context { cl_platform_id platform; std::string platform_name; cl_device_id device; std::string device_name; cl_device_type device_type; std::string device_version; // Initialized by ggml_cl_init(). ggml_backend_opencl_context * backend_ctx = nullptr; // Initialized by ggml_backend_opencl_device_get_buffer_type() ggml_backend_buffer_type buffer_type; cl_context context = nullptr; GPU_FAMILY gpu_family = GPU_FAMILY::UNKNOWN; ADRENO_GPU_GEN adreno_gen = ADRENO_GPU_GEN::ADRENO_UNKNOWN; std::regex *opfilter = nullptr; // regex of ops to not claim std::string opfilter_str = ""; // regex string for opfilter size_t global_mem_size = 0; }; // Lazily-compiled flash-attention kernels and their per-(dk,dv) tile metadata. // One map per (Q/KV dtype, decode/prefill, split) combination; the int maps // hold tile dims (bm/bn), workgroup sizes and the n_kv split thresholds. struct ggml_opencl_fa_kernels { // f16 Q / f16 KV std::map, cl_kernel> f16; std::map, cl_kernel> f16_q1; // f32 Q / f32 KV std::map, cl_kernel> f32; std::map, cl_kernel> f32_q1; // f32 Q / f16 KV (mixed) std::map, cl_kernel> f32_f16; std::map, cl_kernel> f32_f16_split; // N_SPLIT>1 variant std::map, cl_kernel> f32_f16_split_k_img; // DK=512 prefill split, K via image1d_buffer_t std::map, cl_kernel> f32_f16_q1; std::map, cl_kernel> f32_f16_q1_split; // flash-decoding K-split // vec decode std::map, cl_kernel> f32_f16_q1_vec; // kv-head-coalesced vec decode std::map, cl_kernel> f32_f16_q1_vec_mq; // kv-head-coalesced + flash-decoding split std::map, cl_kernel> f32_f16_q1_vec_mq_split; // MQ_GQA=8 specializations std::map, cl_kernel> f32_f16_q1_vec_mq_g8; std::map, cl_kernel> f32_f16_q1_vec_mq_split_g8; // k-image variant of MQ_G8 vec_mq_split std::map, cl_kernel> f32_f16_q1_vec_mq_split_g8_k_img; // k-image variant of MQ_GQA=4 vec_mq_split std::map, cl_kernel> f32_f16_q1_vec_mq_split_k_img; // Cluster-parallel decode std::map, cl_kernel> f32_f16_q1_vec_mq_split_c8; std::map, cl_kernel> f32_f16_q1_vec_mq_split_g8_c8; // NSG_SPLIT=2 specializations (WG=128): the c8 kernel's register footprint // caps its per-kernel WG at 128 on X2, below the stock 256/192 requirement. // 2 subgroups × FA_CL_NCL streams still gives 16 in-flight rows per WG. std::map, cl_kernel> f32_f16_q1_vec_mq_split_c8_ns2; std::map, cl_kernel> f32_f16_q1_vec_mq_split_g8_c8_ns2; // FA_CL_C=32 / MQ_GQA=8 / NSG_SPLIT=2 specialization for the DK=DV=256 // GQA=8 class (Qwen3.5/3.6-35B-A3B: 16 Q heads, 2 KV heads). o_acc = // DV_VEC/32 × 8 = 128B/lane (in budget); the baseline fa1 path for this // shape has NO MQ/FD at all and pays an 8× KV re-read per Q head. std::map, cl_kernel> f32_f16_q1_vec_mq_split_g8_c32; // alternative decode std::map, cl_kernel> f32_f16_q1_local_tile; // hybrid local-tile + MQ + FD-split kernel for DK=DV=128 only std::map, cl_kernel> f32_f16_q1_local_mq_split; std::map, cl_kernel> f32_f16_q1_local_mq_split_g8; std::map, int> f32_f16_bm; std::map, int> f32_f16_bn; std::map, int> f32_f16_wg_size; std::map, int> f32_f16_split_wg_size; std::map, int> f32_f16_split_nkv_threshold; // f32 Q / native q8_0 KV std::map, cl_kernel> f32_q8_0_q1; // decode std::map, cl_kernel> f32_q8_0_q1_vec; // DV-split + multi-subgroup decode std::map, cl_kernel> f32_q8_0_q1_split; // flash-decoding pass 1 // KV-head-coalesced + flash-decoding split for q8_0 KV std::map, cl_kernel> f32_q8_0_q1_vec_mq_split; std::map, cl_kernel> f32_q8_0_q1_vec_mq_split_g8; // Cluster-parallel q8_0 decode std::map, cl_kernel> f32_q8_0_q1_vec_mq_split_c8; std::map, cl_kernel> f32_q8_0; // prefill (baseline) std::map, cl_kernel> f32_q8_0_split; // N_SPLIT>1 variant std::map, int> f32_q8_0_split_wg_size; // wg_size = bm*n_split std::map, int> f32_q8_0_split_nkv_threshold; // use split when n_kv >= this std::map, int> f32_q8_0_split_bm; // per-split BLOCK_M // f32 Q / native q4_0 KV std::map, cl_kernel> f32_q4_0_q1; std::map, cl_kernel> f32_q4_0_q1_vec; // DV-split + multi-subgroup decode std::map, cl_kernel> f32_q4_0_q1_split; // kv-head-coalesced + flash-decoding split for q4_0 kv (dp4a K dot) std::map, cl_kernel> f32_q4_0_q1_vec_mq_split; std::map, cl_kernel> f32_q4_0_q1_vec_mq_split_g8; // Cluster-parallel q4_0 decode std::map, cl_kernel> f32_q4_0_q1_vec_mq_split_g8_c8; std::map, cl_kernel> f32_q4_0_q1_vec_mq_split_c8; std::map, cl_kernel> f32_q4_0; std::map, cl_kernel> f32_q4_0_split; std::map, int> f32_q4_0_split_wg_size; std::map, int> f32_q4_0_split_nkv_threshold; std::map, int> f32_q4_0_split_bm; // shared: flash-decoding merge + prefill prepass (kv-pad, mask-pad, blk class) std::map, cl_kernel> f32_merge; std::map, cl_kernel> kv_pad_f16; std::map, cl_kernel> mask_pad_f16; std::map, cl_kernel> blk_f16; // generic prefill tile dims (f16 / f32 paths) std::map, int> bm; std::map, int> bn; // attempted (variant, (dk, dv)) // all attempted FA kernels appear here, but those not registered failed compilation std::set>> variant_attempted; // FA bin kernels #ifdef GGML_OPENCL_USE_ADRENO_KERNELS cl_kernel kernel_flash_attn_f32_f16_bin; cl_kernel kernel_repack_q_for_wmm; cl_kernel kernel_repack_k_for_wmm; cl_kernel kernel_repack_v_for_wmm; cl_kernel kernel_repack_mask_for_wmm; #endif }; #ifdef GGML_OPENCL_USE_ADRENO_KERNELS struct ggml_cl_adreno_xmem_attn_scratch { cl_mem q_img = nullptr; cl_mem k_img = nullptr; cl_mem v_img = nullptr; cl_mem out_img = nullptr; cl_mem k_transpose_buf = nullptr; cl_mem k_transpose_img1d = nullptr; cl_mem k_packed_buf = nullptr; cl_mem v_packed_buf = nullptr; cl_mem score_buf = nullptr; cl_mem prob_buf = nullptr; cl_mem score_img1d = nullptr; cl_mem prob_img1d = nullptr; cl_mem softmax_stats_img2d = nullptr; cl_mem xmem_qk = nullptr; cl_mem xmem_pv = nullptr; int n_q = 0; int n_kv = 0; int n_kv_padded = 0; int d_head_q = 0; int d_head_v = 0; int q_width = 0; int kv_heads_total = 0; }; struct ggml_cl_adreno_xmem_attn_state { bool compiled = false; bool logged = false; cl_kernel kernel_q_f32_to_img_scaled = nullptr; cl_kernel kernel_kv_f32_to_img_gqa = nullptr; cl_kernel kernel_kv_f16_to_img_gqa = nullptr; cl_kernel kernel_img_to_f32 = nullptr; cl_kernel kernel_k_gather = nullptr; cl_kernel kernel_pack_k = nullptr; cl_kernel kernel_qk_gemm = nullptr; cl_kernel kernel_softmax_reduce_basic = nullptr; cl_kernel kernel_softmax_apply_basic = nullptr; cl_kernel kernel_mask_scores = nullptr; cl_kernel kernel_pack_v = nullptr; cl_kernel kernel_pv_gemm = nullptr; ggml_cl_adreno_xmem_attn_scratch scratch; }; #endif // backend context struct ggml_backend_opencl_context { int ref_count; cl_device_id device; std::string device_name; ggml_cl_version platform_version; ggml_cl_version opencl_c_version; // argsort is loaded in supports_op because its availability depends on how // many workgroups are allowed, which requires kernel compilation. bool kernels_loaded_argsort = false; // rest of the kernels are currently always loaded in alloc_buffer. bool kernels_loaded = false; std::string driver_version; GPU_FAMILY gpu_family; ADRENO_GPU_GEN adreno_gen; cl_int alignment; size_t global_mem_size; size_t max_alloc_size; size_t max_workgroup_size; bool fp16_support; bool has_vector_subgroup_broadcast; bool has_subgroup_shuffle = false; // cl_khr_subgroup_shuffle or cl_qcom_subgroup_shuffle bool has_integer_dot = false; // cl_khr_integer_dot_product or cl_qcom_dot_product8 bool has_qcom_subgroup_shuffle = false; // specifically cl_qcom_subgroup_shuffle bool disable_fusion; bool fuse_mm_glu = true; // opt-out GGML_OPENCL_FUSE_MM_GLU=0 (byte-identical gate+up GEMV + GLU, q4_K FFN) bool fuse_rms_add = true; // opt-out GGML_OPENCL_FUSE_RMS_ADD=0 (fused rms_norm*w + residual) bool f16_mrow = true; // opt-out GGML_OPENCL_F16_MROW=0 (multi-row-per-WG f16 decode GEMV for attn proj + lm_head) int f16_mrow_rpt = 1; // GGML_OPENCL_F16_MROW_RPT={1,2,4,8,16} rows-per-subgroup register blocking // ragged moe, use int to directly pass to kernel cl_uint adreno_use_moe_ragged; cl_uint adreno_moe_ragged_skip_gran; cl_uint adreno_use_moe_ragged_dp4; // whether fuse moe combine cl_uint fuse_moe_combine; // whether to fold the MoE bias adds into swiglu_oai cl_uint fuse_moe_bias_glu; // whether to fold the MoE down-projection bias add into the combine cl_uint fuse_moe_bias_combine; bool adreno_has_large_buffer; bool adreno_use_large_buffer; bool adreno_use_bin_kernels; get_adreno_bin_kernel_func_t get_adreno_bin_kernel_func = nullptr; ggml_cl_compiler_version adreno_cl_compiler_version; // The q6_K flat mul_mat codegen workarounds are needed by old E031 compilers only. bool q6_k_flat_old_compiler; std::string kernel_compile_opts; // cached for lazy-compiled kernels. int adreno_wave_size; cl_bool non_uniform_workgroups; size_t image_max_buffer_size; size_t image2d_max_width; size_t image2d_max_height; cl_device_svm_capabilities svm_caps; cl_context context; cl_command_queue queue; // On-disk compiled-program cache (see GGML_OPENCL_KERNEL_CACHE_DIR). cl_program_cache_state program_cache; bool program_cache_initialized = false; // prealloc buffers for transposing weights and activations ggml_cl_buffer prealloc_quant_trans; ggml_cl_buffer prealloc_scales_trans; ggml_cl_buffer prealloc_act_trans; // q8_1-quantized reordered MoE activations for the dp4a prefill GEMM. ggml_cl_buffer prealloc_moe_qa; // int8 quants [tok_slots * ne00] ggml_cl_buffer prealloc_moe_da; // per-block d [tok_slots * ne00/32] (half) ggml_cl_buffer prealloc_moe_sa; // per-block s [tok_slots * ne00/32] (half) // scratch copy of the router weights to avoid dst aliasing ggml_cl_buffer prealloc_moe_combine_w; ggml_cl_buffer prealloc_splitk_partial; // [ksplit * M] partials for split-K GEMV // pool of persistent image1d_buffer views over kv-cache layers, keyed by // (parent buffer, offset within parent) // used by the img-variant KQ/KQV dispatch paths to avoid per-call // clCreateSubBuffer + clCreateImage + pending-release-queue on long-context decode struct ImagePoolKey { uintptr_t buf; uint64_t offset; bool operator<(const ImagePoolKey & o) const { if (buf != o.buf) return buf < o.buf; return offset < o.offset; } }; struct ImagePoolEntry { cl_mem sub_buffer = nullptr; cl_mem image = nullptr; size_t k_bytes = 0; cl_channel_type channel_data_type = CL_FLOAT; }; std::map kq_img_pool; std::map kqv_img_pool; // pool for the on-device f16 buffer for kv-cache with non-FA quantized-K (q8_0/q4_0) std::map dequant_f16_pool; // prealloc buffers for src0 and src1 ggml_cl_buffer prealloc_src0; ggml_cl_buffer prealloc_src1; #ifdef GGML_OPENCL_USE_ADRENO_KERNELS ggml_cl_buffer prealloc_adreno_xmem_const; bool adreno_xmem_gemm_enabled = false; #endif // prealloc buffers for MoE router table preprocess bool toggle_reorder = false; ggml_cl_buffer prealloc_post_router; ggml_cl_buffer prealloc_emap; ggml_cl_buffer prealloc_hist; ggml_cl_buffer prealloc_tile_offset; ggml_cl_buffer prealloc_total_tiles; ggml_cl_buffer prealloc_slot_counter; cl_program program_add; cl_program program_add_id; cl_program program_moe_add_id_glu; cl_program program_clamp; cl_program program_cvt; cl_program program_diag_mask_inf; cl_program program_gelu; cl_program program_gemv_noshuffle_general; cl_program program_gemv_noshuffle; cl_program program_get_rows; cl_program program_set_rows; cl_program program_glu; cl_program program_im2col_f16; cl_program program_im2col_f32; cl_program program_mul_mat_Ab_Bi_8x4; cl_program program_mul_mv_q4_0_f32; cl_program program_mul_mv_q4_0_f32_v; cl_program program_mul_mv_q4_0_f32_8x_flat; cl_program program_mul_mv_q4_0_f32_1d_8x_flat; cl_program program_mul_mv_q4_0_f32_1d_16x_flat; cl_program program_mul_mv_q6_K; cl_program program_mul_mv_q8_0_f32, program_mul_mv_q8_0_f32_flat; cl_program program_mul_mv_mxfp4_f32; cl_program program_mul_mv_mxfp4_f32_flat; cl_program program_mul_mv_f16_f16; cl_program program_mul_mv_f16_f32_1row; cl_program program_mul_mv_f16_f32_l4; cl_program program_mul_mv_f16_f32; cl_program program_mul_mv_f32_f32; cl_program program_mul; cl_program program_mul_mat_f16_f32_tiled; cl_program program_mul_mm_f16_f32_kqv; cl_program program_mul_mm_f16_f32_kq; cl_program program_div; cl_program program_sub; cl_program program_norm; cl_program program_relu; cl_program program_rms_norm; cl_program program_group_norm; cl_program program_rope; cl_program program_silu; cl_program program_sigmoid; cl_program program_softmax_f32; cl_program program_softmax_f16; cl_program program_softmax_4_f32; cl_program program_softmax_4_f16; cl_program program_argsort_f32_i32; cl_program program_sum_rows_f32; cl_program program_pad; cl_program program_upscale; cl_program program_conv_2d_f16; cl_program program_conv_2d_f32; cl_program program_conv_2d_f16_f32; cl_program program_tsembd; cl_program program_gemv_moe_mxfp4_f32, program_gemm_moe_mxfp4_f32; cl_program program_mul_mv_id_q4_0_f32_8x_flat; cl_program program_mul_mv_id_q8_0_f32, program_mul_mv_id_q8_0_f32_flat; cl_program program_mul_mv_id_mxfp4_f32; cl_program program_mul_mv_id_mxfp4_f32_flat; cl_program program_mul_mm_f32_f32_l4_lm; cl_program program_mul_mm_f16_f32_l4_lm; cl_program program_mul_mm_q8_0_f32_l4_lm; cl_kernel kernel_add, kernel_add_row, kernel_add_f16, kernel_add_row_f16; cl_kernel kernel_mul, kernel_mul_row, kernel_mul_f16, kernel_mul_row_f16; cl_kernel kernel_div, kernel_div_row, kernel_div_f16, kernel_div_row_f16; cl_kernel kernel_sub, kernel_sub_row, kernel_sub_f16, kernel_sub_row_f16; cl_kernel kernel_add_id; cl_kernel kernel_add_id_add_id_swiglu_oai; cl_kernel kernel_scale_f32, kernel_scale_f32_4; cl_kernel kernel_sqr_cont_f32, kernel_sqr_cont_f32_4, kernel_sqr_cont_f16, kernel_sqr_cont_f16_4; cl_kernel kernel_sqrt_cont_f32, kernel_sqrt_cont_f32_4, kernel_sqrt_cont_f16, kernel_sqrt_cont_f16_4; cl_kernel kernel_mean_f32, kernel_mean_f32_4; cl_kernel kernel_silu, kernel_silu_4; cl_kernel kernel_gelu, kernel_gelu_4; cl_kernel kernel_gelu_erf, kernel_gelu_erf_4; cl_kernel kernel_gelu_quick, kernel_gelu_quick_4; cl_kernel kernel_relu; cl_kernel kernel_sigmoid_f32, kernel_sigmoid_f16; cl_kernel kernel_tri; cl_kernel kernel_fill; cl_kernel kernel_clamp; cl_kernel kernel_geglu, kernel_reglu, kernel_swiglu, kernel_swiglu_oai, kernel_swiglu_clamp, kernel_geglu_erf, kernel_geglu_quick, kernel_geglu_f16, kernel_reglu_f16, kernel_swiglu_f16, kernel_swiglu_clamp_f16, kernel_geglu_erf_f16, kernel_geglu_quick_f16; cl_kernel kernel_norm, kernel_norm_mul_add; cl_kernel kernel_rms_norm, kernel_rms_norm_mul; cl_kernel kernel_rms_norm_mul_add = nullptr; // fused rms_norm(x)*w + b (residual) cl_kernel kernel_l2_norm_f32; cl_kernel kernel_group_norm, kernel_group_norm_mul_add; cl_kernel kernel_diag_mask_inf, kernel_diag_mask_inf_8; cl_kernel kernel_diag_f32; cl_kernel kernel_soft_max, kernel_soft_max_4; cl_kernel kernel_soft_max_f16, kernel_soft_max_4_f16; ggml_opencl_fa_kernels fa; #ifdef GGML_OPENCL_USE_ADRENO_KERNELS ggml_cl_adreno_xmem_attn_state adreno_xmem_attn; #endif cl_kernel kernel_get_rows_f32, kernel_get_rows_f16, kernel_get_rows_q4_0; cl_kernel kernel_set_rows_f32_i64, kernel_set_rows_f32_i32, kernel_set_rows_f16_i64, kernel_set_rows_f16_i32; cl_kernel kernel_set_rows_q8_0_i64, kernel_set_rows_q8_0_i32; cl_kernel kernel_set_rows_q8_0_soa_i64, kernel_set_rows_q8_0_soa_i32; cl_kernel kernel_set_rows_q4_0_i64, kernel_set_rows_q4_0_i32; cl_kernel kernel_set_rows_q4_0_soa_i64, kernel_set_rows_q4_0_soa_i32; cl_kernel kernel_rope_norm_f32, kernel_rope_norm_f16, kernel_rope_neox_f32, kernel_rope_neox_f16; cl_kernel kernel_rope_multi_f32, kernel_rope_multi_f16, kernel_rope_vision_f32, kernel_rope_vision_f16; cl_kernel kernel_cpy_f16_f16, kernel_cpy_f16_f32, kernel_cpy_f32_f16, kernel_cpy_f32_f32, kernel_cpy_f32_f32_pack, kernel_cpy_i32_i32; cl_kernel kernel_cpy_f32_f32_flat = nullptr; cl_kernel kernel_mul_mat_f32_f32; cl_kernel kernel_mul_mat_f16_f16; cl_kernel kernel_mul_mat_f16_f32_1row; cl_program program_mul_mv_f16_f32_mrow; cl_kernel kernel_mul_mat_f16_f32_mrow = nullptr; // multi-row decode GEMV (attn proj + lm_head) cl_kernel kernel_mul_mat_f16_f32_mrow_r2 = nullptr; cl_kernel kernel_mul_mat_f16_f32_mrow_r4 = nullptr; cl_kernel kernel_mul_mat_f16_f32_mrow_h8 = nullptr; cl_kernel kernel_mul_mat_f16_f32_mrow_h8r2 = nullptr; cl_kernel kernel_mul_mat_f16_f32; cl_kernel kernel_mul_mat_f16_f32_l4; cl_kernel kernel_mul_mat_f16_f32_l4_dr; cl_kernel kernel_mul_mat_f16_f32_l4_dr_ls; cl_kernel kernel_mul_mat_f16_f32_l4_dr_lq; cl_kernel kernel_mul_mat_f16_f32_l4_x8 = nullptr; cl_kernel kernel_mul_mat_f16_f32_l4_x8_pair = nullptr; cl_kernel kernel_mul_mat_f16_f32_l4_x8_gqa4 = nullptr; cl_kernel kernel_mul_mat_f16_f32_l4_x8_gqa4_img = nullptr; cl_kernel kernel_mul_mat_f16_f32_l4_x8_gqa_r4_img = nullptr; cl_kernel kernel_mul_mat_f16_f32_l4_x8_gqa_r2_dk256_img = nullptr; cl_kernel kernel_mul_mat_f16_f32_l4_y8 = nullptr; cl_kernel kernel_mul_mat_f16_f32_l4_y8_gqa = nullptr; cl_kernel kernel_mul_mat_f16_f32_l4_y8_gqa_img = nullptr; cl_kernel kernel_mul_mat_f16_f32_tiled; cl_kernel kernel_adreno_xmem_pack_src_f32; cl_kernel kernel_adreno_xmem_prepack_weight_f16; cl_kernel kernel_gemm_xmem_f16_f32_os8; cl_kernel kernel_adreno_xmem_store_dst_f32; cl_kernel kernel_mul_mm_f16_f32_kqv; cl_kernel kernel_mul_mm_f16_f32_kq; cl_kernel kernel_mul_mat_q4_0_f32, kernel_mul_mat_q4_0_f32_v; cl_kernel kernel_convert_block_q1_0, kernel_restore_block_q1_0; cl_kernel kernel_convert_block_q4_0, kernel_restore_block_q4_0; cl_kernel kernel_convert_block_q4_0_trans4_ns, kernel_restore_block_q4_0_trans4_ns; cl_kernel kernel_convert_block_q4_1, kernel_restore_block_q4_1; cl_kernel kernel_convert_block_q4_1_trans4_ns, kernel_restore_block_q4_1_trans4_ns; cl_kernel kernel_convert_block_q5_0, kernel_restore_block_q5_0; cl_kernel kernel_convert_block_q5_0_trans4_ns, kernel_restore_block_q5_0_trans4_ns; cl_kernel kernel_convert_block_q5_1, kernel_restore_block_q5_1; cl_kernel kernel_convert_block_q5_1_trans4_ns, kernel_restore_block_q5_1_trans4_ns; cl_kernel kernel_convert_block_q4_k_trans4_ns, kernel_restore_block_q4_k_trans4_ns; cl_kernel kernel_convert_block_q5_k_trans4_ns, kernel_restore_block_q5_k_trans4_ns; cl_kernel kernel_convert_block_q6_k_trans4_ns, kernel_restore_block_q6_k_trans4_ns; cl_kernel kernel_convert_block_mxfp4, kernel_convert_block_mxfp4_trans, kernel_restore_block_mxfp4, kernel_restore_block_mxfp4_trans; cl_kernel kernel_convert_block_mxfp4_trans4_ns, kernel_restore_block_mxfp4_trans4_ns; cl_kernel kernel_convert_block_q8_0, kernel_restore_block_q8_0, kernel_restore_block_q8_0_trans; cl_kernel kernel_dequant_q8_0_f16_view_aos; cl_kernel kernel_dequant_q8_0_f32_view_aos; cl_kernel kernel_dequant_q4_0_f16_view_aos; cl_kernel kernel_dequant_q4_0_f32_view_aos; cl_kernel kernel_convert_block_q6_K_noshuffle, kernel_restore_block_q6_K_noshuffle; cl_kernel kernel_convert_bf16_to_f16, kernel_convert_f16_to_bf16; cl_kernel kernel_mul_mat_q4_0_f32_8x_flat; cl_kernel kernel_convert_block_q4_0_noshuffle; cl_kernel kernel_restore_block_q4_0_noshuffle; cl_kernel kernel_convert_block_q4_1_noshuffle; cl_kernel kernel_restore_block_q4_1_noshuffle; cl_kernel kernel_convert_block_q5_0_noshuffle; cl_kernel kernel_restore_block_q5_0_noshuffle; cl_kernel kernel_convert_block_q5_1_noshuffle; cl_kernel kernel_restore_block_q5_1_noshuffle; cl_kernel kernel_convert_block_q4_K_noshuffle; cl_kernel kernel_restore_block_q4_K_noshuffle; cl_kernel kernel_convert_block_q4_K, kernel_restore_block_q4_K; cl_kernel kernel_convert_block_q5_K, kernel_restore_block_q5_K; cl_kernel kernel_convert_block_q5_K_noshuffle; cl_kernel kernel_restore_block_q5_K_noshuffle; cl_kernel kernel_convert_block_q6_K, kernel_restore_block_q6_K; cl_kernel kernel_convert_block_iq4_nl, kernel_restore_block_iq4_nl; cl_kernel kernel_convert_block_iq4_nl_noshuffle; cl_kernel kernel_restore_block_iq4_nl_noshuffle; cl_kernel kernel_mul_mv_q1_0_f32, kernel_mul_mv_q1_0_f32_flat; cl_kernel kernel_mul_mat_q4_0_f32_1d_8x_flat, kernel_mul_mat_q4_0_f32_1d_16x_flat; cl_kernel kernel_mul_mv_q4_1_f32; cl_kernel kernel_mul_mv_q4_1_f32_flat; cl_kernel kernel_mul_mv_q5_0_f32; cl_kernel kernel_mul_mv_q5_0_f32_flat; cl_kernel kernel_mul_mv_q5_1_f32; cl_kernel kernel_mul_mv_q5_1_f32_flat; cl_kernel kernel_mul_mv_q4_K_f32; cl_kernel kernel_mul_mv_q4_K_f32_flat; cl_kernel kernel_mul_mv_q5_K_f32; cl_kernel kernel_mul_mv_q5_K_f32_flat; cl_kernel kernel_mul_mv_q6_K_f32; cl_kernel kernel_mul_mv_q6_K_f32_flat; cl_kernel kernel_mul_mv_mxfp4_f32, kernel_mul_mv_mxfp4_f32_flat; cl_kernel kernel_mul_mv_q8_0_f32, kernel_mul_mv_q8_0_f32_flat; cl_kernel kernel_mul_mv_iq4_nl_f32; cl_kernel kernel_mul_mv_iq4_nl_f32_flat; cl_kernel kernel_solve_tri_f32; cl_kernel kernel_im2col_f32, kernel_im2col_f16; cl_kernel kernel_argsort_f32_i32; cl_kernel kernel_sum_rows_f32, kernel_sum_rows_f32_4; cl_kernel kernel_cumsum_blk, kernel_cumsum_add; cl_kernel kernel_repeat_f32; cl_kernel kernel_pad; cl_kernel kernel_tanh_f32, kernel_tanh_f32_4, kernel_tanh_f32_nc; cl_kernel kernel_tanh_f16, kernel_tanh_f16_4, kernel_tanh_f16_nc; cl_kernel kernel_neg_f32, kernel_neg_f32_4, kernel_neg_f32_nc; cl_kernel kernel_neg_f16, kernel_neg_f16_4, kernel_neg_f16_nc; cl_kernel kernel_exp_f32, kernel_exp_f32_4, kernel_exp_f32_nc; cl_kernel kernel_exp_f16, kernel_exp_f16_4, kernel_exp_f16_nc; cl_kernel kernel_expm1_f32, kernel_expm1_f32_4, kernel_expm1_f32_nc; cl_kernel kernel_expm1_f16, kernel_expm1_f16_4, kernel_expm1_f16_nc; cl_kernel kernel_abs_f32, kernel_abs_f32_4, kernel_abs_f32_nc; cl_kernel kernel_abs_f16, kernel_abs_f16_4, kernel_abs_f16_nc; cl_kernel kernel_sgn_f32, kernel_sgn_f32_4, kernel_sgn_f32_nc, kernel_sgn_f16, kernel_sgn_f16_4, kernel_sgn_f16_nc; cl_kernel kernel_step_f32, kernel_step_f32_4, kernel_step_f32_nc, kernel_step_f16, kernel_step_f16_4, kernel_step_f16_nc; cl_kernel kernel_elu_f32, kernel_elu_f32_4, kernel_elu_f32_nc, kernel_elu_f16, kernel_elu_f16_4, kernel_elu_f16_nc; cl_kernel kernel_hardswish_f32, kernel_hardswish_f32_4, kernel_hardswish_f32_nc, kernel_hardswish_f16, kernel_hardswish_f16_4, kernel_hardswish_f16_nc; cl_kernel kernel_hardsigmoid_f32, kernel_hardsigmoid_f32_4, kernel_hardsigmoid_f32_nc, kernel_hardsigmoid_f16, kernel_hardsigmoid_f16_4, kernel_hardsigmoid_f16_nc; cl_kernel kernel_floor_f32, kernel_floor_f32_4, kernel_floor_f32_nc, kernel_floor_f16, kernel_floor_f16_4, kernel_floor_f16_nc; cl_kernel kernel_ceil_f32, kernel_ceil_f32_4, kernel_ceil_f32_nc, kernel_ceil_f16, kernel_ceil_f16_4, kernel_ceil_f16_nc; cl_kernel kernel_round_f32, kernel_round_f32_4, kernel_round_f32_nc, kernel_round_f16, kernel_round_f16_4, kernel_round_f16_nc; cl_kernel kernel_trunc_f32, kernel_trunc_f32_4, kernel_trunc_f32_nc, kernel_trunc_f16, kernel_trunc_f16_4, kernel_trunc_f16_nc; cl_kernel kernel_softplus_f32, kernel_softplus_f32_4, kernel_softplus_f32_nc; cl_kernel kernel_softplus_f16, kernel_softplus_f16_4, kernel_softplus_f16_nc; cl_kernel kernel_upscale; cl_kernel kernel_upscale_bilinear; cl_kernel kernel_concat_b1, kernel_concat_b2, kernel_concat_b4, kernel_concat_b8, kernel_concat_b4_pack; cl_kernel kernel_conv_2d_f16; cl_kernel kernel_conv_2d_f32; cl_kernel kernel_conv_2d_f16_f32; cl_kernel kernel_ssm_conv_f32_f32, kernel_ssm_conv_f32_f32_4; // [size_idx][kda][tgpp] where size_idx: 0=S_V=16, 1=32, 2=64, 3=128; kda: 0 or 1. // tgpp 0 = TG variant (COLS_PER_LANE_GROUP=1), tgpp 1 = prefill variant (COLS_PER_LANE_GROUP=4). cl_kernel kernel_gated_delta_net_f32[4][2][2] = {}; cl_kernel kernel_ssm_scan_f32 = nullptr; cl_kernel kernel_ssm_scan_f32_mamba2_d128 = nullptr; cl_kernel kernel_ssm_scan_f32_mamba2_d256 = nullptr; cl_kernel kernel_timestep_embedding; cl_kernel kernel_gemv_moe_q4_0_f32_ns, kernel_gemm_moe_q4_0_f32_ns, kernel_gemm_moe_q4_0_f32_ns_bin; cl_kernel kernel_gemm_moe_q8_0_f32_ns; cl_kernel kernel_gemv_moe_q4_1_f32_ns, kernel_gemm_moe_q4_1_f32_ns, kernel_gemm_moe_q4_1_f32_ns_bin; cl_kernel kernel_gemv_moe_q5_0_f32_ns, kernel_gemm_moe_q5_0_f32_ns; cl_kernel kernel_gemv_moe_q5_1_f32_ns, kernel_gemm_moe_q5_1_f32_ns; cl_kernel kernel_gemv_moe_q4_k_f32_ns, kernel_gemm_moe_q4_k_f32_ns, kernel_gemm_moe_q4_k_f32_ns_bin; cl_kernel kernel_gemv_moe_q4_k_f32_ns_wimg = nullptr; // weight-as-texture MoE decode GEMV (opt-in) cl_kernel kernel_gemm_moe_q4_k_q8_1_dp4a = nullptr; // dp4a (int8) prefill GEMM variant cl_kernel kernel_moe_reorder_quant_a_q8_1; // fused reorder + q8_1 quant for the dp4a GEMM cl_kernel kernel_gemm_moe_q8_1_dp4a_q80 = nullptr; // generic dp4a MoE GEMM (MOE_QT=80), opt-in cl_kernel kernel_moe_expand_scale_q8_0 = nullptr; // q8_0 per-block d -> uniform scale[16] cl_kernel kernel_gemm_moe_q8_1_dp4a_q50 = nullptr; // generic dp4a MoE GEMM (MOE_QT=50, q5_0), opt-in cl_kernel kernel_moe_expand_scale_q5_0 = nullptr; // q5_0 d -> uniform scale[2]/min[1] per 32-block cl_kernel kernel_gemm_moe_q8_1_dp4a_q5k = nullptr; // generic dp4a MoE GEMM (MOE_QT=5, q5_K), opt-in cl_kernel kernel_moe_expand_scale_q5_K = nullptr; // q5_K 6-bit s[] -> uniform scale[16]/min[8] cl_kernel kernel_gemv_moe_q5_k_f32_ns, kernel_gemm_moe_q5_k_f32_ns; cl_kernel kernel_gemv_moe_q6_k_f32_ns, kernel_gemm_moe_q6_k_f32_ns, kernel_gemm_moe_q6_k_f32_ns_bin; cl_kernel kernel_gemm_moe_q6_k_q8_1_dp4a = nullptr; // dp4a (int8) q6_K MoE prefill GEMM cl_kernel kernel_gemv_moe_mxfp4_f32, kernel_gemm_moe_mxfp4_f32; cl_kernel kernel_gemv_moe_mxfp4_f32_ns, kernel_gemm_moe_mxfp4_f32_ns, kernel_gemm_moe_mxfp4_f32_ns_bin; cl_kernel kernel_gemv_moe_mxfp4_f32_ns_wimg = nullptr; // weight-as-texture MoE decode GEMV cl_kernel kernel_gemm_moe_mxfp4_q8_1_dp4a = nullptr; // dp4a (int8) mxfp4 MoE prefill GEMM cl_kernel kernel_gemm_moe_q4_0_q8_1_dp4a = nullptr; // dp4a (int8) q4_0 MoE prefill GEMM cl_kernel kernel_gemm_moe_mxfp4_q8_1_dp4a_bin = nullptr; // binary dp4a (int8) mxfp4 MoE prefill GEMM cl_kernel kernel_gemm_moe_q4_0_q8_1_dp4a_bin = nullptr; // binary dp4a (int8) q4_0 MoE prefill GEMM cl_kernel kernel_moe_reorder_b; cl_kernel kernel_moe_histogram, kernel_moe_scan, kernel_moe_fill, kernel_moe_scatter; cl_kernel kernel_moe_scatter_stable = nullptr; // deterministic slot assignment cl_kernel kernel_moe_combine_f32 = nullptr; // fused router-weight mul + cross-expert sum cl_kernel kernel_moe_combine_bias_f32 = nullptr; // same, with the down-projection bias add folded in cl_kernel kernel_mul_mv_id_q4_0_f32_8x_flat; cl_kernel kernel_mul_mv_id_q8_0_f32, kernel_mul_mv_id_q8_0_f32_flat; cl_kernel kernel_mul_mv_id_mxfp4_f32; cl_kernel kernel_mul_mv_id_mxfp4_f32_flat; cl_kernel kernel_mul_mm_f32_f32_l4_lm; cl_kernel kernel_gemv_f32_f32_mc; // multi-column (small-N) f32 GEMV for spec/MTP verify cl_kernel kernel_mul_mm_f16_f32_l4_lm; cl_kernel kernel_mul_mm_q1_0_f32_l4_lm; cl_kernel kernel_mul_mm_q4_0_f32_l4_lm; cl_kernel kernel_mul_mm_q4_1_f32_l4_lm; cl_kernel kernel_mul_mm_q5_0_f32_l4_lm; cl_kernel kernel_mul_mm_q5_1_f32_l4_lm; cl_kernel kernel_mul_mm_q8_0_f32_l4_lm; cl_kernel kernel_mul_mm_q4_k_f32_l4_lm; cl_kernel kernel_mul_mm_q5_k_f32_l4_lm; cl_kernel kernel_mul_mm_q6_k_f32_l4_lm; cl_kernel kernel_mul_mm_iq4_nl_f32_l4_lm; std::vector profiling_info; std::vector profiling_results; void flush_profiling_batch() { if (profiling_info.empty()) { return; } // Populate profiling info for (ProfilingInfo & info : profiling_info) { cl_ulong cmd_queued; cl_ulong cmd_submit; cl_ulong cmd_start; cl_ulong cmd_end; cl_ulong cmd_complete; CL_CHECK(clWaitForEvents(1, &info.evt)); CL_CHECK(clGetEventProfilingInfo( info.evt, CL_PROFILING_COMMAND_QUEUED, sizeof(cl_ulong), &cmd_queued, NULL)); CL_CHECK(clGetEventProfilingInfo( info.evt, CL_PROFILING_COMMAND_SUBMIT, sizeof(cl_ulong), &cmd_submit, NULL)); CL_CHECK(clGetEventProfilingInfo( info.evt, CL_PROFILING_COMMAND_START, sizeof(cl_ulong), &cmd_start, NULL)); CL_CHECK(clGetEventProfilingInfo( info.evt, CL_PROFILING_COMMAND_END, sizeof(cl_ulong), &cmd_end, NULL)); CL_CHECK(clGetEventProfilingInfo( info.evt, CL_PROFILING_COMMAND_COMPLETE, sizeof(cl_ulong), &cmd_complete, NULL)); CL_CHECK(clReleaseEvent(info.evt)); info.evt = nullptr; char kernel_name[512]; CL_CHECK(clGetKernelInfo(info.kernel, CL_KERNEL_FUNCTION_NAME, sizeof(kernel_name), kernel_name, NULL)); info.kernel_name = kernel_name; info.cmd_queued = cmd_queued; info.cmd_submit = cmd_submit; info.cmd_start = cmd_start; info.cmd_end = cmd_end; info.cmd_queued_duration_ns = cmd_submit - cmd_queued; info.cmd_submit_duration_ns = cmd_start - cmd_submit; info.cmd_duration_ns = cmd_end - cmd_start; info.cmd_complete_duration_ns = cmd_complete - cmd_end; info.cmd_total_duration_ns = cmd_complete - cmd_queued; } profiling_results.insert(profiling_results.end(), std::make_move_iterator(profiling_info.begin()), std::make_move_iterator(profiling_info.end())); profiling_info.clear(); } void write_profiling_info() { if (profiling_results.empty()) { return; } // Dump a csv FILE * fperf = fopen("cl_profiling.csv", "w"); if (!fperf) { GGML_LOG_ERROR("Failed to open cl_profiling.csv\n"); return; } fprintf(fperf, "op name, kernel name, exec duration (ms), global size, local size, output size\n"); for (const ProfilingInfo & info : profiling_results) { fprintf(fperf, "%s,%s,%f,%zux%zux%zu,%zux%zux%zu,%zux%zux%zux%zu\n", info.op_name.c_str(), info.kernel_name.c_str(), info.cmd_duration_ns/1.e6f, info.global_size[0], info.global_size[1], info.global_size[2], info.local_size[0], info.local_size[1], info.local_size[2], info.output_size[0], info.output_size[1], info.output_size[2], info.output_size[3]); } fclose(fperf); // Dump a simple chrome trace FILE * ftrace = fopen("cl_trace.json", "w"); if (!ftrace) { GGML_LOG_ERROR("Failed to open cl_trace.json\n"); return; } fprintf(ftrace, "[\n"); for (const ProfilingInfo & info : profiling_results) { fprintf(ftrace, "{\"name\": \"%s\", \"cat\": \"OpenCL\", \"ph\": \"B\", \"ts\": %" PRIu64 ", \"pid\": \"\", \"tid\": \"Host\"},\n", info.kernel_name.c_str(), info.cmd_queued/1000); fprintf(ftrace, "{\"name\": \"%s\", \"cat\": \"OpenCL\", \"ph\": \"E\", \"ts\": %" PRIu64 ", \"pid\": \"\", \"tid\": \"Host\"},\n", info.kernel_name.c_str(), info.cmd_submit/1000); fprintf(ftrace, "{\"name\": \"%s\", \"cat\": \"OpenCL\", \"ph\": \"B\", \"ts\": %" PRIu64 ", \"pid\": \"\", \"tid\": \"Device\"},\n", info.kernel_name.c_str(), info.cmd_start/1000); fprintf(ftrace, "{\"name\": \"%s\", \"cat\": \"OpenCL\", \"ph\": \"E\", \"ts\": %" PRIu64 ", \"pid\": \"\", \"tid\": \"Device\"},\n", info.kernel_name.c_str(), info.cmd_end/1000); } fprintf(ftrace, "]\n"); fclose(ftrace); } size_t get_kernel_workgroup_size(cl_kernel kernel) const { size_t workgroup_size = 0; size_t ret_size = 0; CL_CHECK( clGetKernelWorkGroupInfo(kernel, device, CL_KERNEL_WORK_GROUP_SIZE, sizeof(size_t), &workgroup_size, &ret_size)); GGML_ASSERT(sizeof(size_t) == ret_size); return workgroup_size; } void enqueue_ndrange_kernel(cl_kernel kernel, cl_uint work_dim, size_t *global_work_size, size_t *local_work_size, const ggml_tensor * tensor) { // From the spec on clEnqueueNDRangeKernel: // If the device associated with command_queue is an OpenCL 2.1 or newer device, // and global_work_size is NULL or the value in any passed dimension is zero, // then the kernel command will trivially succeed after its event dependencies // are satisfied and subsequently update its completion event. // So this ensures such cases always return trivially without causing errors in // case of an older device. for (cl_uint i = 0; i < work_dim; i++) { if (global_work_size[i] == 0) { return; } } #ifdef GGML_OPENCL_PROFILING cl_event evt; CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, work_dim, NULL, global_work_size, local_work_size, 0, NULL, &evt)); profiling_info.emplace_back(); populateProfilingInfo(profiling_info.back(), evt, kernel, work_dim, global_work_size, local_work_size, tensor); if (profiling_info.size() >= 2048) { flush_profiling_batch(); } #else GGML_UNUSED(tensor); CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, work_dim, NULL, global_work_size, local_work_size, 0, NULL, NULL)); #endif } const void * get_adreno_bin_kernel(const std::string &kernel_name, size_t *bin_size) const { if (!get_adreno_bin_kernel_func) { return nullptr; } size_t sz = 0; const void * kernel_bin = get_adreno_bin_kernel_func( kernel_name.c_str(), device_name.c_str(), driver_version.c_str(), &sz); if (bin_size) { *bin_size = sz; } return kernel_bin; } #ifdef GGML_OPENCL_USE_ADRENO_KERNELS // Transpose kernels cl_program program_transpose; cl_kernel kernel_transpose_32; cl_kernel kernel_transpose_32_16; cl_kernel kernel_transpose_16; cl_kernel kernel_transpose_8_buf; cl_kernel kernel_transpose_16_buf; cl_kernel kernel_transpose_32_buf; cl_kernel kernel_transpose_16_4x1; // Gemm and Gemv related programs, kernels, etc cl_kernel kernel_gemm_noshuffle_q4_0_f32; cl_kernel kernel_gemv_noshuffle_q4_0_f32; cl_kernel kernel_gemv_noshuffle_q4_0_f32_mc3; // multi-column (N=3) verify GEMV (spec/MTP) cl_kernel kernel_gemm_noshuffle_q4_0_f32_32b_trans_ila_a8_bin; cl_kernel kernel_gemm_noshuffle_q4_0_q8_1_dp4a_ila_a8_bin; cl_kernel kernel_gemv_noshuffle_q4_0_f32_32b_trans; cl_kernel kernel_gemv_noshuffle_q4_0_f32_4096_1_11008; cl_kernel kernel_gemv_noshuffle_q4_0_f32_4096_1_4096; cl_kernel kernel_gemv_noshuffle_q4_0_f32_11008_1_4096; cl_kernel kernel_gemv_noshuffle_q4_0_f32_32000_1_4096; cl_kernel kernel_gemv_noshuffle_q4_1_f32; cl_kernel kernel_gemv_noshuffle_q4_1_f32_mc3; // multi-column (N=3) verify GEMV (spec/MTP) cl_kernel kernel_gemm_noshuffle_q4_1_f32; cl_kernel kernel_gemm_noshuffle_q8_0_f32, kernel_gemm_noshuffle_q8_0_f32_bin; cl_kernel kernel_gemm_noshuffle_q8_0_q8_1_dp4a = nullptr; // dp4a (int8) dense q8_0 prefill GEMM (opt-in) cl_kernel kernel_gemm_noshuffle_q8_0_q8_1_dp4a_wimg = nullptr; // q8_0 dense dp4a, weights via texture (opt-in) cl_kernel kernel_gemm_noshuffle_q8_0_q8_1_dp4a_ila_a8_bin = nullptr; cl_kernel kernel_gemv_noshuffle_q8_0_f32; cl_kernel kernel_gemv_noshuffle_q8_0_f32_splitk; // split-K across WGs (small-M decode) cl_kernel kernel_gemm_noshuffle_q1_0_f32; cl_kernel kernel_gemv_noshuffle_q1_0_f32; cl_kernel kernel_gemv_noshuffle_q4_k_f32; cl_kernel kernel_gemv_noshuffle_q4_k_f32_o4; // 4-output-per-WI, long-vocab lm_head cl_kernel kernel_gemv_noshuffle_q4_k_f32_tiled; // tiled-wide layout (opt-in) cl_kernel kernel_gemv_noshuffle_q4_k_f32_splitk; // split-K across WGs (small-M decode) cl_kernel kernel_gemv_splitk_reduce_f32; // sums split-K per-slice partials cl_kernel kernel_gemv_noshuffle_q4_k_f32_glu; // fused gate+up GEMV + GLU (FFN) cl_kernel kernel_convert_block_q4_k_tiled_ns; // tiled-wide convert (opt-in) cl_kernel kernel_gemv_noshuffle_q4_k_f32_mc3; // multi-column (N=3) verify GEMV cl_kernel kernel_gemm_noshuffle_q4_k_f32; cl_kernel kernel_gemm_noshuffle_q4_k_f32_32b_trans_ila_a8_bin; cl_kernel kernel_gemm_noshuffle_q4_k_q8_1_dp4a_ila_a8_bin; cl_kernel kernel_gemv_noshuffle_q4_k_f32_32b_trans; cl_kernel kernel_gemm_noshuffle_q4_k_q8_1_dp4a = nullptr; // dp4a (int8) dense prefill GEMM cl_kernel kernel_gemm_noshuffle_q4_k_q8_1_dp4a_wimg = nullptr; // dp4a dense prefill GEMM, weights via texture (X1 opt-in) cl_kernel kernel_gemm_noshuffle_q5_k_q8_1_dp4a = nullptr; // dp4a (int8) dense q5_K prefill GEMM cl_kernel kernel_gemm_noshuffle_q6_k_q8_1_dp4a = nullptr; // dp4a (int8) dense q6_K prefill GEMM cl_kernel kernel_quant_a_q8_1; // plain activation q8_1 pre-pass cl_kernel kernel_gemm_noshuffle_q4_k_f32_r1; cl_kernel kernel_gemm_noshuffle_q4_k_f32_kimg; cl_kernel kernel_gemm_noshuffle_q4_k_f32_cok; cl_kernel kernel_gemv_noshuffle_q6_K_f32; cl_kernel kernel_gemv_noshuffle_q6_K_f32_o4; cl_kernel kernel_gemv_noshuffle_q6_K_f32_o4_global; // weights via __global (opt-in) cl_kernel kernel_gemv_noshuffle_q6_K_f32_tiled; // tiled-wide layout (opt-in) cl_kernel kernel_gemv_noshuffle_q6_K_f32_tiled_mc3; // tiled multi-column (N=3) verify lm_head cl_kernel kernel_gemm_noshuffle_q6_K_f32_tiled; // batched (N>1) over the tiled layout cl_kernel kernel_convert_block_q6_k_tiled_ns; // tiled-wide convert (opt-in) cl_kernel kernel_gemv_noshuffle_q6_K_f32_mc3; // multi-column (N=3) verify GEMV cl_kernel kernel_gemm_noshuffle_q6_K_f32; cl_kernel kernel_gemm_noshuffle_q6_K_f32_cok; cl_kernel kernel_gemm_noshuffle_q6_k_f32_32b_trans_ila_a8_bin; cl_kernel kernel_gemm_noshuffle_q6_k_q8_1_dp4a_ila_a8_bin; cl_kernel kernel_gemv_noshuffle_q6_k_f32_32b_trans; cl_kernel kernel_gemv_noshuffle_q5_k_f32; cl_kernel kernel_gemv_noshuffle_q5_k_f32_mc3; // multi-column (N=3) verify GEMV (spec/MTP) cl_kernel kernel_gemm_noshuffle_q5_k_f32; cl_kernel kernel_gemm_noshuffle_q5_k_f32_32b_trans_ila_a8_bin; cl_kernel kernel_gemm_noshuffle_q5_k_q8_1_dp4a_ila_a8_bin; cl_kernel kernel_gemv_noshuffle_q5_k_f32_32b_trans; cl_kernel kernel_gemv_noshuffle_q5_0_f32; cl_kernel kernel_gemm_noshuffle_q5_0_f32; cl_kernel kernel_gemm_noshuffle_q5_0_q8_1_dp4a = nullptr; // dp4a (int8) dense q5_0 prefill GEMM cl_kernel kernel_gemm_noshuffle_q5_0_q8_1_dp4a_wimg = nullptr; // q5_0 dense dp4a, qs plane via texture (opt-in) cl_kernel kernel_gemv_noshuffle_q5_1_f32; cl_kernel kernel_gemm_noshuffle_q5_1_f32; cl_kernel kernel_gemv_noshuffle_iq4_nl_f32; cl_kernel kernel_gemm_noshuffle_iq4_nl_f32; cl_kernel kernel_gemm_noshuffle_iq4_nl_q8_1_dp4a = nullptr; // dp4a (int8) dense IQ4_NL prefill GEMM cl_kernel kernel_gemm_noshuffle_q4_0_q8_1_dp4a = nullptr; // dp4a (int8) dense q4_0 prefill GEMM #endif // GGML_OPENCL_USE_ADRENO_KERNELS void free() { clFinish(queue); ref_count--; if (ref_count == 0) { #ifdef GGML_OPENCL_PROFILING flush_profiling_batch(); write_profiling_info(); profiling_results.clear(); #endif // release pooled image1d_buffer views over KV cache layers. for (auto & kv : kq_img_pool) { if (kv.second.image) { CL_CHECK(clReleaseMemObject(kv.second.image)); } if (kv.second.sub_buffer) { CL_CHECK(clReleaseMemObject(kv.second.sub_buffer)); } } kq_img_pool.clear(); for (auto & kv : kqv_img_pool) { if (kv.second.image) { CL_CHECK(clReleaseMemObject(kv.second.image)); } if (kv.second.sub_buffer) { CL_CHECK(clReleaseMemObject(kv.second.sub_buffer)); } } kqv_img_pool.clear(); for (auto & kv : dequant_f16_pool) { if (kv.second.image) { CL_CHECK(clReleaseMemObject(kv.second.image)); } } dequant_f16_pool.clear(); #ifdef GGML_OPENCL_USE_ADRENO_KERNELS ggml_cl_adreno_xmem_attn_release_scratch(this); #endif } } }; // All registered devices with a default device in the front. static std::vector g_ggml_backend_opencl_devices; // All device contexts associated with the devices above. // The devices live as long as the process, so do the contexts. static std::vector> g_ggml_backend_opencl_dev_ctxs; inline std::string read_file(const std::string &path) { std::ifstream ifs(path); if (!ifs) { return ""; } std::string text; ifs.seekg(0, std::ios::end); text.resize(ifs.tellg()); ifs.seekg(0, std::ios::beg); ifs.read(&text[0], text.size()); return text; } // fatal=false returns NULL on compile failure instead of aborting; used for // optional FA variants that may exhaust the Adreno compiler at large DK. // when the compiler returns CL_OUT_OF_HOST_MEMORY/CL_OUT_OF_RESOURCES (seen with DK>=256/512) // for FA programs, do clFinish the queue to free up resources, then rebuild (up to 3x) // if retry_queue is provided static cl_program build_program_from_source_ex(cl_context ctx, cl_device_id dev, const char* program_buffer, const std::string &compile_opts, bool fatal, const char *tag = nullptr, cl_command_queue retry_queue = nullptr) { if (tag) { GGML_LOG_INFO("ggml_opencl: compiling %s\n", tag); } cl_program p; char *program_log; size_t program_size; size_t log_size; int err; program_size = strlen(program_buffer); const int max_attempts = retry_queue ? 3 : 1; for (int attempt = 0; attempt < max_attempts; ++attempt) { p = clCreateProgramWithSource(ctx, 1, (const char**)&program_buffer, &program_size, &err); if(err < 0) { GGML_LOG_ERROR("OpenCL error creating program"); if (fatal) exit(1); return NULL; } err = clBuildProgram(p, 0, NULL, compile_opts.c_str(), NULL, NULL); if (err == CL_SUCCESS) { return p; } const bool transient = (err == CL_OUT_OF_HOST_MEMORY || err == CL_OUT_OF_RESOURCES); if (retry_queue && transient && attempt + 1 < max_attempts) { clReleaseProgram(p); GGML_LOG_WARN("ggml_opencl: transient compile failure (err=%d)%s%s — clFinish + retry (%d/%d)\n", err, tag ? " building " : "", tag ? tag : "", attempt + 2, max_attempts); clFinish(retry_queue); // drain in-flight ops holding driver host-heap continue; } clGetProgramBuildInfo(p, dev, CL_PROGRAM_BUILD_LOG, 0, NULL, &log_size); program_log = (char*) malloc(log_size + 1); program_log[log_size] = '\0'; clGetProgramBuildInfo(p, dev, CL_PROGRAM_BUILD_LOG, log_size + 1, program_log, NULL); GGML_LOG_ERROR("ggml_opencl: kernel compile error (err=%d)%s%s:\n\n%s\n", err, tag ? " building " : "", tag ? tag : "", program_log); free(program_log); clReleaseProgram(p); if (fatal) { exit(1); } return nullptr; } return NULL; } static cl_program build_program_from_source(ggml_backend_opencl_context * backend_ctx, const char* program_buffer, const std::string &compile_opts) { cl_context ctx = backend_ctx->context; cl_device_id dev = backend_ctx->device; // Try the on-disk binary cache first. Falls through silently on miss or // any failure; never blocks the build path. Disabled cache => nullptr. cl_program p_cached = cl_program_cache_try_load( backend_ctx->program_cache, ctx, dev, program_buffer, compile_opts); if (p_cached != nullptr) { return p_cached; } cl_program p = build_program_from_source_ex(ctx, dev, program_buffer, compile_opts, /*fatal=*/true); // Best-effort save of the freshly-built binary (no-op if cache disabled). if (p != nullptr) { cl_program_cache_try_save(backend_ctx->program_cache, p, dev, program_buffer, compile_opts); } return p; } static cl_program build_program_from_binary(cl_context ctx, cl_device_id dev, const char* program_buffer, const std::string &compile_opts, size_t bin_size = 0) { cl_program p; char *program_log; size_t log_size; int err; p = clCreateProgramWithBinary(ctx, 1, &dev, &bin_size, (const unsigned char**)&program_buffer, NULL, &err); if(err < 0) { GGML_LOG_ERROR("OpenCL error creating program from binary"); exit(1); } err = clBuildProgram(p, 0, NULL, compile_opts.c_str(), NULL, NULL); if(err < 0) { clGetProgramBuildInfo(p, dev, CL_PROGRAM_BUILD_LOG, 0, NULL, &log_size); program_log = (char*) malloc(log_size + 1); program_log[log_size] = '\0'; clGetProgramBuildInfo(p, dev, CL_PROGRAM_BUILD_LOG, log_size + 1, program_log, NULL); GGML_LOG_ERROR("ggml_opencl: kernel compile error:\n\n%s\n", program_log); free(program_log); exit(1); } return p; } static void load_cl_kernels_argsort(ggml_backend_opencl_context *backend_ctx) { // compiler options for general kernels auto opencl_c_std = std::string("CL") + std::to_string(backend_ctx->opencl_c_version.major) + "." + std::to_string(backend_ctx->opencl_c_version.minor); std::string compile_opts = std::string("-cl-std=") + opencl_c_std + " -cl-mad-enable -cl-unsafe-math-optimizations" " -cl-finite-math-only -cl-fast-relaxed-math"; // argsort if (!backend_ctx->kernels_loaded_argsort) { cl_int err; #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { #include "argsort.cl.h" }; #else const std::string kernel_src = read_file("argsort.cl"); #endif backend_ctx->program_argsort_f32_i32 = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); CL_CHECK((backend_ctx->kernel_argsort_f32_i32 = clCreateKernel(backend_ctx->program_argsort_f32_i32, "kernel_argsort_f32_i32", &err), err)); backend_ctx->kernels_loaded_argsort = true; } } static bool use_adreno_bin_kernels(ggml_backend_opencl_context * backend_ctx) { #ifndef GGML_OPENCL_USE_ADRENO_BIN_KERNELS GGML_UNUSED(backend_ctx); return false; #else if (backend_ctx->gpu_family != GPU_FAMILY::ADRENO) { return false; } return backend_ctx->adreno_use_bin_kernels; #endif // GGML_OPENCL_USE_ADRENO_BIN_KERNELS } static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) { if (backend_ctx->kernels_loaded) { return; } cl_int err; // compiler options for general kernels auto opencl_c_std = std::string("CL") + std::to_string(backend_ctx->opencl_c_version.major) + "." + std::to_string(backend_ctx->opencl_c_version.minor); std::string compile_opts = std::string("-cl-std=") + opencl_c_std + " -cl-mad-enable -cl-unsafe-math-optimizations" " -cl-finite-math-only -cl-fast-relaxed-math"; if (backend_ctx->adreno_use_large_buffer) { compile_opts += " -qcom-enable-large-buffer "; } backend_ctx->kernel_compile_opts = compile_opts; GGML_LOG_INFO("ggml_opencl: loading OpenCL kernels"); // add { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { #include "add.cl.h" }; #else const std::string kernel_src = read_file("add.cl"); #endif backend_ctx->program_add = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); CL_CHECK((backend_ctx->kernel_add = clCreateKernel(backend_ctx->program_add, "kernel_add", &err), err)); CL_CHECK((backend_ctx->kernel_add_row = clCreateKernel(backend_ctx->program_add, "kernel_add_row", &err), err)); CL_CHECK((backend_ctx->kernel_add_f16 = clCreateKernel(backend_ctx->program_add, "kernel_add_f16", &err), err)); CL_CHECK((backend_ctx->kernel_add_row_f16 = clCreateKernel(backend_ctx->program_add, "kernel_add_row_f16", &err), err)); GGML_LOG_CONT("."); } // add_id { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { #include "add_id.cl.h" }; #else const std::string kernel_src = read_file("add_id.cl"); #endif backend_ctx->program_add_id = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); CL_CHECK((backend_ctx->kernel_add_id = clCreateKernel(backend_ctx->program_add_id, "kernel_add_id", &err), err)); GGML_LOG_CONT("."); } // moe_add_id_glu { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { #include "moe_add_id_glu.cl.h" }; #else const std::string kernel_src = read_file("moe_add_id_glu.cl"); #endif backend_ctx->program_moe_add_id_glu = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); CL_CHECK((backend_ctx->kernel_add_id_add_id_swiglu_oai = clCreateKernel(backend_ctx->program_moe_add_id_glu, "kernel_add_id_add_id_swiglu_oai", &err), err)); GGML_LOG_CONT("."); } // tri { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { #include "tri.cl.h" }; #else const std::string kernel_src = read_file("tri.cl"); #endif cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); CL_CHECK((backend_ctx->kernel_tri = clCreateKernel(prog, "kernel_tri_f32", &err), err)); GGML_LOG_CONT("."); CL_CHECK(clReleaseProgram(prog)); } // fill { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { #include "fill.cl.h" }; #else const std::string kernel_src = read_file("fill.cl"); #endif cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); CL_CHECK((backend_ctx->kernel_fill = clCreateKernel(prog, "kernel_fill_f32", &err), err)); GGML_LOG_CONT("."); CL_CHECK(clReleaseProgram(prog)); } // clamp { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { #include "clamp.cl.h" }; #else const std::string kernel_src = read_file("clamp.cl"); #endif backend_ctx->program_clamp = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); CL_CHECK((backend_ctx->kernel_clamp = clCreateKernel(backend_ctx->program_clamp, "kernel_clamp", &err), err)); GGML_LOG_CONT("."); } // cpy { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { #include "cpy.cl.h" }; #else const std::string kernel_src = read_file("cpy.cl"); #endif cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); CL_CHECK((backend_ctx->kernel_cpy_f16_f16 = clCreateKernel(prog, "kernel_cpy_f16_f16", &err), err)); CL_CHECK((backend_ctx->kernel_cpy_f16_f32 = clCreateKernel(prog, "kernel_cpy_f16_f32", &err), err)); CL_CHECK((backend_ctx->kernel_cpy_f32_f16 = clCreateKernel(prog, "kernel_cpy_f32_f16", &err), err)); CL_CHECK((backend_ctx->kernel_cpy_f32_f32 = clCreateKernel(prog, "kernel_cpy_f32_f32", &err), err)); CL_CHECK((backend_ctx->kernel_cpy_f32_f32_pack = clCreateKernel(prog, "kernel_cpy_f32_f32_pack", &err), err)); { // optional: without it ggml_cl_cpy keeps the row-mapped kernel cl_int err_flat = CL_SUCCESS; cl_kernel k = clCreateKernel(prog, "kernel_cpy_f32_f32_flat", &err_flat); if (err_flat == CL_SUCCESS) { backend_ctx->kernel_cpy_f32_f32_flat = k; } } CL_CHECK((backend_ctx->kernel_cpy_i32_i32 = clCreateKernel(prog, "kernel_cpy_i32_i32", &err), err)); GGML_LOG_CONT("."); } // cvt { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { #include "cvt.cl.h" }; #else const std::string kernel_src = read_file("cvt.cl"); #endif backend_ctx->program_cvt = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); CL_CHECK((backend_ctx->kernel_convert_block_q1_0 = clCreateKernel(backend_ctx->program_cvt, "kernel_convert_block_q1_0", &err), err)); CL_CHECK((backend_ctx->kernel_restore_block_q1_0 = clCreateKernel(backend_ctx->program_cvt, "kernel_restore_block_q1_0", &err), err)); CL_CHECK((backend_ctx->kernel_convert_block_q4_0_noshuffle = clCreateKernel(backend_ctx->program_cvt, "kernel_convert_block_q4_0_noshuffle", &err), err)); CL_CHECK((backend_ctx->kernel_restore_block_q4_0_noshuffle = clCreateKernel(backend_ctx->program_cvt, "kernel_restore_block_q4_0_noshuffle", &err), err)); CL_CHECK((backend_ctx->kernel_convert_block_q4_0 = clCreateKernel(backend_ctx->program_cvt, "kernel_convert_block_q4_0", &err), err)); CL_CHECK((backend_ctx->kernel_restore_block_q4_0 = clCreateKernel(backend_ctx->program_cvt, "kernel_restore_block_q4_0", &err), err)); CL_CHECK((backend_ctx->kernel_convert_block_q4_0_trans4_ns = clCreateKernel(backend_ctx->program_cvt, "kernel_convert_block_q4_0_trans4_ns", &err), err)); CL_CHECK((backend_ctx->kernel_restore_block_q4_0_trans4_ns = clCreateKernel(backend_ctx->program_cvt, "kernel_restore_block_q4_0_trans4_ns", &err), err)); CL_CHECK((backend_ctx->kernel_convert_block_q4_1_noshuffle = clCreateKernel(backend_ctx->program_cvt, "kernel_convert_block_q4_1_noshuffle", &err), err)); CL_CHECK((backend_ctx->kernel_restore_block_q4_1_noshuffle = clCreateKernel(backend_ctx->program_cvt, "kernel_restore_block_q4_1_noshuffle", &err), err)); CL_CHECK((backend_ctx->kernel_convert_block_q4_1 = clCreateKernel(backend_ctx->program_cvt, "kernel_convert_block_q4_1", &err), err)); CL_CHECK((backend_ctx->kernel_restore_block_q4_1 = clCreateKernel(backend_ctx->program_cvt, "kernel_restore_block_q4_1", &err), err)); CL_CHECK((backend_ctx->kernel_convert_block_q4_1_trans4_ns = clCreateKernel(backend_ctx->program_cvt, "kernel_convert_block_q4_1_trans4_ns", &err), err)); CL_CHECK((backend_ctx->kernel_restore_block_q4_1_trans4_ns = clCreateKernel(backend_ctx->program_cvt, "kernel_restore_block_q4_1_trans4_ns", &err), err)); CL_CHECK((backend_ctx->kernel_convert_block_q5_0 = clCreateKernel(backend_ctx->program_cvt, "kernel_convert_block_q5_0", &err), err)); CL_CHECK((backend_ctx->kernel_restore_block_q5_0 = clCreateKernel(backend_ctx->program_cvt, "kernel_restore_block_q5_0", &err), err)); CL_CHECK((backend_ctx->kernel_convert_block_q5_0_noshuffle = clCreateKernel(backend_ctx->program_cvt, "kernel_convert_block_q5_0_noshuffle", &err), err)); CL_CHECK((backend_ctx->kernel_restore_block_q5_0_noshuffle = clCreateKernel(backend_ctx->program_cvt, "kernel_restore_block_q5_0_noshuffle", &err), err)); CL_CHECK((backend_ctx->kernel_convert_block_q5_1_noshuffle = clCreateKernel(backend_ctx->program_cvt, "kernel_convert_block_q5_1_noshuffle", &err), err)); CL_CHECK((backend_ctx->kernel_restore_block_q5_1_noshuffle = clCreateKernel(backend_ctx->program_cvt, "kernel_restore_block_q5_1_noshuffle", &err), err)); CL_CHECK((backend_ctx->kernel_convert_block_q5_0_trans4_ns = clCreateKernel(backend_ctx->program_cvt, "kernel_convert_block_q5_0_trans4_ns", &err), err)); CL_CHECK((backend_ctx->kernel_restore_block_q5_0_trans4_ns = clCreateKernel(backend_ctx->program_cvt, "kernel_restore_block_q5_0_trans4_ns", &err), err)); CL_CHECK((backend_ctx->kernel_convert_block_q5_1 = clCreateKernel(backend_ctx->program_cvt, "kernel_convert_block_q5_1", &err), err)); CL_CHECK((backend_ctx->kernel_restore_block_q5_1 = clCreateKernel(backend_ctx->program_cvt, "kernel_restore_block_q5_1", &err), err)); CL_CHECK((backend_ctx->kernel_convert_block_q5_1_trans4_ns = clCreateKernel(backend_ctx->program_cvt, "kernel_convert_block_q5_1_trans4_ns", &err), err)); CL_CHECK((backend_ctx->kernel_restore_block_q5_1_trans4_ns = clCreateKernel(backend_ctx->program_cvt, "kernel_restore_block_q5_1_trans4_ns", &err), err)); CL_CHECK((backend_ctx->kernel_convert_block_q4_k_trans4_ns = clCreateKernel(backend_ctx->program_cvt, "kernel_convert_block_q4_k_trans4_ns", &err), err)); CL_CHECK((backend_ctx->kernel_restore_block_q4_k_trans4_ns = clCreateKernel(backend_ctx->program_cvt, "kernel_restore_block_q4_k_trans4_ns", &err), err)); #ifdef GGML_OPENCL_USE_ADRENO_KERNELS CL_CHECK((backend_ctx->kernel_convert_block_q4_k_tiled_ns = clCreateKernel(backend_ctx->program_cvt, "kernel_convert_block_q4_k_tiled_ns", &err), err)); #endif CL_CHECK((backend_ctx->kernel_convert_block_q5_k_trans4_ns = clCreateKernel(backend_ctx->program_cvt, "kernel_convert_block_q5_k_trans4_ns", &err), err)); CL_CHECK((backend_ctx->kernel_restore_block_q5_k_trans4_ns = clCreateKernel(backend_ctx->program_cvt, "kernel_restore_block_q5_k_trans4_ns", &err), err)); CL_CHECK((backend_ctx->kernel_convert_block_q6_k_trans4_ns = clCreateKernel(backend_ctx->program_cvt, "kernel_convert_block_q6_k_trans4_ns", &err), err)); CL_CHECK((backend_ctx->kernel_restore_block_q6_k_trans4_ns = clCreateKernel(backend_ctx->program_cvt, "kernel_restore_block_q6_k_trans4_ns", &err), err)); #ifdef GGML_OPENCL_USE_ADRENO_KERNELS CL_CHECK((backend_ctx->kernel_convert_block_q6_k_tiled_ns = clCreateKernel(backend_ctx->program_cvt, "kernel_convert_block_q6_k_tiled_ns", &err), err)); #endif CL_CHECK((backend_ctx->kernel_convert_block_mxfp4 = clCreateKernel(backend_ctx->program_cvt, "kernel_convert_block_mxfp4", &err), err)); CL_CHECK((backend_ctx->kernel_convert_block_mxfp4_trans = clCreateKernel(backend_ctx->program_cvt, "kernel_convert_block_mxfp4_trans", &err), err)); CL_CHECK((backend_ctx->kernel_convert_block_mxfp4_trans4_ns = clCreateKernel(backend_ctx->program_cvt, "kernel_convert_block_mxfp4_trans4_ns", &err), err)); CL_CHECK((backend_ctx->kernel_restore_block_mxfp4_trans4_ns = clCreateKernel(backend_ctx->program_cvt, "kernel_restore_block_mxfp4_trans4_ns", &err), err)); CL_CHECK((backend_ctx->kernel_restore_block_mxfp4_trans = clCreateKernel(backend_ctx->program_cvt, "kernel_restore_block_mxfp4_trans", &err), err)); CL_CHECK((backend_ctx->kernel_restore_block_mxfp4 = clCreateKernel(backend_ctx->program_cvt, "kernel_restore_block_mxfp4", &err), err)); CL_CHECK((backend_ctx->kernel_convert_block_q8_0 = clCreateKernel(backend_ctx->program_cvt, "kernel_convert_block_q8_0", &err), err)); CL_CHECK((backend_ctx->kernel_restore_block_q8_0 = clCreateKernel(backend_ctx->program_cvt, "kernel_restore_block_q8_0", &err), err)); CL_CHECK((backend_ctx->kernel_restore_block_q8_0_trans = clCreateKernel(backend_ctx->program_cvt, "kernel_restore_block_q8_0_trans", &err), err)); CL_CHECK((backend_ctx->kernel_dequant_q8_0_f16_view_aos = clCreateKernel(backend_ctx->program_cvt, "kernel_dequant_q8_0_f16_view_aos", &err), err)); CL_CHECK((backend_ctx->kernel_dequant_q8_0_f32_view_aos = clCreateKernel(backend_ctx->program_cvt, "kernel_dequant_q8_0_f32_view_aos", &err), err)); CL_CHECK((backend_ctx->kernel_dequant_q4_0_f16_view_aos = clCreateKernel(backend_ctx->program_cvt, "kernel_dequant_q4_0_f16_view_aos", &err), err)); CL_CHECK((backend_ctx->kernel_dequant_q4_0_f32_view_aos = clCreateKernel(backend_ctx->program_cvt, "kernel_dequant_q4_0_f32_view_aos", &err), err)); CL_CHECK((backend_ctx->kernel_convert_block_q4_K = clCreateKernel(backend_ctx->program_cvt, "kernel_convert_block_q4_K", &err), err)); CL_CHECK((backend_ctx->kernel_restore_block_q4_K = clCreateKernel(backend_ctx->program_cvt, "kernel_restore_block_q4_K", &err), err)); CL_CHECK((backend_ctx->kernel_convert_block_q4_K_noshuffle = clCreateKernel(backend_ctx->program_cvt, "kernel_convert_block_q4_K_noshuffle", &err), err)); CL_CHECK((backend_ctx->kernel_restore_block_q4_K_noshuffle = clCreateKernel(backend_ctx->program_cvt, "kernel_restore_block_q4_K_noshuffle", &err), err)); CL_CHECK((backend_ctx->kernel_convert_block_q5_K = clCreateKernel(backend_ctx->program_cvt, "kernel_convert_block_q5_K", &err), err)); CL_CHECK((backend_ctx->kernel_restore_block_q5_K = clCreateKernel(backend_ctx->program_cvt, "kernel_restore_block_q5_K", &err), err)); CL_CHECK((backend_ctx->kernel_convert_block_q5_K_noshuffle = clCreateKernel(backend_ctx->program_cvt, "kernel_convert_block_q5_K_noshuffle", &err), err)); CL_CHECK((backend_ctx->kernel_restore_block_q5_K_noshuffle = clCreateKernel(backend_ctx->program_cvt, "kernel_restore_block_q5_K_noshuffle", &err), err)); CL_CHECK((backend_ctx->kernel_convert_block_q6_K = clCreateKernel(backend_ctx->program_cvt, "kernel_convert_block_q6_K", &err), err)); CL_CHECK((backend_ctx->kernel_restore_block_q6_K = clCreateKernel(backend_ctx->program_cvt, "kernel_restore_block_q6_K", &err), err)); CL_CHECK((backend_ctx->kernel_convert_block_q6_K_noshuffle = clCreateKernel(backend_ctx->program_cvt, "kernel_convert_block_q6_K_noshuffle", &err), err)); CL_CHECK((backend_ctx->kernel_restore_block_q6_K_noshuffle = clCreateKernel(backend_ctx->program_cvt, "kernel_restore_block_q6_K_noshuffle", &err), err)); CL_CHECK((backend_ctx->kernel_convert_block_iq4_nl = clCreateKernel(backend_ctx->program_cvt, "kernel_convert_block_iq4_nl", &err), err)); CL_CHECK((backend_ctx->kernel_restore_block_iq4_nl = clCreateKernel(backend_ctx->program_cvt, "kernel_restore_block_iq4_nl", &err), err)); CL_CHECK((backend_ctx->kernel_convert_block_iq4_nl_noshuffle = clCreateKernel(backend_ctx->program_cvt, "kernel_convert_block_iq4_nl_noshuffle", &err), err)); CL_CHECK((backend_ctx->kernel_restore_block_iq4_nl_noshuffle = clCreateKernel(backend_ctx->program_cvt, "kernel_restore_block_iq4_nl_noshuffle", &err), err)); CL_CHECK((backend_ctx->kernel_convert_bf16_to_f16 = clCreateKernel(backend_ctx->program_cvt, "kernel_convert_bf16_to_f16", &err), err)); CL_CHECK((backend_ctx->kernel_convert_f16_to_bf16 = clCreateKernel(backend_ctx->program_cvt, "kernel_convert_f16_to_bf16", &err), err)); #ifdef GGML_OPENCL_USE_ADRENO_KERNELS CL_CHECK((backend_ctx->kernel_moe_expand_scale_q8_0 = clCreateKernel(backend_ctx->program_cvt, "kernel_moe_expand_scale_q8_0", &err), err)); CL_CHECK((backend_ctx->kernel_moe_expand_scale_q5_0 = clCreateKernel(backend_ctx->program_cvt, "kernel_moe_expand_scale_q5_0", &err), err)); CL_CHECK((backend_ctx->kernel_moe_expand_scale_q5_K = clCreateKernel(backend_ctx->program_cvt, "kernel_moe_expand_scale_q5_K", &err), err)); #endif GGML_LOG_CONT("."); } // diag_mask_inf { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { #include "diag_mask_inf.cl.h" }; #else const std::string kernel_src = read_file("diag_mask_inf.cl"); #endif backend_ctx->program_diag_mask_inf = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); CL_CHECK((backend_ctx->kernel_diag_mask_inf_8 = clCreateKernel(backend_ctx->program_diag_mask_inf, "kernel_diag_mask_inf_8", &err), err)); CL_CHECK((backend_ctx->kernel_diag_mask_inf = clCreateKernel(backend_ctx->program_diag_mask_inf, "kernel_diag_mask_inf", &err), err)); GGML_LOG_CONT("."); } // diag { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { #include "diag.cl.h" }; #else const std::string kernel_src = read_file("diag.cl"); #endif cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); CL_CHECK((backend_ctx->kernel_diag_f32 = clCreateKernel(prog, "kernel_diag_f32", &err), err)); CL_CHECK(clReleaseProgram(prog)); GGML_LOG_CONT("."); } // gelu { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { #include "gelu.cl.h" }; #else const std::string kernel_src = read_file("gelu.cl"); #endif backend_ctx->program_gelu = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); CL_CHECK((backend_ctx->kernel_gelu = clCreateKernel(backend_ctx->program_gelu, "kernel_gelu", &err), err)); CL_CHECK((backend_ctx->kernel_gelu_4 = clCreateKernel(backend_ctx->program_gelu, "kernel_gelu_4", &err), err)); CL_CHECK((backend_ctx->kernel_gelu_erf = clCreateKernel(backend_ctx->program_gelu, "kernel_gelu_erf", &err), err)); CL_CHECK((backend_ctx->kernel_gelu_erf_4 = clCreateKernel(backend_ctx->program_gelu, "kernel_gelu_erf_4", &err), err)); CL_CHECK((backend_ctx->kernel_gelu_quick = clCreateKernel(backend_ctx->program_gelu, "kernel_gelu_quick", &err), err)); CL_CHECK((backend_ctx->kernel_gelu_quick_4 = clCreateKernel(backend_ctx->program_gelu, "kernel_gelu_quick_4", &err), err)); GGML_LOG_CONT("."); } // glu { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { #include "glu.cl.h" }; #else const std::string kernel_src = read_file("glu.cl"); #endif backend_ctx->program_glu = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); CL_CHECK((backend_ctx->kernel_geglu = clCreateKernel(backend_ctx->program_glu, "kernel_geglu", &err), err)); CL_CHECK((backend_ctx->kernel_reglu = clCreateKernel(backend_ctx->program_glu, "kernel_reglu", &err), err)); CL_CHECK((backend_ctx->kernel_swiglu = clCreateKernel(backend_ctx->program_glu, "kernel_swiglu", &err), err)); CL_CHECK((backend_ctx->kernel_swiglu_oai = clCreateKernel(backend_ctx->program_glu, "kernel_swiglu_oai", &err), err)); CL_CHECK((backend_ctx->kernel_swiglu_clamp = clCreateKernel(backend_ctx->program_glu, "kernel_swiglu_clamp", &err), err)); CL_CHECK((backend_ctx->kernel_geglu_erf = clCreateKernel(backend_ctx->program_glu, "kernel_geglu_erf", &err), err)); CL_CHECK((backend_ctx->kernel_geglu_quick = clCreateKernel(backend_ctx->program_glu, "kernel_geglu_quick", &err), err)); CL_CHECK((backend_ctx->kernel_geglu_f16 = clCreateKernel(backend_ctx->program_glu, "kernel_geglu_f16", &err), err)); CL_CHECK((backend_ctx->kernel_reglu_f16 = clCreateKernel(backend_ctx->program_glu, "kernel_reglu_f16", &err), err)); CL_CHECK((backend_ctx->kernel_swiglu_f16 = clCreateKernel(backend_ctx->program_glu, "kernel_swiglu_f16", &err), err)); CL_CHECK((backend_ctx->kernel_swiglu_clamp_f16 = clCreateKernel(backend_ctx->program_glu, "kernel_swiglu_clamp_f16", &err), err)); CL_CHECK((backend_ctx->kernel_geglu_erf_f16 = clCreateKernel(backend_ctx->program_glu, "kernel_geglu_erf_f16", &err), err)); CL_CHECK((backend_ctx->kernel_geglu_quick_f16 = clCreateKernel(backend_ctx->program_glu, "kernel_geglu_quick_f16", &err), err)); GGML_LOG_CONT("."); } // get_rows { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { #include "get_rows.cl.h" }; #else const std::string kernel_src = read_file("get_rows.cl"); #endif backend_ctx->program_get_rows = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); CL_CHECK((backend_ctx->kernel_get_rows_f32 = clCreateKernel(backend_ctx->program_get_rows, "kernel_get_rows_f32", &err), err)); CL_CHECK((backend_ctx->kernel_get_rows_f16 = clCreateKernel(backend_ctx->program_get_rows, "kernel_get_rows_f16", &err), err)); CL_CHECK((backend_ctx->kernel_get_rows_q4_0 = clCreateKernel(backend_ctx->program_get_rows, "kernel_get_rows_q4_0", &err), err)); GGML_LOG_CONT("."); } // solve_tri_f32 { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { #include "solve_tri.cl.h" }; #else const std::string kernel_src = read_file("solve_tri.cl"); #endif cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); CL_CHECK((backend_ctx->kernel_solve_tri_f32 = clCreateKernel(prog, "kernel_solve_tri_f32", &err), err)); GGML_LOG_CONT("."); CL_CHECK(clReleaseProgram(prog)); } // im2col_f32 { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { #include "im2col_f32.cl.h" }; #else const std::string kernel_src = read_file("im2col_f32.cl"); #endif backend_ctx->program_im2col_f32 = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); CL_CHECK((backend_ctx->kernel_im2col_f32 = clCreateKernel(backend_ctx->program_im2col_f32, "kernel_im2col_f32", &err), err)); GGML_LOG_CONT("."); } // im2col_f16 { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { #include "im2col_f16.cl.h" }; #else const std::string kernel_src = read_file("im2col_f16.cl"); #endif backend_ctx->program_im2col_f16 = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); CL_CHECK((backend_ctx->kernel_im2col_f16 = clCreateKernel(backend_ctx->program_im2col_f16, "kernel_im2col_f16", &err), err)); GGML_LOG_CONT("."); } // mul_mv_q4_0_f32 { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { #include "mul_mv_q4_0_f32.cl.h" }; #else const std::string kernel_src = read_file("mul_mv_q4_0_f32.cl"); #endif backend_ctx->program_mul_mv_q4_0_f32 = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); CL_CHECK((backend_ctx->kernel_mul_mat_q4_0_f32 = clCreateKernel(backend_ctx->program_mul_mv_q4_0_f32, "kernel_mul_mat_q4_0_f32", &err), err)); GGML_LOG_CONT("."); } // mul_mv_q4_0_f32_v { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { #include "mul_mv_q4_0_f32_v.cl.h" }; #else const std::string kernel_src = read_file("mul_mv_q4_0_f32_v.cl"); #endif backend_ctx->program_mul_mv_q4_0_f32_v = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); CL_CHECK((backend_ctx->kernel_mul_mat_q4_0_f32_v = clCreateKernel(backend_ctx->program_mul_mv_q4_0_f32_v, "kernel_mul_mat_q4_0_f32_v", &err), err)); GGML_LOG_CONT("."); } // mul_mv_q4_0_f32_8x_flat { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { #include "mul_mv_q4_0_f32_8x_flat.cl.h" }; #else const std::string kernel_src = read_file("mul_mv_q4_0_f32_8x_flat.cl"); #endif backend_ctx->program_mul_mv_q4_0_f32_8x_flat = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); CL_CHECK((backend_ctx->kernel_mul_mat_q4_0_f32_8x_flat = clCreateKernel(backend_ctx->program_mul_mv_q4_0_f32_8x_flat, "kernel_mul_mat_q4_0_f32_8x_flat", &err), err)); GGML_LOG_CONT("."); } // mul_mv_q4_0_f32_1d_8x_flat // This kernel does not compiler on Adreno cl compiler 38.01. Skip it for // those compiler versions since it is anyway not used for Adreno. if (backend_ctx->gpu_family != ADRENO || backend_ctx->adreno_cl_compiler_version.newer_than_or_same(E031, 38, 11, 0) || backend_ctx->adreno_cl_compiler_version.type == E17 || backend_ctx->adreno_cl_compiler_version.type == DX) { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { #include "mul_mv_q4_0_f32_1d_8x_flat.cl.h" }; #else const std::string kernel_src = read_file("mul_mv_q4_0_f32_1d_8x_flat.cl"); #endif backend_ctx->program_mul_mv_q4_0_f32_1d_8x_flat = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); CL_CHECK((backend_ctx->kernel_mul_mat_q4_0_f32_1d_8x_flat = clCreateKernel(backend_ctx->program_mul_mv_q4_0_f32_1d_8x_flat, "kernel_mul_mat_q4_0_f32_1d_8x_flat", &err), err)); GGML_LOG_CONT("."); } // mul_mv_q4_0_f32_1d_16x_flat // This kernel does not compiler on Adreno cl compiler 38.01. Skip it for // those compiler versions since it is anyway not used for Adreno. if (backend_ctx->gpu_family != ADRENO || backend_ctx->adreno_cl_compiler_version.newer_than_or_same(E031, 38, 11, 0) || backend_ctx->adreno_cl_compiler_version.type == DX) { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { #include "mul_mv_q4_0_f32_1d_16x_flat.cl.h" }; #else const std::string kernel_src = read_file("mul_mv_q4_0_f32_1d_16x_flat.cl"); #endif backend_ctx->program_mul_mv_q4_0_f32_1d_16x_flat = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); CL_CHECK((backend_ctx->kernel_mul_mat_q4_0_f32_1d_16x_flat = clCreateKernel(backend_ctx->program_mul_mv_q4_0_f32_1d_16x_flat, "kernel_mul_mat_q4_0_f32_1d_16x_flat", &err), err)); GGML_LOG_CONT("."); } // mul_mv_q4_1_f32 { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { #include "mul_mv_q4_1_f32.cl.h" }; #else const std::string kernel_src = read_file("mul_mv_q4_1_f32.cl"); #endif cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); CL_CHECK((backend_ctx->kernel_mul_mv_q4_1_f32 = clCreateKernel(prog, "kernel_mul_mv_q4_1_f32", &err), err)); CL_CHECK(clReleaseProgram(prog)); GGML_LOG_CONT("."); } // mul_mv_q4_1_f32_flat { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { #include "mul_mv_q4_1_f32_flat.cl.h" }; #else const std::string kernel_src = read_file("mul_mv_q4_1_f32_flat.cl"); #endif cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); CL_CHECK((backend_ctx->kernel_mul_mv_q4_1_f32_flat = clCreateKernel(prog, "kernel_mul_mv_q4_1_f32_flat", &err), err)); CL_CHECK(clReleaseProgram(prog)); GGML_LOG_CONT("."); } // mul_mv_q4_k_f32 { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { #include "mul_mv_q4_k_f32.cl.h" }; #else const std::string kernel_src = read_file("mul_mv_q4_k_f32.cl"); #endif cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); CL_CHECK((backend_ctx->kernel_mul_mv_q4_K_f32 = clCreateKernel(prog, "kernel_mul_mv_q4_K_f32", &err), err)); CL_CHECK(clReleaseProgram(prog)); GGML_LOG_CONT("."); } // mul_mv_q4_k_f32_flat { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { #include "mul_mv_q4_k_f32_flat.cl.h" }; #else const std::string kernel_src = read_file("mul_mv_q4_k_f32_flat.cl"); #endif cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); CL_CHECK((backend_ctx->kernel_mul_mv_q4_K_f32_flat = clCreateKernel(prog, "kernel_mul_mv_q4_K_f32_flat", &err), err)); CL_CHECK(clReleaseProgram(prog)); GGML_LOG_CONT("."); } // mul_mv_q5_0_f32 { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { #include "mul_mv_q5_0_f32.cl.h" }; #else const std::string kernel_src = read_file("mul_mv_q5_0_f32.cl"); #endif cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); CL_CHECK((backend_ctx->kernel_mul_mv_q5_0_f32 = clCreateKernel(prog, "kernel_mul_mv_q5_0_f32", &err), err)); CL_CHECK(clReleaseProgram(prog)); GGML_LOG_CONT("."); } // mul_mv_q5_0_f32_flat { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { #include "mul_mv_q5_0_f32_flat.cl.h" }; #else const std::string kernel_src = read_file("mul_mv_q5_0_f32_flat.cl"); #endif cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); CL_CHECK((backend_ctx->kernel_mul_mv_q5_0_f32_flat = clCreateKernel(prog, "kernel_mul_mv_q5_0_f32_flat", &err), err)); CL_CHECK(clReleaseProgram(prog)); GGML_LOG_CONT("."); } // mul_mv_q5_1_f32 { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { #include "mul_mv_q5_1_f32.cl.h" }; #else const std::string kernel_src = read_file("mul_mv_q5_1_f32.cl"); #endif cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); CL_CHECK((backend_ctx->kernel_mul_mv_q5_1_f32 = clCreateKernel(prog, "kernel_mul_mv_q5_1_f32", &err), err)); CL_CHECK(clReleaseProgram(prog)); GGML_LOG_CONT("."); } // mul_mv_q5_1_f32_flat { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { #include "mul_mv_q5_1_f32_flat.cl.h" }; #else const std::string kernel_src = read_file("mul_mv_q5_1_f32_flat.cl"); #endif cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); CL_CHECK((backend_ctx->kernel_mul_mv_q5_1_f32_flat = clCreateKernel(prog, "kernel_mul_mv_q5_1_f32_flat", &err), err)); CL_CHECK(clReleaseProgram(prog)); GGML_LOG_CONT("."); } // mul_mv_q5_k_f32 { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { #include "mul_mv_q5_k_f32.cl.h" }; #else const std::string kernel_src = read_file("mul_mv_q5_k_f32.cl"); #endif cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); CL_CHECK((backend_ctx->kernel_mul_mv_q5_K_f32 = clCreateKernel(prog, "kernel_mul_mv_q5_K_f32", &err), err)); CL_CHECK(clReleaseProgram(prog)); GGML_LOG_CONT("."); } // mul_mv_q5_k_f32_flat { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { #include "mul_mv_q5_k_f32_flat.cl.h" }; #else const std::string kernel_src = read_file("mul_mv_q5_k_f32_flat.cl"); #endif cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); CL_CHECK((backend_ctx->kernel_mul_mv_q5_K_f32_flat = clCreateKernel(prog, "kernel_mul_mv_q5_K_f32_flat", &err), err)); CL_CHECK(clReleaseProgram(prog)); } // mul_mv_q6_k_f32 { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { #include "mul_mv_q6_k_f32.cl.h" }; #else const std::string kernel_src = read_file("mul_mv_q6_k_f32.cl"); #endif backend_ctx->program_mul_mv_q6_K = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); CL_CHECK((backend_ctx->kernel_mul_mv_q6_K_f32 = clCreateKernel(backend_ctx->program_mul_mv_q6_K, "kernel_mul_mv_q6_K_f32", &err), err)); GGML_LOG_CONT("."); } // mul_mv_q6_k_f32_flat { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { #include "mul_mv_q6_k_f32_flat.cl.h" }; #else const std::string kernel_src = read_file("mul_mv_q6_k_f32_flat.cl"); #endif // The codegen workarounds in this kernel are a measured 13-20% loss on // compilers that do not need them, so only the affected ones build them; // everyone else gets the original source. const std::string q6k_opts = backend_ctx->q6_k_flat_old_compiler ? compile_opts + " -DADRENO_OLD_COMPILER=1" : compile_opts; cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), q6k_opts); CL_CHECK((backend_ctx->kernel_mul_mv_q6_K_f32_flat = clCreateKernel(prog, "kernel_mul_mv_q6_K_f32_flat", &err), err)); CL_CHECK(clReleaseProgram(prog)); GGML_LOG_CONT("."); } // mul_mv_q8_0_f32 { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { #include "mul_mv_q8_0_f32.cl.h" }; #else const std::string kernel_src = read_file("mul_mv_q8_0_f32.cl"); #endif backend_ctx->program_mul_mv_q8_0_f32 = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); CL_CHECK((backend_ctx->kernel_mul_mv_q8_0_f32 = clCreateKernel(backend_ctx->program_mul_mv_q8_0_f32, "kernel_mul_mv_q8_0_f32", &err), err)); GGML_LOG_CONT("."); } // mul_mv_q8_0_f32_flat { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { #include "mul_mv_q8_0_f32_flat.cl.h" }; #else const std::string kernel_src = read_file("mul_mv_q8_0_f32_flat.cl"); #endif backend_ctx->program_mul_mv_q8_0_f32_flat = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); CL_CHECK((backend_ctx->kernel_mul_mv_q8_0_f32_flat = clCreateKernel(backend_ctx->program_mul_mv_q8_0_f32_flat, "kernel_mul_mv_q8_0_f32_flat", &err), err)); GGML_LOG_CONT("."); } // mul_mv_q1_0_f32 { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { #include "mul_mv_q1_0_f32.cl.h" }; #else const std::string kernel_src = read_file("mul_mv_q1_0_f32.cl"); #endif cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); CL_CHECK((backend_ctx->kernel_mul_mv_q1_0_f32 = clCreateKernel(prog, "kernel_mul_mv_q1_0_f32", &err), err)); CL_CHECK(clReleaseProgram(prog)); GGML_LOG_CONT("."); } // mul_mv_q1_0_f32_flat { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { #include "mul_mv_q1_0_f32_flat.cl.h" }; #else const std::string kernel_src = read_file("mul_mv_q1_0_f32_flat.cl"); #endif cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); CL_CHECK((backend_ctx->kernel_mul_mv_q1_0_f32_flat = clCreateKernel(prog, "kernel_mul_mv_q1_0_f32_flat", &err), err)); CL_CHECK(clReleaseProgram(prog)); GGML_LOG_CONT("."); } // mul_mv_iq4_nl_f32 { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { #include "mul_mv_iq4_nl_f32.cl.h" }; #else const std::string kernel_src = read_file("mul_mv_iq4_nl_f32.cl"); #endif cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); CL_CHECK((backend_ctx->kernel_mul_mv_iq4_nl_f32 = clCreateKernel(prog, "kernel_mul_mv_iq4_nl_f32", &err), err)); CL_CHECK(clReleaseProgram(prog)); GGML_LOG_CONT("."); } // mul_mv_iq4_nl_f32_flat { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { #include "mul_mv_iq4_nl_f32_flat.cl.h" }; #else const std::string kernel_src = read_file("mul_mv_iq4_nl_f32_flat.cl"); #endif cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); CL_CHECK((backend_ctx->kernel_mul_mv_iq4_nl_f32_flat = clCreateKernel(prog, "kernel_mul_mv_iq4_nl_f32_flat", &err), err)); CL_CHECK(clReleaseProgram(prog)); GGML_LOG_CONT("."); } // mul_mv_mxfp4_f32 { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { #include "mul_mv_mxfp4_f32.cl.h" }; #else const std::string kernel_src = read_file("mul_mv_mxfp4_f32.cl"); #endif backend_ctx->program_mul_mv_mxfp4_f32 = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); CL_CHECK((backend_ctx->kernel_mul_mv_mxfp4_f32 = clCreateKernel(backend_ctx->program_mul_mv_mxfp4_f32, "kernel_mul_mv_mxfp4_f32", &err), err)); GGML_LOG_CONT("."); } // mul_mv_mxfp4_f32_flat { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { #include "mul_mv_mxfp4_f32_flat.cl.h" }; #else const std::string kernel_src = read_file("mul_mv_mxfp4_f32_flat.cl"); #endif backend_ctx->program_mul_mv_mxfp4_f32_flat = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); CL_CHECK((backend_ctx->kernel_mul_mv_mxfp4_f32_flat = clCreateKernel(backend_ctx->program_mul_mv_mxfp4_f32_flat, "kernel_mul_mv_mxfp4_f32_flat", &err), err)); GGML_LOG_CONT("."); } // mul_mv_f16_f16 { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { #include "mul_mv_f16_f16.cl.h" }; #else const std::string kernel_src = read_file("mul_mv_f16_f16.cl"); #endif backend_ctx->program_mul_mv_f16_f16 = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); CL_CHECK((backend_ctx->kernel_mul_mat_f16_f16 = clCreateKernel(backend_ctx->program_mul_mv_f16_f16, "kernel_mul_mat_f16_f16", &err), err)); GGML_LOG_CONT("."); } // mul_mv_f16_f32_1row { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { #include "mul_mv_f16_f32_1row.cl.h" }; #else const std::string kernel_src = read_file("mul_mv_f16_f32_1row.cl"); #endif backend_ctx->program_mul_mv_f16_f32_1row = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); CL_CHECK((backend_ctx->kernel_mul_mat_f16_f32_1row = clCreateKernel(backend_ctx->program_mul_mv_f16_f32_1row, "kernel_mul_mat_f16_f32_1row", &err), err)); GGML_LOG_CONT("."); } // mul_mv_f16_f32_mrow (multi-row decode GEMV) { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { #include "mul_mv_f16_f32_mrow.cl.h" }; #else const std::string kernel_src = read_file("mul_mv_f16_f32_mrow.cl"); #endif backend_ctx->program_mul_mv_f16_f32_mrow = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); CL_CHECK((backend_ctx->kernel_mul_mat_f16_f32_mrow = clCreateKernel(backend_ctx->program_mul_mv_f16_f32_mrow, "kernel_mul_mat_f16_f32_mrow", &err), err)); CL_CHECK((backend_ctx->kernel_mul_mat_f16_f32_mrow_r2 = clCreateKernel(backend_ctx->program_mul_mv_f16_f32_mrow, "kernel_mul_mat_f16_f32_mrow_r2", &err), err)); CL_CHECK((backend_ctx->kernel_mul_mat_f16_f32_mrow_r4 = clCreateKernel(backend_ctx->program_mul_mv_f16_f32_mrow, "kernel_mul_mat_f16_f32_mrow_r4", &err), err)); CL_CHECK((backend_ctx->kernel_mul_mat_f16_f32_mrow_h8 = clCreateKernel(backend_ctx->program_mul_mv_f16_f32_mrow, "kernel_mul_mat_f16_f32_mrow_h8", &err), err)); CL_CHECK((backend_ctx->kernel_mul_mat_f16_f32_mrow_h8r2 = clCreateKernel(backend_ctx->program_mul_mv_f16_f32_mrow, "kernel_mul_mat_f16_f32_mrow_h8r2", &err), err)); GGML_LOG_CONT("."); } // mul_mv_f16_f32_l4 { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { #include "mul_mv_f16_f32_l4.cl.h" }; #else const std::string kernel_src = read_file("mul_mv_f16_f32_l4.cl"); #endif backend_ctx->program_mul_mv_f16_f32_l4 = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); CL_CHECK((backend_ctx->kernel_mul_mat_f16_f32_l4 = clCreateKernel(backend_ctx->program_mul_mv_f16_f32_l4, "kernel_mul_mat_f16_f32_l4", &err), err)); CL_CHECK((backend_ctx->kernel_mul_mat_f16_f32_l4_dr = clCreateKernel(backend_ctx->program_mul_mv_f16_f32_l4, "kernel_mul_mat_f16_f32_l4_dr", &err), err)); if (backend_ctx->gpu_family == ADRENO) { CL_CHECK((backend_ctx->kernel_mul_mat_f16_f32_l4_dr_ls = clCreateKernel(backend_ctx->program_mul_mv_f16_f32_l4, "kernel_mul_mat_f16_f32_l4_dr_ls", &err), err)); CL_CHECK((backend_ctx->kernel_mul_mat_f16_f32_l4_dr_lq = clCreateKernel(backend_ctx->program_mul_mv_f16_f32_l4, "kernel_mul_mat_f16_f32_l4_dr_lq", &err), err)); } cl_int err_x8 = CL_SUCCESS; backend_ctx->kernel_mul_mat_f16_f32_l4_x8 = clCreateKernel(backend_ctx->program_mul_mv_f16_f32_l4, "kernel_mul_mat_f16_f32_l4_x8", &err_x8); if (err_x8 != CL_SUCCESS) { backend_ctx->kernel_mul_mat_f16_f32_l4_x8 = nullptr; } cl_int err_x8p = CL_SUCCESS; backend_ctx->kernel_mul_mat_f16_f32_l4_x8_pair = clCreateKernel(backend_ctx->program_mul_mv_f16_f32_l4, "kernel_mul_mat_f16_f32_l4_x8_pair", &err_x8p); if (err_x8p != CL_SUCCESS) { backend_ctx->kernel_mul_mat_f16_f32_l4_x8_pair = nullptr; } cl_int err_x8g = CL_SUCCESS; backend_ctx->kernel_mul_mat_f16_f32_l4_x8_gqa4 = clCreateKernel(backend_ctx->program_mul_mv_f16_f32_l4, "kernel_mul_mat_f16_f32_l4_x8_gqa4", &err_x8g); if (err_x8g != CL_SUCCESS) { backend_ctx->kernel_mul_mat_f16_f32_l4_x8_gqa4 = nullptr; } cl_int err_x8gi = CL_SUCCESS; backend_ctx->kernel_mul_mat_f16_f32_l4_x8_gqa4_img = clCreateKernel(backend_ctx->program_mul_mv_f16_f32_l4, "kernel_mul_mat_f16_f32_l4_x8_gqa4_img", &err_x8gi); if (err_x8gi != CL_SUCCESS) { backend_ctx->kernel_mul_mat_f16_f32_l4_x8_gqa4_img = nullptr; } cl_int err_x8gi_r4 = CL_SUCCESS; backend_ctx->kernel_mul_mat_f16_f32_l4_x8_gqa_r4_img = clCreateKernel(backend_ctx->program_mul_mv_f16_f32_l4, "kernel_mul_mat_f16_f32_l4_x8_gqa_r4_img", &err_x8gi_r4); if (err_x8gi_r4 != CL_SUCCESS) { backend_ctx->kernel_mul_mat_f16_f32_l4_x8_gqa_r4_img = nullptr; } cl_int err_r2dk256 = CL_SUCCESS; backend_ctx->kernel_mul_mat_f16_f32_l4_x8_gqa_r2_dk256_img = clCreateKernel(backend_ctx->program_mul_mv_f16_f32_l4, "kernel_mul_mat_f16_f32_l4_x8_gqa_r2_dk256_img", &err_r2dk256); if (err_r2dk256 != CL_SUCCESS) { backend_ctx->kernel_mul_mat_f16_f32_l4_x8_gqa_r2_dk256_img = nullptr; } cl_int err_y8 = CL_SUCCESS; backend_ctx->kernel_mul_mat_f16_f32_l4_y8 = clCreateKernel(backend_ctx->program_mul_mv_f16_f32_l4, "kernel_mul_mat_f16_f32_l4_y8", &err_y8); if (err_y8 != CL_SUCCESS) { backend_ctx->kernel_mul_mat_f16_f32_l4_y8 = nullptr; } cl_int err_y8g = CL_SUCCESS; backend_ctx->kernel_mul_mat_f16_f32_l4_y8_gqa = clCreateKernel(backend_ctx->program_mul_mv_f16_f32_l4, "kernel_mul_mat_f16_f32_l4_y8_gqa", &err_y8g); if (err_y8g != CL_SUCCESS) { backend_ctx->kernel_mul_mat_f16_f32_l4_y8_gqa = nullptr; } cl_int err_y8gi = CL_SUCCESS; backend_ctx->kernel_mul_mat_f16_f32_l4_y8_gqa_img = clCreateKernel(backend_ctx->program_mul_mv_f16_f32_l4, "kernel_mul_mat_f16_f32_l4_y8_gqa_img", &err_y8gi); if (err_y8gi != CL_SUCCESS) { backend_ctx->kernel_mul_mat_f16_f32_l4_y8_gqa_img = nullptr; } GGML_LOG_CONT("."); } // mul_mv_f16_f32 { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { #include "mul_mv_f16_f32.cl.h" }; #else const std::string kernel_src = read_file("mul_mv_f16_f32.cl"); #endif backend_ctx->program_mul_mv_f16_f32 = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); CL_CHECK((backend_ctx->kernel_mul_mat_f16_f32 = clCreateKernel(backend_ctx->program_mul_mv_f16_f32, "kernel_mul_mat_f16_f32", &err), err)); GGML_LOG_CONT("."); } // mul_mv_f32_f32 { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { #include "mul_mv_f32_f32.cl.h" }; #else const std::string kernel_src = read_file("mul_mv_f32_f32.cl"); #endif backend_ctx->program_mul_mv_f32_f32 = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); CL_CHECK((backend_ctx->kernel_mul_mat_f32_f32 = clCreateKernel(backend_ctx->program_mul_mv_f32_f32, "kernel_mul_mat_f32_f32", &err), err)); GGML_LOG_CONT("."); } // mul_mat_f16_f32_tiled { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { #include "mul_mat_f16_f32.cl.h" }; #else const std::string kernel_src = read_file("mul_mat_f16_f32.cl"); #endif backend_ctx->program_mul_mat_f16_f32_tiled = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); CL_CHECK((backend_ctx->kernel_mul_mat_f16_f32_tiled = clCreateKernel(backend_ctx->program_mul_mat_f16_f32_tiled, "mul_mat_f16_f32", &err), err)); GGML_LOG_CONT("."); } #ifdef GGML_OPENCL_USE_ADRENO_KERNELS // gemm_xmem_f16_f32_os8 { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { #include "gemm_xmem_f16_f32_os8.cl.h" }; #else const std::string kernel_src = read_file("gemm_xmem_f16_f32_os8.cl"); #endif cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); CL_CHECK((backend_ctx->kernel_adreno_xmem_pack_src_f32 = clCreateKernel(prog, "adreno_xmem_pack_src_f32", &err), err)); CL_CHECK((backend_ctx->kernel_adreno_xmem_prepack_weight_f16 = clCreateKernel(prog, "adreno_xmem_prepack_weight_f16", &err), err)); CL_CHECK((backend_ctx->kernel_gemm_xmem_f16_f32_os8 = clCreateKernel(prog, "kernel_gemm_xmem_f16_f32_os8", &err), err)); CL_CHECK((backend_ctx->kernel_adreno_xmem_store_dst_f32 = clCreateKernel(prog, "adreno_xmem_store_dst_f32", &err), err)); CL_CHECK(clReleaseProgram(prog)); GGML_LOG_CONT("."); } #endif // GGML_OPENCL_USE_ADRENO_KERNELS #ifdef GGML_OPENCL_USE_ADRENO_KERNELS // Adreno xmem SDPA if (backend_ctx->gpu_family == GPU_FAMILY::ADRENO) { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { #include "sdpa_xmem_f32_f16_os8.cl.h" }; #else const std::string kernel_src = read_file("sdpa_xmem_f32_f16_os8.cl"); #endif cl_program program = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); auto & xmem_attn = backend_ctx->adreno_xmem_attn; CL_CHECK((xmem_attn.kernel_q_f32_to_img_scaled = clCreateKernel(program, "adreno_xmem_attn_q_f32_to_img_scaled", &err), err)); CL_CHECK((xmem_attn.kernel_kv_f32_to_img_gqa = clCreateKernel(program, "adreno_xmem_attn_kv_f32_to_img_gqa", &err), err)); CL_CHECK((xmem_attn.kernel_kv_f16_to_img_gqa = clCreateKernel(program, "adreno_xmem_attn_kv_f16_to_img_gqa", &err), err)); CL_CHECK((xmem_attn.kernel_img_to_f32 = clCreateKernel(program, "adreno_xmem_attn_img_to_f32", &err), err)); CL_CHECK((xmem_attn.kernel_k_gather = clCreateKernel(program, "adreno_xmem_attn_k_gather", &err), err)); CL_CHECK((xmem_attn.kernel_pack_k = clCreateKernel(program, "adreno_xmem_attn_pack_k", &err), err)); CL_CHECK((xmem_attn.kernel_qk_gemm = clCreateKernel(program, "adreno_xmem_attn_qk_gemm", &err), err)); CL_CHECK((xmem_attn.kernel_softmax_reduce_basic = clCreateKernel(program, "adreno_xmem_attn_softmax_reduce_basic", &err), err)); CL_CHECK((xmem_attn.kernel_softmax_apply_basic = clCreateKernel(program, "adreno_xmem_attn_softmax_apply_basic", &err), err)); CL_CHECK((xmem_attn.kernel_mask_scores = clCreateKernel(program, "adreno_xmem_attn_mask_scores", &err), err)); CL_CHECK((xmem_attn.kernel_pack_v = clCreateKernel(program, "adreno_xmem_attn_pack_v", &err), err)); CL_CHECK((xmem_attn.kernel_pv_gemm = clCreateKernel(program, "adreno_xmem_attn_pv_gemm", &err), err)); CL_CHECK(clReleaseProgram(program)); xmem_attn.compiled = true; GGML_LOG_CONT("."); } #endif // GGML_OPENCL_USE_ADRENO_KERNELS // mul_mm_f32_f32_l4_lm { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { #include "mul_mm_f32_f32_l4_lm.cl.h" }; #else const std::string kernel_src = read_file("mul_mm_f32_f32_l4_lm.cl"); #endif backend_ctx->program_mul_mm_f32_f32_l4_lm = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); CL_CHECK((backend_ctx->kernel_mul_mm_f32_f32_l4_lm = clCreateKernel(backend_ctx->program_mul_mm_f32_f32_l4_lm, "kernel_mul_mm_f32_f32_l4_lm", &err), err)); CL_CHECK((backend_ctx->kernel_gemv_f32_f32_mc = clCreateKernel(backend_ctx->program_mul_mm_f32_f32_l4_lm, "kernel_gemv_f32_f32_mc", &err), err)); GGML_LOG_CONT("."); } // mul_mm_f16_f32_l4_lm { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { #include "mul_mm_f16_f32_l4_lm.cl.h" }; #else const std::string kernel_src = read_file("mul_mm_f16_f32_l4_lm.cl"); #endif backend_ctx->program_mul_mm_f16_f32_l4_lm = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); CL_CHECK((backend_ctx->kernel_mul_mm_f16_f32_l4_lm = clCreateKernel(backend_ctx->program_mul_mm_f16_f32_l4_lm, "kernel_mul_mm_f16_f32_l4_lm", &err), err)); GGML_LOG_CONT("."); } // mul_mm_q4_0_f32_l4_lm { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { #include "mul_mm_q4_0_f32_l4_lm.cl.h" }; #else const std::string kernel_src = read_file("mul_mm_q4_0_f32_l4_lm.cl"); #endif cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); CL_CHECK((backend_ctx->kernel_mul_mm_q4_0_f32_l4_lm = clCreateKernel(prog, "kernel_mul_mm_q4_0_f32_l4_lm", &err), err)); GGML_LOG_CONT("."); } // mul_mm_q4_1_f32_l4_lm { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { #include "mul_mm_q4_1_f32_l4_lm.cl.h" }; #else const std::string kernel_src = read_file("mul_mm_q4_1_f32_l4_lm.cl"); #endif cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); CL_CHECK((backend_ctx->kernel_mul_mm_q4_1_f32_l4_lm = clCreateKernel(prog, "kernel_mul_mm_q4_1_f32_l4_lm", &err), err)); GGML_LOG_CONT("."); } // mul_mm_q5_0_f32_l4_lm { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { #include "mul_mm_q5_0_f32_l4_lm.cl.h" }; #else const std::string kernel_src = read_file("mul_mm_q5_0_f32_l4_lm.cl"); #endif cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); CL_CHECK((backend_ctx->kernel_mul_mm_q5_0_f32_l4_lm = clCreateKernel(prog, "kernel_mul_mm_q5_0_f32_l4_lm", &err), err)); GGML_LOG_CONT("."); } // mul_mm_q5_1_f32_l4_lm { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { #include "mul_mm_q5_1_f32_l4_lm.cl.h" }; #else const std::string kernel_src = read_file("mul_mm_q5_1_f32_l4_lm.cl"); #endif cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); CL_CHECK((backend_ctx->kernel_mul_mm_q5_1_f32_l4_lm = clCreateKernel(prog, "kernel_mul_mm_q5_1_f32_l4_lm", &err), err)); GGML_LOG_CONT("."); } // mul_mm_q8_0_f32_l4_lm { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { #include "mul_mm_q8_0_f32_l4_lm.cl.h" }; #else const std::string kernel_src = read_file("mul_mm_q8_0_f32_l4_lm.cl"); #endif backend_ctx->program_mul_mm_q8_0_f32_l4_lm = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); CL_CHECK((backend_ctx->kernel_mul_mm_q8_0_f32_l4_lm = clCreateKernel(backend_ctx->program_mul_mm_q8_0_f32_l4_lm, "kernel_mul_mm_q8_0_f32_l4_lm", &err), err)); GGML_LOG_CONT("."); } // mul_mm_q1_0_f32_l4_lm { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { #include "mul_mm_q1_0_f32_l4_lm.cl.h" }; #else const std::string kernel_src = read_file("mul_mm_q1_0_f32_l4_lm.cl"); #endif cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); CL_CHECK((backend_ctx->kernel_mul_mm_q1_0_f32_l4_lm = clCreateKernel(prog, "kernel_mul_mm_q1_0_f32_l4_lm", &err), err)); CL_CHECK(clReleaseProgram(prog)); GGML_LOG_CONT("."); } // mul_mm_iq4_nl_f32_l4_lm { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { #include "mul_mm_iq4_nl_f32_l4_lm.cl.h" }; #else const std::string kernel_src = read_file("mul_mm_iq4_nl_f32_l4_lm.cl"); #endif cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); CL_CHECK((backend_ctx->kernel_mul_mm_iq4_nl_f32_l4_lm = clCreateKernel(prog, "kernel_mul_mm_iq4_nl_f32_l4_lm", &err), err)); CL_CHECK(clReleaseProgram(prog)); GGML_LOG_CONT("."); } // mul_mm_q4_k_f32_l4_lm { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { #include "mul_mm_q4_k_f32_l4_lm.cl.h" }; #else const std::string kernel_src = read_file("mul_mm_q4_k_f32_l4_lm.cl"); #endif cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); CL_CHECK((backend_ctx->kernel_mul_mm_q4_k_f32_l4_lm = clCreateKernel(prog, "kernel_mul_mm_q4_k_f32_l4_lm", &err), err)); CL_CHECK(clReleaseProgram(prog)); GGML_LOG_CONT("."); } // mul_mm_q6_k_f32_l4_lm { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { #include "mul_mm_q6_k_f32_l4_lm.cl.h" }; #else const std::string kernel_src = read_file("mul_mm_q6_k_f32_l4_lm.cl"); #endif cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); CL_CHECK((backend_ctx->kernel_mul_mm_q6_k_f32_l4_lm = clCreateKernel(prog, "kernel_mul_mm_q6_k_f32_l4_lm", &err), err)); CL_CHECK(clReleaseProgram(prog)); GGML_LOG_CONT("."); } // mul_mm_q5_k_f32_l4_lm { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { #include "mul_mm_q5_k_f32_l4_lm.cl.h" }; #else const std::string kernel_src = read_file("mul_mm_q5_k_f32_l4_lm.cl"); #endif cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); CL_CHECK((backend_ctx->kernel_mul_mm_q5_k_f32_l4_lm = clCreateKernel(prog, "kernel_mul_mm_q5_k_f32_l4_lm", &err), err)); CL_CHECK(clReleaseProgram(prog)); GGML_LOG_CONT("."); } // mul_mm_f16_f32_kq_kqv { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { #include "mul_mm_f16_f32_kq_kqv.cl.h" }; #else const std::string kernel_src = read_file("mul_mm_f16_f32_kq_kqv.cl"); #endif backend_ctx->program_mul_mm_f16_f32_kqv = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts+" -DKQV "); backend_ctx->program_mul_mm_f16_f32_kq = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); CL_CHECK((backend_ctx->kernel_mul_mm_f16_f32_kqv = clCreateKernel(backend_ctx->program_mul_mm_f16_f32_kqv, "mul_mm_f16_f32_kqv", &err), err)); CL_CHECK((backend_ctx->kernel_mul_mm_f16_f32_kq = clCreateKernel(backend_ctx->program_mul_mm_f16_f32_kq, "mul_mm_f16_f32_kq", &err), err)); GGML_LOG_CONT("."); } // mul { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { #include "mul.cl.h" }; #else const std::string kernel_src = read_file("mul.cl"); #endif backend_ctx->program_mul = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); CL_CHECK((backend_ctx->kernel_mul = clCreateKernel(backend_ctx->program_mul, "kernel_mul", &err), err)); CL_CHECK((backend_ctx->kernel_mul_row = clCreateKernel(backend_ctx->program_mul, "kernel_mul_row", &err), err)); CL_CHECK((backend_ctx->kernel_mul_f16 = clCreateKernel(backend_ctx->program_mul, "kernel_mul_f16", &err), err)); CL_CHECK((backend_ctx->kernel_mul_row_f16 = clCreateKernel(backend_ctx->program_mul, "kernel_mul_row_f16", &err), err)); GGML_LOG_CONT("."); } // norm { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { #include "norm.cl.h" }; #else const std::string kernel_src = read_file("norm.cl"); #endif backend_ctx->program_norm = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); CL_CHECK((backend_ctx->kernel_norm = clCreateKernel(backend_ctx->program_norm, "kernel_norm", &err), err)); CL_CHECK((backend_ctx->kernel_norm_mul_add = clCreateKernel(backend_ctx->program_norm, "kernel_norm_mul_add", &err), err)); GGML_LOG_CONT("."); } // relu { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { #include "relu.cl.h" }; #else const std::string kernel_src = read_file("relu.cl"); #endif backend_ctx->program_relu = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); CL_CHECK((backend_ctx->kernel_relu = clCreateKernel(backend_ctx->program_relu, "kernel_relu", &err), err)); GGML_LOG_CONT("."); } // rms_norm { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { #include "rms_norm.cl.h" }; #else const std::string kernel_src = read_file("rms_norm.cl"); #endif backend_ctx->program_rms_norm = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); CL_CHECK((backend_ctx->kernel_rms_norm = clCreateKernel(backend_ctx->program_rms_norm, "kernel_rms_norm", &err), err)); CL_CHECK((backend_ctx->kernel_rms_norm_mul = clCreateKernel(backend_ctx->program_rms_norm, "kernel_rms_norm_mul", &err), err)); CL_CHECK((backend_ctx->kernel_rms_norm_mul_add = clCreateKernel(backend_ctx->program_rms_norm, "kernel_rms_norm_mul_add", &err), err)); GGML_LOG_CONT("."); } // l2_norm { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { #include "l2_norm.cl.h" }; #else const std::string kernel_src = read_file("l2_norm.cl"); #endif cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); CL_CHECK((backend_ctx->kernel_l2_norm_f32 = clCreateKernel(prog, "kernel_l2_norm_f32", &err), err)); CL_CHECK(clReleaseProgram(prog)); GGML_LOG_CONT("."); } // rope { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { #include "rope.cl.h" }; #else const std::string kernel_src = read_file("rope.cl"); #endif backend_ctx->program_rope = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); CL_CHECK((backend_ctx->kernel_rope_norm_f32 = clCreateKernel(backend_ctx->program_rope, "kernel_rope_norm_f32", &err), err)); CL_CHECK((backend_ctx->kernel_rope_norm_f16 = clCreateKernel(backend_ctx->program_rope, "kernel_rope_norm_f16", &err), err)); CL_CHECK((backend_ctx->kernel_rope_neox_f32 = clCreateKernel(backend_ctx->program_rope, "kernel_rope_neox_f32", &err), err)); CL_CHECK((backend_ctx->kernel_rope_neox_f16 = clCreateKernel(backend_ctx->program_rope, "kernel_rope_neox_f16", &err), err)); CL_CHECK((backend_ctx->kernel_rope_multi_f32 = clCreateKernel(backend_ctx->program_rope, "kernel_rope_multi_f32", &err), err)); CL_CHECK((backend_ctx->kernel_rope_multi_f16 = clCreateKernel(backend_ctx->program_rope, "kernel_rope_multi_f16", &err), err)); CL_CHECK((backend_ctx->kernel_rope_vision_f32 = clCreateKernel(backend_ctx->program_rope, "kernel_rope_vision_f32", &err), err)); CL_CHECK((backend_ctx->kernel_rope_vision_f16 = clCreateKernel(backend_ctx->program_rope, "kernel_rope_vision_f16", &err), err)); GGML_LOG_CONT("."); } // scale { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { #include "scale.cl.h" }; #else const std::string kernel_src = read_file("scale.cl"); #endif cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); CL_CHECK((backend_ctx->kernel_scale_f32 = clCreateKernel(prog, "kernel_scale_f32", &err), err)); CL_CHECK((backend_ctx->kernel_scale_f32_4 = clCreateKernel(prog, "kernel_scale_f32_4", &err), err)); CL_CHECK(clReleaseProgram(prog)); GGML_LOG_CONT("."); } // silu { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { #include "silu.cl.h" }; #else const std::string kernel_src = read_file("silu.cl"); #endif backend_ctx->program_silu = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); CL_CHECK((backend_ctx->kernel_silu = clCreateKernel(backend_ctx->program_silu, "kernel_silu", &err), err)); CL_CHECK((backend_ctx->kernel_silu_4 = clCreateKernel(backend_ctx->program_silu, "kernel_silu_4", &err), err)); GGML_LOG_CONT("."); } // softmax_f32 { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { #include "softmax_f32.cl.h" }; #else const std::string kernel_src = read_file("softmax_f32.cl"); #endif backend_ctx->program_softmax_f32 = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); CL_CHECK((backend_ctx->kernel_soft_max = clCreateKernel(backend_ctx->program_softmax_f32, "kernel_soft_max", &err), err)); GGML_LOG_CONT("."); } // softmax_f16 { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { #include "softmax_f16.cl.h" }; #else const std::string kernel_src = read_file("softmax_f16.cl"); #endif backend_ctx->program_softmax_f16 = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); CL_CHECK((backend_ctx->kernel_soft_max_f16 = clCreateKernel(backend_ctx->program_softmax_f16, "kernel_soft_max_f16", &err), err)); GGML_LOG_CONT("."); } // softmax_4_f32 { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { #include "softmax_4_f32.cl.h" }; #else const std::string kernel_src = read_file("softmax_4_f32.cl"); #endif backend_ctx->program_softmax_4_f32 = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); CL_CHECK((backend_ctx->kernel_soft_max_4 = clCreateKernel(backend_ctx->program_softmax_4_f32, "kernel_soft_max_4", &err), err)); GGML_LOG_CONT("."); } // softmax_4_f16 { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { #include "softmax_4_f16.cl.h" }; #else const std::string kernel_src = read_file("softmax_4_f16.cl"); #endif backend_ctx->program_softmax_4_f16 = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); CL_CHECK((backend_ctx->kernel_soft_max_4_f16 = clCreateKernel(backend_ctx->program_softmax_4_f16, "kernel_soft_max_4_f16", &err), err)); GGML_LOG_CONT("."); } // div { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { #include "div.cl.h" }; #else const std::string kernel_src = read_file("div.cl"); #endif std::string compile_opts = std::string("-cl-std=") + opencl_c_std + " -cl-mad-enable -cl-finite-math-only "; backend_ctx->program_div = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); CL_CHECK((backend_ctx->kernel_div = clCreateKernel(backend_ctx->program_div, "kernel_div", &err), err)); CL_CHECK((backend_ctx->kernel_div_row = clCreateKernel(backend_ctx->program_div, "kernel_div_row", &err), err)); CL_CHECK((backend_ctx->kernel_div_f16 = clCreateKernel(backend_ctx->program_div, "kernel_div_f16", &err), err)); CL_CHECK((backend_ctx->kernel_div_row_f16 = clCreateKernel(backend_ctx->program_div, "kernel_div_row_f16", &err), err)); GGML_LOG_CONT("."); } // sqr { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { #include "sqr.cl.h" }; #else const std::string kernel_src = read_file("sqr.cl"); #endif cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); CL_CHECK((backend_ctx->kernel_sqr_cont_f32 = clCreateKernel(prog, "kernel_sqr_cont_f32", &err), err)); CL_CHECK((backend_ctx->kernel_sqr_cont_f32_4 = clCreateKernel(prog, "kernel_sqr_cont_f32_4", &err), err)); CL_CHECK((backend_ctx->kernel_sqr_cont_f16 = clCreateKernel(prog, "kernel_sqr_cont_f16", &err), err)); CL_CHECK((backend_ctx->kernel_sqr_cont_f16_4 = clCreateKernel(prog, "kernel_sqr_cont_f16_4", &err), err)); CL_CHECK(clReleaseProgram(prog)); GGML_LOG_CONT("."); } // sqrt { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { #include "sqrt.cl.h" }; #else const std::string kernel_src = read_file("sqrt.cl"); #endif cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); CL_CHECK((backend_ctx->kernel_sqrt_cont_f32 = clCreateKernel(prog, "kernel_sqrt_cont_f32", &err), err)); CL_CHECK((backend_ctx->kernel_sqrt_cont_f32_4 = clCreateKernel(prog, "kernel_sqrt_cont_f32_4", &err), err)); CL_CHECK((backend_ctx->kernel_sqrt_cont_f16 = clCreateKernel(prog, "kernel_sqrt_cont_f16", &err), err)); CL_CHECK((backend_ctx->kernel_sqrt_cont_f16_4 = clCreateKernel(prog, "kernel_sqrt_cont_f16_4", &err), err)); CL_CHECK(clReleaseProgram(prog)); GGML_LOG_CONT("."); } // mean { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { #include "mean.cl.h" }; #else const std::string kernel_src = read_file("mean.cl"); #endif cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); CL_CHECK((backend_ctx->kernel_mean_f32 = clCreateKernel(prog, "kernel_mean_f32", &err), err)); CL_CHECK((backend_ctx->kernel_mean_f32_4 = clCreateKernel(prog, "kernel_mean_f32_4", &err), err)); CL_CHECK(clReleaseProgram(prog)); GGML_LOG_CONT("."); } // sub { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { #include "sub.cl.h" }; #else const std::string kernel_src = read_file("sub.cl"); #endif backend_ctx->program_sub = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); CL_CHECK((backend_ctx->kernel_sub = clCreateKernel(backend_ctx->program_sub, "kernel_sub", &err), err)); CL_CHECK((backend_ctx->kernel_sub_row = clCreateKernel(backend_ctx->program_sub, "kernel_sub_row", &err), err)); CL_CHECK((backend_ctx->kernel_sub_f16 = clCreateKernel(backend_ctx->program_sub, "kernel_sub_f16", &err), err)); CL_CHECK((backend_ctx->kernel_sub_row_f16 = clCreateKernel(backend_ctx->program_sub, "kernel_sub_row_f16", &err), err)); GGML_LOG_CONT("."); } // sum_rows { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { #include "sum_rows.cl.h" }; #else const std::string kernel_src = read_file("sum_rows.cl"); #endif backend_ctx->program_sum_rows_f32 = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); CL_CHECK((backend_ctx->kernel_sum_rows_f32 = clCreateKernel(backend_ctx->program_sum_rows_f32, "kernel_sum_rows_f32", &err), err)); CL_CHECK((backend_ctx->kernel_sum_rows_f32_4 = clCreateKernel(backend_ctx->program_sum_rows_f32, "kernel_sum_rows_f32_4", &err), err)); GGML_LOG_CONT("."); } // cumsum { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { #include "cumsum.cl.h" }; #else const std::string kernel_src = read_file("cumsum.cl"); #endif cl_program prog; prog = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); CL_CHECK((backend_ctx->kernel_cumsum_blk = clCreateKernel(prog, "kernel_cumsum_blk", &err), err)); CL_CHECK((backend_ctx->kernel_cumsum_add = clCreateKernel(prog, "kernel_cumsum_add", &err), err)); GGML_LOG_CONT("."); CL_CHECK(clReleaseProgram(prog)); } // sigmoid { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { #include "sigmoid.cl.h" }; #else const std::string kernel_src = read_file("sigmoid.cl"); #endif backend_ctx->program_sigmoid = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); CL_CHECK((backend_ctx->kernel_sigmoid_f32 = clCreateKernel(backend_ctx->program_sigmoid, "kernel_sigmoid_f32", &err), err)); CL_CHECK((backend_ctx->kernel_sigmoid_f16 = clCreateKernel(backend_ctx->program_sigmoid, "kernel_sigmoid_f16", &err), err)); GGML_LOG_CONT("."); } // group_norm { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { #include "group_norm.cl.h" }; #else const std::string kernel_src = read_file("group_norm.cl"); #endif backend_ctx->program_group_norm = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); CL_CHECK((backend_ctx->kernel_group_norm = clCreateKernel(backend_ctx->program_group_norm, "kernel_group_norm", &err), err)); CL_CHECK((backend_ctx->kernel_group_norm_mul_add = clCreateKernel(backend_ctx->program_group_norm, "kernel_group_norm_mul_add", &err), err)); GGML_LOG_CONT("."); } // repeat { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { #include "repeat.cl.h" }; #else const std::string kernel_src = read_file("repeat.cl"); #endif cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); CL_CHECK((backend_ctx->kernel_repeat_f32 = clCreateKernel(prog, "kernel_repeat_f32", &err), err)); CL_CHECK(clReleaseProgram(prog)); GGML_LOG_CONT("."); } // pad { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { #include "pad.cl.h" }; #else const std::string kernel_src = read_file("pad.cl"); #endif if (!kernel_src.empty()) { backend_ctx->program_pad = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); CL_CHECK((backend_ctx->kernel_pad = clCreateKernel(backend_ctx->program_pad, "kernel_pad", &err), err)); GGML_LOG_CONT("."); } else { GGML_LOG_WARN("ggml_opencl: pad kernel source not found or empty. Pad operations will not be available.\n"); backend_ctx->program_pad = nullptr; backend_ctx->kernel_pad = nullptr; } } // tanh { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { #include "tanh.cl.h" }; #else const std::string kernel_src = read_file("tanh.cl"); #endif cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); CL_CHECK((backend_ctx->kernel_tanh_f32 = clCreateKernel(prog, "kernel_tanh_f32", &err), err)); CL_CHECK((backend_ctx->kernel_tanh_f32_4 = clCreateKernel(prog, "kernel_tanh_f32_4", &err), err)); CL_CHECK((backend_ctx->kernel_tanh_f32_nc = clCreateKernel(prog, "kernel_tanh_f32_nc", &err), err)); CL_CHECK((backend_ctx->kernel_tanh_f16 = clCreateKernel(prog, "kernel_tanh_f16", &err), err)); CL_CHECK((backend_ctx->kernel_tanh_f16_4 = clCreateKernel(prog, "kernel_tanh_f16_4", &err), err)); CL_CHECK((backend_ctx->kernel_tanh_f16_nc = clCreateKernel(prog, "kernel_tanh_f16_nc", &err), err)); CL_CHECK(clReleaseProgram(prog)); GGML_LOG_CONT("."); } // neg { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { #include "neg.cl.h" }; #else const std::string kernel_src = read_file("neg.cl"); #endif cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); CL_CHECK((backend_ctx->kernel_neg_f32 = clCreateKernel(prog, "kernel_neg_f32", &err), err)); CL_CHECK((backend_ctx->kernel_neg_f32_4 = clCreateKernel(prog, "kernel_neg_f32_4", &err), err)); CL_CHECK((backend_ctx->kernel_neg_f32_nc = clCreateKernel(prog, "kernel_neg_f32_nc", &err), err)); CL_CHECK((backend_ctx->kernel_neg_f16 = clCreateKernel(prog, "kernel_neg_f16", &err), err)); CL_CHECK((backend_ctx->kernel_neg_f16_4 = clCreateKernel(prog, "kernel_neg_f16_4", &err), err)); CL_CHECK((backend_ctx->kernel_neg_f16_nc = clCreateKernel(prog, "kernel_neg_f16_nc", &err), err)); CL_CHECK(clReleaseProgram(prog)); GGML_LOG_CONT("."); } // exp { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { #include "exp.cl.h" }; #else const std::string kernel_src = read_file("exp.cl"); #endif cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); CL_CHECK((backend_ctx->kernel_exp_f32 = clCreateKernel(prog, "kernel_exp_f32", &err), err)); CL_CHECK((backend_ctx->kernel_exp_f32_4 = clCreateKernel(prog, "kernel_exp_f32_4", &err), err)); CL_CHECK((backend_ctx->kernel_exp_f32_nc = clCreateKernel(prog, "kernel_exp_f32_nc", &err), err)); CL_CHECK((backend_ctx->kernel_exp_f16 = clCreateKernel(prog, "kernel_exp_f16", &err), err)); CL_CHECK((backend_ctx->kernel_exp_f16_4 = clCreateKernel(prog, "kernel_exp_f16_4", &err), err)); CL_CHECK((backend_ctx->kernel_exp_f16_nc = clCreateKernel(prog, "kernel_exp_f16_nc", &err), err)); CL_CHECK(clReleaseProgram(prog)); GGML_LOG_CONT("."); } // expm1 { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { #include "expm1.cl.h" }; #else const std::string kernel_src = read_file("expm1.cl"); #endif cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); CL_CHECK((backend_ctx->kernel_expm1_f32 = clCreateKernel(prog, "kernel_expm1_f32", &err), err)); CL_CHECK((backend_ctx->kernel_expm1_f32_4 = clCreateKernel(prog, "kernel_expm1_f32_4", &err), err)); CL_CHECK((backend_ctx->kernel_expm1_f32_nc = clCreateKernel(prog, "kernel_expm1_f32_nc", &err), err)); CL_CHECK((backend_ctx->kernel_expm1_f16 = clCreateKernel(prog, "kernel_expm1_f16", &err), err)); CL_CHECK((backend_ctx->kernel_expm1_f16_4 = clCreateKernel(prog, "kernel_expm1_f16_4", &err), err)); CL_CHECK((backend_ctx->kernel_expm1_f16_nc = clCreateKernel(prog, "kernel_expm1_f16_nc", &err), err)); CL_CHECK(clReleaseProgram(prog)); GGML_LOG_CONT("."); } // abs { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { #include "abs.cl.h" }; #else const std::string kernel_src = read_file("abs.cl"); #endif cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); CL_CHECK((backend_ctx->kernel_abs_f32 = clCreateKernel(prog, "kernel_abs_f32", &err), err)); CL_CHECK((backend_ctx->kernel_abs_f32_4 = clCreateKernel(prog, "kernel_abs_f32_4", &err), err)); CL_CHECK((backend_ctx->kernel_abs_f32_nc = clCreateKernel(prog, "kernel_abs_f32_nc", &err), err)); CL_CHECK((backend_ctx->kernel_abs_f16 = clCreateKernel(prog, "kernel_abs_f16", &err), err)); CL_CHECK((backend_ctx->kernel_abs_f16_4 = clCreateKernel(prog, "kernel_abs_f16_4", &err), err)); CL_CHECK((backend_ctx->kernel_abs_f16_nc = clCreateKernel(prog, "kernel_abs_f16_nc", &err), err)); CL_CHECK(clReleaseProgram(prog)); GGML_LOG_CONT("."); } // unary_ext (sgn, step, elu, hardswish, hardsigmoid, floor, ceil, round, trunc) { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { #include "unary_ext.cl.h" }; #else const std::string kernel_src = read_file("unary_ext.cl"); #endif cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); #define CL_UNARY_EXT_K(op) \ CL_CHECK((backend_ctx->kernel_##op##_f32 = clCreateKernel(prog, "kernel_" #op "_f32", &err), err)); \ CL_CHECK((backend_ctx->kernel_##op##_f32_4 = clCreateKernel(prog, "kernel_" #op "_f32_4", &err), err)); \ CL_CHECK((backend_ctx->kernel_##op##_f32_nc = clCreateKernel(prog, "kernel_" #op "_f32_nc", &err), err)); \ CL_CHECK((backend_ctx->kernel_##op##_f16 = clCreateKernel(prog, "kernel_" #op "_f16", &err), err)); \ CL_CHECK((backend_ctx->kernel_##op##_f16_4 = clCreateKernel(prog, "kernel_" #op "_f16_4", &err), err)); \ CL_CHECK((backend_ctx->kernel_##op##_f16_nc = clCreateKernel(prog, "kernel_" #op "_f16_nc", &err), err)); CL_UNARY_EXT_K(sgn) CL_UNARY_EXT_K(step) CL_UNARY_EXT_K(elu) CL_UNARY_EXT_K(hardswish) CL_UNARY_EXT_K(hardsigmoid) CL_UNARY_EXT_K(floor) CL_UNARY_EXT_K(ceil) CL_UNARY_EXT_K(round) CL_UNARY_EXT_K(trunc) #undef CL_UNARY_EXT_K CL_CHECK(clReleaseProgram(prog)); GGML_LOG_CONT("."); } // softplus { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { #include "softplus.cl.h" }; #else const std::string kernel_src = read_file("softplus.cl"); #endif cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); CL_CHECK((backend_ctx->kernel_softplus_f32 = clCreateKernel(prog, "kernel_softplus_f32", &err), err)); CL_CHECK((backend_ctx->kernel_softplus_f32_4 = clCreateKernel(prog, "kernel_softplus_f32_4", &err), err)); CL_CHECK((backend_ctx->kernel_softplus_f32_nc = clCreateKernel(prog, "kernel_softplus_f32_nc", &err), err)); CL_CHECK((backend_ctx->kernel_softplus_f16 = clCreateKernel(prog, "kernel_softplus_f16", &err), err)); CL_CHECK((backend_ctx->kernel_softplus_f16_4 = clCreateKernel(prog, "kernel_softplus_f16_4", &err), err)); CL_CHECK((backend_ctx->kernel_softplus_f16_nc = clCreateKernel(prog, "kernel_softplus_f16_nc", &err), err)); CL_CHECK(clReleaseProgram(prog)); GGML_LOG_CONT("."); } // upscale { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { #include "upscale.cl.h" }; #else const std::string kernel_src = read_file("upscale.cl"); #endif if (!kernel_src.empty()) { backend_ctx->program_upscale = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); CL_CHECK((backend_ctx->kernel_upscale = clCreateKernel(backend_ctx->program_upscale, "kernel_upscale", &err), err)); if (backend_ctx->program_upscale) { cl_int err_bilinear; backend_ctx->kernel_upscale_bilinear = clCreateKernel(backend_ctx->program_upscale, "kernel_upscale_bilinear", &err_bilinear); if (err_bilinear != CL_SUCCESS) { GGML_LOG_WARN("ggml_opencl: kernel_upscale_bilinear not found in upscale.cl. Bilinear upscale will not be available. Error: %d\n", err_bilinear); backend_ctx->kernel_upscale_bilinear = nullptr; } } else { backend_ctx->kernel_upscale_bilinear = nullptr; } GGML_LOG_CONT("."); } else { GGML_LOG_WARN("ggml_opencl: upscale kernel source not found or empty. Upscale operations will not be available.\n"); backend_ctx->program_upscale = nullptr; backend_ctx->kernel_upscale = nullptr; backend_ctx->kernel_upscale_bilinear = nullptr; } } // concat { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { #include "concat.cl.h" }; #else const std::string kernel_src = read_file("concat.cl"); #endif cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); CL_CHECK((backend_ctx->kernel_concat_b1 = clCreateKernel(prog, "kernel_concat_b1", &err), err)); CL_CHECK((backend_ctx->kernel_concat_b2 = clCreateKernel(prog, "kernel_concat_b2", &err), err)); CL_CHECK((backend_ctx->kernel_concat_b4 = clCreateKernel(prog, "kernel_concat_b4", &err), err)); CL_CHECK((backend_ctx->kernel_concat_b8 = clCreateKernel(prog, "kernel_concat_b8", &err), err)); CL_CHECK((backend_ctx->kernel_concat_b4_pack = clCreateKernel(prog, "kernel_concat_b4_pack", &err), err)); CL_CHECK(clReleaseProgram(prog)); GGML_LOG_CONT("."); } // timestep_embedding { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { #include "tsembd.cl.h" }; #else const std::string kernel_src = read_file("tsembd.cl"); #endif if (!kernel_src.empty()) { backend_ctx->program_tsembd = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); CL_CHECK((backend_ctx->kernel_timestep_embedding = clCreateKernel(backend_ctx->program_tsembd, "kernel_timestep_embedding", &err), err)); GGML_LOG_CONT("."); } else { GGML_LOG_WARN("ggml_opencl: timestep_embedding kernel source not found or empty. This op will not be available.\n"); backend_ctx->program_tsembd = nullptr; backend_ctx->kernel_timestep_embedding = nullptr; } } // set_rows { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { #include "set_rows.cl.h" }; #else const std::string kernel_src = read_file("set_rows.cl"); #endif backend_ctx->program_set_rows = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); CL_CHECK((backend_ctx->kernel_set_rows_f32_i64 = clCreateKernel(backend_ctx->program_set_rows, "kernel_set_rows_f32_i64", &err), err)); CL_CHECK((backend_ctx->kernel_set_rows_f32_i32 = clCreateKernel(backend_ctx->program_set_rows, "kernel_set_rows_f32_i32", &err), err)); CL_CHECK((backend_ctx->kernel_set_rows_f16_i64 = clCreateKernel(backend_ctx->program_set_rows, "kernel_set_rows_f16_i64", &err), err)); CL_CHECK((backend_ctx->kernel_set_rows_f16_i32 = clCreateKernel(backend_ctx->program_set_rows, "kernel_set_rows_f16_i32", &err), err)); CL_CHECK((backend_ctx->kernel_set_rows_q8_0_i64 = clCreateKernel(backend_ctx->program_set_rows, "kernel_set_rows_q8_0_i64", &err), err)); CL_CHECK((backend_ctx->kernel_set_rows_q8_0_i32 = clCreateKernel(backend_ctx->program_set_rows, "kernel_set_rows_q8_0_i32", &err), err)); CL_CHECK((backend_ctx->kernel_set_rows_q8_0_soa_i64 = clCreateKernel(backend_ctx->program_set_rows, "kernel_set_rows_q8_0_soa_i64", &err), err)); CL_CHECK((backend_ctx->kernel_set_rows_q8_0_soa_i32 = clCreateKernel(backend_ctx->program_set_rows, "kernel_set_rows_q8_0_soa_i32", &err), err)); CL_CHECK((backend_ctx->kernel_set_rows_q4_0_i64 = clCreateKernel(backend_ctx->program_set_rows, "kernel_set_rows_q4_0_i64", &err), err)); CL_CHECK((backend_ctx->kernel_set_rows_q4_0_i32 = clCreateKernel(backend_ctx->program_set_rows, "kernel_set_rows_q4_0_i32", &err), err)); CL_CHECK((backend_ctx->kernel_set_rows_q4_0_soa_i64 = clCreateKernel(backend_ctx->program_set_rows, "kernel_set_rows_q4_0_soa_i64", &err), err)); CL_CHECK((backend_ctx->kernel_set_rows_q4_0_soa_i32 = clCreateKernel(backend_ctx->program_set_rows, "kernel_set_rows_q4_0_soa_i32", &err), err)); GGML_LOG_CONT("."); } // conv2d { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { #include "conv2d.cl.h" }; const std::string kernel_src_f16_f32 { #include "conv2d_f16_f32.cl.h" }; #else const std::string kernel_src = read_file("conv2d.cl"); const std::string kernel_src_f16_f32 = read_file("conv2d_f16_f32.cl"); #endif if (!kernel_src.empty()) { backend_ctx->program_conv_2d_f16 = build_program_from_source(backend_ctx, kernel_src.c_str(), (std::string(compile_opts) + " -DUSE_FP16=1").c_str()); CL_CHECK((backend_ctx->kernel_conv_2d_f16 = clCreateKernel(backend_ctx->program_conv_2d_f16, "kernel_conv_2d", &err), err)); GGML_LOG_CONT("."); backend_ctx->program_conv_2d_f32 = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); CL_CHECK((backend_ctx->kernel_conv_2d_f32 = clCreateKernel(backend_ctx->program_conv_2d_f32, "kernel_conv_2d", &err), err)); GGML_LOG_CONT("."); } else { GGML_LOG_WARN("ggml_opencl: conv2d kernel source not found or empty. This op will not be available.\n"); backend_ctx->program_conv_2d_f16 = nullptr; backend_ctx->kernel_conv_2d_f16 = nullptr; backend_ctx->program_conv_2d_f32 = nullptr; backend_ctx->kernel_conv_2d_f32 = nullptr; } if (!kernel_src_f16_f32.empty()) { backend_ctx->program_conv_2d_f16_f32 = build_program_from_source(backend_ctx, kernel_src_f16_f32.c_str(), compile_opts); CL_CHECK((backend_ctx->kernel_conv_2d_f16_f32 = clCreateKernel(backend_ctx->program_conv_2d_f16_f32, "kernel_conv_2d", &err), err)); GGML_LOG_CONT("."); } else { GGML_LOG_WARN("ggml_opencl: conv2d_f16_f32 kernel source not found or empty. This op will not be available.\n"); backend_ctx->program_conv_2d_f16_f32 = nullptr; backend_ctx->kernel_conv_2d_f16_f32 = nullptr; } } // ssm_conv { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { #include "ssm_conv.cl.h" }; #else const std::string kernel_src = read_file("ssm_conv.cl"); #endif cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); CL_CHECK((backend_ctx->kernel_ssm_conv_f32_f32 = clCreateKernel(prog, "kernel_ssm_conv_f32_f32", &err), err)); CL_CHECK((backend_ctx->kernel_ssm_conv_f32_f32_4 = clCreateKernel(prog, "kernel_ssm_conv_f32_f32_4", &err), err)); CL_CHECK(clReleaseProgram(prog)); GGML_LOG_CONT("."); } // ssm_scan { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { #include "ssm_scan.cl.h" }; #else const std::string kernel_src = read_file("ssm_scan.cl"); #endif cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); CL_CHECK((backend_ctx->kernel_ssm_scan_f32 = clCreateKernel(prog, "kernel_ssm_scan_f32", &err), err)); CL_CHECK((backend_ctx->kernel_ssm_scan_f32_mamba2_d128 = clCreateKernel(prog, "kernel_ssm_scan_f32_mamba2_d128", &err), err)); CL_CHECK((backend_ctx->kernel_ssm_scan_f32_mamba2_d256 = clCreateKernel(prog, "kernel_ssm_scan_f32_mamba2_d256", &err), err)); cl_kernel * kernels[] = { &backend_ctx->kernel_ssm_scan_f32_mamba2_d128, &backend_ctx->kernel_ssm_scan_f32_mamba2_d256 }; // specialized kernels use subgroups and assume subgroup size is 64, // if device does not support subgroups or subgroup size is not 64, // release these kernels for (int i = 0; i < 2; ++i) { size_t subgroup_size = 0; #if CL_TARGET_OPENCL_VERSION >= 210 const size_t local_work_size[] = { 64, 1 }; const cl_int subgroup_err = clGetKernelSubGroupInfo(*kernels[i], backend_ctx->device, CL_KERNEL_MAX_SUB_GROUP_SIZE_FOR_NDRANGE, sizeof(local_work_size), local_work_size, sizeof(subgroup_size), &subgroup_size, nullptr); if (subgroup_err != CL_SUCCESS) { subgroup_size = 0; } #endif // The specialized kernels reduce over one 64-lane subgroup. if (subgroup_size != 64) { CL_CHECK(clReleaseKernel(*kernels[i])); *kernels[i] = nullptr; } } CL_CHECK(clReleaseProgram(prog)); GGML_LOG_CONT("."); } // gated_delta_net: one kernel per (S_V, KDA, tgpp) triple. { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { #include "gated_delta_net.cl.h" }; #else const std::string kernel_src = read_file("gated_delta_net.cl"); #endif const int gdn_sizes[4] = { 16, 32, 64, 128 }; const int sg_size = backend_ctx->gpu_family == GPU_FAMILY::ADRENO ? 64 : backend_ctx->gpu_family == GPU_FAMILY::INTEL ? 32 : -1; if (sg_size < 0) { GGML_LOG_ERROR("Unsupported GPU Family: only Adreno and Intel are supported.\n"); exit(1); } for (int si = 0; si < 4; si++) { const int S_V = gdn_sizes[si]; // MUST match the dispatcher heuristic in ggml_cl_gated_delta_net exactly. int lanes_per_column; if (S_V >= 128) { lanes_per_column = 8; } else { lanes_per_column = std::min(S_V, sg_size); } // Round LANES_PER_COLUMN down until it is: // * power-of-two // * divides both S_V and sg_size while (lanes_per_column > 1 && (((lanes_per_column & (lanes_per_column - 1)) != 0) || (S_V % lanes_per_column) != 0 || (sg_size % lanes_per_column) != 0)) { lanes_per_column >>= 1; } GGML_ASSERT(lanes_per_column >= 1); GGML_ASSERT(((lanes_per_column & (lanes_per_column - 1)) == 0)); GGML_ASSERT((S_V % lanes_per_column) == 0); GGML_ASSERT((sg_size % lanes_per_column) == 0); const bool is_partial_reduce = (lanes_per_column != 1) && (lanes_per_column < sg_size); int use_qcom_shuffle = 0; if (is_partial_reduce) { if (backend_ctx->has_qcom_subgroup_shuffle) { use_qcom_shuffle = 1; } } for (int kda = 0; kda < 2; kda++) { for (int tgpp = 0; tgpp < 2; tgpp++) { const int cpl = (tgpp == 0) ? 1 : 4; const int spw = (tgpp == 0) ? 1 : 1; std::string opts = compile_opts; opts += " -DS_V=" + std::to_string(S_V); opts += " -DKDA=" + std::to_string(kda); opts += " -DSUBGROUP_SIZE=" + std::to_string(sg_size); opts += " -DLANES_PER_COLUMN=" + std::to_string(lanes_per_column); opts += " -DCOLS_PER_LANE_GROUP=" + std::to_string(cpl); opts += " -DUSE_QCOM_SUBGROUP_SHUFFLE=" + std::to_string(use_qcom_shuffle); // Since spw=1 is found to be optimal, SUBGROUPS_PER_WG > 1 code in // the kernel is removed. If you want to experiment with spw > 1, // Please remember to implement code to handle it. opts += " -DSUBGROUPS_PER_WG=" + std::to_string(spw); cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), opts); CL_CHECK((backend_ctx->kernel_gated_delta_net_f32[si][kda][tgpp] = clCreateKernel(prog, "kernel_gated_delta_net", &err), err)); CL_CHECK(clReleaseProgram(prog)); } } } GGML_LOG_CONT("."); } // moe_combine (fused router-weight mul + cross-expert sum) { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { #include "moe_combine.cl.h" }; #else const std::string kernel_src = read_file("moe_combine.cl"); #endif cl_program prog = build_program_from_source( backend_ctx, kernel_src.c_str(), compile_opts); CL_CHECK((backend_ctx->kernel_moe_combine_f32 = clCreateKernel(prog, "kernel_moe_combine_f32", &err), err)); CL_CHECK((backend_ctx->kernel_moe_combine_bias_f32 = clCreateKernel(prog, "kernel_moe_combine_bias_f32", &err), err)); CL_CHECK(clReleaseProgram(prog)); GGML_LOG_CONT("."); } // mul_mv_id_q4_0_f32_8x_flat { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { #include "mul_mv_id_q4_0_f32_8x_flat.cl.h" }; #else const std::string kernel_src = read_file("mul_mv_id_q4_0_f32_8x_flat.cl"); #endif backend_ctx->program_mul_mv_id_q4_0_f32_8x_flat = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); CL_CHECK((backend_ctx->kernel_mul_mv_id_q4_0_f32_8x_flat = clCreateKernel(backend_ctx->program_mul_mv_id_q4_0_f32_8x_flat, "kernel_mul_mv_id_q4_0_f32_8x_flat", &err), err)); GGML_LOG_CONT("."); } // mul_mv_id_q8_0_f32 { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { #include "mul_mv_id_q8_0_f32.cl.h" }; #else const std::string kernel_src = read_file("mul_mv_id_q8_0_f32.cl"); #endif backend_ctx->program_mul_mv_id_q8_0_f32 = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); CL_CHECK((backend_ctx->kernel_mul_mv_id_q8_0_f32 = clCreateKernel(backend_ctx->program_mul_mv_id_q8_0_f32, "kernel_mul_mv_id_q8_0_f32", &err), err)); GGML_LOG_CONT("."); } // mul_mv_id_q8_0_f32_flat { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { #include "mul_mv_id_q8_0_f32_flat.cl.h" }; #else const std::string kernel_src = read_file("mul_mv_id_q8_0_f32_flat.cl"); #endif backend_ctx->program_mul_mv_id_q8_0_f32_flat = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); CL_CHECK((backend_ctx->kernel_mul_mv_id_q8_0_f32_flat = clCreateKernel(backend_ctx->program_mul_mv_id_q8_0_f32_flat, "kernel_mul_mv_id_q8_0_f32_flat", &err), err)); GGML_LOG_CONT("."); } // mul_mv_id_mxfp4_f32 { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { #include "mul_mv_id_mxfp4_f32.cl.h" }; #else const std::string kernel_src = read_file("mul_mv_id_mxfp4_f32.cl"); #endif backend_ctx->program_mul_mv_id_mxfp4_f32 = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); CL_CHECK((backend_ctx->kernel_mul_mv_id_mxfp4_f32 = clCreateKernel(backend_ctx->program_mul_mv_id_mxfp4_f32, "kernel_mul_mv_id_mxfp4_f32", &err), err)); GGML_LOG_CONT("."); } // mul_mv_id_mxfp4_f32_flat { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { #include "mul_mv_id_mxfp4_f32_flat.cl.h" }; #else const std::string kernel_src = read_file("mul_mv_id_mxfp4_f32_flat.cl"); #endif backend_ctx->program_mul_mv_id_mxfp4_f32_flat = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); CL_CHECK((backend_ctx->kernel_mul_mv_id_mxfp4_f32_flat = clCreateKernel(backend_ctx->program_mul_mv_id_mxfp4_f32_flat, "kernel_mul_mv_id_mxfp4_f32_flat", &err), err)); GGML_LOG_CONT("."); } // Adreno kernels #ifdef GGML_OPENCL_USE_ADRENO_KERNELS // transpose { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { #include "transpose.cl.h" }; #else const std::string kernel_src = read_file("transpose.cl"); #endif backend_ctx->program_transpose = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); CL_CHECK((backend_ctx->kernel_transpose_32_16 = clCreateKernel(backend_ctx->program_transpose, "kernel_transpose_32_16", &err), err)); CL_CHECK((backend_ctx->kernel_transpose_32 = clCreateKernel(backend_ctx->program_transpose, "kernel_transpose_32", &err), err)); CL_CHECK((backend_ctx->kernel_transpose_16 = clCreateKernel(backend_ctx->program_transpose, "kernel_transpose_16", &err), err)); CL_CHECK((backend_ctx->kernel_transpose_8_buf = clCreateKernel(backend_ctx->program_transpose, "kernel_transpose_8_buf", &err), err)); CL_CHECK((backend_ctx->kernel_transpose_16_buf = clCreateKernel(backend_ctx->program_transpose, "kernel_transpose_16_buf", &err), err)); CL_CHECK((backend_ctx->kernel_transpose_32_buf = clCreateKernel(backend_ctx->program_transpose, "kernel_transpose_32_buf", &err), err)); CL_CHECK((backend_ctx->kernel_transpose_16_4x1 = clCreateKernel(backend_ctx->program_transpose, "kernel_transpose_16_4x1", &err), err)); GGML_LOG_CONT("."); } // gemm_noshuffle_q1_0_f32 { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { #include "gemm_noshuffle_q1_0_f32.cl.h" }; #else const std::string kernel_src = read_file("gemm_noshuffle_q1_0_f32.cl"); #endif cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q1_0_f32 = clCreateKernel(prog, "kernel_gemm_noshuffle_q1_0_f32", &err), err)); CL_CHECK(clReleaseProgram(prog)); GGML_LOG_CONT("."); } // gemv_noshuffle_q1_0_f32 { std::string CL_gemv_compile_opts = std::string("-cl-std=") + opencl_c_std + " -cl-mad-enable " " -DSIMDGROUP_WIDTH=" + std::to_string(backend_ctx->adreno_wave_size); #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src_CL_gemv_general { #include "gemv_noshuffle_q1_0_f32.cl.h" }; #else const std::string kernel_src_CL_gemv_general = read_file("gemv_noshuffle_q1_0_f32.cl"); #endif cl_program prog = build_program_from_source(backend_ctx, kernel_src_CL_gemv_general.c_str(), CL_gemv_compile_opts); CL_CHECK((backend_ctx->kernel_gemv_noshuffle_q1_0_f32 = clCreateKernel(prog, "kernel_gemv_noshuffle_q1_0_f32", &err), err)); CL_CHECK(clReleaseProgram(prog)); GGML_LOG_CONT("."); } // gemv_noshuffle_general { std::string CL_gemv_compile_opts = std::string("-cl-std=") + opencl_c_std + " -cl-mad-enable " " -DSIMDGROUP_WIDTH=" + std::to_string(backend_ctx->adreno_wave_size); if (backend_ctx->has_vector_subgroup_broadcast) { CL_gemv_compile_opts += " -DVECTOR_SUB_GROUP_BROADCAST "; } #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src_CL_gemv_general { #include "gemv_noshuffle_q4_0_f32.cl.h" }; #else const std::string kernel_src_CL_gemv_general = read_file("gemv_noshuffle_q4_0_f32.cl"); #endif cl_program prog = build_program_from_source(backend_ctx, kernel_src_CL_gemv_general.c_str(), CL_gemv_compile_opts); CL_CHECK((backend_ctx->kernel_gemv_noshuffle_q4_0_f32 = clCreateKernel(prog, "kernel_gemv_noshuffle_q4_0_f32", &err), err)); CL_CHECK((backend_ctx->kernel_gemv_noshuffle_q4_0_f32_mc3 = clCreateKernel(prog, "kernel_gemv_noshuffle_q4_0_f32_mc3", &err), err)); CL_CHECK(clReleaseProgram(prog)); GGML_LOG_CONT("."); } // gemv_noshuffle { // Gemv 2048, 16384 std::string CL_gemv_compile_opts = std::string("-cl-std=") + opencl_c_std + " -cl-mad-enable " " -DLINE_STRIDE_A=2048 " " -DBLOCK_STRIDE_A=16384 " " -DSIMDGROUP_WIDTH=" + std::to_string(backend_ctx->adreno_wave_size); if (backend_ctx->has_vector_subgroup_broadcast) { CL_gemv_compile_opts += " -DVECTOR_SUB_GROUP_BROADCAST "; } #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src_CL_gemv { #include "gemv_noshuffle_q4_0_f32_spec.cl.h" }; #else const std::string kernel_src_CL_gemv = read_file("gemv_noshuffle_q4_0_f32_spec.cl"); #endif cl_program prog = build_program_from_source(backend_ctx, kernel_src_CL_gemv.c_str(), CL_gemv_compile_opts); CL_CHECK((backend_ctx->kernel_gemv_noshuffle_q4_0_f32_4096_1_4096 = clCreateKernel(prog, "kernel_gemv_noshuffle_q4_0_f32", &err), err)); CL_CHECK(clReleaseProgram(prog)); GGML_LOG_CONT("."); // Gemv 2048, 16384 CL_gemv_compile_opts = std::string("-cl-std=") + opencl_c_std + " -cl-mad-enable " " -DLINE_STRIDE_A=2048 " " -DBLOCK_STRIDE_A=16384 " " -DSIMDGROUP_WIDTH=" + std::to_string(backend_ctx->adreno_wave_size); if (backend_ctx->has_vector_subgroup_broadcast) { CL_gemv_compile_opts += " -DVECTOR_SUB_GROUP_BROADCAST "; } prog = build_program_from_source(backend_ctx, kernel_src_CL_gemv.c_str(), CL_gemv_compile_opts); CL_CHECK((backend_ctx->kernel_gemv_noshuffle_q4_0_f32_4096_1_11008 = clCreateKernel(prog, "kernel_gemv_noshuffle_q4_0_f32", &err), err)); CL_CHECK(clReleaseProgram(prog)); GGML_LOG_CONT("."); // Gemv 5504, 44032 CL_gemv_compile_opts = std::string("-cl-std=") + opencl_c_std + " -cl-mad-enable " " -DLINE_STRIDE_A=5504 " " -DBLOCK_STRIDE_A=44032 " " -DSIMDGROUP_WIDTH=" + std::to_string(backend_ctx->adreno_wave_size); if (backend_ctx->has_vector_subgroup_broadcast) { CL_gemv_compile_opts += " -DVECTOR_SUB_GROUP_BROADCAST "; } prog = build_program_from_source(backend_ctx, kernel_src_CL_gemv.c_str(), CL_gemv_compile_opts); CL_CHECK((backend_ctx->kernel_gemv_noshuffle_q4_0_f32_11008_1_4096 = clCreateKernel(prog, "kernel_gemv_noshuffle_q4_0_f32", &err), err)); CL_CHECK(clReleaseProgram(prog)); GGML_LOG_CONT("."); // Gemv 16000, 128000 CL_gemv_compile_opts = std::string("-cl-std=") + opencl_c_std + " -cl-mad-enable " " -DLINE_STRIDE_A=16000 " " -DBLOCK_STRIDE_A=128000 " " -DSIMDGROUP_WIDTH=" + std::to_string(backend_ctx->adreno_wave_size); if (backend_ctx->has_vector_subgroup_broadcast) { CL_gemv_compile_opts += " -DVECTOR_SUB_GROUP_BROADCAST "; } prog = build_program_from_source(backend_ctx, kernel_src_CL_gemv.c_str(), CL_gemv_compile_opts); CL_CHECK((backend_ctx->kernel_gemv_noshuffle_q4_0_f32_32000_1_4096 = clCreateKernel(prog, "kernel_gemv_noshuffle_q4_0_f32", &err), err)); CL_CHECK(clReleaseProgram(prog)); GGML_LOG_CONT("."); } // mul_mat_Ab_Bi_8x4 { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src_CL_gemm { #include "gemm_noshuffle_q4_0_f32.cl.h" }; #else const std::string kernel_src_CL_gemm = read_file("gemm_noshuffle_q4_0_f32.cl"); #endif cl_program prog = build_program_from_source(backend_ctx, kernel_src_CL_gemm.c_str(), compile_opts); CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q4_0_f32 = clCreateKernel(prog, "kernel_gemm_noshuffle_q4_0_f32", &err), err)); CL_CHECK(clReleaseProgram(prog)); GGML_LOG_CONT("."); } backend_ctx->kernel_gemv_noshuffle_q4_0_f32_32b_trans = nullptr; backend_ctx->kernel_gemm_noshuffle_q4_0_f32_32b_trans_ila_a8_bin = nullptr; backend_ctx->kernel_gemm_noshuffle_q4_0_q8_1_dp4a_ila_a8_bin = nullptr; { { std::string opts = std::string("-cl-std=") + opencl_c_std + " -cl-mad-enable " " -DSIMDGROUP_WIDTH=" + std::to_string(backend_ctx->adreno_wave_size); #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { #include "gemv_noshuffle_q4_0_f32_32b_trans.cl.h" }; #else const std::string kernel_src = read_file("gemv_noshuffle_q4_0_f32_32b_trans.cl"); #endif cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), opts); CL_CHECK((backend_ctx->kernel_gemv_noshuffle_q4_0_f32_32b_trans = clCreateKernel(prog, "kernel_gemv_noshuffle_q4_0_f32_32b_trans", &err), err)); CL_CHECK(clReleaseProgram(prog)); GGML_LOG_CONT("."); } if (use_adreno_bin_kernels(backend_ctx)) { size_t bin_size = 0; const char * kernel_bin = (const char *)backend_ctx->get_adreno_bin_kernel("gemm_noshuffle_q4_0_f32_32b_trans_ila_a8", &bin_size); if (kernel_bin && bin_size > 0) { cl_program bin_prog = build_program_from_binary(backend_ctx->context, backend_ctx->device, kernel_bin, "", bin_size); CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q4_0_f32_32b_trans_ila_a8_bin = clCreateKernel(bin_prog, "kernel_gemm_noshuffle_q4_0_f32_32b_trans_ila_a8", &err), err)); CL_CHECK(clReleaseProgram(bin_prog)); GGML_LOG_CONT("."); } kernel_bin = (const char *)backend_ctx->get_adreno_bin_kernel("gemm_noshuffle_q4_0_q8_1_dp4a_ila_a8", &bin_size); if (kernel_bin && bin_size > 0) { cl_program bin_prog = build_program_from_binary(backend_ctx->context, backend_ctx->device, kernel_bin, "", bin_size); CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q4_0_q8_1_dp4a_ila_a8_bin = clCreateKernel(bin_prog, "kernel_gemm_noshuffle_q4_0_q8_1_dp4a_ila_a8", &err), err)); CL_CHECK(clReleaseProgram(bin_prog)); GGML_LOG_CONT("."); } } } // gemm_noshuffle_q4_1_f32 { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { #include "gemm_noshuffle_q4_1_f32.cl.h" }; #else const std::string kernel_src = read_file("gemm_noshuffle_q4_1_f32.cl"); #endif cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q4_1_f32 = clCreateKernel(prog, "kernel_gemm_noshuffle_q4_1_f32", &err), err)); CL_CHECK(clReleaseProgram(prog)); GGML_LOG_CONT("."); } // gemv_noshuffle_q4_1_f32 { std::string CL_gemv_compile_opts = std::string("-cl-std=") + opencl_c_std + " -cl-mad-enable "; if (backend_ctx->has_vector_subgroup_broadcast) { CL_gemv_compile_opts += " -DVECTOR_SUB_GROUP_BROADCAT "; } #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { #include "gemv_noshuffle_q4_1_f32.cl.h" }; #else const std::string kernel_src = read_file("gemv_noshuffle_q4_1_f32.cl"); #endif cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), CL_gemv_compile_opts); CL_CHECK((backend_ctx->kernel_gemv_noshuffle_q4_1_f32 = clCreateKernel(prog, "kernel_gemv_noshuffle_q4_1_f32", &err), err)); CL_CHECK((backend_ctx->kernel_gemv_noshuffle_q4_1_f32_mc3 = clCreateKernel(prog, "kernel_gemv_noshuffle_q4_1_f32_mc3", &err), err)); CL_CHECK(clReleaseProgram(prog)); GGML_LOG_CONT("."); } // gemm_noshuffle_q5_0_f32 { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { #include "gemm_noshuffle_q5_0_f32.cl.h" }; #else const std::string kernel_src = read_file("gemm_noshuffle_q5_0_f32.cl"); #endif cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q5_0_f32 = clCreateKernel(prog, "kernel_gemm_noshuffle_q5_0_f32", &err), err)); CL_CHECK(clReleaseProgram(prog)); GGML_LOG_CONT("."); } // gemm_noshuffle_q5_0_q8_1_dp4a (dp4a dense q5_0 prefill GEMM) if (backend_ctx->has_integer_dot) { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { #include "gemm_noshuffle_q5_0_q8_1_dp4a.cl.h" }; #else const std::string kernel_src = read_file("gemm_noshuffle_q5_0_q8_1_dp4a.cl"); #endif cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q5_0_q8_1_dp4a = clCreateKernel(prog, "kernel_gemm_noshuffle_q5_0_q8_1_dp4a", &err), err)); CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q5_0_q8_1_dp4a_wimg = clCreateKernel(prog, "kernel_gemm_noshuffle_q5_0_q8_1_dp4a_wimg", &err), err)); CL_CHECK(clReleaseProgram(prog)); GGML_LOG_CONT("."); } // gemv_noshuffle_q5_0_f32 { std::string CL_gemv_compile_opts = std::string("-cl-std=") + opencl_c_std + " -cl-mad-enable "; if (backend_ctx->has_vector_subgroup_broadcast) { CL_gemv_compile_opts += " -DVECTOR_SUB_GROUP_BROADCAST "; } #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { #include "gemv_noshuffle_q5_0_f32.cl.h" }; #else const std::string kernel_src = read_file("gemv_noshuffle_q5_0_f32.cl"); #endif cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), CL_gemv_compile_opts); CL_CHECK((backend_ctx->kernel_gemv_noshuffle_q5_0_f32 = clCreateKernel(prog, "kernel_gemv_noshuffle_q5_0_f32", &err), err)); CL_CHECK(clReleaseProgram(prog)); GGML_LOG_CONT("."); } // gemm_noshuffle_q5_1_f32 { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { #include "gemm_noshuffle_q5_1_f32.cl.h" }; #else const std::string kernel_src = read_file("gemm_noshuffle_q5_1_f32.cl"); #endif cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q5_1_f32 = clCreateKernel(prog, "kernel_gemm_noshuffle_q5_1_f32", &err), err)); CL_CHECK(clReleaseProgram(prog)); GGML_LOG_CONT("."); } // gemv_noshuffle_q5_1_f32 { std::string CL_gemv_compile_opts = std::string("-cl-std=") + opencl_c_std + " -cl-mad-enable "; if (backend_ctx->has_vector_subgroup_broadcast) { CL_gemv_compile_opts += " -DVECTOR_SUB_GROUP_BROADCAST "; } #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { #include "gemv_noshuffle_q5_1_f32.cl.h" }; #else const std::string kernel_src = read_file("gemv_noshuffle_q5_1_f32.cl"); #endif cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), CL_gemv_compile_opts); CL_CHECK((backend_ctx->kernel_gemv_noshuffle_q5_1_f32 = clCreateKernel(prog, "kernel_gemv_noshuffle_q5_1_f32", &err), err)); CL_CHECK(clReleaseProgram(prog)); GGML_LOG_CONT("."); } // gemm_noshuffle_iq4_nl_f32 { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { #include "gemm_noshuffle_iq4_nl_f32.cl.h" }; #else const std::string kernel_src = read_file("gemm_noshuffle_iq4_nl_f32.cl"); #endif cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); CL_CHECK((backend_ctx->kernel_gemm_noshuffle_iq4_nl_f32 = clCreateKernel(prog, "kernel_gemm_noshuffle_iq4_nl_f32", &err), err)); CL_CHECK(clReleaseProgram(prog)); GGML_LOG_CONT("."); } // gemm_noshuffle_iq4_nl_q8_1_dp4a (dp4a dense IQ4_NL prefill GEMM) if (backend_ctx->has_integer_dot) { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { #include "gemm_noshuffle_iq4_nl_q8_1_dp4a.cl.h" }; #else const std::string kernel_src = read_file("gemm_noshuffle_iq4_nl_q8_1_dp4a.cl"); #endif cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); CL_CHECK((backend_ctx->kernel_gemm_noshuffle_iq4_nl_q8_1_dp4a = clCreateKernel(prog, "kernel_gemm_noshuffle_iq4_nl_q8_1_dp4a", &err), err)); CL_CHECK(clReleaseProgram(prog)); GGML_LOG_CONT("."); } // gemm_noshuffle_q4_0_q8_1_dp4a (dp4a dense q4_0 prefill GEMM) if (backend_ctx->has_integer_dot) { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { #include "gemm_noshuffle_q4_0_q8_1_dp4a.cl.h" }; #else const std::string kernel_src = read_file("gemm_noshuffle_q4_0_q8_1_dp4a.cl"); #endif cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q4_0_q8_1_dp4a = clCreateKernel(prog, "kernel_gemm_noshuffle_q4_0_q8_1_dp4a", &err), err)); CL_CHECK(clReleaseProgram(prog)); GGML_LOG_CONT("."); } // gemv_noshuffle_iq4_nl_f32 { std::string CL_gemv_compile_opts = std::string("-cl-std=") + opencl_c_std + " -cl-mad-enable "; if (backend_ctx->has_vector_subgroup_broadcast) { CL_gemv_compile_opts += " -DVECTOR_SUB_GROUP_BROADCAST "; } #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { #include "gemv_noshuffle_iq4_nl_f32.cl.h" }; #else const std::string kernel_src = read_file("gemv_noshuffle_iq4_nl_f32.cl"); #endif cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), CL_gemv_compile_opts); CL_CHECK((backend_ctx->kernel_gemv_noshuffle_iq4_nl_f32 = clCreateKernel(prog, "kernel_gemv_noshuffle_iq4_nl_f32", &err), err)); CL_CHECK(clReleaseProgram(prog)); GGML_LOG_CONT("."); } // mul_mm_q8_0_f32_8x4 { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { #include "gemm_noshuffle_q8_0_f32.cl.h" }; #else const std::string kernel_src = read_file("gemm_noshuffle_q8_0_f32.cl"); #endif cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q8_0_f32 = clCreateKernel(prog, "kernel_gemm_noshuffle_q8_0_f32", &err), err)); CL_CHECK(clReleaseProgram(prog)); GGML_LOG_CONT("."); } // gemm_noshuffle_q8_0_f32_bin { size_t bin_size = 0; backend_ctx->kernel_gemm_noshuffle_q8_0_f32_bin = nullptr; if (use_adreno_bin_kernels(backend_ctx)) { const char * kernel_bin = (const char *)backend_ctx->get_adreno_bin_kernel("gemm_noshuffle_q8_0_f32_ila", &bin_size); if (kernel_bin && bin_size > 0) { cl_program prog = build_program_from_binary(backend_ctx->context, backend_ctx->device, kernel_bin, compile_opts, bin_size); CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q8_0_f32_bin = clCreateKernel(prog, "kernel_gemm_noshuffle_q8_0_f32_ila", &err), err)); CL_CHECK(clReleaseProgram(prog)); GGML_LOG_CONT("."); } } } if (backend_ctx->has_integer_dot) { size_t bin_size = 0; backend_ctx->kernel_gemm_noshuffle_q8_0_q8_1_dp4a_ila_a8_bin = nullptr; if (use_adreno_bin_kernels(backend_ctx)) { const char * kernel_bin = (const char *)backend_ctx->get_adreno_bin_kernel("gemm_noshuffle_q8_0_q8_1_dp4a_ila_a8", &bin_size); if (kernel_bin && bin_size > 0) { cl_program prog = build_program_from_binary(backend_ctx->context, backend_ctx->device, kernel_bin, compile_opts, bin_size); CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q8_0_q8_1_dp4a_ila_a8_bin = clCreateKernel(prog, "kernel_gemm_noshuffle_q8_0_q8_1_dp4a_ila_a8", &err), err)); CL_CHECK(clReleaseProgram(prog)); GGML_LOG_CONT("."); } } } // gemv_noshuffle_general_q8_0_f32 { std::string CL_gemv_compile_opts = std::string("-cl-std=") + opencl_c_std + " -cl-mad-enable " " -DSIMDGROUP_WIDTH=" + std::to_string(backend_ctx->adreno_wave_size); if (backend_ctx->has_vector_subgroup_broadcast) { CL_gemv_compile_opts += " -DVECTOR_SUB_GROUP_BROADCAT "; } #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src_CL_gemv_general { #include "gemv_noshuffle_q8_0_f32.cl.h" }; #else const std::string kernel_src_CL_gemv_general = read_file("gemv_noshuffle_q8_0_f32.cl"); #endif cl_program prog = build_program_from_source(backend_ctx, kernel_src_CL_gemv_general.c_str(), CL_gemv_compile_opts); CL_CHECK((backend_ctx->kernel_gemv_noshuffle_q8_0_f32 = clCreateKernel(prog, "kernel_gemv_noshuffle_q8_0_f32", &err), err)); CL_CHECK((backend_ctx->kernel_gemv_noshuffle_q8_0_f32_splitk = clCreateKernel(prog, "kernel_gemv_noshuffle_q8_0_f32_splitk", &err), err)); CL_CHECK(clReleaseProgram(prog)); GGML_LOG_CONT("."); } // gemm_noshuffle_q4_k_f32 { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { #include "gemm_noshuffle_q4_k_f32.cl.h" }; #else const std::string kernel_src = read_file("gemm_noshuffle_q4_k_f32.cl"); #endif cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q4_k_f32 = clCreateKernel(prog, "kernel_gemm_noshuffle_q4_k_f32", &err), err)); CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q4_k_f32_r1 = clCreateKernel(prog, "kernel_gemm_noshuffle_q4_k_f32_r1", &err), err)); CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q4_k_f32_kimg = clCreateKernel(prog, "kernel_gemm_noshuffle_q4_k_f32_kimg", &err), err)); CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q4_k_f32_cok = clCreateKernel(prog, "kernel_gemm_noshuffle_q4_k_f32_cok", &err), err)); CL_CHECK(clReleaseProgram(prog)); GGML_LOG_CONT("."); } // gemm_noshuffle_q4_k_q8_1_dp4a (dp4a dense prefill GEMM) if (backend_ctx->has_integer_dot) { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { #include "gemm_noshuffle_q4_k_q8_1_dp4a.cl.h" }; #else const std::string kernel_src = read_file("gemm_noshuffle_q4_k_q8_1_dp4a.cl"); #endif // Per-device dp4a dense tile. The X2-tuned TILESIZE_N=32 over-occupies LDS on // X1 (1152 B/WG -> few resident WGs); TILESIZE_N=8 (288 B) lifts occupancy on // X1, byte-identical. X2E keeps 32. Env override wins. int q4k_dp4a_ts = (backend_ctx->adreno_gen == ADRENO_GPU_GEN::X1E) ? 8 : 32; if (const char * e = getenv("GGML_OPENCL_Q4K_DP4A_TS")) { q4k_dp4a_ts = atoi(e); } std::string dp4a_opts = compile_opts + " -DTILESIZE_N=" + std::to_string(q4k_dp4a_ts); cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), dp4a_opts); CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q4_k_q8_1_dp4a = clCreateKernel(prog, "kernel_gemm_noshuffle_q4_k_q8_1_dp4a", &err), err)); CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q4_k_q8_1_dp4a_wimg = clCreateKernel(prog, "kernel_gemm_noshuffle_q4_k_q8_1_dp4a_wimg", &err), err)); CL_CHECK(clReleaseProgram(prog)); GGML_LOG_CONT("."); } // gemm_noshuffle_q8_0_q8_1_dp4a (dp4a dense q8_0 prefill GEMM) if (backend_ctx->has_integer_dot) { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { #include "gemm_noshuffle_q8_0_q8_1_dp4a.cl.h" }; #else const std::string kernel_src = read_file("gemm_noshuffle_q8_0_q8_1_dp4a.cl"); #endif cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q8_0_q8_1_dp4a = clCreateKernel(prog, "kernel_gemm_noshuffle_q8_0_q8_1_dp4a", &err), err)); CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q8_0_q8_1_dp4a_wimg = clCreateKernel(prog, "kernel_gemm_noshuffle_q8_0_q8_1_dp4a_wimg", &err), err)); CL_CHECK(clReleaseProgram(prog)); GGML_LOG_CONT("."); } // gemm_noshuffle_q5_k_q8_1_dp4a (dp4a dense prefill GEMM for q5_K) if (backend_ctx->has_integer_dot) { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { #include "gemm_noshuffle_q5_k_q8_1_dp4a.cl.h" }; #else const std::string kernel_src = read_file("gemm_noshuffle_q5_k_q8_1_dp4a.cl"); #endif cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q5_k_q8_1_dp4a = clCreateKernel(prog, "kernel_gemm_noshuffle_q5_k_q8_1_dp4a", &err), err)); CL_CHECK(clReleaseProgram(prog)); GGML_LOG_CONT("."); } // gemm_noshuffle_q6_k_q8_1_dp4a (dp4a dense prefill GEMM for q6_K ffn_down/output) if (backend_ctx->has_integer_dot) { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { #include "gemm_noshuffle_q6_k_q8_1_dp4a.cl.h" }; #else const std::string kernel_src = read_file("gemm_noshuffle_q6_k_q8_1_dp4a.cl"); #endif cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q6_k_q8_1_dp4a = clCreateKernel(prog, "kernel_gemm_noshuffle_q6_k_q8_1_dp4a", &err), err)); CL_CHECK(clReleaseProgram(prog)); GGML_LOG_CONT("."); } // quant_a_q8_1 (plain activation q8_1 pre-pass for the dense dp4a GEMM) { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { #include "quant_a_q8_1.cl.h" }; #else const std::string kernel_src = read_file("quant_a_q8_1.cl"); #endif cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); CL_CHECK((backend_ctx->kernel_quant_a_q8_1 = clCreateKernel(prog, "kernel_quant_a_q8_1", &err), err)); CL_CHECK(clReleaseProgram(prog)); GGML_LOG_CONT("."); } // gemv_noshuffle_q4_k_f32 { std::string CL_gemv_compile_opts = std::string("-cl-std=") + opencl_c_std + " -cl-mad-enable "; if (backend_ctx->has_vector_subgroup_broadcast) { CL_gemv_compile_opts += " -DVECTOR_SUB_GROUP_BROADCAST "; } // Opt-in: dequant-once-per-block mc3 verify GEMV (factors q4_K dequant // out of the 3-column loop; byte-identical, lower spill). A/B vs the // shipped inline mc3 in the same binary. if (getenv("GGML_OPENCL_Q4K_MC3_DQ")) { CL_gemv_compile_opts += " -DQ4K_MC3_DEQUANT_ONCE "; } // Opt-in: LDS-staged dequant mc3 verify GEMV (stages the dequantized // q4_K weights in __local instead of private regs that spill to slow // global on Adreno; byte-identical). A/B vs inline + dequant-once. if (getenv("GGML_OPENCL_Q4K_MC3_LDS")) { CL_gemv_compile_opts += " -DQ4K_MC3_DEQUANT_LDS "; } #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { #include "gemv_noshuffle_q4_k_f32.cl.h" }; #else const std::string kernel_src = read_file("gemv_noshuffle_q4_k_f32.cl"); #endif cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), CL_gemv_compile_opts); CL_CHECK((backend_ctx->kernel_gemv_noshuffle_q4_k_f32 = clCreateKernel(prog, "kernel_gemv_noshuffle_q4_k_f32", &err), err)); CL_CHECK((backend_ctx->kernel_gemv_noshuffle_q4_k_f32_mc3 = clCreateKernel(prog, "kernel_gemv_noshuffle_q4_k_f32_mc3", &err), err)); CL_CHECK((backend_ctx->kernel_gemv_noshuffle_q4_k_f32_splitk = clCreateKernel(prog, "kernel_gemv_noshuffle_q4_k_f32_splitk", &err), err)); CL_CHECK((backend_ctx->kernel_gemv_splitk_reduce_f32 = clCreateKernel(prog, "kernel_gemv_splitk_reduce_f32", &err), err)); CL_CHECK((backend_ctx->kernel_gemv_noshuffle_q4_k_f32_glu = clCreateKernel(prog, "kernel_gemv_noshuffle_q4_k_f32_glu", &err), err)); CL_CHECK(clReleaseProgram(prog)); GGML_LOG_CONT("."); } // gemv_noshuffle_q4_k_f32_o4 — 4-output-per-WI variant for the long-vocab // q4_K lm_head/embed GEMV (shares one activation read across 4 output rows). { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { #include "gemv_noshuffle_q4_k_f32_o4.cl.h" }; #else const std::string kernel_src = read_file("gemv_noshuffle_q4_k_f32_o4.cl"); #endif std::string CL_gemv_compile_opts = std::string("-cl-std=") + opencl_c_std + " -cl-mad-enable "; if (backend_ctx->has_vector_subgroup_broadcast) { CL_gemv_compile_opts += " -DVECTOR_SUB_GROUP_BROADCAST "; } cl_program prog = build_program_from_source( backend_ctx, kernel_src.c_str(), CL_gemv_compile_opts); CL_CHECK((backend_ctx->kernel_gemv_noshuffle_q4_k_f32_o4 = clCreateKernel(prog, "kernel_gemv_noshuffle_q4_k_f32_o4", &err), err)); CL_CHECK(clReleaseProgram(prog)); GGML_LOG_CONT("."); } // gemv_noshuffle_q4_k_f32_tiled — tiled-wide canonical layout, default ON // (opt out: GGML_OPENCL_Q4K_GEMV_TILED=0; separate convert + GEMV; weights via __global). { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { #include "gemv_noshuffle_q4_k_f32_tiled.cl.h" }; #else const std::string kernel_src = read_file("gemv_noshuffle_q4_k_f32_tiled.cl"); #endif std::string compile_opts = std::string("-cl-std=") + opencl_c_std + " -cl-mad-enable "; cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); CL_CHECK((backend_ctx->kernel_gemv_noshuffle_q4_k_f32_tiled = clCreateKernel(prog, "kernel_gemv_noshuffle_q4_k_f32_tiled", &err), err)); CL_CHECK(clReleaseProgram(prog)); GGML_LOG_CONT("."); } backend_ctx->kernel_gemv_noshuffle_q4_k_f32_32b_trans = nullptr; backend_ctx->kernel_gemm_noshuffle_q4_k_f32_32b_trans_ila_a8_bin = nullptr; backend_ctx->kernel_gemm_noshuffle_q4_k_q8_1_dp4a_ila_a8_bin = nullptr; { if (backend_ctx->has_vector_subgroup_broadcast) { std::string opts = std::string("-cl-std=") + opencl_c_std + " -cl-mad-enable " " -DSIMDGROUP_WIDTH=" + std::to_string(backend_ctx->adreno_wave_size); #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { #include "gemv_noshuffle_q4_k_f32_32b_trans.cl.h" }; #else const std::string kernel_src = read_file("gemv_noshuffle_q4_k_f32_32b_trans.cl"); #endif cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), opts); CL_CHECK((backend_ctx->kernel_gemv_noshuffle_q4_k_f32_32b_trans = clCreateKernel(prog, "gemv_noshuffle_q4_k_f32_32b_trans", &err), err)); CL_CHECK(clReleaseProgram(prog)); GGML_LOG_CONT("."); } if (use_adreno_bin_kernels(backend_ctx)) { size_t bin_size = 0; const char * kernel_bin = (const char *)backend_ctx->get_adreno_bin_kernel("gemm_noshuffle_q4_k_f32_32b_trans_ila_a8", &bin_size); if (kernel_bin && bin_size > 0) { cl_program bin_prog = build_program_from_binary(backend_ctx->context, backend_ctx->device, kernel_bin, "", bin_size); CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q4_k_f32_32b_trans_ila_a8_bin = clCreateKernel(bin_prog, "kernel_gemm_noshuffle_q4_k_f32_32b_trans_ila_a8", &err), err)); CL_CHECK(clReleaseProgram(bin_prog)); GGML_LOG_CONT("."); } kernel_bin = (const char *)backend_ctx->get_adreno_bin_kernel("gemm_noshuffle_q4_k_q8_1_dp4a_ila_a8", &bin_size); if (kernel_bin && bin_size > 0) { cl_program bin_prog = build_program_from_binary(backend_ctx->context, backend_ctx->device, kernel_bin, "", bin_size); CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q4_k_q8_1_dp4a_ila_a8_bin = clCreateKernel(bin_prog, "kernel_gemm_noshuffle_q4_k_q8_1_dp4a_ila_a8", &err), err)); CL_CHECK(clReleaseProgram(bin_prog)); GGML_LOG_CONT("."); } } } backend_ctx->kernel_gemv_noshuffle_q6_k_f32_32b_trans = nullptr; backend_ctx->kernel_gemm_noshuffle_q6_k_f32_32b_trans_ila_a8_bin = nullptr; backend_ctx->kernel_gemm_noshuffle_q6_k_q8_1_dp4a_ila_a8_bin = nullptr; { if (backend_ctx->has_vector_subgroup_broadcast) { std::string opts = std::string("-cl-std=") + opencl_c_std + " -cl-mad-enable " " -DSIMDGROUP_WIDTH=" + std::to_string(backend_ctx->adreno_wave_size); #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { #include "gemv_noshuffle_q6_k_f32_32b_trans.cl.h" }; #else const std::string kernel_src = read_file("gemv_noshuffle_q6_k_f32_32b_trans.cl"); #endif cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), opts); CL_CHECK((backend_ctx->kernel_gemv_noshuffle_q6_k_f32_32b_trans = clCreateKernel(prog, "kernel_gemv_noshuffle_q6_k_f32_32b_trans", &err), err)); CL_CHECK(clReleaseProgram(prog)); GGML_LOG_CONT("."); } if (use_adreno_bin_kernels(backend_ctx)) { size_t bin_size = 0; const char * kernel_bin = (const char *)backend_ctx->get_adreno_bin_kernel("gemm_noshuffle_q6_k_f32_32b_trans_ila_a8", &bin_size); if (kernel_bin && bin_size > 0) { cl_program bin_prog = build_program_from_binary(backend_ctx->context, backend_ctx->device, kernel_bin, "", bin_size); CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q6_k_f32_32b_trans_ila_a8_bin = clCreateKernel(bin_prog, "kernel_gemm_noshuffle_q6_k_f32_32b_trans_ila_a8", &err), err)); CL_CHECK(clReleaseProgram(bin_prog)); GGML_LOG_CONT("."); } kernel_bin = (const char *)backend_ctx->get_adreno_bin_kernel("gemm_noshuffle_q6_k_q8_1_dp4a_ila_a8", &bin_size); if (kernel_bin && bin_size > 0) { cl_program bin_prog = build_program_from_binary(backend_ctx->context, backend_ctx->device, kernel_bin, "", bin_size); CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q6_k_q8_1_dp4a_ila_a8_bin = clCreateKernel(bin_prog, "kernel_gemm_noshuffle_q6_k_q8_1_dp4a_ila_a8", &err), err)); CL_CHECK(clReleaseProgram(bin_prog)); GGML_LOG_CONT("."); } } } backend_ctx->kernel_gemv_noshuffle_q5_k_f32_32b_trans = nullptr; backend_ctx->kernel_gemm_noshuffle_q5_k_f32_32b_trans_ila_a8_bin = nullptr; backend_ctx->kernel_gemm_noshuffle_q5_k_q8_1_dp4a_ila_a8_bin = nullptr; { if (backend_ctx->has_vector_subgroup_broadcast) { std::string opts = std::string("-cl-std=") + opencl_c_std + " -cl-mad-enable " " -DSIMDGROUP_WIDTH=" + std::to_string(backend_ctx->adreno_wave_size); #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { #include "gemv_noshuffle_q5_k_f32_32b_trans.cl.h" }; #else const std::string kernel_src = read_file("gemv_noshuffle_q5_k_f32_32b_trans.cl"); #endif cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), opts); CL_CHECK((backend_ctx->kernel_gemv_noshuffle_q5_k_f32_32b_trans = clCreateKernel(prog, "gemv_noshuffle_q5_k_f32_32b_trans", &err), err)); CL_CHECK(clReleaseProgram(prog)); GGML_LOG_CONT("."); } if (use_adreno_bin_kernels(backend_ctx)) { size_t bin_size = 0; const char * kernel_bin = (const char *)backend_ctx->get_adreno_bin_kernel("gemm_noshuffle_q5_k_f32_32b_trans_ila_a8", &bin_size); if (kernel_bin && bin_size > 0) { cl_program bin_prog = build_program_from_binary(backend_ctx->context, backend_ctx->device, kernel_bin, "", bin_size); CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q5_k_f32_32b_trans_ila_a8_bin = clCreateKernel(bin_prog, "kernel_gemm_noshuffle_q5_k_f32_32b_trans_ila_a8", &err), err)); CL_CHECK(clReleaseProgram(bin_prog)); GGML_LOG_CONT("."); } kernel_bin = (const char *)backend_ctx->get_adreno_bin_kernel("gemm_noshuffle_q5_k_q8_1_dp4a_ila_a8", &bin_size); if (kernel_bin && bin_size > 0) { cl_program bin_prog = build_program_from_binary(backend_ctx->context, backend_ctx->device, kernel_bin, "", bin_size); CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q5_k_q8_1_dp4a_ila_a8_bin = clCreateKernel(bin_prog, "kernel_gemm_noshuffle_q5_k_q8_1_dp4a_ila_a8", &err), err)); CL_CHECK(clReleaseProgram(bin_prog)); GGML_LOG_CONT("."); } } } std::string CL_moe_compile_opts = std::string("-cl-std=") + opencl_c_std + " -cl-mad-enable " " -cl-fast-relaxed-math"; // gemv_moe_q4_1_f32_ns { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { #include "gemv_moe_q4_1_f32_ns.cl.h" }; #else const std::string kernel_src = read_file("gemv_moe_q4_1_f32_ns.cl"); #endif cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), CL_moe_compile_opts); CL_CHECK((backend_ctx->kernel_gemv_moe_q4_1_f32_ns = clCreateKernel(prog, "kernel_gemv_moe_q4_1_f32_ns", &err), err)); CL_CHECK(clReleaseProgram(prog)); GGML_LOG_CONT("."); } // gemm_moe_q4_1_f32_ns { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { #include "gemm_moe_q4_1_f32_ns.cl.h" }; #else const std::string kernel_src = read_file("gemm_moe_q4_1_f32_ns.cl"); #endif cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), CL_moe_compile_opts); CL_CHECK((backend_ctx->kernel_gemm_moe_q4_1_f32_ns = clCreateKernel(prog, "kernel_gemm_moe_q4_1_f32_ns", &err), err)); CL_CHECK(clReleaseProgram(prog)); GGML_LOG_CONT("."); } // gemm_moe_q4_1_f32_ns_bin { size_t bin_size = 0; backend_ctx->kernel_gemm_moe_q4_1_f32_ns_bin = nullptr; if (use_adreno_bin_kernels(backend_ctx)) { const char * kernel_bin = (const char *)backend_ctx->get_adreno_bin_kernel("gemm_moe_q4_1_f32_ns_ila", &bin_size); if (kernel_bin && bin_size > 0) { cl_program prog = build_program_from_binary(backend_ctx->context, backend_ctx->device, kernel_bin, CL_moe_compile_opts, bin_size); CL_CHECK((backend_ctx->kernel_gemm_moe_q4_1_f32_ns_bin = clCreateKernel(prog, "kernel_gemm_moe_q4_1_f32_ns_ila", &err), err)); CL_CHECK(clReleaseProgram(prog)); GGML_LOG_CONT("."); } } } // gemv_moe_mxfp4_f32 { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { #include "gemv_moe_mxfp4_f32.cl.h" }; #else const std::string kernel_src = read_file("gemv_moe_mxfp4_f32.cl"); #endif backend_ctx->program_gemv_moe_mxfp4_f32 = build_program_from_source(backend_ctx, kernel_src.c_str(), CL_moe_compile_opts); CL_CHECK((backend_ctx->kernel_gemv_moe_mxfp4_f32 = clCreateKernel(backend_ctx->program_gemv_moe_mxfp4_f32, "kernel_gemv_moe_mxfp4_f32", &err), err)); GGML_LOG_CONT("."); } // gemm_moe_mxfp4_f32 { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { #include "gemm_moe_mxfp4_f32.cl.h" }; #else const std::string kernel_src = read_file("gemm_moe_mxfp4_f32.cl"); #endif backend_ctx->program_gemm_moe_mxfp4_f32 = build_program_from_source(backend_ctx, kernel_src.c_str(), CL_moe_compile_opts); CL_CHECK((backend_ctx->kernel_gemm_moe_mxfp4_f32 = clCreateKernel(backend_ctx->program_gemm_moe_mxfp4_f32, "kernel_gemm_moe_mxfp4_f32", &err), err)); GGML_LOG_CONT("."); } // gemv_moe_q4_0_f32_ns { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { #include "gemv_moe_q4_0_f32_ns.cl.h" }; #else const std::string kernel_src = read_file("gemv_moe_q4_0_f32_ns.cl"); #endif cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), CL_moe_compile_opts); CL_CHECK((backend_ctx->kernel_gemv_moe_q4_0_f32_ns = clCreateKernel(prog, "kernel_gemv_moe_q4_0_f32_ns", &err), err)); CL_CHECK(clReleaseProgram(prog)); GGML_LOG_CONT("."); } // gemm_moe_q4_0_f32_ns { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { #include "gemm_moe_q4_0_f32_ns.cl.h" }; #else const std::string kernel_src = read_file("gemm_moe_q4_0_f32_ns.cl"); #endif cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), CL_moe_compile_opts); CL_CHECK((backend_ctx->kernel_gemm_moe_q4_0_f32_ns = clCreateKernel(prog, "kernel_gemm_moe_q4_0_f32_ns", &err), err)); CL_CHECK(clReleaseProgram(prog)); GGML_LOG_CONT("."); } // gemm_moe_q4_0_f32_ns_bin { size_t bin_size = 0; backend_ctx->kernel_gemm_moe_q4_0_f32_ns_bin = nullptr; if (use_adreno_bin_kernels(backend_ctx)) { const char * kernel_bin = (const char *)backend_ctx->get_adreno_bin_kernel("gemm_moe_q4_0_f32_ns_ila", &bin_size); if (kernel_bin && bin_size > 0) { cl_program prog = build_program_from_binary(backend_ctx->context, backend_ctx->device, kernel_bin, CL_moe_compile_opts, bin_size); CL_CHECK((backend_ctx->kernel_gemm_moe_q4_0_f32_ns_bin = clCreateKernel(prog, "kernel_gemm_moe_q4_0_f32_ns_ila", &err), err)); CL_CHECK(clReleaseProgram(prog)); GGML_LOG_CONT("."); } } } // gemm_moe_q8_0_f32_ns { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { #include "gemm_moe_q8_0_f32_ns.cl.h" }; #else const std::string kernel_src = read_file("gemm_moe_q8_0_f32_ns.cl"); #endif cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), CL_moe_compile_opts); CL_CHECK((backend_ctx->kernel_gemm_moe_q8_0_f32_ns = clCreateKernel(prog, "kernel_gemm_moe_q8_0_f32_ns", &err), err)); CL_CHECK(clReleaseProgram(prog)); GGML_LOG_CONT("."); } // gemv_moe_q5_0_f32_ns { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { #include "gemv_moe_q5_0_f32_ns.cl.h" }; #else const std::string kernel_src = read_file("gemv_moe_q5_0_f32_ns.cl"); #endif cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), CL_moe_compile_opts); CL_CHECK((backend_ctx->kernel_gemv_moe_q5_0_f32_ns = clCreateKernel(prog, "kernel_gemv_moe_q5_0_f32_ns", &err), err)); CL_CHECK(clReleaseProgram(prog)); GGML_LOG_CONT("."); } // gemm_moe_q5_0_f32_ns { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { #include "gemm_moe_q5_0_f32_ns.cl.h" }; #else const std::string kernel_src = read_file("gemm_moe_q5_0_f32_ns.cl"); #endif cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), CL_moe_compile_opts); CL_CHECK((backend_ctx->kernel_gemm_moe_q5_0_f32_ns = clCreateKernel(prog, "kernel_gemm_moe_q5_0_f32_ns", &err), err)); CL_CHECK(clReleaseProgram(prog)); GGML_LOG_CONT("."); } // gemv_moe_q5_1_f32_ns { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { #include "gemv_moe_q5_1_f32_ns.cl.h" }; #else const std::string kernel_src = read_file("gemv_moe_q5_1_f32_ns.cl"); #endif cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), CL_moe_compile_opts); CL_CHECK((backend_ctx->kernel_gemv_moe_q5_1_f32_ns = clCreateKernel(prog, "kernel_gemv_moe_q5_1_f32_ns", &err), err)); CL_CHECK(clReleaseProgram(prog)); GGML_LOG_CONT("."); } // gemm_moe_q5_1_f32_ns { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { #include "gemm_moe_q5_1_f32_ns.cl.h" }; #else const std::string kernel_src = read_file("gemm_moe_q5_1_f32_ns.cl"); #endif cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), CL_moe_compile_opts); CL_CHECK((backend_ctx->kernel_gemm_moe_q5_1_f32_ns = clCreateKernel(prog, "kernel_gemm_moe_q5_1_f32_ns", &err), err)); CL_CHECK(clReleaseProgram(prog)); GGML_LOG_CONT("."); } // gemv_moe_q4_k_f32_ns { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { #include "gemv_moe_q4_k_f32_ns.cl.h" }; #else const std::string kernel_src = read_file("gemv_moe_q4_k_f32_ns.cl"); #endif cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), CL_moe_compile_opts); CL_CHECK((backend_ctx->kernel_gemv_moe_q4_k_f32_ns = clCreateKernel(prog, "kernel_gemv_moe_q4_k_f32_ns", &err), err)); CL_CHECK((backend_ctx->kernel_gemv_moe_q4_k_f32_ns_wimg = clCreateKernel(prog, "kernel_gemv_moe_q4_k_f32_ns_wimg", &err), err)); CL_CHECK(clReleaseProgram(prog)); GGML_LOG_CONT("."); } // gemm_moe_q4_k_f32_ns { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { #include "gemm_moe_q4_k_f32_ns.cl.h" }; #else const std::string kernel_src = read_file("gemm_moe_q4_k_f32_ns.cl"); #endif cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), CL_moe_compile_opts); CL_CHECK((backend_ctx->kernel_gemm_moe_q4_k_f32_ns = clCreateKernel(prog, "kernel_gemm_moe_q4_k_f32_ns", &err), err)); CL_CHECK(clReleaseProgram(prog)); GGML_LOG_CONT("."); } // gemm_moe_q4_k_f32_ns_bin { size_t bin_size = 0; backend_ctx->kernel_gemm_moe_q4_k_f32_ns_bin = nullptr; if (use_adreno_bin_kernels(backend_ctx)) { const char * kernel_bin = (const char *)backend_ctx->get_adreno_bin_kernel("gemm_moe_q4_k_f32_ns_ila", &bin_size); if (kernel_bin && bin_size > 0) { cl_program prog = build_program_from_binary(backend_ctx->context, backend_ctx->device, kernel_bin, CL_moe_compile_opts, bin_size); CL_CHECK((backend_ctx->kernel_gemm_moe_q4_k_f32_ns_bin = clCreateKernel(prog, "kernel_gemm_moe_q4_k_f32_ns_ila", &err), err)); CL_CHECK(clReleaseProgram(prog)); GGML_LOG_CONT("."); } } } // gemm_moe_q4_k_q8_1_dp4a (dp4a prefill GEMM) if (backend_ctx->has_integer_dot) { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { #include "gemm_moe_q4_k_q8_1_dp4a.cl.h" }; #else const std::string kernel_src = read_file("gemm_moe_q4_k_q8_1_dp4a.cl"); #endif cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), CL_moe_compile_opts); CL_CHECK((backend_ctx->kernel_gemm_moe_q4_k_q8_1_dp4a = clCreateKernel(prog, "kernel_gemm_moe_q4_k_q8_1_dp4a", &err), err)); CL_CHECK(clReleaseProgram(prog)); GGML_LOG_CONT("."); } // gemm_moe_mxfp4_q8_1_dp4a (dp4a prefill GEMM) if (backend_ctx->has_integer_dot) { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { #include "gemm_moe_mxfp4_q8_1_dp4a.cl.h" }; #else const std::string kernel_src = read_file("gemm_moe_mxfp4_q8_1_dp4a.cl"); #endif cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), CL_moe_compile_opts); CL_CHECK((backend_ctx->kernel_gemm_moe_mxfp4_q8_1_dp4a = clCreateKernel(prog, "kernel_gemm_moe_mxfp4_q8_1_dp4a", &err), err)); CL_CHECK(clReleaseProgram(prog)); GGML_LOG_CONT("."); } // gemm_moe_mxfp4_q8_1_dp4a_bin (dp4a prefill GEMM) if (backend_ctx->has_integer_dot) { size_t bin_size = 0; backend_ctx->kernel_gemm_moe_mxfp4_q8_1_dp4a_bin = nullptr; if (use_adreno_bin_kernels(backend_ctx)) { const char * kernel_bin = (const char *)backend_ctx->get_adreno_bin_kernel("gemm_moe_mxfp4_q8_1_dp4a_ila", &bin_size); if (kernel_bin && bin_size > 0) { cl_program prog = build_program_from_binary(backend_ctx->context, backend_ctx->device, kernel_bin, CL_moe_compile_opts, bin_size); CL_CHECK((backend_ctx->kernel_gemm_moe_mxfp4_q8_1_dp4a_bin = clCreateKernel(prog, "kernel_gemm_moe_mxfp4_q8_1_dp4a_ila", &err), err)); CL_CHECK(clReleaseProgram(prog)); GGML_LOG_CONT("."); } } } // gemm_moe_q4_0_q8_1_dp4a (dp4a prefill GEMM) if (backend_ctx->has_integer_dot) { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { #include "gemm_moe_q4_0_q8_1_dp4a.cl.h" }; #else const std::string kernel_src = read_file("gemm_moe_q4_0_q8_1_dp4a.cl"); #endif cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), CL_moe_compile_opts); CL_CHECK((backend_ctx->kernel_gemm_moe_q4_0_q8_1_dp4a = clCreateKernel(prog, "kernel_gemm_moe_q4_0_q8_1_dp4a", &err), err)); CL_CHECK(clReleaseProgram(prog)); GGML_LOG_CONT("."); } // gemm_moe_q4_0_q8_1_dp4a_bin (dp4a prefill GEMM) if (backend_ctx->has_integer_dot) { size_t bin_size = 0; backend_ctx->kernel_gemm_moe_q4_0_q8_1_dp4a_bin = nullptr; if (use_adreno_bin_kernels(backend_ctx)) { const char * kernel_bin = (const char *)backend_ctx->get_adreno_bin_kernel("gemm_moe_q4_0_q8_1_dp4a_ila", &bin_size); if (kernel_bin && bin_size > 0) { cl_program prog = build_program_from_binary(backend_ctx->context, backend_ctx->device, kernel_bin, CL_moe_compile_opts, bin_size); CL_CHECK((backend_ctx->kernel_gemm_moe_q4_0_q8_1_dp4a_bin = clCreateKernel(prog, "kernel_gemm_moe_q4_0_q8_1_dp4a_ila", &err), err)); CL_CHECK(clReleaseProgram(prog)); GGML_LOG_CONT("."); } } } // gemm_moe_q8_1_dp4a (generic dp4a MoE GEMM; MOE_QT=80 -> q8_0 expert variant) if (backend_ctx->has_integer_dot) { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { #include "gemm_moe_q8_1_dp4a.cl.h" }; #else const std::string kernel_src = read_file("gemm_moe_q8_1_dp4a.cl"); #endif const std::string opts80 = CL_moe_compile_opts + " -DMOE_QT=80"; cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), opts80.c_str()); CL_CHECK((backend_ctx->kernel_gemm_moe_q8_1_dp4a_q80 = clCreateKernel(prog, "kernel_gemm_moe_q8_1_dp4a", &err), err)); CL_CHECK(clReleaseProgram(prog)); const std::string opts50 = CL_moe_compile_opts + " -DMOE_QT=50"; cl_program prog50 = build_program_from_source(backend_ctx, kernel_src.c_str(), opts50.c_str()); CL_CHECK((backend_ctx->kernel_gemm_moe_q8_1_dp4a_q50 = clCreateKernel(prog50, "kernel_gemm_moe_q8_1_dp4a", &err), err)); CL_CHECK(clReleaseProgram(prog50)); const std::string opts5 = CL_moe_compile_opts + " -DMOE_QT=5"; cl_program prog5 = build_program_from_source(backend_ctx, kernel_src.c_str(), opts5.c_str()); CL_CHECK((backend_ctx->kernel_gemm_moe_q8_1_dp4a_q5k = clCreateKernel(prog5, "kernel_gemm_moe_q8_1_dp4a", &err), err)); CL_CHECK(clReleaseProgram(prog5)); GGML_LOG_CONT("."); } // moe_reorder_quant_a_q8_1 (fused reorder + q8_1 quant) { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { #include "moe_reorder_quant_a_q8_1.cl.h" }; #else const std::string kernel_src = read_file("moe_reorder_quant_a_q8_1.cl"); #endif cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), CL_moe_compile_opts); CL_CHECK((backend_ctx->kernel_moe_reorder_quant_a_q8_1 = clCreateKernel(prog, "kernel_moe_reorder_quant_a_q8_1", &err), err)); CL_CHECK(clReleaseProgram(prog)); GGML_LOG_CONT("."); } // gemv_moe_q5_k_f32_ns { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { #include "gemv_moe_q5_k_f32_ns.cl.h" }; #else const std::string kernel_src = read_file("gemv_moe_q5_k_f32_ns.cl"); #endif cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), CL_moe_compile_opts); CL_CHECK((backend_ctx->kernel_gemv_moe_q5_k_f32_ns = clCreateKernel(prog, "kernel_gemv_moe_q5_k_f32_ns", &err), err)); CL_CHECK(clReleaseProgram(prog)); GGML_LOG_CONT("."); } // gemm_moe_q5_k_f32_ns { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { #include "gemm_moe_q5_k_f32_ns.cl.h" }; #else const std::string kernel_src = read_file("gemm_moe_q5_k_f32_ns.cl"); #endif cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), CL_moe_compile_opts); CL_CHECK((backend_ctx->kernel_gemm_moe_q5_k_f32_ns = clCreateKernel(prog, "kernel_gemm_moe_q5_k_f32_ns", &err), err)); CL_CHECK(clReleaseProgram(prog)); GGML_LOG_CONT("."); } // gemv_moe_q6_k_f32_ns { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { #include "gemv_moe_q6_k_f32_ns.cl.h" }; #else const std::string kernel_src = read_file("gemv_moe_q6_k_f32_ns.cl"); #endif cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), CL_moe_compile_opts); CL_CHECK((backend_ctx->kernel_gemv_moe_q6_k_f32_ns = clCreateKernel(prog, "kernel_gemv_moe_q6_k_f32_ns", &err), err)); CL_CHECK(clReleaseProgram(prog)); GGML_LOG_CONT("."); } // gemm_moe_q6_k_f32_ns { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { #include "gemm_moe_q6_k_f32_ns.cl.h" }; #else const std::string kernel_src = read_file("gemm_moe_q6_k_f32_ns.cl"); #endif cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), CL_moe_compile_opts); CL_CHECK((backend_ctx->kernel_gemm_moe_q6_k_f32_ns = clCreateKernel(prog, "kernel_gemm_moe_q6_k_f32_ns", &err), err)); CL_CHECK(clReleaseProgram(prog)); GGML_LOG_CONT("."); } // gemm_moe_q6_k_f32_ns_bin { size_t bin_size = 0; backend_ctx->kernel_gemm_moe_q6_k_f32_ns_bin = nullptr; if (use_adreno_bin_kernels(backend_ctx)) { const char * kernel_bin = (const char *)backend_ctx->get_adreno_bin_kernel("gemm_moe_q6_k_f32_ns_ila", &bin_size); if (kernel_bin && bin_size > 0) { cl_program prog = build_program_from_binary(backend_ctx->context, backend_ctx->device, kernel_bin, CL_moe_compile_opts, bin_size); CL_CHECK((backend_ctx->kernel_gemm_moe_q6_k_f32_ns_bin = clCreateKernel(prog, "kernel_gemm_moe_q6_k_f32_ns_ila", &err), err)); CL_CHECK(clReleaseProgram(prog)); GGML_LOG_CONT("."); } } } // gemm_moe_q6_k_q8_1_dp4a (dp4a q6_K MoE prefill GEMM) if (backend_ctx->has_integer_dot) { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { #include "gemm_moe_q6_k_q8_1_dp4a.cl.h" }; #else const std::string kernel_src = read_file("gemm_moe_q6_k_q8_1_dp4a.cl"); #endif cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), CL_moe_compile_opts); CL_CHECK((backend_ctx->kernel_gemm_moe_q6_k_q8_1_dp4a = clCreateKernel(prog, "kernel_gemm_moe_q6_k_q8_1_dp4a", &err), err)); CL_CHECK(clReleaseProgram(prog)); GGML_LOG_CONT("."); } // gemv_moe_mxfp4_f32_ns { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { #include "gemv_moe_mxfp4_f32_ns.cl.h" }; #else const std::string kernel_src = read_file("gemv_moe_mxfp4_f32_ns.cl"); #endif cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), CL_moe_compile_opts); CL_CHECK((backend_ctx->kernel_gemv_moe_mxfp4_f32_ns = clCreateKernel(prog, "kernel_gemv_moe_mxfp4_f32_ns", &err), err)); CL_CHECK((backend_ctx->kernel_gemv_moe_mxfp4_f32_ns_wimg = clCreateKernel(prog, "kernel_gemv_moe_mxfp4_f32_ns_wimg", &err), err)); CL_CHECK(clReleaseProgram(prog)); GGML_LOG_CONT("."); } // gemm_moe_mxfp4_f32_ns { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { #include "gemm_moe_mxfp4_f32_ns.cl.h" }; #else const std::string kernel_src = read_file("gemm_moe_mxfp4_f32_ns.cl"); #endif cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), CL_moe_compile_opts); CL_CHECK((backend_ctx->kernel_gemm_moe_mxfp4_f32_ns = clCreateKernel(prog, "kernel_gemm_moe_mxfp4_f32_ns", &err), err)); CL_CHECK(clReleaseProgram(prog)); GGML_LOG_CONT("."); } // gemm_moe_mxfp4_f32_ns_bin { size_t bin_size = 0; backend_ctx->kernel_gemm_moe_mxfp4_f32_ns_bin = nullptr; if (use_adreno_bin_kernels(backend_ctx)) { const char * kernel_bin = (const char *)backend_ctx->get_adreno_bin_kernel("gemm_moe_mxfp4_f32_ns_ila", &bin_size); if (kernel_bin && bin_size > 0) { cl_program prog = build_program_from_binary(backend_ctx->context, backend_ctx->device, kernel_bin, CL_moe_compile_opts, bin_size); CL_CHECK((backend_ctx->kernel_gemm_moe_mxfp4_f32_ns_bin = clCreateKernel(prog, "kernel_gemm_moe_mxfp4_f32_ns_ila", &err), err)); CL_CHECK(clReleaseProgram(prog)); GGML_LOG_CONT("."); } } } // moe_reorder_b { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { #include "moe_reorder_b.cl.h" }; #else const std::string kernel_src = read_file("moe_reorder_b.cl"); #endif cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), CL_moe_compile_opts); CL_CHECK((backend_ctx->kernel_moe_reorder_b = clCreateKernel(prog, "kernel_moe_reorder_b", &err), err)); CL_CHECK(clReleaseProgram(prog)); GGML_LOG_CONT("."); } // moe_sort_by_expert { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { #include "moe_sort_by_expert.cl.h" }; #else const std::string kernel_src = read_file("moe_sort_by_expert.cl"); #endif cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), CL_moe_compile_opts); CL_CHECK((backend_ctx->kernel_moe_histogram = clCreateKernel(prog, "kernel_moe_histogram", &err), err)); CL_CHECK((backend_ctx->kernel_moe_scan = clCreateKernel(prog, "kernel_moe_scan", &err), err)); CL_CHECK((backend_ctx->kernel_moe_fill = clCreateKernel(prog, "kernel_moe_fill", &err), err)); CL_CHECK((backend_ctx->kernel_moe_scatter = clCreateKernel(prog, "kernel_moe_scatter", &err), err)); CL_CHECK((backend_ctx->kernel_moe_scatter_stable = clCreateKernel(prog, "kernel_moe_scatter_stable", &err), err)); CL_CHECK(clReleaseProgram(prog)); GGML_LOG_CONT("."); } // gemv_noshuffle_q6_k_f32 { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { #include "gemv_noshuffle_q6_k_f32.cl.h" }; #else const std::string kernel_src = read_file("gemv_noshuffle_q6_k_f32.cl"); #endif std::string CL_gemv_compile_opts = std::string("-cl-std=") + opencl_c_std + " -cl-mad-enable "; if (backend_ctx->has_vector_subgroup_broadcast) { CL_gemv_compile_opts += " -DVECTOR_SUB_GROUP_BROADCAT "; } cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), CL_gemv_compile_opts); CL_CHECK((backend_ctx->kernel_gemv_noshuffle_q6_K_f32 = clCreateKernel(prog, "kernel_gemv_noshuffle_q6_K_f32", &err), err)); CL_CHECK((backend_ctx->kernel_gemv_noshuffle_q6_K_f32_mc3 = clCreateKernel(prog, "kernel_gemv_noshuffle_q6_K_f32_mc3", &err), err)); if (getenv("GGML_OPENCL_MC3_PROBE")) { cl_ulong pm6 = 0, pm4 = 0; size_t wg6 = 0, wg4 = 0, mult = 0; clGetKernelWorkGroupInfo(backend_ctx->kernel_gemv_noshuffle_q6_K_f32_mc3, backend_ctx->device, CL_KERNEL_PRIVATE_MEM_SIZE, sizeof(pm6), &pm6, NULL); clGetKernelWorkGroupInfo(backend_ctx->kernel_gemv_noshuffle_q6_K_f32_mc3, backend_ctx->device, CL_KERNEL_WORK_GROUP_SIZE, sizeof(wg6), &wg6, NULL); clGetKernelWorkGroupInfo(backend_ctx->kernel_gemv_noshuffle_q4_k_f32_mc3, backend_ctx->device, CL_KERNEL_PRIVATE_MEM_SIZE, sizeof(pm4), &pm4, NULL); clGetKernelWorkGroupInfo(backend_ctx->kernel_gemv_noshuffle_q4_k_f32_mc3, backend_ctx->device, CL_KERNEL_WORK_GROUP_SIZE, sizeof(wg4), &wg4, NULL); clGetKernelWorkGroupInfo(backend_ctx->kernel_gemv_noshuffle_q6_K_f32_mc3, backend_ctx->device, CL_KERNEL_PREFERRED_WORK_GROUP_SIZE_MULTIPLE, sizeof(mult), &mult, NULL); fprintf(stderr, "[MC3-PROBE] q4K_mc3 private=%llu wg_cap=%zu | q6K_mc3 private=%llu wg_cap=%zu | pref_mult=%zu\n", (unsigned long long)pm4, wg4, (unsigned long long)pm6, wg6, mult); fflush(stderr); } GGML_LOG_CONT("."); } // gemv_noshuffle_q6_k_f32_o4 — 4-output-per-WI variant, opt-in via // GGML_OPENCL_Q6K_GEMV_O4=1 (~3x fewer dispatches on long-vocab lm_head). { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { #include "gemv_noshuffle_q6_k_f32_o4.cl.h" }; #else const std::string kernel_src = read_file("gemv_noshuffle_q6_k_f32_o4.cl"); #endif std::string CL_gemv_compile_opts = std::string("-cl-std=") + opencl_c_std + " -cl-mad-enable "; if (backend_ctx->has_vector_subgroup_broadcast) { CL_gemv_compile_opts += " -DVECTOR_SUB_GROUP_BROADCAT "; } cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), CL_gemv_compile_opts); CL_CHECK((backend_ctx->kernel_gemv_noshuffle_q6_K_f32_o4 = clCreateKernel(prog, "kernel_gemv_noshuffle_q6_K_f32_o4", &err), err)); CL_CHECK(clReleaseProgram(prog)); // Global-read variant: weights read from __global coalesced instead of // image1d_buffer (the texture cache caps the streaming lm_head read // bandwidth). Opt-in via GGML_OPENCL_Q6K_GEMV_O4_GLOBAL. cl_program prog_g = build_program_from_source(backend_ctx, kernel_src.c_str(), CL_gemv_compile_opts + " -DQ6K_O4_GLOBAL"); CL_CHECK((backend_ctx->kernel_gemv_noshuffle_q6_K_f32_o4_global = clCreateKernel(prog_g, "kernel_gemv_noshuffle_q6_K_f32_o4_global", &err), err)); CL_CHECK(clReleaseProgram(prog_g)); GGML_LOG_CONT("."); } // gemv_noshuffle_q6_k_f32_tiled — tiled-wide canonical layout, default ON // (opt out: GGML_OPENCL_Q6K_GEMV_TILED=0; separate convert + GEMV; weights via __global). { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { #include "gemv_noshuffle_q6_k_f32_tiled.cl.h" }; #else const std::string kernel_src = read_file("gemv_noshuffle_q6_k_f32_tiled.cl"); #endif std::string compile_opts = std::string("-cl-std=") + opencl_c_std + " -cl-mad-enable "; cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); CL_CHECK((backend_ctx->kernel_gemv_noshuffle_q6_K_f32_tiled = clCreateKernel(prog, "kernel_gemv_noshuffle_q6_K_f32_tiled", &err), err)); CL_CHECK((backend_ctx->kernel_gemv_noshuffle_q6_K_f32_tiled_mc3 = clCreateKernel(prog, "kernel_gemv_noshuffle_q6_K_f32_tiled_mc3", &err), err)); CL_CHECK(clReleaseProgram(prog)); GGML_LOG_CONT("."); } // gemm_noshuffle_q6_k_f32_tiled — batched (N>1) GEMM over the same tiled-wide // canonical layout, so batched lm_head/embed stays correct + on GPU. { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { #include "gemm_noshuffle_q6_k_f32_tiled.cl.h" }; #else const std::string kernel_src = read_file("gemm_noshuffle_q6_k_f32_tiled.cl"); #endif std::string compile_opts = std::string("-cl-std=") + opencl_c_std + " -cl-mad-enable "; cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q6_K_f32_tiled = clCreateKernel(prog, "kernel_gemm_noshuffle_q6_K_f32_tiled", &err), err)); CL_CHECK(clReleaseProgram(prog)); GGML_LOG_CONT("."); } // gemm_noshuffle_q6_k_f32 { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { #include "gemm_noshuffle_q6_k_f32.cl.h" }; #else const std::string kernel_src = read_file("gemm_noshuffle_q6_k_f32.cl"); #endif cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), CL_moe_compile_opts); CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q6_K_f32 = clCreateKernel(prog, "kernel_gemm_noshuffle_q6_K_f32", &err), err)); CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q6_K_f32_cok = clCreateKernel(prog, "kernel_gemm_noshuffle_q6_K_f32_cok", &err), err)); GGML_LOG_CONT("."); } // gemv_noshuffle_q5_k_f32 { std::string CL_gemv_compile_opts = std::string("-cl-std=") + opencl_c_std + " -cl-mad-enable "; if (backend_ctx->has_vector_subgroup_broadcast) { CL_gemv_compile_opts += " -DVECTOR_SUB_GROUP_BROADCAST "; } #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { #include "gemv_noshuffle_q5_k_f32.cl.h" }; #else const std::string kernel_src = read_file("gemv_noshuffle_q5_k_f32.cl"); #endif cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), CL_gemv_compile_opts); CL_CHECK((backend_ctx->kernel_gemv_noshuffle_q5_k_f32 = clCreateKernel(prog, "kernel_gemv_noshuffle_q5_k_f32", &err), err)); CL_CHECK((backend_ctx->kernel_gemv_noshuffle_q5_k_f32_mc3 = clCreateKernel(prog, "kernel_gemv_noshuffle_q5_k_f32_mc3", &err), err)); CL_CHECK(clReleaseProgram(prog)); GGML_LOG_CONT("."); } // gemm_noshuffle_q5_k_f32 { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { #include "gemm_noshuffle_q5_k_f32.cl.h" }; #else const std::string kernel_src = read_file("gemm_noshuffle_q5_k_f32.cl"); #endif cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q5_k_f32 = clCreateKernel(prog, "kernel_gemm_noshuffle_q5_k_f32", &err), err)); CL_CHECK(clReleaseProgram(prog)); GGML_LOG_CONT("."); } // repack { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { #include "flash_attn_repack.cl.h" }; #else const std::string kernel_src = read_file("flash_attn_repack.cl"); #endif cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); CL_CHECK((backend_ctx->fa.kernel_repack_q_for_wmm = clCreateKernel(prog, "kernel_repack_q_for_wmm", &err), err)); CL_CHECK((backend_ctx->fa.kernel_repack_k_for_wmm = clCreateKernel(prog, "kernel_repack_k_for_wmm", &err), err)); CL_CHECK((backend_ctx->fa.kernel_repack_v_for_wmm = clCreateKernel(prog, "kernel_repack_v_for_wmm", &err), err)); CL_CHECK((backend_ctx->fa.kernel_repack_mask_for_wmm = clCreateKernel(prog, "kernel_repack_mask_for_wmm", &err), err)); GGML_LOG_CONT("."); } // kernel_flash_attn_f32_f16_bin { size_t bin_size = 0; backend_ctx->fa.kernel_flash_attn_f32_f16_bin = nullptr; if (use_adreno_bin_kernels(backend_ctx)) { const char * kernel_bin = (const char *)backend_ctx->get_adreno_bin_kernel("flash_attn_f32_f16_wmm", &bin_size); if (kernel_bin && bin_size > 0) { cl_program prog = build_program_from_binary(backend_ctx->context, backend_ctx->device, kernel_bin, CL_moe_compile_opts, bin_size); CL_CHECK((backend_ctx->fa.kernel_flash_attn_f32_f16_bin = clCreateKernel(prog, "flash_attn_f32_f16", &err), err)); CL_CHECK(clReleaseProgram(prog)); GGML_LOG_CONT("."); } } } #endif // GGML_OPENCL_USE_ADRENO_KERNELS GGML_LOG_CONT("\n"); backend_ctx->kernels_loaded = true; } static ggml_backend_opencl_context * ggml_cl_init(ggml_backend_dev_t dev); static bool ggml_opencl_is_device_supported(ggml_backend_dev_t dev); // FA per-(dk,dv) tile tuning table + GGML_OPENCL_FA_TUNE override parsing. #include "fa_tune.h" // FA variant key for the per-(dk,dv,variant) lazy compile cache. // Kernel built on first dispatch to reduce kernel loading time. // NB - a warmup run is recommended to get all necessary FA variants compiled // before actual runs. enum ggml_opencl_fa_variant { FA_VARIANT_PRE = 0, // prepass kernels (kv_pad, mask_pad, blk) FA_VARIANT_F16 = 1, FA_VARIANT_F32 = 2, FA_VARIANT_F32_F16 = 3, FA_VARIANT_Q8_0 = 4, FA_VARIANT_Q4_0 = 5, FA_VARIANT_F32_F16_SPLIT = 6, FA_VARIANT_Q8_0_SPLIT = 7, FA_VARIANT_Q4_0_SPLIT = 8, }; static std::string ggml_opencl_fa_kernel_src(ggml_opencl_fa_variant v) { #ifdef GGML_OPENCL_EMBED_KERNELS switch (v) { case FA_VARIANT_F16: return std::string{ #include "flash_attn_f16.cl.h" }; case FA_VARIANT_F32: return std::string{ #include "flash_attn_f32.cl.h" }; case FA_VARIANT_F32_F16: case FA_VARIANT_F32_F16_SPLIT: return std::string{ #include "flash_attn_f32_f16.cl.h" }; case FA_VARIANT_PRE: return std::string{ #include "flash_attn_pre_f16.cl.h" }; case FA_VARIANT_Q8_0: case FA_VARIANT_Q8_0_SPLIT: return std::string{ #include "flash_attn_f32_q8_0.cl.h" }; case FA_VARIANT_Q4_0: case FA_VARIANT_Q4_0_SPLIT: return std::string{ #include "flash_attn_f32_q4_0.cl.h" }; } return {}; #else switch (v) { case FA_VARIANT_F16: return read_file("flash_attn_f16.cl"); case FA_VARIANT_F32: return read_file("flash_attn_f32.cl"); case FA_VARIANT_F32_F16: case FA_VARIANT_F32_F16_SPLIT: return read_file("flash_attn_f32_f16.cl"); case FA_VARIANT_PRE: return read_file("flash_attn_pre_f16.cl"); case FA_VARIANT_Q8_0: case FA_VARIANT_Q8_0_SPLIT: return read_file("flash_attn_f32_q8_0.cl"); case FA_VARIANT_Q4_0: case FA_VARIANT_Q4_0_SPLIT: return read_file("flash_attn_f32_q4_0.cl"); } return {}; #endif } static std::string ggml_opencl_fa_compile_opts(ggml_backend_opencl_context * backend_ctx, const ggml_opencl_fa_dim * cfg, ggml_opencl_fa_variant variant) { std::string opts = backend_ctx->kernel_compile_opts + " -D DK=" + std::to_string(cfg->dk) + " -D DV=" + std::to_string(cfg->dv) + " -D BLOCK_M=" + std::to_string(cfg->bm) + " -D BLOCK_N=" + std::to_string(cfg->bn); if (backend_ctx->gpu_family == INTEL) { opts += " -D FA_SG=32"; } const bool is_split = variant == FA_VARIANT_F32_F16_SPLIT || variant == FA_VARIANT_Q8_0_SPLIT || variant == FA_VARIANT_Q4_0_SPLIT; if (is_split) { opts += " -D N_SPLIT=" + std::to_string(cfg->n_split); } // Shuffle define for the split tile paths AND the cluster-parallel decode // kernel (q1_vec_mq_split_c8) in the plain F32_F16 program. Without it the // c8 kernel is compiled out (HAS_SUBGROUP_SHUFFLE guard) and dispatch // falls back to the baseline mq_split. if ((is_split || variant == FA_VARIANT_F32_F16) && backend_ctx->has_subgroup_shuffle) { opts += backend_ctx->has_qcom_subgroup_shuffle ? " -D cl_qcom_subgroup_shuffle=1" : " -D cl_khr_subgroup_shuffle=1"; } // X1E drops the explicit sub-group size pin on the c8 kernels, compiler // routes the fp16-heavy kernel to a slow variant with explicit subgroup size if (backend_ctx->adreno_gen == ADRENO_GPU_GEN::X1E) { opts += " -D FA_C8_NO_SG_PIN"; } // Transposed K tile in local memory: the KV rows the QK loop walks together become // adjacent, so a group of them is ONE 128-bit local read instead of several narrow // ones. The QK loop is LDS-read-issue-bound (a wrong-math probe that kept every FMA/dp4a // but removed the LDS reads ran the kernel ~40% faster), so this is worth up to +26% on // fa=1 prefill. Output is bit-identical -- only the layout moves. // // DK <= 128 only. At DK=256 (gemma-3-4b) it measures 1-2% NEGATIVE and reproduces across // rounds; padding the row stride does not recover it, so the cause is not a simple bank // conflict and the wider tile does not want this layout. // // Default on within that gate; GGML_OPENCL_FA_K_LDS_T=0 restores the row-major tile. { const char * e = getenv("GGML_OPENCL_FA_K_LDS_T"); if ((e == nullptr || e[0] != '0') && cfg->dk <= 128) { opts += " -D FA_K_LDS_T"; } } return opts; } // only register when the kernel's required dispatch workgroup size is within // the limit of the device's maximum workgroup size static bool ggml_opencl_fa_kernel_fits_wg(ggml_backend_opencl_context * backend_ctx, cl_kernel kernel, size_t required_wg, const char * name, int dk, int dv) { if (kernel == NULL) { return false; } const size_t dev_max = backend_ctx->max_workgroup_size; if (dev_max < required_wg) { GGML_LOG_INFO("ggml_opencl: %s DK=%d DV=%d requires WG %zu > device max %zu; skipping registration (will fall back)\n", name, dk, dv, required_wg, dev_max); return false; } size_t kwg = 0; cl_int err = clGetKernelWorkGroupInfo(kernel, backend_ctx->device, CL_KERNEL_WORK_GROUP_SIZE, sizeof(kwg), &kwg, NULL); if (err != CL_SUCCESS) { GGML_LOG_INFO("ggml_opencl: clGetKernelWorkGroupInfo failed for %s DK=%d DV=%d (err=%d); skipping registration\n", name, dk, dv, err); return false; } if (kwg < required_wg) { GGML_LOG_INFO("ggml_opencl: %s DK=%d DV=%d per-kernel max %zu < required %zu; skipping registration (will fall back)\n", name, dk, dv, kwg, required_wg); return false; } return true; } // Log private memory for an FA kernel. Enable via `GGML_OPENCL_FA_LOG_SPILL=1`. // On Adreno non-zero private_mem means spilling to global memory due to resource // constraint and usually causes performance degradation. // (per-work-item, no cache locality) — a strong signal to pick a config // with smaller per-thread state (e.g. larger N_SPLIT). static void ggml_opencl_log_fa_kernel_spill(ggml_backend_opencl_context * backend_ctx, cl_kernel kernel, const char * name, int dk, int dv) { static const bool enabled = []{ const char * e = std::getenv("GGML_OPENCL_FA_LOG_SPILL"); return e && e[0] && e[0] != '0'; }(); if (!enabled || kernel == nullptr) { return; } cl_ulong priv_mem = 0; if (clGetKernelWorkGroupInfo(kernel, backend_ctx->device, CL_KERNEL_PRIVATE_MEM_SIZE, sizeof(priv_mem), &priv_mem, NULL) == CL_SUCCESS) { const char * tag = priv_mem > 0 ? "SPILL" : "ok"; GGML_LOG_INFO("ggml_opencl: [%s] %s DK=%d DV=%d private_mem=%llu bytes\n", tag, name, dk, dv, (unsigned long long) priv_mem); } } static void ggml_opencl_ensure_fa_pre_kernels(ggml_backend_opencl_context * backend_ctx, int dk, int dv) { const std::pair dk_dv = {dk, dv}; const ggml_opencl_fa_dim * cfg = nullptr; for (const auto & d : g_opencl_fa_dims) { if (d.dk == dk && d.dv == dv) { cfg = &d; break; } } if (cfg == nullptr) { GGML_ABORT("ggml_opencl: no flash_attn config for DK=%d DV=%d", dk, dv); } // BM-tile metadata is consumed by the prefill dispatch (n_q_blocks / wg // sizing) regardless of whether the prepass kernels are needed for this // n_kv — set it unconditionally backend_ctx->fa.f32_f16_bm[{dk, dv}] = cfg->bm; backend_ctx->fa.f32_f16_bn[{dk, dv}] = cfg->bn; backend_ctx->fa.f32_f16_wg_size[{dk, dv}] = cfg->bm; backend_ctx->fa.bm[{dk, dv}] = cfg->bm; backend_ctx->fa.bn[{dk, dv}] = cfg->bn; if (backend_ctx->fa.kv_pad_f16.count(dk_dv) > 0) { return; } GGML_LOG_INFO("ggml_opencl: lazy-compiling flash_attn prepass for DK=%d DV=%d\n", dk, dv); cl_int err; const std::string src = ggml_opencl_fa_kernel_src(FA_VARIANT_PRE); const std::string opts = ggml_opencl_fa_compile_opts(backend_ctx, cfg, FA_VARIANT_PRE); // retry when kernel compile fails cl_program prog_pre_f16 = build_program_from_source_ex( backend_ctx->context, backend_ctx->device, src.c_str(), opts, /*fatal=*/false, "fa prepass f16", backend_ctx->queue); if (!prog_pre_f16) { return; } cl_kernel k_kv_pad_f16 = clCreateKernel(prog_pre_f16, "flash_attn_kv_pad_f16", &err); if (err != CL_SUCCESS) { clReleaseProgram(prog_pre_f16); return; } cl_kernel k_mask_pad_f16 = clCreateKernel(prog_pre_f16, "flash_attn_mask_pad_f16", &err); if (err != CL_SUCCESS) { clReleaseKernel(k_kv_pad_f16); clReleaseProgram(prog_pre_f16); return; } cl_kernel k_blk_f16 = clCreateKernel(prog_pre_f16, "flash_attn_blk_f16", &err); if (err != CL_SUCCESS) { clReleaseKernel(k_kv_pad_f16); clReleaseKernel(k_mask_pad_f16); clReleaseProgram(prog_pre_f16); return; } backend_ctx->fa.kv_pad_f16[{dk, dv}] = k_kv_pad_f16; backend_ctx->fa.mask_pad_f16[{dk, dv}] = k_mask_pad_f16; backend_ctx->fa.blk_f16[{dk, dv}] = k_blk_f16; clReleaseProgram(prog_pre_f16); } // DK=512 prefill BM-tile static bool ggml_opencl_ensure_fa_f32_f16_prefill_512(ggml_backend_opencl_context * backend_ctx, bool split) { const int dk = 512, dv = 512; const std::pair dk_dv = {dk, dv}; auto & target = split ? backend_ctx->fa.f32_f16_split : backend_ctx->fa.f32_f16; if (target.count(dk_dv) > 0) { return true; } static bool failed[2] = { false, false }; if (failed[split ? 1 : 0]) { return false; } const ggml_opencl_fa_dim * cfg = nullptr; for (const auto & d : g_opencl_fa_dims) { if (d.dk == dk && d.dv == dv) { cfg = &d; break; } } if (cfg == nullptr) { failed[split ? 1 : 0] = true; return false; } if (split && cfg->n_split <= 1) { failed[1] = true; return false; } const ggml_opencl_fa_variant variant = split ? FA_VARIANT_F32_F16_SPLIT : FA_VARIANT_F32_F16; std::string opts = ggml_opencl_fa_compile_opts(backend_ctx, cfg, variant) + " -D FA_PREFILL_ONLY"; cl_program prog = build_program_from_source_ex( backend_ctx->context, backend_ctx->device, ggml_opencl_fa_kernel_src(FA_VARIANT_F32_F16).c_str(), opts, /*fatal=*/false, split ? "fa f32_f16 prefill512 split" : "fa f32_f16 prefill512", backend_ctx->queue); if (!prog) { failed[split ? 1 : 0] = true; return false; } cl_int err; cl_kernel k = clCreateKernel(prog, "flash_attn_f32_f16", &err); if (err != CL_SUCCESS) { clReleaseProgram(prog); failed[split ? 1 : 0] = true; return false; } target[dk_dv] = k; if (split) { backend_ctx->fa.f32_f16_split_wg_size[dk_dv] = cfg->bm * cfg->n_split; backend_ctx->fa.f32_f16_split_nkv_threshold[dk_dv] = cfg->nkv_split_threshold; } ggml_opencl_log_fa_kernel_spill(backend_ctx, k, split ? "flash_attn_f32_f16 (prefill512 split)" : "flash_attn_f32_f16 (prefill512)", dk, dv); clReleaseProgram(prog); // determine whether to use the K-image variant of the split tile static const char * pkimg_build_env = getenv("GGML_OPENCL_FA_PREFILL_K_IMG"); const bool pkimg_build = (pkimg_build_env != NULL) && (pkimg_build_env[0] != '0'); if (split && pkimg_build && backend_ctx->fa.f32_f16_split_k_img.count(dk_dv) == 0) { std::string opts_img = ggml_opencl_fa_compile_opts(backend_ctx, cfg, variant) + " -D FA_PREFILL_ONLY -D FA_K_IMG -D FA_TILE_NAME=flash_attn_f32_f16_k_img"; cl_program prog_img = build_program_from_source_ex( backend_ctx->context, backend_ctx->device, ggml_opencl_fa_kernel_src(FA_VARIANT_F32_F16).c_str(), opts_img, /*fatal=*/false, "fa f32_f16 prefill512 split k_img", backend_ctx->queue); if (prog_img) { cl_int err_img; cl_kernel k_img = clCreateKernel(prog_img, "flash_attn_f32_f16_k_img", &err_img); if (err_img == CL_SUCCESS) { backend_ctx->fa.f32_f16_split_k_img[dk_dv] = k_img; ggml_opencl_log_fa_kernel_spill(backend_ctx, k_img, "flash_attn_f32_f16 (prefill512 split k_img)", dk, dv); } clReleaseProgram(prog_img); } } return true; } // Compile one (variant, dk, dv); memoised. false = compiler rejected. static bool ggml_opencl_ensure_fa_variant(ggml_backend_opencl_context * backend_ctx, int dk, int dv, ggml_opencl_fa_variant variant) { const std::pair dk_dv = {dk, dv}; const ggml_opencl_fa_dim * cfg = nullptr; for (const auto & d : g_opencl_fa_dims) { if (d.dk == dk && d.dv == dv) { cfg = &d; break; } } if (cfg == nullptr) { return false; } // if a variant has already been compiled switch (variant) { case FA_VARIANT_F16: { if (backend_ctx->fa.f16.count(dk_dv)) { return true; } break; } case FA_VARIANT_F32: { if (backend_ctx->fa.f32.count(dk_dv)) { return true; } break; } case FA_VARIANT_F32_F16: { // The DK=512 decode-only program does not create the f32_f16 // prefill kernel; check the q1 kernel instead so that repeated // calls return a consistent result. const bool decode_only = (dk == 512); if (decode_only ? (backend_ctx->fa.f32_f16_q1.count(dk_dv) > 0) : (backend_ctx->fa.f32_f16.count(dk_dv) > 0)) { return true; } break; } case FA_VARIANT_Q8_0: { if (backend_ctx->fa.f32_q8_0.count(dk_dv)) { return true; } break; } case FA_VARIANT_Q4_0: { if (backend_ctx->fa.f32_q4_0.count(dk_dv)) { return true; } break; } case FA_VARIANT_F32_F16_SPLIT: { if (backend_ctx->fa.f32_f16_split.count(dk_dv)) { return true; } break; } case FA_VARIANT_Q8_0_SPLIT: { if (backend_ctx->fa.f32_q8_0_split.count(dk_dv)) { return true; } break; } case FA_VARIANT_Q4_0_SPLIT: { if (backend_ctx->fa.f32_q4_0_split.count(dk_dv)) { return true; } break; } case FA_VARIANT_PRE: { ggml_opencl_ensure_fa_pre_kernels(backend_ctx, dk, dv); return true; } } // not registered but attempted - meaning these kernels failed to compile const auto attempt_key = std::make_pair(variant, dk_dv); if (backend_ctx->fa.variant_attempted.count(attempt_key)) { return false; } backend_ctx->fa.variant_attempted.insert(attempt_key); const bool is_split = variant == FA_VARIANT_F32_F16_SPLIT || variant == FA_VARIANT_Q8_0_SPLIT || variant == FA_VARIANT_Q4_0_SPLIT; const bool is_quant = variant == FA_VARIANT_Q8_0 || variant == FA_VARIANT_Q8_0_SPLIT || variant == FA_VARIANT_Q4_0 || variant == FA_VARIANT_Q4_0_SPLIT; if (is_quant && (dk % 32 != 0 || dv % 32 != 0)) { return false; } if (is_split && cfg->n_split <= 1) { return false; } if ((variant == FA_VARIANT_Q8_0_SPLIT || variant == FA_VARIANT_Q4_0_SPLIT) && ((dk / 32) % cfg->n_split != 0 || (dv / 4) % cfg->n_split != 0)) { return false; } const std::string src = ggml_opencl_fa_kernel_src(variant); if (src.empty()) { return false; } std::string opts = ggml_opencl_fa_compile_opts(backend_ctx, cfg, variant); // bypass kernels for DK=512 const bool fa_decode_only = (variant == FA_VARIANT_F32_F16 && dk == 512); if (fa_decode_only) { opts += " -D FA_DECODE_ONLY -D FA_DECODE_MINIMAL"; } // c8 cluster width (GGML_OPENCL_FA_CL_C overrides): value = GQA4 cluster // width (kernel default 8); the g8 programs use 2x the value (default 16). // Wider clusters halve per-lane o_acc at the cost of position streams per // subgroup static const int fa_cl_c_env = []{ const char * e = std::getenv("GGML_OPENCL_FA_CL_C"); const int x = (e && e[0]) ? atoi(e) : 0; return (x == 8 || x == 16 || x == 32) ? x : 0; // 0 = per-gen default }(); // X2E needs 16 to keep per-lane o_acc at 128B (the compiler spills the // kernel-default width); X1E does not spill, but C=16 is still a measured // +28-30% DK128-GQA4 decode win there (X1-85, kv 4096/8192), neutral on // DK64 / GQA1 / quant-KV. const int fa_cl_c_gqa4 = fa_cl_c_env ? fa_cl_c_env : (backend_ctx->adreno_gen == ADRENO_GPU_GEN::X2E || backend_ctx->adreno_gen == ADRENO_GPU_GEN::X1E ? 16 : 0); const std::string opts_cl_c_gqa4 = fa_cl_c_gqa4 ? " -D FA_CL_C=" + std::to_string(fa_cl_c_gqa4) : std::string(); const std::string fa_cl_c_g8_val = std::to_string(fa_cl_c_gqa4 ? fa_cl_c_gqa4 * 2 : 16); const char * tag = nullptr; switch (variant) { case FA_VARIANT_F16: tag = "fa f16"; break; case FA_VARIANT_F32: tag = "fa f32"; break; case FA_VARIANT_F32_F16: tag = "fa f32_f16"; break; case FA_VARIANT_Q8_0: tag = "fa q8_0"; break; case FA_VARIANT_Q4_0: tag = "fa q4_0"; break; case FA_VARIANT_F32_F16_SPLIT: tag = "fa f32_f16 split"; break; case FA_VARIANT_Q8_0_SPLIT: tag = "fa q8_0 split"; break; case FA_VARIANT_Q4_0_SPLIT: tag = "fa q4_0 split"; break; default: break; } cl_program prog = build_program_from_source_ex( backend_ctx->context, backend_ctx->device, src.c_str(), opts + opts_cl_c_gqa4, /*fatal=*/false, tag, backend_ctx->queue); if (!prog) { return false; } cl_int err; switch (variant) { case FA_VARIANT_F16: { cl_kernel k, kq1; CL_CHECK((k = clCreateKernel(prog, "flash_attn_f16", &err), err)); CL_CHECK((kq1 = clCreateKernel(prog, "flash_attn_f16_q1", &err), err)); backend_ctx->fa.f16[{dk, dv}] = k; backend_ctx->fa.f16_q1[{dk, dv}] = kq1; break; } case FA_VARIANT_F32: { cl_kernel k, kq1; CL_CHECK((k = clCreateKernel(prog, "flash_attn_f32", &err), err)); CL_CHECK((kq1 = clCreateKernel(prog, "flash_attn_f32_q1", &err), err)); backend_ctx->fa.f32[{dk, dv}] = k; backend_ctx->fa.f32_q1[{dk, dv}] = kq1; break; } case FA_VARIANT_F32_F16: { cl_kernel kq1; // BM-tile prefill kernel is excluded from the decode-only (DK=512) if (!fa_decode_only) { cl_kernel k; CL_CHECK((k = clCreateKernel(prog, "flash_attn_f32_f16", &err), err)); backend_ctx->fa.f32_f16[{dk, dv}] = k; ggml_opencl_log_fa_kernel_spill(backend_ctx, k, "flash_attn_f32_f16", dk, dv); } CL_CHECK((kq1 = clCreateKernel(prog, "flash_attn_f32_f16_q1", &err), err)); backend_ctx->fa.f32_f16_q1[{dk, dv}] = kq1; ggml_opencl_log_fa_kernel_spill(backend_ctx, kq1, "flash_attn_f32_f16_q1", dk, dv); cl_kernel k_split = clCreateKernel(prog, "flash_attn_f32_f16_q1_split", &err); if (err == CL_SUCCESS) { backend_ctx->fa.f32_f16_q1_split[{dk, dv}] = k_split; ggml_opencl_log_fa_kernel_spill(backend_ctx, k_split, "flash_attn_f32_f16_q1_split", dk, dv); } // q1_vec decode kernel (DV-split + subgroup reduce) cl_kernel k_q1_vec = clCreateKernel(prog, "flash_attn_f32_f16_q1_vec", &err); if (err == CL_SUCCESS) { if (ggml_opencl_fa_kernel_fits_wg(backend_ctx, k_q1_vec, 256, "flash_attn_f32_f16_q1_vec", dk, dv)) { backend_ctx->fa.f32_f16_q1_vec[{dk, dv}] = k_q1_vec; ggml_opencl_log_fa_kernel_spill(backend_ctx, k_q1_vec, "flash_attn_f32_f16_q1_vec", dk, dv); } else { clReleaseKernel(k_q1_vec); } } // KV-head-coalesced vec for high-GQA small models cl_kernel k_q1_vec_mq = clCreateKernel(prog, "flash_attn_f32_f16_q1_vec_mq", &err); if (err == CL_SUCCESS) { if (ggml_opencl_fa_kernel_fits_wg(backend_ctx, k_q1_vec_mq, 256, "flash_attn_f32_f16_q1_vec_mq", dk, dv)) { backend_ctx->fa.f32_f16_q1_vec_mq[{dk, dv}] = k_q1_vec_mq; ggml_opencl_log_fa_kernel_spill(backend_ctx, k_q1_vec_mq, "flash_attn_f32_f16_q1_vec_mq", dk, dv); } else { clReleaseKernel(k_q1_vec_mq); } } // KV-head-coalesced + flash-decoding split, reuses merge kernel cl_kernel k_q1_vec_mq_split = clCreateKernel(prog, "flash_attn_f32_f16_q1_vec_mq_split", &err); if (err == CL_SUCCESS) { if (ggml_opencl_fa_kernel_fits_wg(backend_ctx, k_q1_vec_mq_split, 256, "flash_attn_f32_f16_q1_vec_mq_split", dk, dv)) { backend_ctx->fa.f32_f16_q1_vec_mq_split[{dk, dv}] = k_q1_vec_mq_split; ggml_opencl_log_fa_kernel_spill(backend_ctx, k_q1_vec_mq_split, "flash_attn_f32_f16_q1_vec_mq_split", dk, dv); } else { clReleaseKernel(k_q1_vec_mq_split); } } // K-image variant of MQ_GQA=4 split cl_kernel k_q1_vec_mq_split_k_img = clCreateKernel(prog, "flash_attn_f32_f16_q1_vec_mq_split_k_img", &err); if (err == CL_SUCCESS) { if (ggml_opencl_fa_kernel_fits_wg(backend_ctx, k_q1_vec_mq_split_k_img, 256, "flash_attn_f32_f16_q1_vec_mq_split_k_img", dk, dv)) { backend_ctx->fa.f32_f16_q1_vec_mq_split_k_img[{dk, dv}] = k_q1_vec_mq_split_k_img; ggml_opencl_log_fa_kernel_spill(backend_ctx, k_q1_vec_mq_split_k_img, "flash_attn_f32_f16_q1_vec_mq_split_k_img", dk, dv); } else { clReleaseKernel(k_q1_vec_mq_split_k_img); } } // Cluster-parallel decode variant cl_kernel k_q1_vec_mq_split_c8 = clCreateKernel(prog, "flash_attn_f32_f16_q1_vec_mq_split_c8", &err); if (err == CL_SUCCESS) { if (ggml_opencl_fa_kernel_fits_wg(backend_ctx, k_q1_vec_mq_split_c8, 256, "flash_attn_f32_f16_q1_vec_mq_split_c8", dk, dv)) { backend_ctx->fa.f32_f16_q1_vec_mq_split_c8[{dk, dv}] = k_q1_vec_mq_split_c8; ggml_opencl_log_fa_kernel_spill(backend_ctx, k_q1_vec_mq_split_c8, "flash_attn_f32_f16_q1_vec_mq_split_c8", dk, dv); } else { clReleaseKernel(k_q1_vec_mq_split_c8); } } cl_kernel k_merge = clCreateKernel(prog, "flash_attn_f32_merge", &err); if (err == CL_SUCCESS) { backend_ctx->fa.f32_merge[{dk, dv}] = k_merge; } // local-tile decode variant if (dk == 128 && dv == 128) { cl_kernel k_lt = clCreateKernel(prog, "flash_attn_f32_f16_q1_local_tile", &err); if (err == CL_SUCCESS) { if (ggml_opencl_fa_kernel_fits_wg(backend_ctx, k_lt, 128, "flash_attn_f32_f16_q1_local_tile", dk, dv)) { backend_ctx->fa.f32_f16_q1_local_tile[{dk, dv}] = k_lt; ggml_opencl_log_fa_kernel_spill(backend_ctx, k_lt, "flash_attn_f32_f16_q1_local_tile", dk, dv); } else { clReleaseKernel(k_lt); } } // hybrid local-tile + MQ + FD-split cl_kernel k_lmq = clCreateKernel(prog, "flash_attn_f32_f16_q1_local_mq_split", &err); if (err == CL_SUCCESS) { if (ggml_opencl_fa_kernel_fits_wg(backend_ctx, k_lmq, 64, "flash_attn_f32_f16_q1_local_mq_split", dk, dv)) { backend_ctx->fa.f32_f16_q1_local_mq_split[{dk, dv}] = k_lmq; ggml_opencl_log_fa_kernel_spill(backend_ctx, k_lmq, "flash_attn_f32_f16_q1_local_mq_split", dk, dv); } else { clReleaseKernel(k_lmq); } } } // second compile of the same source with -DMQ_GQA=8. // FA_MQ_ONLY keeps only the vec_mq kernels so that the program // compiles within the Adreno compiler's memory budget at DK>=256. // FA_CL_C for the g8 program: MQ_GQA=8 doubles the c8 kernel's // per-lane o_acc, so widen the cluster to keep the register // footprint inside the 192-thread WG cap (see fa_cl_c_gqa4 above // for the per-gen default). const std::string opts_g8 = opts + " -D MQ_GQA=8 -D MQ_NSG=3 -D MQ_NSG_SPLIT=3 -D FA_MQ_ONLY -D FA_CL_C=" + fa_cl_c_g8_val; cl_program prog_g8 = fa_decode_only ? nullptr : build_program_from_source_ex( backend_ctx->context, backend_ctx->device, src.c_str(), opts_g8, /*fatal=*/false, "fa f32_f16 MQ_GQA=8", backend_ctx->queue); if (prog_g8) { const size_t mq_g8_required_wg = 192; // Q1_WG_SIZE(64) * MQ_NSG_SPLIT(3) cl_kernel k_q1_vec_mq_g8 = clCreateKernel(prog_g8, "flash_attn_f32_f16_q1_vec_mq", &err); if (err == CL_SUCCESS) { if (ggml_opencl_fa_kernel_fits_wg(backend_ctx, k_q1_vec_mq_g8, mq_g8_required_wg, "flash_attn_f32_f16_q1_vec_mq (g8)", dk, dv)) { backend_ctx->fa.f32_f16_q1_vec_mq_g8[{dk, dv}] = k_q1_vec_mq_g8; ggml_opencl_log_fa_kernel_spill(backend_ctx, k_q1_vec_mq_g8, "flash_attn_f32_f16_q1_vec_mq_g8", dk, dv); } else { clReleaseKernel(k_q1_vec_mq_g8); } } cl_kernel k_q1_vec_mq_split_g8 = clCreateKernel(prog_g8, "flash_attn_f32_f16_q1_vec_mq_split", &err); if (err == CL_SUCCESS) { if (ggml_opencl_fa_kernel_fits_wg(backend_ctx, k_q1_vec_mq_split_g8, mq_g8_required_wg, "flash_attn_f32_f16_q1_vec_mq_split (g8)", dk, dv)) { backend_ctx->fa.f32_f16_q1_vec_mq_split_g8[{dk, dv}] = k_q1_vec_mq_split_g8; ggml_opencl_log_fa_kernel_spill(backend_ctx, k_q1_vec_mq_split_g8, "flash_attn_f32_f16_q1_vec_mq_split_g8", dk, dv); } else { clReleaseKernel(k_q1_vec_mq_split_g8); } } // K-image variant cl_kernel k_q1_vec_mq_split_g8_k_img = clCreateKernel(prog_g8, "flash_attn_f32_f16_q1_vec_mq_split_k_img", &err); if (err == CL_SUCCESS) { if (ggml_opencl_fa_kernel_fits_wg(backend_ctx, k_q1_vec_mq_split_g8_k_img, mq_g8_required_wg, "flash_attn_f32_f16_q1_vec_mq_split_k_img (g8)", dk, dv)) { backend_ctx->fa.f32_f16_q1_vec_mq_split_g8_k_img[{dk, dv}] = k_q1_vec_mq_split_g8_k_img; ggml_opencl_log_fa_kernel_spill(backend_ctx, k_q1_vec_mq_split_g8_k_img, "flash_attn_f32_f16_q1_vec_mq_split_g8_k_img", dk, dv); } else { clReleaseKernel(k_q1_vec_mq_split_g8_k_img); } } // Cluster-parallel decode, MQ_GQA=8 / FA_CL_C=16 specialization cl_kernel k_q1_vec_mq_split_g8_c8 = clCreateKernel(prog_g8, "flash_attn_f32_f16_q1_vec_mq_split_c8", &err); if (err == CL_SUCCESS) { if (ggml_opencl_fa_kernel_fits_wg(backend_ctx, k_q1_vec_mq_split_g8_c8, mq_g8_required_wg, "flash_attn_f32_f16_q1_vec_mq_split_c8 (g8)", dk, dv)) { backend_ctx->fa.f32_f16_q1_vec_mq_split_g8_c8[{dk, dv}] = k_q1_vec_mq_split_g8_c8; ggml_opencl_log_fa_kernel_spill(backend_ctx, k_q1_vec_mq_split_g8_c8, "flash_attn_f32_f16_q1_vec_mq_split_g8_c8", dk, dv); } else { clReleaseKernel(k_q1_vec_mq_split_g8_c8); } } // hybrid local-tile + MQ_GQA=8 if (dk == 128 && dv == 128) { cl_kernel k_lmq_g8 = clCreateKernel(prog_g8, "flash_attn_f32_f16_q1_local_mq_split", &err); if (err == CL_SUCCESS) { if (ggml_opencl_fa_kernel_fits_wg(backend_ctx, k_lmq_g8, 64, "flash_attn_f32_f16_q1_local_mq_split (g8)", dk, dv)) { backend_ctx->fa.f32_f16_q1_local_mq_split_g8[{dk, dv}] = k_lmq_g8; ggml_opencl_log_fa_kernel_spill(backend_ctx, k_lmq_g8, "flash_attn_f32_f16_q1_local_mq_split_g8", dk, dv); } else { clReleaseKernel(k_lmq_g8); } } } clReleaseProgram(prog_g8); } // NSG_SPLIT=2 programs for the cluster-parallel kernel: its register // footprint caps the per-kernel WG at 128 on X2 (< the stock 256/192 // requirement), so it can never register from the stock programs. // With FA_CL_NCL position streams per subgroup, 2 subgroups still // carry 16 in-flight rows per WG (baseline WG has 4). FA_MQ_ONLY // keeps these compiles minimal; skipped when the stock program c8 // registered (some other device) or shuffles are absent. if (!fa_decode_only && backend_ctx->has_subgroup_shuffle && backend_ctx->fa.f32_f16_q1_vec_mq_split_c8.count({dk, dv}) == 0) { const std::string opts_c8_ns2 = opts + " -D FA_MQ_ONLY -D MQ_GQA=4 -D MQ_NSG=2 -D MQ_NSG_SPLIT=2" + opts_cl_c_gqa4; cl_program prog_c8 = build_program_from_source_ex( backend_ctx->context, backend_ctx->device, src.c_str(), opts_c8_ns2, /*fatal=*/false, "fa f32_f16 c8 NSG2", backend_ctx->queue); if (prog_c8) { cl_kernel k_c8 = clCreateKernel(prog_c8, "flash_attn_f32_f16_q1_vec_mq_split_c8", &err); if (err == CL_SUCCESS) { // WG = MQ_NSG(2) × Q1_WG_SIZE(=FA_SG): 128 Adreno (64), 64 Intel (32). const size_t c8_ns2_wg = backend_ctx->gpu_family == INTEL ? 64 : 128; if (ggml_opencl_fa_kernel_fits_wg(backend_ctx, k_c8, c8_ns2_wg, "flash_attn_f32_f16_q1_vec_mq_split_c8 (ns2)", dk, dv)) { backend_ctx->fa.f32_f16_q1_vec_mq_split_c8_ns2[{dk, dv}] = k_c8; ggml_opencl_log_fa_kernel_spill(backend_ctx, k_c8, "flash_attn_f32_f16_q1_vec_mq_split_c8_ns2", dk, dv); } else { clReleaseKernel(k_c8); } } clReleaseProgram(prog_c8); } } // FA_CL_C=32 g8 program for the DK=DV=256 GQA=8 if (!fa_decode_only && backend_ctx->has_subgroup_shuffle && dk == 256 && dv == 256) { const std::string opts_g8_c32 = opts + " -D FA_MQ_ONLY -D MQ_GQA=8 -D MQ_NSG=2 -D MQ_NSG_SPLIT=2 -D FA_CL_C=32"; cl_program prog_g8_c32 = build_program_from_source_ex( backend_ctx->context, backend_ctx->device, src.c_str(), opts_g8_c32, /*fatal=*/false, "fa f32_f16 c32 g8 d256 NSG2", backend_ctx->queue); if (prog_g8_c32) { cl_kernel k_g8_c32 = clCreateKernel(prog_g8_c32, "flash_attn_f32_f16_q1_vec_mq_split_c8", &err); if (err == CL_SUCCESS) { if (ggml_opencl_fa_kernel_fits_wg(backend_ctx, k_g8_c32, 128, "flash_attn_f32_f16_q1_vec_mq_split_c8 (g8 c32 d256)", dk, dv)) { backend_ctx->fa.f32_f16_q1_vec_mq_split_g8_c32[{dk, dv}] = k_g8_c32; ggml_opencl_log_fa_kernel_spill(backend_ctx, k_g8_c32, "flash_attn_f32_f16_q1_vec_mq_split_g8_c32", dk, dv); } else { clReleaseKernel(k_g8_c32); } } clReleaseProgram(prog_g8_c32); } } if (!fa_decode_only && backend_ctx->has_subgroup_shuffle && backend_ctx->fa.f32_f16_q1_vec_mq_split_g8_c8.count({dk, dv}) == 0) { const std::string opts_g8_c8_ns2 = opts + " -D FA_MQ_ONLY -D MQ_GQA=8 -D MQ_NSG=2 -D MQ_NSG_SPLIT=2 -D FA_CL_C=" + fa_cl_c_g8_val; cl_program prog_g8_c8 = build_program_from_source_ex( backend_ctx->context, backend_ctx->device, src.c_str(), opts_g8_c8_ns2, /*fatal=*/false, "fa f32_f16 c8 g8 NSG2", backend_ctx->queue); if (prog_g8_c8) { cl_kernel k_g8_c8 = clCreateKernel(prog_g8_c8, "flash_attn_f32_f16_q1_vec_mq_split_c8", &err); if (err == CL_SUCCESS) { if (ggml_opencl_fa_kernel_fits_wg(backend_ctx, k_g8_c8, 128, "flash_attn_f32_f16_q1_vec_mq_split_c8 (g8 ns2)", dk, dv)) { backend_ctx->fa.f32_f16_q1_vec_mq_split_g8_c8_ns2[{dk, dv}] = k_g8_c8; ggml_opencl_log_fa_kernel_spill(backend_ctx, k_g8_c8, "flash_attn_f32_f16_q1_vec_mq_split_g8_c8_ns2", dk, dv); } else { clReleaseKernel(k_g8_c8); } } clReleaseProgram(prog_g8_c8); } } break; } case FA_VARIANT_Q8_0: case FA_VARIANT_Q4_0: { const bool is_q8 = variant == FA_VARIANT_Q8_0; const std::string base = is_q8 ? "flash_attn_f32_q8_0" : "flash_attn_f32_q4_0"; const std::string name_q1 = base + "_q1"; const std::string name_q1_split = base + "_q1_split"; auto & m_q1 = is_q8 ? backend_ctx->fa.f32_q8_0_q1 : backend_ctx->fa.f32_q4_0_q1; auto & m_prefill = is_q8 ? backend_ctx->fa.f32_q8_0 : backend_ctx->fa.f32_q4_0; auto & m_q1_split = is_q8 ? backend_ctx->fa.f32_q8_0_q1_split : backend_ctx->fa.f32_q4_0_q1_split; cl_kernel k, kq1; CL_CHECK((kq1 = clCreateKernel(prog, name_q1.c_str(), &err), err)); CL_CHECK((k = clCreateKernel(prog, base.c_str(), &err), err)); m_q1[{dk, dv}] = kq1; m_prefill[{dk, dv}] = k; ggml_opencl_log_fa_kernel_spill(backend_ctx, kq1, name_q1.c_str(), dk, dv); ggml_opencl_log_fa_kernel_spill(backend_ctx, k, base.c_str(), dk, dv); cl_kernel k_split = clCreateKernel(prog, name_q1_split.c_str(), &err); if (err == CL_SUCCESS) { m_q1_split[{dk, dv}] = k_split; ggml_opencl_log_fa_kernel_spill(backend_ctx, k_split, name_q1_split.c_str(), dk, dv); } // DV-split decode variant (q1_vec) auto & m_q1_vec = is_q8 ? backend_ctx->fa.f32_q8_0_q1_vec : backend_ctx->fa.f32_q4_0_q1_vec; const std::string name_q1_vec = name_q1 + "_vec"; cl_kernel k_q1_vec = clCreateKernel(prog, name_q1_vec.c_str(), &err); if (err == CL_SUCCESS) { if (ggml_opencl_fa_kernel_fits_wg(backend_ctx, k_q1_vec, 256, name_q1_vec.c_str(), dk, dv)) { m_q1_vec[{dk, dv}] = k_q1_vec; ggml_opencl_log_fa_kernel_spill(backend_ctx, k_q1_vec, name_q1_vec.c_str(), dk, dv); } else { clReleaseKernel(k_q1_vec); } } // KV-head-coalesced + flash-decoding split auto & m_mq_split = is_q8 ? backend_ctx->fa.f32_q8_0_q1_vec_mq_split : backend_ctx->fa.f32_q4_0_q1_vec_mq_split; const std::string name_mq_split = name_q1 + "_vec_mq_split"; cl_kernel k_q1_vec_mq_split = clCreateKernel(prog, name_mq_split.c_str(), &err); if (err == CL_SUCCESS) { if (ggml_opencl_fa_kernel_fits_wg(backend_ctx, k_q1_vec_mq_split, 256, name_mq_split.c_str(), dk, dv)) { m_mq_split[{dk, dv}] = k_q1_vec_mq_split; ggml_opencl_log_fa_kernel_spill(backend_ctx, k_q1_vec_mq_split, name_mq_split.c_str(), dk, dv); } else { clReleaseKernel(k_q1_vec_mq_split); } } if (!backend_ctx->fa.f32_merge.count({dk, dv})) { cl_kernel k_merge = clCreateKernel(prog, "flash_attn_f32_merge", &err); if (err == CL_SUCCESS) { backend_ctx->fa.f32_merge[{dk, dv}] = k_merge; } } // Second compile with MQ_GQA=8, MQ_NSG=3, MQ_NSG_SPLIT=3 auto & m_mq_split_g8 = is_q8 ? backend_ctx->fa.f32_q8_0_q1_vec_mq_split_g8 : backend_ctx->fa.f32_q4_0_q1_vec_mq_split_g8; const std::string opts_mq_g8 = opts + " -D MQ_GQA=8 -D MQ_NSG=3 -D MQ_NSG_SPLIT=3"; cl_program prog_mq_g8 = build_program_from_source_ex( backend_ctx->context, backend_ctx->device, src.c_str(), opts_mq_g8, /*fatal=*/false, is_q8 ? "fa q8_0 MQ_GQA=8" : "fa q4_0 MQ_GQA=8", backend_ctx->queue); if (prog_mq_g8) { const size_t mq_g8_required_wg = 192; cl_kernel k_g8 = clCreateKernel(prog_mq_g8, name_mq_split.c_str(), &err); if (err == CL_SUCCESS) { if (ggml_opencl_fa_kernel_fits_wg(backend_ctx, k_g8, mq_g8_required_wg, name_mq_split.c_str(), dk, dv)) { m_mq_split_g8[{dk, dv}] = k_g8; ggml_opencl_log_fa_kernel_spill(backend_ctx, k_g8, name_mq_split.c_str(), dk, dv); } else { clReleaseKernel(k_g8); } } clReleaseProgram(prog_mq_g8); } // GQA=4 cluster-parallel program (NSG_SPLIT=2 / WG=128) if (backend_ctx->has_subgroup_shuffle) { auto & m_c8_gqa4 = is_q8 ? backend_ctx->fa.f32_q8_0_q1_vec_mq_split_c8 : backend_ctx->fa.f32_q4_0_q1_vec_mq_split_c8; const std::string name_c8_gqa4 = name_q1 + "_vec_mq_split_c8"; const std::string opts_c8_gqa4 = opts + " -D MQ_GQA=4 -D MQ_NSG=2 -D MQ_NSG_SPLIT=2" + opts_cl_c_gqa4; cl_program prog_c8_gqa4 = build_program_from_source_ex( backend_ctx->context, backend_ctx->device, src.c_str(), opts_c8_gqa4, /*fatal=*/false, is_q8 ? "fa q8_0 c8 GQA4 NSG2" : "fa q4_0 c8 GQA4 NSG2", backend_ctx->queue); if (prog_c8_gqa4) { cl_kernel k_c8_gqa4 = clCreateKernel(prog_c8_gqa4, name_c8_gqa4.c_str(), &err); if (err == CL_SUCCESS) { if (ggml_opencl_fa_kernel_fits_wg(backend_ctx, k_c8_gqa4, 128, name_c8_gqa4.c_str(), dk, dv)) { m_c8_gqa4[{dk, dv}] = k_c8_gqa4; ggml_opencl_log_fa_kernel_spill(backend_ctx, k_c8_gqa4, name_c8_gqa4.c_str(), dk, dv); } else { clReleaseKernel(k_c8_gqa4); } } clReleaseProgram(prog_c8_gqa4); } } // Cluster-parallel q4_0 decode kernel if (!is_q8 && backend_ctx->has_subgroup_shuffle) { const std::string opts_c8 = opts + " -D MQ_GQA=8 -D MQ_NSG=2 -D MQ_NSG_SPLIT=2"; cl_program prog_c8 = build_program_from_source_ex( backend_ctx->context, backend_ctx->device, src.c_str(), opts_c8, /*fatal=*/false, "fa q4_0 c8 NSG2", backend_ctx->queue); if (prog_c8) { cl_kernel k_c8 = clCreateKernel(prog_c8, "flash_attn_f32_q4_0_q1_vec_mq_split_c8", &err); if (err == CL_SUCCESS) { if (ggml_opencl_fa_kernel_fits_wg(backend_ctx, k_c8, 128, "flash_attn_f32_q4_0_q1_vec_mq_split_c8 (g8 ns2)", dk, dv)) { backend_ctx->fa.f32_q4_0_q1_vec_mq_split_g8_c8[{dk, dv}] = k_c8; ggml_opencl_log_fa_kernel_spill(backend_ctx, k_c8, "flash_attn_f32_q4_0_q1_vec_mq_split_g8_c8", dk, dv); } else { clReleaseKernel(k_c8); } } clReleaseProgram(prog_c8); } } break; } case FA_VARIANT_F32_F16_SPLIT: { cl_kernel k; CL_CHECK((k = clCreateKernel(prog, "flash_attn_f32_f16", &err), err)); backend_ctx->fa.f32_f16_split[{dk, dv}] = k; backend_ctx->fa.f32_f16_split_wg_size[{dk, dv}] = cfg->bm * cfg->n_split; backend_ctx->fa.f32_f16_split_nkv_threshold[{dk, dv}] = cfg->nkv_split_threshold; break; } case FA_VARIANT_Q8_0_SPLIT: case FA_VARIANT_Q4_0_SPLIT: { const bool is_q8 = variant == FA_VARIANT_Q8_0_SPLIT; cl_kernel k; CL_CHECK((k = clCreateKernel(prog, is_q8 ? "flash_attn_f32_q8_0" : "flash_attn_f32_q4_0", &err), err)); auto & split = is_q8 ? backend_ctx->fa.f32_q8_0_split : backend_ctx->fa.f32_q4_0_split; auto & split_wg = is_q8 ? backend_ctx->fa.f32_q8_0_split_wg_size : backend_ctx->fa.f32_q4_0_split_wg_size; auto & split_bm = is_q8 ? backend_ctx->fa.f32_q8_0_split_bm : backend_ctx->fa.f32_q4_0_split_bm; auto & split_thresh = is_q8 ? backend_ctx->fa.f32_q8_0_split_nkv_threshold : backend_ctx->fa.f32_q4_0_split_nkv_threshold; split[{dk, dv}] = k; split_wg[{dk, dv}] = cfg->bm * cfg->n_split; split_bm[{dk, dv}] = cfg->bm; split_thresh[{dk, dv}] = 0; // quant prefill: always split break; } default: break; } CL_CHECK(clReleaseProgram(prog)); return true; } // Compile a quant FA split kernel with a hand-picked (BLOCK_M, N_SPLIT) that // overrides the default fa_dims tuning, for the DK values where the default // N_SPLIT is degenerate for quant prefill: // DK=256: default N_SPLIT=16 leaves DK/32=8 blocks -> 0 blocks/split. // Override N_SPLIT=8 (1 block/split), BLOCK_M=16. // DK=96 : DK/32 = 3 blocks, not divisible by the default N_SPLIT=2 -> // override N_SPLIT=3. BLOCK_M must be 16, not 32: the N_SPLIT=3 // QK-partial reduction uses sub_group_shuffle, so all 3 split // threads of a query must land in one subgroup — WG_SIZE = // BLOCK_M*N_SPLIT must be <= the 64-lane Adreno subgroup (16*3=48). static bool ggml_opencl_ensure_fa_quant_split_override( ggml_backend_opencl_context * backend_ctx, int dk, int dv, int quant_bm, int quant_n_split, bool is_q8_0 ) { const std::pair dk_dv = {dk, dv}; if (is_q8_0 && backend_ctx->fa.f32_q8_0_split.count(dk_dv)) { return true; } if (!is_q8_0 && backend_ctx->fa.f32_q4_0_split.count(dk_dv)) { return true; } const ggml_opencl_fa_variant variant = is_q8_0 ? FA_VARIANT_Q8_0_SPLIT : FA_VARIANT_Q4_0_SPLIT; const auto attempt_key = std::make_pair(variant, dk_dv); if (backend_ctx->fa.variant_attempted.count(attempt_key)) { return false; } backend_ctx->fa.variant_attempted.insert(attempt_key); std::string shuffle_opts; if (backend_ctx->has_subgroup_shuffle) { shuffle_opts = backend_ctx->has_qcom_subgroup_shuffle ? " -D cl_qcom_subgroup_shuffle=1" : " -D cl_khr_subgroup_shuffle=1"; } const ggml_opencl_fa_dim * cfg = nullptr; for (const auto & d : g_opencl_fa_dims) { if (d.dk == dk && d.dv == dv) { cfg = &d; break; } } if (cfg == nullptr) { return false; } // BLK_PREPASS_BM is the prepass-kernel BLOCK_M, needed so the quant kernel // indexes the blk[] classification buffer correctly. std::string opts = backend_ctx->kernel_compile_opts + shuffle_opts + " -D DK=" + std::to_string(dk) + " -D DV=" + std::to_string(dv) + " -D BLOCK_M=" + std::to_string(quant_bm) + " -D BLOCK_N=" + std::to_string(cfg->bn) + " -D N_SPLIT=" + std::to_string(quant_n_split) + " -D BLK_PREPASS_BM=" + std::to_string(cfg->bm); const std::string src = ggml_opencl_fa_kernel_src(variant); if (src.empty()) { return false; } const std::string tag = std::string("fa ") + (is_q8_0 ? "q8_0" : "q4_0") + " split DK=" + std::to_string(dk); cl_program prog = build_program_from_source_ex( backend_ctx->context, backend_ctx->device, src.c_str(), opts, /*fatal=*/false, tag.c_str(), backend_ctx->queue); if (!prog) { return false; } cl_int err; cl_kernel k; if (is_q8_0) { CL_CHECK((k = clCreateKernel(prog, "flash_attn_f32_q8_0", &err), err)); backend_ctx->fa.f32_q8_0_split[dk_dv] = k; backend_ctx->fa.f32_q8_0_split_wg_size[dk_dv] = quant_bm * quant_n_split; backend_ctx->fa.f32_q8_0_split_bm[dk_dv] = quant_bm; backend_ctx->fa.f32_q8_0_split_nkv_threshold[dk_dv] = 0; } else { CL_CHECK((k = clCreateKernel(prog, "flash_attn_f32_q4_0", &err), err)); backend_ctx->fa.f32_q4_0_split[dk_dv] = k; backend_ctx->fa.f32_q4_0_split_wg_size[dk_dv] = quant_bm * quant_n_split; backend_ctx->fa.f32_q4_0_split_bm[dk_dv] = quant_bm; backend_ctx->fa.f32_q4_0_split_nkv_threshold[dk_dv] = 0; } CL_CHECK(clReleaseProgram(prog)); return true; } namespace /* anonymous */ { extern struct ggml_backend_device_i ggml_backend_opencl_device_i; } // Look for available and suitable devices. static std::vector ggml_opencl_probe_devices(ggml_backend_reg * reg) { std::vector found_devices; #ifdef GGML_OPENCL_PROFILING GGML_LOG_INFO("ggml_opencl: OpenCL profiling enabled\n"); #endif struct cl_device; struct cl_platform { cl_platform_id id; unsigned number; char name[128]; char vendor[128]; struct cl_device * devices; unsigned n_devices; struct cl_device * default_device; }; struct cl_device { struct cl_platform * platform; cl_device_id id; unsigned number; cl_device_type type; char name[128]; char version[128]; }; enum { NPLAT = 16, NDEV = 16 }; struct cl_platform platforms[NPLAT]; unsigned n_platforms = 0; struct cl_device devices[NDEV]; unsigned n_devices = 0; struct cl_device * default_device = NULL; unsigned default_platform_number = 0; cl_platform_id platform_ids[NPLAT]; if (clGetPlatformIDs(NPLAT, platform_ids, &n_platforms) != CL_SUCCESS) { GGML_LOG_ERROR("ggml_opencl: platform IDs not available.\n"); return found_devices; } for (unsigned i = 0; i < n_platforms; i++) { struct cl_platform * p = &platforms[i]; p->number = i; p->id = platform_ids[i]; CL_CHECK(clGetPlatformInfo(p->id, CL_PLATFORM_NAME, sizeof(p->name), &p->name, NULL)); CL_CHECK(clGetPlatformInfo(p->id, CL_PLATFORM_VENDOR, sizeof(p->vendor), &p->vendor, NULL)); cl_device_id device_ids[NDEV]; cl_int clGetDeviceIDsError = clGetDeviceIDs(p->id, CL_DEVICE_TYPE_ALL, NDEV, device_ids, &p->n_devices); if (clGetDeviceIDsError == CL_DEVICE_NOT_FOUND) { p->n_devices = 0; } else { CL_CHECK(clGetDeviceIDsError); } p->devices = p->n_devices > 0 ? &devices[n_devices] : NULL; p->default_device = NULL; for (unsigned j = 0; j < p->n_devices; j++) { struct cl_device * d = &devices[n_devices]; d->number = n_devices++; d->id = device_ids[j]; d->platform = p; CL_CHECK(clGetDeviceInfo(d->id, CL_DEVICE_NAME, sizeof(d->name), &d->name, NULL)); CL_CHECK(clGetDeviceInfo(d->id, CL_DEVICE_TYPE, sizeof(d->type), &d->type, NULL)); CL_CHECK(clGetDeviceInfo(d->id, CL_DEVICE_VERSION, sizeof(d->version), &d->version, NULL)); if (p->default_device == NULL && d->type == CL_DEVICE_TYPE_GPU) { p->default_device = d; } } if (default_device == NULL && p->default_device != NULL) { default_device = p->default_device; default_platform_number = i; } } if (n_devices == 0) { GGML_LOG_ERROR("ggml_opencl: could find any OpenCL devices.\n"); return found_devices; } char * user_platform_string = getenv("GGML_OPENCL_PLATFORM"); char * user_device_string = getenv("GGML_OPENCL_DEVICE"); int user_platform_number = -1; int user_device_number = -1; cl_device * candidate_devices = nullptr; unsigned n_candidate_devices = 0; unsigned n; if (user_platform_string != NULL && sscanf(user_platform_string, " %u", &n) == 1 && n < n_platforms) { user_platform_number = (int)n; } if (user_device_string != NULL && sscanf(user_device_string, " %u", &n) == 1 && n < n_devices) { user_device_number = (int)n; } if (user_platform_number != -1 && user_device_number != -1) { cl_platform* platform = &platforms[user_platform_number]; if ((unsigned)user_device_number >= platform->n_devices) { GGML_LOG_ERROR("ggml_opencl: invalid device number %d\n", user_device_number); exit(1); } default_device = &platform->devices[user_device_number]; candidate_devices = platform->devices; n_candidate_devices = platform->n_devices; } else { // Choose a platform by matching a substring. if (user_platform_number == -1 && user_platform_string != NULL && user_platform_string[0] != 0) { for (unsigned i = 0; i < n_platforms; i++) { struct cl_platform * p = &platforms[i]; if (strstr(p->name, user_platform_string) != NULL || strstr(p->vendor, user_platform_string) != NULL) { user_platform_number = (int)i; break; } } if (user_platform_number == -1) { GGML_LOG_ERROR("ggml_opencl: no platform matching '%s' was found.\n", user_platform_string); exit(1); } } int platform_idx = user_platform_number != -1 ? user_platform_number : default_platform_number; struct cl_platform * p = &platforms[platform_idx]; candidate_devices = p->devices; n_candidate_devices = p->n_devices; default_device = p->default_device; if (n_candidate_devices == 0) { GGML_LOG_ERROR("ggml_opencl: selected platform '%s' does not have any devices.\n", p->name); exit(1); } if (user_device_number == -1 && user_device_string != NULL && user_device_string[0] != 0) { for (unsigned i = 0; i < n_candidate_devices; i++) { struct cl_device * d = &candidate_devices[i]; if (strstr(d->name, user_device_string) != NULL) { user_device_number = d->number; break; } } if (user_device_number == -1) { GGML_LOG_ERROR("ggml_opencl: no device matching '%s' was found.\n", user_device_string); exit(1); } } if (user_device_number != -1) { candidate_devices = &devices[user_device_number]; n_candidate_devices = 1; default_device = &candidate_devices[0]; } GGML_ASSERT(n_candidate_devices > 0); if (default_device == NULL) { default_device = &candidate_devices[0]; } } GGML_ASSERT(n_candidate_devices != 0 && candidate_devices); // Put the default device in front. for (unsigned i = 1; i < n_candidate_devices; i++) { if (&candidate_devices[i] == default_device) { std::swap(candidate_devices[0], candidate_devices[i]); default_device = &candidate_devices[0]; break; } } GGML_LOG_INFO("ggml_opencl: selected platform: '%s'\n", default_device->platform->name); std::vector device_ids; for (auto dev = candidate_devices, dev_end = candidate_devices + n_candidate_devices; dev != dev_end; dev++) { device_ids.push_back(dev->id); } cl_int err; cl_context shared_context; cl_context_properties properties[] = { (intptr_t) CL_CONTEXT_PLATFORM, (intptr_t) default_device->platform->id, 0 }; CL_CHECK( (shared_context = clCreateContext(properties, device_ids.size(), device_ids.data(), NULL, NULL, &err), err)); for (auto dev = candidate_devices, dev_end = candidate_devices + n_candidate_devices; dev != dev_end; dev++) { GGML_LOG_INFO("\nggml_opencl: device: '%s (%s)'\n", dev->name, dev->version); auto dev_ctx = std::unique_ptr(new ggml_backend_opencl_device_context{ /*.platform =*/dev->platform->id, /*.platform_nane =*/dev->platform->name, /*.device =*/dev->id, /*.device_name =*/dev->name, /*.device_type =*/dev->type, /*.device_version =*/dev->version, /*.backend_ctx =*/nullptr, /*.buffer_type =*/{}, /*.context =*/shared_context, }); found_devices.push_back(ggml_backend_device{ /* .iface = */ ggml_backend_opencl_device_i, /* .reg = */ reg, /* .context = */ dev_ctx.get(), }); if (!ggml_opencl_is_device_supported(&found_devices.back())) { found_devices.pop_back(); GGML_LOG_WARN("ggml_opencl: drop unsupported device '%s'.\n", dev->name); continue; } g_ggml_backend_opencl_dev_ctxs.push_back(std::move(dev_ctx)); } if (found_devices.size()) { auto * dev_ctx = static_cast(found_devices.front().context); GGML_LOG_INFO("ggml_opencl: default device: '%s (%s)'\n", dev_ctx->device_name.c_str(), dev_ctx->device_version.c_str()); if (dev_ctx->device_type != CL_DEVICE_TYPE_GPU) { GGML_LOG_WARN("ggml_opencl: warning, the default device is not a GPU: '%s'.\n", dev_ctx->device_name.c_str()); } } return found_devices; } static void ggml_opencl_print_backend_info(ggml_backend_opencl_device_context * dev_ctx) { GGML_ASSERT(dev_ctx); GGML_ASSERT(dev_ctx->backend_ctx); auto * backend_ctx = dev_ctx->backend_ctx; GGML_LOG_INFO("ggml_opencl: OpenCL device: %s\n", backend_ctx->device_name.c_str()); GGML_LOG_INFO("ggml_opencl: OpenCL driver: %s\n", backend_ctx->driver_version.c_str()); GGML_LOG_INFO("ggml_opencl: vector subgroup broadcast support: %s\n", backend_ctx->has_vector_subgroup_broadcast ? "true" : "false"); GGML_LOG_INFO("ggml_opencl: subgroup shuffle support: %s\n", backend_ctx->has_subgroup_shuffle ? "true" : "false"); GGML_LOG_INFO("ggml_opencl: device FP16 support: %s\n", backend_ctx->fp16_support ? "true" : "false"); GGML_LOG_INFO("ggml_opencl: khr dot product support: %s\n", backend_ctx->has_integer_dot ? "true" : "false"); GGML_LOG_INFO("ggml_opencl: mem base addr align: %u\n", backend_ctx->alignment); GGML_LOG_INFO("ggml_opencl: global mem size: %zu MB\n", backend_ctx->global_mem_size/1024/1024); GGML_LOG_INFO("ggml_opencl: max mem alloc size: %zu MB\n", backend_ctx->max_alloc_size/1024/1024); GGML_LOG_INFO("ggml_opencl: device max image buffer size (pixels): %zu\n", backend_ctx->image_max_buffer_size); GGML_LOG_INFO("ggml_opencl: device max image2d size: %zu x %zu\n", backend_ctx->image2d_max_width, backend_ctx->image2d_max_height); GGML_LOG_INFO("ggml_opencl: device max workgroup size: %zu\n", backend_ctx->max_workgroup_size); GGML_LOG_INFO("ggml_opencl: SVM coarse grain buffer support: %s\n", backend_ctx->svm_caps & CL_DEVICE_SVM_COARSE_GRAIN_BUFFER ? "true" : "false"); GGML_LOG_INFO("ggml_opencl: SVM fine grain buffer support: %s\n", backend_ctx->svm_caps & CL_DEVICE_SVM_FINE_GRAIN_BUFFER ? "true" : "false"); GGML_LOG_INFO("ggml_opencl: SVM fine grain system support: %s\n", backend_ctx->svm_caps & CL_DEVICE_SVM_FINE_GRAIN_SYSTEM ? "true" : "false"); GGML_LOG_INFO("ggml_opencl: SVM atomics support: %s\n", backend_ctx->svm_caps & CL_DEVICE_SVM_ATOMICS ? "true" : "false"); GGML_LOG_INFO("ggml_opencl: cl_qcom_subgroup_shuffle support: %s\n", backend_ctx->has_qcom_subgroup_shuffle ? "true" : "false"); // Print out configurations #ifdef GGML_OPENCL_SOA_Q GGML_LOG_INFO("ggml_opencl: flattening quantized weights representation as struct of arrays (GGML_OPENCL_SOA_Q)\n"); #endif // GGML_OPENCL_SOA_Q #ifdef GGML_OPENCL_USE_ADRENO_KERNELS GGML_LOG_INFO("ggml_opencl: using kernels optimized for Adreno (GGML_OPENCL_USE_ADRENO_KERNELS)\n"); if (backend_ctx->adreno_xmem_gemm_enabled) { GGML_LOG_INFO("ggml_opencl: Adreno xmem F16xF32 GEMM enabled (temporary weight prepack)\n"); } #endif // GGML_OPENCL_USE_ADRENO_KERNELS if (backend_ctx->adreno_use_large_buffer) { if (!backend_ctx->adreno_has_large_buffer) { GGML_LOG_INFO("ggml_opencl: Adreno large buffer requested but not supported by driver, will use regular buffer\n"); backend_ctx->adreno_use_large_buffer = false; } else { GGML_LOG_INFO("ggml_opencl: Adreno large buffer enabled\n"); } } if (dev_ctx->opfilter) { // for information only, the actual regex object is created in ggml_opencl_is_device_supported GGML_LOG_INFO("ggml_opencl: opfilter regex = \"%s\"\n", dev_ctx->opfilter_str.c_str()); } } // check if device should be accepted static bool ggml_opencl_is_device_supported(ggml_backend_dev_t dev) { GGML_ASSERT(dev); GGML_ASSERT(dev->context); ggml_backend_opencl_device_context * dev_ctx = (ggml_backend_opencl_device_context *) dev->context; GGML_ASSERT(dev_ctx->platform); GGML_ASSERT(dev_ctx->device); if (strstr(dev_ctx->device_name.c_str(), "Adreno") || strstr(dev_ctx->device_name.c_str(), "Qualcomm") || strstr(dev_ctx->device_version.c_str(), "Adreno")) { dev_ctx->gpu_family = GPU_FAMILY::ADRENO; // Usually device version contains the detailed device name dev_ctx->adreno_gen = get_adreno_gpu_gen(dev_ctx->device_version.c_str()); if (dev_ctx->adreno_gen == ADRENO_GPU_GEN::ADRENO_UNKNOWN) { dev_ctx->adreno_gen = get_adreno_gpu_gen(dev_ctx->device_name.c_str()); } } else if (strstr(dev_ctx->device_name.c_str(), "Intel")) { dev_ctx->gpu_family = GPU_FAMILY::INTEL; } else { GGML_LOG_WARN("ggml_opencl: unsupported GPU '%s'.\n", dev_ctx->device_name.c_str()); dev_ctx->gpu_family = GPU_FAMILY::UNKNOWN; return false; } ggml_cl_version platform_version = get_opencl_platform_version(dev_ctx->platform); // Check device OpenCL version, OpenCL 2.0 or above is required ggml_cl_version opencl_c_version = get_opencl_c_version(platform_version, dev_ctx->device); if (opencl_c_version.major < 2) { GGML_LOG_WARN("ggml_opencl: OpenCL 2.0 or above is required\n"); return false; } #ifdef GGML_OPENCL_USE_ADRENO_KERNELS if (dev_ctx->gpu_family != GPU_FAMILY::ADRENO) { GGML_LOG_WARN("ggml_opencl: Adreno-specific kernels should not be enabled for non-Adreno GPUs; " "run on an Adreno GPU or recompile with CMake option `-DGGML_OPENCL_USE_ADRENO_KERNELS=OFF`\n"); return false; } #endif size_t ext_str_size; CL_CHECK(clGetDeviceInfo(dev_ctx->device, CL_DEVICE_EXTENSIONS, 0, NULL, &ext_str_size)); std::vector ext_buffer(ext_str_size + 1); CL_CHECK(clGetDeviceInfo(dev_ctx->device, CL_DEVICE_EXTENSIONS, ext_str_size, ext_buffer.data(), NULL)); ext_buffer[ext_str_size] = '\0'; // Check if ext_buffer contains cl_khr_fp16 bool fp16_support = strstr(ext_buffer.data(), "cl_khr_fp16") != NULL; if (!fp16_support) { GGML_LOG_WARN("ggml_opencl: device does not support FP16\n"); return false; } // If OpenCL 3.0 is supported, then check for cl_khr_subgroups, which becomes // optional in OpenCL 3.0 (cl_khr_subgroup is mandatory in OpenCL 2.x) if (opencl_c_version.major == 3 && strstr(ext_buffer.data(), "cl_khr_subgroups") == NULL && strstr(ext_buffer.data(), "cl_intel_subgroups") == NULL) { GGML_LOG_WARN("ggml_opencl: device does not support subgroups (cl_khr_subgroups or cl_intel_subgroups) " "(note that subgroups is an optional feature in OpenCL 3.0)\n"); return false; } clGetDeviceInfo(dev_ctx->device, CL_DEVICE_GLOBAL_MEM_SIZE, sizeof(size_t), &dev_ctx->global_mem_size, NULL); const char * str_opfilter = getenv("GGML_OPENCL_OPFILTER"); if (str_opfilter) { dev_ctx->opfilter_str = str_opfilter; dev_ctx->opfilter = new std::regex(str_opfilter, std::regex_constants::icase); } return true; } // Initialize device if it is supported (returns nullptr if it is not). static ggml_backend_opencl_context * ggml_cl_init(ggml_backend_dev_t dev) { GGML_ASSERT(dev); GGML_ASSERT(dev->context); ggml_backend_opencl_device_context * dev_ctx = (ggml_backend_opencl_device_context *) dev->context; GGML_ASSERT(dev_ctx->platform); GGML_ASSERT(dev_ctx->device); if (dev_ctx->backend_ctx) { return dev_ctx->backend_ctx; } auto backend_ctx = std::make_unique(); backend_ctx->device = dev_ctx->device; backend_ctx->gpu_family = GPU_FAMILY::UNKNOWN; // ref_count get increased in ggml_backend_opencl_device_init // This function is also used to retrieve backend context, so we don't want // to increase ref_count for each call. We only want to increase ref_count // when the associated device is initialized backend_ctx->ref_count = 0; backend_ctx->gpu_family = dev_ctx->gpu_family; backend_ctx->adreno_gen = dev_ctx->adreno_gen; if (backend_ctx->gpu_family == GPU_FAMILY::ADRENO) { ggml_cl_init_fa_dims_table(); // Use wave size of 64 for all Adreno GPUs. backend_ctx->adreno_wave_size = 64; } // Populate backend device name backend_ctx->device_name = dev_ctx->device_name; // A local ref of cl_device_id for convenience cl_device_id device = backend_ctx->device; ggml_cl_version platform_version = get_opencl_platform_version(dev_ctx->platform); ggml_cl_version opencl_c_version = get_opencl_c_version(platform_version, device); backend_ctx->platform_version = platform_version; backend_ctx->opencl_c_version = opencl_c_version; // Check driver version size_t driver_version_str_size; CL_CHECK(clGetDeviceInfo(device, CL_DRIVER_VERSION, 0, NULL, &driver_version_str_size)); std::vector driver_version(driver_version_str_size + 1); CL_CHECK(clGetDeviceInfo(device, CL_DRIVER_VERSION, driver_version_str_size, driver_version.data(), NULL)); driver_version[driver_version_str_size] = '\0'; backend_ctx->driver_version = driver_version.data(); backend_ctx->adreno_cl_compiler_version = get_adreno_cl_compiler_version(driver_version.data()); backend_ctx->has_vector_subgroup_broadcast = (backend_ctx->adreno_cl_compiler_version.type == E031 && backend_ctx->adreno_cl_compiler_version.major >= 47) || (backend_ctx->adreno_cl_compiler_version.type == DX && backend_ctx->adreno_cl_compiler_version.major >= 17) || (backend_ctx->adreno_cl_compiler_version.type == E17); // The q6_K flat mul_mat miscompile is a defect of the older E031 compilers, not a // property of any GPU generation: it reproduces on E031.38 (Adreno 642L) and E031.41 // (Adreno 740) and is fixed by E031.45 (Adreno 619). Gate on the compiler so parts // that do not need the workarounds do not pay for them. The explicit type check is // required: newer_than_or_same() is false for every non-E031 compiler, so negating it // alone would enable the workarounds on E17/DX. backend_ctx->q6_k_flat_old_compiler = backend_ctx->adreno_cl_compiler_version.type == E031 && !backend_ctx->adreno_cl_compiler_version.newer_than_or_same(E031, 45, 0, 0); size_t ext_str_size; CL_CHECK(clGetDeviceInfo(device, CL_DEVICE_EXTENSIONS, 0, NULL, &ext_str_size)); std::vector ext_buffer(ext_str_size + 1); CL_CHECK(clGetDeviceInfo(device, CL_DEVICE_EXTENSIONS, ext_str_size, ext_buffer.data(), NULL)); ext_buffer[ext_str_size] = '\0'; // ensure it is null terminated // check support for qcom_subgroup_shuffle if (strstr(ext_buffer.data(), "cl_qcom_subgroup_shuffle") != NULL) { backend_ctx->has_qcom_subgroup_shuffle = true; } // Check if ext_buffer contains cl_khr_fp16 backend_ctx->fp16_support = strstr(ext_buffer.data(), "cl_khr_fp16") != NULL; // check Adreno large buffer support backend_ctx->adreno_has_large_buffer = strstr(ext_buffer.data(), "cl_qcom_large_buffer") != NULL; // subgroup shuffle support (N_SPLIT>1 FA kernel) backend_ctx->has_qcom_subgroup_shuffle = strstr(ext_buffer.data(), "cl_qcom_subgroup_shuffle") != NULL; backend_ctx->has_subgroup_shuffle = strstr(ext_buffer.data(), "cl_khr_subgroup_shuffle") != NULL || backend_ctx->has_qcom_subgroup_shuffle; // check for cl_khr_integer_dot_product // cl_qcom_dot_product8 uses signed * unsigned // while cl_khr_integer_dot_product uses signed * signed -- we stick with khr for now backend_ctx->has_integer_dot = strstr(ext_buffer.data(), "cl_khr_integer_dot_product") != NULL; cl_uint base_align_in_bits; CL_CHECK(clGetDeviceInfo(device, CL_DEVICE_MEM_BASE_ADDR_ALIGN, sizeof(cl_uint), &base_align_in_bits, NULL)); GGML_ASSERT(base_align_in_bits % 8u == 0); backend_ctx->alignment = base_align_in_bits / 8u; backend_ctx->global_mem_size = dev_ctx->global_mem_size; CL_CHECK(clGetDeviceInfo(device, CL_DEVICE_MAX_MEM_ALLOC_SIZE, sizeof(size_t), &backend_ctx->max_alloc_size, NULL)); CL_CHECK(clGetDeviceInfo(device, CL_DEVICE_IMAGE_MAX_BUFFER_SIZE, sizeof(size_t), &backend_ctx->image_max_buffer_size, NULL)); CL_CHECK(clGetDeviceInfo(device, CL_DEVICE_IMAGE2D_MAX_WIDTH, sizeof(size_t), &backend_ctx->image2d_max_width, NULL)); CL_CHECK(clGetDeviceInfo(device, CL_DEVICE_IMAGE2D_MAX_HEIGHT, sizeof(size_t), &backend_ctx->image2d_max_height, NULL)); CL_CHECK(clGetDeviceInfo(device, CL_DEVICE_MAX_WORK_GROUP_SIZE, sizeof(size_t), &backend_ctx->max_workgroup_size, NULL)); CL_CHECK(clGetDeviceInfo(device, CL_DEVICE_SVM_CAPABILITIES, sizeof(cl_device_svm_capabilities), &backend_ctx->svm_caps, 0)); if (opencl_c_version.major >= 3) { // Assume it is not available for 3.0, since it is optional in 3.0. // If compiling against 3.0, then we can query. backend_ctx->non_uniform_workgroups = false; #if CL_TARGET_OPENCL_VERSION >= 300 CL_CHECK(clGetDeviceInfo(device, CL_DEVICE_NON_UNIFORM_WORK_GROUP_SUPPORT, sizeof(cl_bool), &backend_ctx->non_uniform_workgroups, 0)); #endif } else { GGML_ASSERT(opencl_c_version.major == 2); // Non-uniform workgroup sizes is mandatory feature in v2.x. backend_ctx->non_uniform_workgroups = true; } #ifdef GGML_OPENCL_USE_ADRENO_KERNELS // Adreno xmem F16xF32 GEMM, default on adreno, opt out with GGML_OPENCL_ADRENO_XMEM_GEMM=0. // This helps models with f16 attention weights, e.g., gpt-oss-20b-f16 { const char * xmem_env = getenv("GGML_OPENCL_ADRENO_XMEM_GEMM"); backend_ctx->adreno_xmem_gemm_enabled = backend_ctx->gpu_family == GPU_FAMILY::ADRENO && (xmem_env ? atoi(xmem_env) != 0 : true); } #endif // determine whether to use large buffer for Adreno backend_ctx->adreno_use_large_buffer = getenv("GGML_OPENCL_ADRENO_USE_LARGE_BUFFER") != nullptr && backend_ctx->gpu_family == GPU_FAMILY::ADRENO; // ragged moe, unspecified or non-zero means enabled, set to 0 to disable static const char * ragged_fp16_env = getenv("GGML_OPENCL_MOE_RAGGED_FP16"); backend_ctx->adreno_use_moe_ragged = (ragged_fp16_env == NULL) ? 1 : (atoi(ragged_fp16_env) != 0); // ragged moe, tile-skip granularity (columns per skip-group): 8 = quarter (default), // 16 = half (legacy), 32 = disabled. Override with GGML_OPENCL_MOE_RAGGED_GRAN={8,16,32} static const char * ragged_gran_env = getenv("GGML_OPENCL_MOE_RAGGED_GRAN"); backend_ctx->adreno_moe_ragged_skip_gran = (ragged_gran_env != NULL) ? atoi(ragged_gran_env) : 8; // whether fuse moe combine static const char * fuse_moe_bias_glu_env = getenv("GGML_OPENCL_FUSE_MOE_BIAS_GLU"); backend_ctx->fuse_moe_bias_glu = fuse_moe_bias_glu_env == NULL ? 1 : (atoi(fuse_moe_bias_glu_env) != 0); static const char * fuse_moe_bias_combine_env = getenv("GGML_OPENCL_FUSE_MOE_BIAS_COMBINE"); backend_ctx->fuse_moe_bias_combine = fuse_moe_bias_combine_env == NULL ? 1 : (atoi(fuse_moe_bias_combine_env) != 0); static const char * fuse_moe_combine_env = getenv("GGML_OPENCL_FUSE_MOE_COMBINE"); backend_ctx->fuse_moe_combine = fuse_moe_combine_env == NULL ? 1 : (atoi(fuse_moe_combine_env) != 0); // ragged moe dp4 variant static const char * ragged_dp4_env = getenv("GGML_OPENCL_MOE_RAGGED"); backend_ctx->adreno_use_moe_ragged_dp4 = ragged_dp4_env == NULL ? 1 : (atoi(ragged_dp4_env) != 0); #ifdef GGML_OPENCL_USE_ADRENO_BIN_KERNELS // try loading adreno binary kernels if enabled // if fails to load, builtin kernels will be used { dl_handle * kernel_lib_handle = dl_load_library(KERNEL_LIB_NAME); backend_ctx->adreno_use_bin_kernels = false; if (kernel_lib_handle) { backend_ctx->get_adreno_bin_kernel_func = (get_adreno_bin_kernel_func_t)dl_get_sym(kernel_lib_handle, "get_adreno_kernels"); if (backend_ctx->get_adreno_bin_kernel_func) { GGML_LOG_INFO("ggml_opencl: loaded bin kernel library %s\n", KERNEL_LIB_NAME); backend_ctx->adreno_use_bin_kernels = true; } else { GGML_LOG_INFO("ggml_opencl: bin kernel library %s is invalid, will use builtin kernels\n", KERNEL_LIB_NAME); } } else { GGML_LOG_INFO("ggml_opencl: failed to load %s, will use builtin kernels\n", KERNEL_LIB_NAME); } } #endif // GGML_OPENCL_USE_ADRENO_BIN_KERNELS cl_int err; // A local ref of cl_context for convenience cl_context context = backend_ctx->context = dev_ctx->context; //CL_CHECK((queue = clCreateCommandQueue(context, device, CL_QUEUE_OUT_OF_ORDER_EXEC_MODE_ENABLE, &err), // (err != CL_INVALID_QUEUE_PROPERTIES && err != CL_INVALID_VALUE ? err : // (queue = clCreateCommandQueue(context, device, 0, &err), err) //))); cl_command_queue_properties command_queue_props = 0; #ifdef GGML_OPENCL_PROFILING command_queue_props |= CL_QUEUE_PROFILING_ENABLE; #endif CL_CHECK((backend_ctx->queue = clCreateCommandQueue(context, device, command_queue_props, &err), err)); // delay kernel loading until the first buffer is created // load_cl_kernels(backend_ctx.get()); #ifdef GGML_OPENCL_USE_ADRENO_KERNELS // Allocate intermediate buffers and images size_t required_A_q_d_bytes = 311164928; size_t required_A_s_d_bytes = 38895616; size_t required_B_d_bytes = 45088768; // Ensure buffer sizes do not exceed the maximum allocation size size_t max_A_q_d_bytes = MIN(required_A_q_d_bytes, backend_ctx->max_alloc_size); size_t max_A_s_d_bytes = MIN(required_A_s_d_bytes, backend_ctx->max_alloc_size); size_t max_B_d_bytes = MIN(required_B_d_bytes, backend_ctx->max_alloc_size); if (required_A_q_d_bytes > backend_ctx->max_alloc_size) { GGML_LOG_WARN("ggml_opencl: A_q_d buffer size reduced from %zu to %zu due to device limitations.\n", required_A_q_d_bytes, max_A_q_d_bytes); } if (required_A_s_d_bytes > backend_ctx->max_alloc_size) { GGML_LOG_WARN("ggml_opencl: A_s_d buffer size reduced from %zu to %zu due to device limitations.\n", required_A_s_d_bytes, max_A_s_d_bytes); } if (required_B_d_bytes > backend_ctx->max_alloc_size) { GGML_LOG_WARN("ggml_opencl: B_d buffer size reduced from %zu to %zu due to device limitations.\n", required_B_d_bytes, max_B_d_bytes); } backend_ctx->prealloc_quant_trans.allocate(context, max_A_q_d_bytes); backend_ctx->prealloc_scales_trans.allocate(context, max_A_s_d_bytes); backend_ctx->prealloc_act_trans.allocate(context, max_B_d_bytes); #endif // GGML_OPENCL_USE_ADRENO_KERNELS backend_ctx->disable_fusion = getenv("GGML_OPENCL_DISABLE_FUSION") != nullptr; if (const char * env = getenv("GGML_OPENCL_FUSE_MM_GLU")) { backend_ctx->fuse_mm_glu = atoi(env) != 0; } if (const char * env = getenv("GGML_OPENCL_FUSE_RMS_ADD")) { backend_ctx->fuse_rms_add = atoi(env) != 0; } if (const char * env = getenv("GGML_OPENCL_F16_MROW")) { backend_ctx->f16_mrow = atoi(env) != 0; } if (const char * env = getenv("GGML_OPENCL_F16_MROW_RPT")) { const int v = atoi(env); backend_ctx->f16_mrow_rpt = (v == 2 || v == 4 || v == 8 || v == 16) ? v : 1; } dev_ctx->backend_ctx = backend_ctx.release(); return dev_ctx->backend_ctx; } static void ggml_cl_free(ggml_backend_t backend) { ggml_backend_opencl_context * ctx = (ggml_backend_opencl_context *) backend->context; ctx->free(); } #ifdef GGML_OPENCL_USE_ADRENO_KERNELS static void transpose_2d( ggml_backend_opencl_context * backend_ctx, cl_kernel kernel, cl_mem src, cl_mem dst, size_t size, cl_int stride, cl_int rows, bool blocking = true, bool auto_local = false // let driver pick local size for non-uniform workgroups ) { static ggml_cl_buffer buf; cl_event evt; cl_int err; buf.allocate(backend_ctx->context, size); cl_mem trans; cl_buffer_region region; region.origin = 0; region.size = size; CL_CHECK((trans = clCreateSubBuffer( buf.buffer, CL_MEM_READ_WRITE, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &src)); CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &trans)); CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_int), &stride)); CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_int), &rows)); size_t local_size[3] = {64, 1, 1}; size_t global_size[3] = {(size_t)stride, (size_t)rows, 1};; CL_CHECK(clEnqueueNDRangeKernel(backend_ctx->queue, kernel, 3, NULL, global_size, auto_local ? NULL : local_size, 0, NULL, NULL)); if (blocking) { CL_CHECK(clEnqueueCopyBuffer(backend_ctx->queue, trans, dst, 0, 0, size, 0, NULL, &evt)); CL_CHECK(clWaitForEvents(1, &evt)); CL_CHECK(clReleaseEvent(evt)); } else { CL_CHECK(clEnqueueCopyBuffer(backend_ctx->queue, trans, dst, 0, 0, size, 0, NULL, NULL)); } CL_CHECK(clReleaseMemObject(trans)); } static void transpose_2d_as_8b( ggml_backend_opencl_context * backend_ctx, cl_mem src, cl_mem dst, size_t size, cl_int stride, cl_int rows, bool blocking = true, bool auto_local = false ) { transpose_2d(backend_ctx, backend_ctx->kernel_transpose_8_buf, src, dst, size, stride, rows, blocking, auto_local); } static void transpose_2d_as_16b( ggml_backend_opencl_context * backend_ctx, cl_mem src, cl_mem dst, size_t size, cl_int stride, cl_int rows, bool blocking = true ) { transpose_2d(backend_ctx, backend_ctx->kernel_transpose_16_buf, src, dst, size, stride, rows, blocking); } static void transpose_2d_as_32b( ggml_backend_opencl_context * backend_ctx, cl_mem src, cl_mem dst, size_t size, cl_int stride, cl_int rows, bool blocking = true ) { transpose_2d(backend_ctx, backend_ctx->kernel_transpose_32_buf, src, dst, size, stride, rows, blocking); } #endif // GGML_OPENCL_USE_ADRENO_KERNELS //------------------------------------------------------------------------------ // Tensor extra management //------------------------------------------------------------------------------ struct ggml_tensor_extra_cl { // The buffer object that holds the data. cl_mem data_device; // The offset into the buffer object. This is primarily for scratch buffer // and view operation. // NB: this offset no longer includes view offset (view_offs). Whenever this // offset is used, view_offs should be considered. cl_ulong offset; // The actual size of the cl_mem object. This is needed when returning the // block to the pool. size_t actual_size; void reset() { data_device = nullptr; offset = 0; actual_size = 0; } }; struct ggml_tensor_extra_cl_q1_0 { cl_mem q = nullptr; cl_mem q_img = nullptr; cl_mem d = nullptr; cl_mem d_img = nullptr; size_t size_q = 0; size_t size_d = 0; ~ggml_tensor_extra_cl_q1_0() { reset(); } void reset() { // q and d are subbuffers into the bigger buffer allocated in ggml_backend_buffer. // They must be properly released so that the original buffer can be // properly released to avoid memory leak. if (q != nullptr) { CL_CHECK(clReleaseMemObject(q)); q = nullptr; } if (d != nullptr) { CL_CHECK(clReleaseMemObject(d)); d = nullptr; } q_img = nullptr; d_img = nullptr; size_q = 0; size_d = 0; } }; // Additional tensor extra structs for quantized tensors. // These tensors are loaded from files and should not be allocated in scratch -- // they should always be allocated from the pool. Hence, they do not have an // `offset`, which indicate their locations in the scratch buffer. struct ggml_tensor_extra_cl_q4_0 { // Quantized values. cl_mem q = nullptr; // Quantized values in image1d_buffer_t. cl_mem q_img = nullptr; // Scales. cl_mem d = nullptr; // Scales in image1d_buffer_t. cl_mem d_img = nullptr; // Size of quantized values. size_t size_q = 0; // Size of scales. size_t size_d = 0; ~ggml_tensor_extra_cl_q4_0() { reset(); } void reset() { // q and d are subbuffers into the bigger buffer allocated in ggml_backend_buffer. // They must be properly released so that the original buffer can be // properly released to avoid memory leak. if (q != nullptr) { CL_CHECK(clReleaseMemObject(q)); q = nullptr; } if (d != nullptr) { CL_CHECK(clReleaseMemObject(d)); d = nullptr; } if (q_img != nullptr) { CL_CHECK(clReleaseMemObject(q_img)); q_img = nullptr; } if (d_img != nullptr) { CL_CHECK(clReleaseMemObject(d_img)); d_img = nullptr; } size_q = 0; size_d = 0; } }; struct ggml_tensor_extra_cl_q4_1 { // Quantized values. cl_mem q = nullptr; // Quantized values in image1d_buffer_t. cl_mem q_img = nullptr; // Scales. cl_mem d = nullptr; // Scales in image1d_buffer_t. cl_mem d_img = nullptr; // Min cl_mem m = nullptr; // Min in image1d_buffer_t. cl_mem m_img = nullptr; // Size of quantized values. size_t size_q = 0; // Size of scales. size_t size_d = 0; // Size of min values. size_t size_m = 0; ~ggml_tensor_extra_cl_q4_1() { reset(); } void reset() { // q and d are subbuffers into the bigger buffer allocated in ggml_backend_buffer. // They must be properly released so that the original buffer can be // properly released to avoid memory leak. if (q != nullptr) { CL_CHECK(clReleaseMemObject(q)); q = nullptr; } if (d != nullptr) { CL_CHECK(clReleaseMemObject(d)); d = nullptr; } if (m != nullptr) { CL_CHECK(clReleaseMemObject(m)); m = nullptr; } if (q_img != nullptr) { CL_CHECK(clReleaseMemObject(q_img)); q_img = nullptr; } // Currently, q_img and d_img are only initialized when SMALL_ALLOC is // enabled. They point to the images in ggml_backend_opencl_buffer_context. // So, there is no need to release them here. // TODO: initialize them for non SMALL_PATH path, or remove them. d_img = nullptr; m_img = nullptr; size_q = 0; size_d = 0; size_m = 0; } }; struct ggml_tensor_extra_cl_q5_0 { // Quantized values. cl_mem qs = nullptr; // Quantized values in image1d_buffer_t. cl_mem qs_img = nullptr; // 5-th bit values. cl_mem qh = nullptr; // 5-th bit values in image1d_buffer_t. cl_mem qh_img = nullptr; // Scales. cl_mem d = nullptr; // Scales in image1d_buffer_t. cl_mem d_img = nullptr; // Uniform per-32-block scale (2/block) + min (1/block, = d*16 for the -16 centering) // for the generic dp4a MoE GEMM. Built from d. cl_mem scale = nullptr; cl_mem min = nullptr; // Size of quantized values. size_t size_qs = 0; // Size of 5-th bit values. size_t size_qh = 0; // Size of scales. size_t size_d = 0; ~ggml_tensor_extra_cl_q5_0() { reset(); } void reset() { if (qs != nullptr) { CL_CHECK(clReleaseMemObject(qs)); qs = nullptr; } if (qh != nullptr) { CL_CHECK(clReleaseMemObject(qh)); qh = nullptr; } if (d != nullptr) { CL_CHECK(clReleaseMemObject(d)); d = nullptr; } if (qs_img != nullptr) { CL_CHECK(clReleaseMemObject(qs_img)); qs_img = nullptr; } if (scale != nullptr) { CL_CHECK(clReleaseMemObject(scale)); scale = nullptr; } if (min != nullptr) { CL_CHECK(clReleaseMemObject(min)); min = nullptr; } qh_img = nullptr; d_img = nullptr; size_qs = 0; size_qh = 0; size_d = 0; } }; struct ggml_tensor_extra_cl_q5_1 { // Quantized values. cl_mem qs = nullptr; // Quantized values in image1d_buffer_t. cl_mem qs_img = nullptr; // 5-th bit values. cl_mem qh = nullptr; // 5-th bit values in image1d_buffer_t. cl_mem qh_img = nullptr; // Scales. cl_mem d = nullptr; // Scales in image1d_buffer_t. cl_mem d_img = nullptr; // Min cl_mem m = nullptr; // Min in image1d_buffer_t. cl_mem m_img = nullptr; // Size of quantized values. size_t size_qs = 0; // Size of 5-th bit values. size_t size_qh = 0; // Size of scales. size_t size_d = 0; // Size of min values. size_t size_m = 0; ~ggml_tensor_extra_cl_q5_1() { reset(); } void reset() { // q and d are subbuffers into the bigger buffer allocated in ggml_backend_buffer. // They must be properly released so that the original buffer can be // properly released to avoid memory leak. if (qs != nullptr) { CL_CHECK(clReleaseMemObject(qs)); qs = nullptr; } if (qh != nullptr) { CL_CHECK(clReleaseMemObject(qh)); qh = nullptr; } if (d != nullptr) { CL_CHECK(clReleaseMemObject(d)); d = nullptr; } if (m != nullptr) { CL_CHECK(clReleaseMemObject(m)); m = nullptr; } if (qs_img != nullptr) { CL_CHECK(clReleaseMemObject(qs_img)); qs_img = nullptr; } // qh_img, d_img, and m_img are not currently allocated separately. // TODO: initialize them for non SMALL_PATH path, or remove them. qh_img = nullptr; d_img = nullptr; m_img = nullptr; size_qs = 0; size_qh = 0; size_d = 0; size_m = 0; } }; struct ggml_tensor_extra_cl_mxfp4 { // Quantized values. cl_mem q = nullptr; // Quantized values in image1d_buffer_t. cl_mem q_img = nullptr; // Scales in E8M0. cl_mem e = nullptr; // Scales in image1d_buffer_t. cl_mem e_img = nullptr; // Size of quantized values. size_t size_q = 0; // Size of scales. size_t size_e = 0; ~ggml_tensor_extra_cl_mxfp4() { reset(); } void reset() { // q and d are subbuffers into the bigger buffer allocated in ggml_backend_buffer. // They must be properly released so that the original buffer can be // properly released to avoid memory leak. if (q != nullptr) { CL_CHECK(clReleaseMemObject(q)); q = nullptr; } if (e != nullptr) { CL_CHECK(clReleaseMemObject(e)); e = nullptr; } if (q_img != nullptr) { CL_CHECK(clReleaseMemObject(q_img)); q_img = nullptr; } // Currently, e_img is not used. They can be image1d_buffer_t // that wraps around q and d to utilize image access path. e_img = nullptr; size_q = 0; size_e = 0; } }; struct ggml_tensor_extra_cl_q8_0 { cl_mem q = nullptr; cl_mem q_img = nullptr; cl_mem d = nullptr; cl_mem d_img = nullptr; // Uniform per-16-segment scale (16/superblock) for the generic dp4a MoE GEMM. // Expanded from d at set_tensor; the int8 codes are reused from q. // q8_0 is symmetric so no min buffer (has_min=0). cl_mem scale = nullptr; size_t size_q = 0; size_t size_d = 0; ~ggml_tensor_extra_cl_q8_0() { reset(); } void reset() { // q and d are subbuffers into the bigger buffer allocated in ggml_backend_buffer. // They must be properly released so that the original buffer can be // properly released to avoid memory leak. if (q != nullptr) { CL_CHECK(clReleaseMemObject(q)); q = nullptr; } if (d != nullptr) { CL_CHECK(clReleaseMemObject(d)); d = nullptr; } if (scale != nullptr) { CL_CHECK(clReleaseMemObject(scale)); scale = nullptr; } // Currently, q_img and d_img are not used. They can be image1d_buffer_t // that wraps around q and d to utilize image access path. q_img = nullptr; d_img = nullptr; size_q = 0; size_d = 0; } }; struct ggml_tensor_extra_cl_iq4_nl { cl_mem q = nullptr; cl_mem q_img = nullptr; cl_mem d = nullptr; cl_mem d_img = nullptr; size_t size_q = 0; size_t size_d = 0; ~ggml_tensor_extra_cl_iq4_nl() { reset(); } void reset() { if (q != nullptr) { CL_CHECK(clReleaseMemObject(q)); q = nullptr; } if (d != nullptr) { CL_CHECK(clReleaseMemObject(d)); d = nullptr; } q_img = nullptr; d_img = nullptr; size_q = 0; size_d = 0; } }; struct ggml_tensor_extra_cl_q4_K { // Quantized values cl_mem q = nullptr; // Quantized values in image1d_buffer_t. cl_mem q_img = nullptr; // Scales for each super block. cl_mem s = nullptr; // Scales cl_mem d = nullptr; // Min cl_mem dm = nullptr; ~ggml_tensor_extra_cl_q4_K() { reset(); } void reset() { if (q != nullptr) { CL_CHECK(clReleaseMemObject(q)); q = nullptr; } if (s != nullptr) { CL_CHECK(clReleaseMemObject(s)); s = nullptr; } if (d != nullptr) { CL_CHECK(clReleaseMemObject(d)); d = nullptr; } if (dm != nullptr) { CL_CHECK(clReleaseMemObject(dm)); dm = nullptr; } if (q_img != nullptr) { CL_CHECK(clReleaseMemObject(q_img)); q_img = nullptr; } } }; struct ggml_tensor_extra_cl_q5_K { // Lower 4 bits of quantized weights. cl_mem q = nullptr; // Quantized values in image1d_buffer_t. cl_mem q_img = nullptr; // Upper 1 bit of quantized weights. cl_mem qh = nullptr; // Scales for each block. cl_mem s = nullptr; // Scales for each super block. cl_mem d = nullptr; // Min for each super block. cl_mem dm = nullptr; // Uniform per-32-block scale (2/block) + min (1/block, = dm*mn) decoded from the // 6-bit packed s[] for the generic dp4a MoE GEMM kernel_gemm_moe_q8_1_dp4a. // Built from s/d/dm at set_tensor; q/qh are reused as-is. cl_mem scale = nullptr; cl_mem min = nullptr; size_t size_q = 0; size_t size_qh = 0; size_t size_s = 0; size_t size_d = 0; size_t size_dm = 0; ~ggml_tensor_extra_cl_q5_K() { reset(); } void reset() { if (q != nullptr) { CL_CHECK(clReleaseMemObject(q)); q = nullptr; } if (qh != nullptr) { CL_CHECK(clReleaseMemObject(qh)); qh = nullptr; } if (s != nullptr) { CL_CHECK(clReleaseMemObject(s)); s = nullptr; } if (d != nullptr) { CL_CHECK(clReleaseMemObject(d)); d = nullptr; } if (dm != nullptr) { CL_CHECK(clReleaseMemObject(dm)); dm = nullptr; } if (q_img != nullptr) { CL_CHECK(clReleaseMemObject(q_img)); q_img = nullptr; } if (scale != nullptr) { CL_CHECK(clReleaseMemObject(scale)); scale = nullptr; } if (min != nullptr) { CL_CHECK(clReleaseMemObject(min)); min = nullptr; } size_q = 0; size_qh = 0; size_s = 0; size_d = 0; size_dm = 0; } }; struct ggml_tensor_extra_cl_q6_K { // Lower 4 bits of quantized weights. cl_mem ql = nullptr; // Lower 4 bits as image1d_buffer_t cl_mem ql_img = nullptr; // Upper 2 bits of quantized weights. cl_mem qh = nullptr; // Upper 2 bits as image1d_buffer_t cl_mem qh_img = nullptr; // Scales for each block. cl_mem s = nullptr; // Scales for each super block. cl_mem d = nullptr; size_t size_ql = 0; size_t size_qh = 0; size_t size_s = 0; size_t size_d = 0; ~ggml_tensor_extra_cl_q6_K() { reset(); } void reset() { if (ql != nullptr) { CL_CHECK(clReleaseMemObject(ql)); ql = nullptr; } if (qh != nullptr) { CL_CHECK(clReleaseMemObject(qh)); qh = nullptr; } if (s != nullptr) { CL_CHECK(clReleaseMemObject(s)); s = nullptr; } if (d != nullptr) { CL_CHECK(clReleaseMemObject(d)); d = nullptr; } if (ql_img != nullptr) { CL_CHECK(clReleaseMemObject(ql_img)); ql_img = nullptr; } if (qh_img != nullptr) { CL_CHECK(clReleaseMemObject(qh_img)); qh_img = nullptr; } size_ql = 0; size_qh = 0; size_s = 0; size_d = 0; } }; //------------------------------------------------------------------------------ // Backend API //------------------------------------------------------------------------------ // // backend // static const char * ggml_backend_opencl_name(ggml_backend_t backend) { return "OpenCL"; UNUSED(backend); } static void ggml_backend_opencl_free(ggml_backend_t backend) { ggml_cl_free(backend); } static void ggml_backend_opencl_set_tensor_async(ggml_backend_t backend, ggml_tensor * tensor, const void * data, size_t offset, size_t size) { GGML_UNUSED(backend); GGML_UNUSED(tensor); GGML_UNUSED(data); GGML_UNUSED(offset); GGML_UNUSED(size); } static void ggml_backend_opencl_get_tensor_async(ggml_backend_t backend, const ggml_tensor * tensor, void * data, size_t offset, size_t size) { GGML_UNUSED(backend); GGML_UNUSED(tensor); GGML_UNUSED(data); GGML_UNUSED(offset); GGML_UNUSED(size); } static bool ggml_backend_opencl_cpy_tensor_async(ggml_backend_t backend, const ggml_tensor * src, ggml_tensor * dst) { GGML_UNUSED(backend); GGML_UNUSED(src); GGML_UNUSED(dst); return false; } static void ggml_backend_opencl_synchronize(ggml_backend_t backend) { auto * backend_ctx = static_cast(backend->context); cl_event evt; CL_CHECK(clEnqueueBarrierWithWaitList(backend_ctx->queue, 0, nullptr, &evt)); CL_CHECK(clWaitForEvents(1, &evt)); CL_CHECK(clReleaseEvent(evt)); } // Synchronizes the 'backend_ctx's device with others so that commands // enqueued to it won't start until commands in the other devices have // completed. static void sync_with_other_backends(ggml_backend_opencl_context * backend_ctx) { if (g_ggml_backend_opencl_devices.size() < 2) { return; // No other devices to synchronize with. } std::vector events; events.reserve(g_ggml_backend_opencl_devices.size()); for (ggml_backend_device & backend_dev : g_ggml_backend_opencl_devices) { ggml_backend_opencl_device_context * dev_ctx = (ggml_backend_opencl_device_context *) backend_dev.context; auto * other_backend_ctx = dev_ctx->backend_ctx; if (backend_ctx != other_backend_ctx) { cl_event ev; CL_CHECK(clEnqueueMarkerWithWaitList(other_backend_ctx->queue, 0, nullptr, &ev)); CL_CHECK(clFlush(other_backend_ctx->queue)); events.push_back(ev); } } CL_CHECK(clEnqueueBarrierWithWaitList(backend_ctx->queue, events.size(), events.data(), nullptr)); for (auto ev : events) { CL_CHECK(clReleaseEvent(ev)); } } static void sync_with_other_backends(ggml_backend_t backend) { auto * backend_ctx = static_cast(backend->context); sync_with_other_backends(backend_ctx); } // True if two tensors share a device buffer with overlapping byte ranges. The pool // allocator may place a fused op's output over a sequentially-dead input (safe for the // original separate kernels, but a read/write race inside one fused kernel). static bool ggml_cl_tensors_overlap(const ggml_tensor * x, const ggml_tensor * y) { ggml_tensor_extra_cl * ex = (ggml_tensor_extra_cl *)x->extra; ggml_tensor_extra_cl * ey = (ggml_tensor_extra_cl *)y->extra; if (!ex || !ey || ex->data_device != ey->data_device) { return false; } const cl_ulong xo = ex->offset + x->view_offs, xe = xo + ggml_nbytes(x); const cl_ulong yo = ey->offset + y->view_offs, ye = yo + ggml_nbytes(y); return xo < ye && yo < xe; } // Detect the MoE combine epilogue: router-weight MUL ([n_embd,k,nt] * [1,k,nt]) followed // by k VIEWs of it and a (k-1)-long ADD reduction chain producing [n_embd, nt]. When it // matches (and the output does not alias the inputs), the whole subgraph collapses to one // weighted-sum-across-experts kernel. static bool ggml_opencl_can_fuse_moe_combine(const struct ggml_cgraph * cgraph, int node_idx, const ggml_tensor ** out_final_add) { const ggml_tensor * mul = cgraph->nodes[node_idx]; if (mul->op != GGML_OP_MUL) { return false; } const ggml_tensor * experts = mul->src[0]; const ggml_tensor * weights = mul->src[1]; if (!experts || !weights) { return false; } if (experts->type != GGML_TYPE_F32 || weights->type != GGML_TYPE_F32 || mul->type != GGML_TYPE_F32) { return false; } const int64_t n_embd = experts->ne[0]; const int64_t k = experts->ne[1]; const int64_t nt = experts->ne[2]; if (k < 2 || k > 64 || experts->ne[3] != 1 || n_embd % 4 != 0) { return false; } if (weights->ne[0] != 1 || weights->ne[1] != k || weights->ne[2] != nt || weights->ne[3] != 1) { return false; } if (mul->ne[0] != n_embd || mul->ne[1] != k || mul->ne[2] != nt) { return false; } // the fused kernel needs contiguous experts/weights and a contiguous 2D dst if (!ggml_is_contiguous(experts) || !ggml_is_contiguous(weights)) { return false; } const int n_nodes = 1 + (int)k + (int)(k - 1); // MUL + k*VIEW + (k-1)*ADD if (n_nodes >= 32) { return false; } if (node_idx + n_nodes > cgraph->n_nodes) { return false; } enum ggml_op ops[1 + 64 + 63]; int n = 0; ops[n++] = GGML_OP_MUL; for (int j = 0; j < (int)k; ++j) { ops[n++] = GGML_OP_VIEW; } for (int j = 0; j < (int)k - 1; ++j) { ops[n++] = GGML_OP_ADD; } const int outs[] = { node_idx + n_nodes - 1 }; if (!ggml_can_fuse_subgraph(cgraph, node_idx, n_nodes, ops, outs, 1)) { return false; } for (int j = 0; j < (int)k; ++j) { const ggml_tensor * vw = cgraph->nodes[node_idx + 1 + j]; if (vw->op != GGML_OP_VIEW || vw->src[0] != mul || vw->ne[0] != n_embd || vw->ne[1] != nt) { return false; } } const ggml_tensor * final_add = cgraph->nodes[node_idx + n_nodes - 1]; if (final_add->op != GGML_OP_ADD || final_add->type != GGML_TYPE_F32 || final_add->ne[0] != n_embd || final_add->ne[1] != nt || final_add->ne[2] != 1) { return false; } if (!ggml_is_contiguous(final_add)) { return false; } // the fused kernel reads experts + writes final_add in one pass; bail if the // pool allocator overlapped the output with the (large) experts input -- would race. // The small weights input is copied to a private scratch in the dispatch, so its own // aliasing with the output is handled there and does not block the fusion. if (ggml_cl_tensors_overlap(experts, final_add)) { return false; } *out_final_add = final_add; return true; } // Detect the gpt-oss MoE bias+activation epilogue on the PREFILL path: // {MUL_MAT_ID(gate), ADD_ID(gate_bias), MUL_MAT_ID(up), ADD_ID(up_bias), GLU(swiglu_oai)}. // The two matmuls still run as their own dispatches (the prefill GEMM is the vendor's); // what collapses is the epilogue — both add_id passes are in-place read-modify-writes of a // tensor the GLU immediately reads again, so they are three full passes over the same // [n_ff, n_expert_used, n_tokens] f32 tensor where one suffices. // // The decode counterpart is handled by the mxfp4 fused GEMV arm in ggml_opencl_can_fuse, // which folds the matmul too; this one deliberately fires only when that cannot (ne[2] > 1). static bool ggml_opencl_can_fuse_moe_bias_glu(const struct ggml_cgraph * cgraph, int node_idx) { if (node_idx + 4 >= cgraph->n_nodes) { return false; } const enum ggml_op mg_ops[] = { GGML_OP_MUL_MAT_ID, GGML_OP_ADD_ID, GGML_OP_MUL_MAT_ID, GGML_OP_ADD_ID, GGML_OP_GLU }; const int mg_out[] = { node_idx + 4 }; if (!ggml_can_fuse_subgraph(cgraph, node_idx, 5, mg_ops, mg_out, 1)) { return false; } const ggml_tensor * gmm = cgraph->nodes[node_idx]; const ggml_tensor * gad = cgraph->nodes[node_idx+1]; const ggml_tensor * umm = cgraph->nodes[node_idx+2]; const ggml_tensor * uad = cgraph->nodes[node_idx+3]; const ggml_tensor * glu = cgraph->nodes[node_idx+4]; if (ggml_get_glu_op(glu) != GGML_GLU_OP_SWIGLU_OAI) { return false; } // Prefill only — at one token the mxfp4 arm above folds the matmul as well. if (gmm->src[1]->ne[2] == 1) { return false; } // Wiring: both matmuls share the activation and the expert selection, each add_id // biases its own matmul, and the GLU consumes the two biased results as separate // operands (so the same-buffer ne00_off/ne10_off split path is not in play). if (gad->src[0] != gmm || uad->src[0] != umm || glu->src[0] != gad || glu->src[1] != uad || umm->src[1] != gmm->src[1] || umm->src[2] != gmm->src[2]) { return false; } // A swapped GLU would exchange the gate/up roles the fused kernel hard-codes. if (ggml_get_op_params_i32(glu, 1)) { return false; } if (gad->type != GGML_TYPE_F32 || uad->type != GGML_TYPE_F32 || glu->type != GGML_TYPE_F32) { return false; } if (!gad->src[1] || gad->src[1]->type != GGML_TYPE_F32 || !uad->src[1] || uad->src[1]->type != GGML_TYPE_F32) { return false; } if (!gad->src[2] || gad->src[2]->type != GGML_TYPE_I32 || uad->src[2] != gad->src[2]) { return false; } // Full width on both operands: the kernel writes one output element per input pair. if (!ggml_are_same_shape(gad, uad) || glu->ne[0] != gad->ne[0] || glu->ne[1] != gad->ne[1] || glu->ne[2] != gad->ne[2] || glu->ne[3] != gad->ne[3]) { return false; } if (gad->ne[3] != 1) { return false; } // The destination is addressed by (expert slot, token) rather than the GLU's flat row // walk; those agree only for a contiguous destination. if (!ggml_is_contiguous(glu) || !ggml_is_contiguous(gmm) || !ggml_is_contiguous(umm)) { return false; } return true; } static void ggml_cl_mul_mat_id(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst); // Runs the gate and up matmuls unchanged, then one kernel in place of // add_id(gate) + add_id(up) + swiglu_oai. See ggml_opencl_can_fuse_moe_bias_glu. static void ggml_cl_moe_bias_glu_fused(ggml_backend_t backend, ggml_tensor * gate_mm, const ggml_tensor * gate_add, ggml_tensor * up_mm, const ggml_tensor * up_add, const ggml_tensor * glu) { ggml_backend_opencl_context * backend_ctx = (ggml_backend_opencl_context *)backend->context; ggml_cl_mul_mat_id(backend, gate_mm->src[0], gate_mm->src[1], gate_mm); ggml_cl_mul_mat_id(backend, up_mm->src[0], up_mm->src[1], up_mm); const ggml_tensor * gbias = gate_add->src[1]; const ggml_tensor * ubias = up_add->src[1]; const ggml_tensor * ids = gate_add->src[2]; ggml_tensor_extra_cl * eg = (ggml_tensor_extra_cl *)gate_mm->extra; ggml_tensor_extra_cl * egb = (ggml_tensor_extra_cl *)gbias->extra; ggml_tensor_extra_cl * eu = (ggml_tensor_extra_cl *)up_mm->extra; ggml_tensor_extra_cl * eub = (ggml_tensor_extra_cl *)ubias->extra; ggml_tensor_extra_cl * ei = (ggml_tensor_extra_cl *)ids->extra; ggml_tensor_extra_cl * ed = (ggml_tensor_extra_cl *)glu->extra; cl_ulong off_g = eg->offset + gate_mm->view_offs; cl_ulong off_gb = egb->offset + gbias->view_offs; cl_ulong off_u = eu->offset + up_mm->view_offs; cl_ulong off_ub = eub->offset + ubias->view_offs; cl_ulong off_i = ei->offset + ids->view_offs; cl_ulong off_d = ed->offset + glu->view_offs; const cl_ulong nb01_g = gate_mm->nb[1]; const cl_ulong nb02_g = gate_mm->nb[2]; const cl_ulong nb01_u = up_mm->nb[1]; const cl_ulong nb02_u = up_mm->nb[2]; const cl_ulong nb11_g = gbias->nb[1]; const cl_ulong nb11_u = ubias->nb[1]; const cl_ulong nb21 = ids->nb[1]; const cl_ulong nbd1 = glu->nb[1]; const cl_ulong nbd2 = glu->nb[2]; const int ne0 = (int)glu->ne[0]; const float alpha = ggml_get_op_params_f32(glu, 2); const float limit = ggml_get_op_params_f32(glu, 3); cl_kernel kernel = backend_ctx->kernel_add_id_add_id_swiglu_oai; int i = 0; CL_CHECK(clSetKernelArg(kernel, i++, sizeof(cl_mem), &eg->data_device)); CL_CHECK(clSetKernelArg(kernel, i++, sizeof(cl_ulong), &off_g)); CL_CHECK(clSetKernelArg(kernel, i++, sizeof(cl_mem), &egb->data_device)); CL_CHECK(clSetKernelArg(kernel, i++, sizeof(cl_ulong), &off_gb)); CL_CHECK(clSetKernelArg(kernel, i++, sizeof(cl_mem), &eu->data_device)); CL_CHECK(clSetKernelArg(kernel, i++, sizeof(cl_ulong), &off_u)); CL_CHECK(clSetKernelArg(kernel, i++, sizeof(cl_mem), &eub->data_device)); CL_CHECK(clSetKernelArg(kernel, i++, sizeof(cl_ulong), &off_ub)); CL_CHECK(clSetKernelArg(kernel, i++, sizeof(cl_mem), &ei->data_device)); CL_CHECK(clSetKernelArg(kernel, i++, sizeof(cl_ulong), &off_i)); CL_CHECK(clSetKernelArg(kernel, i++, sizeof(cl_mem), &ed->data_device)); CL_CHECK(clSetKernelArg(kernel, i++, sizeof(cl_ulong), &off_d)); CL_CHECK(clSetKernelArg(kernel, i++, sizeof(cl_ulong), &nb01_g)); CL_CHECK(clSetKernelArg(kernel, i++, sizeof(cl_ulong), &nb02_g)); CL_CHECK(clSetKernelArg(kernel, i++, sizeof(cl_ulong), &nb01_u)); CL_CHECK(clSetKernelArg(kernel, i++, sizeof(cl_ulong), &nb02_u)); CL_CHECK(clSetKernelArg(kernel, i++, sizeof(cl_ulong), &nb11_g)); CL_CHECK(clSetKernelArg(kernel, i++, sizeof(cl_ulong), &nb11_u)); CL_CHECK(clSetKernelArg(kernel, i++, sizeof(cl_ulong), &nb21)); CL_CHECK(clSetKernelArg(kernel, i++, sizeof(cl_ulong), &nbd1)); CL_CHECK(clSetKernelArg(kernel, i++, sizeof(cl_ulong), &nbd2)); CL_CHECK(clSetKernelArg(kernel, i++, sizeof(int), &ne0)); CL_CHECK(clSetKernelArg(kernel, i++, sizeof(float), &limit)); CL_CHECK(clSetKernelArg(kernel, i++, sizeof(float), &alpha)); const int nth = MIN(ne0, (int) backend_ctx->get_kernel_workgroup_size(kernel)); size_t global_work_size[] = { (size_t)glu->ne[1]*nth, (size_t)glu->ne[2], 1 }; size_t local_work_size[] = { (size_t)nth, 1, 1 }; backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, glu); } // Fusion B: the MoE down-projection bias add feeding the combine. // // The graph runs ADD_ID(down_bias) and then immediately the combine subgraph // {MUL(router weights), k VIEWs, k-1 ADDs}, and the ADD_ID's only consumer is that // MUL. Since the ADD_ID is an in-place read-modify-write of a tensor the combine // reads once more, the bias can be added inside the combine instead, dropping a // full pass over [n_embd, k, n_tokens]. // // Shape checks for the combine tail are delegated to ggml_opencl_can_fuse_moe_combine // (which also owns the n_nodes >= 32 bail and the experts/dst aliasing bail); what is // added here is the ADD_ID wiring plus a subgraph check over the WHOLE run, so that // the intermediate bias result is confirmed not to escape. static bool ggml_opencl_can_fuse_moe_bias_combine(const struct ggml_cgraph * cgraph, int node_idx, const ggml_tensor ** out_final_add) { if (node_idx + 1 >= cgraph->n_nodes) { return false; } const ggml_tensor * add = cgraph->nodes[node_idx]; if (add->op != GGML_OP_ADD_ID) { return false; } const ggml_tensor * mul = cgraph->nodes[node_idx+1]; if (mul->op != GGML_OP_MUL || mul->src[0] != add) { return false; } const ggml_tensor * final_add = NULL; if (!ggml_opencl_can_fuse_moe_combine(cgraph, node_idx+1, &final_add)) { return false; } const ggml_tensor * raw = add->src[0]; const ggml_tensor * bias = add->src[1]; const ggml_tensor * ids = add->src[2]; if (!raw || !bias || !ids) { return false; } if (raw->type != GGML_TYPE_F32 || bias->type != GGML_TYPE_F32 || ids->type != GGML_TYPE_I32 || add->type != GGML_TYPE_F32) { return false; } // The combine reads the raw matmul output with the strides it computed from the // add_id result, so the two must have the same layout. if (!ggml_are_same_shape(raw, add) || !ggml_is_contiguous(raw)) { return false; } if (raw->nb[1] != add->nb[1] || raw->nb[2] != add->nb[2]) { return false; } // ids is indexed as [expert slot, token]; the combine walks the same two axes. if (ids->ne[0] < add->ne[1] || ids->ne[1] < add->ne[2]) { return false; } // Whole-run escape check: ADD_ID + MUL + k VIEWs + (k-1) ADDs, only the last node escapes. const int k = (int)add->ne[1]; const int n_nodes = 2 + k + (k - 1); if (n_nodes >= 32 || node_idx + n_nodes > cgraph->n_nodes) { return false; } enum ggml_op ops[32]; int n = 0; ops[n++] = GGML_OP_ADD_ID; ops[n++] = GGML_OP_MUL; for (int j = 0; j < k; ++j) ops[n++] = GGML_OP_VIEW; for (int j = 0; j < k - 1; ++j) ops[n++] = GGML_OP_ADD; const int outs[] = { node_idx + n_nodes - 1 }; if (!ggml_can_fuse_subgraph(cgraph, node_idx, n_nodes, ops, outs, 1)) { return false; } *out_final_add = final_add; return true; } // Fusion B dispatch: the combine, reading the RAW matmul output and adding the // per-expert bias row inline. See ggml_opencl_can_fuse_moe_bias_combine. static void ggml_cl_moe_bias_combine_fused(ggml_backend_t backend, const ggml_tensor * add, const ggml_tensor * mul, const ggml_tensor * dst) { ggml_backend_opencl_context * backend_ctx = (ggml_backend_opencl_context *)backend->context; const ggml_tensor * experts = add->src[0]; // raw matmul output, bias not yet applied const ggml_tensor * bias = add->src[1]; const ggml_tensor * ids = add->src[2]; const ggml_tensor * weights = mul->src[1]; ggml_tensor_extra_cl * ee = (ggml_tensor_extra_cl *)experts->extra; ggml_tensor_extra_cl * eb = (ggml_tensor_extra_cl *)bias->extra; ggml_tensor_extra_cl * ei = (ggml_tensor_extra_cl *)ids->extra; ggml_tensor_extra_cl * ew = (ggml_tensor_extra_cl *)weights->extra; ggml_tensor_extra_cl * ed = (ggml_tensor_extra_cl *)dst->extra; cl_ulong off_e = ee->offset + experts->view_offs; cl_ulong off_b = eb->offset + bias->view_offs; cl_ulong off_i = ei->offset + ids->view_offs; cl_ulong off_w = ew->offset + weights->view_offs; cl_ulong off_d = ed->offset + dst->view_offs; const int n_embd4 = (int)(experts->ne[0] / 4); const int k = (int)experts->ne[1]; const int nt = (int)experts->ne[2]; const cl_uint e1 = (cl_uint)(experts->nb[1] / sizeof(float)); const cl_uint e2 = (cl_uint)(experts->nb[2] / sizeof(float)); const cl_uint w1 = (cl_uint)(weights->nb[1] / sizeof(float)); const cl_uint w2 = (cl_uint)(weights->nb[2] / sizeof(float)); const cl_uint d1 = (cl_uint)(dst->nb[1] / sizeof(float)); const cl_ulong nb_b1 = bias->nb[1]; const cl_ulong nb_i1 = ids->nb[1]; const size_t w_bytes = ggml_nbytes(weights); backend_ctx->prealloc_moe_combine_w.allocate(backend_ctx->context, w_bytes); CL_CHECK(clEnqueueCopyBuffer(backend_ctx->queue, ew->data_device, backend_ctx->prealloc_moe_combine_w.buffer, off_w, 0, w_bytes, 0, NULL, NULL)); cl_mem w_dev = backend_ctx->prealloc_moe_combine_w.buffer; cl_ulong w_off = 0; cl_kernel kernel = backend_ctx->kernel_moe_combine_bias_f32; int a = 0; CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_mem), &ee->data_device)); CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_ulong), &off_e)); CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_mem), &w_dev)); CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_ulong), &w_off)); CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_mem), &eb->data_device)); CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_ulong), &off_b)); CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_mem), &ei->data_device)); CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_ulong), &off_i)); CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_mem), &ed->data_device)); CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_ulong), &off_d)); CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &n_embd4)); CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &k)); CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &nt)); CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_uint), &e1)); CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_uint), &e2)); CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_uint), &w1)); CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_uint), &w2)); CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_uint), &d1)); CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_ulong), &nb_b1)); CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_ulong), &nb_i1)); size_t lws[2] = { 64, 1 }; size_t gws[2] = { (size_t)(((n_embd4 + 63) / 64) * 64), (size_t)nt }; backend_ctx->enqueue_ndrange_kernel(kernel, 2, gws, lws, dst); } static void ggml_cl_moe_combine_fused(ggml_backend_t backend, const ggml_tensor * mul, const ggml_tensor * dst) { ggml_backend_opencl_context * backend_ctx = (ggml_backend_opencl_context *)backend->context; const ggml_tensor * experts = mul->src[0]; const ggml_tensor * weights = mul->src[1]; ggml_tensor_extra_cl * ee = (ggml_tensor_extra_cl *)experts->extra; ggml_tensor_extra_cl * ew = (ggml_tensor_extra_cl *)weights->extra; ggml_tensor_extra_cl * ed = (ggml_tensor_extra_cl *)dst->extra; cl_ulong off_e = ee->offset + experts->view_offs; cl_ulong off_w = ew->offset + weights->view_offs; cl_ulong off_d = ed->offset + dst->view_offs; const int n_embd4 = (int)(experts->ne[0] / 4); const int k = (int)experts->ne[1]; const int nt = (int)experts->ne[2]; const cl_uint e1 = (cl_uint)(experts->nb[1] / sizeof(float)); const cl_uint e2 = (cl_uint)(experts->nb[2] / sizeof(float)); const cl_uint w1 = (cl_uint)(weights->nb[1] / sizeof(float)); const cl_uint w2 = (cl_uint)(weights->nb[2] / sizeof(float)); const cl_uint d1 = (cl_uint)(dst->nb[1] / sizeof(float)); // The router weights are tiny ([1,k,nt]) and may share a pool buffer with the output; // copy them into a private scratch so the fused kernel never reads aliased memory. const size_t w_bytes = ggml_nbytes(weights); backend_ctx->prealloc_moe_combine_w.allocate(backend_ctx->context, w_bytes); CL_CHECK(clEnqueueCopyBuffer(backend_ctx->queue, ew->data_device, backend_ctx->prealloc_moe_combine_w.buffer, off_w, 0, w_bytes, 0, NULL, NULL)); cl_mem w_dev = backend_ctx->prealloc_moe_combine_w.buffer; cl_ulong w_off = 0; cl_kernel kernel = backend_ctx->kernel_moe_combine_f32; int a = 0; CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_mem), &ee->data_device)); CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_ulong), &off_e)); CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_mem), &w_dev)); CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_ulong), &w_off)); CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_mem), &ed->data_device)); CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_ulong), &off_d)); CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &n_embd4)); CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &k)); CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &nt)); CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_uint), &e1)); CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_uint), &e2)); CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_uint), &w1)); CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_uint), &w2)); CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_uint), &d1)); size_t lws[2] = { 64, 1 }; size_t gws[2] = { (size_t)(((n_embd4 + 63) / 64) * 64), (size_t)nt }; backend_ctx->enqueue_ndrange_kernel(kernel, 2, gws, lws, dst); } inline bool use_q4k_tiled(const ggml_backend_opencl_context *backend_ctx, const ggml_tensor *tensor); // defined below (used by the GLU-subgraph fuse check) inline bool use_q4_k_bin_kernels(const ggml_backend_opencl_context *backend_ctx, const ggml_tensor *tensor); inline bool use_adreno_kernels(const ggml_backend_opencl_context *backend_ctx, const ggml_tensor *tensor); // defined below static bool ggml_opencl_can_fuse(const ggml_backend_opencl_context * backend_ctx, const struct ggml_cgraph * cgraph, int node_idx, std::initializer_list ops) { // glu(mul_mat(Wg,x), mul_mat(Wu,x)) — the FFN gate/up GEMVs + GLU. This is a // non-linear subgraph (up does NOT consume gate), so the contiguous // ggml_can_fuse below rejects it; use ggml_can_fuse_subgraph with the glu as // the sole output and validate the edges explicitly. q4_K decode only; // byte-identical to the per-op path. if (ops.size() == 3 && ops.begin()[0] == GGML_OP_MUL_MAT && ops.begin()[1] == GGML_OP_MUL_MAT && ops.begin()[2] == GGML_OP_GLU) { const enum ggml_op glu_ops[] = { GGML_OP_MUL_MAT, GGML_OP_MUL_MAT, GGML_OP_GLU }; const int glu_out[] = { node_idx + 2 }; if (!ggml_can_fuse_subgraph(cgraph, node_idx, 3, glu_ops, glu_out, 1)) { return false; } const ggml_tensor *gate = cgraph->nodes[node_idx]; const ggml_tensor *up = cgraph->nodes[node_idx+1]; const ggml_tensor *glu = cgraph->nodes[node_idx+2]; // decode GEMV path only (single token); prefill GEMM is separate if (gate->ne[1] != 1 || up->ne[1] != 1) { return false; } // both projections must be q4_K weights, f32 activation/output if (gate->src[0]->type != GGML_TYPE_Q4_K || up->src[0]->type != GGML_TYPE_Q4_K || gate->src[1]->type != GGML_TYPE_F32 || up->src[1]->type != GGML_TYPE_F32 || gate->type != GGML_TYPE_F32 || up->type != GGML_TYPE_F32 || glu->type != GGML_TYPE_F32) { return false; } // gate and up must share the same activation and have matching shape/stride if (gate->src[1] != up->src[1] || !ggml_are_same_shape(gate->src[0], up->src[0]) || !ggml_are_same_stride(gate->src[0], up->src[0])) { return false; } // GLU must read gate as src[0] and up as src[1], no swap (the fused // epilogue applies the activation to gate, multiplies by up) if (glu->src[0] != gate || glu->src[1] != up) { return false; } if (ggml_get_op_params_i32(glu, 1) /* swapped */) { return false; } // SWIGLU_OAI carries extra alpha/limit params -> not handled by the fused kernel if (ggml_get_glu_op(glu) == GGML_GLU_OP_SWIGLU_OAI) { return false; } // the fused kernel reads the standard noshuffle image layout; the tiled // layout packs weights differently -> defer those to the per-op path if (use_q4k_tiled(backend_ctx, gate->src[0]) || use_q4k_tiled(backend_ctx, up->src[0])) { return false; } // q4_K bin kernel requires 32b transposed layout, not compatible with the fused gemv if (use_q4_k_bin_kernels(backend_ctx, gate->src[0]) || use_q4_k_bin_kernels(backend_ctx, up->src[0])) { return false; } // that noshuffle layout is only produced at set_tensor time when // use_adreno_kernels() accepts the weight (ne0 >= 512 && ne1 >= 512). // Smaller weights stay in the plain q4_K layout, which this kernel would // misread -> defer them to the per-op path. Real FFN gate/up weights are // far above the threshold, so production dispatch is unchanged. if (!use_adreno_kernels(backend_ctx, gate->src[0]) || !use_adreno_kernels(backend_ctx, up->src[0])) { return false; } return true; } if (!ggml_can_fuse(cgraph, node_idx, ops)) { return false; } if (ops.size() == 2 && ops.begin()[0] == GGML_OP_RMS_NORM && ops.begin()[1] == GGML_OP_MUL) { const ggml_tensor *rms_norm = cgraph->nodes[node_idx]; const ggml_tensor *mul = cgraph->nodes[node_idx+1]; GGML_ASSERT(rms_norm->src[0]->type == GGML_TYPE_F32); GGML_ASSERT(rms_norm->type == GGML_TYPE_F32); // rms_norm only supports f32 if (mul->src[0]->type != GGML_TYPE_F32 || mul->src[1]->type != GGML_TYPE_F32 || mul->type != GGML_TYPE_F32) { return false; } // if rms_norm is the B operand, then we don't handle broadcast if (rms_norm == mul->src[1] && !ggml_are_same_shape(mul->src[0], rms_norm)) { return false; } // rms_norm assumes contiguous rows if (!ggml_is_contiguous_rows(mul->src[0]) || !ggml_is_contiguous_rows(mul->src[1])) { return false; } } else if (ops.size() == 3 && ops.begin()[0] == GGML_OP_NORM && ops.begin()[1] == GGML_OP_MUL && ops.begin()[2] == GGML_OP_ADD) { const ggml_tensor *norm = cgraph->nodes[node_idx]; const ggml_tensor *mul = cgraph->nodes[node_idx+1]; const ggml_tensor *add = cgraph->nodes[node_idx+2]; const ggml_tensor *w = mul->src[0] == norm ? mul->src[1] : mul->src[0]; const ggml_tensor *b = add->src[0] == mul ? add->src[1] : add->src[0]; // norm fusion only supports F32 if (norm->src[0]->type != GGML_TYPE_F32 || w->type != GGML_TYPE_F32 || b->type != GGML_TYPE_F32) { return false; } if (norm->src[0]->ne[0] % 4 != 0) { return false; } if (!ggml_is_contiguous(norm->src[0]) || !ggml_is_contiguous(w) || !ggml_is_contiguous(b)) { return false; } } else if (ops.size() == 3 && ops.begin()[0] == GGML_OP_RMS_NORM && ops.begin()[1] == GGML_OP_MUL && ops.begin()[2] == GGML_OP_ADD) { // rms_norm(x) * w + b, fused (residual). Mirrors the RMS_NORM+MUL gate // plus the residual-add operand's constraints. const ggml_tensor *rms_norm = cgraph->nodes[node_idx]; const ggml_tensor *mul = cgraph->nodes[node_idx+1]; const ggml_tensor *add = cgraph->nodes[node_idx+2]; const ggml_tensor *w = mul->src[0] == rms_norm ? mul->src[1] : mul->src[0]; const ggml_tensor *b = add->src[0] == mul ? add->src[1] : add->src[0]; GGML_ASSERT(rms_norm->src[0]->type == GGML_TYPE_F32); GGML_ASSERT(rms_norm->type == GGML_TYPE_F32); if (w->type != GGML_TYPE_F32 || mul->type != GGML_TYPE_F32 || b->type != GGML_TYPE_F32 || add->type != GGML_TYPE_F32) { return false; } if (rms_norm->src[0]->ne[0] % 4 != 0) { return false; } // if rms_norm is the B operand of mul, broadcast is not handled if (rms_norm == mul->src[1] && !ggml_are_same_shape(mul->src[0], rms_norm)) { return false; } // the residual must match the normed output shape (no add broadcast) if (!ggml_are_same_shape(b, add)) { return false; } // rms_norm assumes contiguous rows if (!ggml_is_contiguous_rows(mul->src[0]) || !ggml_is_contiguous_rows(mul->src[1]) || !ggml_is_contiguous_rows(b)) { return false; } } else if (ops.size() == 3 && ops.begin()[0] == GGML_OP_GROUP_NORM && ops.begin()[1] == GGML_OP_MUL && ops.begin()[2] == GGML_OP_ADD) { const ggml_tensor *gn = cgraph->nodes[node_idx]; const ggml_tensor *mul = cgraph->nodes[node_idx+1]; const ggml_tensor *add = cgraph->nodes[node_idx+2]; const ggml_tensor *w = mul->src[0] == gn ? mul->src[1] : mul->src[0]; const ggml_tensor *b = add->src[0] == mul ? add->src[1] : add->src[0]; if (gn->src[0]->type != GGML_TYPE_F32 || w->type != GGML_TYPE_F32 || b->type != GGML_TYPE_F32) { return false; } if (!ggml_is_contiguous(gn->src[0]) || !ggml_is_contiguous(w) || !ggml_is_contiguous(b)) { return false; } } return true; } static void ggml_opencl_op_rms_norm_fused(ggml_backend_t backend, ggml_tensor * rms_norm_tensor, ggml_tensor * mul_tensor); static void ggml_opencl_op_norm_fused(ggml_backend_t backend, ggml_tensor * norm_tensor, ggml_tensor * mul_tensor, ggml_tensor * add_tensor); static void ggml_opencl_op_group_norm_fused(ggml_backend_t backend, ggml_tensor * gn_tensor, ggml_tensor * mul_tensor, ggml_tensor * add_tensor); static void ggml_cl_mul_mat_q4_k_glu_fused(ggml_backend_t backend, ggml_tensor * gate_tensor, ggml_tensor * up_tensor, ggml_tensor * glu_tensor) { #ifdef GGML_OPENCL_USE_ADRENO_KERNELS GGML_ASSERT(gate_tensor && up_tensor && glu_tensor); const ggml_tensor * Wg = gate_tensor->src[0]; const ggml_tensor * Wu = up_tensor->src[0]; const ggml_tensor * src1 = gate_tensor->src[1]; // == up_tensor->src[1] const ggml_tensor * dst = glu_tensor; GGML_ASSERT(Wg && Wg->extra); GGML_ASSERT(Wu && Wu->extra); GGML_ASSERT(src1 && src1->extra); GGML_ASSERT(dst && dst->extra); ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; ggml_tensor_extra_cl * extra1 = (ggml_tensor_extra_cl *)src1->extra; ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; ggml_tensor_extra_cl_q4_K * extra_g = (ggml_tensor_extra_cl_q4_K *)Wg->extra; ggml_tensor_extra_cl_q4_K * extra_u = (ggml_tensor_extra_cl_q4_K *)Wu->extra; cl_ulong offset1 = extra1->offset + src1->view_offs; cl_ulong offsetd = extrad->offset + dst->view_offs; const int K = Wg->ne[0]; // ne00 const int M = Wg->ne[1]; // ne01 (= ffn intermediate width) const int N = 1; // decode GEMV const cl_uchar mask_d6 = 0x3F, mask_d4 = 0x0F, mask_hi2 = 0xC0; const int glu_op = (int)ggml_get_glu_op(dst); cl_context context = backend_ctx->context; cl_int err; cl_image_format img_fmt; cl_image_desc img_desc; cl_buffer_region region; // q images for the two weight matrices (standard noshuffle layout) img_fmt = { CL_R, CL_UNSIGNED_INT32 }; memset(&img_desc, 0, sizeof(img_desc)); img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; img_desc.image_width = (size_t)M * K / 2 / 4; img_desc.buffer = extra_g->q; cl_mem qg_img = nullptr, qu_img = nullptr; CL_CHECK((qg_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err), err)); img_desc.buffer = extra_u->q; CL_CHECK((qu_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err), err)); // shared activation image (one column at decode) region.origin = offset1; region.size = (size_t)K * N * sizeof(float); cl_mem b_sub_buf = nullptr, b_img = nullptr; CL_CHECK((b_sub_buf = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); img_fmt = { CL_RGBA, CL_FLOAT }; memset(&img_desc, 0, sizeof(img_desc)); img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; img_desc.image_width = (size_t)K * N / 4; img_desc.buffer = b_sub_buf; CL_CHECK((b_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err), err)); cl_kernel kernel = backend_ctx->kernel_gemv_noshuffle_q4_k_f32_glu; CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &qg_img)); CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra_g->d)); CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra_g->dm)); CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra_g->s)); CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &qu_img)); CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_mem), &extra_u->d)); CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_mem), &extra_u->dm)); CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_mem), &extra_u->s)); CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_mem), &b_img)); CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_mem), &extrad->data_device)); CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_ulong), &offsetd)); CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_int), &K)); CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_int), &M)); CL_CHECK(clSetKernelArg(kernel, 13, sizeof(cl_int), &glu_op)); CL_CHECK(clSetKernelArg(kernel, 14, sizeof(cl_uchar), &mask_d6)); CL_CHECK(clSetKernelArg(kernel, 15, sizeof(cl_uchar), &mask_d4)); CL_CHECK(clSetKernelArg(kernel, 16, sizeof(cl_uchar), &mask_hi2)); // K-split = nsg_y subgroups. HARD-CAP at 8 (512 work-items): the fused // kernel's cross-subgroup reduce uses a float4 reduceLM (gate+up packed) = // 2x the LDS of the base GEMV's float2 reduce, so 16 co-resident subgroups // exceed the per-CU LDS budget on X2 and the WG barrier DEADLOCKS -> GPU TDR // (reproduced on upstream gemma-4 E4B decode, K=2560 M=10240). This used to // be masked: get_kernel_workgroup_size reported 896 for this kernel (so the // cap loop fell to 8), but it now returns 1024 and the Adreno per-kernel WG // query is unreliable (over-reports), so cap explicitly instead of trusting // it. nsg_y < 16 also means the cross-subgroup accumulation grouping differs // from the standalone wide (nsg=16) GEMV, so the output is coherent but NOT // byte-identical to the per-op path. Keep the maxwg query as a further floor // for any driver that reports < 512. size_t maxwg = backend_ctx->get_kernel_workgroup_size(kernel); size_t nsg_y = 8; while (nsg_y > 1 && 64 * nsg_y > maxwg) { nsg_y >>= 1; } size_t local_work_size[3] = { 64, nsg_y, 1 }; size_t global_work_size[3] = { (size_t)CEIL_DIV(M / 2, 64) * 64, nsg_y, 1 }; if (getenv("GGML_OPENCL_FUSE_DEBUG")) { static int dbg = 0; if (dbg < 3) { fprintf(stderr, "[FUSE_MM_GLU] fired #%d K=%d M=%d glu_op=%d nsg=%zu maxwg=%zu\n", ++dbg, K, M, glu_op, nsg_y, maxwg); fflush(stderr); } } backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); CL_CHECK(clReleaseMemObject(qg_img)); CL_CHECK(clReleaseMemObject(qu_img)); CL_CHECK(clReleaseMemObject(b_img)); CL_CHECK(clReleaseMemObject(b_sub_buf)); #else GGML_UNUSED(backend); GGML_UNUSED(gate_tensor); GGML_UNUSED(up_tensor); GGML_UNUSED(glu_tensor); #endif } static void ggml_opencl_op_rms_norm_mul_add_fused(ggml_backend_t backend, ggml_tensor * rms_norm_tensor, ggml_tensor * mul_tensor, ggml_tensor * add_tensor) { GGML_ASSERT(rms_norm_tensor && mul_tensor && add_tensor); const ggml_tensor * src0 = rms_norm_tensor->src[0]; const ggml_tensor * src1 = mul_tensor->src[0] == rms_norm_tensor ? mul_tensor->src[1] : mul_tensor->src[0]; const ggml_tensor * src2 = add_tensor->src[0] == mul_tensor ? add_tensor->src[1] : add_tensor->src[0]; const ggml_tensor * dst = add_tensor; GGML_ASSERT(src0 && src0->extra); GGML_ASSERT(src1 && src1->extra); GGML_ASSERT(src2 && src2->extra); GGML_ASSERT(dst && dst->extra); ggml_tensor_extra_cl * extra0 = (ggml_tensor_extra_cl *)src0->extra; ggml_tensor_extra_cl * extra1 = (ggml_tensor_extra_cl *)src1->extra; ggml_tensor_extra_cl * extra2 = (ggml_tensor_extra_cl *)src2->extra; ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; cl_ulong offset0 = extra0->offset + src0->view_offs; cl_ulong offset1 = extra1->offset + src1->view_offs; cl_ulong offset2 = extra2->offset + src2->view_offs; cl_ulong offsetd = extrad->offset + dst->view_offs; ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; float eps; memcpy(&eps, rms_norm_tensor->op_params, sizeof(float)); const int ne00 = src0->ne[0], ne01 = src0->ne[1], ne02 = src0->ne[2], ne03 = src0->ne[3]; const cl_ulong nb01 = src0->nb[1], nb02 = src0->nb[2], nb03 = src0->nb[3]; const int ne10 = src1->ne[0], ne11 = src1->ne[1], ne12 = src1->ne[2], ne13 = src1->ne[3]; const cl_ulong nb11 = src1->nb[1], nb12 = src1->nb[2], nb13 = src1->nb[3]; const int ne20 = src2->ne[0], ne21 = src2->ne[1], ne22 = src2->ne[2], ne23 = src2->ne[3]; const cl_ulong nb21 = src2->nb[1], nb22 = src2->nb[2], nb23 = src2->nb[3]; const cl_ulong nb1 = dst->nb[1], nb2 = dst->nb[2], nb3 = dst->nb[3]; GGML_ASSERT(ne00 % 4 == 0); size_t sgs; if (backend_ctx->gpu_family == ADRENO) sgs = 64; else if (backend_ctx->gpu_family == INTEL) sgs = 32; else GGML_ASSERT(false && "Unsupported GPU"); cl_kernel kernel = backend_ctx->kernel_rms_norm_mul_add; int nth = sgs; int max_workgroup_size = backend_ctx->get_kernel_workgroup_size(kernel); while (nth < ne00 && nth < max_workgroup_size) nth *= 2; nth = MIN(nth, max_workgroup_size); nth = MIN(nth, ne00); size_t global_work_size[] = {(size_t)ne01*nth, (size_t)ne02, (size_t)ne03}; size_t local_work_size[] = {(size_t)nth, 1, 1}; CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device)); CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1)); CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extra2->data_device)); CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offset2)); CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_mem), &extrad->data_device)); CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_ulong), &offsetd)); CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne00)); CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne01)); CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &ne02)); CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne03)); CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_ulong), &nb01)); CL_CHECK(clSetKernelArg(kernel, 13, sizeof(cl_ulong), &nb02)); CL_CHECK(clSetKernelArg(kernel, 14, sizeof(cl_ulong), &nb03)); CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &ne10)); CL_CHECK(clSetKernelArg(kernel, 16, sizeof(int), &ne11)); CL_CHECK(clSetKernelArg(kernel, 17, sizeof(int), &ne12)); CL_CHECK(clSetKernelArg(kernel, 18, sizeof(int), &ne13)); CL_CHECK(clSetKernelArg(kernel, 19, sizeof(cl_ulong), &nb11)); CL_CHECK(clSetKernelArg(kernel, 20, sizeof(cl_ulong), &nb12)); CL_CHECK(clSetKernelArg(kernel, 21, sizeof(cl_ulong), &nb13)); CL_CHECK(clSetKernelArg(kernel, 22, sizeof(int), &ne20)); CL_CHECK(clSetKernelArg(kernel, 23, sizeof(int), &ne21)); CL_CHECK(clSetKernelArg(kernel, 24, sizeof(int), &ne22)); CL_CHECK(clSetKernelArg(kernel, 25, sizeof(int), &ne23)); CL_CHECK(clSetKernelArg(kernel, 26, sizeof(cl_ulong), &nb21)); CL_CHECK(clSetKernelArg(kernel, 27, sizeof(cl_ulong), &nb22)); CL_CHECK(clSetKernelArg(kernel, 28, sizeof(cl_ulong), &nb23)); CL_CHECK(clSetKernelArg(kernel, 29, sizeof(cl_ulong), &nb1)); CL_CHECK(clSetKernelArg(kernel, 30, sizeof(cl_ulong), &nb2)); CL_CHECK(clSetKernelArg(kernel, 31, sizeof(cl_ulong), &nb3)); CL_CHECK(clSetKernelArg(kernel, 32, sizeof(float), &eps)); CL_CHECK(clSetKernelArg(kernel, 33, sizeof(float)*sgs, NULL)); backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); } static ggml_status ggml_backend_opencl_graph_compute(ggml_backend_t backend, ggml_cgraph * cgraph) { ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; for (int i = 0; i < cgraph->n_nodes; i++) { ggml_tensor * node = cgraph->nodes[i]; // NOTE: this may oversynchronize by synchronizing with // backends/devices which don't compute 'cgraph's // dependencies. sync_with_other_backends(backend); if (ggml_is_empty(node) || node->op == GGML_OP_RESHAPE || node->op == GGML_OP_TRANSPOSE || node->op == GGML_OP_VIEW || node->op == GGML_OP_PERMUTE || node->op == GGML_OP_NONE) { continue; } if ((node->flags & GGML_TENSOR_FLAG_COMPUTE) == 0) { continue; } if (!backend_ctx->disable_fusion && ggml_opencl_can_fuse(backend_ctx, cgraph, i, { GGML_OP_NORM, GGML_OP_MUL, GGML_OP_ADD })) { ggml_opencl_op_norm_fused(backend, node, cgraph->nodes[i+1], cgraph->nodes[i+2]); i += 2; continue; } if (!backend_ctx->disable_fusion && ggml_opencl_can_fuse(backend_ctx, cgraph, i, { GGML_OP_GROUP_NORM, GGML_OP_MUL, GGML_OP_ADD })) { ggml_opencl_op_group_norm_fused(backend, node, cgraph->nodes[i+1], cgraph->nodes[i+2]); i += 2; continue; } // Fuse the MoE combine: router-weight mul + cross-expert add chain -> // one weighted-sum-across-experts kernel. // Fold the gpt-oss MoE bias epilogue: add_id(gate_bias) + add_id(up_bias) + // glu(swiglu_oai) -> one kernel, leaving the two matmuls as their own dispatches. // Both add_ids are in-place passes over a tensor the GLU reads again, so this // drops two full read+write passes per layer. Opt out GGML_OPENCL_FUSE_MOE_BIAS_GLU=0. if (backend_ctx->fuse_moe_bias_glu && !backend_ctx->disable_fusion && ggml_opencl_can_fuse_moe_bias_glu(cgraph, i)) { ggml_cl_moe_bias_glu_fused(backend, node, cgraph->nodes[i+1], cgraph->nodes[i+2], cgraph->nodes[i+3], cgraph->nodes[i+4]); i += 4; continue; } // Fold the MoE down-projection bias into the combine: add_id(down_bias) + the whole // combine subgraph -> one kernel. Checked before the plain combine arm so the longer // pattern wins. Opt out GGML_OPENCL_FUSE_MOE_BIAS_COMBINE=0. if (backend_ctx->fuse_moe_bias_combine && backend_ctx->fuse_moe_combine && !backend_ctx->disable_fusion) { const ggml_tensor * bias_combine_out = nullptr; if (ggml_opencl_can_fuse_moe_bias_combine(cgraph, i, &bias_combine_out)) { ggml_cl_moe_bias_combine_fused(backend, node, cgraph->nodes[i+1], bias_combine_out); i += 2 * (int)node->ne[1]; // ADD_ID + MUL + k VIEWs + (k-1) ADDs continue; } } if (backend_ctx->fuse_moe_combine && !backend_ctx->disable_fusion) { const ggml_tensor * combine_out = nullptr; if (ggml_opencl_can_fuse_moe_combine(cgraph, i, &combine_out)) { ggml_cl_moe_combine_fused(backend, node, combine_out); i += 2 * (int)node->ne[1] - 1; // skip the k VIEWs + (k-1) ADDs continue; } } // Fuse rms_norm + mul(weight) + add(residual). Checked before the // rms_norm+mul fuse so the 3-op pattern wins over its 2-op prefix. // Default on, opt-out GGML_OPENCL_FUSE_RMS_ADD=0. if (!backend_ctx->disable_fusion && backend_ctx->fuse_rms_add && ggml_opencl_can_fuse(backend_ctx, cgraph, i, { GGML_OP_RMS_NORM, GGML_OP_MUL, GGML_OP_ADD })) { ggml_opencl_op_rms_norm_mul_add_fused(backend, node, cgraph->nodes[i+1], cgraph->nodes[i+2]); i += 2; continue; } if (!backend_ctx->disable_fusion && ggml_opencl_can_fuse(backend_ctx, cgraph, i, { GGML_OP_RMS_NORM, GGML_OP_MUL })) { ggml_opencl_op_rms_norm_fused(backend, node, cgraph->nodes[i+1]); i++; continue; } // Fuse mul_mat(Wg,x) + mul_mat(Wu,x) + glu — fold the FFN's two decode // GEMVs and the GLU into one dispatch. q4_K only (guarded below); the // fused kernel uses the same accumulation/reduction order and the same // scalar GLU formula -> coherent. Default on, opt-out GGML_OPENCL_FUSE_MM_GLU=0. #ifdef GGML_OPENCL_USE_ADRENO_KERNELS // The fused executor (ggml_cl_mul_mat_q4_k_glu_fused) is image-path / // Adreno-only (GGML_ABORT on the non-Adreno #else); gate the dispatch to // match so the FFN GLU subgraph stays dormant on Intel/other drivers. if (backend_ctx->fuse_mm_glu && !backend_ctx->disable_fusion && ggml_opencl_can_fuse(backend_ctx, cgraph, i, { GGML_OP_MUL_MAT, GGML_OP_MUL_MAT, GGML_OP_GLU })) { ggml_cl_mul_mat_q4_k_glu_fused(backend, node, cgraph->nodes[i+1], cgraph->nodes[i+2]); i += 2; continue; } #endif bool ok = ggml_cl_compute_forward(backend, node); if (!ok) { GGML_LOG_ERROR("%s: error: op not supported %s (%s)\n", __func__, node->name, ggml_op_name(node->op)); } GGML_ASSERT(ok); } return GGML_STATUS_SUCCESS; } // The optimized gemm and gemv kernels are used for large matrices without batch. // tensor is the quantized weights matrix. inline bool use_adreno_kernels(const ggml_backend_opencl_context *backend_ctx, const ggml_tensor *tensor) { int64_t threshold_ne0 = 512; int64_t threshold_ne1 = 512; if (!backend_ctx->adreno_cl_compiler_version.newer_than_or_same(E031, 38, 11, 0) && backend_ctx->adreno_cl_compiler_version.type != DX) { threshold_ne0 = 128; threshold_ne1 = 128; } bool threashold_ok = tensor->ne[0] >= threshold_ne0 && tensor->ne[1] >= threshold_ne1 && tensor->ne[2] == 1 && tensor->ne[3] == 1; // The noshuffle layout packs 2 rows per 32-bit texel and the GEMV reads it at an // ne1/2 texel stride with an exact-cover dispatch, so it is only addressable when // ne1 is a multiple of 64; an unaligned ne1 truncates the stride and the weight is // read misaligned. That is a property of the layout, not of one quant -- q4_K, q5_K // and q8_0 read the same packing as q6_K. The bound is 64, not 128: a q8_0 attention // weight of ne1 = 2880 is a multiple of 64 but not 128 and is correct. switch (tensor->type) { case GGML_TYPE_Q4_K: case GGML_TYPE_Q5_K: case GGML_TYPE_Q6_K: case GGML_TYPE_Q8_0: return threashold_ok && tensor->ne[1] % 64 == 0; default: break; } return threashold_ok; } static bool adreno_e17_compiler_quirks(const ggml_backend_opencl_context *backend_ctx) { if (!backend_ctx || backend_ctx->gpu_family != GPU_FAMILY::ADRENO || backend_ctx->adreno_cl_compiler_version.type != ADRENO_CL_COMPILER_TYPE::E17) { return false; } const char * env = getenv("GGML_OPENCL_ADRENO_E17_QUIRKS"); return !(env && env[0] == '0'); } inline bool use_adreno_moe_kernels(const ggml_backend_opencl_context *backend_ctx, const ggml_tensor *tensor) { // The moe weight repack kernels *_trans4_ns alias a private ushort8 through a uchar*. // Certain compilers (found with some A7x and A6x) miscompiles this, corrupting the weights. // So, exclude A6x and A7x from using Adreno MoE kernels for now. // The quants that have a general mul_mat_id kernel fallback to the general version; the // rest fallback to CPU. if (backend_ctx && (backend_ctx->adreno_gen == ADRENO_GPU_GEN::A6X || backend_ctx->adreno_gen == ADRENO_GPU_GEN::A7X || backend_ctx->adreno_gen == ADRENO_GPU_GEN::ADRENO_UNKNOWN)) { return false; } if (adreno_e17_compiler_quirks(backend_ctx)) { return false; } int ne01 = tensor->ne[1]; return (((strstr(tensor->name, "ffn") != NULL) && (strstr(tensor->name, "exps") != NULL)) || (strstr(tensor->name, "as") != NULL)) && (ne01 % 32 == 0); } // Device default for the tiled-wide lm_head/embed GEMV layout: ON for X2E and A8X. // // These kernels were previously off everywhere on the grounds that they compute // wrong values at multi-superblock K. They do not: that NMSE ~2 came from the // backend having no get_tensor restore path for the tiled layout, so // test-backend-ops (which builds its CPU reference by copying the weights back // out of the backend) compared a correct GPU result against a reference // dequantized from tiled bytes. With the restore path added, MUL_MAT passes with // the tiled kernels on, unmodified, on both devices. // // Perf, Qwen3-4B-Q4_K_M (q6_K lm_head 151936x2560), tg128, matched pairs with // alternating lead, tiled vs o4: // // A8X +11.9% 6/6 pairs positive, order bias -0.06% (16.93 vs 15.14 tok/s) // X2E +6.9% 4/4 pairs positive, order bias -0.03% (35.24 vs 32.87 tok/s) // // Measure this one on a COLD device. These kernels are far more clock-sensitive // than the o4 route they replace: on a heat-soaked A8X (CPU cap at 1.5-1.9 GHz) // tiled pins at ~14.2 tok/s while o4 still makes ~14.9, which reads as a 4-5% // LOSS and inverts the ranking. The same box, after a reboot and a gate that // waits for policy6 to return to 4396800, reports the +11.9% above with no // order bias. A7X regresses hard on this layout and stays off. // GGML_OPENCL_{Q4K,Q6K}_GEMV_TILED forces either way (=0 off, any other value on). inline bool tiled_gemv_default_on(const ggml_backend_opencl_context *backend_ctx) { return backend_ctx && (backend_ctx->adreno_gen == ADRENO_GPU_GEN::X2E || backend_ctx->adreno_gen == ADRENO_GPU_GEN::A8X); } // Tiled-wide q6_K GEMV (default OFF; GGML_OPENCL_Q6K_GEMV_TILED forces either // way: =0 off everywhere, any other value on everywhere). // Both the convert (set_tensor) and the GEMV dispatch must agree on this so the // buffer layout matches the kernel. inline bool q6k_gemv_tiled_enabled(const ggml_backend_opencl_context *backend_ctx) { static const char * e = std::getenv("GGML_OPENCL_Q6K_GEMV_TILED"); if (e && e[0] != '\0') { return e[0] != '0'; } return tiled_gemv_default_on(backend_ctx); } // Only the long-vocab lm_head/embed shapes use the tiled layout; ne01 % 64 == 0 // is required by the 64-row tiling (no row padding in the buffers). // use_adreno_kernels is required: only the Adreno GEMV path can read the tiled // layout, so converting a weight it would decline (e.g. ne00 < 512) leaves the // generic kernel reading tiled bytes as plain SOA. inline bool use_q6k_tiled(const ggml_backend_opencl_context *backend_ctx, const ggml_tensor *tensor) { return q6k_gemv_tiled_enabled(backend_ctx) && tensor->type == GGML_TYPE_Q6_K && tensor->ne[1] >= 32768 && tensor->ne[1] % 64 == 0 && use_adreno_kernels(backend_ctx, tensor); } // q4_K analog of the tiled-wide lm_head/embed GEMV (default OFF; // GGML_OPENCL_Q4K_GEMV_TILED forces either way: =0 off, else on). Same gate. inline bool q4k_gemv_tiled_enabled(const ggml_backend_opencl_context *backend_ctx) { static const char * e = std::getenv("GGML_OPENCL_Q4K_GEMV_TILED"); if (e && e[0] != '\0') { return e[0] != '0'; } return tiled_gemv_default_on(backend_ctx); } inline bool use_q4k_tiled(const ggml_backend_opencl_context *backend_ctx, const ggml_tensor *tensor) { return q4k_gemv_tiled_enabled(backend_ctx) && tensor->type == GGML_TYPE_Q4_K && tensor->ne[1] >= 32768 && tensor->ne[1] % 64 == 0 && use_adreno_kernels(backend_ctx, tensor); } inline bool enable_adreno_trans_weight(const ggml_backend_opencl_context *backend_ctx, const ggml_tensor *tensor) { bool adreno_kernel = use_adreno_kernels(backend_ctx, tensor); size_t elem_num = tensor->ne[0] * tensor->ne[1] * tensor->ne[2] * tensor->ne[3]; // The 2D weight transpose (transpose_2d_as_*) tiles rows by 4 over a 2D matrix, // so it requires K(ne0)%32==0, M(ne1)%4==0 and ne2==ne3==1. const bool shape_ok = (tensor->ne[0] % 32 == 0) && (tensor->ne[1] % 4 == 0) && (tensor->ne[2] == 1) && (tensor->ne[3] == 1); return ((elem_num < 128 * 1024 * 1024) && adreno_kernel && shape_ok); // max element num: 2**27 } inline bool enable_adreno_trans_weight_q5_K(const ggml_backend_opencl_context *backend_ctx, const ggml_tensor *tensor) { if (!use_adreno_kernels(backend_ctx, tensor)) { return false; } const size_t elem_num = ggml_nelements(tensor); const size_t q_img_width = elem_num / 8; const size_t qh_img_width = elem_num / 16; const bool shape_ok = tensor->ne[0] % 32 == 0 && tensor->ne[1] % 4 == 0 && tensor->ne[2] == 1 && tensor->ne[3] == 1; return shape_ok && q_img_width <= backend_ctx->image_max_buffer_size && qh_img_width <= backend_ctx->image_max_buffer_size; } inline bool use_q4_0_bin_kernels(const ggml_backend_opencl_context *backend_ctx, const ggml_tensor *tensor) { #ifdef GGML_OPENCL_USE_ADRENO_KERNELS if (!backend_ctx->kernel_gemv_noshuffle_q4_0_f32_32b_trans || !backend_ctx->kernel_gemm_noshuffle_q4_0_f32_32b_trans_ila_a8_bin) { return false; } return (tensor->ne[0] % 32 == 0) && (tensor->ne[1] % 64 == 0); #else GGML_UNUSED(backend_ctx); GGML_UNUSED(tensor); return false; #endif } #ifdef GGML_OPENCL_USE_ADRENO_KERNELS static bool use_fa_bin_kernels_prefill(const ggml_backend_opencl_context * backend_ctx, const ggml_tensor * q, const ggml_tensor * k, const ggml_tensor * v) { if (backend_ctx->fa.kernel_flash_attn_f32_f16_bin == nullptr) { return false; } const bool is_mixed = q->type == GGML_TYPE_F32 && k->type == GGML_TYPE_F16 && v->type == GGML_TYPE_F16; const bool is_q8_0 = q->type == GGML_TYPE_F32 && k->type == GGML_TYPE_Q8_0 && v->type == GGML_TYPE_Q8_0; const int n_q = q->ne[1]; const int dk = q->ne[0]; const int dv = v->ne[0]; constexpr bool prefill_only = true; return (backend_ctx->gpu_family == GPU_FAMILY::ADRENO && (is_mixed || is_q8_0) && (dk == dv) && (dk == 64 || dk == 128 || dk == 256 || dk == 512) && (!prefill_only || n_q != 1)); } #endif // The flat-GEMV large-m escape is OPT-IN (GGML_OPENCL_FLAT_LARGE_M=1) because it // is SLOWER than the route it replaces, not because it is unsafe. It was first // parked on the theory that it out-of-bounds-writes at vocab-scale shapes; that // was a misattribution (the test-backend-ops dst sentinel was tripped by the o4 // GEMV's unguarded tail store, fixed separately - and at the shape it was blamed // for, k=1536, this predicate returns false anyway, so the flat route never ran). // // The escape's original rationale, "gemv_noshuffle perf drops for large M", // predates the o4 kernel, which now covers the same long-vocab shapes and beats // this route on every device measured (Qwen3-4B-Q4_K_M, q6_K lm_head // 151936x2560, tg128, matched pairs vs o4): A8X -10.3% (0/3 pairs), X2E -3.7% // (0/3). Keep it reachable for shapes o4 declines, but do not default it on. static inline bool flat_large_m_enabled() { static const char * e = getenv("GGML_OPENCL_FLAT_LARGE_M"); static const bool en = e != nullptr && atoi(e) != 0; return en; } static inline bool use_flat_gemv_for_large_m_q4_K(const ggml_backend_opencl_context *backend_ctx, const ggml_tensor *tensor) { if (tensor->ne[1] % 4 != 0 && tensor->ne[2] == 1 && tensor->ne[3] == 1) { return true; } if (!flat_large_m_enabled()) { return false; } // gemv_noshuffle variant perf drops for large M, use flat variant for large M. // threshold is well above typical hidden/FFN dims, but below typical vocab sizes. // note that this forces large M weights to use LM GEMM. // EXCEPT when this branch's tiled-canonical lm_head/embed layout is active: the // weight is converted to the 64-row tiled layout, which the flat gemv would // misread as garbage. use_q4k_tiled owns these large-M weights, so defer to it. return tensor->ne[1] >= 32768 && tensor->ne[2] == 1 && tensor->ne[3] == 1 && !use_q4k_tiled(backend_ctx, tensor); } static inline bool use_flat_gemv_for_large_m_q6_K(const ggml_backend_opencl_context *backend_ctx, const ggml_tensor *tensor) { // NOTE on ordering: the ne01 % 128 escape below is a CORRECTNESS guard, not a // performance one, so it must be reachable regardless of flat_large_m_enabled(). // The opt-in gate therefore sits after it, and after the tiled deferral. // gemv_noshuffle variant perf drops for large M, use flat variant for large M. // threshold is well above typical hidden/FFN dims, but below typical vocab sizes. // q6_K flat gemv is worse for smaller K; 2048 seems to be a reasonable threshold. // note that this forces large M weights to use LM GEMM. // When this branch's tiled-canonical lm_head/embed layout is active, the weight is // converted to the 64-row tiled layout, which the flat gemv would misread as // garbage. use_q6k_tiled owns these large-M weights (it requires ne01 % 64 == 0, // so it never claims an odd-vocab weight), so defer to it first. if (use_q6k_tiled(backend_ctx, tensor)) { return false; } // The noshuffle (transposed-weight) layout packs 2 rows per 32-bit texel and the // gemv reads it with a ne01/2 texel stride and an exact-cover dispatch of // ceil(ne01/2 / 64)*64 work-items with no store guard; the gemm uses 4-row tiles. // It is therefore only correct for ne01 % 128 == 0: an odd ne01 (e.g. granitemoe // lm_head [1536, 49155] -- odd vocab) truncates the texel stride, misaligning every // odd column of the transposed layout (gross garbage) and dropping the last row; // other non-multiples over-dispatch and write past the end of dst. Route such // tensors to the flat GEMV + regular convert; the matching GEMM (ne1>1) falls back // to CPU (see supports_op). All standard even-vocab/hidden dims are multiples of // 128 and keep the noshuffle path. if ((tensor->ne[1] % 128 != 0) && tensor->ne[2] == 1 && tensor->ne[3] == 1) { return true; } if (!flat_large_m_enabled()) { return false; } // The gemv_noshuffle slowdown tracks TOTAL weight size, not ne0 alone; ne0 >= 2048 is a // proxy for "large weight" that misses a narrow-hidden vocab-scale lm_head. // Add a direct size escape so such weights also take the flat path, without changing // which weights ne0 >= 2048 already routes there. // The size escape is not taken on the A7X since its compiler miscompiles the flat K-quant GEMV return tensor->ne[1] >= 32768 && (tensor->ne[0] >= 2048 || (backend_ctx->adreno_gen != ADRENO_GPU_GEN::A7X && ggml_nbytes(tensor) >= (256ull << 20))) && tensor->ne[2] == 1 && tensor->ne[3] == 1; } inline bool use_q6_k_bin_kernels(const ggml_backend_opencl_context *backend_ctx, const ggml_tensor *tensor) { #ifdef GGML_OPENCL_USE_ADRENO_KERNELS if (!backend_ctx->kernel_gemv_noshuffle_q6_k_f32_32b_trans || !backend_ctx->kernel_gemm_noshuffle_q6_k_f32_32b_trans_ila_a8_bin) { return false; } return (tensor->ne[0] % 256 == 0) && (tensor->ne[1] % 64 == 0) && !use_q6k_tiled(backend_ctx, tensor) && !use_flat_gemv_for_large_m_q6_K(backend_ctx, tensor); #else GGML_UNUSED(backend_ctx); GGML_UNUSED(tensor); return false; #endif } inline bool use_q4_k_bin_kernels(const ggml_backend_opencl_context *backend_ctx, const ggml_tensor *tensor) { #ifdef GGML_OPENCL_USE_ADRENO_KERNELS if (!backend_ctx->kernel_gemv_noshuffle_q4_k_f32_32b_trans || !backend_ctx->kernel_gemm_noshuffle_q4_k_f32_32b_trans_ila_a8_bin) { return false; } return (tensor->ne[0] % 256 == 0) && (tensor->ne[1] % 64 == 0) && !use_q4k_tiled(backend_ctx, tensor) && !use_flat_gemv_for_large_m_q4_K(backend_ctx, tensor); #else GGML_UNUSED(backend_ctx); GGML_UNUSED(tensor); return false; #endif } inline bool use_q5_k_bin_kernels(const ggml_backend_opencl_context *backend_ctx, const ggml_tensor *tensor) { #ifdef GGML_OPENCL_USE_ADRENO_KERNELS if (!backend_ctx->kernel_gemv_noshuffle_q5_k_f32_32b_trans || !backend_ctx->kernel_gemm_noshuffle_q5_k_f32_32b_trans_ila_a8_bin) { return false; } return (tensor->ne[0] % 256 == 0) && (tensor->ne[1] % 64 == 0); #else GGML_UNUSED(backend_ctx); GGML_UNUSED(tensor); return false; #endif } static bool ggml_opencl_supports_op(ggml_backend_dev_t dev, const struct ggml_tensor * op) { ggml_backend_opencl_device_context * dev_ctx = (ggml_backend_opencl_device_context *)dev->context; ggml_backend_opencl_context * backend_ctx = dev_ctx->backend_ctx; // reject ops that match the opfilter regex if (dev_ctx->opfilter && std::regex_match(std::string(ggml_op_desc(op)), *dev_ctx->opfilter)) { return false; } switch (op->op) { case GGML_OP_NONE: return true; case GGML_OP_GET_ROWS: switch (op->src[0]->type) { case GGML_TYPE_F32: case GGML_TYPE_F16: return true; case GGML_TYPE_Q4_0: #ifdef GGML_OPENCL_SOA_Q // We do not support flattened Q4_0 (and possibly other Q's) return false; #else // GGML_OPENCL_SOA_Q return true; #endif // GGML_OPENCL_SOA_Q default: return false; } case GGML_OP_SET_ROWS: { // TODO: add support // ref: https://github.com/ggml-org/llama.cpp/pull/14274 #pragma message("TODO: implement BF16, Q4_0, Q4_1, Q5_0, Q5_1, Q8_0, IQ4_NL support (https://github.com/ggml-org/llama.cpp/pull/14661)") if (op->src[0]->type != GGML_TYPE_F32) { return false; } switch (op->type) { case GGML_TYPE_F16: case GGML_TYPE_F32: case GGML_TYPE_Q8_0: case GGML_TYPE_Q4_0: return (op->src[1]->type == GGML_TYPE_I64 || op->src[1]->type == GGML_TYPE_I32); default: return false; } } case GGML_OP_CPY: case GGML_OP_DUP: case GGML_OP_CONT: switch (op->src[0]->type) { case GGML_TYPE_F32: switch (op->type) { case GGML_TYPE_F16: case GGML_TYPE_F32: return true; default: return false; } case GGML_TYPE_F16: switch (op->type) { case GGML_TYPE_F16: case GGML_TYPE_F32: return true; default: return false; } case GGML_TYPE_I32: switch (op->type) { case GGML_TYPE_I32: return true; default: return false; } default: return false; } case GGML_OP_SET: { return (op->type == GGML_TYPE_F32 || op->type == GGML_TYPE_I32) && op->type == op->src[0]->type && op->type == op->src[1]->type; } case GGML_OP_SCALE: return op->src[0]->type == GGML_TYPE_F32 && ggml_is_contiguous(op->src[0]); case GGML_OP_ADD: if (op->type == GGML_TYPE_F16) { const bool src0_ok = op->src[0]->type == GGML_TYPE_F16 || op->src[0]->type == GGML_TYPE_F32; const bool src1_ok = op->src[1]->type == GGML_TYPE_F16 || op->src[1]->type == GGML_TYPE_F32; if (src0_ok && src1_ok) { return true; } } case GGML_OP_MUL: case GGML_OP_DIV: case GGML_OP_SUB: return (op->src[0]->type == op->src[1]->type) && (op->src[0]->type == op->type) && (op->src[0]->type == GGML_TYPE_F32 || op->src[0]->type == GGML_TYPE_F16); case GGML_OP_ADD_ID: return op->src[0]->type == GGML_TYPE_F32; case GGML_OP_SQR: case GGML_OP_SQRT: return (op->src[0]->type == GGML_TYPE_F32 || op->src[0]->type == GGML_TYPE_F16) && ggml_is_contiguous(op->src[0]); case GGML_OP_UNARY: switch (ggml_get_unary_op(op)) { case GGML_UNARY_OP_GELU: case GGML_UNARY_OP_SILU: case GGML_UNARY_OP_RELU: case GGML_UNARY_OP_GELU_ERF: case GGML_UNARY_OP_GELU_QUICK: return ggml_is_contiguous(op->src[0]) && op->src[0]->type == GGML_TYPE_F32; case GGML_UNARY_OP_SIGMOID: return ggml_is_contiguous(op->src[0]); case GGML_UNARY_OP_TANH: case GGML_UNARY_OP_NEG: case GGML_UNARY_OP_EXP: // Adreno F16 exp/expm1 overflow even post-half->float convert. return op->src[0]->type == GGML_TYPE_F32; case GGML_UNARY_OP_EXPM1: return op->src[0]->type == GGML_TYPE_F32; case GGML_UNARY_OP_ABS: case GGML_UNARY_OP_SGN: case GGML_UNARY_OP_STEP: case GGML_UNARY_OP_ELU: case GGML_UNARY_OP_HARDSWISH: case GGML_UNARY_OP_HARDSIGMOID: case GGML_UNARY_OP_FLOOR: case GGML_UNARY_OP_CEIL: case GGML_UNARY_OP_ROUND: case GGML_UNARY_OP_TRUNC: return op->src[0]->type == GGML_TYPE_F32 || op->src[0]->type == GGML_TYPE_F16; case GGML_UNARY_OP_SOFTPLUS: return op->src[0]->type == GGML_TYPE_F32 || op->src[0]->type == GGML_TYPE_F16; default: return false; } case GGML_OP_GLU: switch (ggml_get_glu_op(op)) { case GGML_GLU_OP_GEGLU: case GGML_GLU_OP_REGLU: case GGML_GLU_OP_SWIGLU: case GGML_GLU_OP_SWIGLU_OAI: case GGML_GLU_OP_GEGLU_ERF: case GGML_GLU_OP_GEGLU_QUICK: case GGML_GLU_OP_SWIGLU_CLAMP: return ggml_is_contiguous_1(op->src[0]) && (op->type == GGML_TYPE_F32 || op->type == GGML_TYPE_F16); default: return false; } case GGML_OP_TRI: return op->type == GGML_TYPE_F32 && ggml_is_contiguous(op); case GGML_OP_FILL: return op->type == GGML_TYPE_F32 && ggml_is_contiguous(op); case GGML_OP_CLAMP: return op->src[0]->type == GGML_TYPE_F32; case GGML_OP_SOFT_MAX: case GGML_OP_NORM: return true; case GGML_OP_RMS_NORM: return op->ne[0] % 4 == 0 && ggml_is_contiguous_rows(op->src[0]); case GGML_OP_L2_NORM: return ggml_is_contiguous_rows(op->src[0]); case GGML_OP_REPEAT: return op->src[0]->type == GGML_TYPE_F32 && op->type == GGML_TYPE_F32; // Assuming F32 for now, can be expanded case GGML_OP_PAD: // TODO: add circular padding support for opencl, see https://github.com/ggml-org/llama.cpp/pull/16985 if (ggml_get_op_params_i32(op, 8) != 0) { return false; } return op->src[0]->type == GGML_TYPE_F32 && op->type == GGML_TYPE_F32; case GGML_OP_UPSCALE: { ggml_scale_mode mode = (ggml_scale_mode)(ggml_get_op_params_i32(op, 0) & 0xFF); const bool antialias = (ggml_scale_mode)(ggml_get_op_params_i32(op, 0) & GGML_SCALE_FLAG_ANTIALIAS); return op->src[0]->type == GGML_TYPE_F32 && op->type == GGML_TYPE_F32 && (mode == GGML_SCALE_MODE_NEAREST || mode == GGML_SCALE_MODE_BILINEAR) && !antialias; } case GGML_OP_CONV_2D: return (op->src[0]->type == GGML_TYPE_F16 && op->src[1]->type == GGML_TYPE_F16 && op->type == GGML_TYPE_F16) || (op->src[0]->type == GGML_TYPE_F32 && op->src[1]->type == GGML_TYPE_F32 && op->type == GGML_TYPE_F32) || (op->src[0]->type == GGML_TYPE_F16 && op->src[1]->type == GGML_TYPE_F32 && op->type == GGML_TYPE_F32); case GGML_OP_SSM_CONV: return (op->src[0]->type == GGML_TYPE_F32 && op->src[1]->type == GGML_TYPE_F32 && op->type == GGML_TYPE_F32); case GGML_OP_SSM_SCAN: { if (op->type != GGML_TYPE_F32 || op->src[0]->type != GGML_TYPE_F32 || op->src[1]->type != GGML_TYPE_F32 || op->src[2]->type != GGML_TYPE_F32 || op->src[3]->type != GGML_TYPE_F32 || op->src[4]->type != GGML_TYPE_F32 || op->src[5]->type != GGML_TYPE_F32 || op->src[6]->type != GGML_TYPE_I32) { return false; } const int64_t d_state = op->src[0]->ne[0]; return d_state >= 1 && d_state <= 256 && (d_state & (d_state - 1)) == 0; } case GGML_OP_GATED_DELTA_NET: { // Match the Vulkan backend: only F32 -> F32, S_v in {16, 32, 64, 128}. if (op->src[0]->type != GGML_TYPE_F32 || op->type != GGML_TYPE_F32) { return false; } const int64_t S_v = op->src[2]->ne[0]; return S_v == 16 || S_v == 32 || S_v == 64 || S_v == 128; } case GGML_OP_CONCAT: { const ggml_type t = op->src[0]->type; return op->src[1]->type == t && op->type == t && !ggml_is_quantized(t) && ggml_blck_size(t) == 1 && (ggml_type_size(t) == 1 || ggml_type_size(t) == 2 || ggml_type_size(t) == 4 || ggml_type_size(t) == 8); } case GGML_OP_TIMESTEP_EMBEDDING: return op->src[0]->type == GGML_TYPE_F32 && op->type == GGML_TYPE_F32; case GGML_OP_GROUP_NORM: return ggml_is_contiguous(op->src[0]); case GGML_OP_MUL_MAT: if (op->src[0]->type == GGML_TYPE_F16) { return true; } else if (op->src[0]->type == GGML_TYPE_BF16) { return true; } else if (op->src[0]->type == GGML_TYPE_F32) { return op->src[1]->type == GGML_TYPE_F32; } else if (op->src[0]->type == GGML_TYPE_Q1_0) { return op->src[1]->type == GGML_TYPE_F32; } else if (op->src[0]->type == GGML_TYPE_Q4_0) { // Non-contig src0 routes through on-device dequant-to-f16. return op->src[1]->type == GGML_TYPE_F32; } else if (op->src[0]->type == GGML_TYPE_Q4_1 || op->src[0]->type == GGML_TYPE_Q5_0 || op->src[0]->type == GGML_TYPE_Q5_1 || op->src[0]->type == GGML_TYPE_MXFP4 || op->src[0]->type == GGML_TYPE_IQ4_NL || op->src[0]->type == GGML_TYPE_Q4_K || op->src[0]->type == GGML_TYPE_Q5_K || op->src[0]->type == GGML_TYPE_Q6_K) { // The E031.41 compiler (usually with A7x) miscompiles the flat K-quant // GEMV kernels (kernel_mul_mv_q*_K_f32_flat) and makes lm_head run much // slower than it should. So, make it fallback to CPU to preserve performance // for this compiler series. static const char * a7x_lmhead_env = getenv("GGML_OPENCL_A7X_LMHEAD_CPU"); static const bool a7x_lmhead_cpu = (a7x_lmhead_env == nullptr || a7x_lmhead_env[0] != '0'); if (a7x_lmhead_cpu && backend_ctx->adreno_gen == ADRENO_GPU_GEN::A7X && (op->src[0]->type == GGML_TYPE_Q4_K || op->src[0]->type == GGML_TYPE_Q5_K || op->src[0]->type == GGML_TYPE_Q6_K) && op->src[0]->ne[1] >= 32768) { // vocab-scale weight; no FFN/attn weight is this tall return false; } // The generic mul_mv (GEMV) kernels are wrong for large-batch prefill on // Adreno. A quant mul_mat only avoids the GEMV when it reaches the Adreno // trans-weight GEMM, which needs both a GEMM kernel for the type and // use_adreno_kernels(). Decline the large-N shapes that would otherwise // fall through to the GEMV. { const ggml_type t = op->src[0]->type; const bool type_has_gemm = (t == GGML_TYPE_Q4_0 || t == GGML_TYPE_Q4_1 || t == GGML_TYPE_IQ4_NL || t == GGML_TYPE_Q8_0 || t == GGML_TYPE_Q4_K || t == GGML_TYPE_Q5_K || t == GGML_TYPE_Q6_K); const bool uses_gemm = type_has_gemm && use_adreno_kernels(backend_ctx, op->src[0]); if (!uses_gemm && op->src[1]->ne[1] >= 512) { return false; } } return op->src[1]->type == GGML_TYPE_F32 && ggml_is_contiguous(op->src[0]) && ggml_is_contiguous(op->src[1]); } else if (op->src[0]->type == GGML_TYPE_Q8_0) { // ggml_cl_mul_mat_q8_0_f32_adreno now honors src1/dst view_offs (the // activation sub-buffer starts at offset1 and the kernels take offsetd), // so a broadcast q8_0 matmul (src1 batch > src0 batch, e.g. Qwen3.5-9B-UD // / Qwen3.6-35B q8_0 GDN ssm_out) runs on GPU via the per-slice broadcast // iteration in ggml_cl_mul_mat. No special-casing needed. return op->src[1]->type == GGML_TYPE_F32; } return false; case GGML_OP_MUL_MAT_ID: if (op->src[0]->type == GGML_TYPE_Q4_0 || op->src[0]->type == GGML_TYPE_Q8_0 || op->src[0]->type == GGML_TYPE_MXFP4) { if (op->src[1]->type == GGML_TYPE_F32) { return ggml_is_contiguous(op->src[0]) && ggml_is_contiguous(op->src[1]); } } // q4_0, q8_0 and mxfp4 have general MUL_MAT_ID support, // the quantizations here currently do not - they are only supported by Adreno with certain shapes if (op->src[0]->type == GGML_TYPE_Q4_1 || op->src[0]->type == GGML_TYPE_Q5_0 || op->src[0]->type == GGML_TYPE_Q5_1 || op->src[0]->type == GGML_TYPE_Q4_K || op->src[0]->type == GGML_TYPE_Q5_K || op->src[0]->type == GGML_TYPE_Q6_K) { #ifdef GGML_OPENCL_USE_ADRENO_KERNELS if (op->src[1]->type == GGML_TYPE_F32) { return use_adreno_moe_kernels(backend_ctx, op->src[0]) && ggml_is_contiguous(op->src[0]) && ggml_is_contiguous(op->src[1]); } #endif return false; } return false; case GGML_OP_RESHAPE: case GGML_OP_VIEW: case GGML_OP_PERMUTE: case GGML_OP_TRANSPOSE: return true; case GGML_OP_DIAG: return true; case GGML_OP_DIAG_MASK_INF: return op->ne[3] == 1; case GGML_OP_ROPE: { const int mode = ((const int32_t *) op->op_params)[2]; const bool is_mrope = mode & GGML_ROPE_TYPE_MROPE; const bool is_vision = mode == GGML_ROPE_TYPE_VISION; if (is_mrope && !is_vision) { if (op->src[0]->type == GGML_TYPE_F32 || op->src[0]->type == GGML_TYPE_F16) { return true; } return false; } if (is_vision) { if (op->src[0]->type == GGML_TYPE_F32 || op->src[0]->type == GGML_TYPE_F16) { return true; } return false; } return true; } case GGML_OP_SOLVE_TRI: return op->src[0]->type == GGML_TYPE_F32 && ggml_is_contiguous(op->src[0]); case GGML_OP_IM2COL: return true; case GGML_OP_ARGSORT: { load_cl_kernels_argsort(backend_ctx); cl_kernel kernel = backend_ctx->kernel_argsort_f32_i32; int max_workgroup_size = backend_ctx->get_kernel_workgroup_size(kernel); int cols = 1; while (cols < op->ne[0]) { cols *= 2; } return cols <= max_workgroup_size && op->src[0]->type == GGML_TYPE_F32; } case GGML_OP_SUM_ROWS: case GGML_OP_CUMSUM: return op->src[0]->type == GGML_TYPE_F32 && ggml_is_contiguous(op->src[0]); case GGML_OP_MEAN: return op->src[0]->type == GGML_TYPE_F32; case GGML_OP_FLASH_ATTN_EXT: { #ifdef GGML_OPENCL_USE_ADRENO_KERNELS if (use_fa_bin_kernels_prefill(backend_ctx, op->src[0], op->src[1], op->src[2])) { return true; } #endif // The E17 compilers segfault while building FA kernels, skip E17 for now if (adreno_e17_compiler_quirks(backend_ctx)) { return false; } const ggml_tensor * q = op->src[0]; const ggml_tensor * k = op->src[1]; const ggml_tensor * v = op->src[2]; const int dk = q->ne[0]; const int dv = v->ne[0]; const struct { int dk; int dv; } supported_dims[] = { { 40, 40}, { 64, 64}, { 80, 80}, { 96, 96}, {112, 112}, {128, 128}, {192, 128}, {192, 192}, {256, 256}, {512, 512}, }; bool dims_supported = false; for (size_t i = 0; i < sizeof(supported_dims)/sizeof(supported_dims[0]); ++i) { if (supported_dims[i].dk == dk && supported_dims[i].dv == dv) { dims_supported = true; break; } } if (!dims_supported) { return false; } const bool is_f32_f32 = q->type == GGML_TYPE_F32 && k->type == GGML_TYPE_F32 && v->type == GGML_TYPE_F32 && op->type == GGML_TYPE_F32; const bool is_f16_f16 = q->type == GGML_TYPE_F16 && k->type == GGML_TYPE_F16 && v->type == GGML_TYPE_F16 && op->type == GGML_TYPE_F16; const bool is_f32_f16 = q->type == GGML_TYPE_F32 && k->type == GGML_TYPE_F16 && v->type == GGML_TYPE_F16 && op->type == GGML_TYPE_F32; const bool is_f32_q8_0 = q->type == GGML_TYPE_F32 && k->type == GGML_TYPE_Q8_0 && v->type == GGML_TYPE_Q8_0 && op->type == GGML_TYPE_F32 && dk % 32 == 0 && dv % 32 == 0; const bool is_f32_q4_0 = q->type == GGML_TYPE_F32 && k->type == GGML_TYPE_Q4_0 && v->type == GGML_TYPE_Q4_0 && op->type == GGML_TYPE_F32 && dk % 32 == 0 && dv % 32 == 0; // A7X (Adreno 740, compiler E031.41) SIGSEGVs inside clBuildProgram // building the flash_attn programs whose KV path is mixed-type or // dequantized — f32_f16, q8_0, q4_0 (reproduced at DK=40 and DK=64; it // is DK-independent). It is a driver crash, not codegen-wrong-output, so // it cannot be caught in-process (fatal=false only handles clean compile // errors). The uniform f16_f16 / f32_f32 programs compile fine on this // compiler, so decline only the KV-convert variants; ggml then runs // those (f16-KV / quant-KV) attention layers on the CPU backend. // Negative compiler carve-out, same idiom as the Intel DK=512 decline // below and the X1E driver-quirk guards. if (backend_ctx && backend_ctx->adreno_gen == ADRENO_GPU_GEN::A7X && (is_f32_f16 || is_f32_q8_0 || is_f32_q4_0)) { return false; } // Asymmetric KV: host-dequants both sides to F32, uses f32 kernel. auto is_kv_type_ok = [](ggml_type t) { return t == GGML_TYPE_F16 || t == GGML_TYPE_F32 || t == GGML_TYPE_Q4_0 || t == GGML_TYPE_Q8_0; }; const bool is_f32_asym = q->type == GGML_TYPE_F32 && op->type == GGML_TYPE_F32 && k->type != v->type && is_kv_type_ok(k->type) && is_kv_type_ok(v->type); const bool kv_combo_ok = is_f32_f32 || is_f16_f16 || is_f32_f16 || is_f32_q8_0 || is_f32_q4_0 || is_f32_asym; if (!kv_combo_ok) { return false; } // Some compilers for A7x (Adreno 740, compiler E031.41) crashes when // building FA kernels with mixed or quant types (f32_f16, f32_q8_0, f32_q4_0) // Here we skip all A7x for these kernels to avoid crash if (backend_ctx->adreno_gen == ADRENO_GPU_GEN::A7X && (is_f32_f16 || is_f32_q8_0 || is_f32_q4_0)) { return false; } if (dk == 512) { if (backend_ctx->gpu_family == INTEL) { return false; } if (!is_f32_f16) { return false; } if (q->ne[1] == 1) { // DK=512 decode is bandwidth-bound and slower on the GPU // than on the CPU; decline it here so it runs on the CPU. // Prefill (n_q > 1) stays on the GPU. return false; } else { // prefill, BM-tile in its own FA_PREFILL_ONLY program if (!ggml_opencl_ensure_fa_f32_f16_prefill_512(backend_ctx, /*split=*/false)) { return false; } } } return true; } default: return false; } } // Forward declaration - implementation appears later in the file. static const char * ggml_backend_opencl_buffer_type_get_name(ggml_backend_buffer_type_t buffer_type); static ggml_guid_t ggml_backend_opencl_guid() { static ggml_guid guid = { 0xde, 0xe0, 0x70, 0xa2, 0x73, 0x4e, 0x4d, 0xbc, 0xb0, 0xc7, 0x4f, 0xd4, 0x6d, 0x4e, 0x90, 0xfe }; return &guid; } static ggml_backend_i ggml_backend_opencl_i = { /* .get_name = */ ggml_backend_opencl_name, /* .free = */ ggml_backend_opencl_free, /* .set_tensor_async = */ NULL, /* ggml_backend_opencl_set_tensor_async */ /* .get_tensor_async = */ NULL, /* ggml_backend_opencl_get_tensor_async */ /* .set_tensor_2d_async = */ NULL, /* .get_tensor_2d_async = */ NULL, /* .cpy_tensor_async = */ NULL, /* ggml_backend_opencl_cpy_tensor_async */ /* .synchronize = */ ggml_backend_opencl_synchronize, /* .graph_plan_create = */ NULL, /* .graph_plan_free = */ NULL, /* .graph_plan_update = */ NULL, /* .graph_plan_compute = */ NULL, /* .graph_compute = */ ggml_backend_opencl_graph_compute, /* .event_record = */ NULL, /* .event_wait = */ NULL, /* .graph_optimize = */ NULL, }; ggml_backend_t ggml_backend_opencl_init(void) { ggml_backend_dev_t dev = ggml_backend_reg_dev_get(ggml_backend_opencl_reg(), 0); ggml_backend_opencl_context *backend_ctx = ggml_cl_init(dev); backend_ctx->ref_count++; ggml_backend_t backend = new ggml_backend { /* .guid = */ ggml_backend_opencl_guid(), /* .iface = */ ggml_backend_opencl_i, /* .device = */ dev, /* .context = */ backend_ctx }; return backend; } bool ggml_backend_is_opencl(ggml_backend_t backend) { return backend && backend->iface.get_name == ggml_backend_opencl_name; } // // buffer // struct ggml_backend_opencl_buffer_context { // A buffer context can hold multiple cl_mem objects. This is for flattening // quantized weights and should be used with GGML_OPENCL_SMALL_ALLOC where // each tensor is allocated a separate buffer. When flattening is enabled // with small allocation, each tensor is backed by two cl_mem objects (for // quants and scales) packed into a backend_opencl_buffer. ggml_backend_opencl_buffer_context(cl_mem buf) : name("OpenCL") { buffer.push_back(buf); } ~ggml_backend_opencl_buffer_context() { for (cl_mem buf : buffer) { CL_CHECK(clReleaseMemObject(buf)); } for (cl_mem im : img) { CL_CHECK(clReleaseMemObject(im)); } // Delete all extras to trigger their destructors for (ggml_tensor_extra_cl * e : temp_tensor_extras) { delete e; } for (ggml_tensor_extra_cl * e : temp_tensor_extras_in_use) { delete e; } for (ggml_tensor_extra_cl_q4_0 * e : temp_tensor_extras_q4_0) { delete e; } for (ggml_tensor_extra_cl_q4_0 * e : temp_tensor_extras_q4_0_in_use) { delete e; } for (ggml_tensor_extra_cl_q4_1 * e : temp_tensor_extras_q4_1) { delete e; } for (ggml_tensor_extra_cl_q4_1 * e : temp_tensor_extras_q4_1_in_use) { delete e; } for (ggml_tensor_extra_cl_q5_0 * e : temp_tensor_extras_q5_0) { delete e; } for (ggml_tensor_extra_cl_q5_0 * e : temp_tensor_extras_q5_0_in_use) { delete e; } for (ggml_tensor_extra_cl_q5_1 * e : temp_tensor_extras_q5_1) { delete e; } for (ggml_tensor_extra_cl_q5_1 * e : temp_tensor_extras_q5_1_in_use) { delete e; } for (ggml_tensor_extra_cl_mxfp4 * e : temp_tensor_extras_mxfp4) { delete e; } for (ggml_tensor_extra_cl_mxfp4 * e : temp_tensor_extras_mxfp4_in_use) { delete e; } for (ggml_tensor_extra_cl_q8_0 * e : temp_tensor_extras_q8_0) { delete e; } for (ggml_tensor_extra_cl_q8_0 * e : temp_tensor_extras_q8_0_in_use) { delete e; } for (ggml_tensor_extra_cl_q1_0 * e : temp_tensor_extras_q1_0) { delete e; } for (ggml_tensor_extra_cl_q1_0 * e : temp_tensor_extras_q1_0_in_use) { delete e; } for (ggml_tensor_extra_cl_iq4_nl * e : temp_tensor_extras_iq4_nl) { delete e; } for (ggml_tensor_extra_cl_iq4_nl * e : temp_tensor_extras_iq4_nl_in_use) { delete e; } for (ggml_tensor_extra_cl_q4_K * e : temp_tensor_extras_q4_K) { delete e; } for (ggml_tensor_extra_cl_q4_K * e : temp_tensor_extras_q4_K_in_use) { delete e; } for (ggml_tensor_extra_cl_q6_K * e : temp_tensor_extras_q6_K) { delete e; } for (ggml_tensor_extra_cl_q6_K * e : temp_tensor_extras_q6_K_in_use) { delete e; } for (ggml_tensor_extra_cl_q5_K * e : temp_tensor_extras_q5_K) { delete e; } for (ggml_tensor_extra_cl_q5_K * e : temp_tensor_extras_q5_K_in_use) { delete e; } } ggml_tensor_extra_cl * ggml_opencl_alloc_temp_tensor_extra() { ggml_tensor_extra_cl * extra; if (temp_tensor_extras.empty()) { extra = new ggml_tensor_extra_cl(); } else { extra = temp_tensor_extras.back(); temp_tensor_extras.pop_back(); } temp_tensor_extras_in_use.push_back(extra); extra->reset(); return extra; } ggml_tensor_extra_cl_q1_0 * ggml_opencl_alloc_temp_tensor_extra_q1_0() { ggml_tensor_extra_cl_q1_0 * extra; if (temp_tensor_extras_q1_0.empty()) { extra = new ggml_tensor_extra_cl_q1_0(); } else { extra = temp_tensor_extras_q1_0.back(); temp_tensor_extras_q1_0.pop_back(); } temp_tensor_extras_q1_0_in_use.push_back(extra); extra->reset(); return extra; } ggml_tensor_extra_cl_q4_0 * ggml_opencl_alloc_temp_tensor_extra_q4_0() { ggml_tensor_extra_cl_q4_0 * extra; if (temp_tensor_extras_q4_0.empty()) { extra = new ggml_tensor_extra_cl_q4_0(); } else { extra = temp_tensor_extras_q4_0.back(); temp_tensor_extras_q4_0.pop_back(); } temp_tensor_extras_q4_0_in_use.push_back(extra); extra->reset(); return extra; } ggml_tensor_extra_cl_q4_1 * ggml_opencl_alloc_temp_tensor_extra_q4_1() { ggml_tensor_extra_cl_q4_1 * extra; if (temp_tensor_extras_q4_1.empty()) { extra = new ggml_tensor_extra_cl_q4_1(); } else { extra = temp_tensor_extras_q4_1.back(); temp_tensor_extras_q4_1.pop_back(); } temp_tensor_extras_q4_1_in_use.push_back(extra); extra->reset(); return extra; } ggml_tensor_extra_cl_q5_0 * ggml_opencl_alloc_temp_tensor_extra_q5_0() { ggml_tensor_extra_cl_q5_0 * extra; if (temp_tensor_extras_q5_0.empty()) { extra = new ggml_tensor_extra_cl_q5_0(); } else { extra = temp_tensor_extras_q5_0.back(); temp_tensor_extras_q5_0.pop_back(); } temp_tensor_extras_q5_0_in_use.push_back(extra); extra->reset(); return extra; } ggml_tensor_extra_cl_q5_1 * ggml_opencl_alloc_temp_tensor_extra_q5_1() { ggml_tensor_extra_cl_q5_1 * extra; if (temp_tensor_extras_q5_1.empty()) { extra = new ggml_tensor_extra_cl_q5_1(); } else { extra = temp_tensor_extras_q5_1.back(); temp_tensor_extras_q5_1.pop_back(); } temp_tensor_extras_q5_1_in_use.push_back(extra); extra->reset(); return extra; } ggml_tensor_extra_cl_mxfp4 * ggml_opencl_alloc_temp_tensor_extra_mxfp4() { ggml_tensor_extra_cl_mxfp4 * extra; if (temp_tensor_extras_mxfp4.empty()) { extra = new ggml_tensor_extra_cl_mxfp4(); } else { extra = temp_tensor_extras_mxfp4.back(); temp_tensor_extras_mxfp4.pop_back(); } temp_tensor_extras_mxfp4_in_use.push_back(extra); extra->reset(); return extra; } ggml_tensor_extra_cl_q8_0 * ggml_opencl_alloc_temp_tensor_extra_q8_0() { ggml_tensor_extra_cl_q8_0 * extra; if (temp_tensor_extras_q8_0.empty()) { extra = new ggml_tensor_extra_cl_q8_0(); } else { extra = temp_tensor_extras_q8_0.back(); temp_tensor_extras_q8_0.pop_back(); } temp_tensor_extras_q8_0_in_use.push_back(extra); extra->reset(); return extra; } ggml_tensor_extra_cl_iq4_nl * ggml_opencl_alloc_temp_tensor_extra_iq4_nl() { ggml_tensor_extra_cl_iq4_nl * extra; if (temp_tensor_extras_iq4_nl.empty()) { extra = new ggml_tensor_extra_cl_iq4_nl(); } else { extra = temp_tensor_extras_iq4_nl.back(); temp_tensor_extras_iq4_nl.pop_back(); } temp_tensor_extras_iq4_nl_in_use.push_back(extra); extra->reset(); return extra; } ggml_tensor_extra_cl_q4_K * ggml_opencl_alloc_temp_tensor_extra_q4_K() { ggml_tensor_extra_cl_q4_K * extra; if (temp_tensor_extras_q4_K.empty()) { extra = new ggml_tensor_extra_cl_q4_K(); } else { extra = temp_tensor_extras_q4_K.back(); temp_tensor_extras_q4_K.pop_back(); } temp_tensor_extras_q4_K_in_use.push_back(extra); extra->reset(); return extra; } ggml_tensor_extra_cl_q5_K * ggml_opencl_alloc_temp_tensor_extra_q5_K() { ggml_tensor_extra_cl_q5_K * extra; if (temp_tensor_extras_q5_K.empty()) { extra = new ggml_tensor_extra_cl_q5_K(); } else { extra = temp_tensor_extras_q5_K.back(); temp_tensor_extras_q5_K.pop_back(); } temp_tensor_extras_q5_K_in_use.push_back(extra); extra->reset(); return extra; } ggml_tensor_extra_cl_q6_K * ggml_opencl_alloc_temp_tensor_extra_q6_K() { ggml_tensor_extra_cl_q6_K * extra; if (temp_tensor_extras_q6_K.empty()) { extra = new ggml_tensor_extra_cl_q6_K(); } else { extra = temp_tensor_extras_q6_K.back(); temp_tensor_extras_q6_K.pop_back(); } temp_tensor_extras_q6_K_in_use.push_back(extra); extra->reset(); return extra; } void reset() { for (ggml_tensor_extra_cl * e : temp_tensor_extras_in_use) { temp_tensor_extras.push_back(e); } temp_tensor_extras_in_use.clear(); for (ggml_tensor_extra_cl_q1_0 * e : temp_tensor_extras_q1_0_in_use) { temp_tensor_extras_q1_0.push_back(e); } temp_tensor_extras_q1_0_in_use.clear(); for (ggml_tensor_extra_cl_q4_0 * e : temp_tensor_extras_q4_0_in_use) { temp_tensor_extras_q4_0.push_back(e); } temp_tensor_extras_q4_0_in_use.clear(); for (ggml_tensor_extra_cl_q4_1 * e : temp_tensor_extras_q4_1_in_use) { temp_tensor_extras_q4_1.push_back(e); } temp_tensor_extras_q4_1_in_use.clear(); for (ggml_tensor_extra_cl_q5_0 * e : temp_tensor_extras_q5_0_in_use) { temp_tensor_extras_q5_0.push_back(e); } temp_tensor_extras_q5_0_in_use.clear(); for (ggml_tensor_extra_cl_q5_1 * e : temp_tensor_extras_q5_1_in_use) { temp_tensor_extras_q5_1.push_back(e); } temp_tensor_extras_q5_1_in_use.clear(); for (ggml_tensor_extra_cl_mxfp4 * e : temp_tensor_extras_mxfp4_in_use) { temp_tensor_extras_mxfp4.push_back(e); } temp_tensor_extras_mxfp4_in_use.clear(); for (ggml_tensor_extra_cl_q8_0 * e : temp_tensor_extras_q8_0_in_use) { temp_tensor_extras_q8_0.push_back(e); } temp_tensor_extras_q8_0_in_use.clear(); for (ggml_tensor_extra_cl_iq4_nl * e : temp_tensor_extras_iq4_nl_in_use) { temp_tensor_extras_iq4_nl.push_back(e); } temp_tensor_extras_iq4_nl_in_use.clear(); for (ggml_tensor_extra_cl_q4_K * e : temp_tensor_extras_q4_K_in_use) { temp_tensor_extras_q4_K.push_back(e); } temp_tensor_extras_q4_K_in_use.clear(); for (ggml_tensor_extra_cl_q5_K * e : temp_tensor_extras_q5_K_in_use) { temp_tensor_extras_q5_K.push_back(e); } temp_tensor_extras_q5_K_in_use.clear(); for (ggml_tensor_extra_cl_q6_K * e : temp_tensor_extras_q6_K_in_use) { temp_tensor_extras_q6_K.push_back(e); } temp_tensor_extras_q6_K_in_use.clear(); q8_0_soa_tensors.clear(); q4_0_soa_tensors.clear(); } // Pools for extras. Available extras are in `temp_tensor_extras`. Extras // being used are in `temp_tensor_extras_in_use`. At the first run, new // extras get created and put in `in_use`. When the buffer is reset via // the `reset` callback, all extras in `in_use` get moved to available extras // for reuse. std::vector temp_tensor_extras; std::vector temp_tensor_extras_in_use; std::vector temp_tensor_extras_q1_0; std::vector temp_tensor_extras_q1_0_in_use; std::vector temp_tensor_extras_q4_0; std::vector temp_tensor_extras_q4_0_in_use; std::vector temp_tensor_extras_q4_1; std::vector temp_tensor_extras_q4_1_in_use; std::vector temp_tensor_extras_q5_0; std::vector temp_tensor_extras_q5_0_in_use; std::vector temp_tensor_extras_q5_1; std::vector temp_tensor_extras_q5_1_in_use; std::vector temp_tensor_extras_mxfp4; std::vector temp_tensor_extras_mxfp4_in_use; std::vector temp_tensor_extras_q8_0; std::vector temp_tensor_extras_q8_0_in_use; std::vector temp_tensor_extras_iq4_nl; std::vector temp_tensor_extras_iq4_nl_in_use; std::vector temp_tensor_extras_q4_K; std::vector temp_tensor_extras_q4_K_in_use; std::vector temp_tensor_extras_q5_K; std::vector temp_tensor_extras_q5_K_in_use; std::vector temp_tensor_extras_q6_K; std::vector temp_tensor_extras_q6_K_in_use; // q8_0 tensors with AoS->SoA layout conversion installed by set_tensor. // Two types of tensors get SOA'ed - normal weights and MoE weights. // In Q8_0's case, we only have normal weights. If we ever have Q8_0 as MoE // weights, they need to be added to this set in `set_tensors`. std::unordered_set q8_0_soa_tensors; // Same for q4_0. KV-cache q4_0 tensors are allocated but never pass // through set_tensor, so they stay AoS and aren't in this set. // In Q4_0's case, in addition to normal weights, we have MoE weights. std::unordered_set q4_0_soa_tensors; // The buffer_context is initially created by ggml_backend_buft_alloc_buffer // before any tensor is initialized (at the beginning of alloc_tensor_range). // Hence, there is always a buffer object in this vector. When each tensor is // being initialized, this original buffer object will be released if both // flattening and small allocation are enabled, and additional buffer // objects will be created in init_tensor to represent flattened quantized // weights. std::vector buffer; // These are image1d_buffer_t objects that wrap around the quants and scales. // For Q4_0 quantization, there should be two of them - one for quants and // one for scales. They should be populated only when flattening and small // allocation are enabled. std::vector img; std::string name; }; static void ggml_backend_opencl_buffer_free_buffer(ggml_backend_buffer_t buffer) { ggml_backend_opencl_buffer_context * ctx = (ggml_backend_opencl_buffer_context *) buffer->context; delete ctx; } static void * ggml_backend_opencl_buffer_get_base(ggml_backend_buffer_t buffer) { ggml_backend_opencl_device_context * dev_ctx = (ggml_backend_opencl_device_context *) buffer->buft->device->context; return (void *) (uintptr_t) dev_ctx->backend_ctx->alignment; } static enum ggml_status ggml_backend_opencl_buffer_init_tensor(ggml_backend_buffer_t buffer, ggml_tensor * tensor) { ggml_backend_opencl_buffer_context * ctx = (ggml_backend_opencl_buffer_context *) buffer->context; if (tensor->view_src != nullptr) { GGML_ASSERT(tensor->view_src->buffer->buft == buffer->buft); ggml_tensor_extra_cl * view_extra = (ggml_tensor_extra_cl *) tensor->view_src->extra; GGML_ASSERT(view_extra && "view_extra is nullptr?"); // Reuse extra of the parent tensor. The offset of this view tensor // becomes `extra->offset + view_offs` and needs to be calculated when // it is used. This changes is needed because of the change to // ggml_alloc.c in https://github.com/ggml-org/llama.cpp/pull/7640. // `buffer` passed in here will always be `tensor->buffer`. It is OK // to allocate extras from the same buffer context for ordinary // intermediate tensors. But for views into kv cache tensors, doing so // would mess up the extras used by kv cache. // Before #7640, `buffer` is for intermediate tensors, which is always // different from that of kv cache tensors. // // NB: now extra->offset no longer accounts for view_offs. // NB: this should not apply to weight tensors (for end-to-end runs, but // may apply for test-backend-ops). // FIXME: if any unexpected results are seen, double check the offset - // there could be other places that need fix. tensor->extra = view_extra; } else { { size_t offset = (char *) tensor->data - (char *) ggml_backend_opencl_buffer_get_base(buffer); ggml_tensor_extra_cl * extra = ctx->ggml_opencl_alloc_temp_tensor_extra(); extra->offset = offset; extra->data_device = ctx->buffer[0]; extra->actual_size = ggml_nbytes(tensor); tensor->extra = extra; } } return GGML_STATUS_SUCCESS; } // Allocate a temporary upload buffer of `nbytes` and populate it with `data` // from host. On Adreno X1-85 the device-only pool intermittently fails to // allocate at hundreds of MB once model weights fragment the heap (observed // on Qwen3.5-9B output.weight Q6_K at 834 MB). Three-step retry: // 1. CL_MEM_READ_WRITE alloc + clEnqueueWriteBuffer (normal fast path). // 2. clFinish + retry (drains in-flight allocs that may be holding heap; // mirrors the proven pattern at the FD-split partial buffer alloc). // 3. CL_MEM_ALLOC_HOST_PTR + map(WRITE_INVALIDATE) + memcpy + unmap — // different memory pool (host-pinned); true zero-copy on Adreno per // QCOM guidance. (CL_MEM_USE_HOST_PTR is NOT zero-copy on Adreno: the // driver triggers an internal copy because arbitrary host pages aren't // guaranteed mappable/coherent, AND it draws from the same exhausted // device pool — so it doesn't solve the problem.) // Returns the ready-to-read buffer (caller must clReleaseMemObject) or NULL // if all three strategies fail. The buffer is opaque to the caller — it can // be passed as a kernel argument like any normal cl_mem. static cl_mem ggml_cl_create_temp_upload_buffer( cl_context context, cl_command_queue queue, size_t nbytes, const void * data, const char * tensor_name_for_log) { cl_int err; cl_mem buf = clCreateBuffer(context, CL_MEM_READ_WRITE, nbytes, NULL, &err); if (err != CL_SUCCESS) { clFinish(queue); buf = clCreateBuffer(context, CL_MEM_READ_WRITE, nbytes, NULL, &err); } if (err == CL_SUCCESS) { const cl_int werr = clEnqueueWriteBuffer(queue, buf, CL_TRUE, 0, nbytes, data, 0, NULL, NULL); if (werr == CL_SUCCESS) { return buf; } clReleaseMemObject(buf); } buf = clCreateBuffer(context, CL_MEM_READ_ONLY | CL_MEM_ALLOC_HOST_PTR | CL_MEM_HOST_WRITE_ONLY, nbytes, NULL, &err); if (err != CL_SUCCESS) { return NULL; } void * mapped = clEnqueueMapBuffer(queue, buf, CL_TRUE, CL_MAP_WRITE_INVALIDATE_REGION, 0, nbytes, 0, NULL, NULL, &err); if (err != CL_SUCCESS) { clReleaseMemObject(buf); return NULL; } memcpy(mapped, data, nbytes); const cl_int uerr = clEnqueueUnmapMemObject(queue, buf, mapped, 0, NULL, NULL); if (uerr != CL_SUCCESS) { clReleaseMemObject(buf); return NULL; } if (tensor_name_for_log) { GGML_LOG_INFO("ggml_opencl: %s (%.1f MiB) — device alloc failed, using CL_MEM_ALLOC_HOST_PTR fallback\n", tensor_name_for_log, nbytes / 1024.0 / 1024.0); } return buf; } // Allocate a temporary download buffer of `nbytes`. The caller runs a kernel // that writes into it, then reads it back to host via clEnqueueReadBuffer (or // equivalent). Mirrors ggml_cl_create_temp_upload_buffer; the host-pinned // fallback flags are flipped (CL_MEM_WRITE_ONLY | HOST_READ_ONLY) and the // helper doesn't populate the buffer. static cl_mem ggml_cl_create_temp_download_buffer( cl_context context, cl_command_queue queue, size_t nbytes, const char * tensor_name_for_log) { cl_int err; cl_mem buf = clCreateBuffer(context, CL_MEM_READ_WRITE, nbytes, NULL, &err); if (err != CL_SUCCESS) { clFinish(queue); buf = clCreateBuffer(context, CL_MEM_READ_WRITE, nbytes, NULL, &err); } if (err == CL_SUCCESS) { return buf; } buf = clCreateBuffer(context, CL_MEM_WRITE_ONLY | CL_MEM_ALLOC_HOST_PTR | CL_MEM_HOST_READ_ONLY, nbytes, NULL, &err); if (err != CL_SUCCESS) { return NULL; } if (tensor_name_for_log) { GGML_LOG_INFO("ggml_opencl: %s download (%.1f MiB) — device alloc failed, using CL_MEM_ALLOC_HOST_PTR fallback\n", tensor_name_for_log, nbytes / 1024.0 / 1024.0); } return buf; } static void ggml_backend_opencl_buffer_set_tensor(ggml_backend_buffer_t buffer, ggml_tensor * tensor, const void * data, size_t offset, size_t size) { ggml_backend_opencl_device_context * dev_ctx = (ggml_backend_opencl_device_context *) buffer->buft->device->context; ggml_backend_opencl_context * backend_ctx = dev_ctx->backend_ctx; cl_context context = backend_ctx->context; cl_command_queue queue = backend_ctx->queue; #ifdef GGML_OPENCL_SOA_Q if (tensor->type == GGML_TYPE_Q1_0) { ggml_tensor_extra_cl * extra_orig = (ggml_tensor_extra_cl *)tensor->extra; GGML_ASSERT(extra_orig && "Tesnors in OpenCL backend should have been allocated and initialized"); // Allocate the new extra and create aliases from the original. ggml_backend_opencl_buffer_context * ctx = (ggml_backend_opencl_buffer_context *) buffer->context; ggml_tensor_extra_cl_q1_0 * extra = ctx->ggml_opencl_alloc_temp_tensor_extra_q1_0(); // q1_0 block = ggml_half d + (QK1_0/8) quant bytes = 2 + 16 = 18 bytes size_t size_d = ggml_nelements(tensor)/ggml_blck_size(tensor->type)*sizeof(ggml_fp16_t); size_t size_q = ggml_nelements(tensor)/ggml_blck_size(tensor->type)*(ggml_blck_size(tensor->type)/8); GGML_ASSERT(size_d + size_q == ggml_nbytes(tensor) && "Incorrect tensor size"); cl_int err; cl_mem data_device = clCreateBuffer(context, CL_MEM_READ_WRITE, ggml_nbytes(tensor), NULL, &err); CL_CHECK(err); CL_CHECK(clEnqueueWriteBuffer( queue, data_device, CL_TRUE, 0, ggml_nbytes(tensor), data, 0, NULL, NULL)); // The original tensor memory is divided into scales and quants, i.e., // we first store scales, then quants. cl_buffer_region region; // Create subbuffer for scales. region.origin = align_to(extra_orig->offset + tensor->view_offs + offset, backend_ctx->alignment); region.size = size_d; extra->d = clCreateSubBuffer( extra_orig->data_device, CL_MEM_READ_WRITE, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err); CL_CHECK(err); auto previous_origin = region.origin; // Create subbuffer for quants. region.origin = align_to(previous_origin + size_d, backend_ctx->alignment); region.size = size_q; extra->q = clCreateSubBuffer( extra_orig->data_device, CL_MEM_READ_WRITE, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err); CL_CHECK(err); cl_kernel kernel = backend_ctx->kernel_convert_block_q1_0; CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &data_device)); CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra->q)); CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra->d)); size_t global_work_size[] = {(size_t)ggml_nelements(tensor)/ggml_blck_size(tensor->type), 1, 1}; size_t local_work_size[] = {64, 1, 1}; cl_event evt; CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, global_work_size, local_work_size, 0, NULL, &evt)); CL_CHECK(clWaitForEvents(1, &evt)); CL_CHECK(clReleaseMemObject(data_device)); tensor->extra = extra; // q is uint32 (32 sign bits each); d is one half per 128-block. #ifdef GGML_OPENCL_USE_ADRENO_KERNELS if (enable_adreno_trans_weight(backend_ctx, tensor)) { int M = tensor->ne[1]; // ne01 int K = tensor->ne[0]; // ne00 GGML_ASSERT(K % 128 == 0); GGML_ASSERT(M % 4 == 0); GGML_ASSERT(tensor->ne[2] == 1); GGML_ASSERT(tensor->ne[3] == 1); transpose_2d_as_32b(backend_ctx, extra->q, extra->q, size_q, K/32, M); transpose_2d_as_16b(backend_ctx, extra->d, extra->d, size_d, K/128, M); } // end transpose #endif // GGML_OPENCL_USE_ADRENO_KERNELS return; } // We separate the quantized bits and scale from block_q4_0 by using an // additional kernel, where each thread handles a block. We first read the // original weights into a temporary buffer, then create two separate // buffers for quantized bits and scales, which are then populated by the // conversion kernel. if (tensor->type == GGML_TYPE_Q4_0) { // Views can't SoA-ify here — parent owns the layout (see q8_0 guard). if (tensor->view_src != nullptr || !ggml_is_contiguous(tensor)) { return; } // Tensors should have been preallocated, therefore they should // already have ggml_tensor_extra_cl as extra. ggml_tensor_extra_cl * extra_orig = (ggml_tensor_extra_cl *)tensor->extra; GGML_ASSERT(extra_orig && "Tesnors in OpenCL backend should have been allocated and initialized"); // Allocate the new extra and create aliases from the original. ggml_backend_opencl_buffer_context * ctx = (ggml_backend_opencl_buffer_context *) buffer->context; ggml_tensor_extra_cl_q4_0 * extra = ctx->ggml_opencl_alloc_temp_tensor_extra_q4_0(); size_t size_d = ggml_nelements(tensor)/ggml_blck_size(tensor->type)*sizeof(ggml_fp16_t); size_t size_q = ggml_nelements(tensor)/ggml_blck_size(tensor->type)*ggml_blck_size(tensor->type)/2; GGML_ASSERT(size_d + size_q == ggml_nbytes(tensor) && "Incorrect tensor size"); cl_int err; cl_mem data_device = ggml_cl_create_temp_upload_buffer(context, queue, ggml_nbytes(tensor), data, tensor->name); GGML_ASSERT(data_device != NULL && "set_tensor: temp upload buffer alloc failed"); // We consider the specified offset arg as always, although For weights // the offset arg should be 0 (we do not assert this). //GGML_ASSERT(offset == 0); // We create subbuffers from the original tensor buffer for scales and // quants - i.e., scales and quants are aliases into the buffer object // that backs the original tensor. This is a cleaner way to adapt to the // new memory management. // In the old code, we allocate new buffers for scales and quants // respectively, which could still be done but would result in double // allocation; properly deallocating the preallocated buffer that backs // the tensors is tricky and would leak the backend specific information // into the general backend code. // Does this create misaligned subbuffers (alignment is 1024) in certain // cases ? cl_buffer_region region; // The original tensor memory is divided into scales and quants, i.e., // we first store scales, then quants. // Create subbuffer for scales. region.origin = align_to(extra_orig->offset + tensor->view_offs + offset, backend_ctx->alignment); region.size = size_d; extra->d = clCreateSubBuffer( extra_orig->data_device, CL_MEM_READ_WRITE, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err); CL_CHECK(err); auto previous_origin = region.origin; // Create subbuffer for quants. region.origin = align_to(previous_origin + size_d, backend_ctx->alignment); region.size = size_q; extra->q = clCreateSubBuffer( extra_orig->data_device, CL_MEM_READ_WRITE, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err); CL_CHECK(err); #ifdef GGML_OPENCL_USE_ADRENO_KERNELS // Adreno moe q4_0 kernel needs special transpose and unshuffling if (use_adreno_moe_kernels(backend_ctx, tensor)) { cl_kernel kernel = backend_ctx->kernel_convert_block_q4_0_trans4_ns; int ne00 = tensor->ne[0]; int ne01 = tensor->ne[1]; int ne02 = tensor->ne[2]; CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &data_device)); CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra->q)); CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra->d)); CL_CHECK(clSetKernelArg(kernel, 3, sizeof(int), &ne00)); CL_CHECK(clSetKernelArg(kernel, 4, sizeof(int), &ne01)); size_t global_work_size[3] = {static_cast(((ne01 + 63) / 64) * 64), static_cast(ne00 / 32), static_cast(ne02)}; size_t local_work_size[3] = {64, 2, 1}; cl_event evt; CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, global_work_size, local_work_size, 0, NULL, &evt)); CL_CHECK(clWaitForEvents(1, &evt)); CL_CHECK(clReleaseMemObject(data_device)); // Create image for Q cl_image_format img_format_q = {CL_R, CL_UNSIGNED_INT32}; cl_image_desc img_desc_q = { CL_MEM_OBJECT_IMAGE1D_BUFFER, static_cast(ggml_nelements(tensor) / 8), 0, 0, 0, 0, 0, 0, 0, { extra->q } }; extra->q_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_format_q, &img_desc_q, NULL, &err); tensor->extra = extra; // MoE tensors are also SOA'ed ctx->q4_0_soa_tensors.insert(tensor); return; } #endif // GGML_OPENCL_USE_ADRENO_KERNELS #ifdef GGML_OPENCL_USE_ADRENO_KERNELS cl_kernel kernel = backend_ctx->kernel_convert_block_q4_0; // The optimized kernels need weights in natural order, so unshuffle. if (use_adreno_kernels(backend_ctx, tensor)) { kernel = backend_ctx->kernel_convert_block_q4_0_noshuffle; } #else cl_kernel kernel = backend_ctx->kernel_convert_block_q4_0; #endif // GGML_OPENCL_USE_ADRENO_KERNELS CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &data_device)); CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra->q)); CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra->d)); size_t global_work_size[] = {(size_t)ggml_nelements(tensor)/ggml_blck_size(tensor->type), 1, 1}; size_t local_work_size[] = {64, 1, 1}; cl_event evt; CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, global_work_size, local_work_size, 0, NULL, &evt)); CL_CHECK(clWaitForEvents(1, &evt)); CL_CHECK(clReleaseMemObject(data_device)); tensor->extra = extra; ctx->q4_0_soa_tensors.insert(tensor); // transpose the weights and scales #ifdef GGML_OPENCL_USE_ADRENO_KERNELS // Only do transpose for large, non batched matrix // TODO: use preallocated images instead of sub-buffer then image if (use_adreno_kernels(backend_ctx, tensor)) { int M = tensor->ne[1]; int K = tensor->ne[0]; GGML_ASSERT(K % 32 == 0); if (use_q4_0_bin_kernels(backend_ctx, tensor)) { cl_int err; cl_image_format wimg_fmt; cl_image_desc wimg_desc; // transpose quants as 32-bit words (M-first) GGML_ASSERT(M % 64 == 0); transpose_2d_as_32b(backend_ctx, extra->q, extra->q, size_q, K / 8, M); transpose_2d_as_16b(backend_ctx, extra->d, extra->d, size_d, K / 32, M); wimg_fmt = { CL_R, CL_UNSIGNED_INT32 }; memset(&wimg_desc, 0, sizeof(wimg_desc)); wimg_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; wimg_desc.image_width = (size_t)M * K / 8; wimg_desc.buffer = extra->q; CL_CHECK((extra->q_img = clCreateImage(context, CL_MEM_READ_ONLY, &wimg_fmt, &wimg_desc, NULL, &err), err)); wimg_fmt = { CL_R, CL_HALF_FLOAT }; memset(&wimg_desc, 0, sizeof(wimg_desc)); wimg_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; wimg_desc.image_width = (size_t)M * K / 32; wimg_desc.buffer = extra->d; CL_CHECK((extra->d_img = clCreateImage(context, CL_MEM_READ_ONLY, &wimg_fmt, &wimg_desc, NULL, &err), err)); } else { // Transpose q and d as ushort transpose_2d_as_16b(backend_ctx, extra->q, extra->q, size_q, K/4, M); transpose_2d_as_16b(backend_ctx, extra->d, extra->d, size_d, K/32, M); } } #endif // GGML_OPENCL_USE_ADRENO_KERNELS return; } if (tensor->type == GGML_TYPE_Q4_1) { ggml_tensor_extra_cl * extra_orig = (ggml_tensor_extra_cl *)tensor->extra; GGML_ASSERT(extra_orig && "Tesnors in OpenCL backend should have been allocated and initialized"); // Allocate the new extra and create aliases from the original. ggml_backend_opencl_buffer_context * ctx = (ggml_backend_opencl_buffer_context *) buffer->context; ggml_tensor_extra_cl_q4_1 * extra = ctx->ggml_opencl_alloc_temp_tensor_extra_q4_1(); size_t size_d = ggml_nelements(tensor)/ggml_blck_size(tensor->type)*sizeof(ggml_fp16_t); size_t size_m = ggml_nelements(tensor)/ggml_blck_size(tensor->type)*sizeof(ggml_fp16_t); size_t size_q = ggml_nelements(tensor)/ggml_blck_size(tensor->type)*ggml_blck_size(tensor->type)/2; GGML_ASSERT(size_d + size_m + size_q == ggml_nbytes(tensor) && "Incorrect tensor size"); cl_int err; cl_mem data_device = ggml_cl_create_temp_upload_buffer(context, queue, ggml_nbytes(tensor), data, tensor->name); GGML_ASSERT(data_device != NULL && "set_tensor: temp upload buffer alloc failed"); cl_buffer_region region; // The original tensor memory is divided into scales and quants, i.e., // we first store scales, mins, then quants. // Create subbuffer for scales. region.origin = align_to(extra_orig->offset + tensor->view_offs + offset, backend_ctx->alignment); region.size = size_d; extra->d = clCreateSubBuffer( extra_orig->data_device, CL_MEM_READ_WRITE, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err); CL_CHECK(err); auto previous_origin = region.origin; // Create subbuffer for mins. region.origin = align_to(previous_origin + size_d, backend_ctx->alignment); region.size = size_m; extra->m = clCreateSubBuffer( extra_orig->data_device, CL_MEM_READ_WRITE, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err); CL_CHECK(err); previous_origin = region.origin; // Create subbuffer for quants. region.origin = align_to(previous_origin + size_m, backend_ctx->alignment); region.size = size_q; extra->q = clCreateSubBuffer( extra_orig->data_device, CL_MEM_READ_WRITE, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err); CL_CHECK(err); #ifdef GGML_OPENCL_USE_ADRENO_KERNELS // Adreno moe q4_1 kernel needs special transpose and unshuffling if (use_adreno_moe_kernels(backend_ctx, tensor)) { cl_kernel kernel = backend_ctx->kernel_convert_block_q4_1_trans4_ns; int ne00 = tensor->ne[0]; int ne01 = tensor->ne[1]; int ne02 = tensor->ne[2]; CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &data_device)); CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra->q)); CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra->d)); CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra->m)); CL_CHECK(clSetKernelArg(kernel, 4, sizeof(int), &ne00)); CL_CHECK(clSetKernelArg(kernel, 5, sizeof(int), &ne01)); size_t global_work_size[3] = {static_cast(((ne01 + 63) / 64) * 64), static_cast(ne00 / 32), static_cast(ne02)}; size_t local_work_size[3] = {64, 2, 1}; cl_event evt; CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, global_work_size, local_work_size, 0, NULL, &evt)); CL_CHECK(clWaitForEvents(1, &evt)); CL_CHECK(clReleaseMemObject(data_device)); // Create image for Q cl_image_format img_format_q = {CL_R, CL_UNSIGNED_INT32}; cl_image_desc img_desc_q = { CL_MEM_OBJECT_IMAGE1D_BUFFER, static_cast(ggml_nelements(tensor) / 8), 0, 0, 0, 0, 0, 0, 0, { extra->q } }; extra->q_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_format_q, &img_desc_q, NULL, &err); tensor->extra = extra; return; } #endif // GGML_OPENCL_USE_ADRENO_KERNELS // normal q4_1 repack #ifdef GGML_OPENCL_USE_ADRENO_KERNELS cl_kernel kernel = backend_ctx->kernel_convert_block_q4_1; if (use_adreno_kernels(backend_ctx, tensor)) { kernel = backend_ctx->kernel_convert_block_q4_1_noshuffle; } #else cl_kernel kernel = backend_ctx->kernel_convert_block_q4_1; #endif // GGML_OPENCL_USE_ADRENO_KERNELS CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &data_device)); CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra->q)); CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra->d)); CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra->m)); size_t global_work_size[] = {(size_t)ggml_nelements(tensor)/ggml_blck_size(tensor->type), 1, 1}; size_t local_work_size[] = {64, 1, 1}; cl_event evt; CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, global_work_size, local_work_size, 0, NULL, &evt)); CL_CHECK(clWaitForEvents(1, &evt)); CL_CHECK(clReleaseMemObject(data_device)); tensor->extra = extra; #ifdef GGML_OPENCL_USE_ADRENO_KERNELS if (use_adreno_kernels(backend_ctx, tensor)) { int M = tensor->ne[1]; int K = tensor->ne[0]; GGML_ASSERT(K % 32 == 0); // Transpose q as ushort transpose_2d_as_16b(backend_ctx, extra->q, extra->q, size_q, K/4, M); // Transpose d as ushort transpose_2d_as_16b(backend_ctx, extra->d, extra->d, size_d, K/32, M); // Transpose m as ushort transpose_2d_as_16b(backend_ctx, extra->m, extra->m, size_m, K/32, M); } #endif // GGML_OPENCL_USE_ADRENO_KERNELS return; } if (tensor->type == GGML_TYPE_Q5_0) { ggml_tensor_extra_cl * extra_orig = (ggml_tensor_extra_cl *)tensor->extra; GGML_ASSERT(extra_orig && "Tesnors in OpenCL backend should have been allocated and initialized"); // Allocate the new extra and create aliases from the original. ggml_backend_opencl_buffer_context * ctx = (ggml_backend_opencl_buffer_context *) buffer->context; ggml_tensor_extra_cl_q5_0 * extra = ctx->ggml_opencl_alloc_temp_tensor_extra_q5_0(); size_t size_d = ggml_nelements(tensor)/ggml_blck_size(tensor->type)*sizeof(ggml_fp16_t); size_t size_qs = ggml_nelements(tensor)/ggml_blck_size(tensor->type)*ggml_blck_size(tensor->type)/2; size_t size_qh = ggml_nelements(tensor)/ggml_blck_size(tensor->type)*sizeof(int32_t); GGML_ASSERT(size_d + size_qs + size_qh == ggml_nbytes(tensor) && "Incorrect tensor size"); cl_int err; cl_mem data_device = ggml_cl_create_temp_upload_buffer(context, queue, ggml_nbytes(tensor), data, tensor->name); GGML_ASSERT(data_device != NULL && "set_tensor: temp upload buffer alloc failed"); cl_buffer_region region; // Create subbuffer for scales. region.origin = align_to(extra_orig->offset + tensor->view_offs + offset, backend_ctx->alignment); region.size = size_d; extra->d = clCreateSubBuffer( extra_orig->data_device, CL_MEM_READ_WRITE, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err); CL_CHECK(err); auto previous_origin = region.origin; // Create subbuffer for qh. region.origin = align_to(previous_origin + size_d, backend_ctx->alignment); region.size = size_qh; extra->qh = clCreateSubBuffer( extra_orig->data_device, CL_MEM_READ_WRITE, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err); CL_CHECK(err); previous_origin = region.origin; // Create subbuffer for qs. region.origin = align_to(previous_origin + size_qh, backend_ctx->alignment); region.size = size_qs; extra->qs = clCreateSubBuffer( extra_orig->data_device, CL_MEM_READ_WRITE, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err); CL_CHECK(err); #ifdef GGML_OPENCL_USE_ADRENO_KERNELS // Adreno moe q5_0 kernel needs special transpose and unshuffling if (use_adreno_moe_kernels(backend_ctx, tensor)) { cl_kernel kernel = backend_ctx->kernel_convert_block_q5_0_trans4_ns; int ne00 = tensor->ne[0]; int ne01 = tensor->ne[1]; int ne02 = tensor->ne[2]; CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &data_device)); CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra->qs)); CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra->qh)); CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra->d)); CL_CHECK(clSetKernelArg(kernel, 4, sizeof(int), &ne00)); CL_CHECK(clSetKernelArg(kernel, 5, sizeof(int), &ne01)); size_t global_work_size[3] = {static_cast(((ne01 + 63) / 64) * 64), static_cast(ne00 / 32), static_cast(ne02)}; size_t local_work_size[3] = {64, 2, 1}; cl_event evt; CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, global_work_size, local_work_size, 0, NULL, &evt)); CL_CHECK(clWaitForEvents(1, &evt)); CL_CHECK(clReleaseMemObject(data_device)); // Create image for Q cl_image_format img_format_qs = {CL_R, CL_UNSIGNED_INT32}; cl_image_desc img_desc_qs = { CL_MEM_OBJECT_IMAGE1D_BUFFER, static_cast(ggml_nelements(tensor) / 8), 0, 0, 0, 0, 0, 0, 0, { extra->qs } }; extra->qs_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_format_qs, &img_desc_qs, NULL, &err); tensor->extra = extra; // Generic dp4a MoE path { static const char * q5dp4a_env = getenv("GGML_OPENCL_Q5_MOE_DP4A"); const bool q5dp4a = q5dp4a_env ? (atoi(q5dp4a_env) != 0) : (backend_ctx->adreno_gen == ADRENO_GPU_GEN::X2E); if (q5dp4a && ne02 > 1 && (ne00 % 32 == 0)) { size_t nb32 = (size_t)ne00 / 32; size_t sc_elems = (size_t)ne02 * ne01 * nb32 * 2; size_t mn_elems = (size_t)ne02 * ne01 * nb32; extra->scale = clCreateBuffer(context, CL_MEM_READ_WRITE, sc_elems * sizeof(cl_half), NULL, &err); CL_CHECK(err); extra->min = clCreateBuffer(context, CL_MEM_READ_WRITE, mn_elems * sizeof(cl_half), NULL, &err); CL_CHECK(err); cl_kernel ek = backend_ctx->kernel_moe_expand_scale_q5_0; CL_CHECK(clSetKernelArg(ek, 0, sizeof(cl_mem), &extra->d)); CL_CHECK(clSetKernelArg(ek, 1, sizeof(cl_mem), &extra->scale)); CL_CHECK(clSetKernelArg(ek, 2, sizeof(cl_mem), &extra->min)); CL_CHECK(clSetKernelArg(ek, 3, sizeof(int), &ne00)); CL_CHECK(clSetKernelArg(ek, 4, sizeof(int), &ne01)); size_t eg[3] = { (size_t)(((ne01 + 63) / 64) * 64), nb32, (size_t)ne02 }; size_t el[3] = { 64, 1, 1 }; cl_event evt; CL_CHECK(clEnqueueNDRangeKernel(queue, ek, 3, NULL, eg, el, 0, NULL, &evt)); CL_CHECK(clWaitForEvents(1, &evt)); } } return; } #endif // GGML_OPENCL_USE_ADRENO_KERNELS #ifdef GGML_OPENCL_USE_ADRENO_KERNELS if (use_adreno_kernels(backend_ctx, tensor)) { cl_kernel kernel = backend_ctx->kernel_convert_block_q5_0_noshuffle; CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &data_device)); CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra->qs)); CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra->qh)); CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra->d)); size_t global_work_size[] = {(size_t)ggml_nelements(tensor)/ggml_blck_size(tensor->type), 1, 1}; size_t local_work_size[] = {64, 1, 1}; cl_event evt; CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, global_work_size, local_work_size, 0, NULL, &evt)); CL_CHECK(clWaitForEvents(1, &evt)); CL_CHECK(clReleaseMemObject(data_device)); tensor->extra = extra; int M = tensor->ne[1]; int K = tensor->ne[0]; GGML_ASSERT(K % 32 == 0); // Transpose qs as ushort transpose_2d_as_16b(backend_ctx, extra->qs, extra->qs, size_qs, K/4, M); // Transpose qh as uchar transpose_2d_as_8b(backend_ctx, extra->qh, extra->qh, size_qh, K/8, M); // Transpose d as ushort transpose_2d_as_16b(backend_ctx, extra->d, extra->d, size_d, K/32, M); return; } #endif // GGML_OPENCL_USE_ADRENO_KERNELS cl_kernel kernel = backend_ctx->kernel_convert_block_q5_0; cl_ulong n_blk = ggml_nelements(tensor)/ggml_blck_size(tensor->type); CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &data_device)); CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra->qs)); CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra->qh)); CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra->d)); CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_ulong), &n_blk)); size_t global_work_size[] = {(size_t)CEIL_DIV(n_blk, 64) * 64, 1, 1}; size_t local_work_size[] = {64, 1, 1}; cl_event evt; CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, global_work_size, local_work_size, 0, NULL, &evt)); CL_CHECK(clWaitForEvents(1, &evt)); CL_CHECK(clReleaseMemObject(data_device)); tensor->extra = extra; return; } if (tensor->type == GGML_TYPE_Q5_1) { ggml_tensor_extra_cl * extra_orig = (ggml_tensor_extra_cl *)tensor->extra; GGML_ASSERT(extra_orig && "Tesnors in OpenCL backend should have been allocated and initialized"); // Allocate the new extra and create aliases from the original. ggml_backend_opencl_buffer_context * ctx = (ggml_backend_opencl_buffer_context *) buffer->context; ggml_tensor_extra_cl_q5_1 * extra = ctx->ggml_opencl_alloc_temp_tensor_extra_q5_1(); size_t size_d = ggml_nelements(tensor)/ggml_blck_size(tensor->type)*sizeof(ggml_fp16_t); size_t size_m = ggml_nelements(tensor)/ggml_blck_size(tensor->type)*sizeof(ggml_fp16_t); size_t size_qs = ggml_nelements(tensor)/ggml_blck_size(tensor->type)*ggml_blck_size(tensor->type)/2; size_t size_qh = ggml_nelements(tensor)/ggml_blck_size(tensor->type)*sizeof(int32_t); GGML_ASSERT(size_d + size_m + size_qs + size_qh == ggml_nbytes(tensor) && "Incorrect tensor size"); cl_int err; cl_mem data_device = ggml_cl_create_temp_upload_buffer(context, queue, ggml_nbytes(tensor), data, tensor->name); GGML_ASSERT(data_device != NULL && "set_tensor: temp upload buffer alloc failed"); cl_buffer_region region; // The original tensor memory is divided into scales and quants, i.e., // we first store scales, mins, then quants. // Create subbuffer for scales. region.origin = align_to(extra_orig->offset + tensor->view_offs + offset, backend_ctx->alignment); region.size = size_d; extra->d = clCreateSubBuffer( extra_orig->data_device, CL_MEM_READ_WRITE, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err); CL_CHECK(err); auto previous_origin = region.origin; // Create subbuffer for mins. region.origin = align_to(previous_origin + size_d, backend_ctx->alignment); region.size = size_m; extra->m = clCreateSubBuffer( extra_orig->data_device, CL_MEM_READ_WRITE, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err); CL_CHECK(err); previous_origin = region.origin; // Create subbuffer for qh. region.origin = align_to(previous_origin + size_m, backend_ctx->alignment); region.size = size_qh; extra->qh = clCreateSubBuffer( extra_orig->data_device, CL_MEM_READ_WRITE, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err); CL_CHECK(err); previous_origin = region.origin; // Create subbuffer for qs. region.origin = align_to(previous_origin + size_qh, backend_ctx->alignment); region.size = size_qs; extra->qs = clCreateSubBuffer( extra_orig->data_device, CL_MEM_READ_WRITE, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err); CL_CHECK(err); #ifdef GGML_OPENCL_USE_ADRENO_KERNELS // Adreno moe q5_1 kernel needs special transpose and unshuffling if (use_adreno_moe_kernels(backend_ctx, tensor)) { cl_kernel kernel = backend_ctx->kernel_convert_block_q5_1_trans4_ns; int ne00 = tensor->ne[0]; int ne01 = tensor->ne[1]; int ne02 = tensor->ne[2]; CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &data_device)); CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra->qs)); CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra->qh)); CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra->d)); CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extra->m)); CL_CHECK(clSetKernelArg(kernel, 5, sizeof(int), &ne00)); CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne01)); size_t global_work_size[3] = {static_cast(((ne01 + 63) / 64) * 64), static_cast(ne00 / 32), static_cast(ne02)}; size_t local_work_size[3] = {64, 2, 1}; cl_event evt; CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, global_work_size, local_work_size, 0, NULL, &evt)); CL_CHECK(clWaitForEvents(1, &evt)); CL_CHECK(clReleaseMemObject(data_device)); // Create image for Q cl_image_format img_format_qs = {CL_R, CL_UNSIGNED_INT32}; cl_image_desc img_desc_qs = { CL_MEM_OBJECT_IMAGE1D_BUFFER, static_cast(ggml_nelements(tensor) / 8), 0, 0, 0, 0, 0, 0, 0, { extra->qs } }; extra->qs_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_format_qs, &img_desc_qs, NULL, &err); tensor->extra = extra; return; } #endif // GGML_OPENCL_USE_ADRENO_KERNELS #ifdef GGML_OPENCL_USE_ADRENO_KERNELS if (use_adreno_kernels(backend_ctx, tensor)) { cl_kernel kernel = backend_ctx->kernel_convert_block_q5_1_noshuffle; CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &data_device)); CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra->qs)); CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra->qh)); CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra->d)); CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extra->m)); size_t global_work_size[] = {(size_t)ggml_nelements(tensor)/ggml_blck_size(tensor->type), 1, 1}; size_t local_work_size[] = {64, 1, 1}; cl_event evt; CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, global_work_size, local_work_size, 0, NULL, &evt)); CL_CHECK(clWaitForEvents(1, &evt)); CL_CHECK(clReleaseMemObject(data_device)); tensor->extra = extra; int M = tensor->ne[1]; int K = tensor->ne[0]; GGML_ASSERT(K % 32 == 0); // Transpose qs as ushort transpose_2d_as_16b(backend_ctx, extra->qs, extra->qs, size_qs, K/4, M); // Transpose qh as uchar transpose_2d_as_8b(backend_ctx, extra->qh, extra->qh, size_qh, K/8, M); // Transpose d as ushort transpose_2d_as_16b(backend_ctx, extra->d, extra->d, size_d, K/32, M); // Transpose m as ushort transpose_2d_as_16b(backend_ctx, extra->m, extra->m, size_m, K/32, M); return; } #endif // GGML_OPENCL_USE_ADRENO_KERNELS cl_kernel kernel = backend_ctx->kernel_convert_block_q5_1; cl_ulong n_blk = ggml_nelements(tensor)/ggml_blck_size(tensor->type); CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &data_device)); CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra->qs)); CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra->qh)); CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra->d)); CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extra->m)); CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &n_blk)); size_t global_work_size[] = {(size_t)CEIL_DIV(n_blk, 64) * 64, 1, 1}; size_t local_work_size[] = {64, 1, 1}; cl_event evt; CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, global_work_size, local_work_size, 0, NULL, &evt)); CL_CHECK(clWaitForEvents(1, &evt)); CL_CHECK(clReleaseMemObject(data_device)); tensor->extra = extra; return; } if (tensor->type == GGML_TYPE_MXFP4) { ggml_tensor_extra_cl * extra_orig = (ggml_tensor_extra_cl *)tensor->extra; GGML_ASSERT(extra_orig && "Tesnors in OpenCL backend should have been allocated and initialized"); // Allocate the new extra and create aliases from the original. ggml_backend_opencl_buffer_context * ctx = (ggml_backend_opencl_buffer_context *) buffer->context; ggml_tensor_extra_cl_mxfp4 * extra = ctx->ggml_opencl_alloc_temp_tensor_extra_mxfp4(); size_t size_e = ggml_nelements(tensor)/ggml_blck_size(tensor->type)*sizeof(char); size_t size_q = ggml_nelements(tensor)/ggml_blck_size(tensor->type)*ggml_blck_size(tensor->type)/2; GGML_ASSERT(size_e + size_q == ggml_nbytes(tensor) && "Incorrect tensor size"); cl_int err; cl_mem data_device = ggml_cl_create_temp_upload_buffer(context, queue, ggml_nbytes(tensor), data, tensor->name); GGML_ASSERT(data_device != NULL && "set_tensor: temp upload buffer alloc failed"); // The original tensor memory is divided into scales and quants, i.e., // we first store scales, then quants. cl_buffer_region region; // Create subbuffer for scales. region.origin = align_to(extra_orig->offset + tensor->view_offs + offset, backend_ctx->alignment); region.size = size_e; extra->e = clCreateSubBuffer( extra_orig->data_device, CL_MEM_READ_WRITE, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err); CL_CHECK(err); auto previous_origin = region.origin; // Create subbuffer for quants. region.origin = align_to(previous_origin + size_e, backend_ctx->alignment); region.size = size_q; extra->q = clCreateSubBuffer( extra_orig->data_device, CL_MEM_READ_WRITE, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err); CL_CHECK(err); #ifdef GGML_OPENCL_USE_ADRENO_KERNELS // Adreno moe mxfp4 kernel needs special transpose and unshuffling if (use_adreno_moe_kernels(backend_ctx, tensor)) { cl_kernel kernel = backend_ctx->kernel_convert_block_mxfp4_trans4_ns; int ne00 = tensor->ne[0]; int ne01 = tensor->ne[1]; int ne02 = tensor->ne[2]; CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &data_device)); CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra->q)); CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra->e)); CL_CHECK(clSetKernelArg(kernel, 3, sizeof(int), &ne00)); CL_CHECK(clSetKernelArg(kernel, 4, sizeof(int), &ne01)); size_t global_work_size[3] = {static_cast(((ne01 + 63) / 64) * 64), static_cast(ne00 / 32), static_cast(ne02)}; size_t local_work_size[3] = {64, 2, 1}; cl_event evt; CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, global_work_size, local_work_size, 0, NULL, &evt)); CL_CHECK(clWaitForEvents(1, &evt)); CL_CHECK(clReleaseMemObject(data_device)); tensor->extra = extra; // Create image for Q cl_image_format img_format_q = {CL_R, CL_UNSIGNED_INT32}; cl_image_desc img_desc_q = { CL_MEM_OBJECT_IMAGE1D_BUFFER, static_cast(ggml_nelements(tensor) / 8), 0, 0, 0, 0, 0, 0, 0, { extra->q } }; extra->q_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_format_q, &img_desc_q, NULL, &err); tensor->extra = extra; return; } #endif // GGML_OPENCL_USE_ADRENO_KERNELS cl_kernel kernel = backend_ctx->kernel_convert_block_mxfp4; CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &data_device)); CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra->q)); CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra->e)); size_t global_work_size[3] = {(size_t)ggml_nelements(tensor)/ggml_blck_size(tensor->type), 1, 1}; size_t local_work_size[3] = {64, 1, 1}; cl_event evt; CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, global_work_size, local_work_size, 0, NULL, &evt)); CL_CHECK(clWaitForEvents(1, &evt)); CL_CHECK(clReleaseMemObject(data_device)); // Create image for Q cl_image_format img_format_q = {CL_RG, CL_UNSIGNED_INT32}; cl_image_desc img_desc_q = { CL_MEM_OBJECT_IMAGE1D_BUFFER, static_cast(ggml_nelements(tensor)/32*2), 0, 0, 0, 0, 0, 0, 0, { extra->q } }; extra->q_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_format_q, &img_desc_q, NULL, &err); tensor->extra = extra; return; } if (tensor->type == GGML_TYPE_Q8_0) { // Views share the parent's buffer; parent owns SoA conversion. if (tensor->view_src != nullptr || !ggml_is_contiguous(tensor)) { return; } ggml_tensor_extra_cl * extra_orig = (ggml_tensor_extra_cl *)tensor->extra; GGML_ASSERT(extra_orig && "Tesnors in OpenCL backend should have been allocated and initialized"); // Allocate the new extra and create aliases from the original. ggml_backend_opencl_buffer_context * ctx = (ggml_backend_opencl_buffer_context *) buffer->context; ggml_tensor_extra_cl_q8_0 * extra = ctx->ggml_opencl_alloc_temp_tensor_extra_q8_0(); size_t size_d = ggml_nelements(tensor)/ggml_blck_size(tensor->type)*sizeof(ggml_fp16_t); size_t size_q = ggml_nelements(tensor)/ggml_blck_size(tensor->type)*(ggml_blck_size(tensor->type)*sizeof(char)); GGML_ASSERT(size_d + size_q == ggml_nbytes(tensor) && "Incorrect tensor size"); cl_int err; cl_mem data_device = ggml_cl_create_temp_upload_buffer(context, queue, ggml_nbytes(tensor), data, tensor->name); GGML_ASSERT(data_device != NULL && "set_tensor: temp upload buffer alloc failed"); // The original tensor memory is divided into scales and quants, i.e., // we first store scales, then quants. cl_buffer_region region; // Create subbuffer for scales. region.origin = align_to(extra_orig->offset + tensor->view_offs + offset, backend_ctx->alignment); region.size = size_d; extra->d = clCreateSubBuffer( extra_orig->data_device, CL_MEM_READ_WRITE, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err); CL_CHECK(err); auto previous_origin = region.origin; // Create subbuffer for quants. region.origin = align_to(previous_origin + size_d, backend_ctx->alignment); region.size = size_q; extra->q = clCreateSubBuffer( extra_orig->data_device, CL_MEM_READ_WRITE, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err); CL_CHECK(err); cl_kernel kernel = backend_ctx->kernel_convert_block_q8_0; CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &data_device)); CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra->q)); CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra->d)); size_t global_work_size[] = {(size_t)ggml_nelements(tensor)/ggml_blck_size(tensor->type), 1, 1}; size_t local_work_size[] = {64, 1, 1}; cl_event evt; CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, global_work_size, local_work_size, 0, NULL, &evt)); CL_CHECK(clWaitForEvents(1, &evt)); CL_CHECK(clReleaseMemObject(data_device)); tensor->extra = extra; ctx->q8_0_soa_tensors.insert(tensor); // Generic dp4a MoE path (opt-in GGML_OPENCL_Q8_MOE_DP4A) #ifdef GGML_OPENCL_USE_ADRENO_KERNELS { static const char * q8dp4a_env = getenv("GGML_OPENCL_Q8_MOE_DP4A"); const bool q8dp4a = q8dp4a_env ? (atoi(q8dp4a_env) != 0) : (backend_ctx->adreno_gen == ADRENO_GPU_GEN::X2E); if (q8dp4a && tensor->ne[2] > 1 && (tensor->ne[0] % 32 == 0)) { int ne00 = (int)tensor->ne[0]; int ne01 = (int)tensor->ne[1]; int ne02 = (int)tensor->ne[2]; size_t nb32 = (size_t)ne00 / 32; size_t scale_elems = (size_t)ne02 * ne01 * nb32 * 2; // 2 per-16-seg scales / 32-block extra->scale = clCreateBuffer(context, CL_MEM_READ_WRITE, scale_elems * sizeof(cl_half), NULL, &err); CL_CHECK(err); cl_kernel ek = backend_ctx->kernel_moe_expand_scale_q8_0; CL_CHECK(clSetKernelArg(ek, 0, sizeof(cl_mem), &extra->d)); CL_CHECK(clSetKernelArg(ek, 1, sizeof(cl_mem), &extra->scale)); CL_CHECK(clSetKernelArg(ek, 2, sizeof(int), &ne00)); CL_CHECK(clSetKernelArg(ek, 3, sizeof(int), &ne01)); size_t eg[3] = { (size_t)(((ne01 + 63) / 64) * 64), nb32, (size_t)ne02 }; size_t el[3] = { 64, 1, 1 }; cl_event evt; CL_CHECK(clEnqueueNDRangeKernel(queue, ek, 3, NULL, eg, el, 0, NULL, &evt)); CL_CHECK(clWaitForEvents(1, &evt)); } } #endif // Transpose the weights and scales #ifdef GGML_OPENCL_USE_ADRENO_KERNELS if (enable_adreno_trans_weight(backend_ctx, tensor)) { int M = tensor->ne[1]; // ne01 int K = tensor->ne[0]; // ne00 GGML_ASSERT(K % 32 == 0); GGML_ASSERT(M % 4 == 0); GGML_ASSERT(tensor->ne[2] == 1); GGML_ASSERT(tensor->ne[3] == 1); transpose_2d_as_32b(backend_ctx, extra->q, extra->q, size_q, K/4, M); transpose_2d_as_16b(backend_ctx, extra->d, extra->d, size_d, K/32, M); } // end transpose #endif // GGML_OPENCL_USE_ADRENO_KERNELS return; } if (tensor->type == GGML_TYPE_IQ4_NL) { ggml_tensor_extra_cl * extra_orig = (ggml_tensor_extra_cl *)tensor->extra; GGML_ASSERT(extra_orig && "Tensors in OpenCL backend should have been allocated and initialized"); ggml_backend_opencl_buffer_context * ctx = (ggml_backend_opencl_buffer_context *) buffer->context; ggml_tensor_extra_cl_iq4_nl * extra = ctx->ggml_opencl_alloc_temp_tensor_extra_iq4_nl(); size_t size_d = ggml_nelements(tensor)/ggml_blck_size(tensor->type)*sizeof(ggml_fp16_t); size_t size_q = ggml_nelements(tensor)/ggml_blck_size(tensor->type)*(ggml_blck_size(tensor->type)/2); GGML_ASSERT(size_d + size_q == ggml_nbytes(tensor) && "Incorrect tensor size"); cl_int err; cl_mem data_device = ggml_cl_create_temp_upload_buffer(context, queue, ggml_nbytes(tensor), data, tensor->name); GGML_ASSERT(data_device != NULL && "set_tensor: temp upload buffer alloc failed"); cl_buffer_region region; // Create subbuffer for scales. region.origin = align_to(extra_orig->offset + tensor->view_offs + offset, backend_ctx->alignment); region.size = size_d; extra->d = clCreateSubBuffer( extra_orig->data_device, CL_MEM_READ_WRITE, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err); CL_CHECK(err); auto previous_origin = region.origin; // Create subbuffer for quants. region.origin = align_to(previous_origin + size_d, backend_ctx->alignment); region.size = size_q; extra->q = clCreateSubBuffer( extra_orig->data_device, CL_MEM_READ_WRITE, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err); CL_CHECK(err); #ifdef GGML_OPENCL_USE_ADRENO_KERNELS cl_kernel kernel = backend_ctx->kernel_convert_block_iq4_nl; if (use_adreno_kernels(backend_ctx, tensor)) { kernel = backend_ctx->kernel_convert_block_iq4_nl_noshuffle; } #else cl_kernel kernel = backend_ctx->kernel_convert_block_iq4_nl; #endif cl_ulong n_blk = ggml_nelements(tensor)/ggml_blck_size(tensor->type); cl_uchar mask_0F = 0x0F; cl_uchar mask_F0 = 0xF0; CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &data_device)); CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra->q)); CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra->d)); CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_uchar), &mask_0F)); CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_uchar), &mask_F0)); CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &n_blk)); size_t global_work_size[] = {(size_t)CEIL_DIV(n_blk, 64)*64, 1, 1}; size_t local_work_size[] = {64, 1, 1}; cl_event evt; CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, global_work_size, local_work_size, 0, NULL, &evt)); CL_CHECK(clWaitForEvents(1, &evt)); CL_CHECK(clReleaseMemObject(data_device)); tensor->extra = extra; #ifdef GGML_OPENCL_USE_ADRENO_KERNELS if (use_adreno_kernels(backend_ctx, tensor)) { int M = tensor->ne[1]; int K = tensor->ne[0]; GGML_ASSERT(K % 32 == 0); // Transpose q as ushort transpose_2d_as_16b(backend_ctx, extra->q, extra->q, size_q, K/4, M); // Transpose d as ushort transpose_2d_as_16b(backend_ctx, extra->d, extra->d, size_d, K/32, M); } #endif return; } if (tensor->type == GGML_TYPE_Q4_K) { ggml_tensor_extra_cl * extra_orig = (ggml_tensor_extra_cl *)tensor->extra; GGML_ASSERT(extra_orig && "Tesnors in OpenCL backend should have been allocated and initialized"); // Allocate the new extra and create aliases from the original. ggml_backend_opencl_buffer_context * ctx = (ggml_backend_opencl_buffer_context *) buffer->context; ggml_tensor_extra_cl_q4_K * extra = ctx->ggml_opencl_alloc_temp_tensor_extra_q4_K(); size_t size_d = ggml_nelements(tensor)/ggml_blck_size(tensor->type)*sizeof(ggml_fp16_t); size_t size_dm = ggml_nelements(tensor)/ggml_blck_size(tensor->type)*sizeof(ggml_fp16_t); size_t size_s = ggml_nelements(tensor)/ggml_blck_size(tensor->type)*(3 * ggml_blck_size(tensor->type) / 64); size_t size_q = ggml_nelements(tensor)/ggml_blck_size(tensor->type)*ggml_blck_size(tensor->type)/2; GGML_ASSERT(size_d + size_dm + size_s + size_q == ggml_nbytes(tensor) && "Incorrect tensor size"); cl_int err; cl_mem data_device = ggml_cl_create_temp_upload_buffer(context, queue, ggml_nbytes(tensor), data, tensor->name); GGML_ASSERT(data_device != NULL && "q4_K set_tensor: temp upload buffer alloc failed"); cl_buffer_region region; // Create subbuffer for d. region.origin = align_to(extra_orig->offset + tensor->view_offs + offset, backend_ctx->alignment); region.size = size_d; extra->d = clCreateSubBuffer( extra_orig->data_device, CL_MEM_READ_WRITE, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err); CL_CHECK(err); auto previous_origin = region.origin; // Create subbuffer for mins. region.origin = align_to(previous_origin + size_d, backend_ctx->alignment); region.size = size_dm; extra->dm = clCreateSubBuffer( extra_orig->data_device, CL_MEM_READ_WRITE, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err); CL_CHECK(err); previous_origin = region.origin; // Create subbuffer for s. region.origin = align_to(previous_origin + size_dm, backend_ctx->alignment); region.size = size_s; extra->s = clCreateSubBuffer( extra_orig->data_device, CL_MEM_READ_WRITE, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err); CL_CHECK(err); previous_origin = region.origin; // Create subbuffer for quants. region.origin = align_to(previous_origin + size_s, backend_ctx->alignment); region.size = size_q; extra->q = clCreateSubBuffer( extra_orig->data_device, CL_MEM_READ_WRITE, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err); CL_CHECK(err); #ifdef GGML_OPENCL_USE_ADRENO_KERNELS if (use_adreno_moe_kernels(backend_ctx, tensor)) { cl_kernel kernel = backend_ctx->kernel_convert_block_q4_k_trans4_ns; int ne00 = tensor->ne[0]; int ne01 = tensor->ne[1]; int ne02 = tensor->ne[2]; cl_uchar mask_0F = 0x0F; cl_uchar mask_F0 = 0xF0; CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &data_device)); CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra->q)); CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra->d)); CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra->dm)); CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extra->s)); CL_CHECK(clSetKernelArg(kernel, 5, sizeof(int), &ne00)); CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne01)); CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_uchar), &mask_0F)); CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_uchar), &mask_F0)); size_t global_work_size[] = {static_cast(((ne01 + 63) / 64) * 64), static_cast(ne00 / 256), static_cast(ne02)}; size_t local_work_size[] = {64, 1, 1}; cl_event evt; CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, global_work_size, local_work_size, 0, NULL, &evt)); CL_CHECK(clWaitForEvents(1, &evt)); CL_CHECK(clReleaseMemObject(data_device)); cl_image_format img_format_q = {CL_R, CL_UNSIGNED_INT32}; cl_image_desc img_desc_q = { CL_MEM_OBJECT_IMAGE1D_BUFFER, static_cast(ggml_nelements(tensor) / 8), 0, 0, 0, 0, 0, 0, 0, { extra->q } }; extra->q_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_format_q, &img_desc_q, NULL, &err); CL_CHECK(err); tensor->extra = extra; return; } #endif // GGML_OPENCL_USE_ADRENO_KERNELS #ifdef GGML_OPENCL_USE_ADRENO_KERNELS // Tiled-wide convert for the long-vocab lm_head/embed (opt-in). The embed/ // output q4_K weight (token_embd.weight, ne1=vocab) is NOT matched by // use_adreno_moe_kernels, so it lands here in the general branch. Produce // the final 64-row-tiled canonical layout directly into q/d/dm/s (buffer // sizes already match), read back by kernel_gemv_noshuffle_q4_k_f32_tiled. if (use_q4k_tiled(backend_ctx, tensor)) { cl_kernel tk = backend_ctx->kernel_convert_block_q4_k_tiled_ns; int ne00 = tensor->ne[0]; int ne01 = tensor->ne[1]; int ne02 = tensor->ne[2]; CL_CHECK(clSetKernelArg(tk, 0, sizeof(cl_mem), &data_device)); CL_CHECK(clSetKernelArg(tk, 1, sizeof(cl_mem), &extra->q)); CL_CHECK(clSetKernelArg(tk, 2, sizeof(cl_mem), &extra->d)); CL_CHECK(clSetKernelArg(tk, 3, sizeof(cl_mem), &extra->dm)); CL_CHECK(clSetKernelArg(tk, 4, sizeof(cl_mem), &extra->s)); CL_CHECK(clSetKernelArg(tk, 5, sizeof(int), &ne00)); CL_CHECK(clSetKernelArg(tk, 6, sizeof(int), &ne01)); size_t gws[] = {static_cast(((ne01 + 63) / 64) * 64), static_cast(ne00 / 256), static_cast(ne02)}; size_t lws[] = {64, 1, 1}; cl_event tevt; CL_CHECK(clEnqueueNDRangeKernel(queue, tk, 3, NULL, gws, lws, 0, NULL, &tevt)); CL_CHECK(clWaitForEvents(1, &tevt)); CL_CHECK(clReleaseMemObject(data_device)); extra->q_img = nullptr; tensor->extra = extra; return; } cl_kernel kernel = backend_ctx->kernel_convert_block_q4_K; if (use_adreno_kernels(backend_ctx, tensor) && !use_flat_gemv_for_large_m_q4_K(backend_ctx, tensor)) { kernel = backend_ctx->kernel_convert_block_q4_K_noshuffle; } #else cl_kernel kernel = backend_ctx->kernel_convert_block_q4_K; #endif // GGML_OPENCL_USE_ADRENO_KERNELS cl_uchar mask_0F = 0x0F; cl_uchar mask_F0 = 0xF0; CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &data_device)); CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra->q)); CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra->s)); CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra->d)); CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extra->dm)); CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_uchar), &mask_0F)); CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_uchar), &mask_F0)); size_t global_work_size[] = {(size_t)ggml_nelements(tensor)/ggml_blck_size(tensor->type), 1, 1}; size_t local_work_size[] = {64, 1, 1}; cl_event evt; CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, global_work_size, local_work_size, 0, NULL, &evt)); CL_CHECK(clWaitForEvents(1, &evt)); CL_CHECK(clReleaseMemObject(data_device)); tensor->extra = extra; #ifdef GGML_OPENCL_USE_ADRENO_KERNELS if (use_adreno_kernels(backend_ctx, tensor) && !use_flat_gemv_for_large_m_q4_K(backend_ctx, tensor)) { int M = tensor->ne[1]; int K = tensor->ne[0]; GGML_ASSERT(K % 32 == 0); if (use_q4_k_bin_kernels(backend_ctx, tensor)) { cl_int err; cl_image_format wimg_fmt; cl_image_desc wimg_desc; // transpose quants as 32-bit words (M-first) GGML_ASSERT(M % 64 == 0); transpose_2d_as_32b(backend_ctx, extra->q, extra->q, size_q, K/8, M); wimg_fmt = { CL_R, CL_UNSIGNED_INT32 }; memset(&wimg_desc, 0, sizeof(wimg_desc)); wimg_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; wimg_desc.image_width = (size_t)M * K / 8; wimg_desc.buffer = extra->q; CL_CHECK((extra->q_img = clCreateImage(context, CL_MEM_READ_ONLY, &wimg_fmt, &wimg_desc, NULL, &err), err)); } else { // Transpose q as ushort transpose_2d_as_16b(backend_ctx, extra->q, extra->q, size_q, K/4, M); } transpose_2d_as_16b(backend_ctx, extra->d, extra->d, size_d, K/256, M); transpose_2d_as_16b(backend_ctx, extra->dm, extra->dm, size_dm, K/256, M); // Transpose s as uchar transpose_2d_as_8b(backend_ctx, extra->s, extra->s, size_s, K/256*12, M, true, true); } #endif // GGML_OPENCL_USE_ADRENO_KERNELS return; } if (tensor->type == GGML_TYPE_Q5_K) { ggml_tensor_extra_cl * extra_orig = (ggml_tensor_extra_cl *)tensor->extra; GGML_ASSERT(extra_orig && "Tesnors in OpenCL backend should have been allocated and initialized"); // Allocate the new extra and create aliases from the original. ggml_backend_opencl_buffer_context * ctx = (ggml_backend_opencl_buffer_context *) buffer->context; ggml_tensor_extra_cl_q5_K * extra = ctx->ggml_opencl_alloc_temp_tensor_extra_q5_K(); size_t size_q = ggml_nelements(tensor)/ggml_blck_size(tensor->type)*ggml_blck_size(tensor->type)/2; size_t size_qh = ggml_nelements(tensor)/ggml_blck_size(tensor->type)*ggml_blck_size(tensor->type)/8; size_t size_s = ggml_nelements(tensor)/ggml_blck_size(tensor->type)*(3*ggml_blck_size(tensor->type)/64); size_t size_d = ggml_nelements(tensor)/ggml_blck_size(tensor->type)*sizeof(ggml_fp16_t); size_t size_dm = ggml_nelements(tensor)/ggml_blck_size(tensor->type)*sizeof(ggml_fp16_t); GGML_ASSERT(size_q + size_qh + size_s + size_d + size_dm == ggml_nbytes(tensor) && "Incorrect tensor size"); cl_int err; cl_mem data_device = ggml_cl_create_temp_upload_buffer(context, queue, ggml_nbytes(tensor), data, tensor->name); GGML_ASSERT(data_device != NULL && "q5_K set_tensor: temp upload buffer alloc failed"); cl_buffer_region region; // Create subbuffer for d. region.origin = align_to(extra_orig->offset + tensor->view_offs + offset, backend_ctx->alignment); region.size = size_d; extra->d = clCreateSubBuffer( extra_orig->data_device, CL_MEM_READ_WRITE, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err); CL_CHECK(err); auto previous_origin = region.origin; // Create subbuffer for dm. region.origin = align_to(previous_origin + size_d, backend_ctx->alignment); region.size = size_dm; extra->dm = clCreateSubBuffer( extra_orig->data_device, CL_MEM_READ_WRITE, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err); CL_CHECK(err); previous_origin = region.origin; // Create subbuffer for s. region.origin = align_to(previous_origin + size_dm, backend_ctx->alignment); region.size = size_s; extra->s = clCreateSubBuffer( extra_orig->data_device, CL_MEM_READ_WRITE, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err); CL_CHECK(err); previous_origin = region.origin; // Create subbuffer for q (lower 4 bits) region.origin = align_to(previous_origin + size_s, backend_ctx->alignment); region.size = size_q; extra->q = clCreateSubBuffer( extra_orig->data_device, CL_MEM_READ_WRITE, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err); CL_CHECK(err); previous_origin = region.origin; // Create subbuffer for qh (upper 1 bit) region.origin = align_to(previous_origin + size_q, backend_ctx->alignment); region.size = size_qh; CL_CHECK((extra->qh = clCreateSubBuffer(extra_orig->data_device, CL_MEM_READ_WRITE, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); CL_CHECK(err); #ifdef GGML_OPENCL_USE_ADRENO_KERNELS if (use_adreno_moe_kernels(backend_ctx, tensor)) { cl_kernel kernel = backend_ctx->kernel_convert_block_q5_k_trans4_ns; int ne00 = tensor->ne[0]; int ne01 = tensor->ne[1]; int ne02 = tensor->ne[2]; cl_uchar mask_0F = 0x0F; cl_uchar mask_F0 = 0xF0; CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &data_device)); CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra->q)); CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra->qh)); CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra->d)); CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extra->dm)); CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_mem), &extra->s)); CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne00)); CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne01)); CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_uchar), &mask_0F)); CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_uchar), &mask_F0)); size_t global_work_size[] = {static_cast(((ne01 + 63) / 64) * 64), static_cast(ne00 / 256), static_cast(ne02)}; size_t local_work_size[] = {64, 1, 1}; cl_event evt; CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, global_work_size, local_work_size, 0, NULL, &evt)); CL_CHECK(clWaitForEvents(1, &evt)); CL_CHECK(clReleaseMemObject(data_device)); cl_image_format img_format_q = {CL_R, CL_UNSIGNED_INT32}; cl_image_desc img_desc_q = { CL_MEM_OBJECT_IMAGE1D_BUFFER, static_cast(ggml_nelements(tensor) / 8), 0, 0, 0, 0, 0, 0, 0, { extra->q } }; extra->q_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_format_q, &img_desc_q, NULL, &err); CL_CHECK(err); tensor->extra = extra; // Generic dp4a MoE path { static const char * q5kdp4a_env = getenv("GGML_OPENCL_Q5K_MOE_DP4A"); const bool q5kdp4a = q5kdp4a_env ? (atoi(q5kdp4a_env) != 0) : (backend_ctx->adreno_gen == ADRENO_GPU_GEN::X2E); if (q5kdp4a && ne02 > 1 && (ne00 % 256 == 0)) { size_t nb32 = (size_t)ne00 / 32; size_t sc_elems = (size_t)ne02 * ne01 * nb32 * 2; size_t mn_elems = (size_t)ne02 * ne01 * nb32; extra->scale = clCreateBuffer(context, CL_MEM_READ_WRITE, sc_elems * sizeof(cl_half), NULL, &err); CL_CHECK(err); extra->min = clCreateBuffer(context, CL_MEM_READ_WRITE, mn_elems * sizeof(cl_half), NULL, &err); CL_CHECK(err); cl_kernel ek = backend_ctx->kernel_moe_expand_scale_q5_K; CL_CHECK(clSetKernelArg(ek, 0, sizeof(cl_mem), &extra->s)); CL_CHECK(clSetKernelArg(ek, 1, sizeof(cl_mem), &extra->d)); CL_CHECK(clSetKernelArg(ek, 2, sizeof(cl_mem), &extra->dm)); CL_CHECK(clSetKernelArg(ek, 3, sizeof(cl_mem), &extra->scale)); CL_CHECK(clSetKernelArg(ek, 4, sizeof(cl_mem), &extra->min)); CL_CHECK(clSetKernelArg(ek, 5, sizeof(int), &ne00)); CL_CHECK(clSetKernelArg(ek, 6, sizeof(int), &ne01)); size_t eg[3] = { (size_t)(((ne01 + 63) / 64) * 64), (size_t)(ne00 / 256), (size_t)ne02 }; size_t el[3] = { 64, 1, 1 }; cl_event evt; CL_CHECK(clEnqueueNDRangeKernel(queue, ek, 3, NULL, eg, el, 0, NULL, &evt)); CL_CHECK(clWaitForEvents(1, &evt)); } } return; } #endif // GGML_OPENCL_USE_ADRENO_KERNELS #ifdef GGML_OPENCL_USE_ADRENO_KERNELS cl_kernel kernel = backend_ctx->kernel_convert_block_q5_K; if (enable_adreno_trans_weight_q5_K(backend_ctx, tensor)) { kernel = backend_ctx->kernel_convert_block_q5_K_noshuffle; } #else cl_kernel kernel = backend_ctx->kernel_convert_block_q5_K; #endif cl_uchar mask_0F = 0x0F; cl_uchar mask_F0 = 0xF0; CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &data_device)); CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra->q)); CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra->qh)); CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra->s)); CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extra->d)); CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_mem), &extra->dm)); CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_uchar), &mask_0F)); CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_uchar), &mask_F0)); size_t global_work_size[] = {(size_t)ggml_nelements(tensor)/ggml_blck_size(tensor->type), 1, 1}; size_t local_work_size[] = {64, 1, 1}; cl_event evt; CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, global_work_size, local_work_size, 0, NULL, &evt)); CL_CHECK(clWaitForEvents(1, &evt)); CL_CHECK(clReleaseMemObject(data_device)); extra->size_q = size_q; extra->size_qh = size_qh; extra->size_s = size_s; extra->size_d = size_d; extra->size_dm = size_dm; tensor->extra = extra; #ifdef GGML_OPENCL_USE_ADRENO_KERNELS if (enable_adreno_trans_weight_q5_K(backend_ctx, tensor)) { int M = tensor->ne[1]; int K = tensor->ne[0]; GGML_ASSERT(K % 32 == 0); if (use_q5_k_bin_kernels(backend_ctx, tensor)) { cl_int err; cl_image_format wimg_fmt; cl_image_desc wimg_desc; // transpose q as 32-bit words (M-first); qh/d/dm stay in their existing layout // (both new ILA kernels read qh via the existing [K/8][M] uchar plane directly). GGML_ASSERT(M % 64 == 0); transpose_2d_as_32b(backend_ctx, extra->q, extra->q, size_q, K/8, M); wimg_fmt = { CL_R, CL_UNSIGNED_INT32 }; memset(&wimg_desc, 0, sizeof(wimg_desc)); wimg_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; wimg_desc.image_width = (size_t)M * K / 8; wimg_desc.buffer = extra->q; CL_CHECK((extra->q_img = clCreateImage(context, CL_MEM_READ_ONLY, &wimg_fmt, &wimg_desc, NULL, &err), err)); // Transpose s as uchar transpose_2d_as_8b(backend_ctx, extra->s, extra->s, size_s, K/256*12, M, true, true); } else { // Transpose q as ushort transpose_2d_as_16b(backend_ctx, extra->q, extra->q, size_q, K/4, M); } transpose_2d_as_8b (backend_ctx, extra->qh, extra->qh, size_qh, K/8, M); transpose_2d_as_16b(backend_ctx, extra->d, extra->d, size_d, K/256, M); transpose_2d_as_16b(backend_ctx, extra->dm, extra->dm, size_dm, K/256, M); } #endif // GGML_OPENCL_USE_ADRENO_KERNELS return; } if (tensor->type == GGML_TYPE_Q6_K) { ggml_tensor_extra_cl * extra_orig = (ggml_tensor_extra_cl *)tensor->extra; GGML_ASSERT(extra_orig && "Tesnors in OpenCL backend should have been allocated and initialized"); // Allocate the new extra and create aliases from the original. ggml_backend_opencl_buffer_context * ctx = (ggml_backend_opencl_buffer_context *) buffer->context; ggml_tensor_extra_cl_q6_K * extra = ctx->ggml_opencl_alloc_temp_tensor_extra_q6_K(); size_t size_ql = ggml_nelements(tensor)/ggml_blck_size(tensor->type)*ggml_blck_size(tensor->type)/2; size_t size_qh = ggml_nelements(tensor)/ggml_blck_size(tensor->type)*ggml_blck_size(tensor->type)/4; size_t size_s = ggml_nelements(tensor)/ggml_blck_size(tensor->type)*ggml_blck_size(tensor->type)/16; size_t size_d = ggml_nelements(tensor)/ggml_blck_size(tensor->type)*sizeof(ggml_fp16_t); GGML_ASSERT(size_ql + size_qh + size_s + size_d == ggml_nbytes(tensor) && "Incorrect tensor size"); cl_int err; cl_mem data_device = ggml_cl_create_temp_upload_buffer(context, queue, ggml_nbytes(tensor), data, tensor->name); GGML_ASSERT(data_device != NULL && "q6_K set_tensor: temp upload buffer alloc failed"); cl_buffer_region region; #ifdef GGML_OPENCL_USE_ADRENO_KERNELS // Adreno MoE Q6_K kernel needs special transposed layout if (use_adreno_moe_kernels(backend_ctx, tensor)) { size_t moe_size_ql = (size_t)(ggml_nelements(tensor) / 8) * sizeof(uint32_t); // 4 bits per element size_t moe_size_qh = (size_t)(ggml_nelements(tensor) / 16) * sizeof(uint32_t); // 2 bits per element size_t moe_size_s = size_s; size_t moe_size_d = size_d; // Subbuffer for ql region.origin = align_to(extra_orig->offset + tensor->view_offs + offset, backend_ctx->alignment); region.size = moe_size_ql; CL_CHECK((extra->ql = clCreateSubBuffer(extra_orig->data_device, CL_MEM_READ_WRITE, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); auto previous_origin = region.origin; // Subbuffer for qh region.origin = align_to(previous_origin + moe_size_ql, backend_ctx->alignment); region.size = moe_size_qh; CL_CHECK((extra->qh = clCreateSubBuffer(extra_orig->data_device, CL_MEM_READ_WRITE, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); previous_origin = region.origin; // Subbuffer for scales region.origin = align_to(previous_origin + moe_size_qh, backend_ctx->alignment); region.size = moe_size_s; CL_CHECK((extra->s = clCreateSubBuffer(extra_orig->data_device, CL_MEM_READ_WRITE, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); previous_origin = region.origin; // Subbuffer for d region.origin = align_to(previous_origin + moe_size_s, backend_ctx->alignment); region.size = moe_size_d; CL_CHECK((extra->d = clCreateSubBuffer(extra_orig->data_device, CL_MEM_READ_WRITE, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); cl_kernel kernel = backend_ctx->kernel_convert_block_q6_k_trans4_ns; cl_uchar mask_0F = 0x0F; cl_uchar mask_F0 = 0xF0; int ne00 = tensor->ne[0]; int ne01 = tensor->ne[1]; int ne02 = tensor->ne[2]; CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &data_device)); CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra->ql)); CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra->qh)); CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra->d)); CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extra->s)); CL_CHECK(clSetKernelArg(kernel, 5, sizeof(int), &ne00)); CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne01)); CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_uchar), &mask_0F)); CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_uchar), &mask_F0)); size_t global_work_size[] = {static_cast(((ne01 + 63) / 64) * 64), static_cast(ne00 / 256), static_cast(ne02)}; size_t local_work_size[] = {64, 1, 1}; cl_event evt; CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, global_work_size, local_work_size, 0, NULL, &evt)); CL_CHECK(clWaitForEvents(1, &evt)); CL_CHECK(clReleaseMemObject(data_device)); // Create image for ql cl_image_format img_format_ql = {CL_R, CL_UNSIGNED_INT32}; cl_image_desc img_desc_ql = { CL_MEM_OBJECT_IMAGE1D_BUFFER, static_cast(ggml_nelements(tensor) / 8), 0, 0, 0, 0, 0, 0, 0, { extra->ql } }; extra->ql_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_format_ql, &img_desc_ql, NULL, &err); tensor->extra = extra; return; } #endif // GGML_OPENCL_USE_ADRENO_KERNELS // Subbuffer for ql region.origin = align_to(extra_orig->offset + tensor->view_offs + offset, backend_ctx->alignment); region.size = size_ql; CL_CHECK((extra->ql = clCreateSubBuffer(extra_orig->data_device, CL_MEM_READ_WRITE, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); auto previous_origin = region.origin; // Subbuffer for qh region.origin = align_to(previous_origin + size_ql, backend_ctx->alignment); region.size = size_qh; CL_CHECK((extra->qh = clCreateSubBuffer(extra_orig->data_device, CL_MEM_READ_WRITE, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); previous_origin = region.origin; // Subbuffer for scales region.origin = align_to(previous_origin + size_qh, backend_ctx->alignment); region.size = size_s; CL_CHECK((extra->s = clCreateSubBuffer(extra_orig->data_device, CL_MEM_READ_WRITE, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); previous_origin = region.origin; // Create subbuffer for d. region.origin = align_to(previous_origin + size_s, backend_ctx->alignment); region.size = size_d; CL_CHECK((extra->d = clCreateSubBuffer(extra_orig->data_device, CL_MEM_READ_WRITE, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); previous_origin = region.origin; #ifdef GGML_OPENCL_USE_ADRENO_KERNELS // Tiled-wide convert for the long-vocab lm_head/embed (opt-in). The embed // /output q6_K weight (e.g. token_embd.weight, ne1=vocab) is NOT matched by // use_adreno_moe_kernels, so it lands here in the general branch. Produce // the final 64-row-tiled canonical layout directly into ql/qh/s/d (buffer // sizes already match), read back by kernel_gemv_noshuffle_q6_K_f32_tiled. // Bypasses the plain-SOA convert + per-array transpose below. if (use_q6k_tiled(backend_ctx, tensor)) { cl_kernel kernel = backend_ctx->kernel_convert_block_q6_k_tiled_ns; int ne00 = tensor->ne[0]; int ne01 = tensor->ne[1]; int ne02 = tensor->ne[2]; CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &data_device)); CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra->ql)); CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra->qh)); CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra->d)); CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extra->s)); CL_CHECK(clSetKernelArg(kernel, 5, sizeof(int), &ne00)); CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne01)); size_t global_work_size[] = {static_cast(((ne01 + 63) / 64) * 64), static_cast(ne00 / 256), static_cast(ne02)}; size_t local_work_size[] = {64, 1, 1}; cl_event evt; CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, global_work_size, local_work_size, 0, NULL, &evt)); CL_CHECK(clWaitForEvents(1, &evt)); CL_CHECK(clReleaseMemObject(data_device)); extra->size_ql = size_ql; extra->size_qh = size_qh; extra->size_s = size_s; extra->size_d = size_d; tensor->extra = extra; return; } #endif // GGML_OPENCL_USE_ADRENO_KERNELS // Flatten the weights cl_kernel kernel; #ifdef GGML_OPENCL_USE_ADRENO_KERNELS kernel = backend_ctx->kernel_convert_block_q6_K; if (use_adreno_kernels(backend_ctx, tensor) && !use_flat_gemv_for_large_m_q6_K(backend_ctx, tensor)) { kernel = backend_ctx->kernel_convert_block_q6_K_noshuffle; } #else kernel = backend_ctx->kernel_convert_block_q6_K; #endif // GGML_OPENCL_USE_ADRENO_KERNELS cl_uchar mask = 0xff; cl_ulong n_blk = ggml_nelements(tensor)/ggml_blck_size(tensor->type); CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &data_device)); CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra->ql)); CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra->qh)); CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra->s)); CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extra->d)); CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_uchar), &mask)); CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_ulong), &n_blk)); size_t global_work_size[] = {(size_t)CEIL_DIV(n_blk, 64)*64, 1, 1}; size_t local_work_size[] = {64, 1, 1}; cl_event evt; CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, global_work_size, local_work_size, 0, NULL, &evt)); CL_CHECK(clWaitForEvents(1, &evt)); CL_CHECK(clReleaseMemObject(data_device)); extra->size_ql = size_ql; extra->size_qh = size_qh; extra->size_s = size_s; extra->size_d = size_d; tensor->extra = extra; #ifdef GGML_OPENCL_USE_ADRENO_KERNELS if (use_adreno_kernels(backend_ctx, tensor) && !use_flat_gemv_for_large_m_q6_K(backend_ctx, tensor)) { cl_int M = tensor->ne[1]; // ne01 cl_int K = tensor->ne[0]; // ne00 if (use_q6_k_bin_kernels(backend_ctx, tensor)) { GGML_ASSERT(K % 256 == 0); GGML_ASSERT(M % 64 == 0); transpose_2d_as_32b(backend_ctx, extra->ql, extra->ql, size_ql, K/8, M); transpose_2d_as_32b(backend_ctx, extra->qh, extra->qh, size_qh, K/16, M); cl_image_format wimg_fmt = { CL_R, CL_UNSIGNED_INT32 }; cl_image_desc wimg_desc; memset(&wimg_desc, 0, sizeof(wimg_desc)); wimg_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; wimg_desc.image_width = static_cast(ggml_nelements(tensor) / 8); wimg_desc.buffer = extra->ql; CL_CHECK((extra->ql_img = clCreateImage(context, CL_MEM_READ_ONLY, &wimg_fmt, &wimg_desc, NULL, &err), err)); memset(&wimg_desc, 0, sizeof(wimg_desc)); wimg_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; wimg_desc.image_width = static_cast(ggml_nelements(tensor) / 16); wimg_desc.buffer = extra->qh; CL_CHECK((extra->qh_img = clCreateImage(context, CL_MEM_READ_ONLY, &wimg_fmt, &wimg_desc, NULL, &err), err)); } else { // Transpose ql as ushort transpose_2d_as_16b(backend_ctx, extra->ql, extra->ql, size_ql, K/4, M); // Transpose qh as uchar transpose_2d_as_8b(backend_ctx, extra->qh, extra->qh, size_qh, K/4, M); // Transpose s as ushort transpose_2d_as_16b(backend_ctx, extra->s, extra->s, size_s, K/16/2, M); } // Transpose d as ushort transpose_2d_as_16b(backend_ctx, extra->d, extra->d, size_d, K/256, M); } #endif // GGML_OPENCL_USE_ADRENO_KERNELS return; } #endif // GGML_OPENCL_SOA_Q // convert bf16 to f16 and store as f16 in device buffer if (tensor->type == GGML_TYPE_BF16) { GGML_ASSERT(offset % sizeof(ggml_fp16_t) == 0 && size % sizeof(ggml_fp16_t) == 0 && "Offset and size must be multiples of 2 for bf16 tensors"); ggml_tensor_extra_cl * extra = (ggml_tensor_extra_cl *) tensor->extra; GGML_ASSERT(extra); cl_ulong n_elements = size / sizeof(ggml_fp16_t); cl_ulong off_dst = (extra->offset + offset) / sizeof(ggml_fp16_t); cl_int err; cl_mem data_device = clCreateBuffer(context, CL_MEM_READ_ONLY | CL_MEM_COPY_HOST_PTR, size, const_cast(data), &err); CL_CHECK(err); cl_kernel kernel = backend_ctx->kernel_convert_bf16_to_f16; CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &data_device)); CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra->data_device)); CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_ulong), &off_dst)); CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &n_elements)); size_t global_work_size[] = { (size_t)CEIL_DIV(n_elements, 64)*64, 1, 1 }; size_t local_work_size[] = { 64, 1, 1 }; cl_event evt; CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, global_work_size, local_work_size, 0, NULL, &evt)); CL_CHECK(clWaitForEvents(1, &evt)); CL_CHECK(clReleaseMemObject(data_device)); CL_CHECK(clReleaseEvent(evt)); return; } ggml_tensor_extra_cl * extra = (ggml_tensor_extra_cl *) tensor->extra; GGML_ASSERT(extra); CL_CHECK(clEnqueueWriteBuffer( queue, extra->data_device, CL_TRUE, extra->offset + offset, size, data, 0, NULL, NULL)); GGML_UNUSED(buffer); } static void ggml_backend_opencl_buffer_get_tensor(ggml_backend_buffer_t buffer, const ggml_tensor * tensor, void * data, size_t offset, size_t size) { GGML_ASSERT(tensor->extra); ggml_backend_opencl_device_context * dev_ctx = (ggml_backend_opencl_device_context *) buffer->buft->device->context; ggml_backend_opencl_context *backend_ctx = dev_ctx->backend_ctx; cl_context context = backend_ctx->context; cl_command_queue queue = backend_ctx->queue; // Make sure all previously submitted commands in other devices are finished. sync_with_other_backends(backend_ctx); #ifdef GGML_OPENCL_SOA_Q if (tensor->type == GGML_TYPE_Q1_0) { ggml_tensor_extra_cl_q1_0 * extra = (ggml_tensor_extra_cl_q1_0 *)tensor->extra; #ifdef GGML_OPENCL_USE_ADRENO_KERNELS if (enable_adreno_trans_weight(backend_ctx, tensor)) { ggml_cl_buffer buf_trans_q; ggml_cl_buffer buf_trans_d; ggml_cl_buffer buf_unpacked; int M = tensor->ne[1]; int K = tensor->ne[0]; size_t size_d = ggml_nelements(tensor)/ggml_blck_size(tensor->type)*sizeof(ggml_fp16_t); size_t size_q = ggml_nelements(tensor)/ggml_blck_size(tensor->type)*(ggml_blck_size(tensor->type)/8); buf_trans_q.allocate(backend_ctx->context, size_q); buf_trans_d.allocate(backend_ctx->context, size_d); buf_unpacked.allocate(backend_ctx->context, ggml_nbytes(tensor)); transpose_2d_as_32b(backend_ctx, extra->q, buf_trans_q.buffer, size_q, M, K/32); transpose_2d_as_16b(backend_ctx, extra->d, buf_trans_d.buffer, size_d, M, K/128); cl_kernel kernel = backend_ctx->kernel_restore_block_q1_0; CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &buf_trans_q.buffer)); CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &buf_trans_d.buffer)); CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &buf_unpacked.buffer)); size_t global_work_size[] = {(size_t)ggml_nelements(tensor)/ggml_blck_size(tensor->type), 1, 1}; size_t local_work_size[] = {1, 1, 1}; cl_event evt; CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, global_work_size, local_work_size, 0, NULL, &evt)); CL_CHECK(clWaitForEvents(1, &evt)); CL_CHECK(clEnqueueReadBuffer(queue, buf_unpacked.buffer, CL_TRUE, offset, size, data, 0, NULL, NULL)); return; } #endif cl_int err; cl_mem data_device = clCreateBuffer(context, CL_MEM_READ_WRITE, ggml_nbytes(tensor), NULL, &err); CL_CHECK(err); cl_kernel kernel = backend_ctx->kernel_restore_block_q1_0; CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra->q)); CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra->d)); CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &data_device)); size_t global_work_size[] = {(size_t)ggml_nelements(tensor)/ggml_blck_size(tensor->type), 1, 1}; size_t local_work_size[] = {1, 1, 1}; cl_event evt; CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, global_work_size, local_work_size, 0, NULL, &evt)); CL_CHECK(clWaitForEvents(1, &evt)); CL_CHECK(clEnqueueReadBuffer(queue, data_device, CL_TRUE, offset, size, data, 0, NULL, NULL)); CL_CHECK(clReleaseMemObject(data_device)); return; } // In end-to-end runs, get_tensor is usually used to get back the logits, // where we can simply do clEnqueueReadBuffer since they are f32. // However, in test-backend-ops, the GPU graph is copied to the CPU backend, // which requires reading back quantized weight tensors. // To properly support this, we need to restore block_q4_0 struct arrays // from the flattened buffers. if (tensor->type == GGML_TYPE_Q4_0) { // KV-cache q4_0 stays AoS — direct readback, no SoA restore. if (!ggml_cl_is_q4_0_soa(tensor)) { ggml_tensor_extra_cl * extra_aos = (ggml_tensor_extra_cl *) tensor->extra; CL_CHECK(clEnqueueReadBuffer( queue, extra_aos->data_device, CL_TRUE, extra_aos->offset + tensor->view_offs + offset, size, data, 0, NULL, NULL)); return; } // SoA extra lives on the parent tensor — follow view_src. const ggml_tensor * extra_src = tensor->view_src != nullptr ? tensor->view_src : tensor; ggml_tensor_extra_cl_q4_0 * extra = (ggml_tensor_extra_cl_q4_0 *)extra_src->extra; #ifdef GGML_OPENCL_USE_ADRENO_KERNELS if (use_adreno_moe_kernels(backend_ctx, tensor)) { cl_kernel kernel = backend_ctx->kernel_restore_block_q4_0_trans4_ns; cl_mem data_device = ggml_cl_create_temp_download_buffer(context, queue, ggml_nbytes(tensor), tensor->name); GGML_ASSERT(data_device != NULL && "get_tensor: temp download buffer alloc failed"); int ne00 = tensor->ne[0]; int ne01 = tensor->ne[1]; int ne02 = tensor->ne[2]; CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra->q)); CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra->d)); CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &data_device)); CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_int), &ne00)); CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_int), &ne01)); size_t global_work_size[3] = {static_cast(((ne01 + 63) / 64) * 64), static_cast(ne00 / 32), static_cast(ne02)}; size_t local_work_size[3] = {64, 2, 1}; cl_event evt; CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, global_work_size, local_work_size, 0, NULL, &evt)); CL_CHECK(clWaitForEvents(1, &evt)); CL_CHECK(clEnqueueReadBuffer( queue, data_device, CL_TRUE, offset, size, data, 0, NULL, NULL)); CL_CHECK(clReleaseMemObject(data_device)); return; } if (use_adreno_kernels(backend_ctx, tensor)) { ggml_cl_buffer buf_trans_q; ggml_cl_buffer buf_trans_d; ggml_cl_buffer buf_unpacked; cl_int M = tensor->ne[1]; // ne01 cl_int K = tensor->ne[0]; // ne00 GGML_ASSERT(K % 32 == 0); GGML_ASSERT(M % 4 == 0); size_t size_q = (ggml_nelements(tensor)/ggml_blck_size(tensor->type))*ggml_blck_size(tensor->type)/2; size_t size_d = (ggml_nelements(tensor)/ggml_blck_size(tensor->type))*sizeof(ggml_fp16_t); GGML_ASSERT(size_d + size_q == ggml_nbytes(tensor) && "Incorrect tensor size"); buf_trans_q.allocate(backend_ctx->context, size_q); buf_trans_d.allocate(backend_ctx->context, size_d); buf_unpacked.allocate(backend_ctx->context, ggml_nbytes(tensor)); if (use_q4_0_bin_kernels(backend_ctx, tensor)) { transpose_2d_as_32b(backend_ctx, extra->q, buf_trans_q.buffer, size_q, M, K / 8); } else { transpose_2d_as_16b(backend_ctx, extra->q, buf_trans_q.buffer, size_q, M, K / 4); } transpose_2d_as_16b(backend_ctx, extra->d, buf_trans_d.buffer, size_d, M, K/32); cl_uchar mask_0F = 0x0F; cl_uchar mask_F0 = 0xF0; size_t global_work_size[] = {(size_t)ggml_nelements(tensor)/ggml_blck_size(tensor->type), 1, 1}; size_t local_work_size[] = {1, 1, 1}; cl_kernel kernel = backend_ctx->kernel_restore_block_q4_0_noshuffle; CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &buf_trans_q.buffer)); CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &buf_trans_d.buffer)); CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &buf_unpacked.buffer)); CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_uchar), &mask_0F)); CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_uchar), &mask_F0)); CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, global_work_size, local_work_size, 0, NULL, NULL)); CL_CHECK(clEnqueueReadBuffer(queue, buf_unpacked.buffer, CL_TRUE, offset, size, data, 0, NULL, NULL)); return; } #endif cl_mem data_device = ggml_cl_create_temp_download_buffer(context, queue, ggml_nbytes(tensor), tensor->name); GGML_ASSERT(data_device != NULL && "get_tensor: temp download buffer alloc failed"); cl_kernel kernel = backend_ctx->kernel_restore_block_q4_0; CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra->q)); CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra->d)); CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &data_device)); size_t global_work_size[] = {(size_t)ggml_nelements(tensor)/ggml_blck_size(tensor->type), 1, 1}; size_t local_work_size[] = {1, 1, 1}; cl_event evt; CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, global_work_size, local_work_size, 0, NULL, &evt)); CL_CHECK(clWaitForEvents(1, &evt)); CL_CHECK(clEnqueueReadBuffer( queue, data_device, CL_TRUE, offset, size, data, 0, NULL, NULL)); CL_CHECK(clReleaseMemObject(data_device)); return; } if (tensor->type == GGML_TYPE_Q4_1) { ggml_tensor_extra_cl_q4_1 * extra = (ggml_tensor_extra_cl_q4_1 *)tensor->extra; #ifdef GGML_OPENCL_USE_ADRENO_KERNELS if (use_adreno_moe_kernels(backend_ctx, tensor)) { cl_mem data_device = ggml_cl_create_temp_download_buffer(context, queue, ggml_nbytes(tensor), tensor->name); GGML_ASSERT(data_device != NULL && "get_tensor: temp download buffer alloc failed"); cl_kernel kernel = backend_ctx->kernel_restore_block_q4_1_trans4_ns; int ne00 = tensor->ne[0]; int ne01 = tensor->ne[1]; int ne02 = tensor->ne[2]; CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra->q)); CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra->d)); CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra->m)); CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &data_device)); CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_int), &ne00)); CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_int), &ne01)); size_t global_work_size[3] = {static_cast(((ne01 + 63) / 64) * 64), static_cast(ne00 / 32), static_cast(ne02)}; size_t local_work_size[3] = {64, 2, 1}; cl_event evt; CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, global_work_size, local_work_size, 0, NULL, &evt)); CL_CHECK(clWaitForEvents(1, &evt)); CL_CHECK(clEnqueueReadBuffer( queue, data_device, CL_TRUE, offset, size, data, 0, NULL, NULL)); CL_CHECK(clReleaseMemObject(data_device)); return; } if (use_adreno_kernels(backend_ctx, tensor)) { static ggml_cl_buffer buf_trans_q; static ggml_cl_buffer buf_trans_m; static ggml_cl_buffer buf_trans_d; static ggml_cl_buffer buf_unpacked; cl_int M = tensor->ne[1]; cl_int K = tensor->ne[0]; GGML_ASSERT(K % ggml_blck_size(tensor->type) == 0); size_t size_q = (ggml_nelements(tensor)/ggml_blck_size(tensor->type))*ggml_blck_size(tensor->type)/2; size_t size_d = (ggml_nelements(tensor)/ggml_blck_size(tensor->type))*sizeof(ggml_fp16_t); size_t size_m = (ggml_nelements(tensor)/ggml_blck_size(tensor->type))*sizeof(ggml_fp16_t); GGML_ASSERT(size_d + size_q + size_m == ggml_nbytes(tensor) && "Incorrect tensor size"); buf_trans_q.allocate(backend_ctx->context, size_q); buf_trans_m.allocate(backend_ctx->context, size_m); buf_trans_d.allocate(backend_ctx->context, size_d); buf_unpacked.allocate(backend_ctx->context, ggml_nbytes(tensor)); // transpose q, d, m back transpose_2d_as_16b(backend_ctx, extra->q, buf_trans_q.buffer, size_q, M, K/4); transpose_2d_as_16b(backend_ctx, extra->d, buf_trans_d.buffer, size_d, M, K/32); transpose_2d_as_16b(backend_ctx, extra->m, buf_trans_m.buffer, size_m, M, K/32); cl_uchar mask_0F = 0x0F; cl_uchar mask_F0 = 0xF0; size_t global_work_size[] = {(size_t)ggml_nelements(tensor)/ggml_blck_size(tensor->type), 1, 1}; size_t local_work_size[] = {1, 1, 1}; cl_kernel kernel = backend_ctx->kernel_restore_block_q4_1_noshuffle; CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &buf_trans_q.buffer)); CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &buf_trans_d.buffer)); CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &buf_trans_m.buffer)); CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &buf_unpacked.buffer)); CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_uchar), &mask_0F)); CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_uchar), &mask_F0)); CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, global_work_size, local_work_size, 0, NULL, NULL)); CL_CHECK(clEnqueueReadBuffer(queue, buf_unpacked.buffer, CL_TRUE, offset, size, data, 0, NULL, NULL)); return; } #endif cl_mem data_device = ggml_cl_create_temp_download_buffer(context, queue, ggml_nbytes(tensor), tensor->name); GGML_ASSERT(data_device != NULL && "get_tensor: temp download buffer alloc failed"); cl_kernel kernel = backend_ctx->kernel_restore_block_q4_1; CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra->q)); CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra->d)); CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra->m)); CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &data_device)); size_t global_work_size[] = {(size_t)ggml_nelements(tensor)/ggml_blck_size(tensor->type), 1, 1}; size_t local_work_size[] = {1, 1, 1}; cl_event evt; CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, global_work_size, local_work_size, 0, NULL, &evt)); CL_CHECK(clWaitForEvents(1, &evt)); CL_CHECK(clEnqueueReadBuffer( queue, data_device, CL_TRUE, offset, size, data, 0, NULL, NULL)); CL_CHECK(clReleaseMemObject(data_device)); return; } if (tensor->type == GGML_TYPE_Q5_0) { ggml_tensor_extra_cl_q5_0 * extra = (ggml_tensor_extra_cl_q5_0 *)tensor->extra; #ifdef GGML_OPENCL_USE_ADRENO_KERNELS if (use_adreno_moe_kernels(backend_ctx, tensor)) { // TODO: use ggml_cl_buffer to manage this temporary buffer cl_mem data_device = ggml_cl_create_temp_download_buffer(context, queue, ggml_nbytes(tensor), tensor->name); GGML_ASSERT(data_device != NULL && "get_tensor: temp download buffer alloc failed"); cl_kernel kernel = backend_ctx->kernel_restore_block_q5_0_trans4_ns; int ne00 = tensor->ne[0]; int ne01 = tensor->ne[1]; int ne02 = tensor->ne[2]; CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra->qs)); CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra->qh)); CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra->d)); CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &data_device)); CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_int), &ne00)); CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_int), &ne01)); size_t global_work_size[3] = {static_cast(((ne01 + 63) / 64) * 64), static_cast(ne00 / 32), static_cast(ne02)}; size_t local_work_size[3] = {64, 2, 1}; cl_event evt; CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, global_work_size, local_work_size, 0, NULL, &evt)); CL_CHECK(clWaitForEvents(1, &evt)); CL_CHECK(clEnqueueReadBuffer( queue, data_device, CL_TRUE, offset, size, data, 0, NULL, NULL)); CL_CHECK(clReleaseMemObject(data_device)); return; } if (use_adreno_kernels(backend_ctx, tensor)) { ggml_cl_buffer buf_trans_qs; ggml_cl_buffer buf_trans_qh; ggml_cl_buffer buf_trans_d; ggml_cl_buffer buf_unpacked; cl_int M = tensor->ne[1]; cl_int K = tensor->ne[0]; GGML_ASSERT(K % 32 == 0); size_t size_qs = (ggml_nelements(tensor)/ggml_blck_size(tensor->type))*ggml_blck_size(tensor->type)/2; size_t size_qh = (ggml_nelements(tensor)/ggml_blck_size(tensor->type))*sizeof(int32_t); size_t size_d = (ggml_nelements(tensor)/ggml_blck_size(tensor->type))*sizeof(ggml_fp16_t); buf_trans_qs.allocate(backend_ctx->context, size_qs); buf_trans_qh.allocate(backend_ctx->context, size_qh); buf_trans_d.allocate(backend_ctx->context, size_d); buf_unpacked.allocate(backend_ctx->context, ggml_nbytes(tensor)); transpose_2d_as_16b(backend_ctx, extra->qs, buf_trans_qs.buffer, size_qs, M, K/4); transpose_2d_as_8b(backend_ctx, extra->qh, buf_trans_qh.buffer, size_qh, M, K/8); transpose_2d_as_16b(backend_ctx, extra->d, buf_trans_d.buffer, size_d, M, K/32); cl_uchar mask_0F = 0x0F; cl_uchar mask_F0 = 0xF0; size_t global_work_size[] = {(size_t)ggml_nelements(tensor)/ggml_blck_size(tensor->type), 1, 1}; size_t local_work_size[] = {1, 1, 1}; cl_kernel kernel = backend_ctx->kernel_restore_block_q5_0_noshuffle; CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &buf_trans_qs.buffer)); CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &buf_trans_qh.buffer)); CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &buf_trans_d.buffer)); CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &buf_unpacked.buffer)); CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_uchar), &mask_0F)); CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_uchar), &mask_F0)); CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, global_work_size, local_work_size, 0, NULL, NULL)); CL_CHECK(clEnqueueReadBuffer(queue, buf_unpacked.buffer, CL_TRUE, offset, size, data, 0, NULL, NULL)); return; } #endif // GGML_OPENCL_USE_ADRENO_KERNELS cl_int err; cl_mem data_device = clCreateBuffer(context, CL_MEM_READ_WRITE, ggml_nbytes(tensor), NULL, &err); CL_CHECK(err); cl_kernel kernel = backend_ctx->kernel_restore_block_q5_0; CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra->qs)); CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra->qh)); CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra->d)); CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &data_device)); size_t global_work_size[] = {(size_t)ggml_nelements(tensor)/ggml_blck_size(tensor->type), 1, 1}; size_t local_work_size[] = {1, 1, 1}; cl_event evt; CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, global_work_size, local_work_size, 0, NULL, &evt)); CL_CHECK(clWaitForEvents(1, &evt)); CL_CHECK(clEnqueueReadBuffer( queue, data_device, CL_TRUE, offset, size, data, 0, NULL, NULL)); CL_CHECK(clReleaseMemObject(data_device)); return; } if (tensor->type == GGML_TYPE_Q5_1) { ggml_tensor_extra_cl_q5_1 * extra = (ggml_tensor_extra_cl_q5_1 *)tensor->extra; #ifdef GGML_OPENCL_USE_ADRENO_KERNELS if (use_adreno_moe_kernels(backend_ctx, tensor)) { // TODO: use ggml_cl_buffer to manage this temporary buffer cl_mem data_device = ggml_cl_create_temp_download_buffer(context, queue, ggml_nbytes(tensor), tensor->name); GGML_ASSERT(data_device != NULL && "get_tensor: temp download buffer alloc failed"); cl_kernel kernel = backend_ctx->kernel_restore_block_q5_1_trans4_ns; int ne00 = tensor->ne[0]; int ne01 = tensor->ne[1]; int ne02 = tensor->ne[2]; CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra->qs)); CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra->qh)); CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra->d)); CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra->m)); CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &data_device)); CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_int), &ne00)); CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_int), &ne01)); size_t global_work_size[3] = {static_cast(((ne01 + 63) / 64) * 64), static_cast(ne00 / 32), static_cast(ne02)}; size_t local_work_size[3] = {64, 2, 1}; cl_event evt; CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, global_work_size, local_work_size, 0, NULL, &evt)); CL_CHECK(clWaitForEvents(1, &evt)); CL_CHECK(clEnqueueReadBuffer( queue, data_device, CL_TRUE, offset, size, data, 0, NULL, NULL)); CL_CHECK(clReleaseMemObject(data_device)); return; } if (use_adreno_kernels(backend_ctx, tensor)) { ggml_cl_buffer buf_trans_qs; ggml_cl_buffer buf_trans_qh; ggml_cl_buffer buf_trans_d; ggml_cl_buffer buf_trans_m; ggml_cl_buffer buf_unpacked; cl_int M = tensor->ne[1]; cl_int K = tensor->ne[0]; GGML_ASSERT(K % 32 == 0); size_t size_qs = (ggml_nelements(tensor)/ggml_blck_size(tensor->type))*ggml_blck_size(tensor->type)/2; size_t size_qh = (ggml_nelements(tensor)/ggml_blck_size(tensor->type))*sizeof(int32_t); size_t size_d = (ggml_nelements(tensor)/ggml_blck_size(tensor->type))*sizeof(ggml_fp16_t); size_t size_m = (ggml_nelements(tensor)/ggml_blck_size(tensor->type))*sizeof(ggml_fp16_t); buf_trans_qs.allocate(backend_ctx->context, size_qs); buf_trans_qh.allocate(backend_ctx->context, size_qh); buf_trans_d.allocate(backend_ctx->context, size_d); buf_trans_m.allocate(backend_ctx->context, size_m); buf_unpacked.allocate(backend_ctx->context, ggml_nbytes(tensor)); // Transpose back: from col-major to row-major transpose_2d_as_16b(backend_ctx, extra->qs, buf_trans_qs.buffer, size_qs, M, K/4); transpose_2d_as_8b(backend_ctx, extra->qh, buf_trans_qh.buffer, size_qh, M, K/8); transpose_2d_as_16b(backend_ctx, extra->d, buf_trans_d.buffer, size_d, M, K/32); transpose_2d_as_16b(backend_ctx, extra->m, buf_trans_m.buffer, size_m, M, K/32); cl_uchar mask_0F = 0x0F; cl_uchar mask_F0 = 0xF0; size_t global_work_size[] = {(size_t)ggml_nelements(tensor)/ggml_blck_size(tensor->type), 1, 1}; size_t local_work_size[] = {1, 1, 1}; cl_kernel kernel = backend_ctx->kernel_restore_block_q5_1_noshuffle; CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &buf_trans_qs.buffer)); CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &buf_trans_qh.buffer)); CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &buf_trans_d.buffer)); CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &buf_trans_m.buffer)); CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &buf_unpacked.buffer)); CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_uchar), &mask_0F)); CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_uchar), &mask_F0)); CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, global_work_size, local_work_size, 0, NULL, NULL)); CL_CHECK(clEnqueueReadBuffer(queue, buf_unpacked.buffer, CL_TRUE, offset, size, data, 0, NULL, NULL)); return; } #endif // GGML_OPENCL_USE_ADRENO_KERNELS cl_int err; cl_mem data_device = clCreateBuffer(context, CL_MEM_READ_WRITE, ggml_nbytes(tensor), NULL, &err); CL_CHECK(err); cl_kernel kernel = backend_ctx->kernel_restore_block_q5_1; CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra->qs)); CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra->qh)); CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra->d)); CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra->m)); CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &data_device)); size_t global_work_size[] = {(size_t)ggml_nelements(tensor)/ggml_blck_size(tensor->type), 1, 1}; size_t local_work_size[] = {1, 1, 1}; cl_event evt; CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, global_work_size, local_work_size, 0, NULL, &evt)); CL_CHECK(clWaitForEvents(1, &evt)); CL_CHECK(clEnqueueReadBuffer( queue, data_device, CL_TRUE, offset, size, data, 0, NULL, NULL)); CL_CHECK(clReleaseMemObject(data_device)); return; } if (tensor->type == GGML_TYPE_MXFP4) { ggml_tensor_extra_cl_mxfp4 * extra = (ggml_tensor_extra_cl_mxfp4 *)tensor->extra; cl_mem data_device = ggml_cl_create_temp_download_buffer(context, queue, ggml_nbytes(tensor), tensor->name); GGML_ASSERT(data_device != NULL && "get_tensor: temp download buffer alloc failed"); #ifdef GGML_OPENCL_USE_ADRENO_KERNELS if (use_adreno_moe_kernels(backend_ctx, tensor)) { cl_kernel kernel = backend_ctx->kernel_restore_block_mxfp4_trans4_ns; int ne00 = tensor->ne[0]; int ne01 = tensor->ne[1]; int ne02 = tensor->ne[2]; CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra->q)); CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra->e)); CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &data_device)); CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_int), &ne00)); CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_int), &ne01)); size_t global_work_size[3] = {static_cast(((ne01 + 63) / 64) * 64), static_cast(ne00 / 32), static_cast(ne02)}; size_t local_work_size[3] = {64, 2, 1}; cl_event evt; CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, global_work_size, local_work_size, 0, NULL, &evt)); CL_CHECK(clWaitForEvents(1, &evt)); CL_CHECK(clEnqueueReadBuffer( queue, data_device, CL_TRUE, offset, size, data, 0, NULL, NULL)); CL_CHECK(clReleaseMemObject(data_device)); return; } #endif // GGML_OPENCL_USE_ADRENO_KERNELS cl_kernel kernel = backend_ctx->kernel_restore_block_mxfp4; CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra->q)); CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra->e)); CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &data_device)); size_t global_work_size[] = {(size_t)ggml_nelements(tensor)/ggml_blck_size(tensor->type), 1, 1}; size_t local_work_size[] = {1, 1, 1}; cl_event evt; CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, global_work_size, local_work_size, 0, NULL, &evt)); CL_CHECK(clWaitForEvents(1, &evt)); CL_CHECK(clEnqueueReadBuffer( queue, data_device, CL_TRUE, offset, size, data, 0, NULL, NULL)); CL_CHECK(clReleaseMemObject(data_device)); return; } if (tensor->type == GGML_TYPE_Q8_0) { // KV-cache q8_0 stays AoS (see Q4_0 branch). if (!ggml_cl_is_q8_0_soa(tensor)) { ggml_tensor_extra_cl * extra_aos = (ggml_tensor_extra_cl *) tensor->extra; CL_CHECK(clEnqueueReadBuffer( queue, extra_aos->data_device, CL_TRUE, extra_aos->offset + tensor->view_offs + offset, size, data, 0, NULL, NULL)); return; } // SoA extra lives on the parent — follow view_src. const ggml_tensor * extra_src = tensor->view_src != nullptr ? tensor->view_src : tensor; ggml_tensor_extra_cl_q8_0 * extra = (ggml_tensor_extra_cl_q8_0 *)extra_src->extra; cl_mem data_device = ggml_cl_create_temp_download_buffer(context, queue, ggml_nbytes(tensor), tensor->name); GGML_ASSERT(data_device != NULL && "get_tensor: temp download buffer alloc failed"); #ifdef GGML_OPENCL_USE_ADRENO_KERNELS if (enable_adreno_trans_weight(backend_ctx, tensor)) { cl_kernel kernel = backend_ctx->kernel_restore_block_q8_0_trans; int ne00 = tensor->ne[0]; int ne01 = tensor->ne[1]; GGML_ASSERT(tensor->ne[2] == 1); GGML_ASSERT(tensor->ne[3] == 1); CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra->q)); CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra->d)); CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &data_device)); CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_int), &ne00)); CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_int), &ne01)); size_t global_work_size[3] = {static_cast(((ne01 + 63) / 64) * 64), 1, 1}; size_t local_work_size[3] = {64, 1, 1}; cl_event evt; CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, global_work_size, local_work_size, 0, NULL, &evt)); CL_CHECK(clWaitForEvents(1, &evt)); CL_CHECK(clEnqueueReadBuffer( queue, data_device, CL_TRUE, offset, size, data, 0, NULL, NULL)); CL_CHECK(clReleaseMemObject(data_device)); return; } #endif cl_kernel kernel = backend_ctx->kernel_restore_block_q8_0; CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra->q)); CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra->d)); CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &data_device)); size_t global_work_size[] = {(size_t)ggml_nelements(tensor)/ggml_blck_size(tensor->type), 1, 1}; size_t local_work_size[] = {1, 1, 1}; cl_event evt; CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, global_work_size, local_work_size, 0, NULL, &evt)); CL_CHECK(clWaitForEvents(1, &evt)); CL_CHECK(clEnqueueReadBuffer( queue, data_device, CL_TRUE, offset, size, data, 0, NULL, NULL)); CL_CHECK(clReleaseMemObject(data_device)); return; } if (tensor->type == GGML_TYPE_IQ4_NL) { ggml_tensor_extra_cl_iq4_nl * extra = (ggml_tensor_extra_cl_iq4_nl *)tensor->extra; cl_mem data_device = ggml_cl_create_temp_download_buffer(context, queue, ggml_nbytes(tensor), tensor->name); GGML_ASSERT(data_device != NULL && "get_tensor: temp download buffer alloc failed"); #ifdef GGML_OPENCL_USE_ADRENO_KERNELS if (use_adreno_kernels(backend_ctx, tensor)) { static ggml_cl_buffer buf_trans_q; static ggml_cl_buffer buf_trans_d; static ggml_cl_buffer buf_unpacked; cl_int M = tensor->ne[1]; cl_int K = tensor->ne[0]; GGML_ASSERT(K % 32 == 0); size_t size_q = (ggml_nelements(tensor)/ggml_blck_size(tensor->type))*(ggml_blck_size(tensor->type)/2); size_t size_d = (ggml_nelements(tensor)/ggml_blck_size(tensor->type))*sizeof(ggml_fp16_t); GGML_ASSERT(size_d + size_q == ggml_nbytes(tensor) && "Incorrect tensor size"); buf_trans_q.allocate(backend_ctx->context, size_q); buf_trans_d.allocate(backend_ctx->context, size_d); buf_unpacked.allocate(backend_ctx->context, ggml_nbytes(tensor)); // transpose q, d back transpose_2d_as_16b(backend_ctx, extra->q, buf_trans_q.buffer, size_q, M, K/4); transpose_2d_as_16b(backend_ctx, extra->d, buf_trans_d.buffer, size_d, M, K/32); cl_uchar mask_0F = 0x0F; cl_uchar mask_F0 = 0xF0; cl_kernel kernel = backend_ctx->kernel_restore_block_iq4_nl_noshuffle; cl_ulong n_blk = ggml_nelements(tensor)/ggml_blck_size(tensor->type); CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &buf_trans_q.buffer)); CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &buf_trans_d.buffer)); CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &buf_unpacked.buffer)); CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_uchar), &mask_0F)); CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_uchar), &mask_F0)); CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &n_blk)); size_t global_work_size[] = {(size_t)n_blk, 1, 1}; size_t local_work_size[] = {1, 1, 1}; CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, global_work_size, local_work_size, 0, NULL, NULL)); CL_CHECK(clEnqueueReadBuffer(queue, buf_unpacked.buffer, CL_TRUE, offset, size, data, 0, NULL, NULL)); return; } #endif cl_kernel kernel = backend_ctx->kernel_restore_block_iq4_nl; cl_ulong n_blk = ggml_nelements(tensor)/ggml_blck_size(tensor->type); CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra->q)); CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra->d)); CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &data_device)); CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &n_blk)); size_t global_work_size[] = {(size_t)n_blk, 1, 1}; size_t local_work_size[] = {1, 1, 1}; cl_event evt; CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, global_work_size, local_work_size, 0, NULL, &evt)); CL_CHECK(clWaitForEvents(1, &evt)); CL_CHECK(clEnqueueReadBuffer( queue, data_device, CL_TRUE, offset, size, data, 0, NULL, NULL)); CL_CHECK(clReleaseMemObject(data_device)); return; } if (tensor->type == GGML_TYPE_Q4_K) { ggml_tensor_extra_cl_q4_K * extra = (ggml_tensor_extra_cl_q4_K *)tensor->extra; cl_mem data_device = ggml_cl_create_temp_download_buffer(context, queue, ggml_nbytes(tensor), tensor->name); GGML_ASSERT(data_device != NULL && "get_tensor: temp download buffer alloc failed"); cl_uchar mask_0F = 0x0F; cl_uchar mask_F0 = 0xF0; #ifdef GGML_OPENCL_USE_ADRENO_KERNELS // Undo the 64-row-tiled canonical pack (kernel_convert_block_q4_k_tiled_ns). // Without this, a read-back of a tiled weight returns the tiled bytes // reinterpreted as block_q4_K -- which is how test-backend-ops builds its // CPU reference (ggml_backend_graph_copy -> tensor_get), so the tiled path // "failed" the suite while computing the correct product. if (use_q4k_tiled(backend_ctx, tensor)) { const int ne00v = tensor->ne[0]; const int ne01v = tensor->ne[1]; const int nbv = ne00v / 256; const size_t n_blk = (size_t)nbv * ne01v; std::vector tq(n_blk*32); std::vector td(n_blk), tdm(n_blk); std::vector ts(n_blk*12); CL_CHECK(clEnqueueReadBuffer(queue, extra->q, CL_TRUE, 0, tq.size()*4, tq.data(), 0, NULL, NULL)); CL_CHECK(clEnqueueReadBuffer(queue, extra->d, CL_TRUE, 0, td.size()*2, td.data(), 0, NULL, NULL)); CL_CHECK(clEnqueueReadBuffer(queue, extra->dm, CL_TRUE, 0, tdm.size()*2, tdm.data(), 0, NULL, NULL)); CL_CHECK(clEnqueueReadBuffer(queue, extra->s, CL_TRUE, 0, ts.size(), ts.data(), 0, NULL, NULL)); std::vector rebuilt(ggml_nbytes(tensor), 0); for (int i01 = 0; i01 < ne01v; ++i01) { const int rt = i01/64, rit = i01%64; for (int i00 = 0; i00 < nbv; ++i00) { uint8_t * b = rebuilt.data() + ((size_t)i00 + (size_t)i01*nbv)*144; const int tb = rt*nbv + i00; const size_t si = (size_t)tb*64 + rit; memcpy(b + 0, &td [si], 2); memcpy(b + 2, &tdm[si], 2); memcpy(b + 4, &ts[si*12], 12); uint32_t qw[32]; for (int gr = 0; gr < 8; ++gr) { const size_t base = ((size_t)tb*8 + gr)*64 + rit; for (int j = 0; j < 4; ++j) qw[gr*4 + j] = tq[base*4 + j]; } uint8_t * q = b + 16; for (int e = 0; e < 256; ++e) { const int g = e>>6, w = e&63, h = w>>5, l = w&31; const uint32_t code = (qw[e>>3] >> ((e&7)*4)) & 0xF; q[g*32 + l] |= (uint8_t)(h ? (code << 4) : code); } } } memcpy(data, rebuilt.data() + offset, size); CL_CHECK(clReleaseMemObject(data_device)); return; } if (use_adreno_moe_kernels(backend_ctx, tensor)) { cl_mem data_device = ggml_cl_create_temp_download_buffer(context, queue, ggml_nbytes(tensor), tensor->name); GGML_ASSERT(data_device != NULL && "get_tensor: temp download buffer alloc failed"); cl_kernel kernel = backend_ctx->kernel_restore_block_q4_k_trans4_ns; int ne00 = tensor->ne[0]; int ne01 = tensor->ne[1]; int ne02 = tensor->ne[2]; CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra->q)); CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra->d)); CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra->dm)); CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra->s)); CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &data_device)); CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_int), &ne00)); CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_int), &ne01)); CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_uchar), &mask_0F)); CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_uchar), &mask_F0)); size_t global_work_size[] = {static_cast(((ne01 + 63) / 64) * 64), static_cast(ne00 / 256), static_cast(ne02)}; size_t local_work_size[] = {64, 1, 1}; cl_event evt; CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, global_work_size, local_work_size, 0, NULL, &evt)); CL_CHECK(clWaitForEvents(1, &evt)); CL_CHECK(clEnqueueReadBuffer( queue, data_device, CL_TRUE, offset, size, data, 0, NULL, NULL)); CL_CHECK(clReleaseMemObject(data_device)); return; } if (use_adreno_kernels(backend_ctx, tensor) && !use_flat_gemv_for_large_m_q4_K(backend_ctx, tensor)) { int M = tensor->ne[1]; int K = tensor->ne[0]; size_t size_q = ggml_nelements(tensor)/ggml_blck_size(tensor->type)*ggml_blck_size(tensor->type)/2; size_t size_d = ggml_nelements(tensor)/ggml_blck_size(tensor->type)*sizeof(ggml_fp16_t); size_t size_dm = ggml_nelements(tensor)/ggml_blck_size(tensor->type)*sizeof(ggml_fp16_t); size_t size_s = ggml_nelements(tensor)/ggml_blck_size(tensor->type)*12; static ggml_cl_buffer buf_trans_q; static ggml_cl_buffer buf_trans_d; static ggml_cl_buffer buf_trans_dm; static ggml_cl_buffer buf_trans_s; buf_trans_q.allocate(backend_ctx->context, size_q); buf_trans_d.allocate(backend_ctx->context, size_d); buf_trans_dm.allocate(backend_ctx->context, size_dm); buf_trans_s.allocate(backend_ctx->context, size_s); // Transpose q, d, dm, s back if (use_q4_k_bin_kernels(backend_ctx, tensor)) { transpose_2d_as_32b(backend_ctx, extra->q, buf_trans_q.buffer, size_q, M, K/8); } else { transpose_2d_as_16b(backend_ctx, extra->q, buf_trans_q.buffer, size_q, M, K/4); } transpose_2d_as_16b(backend_ctx, extra->d, buf_trans_d.buffer, size_d, M, K/256); transpose_2d_as_16b(backend_ctx, extra->dm, buf_trans_dm.buffer, size_dm, M, K/256); transpose_2d_as_8b (backend_ctx, extra->s, buf_trans_s.buffer, size_s, M, K/256*12, true, true); cl_kernel kernel = backend_ctx->kernel_restore_block_q4_K_noshuffle; CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &buf_trans_q.buffer)); CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &buf_trans_s.buffer)); CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &buf_trans_d.buffer)); CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &buf_trans_dm.buffer)); CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &data_device)); CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_uchar), &mask_0F)); CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_uchar), &mask_F0)); size_t global_work_size[] = {(size_t)ggml_nelements(tensor)/ggml_blck_size(tensor->type), 1, 1}; size_t local_work_size[] = {1, 1, 1}; CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, global_work_size, local_work_size, 0, NULL, NULL)); CL_CHECK(clEnqueueReadBuffer(queue, data_device, CL_TRUE, offset, size, data, 0, NULL, NULL)); CL_CHECK(clReleaseMemObject(data_device)); return; } #endif // GGML_OPENCL_USE_ADRENO_KERNELS cl_kernel kernel = backend_ctx->kernel_restore_block_q4_K; CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra->q)); CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra->s)); CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra->d)); CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra->dm)); CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &data_device)); CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_uchar), &mask_0F)); CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_uchar), &mask_F0)); size_t global_work_size[] = {(size_t)ggml_nelements(tensor)/ggml_blck_size(tensor->type), 1, 1}; size_t local_work_size[] = {1, 1, 1}; cl_event evt; CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, global_work_size, local_work_size, 0, NULL, &evt)); CL_CHECK(clWaitForEvents(1, &evt)); CL_CHECK(clEnqueueReadBuffer( queue, data_device, CL_TRUE, offset, size, data, 0, NULL, NULL)); CL_CHECK(clReleaseMemObject(data_device)); return; } if (tensor->type == GGML_TYPE_Q5_K) { ggml_tensor_extra_cl_q5_K * extra = (ggml_tensor_extra_cl_q5_K *)tensor->extra; cl_mem data_device = ggml_cl_create_temp_download_buffer(context, queue, ggml_nbytes(tensor), tensor->name); GGML_ASSERT(data_device != NULL && "get_tensor: temp download buffer alloc failed"); cl_uchar mask_0F = 0x0F; cl_uchar mask_F0 = 0xF0; #ifdef GGML_OPENCL_USE_ADRENO_KERNELS if (use_adreno_moe_kernels(backend_ctx, tensor)) { cl_mem data_device = ggml_cl_create_temp_download_buffer(context, queue, ggml_nbytes(tensor), tensor->name); GGML_ASSERT(data_device != NULL && "get_tensor: temp download buffer alloc failed"); cl_kernel kernel = backend_ctx->kernel_restore_block_q5_k_trans4_ns; int ne00 = tensor->ne[0]; int ne01 = tensor->ne[1]; int ne02 = tensor->ne[2]; CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra->q)); CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra->qh)); CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra->d)); CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra->dm)); CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extra->s)); CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_mem), &data_device)); CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_int), &ne00)); CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_int), &ne01)); CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_uchar), &mask_0F)); CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_uchar), &mask_F0)); size_t global_work_size[] = {static_cast(((ne01 + 63) / 64) * 64), static_cast(ne00 / 256), static_cast(ne02)}; size_t local_work_size[] = {64, 1, 1}; cl_event evt; CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, global_work_size, local_work_size, 0, NULL, &evt)); CL_CHECK(clWaitForEvents(1, &evt)); CL_CHECK(clEnqueueReadBuffer( queue, data_device, CL_TRUE, offset, size, data, 0, NULL, NULL)); CL_CHECK(clReleaseMemObject(data_device)); return; } if (enable_adreno_trans_weight_q5_K(backend_ctx, tensor)) { int M = tensor->ne[1]; int K = tensor->ne[0]; size_t size_q = extra->size_q; size_t size_qh = extra->size_qh; size_t size_s = extra->size_s; size_t size_d = extra->size_d; size_t size_dm = extra->size_dm; static ggml_cl_buffer buf_trans_q; static ggml_cl_buffer buf_trans_qh; static ggml_cl_buffer buf_trans_s; static ggml_cl_buffer buf_trans_d; static ggml_cl_buffer buf_trans_dm; buf_trans_q.allocate(backend_ctx->context, size_q); buf_trans_qh.allocate(backend_ctx->context, size_qh); buf_trans_s.allocate(backend_ctx->context, size_s); buf_trans_d.allocate(backend_ctx->context, size_d); buf_trans_dm.allocate(backend_ctx->context, size_dm); // bin kernel transposes s but src kernel does not cl_mem buf_s = extra->s; if (use_q5_k_bin_kernels(backend_ctx, tensor)) { transpose_2d_as_32b(backend_ctx, extra->q, buf_trans_q.buffer, size_q, M, K/8); transpose_2d_as_8b (backend_ctx, extra->s, buf_trans_s.buffer, size_s, M, K/256*12, true, true); buf_s = buf_trans_s.buffer; } else { transpose_2d_as_16b(backend_ctx, extra->q, buf_trans_q.buffer, size_q, M, K/4); } transpose_2d_as_8b (backend_ctx, extra->qh, buf_trans_qh.buffer, size_qh, M, K/8); transpose_2d_as_16b(backend_ctx, extra->d, buf_trans_d.buffer, size_d, M, K/256); transpose_2d_as_16b(backend_ctx, extra->dm, buf_trans_dm.buffer, size_dm, M, K/256); cl_kernel kernel = backend_ctx->kernel_restore_block_q5_K_noshuffle; CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &buf_trans_q.buffer)); CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &buf_trans_qh.buffer)); CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &buf_s)); CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &buf_trans_d.buffer)); CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &buf_trans_dm.buffer)); CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_mem), &data_device)); CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_uchar), &mask_0F)); CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_uchar), &mask_F0)); size_t global_work_size[] = {(size_t)ggml_nelements(tensor)/ggml_blck_size(tensor->type), 1, 1}; size_t local_work_size[] = {1, 1, 1}; CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, global_work_size, local_work_size, 0, NULL, NULL)); CL_CHECK(clEnqueueReadBuffer(queue, data_device, CL_TRUE, offset, size, data, 0, NULL, NULL)); CL_CHECK(clReleaseMemObject(data_device)); return; } #endif // GGML_OPENCL_USE_ADRENO_KERNELS cl_kernel kernel = backend_ctx->kernel_restore_block_q5_K; CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra->q)); CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra->qh)); CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra->s)); CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra->d)); CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extra->dm)); CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_mem), &data_device)); CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_uchar), &mask_0F)); CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_uchar), &mask_F0)); size_t global_work_size[] = {(size_t)ggml_nelements(tensor)/ggml_blck_size(tensor->type), 1, 1}; size_t local_work_size[] = {1, 1, 1}; cl_event evt; CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, global_work_size, local_work_size, 0, NULL, &evt)); CL_CHECK(clWaitForEvents(1, &evt)); CL_CHECK(clEnqueueReadBuffer( queue, data_device, CL_TRUE, offset, size, data, 0, NULL, NULL)); CL_CHECK(clReleaseMemObject(data_device)); return; } if (tensor->type == GGML_TYPE_Q6_K) { ggml_tensor_extra_cl_q6_K * extra = (ggml_tensor_extra_cl_q6_K *)tensor->extra; #ifdef GGML_OPENCL_USE_ADRENO_KERNELS // Undo the 64-row-tiled canonical pack (kernel_convert_block_q6_k_tiled_ns). // See the q4_K tiled restore above for why a read-back path is required. if (use_q6k_tiled(backend_ctx, tensor)) { const int ne00v = tensor->ne[0]; const int ne01v = tensor->ne[1]; const int nbv = ne00v / 256; const size_t n_blk = (size_t)nbv * ne01v; std::vector tql(n_blk*32), tqh(n_blk*16); std::vector ts(n_blk*16); std::vector td(n_blk); CL_CHECK(clEnqueueReadBuffer(queue, extra->ql, CL_TRUE, 0, tql.size()*4, tql.data(), 0, NULL, NULL)); CL_CHECK(clEnqueueReadBuffer(queue, extra->qh, CL_TRUE, 0, tqh.size()*4, tqh.data(), 0, NULL, NULL)); CL_CHECK(clEnqueueReadBuffer(queue, extra->s, CL_TRUE, 0, ts.size(), ts.data(), 0, NULL, NULL)); CL_CHECK(clEnqueueReadBuffer(queue, extra->d, CL_TRUE, 0, td.size()*2, td.data(), 0, NULL, NULL)); std::vector rebuilt(ggml_nbytes(tensor), 0); for (int i01 = 0; i01 < ne01v; ++i01) { const int rt = i01/64, rit = i01%64; for (int i00 = 0; i00 < nbv; ++i00) { uint8_t * b = rebuilt.data() + ((size_t)i00 + (size_t)i01*nbv)*210; const int tb = rt*nbv + i00; const size_t si = (size_t)tb*64 + rit; uint32_t qlw[32], qhw[16]; for (int g = 0; g < 8; ++g) { const size_t base = ((size_t)tb*8 + g)*64 + rit; for (int j = 0; j < 4; ++j) qlw[g*4 + j] = tql[base*4 + j]; } for (int g = 0; g < 4; ++g) { const size_t base = ((size_t)tb*4 + g)*64 + rit; for (int j = 0; j < 4; ++j) qhw[g*4 + j] = tqh[base*4 + j]; } uint8_t * ql = b; uint8_t * qh = b + 128; for (int e = 0; e < 256; ++e) { const int n = (e >= 128) ? 1 : 0; const int within = e - n*128, q = within/32, l = within%32; const int off_ql = n*64, off_qh = n*32; const uint8_t low4 = (qlw[e>>3] >> ((e&7)*4)) & 0xF; const uint8_t hi2 = (qhw[e>>4] >> ((e&15)*2)) & 0x3; if (q == 0) ql[off_ql + l] |= low4; else if (q == 1) ql[off_ql + l + 32] |= low4; else if (q == 2) ql[off_ql + l] |= (uint8_t)(low4 << 4); else ql[off_ql + l + 32] |= (uint8_t)(low4 << 4); qh[off_qh + l] |= (uint8_t)(hi2 << (q*2)); } memcpy(b + 192, &ts[si*16], 16); memcpy(b + 208, &td[si], 2); } } memcpy(data, rebuilt.data() + offset, size); return; } if (use_adreno_moe_kernels(backend_ctx, tensor)) { cl_mem data_device = ggml_cl_create_temp_download_buffer(context, queue, ggml_nbytes(tensor), tensor->name); GGML_ASSERT(data_device != NULL && "get_tensor: temp download buffer alloc failed"); cl_kernel kernel = backend_ctx->kernel_restore_block_q6_k_trans4_ns; cl_uchar mask_0F = 0x0F; cl_uchar mask_F0 = 0xF0; int ne00 = tensor->ne[0]; int ne01 = tensor->ne[1]; int ne02 = tensor->ne[2]; CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra->ql)); CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra->qh)); CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra->d)); CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra->s)); CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &data_device)); CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_int), &ne00)); CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_int), &ne01)); CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_uchar), &mask_0F)); CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_uchar), &mask_F0)); size_t global_work_size[] = {static_cast(((ne01 + 63) / 64) * 64), static_cast(ne00 / 256), static_cast(ne02)}; size_t local_work_size[] = {64, 1, 1}; cl_event evt; CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, global_work_size, local_work_size, 0, NULL, &evt)); CL_CHECK(clWaitForEvents(1, &evt)); CL_CHECK(clEnqueueReadBuffer( queue, data_device, CL_TRUE, offset, size, data, 0, NULL, NULL)); CL_CHECK(clReleaseMemObject(data_device)); return; } if (use_adreno_kernels(backend_ctx, tensor) && !use_flat_gemv_for_large_m_q6_K(backend_ctx, tensor)) { static ggml_cl_buffer buf_trans_ql; static ggml_cl_buffer buf_trans_qh; static ggml_cl_buffer buf_trans_s; static ggml_cl_buffer buf_trans_d; static ggml_cl_buffer buf_unpacked; cl_int M = tensor->ne[1]; // ne01 cl_int K = tensor->ne[0]; // ne00 GGML_ASSERT(K % ggml_blck_size(tensor->type) == 0); size_t size_ql = ggml_nelements(tensor)/ggml_blck_size(tensor->type)*ggml_blck_size(tensor->type)/2; size_t size_qh = ggml_nelements(tensor)/ggml_blck_size(tensor->type)*ggml_blck_size(tensor->type)/4; size_t size_s = ggml_nelements(tensor)/ggml_blck_size(tensor->type)*ggml_blck_size(tensor->type)/16; size_t size_d = ggml_nelements(tensor)/ggml_blck_size(tensor->type)*sizeof(ggml_fp16_t); GGML_ASSERT(size_ql + size_qh + size_s + size_d == ggml_nbytes(tensor) && "Incorrect tensor size"); buf_trans_ql.allocate(backend_ctx->context, size_ql); buf_trans_qh.allocate(backend_ctx->context, size_qh); buf_trans_d.allocate(backend_ctx->context, size_d); buf_unpacked.allocate(backend_ctx->context, ggml_nbytes(tensor)); cl_mem s_buffer; if (use_q6_k_bin_kernels(backend_ctx, tensor)) { transpose_2d_as_32b(backend_ctx, extra->ql, buf_trans_ql.buffer, size_ql, M, K/8); transpose_2d_as_32b(backend_ctx, extra->qh, buf_trans_qh.buffer, size_qh, M, K/16); // s is left row-major, untransposed, for the binary layout. s_buffer = extra->s; } else { // transpose ql, qh, s and d back buf_trans_s.allocate(backend_ctx->context, size_s); transpose_2d_as_16b(backend_ctx, extra->ql, buf_trans_ql.buffer, size_ql, M, K/4); transpose_2d_as_8b(backend_ctx, extra->qh, buf_trans_qh.buffer, size_qh, M, K/4); transpose_2d_as_16b(backend_ctx, extra->s, buf_trans_s.buffer, size_s, M, K/16/2); s_buffer = buf_trans_s.buffer; } transpose_2d_as_16b(backend_ctx, extra->d, buf_trans_d.buffer, size_d, M, K/256); // unpack cl_uchar mask = 0xFF; cl_ulong n_blk = ggml_nelements(tensor)/ggml_blck_size(tensor->type); cl_kernel kernel = backend_ctx->kernel_restore_block_q6_K_noshuffle; CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &buf_trans_ql.buffer)); CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &buf_trans_qh.buffer)); CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &s_buffer)); CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &buf_trans_d.buffer)); CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &buf_unpacked.buffer)); CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_uchar), &mask)); CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_ulong), &n_blk)); size_t global_work_size[] = {(size_t)n_blk, 1, 1}; size_t local_work_size[] = {1, 1, 1}; cl_event evt; CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, global_work_size, local_work_size, 0, NULL, &evt)); CL_CHECK(clWaitForEvents(1, &evt)); CL_CHECK(clEnqueueReadBuffer(queue, buf_unpacked.buffer, CL_TRUE, offset, size, data, 0, NULL, NULL)); return; } #endif // GGML_OPENCL_USE_ADRENO_KERNELS cl_mem data_device = ggml_cl_create_temp_download_buffer(context, queue, ggml_nbytes(tensor), tensor->name); GGML_ASSERT(data_device != NULL && "get_tensor: temp download buffer alloc failed"); cl_uchar mask = 0xFF; cl_ulong n_blk = ggml_nelements(tensor)/ggml_blck_size(tensor->type); cl_kernel kernel = backend_ctx->kernel_restore_block_q6_K; CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra->ql)); CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra->qh)); CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra->s)); CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra->d)); CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &data_device)); CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_uchar), &mask)); CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_ulong), &n_blk)); size_t global_work_size[] = {(size_t)n_blk, 1, 1}; size_t local_work_size[] = {1, 1, 1}; cl_event evt; CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, global_work_size, local_work_size, 0, NULL, &evt)); CL_CHECK(clWaitForEvents(1, &evt)); CL_CHECK(clEnqueueReadBuffer( queue, data_device, CL_TRUE, offset, size, data, 0, NULL, NULL)); CL_CHECK(clReleaseMemObject(data_device)); return; } #endif // GGML_OPENCL_SOA_Q if (tensor->type == GGML_TYPE_BF16) { GGML_ASSERT(offset % sizeof(ggml_fp16_t) == 0 && size % sizeof(ggml_fp16_t) == 0 && "Offset and size must be multiples of 2 for bf16 tensors"); ggml_tensor_extra_cl * extra = (ggml_tensor_extra_cl *) tensor->extra; GGML_ASSERT(extra); cl_ulong n_elements = size / sizeof(ggml_fp16_t); cl_ulong off_src = (extra->offset + tensor->view_offs + offset) / sizeof(ggml_fp16_t); cl_int err; cl_mem data_device = clCreateBuffer(context, CL_MEM_READ_WRITE, size, NULL, &err); CL_CHECK(err); cl_kernel kernel = backend_ctx->kernel_convert_f16_to_bf16; CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra->data_device)); CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &off_src)); CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &data_device)); CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &n_elements)); size_t global_work_size[] = { (size_t)CEIL_DIV(n_elements, 64)*64, 1, 1 }; size_t local_work_size[] = { 64, 1, 1 }; cl_event evt; CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, global_work_size, local_work_size, 0, NULL, &evt)); CL_CHECK(clWaitForEvents(1, &evt)); CL_CHECK(clReleaseEvent(evt)); CL_CHECK(clEnqueueReadBuffer( queue, data_device, CL_TRUE, 0, size, data, 0, NULL, NULL)); CL_CHECK(clReleaseMemObject(data_device)); return; } ggml_tensor_extra_cl * extra = (ggml_tensor_extra_cl *) tensor->extra; CL_CHECK(clEnqueueReadBuffer( queue, extra->data_device, CL_TRUE, extra->offset + tensor->view_offs + offset, size, data, 0, NULL, NULL)); GGML_UNUSED(buffer); } static void ggml_backend_opencl_buffer_clear(ggml_backend_buffer_t buffer, uint8_t value) { ggml_backend_opencl_device_context * dev_ctx = (ggml_backend_opencl_device_context *) buffer->buft->device->context; ggml_backend_opencl_context * backend_ctx = dev_ctx->backend_ctx; cl_command_queue queue = backend_ctx->queue; ggml_backend_opencl_buffer_context * ctx = (ggml_backend_opencl_buffer_context *) buffer->context; for (cl_mem buf : ctx->buffer) { CL_CHECK(clEnqueueFillBuffer(queue, buf, &value, sizeof(value), 0, buffer->size, 0, NULL, NULL)); } CL_CHECK(clFinish(queue)); } static void ggml_backend_opencl_buffer_reset(ggml_backend_buffer_t buffer) { ggml_backend_opencl_buffer_context * ctx = (ggml_backend_opencl_buffer_context *) buffer->context; ctx->reset(); } static ggml_backend_buffer_i ggml_backend_opencl_buffer_interface = { /* .free_buffer = */ ggml_backend_opencl_buffer_free_buffer, /* .get_base = */ ggml_backend_opencl_buffer_get_base, /* .init_tensor = */ ggml_backend_opencl_buffer_init_tensor, /* .memset_tensor = */ NULL, /* .set_tensor = */ ggml_backend_opencl_buffer_set_tensor, /* .get_tensor = */ ggml_backend_opencl_buffer_get_tensor, /* .set_tensor_2d = */ NULL, /* .get_tensor_2d = */ NULL, /* .cpy_tensor = */ NULL, /* .clear = */ ggml_backend_opencl_buffer_clear, /* .reset = */ ggml_backend_opencl_buffer_reset, }; // // buffer type // static const char * ggml_backend_opencl_buffer_type_get_name(ggml_backend_buffer_type_t buffer_type) { return "OpenCL"; GGML_UNUSED(buffer_type); } static ggml_backend_buffer_t ggml_backend_opencl_buffer_type_alloc_buffer(ggml_backend_buffer_type_t buffer_type, size_t size) { ggml_backend_opencl_context *backend_ctx = ggml_cl_init(buffer_type->device); if (!backend_ctx->program_cache_initialized) { backend_ctx->program_cache = cl_program_cache_init(backend_ctx->device); backend_ctx->program_cache_initialized = true; } load_cl_kernels(backend_ctx); // clCreateBuffer returns -61 for size 0 size = std::max(size, (size_t)1); cl_int err; cl_mem mem = clCreateBuffer(backend_ctx->context, CL_MEM_READ_WRITE, size, NULL, &err); // On Adreno X1-85 the device pool intermittently fails at hundreds of MB // once the heap fragments (e.g. graph-allocator compute-buffer reserve // after model load). Four-step retry: // 1. normal alloc (fast path) // 2. clFinish + retry (drains in-flight allocs) // 3. cl_qcom_large_buffer (X2-class driver only, OpenCL 3.0 only) // 4. ALLOC_HOST_PTR (host-pinned pool) — last-resort fallback. This // buffer backs compute scratch read/written by every kernel in the // graph, so kernel accesses fall to host memory and runtime perf // degrades meaningfully. Better than failing to load, but the user // should see the warning and consider -ngl reduction. if (err != CL_SUCCESS) { clFinish(backend_ctx->queue); mem = clCreateBuffer(backend_ctx->context, CL_MEM_READ_WRITE, size, NULL, &err); } #if GGML_OPENCL_TARGET_VERSION >= 300 // clCreateBufferWithProperties and cl_mem_properties are OpenCL 3.0. Drivers older than // that do not export the symbol, so a build targeting them fails to link. The large // buffer extension is only ever enabled on drivers that are well past 3.0, so this path // is dead there anyway. if (err != CL_SUCCESS && backend_ctx->adreno_use_large_buffer) { cl_mem_properties props[] = { 0x41A6 /* CL_LARGE_BUFFER_QCOM */, 1, 0 }; mem = clCreateBufferWithProperties(backend_ctx->context, props, CL_MEM_READ_WRITE, size, NULL, &err); } #endif if (err != CL_SUCCESS) { mem = clCreateBuffer(backend_ctx->context, CL_MEM_READ_WRITE | CL_MEM_ALLOC_HOST_PTR, size, NULL, &err); if (err == CL_SUCCESS) { GGML_LOG_WARN("%s: %.2f MiB allocated via CL_MEM_ALLOC_HOST_PTR fallback — " "device pool exhausted; runtime perf will be degraded. " "Consider lowering -ngl or context size.\n", __func__, size / 1024.0 / 1024.0); } } if (err != CL_SUCCESS) { GGML_LOG_ERROR("%s: failed to allocate %.2f MiB (err=%d). " "Consider reducing -ngl, lowering -c / -ub, or using quantized KV cache.\n", __func__, size / 1024.0 / 1024.0, err); return nullptr; } ggml_backend_opencl_buffer_context * ctx = new ggml_backend_opencl_buffer_context(mem); return ggml_backend_buffer_init(buffer_type, ggml_backend_opencl_buffer_interface, ctx, size); } static size_t ggml_backend_opencl_buffer_type_get_alignment(ggml_backend_buffer_type_t buffer_type) { ggml_backend_opencl_device_context * dev_ctx = (ggml_backend_opencl_device_context *) buffer_type->device->context; return dev_ctx->backend_ctx->alignment; } static size_t ggml_backend_opencl_buffer_type_get_max_size(ggml_backend_buffer_type_t buffer_type) { static size_t max_size = -1; if (max_size == (size_t)-1) { ggml_backend_opencl_device_context * dev_ctx = (ggml_backend_opencl_device_context *) buffer_type->device->context; max_size = dev_ctx->backend_ctx->max_alloc_size; } return max_size; } static bool ggml_backend_opencl_buffer_type_supports_backend(ggml_backend_buffer_type_t buft, ggml_backend_t backend) { return ggml_backend_is_opencl(backend); UNUSED(buft); } static size_t ggml_backend_opencl_buffer_type_get_alloc_size(ggml_backend_buffer_type_t buft, const ggml_tensor * tensor) { size_t size = ggml_nbytes(tensor); #ifdef GGML_OPENCL_SOA_Q // set_tensor carves quantized weights into per-component subbuffers (d/q, // ql/qh/s/d, ...) whose origins are each rounded up to the device base // alignment. When a component's size is not a multiple of the alignment // (e.g. q6_K [1536,49155]: size_s = 49155*96 leaves a 96-byte gap at 128-byte // alignment), the aligned carve extends past ggml_nbytes and the last // subbuffer would overlap the next tensor in the pool. Reserve the worst-case // carve slack: at most 5 components (q5_K), i.e. 4 aligned gaps. if (ggml_is_quantized(tensor->type)) { ggml_backend_opencl_device_context * dev_ctx = (ggml_backend_opencl_device_context *) buft->device->context; size += 4 * dev_ctx->backend_ctx->alignment; } #endif // GGML_OPENCL_SOA_Q return size; } static ggml_backend_buffer_type_i ggml_backend_opencl_buffer_type_interface = { /* .get_name = */ ggml_backend_opencl_buffer_type_get_name, /* .alloc_buffer = */ ggml_backend_opencl_buffer_type_alloc_buffer, /* .alloc_buffer_n = */ NULL, /* .get_alignment = */ ggml_backend_opencl_buffer_type_get_alignment, /* .get_max_size = */ ggml_backend_opencl_buffer_type_get_max_size, /* .get_alloc_size = */ ggml_backend_opencl_buffer_type_get_alloc_size, /* .get_alloc_size_n = */ NULL, /* .is_host = */ NULL, }; // // backend device // static const char * ggml_backend_opencl_device_get_name(ggml_backend_dev_t dev) { return "GPUOpenCL"; GGML_UNUSED(dev); } static const char * ggml_backend_opencl_device_get_description(ggml_backend_dev_t dev) { ggml_backend_opencl_device_context *dev_ctx = (ggml_backend_opencl_device_context *) dev->context; return dev_ctx->device_name.c_str(); } static void ggml_backend_opencl_device_get_memory(ggml_backend_dev_t dev, size_t * free, size_t * total) { ggml_backend_opencl_device_context * dev_ctx = (ggml_backend_opencl_device_context *) dev->context; static const size_t opencl_extra_margin = 1024ull*1024ull*1024ull; // OpenCL does not provide reliable currently-free device memory. // Use total/global memory as a best-effort upper bound. // Improved safety: Reduce by a 1GiB extra margin for common --fit *total = dev_ctx->global_mem_size; *free = *total > opencl_extra_margin ? *total - opencl_extra_margin : 0; } static enum ggml_backend_dev_type ggml_backend_opencl_device_get_type(ggml_backend_dev_t dev) { return GGML_BACKEND_DEVICE_TYPE_GPU; GGML_UNUSED(dev); } static void ggml_backend_opencl_device_get_props(ggml_backend_dev_t dev, struct ggml_backend_dev_props * props) { props->name = ggml_backend_opencl_device_get_name(dev); props->description = ggml_backend_opencl_device_get_description(dev); props->type = ggml_backend_opencl_device_get_type(dev); ggml_backend_opencl_device_get_memory(dev, &props->memory_free, &props->memory_total); props->caps = ggml_backend_dev_caps { /* .async = */ false, /* .host_buffer = */ false, /* .buffer_from_host_ptr = */ false, /* .events = */ false, /* .mmap_support = */ false, }; } static ggml_backend_t ggml_backend_opencl_device_init(ggml_backend_dev_t dev, const char * params) { ggml_backend_opencl_context * backend_ctx = ggml_cl_init(dev); // Getting a new reference to the backend, increase ref_count backend_ctx->ref_count++; ggml_backend_t backend = new ggml_backend { /* .guid = */ ggml_backend_opencl_guid(), /* .interface = */ ggml_backend_opencl_i, /* .device = */ dev, /* .context = */ backend_ctx, }; ggml_backend_opencl_device_context * dev_ctx = (ggml_backend_opencl_device_context *) dev->context; ggml_opencl_print_backend_info(dev_ctx); return backend; GGML_UNUSED(params); } static ggml_backend_buffer_type_t ggml_backend_opencl_device_get_buffer_type(ggml_backend_dev_t dev) { auto * dev_ctx = static_cast(dev->context); dev_ctx->buffer_type = ggml_backend_buffer_type{ /* .iface = */ ggml_backend_opencl_buffer_type_interface, /* .device = */ dev, /* .context = */ nullptr, }; return &dev_ctx->buffer_type; } static ggml_backend_buffer_t ggml_backend_opencl_device_buffer_from_ptr(ggml_backend_dev_t dev, void * ptr, size_t size, size_t max_tensor_size) { GGML_UNUSED(dev); GGML_UNUSED(ptr); GGML_UNUSED(size); GGML_UNUSED(max_tensor_size); return nullptr; } static bool ggml_backend_opencl_device_supports_op(ggml_backend_dev_t dev, const struct ggml_tensor * op) { ggml_cl_init(dev); return ggml_opencl_supports_op(dev, op); } static bool ggml_backend_opencl_device_supports_buft(ggml_backend_dev_t dev, ggml_backend_buffer_type_t buft) { // Check 'dev' and 'buffer_type' are not objects belonging to this backend. if (dev->iface.get_name != ggml_backend_opencl_device_get_name || buft->iface.get_name != ggml_backend_opencl_buffer_type_get_name) { return false; } // Check cl_context is the same. clEnqueue* commands may not use // buffers from another cl_context. ggml_backend_opencl_context * backend_ctx0 = ggml_cl_init(dev); ggml_backend_opencl_context * backend_ctx1 = ggml_cl_init(buft->device); return backend_ctx0->context == backend_ctx1->context; } namespace /* anonymous */ { struct ggml_backend_device_i ggml_backend_opencl_device_i = { /* .get_name = */ ggml_backend_opencl_device_get_name, /* .get_description = */ ggml_backend_opencl_device_get_description, /* .get_memory = */ ggml_backend_opencl_device_get_memory, /* .get_type = */ ggml_backend_opencl_device_get_type, /* .get_props = */ ggml_backend_opencl_device_get_props, /* .init_backend = */ ggml_backend_opencl_device_init, /* .get_buffer_type = */ ggml_backend_opencl_device_get_buffer_type, /* .get_host_buffer_type = */ NULL, /* .buffer_from_host_ptr = */ ggml_backend_opencl_device_buffer_from_ptr, /* .supports_op = */ ggml_backend_opencl_device_supports_op, /* .supports_buft = */ ggml_backend_opencl_device_supports_buft, /* .offload_op = */ NULL, /* .event_new = */ NULL, /* .event_free = */ NULL, /* .event_synchronize = */ NULL, }; } // Backend registry static const char * ggml_backend_opencl_reg_get_name(ggml_backend_reg_t reg) { return "OpenCL"; GGML_UNUSED(reg); } static size_t ggml_backend_opencl_reg_device_count(ggml_backend_reg_t reg) { return g_ggml_backend_opencl_devices.size(); GGML_UNUSED(reg); } static ggml_backend_dev_t ggml_backend_opencl_reg_device_get(ggml_backend_reg_t reg, size_t index) { GGML_ASSERT(index < ggml_backend_opencl_reg_device_count(reg)); return &g_ggml_backend_opencl_devices[index]; GGML_UNUSED(reg); GGML_UNUSED(index); } static struct ggml_backend_reg_i ggml_backend_opencl_reg_i = { /* .get_name = */ ggml_backend_opencl_reg_get_name, /* .device_count = */ ggml_backend_opencl_reg_device_count, /* .device_get = */ ggml_backend_opencl_reg_device_get, /* .get_proc_address = */ NULL, }; ggml_backend_reg_t ggml_backend_opencl_reg(void) { static std::mutex mutex; static ggml_backend_reg reg; static bool initialized = false; std::lock_guard lock(mutex); if (initialized) { return ® } initialized = true; g_ggml_backend_opencl_devices = ggml_opencl_probe_devices(®); reg = ggml_backend_reg{ /* .api_version = */ GGML_BACKEND_API_VERSION, /* .iface = */ ggml_backend_opencl_reg_i, /* .context = */ NULL, }; return ® } GGML_BACKEND_DL_IMPL(ggml_backend_opencl_reg) //------------------------------------------------------------------------------ // Debugging utils //------------------------------------------------------------------------------ #if 0 #define QK4_0 32 typedef struct { ggml_fp16_t d; // delta uint8_t qs[QK4_0 / 2]; // nibbles / quants } block_q4_0; static_assert(sizeof(block_q4_0) == sizeof(ggml_fp16_t) + QK4_0 / 2, "wrong q4_0 block size/padding"); #define QK_MXFP4 32 #include #ifdef __cplusplus #include "half.hpp" #endif static void dump_tensor(ggml_backend_t backend, const struct ggml_tensor * tensor) { void * buf = malloc(ggml_nbytes(tensor)); ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; cl_command_queue queue = backend_ctx->queue; #ifdef GGML_OPENCL_SOA_Q void * buf_q; void * buf_d; #endif // Make sure everything is done. CL_CHECK(clFinish(queue)); #ifdef GGML_OPENCL_SOA_Q if (tensor->type == GGML_TYPE_Q4_0) { ggml_tensor_extra_cl_q4_0 * extra = (ggml_tensor_extra_cl_q4_0 *) tensor->extra; GGML_ASSERT(extra); size_t size_q = ggml_nelements(tensor)/QK4_0 * QK4_0/2; size_t size_d = ggml_nelements(tensor)/QK4_0 * sizeof(ggml_fp16_t); GGML_ASSERT(size_q + size_d == ggml_nbytes(tensor)); buf_q = malloc(size_q); buf_d = malloc(size_d); CL_CHECK(clEnqueueReadBuffer(queue, extra->q, CL_TRUE, 0, size_q, buf_q, 0, NULL, NULL)); CL_CHECK(clEnqueueReadBuffer(queue, extra->d, CL_TRUE, 0, size_d, buf_d, 0, NULL, NULL)); CL_CHECK(clFinish(queue)); } else if (tensor->type == GGML_TYPE_MXFP4) { ggml_tensor_extra_cl_mxfp4 * extra = (ggml_tensor_extra_cl_mxfp4 *) tensor->extra; GGML_ASSERT(extra); size_t size_q = ggml_nelements(tensor)/QK_MXFP4 * QK_MXFP4/2; size_t size_e = ggml_nelements(tensor)/QK_MXFP4 * sizeof(char); GGML_ASSERT(size_q + size_e == ggml_nbytes(tensor)); buf_q = malloc(size_q); buf_d = malloc(size_e); CL_CHECK(clEnqueueReadBuffer(queue, extra->q, CL_TRUE, 0, size_q, buf_q, 0, NULL, NULL)); CL_CHECK(clEnqueueReadBuffer(queue, extra->e, CL_TRUE, 0, size_e, buf_d, 0, NULL, NULL)); CL_CHECK(clFinish(queue)); } else { // Read out the tensor from GPU memory. ggml_tensor_extra_cl * extra = (ggml_tensor_extra_cl *) tensor->extra; GGML_ASSERT(extra); CL_CHECK(clEnqueueReadBuffer(queue, extra->data_device, CL_TRUE, extra->offset, ggml_nbytes(tensor), buf, 0, NULL, NULL)); CL_CHECK(clFinish(queue)); } #else // Read out the tensor from GPU memory. ggml_tensor_extra_cl * extra = (ggml_tensor_extra_cl *) tensor->extra; GGML_ASSERT(extra); CL_CHECK(clEnqueueReadBuffer(queue, extra->data_device, CL_TRUE, extra->offset, ggml_nbytes(tensor), buf, 0, NULL, NULL)); CL_CHECK(clFinish(queue)); #endif // GGML_OPENCL_SOA_Q // Open file and dump. char fname[512]; snprintf(fname, sizeof(fname), "./tensor-dumps/%s.txt", tensor->name); FILE * f = fopen(fname, "w"); if (!f) { printf("Failed to open %s\n", fname); return; } if (tensor->type == GGML_TYPE_F32) { float * data = (float *) buf; for (int i = 0; i < ggml_nelements(tensor); ++i) { if (isnan(data[i])) { printf("NaN found: %s\n", tensor->name); break; } fprintf(f, "%f\n", data[i]); } } else if (tensor->type == GGML_TYPE_I32) { int * data = (int *) buf; for (int i = 0; i < ggml_nelements(tensor); ++i) { if (isnan(data[i])) { printf("NaN found: %s\n", tensor->name); break; } fprintf(f, "%d\n", data[i]); } } else if (tensor->type == GGML_TYPE_F16) { #ifdef __cplusplus half_float::half * data = (half_float::half *) buf; for (int i = 0; i < ggml_nelements(tensor); ++i) { if (std::isnan(data[i])) { printf("NaN found: %s\n", tensor->name); break; } fprintf(f, "%f\n", float(data[i])); } #endif } else if (tensor->type == GGML_TYPE_Q4_0) { #ifdef GGML_OPENCL_SOA_Q ggml_fp16_t * data_d = (ggml_fp16_t *)buf_d; unsigned char * data_q = (unsigned char *)buf_q; for (int i = 0; i < ggml_nelements(tensor)/QK4_0; ++i) { fprintf(f, "%04x, ", data_d[i]); for (int k = 0; k < QK4_0/2; ++k) { fprintf(f, "%02x, ", data_q[k]); } fprintf(f, "\n"); data_q += QK4_0/2; } free(buf_d); free(buf_q); #else block_q4_0 * data = (block_q4_0 *) buf; for (int i = 0; i < ggml_nelements(tensor)/QK4_0; ++i) { fprintf(f, "%04x, ", data[i].d); for (int k = 0; k < QK4_0/2; ++k) { fprintf(f, "%02x, ", data[i].qs[k]); } fprintf(f, "\n"); } #endif // GGML_OPENCL_SOA_Q } free(buf); fflush(f); fclose(f); } #else #define dump_tensor(tensor) #endif //------------------------------------------------------------------------------ // Ops //------------------------------------------------------------------------------ static bool ggml_cl_can_mul_mat(const struct ggml_tensor * src0, const struct ggml_tensor * src1, struct ggml_tensor * dst) { const int64_t ne10 = src1->ne[0]; const int64_t ne0 = dst->ne[0]; const int64_t ne1 = dst->ne[1]; // TODO: find the optimal values for these return (src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16 || ggml_is_quantized(src0->type)) && src1->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32 && (ne0 >= 32 && ne1 >= 32 && ne10 >= 32); } // Copy a noncontiguous tensor to contiguous tensor. ne[] remains the same but // nb[] is recalculated such that tensor is contiguous. static void ggml_cl_copy_to_contiguous(ggml_backend_t backend, const ggml_tensor * src, cl_mem dst, cl_ulong &nb0, cl_ulong &nb1, cl_ulong &nb2, cl_ulong &nb3) { ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; const int tensor_type_size = ggml_type_size(src->type); const int ne00 = src->ne[0]; const int ne01 = src->ne[1]; const int ne02 = src->ne[2]; const int ne03 = src->ne[3]; const cl_ulong nb00 = src->nb[0]; const cl_ulong nb01 = src->nb[1]; const cl_ulong nb02 = src->nb[2]; const cl_ulong nb03 = src->nb[3]; const int ne0 = src->ne[0]; const int ne1 = src->ne[1]; const int ne2 = src->ne[2]; const int ne3 = src->ne[3]; nb0 = tensor_type_size; nb1 = tensor_type_size*ne00; nb2 = tensor_type_size*ne00*ne01; nb3 = tensor_type_size*ne00*ne01*ne02; ggml_tensor_extra_cl * extra = (ggml_tensor_extra_cl *)src->extra; cl_ulong offset0 = extra->offset + src->view_offs; cl_ulong offsetd = 0; cl_kernel kernel; switch (src->type) { case GGML_TYPE_F32: kernel = backend_ctx->kernel_cpy_f32_f32; break; case GGML_TYPE_F16: case GGML_TYPE_BF16: // stored as f16 on device kernel = backend_ctx->kernel_cpy_f16_f16; break; default: GGML_ASSERT(false && "not implemented"); } CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra->data_device)); CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &dst)); CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offsetd)); CL_CHECK(clSetKernelArg(kernel, 4, sizeof(int), &ne00)); CL_CHECK(clSetKernelArg(kernel, 5, sizeof(int), &ne01)); CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne02)); CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne03)); CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_ulong), &nb00)); CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_ulong), &nb01)); CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_ulong), &nb02)); CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_ulong), &nb03)); CL_CHECK(clSetKernelArg(kernel, 12, sizeof(int), &ne0)); CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &ne1)); CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &ne2)); CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &ne3)); CL_CHECK(clSetKernelArg(kernel, 16, sizeof(cl_ulong), &nb0)); CL_CHECK(clSetKernelArg(kernel, 17, sizeof(cl_ulong), &nb1)); CL_CHECK(clSetKernelArg(kernel, 18, sizeof(cl_ulong), &nb2)); CL_CHECK(clSetKernelArg(kernel, 19, sizeof(cl_ulong), &nb3)); const int nth = MIN(64, ne00); size_t global_work_size[] = {(size_t)ne01*nth, (size_t)ne02, (size_t)ne03}; size_t local_work_size[] = {(size_t)nth, 1, 1}; backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, src); } static void ggml_cl_nop(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { UNUSED(backend); UNUSED(src0); UNUSED(src1); UNUSED(dst); } static void ggml_cl_get_rows(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { GGML_ASSERT(src0); GGML_ASSERT(src0->extra); GGML_ASSERT(src1); GGML_ASSERT(src1->extra); GGML_ASSERT(dst); GGML_ASSERT(dst->extra); GGML_TENSOR_LOCALS(int, ne0, src0, ne); GGML_TENSOR_LOCALS(cl_ulong, nb0, src0, nb); GGML_TENSOR_LOCALS(int, ne1, src1, ne); GGML_TENSOR_LOCALS(cl_ulong, nb1, src1, nb); GGML_TENSOR_LOCALS(int, ne, dst, ne); GGML_TENSOR_LOCALS(cl_ulong, nb, dst, nb); ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; ggml_tensor_extra_cl * extra0 = (ggml_tensor_extra_cl *)src0->extra; ggml_tensor_extra_cl * extra1 = (ggml_tensor_extra_cl *)src1->extra; ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; cl_ulong offset0 = extra0->offset + src0->view_offs; cl_ulong offset1 = extra1->offset + src1->view_offs; cl_ulong offsetd = extrad->offset + dst->view_offs; cl_kernel kernel; switch (src0->type) { case GGML_TYPE_F32: kernel = backend_ctx->kernel_get_rows_f32; break; case GGML_TYPE_F16: kernel = backend_ctx->kernel_get_rows_f16; break; case GGML_TYPE_Q4_0: kernel = backend_ctx->kernel_get_rows_q4_0; break; default: GGML_ASSERT(false && "not implemented"); } CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device)); CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1)); CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device)); CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd)); CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne00)); CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_ulong), &nb01)); CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_ulong), &nb02)); CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_ulong), &nb03)); CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &ne10)); CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_ulong), &nb10)); CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_ulong), &nb11)); CL_CHECK(clSetKernelArg(kernel, 13, sizeof(cl_ulong), &nb12)); CL_CHECK(clSetKernelArg(kernel, 14, sizeof(cl_ulong), &nb1)); CL_CHECK(clSetKernelArg(kernel, 15, sizeof(cl_ulong), &nb2)); CL_CHECK(clSetKernelArg(kernel, 16, sizeof(cl_ulong), &nb3)); int max_workgroup_size = backend_ctx->get_kernel_workgroup_size(kernel); int nth = 1; while (nth < ne00 && 2*nth <= max_workgroup_size) { nth *= 2; } int nchunks = 1; if (src0->type == GGML_TYPE_F32) { const int chunk_target = nth * 4; nchunks = (ne00 + chunk_target - 1) / chunk_target; nchunks = MAX(1, MIN(nchunks, 64)); } size_t global_work_size[] = {(size_t)ne10*nth*nchunks, (size_t)ne11, (size_t)ne12}; size_t local_work_size[] = {(size_t)nth, 1, 1}; backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); } // check if a Q8_0 tensor has been SOA'ed in set_tensor // we store SOA'ed tensors in a map in set_tensor, check against that map static bool ggml_cl_is_q8_0_soa(const ggml_tensor * tensor) { if (tensor == nullptr || tensor->type != GGML_TYPE_Q8_0 || tensor->buffer == nullptr) { return false; } auto * ctx = (ggml_backend_opencl_buffer_context *) tensor->buffer->context; if (ctx == nullptr) { return false; } const ggml_tensor * key = tensor->view_src != nullptr ? tensor->view_src : tensor; return ctx->q8_0_soa_tensors.count(key) > 0; } // check if a Q4_0 tensor has been SOA'ed in set_tensor // we store SOA'ed tensors in a map in set_tensor, check against that map static bool ggml_cl_is_q4_0_soa(const ggml_tensor * tensor) { if (tensor == nullptr || tensor->type != GGML_TYPE_Q4_0 || tensor->buffer == nullptr) { return false; } auto * ctx = (ggml_backend_opencl_buffer_context *) tensor->buffer->context; if (ctx == nullptr) { return false; } const ggml_tensor * key = tensor->view_src != nullptr ? tensor->view_src : tensor; return ctx->q4_0_soa_tensors.count(key) > 0; } static void ggml_cl_set_rows(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { GGML_ASSERT(src0); GGML_ASSERT(src0->extra); GGML_ASSERT(src1); GGML_ASSERT(src1->extra); GGML_ASSERT(dst); GGML_ASSERT(dst->extra); GGML_ASSERT(src1->type == GGML_TYPE_I64 || src1->type == GGML_TYPE_I32); // ne0 = ne00 // ne2 = ne02 // ne3 = ne03 GGML_TENSOR_LOCALS(int, ne0, src0, ne); GGML_TENSOR_LOCALS(cl_ulong, nb0, src0, nb); GGML_TENSOR_LOCALS(int, ne1, src1, ne); GGML_TENSOR_LOCALS(cl_ulong, nb1, src1, nb); GGML_TENSOR_LOCALS(int, ne, dst, ne); GGML_TENSOR_LOCALS(cl_ulong, nb, dst, nb); const int nblk0 = ne0/ggml_blck_size(dst->type); ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; ggml_tensor_extra_cl * extra0 = (ggml_tensor_extra_cl *)src0->extra; ggml_tensor_extra_cl * extra1 = (ggml_tensor_extra_cl *)src1->extra; cl_ulong offset0 = extra0->offset + src0->view_offs; cl_ulong offset1 = extra1->offset + src1->view_offs; const bool q8_0_soa = dst->type == GGML_TYPE_Q8_0 && ggml_cl_is_q8_0_soa(dst); const bool q4_0_soa = dst->type == GGML_TYPE_Q4_0 && ggml_cl_is_q4_0_soa(dst); const bool is_soa = q8_0_soa || q4_0_soa; cl_kernel kernel; if (q8_0_soa) { kernel = (src1->type == GGML_TYPE_I64) ? backend_ctx->kernel_set_rows_q8_0_soa_i64 : backend_ctx->kernel_set_rows_q8_0_soa_i32; } else if (q4_0_soa) { kernel = (src1->type == GGML_TYPE_I64) ? backend_ctx->kernel_set_rows_q4_0_soa_i64 : backend_ctx->kernel_set_rows_q4_0_soa_i32; } else { switch (dst->type) { case GGML_TYPE_F32: kernel = (src1->type == GGML_TYPE_I64) ? backend_ctx->kernel_set_rows_f32_i64 : backend_ctx->kernel_set_rows_f32_i32; break; case GGML_TYPE_F16: kernel = (src1->type == GGML_TYPE_I64) ? backend_ctx->kernel_set_rows_f16_i64 : backend_ctx->kernel_set_rows_f16_i32; break; case GGML_TYPE_Q8_0: kernel = (src1->type == GGML_TYPE_I64) ? backend_ctx->kernel_set_rows_q8_0_i64 : backend_ctx->kernel_set_rows_q8_0_i32; break; case GGML_TYPE_Q4_0: kernel = (src1->type == GGML_TYPE_I64) ? backend_ctx->kernel_set_rows_q4_0_i64 : backend_ctx->kernel_set_rows_q4_0_i32; break; default: GGML_ABORT("not implemented"); } } fastdiv_vals ne11_ = init_fastdiv_values(ne11); fastdiv_vals ne12_ = init_fastdiv_values(ne12); CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device)); CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1)); if (is_soa) { // The q/d subbuffers in q8_0/q4_0 extras are interchangeable here. // For views (e.g. ggml_set_rows' `out`), follow view_src for the SoA extra. const ggml_tensor * soa_src = dst->view_src != nullptr ? dst->view_src : dst; cl_mem q_mem = nullptr; cl_mem d_mem = nullptr; if (q8_0_soa) { ggml_tensor_extra_cl_q8_0 * e = (ggml_tensor_extra_cl_q8_0 *)soa_src->extra; q_mem = e->q; d_mem = e->d; } else { ggml_tensor_extra_cl_q4_0 * e = (ggml_tensor_extra_cl_q4_0 *)soa_src->extra; q_mem = e->q; d_mem = e->d; } cl_ulong offset_q = 0; cl_ulong offset_d = 0; const int ne1_dst = dst->ne[1]; const int ne2_dst = dst->ne[2]; const int ne3_dst = dst->ne[3]; CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &q_mem)); CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offset_q)); CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_mem), &d_mem)); CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_ulong), &offset_d)); CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne01)); CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_ulong), &nb01)); CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_ulong), &nb02)); CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_ulong), &nb03)); CL_CHECK(clSetKernelArg(kernel, 12, sizeof(fastdiv_vals), &ne11_)); CL_CHECK(clSetKernelArg(kernel, 13, sizeof(fastdiv_vals), &ne12_)); CL_CHECK(clSetKernelArg(kernel, 14, sizeof(cl_ulong), &nb10)); CL_CHECK(clSetKernelArg(kernel, 15, sizeof(cl_ulong), &nb11)); CL_CHECK(clSetKernelArg(kernel, 16, sizeof(cl_ulong), &nb12)); CL_CHECK(clSetKernelArg(kernel, 17, sizeof(int), &nblk0)); CL_CHECK(clSetKernelArg(kernel, 18, sizeof(int), &ne1_dst)); CL_CHECK(clSetKernelArg(kernel, 19, sizeof(int), &ne2_dst)); CL_CHECK(clSetKernelArg(kernel, 20, sizeof(int), &ne3_dst)); } else { ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; cl_ulong offsetd = extrad->offset + dst->view_offs; CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device)); CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd)); CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne01)); CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_ulong), &nb01)); CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_ulong), &nb02)); CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_ulong), &nb03)); CL_CHECK(clSetKernelArg(kernel, 10, sizeof(fastdiv_vals), &ne11_)); CL_CHECK(clSetKernelArg(kernel, 11, sizeof(fastdiv_vals), &ne12_)); CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_ulong), &nb10)); CL_CHECK(clSetKernelArg(kernel, 13, sizeof(cl_ulong), &nb11)); CL_CHECK(clSetKernelArg(kernel, 14, sizeof(cl_ulong), &nb12)); CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &nblk0)); CL_CHECK(clSetKernelArg(kernel, 16, sizeof(cl_ulong), &nb1)); CL_CHECK(clSetKernelArg(kernel, 17, sizeof(cl_ulong), &nb2)); CL_CHECK(clSetKernelArg(kernel, 18, sizeof(cl_ulong), &nb3)); } int nth0 = 64; if (backend_ctx->gpu_family == INTEL) { nth0 = 32; } else if (backend_ctx->gpu_family == ADRENO) { nth0 = 64; } int max_workgroup_size = backend_ctx->get_kernel_workgroup_size(kernel); while (nth0 < nblk0 && nth0 < max_workgroup_size) { nth0 *= 2; } int rows_per_workgroup = 1; if (nth0 > nblk0) { rows_per_workgroup = nth0 / nblk0; nth0 = nblk0; } size_t global_work_size[] = { (size_t)(ne01 + rows_per_workgroup - 1)/rows_per_workgroup*nth0, (size_t)ne02*rows_per_workgroup, (size_t)ne03}; size_t local_work_size[] = {(size_t)nth0, (size_t)rows_per_workgroup, 1}; // ne01 == 0 makes global_work_size[0] zero here; enqueue_ndrange_kernel drops the empty range. backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); } static void ggml_cl_add(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { GGML_ASSERT(src0); GGML_ASSERT(src0->extra); GGML_ASSERT(src1); GGML_ASSERT(src1->extra); GGML_ASSERT(dst); GGML_ASSERT(dst->extra); const int ne00 = src0->ne[0]; const int ne01 = src0->ne[1]; const int ne02 = src0->ne[2]; const int ne03 = src0->ne[3]; const cl_ulong nb00 = src0->nb[0]; const cl_ulong nb01 = src0->nb[1]; const cl_ulong nb02 = src0->nb[2]; const cl_ulong nb03 = src0->nb[3]; const int ne10 = src1->ne[0]; const int ne11 = src1->ne[1]; const int ne12 = src1->ne[2]; const int ne13 = src1->ne[3]; const cl_ulong nb10 = src1->nb[0]; const cl_ulong nb11 = src1->nb[1]; const cl_ulong nb12 = src1->nb[2]; const cl_ulong nb13 = src1->nb[3]; const int ne0 = dst->ne[0]; const int ne1 = dst->ne[1]; const int ne2 = dst->ne[2]; const int ne3 = dst->ne[3]; const cl_ulong nb0 = dst->nb[0]; const cl_ulong nb1 = dst->nb[1]; const cl_ulong nb2 = dst->nb[2]; const cl_ulong nb3 = dst->nb[3]; ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; ggml_tensor_extra_cl * extra0 = (ggml_tensor_extra_cl *)src0->extra; ggml_tensor_extra_cl * extra1 = (ggml_tensor_extra_cl *)src1->extra; ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; cl_ulong offset0 = extra0->offset + src0->view_offs; cl_ulong offset1 = extra1->offset + src1->view_offs; cl_ulong offsetd = extrad->offset + dst->view_offs; cl_kernel kernel; const bool bcast_row = ggml_nelements(src1) == ne10 && ggml_is_contiguous(src1) && ne00 % 4 == 0 && ne10 % 4 == 0; if (bcast_row) { GGML_ASSERT(ggml_is_contiguous(src0)); GGML_ASSERT(ne11 == 1); } if (dst->type == GGML_TYPE_F32) { GGML_ASSERT(src0->type == GGML_TYPE_F32 && src1->type == GGML_TYPE_F32); if (bcast_row) { kernel = backend_ctx->kernel_add_row; const int ne = ne00 / 4; CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device)); CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1)); CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device)); CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd)); CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne)); } else { kernel = backend_ctx->kernel_add; CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device)); CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1)); CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device)); CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd)); CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne00)); CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne01)); CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne02)); CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne03)); CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_ulong), &nb00)); CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_ulong), &nb01)); CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_ulong), &nb02)); CL_CHECK(clSetKernelArg(kernel, 13, sizeof(cl_ulong), &nb03)); CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &ne10)); CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &ne11)); CL_CHECK(clSetKernelArg(kernel, 16, sizeof(int), &ne12)); CL_CHECK(clSetKernelArg(kernel, 17, sizeof(int), &ne13)); CL_CHECK(clSetKernelArg(kernel, 18, sizeof(cl_ulong), &nb10)); CL_CHECK(clSetKernelArg(kernel, 19, sizeof(cl_ulong), &nb11)); CL_CHECK(clSetKernelArg(kernel, 20, sizeof(cl_ulong), &nb12)); CL_CHECK(clSetKernelArg(kernel, 21, sizeof(cl_ulong), &nb13)); CL_CHECK(clSetKernelArg(kernel, 22, sizeof(int), &ne0)); CL_CHECK(clSetKernelArg(kernel, 23, sizeof(int), &ne1)); CL_CHECK(clSetKernelArg(kernel, 24, sizeof(int), &ne2)); CL_CHECK(clSetKernelArg(kernel, 25, sizeof(int), &ne3)); CL_CHECK(clSetKernelArg(kernel, 26, sizeof(cl_ulong), &nb0)); CL_CHECK(clSetKernelArg(kernel, 27, sizeof(cl_ulong), &nb1)); CL_CHECK(clSetKernelArg(kernel, 28, sizeof(cl_ulong), &nb2)); CL_CHECK(clSetKernelArg(kernel, 29, sizeof(cl_ulong), &nb3)); } } else if (dst->type == GGML_TYPE_F16) { GGML_ASSERT(src0->type == GGML_TYPE_F16 || src0->type == GGML_TYPE_F32); GGML_ASSERT(src1->type == GGML_TYPE_F16 || src1->type == GGML_TYPE_F32); const int type_src0 = (src0->type == GGML_TYPE_F32); const int type_src1 = (src1->type == GGML_TYPE_F32); if (bcast_row) { kernel = backend_ctx->kernel_add_row_f16; const int ne = ne00 / 4; CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device)); CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1)); CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device)); CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd)); CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne)); CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &type_src0)); CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &type_src1)); } else { kernel = backend_ctx->kernel_add_f16; CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device)); CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1)); CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device)); CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd)); CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne00)); CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne01)); CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne02)); CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne03)); CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_ulong), &nb00)); CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_ulong), &nb01)); CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_ulong), &nb02)); CL_CHECK(clSetKernelArg(kernel, 13, sizeof(cl_ulong), &nb03)); CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &ne10)); CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &ne11)); CL_CHECK(clSetKernelArg(kernel, 16, sizeof(int), &ne12)); CL_CHECK(clSetKernelArg(kernel, 17, sizeof(int), &ne13)); CL_CHECK(clSetKernelArg(kernel, 18, sizeof(cl_ulong), &nb10)); CL_CHECK(clSetKernelArg(kernel, 19, sizeof(cl_ulong), &nb11)); CL_CHECK(clSetKernelArg(kernel, 20, sizeof(cl_ulong), &nb12)); CL_CHECK(clSetKernelArg(kernel, 21, sizeof(cl_ulong), &nb13)); CL_CHECK(clSetKernelArg(kernel, 22, sizeof(int), &ne0)); CL_CHECK(clSetKernelArg(kernel, 23, sizeof(int), &ne1)); CL_CHECK(clSetKernelArg(kernel, 24, sizeof(int), &ne2)); CL_CHECK(clSetKernelArg(kernel, 25, sizeof(int), &ne3)); CL_CHECK(clSetKernelArg(kernel, 26, sizeof(cl_ulong), &nb0)); CL_CHECK(clSetKernelArg(kernel, 27, sizeof(cl_ulong), &nb1)); CL_CHECK(clSetKernelArg(kernel, 28, sizeof(cl_ulong), &nb2)); CL_CHECK(clSetKernelArg(kernel, 29, sizeof(cl_ulong), &nb3)); CL_CHECK(clSetKernelArg(kernel, 30, sizeof(int), &type_src0)); CL_CHECK(clSetKernelArg(kernel, 31, sizeof(int), &type_src1)); } } else { GGML_ASSERT(false && "unsupported data types for add"); } if (bcast_row) { int n = ggml_nelements(dst)/4; size_t global_work_size[] = {(size_t)n, 1, 1}; size_t local_work_size[] = {64, 1, 1}; size_t * local_work_size_ptr = local_work_size; if (n % 64 != 0 && !backend_ctx->non_uniform_workgroups) { local_work_size_ptr = nullptr; } backend_ctx->enqueue_ndrange_kernel(kernel, 1, global_work_size, local_work_size_ptr, dst); } else { unsigned int nth = MIN(64, ne0); size_t global_work_size[] = {(size_t)ne01*nth, (size_t)ne02, (size_t)ne03}; size_t local_work_size[] = {nth, 1, 1}; backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); } } static void ggml_cl_add_id(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { GGML_ASSERT(src0); GGML_ASSERT(src0->extra); GGML_ASSERT(src1); GGML_ASSERT(src1->extra); GGML_ASSERT(dst); GGML_ASSERT(dst->extra); const ggml_tensor * src2 = dst->src[2]; GGML_ASSERT(src2); GGML_ASSERT(src2->extra); GGML_ASSERT(src0->type == GGML_TYPE_F32); GGML_ASSERT(src1->type == GGML_TYPE_F32); GGML_ASSERT(src2->type == GGML_TYPE_I32); GGML_ASSERT(dst->type == GGML_TYPE_F32); GGML_ASSERT(ggml_is_contiguous_rows(src0)); const int ne00 = src0->ne[0]; const int ne01 = src0->ne[1]; const int ne02 = src0->ne[2]; const cl_ulong nb01 = src0->nb[1]; const cl_ulong nb02 = src0->nb[2]; const cl_ulong nb11 = src1->nb[1]; const cl_ulong nb21 = src2->nb[1]; const int ne0 = dst->ne[0]; const int ne1 = dst->ne[1]; ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; ggml_tensor_extra_cl * extra0 = (ggml_tensor_extra_cl *)src0->extra; ggml_tensor_extra_cl * extra1 = (ggml_tensor_extra_cl *)src1->extra; ggml_tensor_extra_cl * extra2 = (ggml_tensor_extra_cl *)src2->extra; ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; cl_ulong offset0 = extra0->offset + src0->view_offs; cl_ulong offset1 = extra1->offset + src1->view_offs; cl_ulong offset2 = extra2->offset + src2->view_offs; cl_ulong offsetd = extrad->offset + dst->view_offs; cl_kernel kernel = backend_ctx->kernel_add_id; CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device)); CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1)); CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extra2->data_device)); CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offset2)); CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_mem), &extrad->data_device)); CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_ulong), &offsetd)); CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_ulong), &nb01)); CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_ulong), &nb02)); CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_ulong), &nb11)); CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_ulong), &nb21)); CL_CHECK(clSetKernelArg(kernel, 12, sizeof(int), &ne0)); CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &ne1)); int nth = MIN(ne00, (int) backend_ctx->get_kernel_workgroup_size(kernel)); size_t global_work_size[] = { (size_t)ne01*nth, (size_t)ne02, 1 }; size_t local_work_size[] = { (size_t)nth, 1, 1 }; backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); } static void ggml_cl_mul(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { GGML_ASSERT(src0); GGML_ASSERT(src0->extra); GGML_ASSERT(src1); GGML_ASSERT(src1->extra); GGML_ASSERT(dst); GGML_ASSERT(dst->extra); GGML_ASSERT(src0->type == src1->type); GGML_ASSERT(src0->type == dst->type); GGML_ASSERT(src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16); const int ne00 = src0->ne[0]; const int ne01 = src0->ne[1]; const int ne02 = src0->ne[2]; const int ne03 = src0->ne[3]; const cl_ulong nb00 = src0->nb[0]; const cl_ulong nb01 = src0->nb[1]; const cl_ulong nb02 = src0->nb[2]; const cl_ulong nb03 = src0->nb[3]; const int ne10 = src1->ne[0]; const int ne11 = src1->ne[1]; const int ne12 = src1->ne[2]; const int ne13 = src1->ne[3]; UNUSED(ne13); const cl_ulong nb10 = src1->nb[0]; const cl_ulong nb11 = src1->nb[1]; const cl_ulong nb12 = src1->nb[2]; const cl_ulong nb13 = src1->nb[3]; UNUSED(nb13); const int ne0 = dst->ne[0]; const int ne1 = dst->ne[1]; const int ne2 = dst->ne[2]; const int ne3 = dst->ne[3]; const cl_ulong nb0 = dst->nb[0]; const cl_ulong nb1 = dst->nb[1]; const cl_ulong nb2 = dst->nb[2]; const cl_ulong nb3 = dst->nb[3]; ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; ggml_tensor_extra_cl * extra0 = (ggml_tensor_extra_cl *)src0->extra; ggml_tensor_extra_cl * extra1 = (ggml_tensor_extra_cl *)src1->extra; ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; cl_ulong offset0 = extra0->offset + src0->view_offs; cl_ulong offset1 = extra1->offset + src1->view_offs; cl_ulong offsetd = extrad->offset + dst->view_offs; bool bcast_row = false; cl_kernel kernel; if (ggml_nelements(src1) == ne10 && ggml_is_contiguous(src1) && ne00 % 4 == 0 && ne10 % 4 == 0) { GGML_ASSERT(ggml_is_contiguous(src0)); // src1 is a row GGML_ASSERT(ne11 == 1); bcast_row = true; int ne = ne00 / 4; if (src0->type == GGML_TYPE_F32) { kernel = backend_ctx->kernel_mul_row; } else { kernel = backend_ctx->kernel_mul_row_f16; } CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device)); CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1)); CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device)); CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd)); CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne)); } else { if (src0->type == GGML_TYPE_F32) { kernel = backend_ctx->kernel_mul; } else { kernel = backend_ctx->kernel_mul_f16; } CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device)); CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1)); CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device)); CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd)); CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne00)); CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne01)); CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne02)); CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne03)); CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_ulong), &nb00)); CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_ulong), &nb01)); CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_ulong), &nb02)); CL_CHECK(clSetKernelArg(kernel, 13, sizeof(cl_ulong), &nb03)); CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &ne10)); CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &ne11)); CL_CHECK(clSetKernelArg(kernel, 16, sizeof(int), &ne12)); CL_CHECK(clSetKernelArg(kernel, 17, sizeof(int), &ne13)); CL_CHECK(clSetKernelArg(kernel, 18, sizeof(cl_ulong), &nb10)); CL_CHECK(clSetKernelArg(kernel, 19, sizeof(cl_ulong), &nb11)); CL_CHECK(clSetKernelArg(kernel, 20, sizeof(cl_ulong), &nb12)); CL_CHECK(clSetKernelArg(kernel, 21, sizeof(cl_ulong), &nb13)); CL_CHECK(clSetKernelArg(kernel, 22, sizeof(int), &ne0)); CL_CHECK(clSetKernelArg(kernel, 23, sizeof(int), &ne1)); CL_CHECK(clSetKernelArg(kernel, 24, sizeof(int), &ne2)); CL_CHECK(clSetKernelArg(kernel, 25, sizeof(int), &ne3)); CL_CHECK(clSetKernelArg(kernel, 26, sizeof(cl_ulong), &nb0)); CL_CHECK(clSetKernelArg(kernel, 27, sizeof(cl_ulong), &nb1)); CL_CHECK(clSetKernelArg(kernel, 28, sizeof(cl_ulong), &nb2)); CL_CHECK(clSetKernelArg(kernel, 29, sizeof(cl_ulong), &nb3)); } if (bcast_row) { int n = ggml_nelements(dst)/4; size_t global_work_size[] = {(size_t)n, 1, 1}; size_t local_work_size[] = {64, 1, 1}; size_t * local_work_size_ptr = local_work_size; if (n % 64 != 0 && !backend_ctx->non_uniform_workgroups) { local_work_size_ptr = nullptr; // Let driver choose the work-group sizes. } backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size_ptr, dst); } else { unsigned int nth = MIN(64, ne0); size_t global_work_size[] = {ne01*nth, (size_t)ne02, (size_t)ne03}; size_t local_work_size[] = {nth, 1, 1}; backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); } } static void ggml_cl_div(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { GGML_ASSERT(src0); GGML_ASSERT(src0->extra); GGML_ASSERT(src1); GGML_ASSERT(src1->extra); GGML_ASSERT(dst); GGML_ASSERT(dst->extra); GGML_ASSERT(src0->type == src1->type); GGML_ASSERT(src0->type == dst->type); GGML_ASSERT(src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16); const int ne00 = src0->ne[0]; const int ne01 = src0->ne[1]; const int ne02 = src0->ne[2]; const int ne03 = src0->ne[3]; const cl_ulong nb00 = src0->nb[0]; const cl_ulong nb01 = src0->nb[1]; const cl_ulong nb02 = src0->nb[2]; const cl_ulong nb03 = src0->nb[3]; const int ne10 = src1->ne[0]; const int ne11 = src1->ne[1]; const int ne12 = src1->ne[2]; const int ne13 = src1->ne[3]; const cl_ulong nb10 = src1->nb[0]; const cl_ulong nb11 = src1->nb[1]; const cl_ulong nb12 = src1->nb[2]; const cl_ulong nb13 = src1->nb[3]; const int ne0 = dst->ne[0]; const cl_ulong nb0 = dst->nb[0]; const cl_ulong nb1 = dst->nb[1]; const cl_ulong nb2 = dst->nb[2]; const cl_ulong nb3 = dst->nb[3]; ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; ggml_tensor_extra_cl * extra0 = (ggml_tensor_extra_cl *)src0->extra; ggml_tensor_extra_cl * extra1 = (ggml_tensor_extra_cl *)src1->extra; ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; cl_ulong offset0 = extra0->offset + src0->view_offs; cl_ulong offset1 = extra1->offset + src1->view_offs; cl_ulong offsetd = extrad->offset + dst->view_offs; bool bcast_row = false; cl_kernel kernel; if (ggml_nelements(src1) == ne10 && ggml_is_contiguous(src1) && ne00 % 4 == 0 && ne10 % 4 == 0) { GGML_ASSERT(ggml_is_contiguous(src0)); // src1 is a row GGML_ASSERT(ne11 == 1); bcast_row = true; int ne = ne00 / 4; if (src0->type == GGML_TYPE_F32) { kernel = backend_ctx->kernel_div_row; } else { kernel = backend_ctx->kernel_div_row_f16; } CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device)); CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1)); CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device)); CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd)); CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne)); } else { if (src0->type == GGML_TYPE_F32) { kernel = backend_ctx->kernel_div; } else { kernel = backend_ctx->kernel_div_f16; } CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device)); CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1)); CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device)); CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd)); CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_ulong), &nb00)); CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_ulong), &nb01)); CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_ulong), &nb02)); CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_ulong), &nb03)); CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &ne10)); CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne11)); CL_CHECK(clSetKernelArg(kernel, 12, sizeof(int), &ne12)); CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &ne13)); CL_CHECK(clSetKernelArg(kernel, 14, sizeof(cl_ulong), &nb10)); CL_CHECK(clSetKernelArg(kernel, 15, sizeof(cl_ulong), &nb11)); CL_CHECK(clSetKernelArg(kernel, 16, sizeof(cl_ulong), &nb12)); CL_CHECK(clSetKernelArg(kernel, 17, sizeof(cl_ulong), &nb13)); CL_CHECK(clSetKernelArg(kernel, 18, sizeof(int), &ne0)); CL_CHECK(clSetKernelArg(kernel, 19, sizeof(cl_ulong), &nb0)); CL_CHECK(clSetKernelArg(kernel, 20, sizeof(cl_ulong), &nb1)); CL_CHECK(clSetKernelArg(kernel, 21, sizeof(cl_ulong), &nb2)); CL_CHECK(clSetKernelArg(kernel, 22, sizeof(cl_ulong), &nb3)); } if (bcast_row) { int n = ggml_nelements(dst)/4; size_t global_work_size[] = {(size_t)n, 1, 1}; size_t local_work_size[] = {64, 1, 1}; backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); } else { unsigned int nth = MIN(64, ne0); size_t global_work_size[] = {ne01*nth, (size_t)ne02, (size_t)ne03}; size_t local_work_size[] = {nth, 1, 1}; backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); } } static void ggml_cl_sub(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { GGML_ASSERT(src0); GGML_ASSERT(src0->extra); GGML_ASSERT(src1); GGML_ASSERT(src1->extra); GGML_ASSERT(dst); GGML_ASSERT(dst->extra); GGML_ASSERT(src0->type == src1->type); GGML_ASSERT(src0->type == dst->type); GGML_ASSERT(src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16); const int ne00 = src0->ne[0]; const int ne01 = src0->ne[1]; const int ne02 = src0->ne[2]; const int ne03 = src0->ne[3]; const cl_ulong nb00 = src0->nb[0]; const cl_ulong nb01 = src0->nb[1]; const cl_ulong nb02 = src0->nb[2]; const cl_ulong nb03 = src0->nb[3]; const int ne10 = src1->ne[0]; const int ne11 = src1->ne[1]; const int ne12 = src1->ne[2]; const int ne13 = src1->ne[3]; const cl_ulong nb10 = src1->nb[0]; const cl_ulong nb11 = src1->nb[1]; const cl_ulong nb12 = src1->nb[2]; const cl_ulong nb13 = src1->nb[3]; const int ne0 = dst->ne[0]; const cl_ulong nb0 = dst->nb[0]; const cl_ulong nb1 = dst->nb[1]; const cl_ulong nb2 = dst->nb[2]; const cl_ulong nb3 = dst->nb[3]; ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; ggml_tensor_extra_cl * extra0 = (ggml_tensor_extra_cl *)src0->extra; ggml_tensor_extra_cl * extra1 = (ggml_tensor_extra_cl *)src1->extra; ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; cl_ulong offset0 = extra0->offset + src0->view_offs; cl_ulong offset1 = extra1->offset + src1->view_offs; cl_ulong offsetd = extrad->offset + dst->view_offs; bool bcast_row = false; cl_kernel kernel; if (ggml_nelements(src1) == ne10 && ggml_is_contiguous(src1) && ne00 % 4 == 0 && ne10 % 4 == 0) { GGML_ASSERT(ggml_is_contiguous(src0)); // src1 is a row GGML_ASSERT(ne11 == 1); bcast_row = true; int ne = ne00 / 4; if (src0->type == GGML_TYPE_F32) { kernel = backend_ctx->kernel_sub_row; } else { kernel = backend_ctx->kernel_sub_row_f16; } CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device)); CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1)); CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device)); CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd)); CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne)); } else { if (src0->type == GGML_TYPE_F32) { kernel = backend_ctx->kernel_sub; } else { kernel = backend_ctx->kernel_sub_f16; } CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device)); CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1)); CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device)); CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd)); CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_ulong), &nb00)); CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_ulong), &nb01)); CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_ulong), &nb02)); CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_ulong), &nb03)); CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &ne10)); CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne11)); CL_CHECK(clSetKernelArg(kernel, 12, sizeof(int), &ne12)); CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &ne13)); CL_CHECK(clSetKernelArg(kernel, 14, sizeof(cl_ulong), &nb10)); CL_CHECK(clSetKernelArg(kernel, 15, sizeof(cl_ulong), &nb11)); CL_CHECK(clSetKernelArg(kernel, 16, sizeof(cl_ulong), &nb12)); CL_CHECK(clSetKernelArg(kernel, 17, sizeof(cl_ulong), &nb13)); CL_CHECK(clSetKernelArg(kernel, 18, sizeof(int), &ne0)); CL_CHECK(clSetKernelArg(kernel, 19, sizeof(cl_ulong), &nb0)); CL_CHECK(clSetKernelArg(kernel, 20, sizeof(cl_ulong), &nb1)); CL_CHECK(clSetKernelArg(kernel, 21, sizeof(cl_ulong), &nb2)); CL_CHECK(clSetKernelArg(kernel, 22, sizeof(cl_ulong), &nb3)); } if (bcast_row) { int n = ggml_nelements(dst)/4; size_t global_work_size[] = {(size_t)n, 1, 1}; size_t local_work_size[] = {64, 1, 1}; backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); } else { unsigned int nth = MIN(64, ne0); size_t global_work_size[] = {ne01*nth, (size_t)ne02, (size_t)ne03}; size_t local_work_size[] = {nth, 1, 1}; backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); } } static void ggml_cl_sqr(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { GGML_ASSERT(src0); GGML_ASSERT(src0->extra); GGML_ASSERT(dst); GGML_ASSERT(dst->extra); UNUSED(src1); ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; ggml_tensor_extra_cl * extra0 = (ggml_tensor_extra_cl *)src0->extra; ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; cl_ulong offset0 = extra0->offset + src0->view_offs; cl_ulong offsetd = extrad->offset + dst->view_offs; cl_kernel kernel; // Currently assumes src0 is contiguous int n = ggml_nelements(dst); if (n % 4 == 0) { if (src0->type == GGML_TYPE_F32) { kernel = backend_ctx->kernel_sqr_cont_f32_4; } else { kernel = backend_ctx->kernel_sqr_cont_f16_4; } n /= 4; } else { if (src0->type == GGML_TYPE_F32) { kernel = backend_ctx->kernel_sqr_cont_f32; } else { kernel = backend_ctx->kernel_sqr_cont_f16; } } CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extrad->data_device)); CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offsetd)); size_t global_work_size[] = {(size_t)n, 1, 1}; size_t local_work_size[] = {64, 1, 1}; size_t * local_work_size_ptr = local_work_size; if (n % 64 != 0 && !backend_ctx->non_uniform_workgroups) { local_work_size_ptr = nullptr; } backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size_ptr, dst); } static void ggml_cl_sqrt(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { GGML_ASSERT(src0); GGML_ASSERT(src0->extra); GGML_ASSERT(dst); GGML_ASSERT(dst->extra); UNUSED(src1); ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; ggml_tensor_extra_cl * extra0 = (ggml_tensor_extra_cl *)src0->extra; ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; cl_ulong offset0 = extra0->offset + src0->view_offs; cl_ulong offsetd = extrad->offset + dst->view_offs; cl_kernel kernel; // Currently assumes src0 is contiguous int n = ggml_nelements(dst); if (n % 4 == 0) { if (src0->type == GGML_TYPE_F32) { kernel = backend_ctx->kernel_sqrt_cont_f32_4; } else { kernel = backend_ctx->kernel_sqrt_cont_f16_4; } n /= 4; } else { if (src0->type == GGML_TYPE_F32) { kernel = backend_ctx->kernel_sqrt_cont_f32; } else { kernel = backend_ctx->kernel_sqrt_cont_f16; } } CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extrad->data_device)); CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offsetd)); size_t global_work_size[] = {(size_t)n, 1, 1}; size_t local_work_size[] = {64, 1, 1}; size_t * local_work_size_ptr = local_work_size; if (n % 64 != 0 && !backend_ctx->non_uniform_workgroups) { local_work_size_ptr = nullptr; } backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size_ptr, dst); } static void ggml_cl_mean(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { GGML_ASSERT(src0); GGML_ASSERT(src0->extra); GGML_ASSERT(dst); GGML_ASSERT(dst->extra); GGML_UNUSED(src1); GGML_ASSERT(src0->nb[0] == ggml_type_size(src0->type)); ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; ggml_tensor_extra_cl * extra0 = (ggml_tensor_extra_cl *)src0->extra; ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; cl_ulong offset0 = extra0->offset + src0->view_offs; cl_ulong offsetd = extrad->offset + dst->view_offs; const int ne00 = src0->ne[0]; const int ne01 = src0->ne[1]; const int ne02 = src0->ne[2]; const int ne03 = src0->ne[3]; const cl_ulong nb01 = src0->nb[1]; const cl_ulong nb02 = src0->nb[2]; const cl_ulong nb03 = src0->nb[3]; const cl_ulong nb1 = dst->nb[1]; const cl_ulong nb2 = dst->nb[2]; const cl_ulong nb3 = dst->nb[3]; cl_kernel kernel; const bool is_c4 = ne00 % 4 == 0; if (is_c4) { kernel = backend_ctx->kernel_mean_f32_4; } else { kernel = backend_ctx->kernel_mean_f32; } CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extrad->data_device)); CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offsetd)); CL_CHECK(clSetKernelArg(kernel, 4, sizeof(int), &ne00)); CL_CHECK(clSetKernelArg(kernel, 5, sizeof(int), &ne01)); CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne02)); CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne03)); CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_ulong), &nb01)); CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_ulong), &nb02)); CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_ulong), &nb03)); CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_ulong), &nb1)); CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_ulong), &nb2)); CL_CHECK(clSetKernelArg(kernel, 13, sizeof(cl_ulong), &nb3)); size_t global_work_size[] = {64 * (size_t)ne01, (size_t)ne02, (size_t)ne03}; size_t local_work_size[] = {(size_t)64, 1, 1}; backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); } static void ggml_cl_ssm_scan(ggml_backend_t backend, ggml_tensor * dst) { GGML_ASSERT(dst); GGML_ASSERT(dst->extra); GGML_ASSERT(dst->src[0]); GGML_ASSERT(dst->src[0]->extra); GGML_ASSERT(dst->src[1]); GGML_ASSERT(dst->src[1]->extra); GGML_ASSERT(dst->src[2]); GGML_ASSERT(dst->src[2]->extra); GGML_ASSERT(dst->src[3]); GGML_ASSERT(dst->src[3]->extra); GGML_ASSERT(dst->src[4]); GGML_ASSERT(dst->src[4]->extra); GGML_ASSERT(dst->src[5]); GGML_ASSERT(dst->src[5]->extra); GGML_ASSERT(dst->src[6]); GGML_ASSERT(dst->src[6]->extra); ggml_backend_opencl_context * backend_ctx = (ggml_backend_opencl_context *) backend->context; ggml_tensor_extra_cl * extra0 = (ggml_tensor_extra_cl *) dst->src[0]->extra; ggml_tensor_extra_cl * extra1 = (ggml_tensor_extra_cl *) dst->src[1]->extra; ggml_tensor_extra_cl * extra2 = (ggml_tensor_extra_cl *) dst->src[2]->extra; ggml_tensor_extra_cl * extra3 = (ggml_tensor_extra_cl *) dst->src[3]->extra; ggml_tensor_extra_cl * extra4 = (ggml_tensor_extra_cl *) dst->src[4]->extra; ggml_tensor_extra_cl * extra5 = (ggml_tensor_extra_cl *) dst->src[5]->extra; ggml_tensor_extra_cl * extra6 = (ggml_tensor_extra_cl *) dst->src[6]->extra; ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *) dst->extra; const cl_ulong offset0 = extra0->offset + dst->src[0]->view_offs; const cl_ulong offset1 = extra1->offset + dst->src[1]->view_offs; const cl_ulong offset2 = extra2->offset + dst->src[2]->view_offs; const cl_ulong offset3 = extra3->offset + dst->src[3]->view_offs; const cl_ulong offset4 = extra4->offset + dst->src[4]->view_offs; const cl_ulong offset5 = extra5->offset + dst->src[5]->view_offs; const cl_ulong offset6 = extra6->offset + dst->src[6]->view_offs; const cl_ulong offsetd = extrad->offset + dst->view_offs; const ggml_tensor * s = dst->src[0]; const ggml_tensor * x = dst->src[1]; const ggml_tensor * dt = dst->src[2]; const ggml_tensor * A = dst->src[3]; const ggml_tensor * B = dst->src[4]; const ggml_tensor * C = dst->src[5]; const cl_ulong s_nb1 = s->nb[1]; const cl_ulong s_nb2 = s->nb[2]; const cl_ulong s_nb3 = s->nb[3]; const cl_ulong x_nb1 = x->nb[1]; const cl_ulong x_nb2 = x->nb[2]; const cl_ulong x_nb3 = x->nb[3]; const cl_ulong dt_nb1 = dt->nb[1]; const cl_ulong dt_nb2 = dt->nb[2]; const cl_ulong A_nb1 = A->nb[1]; const cl_ulong B_nb1 = B->nb[1]; const cl_ulong B_nb2 = B->nb[2]; const cl_ulong B_nb3 = B->nb[3]; const cl_ulong C_nb1 = C->nb[1]; const cl_ulong C_nb2 = C->nb[2]; const cl_ulong C_nb3 = C->nb[3]; const cl_uint A_ne0 = A->ne[0]; const cl_uint d_state = s->ne[0]; const cl_int head_dim = x->ne[0]; const cl_int n_head = x->ne[1]; const cl_int n_group = B->ne[1]; const cl_int n_tokens = x->ne[2]; const cl_uint n_seqs = x->ne[3]; const cl_uint K = ggml_get_op_params_i32(dst, 0); const cl_ulong s_off_bytes = (cl_ulong) ggml_nelements(x) * sizeof(float); cl_kernel kernel = backend_ctx->kernel_ssm_scan_f32; size_t nth = d_state; if (A_ne0 == 1 && K == 1) { cl_kernel kernel_mamba2 = nullptr; if (d_state == 128) { kernel_mamba2 = backend_ctx->kernel_ssm_scan_f32_mamba2_d128; } else if (d_state == 256) { kernel_mamba2 = backend_ctx->kernel_ssm_scan_f32_mamba2_d256; } if (kernel_mamba2 != nullptr) { kernel = kernel_mamba2; nth = 64; } } CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device)); CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1)); CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extra2->data_device)); CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offset2)); CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_mem), &extra3->data_device)); CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_ulong), &offset3)); CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_mem), &extra4->data_device)); CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_ulong), &offset4)); CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_mem), &extra5->data_device)); CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_ulong), &offset5)); CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_mem), &extra6->data_device)); CL_CHECK(clSetKernelArg(kernel, 13, sizeof(cl_ulong), &offset6)); CL_CHECK(clSetKernelArg(kernel, 14, sizeof(cl_mem), &extrad->data_device)); CL_CHECK(clSetKernelArg(kernel, 15, sizeof(cl_ulong), &offsetd)); CL_CHECK(clSetKernelArg(kernel, 16, sizeof(cl_ulong), &s_nb2)); CL_CHECK(clSetKernelArg(kernel, 17, sizeof(cl_ulong), &s_nb3)); CL_CHECK(clSetKernelArg(kernel, 18, sizeof(cl_ulong), &x_nb2)); CL_CHECK(clSetKernelArg(kernel, 19, sizeof(cl_ulong), &x_nb3)); CL_CHECK(clSetKernelArg(kernel, 20, sizeof(cl_ulong), &dt_nb1)); CL_CHECK(clSetKernelArg(kernel, 21, sizeof(cl_ulong), &dt_nb2)); CL_CHECK(clSetKernelArg(kernel, 22, sizeof(cl_ulong), &A_nb1)); CL_CHECK(clSetKernelArg(kernel, 23, sizeof(cl_ulong), &B_nb2)); CL_CHECK(clSetKernelArg(kernel, 24, sizeof(cl_ulong), &B_nb3)); CL_CHECK(clSetKernelArg(kernel, 25, sizeof(cl_ulong), &C_nb2)); CL_CHECK(clSetKernelArg(kernel, 26, sizeof(cl_ulong), &C_nb3)); CL_CHECK(clSetKernelArg(kernel, 27, sizeof(cl_ulong), &s_off_bytes)); CL_CHECK(clSetKernelArg(kernel, 28, sizeof(cl_int), &head_dim)); CL_CHECK(clSetKernelArg(kernel, 29, sizeof(cl_int), &n_head)); CL_CHECK(clSetKernelArg(kernel, 30, sizeof(cl_int), &n_group)); CL_CHECK(clSetKernelArg(kernel, 31, sizeof(cl_int), &n_tokens)); if (kernel == backend_ctx->kernel_ssm_scan_f32) { CL_CHECK(clSetKernelArg(kernel, 32, sizeof(cl_ulong), &s_nb1)); CL_CHECK(clSetKernelArg(kernel, 33, sizeof(cl_ulong), &x_nb1)); CL_CHECK(clSetKernelArg(kernel, 34, sizeof(cl_ulong), &B_nb1)); CL_CHECK(clSetKernelArg(kernel, 35, sizeof(cl_ulong), &C_nb1)); CL_CHECK(clSetKernelArg(kernel, 36, sizeof(cl_uint), &A_ne0)); CL_CHECK(clSetKernelArg(kernel, 37, sizeof(cl_uint), &d_state)); CL_CHECK(clSetKernelArg(kernel, 38, sizeof(cl_uint), &n_seqs)); CL_CHECK(clSetKernelArg(kernel, 39, sizeof(cl_uint), &K)); CL_CHECK(clSetKernelArg(kernel, 40, d_state * sizeof(float), nullptr)); } size_t global_work_size[] = { (size_t) head_dim * (size_t) n_head * nth, (size_t) n_seqs, }; size_t local_work_size[] = { nth, 1 }; backend_ctx->enqueue_ndrange_kernel(kernel, 2, global_work_size, local_work_size, dst); } static void ggml_cl_ssm_conv(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { GGML_ASSERT(src0); GGML_ASSERT(src0->extra); GGML_ASSERT(src1); GGML_ASSERT(src1->extra); GGML_ASSERT(dst); GGML_ASSERT(dst->extra); ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; ggml_tensor_extra_cl * extra0 = (ggml_tensor_extra_cl *)src0->extra; ggml_tensor_extra_cl * extra1 = (ggml_tensor_extra_cl *)src1->extra; ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; cl_ulong offset0 = extra0->offset + src0->view_offs; cl_ulong offset1 = extra1->offset + src1->view_offs; cl_ulong offsetd = extrad->offset + dst->view_offs; int ne01 = src0->ne[1]; cl_ulong nb00 = src0->nb[0]; cl_ulong nb01 = src0->nb[1]; cl_ulong nb02 = src0->nb[2]; int ne10 = src1->ne[0]; cl_ulong nb11 = src1->nb[1]; int ne1 = dst->ne[1]; int ne2 = dst->ne[2]; cl_ulong nb0 = dst->nb[0]; cl_ulong nb1 = dst->nb[1]; cl_ulong nb2 = dst->nb[2]; cl_kernel kernel = backend_ctx->kernel_ssm_conv_f32_f32; if (ne10 % 4 == 0) { kernel = backend_ctx->kernel_ssm_conv_f32_f32_4; } CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device)); CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1)); CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device)); CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd)); CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_ulong), &nb00)); CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_ulong), &nb01)); CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_ulong), &nb02)); CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne10)); CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_ulong), &nb11)); CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_ulong), &nb0)); CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_ulong), &nb1)); CL_CHECK(clSetKernelArg(kernel, 13, sizeof(cl_ulong), &nb2)); size_t global_work_size[] = {(size_t)ne01, (size_t)ne1, (size_t)ne2}; size_t local_work_size[] = {64, 1, 1}; size_t * local_work_size_ptr = local_work_size; if (ne01 % 64 != 0 && !backend_ctx->non_uniform_workgroups) { local_work_size_ptr = nullptr; } backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size_ptr, dst); } static void ggml_cl_gelu(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { GGML_ASSERT(src0); GGML_ASSERT(src0->extra); GGML_ASSERT(dst); GGML_ASSERT(dst->extra); UNUSED(src1); ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; ggml_tensor_extra_cl * extra0 = (ggml_tensor_extra_cl *)src0->extra; ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; cl_ulong offset0 = extra0->offset + src0->view_offs; cl_ulong offsetd = extrad->offset + dst->view_offs; cl_kernel kernel; int n = ggml_nelements(dst); if (n % 4 == 0) { kernel = backend_ctx->kernel_gelu_4; n /= 4; } else { kernel = backend_ctx->kernel_gelu; } CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extrad->data_device)); CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offsetd)); size_t global_work_size[] = {(size_t)n, 1, 1}; size_t local_work_size[] = {64, 1, 1}; backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); } static void ggml_cl_gelu_erf(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { GGML_ASSERT(src0); GGML_ASSERT(src0->extra); GGML_ASSERT(dst); GGML_ASSERT(dst->extra); UNUSED(src1); ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; ggml_tensor_extra_cl * extra0 = (ggml_tensor_extra_cl *)src0->extra; ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; cl_ulong offset0 = extra0->offset + src0->view_offs; cl_ulong offsetd = extrad->offset + dst->view_offs; cl_kernel kernel; int n = ggml_nelements(dst); if (n % 4 == 0) { kernel = backend_ctx->kernel_gelu_erf_4; n /= 4; } else { kernel = backend_ctx->kernel_gelu_erf; } CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extrad->data_device)); CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offsetd)); size_t global_work_size[] = {(size_t)n, 1, 1}; size_t local_work_size[] = {64, 1, 1}; backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); } static void ggml_cl_gelu_quick(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { GGML_ASSERT(src0); GGML_ASSERT(src0->extra); GGML_ASSERT(dst); GGML_ASSERT(dst->extra); UNUSED(src1); ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; ggml_tensor_extra_cl * extra0 = (ggml_tensor_extra_cl *)src0->extra; ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; cl_ulong offset0 = extra0->offset + src0->view_offs; cl_ulong offsetd = extrad->offset + dst->view_offs; cl_kernel kernel; int n = ggml_nelements(dst); if (n % 4 == 0) { kernel = backend_ctx->kernel_gelu_quick_4; n /= 4; } else { kernel = backend_ctx->kernel_gelu_quick; } CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extrad->data_device)); CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offsetd)); size_t global_work_size[] = {(size_t)n, 1, 1}; size_t local_work_size[] = {64, 1, 1}; backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); } static void ggml_cl_silu(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { GGML_ASSERT(src0); GGML_ASSERT(src0->extra); GGML_ASSERT(dst); GGML_ASSERT(dst->extra); UNUSED(src1); ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; ggml_tensor_extra_cl * extra0 = (ggml_tensor_extra_cl *)src0->extra; ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; cl_ulong offset0 = extra0->offset + src0->view_offs; cl_ulong offsetd = extrad->offset + dst->view_offs; cl_kernel kernel; int n = ggml_nelements(dst); if (n % 4 == 0) { kernel = backend_ctx->kernel_silu_4; n /= 4; } else { kernel = backend_ctx->kernel_silu; } CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extrad->data_device)); CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offsetd)); size_t global_work_size[] = {(size_t)n, 1, 1}; size_t local_work_size[] = {64, 1, 1}; size_t * local_work_size_ptr = local_work_size; if (n % 64 != 0 && !backend_ctx->non_uniform_workgroups) { local_work_size_ptr = nullptr; // Let driver choose the work-group sizes. } backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size_ptr, dst); } static void ggml_cl_relu(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { GGML_ASSERT(src0); GGML_ASSERT(src0->extra); GGML_ASSERT(dst); GGML_ASSERT(dst->extra); UNUSED(src1); ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; ggml_tensor_extra_cl * extra0 = (ggml_tensor_extra_cl *)src0->extra; ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; cl_ulong offset0 = extra0->offset + src0->view_offs; cl_ulong offsetd = extrad->offset + dst->view_offs; cl_kernel kernel = backend_ctx->kernel_relu; CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extrad->data_device)); CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offsetd)); const int64_t n = ggml_nelements(dst); size_t global_work_size[] = {(size_t)n, 1, 1}; size_t local_work_size[] = {64, 1, 1}; size_t * local_work_size_ptr = local_work_size; if (n % 64 != 0 && !backend_ctx->non_uniform_workgroups) { local_work_size_ptr = nullptr; // Let driver choose the work-group sizes. } backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size_ptr, dst); } static void ggml_cl_sigmoid(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { GGML_ASSERT(src0); GGML_ASSERT(src0->extra); GGML_ASSERT(dst); GGML_ASSERT(dst->extra); UNUSED(src1); ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; ggml_tensor_extra_cl * extra0 = (ggml_tensor_extra_cl *)src0->extra; ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; cl_ulong offset0 = extra0->offset + src0->view_offs; cl_ulong offsetd = extrad->offset + dst->view_offs; cl_kernel kernel; if (src0->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32) { kernel = backend_ctx->kernel_sigmoid_f32; } else if (src0->type == GGML_TYPE_F16 && dst->type == GGML_TYPE_F16) { kernel = backend_ctx->kernel_sigmoid_f16; } else if (src0->type == GGML_TYPE_BF16 && dst->type == GGML_TYPE_BF16) { // bf16 converted to f16 kernel = backend_ctx->kernel_sigmoid_f16; } else { GGML_ASSERT(false && "Unsupported data types for sigmoid (input and output must be both f32 or f16)"); } CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extrad->data_device)); CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offsetd)); const int64_t n = ggml_nelements(dst); size_t global_work_size[] = {(size_t)n, 1, 1}; size_t local_work_size[] = {64, 1, 1}; size_t * local_work_size_ptr = local_work_size; if (n % 64 != 0 && !backend_ctx->non_uniform_workgroups) { local_work_size_ptr = nullptr; // Let driver choose the work-group sizes. } backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size_ptr, dst); } static void ggml_cl_tri(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { GGML_ASSERT(src0); GGML_ASSERT(src0->extra); GGML_ASSERT(dst); GGML_ASSERT(dst->extra); UNUSED(src1); ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; ggml_tensor_extra_cl * extra0 = (ggml_tensor_extra_cl *)src0->extra; ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; cl_ulong offset0 = extra0->offset + src0->view_offs; cl_ulong offsetd = extrad->offset + dst->view_offs; const int tri_type = ggml_get_op_params_i32(dst, 0); const int64_t n = ggml_nelements(dst); const int ne0 = dst->ne[0]; const int ne1 = dst->ne[1]; cl_kernel kernel = backend_ctx->kernel_tri; CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extrad->data_device)); CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offsetd)); CL_CHECK(clSetKernelArg(kernel, 4, sizeof(int), &n)); CL_CHECK(clSetKernelArg(kernel, 5, sizeof(int), &ne0)); CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne1)); CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &tri_type)); size_t local_work_size[1] = { 256 }; size_t global_work_size[1] = { ((size_t)n + local_work_size[0] - 1) / local_work_size[0] * local_work_size[0] }; backend_ctx->enqueue_ndrange_kernel(kernel, 1, global_work_size, local_work_size, dst); } static void ggml_cl_fill(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { GGML_ASSERT(dst); GGML_ASSERT(dst->extra); UNUSED(src0); UNUSED(src1); ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; cl_ulong offsetd = extrad->offset + dst->view_offs; float v = 0.0f; memcpy(&v, ((int32_t *) dst->op_params), sizeof(float)); const int64_t n = ggml_nelements(dst); cl_kernel kernel = backend_ctx->kernel_fill; CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extrad->data_device)); CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offsetd)); CL_CHECK(clSetKernelArg(kernel, 2, sizeof(float), &v)); CL_CHECK(clSetKernelArg(kernel, 3, sizeof(float), &n)); size_t local_work_size[1] = { 256 }; size_t global_work_size[1] = { ((size_t)n + local_work_size[0] - 1) / local_work_size[0] * local_work_size[0] }; backend_ctx->enqueue_ndrange_kernel(kernel, 1, global_work_size, local_work_size, dst); } static void ggml_cl_clamp(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { GGML_ASSERT(src0); GGML_ASSERT(src0->extra); GGML_ASSERT(dst); GGML_ASSERT(dst->extra); UNUSED(src1); ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; ggml_tensor_extra_cl * extra0 = (ggml_tensor_extra_cl *)src0->extra; ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; cl_ulong offset0 = extra0->offset + src0->view_offs; cl_ulong offsetd = extrad->offset + dst->view_offs; float min; float max; memcpy(&min, ((int32_t *) dst->op_params) + 0, sizeof(float)); memcpy(&max, ((int32_t *) dst->op_params) + 1, sizeof(float)); cl_kernel kernel = backend_ctx->kernel_clamp; CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extrad->data_device)); CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offsetd)); CL_CHECK(clSetKernelArg(kernel, 4, sizeof(float), &min)); CL_CHECK(clSetKernelArg(kernel, 5, sizeof(float), &max)); const int64_t n = ggml_nelements(dst); size_t global_work_size[] = {(size_t)n, 1, 1}; size_t local_work_size[] = {64, 1, 1}; size_t * local_work_size_ptr = local_work_size; if (n % 64 != 0 && !backend_ctx->non_uniform_workgroups) { local_work_size_ptr = nullptr; // Let driver choose the work-group sizes. } backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size_ptr, dst); } static void ggml_cl_norm(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { GGML_ASSERT(src0); GGML_ASSERT(src0->extra); GGML_ASSERT(dst); GGML_ASSERT(dst->extra); UNUSED(src1); ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; ggml_tensor_extra_cl * extra0 = (ggml_tensor_extra_cl *)src0->extra; ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; cl_ulong offset0 = extra0->offset + src0->view_offs; cl_ulong offsetd = extrad->offset + dst->view_offs; float eps; memcpy(&eps, dst->op_params, sizeof(float)); GGML_TENSOR_LOCALS(int, ne0, src0, ne); GGML_TENSOR_LOCALS(cl_ulong, nb0, src0, nb); int nth = 1; while (nth < ne00 && nth < 64) { nth *= 2; } cl_kernel kernel = backend_ctx->kernel_norm; CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extrad->data_device)); CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offsetd)); CL_CHECK(clSetKernelArg(kernel, 4, sizeof(int), &ne00)); CL_CHECK(clSetKernelArg(kernel, 5, sizeof(int), &ne01)); CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne02)); CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne03)); CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_ulong), &nb00)); CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_ulong), &nb01)); CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_ulong), &nb02)); CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_ulong), &nb03)); CL_CHECK(clSetKernelArg(kernel, 12, sizeof(float), &eps)); CL_CHECK(clSetKernelArg(kernel, 13, sizeof(float)*nth, NULL)); size_t global_work_size[] = {(size_t)ne01*nth, (size_t)ne02, (size_t)ne03}; size_t local_work_size[] = {(size_t)nth, 1, 1}; backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); } static void ggml_cl_rms_norm(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { GGML_ASSERT(src0); GGML_ASSERT(src0->extra); GGML_ASSERT(dst); GGML_ASSERT(dst->extra); UNUSED(src1); ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; //ggml_backend_opencl_device_context * dev_ctx = // (ggml_backend_opencl_device_context *)backend->device->context; ggml_tensor_extra_cl * extra0 = (ggml_tensor_extra_cl *)src0->extra; ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; cl_ulong offset0 = extra0->offset + src0->view_offs; cl_ulong offsetd = extrad->offset + dst->view_offs; float eps; memcpy(&eps, dst->op_params, sizeof(float)); const int ne00 = src0 ? src0->ne[0] : 0; const int ne01 = src0 ? src0->ne[1] : 0; const int ne02 = src0 ? src0->ne[2] : 0; const int ne03 = src0 ? src0->ne[3] : 0; const cl_ulong nb01 = src0 ? src0->nb[1] : 0; const cl_ulong nb02 = src0 ? src0->nb[2] : 0; const cl_ulong nb03 = src0 ? src0->nb[3] : 0; GGML_ASSERT(ne00 % 4 == 0); const int nth = MIN(64, ne00); size_t global_work_size[] = {(size_t)ne01*nth, (size_t)ne02, (size_t)ne03}; size_t local_work_size[] = {(size_t)nth, 1, 1}; cl_kernel kernel = backend_ctx->kernel_rms_norm; // Note, this kernel declares local memory in kernel args and the size // depends on subgroup size. // Note, this requires OpenCL 2.1 and above // For now we use fixed subgroup size to simplify support for OpenCL 2.0. size_t sgs; //CL_CHECK(clGetKernelSubGroupInfo(kernel, dev_ctx->device, // CL_KERNEL_MAX_SUB_GROUP_SIZE_FOR_NDRANGE, // sizeof(local_work_size), local_work_size, // sizeof(size_t), &sgs, NULL)); if (backend_ctx->gpu_family == ADRENO) { sgs = 64; } else if (backend_ctx->gpu_family == INTEL) { sgs = 32; } else { GGML_ASSERT(false && "Unsupported GPU"); } CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extrad->data_device)); CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offsetd)); CL_CHECK(clSetKernelArg(kernel, 4, sizeof(int), &ne00)); CL_CHECK(clSetKernelArg(kernel, 5, sizeof(int), &ne01)); CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne02)); CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne03)); CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_ulong), &nb01)); CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_ulong), &nb02)); CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_ulong), &nb03)); CL_CHECK(clSetKernelArg(kernel, 11, sizeof(float), &eps)); // This is local memory - the size depends on subgroup size. CL_CHECK(clSetKernelArg(kernel, 12, sizeof(float)*nth/sgs, NULL)); backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); } static void ggml_opencl_op_rms_norm_fused(ggml_backend_t backend, ggml_tensor * rms_norm_tensor, ggml_tensor * mul_tensor) { GGML_ASSERT(mul_tensor); GGML_ASSERT(rms_norm_tensor); // src0 is the src of rms_norm, src1 is the other src of mul (one being rms_norm) const ggml_tensor * src0 = rms_norm_tensor->src[0]; const ggml_tensor * src1; if (mul_tensor->src[0] == rms_norm_tensor) { src1 = mul_tensor->src[1]; } else if (mul_tensor->src[1] == rms_norm_tensor) { src1 = mul_tensor->src[0]; } else { GGML_ASSERT(false && "Invalid args for rms_norm and mul"); } const ggml_tensor * dst = mul_tensor; GGML_ASSERT(src0); GGML_ASSERT(src0->extra); GGML_ASSERT(src1); GGML_ASSERT(src1->extra); GGML_ASSERT(dst); GGML_ASSERT(dst->extra); ggml_tensor_extra_cl * extra0 = (ggml_tensor_extra_cl *)src0->extra; ggml_tensor_extra_cl * extra1 = (ggml_tensor_extra_cl *)src1->extra; ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; cl_ulong offset0 = extra0->offset + src0->view_offs; cl_ulong offset1 = extra1->offset + src1->view_offs; cl_ulong offsetd = extrad->offset + dst->view_offs; ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; float eps; memcpy(&eps, rms_norm_tensor->op_params, sizeof(float)); const int ne00 = src0->ne[0]; const int ne01 = src0->ne[1]; const int ne02 = src0->ne[2]; const int ne03 = src0->ne[3]; const cl_ulong nb01 = src0->nb[1]; const cl_ulong nb02 = src0->nb[2]; const cl_ulong nb03 = src0->nb[3]; const int ne10 = src1->ne[0]; const int ne11 = src1->ne[1]; const int ne12 = src1->ne[2]; const int ne13 = src1->ne[3]; const cl_ulong nb11 = src1->nb[1]; const cl_ulong nb12 = src1->nb[2]; const cl_ulong nb13 = src1->nb[3]; const cl_ulong nb1 = dst->nb[1]; const cl_ulong nb2 = dst->nb[2]; const cl_ulong nb3 = dst->nb[3]; GGML_ASSERT(ne00 % 4 == 0); size_t sgs; if (backend_ctx->gpu_family == ADRENO) { sgs = 64; } else if (backend_ctx->gpu_family == INTEL) { sgs = 32; } else { GGML_ASSERT(false && "Unsupported GPU"); } cl_kernel kernel = backend_ctx->kernel_rms_norm_mul; int nth = sgs; int max_workgroup_size = backend_ctx->get_kernel_workgroup_size(kernel); while (nth < ne00 && nth < max_workgroup_size) { nth *= 2; } nth = MIN(nth, max_workgroup_size); nth = MIN(nth, ne00); size_t global_work_size[] = {(size_t)ne01*nth, (size_t)ne02, (size_t)ne03}; size_t local_work_size[] = {(size_t)nth, 1, 1}; CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device)); CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1)); CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device)); CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd)); CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne00)); CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne01)); CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne02)); CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne03)); CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_ulong), &nb01)); CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_ulong), &nb02)); CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_ulong), &nb03)); CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &ne10)); CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &ne11)); CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &ne12)); CL_CHECK(clSetKernelArg(kernel, 16, sizeof(int), &ne13)); CL_CHECK(clSetKernelArg(kernel, 17, sizeof(cl_ulong), &nb11)); CL_CHECK(clSetKernelArg(kernel, 18, sizeof(cl_ulong), &nb12)); CL_CHECK(clSetKernelArg(kernel, 19, sizeof(cl_ulong), &nb13)); CL_CHECK(clSetKernelArg(kernel, 20, sizeof(cl_ulong), &nb1)); CL_CHECK(clSetKernelArg(kernel, 21, sizeof(cl_ulong), &nb2)); CL_CHECK(clSetKernelArg(kernel, 22, sizeof(cl_ulong), &nb3)); CL_CHECK(clSetKernelArg(kernel, 23, sizeof(float), &eps)); CL_CHECK(clSetKernelArg(kernel, 24, sizeof(float)*sgs, NULL)); backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); } static void ggml_opencl_op_norm_fused(ggml_backend_t backend, ggml_tensor * norm_tensor, ggml_tensor * mul_tensor, ggml_tensor * add_tensor) { GGML_ASSERT(norm_tensor && mul_tensor && add_tensor); const ggml_tensor * src0 = norm_tensor->src[0]; const ggml_tensor * src1 = mul_tensor->src[0] == norm_tensor ? mul_tensor->src[1] : mul_tensor->src[0]; const ggml_tensor * src2 = add_tensor->src[0] == mul_tensor ? add_tensor->src[1] : add_tensor->src[0]; const ggml_tensor * dst = add_tensor; ggml_tensor_extra_cl * extra0 = (ggml_tensor_extra_cl *)src0->extra; ggml_tensor_extra_cl * extra1 = (ggml_tensor_extra_cl *)src1->extra; ggml_tensor_extra_cl * extra2 = (ggml_tensor_extra_cl *)src2->extra; ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; cl_ulong offset0 = extra0->offset + src0->view_offs; cl_ulong offset1 = extra1->offset + src1->view_offs; cl_ulong offset2 = extra2->offset + src2->view_offs; cl_ulong offsetd = extrad->offset + dst->view_offs; ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; float eps; memcpy(&eps, norm_tensor->op_params, sizeof(float)); const int ne00 = src0->ne[0], ne01 = src0->ne[1], ne02 = src0->ne[2], ne03 = src0->ne[3]; const cl_ulong nb01 = src0->nb[1], nb02 = src0->nb[2], nb03 = src0->nb[3]; const int ne10 = src1->ne[0], ne11 = src1->ne[1], ne12 = src1->ne[2], ne13 = src1->ne[3]; const cl_ulong nb11 = src1->nb[1], nb12 = src1->nb[2], nb13 = src1->nb[3]; const int ne20 = src2->ne[0], ne21 = src2->ne[1], ne22 = src2->ne[2], ne23 = src2->ne[3]; const cl_ulong nb21 = src2->nb[1], nb22 = src2->nb[2], nb23 = src2->nb[3]; const cl_ulong nbd1 = dst->nb[1], nbd2 = dst->nb[2], nbd3 = dst->nb[3]; size_t sgs; if (backend_ctx->gpu_family == ADRENO) sgs = 64; else if (backend_ctx->gpu_family == INTEL) sgs = 32; else GGML_ASSERT(false && "Unsupported GPU"); cl_kernel kernel = backend_ctx->kernel_norm_mul_add; int nth = sgs; int max_workgroup_size = backend_ctx->get_kernel_workgroup_size(kernel); while (nth < ne00/4 && nth < max_workgroup_size) nth *= 2; nth = MIN(nth, max_workgroup_size); nth = MIN(nth, ne00/4); size_t gws[] = {(size_t)ne01*nth, (size_t)ne02, (size_t)ne03}; size_t lws[] = {(size_t)nth, 1, 1}; size_t num_subgroups = (nth + sgs - 1) / sgs; CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device)); CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1)); CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extra2->data_device)); CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offset2)); CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_mem), &extrad->data_device)); CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_ulong), &offsetd)); CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne00)); CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne01)); CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &ne02)); CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne03)); CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_ulong), &nb01)); CL_CHECK(clSetKernelArg(kernel, 13, sizeof(cl_ulong), &nb02)); CL_CHECK(clSetKernelArg(kernel, 14, sizeof(cl_ulong), &nb03)); CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &ne10)); CL_CHECK(clSetKernelArg(kernel, 16, sizeof(int), &ne11)); CL_CHECK(clSetKernelArg(kernel, 17, sizeof(int), &ne12)); CL_CHECK(clSetKernelArg(kernel, 18, sizeof(int), &ne13)); CL_CHECK(clSetKernelArg(kernel, 19, sizeof(cl_ulong), &nb11)); CL_CHECK(clSetKernelArg(kernel, 20, sizeof(cl_ulong), &nb12)); CL_CHECK(clSetKernelArg(kernel, 21, sizeof(cl_ulong), &nb13)); CL_CHECK(clSetKernelArg(kernel, 22, sizeof(int), &ne20)); CL_CHECK(clSetKernelArg(kernel, 23, sizeof(int), &ne21)); CL_CHECK(clSetKernelArg(kernel, 24, sizeof(int), &ne22)); CL_CHECK(clSetKernelArg(kernel, 25, sizeof(int), &ne23)); CL_CHECK(clSetKernelArg(kernel, 26, sizeof(cl_ulong), &nb21)); CL_CHECK(clSetKernelArg(kernel, 27, sizeof(cl_ulong), &nb22)); CL_CHECK(clSetKernelArg(kernel, 28, sizeof(cl_ulong), &nb23)); CL_CHECK(clSetKernelArg(kernel, 29, sizeof(cl_ulong), &nbd1)); CL_CHECK(clSetKernelArg(kernel, 30, sizeof(cl_ulong), &nbd2)); CL_CHECK(clSetKernelArg(kernel, 31, sizeof(cl_ulong), &nbd3)); CL_CHECK(clSetKernelArg(kernel, 32, sizeof(float), &eps)); CL_CHECK(clSetKernelArg(kernel, 33, sizeof(cl_float2) * num_subgroups, NULL)); backend_ctx->enqueue_ndrange_kernel(kernel, 3, gws, lws, dst); } static void ggml_opencl_op_group_norm_fused(ggml_backend_t backend, ggml_tensor * gn_tensor, ggml_tensor * mul_tensor, ggml_tensor * add_tensor) { GGML_ASSERT(gn_tensor && mul_tensor && add_tensor); const ggml_tensor * src0 = gn_tensor->src[0]; const ggml_tensor * src1 = mul_tensor->src[0] == gn_tensor ? mul_tensor->src[1] : mul_tensor->src[0]; const ggml_tensor * src2 = add_tensor->src[0] == mul_tensor ? add_tensor->src[1] : add_tensor->src[0]; const ggml_tensor * dst = add_tensor; ggml_tensor_extra_cl * extra0 = (ggml_tensor_extra_cl *)src0->extra; ggml_tensor_extra_cl * extra1 = (ggml_tensor_extra_cl *)src1->extra; ggml_tensor_extra_cl * extra2 = (ggml_tensor_extra_cl *)src2->extra; ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; cl_ulong offset0 = extra0->offset + src0->view_offs; cl_ulong offset1 = extra1->offset + src1->view_offs; cl_ulong offset2 = extra2->offset + src2->view_offs; cl_ulong offsetd = extrad->offset + dst->view_offs; ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; int groups; float eps; memcpy(&groups, gn_tensor->op_params, sizeof(int)); memcpy(&eps, (char *)gn_tensor->op_params + sizeof(int), sizeof(float)); cl_kernel kernel = backend_ctx->kernel_group_norm_mul_add; int max_workgroup_size = backend_ctx->get_kernel_workgroup_size(kernel); int ne = ggml_nelements(src0); int group_size = ne / groups; size_t lws[] = { (size_t)MIN(max_workgroup_size, group_size) }; size_t gws[] = { (size_t)groups * lws[0] }; CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device)); CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1)); CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extra2->data_device)); CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offset2)); CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_mem), &extrad->data_device)); CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_ulong), &offsetd)); CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne)); CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &group_size)); CL_CHECK(clSetKernelArg(kernel, 10, sizeof(float), &eps)); backend_ctx->enqueue_ndrange_kernel(kernel, 1, gws, lws, dst); } static void ggml_cl_group_norm(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { GGML_ASSERT(src0); GGML_ASSERT(src0->extra); GGML_ASSERT(dst); GGML_ASSERT(dst->extra); UNUSED(src1); ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; ggml_tensor_extra_cl * extra0 = (ggml_tensor_extra_cl *)src0->extra; ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; cl_ulong offset0 = extra0->offset + src0->view_offs; cl_ulong offsetd = extrad->offset + dst->view_offs; int32_t n_groups = ((const int32_t *) dst->op_params)[0]; int32_t group_size = src0->ne[0] * src0->ne[1] * ((src0->ne[2] + n_groups - 1) / n_groups); float eps = ((const float *) dst->op_params)[1]; const int ne00 = src0->ne[0]; const int ne01 = src0->ne[1]; const int ne02 = src0->ne[2]; const int ne = ne00*ne01*ne02; cl_kernel kernel = backend_ctx->kernel_group_norm; size_t sgs = 64; if (backend_ctx->gpu_family == ADRENO) { sgs = 64; } else if (backend_ctx->gpu_family == INTEL) { sgs = 32; } else { GGML_ASSERT(false && "Unsupported GPU"); } CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extrad->data_device)); CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offsetd)); CL_CHECK(clSetKernelArg(kernel, 4, sizeof(int), &ne)); CL_CHECK(clSetKernelArg(kernel, 5, sizeof(int), &group_size)); CL_CHECK(clSetKernelArg(kernel, 6, sizeof(float), &eps)); size_t global_work_size[] = {(size_t)n_groups*sgs, 1, 1}; size_t local_work_size[] = {(size_t)sgs, 1, 1}; backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); } static void ggml_cl_l2_norm(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { GGML_ASSERT(src0); GGML_ASSERT(src0->extra); GGML_ASSERT(dst); GGML_ASSERT(dst->extra); UNUSED(src1); ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; ggml_tensor_extra_cl * extra0 = (ggml_tensor_extra_cl *)src0->extra; ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; cl_ulong offset0 = extra0->offset + src0->view_offs; cl_ulong offsetd = extrad->offset + dst->view_offs; float eps; memcpy(&eps, dst->op_params, sizeof(float)); GGML_TENSOR_LOCALS(int, ne0, src0, ne); GGML_TENSOR_LOCALS(cl_ulong, nb0, src0, nb); size_t sgs; if (backend_ctx->gpu_family == ADRENO) { sgs = 64; } else if (backend_ctx->gpu_family == INTEL) { sgs = 32; } else { GGML_ASSERT(false && "Unsupported GPU"); } cl_kernel kernel = backend_ctx->kernel_l2_norm_f32; int nth = sgs; while (nth < ne00 && nth < (int)backend_ctx->get_kernel_workgroup_size(kernel)) { nth *= 2; } CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extrad->data_device)); CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offsetd)); CL_CHECK(clSetKernelArg(kernel, 4, sizeof(int), &ne00)); CL_CHECK(clSetKernelArg(kernel, 5, sizeof(int), &ne01)); CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne02)); CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne03)); CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_ulong), &nb01)); CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_ulong), &nb02)); CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_ulong), &nb03)); CL_CHECK(clSetKernelArg(kernel, 11, sizeof(float), &eps)); CL_CHECK(clSetKernelArg(kernel, 12, sizeof(float)*nth/sgs, NULL)); size_t global_work_size[] = {(size_t)ne01*nth, (size_t)ne02, (size_t)ne03}; size_t local_work_size[] = {(size_t)nth, 1, 1}; backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); } static void ggml_cl_tanh(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { GGML_ASSERT(src0); GGML_ASSERT(src0->extra); GGML_ASSERT(dst); GGML_ASSERT(dst->extra); UNUSED(src1); ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; ggml_tensor_extra_cl * extra0 = (ggml_tensor_extra_cl *)src0->extra; ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; cl_ulong offset0 = extra0->offset + src0->view_offs; cl_ulong offsetd = extrad->offset + dst->view_offs; const int ne00 = src0->ne[0]; const int ne01 = src0->ne[1]; const int ne02 = src0->ne[2]; const int ne03 = src0->ne[3]; const cl_ulong nb00 = src0->nb[0]; const cl_ulong nb01 = src0->nb[1]; const cl_ulong nb02 = src0->nb[2]; const cl_ulong nb03 = src0->nb[3]; const cl_ulong nb0 = dst->nb[0]; const cl_ulong nb1 = dst->nb[1]; const cl_ulong nb2 = dst->nb[2]; const cl_ulong nb3 = dst->nb[3]; cl_kernel kernel; if (ggml_is_contiguous(src0)) { // Handle contiguous input int n = ggml_nelements(dst); if (n % 4 == 0) { if (src0->type == GGML_TYPE_F32) { kernel = backend_ctx->kernel_tanh_f32_4; } else { kernel = backend_ctx->kernel_tanh_f16_4; } n /= 4; } else { if (src0->type == GGML_TYPE_F32) { kernel = backend_ctx->kernel_tanh_f32; } else { kernel = backend_ctx->kernel_tanh_f16; } } CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extrad->data_device)); CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offsetd)); size_t global_work_size[] = {(size_t)n, 1, 1}; size_t local_work_size[] = {64, 1, 1}; size_t * local_work_size_ptr = local_work_size; if (n % 64 != 0 && !backend_ctx->non_uniform_workgroups) { local_work_size_ptr = nullptr; } backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size_ptr, dst); } else { // Handle non-contiguous input if (src0->type == GGML_TYPE_F32) { kernel = backend_ctx->kernel_tanh_f32_nc; } else { kernel = backend_ctx->kernel_tanh_f16_nc; } CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extrad->data_device)); CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offsetd)); CL_CHECK(clSetKernelArg(kernel, 4, sizeof(int), &ne00)); CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &nb00)); CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_ulong), &nb01)); CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_ulong), &nb02)); CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_ulong), &nb03)); CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_ulong), &nb0)); CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_ulong), &nb1)); CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_ulong), &nb2)); CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_ulong), &nb3)); int nth = 64; size_t global_work_size[] = {(size_t)ne01*nth, (size_t)ne02, (size_t)ne03}; size_t local_work_size[] = {(size_t)nth, 1, 1}; backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); } } static void ggml_cl_neg(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { GGML_ASSERT(src0); GGML_ASSERT(src0->extra); GGML_ASSERT(dst); GGML_ASSERT(dst->extra); UNUSED(src1); ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; ggml_tensor_extra_cl * extra0 = (ggml_tensor_extra_cl *)src0->extra; ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; cl_ulong offset0 = extra0->offset + src0->view_offs; cl_ulong offsetd = extrad->offset + dst->view_offs; GGML_TENSOR_LOCALS(int, ne0, src0, ne); GGML_TENSOR_LOCALS(cl_ulong, nb0, src0, nb); GGML_TENSOR_LOCALS(int, ne, dst, ne); GGML_TENSOR_LOCALS(cl_ulong, nb, dst, nb); cl_kernel kernel; if (ggml_is_contiguous(src0)) { // Handle contiguous input int n = ggml_nelements(dst); if (n % 4 == 0) { if (src0->type == GGML_TYPE_F32) { kernel = backend_ctx->kernel_neg_f32_4; } else { kernel = backend_ctx->kernel_neg_f16_4; } n /= 4; } else { if (src0->type == GGML_TYPE_F32) { kernel = backend_ctx->kernel_neg_f32; } else { kernel = backend_ctx->kernel_neg_f16; } } CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extrad->data_device)); CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offsetd)); CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_int), &n)); size_t global_work_size[] = {(size_t)CEIL_DIV(n, 64)*64, 1, 1}; size_t local_work_size[] = {64, 1, 1}; backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); } else { // Handle non-contiguous input if (src0->type == GGML_TYPE_F32) { kernel = backend_ctx->kernel_neg_f32_nc; } else { kernel = backend_ctx->kernel_neg_f16_nc; } CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extrad->data_device)); CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offsetd)); CL_CHECK(clSetKernelArg(kernel, 4, sizeof(int), &ne00)); CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &nb00)); CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_ulong), &nb01)); CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_ulong), &nb02)); CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_ulong), &nb03)); CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_ulong), &nb0)); CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_ulong), &nb1)); CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_ulong), &nb2)); CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_ulong), &nb3)); int nth = 64; size_t global_work_size[] = {(size_t)ne01*nth, (size_t)ne02, (size_t)ne03}; size_t local_work_size[] = {(size_t)nth, 1, 1}; backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); } } static void ggml_cl_exp(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { GGML_ASSERT(src0); GGML_ASSERT(src0->extra); GGML_ASSERT(dst); GGML_ASSERT(dst->extra); UNUSED(src1); ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; ggml_tensor_extra_cl * extra0 = (ggml_tensor_extra_cl *)src0->extra; ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; cl_ulong offset0 = extra0->offset + src0->view_offs; cl_ulong offsetd = extrad->offset + dst->view_offs; GGML_TENSOR_LOCALS(int, ne0, src0, ne); GGML_TENSOR_LOCALS(cl_ulong, nb0, src0, nb); GGML_TENSOR_LOCALS(int, ne, dst, ne); GGML_TENSOR_LOCALS(cl_ulong, nb, dst, nb); cl_kernel kernel; if (ggml_is_contiguous(src0)) { // Handle contiguous input int n = ggml_nelements(dst); if (n % 4 == 0) { if (src0->type == GGML_TYPE_F32) { kernel = backend_ctx->kernel_exp_f32_4; } else { kernel = backend_ctx->kernel_exp_f16_4; } n /= 4; } else { if (src0->type == GGML_TYPE_F32) { kernel = backend_ctx->kernel_exp_f32; } else { kernel = backend_ctx->kernel_exp_f16; } } CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extrad->data_device)); CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offsetd)); CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_int), &n)); size_t global_work_size[] = {(size_t)CEIL_DIV(n, 64)*64, 1, 1}; size_t local_work_size[] = {64, 1, 1}; backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); } else { // Handle non-contiguous input if (src0->type == GGML_TYPE_F32) { kernel = backend_ctx->kernel_exp_f32_nc; } else { kernel = backend_ctx->kernel_exp_f16_nc; } CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extrad->data_device)); CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offsetd)); CL_CHECK(clSetKernelArg(kernel, 4, sizeof(int), &ne00)); CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &nb00)); CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_ulong), &nb01)); CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_ulong), &nb02)); CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_ulong), &nb03)); CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_ulong), &nb0)); CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_ulong), &nb1)); CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_ulong), &nb2)); CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_ulong), &nb3)); int nth = 64; size_t global_work_size[] = {(size_t)ne01*nth, (size_t)ne02, (size_t)ne03}; size_t local_work_size[] = {(size_t)nth, 1, 1}; backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); } } static void ggml_cl_expm1(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { GGML_ASSERT(src0); GGML_ASSERT(src0->extra); GGML_ASSERT(dst); GGML_ASSERT(dst->extra); UNUSED(src1); ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; ggml_tensor_extra_cl * extra0 = (ggml_tensor_extra_cl *)src0->extra; ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; cl_ulong offset0 = extra0->offset + src0->view_offs; cl_ulong offsetd = extrad->offset + dst->view_offs; const int ne00 = src0->ne[0]; const int ne01 = src0->ne[1]; const int ne02 = src0->ne[2]; const int ne03 = src0->ne[3]; const cl_ulong nb00 = src0->nb[0]; const cl_ulong nb01 = src0->nb[1]; const cl_ulong nb02 = src0->nb[2]; const cl_ulong nb03 = src0->nb[3]; const cl_ulong nb0 = dst->nb[0]; const cl_ulong nb1 = dst->nb[1]; const cl_ulong nb2 = dst->nb[2]; const cl_ulong nb3 = dst->nb[3]; cl_kernel kernel; if (ggml_is_contiguous(src0)) { // Handle contiguous input int n = ggml_nelements(dst); if (n % 4 == 0) { if (src0->type == GGML_TYPE_F32) { kernel = backend_ctx->kernel_expm1_f32_4; } else { kernel = backend_ctx->kernel_expm1_f16_4; } n /= 4; } else { if (src0->type == GGML_TYPE_F32) { kernel = backend_ctx->kernel_expm1_f32; } else { kernel = backend_ctx->kernel_expm1_f16; } } CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extrad->data_device)); CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offsetd)); size_t global_work_size[] = {(size_t)n, 1, 1}; size_t local_work_size[] = {64, 1, 1}; size_t * local_work_size_ptr = local_work_size; if (n % 64 != 0 && !backend_ctx->non_uniform_workgroups) { local_work_size_ptr = nullptr; } backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size_ptr, dst); } else { // Handle non-contiguous input if (src0->type == GGML_TYPE_F32) { kernel = backend_ctx->kernel_expm1_f32_nc; } else { kernel = backend_ctx->kernel_expm1_f16_nc; } CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extrad->data_device)); CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offsetd)); CL_CHECK(clSetKernelArg(kernel, 4, sizeof(int), &ne00)); CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &nb00)); CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_ulong), &nb01)); CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_ulong), &nb02)); CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_ulong), &nb03)); CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_ulong), &nb0)); CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_ulong), &nb1)); CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_ulong), &nb2)); CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_ulong), &nb3)); int nth = 64; size_t global_work_size[] = {(size_t)ne01*nth, (size_t)ne02, (size_t)ne03}; size_t local_work_size[] = {(size_t)nth, 1, 1}; backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); } } static void ggml_cl_abs(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { GGML_ASSERT(src0); GGML_ASSERT(src0->extra); GGML_ASSERT(dst); GGML_ASSERT(dst->extra); UNUSED(src1); ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; ggml_tensor_extra_cl * extra0 = (ggml_tensor_extra_cl *)src0->extra; ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; cl_ulong offset0 = extra0->offset + src0->view_offs; cl_ulong offsetd = extrad->offset + dst->view_offs; const int ne00 = src0->ne[0]; const int ne01 = src0->ne[1]; const int ne02 = src0->ne[2]; const int ne03 = src0->ne[3]; const cl_ulong nb00 = src0->nb[0]; const cl_ulong nb01 = src0->nb[1]; const cl_ulong nb02 = src0->nb[2]; const cl_ulong nb03 = src0->nb[3]; const cl_ulong nb0 = dst->nb[0]; const cl_ulong nb1 = dst->nb[1]; const cl_ulong nb2 = dst->nb[2]; const cl_ulong nb3 = dst->nb[3]; cl_kernel kernel; if (ggml_is_contiguous(src0)) { // Handle contiguous input int n = ggml_nelements(dst); if (n % 4 == 0) { if (src0->type == GGML_TYPE_F32) { kernel = backend_ctx->kernel_abs_f32_4; } else { kernel = backend_ctx->kernel_abs_f16_4; } n /= 4; } else { if (src0->type == GGML_TYPE_F32) { kernel = backend_ctx->kernel_abs_f32; } else { kernel = backend_ctx->kernel_abs_f16; } } CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extrad->data_device)); CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offsetd)); size_t global_work_size[] = {(size_t)n, 1, 1}; size_t local_work_size[] = {64, 1, 1}; size_t * local_work_size_ptr = local_work_size; if (n % 64 != 0 && !backend_ctx->non_uniform_workgroups) { local_work_size_ptr = nullptr; } backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size_ptr, dst); } else { // Handle non-contiguous input if (src0->type == GGML_TYPE_F32) { kernel = backend_ctx->kernel_abs_f32_nc; } else { kernel = backend_ctx->kernel_abs_f16_nc; } CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extrad->data_device)); CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offsetd)); CL_CHECK(clSetKernelArg(kernel, 4, sizeof(int), &ne00)); CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &nb00)); CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_ulong), &nb01)); CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_ulong), &nb02)); CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_ulong), &nb03)); CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_ulong), &nb0)); CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_ulong), &nb1)); CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_ulong), &nb2)); CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_ulong), &nb3)); int nth = 64; size_t global_work_size[] = {(size_t)ne01*nth, (size_t)ne02, (size_t)ne03}; size_t local_work_size[] = {(size_t)nth, 1, 1}; backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); } } // Shared driver for the extended unary ops (unary_ext.cl), same selection as // ggml_cl_abs: contiguous picks the vec4 kernel when the element count is a // multiple of 4 (else scalar); non-contiguous uses the stride-addressed kernel. static void ggml_cl_unary_ext(ggml_backend_t backend, const ggml_tensor * src0, ggml_tensor * dst, cl_kernel k_f32, cl_kernel k_f32_4, cl_kernel k_f32_nc, cl_kernel k_f16, cl_kernel k_f16_4, cl_kernel k_f16_nc) { GGML_ASSERT(src0); GGML_ASSERT(src0->extra); GGML_ASSERT(dst); GGML_ASSERT(dst->extra); ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; ggml_tensor_extra_cl * extra0 = (ggml_tensor_extra_cl *)src0->extra; ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; cl_ulong offset0 = extra0->offset + src0->view_offs; cl_ulong offsetd = extrad->offset + dst->view_offs; const int ne00 = src0->ne[0], ne01 = src0->ne[1], ne02 = src0->ne[2], ne03 = src0->ne[3]; const cl_ulong nb00 = src0->nb[0], nb01 = src0->nb[1], nb02 = src0->nb[2], nb03 = src0->nb[3]; const cl_ulong nb0 = dst->nb[0], nb1 = dst->nb[1], nb2 = dst->nb[2], nb3 = dst->nb[3]; const bool is_f16 = (src0->type == GGML_TYPE_F16); cl_kernel kernel; if (ggml_is_contiguous(src0)) { int n = ggml_nelements(dst); if (n % 4 == 0) { kernel = is_f16 ? k_f16_4 : k_f32_4; n /= 4; } else { kernel = is_f16 ? k_f16 : k_f32; } CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extrad->data_device)); CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offsetd)); size_t global_work_size[] = {(size_t)n, 1, 1}; size_t local_work_size[] = {64, 1, 1}; size_t * local_work_size_ptr = local_work_size; if (n % 64 != 0 && !backend_ctx->non_uniform_workgroups) { local_work_size_ptr = nullptr; } backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size_ptr, dst); } else { kernel = is_f16 ? k_f16_nc : k_f32_nc; CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extrad->data_device)); CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offsetd)); CL_CHECK(clSetKernelArg(kernel, 4, sizeof(int), &ne00)); CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &nb00)); CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_ulong), &nb01)); CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_ulong), &nb02)); CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_ulong), &nb03)); CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_ulong), &nb0)); CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_ulong), &nb1)); CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_ulong), &nb2)); CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_ulong), &nb3)); int nth = 64; size_t global_work_size[] = {(size_t)ne01*nth, (size_t)ne02, (size_t)ne03}; size_t local_work_size[] = {(size_t)nth, 1, 1}; backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); } } #define GGML_CL_UNARY_EXT_WRAP(FN, OP) \ static void FN(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { \ UNUSED(src1); \ ggml_backend_opencl_context *c = (ggml_backend_opencl_context *)backend->context; \ ggml_cl_unary_ext(backend, src0, dst, c->kernel_##OP##_f32, c->kernel_##OP##_f32_4, c->kernel_##OP##_f32_nc, \ c->kernel_##OP##_f16, c->kernel_##OP##_f16_4, c->kernel_##OP##_f16_nc); \ } GGML_CL_UNARY_EXT_WRAP(ggml_cl_sgn, sgn) GGML_CL_UNARY_EXT_WRAP(ggml_cl_step, step) GGML_CL_UNARY_EXT_WRAP(ggml_cl_elu, elu) GGML_CL_UNARY_EXT_WRAP(ggml_cl_hardswish, hardswish) GGML_CL_UNARY_EXT_WRAP(ggml_cl_hardsigmoid, hardsigmoid) GGML_CL_UNARY_EXT_WRAP(ggml_cl_floor, floor) GGML_CL_UNARY_EXT_WRAP(ggml_cl_ceil, ceil) GGML_CL_UNARY_EXT_WRAP(ggml_cl_round, round) GGML_CL_UNARY_EXT_WRAP(ggml_cl_trunc, trunc) #undef GGML_CL_UNARY_EXT_WRAP static void ggml_cl_softplus(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { GGML_ASSERT(src0); GGML_ASSERT(src0->extra); GGML_ASSERT(dst); GGML_ASSERT(dst->extra); UNUSED(src1); ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; ggml_tensor_extra_cl * extra0 = (ggml_tensor_extra_cl *)src0->extra; ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; cl_ulong offset0 = extra0->offset + src0->view_offs; cl_ulong offsetd = extrad->offset + dst->view_offs; const int ne00 = src0->ne[0]; const int ne01 = src0->ne[1]; const int ne02 = src0->ne[2]; const int ne03 = src0->ne[3]; const cl_ulong nb00 = src0->nb[0]; const cl_ulong nb01 = src0->nb[1]; const cl_ulong nb02 = src0->nb[2]; const cl_ulong nb03 = src0->nb[3]; const cl_ulong nb0 = dst->nb[0]; const cl_ulong nb1 = dst->nb[1]; const cl_ulong nb2 = dst->nb[2]; const cl_ulong nb3 = dst->nb[3]; cl_kernel kernel; if (ggml_is_contiguous(src0)) { // Handle contiguous input int n = ggml_nelements(dst); if (n % 4 == 0) { if (src0->type == GGML_TYPE_F32) { kernel = backend_ctx->kernel_softplus_f32_4; } else { kernel = backend_ctx->kernel_softplus_f16_4; } n /= 4; } else { if (src0->type == GGML_TYPE_F32) { kernel = backend_ctx->kernel_softplus_f32; } else { kernel = backend_ctx->kernel_softplus_f16; } } CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extrad->data_device)); CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offsetd)); size_t global_work_size[] = {(size_t)n, 1, 1}; size_t local_work_size[] = {64, 1, 1}; size_t * local_work_size_ptr = local_work_size; if (n % 64 != 0 && !backend_ctx->non_uniform_workgroups) { local_work_size_ptr = nullptr; } backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size_ptr, dst); } else { // Handle non-contiguous input if (src0->type == GGML_TYPE_F32) { kernel = backend_ctx->kernel_softplus_f32_nc; } else { kernel = backend_ctx->kernel_softplus_f16_nc; } CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extrad->data_device)); CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offsetd)); CL_CHECK(clSetKernelArg(kernel, 4, sizeof(int), &ne00)); CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &nb00)); CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_ulong), &nb01)); CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_ulong), &nb02)); CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_ulong), &nb03)); CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_ulong), &nb0)); CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_ulong), &nb1)); CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_ulong), &nb2)); CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_ulong), &nb3)); int nth = 64; size_t global_work_size[] = {(size_t)ne01*nth, (size_t)ne02, (size_t)ne03}; size_t local_work_size[] = {(size_t)nth, 1, 1}; backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); } } static void ggml_cl_repeat(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1_shape_def, ggml_tensor * dst) { GGML_ASSERT(src0); GGML_ASSERT(src0->extra); GGML_ASSERT(dst); GGML_ASSERT(dst->extra); GGML_ASSERT(dst->type == src0->type); UNUSED(src1_shape_def); ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; ggml_tensor_extra_cl * extra0 = (ggml_tensor_extra_cl *)src0->extra; ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; cl_ulong offset0 = extra0->offset + src0->view_offs; cl_ulong offsetd = extrad->offset + dst->view_offs; const int ne00 = src0->ne[0]; const int ne01 = src0->ne[1]; const int ne02 = src0->ne[2]; const int ne03 = src0->ne[3]; const cl_ulong nb00 = src0->nb[0]; const cl_ulong nb01 = src0->nb[1]; const cl_ulong nb02 = src0->nb[2]; const cl_ulong nb03 = src0->nb[3]; const int ne0 = dst->ne[0]; const int ne1 = dst->ne[1]; const int ne2 = dst->ne[2]; const int ne3 = dst->ne[3]; const cl_ulong nb0 = dst->nb[0]; const cl_ulong nb1 = dst->nb[1]; const cl_ulong nb2 = dst->nb[2]; const cl_ulong nb3 = dst->nb[3]; cl_kernel kernel = backend_ctx->kernel_repeat_f32; CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extrad->data_device)); CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offsetd)); CL_CHECK(clSetKernelArg(kernel, 4, sizeof(int), &ne00)); CL_CHECK(clSetKernelArg(kernel, 5, sizeof(int), &ne01)); CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne02)); CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne03)); CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_ulong), &nb00)); CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_ulong), &nb01)); CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_ulong), &nb02)); CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_ulong), &nb03)); CL_CHECK(clSetKernelArg(kernel, 12, sizeof(int), &ne0)); CL_CHECK(clSetKernelArg(kernel, 13, sizeof(cl_ulong), &nb0)); CL_CHECK(clSetKernelArg(kernel, 14, sizeof(cl_ulong), &nb1)); CL_CHECK(clSetKernelArg(kernel, 15, sizeof(cl_ulong), &nb2)); CL_CHECK(clSetKernelArg(kernel, 16, sizeof(cl_ulong), &nb3)); int nth = 64; size_t global_work_size[] = {(size_t)ne1*nth, (size_t)ne2, (size_t)ne3}; size_t local_work_size[] = {(size_t)nth, 1, 1}; backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); } static void ggml_cl_pad(ggml_backend_t backend, const ggml_tensor * src0, ggml_tensor * dst) { GGML_ASSERT(src0); GGML_ASSERT(src0->extra); GGML_ASSERT(dst); GGML_ASSERT(dst->extra); GGML_ASSERT(src0->type == GGML_TYPE_F32); GGML_ASSERT(dst->type == GGML_TYPE_F32); ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; if (backend_ctx->kernel_pad == nullptr) { GGML_LOG_WARN("%s: pad kernel not available, skipping OpenCL execution.\n", __func__); return; } ggml_tensor_extra_cl * extra_src0 = (ggml_tensor_extra_cl *)src0->extra; ggml_tensor_extra_cl * extra_dst = (ggml_tensor_extra_cl *)dst->extra; cl_ulong off_src0 = extra_src0->offset + src0->view_offs; cl_ulong off_dst = extra_dst->offset + dst->view_offs; const int s_ne0 = src0->ne[0]; const int s_ne1 = src0->ne[1]; const int s_ne2 = src0->ne[2]; const int s_ne3 = src0->ne[3]; const int s_nb0 = src0->nb[0]; const int s_nb1 = src0->nb[1]; const int s_nb2 = src0->nb[2]; const int s_nb3 = src0->nb[3]; const int d_ne0 = dst->ne[0]; const int d_ne1 = dst->ne[1]; const int d_ne2 = dst->ne[2]; const int d_ne3 = dst->ne[3]; const int d_nb0 = dst->nb[0]; const int d_nb1 = dst->nb[1]; const int d_nb2 = dst->nb[2]; const int d_nb3 = dst->nb[3]; const int lp0 = ((const int*)(dst->op_params))[0]; const int rp0 = ((const int*)(dst->op_params))[1]; const int lp1 = ((const int*)(dst->op_params))[2]; const int rp1 = ((const int*)(dst->op_params))[3]; const int lp2 = ((const int*)(dst->op_params))[4]; const int rp2 = ((const int*)(dst->op_params))[5]; const int lp3 = ((const int*)(dst->op_params))[6]; const int rp3 = ((const int*)(dst->op_params))[7]; cl_kernel kernel = backend_ctx->kernel_pad; CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra_src0->data_device)); CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &off_src0)); CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra_dst->data_device)); CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &off_dst)); CL_CHECK(clSetKernelArg(kernel, 4, sizeof(int), &s_ne0)); CL_CHECK(clSetKernelArg(kernel, 5, sizeof(int), &s_ne1)); CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &s_ne2)); CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &s_ne3)); CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_ulong), &s_nb0)); CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_ulong), &s_nb1)); CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_ulong), &s_nb2)); CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_ulong), &s_nb3)); CL_CHECK(clSetKernelArg(kernel, 12, sizeof(int), &d_ne0)); CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &d_ne1)); CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &d_ne2)); CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &d_ne3)); CL_CHECK(clSetKernelArg(kernel, 16, sizeof(cl_ulong), &d_nb0)); CL_CHECK(clSetKernelArg(kernel, 17, sizeof(cl_ulong), &d_nb1)); CL_CHECK(clSetKernelArg(kernel, 18, sizeof(cl_ulong), &d_nb2)); CL_CHECK(clSetKernelArg(kernel, 19, sizeof(cl_ulong), &d_nb3)); CL_CHECK(clSetKernelArg(kernel, 20, sizeof(int), &lp0)); CL_CHECK(clSetKernelArg(kernel, 21, sizeof(int), &rp0)); CL_CHECK(clSetKernelArg(kernel, 22, sizeof(int), &lp1)); CL_CHECK(clSetKernelArg(kernel, 23, sizeof(int), &rp1)); CL_CHECK(clSetKernelArg(kernel, 24, sizeof(int), &lp2)); CL_CHECK(clSetKernelArg(kernel, 25, sizeof(int), &rp2)); CL_CHECK(clSetKernelArg(kernel, 26, sizeof(int), &lp3)); CL_CHECK(clSetKernelArg(kernel, 27, sizeof(int), &rp3)); size_t lws0 = 64; size_t gws0 = (( (size_t)d_ne0 + lws0 - 1 ) / lws0) * lws0; size_t global_work_size[] = { gws0, (size_t)d_ne1, (size_t)d_ne2*d_ne3 }; size_t local_work_size[] = { lws0, 1, 1 }; size_t * local_work_size_ptr = local_work_size; if (d_ne0 % lws0 != 0 && !backend_ctx->non_uniform_workgroups) { local_work_size_ptr = nullptr; } backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size_ptr, dst); } static void ggml_cl_upscale(ggml_backend_t backend, const ggml_tensor * src0, ggml_tensor * dst) { GGML_ASSERT(src0); GGML_ASSERT(src0->extra); GGML_ASSERT(dst); GGML_ASSERT(dst->extra); GGML_ASSERT(src0->type == GGML_TYPE_F32); GGML_ASSERT(dst->type == GGML_TYPE_F32); ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; const int mode_flags = (ggml_scale_mode) ggml_get_op_params_i32(dst, 0); const ggml_scale_mode mode = (ggml_scale_mode) (mode_flags & 0xFF); cl_kernel kernel = nullptr; if (mode == GGML_SCALE_MODE_NEAREST) { kernel = backend_ctx->kernel_upscale; if (kernel == nullptr) { GGML_LOG_WARN("%s: nearest upscale kernel not available, skipping OpenCL execution.\n", __func__); return; } } else if (mode == GGML_SCALE_MODE_BILINEAR) { kernel = backend_ctx->kernel_upscale_bilinear; if (kernel == nullptr) { GGML_LOG_WARN("%s: bilinear upscale kernel not available, skipping OpenCL execution.\n", __func__); return; } } else { GGML_LOG_WARN("%s: unsupported upscale mode %d, skipping OpenCL execution.\n", __func__, mode); return; } ggml_tensor_extra_cl * extra_src0 = (ggml_tensor_extra_cl *)src0->extra; ggml_tensor_extra_cl * extra_dst = (ggml_tensor_extra_cl *)dst->extra; cl_ulong off_src0 = extra_src0->offset + src0->view_offs; cl_ulong off_dst = extra_dst->offset + dst->view_offs; const cl_ulong nb00 = src0->nb[0]; const cl_ulong nb01 = src0->nb[1]; const cl_ulong nb02 = src0->nb[2]; const cl_ulong nb03 = src0->nb[3]; const int ne00 = src0->ne[0]; const int ne01 = src0->ne[1]; const int ne02 = src0->ne[2]; const int ne03 = src0->ne[3]; const int ne0 = dst->ne[0]; const int ne1 = dst->ne[1]; const int ne2 = dst->ne[2]; const int ne3 = dst->ne[3]; float sf0 = (float)ne0 / ne00; float sf1 = (float)ne1 / ne01; float sf2 = (float)ne2 / ne02; float sf3 = (float)ne3 / ne03; float pixel_offset = 0.5f; CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra_src0->data_device)); CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &off_src0)); CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra_dst->data_device)); CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &off_dst)); CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_ulong), &nb00)); CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &nb01)); CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_ulong), &nb02)); CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_ulong), &nb03)); if (mode == GGML_SCALE_MODE_NEAREST) { CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne0)); CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne1)); CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &ne2)); CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne3)); CL_CHECK(clSetKernelArg(kernel, 12, sizeof(float), &sf0)); CL_CHECK(clSetKernelArg(kernel, 13, sizeof(float), &sf1)); CL_CHECK(clSetKernelArg(kernel, 14, sizeof(float), &sf2)); CL_CHECK(clSetKernelArg(kernel, 15, sizeof(float), &sf3)); } else if (mode == GGML_SCALE_MODE_BILINEAR) { if (mode_flags & GGML_SCALE_FLAG_ALIGN_CORNERS) { sf0 = ne0 > 1 && ne00 > 1 ? (float)(ne0 - 1) / (ne00 - 1) : sf0; sf1 = ne1 > 1 && ne01 > 1 ? (float)(ne1 - 1) / (ne01 - 1) : sf1; pixel_offset = 0.0f; } CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne00)); CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne01)); CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &ne0)); CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne1)); CL_CHECK(clSetKernelArg(kernel, 12, sizeof(int), &ne2)); CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &ne3)); CL_CHECK(clSetKernelArg(kernel, 14, sizeof(float), &sf0)); CL_CHECK(clSetKernelArg(kernel, 15, sizeof(float), &sf1)); CL_CHECK(clSetKernelArg(kernel, 16, sizeof(float), &sf2)); CL_CHECK(clSetKernelArg(kernel, 17, sizeof(float), &sf3)); CL_CHECK(clSetKernelArg(kernel, 18, sizeof(float), &pixel_offset)); } size_t dst_total_elements = (size_t)ne0 * ne1 * ne2 * ne3; if (dst_total_elements == 0) { return; } size_t global_work_size[] = { dst_total_elements, 1, 1 }; size_t local_work_size_pref = 256; size_t local_work_size[] = { MIN(local_work_size_pref, dst_total_elements), 1, 1}; size_t * local_work_size_ptr = local_work_size; if (dst_total_elements % local_work_size[0] != 0 && !backend_ctx->non_uniform_workgroups) { local_work_size_ptr = nullptr; } backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size_ptr, dst); } static void ggml_cl_concat(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { GGML_ASSERT(src0); GGML_ASSERT(src0->extra); GGML_ASSERT(src1); GGML_ASSERT(src1->extra); GGML_ASSERT(dst); GGML_ASSERT(dst->extra); GGML_ASSERT(src0->type == src1->type); GGML_ASSERT(src0->type == dst->type); ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; ggml_tensor_extra_cl * extra0 = (ggml_tensor_extra_cl *)src0->extra; ggml_tensor_extra_cl * extra1 = (ggml_tensor_extra_cl *)src1->extra; ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; cl_ulong offset0 = extra0->offset + src0->view_offs; cl_ulong offset1 = extra1->offset + src1->view_offs; cl_ulong offsetd = extrad->offset + dst->view_offs; const int ne00 = src0->ne[0]; const int ne01 = src0->ne[1]; const int ne02 = src0->ne[2]; const int ne03 = src0->ne[3]; const cl_ulong nb00 = src0->nb[0]; const cl_ulong nb01 = src0->nb[1]; const cl_ulong nb02 = src0->nb[2]; const cl_ulong nb03 = src0->nb[3]; const cl_ulong nb10 = src1->nb[0]; const cl_ulong nb11 = src1->nb[1]; const cl_ulong nb12 = src1->nb[2]; const cl_ulong nb13 = src1->nb[3]; const int ne0 = dst->ne[0]; const int ne1 = dst->ne[1]; const int ne2 = dst->ne[2]; const int ne3 = dst->ne[3]; const cl_ulong nb0 = dst->nb[0]; const cl_ulong nb1 = dst->nb[1]; const cl_ulong nb2 = dst->nb[2]; const cl_ulong nb3 = dst->nb[3]; const cl_int dim = ((const int32_t *) dst->op_params)[0]; GGML_ASSERT(dim >= 0 && dim <= 3); int nth = MIN(64, ne0); const size_t ts = ggml_type_size(dst->type); // the pack kernel copies 4-byte elements, so it is only valid for those. const bool concat_pack = (dim == 0 && ne0 < 32 && ts == 4); cl_kernel kernel; if (concat_pack) { kernel = backend_ctx->kernel_concat_b4_pack; } else { switch (ts) { case 1: kernel = backend_ctx->kernel_concat_b1; break; case 2: kernel = backend_ctx->kernel_concat_b2; break; case 4: kernel = backend_ctx->kernel_concat_b4; break; case 8: kernel = backend_ctx->kernel_concat_b8; break; default: GGML_ABORT("unsupported concat element size: %zu", ts); } } CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device)); CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1)); CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device)); CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd)); CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne00)); CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne01)); CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne02)); CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne03)); CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_ulong), &nb00)); CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_ulong), &nb01)); CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_ulong), &nb02)); CL_CHECK(clSetKernelArg(kernel, 13, sizeof(cl_ulong), &nb03)); CL_CHECK(clSetKernelArg(kernel, 14, sizeof(cl_ulong), &nb10)); CL_CHECK(clSetKernelArg(kernel, 15, sizeof(cl_ulong), &nb11)); CL_CHECK(clSetKernelArg(kernel, 16, sizeof(cl_ulong), &nb12)); CL_CHECK(clSetKernelArg(kernel, 17, sizeof(cl_ulong), &nb13)); CL_CHECK(clSetKernelArg(kernel, 18, sizeof(int), &ne0)); CL_CHECK(clSetKernelArg(kernel, 19, sizeof(cl_ulong), &nb0)); CL_CHECK(clSetKernelArg(kernel, 20, sizeof(cl_ulong), &nb1)); CL_CHECK(clSetKernelArg(kernel, 21, sizeof(cl_ulong), &nb2)); CL_CHECK(clSetKernelArg(kernel, 22, sizeof(cl_ulong), &nb3)); CL_CHECK(clSetKernelArg(kernel, 23, sizeof(cl_int), &dim)); if (concat_pack) { // packed kernel needs the dst dims to unflatten its 1-D row index. CL_CHECK(clSetKernelArg(kernel, 24, sizeof(int), &ne1)); CL_CHECK(clSetKernelArg(kernel, 25, sizeof(int), &ne2)); CL_CHECK(clSetKernelArg(kernel, 26, sizeof(int), &ne3)); const int maxwg = (int)backend_ctx->get_kernel_workgroup_size(kernel); const int base = MIN(64, maxwg); const int tpr = MIN(ne0, base); // threads per row const int rpw = MAX(1, base / tpr); // rows per workgroup const int lsz = tpr * rpw; const int nrows = ne1*ne2*ne3; const int nwg = (nrows + rpw - 1) / rpw; size_t global_work_size[] = {(size_t)nwg*lsz, 1, 1}; size_t local_work_size[] = {(size_t)lsz, 1, 1}; backend_ctx->enqueue_ndrange_kernel(kernel, 1, global_work_size, local_work_size, dst); } else { size_t global_work_size[] = {(size_t)ne1*nth, (size_t)ne2, (size_t)ne3}; size_t local_work_size[] = {(size_t)nth, 1, 1}; backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); } } static void ggml_cl_timestep_embedding(ggml_backend_t backend, const ggml_tensor * src0, ggml_tensor * dst) { GGML_ASSERT(src0); GGML_ASSERT(src0->extra); GGML_ASSERT(dst); GGML_ASSERT(dst->extra); GGML_ASSERT(src0->type == GGML_TYPE_F32); GGML_ASSERT(dst->type == GGML_TYPE_F32); ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; if (backend_ctx->kernel_timestep_embedding == nullptr) { GGML_LOG_WARN("%s: timestep_embedding kernel not available, skipping OpenCL execution.\n", __func__); return; } ggml_tensor_extra_cl * extra_src0 = (ggml_tensor_extra_cl *)src0->extra; ggml_tensor_extra_cl * extra_dst = (ggml_tensor_extra_cl *)dst->extra; cl_ulong off_src0 = extra_src0->offset + src0->view_offs; cl_ulong off_dst = extra_dst->offset + dst->view_offs; const int logical_dim = dst->op_params[0]; const int max_period = dst->op_params[1]; const int dst_nb1_bytes = dst->nb[1]; cl_kernel kernel = backend_ctx->kernel_timestep_embedding; CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra_src0->data_device)); CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &off_src0)); CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra_dst->data_device)); CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &off_dst)); CL_CHECK(clSetKernelArg(kernel, 4, sizeof(int), &dst_nb1_bytes)); CL_CHECK(clSetKernelArg(kernel, 5, sizeof(int), &logical_dim)); CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &max_period)); size_t gws0 = (size_t)(((logical_dim + 1) / 2) + 1); size_t gws1 = (size_t)src0->ne[0]; size_t global_work_size[] = {gws0, gws1, 1}; backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, NULL, dst); } struct ggml_cl_flash_attn_temp_buffer { cl_mem data = nullptr; ~ggml_cl_flash_attn_temp_buffer() { if (data != nullptr) { CL_CHECK(clReleaseMemObject(data)); data = nullptr; } } }; // Resolve the source buffer + strides for an FA KV tensor: keep the // caller-supplied AoS buffer if non-NULL, else fall back to tensor->extra. static void ggml_cl_flash_attn_resolve_src( const ggml_tensor * tensor, cl_mem & buf, cl_ulong & offset, cl_ulong & nb1, cl_ulong & nb2, cl_ulong & nb3) { if (buf != NULL) { return; } ggml_tensor_extra_cl * extra = (ggml_tensor_extra_cl *) tensor->extra; GGML_ASSERT(extra && extra->data_device); buf = extra->data_device; offset = extra->offset + tensor->view_offs; nb1 = tensor->nb[1]; nb2 = tensor->nb[2]; nb3 = tensor->nb[3]; } // Read a (possibly strided-view) tensor from device into a tight host buffer. // dim 0 is always tight; a strided view is gathered row-by-row. static void ggml_cl_flash_attn_read_tensor_host( ggml_backend_opencl_context * backend_ctx, const ggml_tensor * tensor, cl_mem src_buffer, cl_ulong src_offset, cl_ulong src_nb1, cl_ulong src_nb2, cl_ulong src_nb3, size_t row_bytes, void * dst, size_t total_bytes ) { const bool contiguous_layout = src_nb1 == row_bytes && src_nb2 == row_bytes * (cl_ulong) tensor->ne[1] && src_nb3 == src_nb2 * (cl_ulong) tensor->ne[2]; if (contiguous_layout) { CL_CHECK(clEnqueueReadBuffer(backend_ctx->queue, src_buffer, CL_TRUE, src_offset, total_bytes, dst, 0, NULL, NULL)); return; } size_t dst_off = 0; for (int64_t i3 = 0; i3 < tensor->ne[3]; ++i3) { for (int64_t i2 = 0; i2 < tensor->ne[2]; ++i2) { for (int64_t i1 = 0; i1 < tensor->ne[1]; ++i1) { const cl_ulong row_src_off = src_offset + (cl_ulong) i3 * src_nb3 + (cl_ulong) i2 * src_nb2 + (cl_ulong) i1 * src_nb1; CL_CHECK(clEnqueueReadBuffer(backend_ctx->queue, src_buffer, CL_TRUE, row_src_off, row_bytes, (uint8_t *) dst + dst_off, 0, NULL, NULL)); dst_off += row_bytes; } } } GGML_ASSERT(dst_off == total_bytes); } // forward decl: used by the FA decode dispatch (K-image variant) below. static cl_mem ggml_cl_img_pool_get_or_create( ggml_backend_opencl_context * backend_ctx, std::map & pool, cl_mem data_device, cl_ulong offset0, size_t required_bytes, cl_channel_type channel_data_type); // Rebuild AoS q8_0/q4_0 bytes from a SoA tensor into a temp buffer. // Returns false if the tensor is not SoA-quantised (already AoS). static bool ggml_cl_flash_attn_reconstruct_aos( ggml_backend_opencl_context * backend_ctx, const ggml_tensor * tensor, ggml_cl_flash_attn_temp_buffer & temp, cl_mem & out_buf, cl_ulong & out_offset, cl_ulong & out_nb1, cl_ulong & out_nb2, cl_ulong & out_nb3 ) { if (tensor == nullptr) { return false; } const bool is_q8_0 = tensor->type == GGML_TYPE_Q8_0 && ggml_cl_is_q8_0_soa(tensor); const bool is_q4_0 = tensor->type == GGML_TYPE_Q4_0 && ggml_cl_is_q4_0_soa(tensor); if (!is_q8_0 && !is_q4_0) { return false; } // For views, SoA extra is on view_src (view->extra is pre-SoA). // Noshuffle layout only applies to 2D weights, as determined by `use_adreno_kernels`, // where ne2 == 1 and ne3 == 1 -- these are never FA inputs. // Therefore, we use `restore_block_qk_0` kernels, not `restore_block_qk_0_noshuffle`. const ggml_tensor * soa_src = tensor->view_src ? tensor->view_src : tensor; cl_mem extra_q = NULL; cl_mem extra_d = NULL; if (is_q8_0) { auto * e = (ggml_tensor_extra_cl_q8_0 *) soa_src->extra; GGML_ASSERT(e && e->q && e->d); extra_q = e->q; extra_d = e->d; } else { auto * e = (ggml_tensor_extra_cl_q4_0 *) soa_src->extra; GGML_ASSERT(e && e->q && e->d); extra_q = e->q; extra_d = e->d; } // Reconstruct the whole parent; view offsets then work naturally. const size_t parent_nbytes = ggml_nbytes(soa_src); cl_int err; temp.data = clCreateBuffer(backend_ctx->context, CL_MEM_READ_WRITE, parent_nbytes, NULL, &err); CL_CHECK(err); cl_kernel kernel = is_q8_0 ? backend_ctx->kernel_restore_block_q8_0 : backend_ctx->kernel_restore_block_q4_0; CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra_q)); CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra_d)); CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &temp.data)); const size_t n_blocks = (size_t) ggml_nelements(soa_src) / ggml_blck_size(soa_src->type); size_t global_work_size[] = { n_blocks, 1, 1 }; size_t local_work_size[] = { 1, 1, 1 }; CL_CHECK(clEnqueueNDRangeKernel(backend_ctx->queue, kernel, 3, NULL, global_work_size, local_work_size, 0, NULL, NULL)); out_buf = temp.data; out_offset = tensor->view_offs; out_nb1 = tensor->nb[1]; out_nb2 = tensor->nb[2]; out_nb3 = tensor->nb[3]; return true; } // GPU dequant of a contiguous q4_0/q8_0 KV tensor to f16/f32. Caller supplies // src_buf when reconstructing from SoA. Returns false for non-contig layouts // (the kernel indexes blocks tightly within ne[0]) so the caller can fall back // to the host path. static bool ggml_cl_flash_attn_dequant_kv_gpu( ggml_backend_opencl_context * backend_ctx, const ggml_tensor * tensor, ggml_type target_type, cl_mem in_src_buf, cl_ulong in_src_offset, cl_ulong in_src_nb1, cl_ulong in_src_nb2, cl_ulong in_src_nb3, ggml_cl_flash_attn_temp_buffer & temp, cl_mem & out_buf, cl_ulong & out_offset, cl_ulong & out_nb1, cl_ulong & out_nb2, cl_ulong & out_nb3 ) { GGML_ASSERT(tensor->type == GGML_TYPE_Q8_0 || tensor->type == GGML_TYPE_Q4_0); GGML_ASSERT(target_type == GGML_TYPE_F16 || target_type == GGML_TYPE_F32); const bool is_q8_0 = tensor->type == GGML_TYPE_Q8_0; cl_mem src_buf = in_src_buf; cl_ulong src_offset = in_src_offset; cl_ulong src_nb1 = in_src_nb1; cl_ulong src_nb2 = in_src_nb2; cl_ulong src_nb3 = in_src_nb3; ggml_cl_flash_attn_resolve_src(tensor, src_buf, src_offset, src_nb1, src_nb2, src_nb3); if (tensor->nb[0] != (cl_ulong) ggml_type_size(tensor->type)) { return false; } const size_t n_blocks = (size_t) ggml_nelements(tensor) / 32; // block size is 32 const size_t elem_size = ggml_type_size(target_type); const size_t out_bytes = n_blocks * 32 * elem_size; const cl_int nblk0_arg = (cl_int) (tensor->ne[0] / 32); const cl_int ne1_arg = (cl_int) tensor->ne[1]; const cl_int ne2_arg = (cl_int) tensor->ne[2]; const cl_int ne3_arg = (cl_int) tensor->ne[3]; cl_int err; temp.data = clCreateBuffer(backend_ctx->context, CL_MEM_READ_WRITE, out_bytes, NULL, &err); CL_CHECK(err); cl_kernel kernel; if (target_type == GGML_TYPE_F16) { kernel = is_q8_0 ? backend_ctx->kernel_dequant_q8_0_f16_view_aos : backend_ctx->kernel_dequant_q4_0_f16_view_aos; } else { kernel = is_q8_0 ? backend_ctx->kernel_dequant_q8_0_f32_view_aos : backend_ctx->kernel_dequant_q4_0_f32_view_aos; } CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &src_buf)); CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &src_offset)); CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_ulong), &src_nb1)); CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &src_nb2)); CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_ulong), &src_nb3)); CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_int), &nblk0_arg)); CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_int), &ne1_arg)); CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_int), &ne2_arg)); CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_int), &ne3_arg)); CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_mem), &temp.data)); size_t global_ws[3] = { (size_t) nblk0_arg, (size_t) ne1_arg, (size_t) ne2_arg * (size_t) ne3_arg }; CL_CHECK(clEnqueueNDRangeKernel(backend_ctx->queue, kernel, 3, NULL, global_ws, NULL, 0, NULL, NULL)); out_buf = temp.data; out_offset = 0; out_nb1 = (cl_ulong) tensor->ne[0] * elem_size; out_nb2 = out_nb1 * (cl_ulong) tensor->ne[1]; out_nb3 = out_nb2 * (cl_ulong) tensor->ne[2]; return true; } static bool ggml_cl_flash_attn_prepare_quantized_tensor( ggml_backend_opencl_context * backend_ctx, const ggml_tensor * tensor, ggml_type target_type, ggml_cl_flash_attn_temp_buffer & temp, cl_mem & data_device, cl_ulong & offset, cl_ulong & nb1, cl_ulong & nb2, cl_ulong & nb3 ) { if (!ggml_is_quantized(tensor->type)) { return false; } // Caller-supplied AoS buffer wins over tensor->extra when present. cl_mem src_buffer = data_device; cl_ulong src_offset = offset; cl_ulong src_nb1 = nb1; cl_ulong src_nb2 = nb2; cl_ulong src_nb3 = nb3; ggml_cl_flash_attn_resolve_src(tensor, src_buffer, src_offset, src_nb1, src_nb2, src_nb3); const int64_t n = ggml_nelements(tensor); const size_t row_bytes = (size_t) (tensor->ne[0] / ggml_blck_size(tensor->type)) * ggml_type_size(tensor->type); // tight-packed byte count (ggml_nbytes includes stride gaps). const size_t total_bytes = (size_t) (n / ggml_blck_size(tensor->type)) * ggml_type_size(tensor->type); std::vector host_quant(total_bytes); sync_with_other_backends(backend_ctx); ggml_cl_flash_attn_read_tensor_host(backend_ctx, tensor, src_buffer, src_offset, src_nb1, src_nb2, src_nb3, row_bytes, host_quant.data(), total_bytes); std::vector host_f32(n); ggml_get_type_traits(tensor->type)->to_float(host_quant.data(), host_f32.data(), n); const size_t bytes_per_elem = ggml_type_size(target_type); const size_t buffer_size = (size_t) n * bytes_per_elem; std::vector host_linear(buffer_size); if (target_type == GGML_TYPE_F32) { memcpy(host_linear.data(), host_f32.data(), buffer_size); } else { GGML_ASSERT(target_type == GGML_TYPE_F16); ggml_fp32_to_fp16_row(host_f32.data(), (ggml_fp16_t *) host_linear.data(), n); } cl_int err; temp.data = clCreateBuffer(backend_ctx->context, CL_MEM_READ_WRITE, buffer_size, NULL, &err); CL_CHECK(err); CL_CHECK(clEnqueueWriteBuffer(backend_ctx->queue, temp.data, CL_TRUE, 0, buffer_size, host_linear.data(), 0, NULL, NULL)); data_device = temp.data; offset = 0; nb1 = (cl_ulong) (tensor->ne[0] * bytes_per_elem); nb2 = (cl_ulong) (tensor->ne[1] * nb1); nb3 = (cl_ulong) (tensor->ne[2] * nb2); static bool warned = false; if (!warned) { GGML_LOG_WARN("ggml_opencl: OpenCL flash attention dequantizes GPU-resident quantized KV cache into temporary linear buffers; performance may be poor\n"); warned = true; } return true; } // Host-side F16 -> F32 for the asymmetric-KV F32 fallback path. static bool ggml_cl_flash_attn_convert_f16_to_f32( ggml_backend_opencl_context * backend_ctx, const ggml_tensor * tensor, ggml_cl_flash_attn_temp_buffer & temp, cl_mem & data_device, cl_ulong & offset, cl_ulong & nb1, cl_ulong & nb2, cl_ulong & nb3 ) { if (tensor->type != GGML_TYPE_F16) { return false; } cl_mem src_buffer = data_device; cl_ulong src_offset = offset; cl_ulong src_nb1 = nb1; cl_ulong src_nb2 = nb2; cl_ulong src_nb3 = nb3; ggml_cl_flash_attn_resolve_src(tensor, src_buffer, src_offset, src_nb1, src_nb2, src_nb3); const int64_t n = ggml_nelements(tensor); const size_t row_bytes = (size_t) tensor->ne[0] * sizeof(ggml_fp16_t); const size_t total_bytes = (size_t) n * sizeof(ggml_fp16_t); std::vector host_f16(total_bytes); sync_with_other_backends(backend_ctx); ggml_cl_flash_attn_read_tensor_host(backend_ctx, tensor, src_buffer, src_offset, src_nb1, src_nb2, src_nb3, row_bytes, host_f16.data(), total_bytes); std::vector host_f32(n); ggml_fp16_to_fp32_row((const ggml_fp16_t *) host_f16.data(), host_f32.data(), n); const size_t f32_bytes = (size_t) n * sizeof(float); cl_int err; temp.data = clCreateBuffer(backend_ctx->context, CL_MEM_READ_WRITE, f32_bytes, NULL, &err); CL_CHECK(err); CL_CHECK(clEnqueueWriteBuffer(backend_ctx->queue, temp.data, CL_TRUE, 0, f32_bytes, host_f32.data(), 0, NULL, NULL)); data_device = temp.data; offset = 0; nb1 = (cl_ulong) (tensor->ne[0] * sizeof(float)); nb2 = (cl_ulong) (tensor->ne[1] * nb1); nb3 = (cl_ulong) (tensor->ne[2] * nb2); static bool warned = false; if (!warned) { GGML_LOG_WARN("ggml_opencl: OpenCL flash attention asymmetric KV converts an F16 cache to F32 host-side; performance may be poor\n"); warned = true; } return true; } // Flash-Decoding (K-split) dispatch thresholds. FD fires for non-causal // attention with n_kv >= FD_MIN_N_KV and d_head <= FD_MAX_DK; the KV range is // split into ~n_kv/FD_KV_PER_SPLIT partials, clamped to [FD_MIN_SPLITS, // FD_MAX_SPLITS]. Multi-query FD is restricted to small heads // (d_head <= FD_MAX_DK_MULTI) and capped at FD_MAX_N_Q_MULTI queries. static constexpr int FD_MIN_N_KV = 2048; static constexpr int FD_KV_PER_SPLIT = 2048; // f16 KV decode wants more splits than the 2048 default; quantized KV keeps 2048. static constexpr int FD_KV_PER_SPLIT_F16 = 512; static constexpr int FD_MIN_SPLITS = 2; static constexpr int FD_MAX_SPLITS = 16; static constexpr int FD_MAX_DK = 128; static constexpr int FD_MAX_DK_MULTI = 64; static constexpr int FD_MAX_N_Q_MULTI = 8; // MQ FD split-groups have few subgroups (MQ_NSG_SPLIT), so use a smaller // kv_per_split to keep the softmax recurrence short; non-MQ keeps FD_KV_PER_SPLIT. static constexpr int FD_MQ_KV_PER_SPLIT = 256; static constexpr int FD_MQ_MAX_SPLITS = 128; #ifdef GGML_OPENCL_USE_ADRENO_KERNELS struct ggml_cl_adreno_xmem_attn_schedule { int qk_lws0 = 256; int qk_lws2 = 1; int softmax_reduce_lws0 = 256; int softmax_apply_lws0 = 64; int softmax_apply_lws2 = 4; int pv_lws0 = 64; int pv_lws2 = 4; }; static inline size_t ggml_cl_round_up(size_t x, size_t a) { return ((x + a - 1) / a) * a; } static inline int ggml_cl_round_up_div(int x, int y) { return (x + y - 1) / y; } static inline void ggml_cl_set_arg_int4(cl_kernel kernel, cl_uint index, int x, int y, int z, int w) { struct { int x, y, z, w; } value { x, y, z, w }; CL_CHECK(clSetKernelArg(kernel, index, sizeof(value), &value)); } static cl_mem ggml_cl_make_image2d_half4(cl_context context, cl_mem_flags flags, size_t width, size_t height) { cl_int err = CL_SUCCESS; cl_image_format format = { CL_RGBA, CL_HALF_FLOAT }; cl_image_desc desc = {}; desc.image_type = CL_MEM_OBJECT_IMAGE2D; desc.image_width = width; desc.image_height = height; cl_mem image = clCreateImage(context, flags, &format, &desc, nullptr, &err); CL_CHECK(err); return image; } static cl_mem ggml_cl_make_image1d_buffer_half4(cl_context context, cl_mem_flags flags, size_t width, cl_mem backing_buffer) { cl_int err = CL_SUCCESS; cl_image_format format = { CL_RGBA, CL_HALF_FLOAT }; cl_image_desc desc = {}; desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; desc.image_width = width; desc.buffer = backing_buffer; cl_mem image = clCreateImage(context, flags, &format, &desc, nullptr, &err); CL_CHECK(err); return image; } static void ggml_cl_release_mem(cl_mem & mem) { if (mem != nullptr) { CL_CHECK(clReleaseMemObject(mem)); mem = nullptr; } } static void ggml_cl_adreno_xmem_attn_release_scratch(ggml_backend_opencl_context * backend_ctx) { auto & s = backend_ctx->adreno_xmem_attn.scratch; ggml_cl_release_mem(s.q_img); ggml_cl_release_mem(s.k_img); ggml_cl_release_mem(s.v_img); ggml_cl_release_mem(s.out_img); ggml_cl_release_mem(s.k_transpose_img1d); ggml_cl_release_mem(s.k_transpose_buf); ggml_cl_release_mem(s.k_packed_buf); ggml_cl_release_mem(s.v_packed_buf); ggml_cl_release_mem(s.score_img1d); ggml_cl_release_mem(s.prob_img1d); ggml_cl_release_mem(s.score_buf); ggml_cl_release_mem(s.prob_buf); ggml_cl_release_mem(s.softmax_stats_img2d); ggml_cl_release_mem(s.xmem_qk); ggml_cl_release_mem(s.xmem_pv); s = {}; } static ggml_cl_adreno_xmem_attn_schedule ggml_cl_adreno_xmem_attn_select_schedule( const ggml_backend_opencl_context * backend_ctx, int n_q, int n_kv, int heads_total, int q_width, int gqa_ratio) { const bool big_h = heads_total >= 8; ggml_cl_adreno_xmem_attn_schedule sched; if (gqa_ratio == 1) { if (n_q >= 512) { sched.qk_lws0 = 512; } else if (n_q >= 256) { sched.qk_lws0 = 128; } else { sched.qk_lws0 = 64; } sched.qk_lws2 = (big_h && n_q >= 512) ? 2 : 1; } else { if (q_width >= 2048) { sched.qk_lws0 = 512; } else if (q_width >= 256) { sched.qk_lws0 = 128; } else { sched.qk_lws0 = 64; } sched.qk_lws2 = MIN(8, (int) backend_ctx->max_workgroup_size / sched.qk_lws0); } if (n_kv >= 2048) { sched.softmax_reduce_lws0 = 1024; } else if (n_kv >= 512) { sched.softmax_reduce_lws0 = big_h ? 256 : 512; } else { sched.softmax_reduce_lws0 = 256; } if (n_kv < 256) { sched.softmax_apply_lws0 = 64; } else { sched.softmax_apply_lws0 = big_h ? 128 : 64; } sched.softmax_apply_lws2 = n_kv >= 512 ? 8 : 4; if (n_q < 256) { sched.pv_lws0 = 64; } else { sched.pv_lws0 = big_h ? 128 : 64; } sched.pv_lws2 = big_h ? 8 : (n_q <= 256 ? 8 : 4); const int max_wg = (int) backend_ctx->max_workgroup_size; auto fix = [&](int & l0, int & l2) { while (l0 * l2 > max_wg) { if (l2 > 1) { l2 /= 2; } else if (l0 > 32) { l0 /= 2; } else { break; } } }; fix(sched.qk_lws0, sched.qk_lws2); fix(sched.softmax_apply_lws0, sched.softmax_apply_lws2); fix(sched.pv_lws0, sched.pv_lws2); while (sched.softmax_reduce_lws0 > max_wg) { sched.softmax_reduce_lws0 /= 2; } return sched; } static bool ggml_cl_adreno_xmem_attn_prepare( ggml_backend_opencl_context * backend_ctx, int n_q, int n_kv, int d_head_q, int d_head_v, int n_head, int n_head_kv, int n_batch) { auto & s = backend_ctx->adreno_xmem_attn.scratch; const int gqa_ratio = n_head / n_head_kv; const int q_width = n_q * gqa_ratio; const int kv_heads_total = n_head_kv * n_batch; const int n_kv_padded = (int) ggml_cl_round_up((size_t) n_kv, 32); if (s.q_img != nullptr && s.n_q == n_q && s.n_kv == n_kv && s.n_kv_padded == n_kv_padded && s.d_head_q == d_head_q && s.d_head_v == d_head_v && s.q_width == q_width && s.kv_heads_total == kv_heads_total) { return true; } ggml_cl_adreno_xmem_attn_release_scratch(backend_ctx); const int qpack = d_head_q / 4; const int vpack = d_head_v / 4; const int npack = n_kv_padded / 4; const size_t q_img_h = (size_t) kv_heads_total * qpack; const size_t v_img_h = (size_t) kv_heads_total * vpack; s.q_img = ggml_cl_make_image2d_half4(backend_ctx->context, CL_MEM_READ_WRITE, (size_t) q_width, q_img_h); s.k_img = ggml_cl_make_image2d_half4(backend_ctx->context, CL_MEM_READ_WRITE, (size_t) n_kv_padded, q_img_h); s.v_img = ggml_cl_make_image2d_half4(backend_ctx->context, CL_MEM_READ_WRITE, (size_t) n_kv_padded, v_img_h); s.out_img = ggml_cl_make_image2d_half4(backend_ctx->context, CL_MEM_READ_WRITE, (size_t) q_width, v_img_h); const size_t k_transpose_half4_elems = (size_t) npack * kv_heads_total * d_head_q; s.k_transpose_buf = clCreateBuffer(backend_ctx->context, CL_MEM_READ_WRITE, k_transpose_half4_elems * sizeof(uint16_t) * 4, nullptr, nullptr); GGML_ASSERT(s.k_transpose_buf != nullptr); s.k_transpose_img1d = ggml_cl_make_image1d_buffer_half4(backend_ctx->context, CL_MEM_READ_ONLY, k_transpose_half4_elems, s.k_transpose_buf); const size_t k_groups16 = (size_t) ggml_cl_round_up_div(kv_heads_total * d_head_q, 16); const size_t v_groups16 = (size_t) ggml_cl_round_up_div(kv_heads_total * d_head_v, 16); const size_t k_packed_half4_elems = (size_t) n_kv_padded * k_groups16 * 4; const size_t v_packed_half4_elems = (size_t) n_kv_padded * v_groups16 * 4; s.k_packed_buf = clCreateBuffer(backend_ctx->context, CL_MEM_READ_WRITE, k_packed_half4_elems * sizeof(uint16_t) * 4, nullptr, nullptr); s.v_packed_buf = clCreateBuffer(backend_ctx->context, CL_MEM_READ_WRITE, v_packed_half4_elems * sizeof(uint16_t) * 4, nullptr, nullptr); GGML_ASSERT(s.k_packed_buf != nullptr && s.v_packed_buf != nullptr); const size_t score_half4_elems = (size_t) npack * kv_heads_total * q_width; const size_t score_bytes = score_half4_elems * sizeof(uint16_t) * 4; s.score_buf = clCreateBuffer(backend_ctx->context, CL_MEM_READ_WRITE, score_bytes, nullptr, nullptr); s.prob_buf = clCreateBuffer(backend_ctx->context, CL_MEM_READ_WRITE, score_bytes, nullptr, nullptr); GGML_ASSERT(s.score_buf != nullptr && s.prob_buf != nullptr); s.score_img1d = ggml_cl_make_image1d_buffer_half4(backend_ctx->context, CL_MEM_READ_ONLY, score_half4_elems, s.score_buf); s.prob_img1d = ggml_cl_make_image1d_buffer_half4(backend_ctx->context, CL_MEM_READ_ONLY, score_half4_elems, s.prob_buf); s.softmax_stats_img2d = ggml_cl_make_image2d_half4(backend_ctx->context, CL_MEM_READ_WRITE, (size_t) q_width, (size_t) kv_heads_total); s.xmem_qk = clCreateBuffer(backend_ctx->context, CL_MEM_READ_ONLY, 6144, nullptr, nullptr); s.xmem_pv = clCreateBuffer(backend_ctx->context, CL_MEM_READ_ONLY, 6144, nullptr, nullptr); GGML_ASSERT(s.softmax_stats_img2d != nullptr && s.xmem_qk != nullptr && s.xmem_pv != nullptr); s.n_q = n_q; s.n_kv = n_kv; s.n_kv_padded = n_kv_padded; s.d_head_q = d_head_q; s.d_head_v = d_head_v; s.q_width = q_width; s.kv_heads_total = kv_heads_total; return true; } static bool ggml_cl_adreno_xmem_attn_can_use( const ggml_backend_opencl_context * backend_ctx, const ggml_tensor * q, const ggml_tensor * k, const ggml_tensor * dst) { static const char * xmem_sdpa_env = getenv("GGML_OPENCL_XMEM_SDPA"); if (xmem_sdpa_env == nullptr || xmem_sdpa_env[0] == '0') { return false; } const ggml_tensor * v = dst->src[2]; const ggml_tensor * mask = dst->src[3]; const ggml_tensor * sinks = dst->src[4]; if (!backend_ctx->adreno_xmem_attn.compiled || backend_ctx->gpu_family != GPU_FAMILY::ADRENO) { return false; } if (q->type != GGML_TYPE_F32 || dst->type != GGML_TYPE_F32 || (k->type != GGML_TYPE_F16 && k->type != GGML_TYPE_F32) || (v->type != GGML_TYPE_F16 && v->type != GGML_TYPE_F32)) { return false; } if (sinks != nullptr) { return false; } if (q->nb[0] != ggml_type_size(q->type) || k->nb[0] != ggml_type_size(k->type) || v->nb[0] != ggml_type_size(v->type) || dst->nb[0] != ggml_type_size(dst->type)) { return false; } if (mask != nullptr && (mask->type != GGML_TYPE_F16 || mask->nb[0] != sizeof(ggml_fp16_t))) { return false; } const int n_q = q->ne[1]; const int n_kv = k->ne[1]; const int d_head_q = q->ne[0]; const int d_head_v = v->ne[0]; const int n_head = q->ne[2]; const int n_head_kv = k->ne[2]; const int n_batch = q->ne[3]; if (n_q <= 1 || n_kv <= 0 || n_kv > 8192) { return false; } if (d_head_q != k->ne[0] || d_head_v != v->ne[0] || k->ne[1] != v->ne[1] || k->ne[3] != v->ne[3]) { return false; } if (q->ne[3] != k->ne[3]) { return false; } if (n_head_kv <= 0 || n_head % n_head_kv != 0 || k->ne[2] != v->ne[2]) { return false; } if (dst->ne[0] != d_head_v || dst->ne[1] != n_head || dst->ne[2] != n_q || dst->ne[3] != n_batch) { return false; } if ((d_head_q % 8) != 0 || (d_head_v % 32) != 0) { return false; } if (mask != nullptr && (mask->ne[0] < n_kv || mask->ne[1] < n_q || mask->ne[2] <= 0 || mask->ne[3] <= 0)) { return false; } float params[3]; memcpy(params, dst->op_params, sizeof(params)); if (params[1] != 0.0f || params[2] != 0.0f) { return false; } const int gqa_ratio = n_head / n_head_kv; const int q_width = n_q * gqa_ratio; const int kv_heads_total = n_head_kv * n_batch; const int n_kv_padded = (int) ggml_cl_round_up((size_t) n_kv, 32); const int qpack = d_head_q / 4; const int vpack = d_head_v / 4; const int npack = n_kv_padded / 4; if ((size_t) q_width > backend_ctx->image2d_max_width || (size_t) n_kv_padded > backend_ctx->image2d_max_width) { return false; } if ((size_t) kv_heads_total * (size_t) qpack > backend_ctx->image2d_max_height || (size_t) kv_heads_total * (size_t) vpack > backend_ctx->image2d_max_height) { return false; } if ((size_t) npack * (size_t) kv_heads_total * (size_t) d_head_q > backend_ctx->image_max_buffer_size || (size_t) npack * (size_t) kv_heads_total * (size_t) q_width > backend_ctx->image_max_buffer_size) { return false; } return true; } static void ggml_cl_adreno_xmem_attn_run( ggml_backend_t backend, const ggml_tensor * q, const ggml_tensor * k, ggml_tensor * dst) { ggml_backend_opencl_context * backend_ctx = (ggml_backend_opencl_context *) backend->context; auto & xstate = backend_ctx->adreno_xmem_attn; auto & s = xstate.scratch; if (!xstate.logged) { GGML_LOG_INFO("ggml_opencl: using Adreno xmem attention path\n"); xstate.logged = true; } const ggml_tensor * v = dst->src[2]; const ggml_tensor * mask = dst->src[3]; ggml_tensor_extra_cl * extra_q = (ggml_tensor_extra_cl *) q->extra; ggml_tensor_extra_cl * extra_k = (ggml_tensor_extra_cl *) k->extra; ggml_tensor_extra_cl * extra_v = (ggml_tensor_extra_cl *) v->extra; ggml_tensor_extra_cl * extra_o = (ggml_tensor_extra_cl *) dst->extra; ggml_tensor_extra_cl * extra_mask = mask ? (ggml_tensor_extra_cl *) mask->extra : nullptr; const cl_ulong offset_q = extra_q->offset + q->view_offs; const cl_ulong offset_k = extra_k->offset + k->view_offs; const cl_ulong offset_v = extra_v->offset + v->view_offs; const cl_ulong offset_o = extra_o->offset + dst->view_offs; const cl_ulong offset_mask = extra_mask ? extra_mask->offset + mask->view_offs : 0; const int n_q = q->ne[1]; const int n_kv = k->ne[1]; const int d_head_q = q->ne[0]; const int d_head_v = v->ne[0]; const int n_head = q->ne[2]; const int n_head_kv = k->ne[2]; const int n_batch = q->ne[3]; const int heads_total = n_head * n_batch; const int gqa_ratio = n_head / n_head_kv; const int q_width = n_q * gqa_ratio; const int kv_heads_total = n_head_kv * n_batch; const int n_kv_padded = (int) ggml_cl_round_up((size_t) n_kv, 32); const int qpack = d_head_q / 4; const int opack = d_head_v / 4; const int npack = n_kv_padded / 4; const float scale = ((const float *) dst->op_params)[0]; GGML_ASSERT(ggml_cl_adreno_xmem_attn_prepare( backend_ctx, n_q, n_kv, d_head_q, d_head_v, n_head, n_head_kv, n_batch)); const ggml_cl_adreno_xmem_attn_schedule sched = ggml_cl_adreno_xmem_attn_select_schedule( backend_ctx, n_q, n_kv_padded, heads_total, q_width, gqa_ratio); { size_t gws[3] = {ggml_cl_round_up((size_t) n_q, 8), (size_t) heads_total, (size_t) qpack}; size_t lws[3] = {8, 1, (size_t) ((qpack <= 32) ? qpack : 1)}; cl_kernel kernel = xstate.kernel_q_f32_to_img_scaled; CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra_q->data_device)); CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset_q)); CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &s.q_img)); CL_CHECK(clSetKernelArg(kernel, 3, sizeof(float), &scale)); CL_CHECK(clSetKernelArg(kernel, 4, sizeof(int), &d_head_q)); CL_CHECK(clSetKernelArg(kernel, 5, sizeof(int), &n_q)); CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &n_head)); CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &n_head_kv)); CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &n_batch)); CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_ulong), &q->nb[1])); CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_ulong), &q->nb[2])); CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_ulong), &q->nb[3])); backend_ctx->enqueue_ndrange_kernel(kernel, 3, gws, lws, dst); } { size_t gws[3] = {(size_t) n_kv_padded, (size_t) kv_heads_total, (size_t) qpack}; size_t lws[3] = {8, 1, (size_t) ((qpack <= 32) ? qpack : 1)}; cl_kernel kernel = k->type == GGML_TYPE_F16 ? xstate.kernel_kv_f16_to_img_gqa : xstate.kernel_kv_f32_to_img_gqa; CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra_k->data_device)); CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset_k)); CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &s.k_img)); CL_CHECK(clSetKernelArg(kernel, 3, sizeof(int), &d_head_q)); CL_CHECK(clSetKernelArg(kernel, 4, sizeof(int), &n_kv)); CL_CHECK(clSetKernelArg(kernel, 5, sizeof(int), &n_kv_padded)); CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &n_head_kv)); CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &n_batch)); CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_ulong), &k->nb[1])); CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_ulong), &k->nb[2])); CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_ulong), &k->nb[3])); backend_ctx->enqueue_ndrange_kernel(kernel, 3, gws, lws, dst); } { size_t gws[3] = {(size_t) n_kv_padded, (size_t) kv_heads_total, (size_t) opack}; size_t lws[3] = {8, 1, (size_t) ((opack <= 32) ? opack : 1)}; cl_kernel kernel = v->type == GGML_TYPE_F16 ? xstate.kernel_kv_f16_to_img_gqa : xstate.kernel_kv_f32_to_img_gqa; CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra_v->data_device)); CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset_v)); CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &s.v_img)); CL_CHECK(clSetKernelArg(kernel, 3, sizeof(int), &d_head_v)); CL_CHECK(clSetKernelArg(kernel, 4, sizeof(int), &n_kv)); CL_CHECK(clSetKernelArg(kernel, 5, sizeof(int), &n_kv_padded)); CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &n_head_kv)); CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &n_batch)); CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_ulong), &v->nb[1])); CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_ulong), &v->nb[2])); CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_ulong), &v->nb[3])); backend_ctx->enqueue_ndrange_kernel(kernel, 3, gws, lws, dst); } { size_t gws[3] = {(size_t) d_head_q, (size_t) kv_heads_total, (size_t) npack}; size_t lws[3] = {(size_t) MIN(64, d_head_q), (size_t) (kv_heads_total >= 2 ? 2 : 1), (size_t) MIN(8, npack)}; if (lws[0] * lws[1] * lws[2] > backend_ctx->max_workgroup_size) { lws[1] = 1; } cl_kernel kernel = xstate.kernel_k_gather; CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &s.k_transpose_buf)); CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &s.k_img)); ggml_cl_set_arg_int4(kernel, 2, n_kv_padded, kv_heads_total, npack, d_head_q); ggml_cl_set_arg_int4(kernel, 3, qpack, 0, 0, 0); backend_ctx->enqueue_ndrange_kernel(kernel, 3, gws, lws, dst); } { const size_t groups16 = (size_t) ggml_cl_round_up_div(kv_heads_total * d_head_q, 16); const size_t packed_linear = (size_t) n_kv_padded * groups16; const size_t lws0 = MIN((size_t) 1024, backend_ctx->max_workgroup_size); size_t gws[3] = {ggml_cl_round_up(packed_linear, lws0), 1, 1}; size_t lws[3] = {lws0, 1, 1}; cl_kernel kernel = xstate.kernel_pack_k; CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &s.k_packed_buf)); CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &s.k_transpose_img1d)); ggml_cl_set_arg_int4(kernel, 2, 8, (int) packed_linear, qpack, d_head_q); ggml_cl_set_arg_int4(kernel, 3, kv_heads_total, kv_heads_total, kv_heads_total, npack); ggml_cl_set_arg_int4(kernel, 4, d_head_q, 0, 0, 0); backend_ctx->enqueue_ndrange_kernel(kernel, 3, gws, lws, dst); } { size_t lws[3] = {(size_t) sched.qk_lws0, 1, (size_t) sched.qk_lws2}; const int slices_per_group = sched.qk_lws2 * 8; const size_t groups_z = (size_t) ggml_cl_round_up_div(npack, slices_per_group); const size_t groups_x = (size_t) ggml_cl_round_up_div(q_width, sched.qk_lws0); size_t gws[3] = { lws[0] * groups_z, groups_x, (size_t) kv_heads_total * lws[2], }; cl_kernel kernel = xstate.kernel_qk_gemm; CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &s.score_buf)); CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &s.k_packed_buf)); CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &s.xmem_qk)); CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &s.q_img)); ggml_cl_set_arg_int4(kernel, 4, kv_heads_total, npack, q_width, 32); ggml_cl_set_arg_int4(kernel, 5, qpack, 0, 0, kv_heads_total); ggml_cl_set_arg_int4(kernel, 6, qpack, 1, 1, 0); backend_ctx->enqueue_ndrange_kernel(kernel, 3, gws, lws, dst); } cl_mem softmax_input_img = s.score_img1d; cl_mem softmax_output_buf = s.prob_buf; cl_mem pv_prob_img = s.prob_img1d; if (mask != nullptr) { const cl_ulong mask_nb1 = mask->nb[1]; const cl_ulong mask_nb2 = mask->nb[2]; const cl_ulong mask_nb3 = mask->nb[3]; const int mask_ne2 = mask->ne[2]; const int mask_ne3 = mask->ne[3]; size_t lws[3] = {(size_t) sched.softmax_apply_lws0, 1, (size_t) sched.softmax_apply_lws2}; size_t gws[3] = { ggml_cl_round_up((size_t) q_width, lws[0]), (size_t) kv_heads_total, ggml_cl_round_up((size_t) npack, lws[2]), }; cl_kernel kernel = xstate.kernel_mask_scores; CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &s.prob_buf)); CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &s.score_img1d)); CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra_mask->data_device)); CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset_mask)); CL_CHECK(clSetKernelArg(kernel, 4, sizeof(int), &q_width)); CL_CHECK(clSetKernelArg(kernel, 5, sizeof(int), &n_q)); CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &n_kv)); CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &n_kv_padded)); CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &kv_heads_total)); CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &n_head)); CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &n_head_kv)); CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_ulong), &mask_nb1)); CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_ulong), &mask_nb2)); CL_CHECK(clSetKernelArg(kernel, 13, sizeof(cl_ulong), &mask_nb3)); CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &mask_ne2)); CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &mask_ne3)); backend_ctx->enqueue_ndrange_kernel(kernel, 3, gws, lws, dst); softmax_input_img = s.prob_img1d; softmax_output_buf = s.score_buf; pv_prob_img = s.score_img1d; } { size_t lws[3] = {(size_t) sched.softmax_reduce_lws0, 1, 1}; size_t gws[3] = {ggml_cl_round_up((size_t) q_width, lws[0]), (size_t) kv_heads_total, 1}; cl_kernel kernel = xstate.kernel_softmax_reduce_basic; CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &softmax_input_img)); CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &s.softmax_stats_img2d)); ggml_cl_set_arg_int4(kernel, 2, kv_heads_total, 1, q_width, n_kv); ggml_cl_set_arg_int4(kernel, 3, kv_heads_total, q_width, 0, 0); backend_ctx->enqueue_ndrange_kernel(kernel, 3, gws, lws, dst); } { size_t lws[3] = {(size_t) sched.softmax_apply_lws0, 1, (size_t) sched.softmax_apply_lws2}; size_t gws[3] = { ggml_cl_round_up((size_t) q_width, lws[0]), (size_t) kv_heads_total, ggml_cl_round_up((size_t) npack, lws[2]), }; cl_kernel kernel = xstate.kernel_softmax_apply_basic; CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &softmax_output_buf)); CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &softmax_input_img)); CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &s.softmax_stats_img2d)); ggml_cl_set_arg_int4(kernel, 3, kv_heads_total, npack, q_width, 1); ggml_cl_set_arg_int4(kernel, 4, kv_heads_total, q_width, n_kv, 0); backend_ctx->enqueue_ndrange_kernel(kernel, 3, gws, lws, dst); } { const size_t groups16 = (size_t) ggml_cl_round_up_div(kv_heads_total * d_head_v, 16); const size_t packed_linear = (size_t) n_kv_padded * groups16; const size_t lws0 = MIN((size_t) 1024, backend_ctx->max_workgroup_size); size_t gws[3] = {ggml_cl_round_up(packed_linear, lws0), 1, 1}; size_t lws[3] = {lws0, 1, 1}; cl_kernel kernel = xstate.kernel_pack_v; CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &s.v_packed_buf)); CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &s.v_img)); ggml_cl_set_arg_int4(kernel, 2, 8, (int) packed_linear, npack, n_kv_padded); ggml_cl_set_arg_int4(kernel, 3, kv_heads_total, kv_heads_total, opack, 0); backend_ctx->enqueue_ndrange_kernel(kernel, 3, gws, lws, dst); } { size_t lws[3] = {(size_t) sched.pv_lws0, 1, (size_t) sched.pv_lws2}; const int blocks = ggml_cl_round_up_div(opack, 8); const size_t groups_z = (size_t) ggml_cl_round_up_div(blocks, sched.pv_lws2); const size_t groups_x = (size_t) ggml_cl_round_up_div(q_width, sched.pv_lws0); size_t gws[3] = { lws[0] * groups_z, groups_x, (size_t) kv_heads_total * lws[2], }; cl_kernel kernel = xstate.kernel_pv_gemm; CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &s.v_packed_buf)); CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &s.xmem_pv)); CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &pv_prob_img)); CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &s.out_img)); ggml_cl_set_arg_int4(kernel, 4, kv_heads_total, opack, q_width, 32); ggml_cl_set_arg_int4(kernel, 5, npack, 0, 0, kv_heads_total); ggml_cl_set_arg_int4(kernel, 6, kv_heads_total * q_width, npack, q_width, 1); ggml_cl_set_arg_int4(kernel, 7, 1, 0, 0, 0); backend_ctx->enqueue_ndrange_kernel(kernel, 3, gws, lws, dst); } { size_t gws[3] = {ggml_cl_round_up((size_t) n_q, 8), (size_t) heads_total, (size_t) opack}; size_t lws[3] = {8, 1, (size_t) ((opack <= 32) ? opack : 1)}; cl_kernel kernel = xstate.kernel_img_to_f32; CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra_o->data_device)); CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset_o)); CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &s.out_img)); CL_CHECK(clSetKernelArg(kernel, 3, sizeof(int), &d_head_v)); CL_CHECK(clSetKernelArg(kernel, 4, sizeof(int), &n_q)); CL_CHECK(clSetKernelArg(kernel, 5, sizeof(int), &n_head)); CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &n_head_kv)); CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &n_batch)); CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_ulong), &dst->nb[1])); CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_ulong), &dst->nb[2])); CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_ulong), &dst->nb[3])); backend_ctx->enqueue_ndrange_kernel(kernel, 3, gws, lws, dst); } } #endif // GGML_OPENCL_USE_ADRENO_KERNELS #ifdef GGML_OPENCL_USE_ADRENO_KERNELS static void ggml_cl_flash_attn_prefill_bin(ggml_backend_t backend, const ggml_tensor * q, const ggml_tensor * k, ggml_tensor * dst) { const ggml_tensor * v = dst->src[2]; const ggml_tensor * mask = dst->src[3]; const ggml_tensor * sinks = dst->src[4]; GGML_ASSERT(q->extra); GGML_ASSERT(k->extra); GGML_ASSERT(v->extra); GGML_ASSERT(dst->extra); if (mask) { GGML_ASSERT(mask->extra); } if (sinks) { GGML_ASSERT(sinks->extra); } ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; cl_context context = backend_ctx->context; const int n_q = q->ne[1]; const int n_kv = k->ne[1]; const int d_head_q = q->ne[0]; const int d_head_v = v->ne[0]; const int n_head = q->ne[2]; const int n_head_kv = k->ne[2]; const int n_batch = q->ne[3]; const std::pair dk_dv = {d_head_q, d_head_v}; cl_kernel kernel = backend_ctx->fa.kernel_flash_attn_f32_f16_bin; GGML_ASSERT(kernel != NULL); ggml_tensor_extra_cl * extra_q = (ggml_tensor_extra_cl *)q->extra; ggml_tensor_extra_cl * extra_k = (ggml_tensor_extra_cl *)k->extra; ggml_tensor_extra_cl * extra_v = (ggml_tensor_extra_cl *)v->extra; ggml_tensor_extra_cl * extra_o = (ggml_tensor_extra_cl *)dst->extra; ggml_tensor_extra_cl * extra_mask = mask ? (ggml_tensor_extra_cl *)mask->extra : NULL; ggml_tensor_extra_cl * extra_sinks = sinks ? (ggml_tensor_extra_cl *)sinks->extra : NULL; cl_ulong offset_q = extra_q->offset + q->view_offs; cl_ulong offset_o = extra_o->offset + dst->view_offs; cl_mem mask_buffer = extra_mask ? extra_mask->data_device : NULL; cl_ulong offset_mask = extra_mask ? extra_mask->offset + mask->view_offs : 0; cl_mem sinks_buffer = extra_sinks ? extra_sinks->data_device : NULL; cl_ulong offset_sinks = extra_sinks ? extra_sinks->offset + sinks->view_offs : 0; const cl_ulong q_nb1 = q->nb[1]; const cl_ulong q_nb2 = q->nb[2]; const cl_ulong q_nb3 = q->nb[3]; cl_mem k_data_device = extra_k->data_device; cl_ulong offset_k = extra_k->offset + k->view_offs; cl_ulong k_nb1 = k->nb[1]; cl_ulong k_nb2 = k->nb[2]; cl_ulong k_nb3 = k->nb[3]; cl_mem v_data_device = extra_v->data_device; cl_ulong offset_v = extra_v->offset + v->view_offs; cl_ulong v_nb1 = v->nb[1]; cl_ulong v_nb2 = v->nb[2]; cl_ulong v_nb3 = v->nb[3]; const cl_ulong o_nb1 = dst->nb[1]; const cl_ulong o_nb2 = dst->nb[2]; const cl_ulong o_nb3 = dst->nb[3]; const cl_ulong mask_nb1 = mask ? mask->nb[1] : 0; const cl_ulong mask_nb2 = mask ? mask->nb[2] : 0; const cl_ulong mask_nb3 = mask ? mask->nb[3] : 0; const int mask_ne2 = mask ? mask->ne[2] : 0; const int mask_ne3 = mask ? mask->ne[3] : 0; float * params = (float *)dst->op_params; float scale = params[0]; float max_bias = params[1]; float logit_softcap = params[2]; const int is_causal = (mask == NULL && n_q > 1 && n_q == n_kv); // redundant n_q > 1 check ? const int n_head_log2_val = n_head > 0 ? 1u << (int)floorf(log2f((float)n_head)) : 0; const float n_head_log2_f = n_head_log2_val > 0 ? (float)n_head_log2_val : 1.0f; const float m0 = powf(2.0f, -(max_bias) / n_head_log2_f); const float m1 = powf(2.0f, -(max_bias / 2.0f) / n_head_log2_f); const bool is_q8_0 = q->type == GGML_TYPE_F32 && k->type == GGML_TYPE_Q8_0 && v->type == GGML_TYPE_Q8_0; ggml_cl_flash_attn_temp_buffer temp_k; ggml_cl_flash_attn_temp_buffer temp_v; ggml_cl_flash_attn_temp_buffer temp_k_aos; ggml_cl_flash_attn_temp_buffer temp_v_aos; if (is_q8_0) { ggml_cl_flash_attn_reconstruct_aos( backend_ctx, k, temp_k_aos, k_data_device, offset_k, k_nb1, k_nb2, k_nb3); ggml_cl_flash_attn_reconstruct_aos( backend_ctx, v, temp_v_aos, v_data_device, offset_v, v_nb1, v_nb2, v_nb3); bool k_done = ggml_cl_flash_attn_dequant_kv_gpu( backend_ctx, k, GGML_TYPE_F16, k_data_device, offset_k, k_nb1, k_nb2, k_nb3, temp_k, k_data_device, offset_k, k_nb1, k_nb2, k_nb3); bool v_done = ggml_cl_flash_attn_dequant_kv_gpu( backend_ctx, v, GGML_TYPE_F16, v_data_device, offset_v, v_nb1, v_nb2, v_nb3, temp_v, v_data_device, offset_v, v_nb1, v_nb2, v_nb3); GGML_ASSERT(k_done && v_done); } // Allocate input/output memory buffers cl_mem mem_matrixQ; cl_mem mem_matrixK; cl_mem mem_matrixV; cl_mem mem_matrixO; cl_buffer_region region; cl_int err; region.origin = offset_q; region.size = ggml_nbytes(q); mem_matrixQ = clCreateSubBuffer(extra_q->data_device, CL_MEM_READ_WRITE, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err); CL_CHECK(err); region.origin = offset_k; region.size = is_q8_0 ? (size_t) k_nb3 * (size_t) k->ne[3] : ggml_nbytes(k); mem_matrixK = clCreateSubBuffer(k_data_device, CL_MEM_READ_WRITE, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err); CL_CHECK(err); region.origin = offset_v; region.size = is_q8_0 ? (size_t) v_nb3 * (size_t) v->ne[3] : ggml_nbytes(v); mem_matrixV = clCreateSubBuffer(v_data_device, CL_MEM_READ_WRITE, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err); CL_CHECK(err); region.origin = offset_o; region.size = ggml_nbytes(dst); mem_matrixO = clCreateSubBuffer(extra_o->data_device, CL_MEM_READ_WRITE, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err); CL_CHECK(err); cl_image_format img_fmt_1d = { CL_RGBA, CL_FLOAT}; cl_image_desc img_desc_1d; // use image 1d buffer used as fallback when on mask is applied cl_mem mem_tex_mask_fallback_1dbuf; img_fmt_1d = { CL_RGBA, CL_HALF_FLOAT}; memset(&img_desc_1d, 0, sizeof(img_desc_1d)); img_desc_1d.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; img_desc_1d.image_width = 1; img_desc_1d.buffer = mem_matrixK; mem_tex_mask_fallback_1dbuf = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt_1d, &img_desc_1d, NULL, &err); CL_CHECK(err); cl_mem mem_tex_matrixO_1dbuf; img_fmt_1d = { CL_RGBA, CL_FLOAT}; memset(&img_desc_1d, 0, sizeof(img_desc_1d)); img_desc_1d.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; img_desc_1d.image_width = ggml_nbytes(dst) / 4 / 4; img_desc_1d.buffer = mem_matrixO; mem_tex_matrixO_1dbuf = clCreateImage(context, CL_MEM_WRITE_ONLY, &img_fmt_1d, &img_desc_1d, NULL, &err); CL_CHECK(err); // The bin kernel requires 2d (or 3d) buffers packed for data loading/multiplication. // These repack kernels launch across all buffers to ensure compatibility cl_mem mem_tex_matrixMask_1dbuf = NULL; cl_mem mem_matrixMask = NULL; cl_mem mem_matrixMask_padded = NULL; cl_ulong mask_nb1_padded = mask_nb1, mask_nb2_padded = mask_nb2, mask_nb3_padded = mask_nb3; if (extra_mask) { // allocate mem_matrixMask w/ new padded size size_t n_kv_padded = GGML_PAD(n_kv, 4); size_t mask_nb_padded = n_kv_padded * sizeof(cl_half) * mask->ne[1] * mask->ne[2] * mask->ne[3]; // apply offset and create subBuffer for mask region.origin = offset_mask; region.size = ggml_nbytes(mask); mem_matrixMask = clCreateSubBuffer(extra_mask->data_device, CL_MEM_READ_WRITE, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err); CL_CHECK(err); { // create padded mask to contain all data mem_matrixMask_padded = clCreateBuffer(context, CL_MEM_ALLOC_HOST_PTR, mask_nb_padded, NULL, &err); CL_CHECK(err); // pass extra_mask->data_device, mem_matrixMask to kernel for copying/padding mask_nb1_padded = (cl_ulong)n_kv_padded * sizeof(cl_half); mask_nb2_padded = mask_nb1_padded * (cl_ulong)mask->ne[1]; mask_nb3_padded = mask_nb2_padded * (cl_ulong)mask->ne[2]; cl_kernel repack_mask = backend_ctx->fa.kernel_repack_mask_for_wmm; CL_CHECK(clSetKernelArg(repack_mask, 0, sizeof(cl_mem), &mem_matrixMask)); CL_CHECK(clSetKernelArg(repack_mask, 1, sizeof(cl_ulong), &mask_nb1)); CL_CHECK(clSetKernelArg(repack_mask, 2, sizeof(cl_ulong), &mask_nb2)); CL_CHECK(clSetKernelArg(repack_mask, 3, sizeof(cl_ulong), &mask_nb3)); CL_CHECK(clSetKernelArg(repack_mask, 4, sizeof(int), &mask_ne2)); CL_CHECK(clSetKernelArg(repack_mask, 5, sizeof(cl_mem), &mem_matrixMask_padded)); CL_CHECK(clSetKernelArg(repack_mask, 6, sizeof(cl_ulong), &mask_nb1_padded)); CL_CHECK(clSetKernelArg(repack_mask, 7, sizeof(cl_ulong), &mask_nb2_padded)); CL_CHECK(clSetKernelArg(repack_mask, 8, sizeof(cl_ulong), &mask_nb3_padded)); size_t repack_mask_gws[3] = {(size_t)n_kv, (size_t)mask->ne[1], (size_t)mask_ne2 * (size_t)mask->ne[3]}; backend_ctx->enqueue_ndrange_kernel(repack_mask, 3, repack_mask_gws, NULL, dst); } // use image 1d buffer for matrix Mask (padded row stride) cl_image_format img_fmt_mask_1d = { CL_RGBA, CL_HALF_FLOAT}; cl_image_desc img_desc_mask_1d; memset(&img_desc_mask_1d, 0, sizeof(img_desc_mask_1d)); img_desc_mask_1d.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; img_desc_mask_1d.image_width = mask_nb_padded / 2 / 4; img_desc_mask_1d.buffer = mem_matrixMask_padded; mem_tex_matrixMask_1dbuf = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt_mask_1d, &img_desc_mask_1d, NULL, &err); CL_CHECK(err); } // WMM QK uses repacked 3D images. // Q image: rows, heads, packed depth. cl_image_format img_fmt_3d = { CL_RGBA, CL_HALF_FLOAT }; cl_image_desc img_desc_3d; memset(&img_desc_3d, 0, sizeof(img_desc_3d)); img_desc_3d.image_type = CL_MEM_OBJECT_IMAGE3D; img_desc_3d.image_width = (size_t)n_q; img_desc_3d.image_height = (size_t)n_batch * (size_t)n_head; img_desc_3d.image_depth = (size_t)d_head_q / 4; cl_mem img_q_wmm = NULL; img_q_wmm = clCreateImage(context, CL_MEM_READ_WRITE, &img_fmt_3d, &img_desc_3d, NULL, &err); CL_CHECK(err); { cl_kernel repack_q = backend_ctx->fa.kernel_repack_q_for_wmm; CL_CHECK(clSetKernelArg(repack_q, 0, sizeof(cl_mem), &mem_matrixQ)); CL_CHECK(clSetKernelArg(repack_q, 1, sizeof(cl_ulong), &q_nb1)); CL_CHECK(clSetKernelArg(repack_q, 2, sizeof(cl_ulong), &q_nb2)); CL_CHECK(clSetKernelArg(repack_q, 3, sizeof(cl_ulong), &q_nb3)); CL_CHECK(clSetKernelArg(repack_q, 4, sizeof(int), &n_head)); CL_CHECK(clSetKernelArg(repack_q, 5, sizeof(cl_mem), &img_q_wmm)); size_t repack_q_gws[3] = {(size_t)d_head_q / 4, (size_t)n_q, (size_t)n_batch * (size_t)n_head}; backend_ctx->enqueue_ndrange_kernel(repack_q, 3, repack_q_gws, NULL, dst); } // K image: columns, row groups, KV heads. const size_t n_kv_row4 = ((size_t)n_kv + 3) / 4; memset(&img_desc_3d, 0, sizeof(img_desc_3d)); img_desc_3d.image_type = CL_MEM_OBJECT_IMAGE3D; img_desc_3d.image_width = (size_t)d_head_q; img_desc_3d.image_height = n_kv_row4; img_desc_3d.image_depth = (size_t)n_batch * (size_t)n_head_kv; cl_mem img_k_wmm = NULL; img_k_wmm = clCreateImage(context, CL_MEM_READ_WRITE, &img_fmt_3d, &img_desc_3d, NULL, &err); CL_CHECK(err); { cl_kernel repack_k = backend_ctx->fa.kernel_repack_k_for_wmm; CL_CHECK(clSetKernelArg(repack_k, 0, sizeof(cl_mem), &mem_matrixK)); CL_CHECK(clSetKernelArg(repack_k, 1, sizeof(cl_ulong), &k_nb1)); CL_CHECK(clSetKernelArg(repack_k, 2, sizeof(cl_ulong), &k_nb2)); CL_CHECK(clSetKernelArg(repack_k, 3, sizeof(cl_ulong), &k_nb3)); CL_CHECK(clSetKernelArg(repack_k, 4, sizeof(int), &n_head_kv)); CL_CHECK(clSetKernelArg(repack_k, 5, sizeof(int), &n_kv)); CL_CHECK(clSetKernelArg(repack_k, 6, sizeof(cl_mem), &img_k_wmm)); size_t repack_k_gws[3] = {(size_t)d_head_q, n_kv_row4, (size_t)n_batch * (size_t)n_head_kv}; backend_ctx->enqueue_ndrange_kernel(repack_k, 3, repack_k_gws, NULL, dst); } // V image: kv-rows (contracted), packed head-dim groups, KV heads. memset(&img_desc_3d, 0, sizeof(img_desc_3d)); img_desc_3d.image_type = CL_MEM_OBJECT_IMAGE3D; img_desc_3d.image_width = (size_t)n_kv; img_desc_3d.image_height = (size_t)d_head_v / 4; img_desc_3d.image_depth = (size_t)n_batch * (size_t)n_head_kv; cl_mem img_v_wmm = NULL; img_v_wmm = clCreateImage(context, CL_MEM_READ_WRITE, &img_fmt_3d, &img_desc_3d, NULL, &err); CL_CHECK(err); { cl_kernel repack_v = backend_ctx->fa.kernel_repack_v_for_wmm; CL_CHECK(clSetKernelArg(repack_v, 0, sizeof(cl_mem), &mem_matrixV)); CL_CHECK(clSetKernelArg(repack_v, 1, sizeof(cl_ulong), &v_nb1)); CL_CHECK(clSetKernelArg(repack_v, 2, sizeof(cl_ulong), &v_nb2)); CL_CHECK(clSetKernelArg(repack_v, 3, sizeof(cl_ulong), &v_nb3)); CL_CHECK(clSetKernelArg(repack_v, 4, sizeof(int), &n_head_kv)); CL_CHECK(clSetKernelArg(repack_v, 5, sizeof(cl_mem), &img_v_wmm)); size_t repack_v_gws[3] = {(size_t)d_head_v / 4, (size_t)n_kv, (size_t)n_batch * (size_t)n_head_kv}; backend_ctx->enqueue_ndrange_kernel(repack_v, 3, repack_v_gws, NULL, dst); } cl_int enable_mask = (extra_mask) ? 1 : 0; mask_buffer = extra_mask ? mem_tex_matrixMask_1dbuf : mem_tex_mask_fallback_1dbuf; cl_mem mem_sinksBuf = NULL; cl_mem mem_tex_sinks_1dbuf = NULL; cl_int enable_sinks = (sinks_buffer != NULL) ? 1 : 0; if (enable_sinks) { region.origin = offset_sinks; region.size = ggml_nbytes(sinks); mem_sinksBuf = clCreateSubBuffer(extra_sinks->data_device, CL_MEM_READ_ONLY, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err); CL_CHECK(err); cl_image_format img_fmt_sinks_1d = { CL_R, CL_FLOAT }; cl_image_desc img_desc_sinks_1d; memset(&img_desc_sinks_1d, 0, sizeof(img_desc_sinks_1d)); img_desc_sinks_1d.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; img_desc_sinks_1d.image_width = (size_t)n_head; img_desc_sinks_1d.buffer = mem_sinksBuf; mem_tex_sinks_1dbuf = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt_sinks_1d, &img_desc_sinks_1d, NULL, &err); CL_CHECK(err); } else { // The image obj cannot be null so we back with buffer of size 1 and use matrixK to back because it always exists cl_image_format img_fmt_sinks_fallback = { CL_R, CL_FLOAT }; cl_image_desc img_desc_sinks_fallback; memset(&img_desc_sinks_fallback, 0, sizeof(img_desc_sinks_fallback)); img_desc_sinks_fallback.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; img_desc_sinks_fallback.image_width = 1; img_desc_sinks_fallback.buffer = mem_matrixK; mem_tex_sinks_1dbuf = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt_sinks_fallback, &img_desc_sinks_fallback, NULL, &err); CL_CHECK(err); } cl_uint arg = 0; CL_CHECK(clSetKernelArg(kernel, arg++, sizeof(cl_mem), &mem_tex_matrixO_1dbuf)); CL_CHECK(clSetKernelArg(kernel, arg++, sizeof(float), &scale)); CL_CHECK(clSetKernelArg(kernel, arg++, sizeof(int), &n_q)); CL_CHECK(clSetKernelArg(kernel, arg++, sizeof(int), &n_kv)); CL_CHECK(clSetKernelArg(kernel, arg++, sizeof(int), &is_causal)); CL_CHECK(clSetKernelArg(kernel, arg++, sizeof(int), &n_head)); CL_CHECK(clSetKernelArg(kernel, arg++, sizeof(cl_ulong), &q_nb1)); CL_CHECK(clSetKernelArg(kernel, arg++, sizeof(cl_ulong), &q_nb2)); CL_CHECK(clSetKernelArg(kernel, arg++, sizeof(cl_ulong), &q_nb3)); CL_CHECK(clSetKernelArg(kernel, arg++, sizeof(cl_ulong), &k_nb1)); CL_CHECK(clSetKernelArg(kernel, arg++, sizeof(cl_ulong), &k_nb2)); CL_CHECK(clSetKernelArg(kernel, arg++, sizeof(cl_ulong), &k_nb3)); CL_CHECK(clSetKernelArg(kernel, arg++, sizeof(cl_ulong), &v_nb1)); CL_CHECK(clSetKernelArg(kernel, arg++, sizeof(cl_ulong), &v_nb2)); CL_CHECK(clSetKernelArg(kernel, arg++, sizeof(cl_ulong), &v_nb3)); CL_CHECK(clSetKernelArg(kernel, arg++, sizeof(cl_ulong), &o_nb1)); CL_CHECK(clSetKernelArg(kernel, arg++, sizeof(cl_ulong), &o_nb2)); CL_CHECK(clSetKernelArg(kernel, arg++, sizeof(cl_ulong), &o_nb3)); CL_CHECK(clSetKernelArg(kernel, arg++, sizeof(float), &max_bias)); CL_CHECK(clSetKernelArg(kernel, arg++, sizeof(float), &m0)); CL_CHECK(clSetKernelArg(kernel, arg++, sizeof(float), &m1)); CL_CHECK(clSetKernelArg(kernel, arg++, sizeof(int), &n_head_log2_val)); CL_CHECK(clSetKernelArg(kernel, arg++, sizeof(float), &logit_softcap)); CL_CHECK(clSetKernelArg(kernel, arg++, sizeof(int), &n_head_kv)); CL_CHECK(clSetKernelArg(kernel, arg++, sizeof(cl_mem), &mask_buffer)); CL_CHECK(clSetKernelArg(kernel, arg++, sizeof(int), &enable_mask)); CL_CHECK(clSetKernelArg(kernel, arg++, sizeof(cl_ulong), &mask_nb1_padded)); CL_CHECK(clSetKernelArg(kernel, arg++, sizeof(cl_ulong), &mask_nb2_padded)); CL_CHECK(clSetKernelArg(kernel, arg++, sizeof(cl_ulong), &mask_nb3_padded)); CL_CHECK(clSetKernelArg(kernel, arg++, sizeof(int), &mask_ne2)); CL_CHECK(clSetKernelArg(kernel, arg++, sizeof(int), &mask_ne3)); CL_CHECK(clSetKernelArg(kernel, arg++, sizeof(cl_mem), &mem_tex_sinks_1dbuf)); CL_CHECK(clSetKernelArg(kernel, arg++, sizeof(int), &enable_sinks)); CL_CHECK(clSetKernelArg(kernel, arg++, sizeof(cl_mem), &img_q_wmm)); CL_CHECK(clSetKernelArg(kernel, arg++, sizeof(cl_mem), &img_k_wmm)); CL_CHECK(clSetKernelArg(kernel, arg++, sizeof(cl_mem), &img_v_wmm)); CL_CHECK(clSetKernelArg(kernel, arg++, sizeof(int), &d_head_q)); size_t global_work_size[3], local_work_size[3]; const int n_waves_v = d_head_q / 64; local_work_size[0] = 64; local_work_size[1] = n_waves_v; local_work_size[2] = 1; global_work_size[0] = 64; global_work_size[1] = ((n_q + 64 - 1) / 64) * n_waves_v; global_work_size[2] = n_batch * n_head; backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); CL_CHECK(clReleaseMemObject(mem_tex_matrixO_1dbuf)); CL_CHECK(clReleaseMemObject(img_q_wmm)); CL_CHECK(clReleaseMemObject(img_k_wmm)); CL_CHECK(clReleaseMemObject(img_v_wmm)); if (mem_tex_matrixMask_1dbuf) { CL_CHECK(clReleaseMemObject(mem_tex_matrixMask_1dbuf)); } if (mem_matrixMask) { CL_CHECK(clReleaseMemObject(mem_matrixMask)); } if (mem_matrixMask_padded) { CL_CHECK(clReleaseMemObject(mem_matrixMask_padded)); } if (mem_tex_sinks_1dbuf) { CL_CHECK(clReleaseMemObject(mem_tex_sinks_1dbuf)); } if (mem_sinksBuf) { CL_CHECK(clReleaseMemObject(mem_sinksBuf)); } CL_CHECK(clReleaseMemObject(mem_matrixQ)); CL_CHECK(clReleaseMemObject(mem_matrixK)); CL_CHECK(clReleaseMemObject(mem_matrixV)); CL_CHECK(clReleaseMemObject(mem_matrixO)); } #endif // GGML_OPENCL_USE_ADRENO_KERNELS static void ggml_cl_flash_attn(ggml_backend_t backend, const ggml_tensor * q, const ggml_tensor * k, ggml_tensor * dst) { const ggml_tensor * v = dst->src[2]; const ggml_tensor * mask = dst->src[3]; const ggml_tensor * sinks = dst->src[4]; GGML_ASSERT(q->extra); GGML_ASSERT(k->extra); GGML_ASSERT(v->extra); GGML_ASSERT(dst->extra); if (mask) { GGML_ASSERT(mask->extra); } if (sinks) { GGML_ASSERT(sinks->extra); } ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; const int n_q = q->ne[1]; const int n_kv = k->ne[1]; const int d_head_q = q->ne[0]; const int d_head_v = v->ne[0]; const int n_head = q->ne[2]; const int n_head_kv = k->ne[2]; const int n_batch = q->ne[3]; #ifdef GGML_OPENCL_USE_ADRENO_KERNELS if (ggml_cl_adreno_xmem_attn_can_use(backend_ctx, q, k, dst)) { ggml_cl_adreno_xmem_attn_run(backend, q, k, dst); return; } #endif // DK=512 (Gemma-4 global layers) runs decode-only (q1 / q1_split) on // Adreno - it never uses the BM-tile path, and the prepass + split-tile // programs OOM the compiler at DK=512; supports_op only admits // n_q==1 here and prefill goes to CPU const bool fa_decode_only_512 = (d_head_q == 512); // per-variant lazy compile for this (dk, dv) // DK=512 decode (n_q==1) needs no prepass // DK=512 prefill (n_q>1) does, so compile it only when needed if (!fa_decode_only_512 || n_q > 1) { ggml_opencl_ensure_fa_pre_kernels(backend_ctx, d_head_q, d_head_v); } cl_kernel kernel = NULL; bool use_prefill_k_img = false; // K is image1d_buffer_t for DK=512 prefill const bool is_f16 = q->type == GGML_TYPE_F16; const bool is_mixed = q->type == GGML_TYPE_F32 && k->type == GGML_TYPE_F16 && v->type == GGML_TYPE_F16; const bool is_q8_0 = q->type == GGML_TYPE_F32 && k->type == GGML_TYPE_Q8_0 && v->type == GGML_TYPE_Q8_0; const bool is_q4_0 = q->type == GGML_TYPE_F32 && k->type == GGML_TYPE_Q4_0 && v->type == GGML_TYPE_Q4_0; #ifdef GGML_OPENCL_USE_ADRENO_KERNELS if (use_fa_bin_kernels_prefill(backend_ctx, q, k, v)) { // We support the prefill path of flash attn with a specialized d_head = 64/128/256 ggml_cl_flash_attn_prefill_bin(backend, q, k, dst); return; } #endif if (is_f16) { ggml_opencl_ensure_fa_variant(backend_ctx, d_head_q, d_head_v, FA_VARIANT_F16); } else if (is_mixed) { ggml_opencl_ensure_fa_variant(backend_ctx, d_head_q, d_head_v, FA_VARIANT_F32_F16); if (fa_decode_only_512) { // DK=512: the BM-tile prefill kernels are specifically compiled from // FA_PREFILL_ONLY if (n_q > 1) { ggml_opencl_ensure_fa_f32_f16_prefill_512(backend_ctx, /*split=*/false); ggml_opencl_ensure_fa_f32_f16_prefill_512(backend_ctx, /*split=*/true); } } else { ggml_opencl_ensure_fa_variant(backend_ctx, d_head_q, d_head_v, FA_VARIANT_F32_F16_SPLIT); } } else if (is_q8_0) { ggml_opencl_ensure_fa_variant(backend_ctx, d_head_q, d_head_v, FA_VARIANT_Q8_0); if (d_head_q == 96 && d_head_v == 96) { ggml_opencl_ensure_fa_quant_split_override(backend_ctx, 96, 96, /*quant_bm=*/16, /*quant_n_split=*/3, /*is_q8_0=*/true); } else if (d_head_q == 256 && d_head_v == 256) { ggml_opencl_ensure_fa_quant_split_override(backend_ctx, 256, 256, /*quant_bm=*/16, /*quant_n_split=*/8, /*is_q8_0=*/true); } else { ggml_opencl_ensure_fa_variant(backend_ctx, d_head_q, d_head_v, FA_VARIANT_Q8_0_SPLIT); } } else if (is_q4_0) { ggml_opencl_ensure_fa_variant(backend_ctx, d_head_q, d_head_v, FA_VARIANT_Q4_0); if (d_head_q == 96 && d_head_v == 96) { ggml_opencl_ensure_fa_quant_split_override(backend_ctx, 96, 96, /*quant_bm=*/16, /*quant_n_split=*/3, /*is_q8_0=*/false); } else if (d_head_q == 256 && d_head_v == 256) { ggml_opencl_ensure_fa_quant_split_override(backend_ctx, 256, 256, /*quant_bm=*/16, /*quant_n_split=*/8, /*is_q8_0=*/false); } else { ggml_opencl_ensure_fa_variant(backend_ctx, d_head_q, d_head_v, FA_VARIANT_Q4_0_SPLIT); } } else { ggml_opencl_ensure_fa_variant(backend_ctx, d_head_q, d_head_v, FA_VARIANT_F32); } const std::pair dk_dv = {d_head_q, d_head_v}; const bool use_native_q8_0_q1 = is_q8_0 && n_q == 1 && backend_ctx->fa.f32_q8_0_q1.count(dk_dv) > 0; // Native q8_0 prefill — reads q8_0 directly, wg_size = cfg->bm. const bool use_native_q8_0 = is_q8_0 && n_q > 1 && backend_ctx->fa.f32_q8_0.count(dk_dv) > 0; const bool use_native_q4_0_q1 = is_q4_0 && n_q == 1 && backend_ctx->fa.f32_q4_0_q1.count(dk_dv) > 0; const bool use_native_q4_0 = is_q4_0 && n_q > 1 && backend_ctx->fa.f32_q4_0.count(dk_dv) > 0; const int block_m = n_q > 1 ? (is_mixed ? backend_ctx->fa.f32_f16_bm.at(dk_dv) : backend_ctx->fa.bm.at(dk_dv)) : 0; // block_n is only used by the n_q > 1 prefill path; its map is not // populated for DK=512 decode, so do not read it for decode. const int block_n = (n_q > 1) ? (is_mixed ? backend_ctx->fa.f32_f16_bn.at(dk_dv) : backend_ctx->fa.bn.at(dk_dv)) : 0; // Pick split variant only when n_kv crosses the per-(dk,dv) threshold. // the N_SPLIT>1 prefill tile reduces DK partials via subgroup shuffle, // on Intel it uses the non-split BM tile and does not depend on subgroup size const bool use_split_kernel = (n_q > 1 && is_mixed && backend_ctx->gpu_family != INTEL && backend_ctx->fa.f32_f16_split.count(dk_dv) > 0 && n_kv >= backend_ctx->fa.f32_f16_split_nkv_threshold.at(dk_dv)); const bool use_split_q8_0 = (use_native_q8_0 && backend_ctx->gpu_family != INTEL && backend_ctx->fa.f32_q8_0_split.count(dk_dv) > 0 && n_kv >= backend_ctx->fa.f32_q8_0_split_nkv_threshold.at(dk_dv)); const bool use_split_q4_0 = (use_native_q4_0 && backend_ctx->gpu_family != INTEL && backend_ctx->fa.f32_q4_0_split.count(dk_dv) > 0 && n_kv >= backend_ctx->fa.f32_q4_0_split_nkv_threshold.at(dk_dv)); const int wg_size_fa = (n_q > 1 && is_mixed) ? (use_split_kernel ? backend_ctx->fa.f32_f16_split_wg_size.at(dk_dv) : backend_ctx->fa.f32_f16_wg_size.at(dk_dv)) : block_m; ggml_tensor_extra_cl * extra_q = (ggml_tensor_extra_cl *)q->extra; ggml_tensor_extra_cl * extra_o = (ggml_tensor_extra_cl *)dst->extra; ggml_tensor_extra_cl * extra_mask = mask ? (ggml_tensor_extra_cl *)mask->extra : NULL; ggml_tensor_extra_cl * extra_sinks = sinks ? (ggml_tensor_extra_cl *)sinks->extra : NULL; // SoA q8_0/q4_0 K/V: data_device aliases the `q` subbuffer; reconstruct // AoS into a temp buffer below. AoS tensors use extra_k/v->data_device. const bool k_soa = ggml_cl_is_q8_0_soa(k) || ggml_cl_is_q4_0_soa(k); const bool v_soa = ggml_cl_is_q8_0_soa(v) || ggml_cl_is_q4_0_soa(v); ggml_tensor_extra_cl * extra_k = k_soa ? nullptr : (ggml_tensor_extra_cl *)k->extra; ggml_tensor_extra_cl * extra_v = v_soa ? nullptr : (ggml_tensor_extra_cl *)v->extra; cl_ulong offset_q = extra_q->offset + q->view_offs; cl_ulong offset_k = k_soa ? 0 : extra_k->offset + k->view_offs; cl_ulong offset_v = v_soa ? 0 : extra_v->offset + v->view_offs; cl_ulong offset_o = extra_o->offset + dst->view_offs; cl_mem mask_buffer = extra_mask ? extra_mask->data_device : NULL; cl_ulong offset_mask = extra_mask ? extra_mask->offset + mask->view_offs : 0; cl_mem sinks_buffer = extra_sinks ? extra_sinks->data_device : NULL; cl_ulong offset_sinks = extra_sinks ? extra_sinks->offset + sinks->view_offs : 0; const cl_ulong q_nb1 = q->nb[1]; const cl_ulong q_nb2 = q->nb[2]; const cl_ulong q_nb3 = q->nb[3]; cl_ulong k_nb1 = k->nb[1]; cl_ulong k_nb2 = k->nb[2]; cl_ulong k_nb3 = k->nb[3]; cl_ulong v_nb1 = v->nb[1]; cl_ulong v_nb2 = v->nb[2]; cl_ulong v_nb3 = v->nb[3]; const cl_ulong o_nb1 = dst->nb[1]; const cl_ulong o_nb2 = dst->nb[2]; const cl_ulong o_nb3 = dst->nb[3]; const cl_ulong mask_nb1 = mask ? mask->nb[1] : 0; const cl_ulong mask_nb2 = mask ? mask->nb[2] : 0; const cl_ulong mask_nb3 = mask ? mask->nb[3] : 0; const int mask_ne2 = mask ? mask->ne[2] : 0; const int mask_ne3 = mask ? mask->ne[3] : 0; float scale; float max_bias; float logit_softcap; const float * params = (const float *)dst->op_params; scale = params[0]; max_bias = params[1]; logit_softcap = params[2]; bool use_q1_vec = false; bool use_q1_vec_mq = false; bool use_local_tile = false; // KV-head-coalesced gate: gqa_ratio == compile-time MQ_GQA // restricts to DK=DV=256 for now due to local memory size const int gqa_ratio_dispatch = n_head_kv > 0 ? (n_head / n_head_kv) : 0; if (n_q == 1) { if (use_native_q8_0_q1) { if (d_head_v >= 256 && backend_ctx->fa.f32_q8_0_q1_vec.count(dk_dv) > 0) { kernel = backend_ctx->fa.f32_q8_0_q1_vec.at(dk_dv); use_q1_vec = true; } else { kernel = backend_ctx->fa.f32_q8_0_q1.at(dk_dv); } } else if (use_native_q4_0_q1) { // q4_0 vec kernel uses per-lane dp4a (cl_khr_integer_dot_product) const char * q4vec_env = getenv("GGML_OPENCL_FA_Q4_VEC"); const bool q4vec_off = (q4vec_env != NULL) && (q4vec_env[0] == '0'); if (!q4vec_off && d_head_v >= 256 && backend_ctx->fa.f32_q4_0_q1_vec.count(dk_dv) > 0) { kernel = backend_ctx->fa.f32_q4_0_q1_vec.at(dk_dv); use_q1_vec = true; } else { kernel = backend_ctx->fa.f32_q4_0_q1.at(dk_dv); } } else if (is_mixed) { static const char * lt_env = getenv("GGML_OPENCL_FA_LOCAL_TILE"); static const bool lt_on = (lt_env != NULL) && (lt_env[0] != '0'); if (lt_on && d_head_q == 128 && d_head_v == 128 && backend_ctx->fa.f32_f16_q1_local_tile.count(dk_dv) > 0) { kernel = backend_ctx->fa.f32_f16_q1_local_tile.at(dk_dv); use_local_tile = true; } else { static const char * f16_vec_dk128_env = getenv("GGML_OPENCL_FA_F16_VEC_DK128"); static const bool f16_vec_dk128_off = (f16_vec_dk128_env != NULL) && (f16_vec_dk128_env[0] == '0'); const int dv_gate = f16_vec_dk128_off ? 256 : 128; if (d_head_v >= dv_gate && backend_ctx->fa.f32_f16_q1_vec.count(dk_dv) > 0) { kernel = backend_ctx->fa.f32_f16_q1_vec.at(dk_dv); use_q1_vec = true; } else { kernel = backend_ctx->fa.f32_f16_q1.at(dk_dv); } } } else if (is_f16) { kernel = backend_ctx->fa.f16_q1.at(dk_dv); } else { kernel = backend_ctx->fa.f32_q1.at(dk_dv); } } else { if (use_native_q8_0) { kernel = use_split_q8_0 ? backend_ctx->fa.f32_q8_0_split.at(dk_dv) : backend_ctx->fa.f32_q8_0.at(dk_dv); } else if (use_native_q4_0) { kernel = use_split_q4_0 ? backend_ctx->fa.f32_q4_0_split.at(dk_dv) : backend_ctx->fa.f32_q4_0.at(dk_dv); } else if (is_mixed) { if (use_split_kernel) { // DK=512 prefill: opt-in texture-cache K reads (image1d_buffer_t). static const char * pkimg_env = getenv("GGML_OPENCL_FA_PREFILL_K_IMG"); const bool pkimg_on = (pkimg_env != NULL) && (pkimg_env[0] != '0'); if (d_head_q == 512 && pkimg_on && backend_ctx->fa.f32_f16_split_k_img.count(dk_dv) > 0) { kernel = backend_ctx->fa.f32_f16_split_k_img.at(dk_dv); use_prefill_k_img = true; } else { kernel = backend_ctx->fa.f32_f16_split.at(dk_dv); } } else { kernel = backend_ctx->fa.f32_f16.at(dk_dv); } } else if (is_f16) { kernel = backend_ctx->fa.f16.at(dk_dv); } else { kernel = backend_ctx->fa.f32.at(dk_dv); } } // Intel goes to the basic q1 kernel if (backend_ctx->gpu_family == INTEL && n_q == 1) { use_q1_vec = use_q1_vec_mq = use_local_tile = false; if (is_mixed && backend_ctx->fa.f32_f16_q1.count(dk_dv)) { kernel = backend_ctx->fa.f32_f16_q1.at(dk_dv); } else if (is_f16 && backend_ctx->fa.f16_q1.count(dk_dv)) { kernel = backend_ctx->fa.f16_q1.at(dk_dv); } else if (is_q8_0 && backend_ctx->fa.f32_q8_0_q1.count(dk_dv)) { kernel = backend_ctx->fa.f32_q8_0_q1.at(dk_dv); } else if (is_q4_0 && backend_ctx->fa.f32_q4_0_q1.count(dk_dv)) { kernel = backend_ctx->fa.f32_q4_0_q1.at(dk_dv); } else if (backend_ctx->fa.f32_q1.count(dk_dv)) { kernel = backend_ctx->fa.f32_q1.at(dk_dv); } } GGML_ASSERT(kernel != NULL); ggml_cl_flash_attn_temp_buffer temp_k; ggml_cl_flash_attn_temp_buffer temp_v; ggml_cl_flash_attn_temp_buffer temp_k_pad; ggml_cl_flash_attn_temp_buffer temp_v_pad; ggml_cl_flash_attn_temp_buffer temp_mask_pad; ggml_cl_flash_attn_temp_buffer temp_blk; const ggml_type kv_target_type = is_f16 ? GGML_TYPE_F16 : GGML_TYPE_F32; cl_mem k_data_device = k_soa ? NULL : extra_k->data_device; cl_mem v_data_device = v_soa ? NULL : extra_v->data_device; // SoA q8_0/q4_0 -> reconstruct AoS for downstream kernels that expect // tight records (no-op when k/v is already AoS). ggml_cl_flash_attn_temp_buffer temp_k_aos; ggml_cl_flash_attn_temp_buffer temp_v_aos; ggml_cl_flash_attn_reconstruct_aos(backend_ctx, k, temp_k_aos, k_data_device, offset_k, k_nb1, k_nb2, k_nb3); ggml_cl_flash_attn_reconstruct_aos(backend_ctx, v, temp_v_aos, v_data_device, offset_v, v_nb1, v_nb2, v_nb3); // currently FA kernels support KV cache with f16, f32, q4_0 and q8_0. // there two cases that these kernels cannot cover, // 1. KV cache types are q4_0 or q8_0, but the FA kernels fail to compile // 2. KV cache types not currently supported by an FA kernel, e.g., q4_1 // these two cases are supported here by dequantizing to f32/f16 and this // causes performance degradation. // For q4_0 or q8_0 cases that fail kernel compilation, dequant happens in GPU; // for types that do not have FA kernels, dequant happens on host. if (!use_native_q8_0_q1 && !use_native_q8_0 && !use_native_q4_0_q1 && !use_native_q4_0) { // for q4_0, q8_0 FA kernels that fail to compile bool k_done = false; bool v_done = false; if (k->type == GGML_TYPE_Q8_0 || k->type == GGML_TYPE_Q4_0) { k_done = ggml_cl_flash_attn_dequant_kv_gpu( backend_ctx, k, kv_target_type, k_data_device, offset_k, k_nb1, k_nb2, k_nb3, temp_k, k_data_device, offset_k, k_nb1, k_nb2, k_nb3); } if (v->type == GGML_TYPE_Q8_0 || v->type == GGML_TYPE_Q4_0) { v_done = ggml_cl_flash_attn_dequant_kv_gpu( backend_ctx, v, kv_target_type, v_data_device, offset_v, v_nb1, v_nb2, v_nb3, temp_v, v_data_device, offset_v, v_nb1, v_nb2, v_nb3); } if (!k_done) { ggml_cl_flash_attn_prepare_quantized_tensor( backend_ctx, k, kv_target_type, temp_k, k_data_device, offset_k, k_nb1, k_nb2, k_nb3); } if (!v_done) { ggml_cl_flash_attn_prepare_quantized_tensor( backend_ctx, v, kv_target_type, temp_v, v_data_device, offset_v, v_nb1, v_nb2, v_nb3); } // Asymmetric KV on the F32 fallback path: convert the F16 side to F32 // too. (Symmetric F16 / mixed paths handle F16 directly.) if (kv_target_type == GGML_TYPE_F32 && !is_mixed && !is_f16) { ggml_cl_flash_attn_convert_f16_to_f32(backend_ctx, k, temp_k, k_data_device, offset_k, k_nb1, k_nb2, k_nb3); ggml_cl_flash_attn_convert_f16_to_f32(backend_ctx, v, temp_v, v_data_device, offset_v, v_nb1, v_nb2, v_nb3); } } cl_mem k_pad_buffer = NULL; cl_mem v_pad_buffer = NULL; cl_mem mask_pad_buffer = NULL; cl_mem blk_buffer = NULL; cl_ulong mask_pad_nb1 = 0; cl_ulong mask_pad_nb2 = 0; cl_ulong mask_pad_nb3 = 0; // Flash-Decoding K-split decision. Resolved here, before the prefill // prepass, because KV-pad and blk prepass are pure overhead when FD fires. // Do not infer causality from tensor shapes: a NULL mask means full // (bidirectional) attention, e.g. ViT encoders, where n_q == n_kv as well. // Causal attention in llama.cpp always comes with an explicit KQ mask. // Inferring is_causal here corrupted mmproj output on OpenCL (see #23800). const int is_causal = 0; const int fd_max_n_q = (d_head_q <= FD_MAX_DK_MULTI) ? FD_MAX_N_Q_MULTI : 1; cl_kernel fd_k_split = NULL; bool use_fd_mq = false; size_t fd_mq_wg = 256; // MQ_GQA=4 kernel: Q1_WG_SIZE(64) * MQ_NSG_SPLIT(4) bool use_fa_k_img = false; // K bound as image1d_buffer_t instead of (buf, offset) { const char * mq_env = getenv("GGML_OPENCL_FA_MQ"); const bool mq_enabled = (mq_env == NULL) ? true : (mq_env[0] != '0'); const bool mq_kv_ok = is_mixed || is_q8_0 || is_q4_0; const char * lmq_env = getenv("GGML_OPENCL_FA_LOCAL_MQ_SPLIT"); const bool lmq_on = (lmq_env != NULL) && (lmq_env[0] != '0'); static const char * vec_nq_env = getenv("GGML_OPENCL_FA_VEC_NQ"); static const int N_MAX_VEC_NQ = (vec_nq_env != NULL && vec_nq_env[0] != '\0') ? atoi(vec_nq_env) : 1; const bool nq_in_vec_range = (n_q >= 1) && (n_q <= N_MAX_VEC_NQ); const bool nq1_only = (n_q == 1); // Cluster-parallel decode default on for Adreno X2E/X1E static const int c8_env_state = []{ const char * e = getenv("GGML_OPENCL_FA_C8"); if (e == NULL || e[0] == '\0') { return -1; } return (e[0] != '0') ? 1 : 0; }(); const bool c8_default_on = backend_ctx->adreno_gen == ADRENO_GPU_GEN::X2E || backend_ctx->adreno_gen == ADRENO_GPU_GEN::X1E; const bool c8_f16_on = (c8_env_state >= 0) ? (c8_env_state == 1) : c8_default_on; // Quant-KV (q4_0/q8_0) GQA4 c8: default-on X2E + X1E const bool c8_quant_on = (c8_env_state >= 0) ? (c8_env_state == 1) : c8_default_on; if (mq_enabled && mq_kv_ok && nq_in_vec_range && !is_causal && backend_ctx->gpu_family != INTEL && !use_local_tile && n_kv >= FD_MIN_N_KV && backend_ctx->fa.f32_merge.count(dk_dv) > 0) { if (nq1_only && lmq_on && is_mixed && d_head_q == 128 && d_head_v == 128 && gqa_ratio_dispatch == 8 && backend_ctx->fa.f32_f16_q1_local_mq_split_g8.count(dk_dv) > 0) { fd_k_split = backend_ctx->fa.f32_f16_q1_local_mq_split_g8.at(dk_dv); use_fd_mq = true; fd_mq_wg = 64; } else if (nq1_only && lmq_on && is_mixed && d_head_q == 128 && d_head_v == 128 && gqa_ratio_dispatch == 4 && backend_ctx->fa.f32_f16_q1_local_mq_split.count(dk_dv) > 0) { fd_k_split = backend_ctx->fa.f32_f16_q1_local_mq_split.at(dk_dv); use_fd_mq = true; fd_mq_wg = 64; } else if (nq1_only && is_mixed && gqa_ratio_dispatch == 4 && ((d_head_q == 256 && d_head_v == 256) || (d_head_q == 128 && d_head_v == 128)) && backend_ctx->fa.f32_f16_q1_vec_mq_split.count(dk_dv) > 0) { const bool k_img_on = d_head_q == 128 && d_head_v == 128 && getenv("GGML_OPENCL_FA_K_IMG") != NULL && getenv("GGML_OPENCL_FA_K_IMG")[0] != '0' && backend_ctx->fa.f32_f16_q1_vec_mq_split_k_img.count(dk_dv) > 0; // Cluster-parallel decode const bool c8_env = d_head_q == 128 && d_head_v == 128 && c8_f16_on; if (c8_env && backend_ctx->fa.f32_f16_q1_vec_mq_split_c8.count(dk_dv) > 0) { fd_k_split = backend_ctx->fa.f32_f16_q1_vec_mq_split_c8.at(dk_dv); use_fd_mq = true; } else if (c8_env && backend_ctx->fa.f32_f16_q1_vec_mq_split_c8_ns2.count(dk_dv) > 0) { fd_k_split = backend_ctx->fa.f32_f16_q1_vec_mq_split_c8_ns2.at(dk_dv); use_fd_mq = true; fd_mq_wg = 128; } else if (k_img_on) { fd_k_split = backend_ctx->fa.f32_f16_q1_vec_mq_split_k_img.at(dk_dv); use_fd_mq = true; use_fa_k_img = true; } else { fd_k_split = backend_ctx->fa.f32_f16_q1_vec_mq_split.at(dk_dv); use_fd_mq = true; } // Cluster-parallel decode, DK=DV=256 GQA=8 } else if (nq1_only && is_mixed && gqa_ratio_dispatch == 8 && d_head_q == 256 && d_head_v == 256 && c8_env_state == 1 && backend_ctx->fa.f32_f16_q1_vec_mq_split_g8_c32.count(dk_dv) > 0) { fd_k_split = backend_ctx->fa.f32_f16_q1_vec_mq_split_g8_c32.at(dk_dv); use_fd_mq = true; fd_mq_wg = 128; // Cluster-parallel decode for the g8 } else if (is_mixed && gqa_ratio_dispatch == 8 && d_head_q == 128 && d_head_v == 128 && c8_f16_on && (backend_ctx->fa.f32_f16_q1_vec_mq_split_g8_c8.count(dk_dv) > 0 || backend_ctx->fa.f32_f16_q1_vec_mq_split_g8_c8_ns2.count(dk_dv) > 0)) { if (backend_ctx->fa.f32_f16_q1_vec_mq_split_g8_c8.count(dk_dv) > 0) { fd_k_split = backend_ctx->fa.f32_f16_q1_vec_mq_split_g8_c8.at(dk_dv); fd_mq_wg = 192; } else { fd_k_split = backend_ctx->fa.f32_f16_q1_vec_mq_split_g8_c8_ns2.at(dk_dv); fd_mq_wg = 128; } use_fd_mq = true; } else if (is_mixed && gqa_ratio_dispatch == 8 && d_head_q == 128 && d_head_v == 128 && getenv("GGML_OPENCL_FA_K_IMG") != NULL && getenv("GGML_OPENCL_FA_K_IMG")[0] != '0' && backend_ctx->fa.f32_f16_q1_vec_mq_split_g8_k_img.count(dk_dv) > 0) { fd_k_split = backend_ctx->fa.f32_f16_q1_vec_mq_split_g8_k_img.at(dk_dv); use_fd_mq = true; fd_mq_wg = 192; use_fa_k_img = true; } else if (is_mixed && gqa_ratio_dispatch == 8 && d_head_q == 128 && d_head_v == 128 && backend_ctx->fa.f32_f16_q1_vec_mq_split_g8.count(dk_dv) > 0) { fd_k_split = backend_ctx->fa.f32_f16_q1_vec_mq_split_g8.at(dk_dv); use_fd_mq = true; fd_mq_wg = 192; } else if (nq1_only && is_q8_0 && gqa_ratio_dispatch == 8 && d_head_q == 128 && d_head_v == 128 && backend_ctx->fa.f32_q8_0_q1_vec_mq_split_g8.count(dk_dv) > 0) { fd_k_split = backend_ctx->fa.f32_q8_0_q1_vec_mq_split_g8.at(dk_dv); use_fd_mq = true; fd_mq_wg = 192; } else if (nq1_only && is_q8_0 && gqa_ratio_dispatch == 4 && d_head_q == 128 && d_head_v == 128 && backend_ctx->fa.f32_q8_0_q1_vec_mq_split.count(dk_dv) > 0) { // Cluster-parallel q8_0 GQA4 if (c8_quant_on && backend_ctx->fa.f32_q8_0_q1_vec_mq_split_c8.count(dk_dv) > 0) { fd_k_split = backend_ctx->fa.f32_q8_0_q1_vec_mq_split_c8.at(dk_dv); fd_mq_wg = 128; } else { fd_k_split = backend_ctx->fa.f32_q8_0_q1_vec_mq_split.at(dk_dv); } use_fd_mq = true; } else if (nq1_only && is_q4_0) { const char * q4_mq_env = getenv("GGML_OPENCL_FA_Q4_MQ"); const bool q4_mq_on = (q4_mq_env != NULL) && (q4_mq_env[0] != '0'); // Cluster-parallel q4_0 const bool q4_c8_on = c8_env_state == 1 && backend_ctx->fa.f32_q4_0_q1_vec_mq_split_g8_c8.count(dk_dv) > 0; if (q4_c8_on && gqa_ratio_dispatch == 8 && d_head_q == 64 && d_head_v == 64) { fd_k_split = backend_ctx->fa.f32_q4_0_q1_vec_mq_split_g8_c8.at(dk_dv); use_fd_mq = true; fd_mq_wg = 128; } else if (q4_mq_on && gqa_ratio_dispatch == 8 && d_head_q == 128 && d_head_v == 128 && backend_ctx->fa.f32_q4_0_q1_vec_mq_split_g8.count(dk_dv) > 0) { fd_k_split = backend_ctx->fa.f32_q4_0_q1_vec_mq_split_g8.at(dk_dv); use_fd_mq = true; fd_mq_wg = 192; } else if (gqa_ratio_dispatch == 4 && d_head_q == 128 && d_head_v == 128 && backend_ctx->fa.f32_q4_0_q1_vec_mq_split.count(dk_dv) > 0) { // Cluster-parallel q4_0 GQA4 if (c8_quant_on && backend_ctx->fa.f32_q4_0_q1_vec_mq_split_c8.count(dk_dv) > 0) { fd_k_split = backend_ctx->fa.f32_q4_0_q1_vec_mq_split_c8.at(dk_dv); fd_mq_wg = 128; } else { fd_k_split = backend_ctx->fa.f32_q4_0_q1_vec_mq_split.at(dk_dv); } use_fd_mq = true; } } } } // Intel cluster-parallel decode FA if (fd_k_split == NULL && backend_ctx->gpu_family == INTEL && n_q == 1 && !is_causal && is_mixed && gqa_ratio_dispatch == 4 && d_head_q == 128 && d_head_v == 128 && n_kv >= FD_MIN_N_KV && getenv("GGML_OPENCL_FA_C8") != NULL && getenv("GGML_OPENCL_FA_C8")[0] != '0' && backend_ctx->fa.f32_merge.count(dk_dv) > 0) { if (backend_ctx->fa.f32_f16_q1_vec_mq_split_c8.count(dk_dv) > 0) { fd_k_split = backend_ctx->fa.f32_f16_q1_vec_mq_split_c8.at(dk_dv); use_fd_mq = true; fd_mq_wg = 128; } else if (backend_ctx->fa.f32_f16_q1_vec_mq_split_c8_ns2.count(dk_dv) > 0) { fd_k_split = backend_ctx->fa.f32_f16_q1_vec_mq_split_c8_ns2.at(dk_dv); use_fd_mq = true; fd_mq_wg = 64; } } if (fd_k_split == NULL && n_q >= 1 && n_q <= fd_max_n_q && n_kv >= FD_MIN_N_KV && !is_causal && d_head_q <= FD_MAX_DK && backend_ctx->fa.f32_merge.count(dk_dv) > 0) { if (is_mixed && backend_ctx->fa.f32_f16_q1_split.count(dk_dv) > 0) { fd_k_split = backend_ctx->fa.f32_f16_q1_split.at(dk_dv); } else if (is_q8_0 && backend_ctx->fa.f32_q8_0_q1_split.count(dk_dv) > 0) { fd_k_split = backend_ctx->fa.f32_q8_0_q1_split.at(dk_dv); } else if (is_q4_0 && backend_ctx->fa.f32_q4_0_q1_split.count(dk_dv) > 0) { fd_k_split = backend_ctx->fa.f32_q4_0_q1_split.at(dk_dv); } } const bool use_fd = (fd_k_split != NULL); const int n_q_blocks = n_q > 1 ? (n_q + block_m - 1) / block_m : 0; const int n_kv_blocks = (n_kv > 0 && block_n > 0) ? (n_kv + block_n - 1) / block_n : 0; // KV pad + blk prepass are pure overhead when FD will fire — skip them. const bool use_mixed_prepass = is_mixed && n_q > 1 && !use_fd; // make sure prepass kernels are compiled const bool have_kv_pad = backend_ctx->fa.kv_pad_f16.count(dk_dv) > 0; const bool have_blk = backend_ctx->fa.blk_f16.count(dk_dv) > 0; const bool use_kv_pad = use_mixed_prepass && (n_kv % block_n != 0) && have_kv_pad; // blk prepass: per-KV-tile mask class (0=masked, 1=mixed, 2=unmasked). // Consumed identically by f32_f16, q8_0 and q4_0 prefill kernels. const bool use_quant_prepass = (use_native_q8_0 || use_native_q4_0) && !use_fd; const bool use_blk_mask = (use_mixed_prepass || use_quant_prepass) && mask_buffer != NULL && have_blk; if (use_kv_pad) { cl_int err; const size_t k_pad_size = (size_t) k_nb1 * (size_t) block_n * (size_t) n_head_kv * (size_t) n_batch; temp_k_pad.data = clCreateBuffer(backend_ctx->context, CL_MEM_READ_WRITE, k_pad_size, NULL, &err); CL_CHECK(err); k_pad_buffer = temp_k_pad.data; const size_t v_pad_size = (size_t) v_nb1 * (size_t) block_n * (size_t) n_head_kv * (size_t) n_batch; temp_v_pad.data = clCreateBuffer(backend_ctx->context, CL_MEM_READ_WRITE, v_pad_size, NULL, &err); CL_CHECK(err); v_pad_buffer = temp_v_pad.data; cl_kernel kernel_kv_pad = backend_ctx->fa.kv_pad_f16.at(dk_dv); CL_CHECK(clSetKernelArg(kernel_kv_pad, 0, sizeof(cl_mem), &k_data_device)); CL_CHECK(clSetKernelArg(kernel_kv_pad, 1, sizeof(cl_ulong), &offset_k)); CL_CHECK(clSetKernelArg(kernel_kv_pad, 2, sizeof(cl_mem), &v_data_device)); CL_CHECK(clSetKernelArg(kernel_kv_pad, 3, sizeof(cl_ulong), &offset_v)); CL_CHECK(clSetKernelArg(kernel_kv_pad, 4, sizeof(cl_mem), &k_pad_buffer)); CL_CHECK(clSetKernelArg(kernel_kv_pad, 5, sizeof(cl_mem), &v_pad_buffer)); CL_CHECK(clSetKernelArg(kernel_kv_pad, 6, sizeof(int), &n_kv)); CL_CHECK(clSetKernelArg(kernel_kv_pad, 7, sizeof(int), &n_head_kv)); CL_CHECK(clSetKernelArg(kernel_kv_pad, 8, sizeof(int), &n_batch)); CL_CHECK(clSetKernelArg(kernel_kv_pad, 9, sizeof(cl_ulong), &k_nb1)); CL_CHECK(clSetKernelArg(kernel_kv_pad, 10, sizeof(cl_ulong), &k_nb2)); CL_CHECK(clSetKernelArg(kernel_kv_pad, 11, sizeof(cl_ulong), &k_nb3)); CL_CHECK(clSetKernelArg(kernel_kv_pad, 12, sizeof(cl_ulong), &v_nb1)); CL_CHECK(clSetKernelArg(kernel_kv_pad, 13, sizeof(cl_ulong), &v_nb2)); CL_CHECK(clSetKernelArg(kernel_kv_pad, 14, sizeof(cl_ulong), &v_nb3)); size_t global_work_size[] = { (size_t) block_n, (size_t) n_head_kv, (size_t) n_batch }; backend_ctx->enqueue_ndrange_kernel(kernel_kv_pad, 3, global_work_size, NULL, dst); if (mask_buffer != NULL) { mask_pad_nb1 = (cl_ulong) block_n * (cl_ulong) sizeof(ggml_fp16_t); mask_pad_nb2 = (cl_ulong) n_q * mask_pad_nb1; mask_pad_nb3 = (cl_ulong) mask_ne2 * mask_pad_nb2; const size_t mask_pad_size = (size_t) mask_ne3 * (size_t) mask_pad_nb3; temp_mask_pad.data = clCreateBuffer(backend_ctx->context, CL_MEM_READ_WRITE, mask_pad_size, NULL, &err); CL_CHECK(err); mask_pad_buffer = temp_mask_pad.data; cl_kernel kernel_mask_pad = backend_ctx->fa.mask_pad_f16.at(dk_dv); CL_CHECK(clSetKernelArg(kernel_mask_pad, 0, sizeof(cl_mem), &mask_buffer)); CL_CHECK(clSetKernelArg(kernel_mask_pad, 1, sizeof(cl_ulong), &offset_mask)); CL_CHECK(clSetKernelArg(kernel_mask_pad, 2, sizeof(cl_mem), &mask_pad_buffer)); CL_CHECK(clSetKernelArg(kernel_mask_pad, 3, sizeof(int), &n_q)); CL_CHECK(clSetKernelArg(kernel_mask_pad, 4, sizeof(int), &n_kv)); CL_CHECK(clSetKernelArg(kernel_mask_pad, 5, sizeof(cl_ulong), &mask_nb1)); CL_CHECK(clSetKernelArg(kernel_mask_pad, 6, sizeof(cl_ulong), &mask_nb2)); CL_CHECK(clSetKernelArg(kernel_mask_pad, 7, sizeof(cl_ulong), &mask_nb3)); CL_CHECK(clSetKernelArg(kernel_mask_pad, 8, sizeof(int), &mask_ne2)); CL_CHECK(clSetKernelArg(kernel_mask_pad, 9, sizeof(int), &mask_ne3)); size_t global_work_size_mask[] = { (size_t) block_n, (size_t) n_q, (size_t) (mask_ne2 * mask_ne3) }; backend_ctx->enqueue_ndrange_kernel(kernel_mask_pad, 3, global_work_size_mask, NULL, dst); } } if (use_blk_mask) { cl_int err; const size_t blk_size = (size_t) n_kv_blocks * (size_t) n_q_blocks * (size_t) mask_ne2 * (size_t) mask_ne3; temp_blk.data = clCreateBuffer(backend_ctx->context, CL_MEM_READ_WRITE, blk_size, NULL, &err); if (err != CL_SUCCESS) { // Flush before retry — reclaim deferred driver deallocations. CL_CHECK(clFinish(backend_ctx->queue)); temp_blk.data = clCreateBuffer(backend_ctx->context, CL_MEM_READ_WRITE, blk_size, NULL, &err); } CL_CHECK(err); blk_buffer = temp_blk.data; cl_kernel kernel_blk = backend_ctx->fa.blk_f16.at(dk_dv); CL_CHECK(clSetKernelArg(kernel_blk, 0, sizeof(cl_mem), &mask_buffer)); CL_CHECK(clSetKernelArg(kernel_blk, 1, sizeof(cl_ulong), &offset_mask)); CL_CHECK(clSetKernelArg(kernel_blk, 2, sizeof(cl_mem), &blk_buffer)); CL_CHECK(clSetKernelArg(kernel_blk, 3, sizeof(int), &n_q)); CL_CHECK(clSetKernelArg(kernel_blk, 4, sizeof(int), &n_kv)); CL_CHECK(clSetKernelArg(kernel_blk, 5, sizeof(cl_ulong), &mask_nb1)); CL_CHECK(clSetKernelArg(kernel_blk, 6, sizeof(cl_ulong), &mask_nb2)); CL_CHECK(clSetKernelArg(kernel_blk, 7, sizeof(cl_ulong), &mask_nb3)); CL_CHECK(clSetKernelArg(kernel_blk, 8, sizeof(int), &mask_ne2)); CL_CHECK(clSetKernelArg(kernel_blk, 9, sizeof(int), &mask_ne3)); size_t global_work_size_blk[] = { (size_t) n_kv_blocks, (size_t) n_q_blocks, (size_t) (mask_ne2 * mask_ne3) }; backend_ctx->enqueue_ndrange_kernel(kernel_blk, 3, global_work_size_blk, NULL, dst); } const int n_head_log2_val = n_head > 0 ? 1u << (int)floorf(log2f((float)n_head)) : 0; const float n_head_log2_f = n_head_log2_val > 0 ? (float)n_head_log2_val : 1.0f; const float m0 = powf(2.0f, -(max_bias) / n_head_log2_f); const float m1 = powf(2.0f, -(max_bias / 2.0f) / n_head_log2_f); if (use_fd) { static const int fd_env_kv_per_split = []{ const char * e = getenv("GGML_OPENCL_FD_KV_PER_SPLIT"); return (e && e[0]) ? atoi(e) : 0; }(); static const int fd_env_max_splits = []{ const char * e = getenv("GGML_OPENCL_FD_MAX_SPLITS"); return (e && e[0]) ? atoi(e) : 0; }(); int fd_kv_per_split = use_fd_mq ? FD_MQ_KV_PER_SPLIT : (is_mixed ? FD_KV_PER_SPLIT_F16 : FD_KV_PER_SPLIT); int fd_max_splits = use_fd_mq ? FD_MQ_MAX_SPLITS : FD_MAX_SPLITS; if (fd_env_kv_per_split > 0) { fd_kv_per_split = fd_env_kv_per_split; } if (fd_env_max_splits > 0) { fd_max_splits = fd_env_max_splits; } int n_splits = (n_kv + fd_kv_per_split - 1) / fd_kv_per_split; if (n_splits < FD_MIN_SPLITS) { n_splits = FD_MIN_SPLITS; } if (n_splits > fd_max_splits) { n_splits = fd_max_splits; } const int kv_per_split = (n_kv + n_splits - 1) / n_splits; const int fa_partial_floats = 2 + d_head_v; const size_t partial_size_bytes = (size_t) n_batch * n_head * n_q * n_splits * fa_partial_floats * sizeof(float); ggml_cl_flash_attn_temp_buffer temp_partial; cl_int err; temp_partial.data = clCreateBuffer(backend_ctx->context, CL_MEM_READ_WRITE, partial_size_bytes, NULL, &err); if (err != CL_SUCCESS) { CL_CHECK(clFinish(backend_ctx->queue)); temp_partial.data = clCreateBuffer(backend_ctx->context, CL_MEM_READ_WRITE, partial_size_bytes, NULL, &err); } CL_CHECK(err); cl_kernel k_split = fd_k_split; int argi = 0; CL_CHECK(clSetKernelArg(k_split, argi++, sizeof(cl_mem), &extra_q->data_device)); CL_CHECK(clSetKernelArg(k_split, argi++, sizeof(cl_ulong), &offset_q)); if (use_fa_k_img) { const size_t nb00_bytes = sizeof(uint16_t); const size_t k_bytes_span = (size_t)(n_kv > 0 ? n_kv - 1 : 0) * (size_t)k_nb1 + (size_t)(n_head_kv > 0 ? n_head_kv - 1 : 0) * (size_t)k_nb2 + (size_t)(n_batch > 0 ? n_batch - 1 : 0) * (size_t)k_nb3 + (size_t)d_head_q * nb00_bytes; const size_t k_bytes = (k_bytes_span + 7) & ~(size_t)7; const size_t k_pixels = k_bytes >> 3; cl_mem k_img = nullptr; if (k_pixels > 0 && k_pixels <= backend_ctx->image_max_buffer_size) { k_img = ggml_cl_img_pool_get_or_create( backend_ctx, backend_ctx->kq_img_pool, k_data_device, offset_k, k_bytes, CL_HALF_FLOAT); } // if image creation fails, fallback to buffer based kernels if (k_img == nullptr) { if (gqa_ratio_dispatch == 4 && backend_ctx->fa.f32_f16_q1_vec_mq_split.count(dk_dv) > 0) { k_split = backend_ctx->fa.f32_f16_q1_vec_mq_split.at(dk_dv); } else { k_split = backend_ctx->fa.f32_f16_q1_vec_mq_split_g8.at(dk_dv); } use_fa_k_img = false; CL_CHECK(clSetKernelArg(k_split, argi++, sizeof(cl_mem), &k_data_device)); CL_CHECK(clSetKernelArg(k_split, argi++, sizeof(cl_ulong), &offset_k)); } else { CL_CHECK(clSetKernelArg(k_split, argi++, sizeof(cl_mem), &k_img)); } } else { CL_CHECK(clSetKernelArg(k_split, argi++, sizeof(cl_mem), &k_data_device)); CL_CHECK(clSetKernelArg(k_split, argi++, sizeof(cl_ulong), &offset_k)); } CL_CHECK(clSetKernelArg(k_split, argi++, sizeof(cl_mem), &v_data_device)); CL_CHECK(clSetKernelArg(k_split, argi++, sizeof(cl_ulong), &offset_v)); CL_CHECK(clSetKernelArg(k_split, argi++, sizeof(float), &scale)); CL_CHECK(clSetKernelArg(k_split, argi++, sizeof(int), &n_q)); CL_CHECK(clSetKernelArg(k_split, argi++, sizeof(int), &n_kv)); CL_CHECK(clSetKernelArg(k_split, argi++, sizeof(int), &n_head)); CL_CHECK(clSetKernelArg(k_split, argi++, sizeof(cl_ulong), &q_nb1)); CL_CHECK(clSetKernelArg(k_split, argi++, sizeof(cl_ulong), &q_nb2)); CL_CHECK(clSetKernelArg(k_split, argi++, sizeof(cl_ulong), &q_nb3)); CL_CHECK(clSetKernelArg(k_split, argi++, sizeof(cl_ulong), &k_nb1)); CL_CHECK(clSetKernelArg(k_split, argi++, sizeof(cl_ulong), &k_nb2)); CL_CHECK(clSetKernelArg(k_split, argi++, sizeof(cl_ulong), &k_nb3)); CL_CHECK(clSetKernelArg(k_split, argi++, sizeof(cl_ulong), &v_nb1)); CL_CHECK(clSetKernelArg(k_split, argi++, sizeof(cl_ulong), &v_nb2)); CL_CHECK(clSetKernelArg(k_split, argi++, sizeof(cl_ulong), &v_nb3)); CL_CHECK(clSetKernelArg(k_split, argi++, sizeof(float), &max_bias)); CL_CHECK(clSetKernelArg(k_split, argi++, sizeof(float), &m0)); CL_CHECK(clSetKernelArg(k_split, argi++, sizeof(float), &m1)); CL_CHECK(clSetKernelArg(k_split, argi++, sizeof(int), &n_head_log2_val)); CL_CHECK(clSetKernelArg(k_split, argi++, sizeof(float), &logit_softcap)); CL_CHECK(clSetKernelArg(k_split, argi++, sizeof(int), &n_head_kv)); CL_CHECK(clSetKernelArg(k_split, argi++, sizeof(cl_mem), &mask_buffer)); CL_CHECK(clSetKernelArg(k_split, argi++, sizeof(cl_ulong), &offset_mask)); CL_CHECK(clSetKernelArg(k_split, argi++, sizeof(cl_ulong), &mask_nb1)); CL_CHECK(clSetKernelArg(k_split, argi++, sizeof(cl_ulong), &mask_nb2)); CL_CHECK(clSetKernelArg(k_split, argi++, sizeof(cl_ulong), &mask_nb3)); CL_CHECK(clSetKernelArg(k_split, argi++, sizeof(int), &mask_ne2)); CL_CHECK(clSetKernelArg(k_split, argi++, sizeof(int), &mask_ne3)); CL_CHECK(clSetKernelArg(k_split, argi++, sizeof(cl_mem), &temp_partial.data)); CL_CHECK(clSetKernelArg(k_split, argi++, sizeof(int), &n_splits)); CL_CHECK(clSetKernelArg(k_split, argi++, sizeof(int), &kv_per_split)); // MQ split kernel uses MQ_NSG_SPLIT subgroups and one WG per (kv_head, batch, split) // matches Q1_WG_SIZE * NSG (MQ_GQA=4 -> 256; MQ_GQA=8 -> 192) const size_t fd_wg = use_fd_mq ? fd_mq_wg : 64; const size_t fd_head_dim = use_fd_mq ? (size_t)(n_head_kv * n_batch) : (size_t)(n_head * n_batch); size_t fd_lws[3] = { fd_wg, 1, 1 }; // gid(2) packs q_idx * n_splits + split_idx. size_t fd_gws[3] = { fd_wg, fd_head_dim, (size_t)(n_splits * n_q) }; backend_ctx->enqueue_ndrange_kernel(k_split, 3, fd_gws, fd_lws, dst); cl_kernel k_merge = backend_ctx->fa.f32_merge.at(dk_dv); argi = 0; CL_CHECK(clSetKernelArg(k_merge, argi++, sizeof(cl_mem), &temp_partial.data)); CL_CHECK(clSetKernelArg(k_merge, argi++, sizeof(cl_mem), &extra_o->data_device)); CL_CHECK(clSetKernelArg(k_merge, argi++, sizeof(cl_ulong), &offset_o)); CL_CHECK(clSetKernelArg(k_merge, argi++, sizeof(int), &n_head)); CL_CHECK(clSetKernelArg(k_merge, argi++, sizeof(int), &n_splits)); CL_CHECK(clSetKernelArg(k_merge, argi++, sizeof(cl_ulong), &o_nb1)); CL_CHECK(clSetKernelArg(k_merge, argi++, sizeof(cl_ulong), &o_nb2)); CL_CHECK(clSetKernelArg(k_merge, argi++, sizeof(cl_ulong), &o_nb3)); CL_CHECK(clSetKernelArg(k_merge, argi++, sizeof(cl_mem), &sinks_buffer)); CL_CHECK(clSetKernelArg(k_merge, argi++, sizeof(cl_ulong), &offset_sinks)); CL_CHECK(clSetKernelArg(k_merge, argi++, sizeof(int), &n_q)); const size_t merge_wg = (size_t) (d_head_v / 4); // one lane per float4 size_t merge_lws[3] = { merge_wg, 1, 1 }; size_t merge_gws[3] = { merge_wg, (size_t)(n_head * n_batch), (size_t) n_q }; backend_ctx->enqueue_ndrange_kernel(k_merge, 3, merge_gws, merge_lws, dst); return; } cl_mem prefill_k_img = nullptr; if (use_prefill_k_img) { const size_t nb00_bytes = sizeof(uint16_t); const size_t k_bytes_span = (size_t)(n_kv > 0 ? n_kv - 1 : 0) * (size_t)k_nb1 + (size_t)(n_head_kv > 0 ? n_head_kv - 1 : 0) * (size_t)k_nb2 + (size_t)(n_batch > 0 ? n_batch - 1 : 0) * (size_t)k_nb3 + (size_t)d_head_q * nb00_bytes; const size_t k_bytes = (k_bytes_span + 7) & ~(size_t)7; const size_t k_pixels = k_bytes >> 3; if (k_pixels > 0 && k_pixels <= backend_ctx->image_max_buffer_size) { prefill_k_img = ggml_cl_img_pool_get_or_create( backend_ctx, backend_ctx->kq_img_pool, k_data_device, offset_k, k_bytes, CL_HALF_FLOAT); } if (prefill_k_img == nullptr) { kernel = backend_ctx->fa.f32_f16_split.at(dk_dv); use_prefill_k_img = false; } } CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra_q->data_device)); CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset_q)); CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), use_prefill_k_img ? &prefill_k_img : &k_data_device)); CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset_k)); CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &v_data_device)); CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offset_v)); CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_mem), &extra_o->data_device)); CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_ulong), &offset_o)); CL_CHECK(clSetKernelArg(kernel, 8, sizeof(float), &scale)); CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &n_q)); CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &n_kv)); CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &is_causal)); CL_CHECK(clSetKernelArg(kernel, 12, sizeof(int), &n_head)); CL_CHECK(clSetKernelArg(kernel, 13, sizeof(cl_ulong), &q_nb1)); CL_CHECK(clSetKernelArg(kernel, 14, sizeof(cl_ulong), &q_nb2)); CL_CHECK(clSetKernelArg(kernel, 15, sizeof(cl_ulong), &q_nb3)); CL_CHECK(clSetKernelArg(kernel, 16, sizeof(cl_ulong), &k_nb1)); CL_CHECK(clSetKernelArg(kernel, 17, sizeof(cl_ulong), &k_nb2)); CL_CHECK(clSetKernelArg(kernel, 18, sizeof(cl_ulong), &k_nb3)); CL_CHECK(clSetKernelArg(kernel, 19, sizeof(cl_ulong), &v_nb1)); CL_CHECK(clSetKernelArg(kernel, 20, sizeof(cl_ulong), &v_nb2)); CL_CHECK(clSetKernelArg(kernel, 21, sizeof(cl_ulong), &v_nb3)); CL_CHECK(clSetKernelArg(kernel, 22, sizeof(cl_ulong), &o_nb1)); CL_CHECK(clSetKernelArg(kernel, 23, sizeof(cl_ulong), &o_nb2)); CL_CHECK(clSetKernelArg(kernel, 24, sizeof(cl_ulong), &o_nb3)); CL_CHECK(clSetKernelArg(kernel, 25, sizeof(float), &max_bias)); CL_CHECK(clSetKernelArg(kernel, 26, sizeof(float), &m0)); CL_CHECK(clSetKernelArg(kernel, 27, sizeof(float), &m1)); CL_CHECK(clSetKernelArg(kernel, 28, sizeof(int), &n_head_log2_val)); CL_CHECK(clSetKernelArg(kernel, 29, sizeof(float), &logit_softcap)); CL_CHECK(clSetKernelArg(kernel, 30, sizeof(int), &n_head_kv)); CL_CHECK(clSetKernelArg(kernel, 31, sizeof(cl_mem), &mask_buffer)); CL_CHECK(clSetKernelArg(kernel, 32, sizeof(cl_ulong), &offset_mask)); CL_CHECK(clSetKernelArg(kernel, 33, sizeof(cl_ulong), &mask_nb1)); CL_CHECK(clSetKernelArg(kernel, 34, sizeof(cl_ulong), &mask_nb2)); CL_CHECK(clSetKernelArg(kernel, 35, sizeof(cl_ulong), &mask_nb3)); CL_CHECK(clSetKernelArg(kernel, 36, sizeof(int), &mask_ne2)); CL_CHECK(clSetKernelArg(kernel, 37, sizeof(int), &mask_ne3)); CL_CHECK(clSetKernelArg(kernel, 38, sizeof(cl_mem), &sinks_buffer)); CL_CHECK(clSetKernelArg(kernel, 39, sizeof(cl_ulong), &offset_sinks)); if (n_q > 1 && is_mixed) { CL_CHECK(clSetKernelArg(kernel, 40, sizeof(cl_mem), &k_pad_buffer)); CL_CHECK(clSetKernelArg(kernel, 41, sizeof(cl_mem), &v_pad_buffer)); CL_CHECK(clSetKernelArg(kernel, 42, sizeof(cl_mem), &mask_pad_buffer)); CL_CHECK(clSetKernelArg(kernel, 43, sizeof(cl_mem), &blk_buffer)); CL_CHECK(clSetKernelArg(kernel, 44, sizeof(int), &n_kv_blocks)); CL_CHECK(clSetKernelArg(kernel, 45, sizeof(cl_ulong), &mask_pad_nb1)); CL_CHECK(clSetKernelArg(kernel, 46, sizeof(cl_ulong), &mask_pad_nb2)); CL_CHECK(clSetKernelArg(kernel, 47, sizeof(cl_ulong), &mask_pad_nb3)); } else if (use_native_q8_0 || use_native_q4_0) { // arg 40 = blk classification buffer (NULL disables prepass opt). CL_CHECK(clSetKernelArg(kernel, 40, sizeof(cl_mem), &blk_buffer)); } if (n_q == 1) { if (use_local_tile) { const size_t lt_wg = 128; size_t local_work_size[] = { lt_wg, 1, 1 }; size_t global_work_size[] = { lt_wg, (size_t) n_head, (size_t) n_batch }; backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); } else { // q1_vec dispatches with NSG subgroups const size_t q1_wg = backend_ctx->gpu_family == INTEL ? 32 : 64; const size_t wg_size = use_q1_vec ? 256 : q1_wg; const size_t head_dim_global = use_q1_vec_mq ? (size_t)(n_head_kv * n_batch) : (size_t)(n_head * n_batch); size_t local_work_size[] = { wg_size, 1 }; size_t global_work_size[] = { wg_size, head_dim_global }; backend_ctx->enqueue_ndrange_kernel(kernel, 2, global_work_size, local_work_size, dst); } } else if (use_native_q8_0 || use_native_q4_0) { // Native quant prefill. The split variant may override BLOCK_M // (e.g. DK=96 quant uses BM=16). const bool use_split = use_native_q8_0 ? use_split_q8_0 : use_split_q4_0; int bm; size_t wg_size; if (use_split) { bm = use_native_q8_0 ? backend_ctx->fa.f32_q8_0_split_bm.at(dk_dv) : backend_ctx->fa.f32_q4_0_split_bm.at(dk_dv); wg_size = use_native_q8_0 ? backend_ctx->fa.f32_q8_0_split_wg_size.at(dk_dv) : backend_ctx->fa.f32_q4_0_split_wg_size.at(dk_dv); } else { bm = backend_ctx->fa.bm.at(dk_dv); wg_size = (size_t) bm; } size_t local_work_size[] = { wg_size, 1 }; size_t global_work_size[] = { (size_t)((n_q + bm - 1) / bm) * wg_size, (size_t)(n_head * n_batch) }; backend_ctx->enqueue_ndrange_kernel(kernel, 2, global_work_size, local_work_size, dst); } else { const size_t wg_size = (size_t) wg_size_fa; size_t local_work_size[] = { wg_size, 1 }; size_t global_work_size[] = { (size_t)((n_q + block_m - 1) / block_m) * wg_size, (size_t)(n_head * n_batch) }; backend_ctx->enqueue_ndrange_kernel(kernel, 2, global_work_size, local_work_size, dst); } } static void ggml_cl_mul_mat_f16_f32_tiled(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; ggml_tensor_extra_cl * extra0 = (ggml_tensor_extra_cl *)src0->extra; ggml_tensor_extra_cl * extra1 = (ggml_tensor_extra_cl *)src1->extra; ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; cl_ulong offset0 = extra0->offset + src0->view_offs; cl_ulong offset1 = extra1->offset + src1->view_offs; cl_ulong offsetd = extrad->offset + dst->view_offs; const int M = src0->ne[1]; const int N = src1->ne[1]; const int K = src0->ne[0]; cl_kernel kernel = backend_ctx->kernel_mul_mat_f16_f32_tiled; CL_CHECK(clSetKernelArg(kernel, 0, sizeof(int), &M)); CL_CHECK(clSetKernelArg(kernel, 1, sizeof(int), &N)); CL_CHECK(clSetKernelArg(kernel, 2, sizeof(int), &K)); CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra0->data_device)); CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_ulong), &offset0)); CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_mem), &extra1->data_device)); CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_ulong), &offset1)); CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_mem), &extrad->data_device)); CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_ulong), &offsetd)); // Tiling parameters. These need to be tuned for optimal performance. // They must match the #defines in the kernel mul_mat_f16_f32.cl. // // OPWM / OPWN: Output tile size per Work-Group. A work-group computes a tile of size OPWM x OPWN. // TPWM / TPWN: Threads per Work-group. This is the work-group size. // OPTM / OPTN: Output elements per Thread. Each thread computes OPTM x OPTN elements. // // The following relationships must hold: // OPWM = TPWM * OPTM // OPWN = TPWN * OPTN // const int OPWM = 64; const int OPWN = 64; const int TPWM = 16; const int TPWN = 8; size_t local_work_size[2] = { TPWM, TPWN }; size_t global_work_size[2] = { (size_t) ((M + OPWM - 1) / OPWM) * TPWM, (size_t) ((N + OPWN - 1) / OPWN) * TPWN, }; backend_ctx->enqueue_ndrange_kernel(kernel, 2, global_work_size, local_work_size, dst); } #ifdef GGML_OPENCL_USE_ADRENO_KERNELS static bool ggml_cl_can_use_adreno_xmem_gemm_f16_f32( const ggml_backend_opencl_context * backend_ctx, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * dst) { if (!backend_ctx->adreno_xmem_gemm_enabled) { return false; } if (backend_ctx->gpu_family != GPU_FAMILY::ADRENO) { return false; } if ((src0->type != GGML_TYPE_F16 && src0->type != GGML_TYPE_BF16) || src1->type != GGML_TYPE_F32 || dst->type != GGML_TYPE_F32) { return false; } if (!ggml_is_contiguous(src0) || !ggml_is_contiguous(src1) || !ggml_is_contiguous(dst)) { return false; } if (src0->ne[2] != 1 || src0->ne[3] != 1 || src1->ne[2] != 1 || src1->ne[3] != 1 || dst->ne[2] != 1 || dst->ne[3] != 1) { return false; } const int K = src0->ne[0]; const int M = src0->ne[1]; const int N = src1->ne[1]; if (src1->ne[0] != K || dst->ne[0] != M || dst->ne[1] != N) { return false; } if (N <= 1 || M < 64 || N < 16 || K < 64) { return false; } if ((K % 8) != 0) { return false; } const int kpack = K / 4; const int npack = CEIL_DIV(M, 4); if (static_cast(N) > backend_ctx->image2d_max_width || static_cast(kpack) > backend_ctx->image2d_max_height) { return false; } if (static_cast(N) > backend_ctx->image2d_max_width || static_cast(npack) > backend_ctx->image2d_max_height) { return false; } return true; } static void ggml_cl_mul_mat_f16_f32_adreno_xmem( ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { ggml_backend_opencl_context * backend_ctx = (ggml_backend_opencl_context *)backend->context; ggml_tensor_extra_cl * extra0 = (ggml_tensor_extra_cl *)src0->extra; ggml_tensor_extra_cl * extra1 = (ggml_tensor_extra_cl *)src1->extra; ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; const cl_ulong offset0 = extra0->offset + src0->view_offs; const cl_ulong offset1 = extra1->offset + src1->view_offs; const cl_ulong offsetd = extrad->offset + dst->view_offs; const int K = src0->ne[0]; const int M = src0->ne[1]; const int N = src1->ne[1]; const int kpack = K / 4; const int npack = CEIL_DIV(M, 4); const int os = 8; const size_t xmem_bytes = 6144; const size_t weight_bytes = static_cast(kpack) * static_cast(npack) * 4u * sizeof(cl_half4); backend_ctx->prealloc_adreno_xmem_const.allocate(backend_ctx->context, xmem_bytes); cl_int err = CL_SUCCESS; cl_image_format fmt = {}; fmt.image_channel_order = CL_RGBA; fmt.image_channel_data_type = CL_HALF_FLOAT; cl_image_desc desc_src = {}; desc_src.image_type = CL_MEM_OBJECT_IMAGE2D; desc_src.image_width = static_cast(N); desc_src.image_height = static_cast(kpack); cl_mem src_img = clCreateImage(backend_ctx->context, CL_MEM_READ_WRITE, &fmt, &desc_src, nullptr, &err); CL_CHECK(err); cl_image_desc desc_dst = {}; desc_dst.image_type = CL_MEM_OBJECT_IMAGE2D; desc_dst.image_width = static_cast(N); desc_dst.image_height = static_cast(npack); cl_mem dst_img = clCreateImage(backend_ctx->context, CL_MEM_READ_WRITE, &fmt, &desc_dst, nullptr, &err); CL_CHECK(err); cl_mem weights = clCreateBuffer(backend_ctx->context, CL_MEM_READ_WRITE, weight_bytes, nullptr, &err); CL_CHECK(err); cl_kernel prepack = backend_ctx->kernel_adreno_xmem_prepack_weight_f16; CL_CHECK(clSetKernelArg(prepack, 0, sizeof(cl_mem), &weights)); CL_CHECK(clSetKernelArg(prepack, 1, sizeof(cl_mem), &extra0->data_device)); CL_CHECK(clSetKernelArg(prepack, 2, sizeof(cl_ulong), &offset0)); CL_CHECK(clSetKernelArg(prepack, 3, sizeof(int), &K)); CL_CHECK(clSetKernelArg(prepack, 4, sizeof(int), &M)); CL_CHECK(clSetKernelArg(prepack, 5, sizeof(int), &kpack)); CL_CHECK(clSetKernelArg(prepack, 6, sizeof(int), &npack)); CL_CHECK(clSetKernelArg(prepack, 7, sizeof(int), &os)); size_t lws = 256; size_t max_wg = backend_ctx->get_kernel_workgroup_size(prepack); if (lws > max_wg) { lws = max_wg; } size_t gws = CEIL_DIV(static_cast(kpack) * static_cast(npack), lws) * lws; backend_ctx->enqueue_ndrange_kernel(prepack, 1, &gws, &lws, dst); cl_kernel pack_src = backend_ctx->kernel_adreno_xmem_pack_src_f32; CL_CHECK(clSetKernelArg(pack_src, 0, sizeof(cl_mem), &extra1->data_device)); CL_CHECK(clSetKernelArg(pack_src, 1, sizeof(cl_ulong), &offset1)); CL_CHECK(clSetKernelArg(pack_src, 2, sizeof(cl_mem), &src_img)); CL_CHECK(clSetKernelArg(pack_src, 3, sizeof(int), &K)); CL_CHECK(clSetKernelArg(pack_src, 4, sizeof(int), &N)); size_t pack_src_lws[2] = { 16, 16 }; size_t pack_src_gws[2] = { CEIL_DIV(static_cast(N), pack_src_lws[0])*pack_src_lws[0], CEIL_DIV(static_cast(kpack), pack_src_lws[1])*pack_src_lws[1] }; backend_ctx->enqueue_ndrange_kernel(pack_src, 2, pack_src_gws, pack_src_lws, dst); cl_kernel gemm = backend_ctx->kernel_gemm_xmem_f16_f32_os8; CL_CHECK(clSetKernelArg(gemm, 0, sizeof(cl_mem), &weights)); CL_CHECK(clSetKernelArg(gemm, 1, sizeof(cl_mem), &backend_ctx->prealloc_adreno_xmem_const.buffer)); CL_CHECK(clSetKernelArg(gemm, 2, sizeof(cl_mem), &src_img)); CL_CHECK(clSetKernelArg(gemm, 3, sizeof(cl_mem), &dst_img)); CL_CHECK(clSetKernelArg(gemm, 4, sizeof(int), &N)); CL_CHECK(clSetKernelArg(gemm, 5, sizeof(int), &npack)); CL_CHECK(clSetKernelArg(gemm, 6, sizeof(int), &kpack)); const size_t z_values = CEIL_DIV(static_cast(npack), static_cast(os)); size_t gemm_lws[3] = { 64, 1, 1 }; size_t gemm_gws[3] = { z_values*gemm_lws[0], CEIL_DIV(static_cast(N), gemm_lws[0]), 1 }; backend_ctx->enqueue_ndrange_kernel(gemm, 3, gemm_gws, gemm_lws, dst); cl_kernel store_dst = backend_ctx->kernel_adreno_xmem_store_dst_f32; CL_CHECK(clSetKernelArg(store_dst, 0, sizeof(cl_mem), &dst_img)); CL_CHECK(clSetKernelArg(store_dst, 1, sizeof(cl_mem), &extrad->data_device)); CL_CHECK(clSetKernelArg(store_dst, 2, sizeof(cl_ulong), &offsetd)); CL_CHECK(clSetKernelArg(store_dst, 3, sizeof(int), &M)); CL_CHECK(clSetKernelArg(store_dst, 4, sizeof(int), &N)); size_t store_lws[2] = { 16, 16 }; size_t store_gws[2] = { CEIL_DIV(static_cast(N), store_lws[0])*store_lws[0], CEIL_DIV(static_cast(npack), store_lws[1])*store_lws[1] }; backend_ctx->enqueue_ndrange_kernel(store_dst, 2, store_gws, store_lws, dst); CL_CHECK(clReleaseMemObject(weights)); CL_CHECK(clReleaseMemObject(dst_img)); CL_CHECK(clReleaseMemObject(src_img)); } #endif // GGML_OPENCL_USE_ADRENO_KERNELS static void ggml_cl_conv_2d(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { GGML_TENSOR_BINARY_OP_LOCALS; ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; ggml_tensor_extra_cl * extra0 = (ggml_tensor_extra_cl *)src0->extra; ggml_tensor_extra_cl * extra1 = (ggml_tensor_extra_cl *)src1->extra; ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; cl_ulong offset0 = extra0->offset + src0->view_offs; cl_ulong offset1 = extra1->offset + src1->view_offs; cl_ulong offsetd = extrad->offset + dst->view_offs; const cl_uint Cout = ne03; const cl_uint Cin = ne02; const cl_uint N = ne13; const cl_uint KW = ne00; const cl_uint KH = ne01; const cl_uint W = ne10; const cl_uint H = ne11; const cl_uint OW = ne0; const cl_uint OH = ne1; const cl_uint s0 = dst->op_params[0]; const cl_uint s1 = dst->op_params[1]; const cl_uint p0 = dst->op_params[2]; const cl_uint p1 = dst->op_params[3]; const cl_uint d0 = dst->op_params[4]; const cl_uint d1 = dst->op_params[5]; const cl_uint cl_nb00 = nb00/ggml_type_size(src0->type); const cl_uint cl_nb01 = nb01/ggml_type_size(src0->type); const cl_uint cl_nb02 = nb02/ggml_type_size(src0->type); const cl_uint cl_nb03 = nb03/ggml_type_size(src0->type); const cl_uint cl_nb10 = nb10/ggml_type_size(src1->type); const cl_uint cl_nb11 = nb11/ggml_type_size(src1->type); const cl_uint cl_nb12 = nb12/ggml_type_size(src1->type); const cl_uint cl_nb13 = nb13/ggml_type_size(src1->type); const cl_uint cl_nb1 = nb1/ggml_type_size(dst->type); const cl_uint cl_nb2 = nb2/ggml_type_size(dst->type); const cl_uint cl_nb3 = nb3/ggml_type_size(dst->type); const int64_t NPQ = (int64_t)N * OW * OH; const uint32_t BS_K = 64; const uint32_t BS_NPQ = 64; const uint32_t BS_CRS = 16; const uint32_t VEC_SIZE = 4; const uint32_t TS_K = 4; const uint32_t TS_NPQ = 8; const uint32_t WG_K = BS_K / TS_K; const uint32_t WG_NPQ = BS_NPQ / TS_NPQ; auto splitWork = [](uint32_t work_size, uint32_t block_size) { return (block_size + work_size - 1) / block_size; }; const uint32_t NB_K = splitWork(Cout, BS_K); const uint32_t NB_NPQ = splitWork(NPQ, BS_NPQ); cl_kernel kernel; size_t shmem_size; if (src0->type == GGML_TYPE_F16 && src1->type == GGML_TYPE_F16) { kernel = backend_ctx->kernel_conv_2d_f16; shmem_size = (size_t)(BS_K * BS_CRS * sizeof(cl_half) + BS_CRS * (BS_NPQ / VEC_SIZE) * sizeof(cl_half4)); } else if (src0->type == GGML_TYPE_F32 && src1->type == GGML_TYPE_F32) { kernel = backend_ctx->kernel_conv_2d_f32; shmem_size = (size_t)(BS_K * BS_CRS * sizeof(cl_float) + BS_CRS * (BS_NPQ / VEC_SIZE) * sizeof(cl_float4)); } else if (src0->type == GGML_TYPE_F16 && src1->type == GGML_TYPE_F32) { kernel = backend_ctx->kernel_conv_2d_f16_f32; shmem_size = (size_t)(BS_K * BS_CRS * sizeof(cl_half) + BS_CRS * (BS_NPQ / VEC_SIZE) * sizeof(cl_float4)); } else { GGML_ASSERT(false && "Unsupported data type combination for conv2d"); } cl_uint idx = 0; CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_mem), &extra0->data_device)); CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_ulong), &offset0)); CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_mem), &extra1->data_device)); CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_ulong), &offset1)); CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_mem), &extrad->data_device)); CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_ulong), &offsetd)); CL_CHECK(clSetKernelArg(kernel, idx++, shmem_size, NULL)); CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_uint), &Cout)); CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_uint), &Cin)); CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_uint), &N)); CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_uint), &KW)); CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_uint), &KH)); CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_uint), &W)); CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_uint), &H)); CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_uint), &OW)); CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_uint), &OH)); CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_uint), &s0)); CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_uint), &s1)); CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_uint), &p0)); CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_uint), &p1)); CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_uint), &d0)); CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_uint), &d1)); CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_uint), &cl_nb00)); CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_uint), &cl_nb01)); CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_uint), &cl_nb02)); CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_uint), &cl_nb03)); CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_uint), &cl_nb10)); CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_uint), &cl_nb11)); CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_uint), &cl_nb12)); CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_uint), &cl_nb13)); CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_uint), &cl_nb1)); CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_uint), &cl_nb2)); CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_uint), &cl_nb3)); size_t global_work_size[] = { (size_t)NB_K * WG_K, (size_t)NB_NPQ * WG_NPQ, 1 }; size_t local_work_size[] = { (size_t)WG_K, (size_t)WG_NPQ, 1 }; backend_ctx->enqueue_ndrange_kernel(kernel, 2, global_work_size, local_work_size, dst); } // is_kq selects which of the two products this call is, and it is decided by the // CALLER -- the two admission arms in ggml_cl_mul_mat, each of which knows which // one it matched. It used to be re-derived here from nb01 > nb02, i.e. "K is // head-major, V^T is not". That discriminator COLLAPSES at n_head_kv == 1, where // the two strides are equal because there is only one head to order, so nothing // here could tell a KQ from a KQV. Pass it in rather than infer it. static void ggml_cl_mul_mat_kq_kqv_adreno(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst, bool is_kq) { ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; ggml_tensor_extra_cl * extra0 = (ggml_tensor_extra_cl *)src0->extra; ggml_tensor_extra_cl * extra1 = (ggml_tensor_extra_cl *)src1->extra; ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; const int ne00 = src0->ne[0]; const int ne01 = src0->ne[1]; const int ne02 = src0->ne[2]; const cl_ulong nb01 = src0->nb[1]; const cl_ulong nb02 = src0->nb[2]; const int ne10 = src1->ne[0]; const int ne11 = src1->ne[1]; const int ne12 = src1->ne[2]; const cl_ulong nb10 = src1->nb[0]; const int ne0 = dst->ne[0]; const int ne1 = dst->ne[1]; GGML_ASSERT(ne00 == ne10); cl_kernel kernel; cl_context context = backend_ctx->context; cl_int status; cl_image_format img_fmt_1d; cl_image_desc img_desc_1d; cl_buffer_region region; cl_mem A_image1d; cl_mem A_sub_buffer; cl_mem B_sub_buffer; cl_mem D_image1d; cl_mem D_sub_buffer; int M = ne01; int N = ne1; int K = ne00; kernel = is_kq ? backend_ctx->kernel_mul_mm_f16_f32_kq : backend_ctx->kernel_mul_mm_f16_f32_kqv; // create sub-buffer for A // <--------------------------------------------> // extra0 = src0->view_src ? (ggml_tensor_extra_cl *)src0->view_src->extra : (ggml_tensor_extra_cl *)src0->extra; region.origin = (extra0->offset + src0->view_offs); if (is_kq) { // KQ region.size = nb01 * ne01; } else { // KQV region.size = nb02 * ne02; } A_sub_buffer = clCreateSubBuffer((extra0->data_device), 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &status); CL_CHECK(status); // <--------------------------------------------> // // create sub-buffer for B // <--------------------------------------------> // region.origin = (extra1->offset + src1->view_offs); region.size = nb10 * ne10 * ne11 * ne12; B_sub_buffer = clCreateSubBuffer((extra1->data_device), 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &status); CL_CHECK(status); // <--------------------------------------------> // img_fmt_1d = {CL_RGBA, CL_FLOAT}; memset(&img_desc_1d, 0, sizeof(img_desc_1d)); img_desc_1d.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; if (is_kq) { img_desc_1d.image_width = (nb01 * ne01 / 4)/4; } else { img_desc_1d.image_width = (nb02 * ne02 / 4)/4; } img_desc_1d.buffer = A_sub_buffer; A_image1d = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt_1d, &img_desc_1d, NULL, &status); CL_CHECK(status); // create sub-buffer for output C // <--------------------------------------------> // region.origin = (extrad->offset + dst->view_offs); region.size = ne0 * ne1 * dst->ne[2] * dst->nb[0]; // size of C in bytes D_sub_buffer = clCreateSubBuffer((extrad->data_device), 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &status); CL_CHECK(status); // <--------------------------------------------> // // create image for C output // <--------------------------------------------> // img_fmt_1d = {CL_R, CL_FLOAT}; memset(&img_desc_1d, 0, sizeof(img_desc_1d)); img_desc_1d.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; img_desc_1d.image_width = ne0 * ne1 * dst->ne[2] * dst->nb[0] / 4; img_desc_1d.buffer = D_sub_buffer; D_image1d = clCreateImage(context, CL_MEM_WRITE_ONLY, &img_fmt_1d, &img_desc_1d, NULL, &status); CL_CHECK(status); // <--------------------------------------------> // int offset_src0 = 0; int offset_src1 = 0; // set kernel args // <--------------------------------------------> // cl_uint k_arg = 0; CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &A_image1d)); CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &offset_src0)); CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &B_sub_buffer)); CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &offset_src1)); CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &D_image1d)); CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &extrad->offset)); CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &M)); CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &K)); CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &N)); CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &ne02)); CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &ne12)); CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &nb01)); size_t global_work_size[3] = {64, static_cast(((M+63)/64)), static_cast(((N+31)/32)*ne12)}; size_t local_work_size[3] = {64, 1, 2}; backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); // deallocate sub buffers and images // <--------------------------------------------> // CL_CHECK(clReleaseMemObject(A_image1d)); CL_CHECK(clReleaseMemObject(D_image1d)); CL_CHECK(clReleaseMemObject(A_sub_buffer)); CL_CHECK(clReleaseMemObject(B_sub_buffer)); CL_CHECK(clReleaseMemObject(D_sub_buffer)); } static void ggml_cl_mul_mat_q1_0_f32_adreno(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { #ifdef GGML_OPENCL_USE_ADRENO_KERNELS GGML_ASSERT(src0); GGML_ASSERT(src0->extra); GGML_ASSERT(src1); GGML_ASSERT(src1->extra); GGML_ASSERT(dst); GGML_ASSERT(dst->extra); GGML_ASSERT(src0->type == GGML_TYPE_Q1_0); GGML_ASSERT(src1->type == GGML_TYPE_F32); ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; ggml_tensor_extra_cl * extra1 = (ggml_tensor_extra_cl *)src1->extra; ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; ggml_tensor_extra_cl_q1_0 * extra0_q1_0 = (ggml_tensor_extra_cl_q1_0 *)src0->extra; cl_ulong offset1 = extra1->offset + src1->view_offs; cl_ulong offsetd = extrad->offset + dst->view_offs; GGML_ASSERT(src1->view_offs == 0); GGML_ASSERT(dst->view_offs == 0); const int ne00 = src0->ne[0]; const int ne01 = src0->ne[1]; const int ne02 = src0->ne[2]; const int ne10 = src1->ne[0]; const int ne12 = src1->ne[2]; const int ne0 = dst->ne[0]; const int ne1 = dst->ne[1]; GGML_ASSERT(ne00 == ne10); GGML_ASSERT((ne00 % 128) == 0); GGML_ASSERT(ne0 == ne01); cl_context context = backend_ctx->context; cl_kernel kernel; cl_int err; cl_image_format img_fmt; cl_image_desc img_desc; cl_buffer_region region; int M = ne01; int N = ne1; int K = ne00; if (ne1 == 1) { cl_mem q_img = nullptr; cl_mem b_sub_buf = nullptr; cl_mem b_img = nullptr; // image for q (uint32: each texel packs 32 sign bits) img_fmt = { CL_R, CL_UNSIGNED_INT32}; memset(&img_desc, 0, sizeof(img_desc)); img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; img_desc.image_width = M * K / 32; img_desc.buffer = extra0_q1_0->q; CL_CHECK((q_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err), err)); // create a sub_buffer for B region.origin = offset1; region.size = K * N * sizeof(float); CL_CHECK((b_sub_buf = clCreateSubBuffer((extra1->data_device), 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); // image for activations img_fmt = {CL_RGBA, CL_FLOAT}; memset(&img_desc, 0, sizeof(img_desc)); img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; img_desc.image_width = K * N / 4; img_desc.buffer = b_sub_buf; CL_CHECK((b_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err), err)); kernel = backend_ctx->kernel_gemv_noshuffle_q1_0_f32; int r2 = 1; int r3 = 1; CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &q_img)); CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q1_0->d)); CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &b_img)); CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &extra1->offset)); CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device)); CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &extrad->offset)); CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne00)); CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne01)); CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne02)); CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne10)); CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &ne12)); CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne0)); CL_CHECK(clSetKernelArg(kernel, 12, sizeof(int), &ne1)); CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &r2)); CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &r3)); size_t wavesize = backend_ctx->adreno_wave_size; size_t local_work_size[] = { wavesize, 4, 1 }; size_t global_work_size[] = { CEIL_DIV(M, wavesize)*wavesize, 4, 1 }; backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); CL_CHECK(clReleaseMemObject(q_img)); CL_CHECK(clReleaseMemObject(b_img)); CL_CHECK(clReleaseMemObject(b_sub_buf)); } else { cl_mem b_sub_buf = nullptr; cl_mem b_sub_buf_trans = nullptr; cl_mem b_img = nullptr; cl_mem b_img_trans = nullptr; // subbuffer for activations region.origin = offset1; region.size = K * N * sizeof(float); CL_CHECK((b_sub_buf = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); // image for activations img_fmt = {CL_RGBA, CL_FLOAT}; memset(&img_desc, 0, sizeof(img_desc)); img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; img_desc.image_width = K * N / 4; img_desc.buffer = b_sub_buf; CL_CHECK((b_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err), err)); // pad N to multiple of 8 int extra_elements = N % 8; int padding = 0; if (extra_elements > 0){ padding = 8 - extra_elements; } // subbuffer for transposed activations region.origin = 0; region.size = K * (N + padding) * sizeof(float)/2; backend_ctx->prealloc_act_trans.allocate(context, region.size); CL_CHECK((b_sub_buf_trans = clCreateSubBuffer(backend_ctx->prealloc_act_trans.buffer, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); // image for transposed activations img_fmt = {CL_RGBA, CL_HALF_FLOAT}; memset(&img_desc, 0, sizeof(img_desc)); img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; img_desc.image_width = K * (N + padding) / 4; img_desc.buffer = b_sub_buf_trans; CL_CHECK((b_img_trans = clCreateImage(context, 0, &img_fmt, &img_desc, NULL, &err), err)); // transpose activations int height_B = N/4; if (height_B == 0) { height_B = 1; } int width_B = K/4; int padded_height_B = (N + padding)/4; kernel = backend_ctx->kernel_transpose_32_16; CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &b_img)); CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &b_img_trans)); CL_CHECK(clSetKernelArg(kernel, 2, sizeof(int), &height_B)); CL_CHECK(clSetKernelArg(kernel, 3, sizeof(int), &width_B)); CL_CHECK(clSetKernelArg(kernel, 4, sizeof(int), &padded_height_B)); size_t local_work_size_t[2] = { 1, 16 }; size_t global_work_size_t[2] = { (size_t)width_B, (size_t)padded_height_B }; backend_ctx->enqueue_ndrange_kernel(kernel, 2, global_work_size_t, local_work_size_t, dst); // gemm kernel = backend_ctx->kernel_gemm_noshuffle_q1_0_f32; int padded_N = N + padding; CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_q1_0->q)); CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q1_0->d)); CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &b_img_trans)); CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extrad->data_device)); CL_CHECK(clSetKernelArg(kernel, 4, sizeof(int), &K)); CL_CHECK(clSetKernelArg(kernel, 5, sizeof(int), &M)); CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &padded_N)); CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &N)); CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_ulong), &offsetd)); size_t global_work_size[] = { (size_t)CEIL_DIV(N, 8), (size_t)CEIL_DIV(M, 4), 1 }; size_t local_work_size[] = { 2, 128, 1 }; backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); CL_CHECK(clReleaseMemObject(b_img_trans)); CL_CHECK(clReleaseMemObject(b_sub_buf_trans)); CL_CHECK(clReleaseMemObject(b_img)); CL_CHECK(clReleaseMemObject(b_sub_buf)); } #else GGML_UNUSED(backend); GGML_UNUSED(src0); GGML_UNUSED(src1); GGML_UNUSED(dst); #endif } #ifdef GGML_OPENCL_USE_ADRENO_KERNELS static void ggml_cl_mul_mat_q4_0_f32_adreno_ila(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { GGML_ASSERT(src0); GGML_ASSERT(src0->extra); GGML_ASSERT(src1); GGML_ASSERT(src1->extra); GGML_ASSERT(dst); GGML_ASSERT(dst->extra); ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; ggml_tensor_extra_cl * extra1 = (ggml_tensor_extra_cl *)src1->extra; ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; ggml_tensor_extra_cl_q4_0 * extra0_q4_0 = (ggml_tensor_extra_cl_q4_0 *)src0->extra; cl_ulong offset1 = extra1->offset + src1->view_offs; cl_ulong offsetd = extrad->offset + dst->view_offs; const int ne00 = src0->ne[0]; const int ne01 = src0->ne[1]; const int ne1 = dst->ne[1]; GGML_ASSERT(ne00 % ggml_blck_size(src0->type) == 0); cl_context context = backend_ctx->context; cl_kernel kernel; cl_int err; cl_image_format img_fmt; cl_image_desc img_desc; cl_buffer_region region; int M = ne01; int N = ne1; int K = ne00; if (ne1 == 1) { cl_mem b_sub_buf = nullptr; cl_mem b_img = nullptr; region.origin = offset1; region.size = (size_t)K * N * sizeof(float); CL_CHECK((b_sub_buf = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); img_fmt = { CL_RGBA, CL_FLOAT }; memset(&img_desc, 0, sizeof(img_desc)); img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; img_desc.image_width = (size_t)K * N / 4; img_desc.buffer = b_sub_buf; CL_CHECK((b_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err), err)); kernel = backend_ctx->kernel_gemv_noshuffle_q4_0_f32_32b_trans; CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_q4_0->q_img)); CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q4_0->d)); CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &b_img)); CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extrad->data_device)); CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_ulong), &offsetd)); CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_int), &K)); CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_int), &M)); size_t wavesize = backend_ctx->adreno_wave_size; size_t local_work_size[3] = { wavesize, 4, 1 }; size_t global_work_size[3] = { (size_t)CEIL_DIV(M, 64) * 64, 4, 1 }; backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); CL_CHECK(clReleaseMemObject(b_sub_buf)); CL_CHECK(clReleaseMemObject(b_img)); } else { const int gemm_tile_n = 64; int N_pad = (N + gemm_tile_n - 1) & ~(gemm_tile_n - 1); cl_mem a_img = extra0_q4_0->q_img; cl_mem s_img = extra0_q4_0->d_img; GGML_ASSERT(a_img && s_img && "ILA Q4_0 weight images missing; set_tensor should have built them"); static const char * q4_0_bin_dp4a_env = getenv("GGML_OPENCL_Q4_0_BIN_DP4A"); bool q4_0_bin_dp4a_on = q4_0_bin_dp4a_env ? (atoi(q4_0_bin_dp4a_env) != 0) : true; // dot prod has to be available q4_0_bin_dp4a_on = backend_ctx->has_integer_dot && q4_0_bin_dp4a_on; if (q4_0_bin_dp4a_on && backend_ctx->kernel_gemm_noshuffle_q4_0_q8_1_dp4a_ila_a8_bin) { const int dp4a_N_pad = CEIL_DIV(N, 32) * 32; const size_t n_blocks = (size_t)dp4a_N_pad * (K / 32); backend_ctx->prealloc_moe_qa.allocate(context, (size_t)dp4a_N_pad * K * sizeof(cl_char)); backend_ctx->prealloc_moe_da.allocate(context, n_blocks * sizeof(cl_half)); backend_ctx->prealloc_moe_sa.allocate(context, n_blocks * sizeof(cl_half)); cl_mem b_sub = nullptr; region.origin = offset1; region.size = (size_t)K * N * sizeof(float); CL_CHECK((b_sub = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); cl_int tb = (cl_int)((size_t)N * (K / 32)); cl_kernel qk = backend_ctx->kernel_quant_a_q8_1; CL_CHECK(clSetKernelArg(qk, 0, sizeof(cl_mem), &b_sub)); CL_CHECK(clSetKernelArg(qk, 1, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer)); CL_CHECK(clSetKernelArg(qk, 2, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer)); CL_CHECK(clSetKernelArg(qk, 3, sizeof(cl_mem), &backend_ctx->prealloc_moe_sa.buffer)); CL_CHECK(clSetKernelArg(qk, 4, sizeof(cl_int), &tb)); size_t q_local[1] = { 64 }; size_t q_global[1] = { (size_t)CEIL_DIV(tb, 64) * 64 }; backend_ctx->enqueue_ndrange_kernel(qk, 1, q_global, q_local, dst); cl_mem d_sub = nullptr; cl_mem d_img = nullptr; region.origin = offsetd; region.size = (size_t)M * N * sizeof(float); CL_CHECK((d_sub = clCreateSubBuffer(extrad->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); img_fmt = { CL_R, CL_FLOAT }; memset(&img_desc, 0, sizeof(img_desc)); img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; img_desc.image_width = (size_t)M * N; img_desc.buffer = d_sub; CL_CHECK((d_img = clCreateImage(context, CL_MEM_WRITE_ONLY, &img_fmt, &img_desc, NULL, &err), err)); kernel = backend_ctx->kernel_gemm_noshuffle_q4_0_q8_1_dp4a_ila_a8_bin; cl_uint k_arg = 0; CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &a_img)); CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &extra0_q4_0->d)); CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer)); CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer)); CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &d_img)); CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &K)); CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &M)); CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &N)); size_t local_work_size[3] = { 64, 1, 1 }; size_t global_work_size[3] = { 64, (size_t)(M / 64), (size_t)(dp4a_N_pad / 32) }; backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); CL_CHECK(clReleaseMemObject(b_sub)); CL_CHECK(clReleaseMemObject(d_img)); CL_CHECK(clReleaseMemObject(d_sub)); return; } // Pad B through a zero-filled scratch buffer when N needs // padding, since the GEMM kernel always reads a full N-tile. const bool need_pad = N_pad > N; cl_mem b_sub_buf = nullptr; cl_mem b_padded = nullptr; if (need_pad) { CL_CHECK((b_padded = clCreateBuffer(context, CL_MEM_READ_WRITE, (size_t)K * N_pad * sizeof(float), NULL, &err), err)); const float zero = 0.0f; CL_CHECK(clEnqueueFillBuffer(backend_ctx->queue, b_padded, &zero, sizeof(zero), 0, (size_t)K * N_pad * sizeof(float), 0, NULL, NULL)); CL_CHECK(clEnqueueCopyBuffer(backend_ctx->queue, extra1->data_device, b_padded, offset1, 0, (size_t)K * N * sizeof(float), 0, NULL, NULL)); } else { region.origin = offset1; region.size = (size_t)K * N * sizeof(float); CL_CHECK((b_sub_buf = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); } img_fmt = { CL_R, CL_FLOAT }; memset(&img_desc, 0, sizeof(img_desc)); img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; img_desc.image_width = need_pad ? (size_t)K * N_pad : (size_t)K * N; img_desc.buffer = need_pad ? b_padded : b_sub_buf; cl_mem b_img; CL_CHECK((b_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err), err)); region.origin = offsetd; region.size = (size_t)M * N * sizeof(float); cl_mem d_sub_buf; CL_CHECK((d_sub_buf = clCreateSubBuffer(extrad->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); img_fmt = { CL_R, CL_FLOAT }; memset(&img_desc, 0, sizeof(img_desc)); img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; img_desc.image_width = (size_t)M * N; img_desc.buffer = d_sub_buf; cl_mem d_img; CL_CHECK((d_img = clCreateImage(context, CL_MEM_WRITE_ONLY, &img_fmt, &img_desc, NULL, &err), err)); int line_stride_matrix_A_in_bytes = M * 4; int line_stride_matrix_S_in_bytes = M * 2; int line_stride_matrix_B_in_bytes = K * 4; int line_stride_matrix_C_in_bytes = M * 4; int c_offset_for_kernel = 0; int b_offset_for_kernel = 0; kernel = backend_ctx->kernel_gemm_noshuffle_q4_0_f32_32b_trans_ila_a8_bin; cl_uint k_arg = 0; CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &a_img)); CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &s_img)); CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &b_img)); CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &b_offset_for_kernel)); CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &d_img)); CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &c_offset_for_kernel)); CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &K)); CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &line_stride_matrix_A_in_bytes)); CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &line_stride_matrix_S_in_bytes)); CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &line_stride_matrix_B_in_bytes)); CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &line_stride_matrix_C_in_bytes)); CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &M)); CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &N)); size_t local_work_size[3] = { 64, 2, 2 }; size_t m_tiles = (size_t)CEIL_DIV(M, 64); size_t global_work_size[3] = { 64, m_tiles, (size_t)CEIL_DIV(N_pad, gemm_tile_n) }; backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); CL_CHECK(clReleaseMemObject(b_img)); if (b_sub_buf) { CL_CHECK(clReleaseMemObject(b_sub_buf)); } if (b_padded) { CL_CHECK(clReleaseMemObject(b_padded)); } CL_CHECK(clReleaseMemObject(d_img)); CL_CHECK(clReleaseMemObject(d_sub_buf)); } } #endif // GGML_OPENCL_USE_ADRENO_KERNELS static void ggml_cl_mul_mat_q4_0_f32_adreno(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { #ifdef GGML_OPENCL_USE_ADRENO_KERNELS GGML_ASSERT(src0); GGML_ASSERT(src0->extra); GGML_ASSERT(src1); GGML_ASSERT(src1->extra); GGML_ASSERT(dst); GGML_ASSERT(dst->extra); ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; ggml_tensor_extra_cl * extra1 = (ggml_tensor_extra_cl *)src1->extra; ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; ggml_tensor_extra_cl_q4_0 * extra0_q4_0 = (ggml_tensor_extra_cl_q4_0 *)src0->extra; cl_ulong offset1 = extra1->offset + src1->view_offs; cl_ulong offsetd = extrad->offset + dst->view_offs; const int ne00 = src0->ne[0]; const int ne01 = src0->ne[1]; const int ne02 = src0->ne[2]; const int ne10 = src1->ne[0]; const int ne12 = src1->ne[2]; const int ne0 = dst->ne[0]; const int ne1 = dst->ne[1]; GGML_ASSERT(ne00 % ggml_blck_size(src0->type) == 0); cl_context context = backend_ctx->context; cl_kernel kernel; cl_int err; cl_image_format img_fmt; cl_image_desc img_desc; cl_buffer_region region; int M = ne01; int N = ne1; int K = ne00; // Multi-column (N=3) verify GEMV for q4_0: route the spec/MTP verify batch // (ne1==3) onto the efficient GEMV path instead of the transposed-GEMM dead- // zone (gemm_noshuffle_q4_0 is ~50% of MTP decode on a Q4_0 model since q4_0 // weights have no cok/mc3, unlike q4_K). Reuses the ne1==1 GEMV image setup // (activation image already sized by N=ne1). Byte-identical. Opt-in via // GGML_OPENCL_Q40_MC3=1. Per-layer only (ne01 < 32768); q4_0 lm_head doesn't // occur (token_embd/output stay Q6_K), guard kept for parity with q4_K mc3. static const bool q40_mc3 = (getenv("GGML_OPENCL_Q40_MC3") != nullptr); const bool use_q40_mc3 = q40_mc3 && (ne1 >= 2 && ne1 <= 4) && (ne01 < 32768); const bool use_bin = use_q4_0_bin_kernels(backend_ctx, src0); if (use_bin) { if (use_q40_mc3) { static bool warned = false; if (!warned) { GGML_LOG_WARN("ggml_opencl: GGML_OPENCL_Q40_MC3 is bypassed by Q4_0 binary kernels\n"); warned = true; } } ggml_cl_mul_mat_q4_0_f32_adreno_ila(backend, src0, src1, dst); return; } if (ne1 == 1 || use_q40_mc3) { cl_mem q_img = nullptr; cl_mem b_sub_buf = nullptr; cl_mem b_img = nullptr; // image for q img_fmt = { CL_R, CL_UNSIGNED_INT32}; memset(&img_desc, 0, sizeof(img_desc)); img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; img_desc.image_width = M * K / 2 / 4; img_desc.buffer = extra0_q4_0->q; CL_CHECK((q_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err), err)); // subbuffer for activations region.origin = offset1; region.size = K * N * sizeof(float); CL_CHECK((b_sub_buf = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); // image for activations img_fmt = {CL_RGBA, CL_FLOAT}; memset(&img_desc, 0, sizeof(img_desc)); img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; img_desc.image_width = K * N / 4; img_desc.buffer = b_sub_buf; CL_CHECK((b_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err), err)); if (use_q40_mc3) { kernel = backend_ctx->kernel_gemv_noshuffle_q4_0_f32_mc3; CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &q_img)); CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q4_0->d)); CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &b_img)); CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extrad->data_device)); CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_ulong), &offsetd)); CL_CHECK(clSetKernelArg(kernel, 5, sizeof(int), &ne00)); CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne01)); CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne1)); } else { kernel = backend_ctx->kernel_gemv_noshuffle_q4_0_f32; if (M == 4096 && K == 4096) { kernel = backend_ctx->kernel_gemv_noshuffle_q4_0_f32_4096_1_4096; } else if (M == 4096 && K == 11008) { kernel = backend_ctx->kernel_gemv_noshuffle_q4_0_f32_4096_1_11008; } else if (M == 11008 && K == 4096) { kernel = backend_ctx->kernel_gemv_noshuffle_q4_0_f32_11008_1_4096; } else if (M == 32000 && K == 4096) { kernel = backend_ctx->kernel_gemv_noshuffle_q4_0_f32_32000_1_4096; } int r2 = 1; int r3 = 1; CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &q_img)); CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q4_0->d)); CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &b_img)); CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1)); CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device)); CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd)); CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne00)); CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne01)); CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne02)); CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne10)); CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &ne12)); CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne0)); CL_CHECK(clSetKernelArg(kernel, 12, sizeof(int), &ne1)); CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &r2)); CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &r3)); } // Small-M mc3 verify is occupancy/latency-bound (too few WGs at small M, so // its bandwidth falls well short of the FFN matmuls'). Use 8 subgroups (512-WI WGs, half the // per-lane K-walk) for small M. Layout stride is fixed (4 uints/block), so only // the K-split count changes; the mc3 kernel reads it via get_local_size(1). The // ne1==1 base kernel hardcodes N_SIMDGROUP=4, so it always stays at 4. const int mc3_nsg = (use_q40_mc3 && ne01 < 4096) ? 8 : 4; size_t local_work_size[3] = {64, (size_t)mc3_nsg, 1}; size_t global_work_size[3] = {(size_t)CEIL_DIV(ne01/2, 64)*64, (size_t)mc3_nsg, 1}; backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); CL_CHECK(clReleaseMemObject(q_img)); CL_CHECK(clReleaseMemObject(b_sub_buf)); CL_CHECK(clReleaseMemObject(b_img)); } else { // dp4a (int8) dense prefill GEMM, default off static const char * q4_0_dense_dp4a_env = getenv("GGML_OPENCL_Q4_0_DENSE_DP4A"); bool q4_0_dense_dp4a_on = q4_0_dense_dp4a_env ? (atoi(q4_0_dense_dp4a_env) != 0) : false; // dot prod has to be available q4_0_dense_dp4a_on = backend_ctx->has_integer_dot && q4_0_dense_dp4a_on; if (q4_0_dense_dp4a_on && backend_ctx->kernel_gemm_noshuffle_q4_0_q8_1_dp4a && N > 8 && (K % 32 == 0) && (M % 64 == 0)) { cl_mem a_sub = nullptr; region.origin = offset1; region.size = (size_t)K * N * sizeof(float); CL_CHECK((a_sub = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); const size_t n_blocks = (size_t)N * (K / 32); backend_ctx->prealloc_moe_qa.allocate(context, (size_t)N * K * sizeof(cl_char)); backend_ctx->prealloc_moe_da.allocate(context, n_blocks * sizeof(cl_half)); backend_ctx->prealloc_moe_sa.allocate(context, n_blocks * sizeof(cl_half)); cl_int tb = (cl_int)n_blocks; cl_kernel qk = backend_ctx->kernel_quant_a_q8_1; CL_CHECK(clSetKernelArg(qk, 0, sizeof(cl_mem), &a_sub)); CL_CHECK(clSetKernelArg(qk, 1, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer)); CL_CHECK(clSetKernelArg(qk, 2, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer)); CL_CHECK(clSetKernelArg(qk, 3, sizeof(cl_mem), &backend_ctx->prealloc_moe_sa.buffer)); CL_CHECK(clSetKernelArg(qk, 4, sizeof(cl_int), &tb)); size_t q_local[1] = { 64 }; size_t q_global[1] = { (size_t)(((n_blocks + 63) / 64) * 64) }; backend_ctx->enqueue_ndrange_kernel(qk, 1, q_global, q_local, dst); cl_kernel dk = backend_ctx->kernel_gemm_noshuffle_q4_0_q8_1_dp4a; int ai = 0; CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &extra0_q4_0->q)); CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &extra0_q4_0->d)); CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer)); CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer)); CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &backend_ctx->prealloc_moe_sa.buffer)); CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &extrad->data_device)); CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_ulong), &offsetd)); CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_int), &M)); CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_int), &N)); CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_int), &K)); size_t d_local[3] = { 64, 1, 1 }; size_t d_global[3] = { 64, (size_t)(M / 64), (size_t)CEIL_DIV(N, 32) }; backend_ctx->enqueue_ndrange_kernel(dk, 3, d_global, d_local, dst); CL_CHECK(clReleaseMemObject(a_sub)); return; } cl_mem b_sub_buf = nullptr; cl_mem b_sub_buf_trans = nullptr; cl_mem b_img = nullptr; cl_mem b_img_trans = nullptr; cl_mem d_sub_buf = nullptr; // subbuffer for activations region.origin = offset1; region.size = K * N * sizeof(float); CL_CHECK((b_sub_buf = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); // image for activations img_fmt = {CL_RGBA, CL_FLOAT}; memset(&img_desc, 0, sizeof(img_desc)); img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; img_desc.image_width = K * N / 4; img_desc.buffer = b_sub_buf; CL_CHECK((b_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err), err)); // pad N to multiple of 8 int extra_elements = N % 8; int padding = 0; if (extra_elements > 0){ padding = 8 - extra_elements; } // subbuffer for transposed activations region.origin = 0; region.size = K * (N + padding) * sizeof(float)/2; backend_ctx->prealloc_act_trans.allocate(context, region.size); CL_CHECK((b_sub_buf_trans = clCreateSubBuffer(backend_ctx->prealloc_act_trans.buffer, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); // image for transposed activations img_fmt = {CL_RGBA, CL_HALF_FLOAT}; memset(&img_desc, 0, sizeof(img_desc)); img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; img_desc.image_width = K * (N + padding) / 4; img_desc.buffer = b_sub_buf_trans; CL_CHECK((b_img_trans = clCreateImage(context, 0, &img_fmt, &img_desc, NULL, &err), err)); // subbuffer for output region.origin = extrad->offset; // Specify the starting offset (in bytes) region.size = M * N * sizeof(float); // Specify the size of the sub-buffer CL_CHECK((d_sub_buf = clCreateSubBuffer(extrad->data_device, CL_MEM_WRITE_ONLY, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); // transpose activations int height_B = N/4; if (height_B == 0) { height_B = 1; } int width_B = K/4; int padded_height_B = (N + padding)/4; kernel = backend_ctx->kernel_transpose_32_16; CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &b_img)); CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &b_img_trans)); CL_CHECK(clSetKernelArg(kernel, 2, sizeof(int), &height_B)); CL_CHECK(clSetKernelArg(kernel, 3, sizeof(int), &width_B)); CL_CHECK(clSetKernelArg(kernel, 4, sizeof(int), &padded_height_B)); size_t local_work_size_t[2] = { 1, 16 }; size_t global_work_size_t[2] = { (size_t)width_B, (size_t)padded_height_B }; if (ne0 == 4096 && ne1 == 128 && ne10 == 4096) { local_work_size_t[0]=4; local_work_size_t[1]=8; } else if (ne0 == 11008 && ne1 == 128 && ne10 == 4096) { local_work_size_t[0]=2; local_work_size_t[1]=8; } else if(ne0 == 4096 && ne1 == 128 && ne10 == 11008) { local_work_size_t[0]=1; local_work_size_t[1]=8; } else if(ne0 == 32000 && ne1 == 128 && ne10 == 4096) { local_work_size_t[0]=2; local_work_size_t[1]=8; } backend_ctx->enqueue_ndrange_kernel(kernel, 2, global_work_size_t, local_work_size_t, dst); // gemm kernel = backend_ctx->kernel_gemm_noshuffle_q4_0_f32; int padded_N = N + padding; CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_q4_0->q)); CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q4_0->d)); CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &b_img_trans)); CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &d_sub_buf)); CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_int), &ne01)); CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_int), &padded_N)); CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_int), &ne00)); CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_int), &ne1)); size_t global_work_size[3] = {(size_t)CEIL_DIV(ne1, 8), (size_t)CEIL_DIV(ne01, 4), 1}; size_t local_work_size[3] = {1, 128, 1}; if (ne0 == 4096 && ne1 == 128 && ne10 == 4096) { local_work_size[0] = 1; local_work_size[1] = 128; } else if (ne0 == 11008 && ne1 == 128 && ne10 == 4096) { local_work_size[0] = 2; local_work_size[1] = 64; } else if (ne0 == 4096 && ne1 == 128 && ne10 == 11008) { local_work_size[0] = 2; local_work_size[1] = 64; } else if (ne0 == 32000 && ne1 == 128 && ne10 == 4096) { local_work_size[0] = 2; local_work_size[1] = 64; } backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); CL_CHECK(clReleaseMemObject(b_sub_buf)); CL_CHECK(clReleaseMemObject(b_sub_buf_trans)); CL_CHECK(clReleaseMemObject(b_img)); CL_CHECK(clReleaseMemObject(b_img_trans)); CL_CHECK(clReleaseMemObject(d_sub_buf)); } #else GGML_UNUSED(backend); GGML_UNUSED(src0); GGML_UNUSED(src1); GGML_UNUSED(dst); #endif } static void ggml_cl_mul_mat_q4_1_f32_adreno(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { #ifdef GGML_OPENCL_USE_ADRENO_KERNELS GGML_ASSERT(src0); GGML_ASSERT(src0->extra); GGML_ASSERT(src1); GGML_ASSERT(src1->extra); GGML_ASSERT(dst); GGML_ASSERT(dst->extra); ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; ggml_tensor_extra_cl * extra1 = (ggml_tensor_extra_cl *)src1->extra; ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; ggml_tensor_extra_cl_q4_1 * extra0_q4_1 = (ggml_tensor_extra_cl_q4_1 *)src0->extra; cl_ulong offset1 = extra1->offset + src1->view_offs; cl_ulong offsetd = extrad->offset + dst->view_offs; const int ne00 = src0->ne[0]; const int ne01 = src0->ne[1]; const int ne1 = dst->ne[1]; GGML_ASSERT(ne00 % ggml_blck_size(src0->type) == 0); cl_context context = backend_ctx->context; cl_kernel kernel; cl_int err; cl_image_format img_fmt; cl_image_desc img_desc; cl_buffer_region region; int M = ne01; int N = ne1; int K = ne00; // Multi-column (N=3) verify GEMV for q4_1: route the spec/MTP verify batch // (ne1==3) onto the efficient GEMV path instead of the transposed-GEMM dead- // zone (gemm_noshuffle_q4_1). Reuses the ne1==1 GEMV image setup. Opt-in via // GGML_OPENCL_Q41_MC3=1. Per-layer only (ne01 < 32768). static const bool q41_mc3 = (getenv("GGML_OPENCL_Q41_MC3") != nullptr); const bool use_q41_mc3 = q41_mc3 && (ne1 >= 2 && ne1 <= 4) && (ne01 < 32768); if (ne1 == 1 || use_q41_mc3) { cl_mem q_img = nullptr; cl_mem b_sub_buf = nullptr; cl_mem b_img = nullptr; // image for q img_fmt = { CL_R, CL_UNSIGNED_INT32}; memset(&img_desc, 0, sizeof(img_desc)); img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; img_desc.image_width = M * K / 2 / 4; img_desc.buffer = extra0_q4_1->q; CL_CHECK((q_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err), err)); // subbuffer for activations region.origin = offset1; region.size = K * N * sizeof(float); CL_CHECK((b_sub_buf = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); // image for activations img_fmt = {CL_RGBA, CL_FLOAT}; memset(&img_desc, 0, sizeof(img_desc)); img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; img_desc.image_width = K * N / 4; img_desc.buffer = b_sub_buf; CL_CHECK((b_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err), err)); kernel = use_q41_mc3 ? backend_ctx->kernel_gemv_noshuffle_q4_1_f32_mc3 : backend_ctx->kernel_gemv_noshuffle_q4_1_f32; CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &q_img)); CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q4_1->d)); CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra0_q4_1->m)); CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &b_img)); CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device)); CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd)); CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_int), &ne00)); CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_int), &ne01)); if (use_q41_mc3) { CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_int), &ne1)); // n_cols } size_t local_work_size[3] = {64, 4, 1}; size_t global_work_size[3] = {(size_t)CEIL_DIV(ne01/2, 64)*64, 4, 1}; backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); CL_CHECK(clReleaseMemObject(q_img)); CL_CHECK(clReleaseMemObject(b_sub_buf)); CL_CHECK(clReleaseMemObject(b_img)); } else { cl_mem b_sub_buf = nullptr; cl_mem b_sub_buf_trans = nullptr; cl_mem b_img = nullptr; cl_mem b_img_trans = nullptr; // subbuffer for activations region.origin = offset1; region.size = K * N * sizeof(float); CL_CHECK((b_sub_buf = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); // image for activations img_fmt = {CL_RGBA, CL_FLOAT}; memset(&img_desc, 0, sizeof(img_desc)); img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; img_desc.image_width = K * N / 4; img_desc.buffer = b_sub_buf; CL_CHECK((b_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err), err)); // pad N to multiple of 8 int extra_elements = N % 8; int padding = 0; if (extra_elements > 0){ padding = 8 - extra_elements; } // subbuffer for transposed activations region.origin = 0; region.size = K * (N + padding) * sizeof(float)/2; backend_ctx->prealloc_act_trans.allocate(context, region.size); CL_CHECK((b_sub_buf_trans = clCreateSubBuffer(backend_ctx->prealloc_act_trans.buffer, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); // image for transposed activations img_fmt = {CL_RGBA, CL_HALF_FLOAT}; memset(&img_desc, 0, sizeof(img_desc)); img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; img_desc.image_width = K * (N + padding) / 4; img_desc.buffer = b_sub_buf_trans; CL_CHECK((b_img_trans = clCreateImage(context, 0, &img_fmt, &img_desc, NULL, &err), err)); // transpose activations int height_B = N/4; if (height_B == 0) { height_B = 1; } int width_B = K/4; int padded_height_B = (N + padding)/4; kernel = backend_ctx->kernel_transpose_32_16; CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &b_img)); CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &b_img_trans)); CL_CHECK(clSetKernelArg(kernel, 2, sizeof(int), &height_B)); CL_CHECK(clSetKernelArg(kernel, 3, sizeof(int), &width_B)); CL_CHECK(clSetKernelArg(kernel, 4, sizeof(int), &padded_height_B)); size_t local_work_size_t[2] = { 1, 16 }; size_t global_work_size_t[2] = { (size_t)width_B, (size_t)padded_height_B }; backend_ctx->enqueue_ndrange_kernel(kernel, 2, global_work_size_t, local_work_size_t, dst); // gemm kernel = backend_ctx->kernel_gemm_noshuffle_q4_1_f32; int padded_N = N + padding; CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_q4_1->q)); CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q4_1->d)); CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra0_q4_1->m)); CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &b_img_trans)); CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device)); CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd)); CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_int), &ne01)); CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_int), &padded_N)); CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_int), &ne00)); CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_int), &ne1)); size_t global_work_size[3] = {(size_t)CEIL_DIV(ne1, 8), (size_t)CEIL_DIV(ne01, 4), 1}; size_t local_work_size[3] = {1, 128, 1}; backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); CL_CHECK(clReleaseMemObject(b_sub_buf)); CL_CHECK(clReleaseMemObject(b_sub_buf_trans)); CL_CHECK(clReleaseMemObject(b_img)); CL_CHECK(clReleaseMemObject(b_img_trans)); } #else GGML_UNUSED(backend); GGML_UNUSED(src0); GGML_UNUSED(src1); GGML_UNUSED(dst); #endif } static void ggml_cl_mul_mat_q5_0_f32_adreno(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { #ifdef GGML_OPENCL_USE_ADRENO_KERNELS GGML_ASSERT(src0); GGML_ASSERT(src0->extra); GGML_ASSERT(src1); GGML_ASSERT(src1->extra); GGML_ASSERT(dst); GGML_ASSERT(dst->extra); ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; ggml_tensor_extra_cl * extra1 = (ggml_tensor_extra_cl *)src1->extra; ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; ggml_tensor_extra_cl_q5_0 * extra0_q5_0 = (ggml_tensor_extra_cl_q5_0 *)src0->extra; cl_ulong offset1 = extra1->offset + src1->view_offs; cl_ulong offsetd = extrad->offset + dst->view_offs; const int ne00 = src0->ne[0]; const int ne01 = src0->ne[1]; const int ne1 = dst->ne[1]; GGML_ASSERT(ne00 % ggml_blck_size(src0->type) == 0); cl_context context = backend_ctx->context; cl_kernel kernel; cl_int err; cl_image_format img_fmt; cl_image_desc img_desc; cl_buffer_region region; int M = ne01; int N = ne1; int K = ne00; if (ne1 == 1) { cl_mem qs_img = nullptr; cl_mem b_sub_buf = nullptr; cl_mem b_img = nullptr; // image for qs img_fmt = { CL_R, CL_UNSIGNED_INT32 }; memset(&img_desc, 0, sizeof(img_desc)); img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; img_desc.image_width = M * K / 2 / 4; img_desc.buffer = extra0_q5_0->qs; CL_CHECK((qs_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err), err)); // subbuffer for activations region.origin = offset1; region.size = K * N * sizeof(float); CL_CHECK((b_sub_buf = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); // image for activations img_fmt = {CL_RGBA, CL_FLOAT}; memset(&img_desc, 0, sizeof(img_desc)); img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; img_desc.image_width = K * N / 4; img_desc.buffer = b_sub_buf; CL_CHECK((b_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err), err)); kernel = backend_ctx->kernel_gemv_noshuffle_q5_0_f32; CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &qs_img)); CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q5_0->qh)); CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra0_q5_0->d)); CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &b_img)); CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device)); CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd)); CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_int), &ne00)); CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_int), &ne01)); size_t local_work_size[3] = {64, 4, 1}; size_t global_work_size[3] = {(size_t)CEIL_DIV(ne01/2, 64)*64, 4, 1}; backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); CL_CHECK(clReleaseMemObject(qs_img)); CL_CHECK(clReleaseMemObject(b_sub_buf)); CL_CHECK(clReleaseMemObject(b_img)); } else { // dp4a (int8) dense q5_0 prefill GEMM, default off static const char * q5_dense_dp4a_env = getenv("GGML_OPENCL_Q5_DENSE_DP4A"); static const char * q5_dense_wimg_env = getenv("GGML_OPENCL_Q5_DENSE_DP4A_WIMG"); const bool q5_dense_wimg_on = q5_dense_wimg_env && (atoi(q5_dense_wimg_env) != 0); bool q5_dense_dp4a_on = q5_dense_wimg_on ? true : (q5_dense_dp4a_env && (atoi(q5_dense_dp4a_env) != 0)); // dot prod has to be available q5_dense_dp4a_on = backend_ctx->has_integer_dot && q5_dense_dp4a_on; if (q5_dense_dp4a_on && backend_ctx->kernel_gemm_noshuffle_q5_0_q8_1_dp4a && N > 8 && (K % 32 == 0) && (M % 64 == 0)) { cl_mem a_sub = nullptr; region.origin = offset1; region.size = (size_t)K * N * sizeof(float); CL_CHECK((a_sub = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); const size_t n_blocks = (size_t)N * (K / 32); backend_ctx->prealloc_moe_qa.allocate(context, (size_t)N * K * sizeof(cl_char)); backend_ctx->prealloc_moe_da.allocate(context, n_blocks * sizeof(cl_half)); backend_ctx->prealloc_moe_sa.allocate(context, n_blocks * sizeof(cl_half)); cl_int tb = (cl_int)n_blocks; cl_kernel qk = backend_ctx->kernel_quant_a_q8_1; CL_CHECK(clSetKernelArg(qk, 0, sizeof(cl_mem), &a_sub)); CL_CHECK(clSetKernelArg(qk, 1, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer)); CL_CHECK(clSetKernelArg(qk, 2, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer)); CL_CHECK(clSetKernelArg(qk, 3, sizeof(cl_mem), &backend_ctx->prealloc_moe_sa.buffer)); CL_CHECK(clSetKernelArg(qk, 4, sizeof(cl_int), &tb)); size_t q_local[1] = { 64 }; size_t q_global[1] = { (size_t)(((n_blocks + 63) / 64) * 64) }; backend_ctx->enqueue_ndrange_kernel(qk, 1, q_global, q_local, dst); // optional qs texture (image1d_buffer over the nibble plane; the same // CL_R/UINT32 view, width M*K/8, the GEMV path builds). cl_mem q5_qs_img = nullptr; bool use_wimg = q5_dense_wimg_on; if (use_wimg) { const size_t tex = (size_t)M * (size_t)K / 8; // uint32 texels (2 ushorts/texel) if (tex == 0 || tex > backend_ctx->image_max_buffer_size) { use_wimg = false; } else { img_fmt = { CL_R, CL_UNSIGNED_INT32 }; memset(&img_desc, 0, sizeof(img_desc)); img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; img_desc.image_width = tex; img_desc.buffer = extra0_q5_0->qs; q5_qs_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err); if (err != CL_SUCCESS || q5_qs_img == nullptr) { use_wimg = false; q5_qs_img = nullptr; } } } cl_kernel dk = use_wimg ? backend_ctx->kernel_gemm_noshuffle_q5_0_q8_1_dp4a_wimg : backend_ctx->kernel_gemm_noshuffle_q5_0_q8_1_dp4a; int ai = 0; if (use_wimg) { CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &q5_qs_img)); } else { CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &extra0_q5_0->qs)); } CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &extra0_q5_0->qh)); CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &extra0_q5_0->d)); CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer)); CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer)); CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &backend_ctx->prealloc_moe_sa.buffer)); CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &extrad->data_device)); CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_ulong), &offsetd)); CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_int), &M)); CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_int), &N)); CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_int), &K)); size_t d_local[3] = { 64, 1, 1 }; size_t d_global[3] = { 64, (size_t)(M / 64), (size_t)CEIL_DIV(N, 32) }; backend_ctx->enqueue_ndrange_kernel(dk, 3, d_global, d_local, dst); if (q5_qs_img != nullptr) { CL_CHECK(clReleaseMemObject(q5_qs_img)); } CL_CHECK(clReleaseMemObject(a_sub)); return; } cl_mem b_sub_buf = nullptr; cl_mem b_sub_buf_trans = nullptr; cl_mem b_img = nullptr; cl_mem b_img_trans = nullptr; cl_mem d_sub_buf = nullptr; // subbuffer for activations region.origin = offset1; region.size = K * N * sizeof(float); CL_CHECK((b_sub_buf = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); // image for activations img_fmt = {CL_RGBA, CL_FLOAT}; memset(&img_desc, 0, sizeof(img_desc)); img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; img_desc.image_width = K * N / 4; img_desc.buffer = b_sub_buf; CL_CHECK((b_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err), err)); // pad N to multiple of 8 int extra_elements = N % 8; int padding = 0; if (extra_elements > 0){ padding = 8 - extra_elements; } // subbuffer for transposed activations region.origin = 0; region.size = K * (N + padding) * sizeof(float)/2; backend_ctx->prealloc_act_trans.allocate(context, region.size); CL_CHECK((b_sub_buf_trans = clCreateSubBuffer(backend_ctx->prealloc_act_trans.buffer, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); // image for transposed activations img_fmt = {CL_RGBA, CL_HALF_FLOAT}; memset(&img_desc, 0, sizeof(img_desc)); img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; img_desc.image_width = K * (N + padding) / 4; img_desc.buffer = b_sub_buf_trans; CL_CHECK((b_img_trans = clCreateImage(context, 0, &img_fmt, &img_desc, NULL, &err), err)); // subbuffer for output region.origin = extrad->offset; region.size = M * N * sizeof(float); CL_CHECK((d_sub_buf = clCreateSubBuffer(extrad->data_device, CL_MEM_WRITE_ONLY, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); // transpose activations int height_B = N/4; if (height_B == 0) { height_B = 1; } int width_B = K/4; int padded_height_B = (N + padding)/4; kernel = backend_ctx->kernel_transpose_32_16; CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &b_img)); CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &b_img_trans)); CL_CHECK(clSetKernelArg(kernel, 2, sizeof(int), &height_B)); CL_CHECK(clSetKernelArg(kernel, 3, sizeof(int), &width_B)); CL_CHECK(clSetKernelArg(kernel, 4, sizeof(int), &padded_height_B)); size_t local_work_size_t[2] = { 1, 16 }; size_t global_work_size_t[2] = { (size_t)width_B, (size_t)padded_height_B }; backend_ctx->enqueue_ndrange_kernel(kernel, 2, global_work_size_t, local_work_size_t, dst); // gemm kernel = backend_ctx->kernel_gemm_noshuffle_q5_0_f32; int padded_N = N + padding; CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_q5_0->qs)); CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q5_0->qh)); CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra0_q5_0->d)); CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &b_img_trans)); CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &d_sub_buf)); CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_int), &ne01)); CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_int), &padded_N)); CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_int), &ne00)); CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_int), &ne1)); size_t global_work_size[3] = {(size_t)CEIL_DIV(ne1, 8), (size_t)CEIL_DIV(ne01, 4), 1}; size_t local_work_size[3] = {1, 128, 1}; backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); CL_CHECK(clReleaseMemObject(b_sub_buf)); CL_CHECK(clReleaseMemObject(b_sub_buf_trans)); CL_CHECK(clReleaseMemObject(b_img)); CL_CHECK(clReleaseMemObject(b_img_trans)); CL_CHECK(clReleaseMemObject(d_sub_buf)); } #else GGML_UNUSED(backend); GGML_UNUSED(src0); GGML_UNUSED(src1); GGML_UNUSED(dst); #endif } static void ggml_cl_mul_mat_q5_1_f32_adreno(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { #ifdef GGML_OPENCL_USE_ADRENO_KERNELS GGML_ASSERT(src0); GGML_ASSERT(src0->extra); GGML_ASSERT(src1); GGML_ASSERT(src1->extra); GGML_ASSERT(dst); GGML_ASSERT(dst->extra); ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; ggml_tensor_extra_cl * extra1 = (ggml_tensor_extra_cl *)src1->extra; ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; ggml_tensor_extra_cl_q5_1 * extra0_q5_1 = (ggml_tensor_extra_cl_q5_1 *)src0->extra; cl_ulong offset1 = extra1->offset + src1->view_offs; cl_ulong offsetd = extrad->offset + dst->view_offs; const int ne00 = src0->ne[0]; const int ne01 = src0->ne[1]; const int ne1 = dst->ne[1]; GGML_ASSERT(ne00 % ggml_blck_size(src0->type) == 0); cl_context context = backend_ctx->context; cl_kernel kernel; cl_int err; cl_image_format img_fmt; cl_image_desc img_desc; cl_buffer_region region; int M = ne01; int N = ne1; int K = ne00; if (ne1 == 1) { cl_mem qs_img = nullptr; cl_mem b_sub_buf = nullptr; cl_mem b_img = nullptr; // image for qs img_fmt = { CL_R, CL_UNSIGNED_INT32 }; memset(&img_desc, 0, sizeof(img_desc)); img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; img_desc.image_width = M * K / 2 / 4; img_desc.buffer = extra0_q5_1->qs; CL_CHECK((qs_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err), err)); // subbuffer for activations region.origin = offset1; region.size = K * N * sizeof(float); CL_CHECK((b_sub_buf = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); // image for activations img_fmt = {CL_RGBA, CL_FLOAT}; memset(&img_desc, 0, sizeof(img_desc)); img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; img_desc.image_width = K * N / 4; img_desc.buffer = b_sub_buf; CL_CHECK((b_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err), err)); kernel = backend_ctx->kernel_gemv_noshuffle_q5_1_f32; CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &qs_img)); CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q5_1->qh)); CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra0_q5_1->d)); CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra0_q5_1->m)); CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &b_img)); CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_mem), &extrad->data_device)); CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_ulong), &offsetd)); CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_int), &ne00)); CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_int), &ne01)); size_t local_work_size[3] = {64, 4, 1}; size_t global_work_size[3] = {(size_t)CEIL_DIV(ne01/2, 64)*64, 4, 1}; backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); CL_CHECK(clReleaseMemObject(qs_img)); CL_CHECK(clReleaseMemObject(b_sub_buf)); CL_CHECK(clReleaseMemObject(b_img)); } else { cl_mem b_sub_buf = nullptr; cl_mem b_sub_buf_trans = nullptr; cl_mem b_img = nullptr; cl_mem b_img_trans = nullptr; cl_mem d_sub_buf = nullptr; // subbuffer for activations region.origin = offset1; region.size = K * N * sizeof(float); CL_CHECK((b_sub_buf = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); // image for activations img_fmt = {CL_RGBA, CL_FLOAT}; memset(&img_desc, 0, sizeof(img_desc)); img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; img_desc.image_width = K * N / 4; img_desc.buffer = b_sub_buf; CL_CHECK((b_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err), err)); // pad N to multiple of 8 int extra_elements = N % 8; int padding = 0; if (extra_elements > 0){ padding = 8 - extra_elements; } // subbuffer for transposed activations region.origin = 0; region.size = K * (N + padding) * sizeof(float)/2; backend_ctx->prealloc_act_trans.allocate(context, region.size); CL_CHECK((b_sub_buf_trans = clCreateSubBuffer(backend_ctx->prealloc_act_trans.buffer, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); // image for transposed activations img_fmt = {CL_RGBA, CL_HALF_FLOAT}; memset(&img_desc, 0, sizeof(img_desc)); img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; img_desc.image_width = K * (N + padding) / 4; img_desc.buffer = b_sub_buf_trans; CL_CHECK((b_img_trans = clCreateImage(context, 0, &img_fmt, &img_desc, NULL, &err), err)); // subbuffer for output region.origin = extrad->offset; region.size = M * N * sizeof(float); CL_CHECK((d_sub_buf = clCreateSubBuffer(extrad->data_device, CL_MEM_WRITE_ONLY, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); // transpose activations int height_B = N/4; if (height_B == 0) { height_B = 1; } int width_B = K/4; int padded_height_B = (N + padding)/4; kernel = backend_ctx->kernel_transpose_32_16; CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &b_img)); CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &b_img_trans)); CL_CHECK(clSetKernelArg(kernel, 2, sizeof(int), &height_B)); CL_CHECK(clSetKernelArg(kernel, 3, sizeof(int), &width_B)); CL_CHECK(clSetKernelArg(kernel, 4, sizeof(int), &padded_height_B)); size_t local_work_size_t[2] = { 1, 16 }; size_t global_work_size_t[2] = { (size_t)width_B, (size_t)padded_height_B }; backend_ctx->enqueue_ndrange_kernel(kernel, 2, global_work_size_t, local_work_size_t, dst); // gemm kernel = backend_ctx->kernel_gemm_noshuffle_q5_1_f32; int padded_N = N + padding; CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_q5_1->qs)); CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q5_1->qh)); CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra0_q5_1->d)); CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra0_q5_1->m)); CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &b_img_trans)); CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_mem), &d_sub_buf)); CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_int), &ne01)); CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_int), &padded_N)); CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_int), &ne00)); CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_int), &ne1)); size_t global_work_size[3] = {(size_t)CEIL_DIV(ne1, 8), (size_t)CEIL_DIV(ne01, 4), 1}; size_t local_work_size[3] = {1, 128, 1}; backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); CL_CHECK(clReleaseMemObject(b_sub_buf)); CL_CHECK(clReleaseMemObject(b_sub_buf_trans)); CL_CHECK(clReleaseMemObject(b_img)); CL_CHECK(clReleaseMemObject(b_img_trans)); CL_CHECK(clReleaseMemObject(d_sub_buf)); } #else GGML_UNUSED(backend); GGML_UNUSED(src0); GGML_UNUSED(src1); GGML_UNUSED(dst); #endif } static void ggml_cl_mul_mat_iq4_nl_f32_adreno(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { #ifdef GGML_OPENCL_USE_ADRENO_KERNELS GGML_ASSERT(src0); GGML_ASSERT(src0->extra); GGML_ASSERT(src1); GGML_ASSERT(src1->extra); GGML_ASSERT(dst); GGML_ASSERT(dst->extra); ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; ggml_tensor_extra_cl * extra1 = (ggml_tensor_extra_cl *)src1->extra; ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; ggml_tensor_extra_cl_iq4_nl * extra0_iq4_nl = (ggml_tensor_extra_cl_iq4_nl *)src0->extra; cl_ulong offset1 = extra1->offset + src1->view_offs; cl_ulong offsetd = extrad->offset + dst->view_offs; const int ne00 = src0->ne[0]; const int ne01 = src0->ne[1]; const int ne1 = dst->ne[1]; GGML_ASSERT(ne00 % 32 == 0); cl_context context = backend_ctx->context; cl_kernel kernel; cl_int err; cl_image_format img_fmt; cl_image_desc img_desc; cl_buffer_region region; int M = ne01; int N = ne1; int K = ne00; if (ne1 == 1) { cl_mem q_img = nullptr; cl_mem b_sub_buf = nullptr; cl_mem b_img = nullptr; // image for q img_fmt = { CL_R, CL_UNSIGNED_INT32}; memset(&img_desc, 0, sizeof(img_desc)); img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; img_desc.image_width = M * K / 2 / 4; img_desc.buffer = extra0_iq4_nl->q; CL_CHECK((q_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err), err)); // subbuffer for activations region.origin = offset1; region.size = K * N * sizeof(float); CL_CHECK((b_sub_buf = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); // image for activations img_fmt = {CL_RGBA, CL_FLOAT}; memset(&img_desc, 0, sizeof(img_desc)); img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; img_desc.image_width = K * N / 4; img_desc.buffer = b_sub_buf; CL_CHECK((b_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err), err)); kernel = backend_ctx->kernel_gemv_noshuffle_iq4_nl_f32; CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &q_img)); CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_iq4_nl->d)); CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &b_img)); CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extrad->data_device)); CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_ulong), &offsetd)); CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_int), &ne00)); CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_int), &ne01)); size_t local_work_size[3] = {64, 4, 1}; size_t global_work_size[3] = {(size_t)CEIL_DIV(ne01/2, 64)*64, 4, 1}; backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); CL_CHECK(clReleaseMemObject(q_img)); CL_CHECK(clReleaseMemObject(b_sub_buf)); CL_CHECK(clReleaseMemObject(b_img)); } else { // dp4a (int8) dense IQ4_NL prefill GEMM. Quantizes the [N,K] activations to // q8_1 and runs the int8 dot instead of the f16 half-dot. Large-batch // (ne1>8) only static const char * iq4nl_dense_dp4a_env = getenv("GGML_OPENCL_IQ4NL_DENSE_DP4A"); bool iq4nl_dense_dp4a_on = iq4nl_dense_dp4a_env ? (atoi(iq4nl_dense_dp4a_env) != 0) : (backend_ctx->adreno_gen == ADRENO_GPU_GEN::X2E); // dot prod has to be available iq4nl_dense_dp4a_on = backend_ctx->has_integer_dot && iq4nl_dense_dp4a_on; if (iq4nl_dense_dp4a_on && backend_ctx->kernel_gemm_noshuffle_iq4_nl_q8_1_dp4a && N > 8 && (K % 32 == 0) && (M % 64 == 0)) { cl_mem a_sub = nullptr; region.origin = offset1; region.size = (size_t)K * N * sizeof(float); CL_CHECK((a_sub = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); const size_t n_blocks = (size_t)N * (K / 32); backend_ctx->prealloc_moe_qa.allocate(context, (size_t)N * K * sizeof(cl_char)); backend_ctx->prealloc_moe_da.allocate(context, n_blocks * sizeof(cl_half)); backend_ctx->prealloc_moe_sa.allocate(context, n_blocks * sizeof(cl_half)); cl_int tb = (cl_int)n_blocks; cl_kernel qk = backend_ctx->kernel_quant_a_q8_1; CL_CHECK(clSetKernelArg(qk, 0, sizeof(cl_mem), &a_sub)); CL_CHECK(clSetKernelArg(qk, 1, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer)); CL_CHECK(clSetKernelArg(qk, 2, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer)); CL_CHECK(clSetKernelArg(qk, 3, sizeof(cl_mem), &backend_ctx->prealloc_moe_sa.buffer)); CL_CHECK(clSetKernelArg(qk, 4, sizeof(cl_int), &tb)); size_t q_local[1] = { 64 }; size_t q_global[1] = { (size_t)(((n_blocks + 63) / 64) * 64) }; backend_ctx->enqueue_ndrange_kernel(qk, 1, q_global, q_local, dst); cl_kernel dk = backend_ctx->kernel_gemm_noshuffle_iq4_nl_q8_1_dp4a; int ai = 0; CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &extra0_iq4_nl->q)); CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &extra0_iq4_nl->d)); CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer)); CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer)); CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &extrad->data_device)); CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_ulong), &offsetd)); CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_int), &M)); CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_int), &N)); CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_int), &K)); size_t d_local[3] = { 64, 1, 1 }; size_t d_global[3] = { 64, (size_t)(M / 64), (size_t)CEIL_DIV(N, 32) }; backend_ctx->enqueue_ndrange_kernel(dk, 3, d_global, d_local, dst); CL_CHECK(clReleaseMemObject(a_sub)); return; } cl_mem b_sub_buf = nullptr; cl_mem b_sub_buf_trans = nullptr; cl_mem b_img = nullptr; cl_mem b_img_trans = nullptr; // subbuffer for activations region.origin = offset1; region.size = K * N * sizeof(float); CL_CHECK((b_sub_buf = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); // image for activations img_fmt = {CL_RGBA, CL_FLOAT}; memset(&img_desc, 0, sizeof(img_desc)); img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; img_desc.image_width = K * N / 4; img_desc.buffer = b_sub_buf; CL_CHECK((b_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err), err)); // pad N to multiple of 8 int extra_elements = N % 8; int padding = 0; if (extra_elements > 0){ padding = 8 - extra_elements; } // subbuffer for transposed activations region.origin = 0; region.size = K * (N + padding) * sizeof(float)/2; backend_ctx->prealloc_act_trans.allocate(context, region.size); CL_CHECK((b_sub_buf_trans = clCreateSubBuffer(backend_ctx->prealloc_act_trans.buffer, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); // image for transposed activations img_fmt = {CL_RGBA, CL_HALF_FLOAT}; memset(&img_desc, 0, sizeof(img_desc)); img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; img_desc.image_width = K * (N + padding) / 4; img_desc.buffer = b_sub_buf_trans; CL_CHECK((b_img_trans = clCreateImage(context, 0, &img_fmt, &img_desc, NULL, &err), err)); // transpose activations int height_B = N/4; if (height_B == 0) { height_B = 1; } int width_B = K/4; int padded_height_B = (N + padding)/4; kernel = backend_ctx->kernel_transpose_32_16; CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &b_img)); CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &b_img_trans)); CL_CHECK(clSetKernelArg(kernel, 2, sizeof(int), &height_B)); CL_CHECK(clSetKernelArg(kernel, 3, sizeof(int), &width_B)); CL_CHECK(clSetKernelArg(kernel, 4, sizeof(int), &padded_height_B)); size_t local_work_size_t[2] = { 1, 16 }; size_t global_work_size_t[2] = { (size_t)width_B, (size_t)padded_height_B }; backend_ctx->enqueue_ndrange_kernel(kernel, 2, global_work_size_t, local_work_size_t, dst); // gemm kernel = backend_ctx->kernel_gemm_noshuffle_iq4_nl_f32; int padded_N = N + padding; CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_iq4_nl->q)); CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_iq4_nl->d)); CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &b_img_trans)); CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extrad->data_device)); CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_ulong), &offsetd)); CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_int), &ne01)); CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_int), &padded_N)); CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_int), &ne00)); CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_int), &ne1)); size_t global_work_size[3] = {(size_t)CEIL_DIV(ne1, 8), (size_t)CEIL_DIV(ne01, 4), 1}; size_t local_work_size[3] = {1, 128, 1}; backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); CL_CHECK(clReleaseMemObject(b_sub_buf)); CL_CHECK(clReleaseMemObject(b_sub_buf_trans)); CL_CHECK(clReleaseMemObject(b_img)); CL_CHECK(clReleaseMemObject(b_img_trans)); } #else GGML_UNUSED(backend); GGML_UNUSED(src0); GGML_UNUSED(src1); GGML_UNUSED(dst); #endif } static void ggml_cl_mul_mat_q8_0_f32_adreno(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { #ifdef GGML_OPENCL_USE_ADRENO_KERNELS GGML_ASSERT(src0); GGML_ASSERT(src0->extra); GGML_ASSERT(src1); GGML_ASSERT(src1->extra); GGML_ASSERT(dst); GGML_ASSERT(dst->extra); GGML_ASSERT(src0->type == GGML_TYPE_Q8_0); GGML_ASSERT(src1->type == GGML_TYPE_F32); ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; ggml_tensor_extra_cl * extra1 = (ggml_tensor_extra_cl *)src1->extra; ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; // SoA extra lives on view_src (view->extra is pre-SoA). const ggml_tensor * soa0_src = src0->view_src != nullptr ? src0->view_src : src0; ggml_tensor_extra_cl_q8_0 * extra0_q8_0 = (ggml_tensor_extra_cl_q8_0 *)soa0_src->extra; cl_ulong offset1 = extra1->offset + src1->view_offs; cl_ulong offsetd = extrad->offset + dst->view_offs; const int ne00 = src0->ne[0]; const int ne01 = src0->ne[1]; const int ne02 = src0->ne[2]; const int ne10 = src1->ne[0]; const int ne12 = src1->ne[2]; const int ne0 = dst->ne[0]; const int ne1 = dst->ne[1]; GGML_ASSERT(ne00 == ne10); GGML_ASSERT((ne00 % 32) == 0); GGML_ASSERT(ne0 == ne01); cl_context context = backend_ctx->context; cl_kernel kernel; cl_int err; cl_image_format img_fmt; cl_image_desc img_desc; cl_buffer_region region; int M = ne01; int N = ne1; int K = ne00; if (ne1 == 1) { cl_mem q_img = nullptr; cl_mem b_sub_buf = nullptr; cl_mem b_img = nullptr; // image for q img_fmt = { CL_R, CL_UNSIGNED_INT32}; memset(&img_desc, 0, sizeof(img_desc)); img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; img_desc.image_width = M * K / 4; img_desc.buffer = extra0_q8_0->q; CL_CHECK((q_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err), err)); // create a sub_buffer for B region.origin = offset1; region.size = K * N * sizeof(float); CL_CHECK((b_sub_buf = clCreateSubBuffer((extra1->data_device), 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); // image for activations img_fmt = {CL_RGBA, CL_FLOAT}; memset(&img_desc, 0, sizeof(img_desc)); img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; img_desc.image_width = K * N / 4; img_desc.buffer = b_sub_buf; CL_CHECK((b_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err), err)); // Split-K for small-M decode GEMVs. The base kernel puts one output row // per lane and splits K only inside one workgroup, so M is the sole source // of workgroup parallelism: gpt-oss's K and V projections are M=512 = 8 // workgroups on a 16-CU X2, and the kernel measures 48 GB/s where the // M=2880/4096 projections in the same decode graph reach 122-123. Mirrors // the q4_0/q4_K split-K above and reuses their reduce kernel. // // Enabled where it is measured to win, like the q4_K gate: X2-90 +2.8% // tg32 @d4096 on gpt-oss; Adreno 840 (12 CU) NEUTRAL on Llama-3.2-3B-Q8_0 // (0.0% @d4096 -- its K/V proj is M=1024 = 16 workgroups, which already // fills 12 CUs). Unmeasured on X1E/A7X/A6X and the q4_K split-K measured // -0.7% on X1E, so the default is not widened on absence of evidence. static const bool q8_splitk_env_set = []{ const char * e = std::getenv("GGML_OPENCL_Q8_GEMV_SPLITK"); return e && e[0] != '\0'; }(); static const bool q8_splitk_env_on = []{ const char * e = std::getenv("GGML_OPENCL_Q8_GEMV_SPLITK"); return !(e && e[0] == '0'); }(); const bool q8_splitk_on = q8_splitk_env_set ? q8_splitk_env_on : (backend_ctx->adreno_gen == ADRENO_GPU_GEN::X2E); if (q8_splitk_on && backend_ctx->kernel_gemv_noshuffle_q8_0_f32_splitk && ne01 <= 1024 && ne01 % 64 == 0) { const int nsg = 8; const int ksplit = 8; // -> 8 * M/64 workgroups const size_t gx = (size_t) CEIL_DIV(ne01, 64) * 64; backend_ctx->prealloc_splitk_partial.allocate( backend_ctx->context, (size_t) ksplit * ne01 * sizeof(float)); cl_mem partial = backend_ctx->prealloc_splitk_partial.buffer; cl_kernel ks = backend_ctx->kernel_gemv_noshuffle_q8_0_f32_splitk; CL_CHECK(clSetKernelArg(ks, 0, sizeof(cl_mem), &q_img)); CL_CHECK(clSetKernelArg(ks, 1, sizeof(cl_mem), &extra0_q8_0->d)); CL_CHECK(clSetKernelArg(ks, 2, sizeof(cl_mem), &b_img)); CL_CHECK(clSetKernelArg(ks, 3, sizeof(cl_mem), &partial)); CL_CHECK(clSetKernelArg(ks, 4, sizeof(cl_int), &ne00)); CL_CHECK(clSetKernelArg(ks, 5, sizeof(cl_int), &ne01)); size_t lsk[3] = { 64, (size_t) nsg, 1 }; size_t gsk[3] = { gx, (size_t) (nsg * ksplit), 1 }; backend_ctx->enqueue_ndrange_kernel(ks, 3, gsk, lsk, dst); cl_kernel kr = backend_ctx->kernel_gemv_splitk_reduce_f32; CL_CHECK(clSetKernelArg(kr, 0, sizeof(cl_mem), &partial)); CL_CHECK(clSetKernelArg(kr, 1, sizeof(cl_mem), &extrad->data_device)); CL_CHECK(clSetKernelArg(kr, 2, sizeof(cl_ulong), &offsetd)); CL_CHECK(clSetKernelArg(kr, 3, sizeof(cl_int), &ne01)); CL_CHECK(clSetKernelArg(kr, 4, sizeof(cl_int), &ksplit)); size_t lr[3] = { 64, 1, 1 }; size_t gr[3] = { (size_t) CEIL_DIV(ne01, 64) * 64, 1, 1 }; backend_ctx->enqueue_ndrange_kernel(kr, 3, gr, lr, dst); CL_CHECK(clReleaseMemObject(q_img)); CL_CHECK(clReleaseMemObject(b_img)); CL_CHECK(clReleaseMemObject(b_sub_buf)); return; } kernel = backend_ctx->kernel_gemv_noshuffle_q8_0_f32; int r2 = 1; int r3 = 1; CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &q_img)); CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q8_0->d)); CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &b_img)); CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1)); CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device)); CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd)); CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne00)); CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne01)); CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne02)); CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne10)); CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &ne12)); CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne0)); CL_CHECK(clSetKernelArg(kernel, 12, sizeof(int), &ne1)); CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &r2)); CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &r3)); size_t wavesize = backend_ctx->adreno_wave_size; size_t local_work_size[] = { wavesize, 4, 1 }; size_t global_work_size[] = { CEIL_DIV(M, wavesize)*wavesize, 4, 1 }; backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); CL_CHECK(clReleaseMemObject(q_img)); CL_CHECK(clReleaseMemObject(b_img)); CL_CHECK(clReleaseMemObject(b_sub_buf)); } else { static const char * q8_bin_dp4a_env = getenv("GGML_OPENCL_Q8_0_BIN_DP4A"); const bool q8_bin_dp4a_on = backend_ctx->has_integer_dot && (q8_bin_dp4a_env ? (atoi(q8_bin_dp4a_env) != 0) : true); if (q8_bin_dp4a_on && backend_ctx->kernel_gemm_noshuffle_q8_0_q8_1_dp4a_ila_a8_bin && M % 64 == 0) { cl_mem a_sub = nullptr; region.origin = offset1; region.size = (size_t)K * N * sizeof(float); CL_CHECK((a_sub = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); const size_t n_blocks = (size_t)N * (K / 32); const size_t N_pad = ((size_t)N + 31) / 32 * 32; const size_t n_blocks_pad = N_pad * (K / 32); backend_ctx->prealloc_moe_qa.allocate(context, (size_t)N_pad * K * sizeof(cl_char)); backend_ctx->prealloc_moe_da.allocate(context, n_blocks_pad * sizeof(cl_half)); backend_ctx->prealloc_moe_sa.allocate(context, n_blocks_pad * sizeof(cl_half)); cl_int tb = (cl_int)n_blocks; cl_kernel qk = backend_ctx->kernel_quant_a_q8_1; CL_CHECK(clSetKernelArg(qk, 0, sizeof(cl_mem), &a_sub)); CL_CHECK(clSetKernelArg(qk, 1, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer)); CL_CHECK(clSetKernelArg(qk, 2, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer)); CL_CHECK(clSetKernelArg(qk, 3, sizeof(cl_mem), &backend_ctx->prealloc_moe_sa.buffer)); CL_CHECK(clSetKernelArg(qk, 4, sizeof(cl_int), &tb)); size_t q_local[1] = { 64 }; size_t q_global[1] = { (size_t)(((n_blocks + 63) / 64) * 64) }; backend_ctx->enqueue_ndrange_kernel(qk, 1, q_global, q_local, dst); cl_mem q_img = nullptr; img_fmt = { CL_R, CL_UNSIGNED_INT32 }; memset(&img_desc, 0, sizeof(img_desc)); img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; img_desc.image_width = (size_t)M * (size_t)K / 4; img_desc.buffer = extra0_q8_0->q; CL_CHECK((q_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err), err)); cl_mem d_sub_buf = nullptr; cl_mem d_img = nullptr; region.origin = extrad->offset; region.size = (size_t)M * N * sizeof(float); CL_CHECK((d_sub_buf = clCreateSubBuffer(extrad->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); img_fmt = { CL_R, CL_FLOAT }; memset(&img_desc, 0, sizeof(img_desc)); img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; img_desc.image_width = (size_t)M * N; img_desc.buffer = d_sub_buf; CL_CHECK((d_img = clCreateImage(context, CL_MEM_WRITE_ONLY, &img_fmt, &img_desc, NULL, &err), err)); cl_kernel dk = backend_ctx->kernel_gemm_noshuffle_q8_0_q8_1_dp4a_ila_a8_bin; cl_uint ne00_u = (cl_uint)K; cl_uint ne01_u = (cl_uint)M; CL_CHECK(clSetKernelArg(dk, 0, sizeof(cl_mem), &q_img)); CL_CHECK(clSetKernelArg(dk, 1, sizeof(cl_mem), &extra0_q8_0->d)); CL_CHECK(clSetKernelArg(dk, 2, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer)); CL_CHECK(clSetKernelArg(dk, 3, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer)); CL_CHECK(clSetKernelArg(dk, 4, sizeof(cl_mem), &d_img)); CL_CHECK(clSetKernelArg(dk, 5, sizeof(cl_uint), &ne00_u)); CL_CHECK(clSetKernelArg(dk, 6, sizeof(cl_uint), &ne01_u)); CL_CHECK(clSetKernelArg(dk, 7, sizeof(int), &N)); size_t d_local[3] = { 64, 1, 1 }; size_t d_global[3] = { 64, (size_t)(M / 64), (size_t)CEIL_DIV(N, 32) }; backend_ctx->enqueue_ndrange_kernel(dk, 3, d_global, d_local, dst); CL_CHECK(clReleaseMemObject(q_img)); CL_CHECK(clReleaseMemObject(d_img)); CL_CHECK(clReleaseMemObject(d_sub_buf)); CL_CHECK(clReleaseMemObject(a_sub)); return; } // dp4a dense q8_0 prefill GEMM. Quantizes the [N,K] activations to // q8_1 and runs the int8 dot instead of the f16 half-dot. Large-batch // (ne1>8) only; q8_0 weights are already int8 (no requant) and symmetric // (no min term) static const char * q8_dense_dp4a_env = getenv("GGML_OPENCL_Q8_DENSE_DP4A"); static const char * q8_dense_wimg_env = getenv("GGML_OPENCL_Q8_DENSE_DP4A_WIMG"); const bool q8_dense_wimg_on = q8_dense_wimg_env && (atoi(q8_dense_wimg_env) != 0); const bool q8_bin_loaded = (backend_ctx->kernel_gemm_noshuffle_q8_0_f32_bin != nullptr); // bin kernel takes precedence bool q8_dense_dp4a_on = q8_dense_wimg_on ? true : q8_dense_dp4a_env ? (atoi(q8_dense_dp4a_env) != 0) : (backend_ctx->adreno_gen == ADRENO_GPU_GEN::X2E && !q8_bin_loaded); // dot prod has to be available q8_dense_dp4a_on = backend_ctx->has_integer_dot && q8_dense_dp4a_on; if (q8_dense_dp4a_on && backend_ctx->kernel_gemm_noshuffle_q8_0_q8_1_dp4a && N > 8 && (K % 32 == 0) && (M % 64 == 0)) { cl_mem a_sub = nullptr; region.origin = offset1; region.size = (size_t)K * N * sizeof(float); CL_CHECK((a_sub = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); const size_t n_blocks = (size_t)N * (K / 32); backend_ctx->prealloc_moe_qa.allocate(context, (size_t)N * K * sizeof(cl_char)); backend_ctx->prealloc_moe_da.allocate(context, n_blocks * sizeof(cl_half)); backend_ctx->prealloc_moe_sa.allocate(context, n_blocks * sizeof(cl_half)); cl_int tb = (cl_int)n_blocks; cl_kernel qk = backend_ctx->kernel_quant_a_q8_1; CL_CHECK(clSetKernelArg(qk, 0, sizeof(cl_mem), &a_sub)); CL_CHECK(clSetKernelArg(qk, 1, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer)); CL_CHECK(clSetKernelArg(qk, 2, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer)); CL_CHECK(clSetKernelArg(qk, 3, sizeof(cl_mem), &backend_ctx->prealloc_moe_sa.buffer)); CL_CHECK(clSetKernelArg(qk, 4, sizeof(cl_int), &tb)); size_t q_local[1] = { 64 }; size_t q_global[1] = { (size_t)(((n_blocks + 63) / 64) * 64) }; backend_ctx->enqueue_ndrange_kernel(qk, 1, q_global, q_local, dst); // optional weight texture, the same CL_R/UINT32 view, width M*K/4 cl_mem q8_q_img = nullptr; bool use_wimg = q8_dense_wimg_on; if (use_wimg) { const size_t tex = (size_t)M * (size_t)K / 4; // uint32 texels if (tex == 0 || tex > backend_ctx->image_max_buffer_size) { use_wimg = false; } else { img_fmt = { CL_R, CL_UNSIGNED_INT32 }; memset(&img_desc, 0, sizeof(img_desc)); img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; img_desc.image_width = tex; img_desc.buffer = extra0_q8_0->q; q8_q_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err); if (err != CL_SUCCESS || q8_q_img == nullptr) { use_wimg = false; q8_q_img = nullptr; } } } cl_kernel dk = use_wimg ? backend_ctx->kernel_gemm_noshuffle_q8_0_q8_1_dp4a_wimg : backend_ctx->kernel_gemm_noshuffle_q8_0_q8_1_dp4a; int ai = 0; if (use_wimg) { CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &q8_q_img)); } else { CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &extra0_q8_0->q)); } CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &extra0_q8_0->d)); CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer)); CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer)); CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &extrad->data_device)); CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_ulong), &offsetd)); CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_int), &M)); CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_int), &N)); CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_int), &K)); size_t d_local[3] = { 64, 1, 1 }; size_t d_global[3] = { 64, (size_t)(M / 64), (size_t)CEIL_DIV(N, 32) }; backend_ctx->enqueue_ndrange_kernel(dk, 3, d_global, d_local, dst); if (q8_q_img != nullptr) { CL_CHECK(clReleaseMemObject(q8_q_img)); } CL_CHECK(clReleaseMemObject(a_sub)); return; } // use bin kernel if available if (backend_ctx->kernel_gemm_noshuffle_q8_0_f32_bin) { int K_pad = K; cl_mem b_sub_buf = nullptr; cl_mem d_sub_buf = nullptr; cl_mem a_img = nullptr; cl_mem s_img = nullptr; cl_mem b_img = nullptr; cl_mem d_img = nullptr; // subbuffer for activations region.origin = offset1; region.size = K_pad * N * sizeof(float); CL_CHECK((b_sub_buf = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); // Create subbuffer and image1d_buffer for dst region.origin = (extrad->offset); // + dst->view_offs; region.size = M * N * sizeof(float); CL_CHECK((d_sub_buf = clCreateSubBuffer((extrad->data_device), 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); // create an image for A img_fmt = { CL_R, CL_FLOAT}; memset(&img_desc, 0, sizeof(img_desc)); img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; img_desc.image_width = M * K / 4; // Divide by 4 for char -> float img_desc.buffer = extra0_q8_0->q; CL_CHECK((a_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err), err)); // create an image for Scale img_fmt = { CL_R, CL_HALF_FLOAT}; memset(&img_desc, 0, sizeof(img_desc)); img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; img_desc.image_width = M * K / 32; // Block size is 32 img_desc.buffer = extra0_q8_0->d; CL_CHECK((s_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err), err)); // create an image for B from sub_buffer img_fmt = {CL_R, CL_FLOAT}; memset(&img_desc, 0, sizeof(img_desc)); img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; img_desc.image_width = K_pad * N; img_desc.buffer = b_sub_buf; CL_CHECK((b_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err), err)); // img for d img_fmt = {CL_R, CL_FLOAT}; memset(&img_desc, 0, sizeof(img_desc)); img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; img_desc.image_width = M * N; img_desc.buffer = d_sub_buf; CL_CHECK((d_img = clCreateImage(context, CL_MEM_WRITE_ONLY, &img_fmt, &img_desc, NULL, &err), err)); // gemm kernel = backend_ctx->kernel_gemm_noshuffle_q8_0_f32_bin; bool layoutA_Mfirst = true; bool layoutS_Mfirst = true; bool layoutB_Nfirst = false; bool layoutC_Mfirst = true; cl_uint lineStrideMatrixAinBytes = layoutA_Mfirst ? M * 4 : K; // int8 cl_uint lineStrideMatrixSinBytes = layoutS_Mfirst ? M * 2 : (K / 32) * 2; // fp16 cl_uint lineStrideMatrixBinBytes = layoutB_Nfirst ? N * 4 : K_pad * 4; // fp32 cl_uint lineStrideMatrixCinBytes = layoutC_Mfirst ? M * 4 : N * 4; // fp32 CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &a_img)); CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &s_img)); CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &b_img)); CL_CHECK(clSetKernelArg(kernel, 3, sizeof(int), &extra1->offset)); CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &d_img)); CL_CHECK(clSetKernelArg(kernel, 5, sizeof(int), &extrad->offset)); CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &K)); CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &lineStrideMatrixAinBytes)); CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &lineStrideMatrixSinBytes)); CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &lineStrideMatrixBinBytes)); CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &lineStrideMatrixCinBytes)); size_t global_work_size[] = { 64, (size_t)CEIL_DIV(M, 64), (size_t)CEIL_DIV(N, 64)}; size_t local_work_size[] = { 64, 2, 2 }; backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); CL_CHECK(clReleaseMemObject(b_sub_buf)); CL_CHECK(clReleaseMemObject(d_sub_buf)); CL_CHECK(clReleaseMemObject(a_img)); CL_CHECK(clReleaseMemObject(s_img)); CL_CHECK(clReleaseMemObject(b_img)); CL_CHECK(clReleaseMemObject(d_img)); return; } cl_mem b_sub_buf = nullptr; cl_mem b_sub_buf_trans = nullptr; cl_mem b_img = nullptr; cl_mem b_img_trans = nullptr; // subbuffer for activations region.origin = offset1; region.size = K * N * sizeof(float); CL_CHECK((b_sub_buf = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); // image for activations img_fmt = {CL_RGBA, CL_FLOAT}; memset(&img_desc, 0, sizeof(img_desc)); img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; img_desc.image_width = K * N / 4; img_desc.buffer = b_sub_buf; CL_CHECK((b_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err), err)); // pad N to multiple of 8 int extra_elements = N % 8; int padding = 0; if (extra_elements > 0){ padding = 8 - extra_elements; } // subbuffer for transposed activations region.origin = 0; region.size = K * (N + padding) * sizeof(float)/2; backend_ctx->prealloc_act_trans.allocate(context, region.size); CL_CHECK((b_sub_buf_trans = clCreateSubBuffer(backend_ctx->prealloc_act_trans.buffer, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); // image for transposed activations img_fmt = {CL_RGBA, CL_HALF_FLOAT}; memset(&img_desc, 0, sizeof(img_desc)); img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; img_desc.image_width = K * (N + padding) / 4; img_desc.buffer = b_sub_buf_trans; CL_CHECK((b_img_trans = clCreateImage(context, 0, &img_fmt, &img_desc, NULL, &err), err)); // transpose activations int height_B = N/4; if (height_B == 0) { height_B = 1; } int width_B = K/4; int padded_height_B = (N + padding)/4; kernel = backend_ctx->kernel_transpose_32_16; CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &b_img)); CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &b_img_trans)); CL_CHECK(clSetKernelArg(kernel, 2, sizeof(int), &height_B)); CL_CHECK(clSetKernelArg(kernel, 3, sizeof(int), &width_B)); CL_CHECK(clSetKernelArg(kernel, 4, sizeof(int), &padded_height_B)); size_t local_work_size_t[2] = { 1, 16 }; size_t global_work_size_t[2] = { (size_t)width_B, (size_t)padded_height_B }; backend_ctx->enqueue_ndrange_kernel(kernel, 2, global_work_size_t, local_work_size_t, dst); // gemm kernel = backend_ctx->kernel_gemm_noshuffle_q8_0_f32; int padded_N = N + padding; CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_q8_0->q)); CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q8_0->d)); CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &b_img_trans)); CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extrad->data_device)); CL_CHECK(clSetKernelArg(kernel, 4, sizeof(int), &K)); CL_CHECK(clSetKernelArg(kernel, 5, sizeof(int), &M)); CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &padded_N)); CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &N)); CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_ulong), &offsetd)); size_t global_work_size[] = { (size_t)CEIL_DIV(N, 8), (size_t)CEIL_DIV(M, 4), 1 }; size_t local_work_size[] = { 2, 128, 1 }; backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); CL_CHECK(clReleaseMemObject(b_img_trans)); CL_CHECK(clReleaseMemObject(b_sub_buf_trans)); CL_CHECK(clReleaseMemObject(b_img)); CL_CHECK(clReleaseMemObject(b_sub_buf)); } #else GGML_UNUSED(backend); GGML_UNUSED(src0); GGML_UNUSED(src1); GGML_UNUSED(dst); #endif } #ifdef GGML_OPENCL_USE_ADRENO_KERNELS static void ggml_cl_mul_mat_q4_k_f32_adreno_ila(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { GGML_ASSERT(src0); GGML_ASSERT(src0->extra); GGML_ASSERT(src1); GGML_ASSERT(src1->extra); GGML_ASSERT(dst); GGML_ASSERT(dst->extra); ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; ggml_tensor_extra_cl * extra1 = (ggml_tensor_extra_cl *)src1->extra; ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; ggml_tensor_extra_cl_q4_K * extra0_q4_k = (ggml_tensor_extra_cl_q4_K *)src0->extra; cl_ulong offset1 = extra1->offset + src1->view_offs; cl_ulong offsetd = extrad->offset + dst->view_offs; const int ne00 = src0->ne[0]; const int ne01 = src0->ne[1]; const int ne1 = dst->ne[1]; GGML_ASSERT(ne00 % ggml_blck_size(src0->type) == 0); cl_context context = backend_ctx->context; cl_kernel kernel; cl_int err; cl_image_format img_fmt; cl_image_desc img_desc; cl_buffer_region region; int M = ne01; int N = ne1; int K = ne00; if (ne1 == 1) { cl_mem b_sub_buf = nullptr; cl_mem b_img = nullptr; region.origin = offset1; region.size = (size_t)K * N * sizeof(float); CL_CHECK((b_sub_buf = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); img_fmt = { CL_RGBA, CL_FLOAT }; memset(&img_desc, 0, sizeof(img_desc)); img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; img_desc.image_width = (size_t)K * N / 4; img_desc.buffer = b_sub_buf; CL_CHECK((b_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err), err)); kernel = backend_ctx->kernel_gemv_noshuffle_q4_k_f32_32b_trans; CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_q4_k->q_img)); CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q4_k->d)); CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra0_q4_k->dm)); CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra0_q4_k->s)); CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &b_img)); CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_mem), &extrad->data_device)); CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_ulong), &offsetd)); CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_int), &ne00)); CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_int), &ne01)); size_t local_work_size[3] = { 64, 8, 1 }; size_t global_work_size[3] = { (size_t)ne01, 8, 1 }; backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); CL_CHECK(clReleaseMemObject(b_sub_buf)); CL_CHECK(clReleaseMemObject(b_img)); } else { const int gemm_tile_n = 64; int N_pad = CEIL_DIV(N, gemm_tile_n) * gemm_tile_n; static const char * q4_k_bin_dp4a_env = getenv("GGML_OPENCL_Q4_K_BIN_DP4A"); bool q4_k_bin_dp4a_on = q4_k_bin_dp4a_env ? (atoi(q4_k_bin_dp4a_env) != 0) : true; // dot prod has to be available q4_k_bin_dp4a_on = backend_ctx->has_integer_dot && q4_k_bin_dp4a_on; if (q4_k_bin_dp4a_on && backend_ctx->kernel_gemm_noshuffle_q4_k_q8_1_dp4a_ila_a8_bin) { const int dp4a_N_pad = CEIL_DIV(N, 32) * 32; const size_t n_blocks = (size_t)dp4a_N_pad * (K / 32); backend_ctx->prealloc_moe_qa.allocate(context, (size_t)dp4a_N_pad * K * sizeof(cl_char)); backend_ctx->prealloc_moe_da.allocate(context, n_blocks * sizeof(cl_half)); backend_ctx->prealloc_moe_sa.allocate(context, n_blocks * sizeof(cl_half)); cl_mem b_sub = nullptr; region.origin = offset1; region.size = (size_t)K * N * sizeof(float); CL_CHECK((b_sub = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); cl_int tb = (cl_int)((size_t)N * (K / 32)); cl_kernel qk = backend_ctx->kernel_quant_a_q8_1; CL_CHECK(clSetKernelArg(qk, 0, sizeof(cl_mem), &b_sub)); CL_CHECK(clSetKernelArg(qk, 1, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer)); CL_CHECK(clSetKernelArg(qk, 2, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer)); CL_CHECK(clSetKernelArg(qk, 3, sizeof(cl_mem), &backend_ctx->prealloc_moe_sa.buffer)); CL_CHECK(clSetKernelArg(qk, 4, sizeof(cl_int), &tb)); size_t q_local[1] = { 64 }; size_t q_global[1] = { (size_t)CEIL_DIV(tb, 64) * 64 }; backend_ctx->enqueue_ndrange_kernel(qk, 1, q_global, q_local, dst); cl_mem d_sub = nullptr; cl_mem d_img = nullptr; region.origin = offsetd; region.size = (size_t)M * N * sizeof(float); CL_CHECK((d_sub = clCreateSubBuffer(extrad->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); img_fmt = { CL_R, CL_FLOAT }; memset(&img_desc, 0, sizeof(img_desc)); img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; img_desc.image_width = (size_t)M * N; img_desc.buffer = d_sub; CL_CHECK((d_img = clCreateImage(context, CL_MEM_WRITE_ONLY, &img_fmt, &img_desc, NULL, &err), err)); kernel = backend_ctx->kernel_gemm_noshuffle_q4_k_q8_1_dp4a_ila_a8_bin; cl_uint k_arg = 0; CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &extra0_q4_k->q_img)); CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &extra0_q4_k->d)); CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &extra0_q4_k->dm)); CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &extra0_q4_k->s)); CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer)); CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer)); CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &backend_ctx->prealloc_moe_sa.buffer)); CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &d_img)); CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_uint), &ne00)); CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_uint), &ne01)); CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_int), &N)); size_t local_work_size[3] = { 64, 1, 1 }; size_t global_work_size[3] = { 64, (size_t)(M / 64), (size_t)(dp4a_N_pad / 32) }; backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); CL_CHECK(clReleaseMemObject(b_sub)); CL_CHECK(clReleaseMemObject(d_img)); CL_CHECK(clReleaseMemObject(d_sub)); return; } cl_mem b_sub_buf = nullptr; cl_mem b_padded = nullptr; cl_mem b_buf = nullptr; if (N_pad == N) { region.origin = offset1; region.size = (size_t)K * N * sizeof(float); CL_CHECK((b_sub_buf = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); b_buf = b_sub_buf; } else { CL_CHECK((b_padded = clCreateBuffer(context, CL_MEM_READ_WRITE, (size_t)K * N_pad * sizeof(float), NULL, &err), err)); const float zero = 0.0f; CL_CHECK(clEnqueueFillBuffer(backend_ctx->queue, b_padded, &zero, sizeof(zero), 0, (size_t)K * N_pad * sizeof(float), 0, NULL, NULL)); CL_CHECK(clEnqueueCopyBuffer(backend_ctx->queue, extra1->data_device, b_padded, offset1, 0, (size_t)K * N * sizeof(float), 0, NULL, NULL)); b_buf = b_padded; } img_fmt = { CL_R, CL_FLOAT }; memset(&img_desc, 0, sizeof(img_desc)); img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; img_desc.image_width = (size_t)K * N_pad; img_desc.buffer = b_buf; cl_mem b_img; CL_CHECK((b_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err), err)); region.origin = offsetd; region.size = (size_t)M * N * sizeof(float); cl_mem d_sub_buf; CL_CHECK((d_sub_buf = clCreateSubBuffer(extrad->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); img_fmt = { CL_R, CL_FLOAT }; memset(&img_desc, 0, sizeof(img_desc)); img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; img_desc.image_width = (size_t)M * N; img_desc.buffer = d_sub_buf; cl_mem d_img; CL_CHECK((d_img = clCreateImage(context, CL_MEM_WRITE_ONLY, &img_fmt, &img_desc, NULL, &err), err)); kernel = backend_ctx->kernel_gemm_noshuffle_q4_k_f32_32b_trans_ila_a8_bin; CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_q4_k->q_img)); CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q4_k->d)); CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra0_q4_k->dm)); CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra0_q4_k->s)); CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &b_img)); CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_mem), &d_img)); CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_uint), &ne00)); CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_uint), &ne01)); CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &N)); size_t local_work_size[3] = { 64, 2, 2 }; size_t m_tiles = (size_t)CEIL_DIV(M, 64); size_t global_work_size[3] = { 64, m_tiles, (size_t)CEIL_DIV(N_pad, gemm_tile_n) }; backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); CL_CHECK(clReleaseMemObject(b_img)); if (b_sub_buf) { CL_CHECK(clReleaseMemObject(b_sub_buf)); } if (b_padded) { CL_CHECK(clReleaseMemObject(b_padded)); } CL_CHECK(clReleaseMemObject(d_img)); CL_CHECK(clReleaseMemObject(d_sub_buf)); } } #endif // GGML_OPENCL_USE_ADRENO_KERNELS static void ggml_cl_mul_mat_q4_k_f32_adreno(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { #ifdef GGML_OPENCL_USE_ADRENO_KERNELS GGML_ASSERT(src0); GGML_ASSERT(src0->extra); GGML_ASSERT(src1); GGML_ASSERT(src1->extra); GGML_ASSERT(dst); GGML_ASSERT(dst->extra); ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; ggml_tensor_extra_cl * extra1 = (ggml_tensor_extra_cl *)src1->extra; ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; ggml_tensor_extra_cl_q4_K * extra0_q4_k = (ggml_tensor_extra_cl_q4_K *)src0->extra; cl_ulong offset1 = extra1->offset + src1->view_offs; cl_ulong offsetd = extrad->offset + dst->view_offs; const int ne00 = src0->ne[0]; const int ne01 = src0->ne[1]; const int ne1 = dst->ne[1]; GGML_ASSERT(ne00 % ggml_blck_size(src0->type) == 0); cl_context context = backend_ctx->context; cl_kernel kernel; cl_int err; cl_image_format img_fmt; cl_image_desc img_desc; cl_buffer_region region; int M = ne01; int N = ne1; int K = ne00; cl_uchar mask_d6 = 0x3F; cl_uchar mask_d4 = 0x0F; cl_uchar mask_hi2 = 0xC0; // Multi-column verify GEMV: route the spec/MTP verify batch (ne1==3 = 2 // drafts + 1 bonus) onto the efficient GEMV path (subgroup-broadcast, no // transpose) instead of the transposed-GEMM dead-zone. Reuses the ne1==1 // GEMV setup (the activation image is already sized by N=ne1). Byte- // identical. Opt-in via GGML_OPENCL_Q4K_MC3=1 while validating. static const bool q4k_mc3 = (getenv("GGML_OPENCL_Q4K_MC3") != nullptr); // Per-layer only (ne01 < 32768): the batched large-vocab lm_head at ne1==3 // is left to the existing routing (corrupts on the Adreno GEMV path; x2- // unified routes batched Q6_K lm_head to CPU). Per-layer mc3 is byte-identical. const bool use_mc3 = q4k_mc3 && (ne1 == 3) && (ne01 < 32768); const bool use_bin = use_q4_k_bin_kernels(backend_ctx, src0); if (use_bin) { if (use_mc3) { static bool warned = false; if (!warned) { GGML_LOG_WARN("ggml_opencl: GGML_OPENCL_Q4K_MC3 is bypassed by Q4_K binary kernels\n"); warned = true; } } ggml_cl_mul_mat_q4_k_f32_adreno_ila(backend, src0, src1, dst); return; } if (ne1 == 1 || use_mc3) { cl_mem q_img = nullptr; cl_mem b_sub_buf = nullptr; cl_mem b_img = nullptr; const bool use_tiled = !use_mc3 && use_q4k_tiled(backend_ctx, src0); // image for q (not needed for the tiled path, which reads __global) if (!use_tiled) { img_fmt = { CL_R, CL_UNSIGNED_INT32}; memset(&img_desc, 0, sizeof(img_desc)); img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; img_desc.image_width = M * K / 2 / 4; img_desc.buffer = extra0_q4_k->q; CL_CHECK((q_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err), err)); } // subbuffer for activations region.origin = offset1; region.size = K * N * sizeof(float); CL_CHECK((b_sub_buf = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); // image for activations img_fmt = {CL_RGBA, CL_FLOAT}; memset(&img_desc, 0, sizeof(img_desc)); img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; img_desc.image_width = K * N / 4; img_desc.buffer = b_sub_buf; CL_CHECK((b_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err), err)); // 4-output-per-WI o4 variant for the long-vocab lm_head/embed GEMV // (ne01 = vocab ~256K on Gemma): shares one activation read across 4 // output rows. Gated to large ne01 (lm_head/embed). Default on; opt-out // GGML_OPENCL_Q4K_GEMV_O4=0. (Skipped when mc3 handles the ne1==3 verify.) static const bool q4k_o4_env = []{ const char * e = std::getenv("GGML_OPENCL_Q4K_GEMV_O4"); return !e || e[0] == '\0' || e[0] != '0'; }(); const bool use_q4k_o4 = !use_tiled && !use_mc3 && q4k_o4_env && (ne01 % 4 == 0) && (ne01 >= 32768); // Split-K across workgroups for small-M decode GEMVs. A single-token GEMV // makes only CEIL_DIV(M/2,64) workgroups; even with the wide intra-WG split // (16 subgroups) those all land on ONE CU, so small-M matmuls under-fill the // 16 CUs and their bandwidth falls well short of what the large-M FFN matmuls // reach. Adding a `ksplit` second grid dim that spreads K across WGs (+ a // reduce pass) fills the CUs. Gate is M<=2560: the tiny M<=1024 ones only // break even (the reduce dispatch eats the kernel win), but the big-K M=2560 // cases (ffn_down, attn_output) make the per-call win dwarf the reduce, and // are byte-identical. ffn_gate/up (large M) fill the CUs already and are excluded. // // DEVICE-GATED. Split-K buys GPU time by spending an extra kernel LAUNCH (the // reduce), so it only pays where launches are cheap. That is a per-device // property and it does not travel from the X2-90 this was tuned on. Measured // with one binary, env A/B (tg32, GGML_OPENCL_Q4K_GEMV_SPLITK=0/1): // // X2-90 +3.36% gemma-4 E4B (the number this gate was built on) // 840 -1.3% Qwen3.5-4B-Q4_K_M 14.00 -> 13.85 // 850 -20.0% Qwen3-1.7B-Q4_K_M 6.97 -> 5.58 (6 interleaved reps) // // The kernel is not the problem. On the 850 split-K makes the GPU strictly // faster -- total busy 537 -> 485 ms, this GEMV 43.7 -> 34.0 us/call (-22%) -- // and still costs a fifth of decode, because the +3696 reduce dispatches cost // ~550 us of HOST round-trip each against 2.7 us of GPU work (~200x; that part // is ~95% host-bound at decode). The 840 pays the same tax at ~42 us/dispatch. // Break-even needs launch cost below the ~9.7 us/call the split actually saves, // so this is not a "the 850 is slow" adjustment that a faster part would fix -- // the 840 is 13x cheaper per launch and still loses. // // Enabled where it is measured to win, i.e. X2E only. The X1-85 was measured // afterwards and is NOT a win either: Qwen3.5-4B-Q4_K_M tg32, split-K off // 17.98/18.10/18.19 vs on 18.03/17.91/17.97 = -0.7%, so X1E stays excluded on // evidence rather than on absence of it. Do not widen this without a NEW // measurement. The env still forces either way so every device stays measurable. static const bool splitk_env_set = []{ const char * e = std::getenv("GGML_OPENCL_Q4K_GEMV_SPLITK"); return e && e[0] != '\0'; }(); static const bool splitk_env_on = []{ const char * e = std::getenv("GGML_OPENCL_Q4K_GEMV_SPLITK"); return !(e && e[0] == '0'); }(); const bool splitk_wg_env = splitk_env_set ? splitk_env_on : (backend_ctx->adreno_gen == ADRENO_GPU_GEN::X2E); // Gate: small-M decode GEMVs that under-fill the 16 CUs even with the wide // intra-WG split (all 16 subgroups land on one CU). M<=2560 covers Kcur/Vcur // (M=1024), Qcur (2048), attn_output + ffn_down (2560). The tiny ones // (M<=1024) only break even (reduce dispatch eats the kernel win), but the // big-K M=2560 cases (ffn_down K=10240 @182us, attn_output @42us) have a // large per-call win that dwarfs the ~5us reduce, so extending to 2560 nets // positive end-to-end. ffn_gate/up (M=10240) already fill the CUs -> excluded. const bool use_splitk = splitk_wg_env && !use_tiled && !use_q4k_o4 && !use_mc3 && ne01 <= 2560; if (use_splitk) { const int nsg = 8; const int ksplit = (ne01 <= 512) ? 8 : 4; // -> ~32 total WGs const size_t gx = (size_t)CEIL_DIV(ne01/2, 64) * 64; backend_ctx->prealloc_splitk_partial.allocate( backend_ctx->context, (size_t)ksplit * ne01 * sizeof(float)); cl_mem partial = backend_ctx->prealloc_splitk_partial.buffer; cl_kernel ks = backend_ctx->kernel_gemv_noshuffle_q4_k_f32_splitk; CL_CHECK(clSetKernelArg(ks, 0, sizeof(cl_mem), &q_img)); CL_CHECK(clSetKernelArg(ks, 1, sizeof(cl_mem), &extra0_q4_k->d)); CL_CHECK(clSetKernelArg(ks, 2, sizeof(cl_mem), &extra0_q4_k->dm)); CL_CHECK(clSetKernelArg(ks, 3, sizeof(cl_mem), &extra0_q4_k->s)); CL_CHECK(clSetKernelArg(ks, 4, sizeof(cl_mem), &b_img)); CL_CHECK(clSetKernelArg(ks, 5, sizeof(cl_mem), &partial)); CL_CHECK(clSetKernelArg(ks, 6, sizeof(cl_int), &ne00)); CL_CHECK(clSetKernelArg(ks, 7, sizeof(cl_int), &ne01)); CL_CHECK(clSetKernelArg(ks, 8, sizeof(cl_uchar), &mask_d6)); CL_CHECK(clSetKernelArg(ks, 9, sizeof(cl_uchar), &mask_d4)); CL_CHECK(clSetKernelArg(ks, 10, sizeof(cl_uchar), &mask_hi2)); size_t lsk[3] = {64, (size_t)nsg, 1}; size_t gsk[3] = {gx, (size_t)(nsg * ksplit), 1}; backend_ctx->enqueue_ndrange_kernel(ks, 3, gsk, lsk, dst); cl_kernel kr = backend_ctx->kernel_gemv_splitk_reduce_f32; CL_CHECK(clSetKernelArg(kr, 0, sizeof(cl_mem), &partial)); CL_CHECK(clSetKernelArg(kr, 1, sizeof(cl_mem), &extrad->data_device)); CL_CHECK(clSetKernelArg(kr, 2, sizeof(cl_ulong), &offsetd)); CL_CHECK(clSetKernelArg(kr, 3, sizeof(cl_int), &ne01)); CL_CHECK(clSetKernelArg(kr, 4, sizeof(cl_int), &ksplit)); size_t lr[3] = {64, 1, 1}; size_t gr[3] = {(size_t)CEIL_DIV(ne01, 64) * 64, 1, 1}; backend_ctx->enqueue_ndrange_kernel(kr, 3, gr, lr, dst); if (q_img) CL_CHECK(clReleaseMemObject(q_img)); CL_CHECK(clReleaseMemObject(b_sub_buf)); CL_CHECK(clReleaseMemObject(b_img)); return; } kernel = use_mc3 ? backend_ctx->kernel_gemv_noshuffle_q4_k_f32_mc3 : use_tiled ? backend_ctx->kernel_gemv_noshuffle_q4_k_f32_tiled : use_q4k_o4 ? backend_ctx->kernel_gemv_noshuffle_q4_k_f32_o4 : backend_ctx->kernel_gemv_noshuffle_q4_k_f32; if (use_tiled) { CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_q4_k->q)); CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q4_k->d)); CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra0_q4_k->dm)); CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra0_q4_k->s)); CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &b_img)); CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_mem), &extrad->data_device)); CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_ulong), &offsetd)); CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_int), &ne00)); CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_int), &ne01)); } else { CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &q_img)); CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q4_k->d)); CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra0_q4_k->dm)); CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra0_q4_k->s)); CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &b_img)); CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_mem), &extrad->data_device)); CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_ulong), &offsetd)); CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_int), &ne00)); CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_int), &ne01)); CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_uchar), &mask_d6)); CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_uchar), &mask_d4)); CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_uchar), &mask_hi2)); } // Wide K-split for the decode GEMV: the default 4-subgroup K-split leaves // each Adreno SP with only ~4 waves, too few to hide LPDDR weight-load // latency, so even the large FFN matmuls run well below the achievable // bandwidth. Widen to 16 subgroups/WG (= the 1024-lane Adreno WG max) so // each SP holds enough in-flight memory requests. Prefill is unaffected (the // GEMM path is separate) and coherence-identical (greedy output unchanged). // Applies to the plain base // GEMV only; tiled/o4/mc3 keep 4 (their reductions are hard-coded to 4). // Layout-safe: the base kernel derives its K-split from get_local_size(1) // and the packed block stride is a physical constant (independent of it). // Opt-out: GGML_OPENCL_Q4K_GEMV_WIDE=0. static const bool splitk_wide_env = []{ const char * e = std::getenv("GGML_OPENCL_Q4K_GEMV_WIDE"); return !e || e[0] == '\0' || e[0] != '0'; }(); const bool splitk_wide = splitk_wide_env && !use_tiled && !use_q4k_o4 && !use_mc3; size_t nsg_y = splitk_wide ? 16 : 4; // Cap the wide K-split by the kernel's real max WG. X1-class drivers cap // this GEMV at 768 (< 64*16 = 1024), so an uncapped lws aborts the // dispatch with CL_INVALID_WORK_GROUP_SIZE (-54) and breaks ALL q4_K // decode for M>2560. nsg_y is a pure K-split (the base kernel reads it // from get_local_size(1); the packed block stride is a physical constant), // so halving it stays coherent — just a narrower split. X2 keeps 16 // (maxwg 1024); X1 falls to 8. if (splitk_wide) { const size_t maxwg = backend_ctx->get_kernel_workgroup_size(kernel); while (nsg_y > 4 && 64 * nsg_y > maxwg) { nsg_y >>= 1; } } size_t local_work_size[3] = {64, nsg_y, 1}; size_t global_work_size[3] = {(size_t)CEIL_DIV(use_tiled ? ne01 : (use_q4k_o4 ? ne01/4 : ne01/2), 64)*64, nsg_y, 1}; backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); if (q_img) CL_CHECK(clReleaseMemObject(q_img)); CL_CHECK(clReleaseMemObject(b_sub_buf)); CL_CHECK(clReleaseMemObject(b_img)); } else { cl_mem b_sub_buf = nullptr; cl_mem b_sub_buf_trans = nullptr; cl_mem b_img = nullptr; cl_mem b_img_trans = nullptr; // subbuffer for activations region.origin = offset1; region.size = K * N * sizeof(float); CL_CHECK((b_sub_buf = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); // image for activations img_fmt = {CL_RGBA, CL_FLOAT}; memset(&img_desc, 0, sizeof(img_desc)); img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; img_desc.image_width = K * N / 4; img_desc.buffer = b_sub_buf; CL_CHECK((b_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err), err)); // pad N to multiple of 8 int extra_elements = N % 8; int padding = 0; if (extra_elements > 0){ padding = 8 - extra_elements; } // subbuffer for transposed activations region.origin = 0; region.size = K * (N + padding) * sizeof(float)/2; backend_ctx->prealloc_act_trans.allocate(context, region.size); CL_CHECK((b_sub_buf_trans = clCreateSubBuffer(backend_ctx->prealloc_act_trans.buffer, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); // image for transposed activations img_fmt = {CL_RGBA, CL_HALF_FLOAT}; memset(&img_desc, 0, sizeof(img_desc)); img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; img_desc.image_width = K * (N + padding) / 4; img_desc.buffer = b_sub_buf_trans; CL_CHECK((b_img_trans = clCreateImage(context, 0, &img_fmt, &img_desc, NULL, &err), err)); // transpose activations int height_B = N/4; if (height_B == 0) { height_B = 1; } int width_B = K/4; int padded_height_B = (N + padding)/4; kernel = backend_ctx->kernel_transpose_32_16; CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &b_img)); CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &b_img_trans)); CL_CHECK(clSetKernelArg(kernel, 2, sizeof(int), &height_B)); CL_CHECK(clSetKernelArg(kernel, 3, sizeof(int), &width_B)); CL_CHECK(clSetKernelArg(kernel, 4, sizeof(int), &padded_height_B)); size_t local_work_size_t[2] = { 1, 16 }; size_t global_work_size_t[2] = { (size_t)width_B, (size_t)padded_height_B }; backend_ctx->enqueue_ndrange_kernel(kernel, 2, global_work_size_t, local_work_size_t, dst); // dp4a (int8) dense prefill GEMM and weight via texture static const char * q4k_dense_dp4a_env = getenv("GGML_OPENCL_Q4K_DENSE_DP4A"); static const char * q4k_dense_wimg_env = getenv("GGML_OPENCL_Q4K_DENSE_DP4A_WIMG"); const bool q4k_dense_wimg_on = q4k_dense_wimg_env && (atoi(q4k_dense_wimg_env) != 0); bool q4k_dense_dp4a_on = q4k_dense_wimg_on ? true : q4k_dense_dp4a_env ? (atoi(q4k_dense_dp4a_env) != 0) : (backend_ctx->adreno_gen == ADRENO_GPU_GEN::X2E); // dp4 has to be available q4k_dense_dp4a_on = backend_ctx->has_integer_dot && q4k_dense_dp4a_on; // Min N for the dp4a prefill GEMM, default 9, i.e., ne1 > 8 static const char * q4k_dp4a_minn_env = getenv("GGML_OPENCL_Q4K_DP4A_MINN"); const int q4k_dp4a_minn = q4k_dp4a_minn_env ? atoi(q4k_dp4a_minn_env) : 9; if (q4k_dense_dp4a_on && N >= q4k_dp4a_minn && (K % 32 == 0) && (M % 64 == 0)) { const size_t n_blocks = (size_t)N * (K / 32); backend_ctx->prealloc_moe_qa.allocate(context, (size_t)N * K * sizeof(cl_char)); backend_ctx->prealloc_moe_da.allocate(context, n_blocks * sizeof(cl_half)); backend_ctx->prealloc_moe_sa.allocate(context, n_blocks * sizeof(cl_half)); cl_int tb = (cl_int)n_blocks; cl_kernel qk = backend_ctx->kernel_quant_a_q8_1; CL_CHECK(clSetKernelArg(qk, 0, sizeof(cl_mem), &b_sub_buf)); CL_CHECK(clSetKernelArg(qk, 1, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer)); CL_CHECK(clSetKernelArg(qk, 2, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer)); CL_CHECK(clSetKernelArg(qk, 3, sizeof(cl_mem), &backend_ctx->prealloc_moe_sa.buffer)); CL_CHECK(clSetKernelArg(qk, 4, sizeof(cl_int), &tb)); size_t q_local[1] = { 64 }; size_t q_global[1] = { (size_t)(((n_blocks + 63) / 64) * 64) }; backend_ctx->enqueue_ndrange_kernel(qk, 1, q_global, q_local, dst); // check if weights go through texture cl_mem q4k_q_img = nullptr; bool use_wimg = q4k_dense_wimg_on; if (use_wimg) { const size_t tex = (size_t)M * (size_t)K / 8; // uint32 texels = bytes/4 if (tex == 0 || tex > backend_ctx->image_max_buffer_size) { use_wimg = false; } else { img_fmt = { CL_R, CL_UNSIGNED_INT32 }; memset(&img_desc, 0, sizeof(img_desc)); img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; img_desc.image_width = tex; img_desc.buffer = extra0_q4_k->q; q4k_q_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err); if (err != CL_SUCCESS || q4k_q_img == nullptr) { use_wimg = false; q4k_q_img = nullptr; } } } cl_kernel dk = use_wimg ? backend_ctx->kernel_gemm_noshuffle_q4_k_q8_1_dp4a_wimg : backend_ctx->kernel_gemm_noshuffle_q4_k_q8_1_dp4a; int ai = 0; if (use_wimg) { CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &q4k_q_img)); } else { CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &extra0_q4_k->q)); } CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &extra0_q4_k->s)); CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &extra0_q4_k->d)); CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &extra0_q4_k->dm)); CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer)); CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer)); CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &backend_ctx->prealloc_moe_sa.buffer)); CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &extrad->data_device)); CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_ulong), &offsetd)); CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_int), &M)); CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_int), &N)); CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_int), &K)); CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_uchar), &mask_d6)); CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_uchar), &mask_d4)); CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_uchar), &mask_hi2)); // Must match the compile-time TILESIZE_N chosen at program build (per-device, // X1E=8 else 32; env override). Same inputs -> same value. int q4k_dp4a_ts = (backend_ctx->adreno_gen == ADRENO_GPU_GEN::X1E) ? 8 : 32; if (const char * e = getenv("GGML_OPENCL_Q4K_DP4A_TS")) q4k_dp4a_ts = atoi(e); size_t d_local[3] = { 64, 1, 1 }; size_t d_global[3] = { 64, (size_t)(M / 64), (size_t)CEIL_DIV(N, q4k_dp4a_ts) }; backend_ctx->enqueue_ndrange_kernel(dk, 3, d_global, d_local, dst); if (q4k_q_img != nullptr) { CL_CHECK(clReleaseMemObject(q4k_q_img)); } CL_CHECK(clReleaseMemObject(b_sub_buf)); CL_CHECK(clReleaseMemObject(b_sub_buf_trans)); CL_CHECK(clReleaseMemObject(b_img)); CL_CHECK(clReleaseMemObject(b_img_trans)); return; } // gemm // Small-batch (medium n_q) occupancy fix: at ne1<=8 the 2x8 grid is // (1, ceil(M/2)) -> ~M/256 workgroups, which under-occupies the SP and // makes the GEMM much slower than the ne1==1 GEMV at the same weight // traffic. The _r1 (1-row) kernel doubles the M-axis workgroup count // and removes the accumulator spill. Opt-in via env while validating. static const bool q4k_gemm_r1 = (getenv("GGML_OPENCL_Q4K_GEMM_R1") != nullptr); static const bool q4k_gemm_kimg = (getenv("GGML_OPENCL_Q4K_GEMM_KIMG") != nullptr); // Cooperative-K (intra-WG K-split + reduction) for the small-batch // (n_q in [2..8]) path: DEFAULT ON, opt out with GGML_OPENCL_Q4K_GEMM_COK=0. // Byte-identical greedy output; large-batch (ne1>8) untouched. static const char * q4k_cok_env = getenv("GGML_OPENCL_Q4K_GEMM_COK"); static const bool q4k_gemm_cok = (q4k_cok_env == nullptr) || (atoi(q4k_cok_env) != 0); const bool use_cok = q4k_gemm_cok && (ne1 <= 8); const bool use_r1 = !use_cok && q4k_gemm_r1 && (ne1 <= 8); // Weights-as-image (L1/TPL1) for the small-batch weight-read-bound path. const bool use_kimg = !use_cok && !use_r1 && q4k_gemm_kimg && (ne1 <= 8); cl_mem q_img = nullptr; if (use_kimg) { img_fmt = { CL_R, CL_UNSIGNED_INT32 }; memset(&img_desc, 0, sizeof(img_desc)); img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; img_desc.image_width = M * K / 2 / 4; img_desc.buffer = extra0_q4_k->q; CL_CHECK((q_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err), err)); } kernel = use_cok ? backend_ctx->kernel_gemm_noshuffle_q4_k_f32_cok : use_r1 ? backend_ctx->kernel_gemm_noshuffle_q4_k_f32_r1 : use_kimg ? backend_ctx->kernel_gemm_noshuffle_q4_k_f32_kimg : backend_ctx->kernel_gemm_noshuffle_q4_k_f32; int padded_N = N + padding; if (use_kimg) { CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &q_img)); } else { CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_q4_k->q)); } CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q4_k->s)); CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra0_q4_k->d)); CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra0_q4_k->dm)); CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &b_img_trans)); CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_mem), &extrad->data_device)); CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_ulong), &offsetd)); CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_int), &ne01)); CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_int), &padded_N)); CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_int), &ne00)); CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_int), &ne1)); CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_uchar), &mask_d6)); CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_uchar), &mask_d4)); CL_CHECK(clSetKernelArg(kernel, 13, sizeof(cl_uchar), &mask_hi2)); size_t global_work_size[3]; size_t local_work_size[3]; if (use_cok) { // (COK_SG lanes x COK_NSG subgroups): one row per lane, K split // across the COK_NSG subgroups. ne01 is a multiple of 64. global_work_size[0] = (size_t)ne01; // rows global_work_size[1] = 8; // COK_NSG global_work_size[2] = 1; local_work_size[0] = 64; // COK_SG local_work_size[1] = 8; // COK_NSG local_work_size[2] = 1; } else if (use_r1) { // 1 row per WI (opt-in occupancy experiment). global_work_size[0] = (size_t)CEIL_DIV(ne1, 8); global_work_size[1] = (size_t)ne01; global_work_size[2] = 1; local_work_size[0] = 1; local_work_size[1] = 128; local_work_size[2] = 1; } else if (use_kimg) { // kimg is a 2-row tile (opt-in weights-as-image experiment). global_work_size[0] = (size_t)CEIL_DIV(ne1, 8); global_work_size[1] = (size_t)CEIL_DIV(ne01, 2); global_work_size[2] = 1; local_work_size[0] = 1; local_work_size[1] = 128; local_work_size[2] = 1; } else { // Default: x2-unified base kernel is the 4-row (gx<<2) tile. global_work_size[0] = (size_t)CEIL_DIV(ne1, 8); global_work_size[1] = (size_t)CEIL_DIV(ne01, 4); global_work_size[2] = 1; local_work_size[0] = 1; local_work_size[1] = 128; local_work_size[2] = 1; } backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); if (q_img) CL_CHECK(clReleaseMemObject(q_img)); CL_CHECK(clReleaseMemObject(b_sub_buf)); CL_CHECK(clReleaseMemObject(b_sub_buf_trans)); CL_CHECK(clReleaseMemObject(b_img)); CL_CHECK(clReleaseMemObject(b_img_trans)); } #else GGML_UNUSED(backend); GGML_UNUSED(src0); GGML_UNUSED(src1); GGML_UNUSED(dst); #endif } #ifdef GGML_OPENCL_USE_ADRENO_KERNELS static void ggml_cl_mul_mat_q6_K_f32_adreno_ila(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { GGML_ASSERT(src0); GGML_ASSERT(src0->extra); GGML_ASSERT(src1); GGML_ASSERT(src1->extra); GGML_ASSERT(dst); GGML_ASSERT(dst->extra); ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; ggml_tensor_extra_cl_q6_K * extra0_q6_K = (ggml_tensor_extra_cl_q6_K *)src0->extra; ggml_tensor_extra_cl * extra1 = (ggml_tensor_extra_cl *)src1->extra; ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; cl_ulong offset1 = extra1->offset + src1->view_offs; cl_ulong offsetd = extrad->offset + dst->view_offs; const int ne00 = src0->ne[0]; const int ne01 = src0->ne[1]; const int ne1 = dst->ne[1]; GGML_ASSERT(ne00 % ggml_blck_size(src0->type) == 0); cl_context context = backend_ctx->context; cl_kernel kernel; cl_int err; cl_buffer_region region; cl_image_format img_fmt; cl_image_desc img_desc; const int M = ne01; const int N = ne1; const int K = ne00; if (ne1 == 1) { cl_mem b_sub_buf = nullptr; cl_mem b_img = nullptr; region.origin = offset1; region.size = (size_t)K * N * sizeof(float); CL_CHECK((b_sub_buf = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); img_fmt = { CL_RGBA, CL_FLOAT }; memset(&img_desc, 0, sizeof(img_desc)); img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; img_desc.image_width = (size_t)K * N / 4; img_desc.buffer = b_sub_buf; CL_CHECK((b_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err), err)); kernel = backend_ctx->kernel_gemv_noshuffle_q6_k_f32_32b_trans; CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_q6_K->ql_img)); CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q6_K->qh_img)); CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra0_q6_K->s)); CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra0_q6_K->d)); CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &b_img)); CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_mem), &extrad->data_device)); CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_ulong), &offsetd)); CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_int), &ne00)); CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_int), &ne01)); size_t local_work_size[3] = { 64, 8, 1 }; size_t global_work_size[3] = { (size_t)ne01, 8, 1 }; backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); CL_CHECK(clReleaseMemObject(b_img)); CL_CHECK(clReleaseMemObject(b_sub_buf)); } else { const int gemm_tile_n = 64; int N_pad = CEIL_DIV(N, gemm_tile_n) * gemm_tile_n; static const char * q6_k_bin_dp4a_env = getenv("GGML_OPENCL_Q6_K_BIN_DP4A"); bool q6_k_bin_dp4a_on = q6_k_bin_dp4a_env ? (atoi(q6_k_bin_dp4a_env) != 0) : true; // dot prod has to be available q6_k_bin_dp4a_on = backend_ctx->has_integer_dot && q6_k_bin_dp4a_on; if (q6_k_bin_dp4a_on && backend_ctx->kernel_gemm_noshuffle_q6_k_q8_1_dp4a_ila_a8_bin) { const int dp4a_N_pad = CEIL_DIV(N, 32) * 32; const size_t n_blocks = (size_t)dp4a_N_pad * (K / 32); backend_ctx->prealloc_moe_qa.allocate(context, (size_t)dp4a_N_pad * K * sizeof(cl_char)); backend_ctx->prealloc_moe_da.allocate(context, n_blocks * sizeof(cl_half)); backend_ctx->prealloc_moe_sa.allocate(context, n_blocks * sizeof(cl_half)); cl_mem b_sub = nullptr; region.origin = offset1; region.size = (size_t)K * N * sizeof(float); CL_CHECK((b_sub = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); cl_int tb = (cl_int)((size_t)N * (K / 32)); cl_kernel qk = backend_ctx->kernel_quant_a_q8_1; CL_CHECK(clSetKernelArg(qk, 0, sizeof(cl_mem), &b_sub)); CL_CHECK(clSetKernelArg(qk, 1, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer)); CL_CHECK(clSetKernelArg(qk, 2, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer)); CL_CHECK(clSetKernelArg(qk, 3, sizeof(cl_mem), &backend_ctx->prealloc_moe_sa.buffer)); CL_CHECK(clSetKernelArg(qk, 4, sizeof(cl_int), &tb)); size_t q_local[1] = { 64 }; size_t q_global[1] = { (size_t)CEIL_DIV(tb, 64) * 64 }; backend_ctx->enqueue_ndrange_kernel(qk, 1, q_global, q_local, dst); cl_mem d_sub = nullptr; cl_mem d_img = nullptr; region.origin = offsetd; region.size = (size_t)M * N * sizeof(float); CL_CHECK((d_sub = clCreateSubBuffer(extrad->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); img_fmt = { CL_R, CL_FLOAT }; memset(&img_desc, 0, sizeof(img_desc)); img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; img_desc.image_width = (size_t)M * N; img_desc.buffer = d_sub; CL_CHECK((d_img = clCreateImage(context, CL_MEM_WRITE_ONLY, &img_fmt, &img_desc, NULL, &err), err)); kernel = backend_ctx->kernel_gemm_noshuffle_q6_k_q8_1_dp4a_ila_a8_bin; cl_uint k_arg = 0; CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &extra0_q6_K->ql_img)); CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &extra0_q6_K->qh)); CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &extra0_q6_K->s)); CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &extra0_q6_K->d)); CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer)); CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer)); CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &d_img)); CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &K)); CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &M)); CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &N)); size_t local_work_size[3] = { 64, 1, 1 }; size_t global_work_size[3] = { 64, (size_t)(M / 64), (size_t)(dp4a_N_pad / 32) }; backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); CL_CHECK(clReleaseMemObject(b_sub)); CL_CHECK(clReleaseMemObject(d_img)); CL_CHECK(clReleaseMemObject(d_sub)); return; } cl_mem b_sub_buf = nullptr; cl_mem b_padded = nullptr; cl_mem b_buf = nullptr; if (N_pad == N) { region.origin = offset1; region.size = (size_t)K * N * sizeof(float); CL_CHECK((b_sub_buf = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); b_buf = b_sub_buf; } else { CL_CHECK((b_padded = clCreateBuffer(context, CL_MEM_READ_WRITE, (size_t)K * N_pad * sizeof(float), NULL, &err), err)); const float zero = 0.0f; CL_CHECK(clEnqueueFillBuffer(backend_ctx->queue, b_padded, &zero, sizeof(zero), 0, (size_t)K * N_pad * sizeof(float), 0, NULL, NULL)); CL_CHECK(clEnqueueCopyBuffer(backend_ctx->queue, extra1->data_device, b_padded, offset1, 0, (size_t)K * N * sizeof(float), 0, NULL, NULL)); b_buf = b_padded; } img_fmt = { CL_R, CL_FLOAT }; memset(&img_desc, 0, sizeof(img_desc)); img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; img_desc.image_width = (size_t)K * N_pad; img_desc.buffer = b_buf; cl_mem b_img; CL_CHECK((b_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err), err)); region.origin = offsetd; region.size = (size_t)M * N * sizeof(float); cl_mem d_sub_buf; CL_CHECK((d_sub_buf = clCreateSubBuffer(extrad->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); img_fmt = { CL_R, CL_FLOAT }; memset(&img_desc, 0, sizeof(img_desc)); img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; img_desc.image_width = (size_t)M * N; img_desc.buffer = d_sub_buf; cl_mem d_img; CL_CHECK((d_img = clCreateImage(context, CL_MEM_WRITE_ONLY, &img_fmt, &img_desc, NULL, &err), err)); kernel = backend_ctx->kernel_gemm_noshuffle_q6_k_f32_32b_trans_ila_a8_bin; CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_q6_K->ql_img)); CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q6_K->qh)); CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra0_q6_K->s)); CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra0_q6_K->d)); CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &b_img)); CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_mem), &d_img)); CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_uint), &ne00)); CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_uint), &ne01)); CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &N)); size_t local_work_size[3] = { 64, 2, 2 }; size_t m_tiles = (size_t)CEIL_DIV(M, 64); size_t global_work_size[3] = { 64, m_tiles, (size_t)CEIL_DIV(N_pad, gemm_tile_n) }; backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); CL_CHECK(clReleaseMemObject(b_img)); if (b_sub_buf) { CL_CHECK(clReleaseMemObject(b_sub_buf)); } if (b_padded) { CL_CHECK(clReleaseMemObject(b_padded)); } CL_CHECK(clReleaseMemObject(d_img)); CL_CHECK(clReleaseMemObject(d_sub_buf)); } } #endif // GGML_OPENCL_USE_ADRENO_KERNELS static void ggml_cl_mul_mat_q6_K_f32_adreno(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { #ifdef GGML_OPENCL_USE_ADRENO_KERNELS GGML_ASSERT(src0); GGML_ASSERT(src0->extra); GGML_ASSERT(src1); GGML_ASSERT(src1->extra); GGML_ASSERT(dst); GGML_ASSERT(dst->extra); ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; ggml_tensor_extra_cl_q6_K * extra0_q6_K = (ggml_tensor_extra_cl_q6_K *)src0->extra; ggml_tensor_extra_cl * extra1 = (ggml_tensor_extra_cl *)src1->extra; ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; cl_ulong offset1 = extra1->offset + src1->view_offs; cl_ulong offsetd = extrad->offset + dst->view_offs; const int ne00 = src0->ne[0]; const int ne01 = src0->ne[1]; const int ne1 = dst->ne[1]; GGML_ASSERT(ne00 % ggml_blck_size(src0->type) == 0); cl_context context = backend_ctx->context; cl_kernel kernel; cl_int err; cl_buffer_region region; cl_image_format img_fmt; cl_image_desc img_desc; // subbuffer and image for activation // Multi-column verify GEMV: route the spec/MTP verify q6_K matmuls (ne1==3) // onto the efficient GEMV path instead of the transposed-GEMM dead-zone. // Reuses the ne1==1 image setup (activation image sized by N=ne1). Byte- // identical. Opt-in via GGML_OPENCL_Q6K_MC3=1 while validating. static const bool q6k_mc3 = (getenv("GGML_OPENCL_Q6K_MC3") != nullptr); // Per-layer only (ne01 < 32768): batched large-vocab lm_head stays on the // existing path (x2-unified routes batched Q6_K lm_head to CPU; the Adreno // GEMV corrupts it). Per-layer mc3 is byte-identical. const bool use_q6k_mc3 = q6k_mc3 && (ne1 == 3) && (ne01 < 32768); // Batched verify lm_head/embed (ne1==3, tiled layout): multi-column tiled // GEMV — streams the large lm_head weight once across the 3 verify columns // (the #1 MTP bottleneck; mc3 above can't, it reads the noshuffle layout). const bool use_q6k_tiled_mc = q6k_mc3 && (ne1 == 3) && (ne01 >= 32768) && use_q6k_tiled(backend_ctx, src0); const bool use_bin = use_q6_k_bin_kernels(backend_ctx, src0); if (use_bin) { if (use_q6k_mc3 || use_q6k_tiled_mc) { static bool warned = false; if (!warned) { GGML_LOG_WARN("ggml_opencl: GGML_OPENCL_Q6K_MC3 is bypassed by Q6_K binary kernels\n"); warned = true; } } ggml_cl_mul_mat_q6_K_f32_adreno_ila(backend, src0, src1, dst); return; } if (ne1 == 1 || use_q6k_mc3 || use_q6k_tiled_mc) { cl_mem ql_img = nullptr; cl_mem qh_img = nullptr; cl_mem b_sub_buffer = nullptr; cl_mem b_img = nullptr; // o4 = 4-output-per-WI variant for long-vocab lm_head/embed; gated to // ne01 >= 32768 so per-layer q6_K (ne01=hidden 2-8K) keeps the 2-output // kernel (o4 regresses there). o4_global reads the weights from __global // coalesced instead of image1d_buffer -- the texture cache caps the // read-once-per-token lm_head bandwidth, while __global reaches the higher // rate the rest of the model gets. Both default ON; opt out via // GGML_OPENCL_Q6K_GEMV_O4 / GGML_OPENCL_Q6K_GEMV_O4_GLOBAL = 0. static const bool gemv_o4_env = []{ const char * e = std::getenv("GGML_OPENCL_Q6K_GEMV_O4"); return !e || e[0] == '\0' || e[0] != '0'; }(); static const bool o4_global_env = []{ const char * e = std::getenv("GGML_OPENCL_Q6K_GEMV_O4_GLOBAL"); return !e || e[0] == '\0' || e[0] != '0'; }(); const bool use_tiled = !use_q6k_mc3 && use_q6k_tiled(backend_ctx, src0); const bool use_o4 = !use_tiled && !use_q6k_mc3 && gemv_o4_env && (ne01 % 4 == 0) && (ne01 >= 32768); const bool use_o4_global = use_o4 && o4_global_env; // ql/qh image views are only needed when NOT reading weights from global. if (!use_o4_global && !use_tiled) { // image for ql img_fmt.image_channel_order = CL_R; img_fmt.image_channel_data_type = CL_FLOAT; memset(&img_desc, 0, sizeof(img_desc)); img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; img_desc.image_width = ne01 * ne00 / 8; img_desc.buffer = extra0_q6_K->ql; CL_CHECK((ql_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err), err)); // image for qh img_fmt.image_channel_order = CL_R; img_fmt.image_channel_data_type = CL_HALF_FLOAT; memset(&img_desc, 0, sizeof(img_desc)); img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; img_desc.image_width = ne01 * ne00 / 8; img_desc.buffer = extra0_q6_K->qh; CL_CHECK((qh_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err), err)); } region.origin = offset1; region.size = ne00 * ne1 * sizeof(float); CL_CHECK((b_sub_buffer = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); img_fmt.image_channel_order = CL_RGBA; img_fmt.image_channel_data_type = CL_FLOAT; memset(&img_desc, 0, sizeof(img_desc)); img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; img_desc.image_width = ne00 * ne1 / 4; img_desc.buffer = b_sub_buffer; CL_CHECK((b_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err), err)); kernel = use_q6k_mc3 ? backend_ctx->kernel_gemv_noshuffle_q6_K_f32_mc3 : use_q6k_tiled_mc ? backend_ctx->kernel_gemv_noshuffle_q6_K_f32_tiled_mc3 : use_tiled ? backend_ctx->kernel_gemv_noshuffle_q6_K_f32_tiled : use_o4_global ? backend_ctx->kernel_gemv_noshuffle_q6_K_f32_o4_global : use_o4 ? backend_ctx->kernel_gemv_noshuffle_q6_K_f32_o4 : backend_ctx->kernel_gemv_noshuffle_q6_K_f32; if (use_o4_global || use_tiled) { CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_q6_K->ql)); CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q6_K->qh)); } else { CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &ql_img)); CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &qh_img)); } CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra0_q6_K->s)); CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra0_q6_K->d)); CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &b_img)); CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_mem), &extrad->data_device)); CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_ulong), &offsetd)); CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_int), &ne00)); CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_int), &ne01)); const size_t gws_x = use_tiled ? (size_t) CEIL_DIV(ne01, 64) * 64 : use_o4 ? (size_t) CEIL_DIV(ne01/4, 64) * 64 : (size_t) CEIL_DIV(ne01/2, 64) * 64; size_t local_work_size[3] = {64, 4, 1}; size_t global_work_size[3] = {gws_x, 4, 1}; backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); if (ql_img) CL_CHECK(clReleaseMemObject(ql_img)); if (qh_img) CL_CHECK(clReleaseMemObject(qh_img)); CL_CHECK(clReleaseMemObject(b_sub_buffer)); CL_CHECK(clReleaseMemObject(b_img)); } else { // Tiled-layout batched GEMM. When the weight was converted to the 64-row // tiled canonical layout (use_q6k_tiled — the default for lm_head/embed), // the plain noshuffle GEMM below reads it as plain-transposed and produces // garbage. Use the batched GEMM that matches the decode tiled GEMV's // layout; it reads the f32 activation directly (column-major, no transpose). if (use_q6k_tiled(backend_ctx, src0)) { cl_mem b_sub_buf_t = nullptr; cl_mem b_img_t = nullptr; region.origin = offset1; region.size = ne00 * ne1 * sizeof(float); CL_CHECK((b_sub_buf_t = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); img_fmt.image_channel_order = CL_RGBA; img_fmt.image_channel_data_type = CL_FLOAT; memset(&img_desc, 0, sizeof(img_desc)); img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; img_desc.image_width = ne00 * ne1 / 4; img_desc.buffer = b_sub_buf_t; CL_CHECK((b_img_t = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err), err)); cl_kernel kt = backend_ctx->kernel_gemm_noshuffle_q6_K_f32_tiled; CL_CHECK(clSetKernelArg(kt, 0, sizeof(cl_mem), &extra0_q6_K->ql)); CL_CHECK(clSetKernelArg(kt, 1, sizeof(cl_mem), &extra0_q6_K->qh)); CL_CHECK(clSetKernelArg(kt, 2, sizeof(cl_mem), &extra0_q6_K->s)); CL_CHECK(clSetKernelArg(kt, 3, sizeof(cl_mem), &extra0_q6_K->d)); CL_CHECK(clSetKernelArg(kt, 4, sizeof(cl_mem), &b_img_t)); CL_CHECK(clSetKernelArg(kt, 5, sizeof(cl_mem), &extrad->data_device)); CL_CHECK(clSetKernelArg(kt, 6, sizeof(cl_ulong), &offsetd)); CL_CHECK(clSetKernelArg(kt, 7, sizeof(int), &ne00)); CL_CHECK(clSetKernelArg(kt, 8, sizeof(int), &ne01)); CL_CHECK(clSetKernelArg(kt, 9, sizeof(int), &ne1)); // Must match the kernel: NTILES=4 64-row tiles per work-group (256 rows), // BN=8 output columns per work-group. const int BN_T = 16; const int WROWS = 4 * 64; // NTILES * TILE_ROWS size_t local_work_size[3] = {64, 4, 1}; size_t global_work_size[3] = {(size_t)CEIL_DIV(ne01, WROWS) * 64, 4, (size_t)CEIL_DIV(ne1, BN_T)}; backend_ctx->enqueue_ndrange_kernel(kt, 3, global_work_size, local_work_size, dst); CL_CHECK(clReleaseMemObject(b_img_t)); CL_CHECK(clReleaseMemObject(b_sub_buf_t)); return; } cl_mem b_sub_buf; cl_mem b_buf_trans; cl_mem b_img; cl_mem b_img_trans; // subbuffer for activation region.origin = offset1; region.size = ne00 * ne1 * sizeof(float); CL_CHECK((b_sub_buf = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); // dp4a (int8) dense q6_K prefill GEMM static const char * q6k_dense_dp4a_env = getenv("GGML_OPENCL_Q6K_DENSE_DP4A"); bool q6k_dense_dp4a_on = (q6k_dense_dp4a_env != nullptr) ? (atoi(q6k_dense_dp4a_env) != 0) : (backend_ctx->adreno_gen != ADRENO_GPU_GEN::X1E); // dot prod has to be available q6k_dense_dp4a_on = backend_ctx->has_integer_dot && q6k_dense_dp4a_on; const bool is_output_w_dp4a = strncmp(src0->name, "output", 6) == 0 || strncmp(src0->name, "token_embd", 10) == 0; if (q6k_dense_dp4a_on && !is_output_w_dp4a && ne1 > 8 && (ne00 % 32 == 0) && (ne01 % 64 == 0)) { const int M = ne01, N = ne1, K = ne00; const size_t n_blocks = (size_t)N * (K / 32); backend_ctx->prealloc_moe_qa.allocate(context, (size_t)N * K * sizeof(cl_char)); backend_ctx->prealloc_moe_da.allocate(context, n_blocks * sizeof(cl_half)); backend_ctx->prealloc_moe_sa.allocate(context, n_blocks * sizeof(cl_half)); cl_int tb = (cl_int)n_blocks; cl_kernel qk = backend_ctx->kernel_quant_a_q8_1; CL_CHECK(clSetKernelArg(qk, 0, sizeof(cl_mem), &b_sub_buf)); CL_CHECK(clSetKernelArg(qk, 1, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer)); CL_CHECK(clSetKernelArg(qk, 2, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer)); CL_CHECK(clSetKernelArg(qk, 3, sizeof(cl_mem), &backend_ctx->prealloc_moe_sa.buffer)); CL_CHECK(clSetKernelArg(qk, 4, sizeof(cl_int), &tb)); size_t q_local[1] = { 64 }; size_t q_global[1] = { (size_t)(((n_blocks + 63) / 64) * 64) }; backend_ctx->enqueue_ndrange_kernel(qk, 1, q_global, q_local, dst); cl_kernel dk = backend_ctx->kernel_gemm_noshuffle_q6_k_q8_1_dp4a; int ai = 0; CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &extra0_q6_K->ql)); CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &extra0_q6_K->qh)); CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &extra0_q6_K->s)); CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &extra0_q6_K->d)); CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer)); CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer)); CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &extrad->data_device)); CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_ulong), &offsetd)); CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_int), &M)); CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_int), &N)); CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_int), &K)); size_t d_local[3] = { 64, 1, 1 }; size_t d_global[3] = { 64, (size_t)(M / 64), (size_t)CEIL_DIV(N, 32) }; backend_ctx->enqueue_ndrange_kernel(dk, 3, d_global, d_local, dst); CL_CHECK(clReleaseMemObject(b_sub_buf)); return; } // image for activation img_fmt.image_channel_order = CL_RGBA; img_fmt.image_channel_data_type = CL_FLOAT; memset(&img_desc, 0, sizeof(img_desc)); img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; img_desc.image_width = ne00 * ne1 / 4; img_desc.buffer = b_sub_buf; CL_CHECK((b_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err), err)); // pad N to multiple of 8 int extra_elements = ne1 % 8; int padding = 0; if (extra_elements > 0){ padding = 8 - extra_elements; } // subbuffer for transposed activation region.origin = 0; region.size = ne00 * (ne1 + padding) * sizeof(float)/2; backend_ctx->prealloc_act_trans.allocate(context, region.size); CL_CHECK((b_buf_trans = clCreateSubBuffer(backend_ctx->prealloc_act_trans.buffer, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); // image for transposed activation img_fmt.image_channel_order = CL_RGBA; img_fmt.image_channel_data_type = CL_HALF_FLOAT; memset(&img_desc, 0, sizeof(img_desc)); img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; img_desc.image_width = ne00 * (ne1 + padding) / 4; img_desc.buffer = b_buf_trans; CL_CHECK((b_img_trans = clCreateImage(context, 0, &img_fmt, &img_desc, NULL, &err), err)); // transpose activation int height_B = ne1/4; if (height_B == 0) { height_B = 1; } int width_B = ne00/4; int padded_height_B = (ne1 + padding) / 4; kernel = backend_ctx->kernel_transpose_32_16; CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &b_img)); CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &b_img_trans)); CL_CHECK(clSetKernelArg(kernel, 2, sizeof(int), &height_B)); CL_CHECK(clSetKernelArg(kernel, 3, sizeof(int), &width_B)); CL_CHECK(clSetKernelArg(kernel, 4, sizeof(int), &padded_height_B)); size_t local_size_t[2] = { 1, 16 }; size_t global_size_t[2] = { (size_t)width_B, (size_t)padded_height_B }; backend_ctx->enqueue_ndrange_kernel(kernel, 2, global_size_t, local_size_t, dst); // gemm // Cooperative-K small-batch (n_q in [2..8]) path: intra-WG K-split, // mirrors the q4_K _cok path (batched serving). OPT-IN // (GGML_OPENCL_Q6K_GEMM_COK=1), DEFAULT OFF: q6_K is the tied lm_head/ // output projection, so the K-reassociation perturbs final logits and // greedy is NOT byte-identical (op-tests pass, output coherent, but not // bit-exact). It is also NEUTRAL on end-to-end MTP (q4_K cok already // captured that; the MTP bottleneck moved off the GEMMs). Keep opt-in // for batched serving until PPL-validated on a non-GDN q6_K model. static const char * q6k_cok_env = getenv("GGML_OPENCL_Q6K_GEMM_COK"); static const bool q6k_gemm_cok = (q6k_cok_env != nullptr) && (atoi(q6k_cok_env) != 0); const bool use_q6k_cok = q6k_gemm_cok && (ne1 <= 8); kernel = use_q6k_cok ? backend_ctx->kernel_gemm_noshuffle_q6_K_f32_cok : backend_ctx->kernel_gemm_noshuffle_q6_K_f32; int padded_N = ne1 + padding; cl_ushort mask_f000 = 0xF000; cl_uchar mask_c0 = 0xC0; CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_q6_K->ql)); CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q6_K->qh)); CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra0_q6_K->s)); CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra0_q6_K->d)); CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &b_img_trans)); CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_mem), &extrad->data_device)); CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_ulong), &offsetd)); CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne01)); CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &padded_N)); CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne00)); CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &ne1)); CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_ushort),&mask_f000)); CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_uchar), &mask_c0)); size_t global_work_size[3]; size_t local_work_size[3]; if (use_q6k_cok) { global_work_size[0] = (size_t)ne01; // rows (1 per lane) global_work_size[1] = 8; // COK_NSG global_work_size[2] = 1; local_work_size[0] = 64; // COK_SG local_work_size[1] = 8; // COK_NSG local_work_size[2] = 1; } else { global_work_size[0] = (size_t)CEIL_DIV(ne1, 8); global_work_size[1] = (size_t)CEIL_DIV(ne01, 4); global_work_size[2] = 1; local_work_size[0] = 2; local_work_size[1] = 128; local_work_size[2] = 1; } backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); CL_CHECK(clReleaseMemObject(b_sub_buf)); CL_CHECK(clReleaseMemObject(b_img)); CL_CHECK(clReleaseMemObject(b_buf_trans)); CL_CHECK(clReleaseMemObject(b_img_trans)); } #else GGML_UNUSED(backend); GGML_UNUSED(src0); GGML_UNUSED(src1); GGML_UNUSED(dst); #endif } #ifdef GGML_OPENCL_USE_ADRENO_KERNELS static void ggml_cl_mul_mat_q5_K_f32_adreno_ila(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { GGML_ASSERT(src0); GGML_ASSERT(src0->extra); GGML_ASSERT(src1); GGML_ASSERT(src1->extra); GGML_ASSERT(dst); GGML_ASSERT(dst->extra); ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; ggml_tensor_extra_cl_q5_K * extra0_q5_k = (ggml_tensor_extra_cl_q5_K *)src0->extra; ggml_tensor_extra_cl * extra1 = (ggml_tensor_extra_cl *)src1->extra; ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; cl_ulong offset1 = extra1->offset + src1->view_offs; cl_ulong offsetd = extrad->offset + dst->view_offs; const int ne00 = src0->ne[0]; const int ne01 = src0->ne[1]; const int ne1 = dst->ne[1]; GGML_ASSERT(ne00 % ggml_blck_size(src0->type) == 0); cl_context context = backend_ctx->context; cl_kernel kernel; cl_int err; cl_buffer_region region; cl_image_format img_fmt; cl_image_desc img_desc; const int M = ne01; const int N = ne1; const int K = ne00; if (ne1 == 1) { cl_mem b_sub_buf = nullptr; cl_mem b_img = nullptr; region.origin = offset1; region.size = (size_t)K * N * sizeof(float); CL_CHECK((b_sub_buf = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); img_fmt = { CL_RGBA, CL_FLOAT }; memset(&img_desc, 0, sizeof(img_desc)); img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; img_desc.image_width = (size_t)K * N / 4; img_desc.buffer = b_sub_buf; CL_CHECK((b_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err), err)); kernel = backend_ctx->kernel_gemv_noshuffle_q5_k_f32_32b_trans; CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_q5_k->q_img)); CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q5_k->qh)); CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra0_q5_k->d)); CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra0_q5_k->dm)); CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extra0_q5_k->s)); CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_mem), &b_img)); CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_mem), &extrad->data_device)); CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_ulong), &offsetd)); CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_int), &ne00)); CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_int), &ne01)); size_t local_work_size[3] = { 64, 8, 1 }; size_t global_work_size[3] = { (size_t)ne01, 8, 1 }; backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); CL_CHECK(clReleaseMemObject(b_img)); CL_CHECK(clReleaseMemObject(b_sub_buf)); } else { static const char * q5_k_bin_dp4a_env = getenv("GGML_OPENCL_Q5_K_BIN_DP4A"); bool q5_k_bin_dp4a_on = q5_k_bin_dp4a_env ? (atoi(q5_k_bin_dp4a_env) != 0) : true; // dot prod has to be available q5_k_bin_dp4a_on = backend_ctx->has_integer_dot && q5_k_bin_dp4a_on; if (q5_k_bin_dp4a_on && backend_ctx->kernel_gemm_noshuffle_q5_k_q8_1_dp4a_ila_a8_bin) { const int dp4a_N_pad = CEIL_DIV(N, 32) * 32; const size_t n_blocks = (size_t)dp4a_N_pad * (K / 32); backend_ctx->prealloc_moe_qa.allocate(context, (size_t)dp4a_N_pad * K * sizeof(cl_char)); backend_ctx->prealloc_moe_da.allocate(context, n_blocks * sizeof(cl_half)); backend_ctx->prealloc_moe_sa.allocate(context, n_blocks * sizeof(cl_half)); cl_mem b_sub = nullptr; region.origin = offset1; region.size = (size_t)K * N * sizeof(float); CL_CHECK((b_sub = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); cl_int tb = (cl_int)((size_t)N * (K / 32)); cl_kernel qk = backend_ctx->kernel_quant_a_q8_1; CL_CHECK(clSetKernelArg(qk, 0, sizeof(cl_mem), &b_sub)); CL_CHECK(clSetKernelArg(qk, 1, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer)); CL_CHECK(clSetKernelArg(qk, 2, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer)); CL_CHECK(clSetKernelArg(qk, 3, sizeof(cl_mem), &backend_ctx->prealloc_moe_sa.buffer)); CL_CHECK(clSetKernelArg(qk, 4, sizeof(cl_int), &tb)); size_t q_local[1] = { 64 }; size_t q_global[1] = { (size_t)CEIL_DIV(tb, 64) * 64 }; backend_ctx->enqueue_ndrange_kernel(qk, 1, q_global, q_local, dst); cl_mem d_sub = nullptr; cl_mem d_img = nullptr; region.origin = offsetd; region.size = (size_t)M * N * sizeof(float); CL_CHECK((d_sub = clCreateSubBuffer(extrad->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); img_fmt = { CL_R, CL_FLOAT }; memset(&img_desc, 0, sizeof(img_desc)); img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; img_desc.image_width = (size_t)M * N; img_desc.buffer = d_sub; CL_CHECK((d_img = clCreateImage(context, CL_MEM_WRITE_ONLY, &img_fmt, &img_desc, NULL, &err), err)); kernel = backend_ctx->kernel_gemm_noshuffle_q5_k_q8_1_dp4a_ila_a8_bin; cl_uint k_arg = 0; CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &extra0_q5_k->q_img)); CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &extra0_q5_k->qh)); CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &extra0_q5_k->d)); CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &extra0_q5_k->dm)); CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &extra0_q5_k->s)); CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer)); CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer)); CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &backend_ctx->prealloc_moe_sa.buffer)); CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &d_img)); CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_uint), &ne00)); CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_uint), &ne01)); CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_int), &N)); size_t local_work_size_dp4a[3] = { 64, 1, 1 }; size_t global_work_size_dp4a[3] = { 64, (size_t)(M / 64), (size_t)(dp4a_N_pad / 32) }; backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size_dp4a, local_work_size_dp4a, dst); CL_CHECK(clReleaseMemObject(b_sub)); CL_CHECK(clReleaseMemObject(d_img)); CL_CHECK(clReleaseMemObject(d_sub)); return; } const int gemm_tile_n = 64; int N_pad = CEIL_DIV(N, gemm_tile_n) * gemm_tile_n; cl_mem b_sub_buf = nullptr; cl_mem b_padded = nullptr; cl_mem b_buf = nullptr; if (N_pad == N) { region.origin = offset1; region.size = (size_t)K * N * sizeof(float); CL_CHECK((b_sub_buf = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); b_buf = b_sub_buf; } else { CL_CHECK((b_padded = clCreateBuffer(context, CL_MEM_READ_WRITE, (size_t)K * N_pad * sizeof(float), NULL, &err), err)); const float zero = 0.0f; CL_CHECK(clEnqueueFillBuffer(backend_ctx->queue, b_padded, &zero, sizeof(zero), 0, (size_t)K * N_pad * sizeof(float), 0, NULL, NULL)); CL_CHECK(clEnqueueCopyBuffer(backend_ctx->queue, extra1->data_device, b_padded, offset1, 0, (size_t)K * N * sizeof(float), 0, NULL, NULL)); b_buf = b_padded; } img_fmt = { CL_R, CL_FLOAT }; memset(&img_desc, 0, sizeof(img_desc)); img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; img_desc.image_width = (size_t)K * N_pad; img_desc.buffer = b_buf; cl_mem b_img; CL_CHECK((b_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err), err)); region.origin = offsetd; region.size = (size_t)M * N * sizeof(float); cl_mem d_sub_buf; CL_CHECK((d_sub_buf = clCreateSubBuffer(extrad->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); img_fmt = { CL_R, CL_FLOAT }; memset(&img_desc, 0, sizeof(img_desc)); img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; img_desc.image_width = (size_t)M * N; img_desc.buffer = d_sub_buf; cl_mem d_img; CL_CHECK((d_img = clCreateImage(context, CL_MEM_WRITE_ONLY, &img_fmt, &img_desc, NULL, &err), err)); kernel = backend_ctx->kernel_gemm_noshuffle_q5_k_f32_32b_trans_ila_a8_bin; CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_q5_k->q_img)); CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q5_k->qh)); CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra0_q5_k->d)); CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra0_q5_k->dm)); CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extra0_q5_k->s)); CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_mem), &b_img)); CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_mem), &d_img)); CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_uint), &ne00)); CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_uint), &ne01)); CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &N)); size_t local_work_size[3] = { 64, 2, 2 }; size_t m_tiles = (size_t)CEIL_DIV(M, 64); size_t global_work_size[3] = { 64, m_tiles, (size_t)CEIL_DIV(N_pad, gemm_tile_n) }; backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); CL_CHECK(clReleaseMemObject(b_img)); if (b_sub_buf) { CL_CHECK(clReleaseMemObject(b_sub_buf)); } if (b_padded) { CL_CHECK(clReleaseMemObject(b_padded)); } CL_CHECK(clReleaseMemObject(d_img)); CL_CHECK(clReleaseMemObject(d_sub_buf)); } } #endif // GGML_OPENCL_USE_ADRENO_KERNELS static void ggml_cl_mul_mat_q5_K_f32_adreno(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { #ifdef GGML_OPENCL_USE_ADRENO_KERNELS GGML_ASSERT(src0); GGML_ASSERT(src0->extra); GGML_ASSERT(src1); GGML_ASSERT(src1->extra); GGML_ASSERT(dst); GGML_ASSERT(dst->extra); ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; ggml_tensor_extra_cl * extra1 = (ggml_tensor_extra_cl *)src1->extra; ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; ggml_tensor_extra_cl_q5_K * extra0_q5_k = (ggml_tensor_extra_cl_q5_K *)src0->extra; cl_ulong offset1 = extra1->offset + src1->view_offs; cl_ulong offsetd = extrad->offset + dst->view_offs; const int ne00 = src0->ne[0]; const int ne01 = src0->ne[1]; const int ne1 = dst->ne[1]; GGML_ASSERT(ne00 % ggml_blck_size(src0->type) == 0); cl_context context = backend_ctx->context; cl_kernel kernel; cl_int err; cl_image_format img_fmt; cl_image_desc img_desc; cl_buffer_region region; int M = ne01; int N = ne1; int K = ne00; cl_uchar mask_d6 = 0x3F; cl_uchar mask_d4 = 0x0F; cl_uchar mask_hi2 = 0xC0; // Multi-column (N=3) verify GEMV for q5_K: route the spec/MTP verify batch // (ne1==3) onto the efficient GEMV path instead of the transposed-GEMM dead- // zone (gemm_noshuffle_q5_k, the #2 chunk of MTP decode on a Q4_0-mix model // after q4_0 mc3). Reuses the ne1==1 GEMV image setup (q + qh + activations). // Opt-in via GGML_OPENCL_Q5K_MC3=1. Per-layer only (ne01 < 32768). static const bool q5k_mc3 = (getenv("GGML_OPENCL_Q5K_MC3") != nullptr); const bool use_q5k_mc3 = q5k_mc3 && (ne1 >= 2 && ne1 <= 4) && (ne01 < 32768); const bool use_bin = use_q5_k_bin_kernels(backend_ctx, src0); if (use_bin) { if (use_q5k_mc3) { static bool warned = false; if (!warned) { GGML_LOG_WARN("ggml_opencl: GGML_OPENCL_Q5K_MC3 is bypassed by Q5_K binary kernels\n"); warned = true; } } ggml_cl_mul_mat_q5_K_f32_adreno_ila(backend, src0, src1, dst); return; } if (ne1 == 1 || use_q5k_mc3) { cl_mem q_img = nullptr; cl_mem qh_img = nullptr; cl_mem b_sub_buf = nullptr; cl_mem b_img = nullptr; // image for q (CL_R, CL_UNSIGNED_INT32): width = M*K/2/4 img_fmt = {CL_R, CL_UNSIGNED_INT32}; memset(&img_desc, 0, sizeof(img_desc)); img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; img_desc.image_width = M * K / 2 / 4; img_desc.buffer = extra0_q5_k->q; CL_CHECK((q_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err), err)); // image for qh (CL_R, CL_HALF_FLOAT): width = M*K/16 img_fmt = {CL_R, CL_HALF_FLOAT}; memset(&img_desc, 0, sizeof(img_desc)); img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; img_desc.image_width = M * K / 16; img_desc.buffer = extra0_q5_k->qh; CL_CHECK((qh_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err), err)); // subbuffer for activations region.origin = offset1; region.size = K * N * sizeof(float); CL_CHECK((b_sub_buf = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); // image for activations (CL_RGBA, CL_FLOAT): width = K*N/4 img_fmt = {CL_RGBA, CL_FLOAT}; memset(&img_desc, 0, sizeof(img_desc)); img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; img_desc.image_width = K * N / 4; img_desc.buffer = b_sub_buf; CL_CHECK((b_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err), err)); kernel = use_q5k_mc3 ? backend_ctx->kernel_gemv_noshuffle_q5_k_f32_mc3 : backend_ctx->kernel_gemv_noshuffle_q5_k_f32; CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &q_img)); CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &qh_img)); CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra0_q5_k->d)); CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra0_q5_k->dm)); CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extra0_q5_k->s)); CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_mem), &b_img)); CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_mem), &extrad->data_device)); CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_ulong), &offsetd)); CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_int), &ne00)); CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_int), &ne01)); CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_uchar), &mask_d6)); CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_uchar), &mask_d4)); CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_uchar), &mask_hi2)); if (use_q5k_mc3) { CL_CHECK(clSetKernelArg(kernel, 13, sizeof(cl_int), &ne1)); // n_cols } size_t local_work_size[3] = {64, 4, 1}; size_t global_work_size[3] = {(size_t)CEIL_DIV(ne01/2, 64)*64, 4, 1}; backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); CL_CHECK(clReleaseMemObject(q_img)); CL_CHECK(clReleaseMemObject(qh_img)); CL_CHECK(clReleaseMemObject(b_sub_buf)); CL_CHECK(clReleaseMemObject(b_img)); } else { cl_mem b_sub_buf = nullptr; cl_mem b_sub_buf_trans = nullptr; cl_mem b_img = nullptr; cl_mem b_img_trans = nullptr; // subbuffer for activations region.origin = offset1; region.size = K * N * sizeof(float); CL_CHECK((b_sub_buf = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); // image for activations img_fmt = {CL_RGBA, CL_FLOAT}; memset(&img_desc, 0, sizeof(img_desc)); img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; img_desc.image_width = K * N / 4; img_desc.buffer = b_sub_buf; CL_CHECK((b_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err), err)); // pad N to multiple of 8 int extra_elements = N % 8; int padding = 0; if (extra_elements > 0) { padding = 8 - extra_elements; } // subbuffer for transposed activations region.origin = 0; region.size = K * (N + padding) * sizeof(float) / 2; backend_ctx->prealloc_act_trans.allocate(context, region.size); CL_CHECK((b_sub_buf_trans = clCreateSubBuffer(backend_ctx->prealloc_act_trans.buffer, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); // image for transposed activations img_fmt = {CL_RGBA, CL_HALF_FLOAT}; memset(&img_desc, 0, sizeof(img_desc)); img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; img_desc.image_width = K * (N + padding) / 4; img_desc.buffer = b_sub_buf_trans; CL_CHECK((b_img_trans = clCreateImage(context, 0, &img_fmt, &img_desc, NULL, &err), err)); // transpose activations int height_B = N / 4; if (height_B == 0) height_B = 1; int width_B = K / 4; int padded_height_B = (N + padding) / 4; kernel = backend_ctx->kernel_transpose_32_16; CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &b_img)); CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &b_img_trans)); CL_CHECK(clSetKernelArg(kernel, 2, sizeof(int), &height_B)); CL_CHECK(clSetKernelArg(kernel, 3, sizeof(int), &width_B)); CL_CHECK(clSetKernelArg(kernel, 4, sizeof(int), &padded_height_B)); size_t local_work_size_t[2] = {1, 16}; size_t global_work_size_t[2] = {(size_t)width_B, (size_t)padded_height_B}; backend_ctx->enqueue_ndrange_kernel(kernel, 2, global_work_size_t, local_work_size_t, dst); // dp4a (int8) dense q5_K prefill GEMM static const char * q5k_dense_dp4a_env = getenv("GGML_OPENCL_Q5K_DENSE_DP4A"); bool q5k_dense_dp4a_on = q5k_dense_dp4a_env ? (atoi(q5k_dense_dp4a_env) != 0) : (backend_ctx->adreno_gen == ADRENO_GPU_GEN::X2E); // dot prod has to be available q5k_dense_dp4a_on = backend_ctx->has_integer_dot && q5k_dense_dp4a_on; if (q5k_dense_dp4a_on && ne1 > 8 && (ne00 % 32 == 0) && (ne01 % 64 == 0)) { const int Mm = ne01, Nn = ne1, Kk = ne00; const size_t n_blocks = (size_t)Nn * (Kk / 32); backend_ctx->prealloc_moe_qa.allocate(context, (size_t)Nn * Kk * sizeof(cl_char)); backend_ctx->prealloc_moe_da.allocate(context, n_blocks * sizeof(cl_half)); backend_ctx->prealloc_moe_sa.allocate(context, n_blocks * sizeof(cl_half)); cl_int tb = (cl_int)n_blocks; cl_kernel qk = backend_ctx->kernel_quant_a_q8_1; CL_CHECK(clSetKernelArg(qk, 0, sizeof(cl_mem), &b_sub_buf)); CL_CHECK(clSetKernelArg(qk, 1, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer)); CL_CHECK(clSetKernelArg(qk, 2, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer)); CL_CHECK(clSetKernelArg(qk, 3, sizeof(cl_mem), &backend_ctx->prealloc_moe_sa.buffer)); CL_CHECK(clSetKernelArg(qk, 4, sizeof(cl_int), &tb)); size_t q_local[1] = { 64 }; size_t q_global[1] = { (size_t)(((n_blocks + 63) / 64) * 64) }; backend_ctx->enqueue_ndrange_kernel(qk, 1, q_global, q_local, dst); cl_kernel dk = backend_ctx->kernel_gemm_noshuffle_q5_k_q8_1_dp4a; int ai = 0; CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &extra0_q5_k->q)); CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &extra0_q5_k->qh)); CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &extra0_q5_k->s)); CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &extra0_q5_k->d)); CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &extra0_q5_k->dm)); CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer)); CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer)); CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &backend_ctx->prealloc_moe_sa.buffer)); CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &extrad->data_device)); CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_ulong), &offsetd)); CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_int), &Mm)); CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_int), &Nn)); CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_int), &Kk)); CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_uchar), &mask_d6)); CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_uchar), &mask_d4)); CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_uchar), &mask_hi2)); size_t d_local[3] = { 64, 1, 1 }; size_t d_global[3] = { 64, (size_t)(Mm / 64), (size_t)CEIL_DIV(Nn, 32) }; backend_ctx->enqueue_ndrange_kernel(dk, 3, d_global, d_local, dst); CL_CHECK(clReleaseMemObject(b_sub_buf)); CL_CHECK(clReleaseMemObject(b_sub_buf_trans)); CL_CHECK(clReleaseMemObject(b_img)); CL_CHECK(clReleaseMemObject(b_img_trans)); return; } // gemm kernel = backend_ctx->kernel_gemm_noshuffle_q5_k_f32; int padded_N = N + padding; CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_q5_k->q)); CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q5_k->qh)); CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra0_q5_k->s)); CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra0_q5_k->d)); CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extra0_q5_k->dm)); CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_mem), &b_img_trans)); CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_mem), &extrad->data_device)); CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_ulong), &offsetd)); CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_int), &ne01)); CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_int), &padded_N)); CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_int), &ne00)); CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_int), &ne1)); CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_uchar), &mask_d6)); CL_CHECK(clSetKernelArg(kernel, 13, sizeof(cl_uchar), &mask_d4)); CL_CHECK(clSetKernelArg(kernel, 14, sizeof(cl_uchar), &mask_hi2)); size_t global_work_size[3] = {(size_t)CEIL_DIV(ne1, 8), (size_t)CEIL_DIV(ne01, 4), 1}; size_t local_work_size[3] = {1, 128, 1}; backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); CL_CHECK(clReleaseMemObject(b_sub_buf)); CL_CHECK(clReleaseMemObject(b_sub_buf_trans)); CL_CHECK(clReleaseMemObject(b_img)); CL_CHECK(clReleaseMemObject(b_img_trans)); } #else GGML_UNUSED(backend); GGML_UNUSED(src0); GGML_UNUSED(src1); GGML_UNUSED(dst); #endif } // Dequant a possibly-strided q4_0/q8_0 tensor to tight-packed f16. Returns a // temp cl_mem the caller must release. SoA inputs are reconstructed into a // temp AoS buffer reported via *extra_reconstruct (also caller-released). // this is for quantized K cache without FA. static cl_mem ggml_cl_mul_mat_dequant_quant_to_f16( ggml_backend_opencl_context * backend_ctx, const ggml_tensor * tensor, cl_mem * extra_reconstruct /* out, may be NULL */ ) { GGML_ASSERT(tensor->type == GGML_TYPE_Q4_0 || tensor->type == GGML_TYPE_Q8_0); if (extra_reconstruct) { *extra_reconstruct = NULL; } cl_mem src_buf; cl_ulong src_offset; cl_ulong src_nb1; cl_ulong src_nb2; cl_ulong src_nb3; uintptr_t pool_key_buf = 0; cl_ulong pool_key_off = (cl_ulong) tensor->view_offs; const bool is_soa = tensor->type == GGML_TYPE_Q4_0 ? ggml_cl_is_q4_0_soa(tensor) : ggml_cl_is_q8_0_soa(tensor); cl_mem aos = nullptr; if (is_soa) { // Reconstruct full parent AoS; view's own nb[] then index it correctly. const ggml_tensor * parent = tensor->view_src ? tensor->view_src : tensor; const ggml_tensor * soa_src = parent; const size_t block_bytes = (size_t) ggml_type_size(tensor->type); const size_t blck_size = (size_t) ggml_blck_size(tensor->type); const size_t parent_row_blocks = (size_t) parent->ne[0] / blck_size; const size_t parent_row_bytes = parent_row_blocks * block_bytes; const size_t parent_nbytes = (size_t) ggml_nelements(parent) / blck_size * block_bytes; cl_int err; aos = clCreateBuffer(backend_ctx->context, CL_MEM_READ_WRITE, parent_nbytes, NULL, &err); CL_CHECK(err); // large q4_0/q8_0 WEIGHTS are stored transposed and small weights // (and the AoS KV-cache, handled in the else branch above) are not. // choose a proper restore kernel based on this. bool restored = false; #ifdef GGML_OPENCL_USE_ADRENO_KERNELS const int p_ne00 = (int) parent->ne[0]; const int p_ne01 = (int) parent->ne[1]; if (tensor->type == GGML_TYPE_Q8_0 && enable_adreno_trans_weight(backend_ctx, parent)) { auto * extra = (ggml_tensor_extra_cl_q8_0 *) soa_src->extra; pool_key_buf = (uintptr_t) extra->q; cl_kernel kernel = backend_ctx->kernel_restore_block_q8_0_trans; CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra->q)); CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra->d)); CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &aos)); CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_int), &p_ne00)); CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_int), &p_ne01)); size_t gws[] = { (size_t)(((p_ne01 + 63) / 64) * 64), 1, 1 }; size_t lws[] = { 64, 1, 1 }; CL_CHECK(clEnqueueNDRangeKernel(backend_ctx->queue, kernel, 3, NULL, gws, lws, 0, NULL, NULL)); restored = true; } else if (tensor->type == GGML_TYPE_Q4_0 && use_adreno_kernels(backend_ctx, parent) && !use_adreno_moe_kernels(backend_ctx, parent)) { auto * extra = (ggml_tensor_extra_cl_q4_0 *) soa_src->extra; pool_key_buf = (uintptr_t) extra->q; const size_t size_q = (size_t) ggml_nelements(parent) / blck_size * (blck_size / 2); const size_t size_d = (size_t) ggml_nelements(parent) / blck_size * sizeof(ggml_fp16_t); cl_int err2 = CL_SUCCESS; cl_mem buf_tq = clCreateBuffer(backend_ctx->context, CL_MEM_READ_WRITE, size_q, NULL, &err2); CL_CHECK(err2); cl_mem buf_td = clCreateBuffer(backend_ctx->context, CL_MEM_READ_WRITE, size_d, NULL, &err2); CL_CHECK(err2); transpose_2d_as_16b(backend_ctx, extra->q, buf_tq, size_q, p_ne01, p_ne00 / 4); transpose_2d_as_16b(backend_ctx, extra->d, buf_td, size_d, p_ne01, p_ne00 / 32); cl_uchar mask_0F = 0x0F, mask_F0 = 0xF0; cl_kernel kernel = backend_ctx->kernel_restore_block_q4_0_noshuffle; CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &buf_tq)); CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &buf_td)); CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &aos)); CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_uchar), &mask_0F)); CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_uchar), &mask_F0)); const size_t n_blk = parent_nbytes / block_bytes; size_t gws[] = { n_blk, 1, 1 }; size_t lws[] = { 1, 1, 1 }; CL_CHECK(clEnqueueNDRangeKernel(backend_ctx->queue, kernel, 3, NULL, gws, lws, 0, NULL, NULL)); CL_CHECK(clReleaseMemObject(buf_tq)); CL_CHECK(clReleaseMemObject(buf_td)); restored = true; } #endif // GGML_OPENCL_USE_ADRENO_KERNELS if (!restored) { cl_kernel kernel; if (tensor->type == GGML_TYPE_Q8_0) { auto * extra = (ggml_tensor_extra_cl_q8_0 *) soa_src->extra; kernel = backend_ctx->kernel_restore_block_q8_0; CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra->q)); CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra->d)); CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &aos)); pool_key_buf = (uintptr_t) extra->q; } else { auto * extra = (ggml_tensor_extra_cl_q4_0 *) soa_src->extra; kernel = backend_ctx->kernel_restore_block_q4_0; CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra->q)); CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra->d)); CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &aos)); pool_key_buf = (uintptr_t) extra->q; } const size_t n_blocks = parent_nbytes / block_bytes; size_t gws_rec[] = { n_blocks, 1, 1 }; size_t lws_rec[] = { 1, 1, 1 }; CL_CHECK(clEnqueueNDRangeKernel(backend_ctx->queue, kernel, 3, NULL, gws_rec, lws_rec, 0, NULL, NULL)); } (void) parent_row_blocks; (void) parent_row_bytes; src_buf = aos; src_offset = tensor->view_offs; src_nb1 = tensor->nb[1]; src_nb2 = tensor->nb[2]; src_nb3 = tensor->nb[3]; if (extra_reconstruct) { *extra_reconstruct = aos; } } else { auto * extra = (ggml_tensor_extra_cl *) tensor->extra; GGML_ASSERT(extra && extra->data_device); src_buf = extra->data_device; src_offset = extra->offset + tensor->view_offs; src_nb1 = tensor->nb[1]; src_nb2 = tensor->nb[2]; src_nb3 = tensor->nb[3]; pool_key_buf = (uintptr_t) extra->data_device; pool_key_off = (cl_ulong) src_offset; } const cl_int nblk0 = (cl_int) (tensor->ne[0] / ggml_blck_size(tensor->type)); const cl_int ne1_ = (cl_int) tensor->ne[1]; const cl_int ne2_ = (cl_int) tensor->ne[2]; const cl_int ne3_ = (cl_int) tensor->ne[3]; const size_t out_bytes = (size_t) ggml_nelements(tensor) * sizeof(ggml_fp16_t); // reuse a pooled f16 buffer for this KV-cache view across decode steps instead of // allocating new one per attention op cl_mem out = nullptr; { auto & pool = backend_ctx->dequant_f16_pool; ggml_backend_opencl_context::ImagePoolKey key{pool_key_buf, (uint64_t) pool_key_off}; auto it = pool.find(key); if (it != pool.end() && it->second.k_bytes >= out_bytes && it->second.image) { out = it->second.image; } else { if (it != pool.end()) { if (it->second.image) { CL_CHECK(clReleaseMemObject(it->second.image)); } pool.erase(it); } cl_int err = CL_SUCCESS; out = clCreateBuffer(backend_ctx->context, CL_MEM_READ_WRITE, out_bytes, NULL, &err); CL_CHECK(err); ggml_backend_opencl_context::ImagePoolEntry entry; entry.image = out; entry.k_bytes = out_bytes; pool[key] = entry; } } cl_kernel dq_kernel = tensor->type == GGML_TYPE_Q8_0 ? backend_ctx->kernel_dequant_q8_0_f16_view_aos : backend_ctx->kernel_dequant_q4_0_f16_view_aos; CL_CHECK(clSetKernelArg(dq_kernel, 0, sizeof(cl_mem), &src_buf)); CL_CHECK(clSetKernelArg(dq_kernel, 1, sizeof(cl_ulong), &src_offset)); CL_CHECK(clSetKernelArg(dq_kernel, 2, sizeof(cl_ulong), &src_nb1)); CL_CHECK(clSetKernelArg(dq_kernel, 3, sizeof(cl_ulong), &src_nb2)); CL_CHECK(clSetKernelArg(dq_kernel, 4, sizeof(cl_ulong), &src_nb3)); CL_CHECK(clSetKernelArg(dq_kernel, 5, sizeof(cl_int), &nblk0)); CL_CHECK(clSetKernelArg(dq_kernel, 6, sizeof(cl_int), &ne1_)); CL_CHECK(clSetKernelArg(dq_kernel, 7, sizeof(cl_int), &ne2_)); CL_CHECK(clSetKernelArg(dq_kernel, 8, sizeof(cl_int), &ne3_)); CL_CHECK(clSetKernelArg(dq_kernel, 9, sizeof(cl_mem), &out)); size_t gws[3] = { (size_t) nblk0, (size_t) ne1_, (size_t) (ne2_ * ne3_) }; size_t lws[3] = { 1, 1, 1 }; CL_CHECK(clEnqueueNDRangeKernel(backend_ctx->queue, dq_kernel, 3, NULL, gws, lws, 0, NULL, NULL)); // release the reconstructed aos if // 1. it was actually reconstructed // 2. the caller didn't request it to be returned // src_buf may refer to aos, so we should release after this enqueue if (aos && !extra_reconstruct) { CL_CHECK(clReleaseMemObject(aos)); } return out; } // look up or create a pooled image1d_buffer over a KV-cache view. static cl_mem ggml_cl_img_pool_get_or_create( ggml_backend_opencl_context * backend_ctx, std::map & pool, cl_mem data_device, cl_ulong offset0, size_t required_bytes, cl_channel_type channel_data_type ) { ggml_backend_opencl_context::ImagePoolKey key{(uintptr_t)data_device, (uint64_t)offset0}; auto it = pool.find(key); if (it != pool.end() && it->second.k_bytes >= required_bytes && it->second.channel_data_type == channel_data_type && it->second.image != nullptr) { return it->second.image; } // need to create or recreate and release any stale entry first. if (it != pool.end()) { if (it->second.image) { CL_CHECK(clReleaseMemObject(it->second.image)); } if (it->second.sub_buffer) {CL_CHECK(clReleaseMemObject(it->second.sub_buffer)); } pool.erase(it); } cl_int status = CL_SUCCESS; cl_buffer_region region = {}; region.origin = (size_t)offset0; region.size = required_bytes; cl_mem sub = clCreateSubBuffer(data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &status); if (status != CL_SUCCESS) { return nullptr; } const size_t pixel_size = (channel_data_type == CL_HALF_FLOAT) ? 8 : 16; cl_image_format fmt = {CL_RGBA, channel_data_type}; cl_image_desc desc = {}; desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; desc.image_width = required_bytes / pixel_size; desc.buffer = sub; cl_mem img = clCreateImage(backend_ctx->context, CL_MEM_READ_ONLY, &fmt, &desc, NULL, &status); if (status != CL_SUCCESS) { CL_CHECK(clReleaseMemObject(sub)); return nullptr; } ggml_backend_opencl_context::ImagePoolEntry entry; entry.sub_buffer = sub; entry.image = img; entry.k_bytes = required_bytes; entry.channel_data_type = channel_data_type; pool[key] = entry; return img; } static void ggml_cl_mul_mat(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { GGML_ASSERT(src0); GGML_ASSERT(src0->extra); GGML_ASSERT(src1); GGML_ASSERT(src1->extra); GGML_ASSERT(dst); GGML_ASSERT(dst->extra); // bf16 is stored as f16 on device const enum ggml_type src0t = (src0->type == GGML_TYPE_BF16) ? GGML_TYPE_F16 : src0->type; const enum ggml_type src1t = src1->type; ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; // quant kv without FA // used for non-contiguous src0 (the usual head-major permuted K view when n_head_kv>1) // AND for the contiguous case that occurs when n_head_kv==1 (e.g. Gemma-4 E2B) if ((src0t == GGML_TYPE_Q4_0 || src0t == GGML_TYPE_Q8_0) && (!ggml_is_contiguous(src0) || src1->ne[2] > src0->ne[2])) { cl_mem f16_buf = ggml_cl_mul_mat_dequant_quant_to_f16(backend_ctx, src0, nullptr); ggml_tensor fake_src0 = *src0; ggml_tensor_extra_cl fake_extra = {}; fake_extra.data_device = f16_buf; fake_extra.offset = 0; fake_src0.type = GGML_TYPE_F16; fake_src0.extra = &fake_extra; fake_src0.view_src = nullptr; fake_src0.view_offs = 0; fake_src0.nb[0] = sizeof(ggml_fp16_t); fake_src0.nb[1] = fake_src0.nb[0] * src0->ne[0]; fake_src0.nb[2] = fake_src0.nb[1] * src0->ne[1]; fake_src0.nb[3] = fake_src0.nb[2] * src0->ne[2]; ggml_cl_mul_mat(backend, &fake_src0, src1, dst); return; } ggml_tensor_extra_cl * extra0 = (ggml_tensor_extra_cl *)src0->extra; ggml_tensor_extra_cl * extra1 = (ggml_tensor_extra_cl *)src1->extra; ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; cl_ulong offset0 = extra0->offset + src0->view_offs; cl_ulong offset1 = extra1->offset + src1->view_offs; cl_ulong offsetd = extrad->offset + dst->view_offs; #ifdef GGML_OPENCL_SOA_Q // view->extra stays pre-SoA; cast to the SoA struct would SIGSEGV. // Follow view_src to reach the real SoA extra. const ggml_tensor * soa0_src = src0->view_src != nullptr ? src0->view_src : src0; ggml_tensor_extra_cl_q1_0 * extra0_q1_0 = (ggml_tensor_extra_cl_q1_0 *)src0->extra; ggml_tensor_extra_cl_q4_0 * extra0_q4_0 = (ggml_tensor_extra_cl_q4_0 *)soa0_src->extra; ggml_tensor_extra_cl_q4_1 * extra0_q4_1 = (ggml_tensor_extra_cl_q4_1 *)soa0_src->extra; ggml_tensor_extra_cl_q5_0 * extra0_q5_0 = (ggml_tensor_extra_cl_q5_0 *)soa0_src->extra; ggml_tensor_extra_cl_q5_1 * extra0_q5_1 = (ggml_tensor_extra_cl_q5_1 *)soa0_src->extra; ggml_tensor_extra_cl_mxfp4 * extra0_mxfp4 = (ggml_tensor_extra_cl_mxfp4 *)soa0_src->extra; ggml_tensor_extra_cl_q8_0 * extra0_q8_0 = (ggml_tensor_extra_cl_q8_0 *)soa0_src->extra; ggml_tensor_extra_cl_iq4_nl * extra0_iq4_nl = (ggml_tensor_extra_cl_iq4_nl *)soa0_src->extra; ggml_tensor_extra_cl_q4_K * extra0_q4_K = (ggml_tensor_extra_cl_q4_K *)soa0_src->extra; ggml_tensor_extra_cl_q5_K * extra0_q5_K = (ggml_tensor_extra_cl_q5_K *)soa0_src->extra; ggml_tensor_extra_cl_q6_K * extra0_q6_K = (ggml_tensor_extra_cl_q6_K *)soa0_src->extra; #endif GGML_TENSOR_LOCALS(int, ne0, src0, ne); GGML_TENSOR_LOCALS(cl_ulong, nb0, src0, nb); GGML_TENSOR_LOCALS(int, ne1, src1, ne); GGML_TENSOR_LOCALS(cl_ulong, nb1, src1, nb); GGML_TENSOR_LOCALS(int, ne, dst, ne); GGML_TENSOR_LOCALS(cl_ulong, nb, dst, nb); int r2 = ne12/ne02; int r3 = ne13/ne03; GGML_ASSERT(ne00 == ne10); #ifdef GGML_OPENCL_USE_ADRENO_KERNELS // adreno GEMM/GEMV kernels do not support broadcast, assuming ne2 and ne3 are 1 for src1 // so we handle broadcast here if ((ne12 > 1 || ne13 > 1) && ne02 == 1 && ne03 == 1 && src0t != GGML_TYPE_F16 && src0t != GGML_TYPE_F32) { for (int i13 = 0; i13 < ne13; ++i13) { for (int i12 = 0; i12 < ne12; ++i12) { ggml_tensor s1 = *src1; s1.ne[2] = 1; s1.ne[3] = 1; s1.view_offs = src1->view_offs + (size_t)i12*nb12 + (size_t)i13*nb13; ggml_tensor d = *dst; d.ne[2] = 1; d.ne[3] = 1; d.view_offs = dst->view_offs + (size_t)i12*nb2 + (size_t)i13*nb3; ggml_cl_mul_mat(backend, src0, &s1, &d); } } return; } #endif int nth0 = 32; int nth1 = 1; int nrows = 1; // The number of values produced by each subgroup int ndst = 4; cl_kernel kernel; #ifdef GGML_OPENCL_USE_ADRENO_KERNELS if(src0t == GGML_TYPE_F16 && src1t == GGML_TYPE_F32){ // Two tiling assumptions these kernels make but nothing enforced: // // ne00 % TILESIZE_K(16): the K loop has no tail, so a K that does not // divide folds 1-15 rows of whatever follows the operands into every // output. // // ne01 % TILESIZE_M(64): mm_store_c_N guards the n direction with its // `mask` argument but nothing guards m -- the store walks all 64 rows // of the tile at a stride of M. When M does not divide, the last tile // does not run off the end of the buffer, it writes 64 - (M % 64) // values ON TOP OF the next column, so the result is silently wrong. // Reachable on the KQV side for any head size >= 64 that is not a // multiple of it (80, 96, 112). // // Attention shapes in the graph satisfy both -- head sizes are multiples // of 64 and n_kv is padded -- which is why this has stayed latent. // Declining leaves the odd shapes on the generic GEMM, which handles them. if (ne01 >= 64 && ne1 >= 32 && ne00 >= 16 && (ne00 % 16) == 0 && (ne01 % 64) == 0 && (ne12 % ne02) == 0 && // the KQ/KQV image kernels do not handle dim 3 (multi-stream batches) ne03 == 1 && ne13 == 1 && // dst is wrapped with image1d_buffer, the size limit applies, also src0 (ne0 * ne1 * dst->ne[2] * dst->nb[0] / 4 <= backend_ctx->image_max_buffer_size)) { // For KQ. // // Layout admission, mirroring the KQV arm below. The KQ kernel takes // no stride arguments for A or B: it derives them as K*D_A*2 and // K*D_B*4, i.e. it assumes both operands pack exactly D heads of K // elements per row. Every real KV-cache view and permuted-Q view // does, but a view spanning part of a wider allocation does not, and // the kernel then walks the wrong rows with nothing to range-check // it. Gate on the packed layout itself rather than on the stride // ORDERING, which a wider parent satisfies just as well. const bool kq_packed_a = (nb01 == (cl_ulong)ne00 * ne02 * ggml_type_size(src0t)) && (nb02 == (cl_ulong)ne00 * ggml_type_size(src0t)); const bool kq_packed_b = (nb11 == (cl_ulong)ne10 * ne12 * ggml_type_size(src1t)) && (nb12 == (cl_ulong)ne10 * ggml_type_size(src1t)); // // ggml_is_permuted(src0) stands in for "K is head-major", but it is // only a proxy and it COLLAPSES at n_head_kv == 1: with a single // head there is no head stride to be out of order, so nb01 == nb02 // and the view reports itself unpermuted. Such a KQ was declined // here and fell through to the generic GEMM (gemma-4 E2B, and any // other multi-query model). The packed check above is the contract // the kernel actually needs -- it pins both strides exactly -- so // require permutedness only where there is more than one head for // it to mean anything. // // Default on; GGML_OPENCL_KQ_NHEAD_KV1=0 restores the old proxy so // the two routings can be compared in one binary. static const char * kq_nhkv1_env = getenv("GGML_OPENCL_KQ_NHEAD_KV1"); static const bool kq_nhkv1_on = (kq_nhkv1_env == nullptr || kq_nhkv1_env[0] != '0'); if ((ggml_is_permuted(src0) || (ne02 == 1 && kq_nhkv1_on)) && ggml_is_permuted(src1) && kq_packed_a && kq_packed_b && ((nb01 * ne01 / 4)/4 <= backend_ctx->image_max_buffer_size) && nb00 <= nb02 && nb02 <= nb01 && nb01 <= nb03 && nb10 <= nb12 && nb12 <= nb11 && nb11 <= nb13) { ggml_cl_mul_mat_kq_kqv_adreno(backend, src0, src1, dst, /*is_kq =*/ true); return; } // For KQV. Reaching this arm is what makes the op a KQV; the callee // is told so explicitly rather than re-deriving it from the strides // the arm above has already ruled on. if (!ggml_is_contiguous(src0) && ggml_is_contiguous(src1) && ((nb02 * ne02 / 4)/4 <= backend_ctx->image_max_buffer_size)) { ggml_cl_mul_mat_kq_kqv_adreno(backend, src0, src1, dst, /*is_kq =*/ false); return; } } static const char * mm_kq_gqa_img_env = getenv("GGML_OPENCL_MM_KQ_GQA_IMG"); static const bool mm_kq_gqa_img_on = (mm_kq_gqa_img_env == nullptr || mm_kq_gqa_img_env[0] != '0'); static const char * mm_kq_gqa_r4_img_env = getenv("GGML_OPENCL_MM_KQ_GQA_R4_IMG"); static const bool mm_kq_gqa_r4_img_on = (mm_kq_gqa_r4_img_env == nullptr || mm_kq_gqa_r4_img_env[0] != '0'); const bool img_r4_gate = mm_kq_gqa_r4_img_on && backend_ctx->kernel_mul_mat_f16_f32_l4_x8_gqa_r4_img != nullptr && ne11 == 1 && ne01 >= 64 && (ne01 % 16) == 0 && ne00 == 128 && (ne12 % ne02) == 0 && (ne12 / ne02) == 4 && (ne13 / ne03) == 1; if (mm_kq_gqa_img_on && backend_ctx->kernel_mul_mat_f16_f32_l4_x8_gqa4_img != nullptr && ne11 == 1 && ne01 >= 64 && (ne01 % 16) == 0 && ne00 == 128 && (ne12 % ne02) == 0 && (ne12 / ne02) == 8 && (ne13 / ne03) == 1) { const size_t nb00_bytes = sizeof(uint16_t); const size_t k_bytes_span = (size_t)(ne01 > 0 ? ne01 - 1 : 0) * (size_t)nb01 + (size_t)(ne02 > 0 ? ne02 - 1 : 0) * (size_t)nb02 + (size_t)(ne03 > 0 ? ne03 - 1 : 0) * (size_t)nb03 + (size_t)ne00 * nb00_bytes; const size_t k_bytes = (k_bytes_span + 15) & ~(size_t)15; const size_t k_pixels = k_bytes >> 4; if (k_pixels > 0 && k_pixels <= backend_ctx->image_max_buffer_size) { cl_kernel kernel = backend_ctx->kernel_mul_mat_f16_f32_l4_x8_gqa4_img; cl_mem K_img = ggml_cl_img_pool_get_or_create( backend_ctx, backend_ctx->kq_img_pool, extra0->data_device, offset0, k_bytes, CL_FLOAT); if (K_img != nullptr) { cl_uint k_arg = 0; CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &K_img)); CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &extra1->data_device)); CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_ulong), &offset1)); CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &extrad->data_device)); CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_ulong), &offsetd)); CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &ne00)); CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &ne01)); CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &ne02)); CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_ulong), &nb01)); CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_ulong), &nb02)); CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_ulong), &nb03)); CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &ne10)); CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &ne11)); CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &ne12)); CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_ulong), &nb10)); CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_ulong), &nb11)); CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_ulong), &nb12)); CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_ulong), &nb13)); CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &ne0)); CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &ne1)); CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &r2)); CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &r3)); const int nth0_d = 64; const int64_t n_wg_x = ne01 / 16; size_t global_work_size[] = {(size_t)n_wg_x * nth0_d, (size_t)1, (size_t)ne02 * ne13}; size_t local_work_size[] = {(size_t)nth0_d, (size_t)1, 1}; backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); return; } } } // r2=4 specialization if (img_r4_gate) { const size_t nb00_bytes = sizeof(uint16_t); const size_t k_bytes_span = (size_t)(ne01 > 0 ? ne01 - 1 : 0) * (size_t)nb01 + (size_t)(ne02 > 0 ? ne02 - 1 : 0) * (size_t)nb02 + (size_t)(ne03 > 0 ? ne03 - 1 : 0) * (size_t)nb03 + (size_t)ne00 * nb00_bytes; const size_t k_bytes = (k_bytes_span + 15) & ~(size_t)15; const size_t k_pixels = k_bytes >> 4; if (k_pixels > 0 && k_pixels <= backend_ctx->image_max_buffer_size) { cl_kernel kernel = backend_ctx->kernel_mul_mat_f16_f32_l4_x8_gqa_r4_img; cl_mem K_img = ggml_cl_img_pool_get_or_create( backend_ctx, backend_ctx->kq_img_pool, extra0->data_device, offset0, k_bytes, CL_FLOAT); if (K_img != nullptr) { cl_uint k_arg = 0; CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &K_img)); CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &extra1->data_device)); CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_ulong), &offset1)); CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &extrad->data_device)); CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_ulong), &offsetd)); CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &ne00)); CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &ne01)); CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &ne02)); CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_ulong), &nb01)); CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_ulong), &nb02)); CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_ulong), &nb03)); CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &ne10)); CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &ne11)); CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &ne12)); CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_ulong), &nb10)); CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_ulong), &nb11)); CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_ulong), &nb12)); CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_ulong), &nb13)); CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &ne0)); CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &ne1)); CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &r2)); CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &r3)); const int nth0_d = 64; const int64_t n_wg_x = ne01 / 16; size_t global_work_size[] = {(size_t)n_wg_x * nth0_d, (size_t)1, (size_t)ne02 * ne13}; size_t local_work_size[] = {(size_t)nth0_d, (size_t)1, 1}; backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); return; } } } // DK=256, r2=2 specialization static const char * mm_kq_r2_dk256_env = getenv("GGML_OPENCL_MM_KQ_GQA_R2_DK256_IMG"); static const bool mm_kq_r2_dk256_on = (mm_kq_r2_dk256_env != nullptr && mm_kq_r2_dk256_env[0] != '0'); if (mm_kq_r2_dk256_on && backend_ctx->kernel_mul_mat_f16_f32_l4_x8_gqa_r2_dk256_img != nullptr && ne11 == 1 && ne01 >= 64 && (ne01 % 16) == 0 && ne00 == 256 && (ne12 % ne02) == 0 && (ne12 / ne02) == 2 && (ne13 / ne03) == 1) { const size_t nb00_bytes = sizeof(uint16_t); const size_t k_bytes_span = (size_t)(ne01 > 0 ? ne01 - 1 : 0) * (size_t)nb01 + (size_t)(ne02 > 0 ? ne02 - 1 : 0) * (size_t)nb02 + (size_t)(ne03 > 0 ? ne03 - 1 : 0) * (size_t)nb03 + (size_t)ne00 * nb00_bytes; const size_t k_bytes = (k_bytes_span + 15) & ~(size_t)15; const size_t k_pixels = k_bytes >> 4; if (k_pixels > 0 && k_pixels <= backend_ctx->image_max_buffer_size) { cl_kernel kernel = backend_ctx->kernel_mul_mat_f16_f32_l4_x8_gqa_r2_dk256_img; cl_mem K_img = ggml_cl_img_pool_get_or_create( backend_ctx, backend_ctx->kq_img_pool, extra0->data_device, offset0, k_bytes, CL_FLOAT); if (K_img != nullptr) { cl_uint k_arg = 0; CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &K_img)); CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &extra1->data_device)); CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_ulong), &offset1)); CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &extrad->data_device)); CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_ulong), &offsetd)); CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &ne00)); CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &ne01)); CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &ne02)); CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_ulong), &nb01)); CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_ulong), &nb02)); CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_ulong), &nb03)); CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &ne10)); CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &ne11)); CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &ne12)); CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_ulong), &nb10)); CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_ulong), &nb11)); CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_ulong), &nb12)); CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_ulong), &nb13)); CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &ne0)); CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &ne1)); CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &r2)); CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &r3)); const int nth0_d = 64; const int64_t n_wg_x = ne01 / 16; size_t global_work_size[] = {(size_t)n_wg_x * nth0_d, (size_t)1, (size_t)ne02 * ne13}; size_t local_work_size[] = {(size_t)nth0_d, (size_t)1, 1}; backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); return; } } } // GQA-coalesced KQV for decode using image1d_buffer_t static const char * mm_kqv_gqa_img_env = getenv("GGML_OPENCL_MM_KQV_GQA_IMG"); static const bool mm_kqv_gqa_img_on = (mm_kqv_gqa_img_env != nullptr && mm_kqv_gqa_img_env[0] != '0'); if (mm_kqv_gqa_img_on && backend_ctx->kernel_mul_mat_f16_f32_l4_y8_gqa_img != nullptr && ne11 == 1 && ne01 == 128 && (ne12 % ne02) == 0 && (ne12 / ne02) == 8 && (ne13 / ne03) == 1) { const size_t nb00_bytes = sizeof(uint16_t); const size_t v_bytes_span = (size_t)(ne01 > 0 ? ne01 - 1 : 0) * (size_t)nb01 + (size_t)(ne02 > 0 ? ne02 - 1 : 0) * (size_t)nb02 + (size_t)(ne03 > 0 ? ne03 - 1 : 0) * (size_t)nb03 + (size_t)ne00 * nb00_bytes; const size_t v_bytes = (v_bytes_span + 7) & ~(size_t)7; const size_t v_pixels = v_bytes >> 3; if (v_pixels > 0 && v_pixels <= backend_ctx->image_max_buffer_size) { cl_kernel kernel = backend_ctx->kernel_mul_mat_f16_f32_l4_y8_gqa_img; cl_mem V_img = ggml_cl_img_pool_get_or_create( backend_ctx, backend_ctx->kqv_img_pool, extra0->data_device, offset0, v_bytes, CL_HALF_FLOAT); if (V_img != nullptr) { cl_uint k_arg = 0; CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &V_img)); CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &extra1->data_device)); CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_ulong), &offset1)); CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &extrad->data_device)); CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_ulong), &offsetd)); CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &ne00)); CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &ne01)); CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &ne02)); CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_ulong), &nb01)); CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_ulong), &nb02)); CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_ulong), &nb03)); CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &ne10)); CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &ne11)); CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &ne12)); CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_ulong), &nb10)); CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_ulong), &nb11)); CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_ulong), &nb12)); CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_ulong), &nb13)); CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &ne0)); CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &ne1)); CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &r2)); CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &r3)); const int nth0_d = 64; const int64_t n_wg_x = ne01 / 8; size_t global_work_size[] = {(size_t)n_wg_x * nth0_d, (size_t)1, (size_t)ne02 * ne13}; size_t local_work_size[] = {(size_t)nth0_d, (size_t)1, 1}; backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); return; } } } } if (ne01 && ne1 && use_adreno_kernels(backend_ctx, src0)) { // NOTE: Kernels using image1d_buffer_t (e.g., src0_q) would normally require // a limit check, but q4_0 / q4_1 tensors are very unlikely to exceed that // limit, so the check is omitted. // q1_0 x fp32 if (src0t == GGML_TYPE_Q1_0 && src1t == GGML_TYPE_F32 && enable_adreno_trans_weight(backend_ctx, src0)) { ggml_cl_mul_mat_q1_0_f32_adreno(backend, src0, src1, dst); return; } // q4_0 x fp32 if(src0t == GGML_TYPE_Q4_0 && src1t == GGML_TYPE_F32) { ggml_cl_mul_mat_q4_0_f32_adreno(backend, src0, src1, dst); return; } // q4_1 x fp32 if (src0t == GGML_TYPE_Q4_1 && src1t == GGML_TYPE_F32) { ggml_cl_mul_mat_q4_1_f32_adreno(backend, src0, src1, dst); return; } // q5_0 x fp32 if (src0t == GGML_TYPE_Q5_0 && src1t == GGML_TYPE_F32) { ggml_cl_mul_mat_q5_0_f32_adreno(backend, src0, src1, dst); return; } // q5_1 x fp32 if (src0t == GGML_TYPE_Q5_1 && src1t == GGML_TYPE_F32) { ggml_cl_mul_mat_q5_1_f32_adreno(backend, src0, src1, dst); return; } // iq4_nl x fp32 if (src0t == GGML_TYPE_IQ4_NL && src1t == GGML_TYPE_F32) { ggml_cl_mul_mat_iq4_nl_f32_adreno(backend, src0, src1, dst); return; } // q8_0 x fp32 if (src0t == GGML_TYPE_Q8_0 && src1t == GGML_TYPE_F32 && enable_adreno_trans_weight(backend_ctx, src0)) { ggml_cl_mul_mat_q8_0_f32_adreno(backend, src0, src1, dst); return; } // q4_k x fp32 if (src0t == GGML_TYPE_Q4_K && src1t == GGML_TYPE_F32 && !use_flat_gemv_for_large_m_q4_K(backend_ctx, src0)) { ggml_cl_mul_mat_q4_k_f32_adreno(backend, src0, src1, dst); return; } // q6_K x fp32 if (src0t == GGML_TYPE_Q6_K && src1t == GGML_TYPE_F32 && !use_flat_gemv_for_large_m_q6_K(backend_ctx, src0)) { ggml_cl_mul_mat_q6_K_f32_adreno(backend, src0, src1, dst); return; } // q5_K x fp32 if (src0t == GGML_TYPE_Q5_K && src1t == GGML_TYPE_F32 && enable_adreno_trans_weight_q5_K(backend_ctx, src0)) { ggml_cl_mul_mat_q5_K_f32_adreno(backend, src0, src1, dst); return; } } // if (ne01 && ne1) #endif // GGML_OPENCL_USE_ADRENO_KERNELS // GEMM using local memory // Current BK = 16, so ne00 % 16 == 0 // // Certain A7X compiler (E031.41) executes kernel_mul_mm_f32_f32_l4_lm poorly; // matrices with ne11 <= 8 appears OK. // Fallback to the MV style kernels for A7x and ne11 > 8. // Override with GGML_OPENCL_A7X_F32_LM_BYPASS=0. static const char * a7x_f32lm_env = getenv("GGML_OPENCL_A7X_F32_LM_BYPASS"); static const bool a7x_f32lm_bypass = (a7x_f32lm_env == nullptr || a7x_f32lm_env[0] != '0'); if (src1t == GGML_TYPE_F32 && ne00 % 16 == 0 && ne11 > 1 && !(a7x_f32lm_bypass && src0t == GGML_TYPE_F32 && ne11 > 8 && backend_ctx->adreno_gen == ADRENO_GPU_GEN::A7X)) { switch(src0t) { case GGML_TYPE_F32: { // Small-N f32 GEMV for the spec/MTP verify batch: the tiled GEMM // below always computes a full 64x64 tile, so at ne11=3 with a // skinny f32 weight (GDN ssm_alpha/ssm_beta, M=32) it launches one // under-occupied WG at ~2.3% tile utilization. Route to a per-output // (m,n) GEMV (64-thread WG, K-split + __local reduce) instead. // Opt-in GGML_OPENCL_F32_MC=1; 2D contiguous, small N + skinny M only. static const bool f32_mc = (getenv("GGML_OPENCL_F32_MC") != nullptr); if (f32_mc && ne11 >= 2 && ne11 <= 8 && ne01 <= 512 && (ne00 % 4 == 0) && ne02 == 1 && ne12 == 1 && ne13 == 1 && ggml_is_contiguous(src0) && ggml_is_contiguous(src1)) { cl_kernel kmc = backend_ctx->kernel_gemv_f32_f32_mc; int stride_a = ne00, stride_b = ne00, stride_d = ne01; CL_CHECK(clSetKernelArg(kmc, 0, sizeof(cl_mem), &extra0->data_device)); CL_CHECK(clSetKernelArg(kmc, 1, sizeof(cl_ulong), &offset0)); CL_CHECK(clSetKernelArg(kmc, 2, sizeof(cl_mem), &extra1->data_device)); CL_CHECK(clSetKernelArg(kmc, 3, sizeof(cl_ulong), &offset1)); CL_CHECK(clSetKernelArg(kmc, 4, sizeof(cl_mem), &extrad->data_device)); CL_CHECK(clSetKernelArg(kmc, 5, sizeof(cl_ulong), &offsetd)); CL_CHECK(clSetKernelArg(kmc, 6, sizeof(int), &ne00)); CL_CHECK(clSetKernelArg(kmc, 7, sizeof(int), &ne01)); CL_CHECK(clSetKernelArg(kmc, 8, sizeof(int), &ne11)); CL_CHECK(clSetKernelArg(kmc, 9, sizeof(int), &stride_a)); CL_CHECK(clSetKernelArg(kmc, 10, sizeof(int), &stride_b)); CL_CHECK(clSetKernelArg(kmc, 11, sizeof(int), &stride_d)); size_t gws[3] = {64, (size_t)ne01 * (size_t)ne11, 1}; size_t lws[3] = {64, 1, 1}; backend_ctx->enqueue_ndrange_kernel(kmc, 3, gws, lws, dst); return; } kernel = backend_ctx->kernel_mul_mm_f32_f32_l4_lm; nth0 = 128; // calculated as (BM*BN)/(TM*TN) int batch_stride_a = ne00*ne01; int batch_stride_b = ne10*ne11; int batch_stride_d = ne0*ne1; cl_mem mem_src0 = extra0->data_device; cl_mem mem_src1 = extra1->data_device; cl_ulong nb00_cont = nb00; cl_ulong nb01_cont = nb01; cl_ulong nb02_cont = nb02; cl_ulong nb03_cont = nb03; cl_ulong nb10_cont = nb10; cl_ulong nb11_cont = nb11; cl_ulong nb12_cont = nb12; cl_ulong nb13_cont = nb13; cl_ulong offset0_cont = offset0; cl_ulong offset1_cont = offset1; if (!ggml_is_contiguous(src0)) { backend_ctx->prealloc_src0.allocate(backend_ctx->context, ggml_nbytes(src0)); ggml_cl_copy_to_contiguous(backend, src0, backend_ctx->prealloc_src0.buffer, nb00_cont, nb01_cont, nb02_cont, nb03_cont); mem_src0 = backend_ctx->prealloc_src0.buffer; offset0_cont = 0; } if (!ggml_is_contiguous(src1)) { backend_ctx->prealloc_src1.allocate(backend_ctx->context, ggml_nbytes(src1)); ggml_cl_copy_to_contiguous(backend, src1, backend_ctx->prealloc_src1.buffer, nb10_cont, nb11_cont, nb12_cont, nb13_cont); mem_src1 = backend_ctx->prealloc_src1.buffer; offset1_cont = 0; } CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &mem_src0)); CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0_cont)); CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &mem_src1)); CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1_cont)); CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device)); CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd)); CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne00)); CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne01)); CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne02)); CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne11)); CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &ne12)); CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne10)); // stride_a CL_CHECK(clSetKernelArg(kernel, 12, sizeof(int), &ne10)); // stride_b CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &ne01)); // stride_d CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &batch_stride_a)); CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &batch_stride_b)); CL_CHECK(clSetKernelArg(kernel, 16, sizeof(int), &batch_stride_d)); CL_CHECK(clSetKernelArg(kernel, 17, sizeof(int), &r2)); CL_CHECK(clSetKernelArg(kernel, 18, sizeof(int), &r3)); // 64 is block tile size BM and BN - change here when BM and BN in the kernel are changed. size_t global_work_size[] = {(size_t)(CEIL_DIV(ne01, 64)*nth0), (size_t)(CEIL_DIV(ne11, 64)), (size_t)ne12*ne13}; size_t local_work_size[] = {(size_t)nth0, 1, 1}; backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); return; } case GGML_TYPE_F16: { #ifdef GGML_OPENCL_USE_ADRENO_KERNELS if (ggml_cl_can_use_adreno_xmem_gemm_f16_f32(backend_ctx, src0, src1, dst)) { ggml_cl_mul_mat_f16_f32_adreno_xmem(backend, src0, src1, dst); return; } #endif kernel = backend_ctx->kernel_mul_mm_f16_f32_l4_lm; nth0 = 128; // calculated as (BM*BN)/(TM*TN) int batch_stride_a = ne00*ne01; int batch_stride_b = ne10*ne11; int batch_stride_d = ne0*ne1; cl_mem mem_src0 = extra0->data_device; cl_mem mem_src1 = extra1->data_device; cl_ulong nb00_cont = nb00; cl_ulong nb01_cont = nb01; cl_ulong nb02_cont = nb02; cl_ulong nb03_cont = nb03; cl_ulong nb10_cont = nb10; cl_ulong nb11_cont = nb11; cl_ulong nb12_cont = nb12; cl_ulong nb13_cont = nb13; cl_ulong offset0_cont = offset0; cl_ulong offset1_cont = offset1; if (!ggml_is_contiguous(src0)) { backend_ctx->prealloc_src0.allocate(backend_ctx->context, ggml_nbytes(src0)); ggml_cl_copy_to_contiguous(backend, src0, backend_ctx->prealloc_src0.buffer, nb00_cont, nb01_cont, nb02_cont, nb03_cont); mem_src0 = backend_ctx->prealloc_src0.buffer; offset0_cont = 0; } if (!ggml_is_contiguous(src1)) { backend_ctx->prealloc_src1.allocate(backend_ctx->context, ggml_nbytes(src1)); ggml_cl_copy_to_contiguous(backend, src1, backend_ctx->prealloc_src1.buffer, nb10_cont, nb11_cont, nb12_cont, nb13_cont); mem_src1 = backend_ctx->prealloc_src1.buffer; offset1_cont = 0; } CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &mem_src0)); CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0_cont)); CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &mem_src1)); CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1_cont)); CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device)); CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd)); CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne00)); CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne01)); CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne02)); CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne11)); CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &ne12)); CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne10)); // stride_a CL_CHECK(clSetKernelArg(kernel, 12, sizeof(int), &ne10)); // stride_b CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &ne01)); // stride_d CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &batch_stride_a)); CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &batch_stride_b)); CL_CHECK(clSetKernelArg(kernel, 16, sizeof(int), &batch_stride_d)); CL_CHECK(clSetKernelArg(kernel, 17, sizeof(int), &r2)); CL_CHECK(clSetKernelArg(kernel, 18, sizeof(int), &r3)); // 64 is block tile size BM and BN - change here when BM and BN in the kernel are changed. size_t global_work_size[] = {(size_t)(CEIL_DIV(ne01, 64)*nth0), (size_t)(CEIL_DIV(ne11, 64)), (size_t)ne12*ne13}; size_t local_work_size[] = {(size_t)nth0, 1, 1}; backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); return; } case GGML_TYPE_Q1_0: { if (ne11 < 32) { break; } if (!ggml_is_contiguous(src0) || !ggml_is_contiguous(src1)) { break; } kernel = backend_ctx->kernel_mul_mm_q1_0_f32_l4_lm; nth0 = 128; // calculated as (BM*BN)/(TM*TN) int batch_stride_a = ne00*ne01; int batch_stride_b = ne10*ne11; int batch_stride_d = ne0*ne1; CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_q1_0->q)); CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q1_0->d)); CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device)); CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1)); CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device)); CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd)); CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne00)); CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne01)); CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne02)); CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne11)); CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &ne12)); CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne10)); // stride_a CL_CHECK(clSetKernelArg(kernel, 12, sizeof(int), &ne10)); // stride_b CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &ne01)); // stride_d CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &batch_stride_a)); CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &batch_stride_b)); CL_CHECK(clSetKernelArg(kernel, 16, sizeof(int), &batch_stride_d)); CL_CHECK(clSetKernelArg(kernel, 17, sizeof(int), &r2)); CL_CHECK(clSetKernelArg(kernel, 18, sizeof(int), &r3)); // 64 is block tile size BM and BN - change here when BM and BN in the kernel are changed. size_t global_work_size[] = {(size_t)(CEIL_DIV(ne01, 64)*nth0), (size_t)(CEIL_DIV(ne11, 64)), (size_t)ne12*ne13}; size_t local_work_size[] = {(size_t)nth0, 1, 1}; backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); return; } case GGML_TYPE_Q4_0: { if (ne11 < 32) { break; } if (!ggml_is_contiguous(src0) || !ggml_is_contiguous(src1)) { break; } kernel = backend_ctx->kernel_mul_mm_q4_0_f32_l4_lm; nth0 = 128; // calculated as (BM*BN)/(TM*TN) int batch_stride_a = ne00*ne01; int batch_stride_b = ne10*ne11; int batch_stride_d = ne0*ne1; CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_q4_0->q)); CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q4_0->d)); CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device)); CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1)); CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device)); CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd)); CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne00)); CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne01)); CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne02)); CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne11)); CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &ne12)); CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne10)); // stride_a CL_CHECK(clSetKernelArg(kernel, 12, sizeof(int), &ne10)); // stride_b CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &ne01)); // stride_d CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &batch_stride_a)); CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &batch_stride_b)); CL_CHECK(clSetKernelArg(kernel, 16, sizeof(int), &batch_stride_d)); CL_CHECK(clSetKernelArg(kernel, 17, sizeof(int), &r2)); CL_CHECK(clSetKernelArg(kernel, 18, sizeof(int), &r3)); // 64 is block tile size BM and BN - change here when BM and BN in the kernel are changed. size_t global_work_size[] = {(size_t)(CEIL_DIV(ne01, 64)*nth0), (size_t)(CEIL_DIV(ne11, 64)), (size_t)ne12*ne13}; size_t local_work_size[] = {(size_t)nth0, 1, 1}; backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); return; } case GGML_TYPE_Q4_1: { if (ne11 < 32) { break; } if (!ggml_is_contiguous(src0) || !ggml_is_contiguous(src1)) { break; } kernel = backend_ctx->kernel_mul_mm_q4_1_f32_l4_lm; nth0 = 128; // calculated as (BM*BN)/(TM*TN) int batch_stride_a = ne00*ne01; int batch_stride_b = ne10*ne11; int batch_stride_d = ne0*ne1; CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_q4_1->q)); CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q4_1->d)); CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra0_q4_1->m)); CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra1->data_device)); CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_ulong), &offset1)); CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_mem), &extrad->data_device)); CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_ulong), &offsetd)); CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne00)); CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne01)); CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne02)); CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &ne11)); CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne12)); CL_CHECK(clSetKernelArg(kernel, 12, sizeof(int), &ne10)); // stride_a CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &ne10)); // stride_b CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &ne01)); // stride_d CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &batch_stride_a)); CL_CHECK(clSetKernelArg(kernel, 16, sizeof(int), &batch_stride_b)); CL_CHECK(clSetKernelArg(kernel, 17, sizeof(int), &batch_stride_d)); CL_CHECK(clSetKernelArg(kernel, 18, sizeof(int), &r2)); CL_CHECK(clSetKernelArg(kernel, 19, sizeof(int), &r3)); // 64 is block tile size BM and BN - change here when BM and BN in the kernel are changed. size_t global_work_size[] = {(size_t)(CEIL_DIV(ne01, 64)*nth0), (size_t)(CEIL_DIV(ne11, 64)), (size_t)ne12*ne13}; size_t local_work_size[] = {(size_t)nth0, 1, 1}; backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); return; } case GGML_TYPE_Q5_0: { if (ne11 < 32) { break; } if (!ggml_is_contiguous(src0) || !ggml_is_contiguous(src1)) { break; } kernel = backend_ctx->kernel_mul_mm_q5_0_f32_l4_lm; nth0 = 128; // calculated as (BM*BN)/(TM*TN) int batch_stride_a = ne00*ne01; int batch_stride_b = ne10*ne11; int batch_stride_d = ne0*ne1; CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_q5_0->qs)); CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q5_0->qh)); CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra0_q5_0->d)); CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra1->data_device)); CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_ulong), &offset1)); CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_mem), &extrad->data_device)); CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_ulong), &offsetd)); CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne00)); CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne01)); CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne02)); CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &ne11)); CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne12)); CL_CHECK(clSetKernelArg(kernel, 12, sizeof(int), &ne10)); // stride_a CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &ne10)); // stride_b CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &ne01)); // stride_d CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &batch_stride_a)); CL_CHECK(clSetKernelArg(kernel, 16, sizeof(int), &batch_stride_b)); CL_CHECK(clSetKernelArg(kernel, 17, sizeof(int), &batch_stride_d)); CL_CHECK(clSetKernelArg(kernel, 18, sizeof(int), &r2)); CL_CHECK(clSetKernelArg(kernel, 19, sizeof(int), &r3)); // 64 is block tile size BM and BN - change here when BM and BN in the kernel are changed. size_t global_work_size[] = {(size_t)(CEIL_DIV(ne01, 64)*nth0), (size_t)(CEIL_DIV(ne11, 64)), (size_t)ne12*ne13}; size_t local_work_size[] = {(size_t)nth0, 1, 1}; backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); return; } case GGML_TYPE_Q5_1: { if (ne11 < 32) { break; } if (!ggml_is_contiguous(src0) || !ggml_is_contiguous(src1)) { break; } kernel = backend_ctx->kernel_mul_mm_q5_1_f32_l4_lm; nth0 = 128; // calculated as (BM*BN)/(TM*TN) int batch_stride_a = ne00*ne01; int batch_stride_b = ne10*ne11; int batch_stride_d = ne0*ne1; CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_q5_1->qs)); CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q5_1->qh)); CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra0_q5_1->d)); CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra0_q5_1->m)); CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extra1->data_device)); CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offset1)); CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_mem), &extrad->data_device)); CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_ulong), &offsetd)); CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne00)); CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne01)); CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &ne02)); CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne11)); CL_CHECK(clSetKernelArg(kernel, 12, sizeof(int), &ne12)); CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &ne10)); // stride_a CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &ne10)); // stride_b CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &ne01)); // stride_d CL_CHECK(clSetKernelArg(kernel, 16, sizeof(int), &batch_stride_a)); CL_CHECK(clSetKernelArg(kernel, 17, sizeof(int), &batch_stride_b)); CL_CHECK(clSetKernelArg(kernel, 18, sizeof(int), &batch_stride_d)); CL_CHECK(clSetKernelArg(kernel, 19, sizeof(int), &r2)); CL_CHECK(clSetKernelArg(kernel, 20, sizeof(int), &r3)); // 64 is block tile size BM and BN - change here when BM and BN in the kernel are changed. size_t global_work_size[] = {(size_t)(CEIL_DIV(ne01, 64)*nth0), (size_t)(CEIL_DIV(ne11, 64)), (size_t)ne12*ne13}; size_t local_work_size[] = {(size_t)nth0, 1, 1}; backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); return; } case GGML_TYPE_Q8_0: { if (ne11 < 32) { break; } if (!ggml_is_contiguous(src0) || !ggml_is_contiguous(src1)) { break; } kernel = backend_ctx->kernel_mul_mm_q8_0_f32_l4_lm; nth0 = 128; // calculated as (BM*BN)/(TM*TN) int batch_stride_a = ne00*ne01; int batch_stride_b = ne10*ne11; int batch_stride_d = ne0*ne1; CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_q8_0->q)); CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q8_0->d)); CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device)); CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1)); CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device)); CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd)); CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne00)); CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne01)); CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne02)); CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne11)); CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &ne12)); CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne10)); // stride_a CL_CHECK(clSetKernelArg(kernel, 12, sizeof(int), &ne10)); // stride_b CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &ne01)); // stride_d CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &batch_stride_a)); CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &batch_stride_b)); CL_CHECK(clSetKernelArg(kernel, 16, sizeof(int), &batch_stride_d)); CL_CHECK(clSetKernelArg(kernel, 17, sizeof(int), &r2)); CL_CHECK(clSetKernelArg(kernel, 18, sizeof(int), &r3)); // 64 is block tile size BM and BN - change here when BM and BN in the kernel are changed. size_t global_work_size[] = {(size_t)(CEIL_DIV(ne01, 64)*nth0), (size_t)(CEIL_DIV(ne11, 64)), (size_t)ne12*ne13}; size_t local_work_size[] = {(size_t)nth0, 1, 1}; backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); return; } case GGML_TYPE_IQ4_NL: { if (ne11 < 32) { break; } if (!ggml_is_contiguous(src0) || !ggml_is_contiguous(src1)) { break; } kernel = backend_ctx->kernel_mul_mm_iq4_nl_f32_l4_lm; nth0 = 128; // calculated as (BM*BN)/(TM*TN) int batch_stride_a = ne00*ne01; int batch_stride_b = ne10*ne11; int batch_stride_d = ne0*ne1; CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_iq4_nl->q)); CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_iq4_nl->d)); CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device)); CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1)); CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device)); CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd)); CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne00)); CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne01)); CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne02)); CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne11)); CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &ne12)); CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne10)); // stride_a CL_CHECK(clSetKernelArg(kernel, 12, sizeof(int), &ne10)); // stride_b CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &ne01)); // stride_d CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &batch_stride_a)); CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &batch_stride_b)); CL_CHECK(clSetKernelArg(kernel, 16, sizeof(int), &batch_stride_d)); CL_CHECK(clSetKernelArg(kernel, 17, sizeof(int), &r2)); CL_CHECK(clSetKernelArg(kernel, 18, sizeof(int), &r3)); // 64 is block tile size BM and BN - change here when BM and BN in the kernel are changed. size_t global_work_size[] = {(size_t)(CEIL_DIV(ne01, 64)*nth0), (size_t)(CEIL_DIV(ne11, 64)), (size_t)ne12*ne13}; size_t local_work_size[] = {(size_t)nth0, 1, 1}; backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); return; } case GGML_TYPE_Q4_K: { if (ne11 < 32) { break; } if (!ggml_is_contiguous(src0) || !ggml_is_contiguous(src1)) { break; } kernel = backend_ctx->kernel_mul_mm_q4_k_f32_l4_lm; // (BM*BN)/(TM*TN): Intel uses an 8x8 microtile (WG=64), others 4x8 (WG=128) nth0 = (backend_ctx->gpu_family == INTEL) ? 64 : 128; int batch_stride_a = ne00*ne01; int batch_stride_b = ne10*ne11; int batch_stride_d = ne0*ne1; CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_q4_K->q)); CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q4_K->s)); CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra0_q4_K->d)); CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra0_q4_K->dm)); CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extra1->data_device)); CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offset1)); CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_mem), &extrad->data_device)); CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_ulong), &offsetd)); CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne00)); CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne01)); CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &ne02)); CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne11)); CL_CHECK(clSetKernelArg(kernel, 12, sizeof(int), &ne12)); CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &ne10)); // stride_a CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &ne10)); // stride_b CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &ne01)); // stride_d CL_CHECK(clSetKernelArg(kernel, 16, sizeof(int), &batch_stride_a)); CL_CHECK(clSetKernelArg(kernel, 17, sizeof(int), &batch_stride_b)); CL_CHECK(clSetKernelArg(kernel, 18, sizeof(int), &batch_stride_d)); CL_CHECK(clSetKernelArg(kernel, 19, sizeof(int), &r2)); CL_CHECK(clSetKernelArg(kernel, 20, sizeof(int), &r3)); // 64 is block tile size BM and BN - change here when BM and BN in the kernel are changed. size_t global_work_size[] = {(size_t)(CEIL_DIV(ne01, 64)*nth0), (size_t)(CEIL_DIV(ne11, 64)), (size_t)ne12*ne13}; size_t local_work_size[] = {(size_t)nth0, 1, 1}; backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); return; } case GGML_TYPE_Q5_K: { if (ne11 < 32) { break; } if (!ggml_is_contiguous(src0) || !ggml_is_contiguous(src1)) { break; } kernel = backend_ctx->kernel_mul_mm_q5_k_f32_l4_lm; nth0 = (backend_ctx->gpu_family == INTEL) ? 64 : 128; // Intel 8x8 microtile int batch_stride_a = ne00*ne01; int batch_stride_b = ne10*ne11; int batch_stride_d = ne0*ne1; CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_q5_K->q)); CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q5_K->qh)); CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra0_q5_K->s)); CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra0_q5_K->d)); CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extra0_q5_K->dm)); CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_mem), &extra1->data_device)); CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_ulong), &offset1)); CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_mem), &extrad->data_device)); CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_ulong), &offsetd)); CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne00)); CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &ne01)); CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne02)); CL_CHECK(clSetKernelArg(kernel, 12, sizeof(int), &ne11)); CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &ne12)); CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &ne10)); // stride_a CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &ne10)); // stride_b CL_CHECK(clSetKernelArg(kernel, 16, sizeof(int), &ne01)); // stride_d CL_CHECK(clSetKernelArg(kernel, 17, sizeof(int), &batch_stride_a)); CL_CHECK(clSetKernelArg(kernel, 18, sizeof(int), &batch_stride_b)); CL_CHECK(clSetKernelArg(kernel, 19, sizeof(int), &batch_stride_d)); CL_CHECK(clSetKernelArg(kernel, 20, sizeof(int), &r2)); CL_CHECK(clSetKernelArg(kernel, 21, sizeof(int), &r3)); // 64 is block tile size BM and BN - change here when BM and BN in the kernel are changed. size_t global_work_size[] = {(size_t)(CEIL_DIV(ne01, 64)*nth0), (size_t)(CEIL_DIV(ne11, 64)), (size_t)ne12*ne13}; size_t local_work_size[] = {(size_t)nth0, 1, 1}; backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); return; } case GGML_TYPE_Q6_K: { if (ne11 < 32) { break; } if (!ggml_is_contiguous(src0) || !ggml_is_contiguous(src1)) { break; } kernel = backend_ctx->kernel_mul_mm_q6_k_f32_l4_lm; nth0 = 128; // calculated as (BM*BN)/(TM*TN) int batch_stride_a = ne00*ne01; int batch_stride_b = ne10*ne11; int batch_stride_d = ne0*ne1; CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_q6_K->ql)); CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q6_K->qh)); CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra0_q6_K->s)); CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra0_q6_K->d)); CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extra1->data_device)); CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offset1)); CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_mem), &extrad->data_device)); CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_ulong), &offsetd)); CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne00)); CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne01)); CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &ne02)); CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne11)); CL_CHECK(clSetKernelArg(kernel, 12, sizeof(int), &ne12)); CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &ne10)); // stride_a CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &ne10)); // stride_b CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &ne01)); // stride_d CL_CHECK(clSetKernelArg(kernel, 16, sizeof(int), &batch_stride_a)); CL_CHECK(clSetKernelArg(kernel, 17, sizeof(int), &batch_stride_b)); CL_CHECK(clSetKernelArg(kernel, 18, sizeof(int), &batch_stride_d)); CL_CHECK(clSetKernelArg(kernel, 19, sizeof(int), &r2)); CL_CHECK(clSetKernelArg(kernel, 20, sizeof(int), &r3)); // 64 is block tile size BM and BN - change here when BM and BN in the kernel are changed. size_t global_work_size[] = {(size_t)(CEIL_DIV(ne01, 64)*nth0), (size_t)(CEIL_DIV(ne11, 64)), (size_t)ne12*ne13}; size_t local_work_size[] = {(size_t)nth0, 1, 1}; backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); return; } default: break; } } if (src0t == GGML_TYPE_F16 && src1t == GGML_TYPE_F32 && src0->ne[1] > 32 && // M > 32 src1->ne[1] > 32 && // N > 32 src0->ne[0] > 32 && // K > 32 src0->ne[2] == 1 && src0->ne[3] == 1 && src1->ne[2] == 1 && src1->ne[3] == 1 && ggml_is_contiguous(src0) && ggml_is_contiguous(src1) && backend_ctx->kernel_mul_mat_f16_f32_tiled != NULL) { ggml_cl_mul_mat_f16_f32_tiled(backend, src0, src1, dst); return; } if (!ggml_is_transposed(src0) && !ggml_is_transposed(src1) && src1t == GGML_TYPE_F32 && ne00%32 == 0 && ne11 > 2) { #ifdef GGML_OPENCL_SOA_Q // Set up kernel. switch(src0t) { case GGML_TYPE_Q4_0: // This should have been satisfied. GGML_ASSERT(ne11 == ne1); GGML_ASSERT(ne01 == ne0); if (backend_ctx->gpu_family == INTEL) { nth0 = 16; nth1 = 1; kernel = backend_ctx->kernel_mul_mat_q4_0_f32_1d_16x_flat; } else if (backend_ctx->gpu_family == ADRENO) { nth0 = 64; nth1 = 1; kernel = backend_ctx->kernel_mul_mat_q4_0_f32_1d_8x_flat; } else { GGML_ASSERT(false && "TODO: Unknown GPU"); } CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_q4_0->q)); CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q4_0->d)); CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device)); CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1)); CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device)); CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd)); CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne00)); CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne01)); CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne02)); CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne10)); CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &ne12)); CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne0)); CL_CHECK(clSetKernelArg(kernel, 12, sizeof(int), &ne1)); CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &r2)); CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &r3)); break; default: break; } // Launch kernel. if (src0t == GGML_TYPE_Q4_0) { size_t global_work_size[] = {(size_t)(ne01 + 7)/8*nth0, (size_t)ne11*nth1, (size_t)ne12*ne13}; size_t local_work_size[] = {(size_t)nth0, (size_t)nth1, 1}; if (backend_ctx->gpu_family == INTEL) { // Set global size for Intel. It uses 16x output values. global_work_size[0] = (size_t)(ne01 + 15)/16*nth0; global_work_size[1] = (size_t)ne11*nth1; global_work_size[2] = (size_t)ne12*ne13; } backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); return; } #else // GGML_OPENCL_SOA_Q // TODO: add block_q4_0 variant. #endif // GGML_OPENCL_SOA_Q } // use custom matrix x vector kernel bool use_f16_mrow = false; switch (src0t) { case GGML_TYPE_F32: //GGML_ASSERT(ne02 == ne12); GGML_ASSERT(src1t == GGML_TYPE_F32); kernel = backend_ctx->kernel_mul_mat_f32_f32; nrows = 4; if (backend_ctx->gpu_family == INTEL) { nth0 = 32; nth1 = 1; } else if (backend_ctx->gpu_family == ADRENO) { nth0 = 64; nth1 = 1; } else { GGML_ASSERT(false && "TODO: Unknown GPU"); } CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device)); CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1)); CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device)); CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd)); CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne00)); CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne01)); CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne02)); CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_ulong), &nb00)); CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_ulong), &nb01)); CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_ulong), &nb02)); CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_ulong), &nb03)); CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &ne10)); CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &ne11)); CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &ne12)); CL_CHECK(clSetKernelArg(kernel, 16, sizeof(cl_ulong), &nb10)); CL_CHECK(clSetKernelArg(kernel, 17, sizeof(cl_ulong), &nb11)); CL_CHECK(clSetKernelArg(kernel, 18, sizeof(cl_ulong), &nb12)); CL_CHECK(clSetKernelArg(kernel, 19, sizeof(cl_ulong), &nb13)); CL_CHECK(clSetKernelArg(kernel, 20, sizeof(int), &ne0)); CL_CHECK(clSetKernelArg(kernel, 21, sizeof(int), &ne1)); CL_CHECK(clSetKernelArg(kernel, 22, sizeof(int), &r2)); CL_CHECK(clSetKernelArg(kernel, 23, sizeof(int), &r3)); break; case GGML_TYPE_F16: //GGML_ASSERT(ne02 == ne12); if (backend_ctx->gpu_family == INTEL) { nth0 = 32; nth1 = 1; } else if (backend_ctx->gpu_family == ADRENO) { nth0 = 64; nth1 = 1; } else { GGML_ASSERT(false && "TODO: Unknown GPU"); } if (src1t == GGML_TYPE_F32) { // heuristic for packing more work for Adreno const bool adreno_use_lane_split = backend_ctx->gpu_family == ADRENO && ne11 == 1 && ne01 >= 8 && ne00 % 4 == 0 && r3 == 1 && r2 >= 1 && r2 <= 8 && (ne12 % r2) == 0; if (ne11 * ne12 < 4) { // Decode (single token): the legacy _1row runs one 64-lane // subgroup per WG (one output row), under-utilizing BW. Route the // wide f16 weight matmuls (attn proj + lm_head) to the multi-row // variant: MROW rows per WG -> more loads in flight + activation // staged once in __local. ne00<=8192 bounds the LDS. The mrow WG // is 64 x MROW = 1024 work-items (> Intel's 512 max) and reduces // within a 64-wide subgroup, so skip on Intel. if (backend_ctx->f16_mrow && backend_ctx->gpu_family != INTEL && backend_ctx->kernel_mul_mat_f16_f32_mrow != nullptr && ne00 >= 128 && ne01 >= 8 && ne00 % 4 == 0 && ne00 <= 8192) { // The register-blocked / half8 variants cast the src0 row pointer to // half4 / half8 (8- and 16-byte loads) with no scalar fallback inside // the kernel. ne00 % 4 == 0 constrains the element count per row, NOT // the byte stride between rows: a permuted or strided src0 (or a view // at an odd offset) can leave nb01/nb02/nb03 unaligned. Only take them // when every row this dispatch touches is aligned; the base mrow kernel // re-checks per row and falls back to its scalar loop. const cl_ulong row_addr_bits = offset0 | nb01 | nb02 | nb03; const bool aligned8 = (row_addr_bits & 7) == 0; const bool aligned16 = (row_addr_bits & 15) == 0; // Register-blocked variants: each subgroup does RPT rows (more // weight loads in flight per lane). 8/16 use half8 (128-bit) // loads, gated on ne00 % 8 == 0. const int rpt = backend_ctx->f16_mrow_rpt; if (rpt == 16 && ne00 % 8 == 0 && aligned16 && backend_ctx->kernel_mul_mat_f16_f32_mrow_h8r2 != nullptr) { kernel = backend_ctx->kernel_mul_mat_f16_f32_mrow_h8r2; } else if (rpt == 8 && ne00 % 8 == 0 && aligned16 && backend_ctx->kernel_mul_mat_f16_f32_mrow_h8 != nullptr) { kernel = backend_ctx->kernel_mul_mat_f16_f32_mrow_h8; } else if (rpt == 4 && aligned8 && backend_ctx->kernel_mul_mat_f16_f32_mrow_r4 != nullptr) { kernel = backend_ctx->kernel_mul_mat_f16_f32_mrow_r4; } else if (rpt == 2 && aligned8 && backend_ctx->kernel_mul_mat_f16_f32_mrow_r2 != nullptr) { kernel = backend_ctx->kernel_mul_mat_f16_f32_mrow_r2; } else { kernel = backend_ctx->kernel_mul_mat_f16_f32_mrow; } use_f16_mrow = true; } else { kernel = backend_ctx->kernel_mul_mat_f16_f32_1row; } } else if (adreno_use_lane_split && ne00 >= 64 && ne00 <= 128) { kernel = backend_ctx->kernel_mul_mat_f16_f32_l4_dr_lq; nrows = 1; } else if (adreno_use_lane_split && r2 >= 2 && ne00 > 128 && ne00 <= 256) { kernel = backend_ctx->kernel_mul_mat_f16_f32_l4_dr_ls; nrows = 1; } else if (ne00 >= 128 && ne01 >= 8 && ne00%4 == 0) { // multi-output decode variants when Q is a single row static const char * mm_force_l4_env = getenv("GGML_OPENCL_MM_F16_FORCE_L4"); static const bool mm_force_l4_on = (mm_force_l4_env != nullptr && mm_force_l4_env[0] != '0'); const bool can_multi_out = !mm_force_l4_on && ne11 == 1 && ne01 >= 64 && ne01 % 8 == 0; // paired-K-row variant that doubles per-wave-cycle static const char * mm_kq_pair_env = getenv("GGML_OPENCL_MM_KQ_PAIR"); static const bool mm_kq_pair_on = (mm_kq_pair_env != nullptr && mm_kq_pair_env[0] != '0'); // GQA-coalesced variant that reads each K-row once and // emits gqa_ratio outputs static const char * mm_kq_gqa_env = getenv("GGML_OPENCL_MM_KQ_GQA"); static const bool mm_kq_gqa_on = (mm_kq_gqa_env != nullptr && mm_kq_gqa_env[0] != '0'); // GQA-coalesced KQV variant (DK=128/r2=8/r3=1) that reads // each V slab once per K-head and emits all r2 Q-heads static const char * mm_kqv_gqa_env = getenv("GGML_OPENCL_MM_KQV_GQA"); static const bool mm_kqv_gqa_on = (mm_kqv_gqa_env != nullptr && mm_kqv_gqa_env[0] != '0'); if (can_multi_out && (ne01 % 16) == 0 && ne00 == 128 && r2 == 8 && r3 == 1 && mm_kq_gqa_on && backend_ctx->kernel_mul_mat_f16_f32_l4_x8_gqa4 != nullptr) { kernel = backend_ctx->kernel_mul_mat_f16_f32_l4_x8_gqa4; nrows = 1; } else if (can_multi_out && ne00 <= 256 && mm_kq_pair_on && backend_ctx->kernel_mul_mat_f16_f32_l4_x8_pair != nullptr) { kernel = backend_ctx->kernel_mul_mat_f16_f32_l4_x8_pair; nrows = 1; } else if (can_multi_out && ne00 <= 256 && backend_ctx->kernel_mul_mat_f16_f32_l4_x8 != nullptr) { kernel = backend_ctx->kernel_mul_mat_f16_f32_l4_x8; nrows = 1; } else if (can_multi_out && ne01 == 128 && r2 == 8 && r3 == 1 && mm_kqv_gqa_on && backend_ctx->kernel_mul_mat_f16_f32_l4_y8_gqa != nullptr) { kernel = backend_ctx->kernel_mul_mat_f16_f32_l4_y8_gqa; nrows = 1; } else if (can_multi_out && backend_ctx->kernel_mul_mat_f16_f32_l4_y8 != nullptr) { kernel = backend_ctx->kernel_mul_mat_f16_f32_l4_y8; nrows = 1; } else if (ne11 == 1) { // Decode shapes that don't satisfy the x8/y8 row // constraints (ne01 < 64 or ne01 % 8 != 0) fall back to // upstream's 4-output _dr kernel. kernel = backend_ctx->kernel_mul_mat_f16_f32_l4_dr; nrows = 1; // not used by this kernel } else { kernel = backend_ctx->kernel_mul_mat_f16_f32_l4; nrows = ne11; } } else { kernel = backend_ctx->kernel_mul_mat_f16_f32; nrows = 4; } } else { kernel = backend_ctx->kernel_mul_mat_f16_f16; nrows = 4; } CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device)); CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1)); CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device)); CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd)); CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne00)); CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne01)); CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne02)); CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_ulong), &nb00)); CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_ulong), &nb01)); CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_ulong), &nb02)); CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_ulong), &nb03)); CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &ne10)); CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &ne11)); CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &ne12)); CL_CHECK(clSetKernelArg(kernel, 16, sizeof(cl_ulong), &nb10)); CL_CHECK(clSetKernelArg(kernel, 17, sizeof(cl_ulong), &nb11)); CL_CHECK(clSetKernelArg(kernel, 18, sizeof(cl_ulong), &nb12)); CL_CHECK(clSetKernelArg(kernel, 19, sizeof(cl_ulong), &nb13)); CL_CHECK(clSetKernelArg(kernel, 20, sizeof(int), &ne0)); CL_CHECK(clSetKernelArg(kernel, 21, sizeof(int), &ne1)); CL_CHECK(clSetKernelArg(kernel, 22, sizeof(int), &r2)); CL_CHECK(clSetKernelArg(kernel, 23, sizeof(int), &r3)); if (use_f16_mrow) { const int MROW = 16; // must match MROW in mul_mv_f16_f32_mrow.cl // rows-per-subgroup multiplier for the selected variant: // 1/2/4 -> half4 register blocking; 8 -> half8(1 row); 16 -> half8(2 rows) const int rpt = backend_ctx->f16_mrow_rpt; int rmul; if (rpt == 16) rmul = (ne00 % 8 == 0) ? 2 : 1; else if (rpt == 8) rmul = 1; else rmul = rpt; // 1,2,4 const int rows_per_wg = MROW * rmul; // __local activation buffer: ne00 floats, rounded up for float4 access CL_CHECK(clSetKernelArg(kernel, 24, sizeof(float) * ((ne00 + 3) / 4 * 4), nullptr)); size_t mrow_global[] = { (size_t)((ne01 + rows_per_wg - 1) / rows_per_wg) * 64, (size_t)ne11 * MROW, (size_t)ne12 * ne13 }; size_t mrow_local[] = { 64, (size_t)MROW, 1 }; backend_ctx->enqueue_ndrange_kernel(kernel, 3, mrow_global, mrow_local, dst); return; } break; case GGML_TYPE_Q1_0: { #ifdef GGML_OPENCL_SOA_Q kernel = backend_ctx->kernel_mul_mv_q1_0_f32_flat; // nth0 - subgroup size // nth1 - number of subgroups per workgroup // ndst - number of output values per workgroup = output per subgroup * number of subgroups if (backend_ctx->gpu_family == INTEL) { nth0 = 16; nth1 = 2; ndst = nth1*4; } else if (backend_ctx->gpu_family == ADRENO) { nth0 = 64; nth1 = 2; ndst = nth1*4; } else { GGML_ASSERT(false && "TODO: Unknown GPU"); } CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_q1_0->q)); CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q1_0->d)); CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device)); CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1)); CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device)); CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd)); CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne00)); CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne01)); CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_ulong), &nb01)); CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_ulong), &nb02)); CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_ulong), &nb03)); CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne12)); CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_ulong), &nb11)); CL_CHECK(clSetKernelArg(kernel, 13, sizeof(cl_ulong), &nb12)); CL_CHECK(clSetKernelArg(kernel, 14, sizeof(cl_ulong), &nb13)); CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &ne0)); CL_CHECK(clSetKernelArg(kernel, 16, sizeof(int), &ne1)); CL_CHECK(clSetKernelArg(kernel, 17, sizeof(int), &r2)); CL_CHECK(clSetKernelArg(kernel, 18, sizeof(int), &r3)); #else kernel = backend_ctx->kernel_mul_mv_q1_0_f32; if (backend_ctx->gpu_family == INTEL) { nth0 = 16; nth1 = 2; ndst = nth1*4; } else if (backend_ctx->gpu_family == ADRENO) { nth0 = 64; nth1 = 2; ndst = nth1*4; } else { GGML_ASSERT(false && "TODO: Unknown GPU"); } CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device)); CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1)); CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device)); CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd)); CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne00)); CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne01)); CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_ulong), &nb01)); CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_ulong), &nb02)); CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_ulong), &nb03)); CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne12)); CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_ulong), &nb11)); CL_CHECK(clSetKernelArg(kernel, 13, sizeof(cl_ulong), &nb12)); CL_CHECK(clSetKernelArg(kernel, 14, sizeof(cl_ulong), &nb13)); CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &ne0)); CL_CHECK(clSetKernelArg(kernel, 16, sizeof(int), &ne1)); CL_CHECK(clSetKernelArg(kernel, 17, sizeof(int), &r2)); CL_CHECK(clSetKernelArg(kernel, 18, sizeof(int), &r3)); #endif // GGML_OPENCL_SOA_Q break; } case GGML_TYPE_Q4_0: // This should have been satisfied. GGML_ASSERT(ne11 == ne1); GGML_ASSERT(ne01 == ne0); #ifdef GGML_OPENCL_SOA_Q if (backend_ctx->gpu_family == INTEL) { nth0 = 16; nth1 = 1; kernel = backend_ctx->kernel_mul_mat_q4_0_f32_8x_flat; ndst = 8; } else if (backend_ctx->gpu_family == ADRENO) { nth0 = 64; nth1 = 1; kernel = backend_ctx->kernel_mul_mat_q4_0_f32_8x_flat; ndst =8; } else { GGML_ASSERT(false && "TODO: Unknown GPU"); } CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_q4_0->q)); CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q4_0->d)); CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device)); CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1)); CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device)); CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd)); CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne00)); CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne01)); CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne02)); CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne10)); CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &ne12)); CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne0)); CL_CHECK(clSetKernelArg(kernel, 12, sizeof(int), &ne1)); CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &r2)); CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &r3)); #else // GGML_OPENCL_SOA_Q if (backend_ctx->gpu_family == INTEL) { // Use 1D local size. Each workgroup is a SIMD group. Each SIMD // group produces N_DST (4 for Q4_0 kernel) values in the result. // The number of workgroups on dim 0 (the leading dimension) is // the nearest multiple of 4 that covers ne0 (equals ne01). nth0 = 16; nth1 = 1; kernel = backend_ctx->kernel_mul_mat_q4_0_f32; ndst = 4; } else if (backend_ctx->gpu_family == ADRENO) { nth0 = 64; nth1 = 1; kernel = backend_ctx->kernel_mul_mat_q4_0_f32_v; ndst = 4; } else { GGML_ASSERT(false && "TODO: Unknown GPU"); } CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device)); CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1)); CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device)); CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd)); CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne00)); CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne01)); CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne02)); CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne10)); CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &ne12)); CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne0)); CL_CHECK(clSetKernelArg(kernel, 12, sizeof(int), &ne1)); CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &r2)); CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &r3)); #endif // GGML_OPENCL_SOA_Q break; case GGML_TYPE_Q4_1: { #ifdef GGML_OPENCL_SOA_Q if (backend_ctx->gpu_family == INTEL) { nth0 = 16; nth1 = 1; ndst = 4; } else if (backend_ctx->gpu_family == ADRENO) { nth0 = 64; nth1 = 1; ndst = 4; } else { GGML_ASSERT(false && "TODO: Unknown GPU"); } kernel = backend_ctx->kernel_mul_mv_q4_1_f32_flat; CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_q4_1->q)); CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q4_1->d)); CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra0_q4_1->m)); CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra1->data_device)); CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_ulong), &offset1)); CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_mem), &extrad->data_device)); CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_ulong), &offsetd)); CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne00)); CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne01)); CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne02)); CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &ne10)); CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne12)); CL_CHECK(clSetKernelArg(kernel, 12, sizeof(int), &ne0)); CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &ne1)); CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &r2)); CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &r3)); #else if (backend_ctx->gpu_family == INTEL) { nth0 = 16; nth1 = 1; ndst = 4; } else if (backend_ctx->gpu_family == ADRENO) { nth0 = 64; nth1 = 1; ndst = 4; } else { GGML_ASSERT(false && "TODO: Unknown GPU"); } kernel = backend_ctx->kernel_mul_mv_q4_1_f32; CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device)); CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1)); CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device)); CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd)); CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne00)); CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne01)); CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne02)); CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne10)); CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &ne12)); CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne0)); CL_CHECK(clSetKernelArg(kernel, 12, sizeof(int), &ne1)); CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &r2)); CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &r3)); #endif // GGML_OPENCL_SOA_Q break; } case GGML_TYPE_Q5_0: { #ifdef GGML_OPENCL_SOA_Q if (backend_ctx->gpu_family == INTEL) { nth0 = 16; nth1 = 1; ndst = 4; } else if (backend_ctx->gpu_family == ADRENO) { nth0 = 64; nth1 = 1; ndst = 4; } else { GGML_ASSERT(false && "TODO: Unknown GPU"); } kernel = backend_ctx->kernel_mul_mv_q5_0_f32_flat; CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_q5_0->qs)); CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q5_0->qh)); CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra0_q5_0->d)); CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra1->data_device)); CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_ulong), &offset1)); CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_mem), &extrad->data_device)); CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_ulong), &offsetd)); CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne00)); CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne01)); CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne02)); CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &ne10)); CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne12)); CL_CHECK(clSetKernelArg(kernel, 12, sizeof(int), &ne0)); CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &ne1)); CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &r2)); CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &r3)); #else if (backend_ctx->gpu_family == INTEL) { nth0 = 16; nth1 = 1; ndst = 4; } else if (backend_ctx->gpu_family == ADRENO) { nth0 = 64; nth1 = 1; ndst = 4; } else { GGML_ASSERT(false && "TODO: Unknown GPU"); } kernel = backend_ctx->kernel_mul_mv_q5_0_f32; CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device)); CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1)); CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device)); CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd)); CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne00)); CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne01)); CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne02)); CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne10)); CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &ne12)); CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne0)); CL_CHECK(clSetKernelArg(kernel, 12, sizeof(int), &ne1)); CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &r2)); CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &r3)); #endif // GGML_OPENCL_SOA_Q break; } case GGML_TYPE_Q5_1: { #ifdef GGML_OPENCL_SOA_Q if (backend_ctx->gpu_family == INTEL) { nth0 = 16; nth1 = 1; ndst = 4; } else if (backend_ctx->gpu_family == ADRENO) { nth0 = 64; nth1 = 1; ndst = 4; } else { GGML_ASSERT(false && "TODO: Unknown GPU"); } kernel = backend_ctx->kernel_mul_mv_q5_1_f32_flat; CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_q5_1->qs)); CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q5_1->qh)); CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra0_q5_1->d)); CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra0_q5_1->m)); CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extra1->data_device)); CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offset1)); CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_mem), &extrad->data_device)); CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_ulong), &offsetd)); CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne00)); CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne01)); CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &ne02)); CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne10)); CL_CHECK(clSetKernelArg(kernel, 12, sizeof(int), &ne12)); CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &ne0)); CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &ne1)); CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &r2)); CL_CHECK(clSetKernelArg(kernel, 16, sizeof(int), &r3)); #else if (backend_ctx->gpu_family == INTEL) { nth0 = 16; nth1 = 1; ndst = 4; } else if (backend_ctx->gpu_family == ADRENO) { nth0 = 64; nth1 = 1; ndst = 4; } else { GGML_ASSERT(false && "TODO: Unknown GPU"); } kernel = backend_ctx->kernel_mul_mv_q5_1_f32; CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device)); CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1)); CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device)); CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd)); CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne00)); CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne01)); CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne02)); CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne10)); CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &ne12)); CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne0)); CL_CHECK(clSetKernelArg(kernel, 12, sizeof(int), &ne1)); CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &r2)); CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &r3)); #endif // GGML_OPENCL_SOA_Q break; } case GGML_TYPE_Q8_0: { #ifdef GGML_OPENCL_SOA_Q kernel = backend_ctx->kernel_mul_mv_q8_0_f32_flat; // nth0 - subgroup size // nth1 - number of subgroups per workgroup // ndst - number of output values per workgroup = output per subgroup * number of subgroups if (backend_ctx->gpu_family == INTEL) { nth0 = 16; nth1 = 2; ndst = nth1*4; } else if (backend_ctx->gpu_family == ADRENO) { nth0 = 64; nth1 = 2; ndst = nth1*4; } else { GGML_ASSERT(false && "TODO: Unknown GPU"); } CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_q8_0->q)); CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q8_0->d)); CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device)); CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1)); CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device)); CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd)); CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne00)); CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne01)); CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_ulong), &nb01)); CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_ulong), &nb02)); CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_ulong), &nb03)); CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne12)); CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_ulong), &nb11)); CL_CHECK(clSetKernelArg(kernel, 13, sizeof(cl_ulong), &nb12)); CL_CHECK(clSetKernelArg(kernel, 14, sizeof(cl_ulong), &nb13)); CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &ne0)); CL_CHECK(clSetKernelArg(kernel, 16, sizeof(int), &ne1)); CL_CHECK(clSetKernelArg(kernel, 17, sizeof(int), &r2)); CL_CHECK(clSetKernelArg(kernel, 18, sizeof(int), &r3)); #else kernel = backend_ctx->kernel_mul_mv_q8_0_f32; // nth0 - subgroup size // nth1 - number of subgroups per workgroup // ndst - number of output values per workgroup = output per subgroup * number of subgroups if (backend_ctx->gpu_family == INTEL) { nth0 = 16; nth1 = 2; ndst = nth1*4; } else if (backend_ctx->gpu_family == ADRENO) { nth0 = 64; nth1 = 2; ndst = nth1*4; } else { GGML_ASSERT(false && "TODO: Unknown GPU"); } CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device)); CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1)); CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device)); CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd)); CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne00)); CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne01)); CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_ulong), &nb01)); CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_ulong), &nb02)); CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_ulong), &nb03)); CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne12)); CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_ulong), &nb11)); CL_CHECK(clSetKernelArg(kernel, 13, sizeof(cl_ulong), &nb12)); CL_CHECK(clSetKernelArg(kernel, 14, sizeof(cl_ulong), &nb13)); CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &ne0)); CL_CHECK(clSetKernelArg(kernel, 16, sizeof(int), &ne1)); CL_CHECK(clSetKernelArg(kernel, 17, sizeof(int), &r2)); CL_CHECK(clSetKernelArg(kernel, 18, sizeof(int), &r3)); #endif // GGML_OPENCL_SOA_Q break; } case GGML_TYPE_IQ4_NL: { #ifdef GGML_OPENCL_SOA_Q kernel = backend_ctx->kernel_mul_mv_iq4_nl_f32_flat; if (backend_ctx->gpu_family == INTEL) { nth0 = 16; nth1 = 1; ndst = 8; } else if (backend_ctx->gpu_family == ADRENO) { nth0 = 64; nth1 = 1; ndst = 8; } else { GGML_ASSERT(false && "TODO: Unknown GPU"); } CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_iq4_nl->q)); CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_iq4_nl->d)); CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device)); CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1)); CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device)); CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd)); CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne00)); CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne01)); CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne02)); CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne10)); CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &ne12)); CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne0)); CL_CHECK(clSetKernelArg(kernel, 12, sizeof(int), &ne1)); CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &r2)); CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &r3)); #else kernel = backend_ctx->kernel_mul_mv_iq4_nl_f32; if (backend_ctx->gpu_family == INTEL) { nth0 = 16; nth1 = 1; ndst = 4; } else if (backend_ctx->gpu_family == ADRENO) { nth0 = 64; nth1 = 1; ndst = 4; } else { GGML_ASSERT(false && "TODO: Unknown GPU"); } CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device)); CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1)); CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device)); CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd)); CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne00)); CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne01)); CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne02)); CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne10)); CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &ne12)); CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne0)); CL_CHECK(clSetKernelArg(kernel, 12, sizeof(int), &ne1)); CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &r2)); CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &r3)); #endif // GGML_OPENCL_SOA_Q break; } case GGML_TYPE_Q2_K: case GGML_TYPE_Q3_K: case GGML_TYPE_Q4_K: { #ifdef GGML_OPENCL_SOA_Q kernel = backend_ctx->kernel_mul_mv_q4_K_f32_flat; if (backend_ctx->gpu_family == INTEL) { nth0 = 16; nth1 = 1; ndst = 16; // 8->16 rows per subgroup — matches N_DST in mul_mv_q4_k_f32_flat.cl (32 spills) } else if (backend_ctx->gpu_family == ADRENO) { nth0 = 64; nth1 = 2; ndst = 16; } else { GGML_ASSERT(false && "TODO: Unknown GPU"); } CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_q4_K->q)); CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q4_K->s)); CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra0_q4_K->d)); CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra0_q4_K->dm)); CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extra1->data_device)); CL_CHECK(clSetKernelArg(kernel, 5, sizeof(int), &offset1)); CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_mem), &extrad->data_device)); CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &offsetd)); CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne00)); CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne01)); CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_ulong), &nb01)); CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_ulong), &nb02)); CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_ulong), &nb03)); CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &ne12)); CL_CHECK(clSetKernelArg(kernel, 14, sizeof(cl_ulong), &nb11)); CL_CHECK(clSetKernelArg(kernel, 15, sizeof(cl_ulong), &nb12)); CL_CHECK(clSetKernelArg(kernel, 16, sizeof(cl_ulong), &nb13)); CL_CHECK(clSetKernelArg(kernel, 17, sizeof(int), &ne0)); CL_CHECK(clSetKernelArg(kernel, 18, sizeof(int), &ne1)); CL_CHECK(clSetKernelArg(kernel, 19, sizeof(int), &r2)); CL_CHECK(clSetKernelArg(kernel, 20, sizeof(int), &r3)); #else kernel = backend_ctx->kernel_mul_mv_q4_K_f32; if (backend_ctx->gpu_family == INTEL) { nth0 = 16; nth1 = 1; ndst = 4; } else if (backend_ctx->gpu_family == ADRENO) { nth0 = 64; nth1 = 1; ndst = 4; } else { GGML_ASSERT(false && "TODO: Unknown GPU"); } CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); CL_CHECK(clSetKernelArg(kernel, 1, sizeof(int), &offset0)); CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device)); CL_CHECK(clSetKernelArg(kernel, 3, sizeof(int), &offset1)); CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device)); CL_CHECK(clSetKernelArg(kernel, 5, sizeof(int), &offsetd)); CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne00)); CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne01)); CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_ulong), &nb01)); CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_ulong), &nb02)); CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_ulong), &nb03)); CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne12)); CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_ulong), &nb11)); CL_CHECK(clSetKernelArg(kernel, 13, sizeof(cl_ulong), &nb12)); CL_CHECK(clSetKernelArg(kernel, 14, sizeof(cl_ulong), &nb13)); CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &ne0)); CL_CHECK(clSetKernelArg(kernel, 16, sizeof(int), &ne1)); CL_CHECK(clSetKernelArg(kernel, 17, sizeof(int), &r2)); CL_CHECK(clSetKernelArg(kernel, 18, sizeof(int), &r3)); #endif // GGML_OPENCL_SOA_Q break; } case GGML_TYPE_Q5_K: { #ifdef GGML_OPENCL_SOA_Q kernel = backend_ctx->kernel_mul_mv_q5_K_f32_flat; if (backend_ctx->gpu_family == INTEL) { nth0 = 16; nth1 = 1; ndst = 8; // 4->8 rows per subgroup (2x activation reuse) } else if (backend_ctx->gpu_family == ADRENO) { nth0 = 64; nth1 = 2; ndst = 16; } else { GGML_ASSERT(false && "TODO: Unknown GPU"); } CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_q5_K->q)); CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q5_K->qh)); CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra0_q5_K->s)); CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra0_q5_K->d)); CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extra0_q5_K->dm)); CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_mem), &extra1->data_device)); CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &offset1)); CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_mem), &extrad->data_device)); CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &offsetd)); CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne00)); CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &ne01)); CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_ulong), &nb01)); CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_ulong), &nb02)); CL_CHECK(clSetKernelArg(kernel, 13, sizeof(cl_ulong), &nb03)); CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &ne12)); CL_CHECK(clSetKernelArg(kernel, 15, sizeof(cl_ulong), &nb11)); CL_CHECK(clSetKernelArg(kernel, 16, sizeof(cl_ulong), &nb12)); CL_CHECK(clSetKernelArg(kernel, 17, sizeof(cl_ulong), &nb13)); CL_CHECK(clSetKernelArg(kernel, 18, sizeof(int), &ne0)); CL_CHECK(clSetKernelArg(kernel, 19, sizeof(int), &ne1)); CL_CHECK(clSetKernelArg(kernel, 20, sizeof(int), &r2)); CL_CHECK(clSetKernelArg(kernel, 21, sizeof(int), &r3)); #else kernel = backend_ctx->kernel_mul_mv_q5_K_f32; if (backend_ctx->gpu_family == INTEL) { nth0 = 16; nth1 = 1; ndst = 4; } else if (backend_ctx->gpu_family == ADRENO) { nth0 = 64; nth1 = 1; ndst = 4; } else { GGML_ASSERT(false && "TODO: Unknown GPU"); } CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); CL_CHECK(clSetKernelArg(kernel, 1, sizeof(int), &offset0)); CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device)); CL_CHECK(clSetKernelArg(kernel, 3, sizeof(int), &offset1)); CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device)); CL_CHECK(clSetKernelArg(kernel, 5, sizeof(int), &offsetd)); CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne00)); CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne01)); CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_ulong), &nb01)); CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_ulong), &nb02)); CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_ulong), &nb03)); CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne12)); CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_ulong), &nb11)); CL_CHECK(clSetKernelArg(kernel, 13, sizeof(cl_ulong), &nb12)); CL_CHECK(clSetKernelArg(kernel, 14, sizeof(cl_ulong), &nb13)); CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &ne0)); CL_CHECK(clSetKernelArg(kernel, 16, sizeof(int), &ne1)); CL_CHECK(clSetKernelArg(kernel, 17, sizeof(int), &r2)); CL_CHECK(clSetKernelArg(kernel, 18, sizeof(int), &r3)); #endif // GGML_OPENCL_SOA_Q break; } case GGML_TYPE_Q6_K: #ifdef GGML_OPENCL_SOA_Q kernel = backend_ctx->kernel_mul_mv_q6_K_f32_flat; if (backend_ctx->gpu_family == INTEL) { nth0 = 16; nth1 = 2; ndst = 4; } else if (backend_ctx->gpu_family == ADRENO) { nth0 = 64; nth1 = 2; ndst = 16; } else { GGML_ASSERT(false && "TODO: Unknown GPU"); } CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_q6_K->ql)); CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q6_K->qh)); CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra0_q6_K->s)); CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra0_q6_K->d)); CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extra1->data_device)); CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offset1)); CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_mem), &extrad->data_device)); CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_ulong), &offsetd)); CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne00)); CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne01)); CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &ne02)); CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne10)); CL_CHECK(clSetKernelArg(kernel, 12, sizeof(int), &ne12)); CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &ne0)); CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &ne1)); CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &r2)); CL_CHECK(clSetKernelArg(kernel, 16, sizeof(int), &r3)); // The optimizer-barrier arg exists only in the ADRENO_OLD_COMPILER build of // this kernel; conformant compilers get the original 17-arg signature. if (backend_ctx->q6_k_flat_old_compiler) { cl_uchar q6k_mask = 0xFF; // never 0xFE in prod; see the kernel note CL_CHECK(clSetKernelArg(kernel, 17, sizeof(cl_uchar), &q6k_mask)); } #else kernel = backend_ctx->kernel_mul_mv_q6_K_f32; if (backend_ctx->gpu_family == INTEL) { nth0 = 16; nth1 = 2; ndst = 1; } else if (backend_ctx->gpu_family == ADRENO) { nth0 = 64; nth1 = 2; ndst = 1; } else { GGML_ASSERT(false && "TODO: Unknown GPU"); } CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device)); CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1)); CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device)); CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd)); CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne00)); CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne01)); CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne02)); CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne10)); CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &ne12)); CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne0)); CL_CHECK(clSetKernelArg(kernel, 12, sizeof(int), &ne1)); CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &r2)); CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &r3)); #endif // GGML_OPENCL_SOA_Q break; case GGML_TYPE_MXFP4: { #ifdef GGML_OPENCL_SOA_Q kernel = backend_ctx->kernel_mul_mv_mxfp4_f32_flat; cl_mem q; if (backend_ctx->gpu_family == INTEL) { nth0 = 16; nth1 = 2; ndst = nth1*2; q = extra0_mxfp4->q; } else if (backend_ctx->gpu_family == ADRENO) { nth0 = 64; nth1 = 2; ndst = nth1*2; q = extra0_mxfp4->q_img; } else { GGML_ASSERT(false && "TODO: Unknown GPU"); } CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &q)); CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_mxfp4->e)); CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device)); CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1)); CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device)); CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd)); CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne00)); CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_ulong), &nb01)); CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_ulong), &nb02)); CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_ulong), &nb03)); CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &ne12)); CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_ulong), &nb11)); CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_ulong), &nb12)); CL_CHECK(clSetKernelArg(kernel, 13, sizeof(cl_ulong), &nb13)); CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &ne0)); CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &ne1)); CL_CHECK(clSetKernelArg(kernel, 16, sizeof(int), &r2)); CL_CHECK(clSetKernelArg(kernel, 17, sizeof(int), &r3)); #else kernel = backend_ctx->kernel_mul_mv_mxfp4_f32; if (backend_ctx->gpu_family == INTEL) { nth0 = 16; nth1 = 2; ndst = nth1*2; } else if (backend_ctx->gpu_family == ADRENO) { nth0 = 64; nth1 = 2; ndst = nth1*2; } else { GGML_ASSERT(false && "TODO: Unknown GPU"); } CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device)); CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1)); CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device)); CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd)); CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne00)); CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_ulong), &nb01)); CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_ulong), &nb02)); CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_ulong), &nb03)); CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &ne12)); CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_ulong), &nb11)); CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_ulong), &nb12)); CL_CHECK(clSetKernelArg(kernel, 13, sizeof(cl_ulong), &nb13)); CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &ne0)); CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &ne1)); CL_CHECK(clSetKernelArg(kernel, 16, sizeof(int), &r2)); CL_CHECK(clSetKernelArg(kernel, 17, sizeof(int), &r3)); CL_CHECK(clSetKernelArg(kernel, 18, sizeof(float)*nth0,nullptr)); #endif break; } default: GGML_ASSERT(false && "not implemented"); } if (src0t == GGML_TYPE_Q4_0 || src0t == GGML_TYPE_MXFP4 || src0t == GGML_TYPE_Q4_1 || src0t == GGML_TYPE_Q5_0 || src0t == GGML_TYPE_Q5_1 || src0t == GGML_TYPE_Q8_0 || src0t == GGML_TYPE_Q1_0 || src0t == GGML_TYPE_IQ4_NL || src0t == GGML_TYPE_Q2_K) { // Each SIMD group produces N_DST values in the result. Assuming each // workgroup has N_SIMDGROUP SIMD groups, then each workgroup will // produce N_DST*N_SIMDGROUP values in the result. Hence, the grid size // (number of workgroups) will be a nearest multiple of // N_DST*N_SIMDGROUP to cover the size of the dimension. Below, 4 is // N_DST*N_SIMDGROUP (see the kernel for Q4_0 matmul). size_t global_work_size[] = {(size_t)(ne01 + ndst-1)/ndst*nth0, (size_t)ne11*nth1, (size_t)ne12*ne13}; size_t local_work_size[] = {(size_t)nth0, (size_t)nth1, 1}; backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); } else if (src0t == GGML_TYPE_Q4_K) { size_t global_work_size[] = {(size_t)(ne01+ndst*nth1-1)/(ndst*nth1)*nth0, (size_t)ne11*nth1, (size_t)ne12*ne13}; size_t local_work_size[] = {(size_t)nth0, (size_t)nth1, 1}; backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); } else if (src0t == GGML_TYPE_Q3_K) { GGML_ASSERT(false && "not implemented"); } else if (src0t == GGML_TYPE_Q5_K) { size_t global_work_size[] = {(size_t)(ne01+ndst*nth1-1)/(ndst*nth1)*nth0, (size_t)ne11*nth1, (size_t)ne12*ne13}; size_t local_work_size[] = {(size_t)nth0, (size_t)nth1, 1}; backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); } else if (src0t == GGML_TYPE_Q6_K) { size_t global_work_size[] = {(size_t)(ne01+ndst*nth1-1)/(ndst*nth1)*nth0, (size_t)ne11*nth1, (size_t)ne12*ne13}; size_t local_work_size[] = {(size_t)nth0, (size_t)nth1, 1}; backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); } else if (kernel == backend_ctx->kernel_mul_mat_f16_f32_l4_x8 || kernel == backend_ctx->kernel_mul_mat_f16_f32_l4_x8_pair || kernel == backend_ctx->kernel_mul_mat_f16_f32_l4_y8) { // multi-output decode variants: each WG processes 8 outputs along ne01, ne11 == 1 const int64_t n_wg_x = ne01 / 8; size_t global_work_size[] = {(size_t)n_wg_x*nth0, (size_t)nth1, (size_t)ne12*ne13}; size_t local_work_size[] = {(size_t)nth0, (size_t)nth1, 1}; backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); } else if (kernel == backend_ctx->kernel_mul_mat_f16_f32_l4_x8_gqa4) { // GQA-coalesced KQ: one WG per K-head emits N_K_ROWS_GQA=16 K-rows * r2 Q-heads const int64_t n_wg_x = ne01 / 16; size_t global_work_size[] = {(size_t)n_wg_x*nth0, (size_t)nth1, (size_t)ne02*ne13}; size_t local_work_size[] = {(size_t)nth0, (size_t)nth1, 1}; backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); } else if (kernel == backend_ctx->kernel_mul_mat_f16_f32_l4_y8_gqa) { // GQA-coalesced KQV: one WG per K-head emits 8 DV-rows * r2 Q-heads const int64_t n_wg_x = ne01 / 8; size_t global_work_size[] = {(size_t)n_wg_x*nth0, (size_t)nth1, (size_t)ne02*ne13}; size_t local_work_size[] = {(size_t)nth0, (size_t)nth1, 1}; backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); } else { if (kernel == backend_ctx->kernel_mul_mat_f16_f32_l4_dr) { const int NDST_DR = 4; size_t global_work_size[] = {(size_t)CEIL_DIV(ne01, NDST_DR)*nth0, (size_t)nth1, (size_t)ne12*ne13}; size_t local_work_size[] = {(size_t)nth0, (size_t)nth1, 1}; backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); } else if (kernel == backend_ctx->kernel_mul_mat_f16_f32_l4_dr_ls) { size_t global_work_size[] = {(size_t)CEIL_DIV(ne01, 2)*nth0, (size_t)nth1, (size_t)ne02*ne03}; size_t local_work_size[] = {(size_t)nth0, (size_t)nth1, 1}; backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); } else if (kernel == backend_ctx->kernel_mul_mat_f16_f32_l4_dr_lq) { size_t global_work_size[] = {(size_t)CEIL_DIV(ne01, 4)*nth0, (size_t)nth1, (size_t)ne02*ne03}; size_t local_work_size[] = {(size_t)nth0, (size_t)nth1, 1}; backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); } else { int64_t ny = (ne11 + nrows - 1)/nrows; size_t global_work_size[] = {(size_t)ne01*nth0, (size_t)ny*nth1, (size_t)ne12*ne13}; size_t local_work_size[] = {(size_t)nth0, (size_t)nth1, 1}; backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); } } } static void moe_router_reoerder(ggml_backend_t backend, const ggml_tensor * src, int ne20) { cl_int err; ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; ggml_tensor_extra_cl * extra = (ggml_tensor_extra_cl *)src->extra; cl_ulong offset = extra->offset + src->view_offs; const int ne21 = src->ne[1]; const int nb21 = src->nb[1]; const int ne02 = nb21 / src->nb[0]; const int n_tile_size = 32; const int max_post_router_tile = (ne20 * ne21 / n_tile_size) + ne02; cl_buffer_region region; region.origin = offset; region.size = nb21 * ne21; cl_mem original_router_buf = clCreateSubBuffer(extra->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err); CL_CHECK(err); backend_ctx->prealloc_post_router.allocate(backend_ctx->context, sizeof(int) * max_post_router_tile * n_tile_size); region.origin = 0; region.size = sizeof(int) * max_post_router_tile * n_tile_size; cl_mem post_router_buf = clCreateSubBuffer(backend_ctx->prealloc_post_router.buffer, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err); CL_CHECK(err); backend_ctx->prealloc_emap.allocate(backend_ctx->context, sizeof(short) * max_post_router_tile); region.origin = 0; region.size = sizeof(short) * max_post_router_tile; cl_mem emap_buf = clCreateSubBuffer(backend_ctx->prealloc_emap.buffer, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err); CL_CHECK(err); backend_ctx->prealloc_hist.allocate(backend_ctx->context, sizeof(int) * ne02); region.origin = 0; region.size = sizeof(int) * ne02; cl_mem hist_buf = clCreateSubBuffer(backend_ctx->prealloc_hist.buffer, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err); CL_CHECK(err); backend_ctx->prealloc_tile_offset.allocate(backend_ctx->context, sizeof(int) * ne02); region.origin = 0; region.size = sizeof(int) * ne02; cl_mem tile_offset_buf = clCreateSubBuffer(backend_ctx->prealloc_tile_offset.buffer, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err); CL_CHECK(err); backend_ctx->prealloc_slot_counter.allocate(backend_ctx->context, sizeof(int) * ne02); region.origin = 0; region.size = sizeof(int) * ne02; cl_mem slot_counter_buf = clCreateSubBuffer(backend_ctx->prealloc_slot_counter.buffer, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err); CL_CHECK(err); backend_ctx->prealloc_total_tiles.allocate(backend_ctx->context, sizeof(int)); region.origin = 0; region.size = sizeof(int); cl_mem total_tiles_buf = clCreateSubBuffer(backend_ctx->prealloc_total_tiles.buffer, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err); CL_CHECK(err); // Histogram cl_kernel kernel = backend_ctx->kernel_moe_histogram; CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &original_router_buf)); CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &hist_buf)); CL_CHECK(clSetKernelArg(kernel, 2, sizeof(int), &ne21)); CL_CHECK(clSetKernelArg(kernel, 3, sizeof(int), &ne20)); CL_CHECK(clSetKernelArg(kernel, 4, sizeof(int), &ne02)); size_t histogram_global_size[] = {(size_t)(((ne21 + 63) / 64) * 64), static_cast(ne20), 1}; size_t histogram_local_size[] = {64, 1, 1}; backend_ctx->enqueue_ndrange_kernel(kernel, 3, histogram_global_size, histogram_local_size, src); // Scan kernel = backend_ctx->kernel_moe_scan; CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &hist_buf)); CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &tile_offset_buf)); CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &total_tiles_buf)); CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &slot_counter_buf)); CL_CHECK(clSetKernelArg(kernel, 4, sizeof(int), &n_tile_size)); CL_CHECK(clSetKernelArg(kernel, 5, sizeof(int), &ne02)); size_t scan_global_size[] = {1}; size_t scan_local_size[] = {1}; backend_ctx->enqueue_ndrange_kernel(kernel, 1, scan_global_size, scan_local_size, src); // Fill kernel = backend_ctx->kernel_moe_fill; CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &post_router_buf)); CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &total_tiles_buf)); CL_CHECK(clSetKernelArg(kernel, 2, sizeof(int), &n_tile_size)); size_t fill_global_size[] = {(size_t)(((max_post_router_tile + 63) / 64) * 64), n_tile_size, 1}; size_t fill_local_size[] = {64, 1, 1}; backend_ctx->enqueue_ndrange_kernel(kernel, 3, fill_global_size, fill_local_size, src); // Scatter. The deterministic variant is the default: kernel_moe_scatter derives // each token's slot from an atomic counter, so the packing inside an expert - and // with it the output of the ragged prefill GEMM - changes from run to run. Set // GGML_OPENCL_MOE_STABLE_SCATTER=0 to restore the atomic version. static const bool stable_scatter = []{ const char * e = getenv("GGML_OPENCL_MOE_STABLE_SCATTER"); return !e || e[0] == '\0' || e[0] != '0'; }(); if (stable_scatter) { kernel = backend_ctx->kernel_moe_scatter_stable; CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &original_router_buf)); CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &post_router_buf)); CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &emap_buf)); CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &tile_offset_buf)); CL_CHECK(clSetKernelArg(kernel, 4, sizeof(int), &ne21)); CL_CHECK(clSetKernelArg(kernel, 5, sizeof(int), &ne20)); CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne02)); // one workgroup (one wave) per expert; each ranks its own tokens size_t scatter_global_size[] = {64, (size_t)ne02}; size_t scatter_local_size[] = {64, 1}; backend_ctx->enqueue_ndrange_kernel(kernel, 2, scatter_global_size, scatter_local_size, src); } else { kernel = backend_ctx->kernel_moe_scatter; CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &original_router_buf)); CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &post_router_buf)); CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &emap_buf)); CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &tile_offset_buf)); CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &slot_counter_buf)); CL_CHECK(clSetKernelArg(kernel, 5, sizeof(int), &ne21)); CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne20)); CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne02)); backend_ctx->enqueue_ndrange_kernel(kernel, 3, histogram_global_size, histogram_local_size, src); } // [MOE_TILES] env-gated padding probe: read back total_tiles (= Sum_e // ceil(k_e/n_tile_size)) and compare to the ideal tile count for the real // routing count. Quantifies the per-expert tile-padding waste. Blocking // readback perturbs timing -> diagnostic only. if (getenv("GGML_OPENCL_MOE_TILES_DEBUG")) { int h_total = 0; clFinish(backend_ctx->queue); CL_CHECK(clEnqueueReadBuffer(backend_ctx->queue, total_tiles_buf, CL_TRUE, 0, sizeof(int), &h_total, 0, NULL, NULL)); const int routings = ne20 * ne21; const int ideal = (routings + n_tile_size - 1) / n_tile_size; const int slots = h_total * n_tile_size; fprintf(stderr, "[MOE_TILES] routings=%d (ne20=%d ne21=%d nexp=%d) total_tiles=%d ideal=%d slots=%d pad=%.1f%%\n", routings, ne20, ne21, ne02, h_total, ideal, slots, routings > 0 ? 100.0 * (slots - routings) / routings : 0.0); fflush(stderr); } CL_CHECK(clReleaseMemObject(original_router_buf)); CL_CHECK(clReleaseMemObject(hist_buf)); CL_CHECK(clReleaseMemObject(tile_offset_buf)); CL_CHECK(clReleaseMemObject(total_tiles_buf)); CL_CHECK(clReleaseMemObject(slot_counter_buf)); CL_CHECK(clReleaseMemObject(post_router_buf)); CL_CHECK(clReleaseMemObject(emap_buf)); } static void ggml_cl_mul_mat_id(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { GGML_ASSERT(src0); GGML_ASSERT(src0->extra); GGML_ASSERT(src1); GGML_ASSERT(src1->extra); GGML_ASSERT(dst); GGML_ASSERT(dst->extra); const ggml_tensor * src2 = dst->src[2]; GGML_ASSERT(src2); GGML_ASSERT(src2->extra); ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; ggml_tensor_extra_cl * extra0 = (ggml_tensor_extra_cl *)src0->extra; ggml_tensor_extra_cl * extra1 = (ggml_tensor_extra_cl *)src1->extra; ggml_tensor_extra_cl * extra2 = (ggml_tensor_extra_cl *)src2->extra; ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; cl_ulong offset0 = extra0->offset + src0->view_offs; cl_ulong offset1 = extra1->offset + src1->view_offs; cl_ulong offset2 = extra2->offset + src2->view_offs; cl_ulong offsetd = extrad->offset + dst->view_offs; GGML_UNUSED(offset0); #ifdef GGML_OPENCL_SOA_Q // SoA extra lives on view_src (view->extra is pre-SoA). const ggml_tensor * soa0_src = src0->view_src != nullptr ? src0->view_src : src0; ggml_tensor_extra_cl_q4_0 * extra0_q4_0 = (ggml_tensor_extra_cl_q4_0 *)soa0_src->extra; ggml_tensor_extra_cl_q4_1 * extra0_q4_1 = (ggml_tensor_extra_cl_q4_1 *)soa0_src->extra; ggml_tensor_extra_cl_q5_0 * extra0_q5_0 = (ggml_tensor_extra_cl_q5_0 *)soa0_src->extra; ggml_tensor_extra_cl_q5_1 * extra0_q5_1 = (ggml_tensor_extra_cl_q5_1 *)soa0_src->extra; ggml_tensor_extra_cl_q4_K * extra0_q4_K = (ggml_tensor_extra_cl_q4_K *)soa0_src->extra; ggml_tensor_extra_cl_q5_K * extra0_q5_K = (ggml_tensor_extra_cl_q5_K *)soa0_src->extra; ggml_tensor_extra_cl_q6_K * extra0_q6_K = (ggml_tensor_extra_cl_q6_K *)soa0_src->extra; ggml_tensor_extra_cl_mxfp4 * extra0_mxfp4 = (ggml_tensor_extra_cl_mxfp4 *)soa0_src->extra; ggml_tensor_extra_cl_q8_0 * extra0_q8_0 = (ggml_tensor_extra_cl_q8_0 *)soa0_src->extra; #endif // TODO: general MoE for the following types (void)extra0_q4_1; (void)extra0_q5_0; (void)extra0_q5_1; (void)extra0_q4_K; (void)extra0_q5_K; (void)extra0_q6_K; const int ne00 = src0->ne[0]; const int ne01 = src0->ne[1]; const int ne02 = src0->ne[2]; const int ne03 = src0->ne[3]; const cl_ulong nb00 = src0->nb[0]; const cl_ulong nb01 = src0->nb[1]; const cl_ulong nb02 = src0->nb[2]; const cl_ulong nb03 = src0->nb[3]; const int ne10 = src1->ne[0]; const int ne11 = src1->ne[1]; const int ne12 = src1->ne[2]; const int ne13 = src1->ne[3]; const cl_ulong nb11 = src1->nb[1]; const cl_ulong nb12 = src1->nb[2]; const cl_ulong nb13 = src1->nb[3]; const int ne20 = src2->ne[0]; const int ne21 = src2->ne[1]; const cl_ulong nb21 = src2->nb[1]; const cl_ulong nb20 = src2->nb[0]; UNUSED(nb20); const int ne0 = dst->ne[0]; const int ne1 = dst->ne[1]; const int ne2 = dst->ne[2]; GGML_UNUSED(ne2); const int r2 = ne12/ne02; const int r3 = ne13/ne03; const int dst_rows = ne20*ne21; // ne20 = n_used_experts, ne21 = n_rows GGML_ASSERT(ne00 == ne10); int sgs = 32; // subgroup size int nsg = 1; // number of subgroups int nrows = 1; // number of row in src1 int ndst = 4; // number of values produced by each subgroup const int n_tile_size = 32; const int max_post_router_tile = (ne20 * ne21 / n_tile_size) + ne02; GGML_UNUSED(max_post_router_tile); cl_kernel kernel; // subgroup mat vec switch (src0->type) { case GGML_TYPE_Q4_0: { #ifdef GGML_OPENCL_USE_ADRENO_KERNELS if (use_adreno_moe_kernels(backend_ctx, src0)) { cl_int status; size_t local_size[3] = {64, 2, 1}; size_t global_size[3] = {64, 2, 1}; if (ne12 == 1) { // for gemv kernel = backend_ctx->kernel_gemv_moe_q4_0_f32_ns; cl_mem src1_sub_buffer, buf_src1_image, buf_src2; // create a sub_buffer for src2 cl_buffer_region region; region.origin = offset2; region.size = ne20 * ne21 * sizeof(int); buf_src2 = clCreateSubBuffer(extra2->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &status); CL_CHECK(status); // set thread grid global_size[0] = static_cast(((ne01 + 63) / 64) * 64); global_size[1] = 4; global_size[2] = static_cast(ne20); local_size[1] = 4; // create a sub_buffer for src1 region.origin = offset1; region.size = ne10 * ne11 * ne12 * sizeof(float); src1_sub_buffer = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &status); CL_CHECK(status); // create image for src1 cl_image_format image_format_buf_src1 = {CL_RGBA, CL_FLOAT}; cl_image_desc image_desc_buf_src1 = {CL_MEM_OBJECT_IMAGE1D_BUFFER, static_cast(ne10 * ne11 * ne12 / 4), 0,0,0,0,0,0,0, {src1_sub_buffer}}; buf_src1_image = clCreateImage(backend_ctx->context, CL_MEM_READ_ONLY, &image_format_buf_src1, &image_desc_buf_src1, NULL, &status); CL_CHECK(status); // Set kernel args int arg_idx = 0; CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &extra0_q4_0->q)); CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &extra0_q4_0->d)); CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &buf_src1_image)); CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &buf_src2)); CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &extrad->data_device)); CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_ulong), &offsetd)); CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(int), &ne00)); CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(int), &ne01)); CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(int), &ne11)); // launch kernel backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_size, local_size, dst); // deallocate sub buffers and images CL_CHECK(clReleaseMemObject(src1_sub_buffer)); CL_CHECK(clReleaseMemObject(buf_src1_image)); CL_CHECK(clReleaseMemObject(buf_src2)); } else { // for gemm // dp4a (int8) prefill GEMM variant static const char * q4_0_moe_dp4a_env = getenv("GGML_OPENCL_Q4_0_MOE_DP4A"); // It turns out that the prebuilt kernel only outperforms the dp4a variant (on X2-90) // at very large routing counts, so we gate its use accordingly using moe_bin_min, // which can be overridden via the GGML_OPENCL_MOE_BIN_MIN_ROUTINGS environment variable. // The routing count is ne20 * ne21 (n_expert_used * n_tokens). static const char * moe_bin_min_env = getenv("GGML_OPENCL_MOE_BIN_MIN_ROUTINGS"); const int moe_bin_min = moe_bin_min_env ? atoi(moe_bin_min_env) : 4096; // whether bin kernels are available const bool bin_available = backend_ctx->kernel_gemm_moe_q4_0_f32_ns_bin != nullptr; const bool dp4a_bin_available = backend_ctx->kernel_gemm_moe_q4_0_q8_1_dp4a_bin != nullptr; bool use_moe_dp4a = q4_0_moe_dp4a_env ? (atoi(q4_0_moe_dp4a_env) != 0) : (backend_ctx->adreno_gen == ADRENO_GPU_GEN::X2E && (dp4a_bin_available || !bin_available || (int)(ne20 * ne21) < moe_bin_min)); // dot prod has to be available use_moe_dp4a = backend_ctx->has_integer_dot && use_moe_dp4a; const bool use_bin_kernel = bin_available && !use_moe_dp4a; kernel = use_bin_kernel ? backend_ctx->kernel_gemm_moe_q4_0_f32_ns_bin : backend_ctx->kernel_gemm_moe_q4_0_f32_ns; // Reorder router if called from test-backend-ops or when new router is generated. // Otherwise reuse the reordered result from previous mul_mat_id call. if ((strstr(src0->name, "as") != NULL) || backend_ctx->toggle_reorder) { moe_router_reoerder(backend, src2, ne20); backend_ctx->toggle_reorder = false; } cl_mem sub_buf_src1_pre, sub_buf_dst, buf_dst_image; cl_mem buf_src1_reordered = nullptr, image_src1_reordered = nullptr; cl_mem buf_src2, buf_src2_emap; cl_buffer_region region; region.origin = 0; region.size = sizeof(int) * max_post_router_tile * n_tile_size; buf_src2 = clCreateSubBuffer(backend_ctx->prealloc_post_router.buffer, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &status); CL_CHECK(status); region.origin = 0; region.size = sizeof(short) * max_post_router_tile; buf_src2_emap = clCreateSubBuffer(backend_ctx->prealloc_emap.buffer, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &status); CL_CHECK(status); // Reorder activations // create a sub_buffer for src1 region.origin = offset1; region.size = ne10 * ne11 * ne12 * sizeof(float); sub_buf_src1_pre = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &status); CL_CHECK(status); unsigned short map_ratio = ne20 / ne11; GGML_ASSERT(((map_ratio == 1) || (map_ratio == ne20)) && "Map ratio not supported\n"); if (!use_moe_dp4a) { // Create image for reordered src1 // Use pre-allocated placeholder region.origin = 0; region.size = ne00 * max_post_router_tile * n_tile_size * sizeof(float); backend_ctx->prealloc_act_trans.allocate(backend_ctx->context, region.size); buf_src1_reordered = clCreateSubBuffer( backend_ctx->prealloc_act_trans.buffer, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &status); CL_CHECK(status); cl_image_format image_format_buf_src1; cl_image_desc image_desc_buf_src1; image_format_buf_src1 = {CL_RGBA, CL_FLOAT}; image_desc_buf_src1 = {CL_MEM_OBJECT_IMAGE1D_BUFFER, static_cast(ne00 * max_post_router_tile * n_tile_size / 4), 0,0,0,0,0,0,0, {buf_src1_reordered}}; if (use_bin_kernel) { // bin kernel uses slightly different image format image_format_buf_src1 = {CL_R, CL_FLOAT}; image_desc_buf_src1.image_width = static_cast(ne00 * max_post_router_tile * n_tile_size); } image_src1_reordered = clCreateImage(backend_ctx->context, CL_MEM_READ_ONLY, &image_format_buf_src1, &image_desc_buf_src1, NULL, &status); CL_CHECK(status); CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 0, sizeof(cl_mem), &sub_buf_src1_pre)); CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 1, sizeof(cl_mem), &buf_src2)); CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 2, sizeof(cl_mem), &buf_src1_reordered)); CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 3, sizeof(cl_mem), &(backend_ctx->prealloc_total_tiles.buffer))); CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 4, sizeof(unsigned int), &ne00)); CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 5, sizeof(unsigned short), &map_ratio)); CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 6, sizeof(unsigned int), &n_tile_size)); size_t reorder_b_local_size[3] = {256, 1, 1}; size_t reorder_b_global_size[3] = {static_cast(((ne00 / 4) + 255) / 256 * 256), static_cast(max_post_router_tile * n_tile_size), 1}; // Dispatch reorder kernel backend_ctx->enqueue_ndrange_kernel(backend_ctx->kernel_moe_reorder_b, 3, reorder_b_global_size, reorder_b_local_size, dst); } // MoE kernel prepare // Create sub buffer for dst region.origin = offsetd; region.size = ne0 * ne1 * ne2 * sizeof(float); sub_buf_dst = clCreateSubBuffer( extrad->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &status); CL_CHECK(status); // Create image for dst cl_image_format image_format_buf_dst = {CL_R, CL_FLOAT}; cl_image_desc image_desc_buf_dst = {CL_MEM_OBJECT_IMAGE1D_BUFFER, static_cast(ne0 * ne1 * ne2), 0,0,0,0,0,0,0, {sub_buf_dst}}; buf_dst_image = clCreateImage(backend_ctx->context, CL_MEM_WRITE_ONLY, &image_format_buf_dst, &image_desc_buf_dst, NULL, &status); CL_CHECK(status); if (use_moe_dp4a) { const size_t tok_slots = (size_t)max_post_router_tile * n_tile_size; const size_t n_blocks = tok_slots * (ne00 / 32); backend_ctx->prealloc_moe_qa.allocate(backend_ctx->context, tok_slots * ne00 * sizeof(cl_char)); backend_ctx->prealloc_moe_da.allocate(backend_ctx->context, n_blocks * sizeof(cl_half)); backend_ctx->prealloc_moe_sa.allocate(backend_ctx->context, n_blocks * sizeof(cl_half)); // fused reorder + q8_1 quant straight from the original activations const cl_uint n_kblocks = (cl_uint)(ne00 / 32); cl_kernel rq = backend_ctx->kernel_moe_reorder_quant_a_q8_1; CL_CHECK(clSetKernelArg(rq, 0, sizeof(cl_mem), &sub_buf_src1_pre)); CL_CHECK(clSetKernelArg(rq, 1, sizeof(cl_mem), &buf_src2)); CL_CHECK(clSetKernelArg(rq, 2, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer)); CL_CHECK(clSetKernelArg(rq, 3, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer)); CL_CHECK(clSetKernelArg(rq, 4, sizeof(cl_mem), &backend_ctx->prealloc_moe_sa.buffer)); CL_CHECK(clSetKernelArg(rq, 5, sizeof(cl_mem), &(backend_ctx->prealloc_total_tiles.buffer))); CL_CHECK(clSetKernelArg(rq, 6, sizeof(cl_uint), &ne00)); CL_CHECK(clSetKernelArg(rq, 7, sizeof(unsigned short), &map_ratio)); CL_CHECK(clSetKernelArg(rq, 8, sizeof(cl_uint), &n_tile_size)); CL_CHECK(clSetKernelArg(rq, 9, sizeof(cl_uint), &n_kblocks)); size_t rq_local[2] = { 32, 1 }; size_t rq_global[2] = { (size_t)(((n_kblocks + 31) / 32) * 32), tok_slots }; backend_ctx->enqueue_ndrange_kernel(rq, 2, rq_global, rq_local, dst); // dp4a GEMM cl_kernel dk = backend_ctx->kernel_gemm_moe_q4_0_q8_1_dp4a; if (backend_ctx->kernel_gemm_moe_q4_0_q8_1_dp4a_bin) { dk = backend_ctx->kernel_gemm_moe_q4_0_q8_1_dp4a_bin; } int aidx = 0; CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &extra0_q4_0->q_img)); CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &extra0_q4_0->d)); CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer)); CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer)); CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &backend_ctx->prealloc_moe_sa.buffer)); CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &buf_src2)); CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &buf_src2_emap)); CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &buf_dst_image)); CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &(backend_ctx->prealloc_total_tiles.buffer))); CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(int), &ne00)); CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(int), &ne01)); CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(int), &backend_ctx->adreno_use_moe_ragged_dp4)); size_t dp_global[3] = { 64, (size_t)((ne01 + 63) / 64), (size_t)max_post_router_tile }; size_t dp_local[3] = { 64, 1, 1 }; backend_ctx->enqueue_ndrange_kernel(dk, 3, dp_global, dp_local, dst); clReleaseMemObject(sub_buf_src1_pre); clReleaseMemObject(buf_src2); clReleaseMemObject(buf_src2_emap); clReleaseMemObject(sub_buf_dst); clReleaseMemObject(buf_dst_image); return; } // Set kernel args int arg_idx = 0; CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &extra0_q4_0->q_img)); CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &extra0_q4_0->d)); CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &image_src1_reordered)); CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &buf_src2)); CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &buf_src2_emap)); CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &buf_dst_image)); CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &(backend_ctx->prealloc_total_tiles.buffer))); CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(int), &ne00)); CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(int), &ne01)); CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_uint), &backend_ctx->adreno_use_moe_ragged)); CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_uint), &backend_ctx->adreno_moe_ragged_skip_gran)); // set thread grid global_size[1] = static_cast((ne01 + 63) / 64); global_size[2] = static_cast(max_post_router_tile); local_size[1] = 1; local_size[2] = 1; // Dispatch kernel backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_size, local_size, dst); clReleaseMemObject(sub_buf_src1_pre); clReleaseMemObject(buf_src1_reordered); clReleaseMemObject(image_src1_reordered); clReleaseMemObject(buf_src2); clReleaseMemObject(buf_src2_emap); clReleaseMemObject(sub_buf_dst); clReleaseMemObject(buf_dst_image); } return; } // fallback to generic Q4_0 MoE kernel #endif // GGML_OPENCL_USE_ADRENO_KERNELS kernel = backend_ctx->kernel_mul_mv_id_q4_0_f32_8x_flat; if (backend_ctx->gpu_family == INTEL) { sgs = 16; nsg = 1; ndst = 8; } else if (backend_ctx->gpu_family == ADRENO) { sgs = 64; nsg = 1; ndst = 8; } else { GGML_ASSERT(false && "TODO: Unknown GPU"); } CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_q4_0->q)); CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q4_0->d)); CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device)); CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1)); CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extra2->data_device)); CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offset2)); CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_mem), &extrad->data_device)); CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_ulong), &offsetd)); CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne00)); CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne01)); CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &ne02)); CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_ulong), &nb00)); CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_ulong), &nb02)); CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &ne10)); CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &ne11)); CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &ne12)); CL_CHECK(clSetKernelArg(kernel, 16, sizeof(cl_ulong), &nb11)); CL_CHECK(clSetKernelArg(kernel, 17, sizeof(cl_ulong), &nb12)); CL_CHECK(clSetKernelArg(kernel, 18, sizeof(int), &ne20)); CL_CHECK(clSetKernelArg(kernel, 19, sizeof(int), &ne21)); CL_CHECK(clSetKernelArg(kernel, 20, sizeof(cl_ulong), &nb21)); CL_CHECK(clSetKernelArg(kernel, 21, sizeof(int), &ne0)); CL_CHECK(clSetKernelArg(kernel, 22, sizeof(int), &ne1)); CL_CHECK(clSetKernelArg(kernel, 23, sizeof(int), &r2)); CL_CHECK(clSetKernelArg(kernel, 24, sizeof(int), &r3)); break; } case GGML_TYPE_Q4_1: { #ifdef GGML_OPENCL_USE_ADRENO_KERNELS if (use_adreno_moe_kernels(backend_ctx, src0)) { cl_int status; size_t local_size[3] = {64, 2, 1}; size_t global_size[3] = {64, 2, 1}; if (ne12 == 1) { // for gemv kernel = backend_ctx->kernel_gemv_moe_q4_1_f32_ns; cl_mem src1_sub_buffer, buf_src1_image, buf_src2; // create a sub_buffer for src2 cl_buffer_region region; region.origin = offset2; region.size = ne20 * ne21 * sizeof(int); buf_src2 = clCreateSubBuffer(extra2->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &status); CL_CHECK(status); // set thread grid global_size[0] = static_cast(((ne01 + 63) / 64) * 64); global_size[1] = 4; global_size[2] = static_cast(ne20); local_size[1] = 4; // create a sub_buffer for src1 region.origin = offset1; region.size = ne10 * ne11 * ne12 * sizeof(float); src1_sub_buffer = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &status); CL_CHECK(status); // create image for src1 cl_image_format image_format_buf_src1 = {CL_RGBA, CL_FLOAT}; cl_image_desc image_desc_buf_src1 = {CL_MEM_OBJECT_IMAGE1D_BUFFER, static_cast(ne10 * ne11 * ne12 / 4), 0,0,0,0,0,0,0, {src1_sub_buffer}}; buf_src1_image = clCreateImage(backend_ctx->context, CL_MEM_READ_ONLY, &image_format_buf_src1, &image_desc_buf_src1, NULL, &status); CL_CHECK(status); // Set kernel args int arg_idx = 0; CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &extra0_q4_1->q)); CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &extra0_q4_1->d)); CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &extra0_q4_1->m)); CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &buf_src1_image)); CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &buf_src2)); CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &extrad->data_device)); CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_ulong), &offsetd)); CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(int), &ne00)); CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(int), &ne01)); CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(int), &ne11)); // launch kernel backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_size, local_size, dst); // deallocate sub buffers and images CL_CHECK(clReleaseMemObject(src1_sub_buffer)); CL_CHECK(clReleaseMemObject(buf_src1_image)); CL_CHECK(clReleaseMemObject(buf_src2)); } else { // for gemm kernel = backend_ctx->kernel_gemm_moe_q4_1_f32_ns; if (backend_ctx->kernel_gemm_moe_q4_1_f32_ns_bin) { kernel = backend_ctx->kernel_gemm_moe_q4_1_f32_ns_bin; } // Reorder router if called from test-backend-ops or when new router is generated. // Otherwise reuse the reordered result from previous mul_mat_id call. if ((strstr(src0->name, "as") != NULL) || backend_ctx->toggle_reorder) { moe_router_reoerder(backend, src2, ne20); backend_ctx->toggle_reorder = false; } cl_mem sub_buf_src1_pre, buf_src1_reordered, image_src1_reordered, sub_buf_dst, buf_dst_image; cl_mem buf_src2, buf_src2_emap; cl_buffer_region region; region.origin = 0; region.size = sizeof(int) * max_post_router_tile * n_tile_size; buf_src2 = clCreateSubBuffer(backend_ctx->prealloc_post_router.buffer, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &status); CL_CHECK(status); region.origin = 0; region.size = sizeof(short) * max_post_router_tile; buf_src2_emap = clCreateSubBuffer(backend_ctx->prealloc_emap.buffer, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &status); CL_CHECK(status); // Reorder activations // create a sub_buffer for src1 region.origin = offset1; region.size = ne10 * ne11 * ne12 * sizeof(float); sub_buf_src1_pre = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &status); CL_CHECK(status); // Create image for reordered src1 // Use pre-allocated placeholder region.origin = 0; region.size = ne00 * max_post_router_tile * n_tile_size * sizeof(float); backend_ctx->prealloc_act_trans.allocate(backend_ctx->context, region.size); buf_src1_reordered = clCreateSubBuffer( backend_ctx->prealloc_act_trans.buffer, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &status); CL_CHECK(status); cl_image_format image_format_buf_src1; cl_image_desc image_desc_buf_src1; image_format_buf_src1 = {CL_RGBA, CL_FLOAT}; image_desc_buf_src1 = {CL_MEM_OBJECT_IMAGE1D_BUFFER, static_cast(ne00 * max_post_router_tile * n_tile_size / 4), 0,0,0,0,0,0,0, {buf_src1_reordered}}; if (backend_ctx->kernel_gemm_moe_q4_1_f32_ns_bin) { // bin kernel uses slightly different image format image_format_buf_src1 = {CL_R, CL_FLOAT}; image_desc_buf_src1.image_width = static_cast(ne00 * max_post_router_tile * n_tile_size); } image_src1_reordered = clCreateImage(backend_ctx->context, CL_MEM_READ_ONLY, &image_format_buf_src1, &image_desc_buf_src1, NULL, &status); CL_CHECK(status); unsigned short map_ratio = ne20 / ne11; GGML_ASSERT(((map_ratio == 1) || (map_ratio == ne20)) && "Map ratio not supported\n"); CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 0, sizeof(cl_mem), &sub_buf_src1_pre)); CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 1, sizeof(cl_mem), &buf_src2)); CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 2, sizeof(cl_mem), &buf_src1_reordered)); CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 3, sizeof(cl_mem), &(backend_ctx->prealloc_total_tiles.buffer))); CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 4, sizeof(unsigned int), &ne00)); CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 5, sizeof(unsigned short), &map_ratio)); CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 6, sizeof(unsigned int), &n_tile_size)); size_t reorder_b_local_size[3] = {256, 1, 1}; size_t reorder_b_global_size[3] = {static_cast(((ne00 / 4) + 255) / 256 * 256), static_cast(max_post_router_tile * n_tile_size), 1}; // Dispatch reorder kernel backend_ctx->enqueue_ndrange_kernel(backend_ctx->kernel_moe_reorder_b, 3, reorder_b_global_size, reorder_b_local_size, dst); // MoE kernel prepare // Create sub buffer for dst region.origin = offsetd; region.size = ne0 * ne1 * ne2 * sizeof(float); sub_buf_dst = clCreateSubBuffer( extrad->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &status); CL_CHECK(status); // Create image for dst cl_image_format image_format_buf_dst = {CL_R, CL_FLOAT}; cl_image_desc image_desc_buf_dst = {CL_MEM_OBJECT_IMAGE1D_BUFFER, static_cast(ne0 * ne1 * ne2), 0,0,0,0,0,0,0, {sub_buf_dst}}; buf_dst_image = clCreateImage(backend_ctx->context, CL_MEM_WRITE_ONLY, &image_format_buf_dst, &image_desc_buf_dst, NULL, &status); CL_CHECK(status); // Set kernel args int arg_idx = 0; CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &extra0_q4_1->q_img)); CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &extra0_q4_1->d)); CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &extra0_q4_1->m)); CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &image_src1_reordered)); CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &buf_src2)); CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &buf_src2_emap)); CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &buf_dst_image)); CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &(backend_ctx->prealloc_total_tiles.buffer))); CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(int), &ne00)); CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(int), &ne01)); CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_uint), &backend_ctx->adreno_use_moe_ragged)); CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_uint), &backend_ctx->adreno_moe_ragged_skip_gran)); // set thread grid global_size[1] = static_cast((ne01 + 63) / 64); global_size[2] = static_cast(max_post_router_tile); local_size[1] = 1; local_size[2] = 1; // Dispatch kernel backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_size, local_size, dst); clReleaseMemObject(sub_buf_src1_pre); clReleaseMemObject(buf_src1_reordered); clReleaseMemObject(image_src1_reordered); clReleaseMemObject(buf_src2); clReleaseMemObject(buf_src2_emap); clReleaseMemObject(sub_buf_dst); clReleaseMemObject(buf_dst_image); } return; } #endif //GGML_OPENCL_USE_ADRENO_KERNELS } case GGML_TYPE_Q5_0: { #ifdef GGML_OPENCL_USE_ADRENO_KERNELS if (use_adreno_moe_kernels(backend_ctx, src0)) { cl_int status; size_t local_size[3] = {64, 2, 1}; size_t global_size[3] = {64, 2, 1}; if (ne12 == 1) { // for gemv kernel = backend_ctx->kernel_gemv_moe_q5_0_f32_ns; cl_mem src1_sub_buffer, buf_src1_image, buf_src2; // create a sub_buffer for src2 cl_buffer_region region; region.origin = offset2; region.size = ne20 * ne21 * sizeof(int); buf_src2 = clCreateSubBuffer(extra2->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &status); CL_CHECK(status); // set thread grid global_size[0] = static_cast(((ne01 + 63) / 64) * 64); global_size[1] = 4; global_size[2] = static_cast(ne20); local_size[1] = 4; // create a sub_buffer for src1 region.origin = offset1; region.size = ne10 * ne11 * ne12 * sizeof(float); src1_sub_buffer = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &status); CL_CHECK(status); // create image for src1 cl_image_format image_format_buf_src1 = {CL_RGBA, CL_FLOAT}; cl_image_desc image_desc_buf_src1 = {CL_MEM_OBJECT_IMAGE1D_BUFFER, static_cast(ne10 * ne11 * ne12 / 4), 0,0,0,0,0,0,0, {src1_sub_buffer}}; buf_src1_image = clCreateImage(backend_ctx->context, CL_MEM_READ_ONLY, &image_format_buf_src1, &image_desc_buf_src1, NULL, &status); CL_CHECK(status); // Set kernel args int arg_idx = 0; CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &extra0_q5_0->qs)); CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &extra0_q5_0->qh)); CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &extra0_q5_0->d)); CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &buf_src1_image)); CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &buf_src2)); CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &extrad->data_device)); CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_ulong), &offsetd)); CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(int), &ne00)); CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(int), &ne01)); CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(int), &ne11)); // launch kernel backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_size, local_size, dst); // deallocate sub buffers and images CL_CHECK(clReleaseMemObject(src1_sub_buffer)); CL_CHECK(clReleaseMemObject(buf_src1_image)); CL_CHECK(clReleaseMemObject(buf_src2)); } else { // for gemm kernel = backend_ctx->kernel_gemm_moe_q5_0_f32_ns; // Reorder router if called from test-backend-ops or when new router is generated. // Otherwise reuse the reordered result from previous mul_mat_id call. if ((strstr(src0->name, "as") != NULL) || backend_ctx->toggle_reorder) { moe_router_reoerder(backend, src2, ne20); backend_ctx->toggle_reorder = false; } cl_mem sub_buf_src1_pre, buf_src1_reordered, image_src1_reordered, sub_buf_dst, buf_dst_image; cl_mem buf_src2, buf_src2_emap; cl_buffer_region region; region.origin = 0; region.size = sizeof(int) * max_post_router_tile * n_tile_size; buf_src2 = clCreateSubBuffer(backend_ctx->prealloc_post_router.buffer, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &status); CL_CHECK(status); region.origin = 0; region.size = sizeof(short) * max_post_router_tile; buf_src2_emap = clCreateSubBuffer(backend_ctx->prealloc_emap.buffer, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &status); CL_CHECK(status); // Reorder activations // create a sub_buffer for src1 region.origin = offset1; region.size = ne10 * ne11 * ne12 * sizeof(float); sub_buf_src1_pre = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &status); CL_CHECK(status); // Generic dp4a MoE GEMM { static const char * q5mdp4a_env = getenv("GGML_OPENCL_Q5_MOE_DP4A"); const bool q5mdp4a_on = q5mdp4a_env ? (atoi(q5mdp4a_env) != 0) : (backend_ctx->adreno_gen == ADRENO_GPU_GEN::X2E); const bool use_q5_moe_dp4a = q5mdp4a_on && backend_ctx->kernel_gemm_moe_q8_1_dp4a_q50 != nullptr && extra0_q5_0->scale != nullptr; if (use_q5_moe_dp4a) { const size_t tok_slots = (size_t)max_post_router_tile * n_tile_size; const size_t n_blocks = tok_slots * (ne00 / 32); backend_ctx->prealloc_moe_qa.allocate(backend_ctx->context, tok_slots * ne00 * sizeof(cl_char)); backend_ctx->prealloc_moe_da.allocate(backend_ctx->context, n_blocks * sizeof(cl_half)); backend_ctx->prealloc_moe_sa.allocate(backend_ctx->context, n_blocks * sizeof(cl_half)); const cl_uint n_kblocks = (cl_uint)(ne00 / 32); unsigned short map_ratio_q5 = ne20 / ne11; cl_kernel rq = backend_ctx->kernel_moe_reorder_quant_a_q8_1; CL_CHECK(clSetKernelArg(rq, 0, sizeof(cl_mem), &sub_buf_src1_pre)); CL_CHECK(clSetKernelArg(rq, 1, sizeof(cl_mem), &buf_src2)); CL_CHECK(clSetKernelArg(rq, 2, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer)); CL_CHECK(clSetKernelArg(rq, 3, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer)); CL_CHECK(clSetKernelArg(rq, 4, sizeof(cl_mem), &backend_ctx->prealloc_moe_sa.buffer)); CL_CHECK(clSetKernelArg(rq, 5, sizeof(cl_mem), &(backend_ctx->prealloc_total_tiles.buffer))); CL_CHECK(clSetKernelArg(rq, 6, sizeof(cl_uint), &ne00)); CL_CHECK(clSetKernelArg(rq, 7, sizeof(unsigned short), &map_ratio_q5)); CL_CHECK(clSetKernelArg(rq, 8, sizeof(cl_uint), &n_tile_size)); CL_CHECK(clSetKernelArg(rq, 9, sizeof(cl_uint), &n_kblocks)); size_t rq_local[2] = { 32, 1 }; size_t rq_global[2] = { (size_t)(((n_kblocks + 31) / 32) * 32), tok_slots }; backend_ctx->enqueue_ndrange_kernel(rq, 2, rq_global, rq_local, dst); region.origin = offsetd; region.size = ne0 * ne1 * ne2 * sizeof(float); cl_mem dp_sub_buf_dst = clCreateSubBuffer(extrad->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &status); CL_CHECK(status); cl_image_format dp_ifd = {CL_R, CL_FLOAT}; cl_image_desc dp_idd = {CL_MEM_OBJECT_IMAGE1D_BUFFER, static_cast(ne0 * ne1 * ne2), 0,0,0,0,0,0,0, {dp_sub_buf_dst}}; cl_mem dp_buf_dst_image = clCreateImage(backend_ctx->context, CL_MEM_WRITE_ONLY, &dp_ifd, &dp_idd, NULL, &status); CL_CHECK(status); int ne00i = (int)ne00, ne01i = (int)ne01; cl_kernel dk = backend_ctx->kernel_gemm_moe_q8_1_dp4a_q50; int has_min_q5 = 1; int aidx = 0; CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &extra0_q5_0->qs_img)); CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &extra0_q5_0->qh)); CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &extra0_q5_0->scale)); CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &extra0_q5_0->min)); CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer)); CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer)); CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &backend_ctx->prealloc_moe_sa.buffer)); CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &buf_src2)); CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &buf_src2_emap)); CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &dp_buf_dst_image)); CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &(backend_ctx->prealloc_total_tiles.buffer))); CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(int), &ne00i)); CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(int), &ne01i)); CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(int), &backend_ctx->adreno_use_moe_ragged_dp4)); CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(int), &has_min_q5)); size_t dp_global[3] = { 64, (size_t)((ne01 + 63) / 64), (size_t)max_post_router_tile }; size_t dp_local[3] = { 64, 1, 1 }; backend_ctx->enqueue_ndrange_kernel(dk, 3, dp_global, dp_local, dst); clReleaseMemObject(sub_buf_src1_pre); clReleaseMemObject(buf_src2); clReleaseMemObject(buf_src2_emap); clReleaseMemObject(dp_sub_buf_dst); clReleaseMemObject(dp_buf_dst_image); return; } } // Create image for reordered src1 // Use pre-allocated placeholder region.origin = 0; region.size = ne00 * max_post_router_tile * n_tile_size * sizeof(float); backend_ctx->prealloc_act_trans.allocate(backend_ctx->context, region.size); buf_src1_reordered = clCreateSubBuffer( backend_ctx->prealloc_act_trans.buffer, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &status); CL_CHECK(status); cl_image_format image_format_buf_src1; cl_image_desc image_desc_buf_src1; image_format_buf_src1 = {CL_RGBA, CL_FLOAT}; image_desc_buf_src1 = {CL_MEM_OBJECT_IMAGE1D_BUFFER, static_cast(ne00 * max_post_router_tile * n_tile_size / 4), 0,0,0,0,0,0,0, {buf_src1_reordered}}; image_src1_reordered = clCreateImage(backend_ctx->context, CL_MEM_READ_ONLY, &image_format_buf_src1, &image_desc_buf_src1, NULL, &status); CL_CHECK(status); unsigned short map_ratio = ne20 / ne11; GGML_ASSERT(((map_ratio == 1) || (map_ratio == ne20)) && "Map ratio not supported\n"); CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 0, sizeof(cl_mem), &sub_buf_src1_pre)); CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 1, sizeof(cl_mem), &buf_src2)); CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 2, sizeof(cl_mem), &buf_src1_reordered)); CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 3, sizeof(cl_mem), &(backend_ctx->prealloc_total_tiles.buffer))); CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 4, sizeof(unsigned int), &ne00)); CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 5, sizeof(unsigned short), &map_ratio)); CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 6, sizeof(unsigned int), &n_tile_size)); size_t reorder_b_local_size[3] = {256, 1, 1}; size_t reorder_b_global_size[3] = {static_cast(((ne00 / 4) + 255) / 256 * 256), static_cast(max_post_router_tile * n_tile_size), 1}; // Dispatch reorder kernel backend_ctx->enqueue_ndrange_kernel(backend_ctx->kernel_moe_reorder_b, 3, reorder_b_global_size, reorder_b_local_size, dst); // MoE kernel prepare // Create sub buffer for dst region.origin = offsetd; region.size = ne0 * ne1 * ne2 * sizeof(float); sub_buf_dst = clCreateSubBuffer( extrad->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &status); CL_CHECK(status); // Create image for dst cl_image_format image_format_buf_dst = {CL_R, CL_FLOAT}; cl_image_desc image_desc_buf_dst = {CL_MEM_OBJECT_IMAGE1D_BUFFER, static_cast(ne0 * ne1 * ne2), 0,0,0,0,0,0,0, {sub_buf_dst}}; buf_dst_image = clCreateImage(backend_ctx->context, CL_MEM_WRITE_ONLY, &image_format_buf_dst, &image_desc_buf_dst, NULL, &status); CL_CHECK(status); // Set kernel args int arg_idx = 0; CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &extra0_q5_0->qs_img)); CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &extra0_q5_0->qh)); CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &extra0_q5_0->d)); CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &image_src1_reordered)); CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &buf_src2)); CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &buf_src2_emap)); CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &buf_dst_image)); CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &(backend_ctx->prealloc_total_tiles.buffer))); CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(int), &ne00)); CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(int), &ne01)); CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_uint), &backend_ctx->adreno_use_moe_ragged)); CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_uint), &backend_ctx->adreno_moe_ragged_skip_gran)); // set thread grid global_size[1] = static_cast((ne01 + 63) / 64); global_size[2] = static_cast(max_post_router_tile); local_size[1] = 1; local_size[2] = 1; // Dispatch kernel backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_size, local_size, dst); clReleaseMemObject(sub_buf_src1_pre); clReleaseMemObject(buf_src1_reordered); clReleaseMemObject(image_src1_reordered); clReleaseMemObject(buf_src2); clReleaseMemObject(buf_src2_emap); clReleaseMemObject(sub_buf_dst); clReleaseMemObject(buf_dst_image); } return; } #endif //GGML_OPENCL_USE_ADRENO_KERNELS } case GGML_TYPE_Q5_1: { #ifdef GGML_OPENCL_USE_ADRENO_KERNELS if (use_adreno_moe_kernels(backend_ctx, src0)) { cl_int status; size_t local_size[3] = {64, 2, 1}; size_t global_size[3] = {64, 2, 1}; if (ne12 == 1) { // for gemv kernel = backend_ctx->kernel_gemv_moe_q5_1_f32_ns; cl_mem src1_sub_buffer, buf_src1_image, buf_src2; // create a sub_buffer for src2 cl_buffer_region region; region.origin = offset2; region.size = ne20 * ne21 * sizeof(int); buf_src2 = clCreateSubBuffer(extra2->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &status); CL_CHECK(status); // set thread grid global_size[0] = static_cast(((ne01 + 63) / 64) * 64); global_size[1] = 4; global_size[2] = static_cast(ne20); local_size[1] = 4; // create a sub_buffer for src1 region.origin = offset1; region.size = ne10 * ne11 * ne12 * sizeof(float); src1_sub_buffer = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &status); CL_CHECK(status); // create image for src1 cl_image_format image_format_buf_src1 = {CL_RGBA, CL_FLOAT}; cl_image_desc image_desc_buf_src1 = {CL_MEM_OBJECT_IMAGE1D_BUFFER, static_cast(ne10 * ne11 * ne12 / 4), 0,0,0,0,0,0,0, {src1_sub_buffer}}; buf_src1_image = clCreateImage(backend_ctx->context, CL_MEM_READ_ONLY, &image_format_buf_src1, &image_desc_buf_src1, NULL, &status); CL_CHECK(status); // Set kernel args int arg_idx = 0; CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &extra0_q5_1->qs)); CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &extra0_q5_1->qh)); CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &extra0_q5_1->d)); CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &extra0_q5_1->m)); CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &buf_src1_image)); CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &buf_src2)); CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &extrad->data_device)); CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_ulong), &offsetd)); CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(int), &ne00)); CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(int), &ne01)); CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(int), &ne11)); // launch kernel backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_size, local_size, dst); // deallocate sub buffers and images CL_CHECK(clReleaseMemObject(src1_sub_buffer)); CL_CHECK(clReleaseMemObject(buf_src1_image)); CL_CHECK(clReleaseMemObject(buf_src2)); } else { // for gemm kernel = backend_ctx->kernel_gemm_moe_q5_1_f32_ns; // Reorder router if called from test-backend-ops or when new router is generated. // Otherwise reuse the reordered result from previous mul_mat_id call. if ((strstr(src0->name, "as") != NULL) || backend_ctx->toggle_reorder) { moe_router_reoerder(backend, src2, ne20); backend_ctx->toggle_reorder = false; } cl_mem sub_buf_src1_pre, buf_src1_reordered, image_src1_reordered, sub_buf_dst, buf_dst_image; cl_mem buf_src2, buf_src2_emap; cl_buffer_region region; region.origin = 0; region.size = sizeof(int) * max_post_router_tile * n_tile_size; buf_src2 = clCreateSubBuffer(backend_ctx->prealloc_post_router.buffer, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &status); CL_CHECK(status); region.origin = 0; region.size = sizeof(short) * max_post_router_tile; buf_src2_emap = clCreateSubBuffer(backend_ctx->prealloc_emap.buffer, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &status); CL_CHECK(status); // Reorder activations // create a sub_buffer for src1 region.origin = offset1; region.size = ne10 * ne11 * ne12 * sizeof(float); sub_buf_src1_pre = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &status); CL_CHECK(status); // Create image for reordered src1 // Use pre-allocated placeholder region.origin = 0; region.size = ne00 * max_post_router_tile * n_tile_size * sizeof(float); backend_ctx->prealloc_act_trans.allocate(backend_ctx->context, region.size); buf_src1_reordered = clCreateSubBuffer( backend_ctx->prealloc_act_trans.buffer, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &status); CL_CHECK(status); cl_image_format image_format_buf_src1; cl_image_desc image_desc_buf_src1; image_format_buf_src1 = {CL_RGBA, CL_FLOAT}; image_desc_buf_src1 = {CL_MEM_OBJECT_IMAGE1D_BUFFER, static_cast(ne00 * max_post_router_tile * n_tile_size / 4), 0,0,0,0,0,0,0, {buf_src1_reordered}}; image_src1_reordered = clCreateImage(backend_ctx->context, CL_MEM_READ_ONLY, &image_format_buf_src1, &image_desc_buf_src1, NULL, &status); CL_CHECK(status); unsigned short map_ratio = ne20 / ne11; GGML_ASSERT(((map_ratio == 1) || (map_ratio == ne20)) && "Map ratio not supported\n"); CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 0, sizeof(cl_mem), &sub_buf_src1_pre)); CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 1, sizeof(cl_mem), &buf_src2)); CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 2, sizeof(cl_mem), &buf_src1_reordered)); CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 3, sizeof(cl_mem), &(backend_ctx->prealloc_total_tiles.buffer))); CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 4, sizeof(unsigned int), &ne00)); CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 5, sizeof(unsigned short), &map_ratio)); CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 6, sizeof(unsigned int), &n_tile_size)); size_t reorder_b_local_size[3] = {256, 1, 1}; size_t reorder_b_global_size[3] = {static_cast(((ne00 / 4) + 255) / 256 * 256), static_cast(max_post_router_tile * n_tile_size), 1}; // Dispatch reorder kernel backend_ctx->enqueue_ndrange_kernel(backend_ctx->kernel_moe_reorder_b, 3, reorder_b_global_size, reorder_b_local_size, dst); // MoE kernel prepare // Create sub buffer for dst region.origin = offsetd; region.size = ne0 * ne1 * ne2 * sizeof(float); sub_buf_dst = clCreateSubBuffer( extrad->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &status); CL_CHECK(status); // Create image for dst cl_image_format image_format_buf_dst = {CL_R, CL_FLOAT}; cl_image_desc image_desc_buf_dst = {CL_MEM_OBJECT_IMAGE1D_BUFFER, static_cast(ne0 * ne1 * ne2), 0,0,0,0,0,0,0, {sub_buf_dst}}; buf_dst_image = clCreateImage(backend_ctx->context, CL_MEM_WRITE_ONLY, &image_format_buf_dst, &image_desc_buf_dst, NULL, &status); CL_CHECK(status); // Set kernel args int arg_idx = 0; CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &extra0_q5_1->qs_img)); CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &extra0_q5_1->qh)); CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &extra0_q5_1->d)); CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &extra0_q5_1->m)); CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &image_src1_reordered)); CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &buf_src2)); CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &buf_src2_emap)); CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &buf_dst_image)); CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &(backend_ctx->prealloc_total_tiles.buffer))); CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(int), &ne00)); CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(int), &ne01)); CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_uint), &backend_ctx->adreno_use_moe_ragged)); CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_uint), &backend_ctx->adreno_moe_ragged_skip_gran)); // set thread grid global_size[1] = static_cast((ne01 + 63) / 64); global_size[2] = static_cast(max_post_router_tile); local_size[1] = 1; local_size[2] = 1; // Dispatch kernel backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_size, local_size, dst); clReleaseMemObject(sub_buf_src1_pre); clReleaseMemObject(buf_src1_reordered); clReleaseMemObject(image_src1_reordered); clReleaseMemObject(buf_src2); clReleaseMemObject(buf_src2_emap); clReleaseMemObject(sub_buf_dst); clReleaseMemObject(buf_dst_image); } return; } #endif //GGML_OPENCL_USE_ADRENO_KERNELS } case GGML_TYPE_Q8_0: { #ifdef GGML_OPENCL_USE_ADRENO_KERNELS // MoE GEMM for q8_0 at prefill (ne12>1) // There is no corresponding gemv_moe, so the code path is different here static const char * moe_gemm_q8_env = getenv("GGML_OPENCL_MOE_GEMM_Q8"); const bool moe_gemm_q8 = moe_gemm_q8_env ? (atoi(moe_gemm_q8_env) != 0) : (backend_ctx->adreno_gen == ADRENO_GPU_GEN::X2E); if (moe_gemm_q8 && use_adreno_moe_kernels(backend_ctx, src0) && ne12 > 1) { cl_int status; size_t local_size[3] = {64, 2, 1}; size_t global_size[3] = {64, 2, 1}; kernel = backend_ctx->kernel_gemm_moe_q8_0_f32_ns; if ((strstr(src0->name, "as") != NULL) || backend_ctx->toggle_reorder) { moe_router_reoerder(backend, src2, ne20); backend_ctx->toggle_reorder = false; } cl_mem sub_buf_src1_pre, buf_src1_reordered, image_src1_reordered, sub_buf_dst, buf_dst_image; cl_mem buf_src2, buf_src2_emap; cl_buffer_region region; region.origin = 0; region.size = sizeof(int) * max_post_router_tile * n_tile_size; buf_src2 = clCreateSubBuffer(backend_ctx->prealloc_post_router.buffer, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &status); CL_CHECK(status); region.origin = 0; region.size = sizeof(short) * max_post_router_tile; buf_src2_emap = clCreateSubBuffer(backend_ctx->prealloc_emap.buffer, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &status); CL_CHECK(status); // Reorder activations (group tokens by expert into tiles of 32) region.origin = offset1; region.size = ne10 * ne11 * ne12 * sizeof(float); sub_buf_src1_pre = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &status); CL_CHECK(status); // Generic dp4a MoE GEMM { static const char * q8mdp4a_env = getenv("GGML_OPENCL_Q8_MOE_DP4A"); const bool q8mdp4a_on = q8mdp4a_env ? (atoi(q8mdp4a_env) != 0) : (backend_ctx->adreno_gen == ADRENO_GPU_GEN::X2E); const bool use_q8_moe_dp4a = q8mdp4a_on && backend_ctx->kernel_gemm_moe_q8_1_dp4a_q80 != nullptr && extra0_q8_0->scale != nullptr; if (use_q8_moe_dp4a) { const size_t tok_slots = (size_t)max_post_router_tile * n_tile_size; const size_t n_blocks = tok_slots * (ne00 / 32); backend_ctx->prealloc_moe_qa.allocate(backend_ctx->context, tok_slots * ne00 * sizeof(cl_char)); backend_ctx->prealloc_moe_da.allocate(backend_ctx->context, n_blocks * sizeof(cl_half)); backend_ctx->prealloc_moe_sa.allocate(backend_ctx->context, n_blocks * sizeof(cl_half)); const cl_uint n_kblocks = (cl_uint)(ne00 / 32); unsigned short map_ratio_q8 = ne20 / ne11; cl_kernel rq = backend_ctx->kernel_moe_reorder_quant_a_q8_1; CL_CHECK(clSetKernelArg(rq, 0, sizeof(cl_mem), &sub_buf_src1_pre)); CL_CHECK(clSetKernelArg(rq, 1, sizeof(cl_mem), &buf_src2)); CL_CHECK(clSetKernelArg(rq, 2, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer)); CL_CHECK(clSetKernelArg(rq, 3, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer)); CL_CHECK(clSetKernelArg(rq, 4, sizeof(cl_mem), &backend_ctx->prealloc_moe_sa.buffer)); CL_CHECK(clSetKernelArg(rq, 5, sizeof(cl_mem), &(backend_ctx->prealloc_total_tiles.buffer))); CL_CHECK(clSetKernelArg(rq, 6, sizeof(cl_uint), &ne00)); CL_CHECK(clSetKernelArg(rq, 7, sizeof(unsigned short), &map_ratio_q8)); CL_CHECK(clSetKernelArg(rq, 8, sizeof(cl_uint), &n_tile_size)); CL_CHECK(clSetKernelArg(rq, 9, sizeof(cl_uint), &n_kblocks)); size_t rq_local[2] = { 32, 1 }; size_t rq_global[2] = { (size_t)(((n_kblocks + 31) / 32) * 32), tok_slots }; backend_ctx->enqueue_ndrange_kernel(rq, 2, rq_global, rq_local, dst); // dst image region.origin = offsetd; region.size = ne0 * ne1 * ne2 * sizeof(float); cl_mem dp_sub_buf_dst = clCreateSubBuffer(extrad->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &status); CL_CHECK(status); cl_image_format dp_ifd = {CL_R, CL_FLOAT}; cl_image_desc dp_idd = {CL_MEM_OBJECT_IMAGE1D_BUFFER, static_cast(ne0 * ne1 * ne2), 0,0,0,0,0,0,0, {dp_sub_buf_dst}}; cl_mem dp_buf_dst_image = clCreateImage(backend_ctx->context, CL_MEM_WRITE_ONLY, &dp_ifd, &dp_idd, NULL, &status); CL_CHECK(status); int ne00i = (int)ne00, ne01i = (int)ne01; cl_kernel dk = backend_ctx->kernel_gemm_moe_q8_1_dp4a_q80; int has_min_q8 = 0; int aidx = 0; CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &extra0_q8_0->q)); // flat int8 codes [expert][row][K] CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &extra0_q8_0->scale)); // uniform scale[16] CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &extra0_q8_0->scale)); // dummy min (has_min=0, unread) CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer)); CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer)); CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &backend_ctx->prealloc_moe_sa.buffer)); CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &buf_src2)); CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &buf_src2_emap)); CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &dp_buf_dst_image)); CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &(backend_ctx->prealloc_total_tiles.buffer))); CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(int), &ne00i)); CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(int), &ne01i)); CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(int), &backend_ctx->adreno_use_moe_ragged_dp4)); CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(int), &has_min_q8)); size_t dp_global[3] = { 64, (size_t)((ne01 + 63) / 64), (size_t)max_post_router_tile }; size_t dp_local[3] = { 64, 1, 1 }; backend_ctx->enqueue_ndrange_kernel(dk, 3, dp_global, dp_local, dst); clReleaseMemObject(sub_buf_src1_pre); clReleaseMemObject(buf_src2); clReleaseMemObject(buf_src2_emap); clReleaseMemObject(dp_sub_buf_dst); clReleaseMemObject(dp_buf_dst_image); return; } } region.origin = 0; region.size = ne00 * max_post_router_tile * n_tile_size * sizeof(float); backend_ctx->prealloc_act_trans.allocate(backend_ctx->context, region.size); buf_src1_reordered = clCreateSubBuffer( backend_ctx->prealloc_act_trans.buffer, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &status); CL_CHECK(status); cl_image_format image_format_buf_src1 = {CL_RGBA, CL_FLOAT}; cl_image_desc image_desc_buf_src1 = {CL_MEM_OBJECT_IMAGE1D_BUFFER, static_cast(ne00 * max_post_router_tile * n_tile_size / 4), 0,0,0,0,0,0,0, {buf_src1_reordered}}; image_src1_reordered = clCreateImage(backend_ctx->context, CL_MEM_READ_ONLY, &image_format_buf_src1, &image_desc_buf_src1, NULL, &status); CL_CHECK(status); unsigned short map_ratio = ne20 / ne11; GGML_ASSERT(((map_ratio == 1) || (map_ratio == ne20)) && "Map ratio not supported\n"); CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 0, sizeof(cl_mem), &sub_buf_src1_pre)); CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 1, sizeof(cl_mem), &buf_src2)); CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 2, sizeof(cl_mem), &buf_src1_reordered)); CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 3, sizeof(cl_mem), &(backend_ctx->prealloc_total_tiles.buffer))); CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 4, sizeof(unsigned int), &ne00)); CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 5, sizeof(unsigned short), &map_ratio)); CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 6, sizeof(unsigned int), &n_tile_size)); size_t reorder_b_local_size[3] = {256, 1, 1}; size_t reorder_b_global_size[3] = {static_cast(((ne00 / 4) + 255) / 256 * 256), static_cast(max_post_router_tile * n_tile_size), 1}; backend_ctx->enqueue_ndrange_kernel(backend_ctx->kernel_moe_reorder_b, 3, reorder_b_global_size, reorder_b_local_size, dst); // dst image region.origin = offsetd; region.size = ne0 * ne1 * ne2 * sizeof(float); sub_buf_dst = clCreateSubBuffer(extrad->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &status); CL_CHECK(status); cl_image_format image_format_buf_dst = {CL_R, CL_FLOAT}; cl_image_desc image_desc_buf_dst = {CL_MEM_OBJECT_IMAGE1D_BUFFER, static_cast(ne0 * ne1 * ne2), 0,0,0,0,0,0,0, {sub_buf_dst}}; buf_dst_image = clCreateImage(backend_ctx->context, CL_MEM_WRITE_ONLY, &image_format_buf_dst, &image_desc_buf_dst, NULL, &status); CL_CHECK(status); int arg_idx = 0; CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &extra0_q8_0->q)); // flat q8_0 quants CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &extra0_q8_0->d)); // flat q8_0 scales CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &image_src1_reordered)); CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &buf_src2)); CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &buf_src2_emap)); CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &buf_dst_image)); CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &(backend_ctx->prealloc_total_tiles.buffer))); CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(int), &ne00)); CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(int), &ne01)); global_size[1] = static_cast((ne01 + 63) / 64); global_size[2] = static_cast(max_post_router_tile); local_size[1] = 1; local_size[2] = 1; backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_size, local_size, dst); clReleaseMemObject(sub_buf_src1_pre); clReleaseMemObject(buf_src1_reordered); clReleaseMemObject(image_src1_reordered); clReleaseMemObject(buf_src2); clReleaseMemObject(buf_src2_emap); clReleaseMemObject(sub_buf_dst); clReleaseMemObject(buf_dst_image); return; } #endif // GGML_OPENCL_USE_ADRENO_KERNELS #ifdef GGML_OPENCL_SOA_Q kernel = backend_ctx->kernel_mul_mv_id_q8_0_f32_flat; if (backend_ctx->gpu_family == INTEL) { sgs = 16; nsg = 2; ndst = 4; } else if (backend_ctx->gpu_family == ADRENO) { sgs = 64; nsg = 2; ndst = 4; } else { GGML_ASSERT(false && "TODO: Unknown GPU"); } CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_q8_0->q)); CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q8_0->d)); CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device)); CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1)); CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extra2->data_device)); CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offset2)); CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_mem), &extrad->data_device)); CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_ulong), &offsetd)); CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne00)); CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne01)); CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_ulong), &nb01)); CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_ulong), &nb02)); CL_CHECK(clSetKernelArg(kernel, 12, sizeof(int), &ne11)); CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &ne12)); CL_CHECK(clSetKernelArg(kernel, 14, sizeof(cl_ulong), &nb11)); CL_CHECK(clSetKernelArg(kernel, 15, sizeof(cl_ulong), &nb12)); CL_CHECK(clSetKernelArg(kernel, 16, sizeof(int), &ne20)); CL_CHECK(clSetKernelArg(kernel, 17, sizeof(int), &ne21)); CL_CHECK(clSetKernelArg(kernel, 18, sizeof(cl_ulong), &nb21)); CL_CHECK(clSetKernelArg(kernel, 19, sizeof(int), &ne0)); CL_CHECK(clSetKernelArg(kernel, 20, sizeof(int), &ne1)); #else kernel = backend_ctx->kernel_mul_mv_id_q8_0_f32; if (backend_ctx->gpu_family == INTEL) { sgs = 16; nsg = 2; ndst = 4; } else if (backend_ctx->gpu_family == ADRENO) { sgs = 64; nsg = 2; ndst = 4; } else { GGML_ASSERT(false && "TODO: Unknown GPU"); } CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device)); CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1)); CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extra2->data_device)); CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offset2)); CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_mem), &extrad->data_device)); CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_ulong), &offsetd)); CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne00)); CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne01)); CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_ulong), &nb01)); CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_ulong), &nb02)); CL_CHECK(clSetKernelArg(kernel, 12, sizeof(int), &ne11)); CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &ne12)); CL_CHECK(clSetKernelArg(kernel, 14, sizeof(cl_ulong), &nb11)); CL_CHECK(clSetKernelArg(kernel, 15, sizeof(cl_ulong), &nb12)); CL_CHECK(clSetKernelArg(kernel, 16, sizeof(int), &ne20)); CL_CHECK(clSetKernelArg(kernel, 17, sizeof(int), &ne21)); CL_CHECK(clSetKernelArg(kernel, 18, sizeof(cl_ulong), &nb21)); CL_CHECK(clSetKernelArg(kernel, 19, sizeof(int), &ne0)); CL_CHECK(clSetKernelArg(kernel, 20, sizeof(int), &ne1)); #endif // GGML_OPENCL_SOA_Q break; } case GGML_TYPE_Q4_K: { #ifdef GGML_OPENCL_USE_ADRENO_KERNELS if (use_adreno_moe_kernels(backend_ctx, src0)) { cl_int status; size_t local_size[3] = {64, 2, 1}; size_t global_size[3] = {64, 2, 1}; if (ne12 == 1) { // for gemv kernel = backend_ctx->kernel_gemv_moe_q4_k_f32_ns; // Weight-as-texture MoE decode GEMV static const char * moe_decode_wimg_env = getenv("GGML_OPENCL_MOE_DECODE_WIMG"); const bool moe_decode_wimg_on = moe_decode_wimg_env ? (atoi(moe_decode_wimg_env) != 0) : (backend_ctx->adreno_gen == ADRENO_GPU_GEN::X2E); const bool use_moe_decode_wimg = moe_decode_wimg_on && backend_ctx->kernel_gemv_moe_q4_k_f32_ns_wimg != nullptr && extra0_q4_K->q_img != nullptr; if (use_moe_decode_wimg) { kernel = backend_ctx->kernel_gemv_moe_q4_k_f32_ns_wimg; } cl_mem src1_sub_buffer, buf_src1_image, buf_src2; // create a sub_buffer for src2 cl_buffer_region region; region.origin = offset2; region.size = ne20 * ne21 * sizeof(int); buf_src2 = clCreateSubBuffer(extra2->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &status); CL_CHECK(status); // set thread grid global_size[0] = static_cast(((ne01 + 63) / 64) * 64); global_size[1] = 4; global_size[2] = static_cast(ne20); local_size[1] = 4; // create a sub_buffer for src1 region.origin = offset1; region.size = ne10 * ne11 * ne12 * sizeof(float); src1_sub_buffer = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &status); CL_CHECK(status); // create image for src1 cl_image_format image_format_buf_src1 = {CL_RGBA, CL_FLOAT}; cl_image_desc image_desc_buf_src1 = {CL_MEM_OBJECT_IMAGE1D_BUFFER, static_cast(ne10 * ne11 * ne12 / 4), 0,0,0,0,0,0,0, {src1_sub_buffer}}; buf_src1_image = clCreateImage(backend_ctx->context, CL_MEM_READ_ONLY, &image_format_buf_src1, &image_desc_buf_src1, NULL, &status); CL_CHECK(status); // Set kernel args int arg_idx = 0; CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), use_moe_decode_wimg ? &extra0_q4_K->q_img : &extra0_q4_K->q)); CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &extra0_q4_K->d)); CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &extra0_q4_K->dm)); CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &extra0_q4_K->s)); CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &buf_src1_image)); CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &buf_src2)); CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &extrad->data_device)); CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_ulong), &offsetd)); CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(int), &ne00)); CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(int), &ne01)); CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(int), &ne11)); // launch kernel backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_size, local_size, dst); // deallocate sub buffers and images CL_CHECK(clReleaseMemObject(src1_sub_buffer)); CL_CHECK(clReleaseMemObject(buf_src1_image)); CL_CHECK(clReleaseMemObject(buf_src2)); } else { // for gemm kernel = backend_ctx->kernel_gemm_moe_q4_k_f32_ns; if (backend_ctx->kernel_gemm_moe_q4_k_f32_ns_bin) { kernel = backend_ctx->kernel_gemm_moe_q4_k_f32_ns_bin; } // Reorder router if called from test-backend-ops or when new router is generated. // Otherwise reuse the reordered result from previous mul_mat_id call. if ((strstr(src0->name, "as") != NULL) || backend_ctx->toggle_reorder) { moe_router_reoerder(backend, src2, ne20); backend_ctx->toggle_reorder = false; } cl_mem sub_buf_src1_pre, sub_buf_dst, buf_dst_image; cl_mem buf_src1_reordered = nullptr, image_src1_reordered = nullptr; cl_mem buf_src2, buf_src2_emap; // dp4a (int8) prefill GEMM variant static const char * q4k_moe_dp4a_env = getenv("GGML_OPENCL_Q4K_MOE_DP4A"); bool use_moe_dp4a = (q4k_moe_dp4a_env != nullptr) ? (atoi(q4k_moe_dp4a_env) != 0) : (backend_ctx->adreno_gen == ADRENO_GPU_GEN::X2E || backend_ctx->adreno_gen == ADRENO_GPU_GEN::X1E); // dot prod has to be available use_moe_dp4a = backend_ctx->has_integer_dot && use_moe_dp4a; // bin kernel takes precedence use_moe_dp4a = use_moe_dp4a && backend_ctx->kernel_gemm_moe_q4_k_f32_ns_bin == nullptr; cl_buffer_region region; region.origin = 0; region.size = sizeof(int) * max_post_router_tile * n_tile_size; buf_src2 = clCreateSubBuffer(backend_ctx->prealloc_post_router.buffer, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &status); CL_CHECK(status); region.origin = 0; region.size = sizeof(short) * max_post_router_tile; buf_src2_emap = clCreateSubBuffer(backend_ctx->prealloc_emap.buffer, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &status); CL_CHECK(status); // Reorder activations region.origin = offset1; region.size = ne10 * ne11 * ne12 * sizeof(float); sub_buf_src1_pre = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &status); CL_CHECK(status); unsigned short map_ratio = ne20 / ne11; GGML_ASSERT(((map_ratio == 1) || (map_ratio == ne20)) && "Map ratio not supported\n"); if (!use_moe_dp4a) { // Create image for reordered src1 region.origin = 0; region.size = ne00 * max_post_router_tile * n_tile_size * sizeof(float); backend_ctx->prealloc_act_trans.allocate(backend_ctx->context, region.size); buf_src1_reordered = clCreateSubBuffer( backend_ctx->prealloc_act_trans.buffer, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &status); CL_CHECK(status); cl_image_format image_format_buf_src1 = {CL_RGBA, CL_FLOAT}; cl_image_desc image_desc_buf_src1 = {CL_MEM_OBJECT_IMAGE1D_BUFFER, static_cast(ne00 * max_post_router_tile * n_tile_size / 4), 0,0,0,0,0,0,0, {buf_src1_reordered}}; if (backend_ctx->kernel_gemm_moe_q4_k_f32_ns_bin) { // bin kernel uses slightly different image format image_format_buf_src1 = {CL_R, CL_FLOAT}; image_desc_buf_src1.image_width = static_cast(ne00 * max_post_router_tile * n_tile_size); } image_src1_reordered = clCreateImage(backend_ctx->context, CL_MEM_READ_ONLY, &image_format_buf_src1, &image_desc_buf_src1, NULL, &status); CL_CHECK(status); CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 0, sizeof(cl_mem), &sub_buf_src1_pre)); CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 1, sizeof(cl_mem), &buf_src2)); CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 2, sizeof(cl_mem), &buf_src1_reordered)); CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 3, sizeof(cl_mem), &(backend_ctx->prealloc_total_tiles.buffer))); CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 4, sizeof(unsigned int), &ne00)); CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 5, sizeof(unsigned short), &map_ratio)); CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 6, sizeof(unsigned int), &n_tile_size)); size_t reorder_b_local_size[3] = {256, 1, 1}; size_t reorder_b_global_size[3] = {static_cast(((ne00 / 4) + 255) / 256 * 256), static_cast(max_post_router_tile * n_tile_size), 1}; // Dispatch reorder kernel backend_ctx->enqueue_ndrange_kernel(backend_ctx->kernel_moe_reorder_b, 3, reorder_b_global_size, reorder_b_local_size, dst); } // MoE kernel prepare region.origin = offsetd; region.size = ne0 * ne1 * ne2 * sizeof(float); sub_buf_dst = clCreateSubBuffer( extrad->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &status); CL_CHECK(status); // Create image for dst cl_image_format image_format_buf_dst = {CL_R, CL_FLOAT}; cl_image_desc image_desc_buf_dst = {CL_MEM_OBJECT_IMAGE1D_BUFFER, static_cast(ne0 * ne1 * ne2), 0,0,0,0,0,0,0, {sub_buf_dst}}; buf_dst_image = clCreateImage(backend_ctx->context, CL_MEM_WRITE_ONLY, &image_format_buf_dst, &image_desc_buf_dst, NULL, &status); CL_CHECK(status); if (use_moe_dp4a) { const size_t tok_slots = (size_t)max_post_router_tile * n_tile_size; const size_t n_blocks = tok_slots * (ne00 / 32); backend_ctx->prealloc_moe_qa.allocate(backend_ctx->context, tok_slots * ne00 * sizeof(cl_char)); backend_ctx->prealloc_moe_da.allocate(backend_ctx->context, n_blocks * sizeof(cl_half)); backend_ctx->prealloc_moe_sa.allocate(backend_ctx->context, n_blocks * sizeof(cl_half)); // fused reorder + q8_1 quant straight from the original // activations (no intermediate f32 reorder buffer) const cl_uint n_kblocks = (cl_uint)(ne00 / 32); cl_kernel rq = backend_ctx->kernel_moe_reorder_quant_a_q8_1; CL_CHECK(clSetKernelArg(rq, 0, sizeof(cl_mem), &sub_buf_src1_pre)); CL_CHECK(clSetKernelArg(rq, 1, sizeof(cl_mem), &buf_src2)); CL_CHECK(clSetKernelArg(rq, 2, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer)); CL_CHECK(clSetKernelArg(rq, 3, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer)); CL_CHECK(clSetKernelArg(rq, 4, sizeof(cl_mem), &backend_ctx->prealloc_moe_sa.buffer)); CL_CHECK(clSetKernelArg(rq, 5, sizeof(cl_mem), &(backend_ctx->prealloc_total_tiles.buffer))); CL_CHECK(clSetKernelArg(rq, 6, sizeof(cl_uint), &ne00)); CL_CHECK(clSetKernelArg(rq, 7, sizeof(unsigned short), &map_ratio)); CL_CHECK(clSetKernelArg(rq, 8, sizeof(cl_uint), &n_tile_size)); CL_CHECK(clSetKernelArg(rq, 9, sizeof(cl_uint), &n_kblocks)); size_t rq_local[2] = { 32, 1 }; size_t rq_global[2] = { (size_t)(((n_kblocks + 31) / 32) * 32), tok_slots }; backend_ctx->enqueue_ndrange_kernel(rq, 2, rq_global, rq_local, dst); // dp4a GEMM cl_kernel dk = backend_ctx->kernel_gemm_moe_q4_k_q8_1_dp4a; int aidx = 0; CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &extra0_q4_K->q_img)); CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &extra0_q4_K->d)); CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &extra0_q4_K->dm)); CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &extra0_q4_K->s)); CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer)); CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer)); CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &backend_ctx->prealloc_moe_sa.buffer)); CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &buf_src2)); CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &buf_src2_emap)); CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &buf_dst_image)); CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &(backend_ctx->prealloc_total_tiles.buffer))); CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(int), &ne00)); CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(int), &ne01)); CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(int), &backend_ctx->adreno_use_moe_ragged_dp4)); size_t dp_global[3] = { 64, (size_t)((ne01 + 63) / 64), (size_t)max_post_router_tile }; size_t dp_local[3] = { 64, 1, 1 }; backend_ctx->enqueue_ndrange_kernel(dk, 3, dp_global, dp_local, dst); clReleaseMemObject(sub_buf_src1_pre); clReleaseMemObject(buf_src2); clReleaseMemObject(buf_src2_emap); clReleaseMemObject(sub_buf_dst); clReleaseMemObject(buf_dst_image); return; } // Set kernel args int arg_idx = 0; CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &extra0_q4_K->q_img)); CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &extra0_q4_K->d)); CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &extra0_q4_K->dm)); CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &extra0_q4_K->s)); CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &image_src1_reordered)); CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &buf_src2)); CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &buf_src2_emap)); CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &buf_dst_image)); CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &(backend_ctx->prealloc_total_tiles.buffer))); CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(int), &ne00)); CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(int), &ne01)); CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_uint), &backend_ctx->adreno_use_moe_ragged)); CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_uint), &backend_ctx->adreno_moe_ragged_skip_gran)); // set thread grid global_size[1] = static_cast((ne01 + 63) / 64); global_size[2] = static_cast(max_post_router_tile); local_size[1] = 1; local_size[2] = 1; // Dispatch kernel backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_size, local_size, dst); clReleaseMemObject(sub_buf_src1_pre); clReleaseMemObject(buf_src1_reordered); clReleaseMemObject(image_src1_reordered); clReleaseMemObject(buf_src2); clReleaseMemObject(buf_src2_emap); clReleaseMemObject(sub_buf_dst); clReleaseMemObject(buf_dst_image); } return; } #endif //GGML_OPENCL_USE_ADRENO_KERNELS } case GGML_TYPE_Q5_K: { #ifdef GGML_OPENCL_USE_ADRENO_KERNELS if (use_adreno_moe_kernels(backend_ctx, src0)) { cl_int status; size_t local_size[3] = {64, 2, 1}; size_t global_size[3] = {64, 2, 1}; if (ne12 == 1) { // for gemv kernel = backend_ctx->kernel_gemv_moe_q5_k_f32_ns; cl_mem src1_sub_buffer, buf_src1_image, buf_src2; // create a sub_buffer for src2 cl_buffer_region region; region.origin = offset2; region.size = ne20 * ne21 * sizeof(int); buf_src2 = clCreateSubBuffer(extra2->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &status); CL_CHECK(status); // set thread grid global_size[0] = static_cast(((ne01 + 63) / 64) * 64); global_size[1] = 4; global_size[2] = static_cast(ne20); local_size[1] = 4; // create a sub_buffer for src1 region.origin = offset1; region.size = ne10 * ne11 * ne12 * sizeof(float); src1_sub_buffer = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &status); CL_CHECK(status); // create image for src1 cl_image_format image_format_buf_src1 = {CL_RGBA, CL_FLOAT}; cl_image_desc image_desc_buf_src1 = {CL_MEM_OBJECT_IMAGE1D_BUFFER, static_cast(ne10 * ne11 * ne12 / 4), 0,0,0,0,0,0,0, {src1_sub_buffer}}; buf_src1_image = clCreateImage(backend_ctx->context, CL_MEM_READ_ONLY, &image_format_buf_src1, &image_desc_buf_src1, NULL, &status); CL_CHECK(status); // Set kernel args int arg_idx = 0; CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &extra0_q5_K->q)); CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &extra0_q5_K->qh)); CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &extra0_q5_K->d)); CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &extra0_q5_K->dm)); CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &extra0_q5_K->s)); CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &buf_src1_image)); CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &buf_src2)); CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &extrad->data_device)); CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_ulong), &offsetd)); CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(int), &ne00)); CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(int), &ne01)); CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(int), &ne11)); // launch kernel backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_size, local_size, dst); // deallocate sub buffers and images CL_CHECK(clReleaseMemObject(src1_sub_buffer)); CL_CHECK(clReleaseMemObject(buf_src1_image)); CL_CHECK(clReleaseMemObject(buf_src2)); } else { // for gemm kernel = backend_ctx->kernel_gemm_moe_q5_k_f32_ns; // Reorder router if called from test-backend-ops or when new router is generated. // Otherwise reuse the reordered result from previous mul_mat_id call. if ((strstr(src0->name, "as") != NULL) || backend_ctx->toggle_reorder) { moe_router_reoerder(backend, src2, ne20); backend_ctx->toggle_reorder = false; } cl_mem sub_buf_src1_pre, buf_src1_reordered, image_src1_reordered, sub_buf_dst, buf_dst_image; cl_mem buf_src2, buf_src2_emap; cl_buffer_region region; region.origin = 0; region.size = sizeof(int) * max_post_router_tile * n_tile_size; buf_src2 = clCreateSubBuffer(backend_ctx->prealloc_post_router.buffer, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &status); CL_CHECK(status); region.origin = 0; region.size = sizeof(short) * max_post_router_tile; buf_src2_emap = clCreateSubBuffer(backend_ctx->prealloc_emap.buffer, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &status); CL_CHECK(status); // Reorder activations // create a sub_buffer for src1 region.origin = offset1; region.size = ne10 * ne11 * ne12 * sizeof(float); sub_buf_src1_pre = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &status); CL_CHECK(status); // Generic dp4a MoE GEMM { static const char * q5kmdp4a_env = getenv("GGML_OPENCL_Q5K_MOE_DP4A"); const bool q5kmdp4a_on = q5kmdp4a_env ? (atoi(q5kmdp4a_env) != 0) : (backend_ctx->adreno_gen == ADRENO_GPU_GEN::X2E); bool use_moe_dp4a = q5kmdp4a_on && backend_ctx->kernel_gemm_moe_q8_1_dp4a_q5k != nullptr && extra0_q5_K->scale != nullptr; // dot prod has to be available use_moe_dp4a = backend_ctx->has_integer_dot && use_moe_dp4a; if (use_moe_dp4a) { const size_t tok_slots = (size_t)max_post_router_tile * n_tile_size; const size_t n_blocks = tok_slots * (ne00 / 32); backend_ctx->prealloc_moe_qa.allocate(backend_ctx->context, tok_slots * ne00 * sizeof(cl_char)); backend_ctx->prealloc_moe_da.allocate(backend_ctx->context, n_blocks * sizeof(cl_half)); backend_ctx->prealloc_moe_sa.allocate(backend_ctx->context, n_blocks * sizeof(cl_half)); const cl_uint n_kblocks = (cl_uint)(ne00 / 32); unsigned short map_ratio_q5k = ne20 / ne11; cl_kernel rq = backend_ctx->kernel_moe_reorder_quant_a_q8_1; CL_CHECK(clSetKernelArg(rq, 0, sizeof(cl_mem), &sub_buf_src1_pre)); CL_CHECK(clSetKernelArg(rq, 1, sizeof(cl_mem), &buf_src2)); CL_CHECK(clSetKernelArg(rq, 2, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer)); CL_CHECK(clSetKernelArg(rq, 3, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer)); CL_CHECK(clSetKernelArg(rq, 4, sizeof(cl_mem), &backend_ctx->prealloc_moe_sa.buffer)); CL_CHECK(clSetKernelArg(rq, 5, sizeof(cl_mem), &(backend_ctx->prealloc_total_tiles.buffer))); CL_CHECK(clSetKernelArg(rq, 6, sizeof(cl_uint), &ne00)); CL_CHECK(clSetKernelArg(rq, 7, sizeof(unsigned short), &map_ratio_q5k)); CL_CHECK(clSetKernelArg(rq, 8, sizeof(cl_uint), &n_tile_size)); CL_CHECK(clSetKernelArg(rq, 9, sizeof(cl_uint), &n_kblocks)); size_t rq_local[2] = { 32, 1 }; size_t rq_global[2] = { (size_t)(((n_kblocks + 31) / 32) * 32), tok_slots }; backend_ctx->enqueue_ndrange_kernel(rq, 2, rq_global, rq_local, dst); region.origin = offsetd; region.size = ne0 * ne1 * ne2 * sizeof(float); cl_mem dp_sub_buf_dst = clCreateSubBuffer(extrad->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &status); CL_CHECK(status); cl_image_format dp_ifd = {CL_R, CL_FLOAT}; cl_image_desc dp_idd = {CL_MEM_OBJECT_IMAGE1D_BUFFER, static_cast(ne0 * ne1 * ne2), 0,0,0,0,0,0,0, {dp_sub_buf_dst}}; cl_mem dp_buf_dst_image = clCreateImage(backend_ctx->context, CL_MEM_WRITE_ONLY, &dp_ifd, &dp_idd, NULL, &status); CL_CHECK(status); int ne00i = (int)ne00, ne01i = (int)ne01; cl_kernel dk = backend_ctx->kernel_gemm_moe_q8_1_dp4a_q5k; int has_min_q5k = 1; int aidx = 0; CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &extra0_q5_K->q_img)); CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &extra0_q5_K->qh)); CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &extra0_q5_K->scale)); CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &extra0_q5_K->min)); CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer)); CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer)); CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &backend_ctx->prealloc_moe_sa.buffer)); CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &buf_src2)); CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &buf_src2_emap)); CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &dp_buf_dst_image)); CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &(backend_ctx->prealloc_total_tiles.buffer))); CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(int), &ne00i)); CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(int), &ne01i)); CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(int), &backend_ctx->adreno_use_moe_ragged_dp4)); CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(int), &has_min_q5k)); size_t dp_global[3] = { 64, (size_t)((ne01 + 63) / 64), (size_t)max_post_router_tile }; size_t dp_local[3] = { 64, 1, 1 }; backend_ctx->enqueue_ndrange_kernel(dk, 3, dp_global, dp_local, dst); clReleaseMemObject(sub_buf_src1_pre); clReleaseMemObject(buf_src2); clReleaseMemObject(buf_src2_emap); clReleaseMemObject(dp_sub_buf_dst); clReleaseMemObject(dp_buf_dst_image); return; } } // Create image for reordered src1 // Use pre-allocated placeholder region.origin = 0; region.size = ne00 * max_post_router_tile * n_tile_size * sizeof(float); backend_ctx->prealloc_act_trans.allocate(backend_ctx->context, region.size); buf_src1_reordered = clCreateSubBuffer( backend_ctx->prealloc_act_trans.buffer, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &status); CL_CHECK(status); cl_image_format image_format_buf_src1 = {CL_RGBA, CL_FLOAT}; cl_image_desc image_desc_buf_src1 = {CL_MEM_OBJECT_IMAGE1D_BUFFER, static_cast(ne00 * max_post_router_tile * n_tile_size / 4), 0,0,0,0,0,0,0, {buf_src1_reordered}}; image_src1_reordered = clCreateImage(backend_ctx->context, CL_MEM_READ_ONLY, &image_format_buf_src1, &image_desc_buf_src1, NULL, &status); CL_CHECK(status); unsigned short map_ratio = ne20 / ne11; GGML_ASSERT(((map_ratio == 1) || (map_ratio == ne20)) && "Map ratio not supported\n"); CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 0, sizeof(cl_mem), &sub_buf_src1_pre)); CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 1, sizeof(cl_mem), &buf_src2)); CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 2, sizeof(cl_mem), &buf_src1_reordered)); CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 3, sizeof(cl_mem), &(backend_ctx->prealloc_total_tiles.buffer))); CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 4, sizeof(unsigned int), &ne00)); CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 5, sizeof(unsigned short), &map_ratio)); CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 6, sizeof(unsigned int), &n_tile_size)); size_t reorder_b_local_size[3] = {256, 1, 1}; size_t reorder_b_global_size[3] = {static_cast(((ne00 / 4) + 255) / 256 * 256), static_cast(max_post_router_tile * n_tile_size), 1}; // Dispatch reorder kernel backend_ctx->enqueue_ndrange_kernel(backend_ctx->kernel_moe_reorder_b, 3, reorder_b_global_size, reorder_b_local_size, dst); // MoE kernel prepare // Create sub buffer for dst region.origin = offsetd; region.size = ne0 * ne1 * ne2 * sizeof(float); sub_buf_dst = clCreateSubBuffer( extrad->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &status); CL_CHECK(status); // Create image for dst cl_image_format image_format_buf_dst = {CL_R, CL_FLOAT}; cl_image_desc image_desc_buf_dst = {CL_MEM_OBJECT_IMAGE1D_BUFFER, static_cast(ne0 * ne1 * ne2), 0,0,0,0,0,0,0, {sub_buf_dst}}; buf_dst_image = clCreateImage(backend_ctx->context, CL_MEM_WRITE_ONLY, &image_format_buf_dst, &image_desc_buf_dst, NULL, &status); CL_CHECK(status); // Set kernel args int arg_idx = 0; CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &extra0_q5_K->q_img)); CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &extra0_q5_K->qh)); CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &extra0_q5_K->s)); CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &extra0_q5_K->d)); CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &extra0_q5_K->dm)); CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &image_src1_reordered)); CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &buf_src2)); CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &buf_src2_emap)); CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &buf_dst_image)); CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &(backend_ctx->prealloc_total_tiles.buffer))); CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(int), &ne00)); CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(int), &ne01)); CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_uint), &backend_ctx->adreno_use_moe_ragged)); CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_uint), &backend_ctx->adreno_moe_ragged_skip_gran)); // set thread grid global_size[1] = static_cast((ne01 + 63) / 64); global_size[2] = static_cast(max_post_router_tile); local_size[1] = 1; local_size[2] = 1; // Dispatch kernel backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_size, local_size, dst); clReleaseMemObject(sub_buf_src1_pre); clReleaseMemObject(buf_src1_reordered); clReleaseMemObject(image_src1_reordered); clReleaseMemObject(buf_src2); clReleaseMemObject(buf_src2_emap); clReleaseMemObject(sub_buf_dst); clReleaseMemObject(buf_dst_image); } return; } #endif //GGML_OPENCL_USE_ADRENO_KERNELS } case GGML_TYPE_Q6_K: { #ifdef GGML_OPENCL_USE_ADRENO_KERNELS if (use_adreno_moe_kernels(backend_ctx, src0)) { cl_int status; size_t local_size[3] = {64, 2, 1}; size_t global_size[3] = {64, 2, 1}; if (ne12 == 1) { // for gemv kernel = backend_ctx->kernel_gemv_moe_q6_k_f32_ns; cl_mem src1_sub_buffer, buf_src1_image, buf_src2; // create a sub_buffer for src2 cl_buffer_region region; region.origin = offset2; region.size = ne20 * ne21 * sizeof(int); buf_src2 = clCreateSubBuffer(extra2->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &status); CL_CHECK(status); // set thread grid global_size[0] = static_cast(((ne01 + 63) / 64) * 64); global_size[1] = 4; global_size[2] = static_cast(ne20); local_size[1] = 4; // create a sub_buffer for src1 region.origin = offset1; region.size = ne10 * ne11 * ne12 * sizeof(float); src1_sub_buffer = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &status); CL_CHECK(status); // create image for src1 cl_image_format image_format_buf_src1 = {CL_RGBA, CL_FLOAT}; cl_image_desc image_desc_buf_src1 = {CL_MEM_OBJECT_IMAGE1D_BUFFER, static_cast(ne10 * ne11 * ne12 / 4), 0,0,0,0,0,0,0, {src1_sub_buffer}}; buf_src1_image = clCreateImage(backend_ctx->context, CL_MEM_READ_ONLY, &image_format_buf_src1, &image_desc_buf_src1, NULL, &status); CL_CHECK(status); // Set kernel args int arg_idx = 0; CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &extra0_q6_K->ql)); CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &extra0_q6_K->qh)); CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &extra0_q6_K->s)); CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &extra0_q6_K->d)); CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &buf_src1_image)); CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &buf_src2)); CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &extrad->data_device)); CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_ulong), &offsetd)); CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(int), &ne00)); CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(int), &ne01)); CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(int), &ne11)); // launch kernel backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_size, local_size, dst); // deallocate sub buffers and images CL_CHECK(clReleaseMemObject(src1_sub_buffer)); CL_CHECK(clReleaseMemObject(buf_src1_image)); CL_CHECK(clReleaseMemObject(buf_src2)); } else { // for gemm kernel = backend_ctx->kernel_gemm_moe_q6_k_f32_ns; if (backend_ctx->kernel_gemm_moe_q6_k_f32_ns_bin) { kernel = backend_ctx->kernel_gemm_moe_q6_k_f32_ns_bin; } // Reorder router if called from test-backend-ops or when new router is generated. // Otherwise reuse the reordered result from previous mul_mat_id call. if ((strstr(src0->name, "as") != NULL) || backend_ctx->toggle_reorder) { moe_router_reoerder(backend, src2, ne20); backend_ctx->toggle_reorder = false; } cl_mem sub_buf_src1_pre, sub_buf_dst, buf_dst_image; cl_mem buf_src1_reordered = nullptr, image_src1_reordered = nullptr; cl_mem buf_src2, buf_src2_emap; // dp4a (int8) q6_K MoE prefill GEMM static const char * q6k_moe_dp4a_env = getenv("GGML_OPENCL_Q6K_MOE_DP4A"); bool use_moe_dp4a = (q6k_moe_dp4a_env != nullptr) ? (atoi(q6k_moe_dp4a_env) != 0) : (backend_ctx->adreno_gen == ADRENO_GPU_GEN::X2E || backend_ctx->adreno_gen == ADRENO_GPU_GEN::X1E); // dot prod has to be available use_moe_dp4a = backend_ctx->has_integer_dot && use_moe_dp4a; // bin kernel takes precedence use_moe_dp4a = use_moe_dp4a && backend_ctx->kernel_gemm_moe_q6_k_f32_ns_bin == nullptr; cl_buffer_region region; region.origin = 0; region.size = sizeof(int) * max_post_router_tile * n_tile_size; buf_src2 = clCreateSubBuffer(backend_ctx->prealloc_post_router.buffer, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &status); CL_CHECK(status); region.origin = 0; region.size = sizeof(short) * max_post_router_tile; buf_src2_emap = clCreateSubBuffer(backend_ctx->prealloc_emap.buffer, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &status); CL_CHECK(status); // Reorder activations // create a sub_buffer for src1 region.origin = offset1; region.size = ne10 * ne11 * ne12 * sizeof(float); sub_buf_src1_pre = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &status); CL_CHECK(status); unsigned short map_ratio = ne20 / ne11; GGML_ASSERT(((map_ratio == 1) || (map_ratio == ne20)) && "Map ratio not supported\n"); if (!use_moe_dp4a) { // Create image for reordered src1 region.origin = 0; region.size = ne00 * max_post_router_tile * n_tile_size * sizeof(float); backend_ctx->prealloc_act_trans.allocate(backend_ctx->context, region.size); buf_src1_reordered = clCreateSubBuffer( backend_ctx->prealloc_act_trans.buffer, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &status); CL_CHECK(status); cl_image_format image_format_buf_src1 = {CL_RGBA, CL_FLOAT}; cl_image_desc image_desc_buf_src1 = {CL_MEM_OBJECT_IMAGE1D_BUFFER, static_cast(ne00 * max_post_router_tile * n_tile_size / 4), 0,0,0,0,0,0,0, {buf_src1_reordered}}; if (backend_ctx->kernel_gemm_moe_q6_k_f32_ns_bin) { // bin kernel uses slightly different image format image_format_buf_src1 = {CL_R, CL_FLOAT}; image_desc_buf_src1.image_width = static_cast(ne00 * max_post_router_tile * n_tile_size); } image_src1_reordered = clCreateImage(backend_ctx->context, CL_MEM_READ_ONLY, &image_format_buf_src1, &image_desc_buf_src1, NULL, &status); CL_CHECK(status); CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 0, sizeof(cl_mem), &sub_buf_src1_pre)); CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 1, sizeof(cl_mem), &buf_src2)); CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 2, sizeof(cl_mem), &buf_src1_reordered)); CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 3, sizeof(cl_mem), &(backend_ctx->prealloc_total_tiles.buffer))); CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 4, sizeof(unsigned int), &ne00)); CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 5, sizeof(unsigned short), &map_ratio)); CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 6, sizeof(unsigned int), &n_tile_size)); size_t reorder_b_local_size[3] = {256, 1, 1}; size_t reorder_b_global_size[3] = {static_cast(((ne00 / 4) + 255) / 256 * 256), static_cast(max_post_router_tile * n_tile_size), 1}; // Dispatch reorder kernel backend_ctx->enqueue_ndrange_kernel(backend_ctx->kernel_moe_reorder_b, 3, reorder_b_global_size, reorder_b_local_size, dst); } // MoE kernel prepare // Create sub buffer for dst region.origin = offsetd; region.size = ne0 * ne1 * ne2 * sizeof(float); sub_buf_dst = clCreateSubBuffer( extrad->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &status); CL_CHECK(status); // Create image for dst cl_image_format image_format_buf_dst = {CL_R, CL_FLOAT}; cl_image_desc image_desc_buf_dst = {CL_MEM_OBJECT_IMAGE1D_BUFFER, static_cast(ne0 * ne1 * ne2), 0,0,0,0,0,0,0, {sub_buf_dst}}; buf_dst_image = clCreateImage(backend_ctx->context, CL_MEM_WRITE_ONLY, &image_format_buf_dst, &image_desc_buf_dst, NULL, &status); CL_CHECK(status); if (use_moe_dp4a) { const size_t tok_slots = (size_t)max_post_router_tile * n_tile_size; const size_t n_blocks = tok_slots * (ne00 / 32); backend_ctx->prealloc_moe_qa.allocate(backend_ctx->context, tok_slots * ne00 * sizeof(cl_char)); backend_ctx->prealloc_moe_da.allocate(backend_ctx->context, n_blocks * sizeof(cl_half)); backend_ctx->prealloc_moe_sa.allocate(backend_ctx->context, n_blocks * sizeof(cl_half)); // fused reorder + q8_1 quant from the original activations const cl_uint n_kblocks = (cl_uint)(ne00 / 32); cl_kernel rq = backend_ctx->kernel_moe_reorder_quant_a_q8_1; CL_CHECK(clSetKernelArg(rq, 0, sizeof(cl_mem), &sub_buf_src1_pre)); CL_CHECK(clSetKernelArg(rq, 1, sizeof(cl_mem), &buf_src2)); CL_CHECK(clSetKernelArg(rq, 2, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer)); CL_CHECK(clSetKernelArg(rq, 3, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer)); CL_CHECK(clSetKernelArg(rq, 4, sizeof(cl_mem), &backend_ctx->prealloc_moe_sa.buffer)); CL_CHECK(clSetKernelArg(rq, 5, sizeof(cl_mem), &(backend_ctx->prealloc_total_tiles.buffer))); CL_CHECK(clSetKernelArg(rq, 6, sizeof(cl_uint), &ne00)); CL_CHECK(clSetKernelArg(rq, 7, sizeof(unsigned short), &map_ratio)); CL_CHECK(clSetKernelArg(rq, 8, sizeof(cl_uint), &n_tile_size)); CL_CHECK(clSetKernelArg(rq, 9, sizeof(cl_uint), &n_kblocks)); size_t rq_local[2] = { 32, 1 }; size_t rq_global[2] = { (size_t)(((n_kblocks + 31) / 32) * 32), tok_slots }; backend_ctx->enqueue_ndrange_kernel(rq, 2, rq_global, rq_local, dst); cl_kernel dk = backend_ctx->kernel_gemm_moe_q6_k_q8_1_dp4a; int qi = 0; CL_CHECK(clSetKernelArg(dk, qi++, sizeof(cl_mem), &extra0_q6_K->ql_img)); CL_CHECK(clSetKernelArg(dk, qi++, sizeof(cl_mem), &extra0_q6_K->qh)); CL_CHECK(clSetKernelArg(dk, qi++, sizeof(cl_mem), &extra0_q6_K->s)); CL_CHECK(clSetKernelArg(dk, qi++, sizeof(cl_mem), &extra0_q6_K->d)); CL_CHECK(clSetKernelArg(dk, qi++, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer)); CL_CHECK(clSetKernelArg(dk, qi++, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer)); CL_CHECK(clSetKernelArg(dk, qi++, sizeof(cl_mem), &buf_src2)); CL_CHECK(clSetKernelArg(dk, qi++, sizeof(cl_mem), &buf_src2_emap)); CL_CHECK(clSetKernelArg(dk, qi++, sizeof(cl_mem), &buf_dst_image)); CL_CHECK(clSetKernelArg(dk, qi++, sizeof(cl_mem), &(backend_ctx->prealloc_total_tiles.buffer))); CL_CHECK(clSetKernelArg(dk, qi++, sizeof(int), &ne00)); CL_CHECK(clSetKernelArg(dk, qi++, sizeof(int), &ne01)); CL_CHECK(clSetKernelArg(dk, qi++, sizeof(int), &backend_ctx->adreno_use_moe_ragged_dp4)); size_t dp_global[3] = { 64, (size_t)((ne01 + 63) / 64), (size_t)max_post_router_tile }; size_t dp_local[3] = { 64, 1, 1 }; backend_ctx->enqueue_ndrange_kernel(dk, 3, dp_global, dp_local, dst); clReleaseMemObject(sub_buf_src1_pre); clReleaseMemObject(buf_src2); clReleaseMemObject(buf_src2_emap); clReleaseMemObject(sub_buf_dst); clReleaseMemObject(buf_dst_image); return; } // Set kernel args int arg_idx = 0; CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &extra0_q6_K->ql_img)); CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &extra0_q6_K->qh)); CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &extra0_q6_K->s)); CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &extra0_q6_K->d)); CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &image_src1_reordered)); CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &buf_src2)); CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &buf_src2_emap)); CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &buf_dst_image)); CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &(backend_ctx->prealloc_total_tiles.buffer))); CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(int), &ne00)); CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(int), &ne01)); CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_uint), &backend_ctx->adreno_use_moe_ragged)); CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_uint), &backend_ctx->adreno_moe_ragged_skip_gran)); // set thread grid global_size[1] = static_cast((ne01 + 63) / 64); global_size[2] = static_cast(max_post_router_tile); local_size[1] = 1; local_size[2] = 1; // Dispatch kernel backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_size, local_size, dst); clReleaseMemObject(sub_buf_src1_pre); clReleaseMemObject(buf_src1_reordered); clReleaseMemObject(image_src1_reordered); clReleaseMemObject(buf_src2); clReleaseMemObject(buf_src2_emap); clReleaseMemObject(sub_buf_dst); clReleaseMemObject(buf_dst_image); } return; } #endif //GGML_OPENCL_USE_ADRENO_KERNELS } case GGML_TYPE_MXFP4: { #ifdef GGML_OPENCL_USE_ADRENO_KERNELS if (use_adreno_moe_kernels(backend_ctx, src0)) { cl_int status; size_t local_size[3] = {64, 2, 1}; size_t global_size[3] = {64, 2, 1}; if (ne12 == 1) { // for gemv kernel = backend_ctx->kernel_gemv_moe_mxfp4_f32_ns; // Weight-as-texture MoE decode GEMV (see q4_K _wimg) static const char * moe_decode_wimg_env = getenv("GGML_OPENCL_MOE_DECODE_WIMG"); const bool use_moe_decode_wimg = (moe_decode_wimg_env && (atoi(moe_decode_wimg_env) != 0)) && backend_ctx->kernel_gemv_moe_mxfp4_f32_ns_wimg != nullptr && extra0_mxfp4->q_img != nullptr; if (use_moe_decode_wimg) { kernel = backend_ctx->kernel_gemv_moe_mxfp4_f32_ns_wimg; } cl_mem src1_sub_buffer, buf_src1_image, buf_src2; // create a sub_buffer for src2 cl_buffer_region region; region.origin = offset2; region.size = ne20 * ne21 * sizeof(int); buf_src2 = clCreateSubBuffer(extra2->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &status); CL_CHECK(status); // set thread grid global_size[0] = static_cast(((ne01 + 63) / 64) * 64); global_size[1] = 4; global_size[2] = static_cast(ne20); local_size[1] = 4; // create a sub_buffer for src1 region.origin = offset1; region.size = ne10 * ne11 * ne12 * sizeof(float); src1_sub_buffer = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &status); CL_CHECK(status); // create image for src1 cl_image_format image_format_buf_src1 = {CL_RGBA, CL_FLOAT}; cl_image_desc image_desc_buf_src1 = {CL_MEM_OBJECT_IMAGE1D_BUFFER, static_cast(ne10 * ne11 * ne12 / 4), 0,0,0,0,0,0,0, {src1_sub_buffer}}; buf_src1_image = clCreateImage(backend_ctx->context, CL_MEM_READ_ONLY, &image_format_buf_src1, &image_desc_buf_src1, NULL, &status); CL_CHECK(status); // Set kernel args int arg_idx = 0; CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), use_moe_decode_wimg ? &extra0_mxfp4->q_img : &extra0_mxfp4->q)); CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &extra0_mxfp4->e)); CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &buf_src1_image)); CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &buf_src2)); CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &extrad->data_device)); CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_ulong), &offsetd)); CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(int), &ne00)); CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(int), &ne01)); CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(int), &ne11)); // launch kernel backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_size, local_size, dst); // deallocate sub buffers and images CL_CHECK(clReleaseMemObject(src1_sub_buffer)); CL_CHECK(clReleaseMemObject(buf_src1_image)); CL_CHECK(clReleaseMemObject(buf_src2)); } else { // for gemm kernel = backend_ctx->kernel_gemm_moe_mxfp4_f32_ns; if (backend_ctx->kernel_gemm_moe_mxfp4_f32_ns_bin) { kernel = backend_ctx->kernel_gemm_moe_mxfp4_f32_ns_bin; } // Reorder router if called from test-backend-ops or when new router is generated. // Otherwise reuse the reordered result from previous mul_mat_id call. if ((strstr(src0->name, "as") != NULL) || backend_ctx->toggle_reorder) { moe_router_reoerder(backend, src2, ne20); backend_ctx->toggle_reorder = false; } cl_mem sub_buf_src1_pre, sub_buf_dst, buf_dst_image; cl_mem buf_src1_reordered = nullptr, image_src1_reordered = nullptr; cl_mem buf_src2, buf_src2_emap; // dp4a (int8) prefill GEMM variant static const char * mxfp4_moe_dp4a_env = getenv("GGML_OPENCL_MXFP4_MOE_DP4A"); bool use_moe_dp4a = mxfp4_moe_dp4a_env ? (atoi(mxfp4_moe_dp4a_env) != 0) : (backend_ctx->adreno_gen == ADRENO_GPU_GEN::X2E); // dot prod has to be available use_moe_dp4a = backend_ctx->has_integer_dot && use_moe_dp4a; // bin kernel takes precedence, dp4a bin kernel has higher priority than normal bin kernel if (backend_ctx->kernel_gemm_moe_mxfp4_q8_1_dp4a_bin == nullptr) { use_moe_dp4a = use_moe_dp4a && backend_ctx->kernel_gemm_moe_mxfp4_f32_ns_bin == nullptr; } cl_buffer_region region; region.origin = 0; region.size = sizeof(int) * max_post_router_tile * n_tile_size; GGML_ASSERT(backend_ctx->prealloc_post_router.buffer); buf_src2 = clCreateSubBuffer(backend_ctx->prealloc_post_router.buffer, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &status); CL_CHECK(status); region.origin = 0; region.size = sizeof(short) * max_post_router_tile; buf_src2_emap = clCreateSubBuffer(backend_ctx->prealloc_emap.buffer, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &status); CL_CHECK(status); // Reorder activations // create a sub_buffer for src1 region.origin = offset1; region.size = ne10 * ne11 * ne12 * sizeof(float); sub_buf_src1_pre = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &status); CL_CHECK(status); unsigned short map_ratio = ne20 / ne11; GGML_ASSERT(((map_ratio == 1) || (map_ratio == ne20)) && "Map ratio not supported\n"); if (!use_moe_dp4a) { // Create image for reordered src1 // Use pre-allocated placeholder region.origin = 0; region.size = ne00 * max_post_router_tile * n_tile_size * sizeof(float); backend_ctx->prealloc_act_trans.allocate(backend_ctx->context, region.size); buf_src1_reordered = clCreateSubBuffer( backend_ctx->prealloc_act_trans.buffer, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &status); CL_CHECK(status); cl_image_format image_format_buf_src1; cl_image_desc image_desc_buf_src1; image_format_buf_src1 = {CL_RGBA, CL_FLOAT}; image_desc_buf_src1 = {CL_MEM_OBJECT_IMAGE1D_BUFFER, static_cast(ne00 * max_post_router_tile * n_tile_size / 4), 0,0,0,0,0,0,0, {buf_src1_reordered}}; if (backend_ctx->kernel_gemm_moe_mxfp4_f32_ns_bin) { // bin kernel uses slightly different image format image_format_buf_src1 = {CL_R, CL_FLOAT}; image_desc_buf_src1.image_width = static_cast(ne00 * max_post_router_tile * n_tile_size); } image_src1_reordered = clCreateImage(backend_ctx->context, CL_MEM_READ_ONLY, &image_format_buf_src1, &image_desc_buf_src1, NULL, &status); CL_CHECK(status); CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 0, sizeof(cl_mem), &sub_buf_src1_pre)); CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 1, sizeof(cl_mem), &buf_src2)); CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 2, sizeof(cl_mem), &buf_src1_reordered)); CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 3, sizeof(cl_mem), &(backend_ctx->prealloc_total_tiles.buffer))); CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 4, sizeof(unsigned int), &ne00)); CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 5, sizeof(unsigned short), &map_ratio)); CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 6, sizeof(unsigned int), &n_tile_size)); size_t reorder_b_local_size[3] = {256, 1, 1}; size_t reorder_b_global_size[3] = {static_cast(((ne00 / 4) + 255) / 256 * 256), static_cast(max_post_router_tile * n_tile_size), 1}; // Dispatch reorder kernel backend_ctx->enqueue_ndrange_kernel(backend_ctx->kernel_moe_reorder_b, 3, reorder_b_global_size, reorder_b_local_size, dst); } // MoE kernel prepare // Create sub buffer for dst region.origin = offsetd; region.size = ne0 * ne1 * ne2 * sizeof(float); sub_buf_dst = clCreateSubBuffer( extrad->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &status); CL_CHECK(status); // Create image for dst cl_image_format image_format_buf_dst = {CL_R, CL_FLOAT}; cl_image_desc image_desc_buf_dst = {CL_MEM_OBJECT_IMAGE1D_BUFFER, static_cast(ne0 * ne1 * ne2), 0,0,0,0,0,0,0, {sub_buf_dst}}; buf_dst_image = clCreateImage(backend_ctx->context, CL_MEM_WRITE_ONLY, &image_format_buf_dst, &image_desc_buf_dst, NULL, &status); CL_CHECK(status); if (use_moe_dp4a) { const size_t tok_slots = (size_t)max_post_router_tile * n_tile_size; const size_t n_blocks = tok_slots * (ne00 / 32); backend_ctx->prealloc_moe_qa.allocate(backend_ctx->context, tok_slots * ne00 * sizeof(cl_char)); backend_ctx->prealloc_moe_da.allocate(backend_ctx->context, n_blocks * sizeof(cl_half)); backend_ctx->prealloc_moe_sa.allocate(backend_ctx->context, n_blocks * sizeof(cl_half)); // fused reorder + q8_1 quant straight from the original // activations (no intermediate f32 reorder buffer). mxfp4 has no // min term so the GEMM ignores sa, but reorder_quant still writes it. const cl_uint n_kblocks = (cl_uint)(ne00 / 32); cl_kernel rq = backend_ctx->kernel_moe_reorder_quant_a_q8_1; CL_CHECK(clSetKernelArg(rq, 0, sizeof(cl_mem), &sub_buf_src1_pre)); CL_CHECK(clSetKernelArg(rq, 1, sizeof(cl_mem), &buf_src2)); CL_CHECK(clSetKernelArg(rq, 2, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer)); CL_CHECK(clSetKernelArg(rq, 3, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer)); CL_CHECK(clSetKernelArg(rq, 4, sizeof(cl_mem), &backend_ctx->prealloc_moe_sa.buffer)); CL_CHECK(clSetKernelArg(rq, 5, sizeof(cl_mem), &(backend_ctx->prealloc_total_tiles.buffer))); CL_CHECK(clSetKernelArg(rq, 6, sizeof(cl_uint), &ne00)); CL_CHECK(clSetKernelArg(rq, 7, sizeof(unsigned short), &map_ratio)); CL_CHECK(clSetKernelArg(rq, 8, sizeof(cl_uint), &n_tile_size)); CL_CHECK(clSetKernelArg(rq, 9, sizeof(cl_uint), &n_kblocks)); size_t rq_local[2] = { 32, 1 }; size_t rq_global[2] = { (size_t)(((n_kblocks + 31) / 32) * 32), tok_slots }; backend_ctx->enqueue_ndrange_kernel(rq, 2, rq_global, rq_local, dst); // dp4a GEMM cl_kernel dk = backend_ctx->kernel_gemm_moe_mxfp4_q8_1_dp4a; if (backend_ctx->kernel_gemm_moe_mxfp4_q8_1_dp4a_bin) { dk = backend_ctx->kernel_gemm_moe_mxfp4_q8_1_dp4a_bin; } int aidx = 0; CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &extra0_mxfp4->q_img)); CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &extra0_mxfp4->e)); CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer)); CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer)); CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &buf_src2)); CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &buf_src2_emap)); CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &buf_dst_image)); CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &(backend_ctx->prealloc_total_tiles.buffer))); CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(int), &ne00)); CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(int), &ne01)); CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(int), &backend_ctx->adreno_use_moe_ragged_dp4)); size_t dp_global[3] = { 64, (size_t)((ne01 + 63) / 64), (size_t)max_post_router_tile }; size_t dp_local[3] = { 64, 1, 1 }; backend_ctx->enqueue_ndrange_kernel(dk, 3, dp_global, dp_local, dst); clReleaseMemObject(sub_buf_src1_pre); clReleaseMemObject(buf_src2); clReleaseMemObject(buf_src2_emap); clReleaseMemObject(sub_buf_dst); clReleaseMemObject(buf_dst_image); return; } // Set kernel args int arg_idx = 0; CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &extra0_mxfp4->q_img)); CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &extra0_mxfp4->e)); CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &image_src1_reordered)); CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &buf_src2)); CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &buf_src2_emap)); CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &buf_dst_image)); CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &(backend_ctx->prealloc_total_tiles.buffer))); CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(int), &ne00)); CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(int), &ne01)); CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_uint), &backend_ctx->adreno_use_moe_ragged)); CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_uint), &backend_ctx->adreno_moe_ragged_skip_gran)); // set thread grid global_size[1] = static_cast((ne01 + 63) / 64); global_size[2] = static_cast(max_post_router_tile); local_size[1] = 1; local_size[2] = 1; // Dispatch kernel backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_size, local_size, dst); clReleaseMemObject(sub_buf_src1_pre); clReleaseMemObject(buf_src1_reordered); clReleaseMemObject(image_src1_reordered); clReleaseMemObject(buf_src2); clReleaseMemObject(buf_src2_emap); clReleaseMemObject(sub_buf_dst); clReleaseMemObject(buf_dst_image); } return; } // fallback to generic MoE mxfp4 kernel #endif // GGML_OPENCL_USE_ADRENO_KERNELS #ifdef GGML_OPENCL_SOA_Q kernel = backend_ctx->kernel_mul_mv_id_mxfp4_f32_flat; cl_mem q; if (backend_ctx->gpu_family == INTEL) { sgs = 16; nsg = 2; ndst = 2; q = extra0_mxfp4->q; } else if (backend_ctx->gpu_family == ADRENO) { sgs = 64; nsg = 1; ndst = 4; q = extra0_mxfp4->q_img; } else { GGML_ASSERT(false && "TODO: Unknown GPU"); } CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &q)); CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_mxfp4->e)); CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device)); CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1)); CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extra2->data_device)); CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offset2)); CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_mem), &extrad->data_device)); CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_ulong), &offsetd)); CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne00)); CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_ulong), &nb01)); CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_ulong), &nb02)); CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_ulong), &nb03)); CL_CHECK(clSetKernelArg(kernel, 12, sizeof(int), &ne11)); CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &ne12)); CL_CHECK(clSetKernelArg(kernel, 14, sizeof(cl_ulong), &nb11)); CL_CHECK(clSetKernelArg(kernel, 15, sizeof(cl_ulong), &nb12)); CL_CHECK(clSetKernelArg(kernel, 16, sizeof(cl_ulong), &nb13)); CL_CHECK(clSetKernelArg(kernel, 17, sizeof(int), &ne20)); CL_CHECK(clSetKernelArg(kernel, 18, sizeof(int), &ne21)); CL_CHECK(clSetKernelArg(kernel, 19, sizeof(cl_ulong), &nb21)); CL_CHECK(clSetKernelArg(kernel, 20, sizeof(int), &ne0)); CL_CHECK(clSetKernelArg(kernel, 21, sizeof(int), &ne1)); CL_CHECK(clSetKernelArg(kernel, 22, sizeof(int), &r2)); CL_CHECK(clSetKernelArg(kernel, 23, sizeof(int), &r3)); #else // GGML_OPENCL_SOA_Q kernel = backend_ctx->kernel_mul_mv_id_mxfp4_f32; if (backend_ctx->gpu_family == INTEL) { sgs = 16; nsg = 2; ndst = 2; } else if (backend_ctx->gpu_family == ADRENO) { sgs = 64; nsg = 2; ndst = 2; } else { GGML_ASSERT(false && "TODO: Unknown GPU"); } CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device)); CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1)); CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extra2->data_device)); CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offset2)); CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_mem), &extrad->data_device)); CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_ulong), &offsetd)); CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne00)); CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_ulong), &nb01)); CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_ulong), &nb02)); CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_ulong), &nb03)); CL_CHECK(clSetKernelArg(kernel, 12, sizeof(int), &ne11)); CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &ne12)); CL_CHECK(clSetKernelArg(kernel, 14, sizeof(cl_ulong), &nb11)); CL_CHECK(clSetKernelArg(kernel, 15, sizeof(cl_ulong), &nb12)); CL_CHECK(clSetKernelArg(kernel, 16, sizeof(cl_ulong), &nb13)); CL_CHECK(clSetKernelArg(kernel, 17, sizeof(int), &ne20)); CL_CHECK(clSetKernelArg(kernel, 18, sizeof(int), &ne21)); CL_CHECK(clSetKernelArg(kernel, 19, sizeof(cl_ulong), &nb21)); CL_CHECK(clSetKernelArg(kernel, 20, sizeof(int), &ne0)); CL_CHECK(clSetKernelArg(kernel, 21, sizeof(int), &ne1)); CL_CHECK(clSetKernelArg(kernel, 22, sizeof(int), &r2)); CL_CHECK(clSetKernelArg(kernel, 23, sizeof(int), &r3)); CL_CHECK(clSetKernelArg(kernel, 24, sizeof(float)*sgs,nullptr)); #endif // GGML_OPENCL_SOA_Q break; } default: GGML_ASSERT(false && "not implemented");; } int _ne1 = 1; int ne123 = dst_rows; size_t global_work_size[] = {(size_t)(ne01+ndst*nsg-1)/(ndst*nsg)*sgs, (size_t)(_ne1+nrows-1)/nrows*nsg, (size_t)ne123}; size_t local_work_size[] = {(size_t)sgs, (size_t)nsg, 1}; backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); } static void ggml_cl_scale(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { GGML_ASSERT(src0); GGML_ASSERT(src0->extra); GGML_ASSERT(dst); GGML_ASSERT(dst->extra); GGML_UNUSED(src1); GGML_ASSERT(ggml_is_contiguous(src0)); ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; float scale; float bias; memcpy(&scale, ((int32_t *) dst->op_params) + 0, sizeof(float)); memcpy(&bias, ((int32_t *) dst->op_params) + 1, sizeof(float)); ggml_tensor_extra_cl * extra0 = (ggml_tensor_extra_cl *)src0->extra; ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; cl_ulong offset0 = extra0->offset + src0->view_offs; cl_ulong offsetd = extrad->offset + dst->view_offs; cl_kernel kernel; int n = ggml_nelements(dst); if (n % 4 == 0) { kernel = backend_ctx->kernel_scale_f32_4; n /= 4; } else { kernel = backend_ctx->kernel_scale_f32; } CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extrad->data_device)); CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offsetd)); CL_CHECK(clSetKernelArg(kernel, 4, sizeof(float), &scale)); CL_CHECK(clSetKernelArg(kernel, 5, sizeof(float), &bias)); size_t global_work_size[] = {(size_t)n, 1, 1}; size_t local_work_size[] = {64, 1, 1}; size_t * local_work_size_ptr = local_work_size; if (n % 64 != 0 && !backend_ctx->non_uniform_workgroups) { local_work_size_ptr = nullptr; // Let driver choose the work-group sizes. } backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size_ptr, dst); } static void ggml_cl_cpy(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { GGML_ASSERT(src0); GGML_ASSERT(src0->extra); GGML_ASSERT(src1); GGML_ASSERT(src1->extra); // GGML_OP_CPY happens between src0 and src1. // GGML_OP_DUP and GGML_OP_CONT happen between src0 and dst. UNUSED(dst); GGML_TENSOR_LOCALS(int, ne0, src0, ne); GGML_TENSOR_LOCALS(cl_ulong, nb0, src0, nb); GGML_TENSOR_LOCALS(int, ne1, src1, ne); GGML_TENSOR_LOCALS(cl_ulong, nb1, src1, nb); const enum ggml_type src0t = src0->type; const enum ggml_type src1t = src1->type; ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; ggml_tensor_extra_cl * extra0 = (ggml_tensor_extra_cl *)src0->extra; ggml_tensor_extra_cl * extra1 = (ggml_tensor_extra_cl *)src1->extra; cl_ulong offset0 = extra0->offset + src0->view_offs; cl_ulong offset1 = extra1->offset + src1->view_offs; // A contiguous f32 -> f32 copy is a linear move. The kernel below maps one workgroup to // each row, so a tensor with few long rows runs on a single compute unit; dispatch those // over the whole device instead. GGML_OPENCL_CPY_FLAT=0 restores the row-mapped path. static const bool cpy_flat_on = []{ const char * e = getenv("GGML_OPENCL_CPY_FLAT"); return !(e && e[0] == '0'); }(); if (cpy_flat_on && backend_ctx->kernel_cpy_f32_f32_flat != nullptr && src0t == GGML_TYPE_F32 && src1t == GGML_TYPE_F32 && ggml_is_contiguous(src0) && ggml_is_contiguous(src1) && ggml_nelements(src0) == ggml_nelements(src1)) { cl_kernel k = backend_ctx->kernel_cpy_f32_f32_flat; const cl_ulong nelem = (cl_ulong) ggml_nelements(src0); const cl_ulong n4 = nelem / 4; CL_CHECK(clSetKernelArg(k, 0, sizeof(cl_mem), &extra0->data_device)); CL_CHECK(clSetKernelArg(k, 1, sizeof(cl_ulong), &offset0)); CL_CHECK(clSetKernelArg(k, 2, sizeof(cl_mem), &extra1->data_device)); CL_CHECK(clSetKernelArg(k, 3, sizeof(cl_ulong), &offset1)); CL_CHECK(clSetKernelArg(k, 4, sizeof(cl_ulong), &nelem)); CL_CHECK(clSetKernelArg(k, 5, sizeof(cl_ulong), &n4)); // one work item per float4, plus one for the trailing scalars const size_t items = (size_t) n4 + ((nelem % 4) ? 1 : 0); const size_t lsz = MIN((size_t) 64, backend_ctx->max_workgroup_size); size_t global_work_size[] = { ((items + lsz - 1) / lsz) * lsz, 1, 1 }; size_t local_work_size[] = { lsz, 1, 1 }; backend_ctx->enqueue_ndrange_kernel(k, 1, global_work_size, local_work_size, src1); return; } cl_kernel kernel; switch (src0t) { case GGML_TYPE_F32: switch (src1t) { case GGML_TYPE_F16: kernel = backend_ctx->kernel_cpy_f32_f16; break; case GGML_TYPE_F32: kernel = ne00 < 32 ? backend_ctx->kernel_cpy_f32_f32_pack : backend_ctx->kernel_cpy_f32_f32; break; default: GGML_ASSERT(false && "not implemented"); } break; case GGML_TYPE_F16: switch (src1t) { case GGML_TYPE_F16: kernel = backend_ctx->kernel_cpy_f16_f16; break; case GGML_TYPE_F32: kernel = backend_ctx->kernel_cpy_f16_f32; break; default: GGML_ASSERT(false && "not implemented"); } break; case GGML_TYPE_I32: switch (src1t) { case GGML_TYPE_I32: kernel = backend_ctx->kernel_cpy_i32_i32; break; default: GGML_ASSERT(false && "not implemented"); } break; default: GGML_ASSERT(false && "not implemented"); } CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device)); CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1)); CL_CHECK(clSetKernelArg(kernel, 4, sizeof(int), &ne00)); CL_CHECK(clSetKernelArg(kernel, 5, sizeof(int), &ne01)); CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne02)); CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne03)); CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_ulong), &nb00)); CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_ulong), &nb01)); CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_ulong), &nb02)); CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_ulong), &nb03)); CL_CHECK(clSetKernelArg(kernel, 12, sizeof(int), &ne10)); CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &ne11)); CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &ne12)); CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &ne13)); CL_CHECK(clSetKernelArg(kernel, 16, sizeof(cl_ulong), &nb10)); CL_CHECK(clSetKernelArg(kernel, 17, sizeof(cl_ulong), &nb11)); CL_CHECK(clSetKernelArg(kernel, 18, sizeof(cl_ulong), &nb12)); CL_CHECK(clSetKernelArg(kernel, 19, sizeof(cl_ulong), &nb13)); if (kernel == backend_ctx->kernel_cpy_f32_f32_pack) { const int maxwg = (int)backend_ctx->get_kernel_workgroup_size(kernel); const int base = MIN(64, maxwg); const int tpr = MIN(ne00, base); // threads per row const int rpw = MAX(1, base / tpr); // rows per workgroup const int lsz = tpr * rpw; // <= base <= maxwg const int nrows = ne01*ne02*ne03; const int nwg = (nrows + rpw - 1) / rpw; size_t global_work_size[] = {(size_t)nwg*lsz, 1, 1}; size_t local_work_size[] = {(size_t)lsz, 1, 1}; backend_ctx->enqueue_ndrange_kernel(kernel, 1, global_work_size, local_work_size, src1); } else { const int nth = MIN(64, ne00); size_t global_work_size[] = {(size_t)ne01*nth, (size_t)ne02, (size_t)ne03}; size_t local_work_size[] = {(size_t)nth, 1, 1}; backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, src1); } } static void ggml_cl_dup(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { ggml_cl_cpy(backend, src0, dst, nullptr); UNUSED(src1); } static void ggml_cl_set(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { GGML_ASSERT(src0); GGML_ASSERT(src0->extra); GGML_ASSERT(src1); GGML_ASSERT(src1->extra); GGML_ASSERT(dst); GGML_ASSERT(dst->extra); GGML_ASSERT((src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_I32) && src1->type == src0->type && dst->type == src0->type); GGML_TENSOR_LOCALS(int, ne0, src0, ne); GGML_TENSOR_LOCALS(cl_ulong, nb0, src0, nb); GGML_TENSOR_LOCALS(int, ne1, src1, ne); GGML_TENSOR_LOCALS(cl_ulong, nb1, src1, nb); GGML_TENSOR_LOCALS(int, ne, dst, ne); GGML_TENSOR_LOCALS(cl_ulong, nb, dst, nb); ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; ggml_tensor_extra_cl * extra1 = (ggml_tensor_extra_cl *)src1->extra; ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; cl_ulong offset1 = extra1->offset + src1->view_offs; cl_ulong offsetd = extrad->offset + dst->view_offs; const cl_ulong pnb1 = ((const int32_t *)dst->op_params)[0]; const cl_ulong pnb2 = ((const int32_t *)dst->op_params)[1]; const cl_ulong pnb3 = ((const int32_t *)dst->op_params)[2]; const cl_ulong offs = ((const int32_t *)dst->op_params)[3]; const bool inplace = (bool)((const int32_t *)dst->op_params)[4]; cl_kernel kernel = nullptr; // for inplace case, dst is a view of src0 and is updated on top of it // so for non-inplace case, copy src0 to dst first if (!inplace) { ggml_cl_cpy(backend, src0, dst, nullptr); } // then copy src1 to dst with specified offset if (src1->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32) { kernel = backend_ctx->kernel_cpy_f32_f32; } else if (src1->type == GGML_TYPE_I32 && dst->type == GGML_TYPE_I32) { kernel = backend_ctx->kernel_cpy_i32_i32; } else { GGML_ASSERT(false && "not implemented"); } offsetd += offs; cl_ulong nb = ggml_element_size(dst); CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra1->data_device)); CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset1)); CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extrad->data_device)); CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offsetd)); CL_CHECK(clSetKernelArg(kernel, 4, sizeof(int), &ne10)); CL_CHECK(clSetKernelArg(kernel, 5, sizeof(int), &ne11)); CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne12)); CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne13)); CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_ulong), &nb10)); CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_ulong), &nb11)); CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_ulong), &nb12)); CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_ulong), &nb13)); CL_CHECK(clSetKernelArg(kernel, 12, sizeof(int), &ne10)); CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &ne11)); CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &ne12)); CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &ne13)); CL_CHECK(clSetKernelArg(kernel, 16, sizeof(cl_ulong), &nb)); CL_CHECK(clSetKernelArg(kernel, 17, sizeof(cl_ulong), &pnb1)); CL_CHECK(clSetKernelArg(kernel, 18, sizeof(cl_ulong), &pnb2)); CL_CHECK(clSetKernelArg(kernel, 19, sizeof(cl_ulong), &pnb3)); int max_local_size = backend_ctx->get_kernel_workgroup_size(kernel); const int nth = MIN(max_local_size, ne00); size_t global_work_size[] = {(size_t)ne11*nth, (size_t)ne12, (size_t)ne13}; size_t local_work_size[] = {(size_t)nth, 1, 1}; backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); } static void ggml_cl_diag_mask_inf(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { GGML_ASSERT(src0); GGML_ASSERT(src0->extra); GGML_ASSERT(dst); GGML_ASSERT(dst->extra); UNUSED(src1); int n_past = ((int32_t *)(dst->op_params))[0]; const int ne00 = src0 ? src0->ne[0] : 0; const int ne01 = src0 ? src0->ne[1] : 0; const int ne02 = src0 ? src0->ne[2] : 0; ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; ggml_tensor_extra_cl * extra0 = (ggml_tensor_extra_cl *)src0->extra; ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; cl_ulong offset0 = extra0->offset + src0->view_offs; cl_ulong offsetd = extrad->offset + dst->view_offs; cl_kernel kernel; if (ne00%8 == 0) { kernel = backend_ctx->kernel_diag_mask_inf_8; CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extrad->data_device)); CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offsetd)); CL_CHECK(clSetKernelArg(kernel, 4, sizeof(int), &ne00)); CL_CHECK(clSetKernelArg(kernel, 5, sizeof(int), &ne01)); CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &n_past)); size_t global_work_size[] = {(size_t)ne00*ne01*ne02/8, 1, 1}; size_t local_work_size[] = {64, 1, 1}; backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); } else { kernel = backend_ctx->kernel_diag_mask_inf; CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extrad->data_device)); CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offsetd)); CL_CHECK(clSetKernelArg(kernel, 4, sizeof(int), &ne00)); CL_CHECK(clSetKernelArg(kernel, 5, sizeof(int), &ne01)); CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &n_past)); size_t global_work_size[] = {(size_t)ne00, (size_t)ne01, (size_t)ne02}; size_t local_work_size[] = {64, 1, 1}; size_t * local_work_size_ptr = local_work_size; if (ne00 % 64 != 0 && !backend_ctx->non_uniform_workgroups) { local_work_size_ptr = nullptr; // Let driver choose the work-group sizes. } backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size_ptr, dst); } } static void ggml_cl_diag(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { GGML_ASSERT(src0); GGML_ASSERT(src0->extra); GGML_ASSERT(dst); GGML_ASSERT(dst->extra); UNUSED(src1); ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; ggml_tensor_extra_cl * extra0 = (ggml_tensor_extra_cl *)src0->extra; ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; cl_ulong offset0 = extra0->offset + src0->view_offs; cl_ulong offsetd = extrad->offset + dst->view_offs; GGML_TENSOR_LOCALS(int, ne0, src0, ne); GGML_TENSOR_LOCALS(cl_ulong, nb0, src0, nb); GGML_TENSOR_LOCALS(int, ne, dst, ne); GGML_TENSOR_LOCALS(cl_ulong, nb, dst, nb); cl_kernel kernel = backend_ctx->kernel_diag_f32; CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extrad->data_device)); CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offsetd)); CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_ulong), &nb01)); CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &nb02)); CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_ulong), &nb03)); CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_int), &ne0)); CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_ulong), &nb0)); CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_ulong), &nb2)); CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_ulong), &nb3)); int nth = 64; size_t global_work_size[] = {(size_t)ne1*nth, (size_t)ne2, (size_t)ne3}; size_t local_work_size[] = {(size_t)nth, 1, 1}; backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); } static void ggml_cl_soft_max(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { GGML_ASSERT(src0); GGML_ASSERT(src0->extra); GGML_ASSERT(dst); GGML_ASSERT(dst->extra); // Softmax can now fuse KQ mask and KQ scale, which used to be two additional // ops before softmax. It now also fuses alibi if `max_bias > 0`. For llama, // alibi is not used; however, for some other models, it is used. // KQ_mask if (src1) { GGML_ASSERT(src1); GGML_ASSERT(src1->extra); } const ggml_tensor * src2 = dst->src[2]; if (src2) { GGML_ASSERT(src2->extra); } ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; ggml_tensor_extra_cl * extra0 = (ggml_tensor_extra_cl *)src0->extra; ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; ggml_tensor_extra_cl * extra1 = src1 ? (ggml_tensor_extra_cl *)src1->extra : nullptr; ggml_tensor_extra_cl * extra2 = src2 ? (ggml_tensor_extra_cl *)src2->extra : nullptr; cl_ulong offset0 = extra0->offset + src0->view_offs; cl_ulong offsetd = extrad->offset + dst->view_offs; cl_ulong offset1 = extra1 ? extra1->offset + src1->view_offs : offset0; cl_ulong offset2 = extra2 ? extra2->offset + src2->view_offs : offset0; const int ne00 = src0->ne[0]; const int ne01 = src0->ne[1]; const int ne02 = src0->ne[2]; const int ne03 = src0->ne[3]; const cl_long nb01 = src0->nb[1]; const cl_long nb02 = src0->nb[2]; const cl_long nb03 = src0->nb[3]; const int ne12 = src1 ? src1->ne[2] : 0; const int ne13 = src1 ? src1->ne[3] : 0; const cl_long nb11 = src1 ? src1->nb[1] : 0; const cl_long nb12 = src1 ? src1->nb[2] : 0; const cl_long nb13 = src1 ? src1->nb[3] : 0; const cl_long nb1 = dst->nb[1]; const cl_long nb2 = dst->nb[2]; const cl_long nb3 = dst->nb[3]; float scale, max_bias; memcpy(&scale, dst->op_params + 0, sizeof(float)); memcpy(&max_bias, dst->op_params + 1, sizeof(float)); const int n_head = src0->ne[2]; const int n_head_log2 = 1u << (uint32_t) floorf(log2f((float) n_head)); const float m0 = powf(2.0f, -(max_bias ) / n_head_log2); const float m1 = powf(2.0f, -(max_bias / 2.0f) / n_head_log2); const bool use_f16 = (src1 && src1->type == GGML_TYPE_F16); // Local size must be wave size. Each workgroup is a wave, working on a row, // where a row corresponds to leading dimension. int nth = MIN(32, ne00); if (backend_ctx->gpu_family == INTEL) { // This is the same as the initial value. nth = MIN(32, ne00); } else if (backend_ctx->gpu_family == ADRENO) { nth = 64; } else { GGML_ASSERT(false && "TODO: Unknown GPU"); } cl_kernel kernel; if (ne00%4 == 0) { if (use_f16) { kernel = backend_ctx->kernel_soft_max_4_f16; } else { kernel = backend_ctx->kernel_soft_max_4; } } else { if (use_f16) { kernel = backend_ctx->kernel_soft_max_f16; } else { kernel = backend_ctx->kernel_soft_max; } } CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), extra1 ? &extra1->data_device : &extra0->data_device)); CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1)); CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), extra2 ? &extra2->data_device : &extra0->data_device)); CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offset2)); CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_mem), &extrad->data_device)); CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_ulong), &offsetd)); CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne00)); CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_ulong), &nb01)); CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_ulong), &nb02)); CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_ulong), &nb03)); CL_CHECK(clSetKernelArg(kernel, 12, sizeof(int), &ne12)); CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &ne13)); CL_CHECK(clSetKernelArg(kernel, 14, sizeof(cl_ulong), &nb11)); CL_CHECK(clSetKernelArg(kernel, 15, sizeof(cl_ulong), &nb12)); CL_CHECK(clSetKernelArg(kernel, 16, sizeof(cl_ulong), &nb13)); CL_CHECK(clSetKernelArg(kernel, 17, sizeof(cl_ulong), &nb1)); CL_CHECK(clSetKernelArg(kernel, 18, sizeof(cl_ulong), &nb2)); CL_CHECK(clSetKernelArg(kernel, 19, sizeof(cl_ulong), &nb3)); CL_CHECK(clSetKernelArg(kernel, 20, sizeof(float), &scale)); CL_CHECK(clSetKernelArg(kernel, 21, sizeof(float), &max_bias)); CL_CHECK(clSetKernelArg(kernel, 22, sizeof(float), &m0)); CL_CHECK(clSetKernelArg(kernel, 23, sizeof(float), &m1)); CL_CHECK(clSetKernelArg(kernel, 24, sizeof(int), &n_head_log2)); size_t global_work_size[] = {(size_t)ne01*nth, (size_t)ne02, (size_t)ne03}; size_t local_work_size[] = {(size_t)nth, 1, 1}; backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); } static void ggml_cl_rope(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { GGML_ASSERT(src0); GGML_ASSERT(src0->extra); GGML_ASSERT(src1); GGML_ASSERT(src1->extra); GGML_ASSERT(dst); GGML_ASSERT(dst->extra); ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; ggml_tensor_extra_cl * extra0 = (ggml_tensor_extra_cl *)src0->extra; ggml_tensor_extra_cl * extra1 = (ggml_tensor_extra_cl *)src1->extra; ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; cl_ulong offset0 = extra0->offset + src0->view_offs; cl_ulong offset1 = extra1->offset + src1->view_offs; cl_ulong offsetd = extrad->offset + dst->view_offs; ggml_tensor * src2 = dst->src[2]; ggml_tensor_extra_cl * extra2 = src2 ? (ggml_tensor_extra_cl *)src2->extra : nullptr; cl_ulong offset2 = extra2 ? extra2->offset + src2->view_offs : offset0; const int ne00 = src0 ? src0->ne[0] : 0; const int ne01 = src0 ? src0->ne[1] : 0; const int ne02 = src0 ? src0->ne[2] : 0; const int ne03 = src0 ? src0->ne[3] : 0; const cl_ulong nb00 = src0 ? src0->nb[0] : 0; const cl_ulong nb01 = src0 ? src0->nb[1] : 0; const cl_ulong nb02 = src0 ? src0->nb[2] : 0; const cl_ulong nb03 = src0 ? src0->nb[3] : 0; const int ne10 = src1 ? src1->ne[0] : 0; const int ne11 = src1 ? src1->ne[1] : 0; UNUSED(ne11); const int ne12 = src1 ? src1->ne[2] : 0; UNUSED(ne12); const int ne13 = src1 ? src1->ne[3] : 0; UNUSED(ne13); const int ne0 = dst ? dst->ne[0] : 0; const int ne1 = dst ? dst->ne[1] : 0; const int ne2 = dst ? dst->ne[2] : 0; const int ne3 = dst ? dst->ne[3] : 0; const cl_ulong nb0 = dst ? dst->nb[0] : 0; const cl_ulong nb1 = dst ? dst->nb[1] : 0; const cl_ulong nb2 = dst ? dst->nb[2] : 0; const cl_ulong nb3 = dst ? dst->nb[3] : 0; GGML_ASSERT(ne10 % ne02 == 0); GGML_ASSERT(ne10 >= ne02); int nth = MIN(64, ne00); const int n_past = ((int *) dst->op_params)[0]; const int n_dims = ((int *) dst->op_params)[1]; const int mode = ((int *) dst->op_params)[2]; const int n_ctx_orig = ((int32_t *) dst->op_params)[4]; const int n_offs = ((int32_t *) dst->op_params)[15]; float freq_base; float freq_scale; float ext_factor; float attn_factor; float beta_fast; float beta_slow; int32_t sections[4]; memcpy(&freq_base, (int32_t *) dst->op_params + 5, sizeof(float)); memcpy(&freq_scale, (int32_t *) dst->op_params + 6, sizeof(float)); memcpy(&ext_factor, (int32_t *) dst->op_params + 7, sizeof(float)); memcpy(&attn_factor, (int32_t *) dst->op_params + 8, sizeof(float)); memcpy(&beta_fast, (int32_t *) dst->op_params + 9, sizeof(float)); memcpy(&beta_slow, (int32_t *) dst->op_params + 10, sizeof(float)); memcpy(§ions, (int32_t *) dst->op_params + 11, sizeof(int32_t)*4); const bool is_neox = mode & 2; const bool is_mrope = mode & GGML_ROPE_TYPE_MROPE; const bool is_vision = mode == GGML_ROPE_TYPE_VISION; const int is_imrope = mode == GGML_ROPE_TYPE_IMROPE; if (is_mrope) { GGML_ASSERT(sections[0] > 0 || sections[1] > 0 || sections[2] > 0); } if (is_vision) { GGML_ASSERT(n_dims == ne00/2); GGML_ASSERT(n_offs == 0); // offset not supported for vision, as the rotated pairs span the whole row } cl_kernel kernel; if (is_neox) { switch (src0->type) { case GGML_TYPE_F32: kernel = backend_ctx->kernel_rope_neox_f32; break; case GGML_TYPE_F16: kernel = backend_ctx->kernel_rope_neox_f16; break; default: GGML_ASSERT(false); }; } else if (is_mrope && !is_vision) { switch (src0->type) { case GGML_TYPE_F32: kernel = backend_ctx->kernel_rope_multi_f32; break; case GGML_TYPE_F16: kernel = backend_ctx->kernel_rope_multi_f16; break; default: GGML_ASSERT(false); }; } else if (is_vision) { switch (src0->type) { case GGML_TYPE_F32: kernel = backend_ctx->kernel_rope_vision_f32; break; case GGML_TYPE_F16: kernel = backend_ctx->kernel_rope_vision_f16; break; default: GGML_ASSERT(false); } } else { switch (src0->type) { case GGML_TYPE_F32: kernel = backend_ctx->kernel_rope_norm_f32; break; case GGML_TYPE_F16: kernel = backend_ctx->kernel_rope_norm_f16; break; default: GGML_ASSERT(false); }; } CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device)); CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1)); CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), extra2 ? &extra2->data_device : &extra0->data_device)); CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offset2)); CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_mem), &extrad->data_device)); CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_ulong), &offsetd)); CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne00)); CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne01)); CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &ne02)); CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne03)); CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_ulong), &nb00)); CL_CHECK(clSetKernelArg(kernel, 13, sizeof(cl_ulong), &nb01)); CL_CHECK(clSetKernelArg(kernel, 14, sizeof(cl_ulong), &nb02)); CL_CHECK(clSetKernelArg(kernel, 15, sizeof(cl_ulong), &nb03)); CL_CHECK(clSetKernelArg(kernel, 16, sizeof(int), &ne0)); CL_CHECK(clSetKernelArg(kernel, 17, sizeof(int), &ne1)); CL_CHECK(clSetKernelArg(kernel, 18, sizeof(int), &ne2)); CL_CHECK(clSetKernelArg(kernel, 19, sizeof(int), &ne3)); CL_CHECK(clSetKernelArg(kernel, 20, sizeof(cl_ulong), &nb0)); CL_CHECK(clSetKernelArg(kernel, 21, sizeof(cl_ulong), &nb1)); CL_CHECK(clSetKernelArg(kernel, 22, sizeof(cl_ulong), &nb2)); CL_CHECK(clSetKernelArg(kernel, 23, sizeof(cl_ulong), &nb3)); CL_CHECK(clSetKernelArg(kernel, 24, sizeof(int), &n_past)); CL_CHECK(clSetKernelArg(kernel, 25, sizeof(int), &n_dims)); CL_CHECK(clSetKernelArg(kernel, 26, sizeof(int), &n_ctx_orig)); CL_CHECK(clSetKernelArg(kernel, 27, sizeof(float), &freq_base)); CL_CHECK(clSetKernelArg(kernel, 28, sizeof(float), &freq_scale)); CL_CHECK(clSetKernelArg(kernel, 29, sizeof(float), &ext_factor)); CL_CHECK(clSetKernelArg(kernel, 30, sizeof(float), &attn_factor)); CL_CHECK(clSetKernelArg(kernel, 31, sizeof(float), &beta_fast)); CL_CHECK(clSetKernelArg(kernel, 32, sizeof(float), &beta_slow)); // both mrope and vision kernels have sections if (is_mrope || is_vision) { CL_CHECK(clSetKernelArg(kernel, 33, sizeof(int32_t)*4, §ions)); } // only mrope has is_imrope if (is_mrope && !is_vision) { CL_CHECK(clSetKernelArg(kernel, 34, sizeof(int), &is_imrope)); } // norm and neox have n_offs after beta_slow, mrope has it after is_imrope if (!is_mrope && !is_vision) { CL_CHECK(clSetKernelArg(kernel, 33, sizeof(int), &n_offs)); } else if (is_mrope && !is_vision) { CL_CHECK(clSetKernelArg(kernel, 35, sizeof(int), &n_offs)); } size_t global_work_size[] = {(size_t)ne01*nth, (size_t)ne02, (size_t)ne03}; size_t local_work_size[] = {(size_t)nth, 1, 1}; backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); } static void ggml_cl_solve_tri(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { GGML_ASSERT(src0); GGML_ASSERT(src0->extra); GGML_ASSERT(src1); GGML_ASSERT(src1->extra); GGML_ASSERT(dst); GGML_ASSERT(dst->extra); ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; ggml_tensor_extra_cl * extra0 = (ggml_tensor_extra_cl *)src0->extra; ggml_tensor_extra_cl * extra1 = (ggml_tensor_extra_cl *)src1->extra; ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; cl_ulong offset0 = extra0->offset + src0->view_offs; cl_ulong offset1 = extra1->offset + src1->view_offs; cl_ulong offsetd = extrad->offset + dst->view_offs; cl_kernel kernel = backend_ctx->kernel_solve_tri_f32; GGML_ASSERT(kernel != nullptr); const int n = src0->ne[0]; const int k = src1->ne[0]; const cl_ulong nb00 = src0->nb[0]; const cl_ulong nb01 = src0->nb[1]; const cl_ulong nb02 = src0->nb[2]; const cl_ulong nb03 = src0->nb[3]; const cl_ulong nb10 = src1->nb[0]; const cl_ulong nb11 = src1->nb[1]; const cl_ulong nb12 = src1->nb[2]; const cl_ulong nb13 = src1->nb[3]; const cl_ulong nb0 = dst->nb[0]; const cl_ulong nb1 = dst->nb[1]; const cl_ulong nb2 = dst->nb[2]; const cl_ulong nb3 = dst->nb[3]; CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device)); CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1)); CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device)); CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd)); CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &n)); CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &k)); CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_ulong), &nb00)); CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_ulong), &nb01)); CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_ulong),&nb02)); CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_ulong),&nb03)); CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_ulong),&nb10)); CL_CHECK(clSetKernelArg(kernel, 13, sizeof(cl_ulong),&nb11)); CL_CHECK(clSetKernelArg(kernel, 14, sizeof(cl_ulong),&nb12)); CL_CHECK(clSetKernelArg(kernel, 15, sizeof(cl_ulong),&nb13)); CL_CHECK(clSetKernelArg(kernel, 16, sizeof(cl_ulong),&nb0)); CL_CHECK(clSetKernelArg(kernel, 17, sizeof(cl_ulong),&nb1)); CL_CHECK(clSetKernelArg(kernel, 18, sizeof(cl_ulong),&nb2)); CL_CHECK(clSetKernelArg(kernel, 19, sizeof(cl_ulong),&nb3)); size_t global_work_size[3]= { (size_t)k, (size_t)dst->ne[2], (size_t)dst->ne[3]}; size_t local_work_size[] = {16, 4, 1}; backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); } static void ggml_cl_im2col(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { GGML_ASSERT(src0); GGML_ASSERT(src1); GGML_ASSERT(src1->extra); GGML_ASSERT(dst); GGML_ASSERT(dst->extra); // src0 - filter, src1 - input GGML_ASSERT(src1->type == GGML_TYPE_F32); GGML_ASSERT(dst->type == GGML_TYPE_F16 || dst->type == GGML_TYPE_F32); ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; ggml_tensor_extra_cl * extra1 = (ggml_tensor_extra_cl *)src1->extra; ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; cl_ulong offset1 = extra1->offset + src1->view_offs; cl_ulong offsetd = extrad->offset + dst->view_offs; const int32_t s0 = ((const int32_t*)(dst->op_params))[0]; const int32_t s1 = ((const int32_t*)(dst->op_params))[1]; const int32_t p0 = ((const int32_t*)(dst->op_params))[2]; const int32_t p1 = ((const int32_t*)(dst->op_params))[3]; const int32_t d0 = ((const int32_t*)(dst->op_params))[4]; const int32_t d1 = ((const int32_t*)(dst->op_params))[5]; const bool is_2D = ((const int32_t*)(dst->op_params))[6] == 1; const cl_long IC = src1->ne[is_2D ? 2 : 1]; const cl_long IH = is_2D ? src1->ne[1] : 1; const cl_long IW = src1->ne[0]; const cl_long KH = is_2D ? src0->ne[1] : 1; const cl_long KW = src0->ne[0]; const cl_long OH = is_2D ? dst->ne[2] : 1; const cl_long OW = dst->ne[1]; // nb is byte offset, src is type float32 const cl_ulong delta_offset = src1->nb[is_2D ? 2 : 1]/4; const cl_long batch = src1->ne[is_2D ? 3 : 2]; const cl_ulong batch_offset = src1->nb[is_2D ? 3 : 2]/4; const cl_long pelements = OW*KW*KH; const cl_long CHW = IC*KH*KW; cl_kernel kernel; if(dst->type == GGML_TYPE_F16) { kernel = backend_ctx->kernel_im2col_f16; } else { kernel = backend_ctx->kernel_im2col_f32; } CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra1->data_device)); CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset1)); CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extrad->data_device)); CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offsetd)); CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_ulong), &batch_offset)); CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &delta_offset)); CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_long), &IW)); CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_long), &IH)); CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_long), &IC)); CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_long), &OW)); CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_long), &OH)); CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_long), &KW)); CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_long), &KH)); CL_CHECK(clSetKernelArg(kernel, 13, sizeof(cl_long), &pelements)); CL_CHECK(clSetKernelArg(kernel, 14, sizeof(cl_long), &CHW)); CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &s0)); CL_CHECK(clSetKernelArg(kernel, 16, sizeof(int), &s1)); CL_CHECK(clSetKernelArg(kernel, 17, sizeof(int), &p0)); CL_CHECK(clSetKernelArg(kernel, 18, sizeof(int), &p1)); CL_CHECK(clSetKernelArg(kernel, 19, sizeof(int), &d0)); CL_CHECK(clSetKernelArg(kernel, 20, sizeof(int), &d1)); const int num_blocks = (pelements + 256 - 1) / 256; size_t global_work_size[] = {(size_t)num_blocks*256, (size_t)OH, (size_t)batch*IC}; size_t local_work_size[] = {256, 1, 1}; backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); } static void ggml_cl_argsort(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { GGML_ASSERT(src0); GGML_ASSERT(src0->extra); GGML_ASSERT(dst); GGML_ASSERT(dst->extra); GGML_UNUSED(src1); GGML_ASSERT(src0->type == GGML_TYPE_F32); GGML_ASSERT( dst->type == GGML_TYPE_I32); GGML_ASSERT(ggml_is_contiguous(src0)); ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; ggml_tensor_extra_cl * extra0 = (ggml_tensor_extra_cl *)src0->extra; ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; cl_ulong offset0 = extra0->offset + src0->view_offs; cl_ulong offsetd = extrad->offset + dst->view_offs; const int ne00 = src0->ne[0]; const int nrows = ggml_nrows(src0); int ne00_padded = 1; while (ne00_padded < ne00) { ne00_padded *= 2; } int order = (enum ggml_sort_order) dst->op_params[0]; cl_kernel kernel = backend_ctx->kernel_argsort_f32_i32; CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extrad->data_device)); CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offsetd)); CL_CHECK(clSetKernelArg(kernel, 4, sizeof(int), &ne00)); CL_CHECK(clSetKernelArg(kernel, 5, sizeof(int), &ne00_padded)); CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &order)); CL_CHECK(clSetKernelArg(kernel, 7, ne00_padded*sizeof(int), NULL)); size_t global_work_size[] = {(size_t)ne00_padded, (size_t)nrows, (size_t)1}; size_t local_work_size[] = {(size_t)ne00_padded, 1, 1}; backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); #ifdef GGML_OPENCL_USE_ADRENO_KERNELS const int ne21 = dst->ne[1]; if ((strstr(src0->name, "_moe") != NULL) && (ne21 != 1)) { backend_ctx->toggle_reorder = true; } #endif // GGML_OPENCL_USE_ADRENO_KERNELS } static void ggml_cl_sum_rows(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { GGML_ASSERT(src0); GGML_ASSERT(src0->extra); GGML_ASSERT(dst); GGML_ASSERT(dst->extra); GGML_UNUSED(src1); GGML_ASSERT(src0->nb[0] == ggml_type_size(src0->type)); ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; ggml_tensor_extra_cl * extra0 = (ggml_tensor_extra_cl *)src0->extra; ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; cl_ulong offset0 = extra0->offset + src0->view_offs; cl_ulong offsetd = extrad->offset + dst->view_offs; const int ne00 = src0->ne[0]; const int ne01 = src0->ne[1]; const int ne02 = src0->ne[2]; const int ne03 = src0->ne[3]; const cl_ulong nb01 = src0->nb[1]; const cl_ulong nb02 = src0->nb[2]; const cl_ulong nb03 = src0->nb[3]; const cl_ulong nb1 = dst->nb[1]; const cl_ulong nb2 = dst->nb[2]; const cl_ulong nb3 = dst->nb[3]; cl_kernel kernel; const bool is_c4 = ne00 % 4 == 0; if (is_c4) { kernel = backend_ctx->kernel_sum_rows_f32_4; } else { kernel = backend_ctx->kernel_sum_rows_f32; } CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extrad->data_device)); CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offsetd)); CL_CHECK(clSetKernelArg(kernel, 4, sizeof(int), &ne00)); CL_CHECK(clSetKernelArg(kernel, 5, sizeof(int), &ne01)); CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne02)); CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne03)); CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_ulong), &nb01)); CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_ulong), &nb02)); CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_ulong), &nb03)); CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_ulong), &nb1)); CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_ulong), &nb2)); CL_CHECK(clSetKernelArg(kernel, 13, sizeof(cl_ulong), &nb3)); size_t global_work_size[] = {64 * (size_t)ne01, (size_t)ne02, (size_t)ne03}; size_t local_work_size[] = {(size_t)64, 1, 1}; backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); } static void ggml_cl_cumsum(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { GGML_ASSERT(src0); GGML_ASSERT(src0->extra); GGML_ASSERT(dst); GGML_ASSERT(dst->extra); GGML_UNUSED(src1); GGML_ASSERT(src0->nb[0] == ggml_type_size(src0->type)); GGML_ASSERT(ggml_is_contiguous(src0)); ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; ggml_tensor_extra_cl * extra0 = (ggml_tensor_extra_cl *)src0->extra; ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; cl_ulong offset0 = extra0->offset + src0->view_offs; cl_ulong offsetd = extrad->offset + dst->view_offs; GGML_TENSOR_LOCALS(int, ne0, src0, ne); GGML_TENSOR_LOCALS(cl_ulong, nb0, src0, nb); cl_kernel kernel = backend_ctx->kernel_cumsum_blk; int max_workgroup_size = backend_ctx->get_kernel_workgroup_size(kernel); int nth = 1; while (nth < ne00 && 2*nth <= max_workgroup_size) { nth *= 2; } GGML_ASSERT(ne00 <= nth*nth); const int net0 = CEIL_DIV(ne00, nth); const int net1 = ne01; const int net2 = ne02; const int net3 = ne03; const cl_ulong nbt0 = sizeof(float); const cl_ulong nbt1 = net0*nbt0; const cl_ulong nbt2 = net1*nbt1; const cl_ulong nbt3 = net2*nbt2; static ggml_cl_buffer tmp_buffer; tmp_buffer.allocate(backend_ctx->context, net0*ne01*ne02*ne03*sizeof(float)); CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &tmp_buffer.buffer)); CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extrad->data_device)); CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_ulong), &offsetd)); CL_CHECK(clSetKernelArg(kernel, 5, sizeof(int), &ne00)); CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne01)); CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne02)); CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne03)); CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_ulong), &nb00)); CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_ulong), &nb01)); CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_ulong), &nb02)); CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_ulong), &nb03)); CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &net0)); CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &net1)); CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &net2)); size_t global_work_size[] = { (size_t)(nth*net0*ne01), (size_t)ne02, (size_t)ne03}; size_t local_work_size[] = { (size_t)nth, 1, 1}; backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); if(ne00 > nth) { // if a single workgroup cannot handle an entire row, each workgroup // computes a partial sum and stores to dst, tmp_buffer contains the sum // of the each workgroup; cumsum this buffer and add to the partial sums in dst cl_ulong offsett = 0; kernel = backend_ctx->kernel_cumsum_blk; CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &tmp_buffer.buffer)); CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offsett)); CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &tmp_buffer.buffer)); CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &tmp_buffer.buffer)); CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_ulong), &offsett)); CL_CHECK(clSetKernelArg(kernel, 5, sizeof(int), &net0)); CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne01)); CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne02)); CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne03)); CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_ulong), &nbt0)); CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_ulong), &nbt1)); CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_ulong), &nbt2)); CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_ulong), &nbt3)); CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &net0)); CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &net1)); CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &net2)); size_t global_work_size_1[] = { (size_t)net1*nth, (size_t)net2, (size_t)net3}; size_t local_work_size_1[] = { (size_t)nth, 1, 1}; backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size_1, local_work_size_1, dst); kernel = backend_ctx->kernel_cumsum_add; CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &tmp_buffer.buffer)); CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extrad->data_device)); CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_ulong), &offsetd)); CL_CHECK(clSetKernelArg(kernel, 3, sizeof(int), &ne00)); CL_CHECK(clSetKernelArg(kernel, 4, sizeof(int), &ne01)); CL_CHECK(clSetKernelArg(kernel, 5, sizeof(int), &ne02)); CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne03)); CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &nbt0)); CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &nbt1)); CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &nbt2)); CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &nbt3)); size_t global_work_size_2[] = { (size_t)(nth*net0*ne01), (size_t)ne02, (size_t)ne03}; size_t local_work_size_2[] = { (size_t)nth, 1, 1}; backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size_2, local_work_size_2, dst); } } static void ggml_cl_glu(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { GGML_ASSERT(src0); GGML_ASSERT(src0->extra); GGML_ASSERT(dst); GGML_ASSERT(dst->extra); GGML_ASSERT(ggml_is_contiguous_1(src0)); if (src1) { GGML_ASSERT(src1); GGML_ASSERT(src1->extra); GGML_ASSERT(ggml_are_same_shape(src0, src1)); } ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; cl_kernel kernel; switch (ggml_get_glu_op(dst)) { case GGML_GLU_OP_GEGLU: if (dst->type == GGML_TYPE_F32) { kernel = backend_ctx->kernel_geglu; } else { kernel = backend_ctx->kernel_geglu_f16; } break; case GGML_GLU_OP_REGLU: if (dst->type == GGML_TYPE_F32) { kernel = backend_ctx->kernel_reglu; } else { kernel = backend_ctx->kernel_reglu_f16; } break; case GGML_GLU_OP_SWIGLU: if (dst->type == GGML_TYPE_F32) { kernel = backend_ctx->kernel_swiglu; } else { kernel = backend_ctx->kernel_swiglu_f16; } break; case GGML_GLU_OP_SWIGLU_OAI: kernel = backend_ctx->kernel_swiglu_oai; break; case GGML_GLU_OP_SWIGLU_CLAMP: if (dst->type == GGML_TYPE_F32) { kernel = backend_ctx->kernel_swiglu_clamp; } else { kernel = backend_ctx->kernel_swiglu_clamp_f16; } break; case GGML_GLU_OP_GEGLU_ERF: if (dst->type == GGML_TYPE_F32) { kernel = backend_ctx->kernel_geglu_erf; } else { kernel = backend_ctx->kernel_geglu_erf_f16; } break; case GGML_GLU_OP_GEGLU_QUICK: if (dst->type == GGML_TYPE_F32) { kernel = backend_ctx->kernel_geglu_quick; } else { kernel = backend_ctx->kernel_geglu_quick_f16; } break; default: GGML_ABORT("Unsupported glu op"); } ggml_tensor_extra_cl * extra0 = (ggml_tensor_extra_cl *)src0->extra; ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; ggml_tensor_extra_cl * extra1 = src1 ? (ggml_tensor_extra_cl *)src1->extra : nullptr; cl_ulong offset0 = extra0->offset + src0->view_offs; cl_ulong offsetd = extrad->offset + dst->view_offs; cl_ulong offset1 = extra1 ? extra1->offset + src1->view_offs : offset0; const int ne0 = dst->ne[0]; const cl_ulong nb01 = src0->nb[1]; const cl_ulong nb11 = src1 ? src1->nb[1] : nb01; const cl_ulong nb1 = dst->nb[1]; const int swp = ggml_get_op_params_i32(dst, 1); const float alpha = ggml_get_op_params_f32(dst, 2); const float limit = ggml_get_op_params_f32(dst, 3); const int ne00_off = src1 ? 0 : (swp ? ne0 : 0); const int ne10_off = src1 ? 0 : (swp ? 0 : ne0); CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), src1 ? &extra1->data_device : &extra0->data_device)); CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1)); CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device)); CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd)); CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_ulong), &nb01)); CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_ulong), &nb11)); CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne0)); CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_ulong), &nb1)); CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &ne00_off)); CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne10_off)); if (ggml_get_glu_op(dst) == GGML_GLU_OP_SWIGLU_OAI || ggml_get_glu_op(dst) == GGML_GLU_OP_SWIGLU_CLAMP) { CL_CHECK(clSetKernelArg(kernel, 12, sizeof(float), &limit)); } if (ggml_get_glu_op(dst) == GGML_GLU_OP_SWIGLU_OAI) { CL_CHECK(clSetKernelArg(kernel, 13, sizeof(float), &alpha)); } const size_t nrows = ggml_nrows(src0); size_t nth = backend_ctx->max_workgroup_size < 512 ? backend_ctx->max_workgroup_size : 512; size_t global_work_size[] = {nrows*nth, 1, 1}; size_t local_work_size[] = {nth, 1, 1}; backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); } static void ggml_cl_gated_delta_net(ggml_backend_t backend, ggml_tensor * dst) { GGML_ASSERT(dst); GGML_ASSERT(dst->extra); const ggml_tensor * src_q = dst->src[0]; const ggml_tensor * src_k = dst->src[1]; const ggml_tensor * src_v = dst->src[2]; const ggml_tensor * src_g = dst->src[3]; const ggml_tensor * src_beta = dst->src[4]; const ggml_tensor * src_state = dst->src[5]; GGML_ASSERT(src_q && src_q->extra); GGML_ASSERT(src_k && src_k->extra); GGML_ASSERT(src_v && src_v->extra); GGML_ASSERT(src_g && src_g->extra); GGML_ASSERT(src_beta && src_beta->extra); GGML_ASSERT(src_state && src_state->extra); ggml_backend_opencl_context * backend_ctx = (ggml_backend_opencl_context *) backend->context; const cl_uint S_v = (cl_uint) src_v->ne[0]; const cl_uint H_v = (cl_uint) src_v->ne[1]; const cl_uint n_tokens = (cl_uint) src_v->ne[2]; const cl_uint n_seqs = (cl_uint) src_v->ne[3]; const cl_uint K = (cl_uint) ggml_get_op_params_i32(dst, 0); int si; switch (S_v) { case 16: si = 0; break; case 32: si = 1; break; case 64: si = 2; break; case 128: si = 3; break; default: GGML_ASSERT(false && "ggml_cl_gated_delta_net: unsupported S_v"); } const int kda = (src_g->ne[0] == (int64_t) S_v) ? 1 : 0; // TODO: Optimize when S_v!=128. Not necessary for now as Qwen3.5/6 are all S_v=128 // token generation mode (tgpp=0): // process 1 token at a time, so columns per lane (cpl) == 1 // prompt processing mode (tgpp=1): // cpl=4 to process 4 tokens for single-token. 4 is chosen for Adreno 750 as per // work-item/thread has at most 128 registers. // All Qwen3.5/6 models are S_v == 128, so LANES_PER_COLUMN == 8 // such that ROWS_PER_LANE = 128/8 = 16 // Variables in the kernel: // k_reg, q_reg, g_exp are all 16 floats // s_shard has cpl*ROWS_PER_LANE = 4*16 = 64 floats // Total 112 registers used. // subgroups_per_workgroup (spw) can be set to 1,2,4,8,16 for tg and 1,2,4 for pp // for S_v=128. // Empirically found that when spw=1, we get the best performance for both tg and pp const int tgpp = (n_tokens == 1) ? 0 : 1; const int cpl = (tgpp == 0) ? 1 : 4; // spw needs adjustment when S_v != 128 const int spw = (tgpp == 0) ? 1 : 1; cl_kernel kernel = backend_ctx->kernel_gated_delta_net_f32[si][kda][tgpp]; GGML_ASSERT(kernel != nullptr); const cl_uint s_off = S_v * H_v * n_tokens * n_seqs; const cl_uint sq1 = (cl_uint)(src_q->nb[1] / sizeof(float)); const cl_uint sq2 = (cl_uint)(src_q->nb[2] / sizeof(float)); const cl_uint sq3 = (cl_uint)(src_q->nb[3] / sizeof(float)); const cl_uint sv1 = (cl_uint)(src_v->nb[1] / sizeof(float)); const cl_uint sv2 = (cl_uint)(src_v->nb[2] / sizeof(float)); const cl_uint sv3 = (cl_uint)(src_v->nb[3] / sizeof(float)); const cl_uint sb1 = (cl_uint)(src_beta->nb[1] / sizeof(float)); const cl_uint sb2 = (cl_uint)(src_beta->nb[2] / sizeof(float)); const cl_uint sb3 = (cl_uint)(src_beta->nb[3] / sizeof(float)); const cl_uint H_k = (cl_uint) src_q->ne[1]; const cl_uint rq3 = (cl_uint)(src_v->ne[3] / src_q->ne[3]); const float scale = 1.0f / sqrtf((float) S_v); ggml_tensor_extra_cl * extra_q = (ggml_tensor_extra_cl *) src_q->extra; ggml_tensor_extra_cl * extra_k = (ggml_tensor_extra_cl *) src_k->extra; ggml_tensor_extra_cl * extra_v = (ggml_tensor_extra_cl *) src_v->extra; ggml_tensor_extra_cl * extra_g = (ggml_tensor_extra_cl *) src_g->extra; ggml_tensor_extra_cl * extra_beta = (ggml_tensor_extra_cl *) src_beta->extra; ggml_tensor_extra_cl * extra_state = (ggml_tensor_extra_cl *) src_state->extra; ggml_tensor_extra_cl * extra_dst = (ggml_tensor_extra_cl *) dst->extra; const cl_ulong off_q = extra_q->offset + src_q->view_offs; const cl_ulong off_k = extra_k->offset + src_k->view_offs; const cl_ulong off_v = extra_v->offset + src_v->view_offs; const cl_ulong off_g = extra_g->offset + src_g->view_offs; const cl_ulong off_beta = extra_beta->offset + src_beta->view_offs; const cl_ulong off_state = extra_state->offset + src_state->view_offs; const cl_ulong off_dst = extra_dst->offset + dst->view_offs; int idx = 0; CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_mem), &extra_q->data_device)); CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_ulong), &off_q)); CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_mem), &extra_k->data_device)); CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_ulong), &off_k)); CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_mem), &extra_v->data_device)); CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_ulong), &off_v)); CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_mem), &extra_g->data_device)); CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_ulong), &off_g)); CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_mem), &extra_beta->data_device)); CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_ulong), &off_beta)); CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_mem), &extra_state->data_device)); CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_ulong), &off_state)); CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_mem), &extra_dst->data_device)); CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_ulong), &off_dst)); CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_uint), &H_v)); CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_uint), &n_tokens)); CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_uint), &n_seqs)); CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_uint), &s_off)); CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_uint), &sq1)); CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_uint), &sq2)); CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_uint), &sq3)); CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_uint), &sv1)); CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_uint), &sv2)); CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_uint), &sv3)); CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_uint), &sb1)); CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_uint), &sb2)); CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_uint), &sb3)); CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_uint), &H_k)); CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_uint), &rq3)); CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(float), &scale)); CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_uint), &K)); // Subgroup size is 64 for Adreno and 32 for Intel const int sg_size = backend_ctx->gpu_family == GPU_FAMILY::ADRENO ? 64 : backend_ctx->gpu_family == GPU_FAMILY::INTEL ? 32 : -1; if (sg_size < 0) { GGML_LOG_ERROR("Unsupported GPU Family: only Adreno and Intel are supported.\n"); exit(1); } // For the subgroup-shuffle kernel, we can safely prefer 8 lanes/column for S_v>=128 // For the subgroup-shuffle kernel: // S_v >= 128 -> prefer 8 lanes/column (good occupancy & register pressure tradeoff) // else -> min(S_v, subgroup_size) int lanes_per_column; if ((int)S_v >= 128) { lanes_per_column = 8; } else { lanes_per_column = std::min((int)S_v, sg_size); } // Max workgroup size for Adreno 750 is 1024 const int wg_size = sg_size * spw; // Ensure lanes_per_column is a power-of-two and divides both S_v and subgroup_size. // (Required for lane-group shuffle-xor reduction correctness.) while (lanes_per_column > 1 && (((lanes_per_column & (lanes_per_column - 1)) != 0) || (((int)S_v % lanes_per_column) != 0) || (sg_size % lanes_per_column) != 0)) { lanes_per_column >>= 1; } GGML_ASSERT(lanes_per_column >= 1); GGML_ASSERT(((lanes_per_column & (lanes_per_column - 1)) == 0)); GGML_ASSERT(((int)S_v % lanes_per_column) == 0); GGML_ASSERT((sg_size % lanes_per_column) == 0); const int cols_per_wg = spw * (sg_size / lanes_per_column) * cpl; GGML_ASSERT(cols_per_wg > 0); GGML_ASSERT(((int)S_v % cols_per_wg) == 0); size_t global_work_size[3]; size_t local_work_size[3]; global_work_size[0] = (size_t) H_v * (size_t) wg_size; global_work_size[1] = (size_t) n_seqs; global_work_size[2] = (size_t) S_v / (size_t) cols_per_wg; local_work_size[0] = (size_t) wg_size; local_work_size[1] = 1; local_work_size[2] = 1; backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); } //------------------------------------------------------------------------------ // Op offloading //------------------------------------------------------------------------------ typedef void (*ggml_cl_func_t)(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst); bool ggml_cl_compute_forward(ggml_backend_t backend, struct ggml_tensor * tensor) { ggml_cl_func_t func = nullptr; ggml_tensor * src0 = tensor->src[0]; ggml_tensor * src1 = tensor->src[1]; const bool any_on_device = tensor->extra || (src0 != nullptr && src0->extra) || (src1 != nullptr && src1->extra); switch (tensor->op) { case GGML_OP_GET_ROWS: if (!any_on_device) { return false; } func = ggml_cl_get_rows; break; case GGML_OP_SET_ROWS: if (!any_on_device) { return false; } func = ggml_cl_set_rows; break; case GGML_OP_CPY: if (!any_on_device) { return false; } func = ggml_cl_cpy; break; case GGML_OP_SET: if (!any_on_device) { return false; } func = ggml_cl_set; break; case GGML_OP_DUP: case GGML_OP_CONT: if (!any_on_device) { return false; } func = ggml_cl_dup; break; case GGML_OP_ADD: if (!any_on_device) { return false; } func = ggml_cl_add; break; case GGML_OP_ADD_ID: if (!any_on_device) { return false; } func = ggml_cl_add_id; break; case GGML_OP_MUL: if (!any_on_device) { return false; } func = ggml_cl_mul; break; case GGML_OP_DIV: if (!any_on_device) { return false; } func = ggml_cl_div; break; case GGML_OP_SUB: if (!any_on_device) { return false; } func = ggml_cl_sub; break; case GGML_OP_SQR: if (!any_on_device) { return false; } func = ggml_cl_sqr; break; case GGML_OP_SQRT: if (!any_on_device) { return false; } func = ggml_cl_sqrt; break; case GGML_OP_MEAN: if (!any_on_device) { return false; } func = ggml_cl_mean; break; case GGML_OP_UNARY: switch (ggml_get_unary_op(tensor)) { case GGML_UNARY_OP_GELU: if (!any_on_device) { return false; } func = ggml_cl_gelu; break; case GGML_UNARY_OP_GELU_ERF: if (!any_on_device) { return false; } func = ggml_cl_gelu_erf; break; case GGML_UNARY_OP_GELU_QUICK: if (!any_on_device) { return false; } func = ggml_cl_gelu_quick; break; case GGML_UNARY_OP_SILU: if (!any_on_device) { return false; } func = ggml_cl_silu; break; case GGML_UNARY_OP_RELU: if (!any_on_device) { return false; } func = ggml_cl_relu; break; case GGML_UNARY_OP_SIGMOID: if (!any_on_device) { return false; } func = ggml_cl_sigmoid; break; case GGML_UNARY_OP_TANH: if (!any_on_device) { return false; } func = ggml_cl_tanh; break; case GGML_UNARY_OP_NEG: if (!any_on_device) { return false; } func = ggml_cl_neg; break; case GGML_UNARY_OP_EXP: if (!any_on_device) { return false; } func = ggml_cl_exp; break; case GGML_UNARY_OP_EXPM1: if (!any_on_device) { return false; } func = ggml_cl_expm1; break; case GGML_UNARY_OP_ABS: if (!any_on_device) { return false; } func = ggml_cl_abs; break; case GGML_UNARY_OP_SGN: if (!any_on_device) { return false; } func = ggml_cl_sgn; break; case GGML_UNARY_OP_STEP: if (!any_on_device) { return false; } func = ggml_cl_step; break; case GGML_UNARY_OP_ELU: if (!any_on_device) { return false; } func = ggml_cl_elu; break; case GGML_UNARY_OP_HARDSWISH: if (!any_on_device) { return false; } func = ggml_cl_hardswish; break; case GGML_UNARY_OP_HARDSIGMOID: if (!any_on_device) { return false; } func = ggml_cl_hardsigmoid; break; case GGML_UNARY_OP_FLOOR: if (!any_on_device) { return false; } func = ggml_cl_floor; break; case GGML_UNARY_OP_CEIL: if (!any_on_device) { return false; } func = ggml_cl_ceil; break; case GGML_UNARY_OP_ROUND: if (!any_on_device) { return false; } func = ggml_cl_round; break; case GGML_UNARY_OP_TRUNC: if (!any_on_device) { return false; } func = ggml_cl_trunc; break; case GGML_UNARY_OP_SOFTPLUS: if (!any_on_device) { return false; } func = ggml_cl_softplus; break; default: return false; } break; case GGML_OP_GLU: if (!any_on_device) { return false; } func = ggml_cl_glu; break; case GGML_OP_TRI: if (!any_on_device) { return false; } func = ggml_cl_tri; break; case GGML_OP_FILL: if (!any_on_device) { return false; } func = ggml_cl_fill; break; case GGML_OP_CLAMP: if (!any_on_device) { return false; } func = ggml_cl_clamp; break; case GGML_OP_NORM: if (!any_on_device) { return false; } func = ggml_cl_norm; break; case GGML_OP_RMS_NORM: if (!any_on_device) { return false; } func = ggml_cl_rms_norm; break; case GGML_OP_L2_NORM: if (!any_on_device) { return false; } func = ggml_cl_l2_norm; break; case GGML_OP_GROUP_NORM: if (!any_on_device) { return false; } func = ggml_cl_group_norm; break; case GGML_OP_REPEAT: if (!any_on_device) { return false; } func = ggml_cl_repeat; break; case GGML_OP_PAD: if (!any_on_device) { return false; } ggml_cl_pad(backend, tensor->src[0], tensor); return true; case GGML_OP_UPSCALE: if (!any_on_device) { return false; } ggml_cl_upscale(backend, tensor->src[0], tensor); return true; case GGML_OP_CONV_2D: if (!any_on_device) { return false; } func = ggml_cl_conv_2d; break; case GGML_OP_SSM_CONV: if (!any_on_device) { return false; } func = ggml_cl_ssm_conv; break; case GGML_OP_SSM_SCAN: if (!any_on_device) { return false; } // SSM_SCAN has 7 source tensors, so it cannot use the standard // (src0, src1, dst) func signature. Dispatch directly and return. ggml_cl_ssm_scan(backend, tensor); return true; case GGML_OP_GATED_DELTA_NET: if (!any_on_device) { return false; } // GDN has 6 source tensors, so it cannot use the standard // (src0, src1, dst) func signature. Dispatch directly and return. ggml_cl_gated_delta_net(backend, tensor); return true; case GGML_OP_CONCAT: if (!any_on_device) { return false; } func = ggml_cl_concat; break; case GGML_OP_TIMESTEP_EMBEDDING: if (!any_on_device) { return false; } ggml_cl_timestep_embedding(backend, tensor->src[0], tensor); return true; case GGML_OP_MUL_MAT: if (!any_on_device && !ggml_cl_can_mul_mat(tensor->src[0], tensor->src[1], tensor)) { return false; } func = ggml_cl_mul_mat; break; case GGML_OP_MUL_MAT_ID: if (!any_on_device) { return false; } func = ggml_cl_mul_mat_id; break; case GGML_OP_SCALE: if (!any_on_device) { return false; } func = ggml_cl_scale; break; case GGML_OP_RESHAPE: case GGML_OP_VIEW: case GGML_OP_PERMUTE: case GGML_OP_TRANSPOSE: if (!any_on_device) { return false; } func = ggml_cl_nop; break; case GGML_OP_DIAG: if (!any_on_device) { return false; } func = ggml_cl_diag; break; case GGML_OP_DIAG_MASK_INF: if (!any_on_device) { return false; } func = ggml_cl_diag_mask_inf; break; case GGML_OP_SOFT_MAX: if (!any_on_device) { return false; } func = ggml_cl_soft_max; break; case GGML_OP_ROPE: if (!any_on_device) { return false; } func = ggml_cl_rope; break; case GGML_OP_SOLVE_TRI: if (!any_on_device) { return false; } func = ggml_cl_solve_tri; break; case GGML_OP_IM2COL: if (!any_on_device) { return false; } func = ggml_cl_im2col; break; case GGML_OP_ARGSORT: if (!any_on_device) { return false; } func = ggml_cl_argsort; break; case GGML_OP_SUM_ROWS: if (!any_on_device) { return false; } func = ggml_cl_sum_rows; break; case GGML_OP_CUMSUM: if (!any_on_device) { return false; } func = ggml_cl_cumsum; break; case GGML_OP_FLASH_ATTN_EXT: if (!any_on_device) { return false; } ggml_cl_flash_attn(backend, tensor->src[0], tensor->src[1], tensor); return true; default: return false; } func(backend, tensor->src[0], tensor->src[1], tensor); return true; }