From 1b88cbadd59760e74b1229a1b7bd2e19d97ad349 Mon Sep 17 00:00:00 2001 From: Jhen Date: Sat, 7 Oct 2023 10:52:05 +0800 Subject: [PATCH 1/2] metal : support load default.metallib & reuse code for swift package --- Package.swift | 7 +++++- ggml-metal.m | 70 +++++++++++++++++++++------------------------------ 2 files changed, 35 insertions(+), 42 deletions(-) diff --git a/Package.swift b/Package.swift index 3ee3b2a209f75..5d1400d9b75e7 100644 --- a/Package.swift +++ b/Package.swift @@ -10,7 +10,10 @@ let platforms: [SupportedPlatform]? = [ .tvOS(.v14) ] let exclude: [String] = [] -let additionalSources: [String] = ["ggml-metal.m", "ggml-metal.metal"] +let resources: [Resource] = [ + .process("ggml-metal.metal") +] +let additionalSources: [String] = ["ggml-metal.m"] let additionalSettings: [CSetting] = [ .unsafeFlags(["-fno-objc-arc"]), .define("GGML_SWIFT"), @@ -19,6 +22,7 @@ let additionalSettings: [CSetting] = [ #else let platforms: [SupportedPlatform]? = nil let exclude: [String] = ["ggml-metal.metal"] +let resources: [String] = [] let additionalSources: [String] = [] let additionalSettings: [CSetting] = [] #endif @@ -40,6 +44,7 @@ let package = Package( "ggml-alloc.c", "k_quants.c", ] + additionalSources, + resources: resources, publicHeadersPath: "spm-headers", cSettings: [ .unsafeFlags(["-Wno-shorten-64-to-32"]), diff --git a/ggml-metal.m b/ggml-metal.m index 866fed43442a8..6fbb8e959dfba 100644 --- a/ggml-metal.m +++ b/ggml-metal.m @@ -183,56 +183,44 @@ static void ggml_metal_log(enum ggml_log_level level, const char* format, ...){ ctx->d_queue = dispatch_queue_create("ggml-metal", DISPATCH_QUEUE_CONCURRENT); -#ifdef GGML_SWIFT - // load the default.metallib file + // load library { - NSError * error = nil; - - NSBundle * bundle = [NSBundle bundleForClass:[GGMLMetalClass class]]; - NSString * llamaBundlePath = [bundle pathForResource:@"llama_llama" ofType:@"bundle"]; - NSBundle * llamaBundle = [NSBundle bundleWithPath:llamaBundlePath]; - NSString * libPath = [llamaBundle pathForResource:@"default" ofType:@"metallib"]; - NSURL * libURL = [NSURL fileURLWithPath:libPath]; - - // Load the metallib file into a Metal library - ctx->library = [ctx->device newLibraryWithURL:libURL error:&error]; - - if (error) { - GGML_METAL_LOG_ERROR("%s: error: %s\n", __func__, [[error description] UTF8String]); - return NULL; - } - } + NSBundle * bundle = nil; +#ifdef GGML_SWIFT + bundle = SWIFTPM_MODULE_BUNDLE; #else - UNUSED(msl_library_source); - - // read the source from "ggml-metal.metal" into a string and use newLibraryWithSource - { + bundle = [NSBundle bundleForClass:[GGMLMetalClass class]]; +#endif NSError * error = nil; + NSString * libPath = [bundle pathForResource:@"default" ofType:@"metallib"]; + if (libPath != nil) { + NSURL * libURL = [NSURL fileURLWithPath:libPath]; + GGML_METAL_LOG_INFO("%s: loading '%s'\n", __func__, [libPath UTF8String]); + ctx->library = [ctx->device newLibraryWithURL:libURL error:&error]; + } else { + GGML_METAL_LOG_INFO("%s: default.metallib not found, loading from source\n", __func__); + + NSString * sourcePath = [bundle pathForResource:@"ggml-metal" ofType:@"metal"]; + GGML_METAL_LOG_INFO("%s: loading '%s'\n", __func__, [sourcePath UTF8String]); + NSString * src = [NSString stringWithContentsOfFile:sourcePath encoding:NSUTF8StringEncoding error:&error]; + if (error) { + GGML_METAL_LOG_INFO("%s: error: %s\n", __func__, [[error description] UTF8String]); + return NULL; + } - //NSString * path = [[NSBundle mainBundle] pathForResource:@"../../examples/metal/metal" ofType:@"metal"]; - NSBundle * bundle = [NSBundle bundleForClass:[GGMLMetalClass class]]; - NSString * path = [bundle pathForResource:@"ggml-metal" ofType:@"metal"]; - GGML_METAL_LOG_INFO("%s: loading '%s'\n", __func__, [path UTF8String]); - - NSString * src = [NSString stringWithContentsOfFile:path encoding:NSUTF8StringEncoding error:&error]; - if (error) { - GGML_METAL_LOG_ERROR("%s: error: %s\n", __func__, [[error description] UTF8String]); - return NULL; - } - + MTLCompileOptions* options = nil; #ifdef GGML_QKK_64 - MTLCompileOptions* options = [MTLCompileOptions new]; - options.preprocessorMacros = @{ @"QK_K" : @(64) }; - ctx->library = [ctx->device newLibraryWithSource:src options:options error:&error]; -#else - ctx->library = [ctx->device newLibraryWithSource:src options:nil error:&error]; + options = [MTLCompileOptions new]; + options.preprocessorMacros = @{ @"QK_K" : @(64) }; #endif + ctx->library = [ctx->device newLibraryWithSource:src options:options error:&error]; + } + if (error) { - GGML_METAL_LOG_ERROR("%s: error: %s\n", __func__, [[error description] UTF8String]); + GGML_METAL_LOG_INFO("%s: error: %s\n", __func__, [[error description] UTF8String]); return NULL; } } -#endif // load kernels { @@ -431,7 +419,7 @@ int ggml_metal_if_optimized(struct ggml_metal_context * ctx) { for (int i = 0; i < ctx->n_buffers; ++i) { const int64_t ioffs = (int64_t) t->data - (int64_t) ctx->buffers[i].data; - //metal_printf("ioffs = %10ld, tsize = %10ld, sum = %10ld, ctx->buffers[%d].size = %10ld, name = %s\n", ioffs, tsize, ioffs + tsize, i, ctx->buffers[i].size, ctx->buffers[i].name); + //GGML_METAL_LOG_INFO("ioffs = %10ld, tsize = %10ld, sum = %10ld, ctx->buffers[%d].size = %10ld, name = %s\n", ioffs, tsize, ioffs + tsize, i, ctx->buffers[i].size, ctx->buffers[i].name); if (ioffs >= 0 && ioffs + tsize <= (int64_t) ctx->buffers[i].size) { *offs = (size_t) ioffs; From f134d64bbdbab6b52c3cb6fe7fc19018fb7bca42 Mon Sep 17 00:00:00 2001 From: Jhen Date: Sat, 7 Oct 2023 11:20:11 +0800 Subject: [PATCH 2/2] metal : use SWIFT_PACKAGE def instead of define GGML_SWIFT --- .gitignore | 1 + Package.swift | 3 +-- ggml-metal.m | 8 ++++---- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.gitignore b/.gitignore index 4d5767d228b86..420e0d6d016a2 100644 --- a/.gitignore +++ b/.gitignore @@ -10,6 +10,7 @@ *.gcno *.gcda *.dot +*.metallib .DS_Store .build/ .cache/ diff --git a/Package.swift b/Package.swift index 5d1400d9b75e7..1ea414cc149fd 100644 --- a/Package.swift +++ b/Package.swift @@ -16,13 +16,12 @@ let resources: [Resource] = [ let additionalSources: [String] = ["ggml-metal.m"] let additionalSettings: [CSetting] = [ .unsafeFlags(["-fno-objc-arc"]), - .define("GGML_SWIFT"), .define("GGML_USE_METAL") ] #else let platforms: [SupportedPlatform]? = nil let exclude: [String] = ["ggml-metal.metal"] -let resources: [String] = [] +let resources: [Resource] = [] let additionalSources: [String] = [] let additionalSettings: [CSetting] = [] #endif diff --git a/ggml-metal.m b/ggml-metal.m index 6fbb8e959dfba..1bf6b965bb82f 100644 --- a/ggml-metal.m +++ b/ggml-metal.m @@ -186,7 +186,7 @@ static void ggml_metal_log(enum ggml_log_level level, const char* format, ...){ // load library { NSBundle * bundle = nil; -#ifdef GGML_SWIFT +#ifdef SWIFT_PACKAGE bundle = SWIFTPM_MODULE_BUNDLE; #else bundle = [NSBundle bundleForClass:[GGMLMetalClass class]]; @@ -204,7 +204,7 @@ static void ggml_metal_log(enum ggml_log_level level, const char* format, ...){ GGML_METAL_LOG_INFO("%s: loading '%s'\n", __func__, [sourcePath UTF8String]); NSString * src = [NSString stringWithContentsOfFile:sourcePath encoding:NSUTF8StringEncoding error:&error]; if (error) { - GGML_METAL_LOG_INFO("%s: error: %s\n", __func__, [[error description] UTF8String]); + GGML_METAL_LOG_ERROR("%s: error: %s\n", __func__, [[error description] UTF8String]); return NULL; } @@ -215,9 +215,9 @@ static void ggml_metal_log(enum ggml_log_level level, const char* format, ...){ #endif ctx->library = [ctx->device newLibraryWithSource:src options:options error:&error]; } - + if (error) { - GGML_METAL_LOG_INFO("%s: error: %s\n", __func__, [[error description] UTF8String]); + GGML_METAL_LOG_ERROR("%s: error: %s\n", __func__, [[error description] UTF8String]); return NULL; } }