From e6b3b4a8077c94c434cf7529d621778e9c2c2f6b Mon Sep 17 00:00:00 2001 From: Rinne Date: Wed, 24 Apr 2024 00:20:15 +0800 Subject: [PATCH 1/7] feat: support auto-download for native libraries. --- LLama.Examples/Program.cs | 25 +- LLama/Abstractions/INativeLibrary.cs | 30 + .../INativeLibrarySelectingPolicy.cs | 24 + LLama/LLamaSharp.csproj | 5 +- .../DefaultNativeLibrarySelectingPolicy.cs | 69 ++ LLama/Native/Load/NativeLibraryConfig.cs | 664 ++++++++++++++++++ .../Load/NativeLibraryDownloadManager.cs | 285 ++++++++ LLama/Native/Load/NativeLibraryFromPath.cs | 31 + LLama/Native/Load/NativeLibraryMetadata.cs | 43 ++ LLama/Native/Load/NativeLibraryUtils.cs | 182 +++++ LLama/Native/Load/NativeLibraryWithAvx.cs | 75 ++ .../Native/Load/NativeLibraryWithCpuOrMac.cs | 77 ++ LLama/Native/Load/NativeLibraryWithCuda.cs | 91 +++ LLama/Native/Load/SystemInfo.cs | 129 ++++ LLama/Native/NativeApi.Load.cs | 358 +--------- LLama/Native/NativeApi.cs | 3 - LLama/Native/NativeLibraryConfig.cs | 332 --------- LLama/Native/NativeLogConfig.cs | 2 +- 18 files changed, 1735 insertions(+), 690 deletions(-) create mode 100644 LLama/Abstractions/INativeLibrary.cs create mode 100644 LLama/Abstractions/INativeLibrarySelectingPolicy.cs create mode 100644 LLama/Native/Load/DefaultNativeLibrarySelectingPolicy.cs create mode 100644 LLama/Native/Load/NativeLibraryConfig.cs create mode 100644 LLama/Native/Load/NativeLibraryDownloadManager.cs create mode 100644 LLama/Native/Load/NativeLibraryFromPath.cs create mode 100644 LLama/Native/Load/NativeLibraryMetadata.cs create mode 100644 LLama/Native/Load/NativeLibraryUtils.cs create mode 100644 LLama/Native/Load/NativeLibraryWithAvx.cs create mode 100644 LLama/Native/Load/NativeLibraryWithCpuOrMac.cs create mode 100644 LLama/Native/Load/NativeLibraryWithCuda.cs create mode 100644 LLama/Native/Load/SystemInfo.cs delete mode 100644 LLama/Native/NativeLibraryConfig.cs diff --git a/LLama.Examples/Program.cs b/LLama.Examples/Program.cs index b24ef406b..cee9540be 100644 --- a/LLama.Examples/Program.cs +++ b/LLama.Examples/Program.cs @@ -1,5 +1,6 @@ using LLama.Native; using Spectre.Console; +using System.Runtime.InteropServices; AnsiConsole.MarkupLineInterpolated( $""" @@ -16,23 +17,23 @@ __ __ ____ __ """); -// Configure native library to use. This must be done before any other llama.cpp methods are called! -NativeLibraryConfig - .Instance - .WithCuda(); - // Configure logging. Change this to `true` to see log messages from llama.cpp var showLLamaCppLogs = false; NativeLibraryConfig - .Instance + .All .WithLogCallback((level, message) => - { - if (showLLamaCppLogs) - Console.WriteLine($"[llama {level}]: {message.TrimEnd('\n')}"); - }); + { + if (showLLamaCppLogs) + Console.WriteLine($"[llama {level}]: {message.TrimEnd('\n')}"); + }); + +// Configure native library to use. This must be done before any other llama.cpp methods are called! +NativeLibraryConfig + .All + .WithCuda() + .WithAutoDownload().DryRun(); // Calling this method forces loading to occur now. NativeApi.llama_empty_call(); -await ExampleRunner.Run(); - +await ExampleRunner.Run(); \ No newline at end of file diff --git a/LLama/Abstractions/INativeLibrary.cs b/LLama/Abstractions/INativeLibrary.cs new file mode 100644 index 000000000..64ba182e0 --- /dev/null +++ b/LLama/Abstractions/INativeLibrary.cs @@ -0,0 +1,30 @@ +using LLama.Native; +using System; +using System.Collections.Generic; +using System.Text; + +namespace LLama.Abstractions +{ + /// + /// Descriptor of a native library. + /// + public interface INativeLibrary + { + /// + /// Metadata of this library. + /// + NativeLibraryMetadata? Metadata { get; } + + /// + /// Prepare the native library file and returns the local path of it. + /// If it's a relative path, LLamaSharp will search the path in the search directies you set. + /// + /// The system information of the current machine. + /// Whether it's allowed to download from remote. + /// The log callback. + /// + /// The relative paths of the library. You could return multiple paths to try them one by one. If no file is available, please return an empty array. + /// + IEnumerable Prepare(SystemInfo systemInfo, bool fromRemote = false, NativeLogConfig.LLamaLogCallback? logCallback = null); + } +} diff --git a/LLama/Abstractions/INativeLibrarySelectingPolicy.cs b/LLama/Abstractions/INativeLibrarySelectingPolicy.cs new file mode 100644 index 000000000..f57ae6d1a --- /dev/null +++ b/LLama/Abstractions/INativeLibrarySelectingPolicy.cs @@ -0,0 +1,24 @@ +using LLama.Native; +using System; +using System.Collections.Generic; +using System.Text; + +namespace LLama.Abstractions +{ +#if NET6_0_OR_GREATER + /// + /// Decides the selected native library that should be loaded according to the configurations. + /// + public interface INativeLibrarySelectingPolicy + { + /// + /// Select the native library. + /// + /// + /// The system information of the current machine. + /// The log callback. + /// The information of the selected native library files, in order by priority from the beginning to the end. + IEnumerable Select(NativeLibraryConfig.Description description, SystemInfo systemInfo, NativeLogConfig.LLamaLogCallback? logCallback = null); + } +#endif +} diff --git a/LLama/LLamaSharp.csproj b/LLama/LLamaSharp.csproj index 3947b7c31..8277ce9eb 100644 --- a/LLama/LLamaSharp.csproj +++ b/LLama/LLamaSharp.csproj @@ -1,9 +1,9 @@ - netstandard2.0;net6.0;net7.0;net8.0 + net6.0;net7.0;net8.0;netstandard2.0 LLama enable - 10 + 12 AnyCPU;x64;Arm64 True @@ -49,6 +49,7 @@ + diff --git a/LLama/Native/Load/DefaultNativeLibrarySelectingPolicy.cs b/LLama/Native/Load/DefaultNativeLibrarySelectingPolicy.cs new file mode 100644 index 000000000..42433cecb --- /dev/null +++ b/LLama/Native/Load/DefaultNativeLibrarySelectingPolicy.cs @@ -0,0 +1,69 @@ +using LLama.Abstractions; +using System.Collections.Generic; +using System.Runtime.InteropServices; + +namespace LLama.Native +{ +#if NET6_0_OR_GREATER + /// + public class DefaultNativeLibrarySelectingPolicy: INativeLibrarySelectingPolicy + { + /// + public IEnumerable Select(NativeLibraryConfig.Description description, SystemInfo systemInfo, NativeLogConfig.LLamaLogCallback? logCallback) + { + List results = new(); + + // Show the configuration we're working with + Log(description.ToString(), LLamaLogLevel.Info, logCallback); + + // If a specific path is requested, only use it, no fall back. + if (!string.IsNullOrEmpty(description.Path)) + { + yield return new NativeLibraryFromPath(description.Path); + } + else + { + if (description.UseCuda) + { + yield return new NativeLibraryWithCuda(systemInfo.CudaMajorVersion, description.Library, description.SkipCheck, description.DownloadSettings); + } + + if(!description.UseCuda || description.AllowFallback) + { + if (description.AllowFallback) + { + // Try all of the AVX levels we can support. + if (description.AvxLevel >= AvxLevel.Avx512) + yield return new NativeLibraryWithAvx(description.Library, AvxLevel.Avx512, description.SkipCheck, description.DownloadSettings); + + if (description.AvxLevel >= AvxLevel.Avx2) + yield return new NativeLibraryWithAvx(description.Library, AvxLevel.Avx2, description.SkipCheck, description.DownloadSettings); + + if (description.AvxLevel >= AvxLevel.Avx) + yield return new NativeLibraryWithAvx(description.Library, AvxLevel.Avx, description.SkipCheck, description.DownloadSettings); + + yield return new NativeLibraryWithAvx(description.Library, AvxLevel.None, description.SkipCheck, description.DownloadSettings); + } + else + { + yield return new NativeLibraryWithAvx(description.Library, description.AvxLevel, description.SkipCheck, description.DownloadSettings); + } + } + + if(systemInfo.OSPlatform == OSPlatform.OSX || description.AllowFallback) + { + yield return new NativeLibraryWithCpuOrMac(description.Library, description.SkipCheck, description.DownloadSettings); + } + } + } + + private void Log(string message, LLamaLogLevel level, NativeLogConfig.LLamaLogCallback? logCallback) + { + if (!message.EndsWith("\n")) + message += "\n"; + + logCallback?.Invoke(level, message); + } + } +#endif +} diff --git a/LLama/Native/Load/NativeLibraryConfig.cs b/LLama/Native/Load/NativeLibraryConfig.cs new file mode 100644 index 000000000..6f77927d5 --- /dev/null +++ b/LLama/Native/Load/NativeLibraryConfig.cs @@ -0,0 +1,664 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using LLama.Abstractions; +using Microsoft.Extensions.Logging; + +namespace LLama.Native +{ +#if NET6_0_OR_GREATER + /// + /// Allows configuration of the native llama.cpp libraries to load and use. + /// All configuration must be done before using **any** other LLamaSharp methods! + /// + public sealed partial class NativeLibraryConfig + { + private string? _libraryPath; + + private bool _useCuda = true; + private AvxLevel _avxLevel; + private bool _allowFallback = true; + private bool _skipCheck = false; + private bool _allowAutoDownload = false; + private NativeLibraryDownloadSettings _downloadSettings = NativeLibraryDownloadSettings.Create(); + + /// + /// search directory -> priority level, 0 is the lowest. + /// + private readonly List _searchDirectories = new List(); + + internal INativeLibrarySelectingPolicy SelectingPolicy { get; private set; } = new DefaultNativeLibrarySelectingPolicy(); + + internal bool AllowAutoDownload => _allowAutoDownload; + + #region configurators + /// + /// Load a specified native library as backend for LLamaSharp. + /// When this method is called, all the other configurations will be ignored. + /// + /// The full path to the native library to load. + /// Thrown if `LibraryHasLoaded` is true. + public NativeLibraryConfig WithLibrary(string? libraryPath) + { + ThrowIfLoaded(); + + _libraryPath = libraryPath; + return this; + } + + /// + /// Configure whether to use cuda backend if possible. Default is true. + /// + /// + /// + /// Thrown if `LibraryHasLoaded` is true. + public NativeLibraryConfig WithCuda(bool enable = true) + { + ThrowIfLoaded(); + + _useCuda = enable; + return this; + } + + /// + /// Configure the prefferred avx support level of the backend. + /// Default value is detected automatically due to your operating system. + /// + /// + /// + /// Thrown if `LibraryHasLoaded` is true. + public NativeLibraryConfig WithAvx(AvxLevel level) + { + ThrowIfLoaded(); + + _avxLevel = level; + return this; + } + + /// + /// Configure whether to allow fallback when there's no match for preferred settings. Default is true. + /// + /// + /// + /// Thrown if `LibraryHasLoaded` is true. + public NativeLibraryConfig WithAutoFallback(bool enable = true) + { + ThrowIfLoaded(); + + _allowFallback = enable; + return this; + } + + /// + /// Whether to skip the check when you don't allow fallback. This option + /// may be useful under some complex conditions. For example, you're sure + /// you have your cublas configured but LLamaSharp take it as invalid by mistake. Default is false; + /// + /// + /// + /// Thrown if `LibraryHasLoaded` is true. + public NativeLibraryConfig SkipCheck(bool enable = true) + { + ThrowIfLoaded(); + + _skipCheck = enable; + return this; + } + + /// + /// Add self-defined search directories. Note that the file stucture of the added + /// directories must be the same as the default directory. Besides, the directory + /// won't be used recursively. + /// + /// + /// + public NativeLibraryConfig WithSearchDirectories(IEnumerable directories) + { + ThrowIfLoaded(); + + _searchDirectories.AddRange(directories); + return this; + } + + /// + /// Add self-defined search directories. Note that the file stucture of the added + /// directories must be the same as the default directory. Besides, the directory + /// won't be used recursively. + /// + /// + /// + public NativeLibraryConfig WithSearchDirectory(string directory) + { + ThrowIfLoaded(); + + _searchDirectories.Add(directory); + return this; + } + + /// + /// Set whether to download the best-matched native library file automatically if there's no backend or specified file to load. + /// You could add a setting here to customize the behavior of the download. + /// + /// If auto-download is enabled, please call after you have finished setting your configurations. + /// + /// + /// + /// + public NativeLibraryConfig WithAutoDownload(bool enable = true, NativeLibraryDownloadSettings? settings = null) + { + ThrowIfLoaded(); + + _allowAutoDownload = enable; + if (settings is not null) + _downloadSettings = settings; + return this; + } + + /// + /// Set the policy which decides how to select the desired native libraries and order them by priority. + /// By default we use . + /// + /// + /// + public NativeLibraryConfig WithSelectingPolicy(INativeLibrarySelectingPolicy policy) + { + ThrowIfLoaded(); + + SelectingPolicy = policy; + return this; + } + + #endregion + + internal Description CheckAndGatherDescription() + { + if (_allowFallback && _skipCheck) + throw new ArgumentException("Cannot skip the check when fallback is allowed."); + + var path = _libraryPath; + + // Don't modify and pass the original object to `Description`, create a new one instead. + // Also, we need to set the default local directory if the user does not. + var defaultLocalDir = NativeLibraryDownloadSettings.GetDefaultLocalDir(GetCommitHash(_downloadSettings.Tag)); + var downloadSettings = NativeLibraryDownloadSettings.Create() + .WithEndpoint(_downloadSettings.Endpoint).WithEndpointFallbacks(_downloadSettings.EndpointFallbacks ?? []) + .WithRepoId(_downloadSettings.RepoId).WithToken(_downloadSettings.Token).WithTag(_downloadSettings.Tag) + .WithTimeout(_downloadSettings.Timeout).WithLocalDir(_downloadSettings.LocalDir ?? defaultLocalDir); + + return new Description( + path, + NativeLibraryName, + _useCuda, + _avxLevel, + _allowFallback, + _skipCheck, + _searchDirectories.Concat(new[] { "./" }).ToArray(), + _allowAutoDownload, + downloadSettings + ); + } + + internal static string AvxLevelToString(AvxLevel level) + { + return level switch + { + AvxLevel.None => string.Empty, + AvxLevel.Avx => "avx", + AvxLevel.Avx2 => "avx2", + AvxLevel.Avx512 => "avx512", + _ => throw new ArgumentException($"Unknown AvxLevel '{level}'") + }; + } + + /// + /// Private constructor prevents new instances of this class being created + /// + private NativeLibraryConfig(NativeLibraryName nativeLibraryName) + { + NativeLibraryName = nativeLibraryName; + // This value should be changed when we're going to publish new release. (any better approach?) + _downloadSettings = new NativeLibraryDownloadSettings().WithTag(GetCommitHash("master")); + + // Automatically detect the highest supported AVX level + if (System.Runtime.Intrinsics.X86.Avx.IsSupported) + _avxLevel = AvxLevel.Avx; + if (System.Runtime.Intrinsics.X86.Avx2.IsSupported) + _avxLevel = AvxLevel.Avx2; + + if (CheckAVX512()) + _avxLevel = AvxLevel.Avx512; + } + + private static bool CheckAVX512() + { + if (!System.Runtime.Intrinsics.X86.X86Base.IsSupported) + return false; + + // ReSharper disable UnusedVariable (ebx is used when < NET8) + var (_, ebx, ecx, _) = System.Runtime.Intrinsics.X86.X86Base.CpuId(7, 0); + // ReSharper restore UnusedVariable + + var vnni = (ecx & 0b_1000_0000_0000) != 0; + +#if NET8_0_OR_GREATER + var f = System.Runtime.Intrinsics.X86.Avx512F.IsSupported; + var bw = System.Runtime.Intrinsics.X86.Avx512BW.IsSupported; + var vbmi = System.Runtime.Intrinsics.X86.Avx512Vbmi.IsSupported; +#else + var f = (ebx & (1 << 16)) != 0; + var bw = (ebx & (1 << 30)) != 0; + var vbmi = (ecx & 0b_0000_0000_0010) != 0; +#endif + + return vnni && vbmi && bw && f; + } + + /// + /// The description of the native library configurations that's already specified. + /// + /// + /// + /// + /// + /// + /// + /// + /// + /// + public record Description(string? Path, NativeLibraryName Library, bool UseCuda, AvxLevel AvxLevel, bool AllowFallback, bool SkipCheck, + string[] SearchDirectories, bool AllowAutoDownload, NativeLibraryDownloadSettings DownloadSettings) + { + /// + public override string ToString() + { + string avxLevelString = AvxLevel switch + { + AvxLevel.None => "NoAVX", + AvxLevel.Avx => "AVX", + AvxLevel.Avx2 => "AVX2", + AvxLevel.Avx512 => "AVX512", + _ => "Unknown" + }; + + string searchDirectoriesString = "{ " + string.Join(", ", SearchDirectories) + " }"; + + return $"NativeLibraryConfig Description:\n" + + $"- LibraryName: {Library}\n" + + $"- Path: '{Path}'\n" + + $"- PreferCuda: {UseCuda}\n" + + $"- PreferredAvxLevel: {avxLevelString}\n" + + $"- AllowFallback: {AllowFallback}\n" + + $"- SkipCheck: {SkipCheck}\n" + + $"- SearchDirectories and Priorities: {searchDirectoriesString}" + + $"- AllowAutoDownload: {AllowAutoDownload}\n" + + $"- DownloadSettings: {DownloadSettings}\n"; + } + } + } +#endif + + public sealed partial class NativeLibraryConfig + { + /// + /// Set configurations for all the native libraries, including LLama and LLava + /// + [Obsolete("Please use NativeLibraryConfig.All instead, or set configurations for NativeLibraryConfig.LLama and NativeLibraryConfig.LLavaShared respectively.")] + public static NativeLibraryConfigContainer Instance { get; } + + /// + /// Set configurations for all the native libraries, including LLama and LLava + /// + public static NativeLibraryConfigContainer All { get; } + + /// + /// Configuration for LLama native library + /// + public static NativeLibraryConfig LLama { get; } + + /// + /// Configuration for LLava native library + /// + public static NativeLibraryConfig LLavaShared { get; } + + /// + /// A dictionary mapping from version to corresponding llama.cpp commit hash. + /// The version should be formatted int `[major].[minor].[patch]`. But there's an exceptance that you can + /// use `master` as a version to get the llama.cpp commit hash from the master branch. + /// + public static Dictionary VersionMap { get; } = new Dictionary() + // This value should be changed when we're going to publish new release. (any better approach?) + { + {"master", "f7001c"} + }; + + internal static string GetCommitHash(string version) + { + if(VersionMap.TryGetValue(version, out var hash)) + { + return hash; + } + else + { + return version; + } + } + + static NativeLibraryConfig() + { + LLama = new(NativeLibraryName.Llama); + LLavaShared = new(NativeLibraryName.LlavaShared); + All = new(LLama, LLavaShared); + Instance = All; + } + +#if NETSTANDARD2_0 + private NativeLibraryConfig(NativeLibraryName nativeLibraryName) + { + NativeLibraryName = nativeLibraryName; + } +#endif + + /// + /// Check if the native library has already been loaded. Configuration cannot be modified if this is true. + /// + public bool LibraryHasLoaded { get; internal set; } + + /// + /// Whether has been called. + /// + internal bool HasCalledDryRun { get; private set; } = false; + + internal NativeLibraryName NativeLibraryName { get; } + + internal NativeLogConfig.LLamaLogCallback? LogCallback { get; private set; } = null; + + private void ThrowIfLoaded() + { + if (LibraryHasLoaded) + throw new InvalidOperationException("NativeLibraryConfig must be configured before using **any** other LLamaSharp methods!"); + } + + /// + /// Set the log callback that will be used for all llama.cpp log messages + /// + /// + /// + public NativeLibraryConfig WithLogCallback(NativeLogConfig.LLamaLogCallback? callback) + { + ThrowIfLoaded(); + + LogCallback = callback; + return this; + } + + /// + /// Set the log callback that will be used for all llama.cpp log messages + /// + /// + /// + public NativeLibraryConfig WithLogCallback(ILogger? logger) + { + ThrowIfLoaded(); + + // Redirect to llama_log_set. This will wrap the logger in a delegate and bind that as the log callback instead. + NativeLogConfig.llama_log_set(logger); + + return this; + } + + /// + /// Try to load the native library with the current configurations, + /// but do not actually set it to . + /// + /// You can still modify the configuration after this calling but only before any call from . + /// + /// Whether the running is successful. + public bool DryRun() + { + LogCallback?.Invoke(LLamaLogLevel.Debug, $"Beginning dry run for {this.NativeLibraryName.GetLibraryName()}..."); + HasCalledDryRun = true; + return NativeLibraryUtils.TryLoadLibrary(this) != IntPtr.Zero; + } + } + + /// + /// A class to set same configurations to multiple libraries at the same time. + /// + public sealed partial class NativeLibraryConfigContainer + { + private NativeLibraryConfig[] _configs; + + internal NativeLibraryConfigContainer(params NativeLibraryConfig[] configs) + { + _configs = configs; + } + + #region configurators + +#if NET6_0_OR_GREATER + /// + /// Load a specified native library as backend for LLamaSharp. + /// When this method is called, all the other configurations will be ignored. + /// + /// The full path to the llama library to load. + /// The full path to the llava library to load. + /// Thrown if `LibraryHasLoaded` is true. + public NativeLibraryConfigContainer WithLibrary(string? llamaPath, string? llavaPath) + { + foreach(var config in _configs) + { + if(config.NativeLibraryName == NativeLibraryName.Llama && llamaPath is not null) + { + config.WithLibrary(llamaPath); + } + if(config.NativeLibraryName == NativeLibraryName.LlavaShared && llavaPath is not null) + { + config.WithLibrary(llavaPath); + } + } + + return this; + } + + /// + /// Configure whether to use cuda backend if possible. + /// + /// + /// + /// Thrown if `LibraryHasLoaded` is true. + public NativeLibraryConfigContainer WithCuda(bool enable = true) + { + foreach(var config in _configs) + { + config.WithCuda(enable); + } + return this; + } + + /// + /// Configure the prefferred avx support level of the backend. + /// + /// + /// + /// Thrown if `LibraryHasLoaded` is true. + public NativeLibraryConfigContainer WithAvx(AvxLevel level) + { + foreach (var config in _configs) + { + config.WithAvx(level); + } + return this; + } + + /// + /// Configure whether to allow fallback when there's no match for preferred settings. + /// + /// + /// + /// Thrown if `LibraryHasLoaded` is true. + public NativeLibraryConfigContainer WithAutoFallback(bool enable = true) + { + foreach (var config in _configs) + { + config.WithAutoFallback(enable); + } + return this; + } + + /// + /// Whether to skip the check when you don't allow fallback. This option + /// may be useful under some complex conditions. For example, you're sure + /// you have your cublas configured but LLamaSharp take it as invalid by mistake. + /// + /// + /// + /// Thrown if `LibraryHasLoaded` is true. + public NativeLibraryConfigContainer SkipCheck(bool enable = true) + { + foreach (var config in _configs) + { + config.SkipCheck(enable); + } + return this; + } + + /// + /// Add self-defined search directories. Note that the file stucture of the added + /// directories must be the same as the default directory. Besides, the directory + /// won't be used recursively. + /// + /// + /// + public NativeLibraryConfigContainer WithSearchDirectories(IEnumerable directories) + { + foreach (var config in _configs) + { + config.WithSearchDirectories(directories); + } + return this; + } + + /// + /// Add self-defined search directories. Note that the file stucture of the added + /// directories must be the same as the default directory. Besides, the directory + /// won't be used recursively. + /// + /// + /// + public NativeLibraryConfigContainer WithSearchDirectory(string directory) + { + foreach (var config in _configs) + { + config.WithSearchDirectory(directory); + } + return this; + } + + /// + /// Set whether to download the best-matched native library file automatically if there's no backend or specified file to load. + /// You could add a setting here to customize the behavior of the download. + /// + /// If auto-download is enabled, please call after you have finished setting your configurations. + /// + /// + /// + /// + public NativeLibraryConfigContainer WithAutoDownload(bool enable = true, NativeLibraryDownloadSettings? settings = null) + { + foreach (var config in _configs) + { + config.WithAutoDownload(enable, settings); + } + return this; + } + + /// + /// Set the policy which decides how to select the desired native libraries and order them by priority. + /// By default we use . + /// + /// + /// + public NativeLibraryConfigContainer WithSelectingPolicy(INativeLibrarySelectingPolicy policy) + { + foreach (var config in _configs) + { + config.WithSelectingPolicy(policy); + } + return this; + } +#endif + + /// + /// Set the log callback that will be used for all llama.cpp log messages + /// + /// + /// + public NativeLibraryConfigContainer WithLogCallback(NativeLogConfig.LLamaLogCallback? callback) + { + foreach (var config in _configs) + { + config.WithLogCallback(callback); + } + return this; + } + + /// + /// Set the log callback that will be used for all llama.cpp log messages + /// + /// + /// + public NativeLibraryConfigContainer WithLogCallback(ILogger? logger) + { + foreach (var config in _configs) + { + config.WithLogCallback(logger); + } + return this; + } + + #endregion + + /// + /// Try to load the native library with the current configurations, + /// but do not actually set it to . + /// + /// You can still modify the configuration after this calling but only before any call from . + /// + /// Whether the running is successful. + public bool DryRun() + { + return _configs.All(config => config.DryRun()); + } + } + + /// + /// The name of the native library + /// + public enum NativeLibraryName + { + /// + /// The native library compiled from llama.cpp. + /// + Llama, + /// + /// The native library compiled from the LLaVA example of llama.cpp. + /// + LlavaShared + } + + internal static class LibraryNameExtensions + { + public static string GetLibraryName(this NativeLibraryName name) + { + switch (name) + { + case NativeLibraryName.Llama: + return NativeApi.libraryName; + case NativeLibraryName.LlavaShared: + return NativeApi.llavaLibraryName; + default: + throw new ArgumentOutOfRangeException(nameof(name), name, null); + } + } + } +} diff --git a/LLama/Native/Load/NativeLibraryDownloadManager.cs b/LLama/Native/Load/NativeLibraryDownloadManager.cs new file mode 100644 index 000000000..b9a136991 --- /dev/null +++ b/LLama/Native/Load/NativeLibraryDownloadManager.cs @@ -0,0 +1,285 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Threading.Tasks; +using Huggingface; + +namespace LLama.Native +{ + internal class NativeLibraryDownloadManager + { + /// + /// Download the library file + /// + /// + /// + /// + /// The local path of the file if successful otherwise null. + public static async Task DownloadLibraryFile(NativeLibraryDownloadSettings settings, string remoteFilePath, NativeLogConfig.LLamaLogCallback? logCallback = null) + { + HFGlobalConfig.DefaultDownloadTimeout = settings.Timeout; + + HashSet endpointSet = new([settings.Endpoint]); + if (settings.EndpointFallbacks is not null) + { + foreach (var endpoint in settings.EndpointFallbacks) + { + endpointSet.Add(endpoint); + } + } + var endpoints = endpointSet.ToArray(); + + Dictionary exceptionMap = new(); + foreach(var endpoint in endpoints) + { + logCallback?.Invoke(LLamaLogLevel.Debug, $"Downloading the native library file '{remoteFilePath}' from {endpoint} with repo = {settings.RepoId}, tag = {settings.Tag}"); + var path = await HFDownloader.DownloadFileAsync(settings.RepoId, remoteFilePath, revision: settings.Tag, cacheDir: settings.CacheDir, + localDir: settings.LocalDir, token: settings.Token, endpoint: endpoint); + if (path is not null) + { + logCallback?.Invoke(LLamaLogLevel.Debug, $"Successfully downloaded the native library file to {path}"); + return path; + } + else + { + logCallback?.Invoke(LLamaLogLevel.Warning, "The download failed without an explicit error, please check your configuration or report an issue to LLamaSharp."); + } + //try + //{ + // logCallback?.Invoke(LLamaLogLevel.Debug, $"Downloading the native library file '{remoteFilePath}' from {endpoint} with repo = {settings.RepoId}, tag = {settings.Tag}"); + // var path = await HFDownloader.DownloadFileAsync(settings.RepoId, remoteFilePath, revision: settings.Tag, cacheDir: settings.CacheDir, + // localDir: settings.LocalDir, token: settings.Token, endpoint: endpoint); + // if(path is not null) + // { + // logCallback?.Invoke(LLamaLogLevel.Debug, $"Successfully downloaded the native library file to {path}"); + // return path; + // } + // else + // { + // logCallback?.Invoke(LLamaLogLevel.Warning, "The download failed without an explicit error, please check your configuration or report an issue to LLamaSharp."); + // } + //} + //catch(Exception ex) + //{ + // logCallback?.Invoke(LLamaLogLevel.Warning, $"An exception was thrown when downloading the native library file from {endpoint}: {ex.Message}"); + //} + } + + // means that the download finally fails. + return null; + } + } + + /// + /// Settings for downloading the native library. + /// + public class NativeLibraryDownloadSettings + { + /// + /// The endpoint to download from, by default the official site of HuggingFace. + /// + public string Endpoint { get; private set; } = "https://huggingface.co"; + + /// + /// Endpoints to fallback to if downloading with the main endpoint fails. + /// + /// Generally this is an option for those countries or regions where the main endpoint is blocked. + /// You should not put too many endpoints here, as it will slow down the downloading process. + /// + public string[]? EndpointFallbacks { get; private set; } = null; + + /// + /// The version of the library to download. Please use LLamaSharp version in format `[major].[minor].[patch]` as tag + /// or go to https://huggingface.co/AsakusaRinne/LLamaSharpNative + /// to see all available tags, or use your own repo and tags. + /// + public string Tag { get; private set; } = string.Empty; + + /// + /// The repo id to download the native library files. + /// + public string RepoId { get; private set; } = "AsakusaRinne/LLamaSharpNative"; + + /// + /// The directory to cache the downloaded files. If you only want to make the downloaded files appear in a directory, + /// regardless of whether the file will have a copy in another place, please set instead. + /// + public string CacheDir { get; private set; } + + /// + /// If provided, the downloaded file will be placed under this directory, + /// either as a symlink (default) or a regular file. + /// + public string? LocalDir { get; private set; } = null; + + /// + /// If you are using your own private repo as remote source, you could set the token to get the access. + /// + public string? Token { get; private set; } = null; + + /// + /// The timeout (second) of the native library file download. + /// + public int Timeout { get; private set; } = 10; + + /// + /// Extra search directories. They will only be used when finding files downloaded from remote. + /// Generally it will be useful when you wnat to replace the downloading process with your custom implementation. + /// If you are not sure how it works, please leave it empty. + /// + public string[]? ExtraSearchDirectories { get;private set; } = null; + + internal NativeLibraryDownloadSettings() + { + var home = Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.UserProfile), ".cache"); + CacheDir = Path.Combine(home, "llama_sharp"); + } + + internal static string GetDefaultLocalDir(string tag) + { + var home = Environment.GetFolderPath(Environment.SpecialFolder.UserProfile); + return Path.Combine(home, ".llama_sharp", tag); + } + + /// + /// Create a with default settings. + /// + /// + public static NativeLibraryDownloadSettings Create() + { + return new NativeLibraryDownloadSettings(); + } + + /// + /// Set the default endpoint to download file from. + /// + /// + /// + public NativeLibraryDownloadSettings WithEndpoint(string endpoint) + { + Endpoint = endpoint; + return this; + } + + /// + /// Set the endpoints to try when the download fails with the default endpoint. + /// + /// + /// + public NativeLibraryDownloadSettings WithEndpointFallbacks(params string[] endpoints) + { + EndpointFallbacks = endpoints; + return this; + } + + /// + /// Set the + /// + /// + /// + public NativeLibraryDownloadSettings WithTag(string tag) + { + Tag = tag; + return this; + } + + /// + /// Set the + /// + /// + /// + public NativeLibraryDownloadSettings WithRepoId(string repoId) + { + RepoId = repoId; + return this; + } + + /// + /// Set the . If you only want to make the downloaded files appear in a directory, + /// regardless of whether the file may have a copy in another place, please use instead. + /// + /// + /// + public NativeLibraryDownloadSettings WithCacheDir(string cacheDir) + { + CacheDir = cacheDir; + return this; + } + + /// + /// Set the + /// + /// + /// + public NativeLibraryDownloadSettings WithLocalDir(string localDir) + { + LocalDir = localDir; + return this; + } + + /// + /// Set the + /// + /// + /// + public NativeLibraryDownloadSettings WithToken(string token) + { + Token = token; + return this; + } + + /// + /// Set the + /// + /// + /// + public NativeLibraryDownloadSettings WithTimeout(int timeout) + { + Timeout = timeout; + return this; + } + + /// + /// Set . They will only be used when finding files downloaded from remote. + /// Generally it will be useful when you wnat to replace the downloading process with your custom implementation. + /// If you are not sure how it works, please ignore this method. + /// + /// + /// + public NativeLibraryDownloadSettings WithExtraSearchDirectories(string[] directories) + { + ExtraSearchDirectories = directories; + return this; + } + + /// + public override string ToString() + { + // Token should be hidden when printing it. + string hiddenToken = ""; + if(Token is not null) + { + if(Token.Length <= 10) + { + hiddenToken = new string('*', Token.Length - 1) + Token.Last(); + } + else + { + hiddenToken += Token.Substring(0, 2); + hiddenToken += new string('*', Token.Length - 3); + hiddenToken += Token.Last(); + } + } + + return $"(Endpoint = {Endpoint}, " + + $"EndpointFallbacks = {string.Join(", ", EndpointFallbacks ?? new string[0])}, " + + $"Tag = {Tag}, " + + $"RepoId = {RepoId}, " + + $"CacheDir = {CacheDir}, " + + $"LocalDir = {LocalDir}, " + + $"Token = {hiddenToken}, " + + $"Timeout = {Timeout}s)"; + } + } +} diff --git a/LLama/Native/Load/NativeLibraryFromPath.cs b/LLama/Native/Load/NativeLibraryFromPath.cs new file mode 100644 index 000000000..45c7e48ec --- /dev/null +++ b/LLama/Native/Load/NativeLibraryFromPath.cs @@ -0,0 +1,31 @@ +using LLama.Abstractions; +using System.Collections.Generic; + +namespace LLama.Native +{ + /// + /// A native library specified with a lcoal file path. + /// + public class NativeLibraryFromPath: INativeLibrary + { + private string _path; + + /// + public NativeLibraryMetadata? Metadata => null; + + /// + /// + /// + /// + public NativeLibraryFromPath(string path) + { + _path = path; + } + + /// + public IEnumerable Prepare(SystemInfo systemInfo, bool fromRemote, NativeLogConfig.LLamaLogCallback? logCallback) + { + return fromRemote? [] : [_path]; + } + } +} diff --git a/LLama/Native/Load/NativeLibraryMetadata.cs b/LLama/Native/Load/NativeLibraryMetadata.cs new file mode 100644 index 000000000..654c9002f --- /dev/null +++ b/LLama/Native/Load/NativeLibraryMetadata.cs @@ -0,0 +1,43 @@ + +namespace LLama.Native +{ + /// + /// Information of a native library file. + /// + /// Which kind of library it is. + /// Whether it's compiled with cublas. + /// Which AvxLevel it's compiled with. + public record class NativeLibraryMetadata(NativeLibraryName NativeLibraryName, bool UseCuda, AvxLevel AvxLevel) + { + public override string ToString() + { + return $"(NativeLibraryName: {NativeLibraryName}, UseCuda: {UseCuda}, AvxLevel: {AvxLevel})"; + } + } + + /// + /// Avx support configuration + /// + public enum AvxLevel + { + /// + /// No AVX + /// + None, + + /// + /// Advanced Vector Extensions (supported by most processors after 2011) + /// + Avx, + + /// + /// AVX2 (supported by most processors after 2013) + /// + Avx2, + + /// + /// AVX512 (supported by some processors after 2016, not widely supported) + /// + Avx512, + } +} diff --git a/LLama/Native/Load/NativeLibraryUtils.cs b/LLama/Native/Load/NativeLibraryUtils.cs new file mode 100644 index 000000000..513f6ccc2 --- /dev/null +++ b/LLama/Native/Load/NativeLibraryUtils.cs @@ -0,0 +1,182 @@ +using LLama.Exceptions; +using System; +using System.Collections.Generic; +using System.IO; +using System.Runtime.InteropServices; + +namespace LLama.Native +{ + internal static class NativeLibraryUtils + { + /// + /// Try to load libllama/llava_shared, using CPU feature detection to try and load a more specialised DLL if possible + /// + /// The library handle to unload later, or IntPtr.Zero if no library was loaded + internal static IntPtr TryLoadLibrary(NativeLibraryConfig config) + { +#if NET6_0_OR_GREATER + var description = config.CheckAndGatherDescription(); + var systemInfo = SystemInfo.Get(); + Log($"Loading library: '{config.NativeLibraryName.GetLibraryName()}'", LLamaLogLevel.Debug, config.LogCallback); + + // Get platform specific parts of the path (e.g. .so/.dll/.dylib, libName prefix or not) + NativeLibraryUtils.GetPlatformPathParts(systemInfo.OSPlatform, out var os, out var ext, out var libPrefix); + Log($"Detected OS Platform: '{systemInfo.OSPlatform}'", LLamaLogLevel.Info, config.LogCallback); + Log($"Detected OS string: '{os}'", LLamaLogLevel.Debug, config.LogCallback); + Log($"Detected extension string: '{ext}'", LLamaLogLevel.Debug, config.LogCallback); + Log($"Detected prefix string: '{libPrefix}'", LLamaLogLevel.Debug, config.LogCallback); + + // Set the flag to ensure this config can no longer be modified + config.LibraryHasLoaded = true; + + // Show the configuration we're working with + Log(description.ToString(), LLamaLogLevel.Info, config.LogCallback); + + // Get the libraries ordered by priority from the selecting policy. + var libraries = config.SelectingPolicy.Select(description, systemInfo, config.LogCallback); + + foreach (var library in libraries) + { + // Prepare the local library file and get the path. + var paths = library.Prepare(systemInfo, false, config.LogCallback); + foreach (var path in paths) + { + Log($"Got relative library path '{path}' from local with {library.Metadata}, trying to load it...", LLamaLogLevel.Debug, config.LogCallback); + + var result = TryLoad(path, description.SearchDirectories, config.LogCallback); + if (result != IntPtr.Zero) + { + return result; + } + } + // If we failed but auto-download is allowed, try to prepare the file from remote. + if (description.AllowAutoDownload) + { + paths = library.Prepare(systemInfo, true, config.LogCallback); + if (description.DownloadSettings.LocalDir is null) + { + // Null local directory is not expected here (it will make things more complex if we want to handle it). + // It should always be set when gathering the description. + throw new RuntimeError("Auto-download is enabled for native library but the `LocalDir` is null. " + + "It's an unexpected behavior and please report an issue to LLamaSharp."); + } + // When using auto-download, this should be the only search directory. + List searchDirectoriesForDownload = [description.DownloadSettings.LocalDir]; + // unless extra search paths are added by the user. + searchDirectoriesForDownload.AddRange(description.DownloadSettings.ExtraSearchDirectories ?? []); + + foreach (var path in paths) + { + Log($"Got relative library path '{path}' from remote with {library.Metadata}, trying to load it...", LLamaLogLevel.Debug, config.LogCallback); + + var result = TryLoad(path, searchDirectoriesForDownload, config.LogCallback); + if (result != IntPtr.Zero) + { + return result; + } + } + } + } + + // If fallback is allowed, we will make the last try (the default system loading) when calling the native api. + // Otherwise we throw an exception here. + if (!description.AllowFallback) + { + throw new RuntimeError("Failed to load the native library. Please check the log for more information."); + } +#endif + + Log($"No library was loaded before calling native apis. " + + $"This is not an error under netstandard2.0 but needs attention with net6 or higher.", LLamaLogLevel.Warning, config.LogCallback); + return IntPtr.Zero; + +#if NET6_0_OR_GREATER + // Try to load a DLL from the path. + // Returns null if nothing is loaded. + static IntPtr TryLoad(string path, IEnumerable searchDirectories, NativeLogConfig.LLamaLogCallback? logCallback) + { + var fullPath = TryFindPath(path, searchDirectories); + Log($"Found full path file '{fullPath}' for relative path '{path}'", LLamaLogLevel.Debug, logCallback); + if (NativeLibrary.TryLoad(fullPath, out var handle)) + { + Log($"Successfully loaded '{fullPath}'", LLamaLogLevel.Info, logCallback); + return handle; + } + + Log($"Failed Loading '{fullPath}'", LLamaLogLevel.Info, logCallback); + return IntPtr.Zero; + } +#endif + } + + // Try to find the given file in any of the possible search paths + private static string TryFindPath(string filename, IEnumerable searchDirectories) + { + // Try the configured search directories in the configuration + foreach (var path in searchDirectories) + { + var candidate = Path.Combine(path, filename); + if (File.Exists(candidate)) + return candidate; + } + + // Try a few other possible paths + var possiblePathPrefix = new[] { + AppDomain.CurrentDomain.BaseDirectory, + Path.GetDirectoryName(System.Reflection.Assembly.GetExecutingAssembly().Location) ?? "" + }; + + foreach (var path in possiblePathPrefix) + { + var candidate = Path.Combine(path, filename); + if (File.Exists(candidate)) + return candidate; + } + + return filename; + } + + private static void Log(string message, LLamaLogLevel level, NativeLogConfig.LLamaLogCallback? logCallback) + { + if (!message.EndsWith("\n")) + message += "\n"; + + logCallback?.Invoke(level, message); + } + +#if NET6_0_OR_GREATER + public static void GetPlatformPathParts(OSPlatform platform, out string os, out string fileExtension, out string libPrefix) + { + if (platform == OSPlatform.Windows) + { + os = "win-x64"; + fileExtension = ".dll"; + libPrefix = ""; + return; + } + + if (platform == OSPlatform.Linux) + { + os = "linux-x64"; + fileExtension = ".so"; + libPrefix = "lib"; + return; + } + + if (platform == OSPlatform.OSX) + { + fileExtension = ".dylib"; + + os = System.Runtime.Intrinsics.Arm.ArmBase.Arm64.IsSupported + ? "osx-arm64" + : "osx-x64"; + libPrefix = "lib"; + } + else + { + throw new RuntimeError("Your operating system is not supported, please open an issue in LLamaSharp."); + } + } +#endif + } +} diff --git a/LLama/Native/Load/NativeLibraryWithAvx.cs b/LLama/Native/Load/NativeLibraryWithAvx.cs new file mode 100644 index 000000000..5b1f65142 --- /dev/null +++ b/LLama/Native/Load/NativeLibraryWithAvx.cs @@ -0,0 +1,75 @@ +using LLama.Abstractions; +using System.Collections.Generic; +using System.Runtime.InteropServices; + +namespace LLama.Native +{ +#if NET6_0_OR_GREATER + /// + /// A native library compiled with avx support but without cuda/cublas. + /// + public class NativeLibraryWithAvx : INativeLibrary + { + private NativeLibraryName _libraryName; + private AvxLevel _avxLevel; + private bool _skipCheck; + private NativeLibraryDownloadSettings _downloadSettings; + + /// + public NativeLibraryMetadata? Metadata + { + get + { + return new NativeLibraryMetadata(_libraryName, false, _avxLevel); + } + } + + /// + /// + /// + /// + /// + /// + /// + public NativeLibraryWithAvx(NativeLibraryName libraryName, AvxLevel avxLevel, bool skipCheck, NativeLibraryDownloadSettings downloadSettings) + { + _libraryName = libraryName; + _avxLevel = avxLevel; + _skipCheck = skipCheck; + _downloadSettings = downloadSettings; + } + + /// + public IEnumerable Prepare(SystemInfo systemInfo, bool fromRemote, NativeLogConfig.LLamaLogCallback? logCallback) + { + if (systemInfo.OSPlatform != OSPlatform.Windows && systemInfo.OSPlatform != OSPlatform.Linux && !_skipCheck) + { + // Not supported on systems other than Windows and Linux. + return []; + } + var path = GetAvxPath(systemInfo, _avxLevel, fromRemote, logCallback); + return path is null ? [] : [path]; + } + + private string? GetAvxPath(SystemInfo systemInfo, AvxLevel avxLevel, bool fromRemote, NativeLogConfig.LLamaLogCallback? logCallback) + { + NativeLibraryUtils.GetPlatformPathParts(systemInfo.OSPlatform, out var os, out var fileExtension, out var libPrefix); + var avxStr = NativeLibraryConfig.AvxLevelToString(avxLevel); + if (!string.IsNullOrEmpty(avxStr)) + avxStr += "/"; + var relativePath = $"runtimes/{os}/native/{avxStr}{libPrefix}{_libraryName.GetLibraryName()}{fileExtension}"; + + if (fromRemote) + { + // Download and return the local path. + // We make it sychronize because we c'd better not use async method when loading library later. + return NativeLibraryDownloadManager.DownloadLibraryFile(_downloadSettings, relativePath, logCallback).Result; + } + else + { + return relativePath; + } + } + } +#endif +} diff --git a/LLama/Native/Load/NativeLibraryWithCpuOrMac.cs b/LLama/Native/Load/NativeLibraryWithCpuOrMac.cs new file mode 100644 index 000000000..3bc09b13e --- /dev/null +++ b/LLama/Native/Load/NativeLibraryWithCpuOrMac.cs @@ -0,0 +1,77 @@ +using LLama.Abstractions; +using System.Collections.Generic; +using System.Runtime.InteropServices; + +namespace LLama.Native +{ +#if NET6_0_OR_GREATER + /// + /// A native library compiled on Mac, or fallbacks from all other libraries in the selection. + /// + public class NativeLibraryWithCpuOrMac + : INativeLibrary + { + private NativeLibraryName _libraryName; + private bool _skipCheck; + private NativeLibraryDownloadSettings _downloadSettings; + + /// + public NativeLibraryMetadata? Metadata + { + get + { + return new NativeLibraryMetadata(_libraryName, false, AvxLevel.None); + } + } + + /// + /// + /// + /// + /// + /// + public NativeLibraryWithCpuOrMac(NativeLibraryName libraryName, bool skipCheck, NativeLibraryDownloadSettings downloadSettings) + { + _libraryName = libraryName; + _skipCheck = skipCheck; + _downloadSettings = downloadSettings; + } + + /// + public IEnumerable Prepare(SystemInfo systemInfo, bool fromRemote, NativeLogConfig.LLamaLogCallback? logCallback) + { + var path = GetPath(systemInfo, AvxLevel.None, fromRemote, logCallback); + return path is null ?[] : [path]; + } + + private string? GetPath(SystemInfo systemInfo, AvxLevel avxLevel, bool fromRemote, NativeLogConfig.LLamaLogCallback? logCallback) + { + NativeLibraryUtils.GetPlatformPathParts(systemInfo.OSPlatform, out var os, out var fileExtension, out var libPrefix); + string relativePath; + if (systemInfo.OSPlatform == OSPlatform.OSX) + { + relativePath = $"runtimes/{os}/native/{libPrefix}{_libraryName.GetLibraryName()}{fileExtension}"; + } + else + { + var avxStr = NativeLibraryConfig.AvxLevelToString(AvxLevel.None); + if (!string.IsNullOrEmpty(avxStr)) + avxStr += "/"; + + relativePath = $"runtimes/{os}/native/{avxStr}{libPrefix}{_libraryName.GetLibraryName()}{fileExtension}"; + } + + if (fromRemote) + { + // Download and return the local path. + // We make it sychronize because we c'd better not use async method when loading library later. + return NativeLibraryDownloadManager.DownloadLibraryFile(_downloadSettings, relativePath, logCallback).Result; + } + else + { + return relativePath; + } + } + } +#endif +} diff --git a/LLama/Native/Load/NativeLibraryWithCuda.cs b/LLama/Native/Load/NativeLibraryWithCuda.cs new file mode 100644 index 000000000..1e2840d5e --- /dev/null +++ b/LLama/Native/Load/NativeLibraryWithCuda.cs @@ -0,0 +1,91 @@ +using LLama.Abstractions; +using System.Collections.Generic; +using System.Runtime.InteropServices; + +namespace LLama.Native +{ +#if NET6_0_OR_GREATER + /// + /// A native library compiled with cublas/cuda. + /// + public class NativeLibraryWithCuda : INativeLibrary + { + private int _majorCudaVersion; + private NativeLibraryName _libraryName; + private AvxLevel _avxLevel; + private bool _skipCheck; + private NativeLibraryDownloadSettings _downloadSettings; + + /// + public NativeLibraryMetadata? Metadata + { + get + { + return new NativeLibraryMetadata(_libraryName, true, _avxLevel); + } + } + + /// + /// + /// + /// + /// + /// + /// + public NativeLibraryWithCuda(int majorCudaVersion, NativeLibraryName libraryName, bool skipCheck, NativeLibraryDownloadSettings downloadSettings) + { + _majorCudaVersion = majorCudaVersion; + _libraryName = libraryName; + _skipCheck = skipCheck; + _downloadSettings = downloadSettings; + } + + /// + public IEnumerable Prepare(SystemInfo systemInfo, bool fromRemote, NativeLogConfig.LLamaLogCallback? logCallback) + { + // TODO: Avx level is ignored now, needs to be implemented in the future. + if (systemInfo.OSPlatform == OSPlatform.Windows || systemInfo.OSPlatform == OSPlatform.Linux || _skipCheck) + { + if (_majorCudaVersion == -1 && _skipCheck) + { + // Currently only 11 and 12 are supported. + var cuda12LibraryPath = GetCudaPath(systemInfo, 12, fromRemote, logCallback); + if (cuda12LibraryPath is not null) + { + yield return cuda12LibraryPath; + } + var cuda11LibraryPath = GetCudaPath(systemInfo, 11, fromRemote, logCallback); + if (cuda11LibraryPath is not null) + { + yield return cuda11LibraryPath; + } + } + else if (_majorCudaVersion != -1) + { + var cudaLibraryPath = GetCudaPath(systemInfo, _majorCudaVersion, fromRemote, logCallback); + if (cudaLibraryPath is not null) + { + yield return cudaLibraryPath; + } + } + } + } + + private string? GetCudaPath(SystemInfo systemInfo, int cudaVersion, bool remote, NativeLogConfig.LLamaLogCallback? logCallback) + { + NativeLibraryUtils.GetPlatformPathParts(systemInfo.OSPlatform, out var os, out var fileExtension, out var libPrefix); + var relativePath = $"runtimes/{os}/native/cuda{cudaVersion}/{libPrefix}{_libraryName.GetLibraryName()}{fileExtension}"; + if (remote) + { + // Download and return the local path. + // We make it sychronize because we c'd better not use async method when loading library later. + return NativeLibraryDownloadManager.DownloadLibraryFile(_downloadSettings, relativePath, logCallback).Result; + } + else + { + return relativePath; + } + } + } +#endif +} diff --git a/LLama/Native/Load/SystemInfo.cs b/LLama/Native/Load/SystemInfo.cs new file mode 100644 index 000000000..0ffc67e91 --- /dev/null +++ b/LLama/Native/Load/SystemInfo.cs @@ -0,0 +1,129 @@ +using System; +using System.IO; +using System.Runtime.InteropServices; +using System.Text.Json; + +namespace LLama.Native +{ + /// + /// Operating system information. + /// + /// + /// + public record class SystemInfo(OSPlatform OSPlatform, int CudaMajorVersion) + { + /// + /// Get the system information of the current machine. + /// + /// + /// + public static SystemInfo Get() + { + OSPlatform platform; + if(RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) + { + platform = OSPlatform.Windows; + } + else if(RuntimeInformation.IsOSPlatform(OSPlatform.Linux)) + { + platform = OSPlatform.Linux; + } + else if(RuntimeInformation.IsOSPlatform(OSPlatform.OSX)) + { + platform = OSPlatform.OSX; + } + else + { + throw new PlatformNotSupportedException(); + } + + return new SystemInfo(platform, GetCudaMajorVersion()); + } + + #region CUDA version + private static int GetCudaMajorVersion() + { + string? cudaPath; + string version = ""; + if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) + { + cudaPath = Environment.GetEnvironmentVariable("CUDA_PATH"); + if (cudaPath is null) + { + return -1; + } + + //Ensuring cuda bin path is reachable. Especially for MAUI environment. + string cudaBinPath = Path.Combine(cudaPath, "bin"); + + if (Directory.Exists(cudaBinPath)) + { + AddDllDirectory(cudaBinPath); + } + + version = GetCudaVersionFromPath(cudaPath); + } + else if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux)) + { + // Try the default first + cudaPath = "/usr/local/bin/cuda"; + version = GetCudaVersionFromPath(cudaPath); + if (string.IsNullOrEmpty(version)) + { + cudaPath = Environment.GetEnvironmentVariable("LD_LIBRARY_PATH"); + if (cudaPath is null) + { + return -1; + } + foreach (var path in cudaPath.Split(':')) + { + version = GetCudaVersionFromPath(Path.Combine(path, "..")); + if (string.IsNullOrEmpty(version)) + { + break; + } + } + } + } + + if (string.IsNullOrEmpty(version)) + return -1; + + version = version.Split('.')[0]; + if (int.TryParse(version, out var majorVersion)) + return majorVersion; + + return -1; + } + + private static string GetCudaVersionFromPath(string cudaPath) + { + try + { + string json = File.ReadAllText(Path.Combine(cudaPath, cudaVersionFile)); + using (JsonDocument document = JsonDocument.Parse(json)) + { + JsonElement root = document.RootElement; + JsonElement cublasNode = root.GetProperty("libcublas"); + JsonElement versionNode = cublasNode.GetProperty("version"); + if (versionNode.ValueKind == JsonValueKind.Undefined) + { + return string.Empty; + } + return versionNode.GetString() ?? ""; + } + } + catch (Exception) + { + return string.Empty; + } + } + + // Put it here to avoid calling NativeApi when getting the cuda version. + [DllImport("kernel32.dll", CharSet = CharSet.Unicode, SetLastError = true)] + internal static extern int AddDllDirectory(string NewDirectory); + + private const string cudaVersionFile = "version.json"; + #endregion + } +} diff --git a/LLama/Native/NativeApi.Load.cs b/LLama/Native/NativeApi.Load.cs index 4b4beea2e..b3c9a6ad2 100644 --- a/LLama/Native/NativeApi.Load.cs +++ b/LLama/Native/NativeApi.Load.cs @@ -18,7 +18,8 @@ static NativeApi() SetDllImportResolver(); // Set flag to indicate that this point has been passed. No native library config can be done after this point. - NativeLibraryConfig.LibraryHasLoaded = true; + NativeLibraryConfig.LLama.LibraryHasLoaded = true; + NativeLibraryConfig.LLavaShared.LibraryHasLoaded = true; // Immediately make a call which requires loading the llama DLL. This method call // can't fail unless the DLL hasn't been loaded. @@ -38,8 +39,8 @@ static NativeApi() } // Now that the "loaded" flag is set configure logging in llama.cpp - if (NativeLibraryConfig.Instance.LogCallback != null) - NativeLogConfig.llama_log_set(NativeLibraryConfig.Instance.LogCallback); + if (NativeLibraryConfig.LLama.LogCallback != null) + NativeLogConfig.llama_log_set(NativeLibraryConfig.LLama.LogCallback); // Init llama.cpp backend llama_backend_init(); @@ -63,8 +64,14 @@ private static void SetDllImportResolver() if (_loadedLlamaHandle != IntPtr.Zero) return _loadedLlamaHandle; + // We don't allow downloading in static constructor to avoid potentially uncertain behaviors now. + if (NativeLibraryConfig.LLama.AllowAutoDownload && !NativeLibraryConfig.LLama.HasCalledDryRun) + { + throw new RuntimeError("When using auto-download, please call NativeLibraryConfig.DryRun() right after finishing you configurations."); + } + // Try to load a preferred library, based on CPU feature detection - _loadedLlamaHandle = TryLoadLibraries(LibraryName.Llama); + _loadedLlamaHandle = NativeLibraryUtils.TryLoadLibrary(NativeLibraryConfig.LLama); return _loadedLlamaHandle; } @@ -74,8 +81,14 @@ private static void SetDllImportResolver() if (_loadedLlavaSharedHandle != IntPtr.Zero) return _loadedLlavaSharedHandle; + // We don't allow downloading in static constructor to avoid potentially uncertain behaviors now. + if (NativeLibraryConfig.LLavaShared.AllowAutoDownload && !NativeLibraryConfig.LLavaShared.HasCalledDryRun) + { + throw new RuntimeError("When using auto-download, please call NativeLibraryConfig.DryRun() right after finishing you configurations."); + } + // Try to load a preferred library, based on CPU feature detection - _loadedLlavaSharedHandle = TryLoadLibraries(LibraryName.LlavaShared); + _loadedLlavaSharedHandle = NativeLibraryUtils.TryLoadLibrary(NativeLibraryConfig.LLavaShared); return _loadedLlavaSharedHandle; } @@ -85,341 +98,6 @@ private static void SetDllImportResolver() #endif } - private static void Log(string message, LLamaLogLevel level) - { - if (!message.EndsWith("\n")) - message += "\n"; - - NativeLibraryConfig.Instance.LogCallback?.Invoke(level, message); - } - - #region CUDA version - private static int GetCudaMajorVersion() - { - string? cudaPath; - string version = ""; - if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) - { - cudaPath = Environment.GetEnvironmentVariable("CUDA_PATH"); - if (cudaPath is null) - { - return -1; - } - - //Ensuring cuda bin path is reachable. Especially for MAUI environment. - string cudaBinPath = Path.Combine(cudaPath, "bin"); - - if (Directory.Exists(cudaBinPath)) - { - AddDllDirectory(cudaBinPath); - } - - version = GetCudaVersionFromPath(cudaPath); - } - else if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux)) - { - // Try the default first - cudaPath = "/usr/local/bin/cuda"; - version = GetCudaVersionFromPath(cudaPath); - if (string.IsNullOrEmpty(version)) - { - cudaPath = Environment.GetEnvironmentVariable("LD_LIBRARY_PATH"); - if (cudaPath is null) - { - return -1; - } - foreach (var path in cudaPath.Split(':')) - { - version = GetCudaVersionFromPath(Path.Combine(path, "..")); - if (string.IsNullOrEmpty(version)) - { - break; - } - } - } - } - - if (string.IsNullOrEmpty(version)) - return -1; - - version = version.Split('.')[0]; - if (int.TryParse(version, out var majorVersion)) - return majorVersion; - - return -1; - } - - private static string GetCudaVersionFromPath(string cudaPath) - { - try - { - string json = File.ReadAllText(Path.Combine(cudaPath, cudaVersionFile)); - using (JsonDocument document = JsonDocument.Parse(json)) - { - JsonElement root = document.RootElement; - JsonElement cublasNode = root.GetProperty("libcublas"); - JsonElement versionNode = cublasNode.GetProperty("version"); - if (versionNode.ValueKind == JsonValueKind.Undefined) - { - return string.Empty; - } - return versionNode.GetString() ?? ""; - } - } - catch (Exception) - { - return string.Empty; - } - } - #endregion - -#if NET6_0_OR_GREATER - private static IEnumerable GetLibraryTryOrder(NativeLibraryConfig.Description configuration) - { - var loadingName = configuration.Library.GetLibraryName(); - Log($"Loading library: '{loadingName}'", LLamaLogLevel.Debug); - - // Get platform specific parts of the path (e.g. .so/.dll/.dylib, libName prefix or not) - GetPlatformPathParts(out var platform, out var os, out var ext, out var libPrefix); - Log($"Detected OS Platform: '{platform}'", LLamaLogLevel.Info); - Log($"Detected OS string: '{os}'", LLamaLogLevel.Debug); - Log($"Detected extension string: '{ext}'", LLamaLogLevel.Debug); - Log($"Detected prefix string: '{libPrefix}'", LLamaLogLevel.Debug); - - if (configuration.UseCuda && (platform == OSPlatform.Windows || platform == OSPlatform.Linux)) - { - var cudaVersion = GetCudaMajorVersion(); - Log($"Detected cuda major version {cudaVersion}.", LLamaLogLevel.Info); - - if (cudaVersion == -1 && !configuration.AllowFallback) - { - // if check skipped, we just try to load cuda libraries one by one. - if (configuration.SkipCheck) - { - yield return GetCudaLibraryPath(loadingName, "cuda12"); - yield return GetCudaLibraryPath(loadingName, "cuda11"); - } - else - { - throw new RuntimeError("Configured to load a cuda library but no cuda detected on your device."); - } - } - else if (cudaVersion == 11) - { - yield return GetCudaLibraryPath(loadingName, "cuda11"); - } - else if (cudaVersion == 12) - { - yield return GetCudaLibraryPath(loadingName, "cuda12"); - } - else if (cudaVersion > 0) - { - throw new RuntimeError($"Cuda version {cudaVersion} hasn't been supported by LLamaSharp, please open an issue for it."); - } - - // otherwise no cuda detected but allow fallback - } - - // Add the CPU/Metal libraries - if (platform == OSPlatform.OSX) - { - // On Mac it's very simple, there's no AVX to consider. - yield return GetMacLibraryPath(loadingName); - } - else - { - if (configuration.AllowFallback) - { - // Try all of the AVX levels we can support. - if (configuration.AvxLevel >= NativeLibraryConfig.AvxLevel.Avx512) - yield return GetAvxLibraryPath(loadingName, NativeLibraryConfig.AvxLevel.Avx512); - - if (configuration.AvxLevel >= NativeLibraryConfig.AvxLevel.Avx2) - yield return GetAvxLibraryPath(loadingName, NativeLibraryConfig.AvxLevel.Avx2); - - if (configuration.AvxLevel >= NativeLibraryConfig.AvxLevel.Avx) - yield return GetAvxLibraryPath(loadingName, NativeLibraryConfig.AvxLevel.Avx); - - yield return GetAvxLibraryPath(loadingName, NativeLibraryConfig.AvxLevel.None); - } - else - { - // Fallback is not allowed - use the exact specified AVX level - yield return GetAvxLibraryPath(loadingName, configuration.AvxLevel); - } - } - } - - private static string GetMacLibraryPath(string libraryName) - { - GetPlatformPathParts(out _, out var os, out var fileExtension, out var libPrefix); - - return $"runtimes/{os}/native/{libPrefix}{libraryName}{fileExtension}"; - } - - /// - /// Given a CUDA version and some path parts, create a complete path to the library file - /// - /// Library being loaded (e.g. "llama") - /// CUDA version (e.g. "cuda11") - /// - private static string GetCudaLibraryPath(string libraryName, string cuda) - { - GetPlatformPathParts(out _, out var os, out var fileExtension, out var libPrefix); - - return $"runtimes/{os}/native/{cuda}/{libPrefix}{libraryName}{fileExtension}"; - } - - /// - /// Given an AVX level and some path parts, create a complete path to the library file - /// - /// Library being loaded (e.g. "llama") - /// - /// - private static string GetAvxLibraryPath(string libraryName, NativeLibraryConfig.AvxLevel avx) - { - GetPlatformPathParts(out _, out var os, out var fileExtension, out var libPrefix); - - var avxStr = NativeLibraryConfig.AvxLevelToString(avx); - if (!string.IsNullOrEmpty(avxStr)) - avxStr += "/"; - - return $"runtimes/{os}/native/{avxStr}{libPrefix}{libraryName}{fileExtension}"; - } - - private static void GetPlatformPathParts(out OSPlatform platform, out string os, out string fileExtension, out string libPrefix) - { - if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) - { - platform = OSPlatform.Windows; - os = "win-x64"; - fileExtension = ".dll"; - libPrefix = ""; - return; - } - - if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux)) - { - platform = OSPlatform.Linux; - os = "linux-x64"; - fileExtension = ".so"; - libPrefix = "lib"; - return; - } - - if (RuntimeInformation.IsOSPlatform(OSPlatform.OSX)) - { - platform = OSPlatform.OSX; - fileExtension = ".dylib"; - - os = System.Runtime.Intrinsics.Arm.ArmBase.Arm64.IsSupported - ? "osx-arm64" - : "osx-x64"; - libPrefix = "lib"; - } - else - { - throw new RuntimeError("Your operating system is not supported, please open an issue in LLamaSharp."); - } - } -#endif - - /// - /// Try to load libllama/llava_shared, using CPU feature detection to try and load a more specialised DLL if possible - /// - /// The library handle to unload later, or IntPtr.Zero if no library was loaded - private static IntPtr TryLoadLibraries(LibraryName lib) - { -#if NET6_0_OR_GREATER - var configuration = NativeLibraryConfig.CheckAndGatherDescription(lib); - - // Set the flag to ensure the NativeLibraryConfig can no longer be modified - NativeLibraryConfig.LibraryHasLoaded = true; - - // Show the configuration we're working with - Log(configuration.ToString(), LLamaLogLevel.Info); - - // If a specific path is requested, load that or immediately fail - if (!string.IsNullOrEmpty(configuration.Path)) - { - if (!NativeLibrary.TryLoad(configuration.Path, out var handle)) - throw new RuntimeError($"Failed to load the native library [{configuration.Path}] you specified."); - - Log($"Successfully loaded the library [{configuration.Path}] specified by user", LLamaLogLevel.Info); - return handle; - } - - // Get a list of locations to try loading (in order of preference) - var libraryTryLoadOrder = GetLibraryTryOrder(configuration); - - foreach (var libraryPath in libraryTryLoadOrder) - { - var fullPath = TryFindPath(libraryPath); - Log($"Trying '{fullPath}'", LLamaLogLevel.Debug); - - var result = TryLoad(fullPath); - if (result != IntPtr.Zero) - { - Log($"Loaded '{fullPath}'", LLamaLogLevel.Info); - return result; - } - - Log($"Failed Loading '{fullPath}'", LLamaLogLevel.Info); - } - - if (!configuration.AllowFallback) - { - throw new RuntimeError("Failed to load the library that match your rule, please" + - " 1) check your rule." + - " 2) try to allow fallback." + - " 3) or open an issue if it's expected to be successful."); - } -#endif - - Log($"No library was loaded before calling native apis. " + - $"This is not an error under netstandard2.0 but needs attention with net6 or higher.", LLamaLogLevel.Warning); - return IntPtr.Zero; - -#if NET6_0_OR_GREATER - // Try to load a DLL from the path. - // Returns null if nothing is loaded. - static IntPtr TryLoad(string path) - { - if (NativeLibrary.TryLoad(path, out var handle)) - return handle; - - return IntPtr.Zero; - } - - // Try to find the given file in any of the possible search paths - string TryFindPath(string filename) - { - // Try the configured search directories in the configuration - foreach (var path in configuration.SearchDirectories) - { - var candidate = Path.Combine(path, filename); - if (File.Exists(candidate)) - return candidate; - } - - // Try a few other possible paths - var possiblePathPrefix = new[] { - AppDomain.CurrentDomain.BaseDirectory, - Path.GetDirectoryName(System.Reflection.Assembly.GetExecutingAssembly().Location) ?? "" - }; - - foreach (var path in possiblePathPrefix) - { - var candidate = Path.Combine(path, filename); - if (File.Exists(candidate)) - return candidate; - } - - return filename; - } -#endif - } - internal const string libraryName = "llama"; internal const string llavaLibraryName = "llava_shared"; private const string cudaVersionFile = "version.json"; diff --git a/LLama/Native/NativeApi.cs b/LLama/Native/NativeApi.cs index ed4561517..d57b00e77 100644 --- a/LLama/Native/NativeApi.cs +++ b/LLama/Native/NativeApi.cs @@ -19,9 +19,6 @@ public static void llama_empty_call() llama_max_devices(); } - [DllImport("kernel32.dll", CharSet = CharSet.Unicode, SetLastError = true)] - private static extern int AddDllDirectory(string NewDirectory); - /// /// Get the maximum number of devices supported by llama.cpp /// diff --git a/LLama/Native/NativeLibraryConfig.cs b/LLama/Native/NativeLibraryConfig.cs deleted file mode 100644 index ef7cd7c19..000000000 --- a/LLama/Native/NativeLibraryConfig.cs +++ /dev/null @@ -1,332 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using Microsoft.Extensions.Logging; - -namespace LLama.Native -{ -#if NET6_0_OR_GREATER - /// - /// Allows configuration of the native llama.cpp libraries to load and use. - /// All configuration must be done before using **any** other LLamaSharp methods! - /// - public sealed partial class NativeLibraryConfig - { - private string? _libraryPath; - private string? _libraryPathLLava; - - private bool _useCuda = true; - private AvxLevel _avxLevel; - private bool _allowFallback = true; - private bool _skipCheck = false; - - /// - /// search directory -> priority level, 0 is the lowest. - /// - private readonly List _searchDirectories = new List(); - - #region configurators - /// - /// Load a specified native library as backend for LLamaSharp. - /// When this method is called, all the other configurations will be ignored. - /// - /// The full path to the llama library to load. - /// The full path to the llava library to load. - /// Thrown if `LibraryHasLoaded` is true. - public NativeLibraryConfig WithLibrary(string? llamaPath, string? llavaPath) - { - ThrowIfLoaded(); - - _libraryPath = llamaPath; - _libraryPathLLava = llavaPath; - return this; - } - - /// - /// Configure whether to use cuda backend if possible. - /// - /// - /// - /// Thrown if `LibraryHasLoaded` is true. - public NativeLibraryConfig WithCuda(bool enable = true) - { - ThrowIfLoaded(); - - _useCuda = enable; - return this; - } - - /// - /// Configure the prefferred avx support level of the backend. - /// - /// - /// - /// Thrown if `LibraryHasLoaded` is true. - public NativeLibraryConfig WithAvx(AvxLevel level) - { - ThrowIfLoaded(); - - _avxLevel = level; - return this; - } - - /// - /// Configure whether to allow fallback when there's no match for preferred settings. - /// - /// - /// - /// Thrown if `LibraryHasLoaded` is true. - public NativeLibraryConfig WithAutoFallback(bool enable = true) - { - ThrowIfLoaded(); - - _allowFallback = enable; - return this; - } - - /// - /// Whether to skip the check when you don't allow fallback. This option - /// may be useful under some complex conditions. For example, you're sure - /// you have your cublas configured but LLamaSharp take it as invalid by mistake. - /// - /// - /// - /// Thrown if `LibraryHasLoaded` is true. - public NativeLibraryConfig SkipCheck(bool enable = true) - { - ThrowIfLoaded(); - - _skipCheck = enable; - return this; - } - - /// - /// Add self-defined search directories. Note that the file stucture of the added - /// directories must be the same as the default directory. Besides, the directory - /// won't be used recursively. - /// - /// - /// - public NativeLibraryConfig WithSearchDirectories(IEnumerable directories) - { - ThrowIfLoaded(); - - _searchDirectories.AddRange(directories); - return this; - } - - /// - /// Add self-defined search directories. Note that the file stucture of the added - /// directories must be the same as the default directory. Besides, the directory - /// won't be used recursively. - /// - /// - /// - public NativeLibraryConfig WithSearchDirectory(string directory) - { - ThrowIfLoaded(); - - _searchDirectories.Add(directory); - return this; - } - #endregion - - internal static Description CheckAndGatherDescription(LibraryName library) - { - if (Instance._allowFallback && Instance._skipCheck) - throw new ArgumentException("Cannot skip the check when fallback is allowed."); - - var path = library switch - { - LibraryName.Llama => Instance._libraryPath, - LibraryName.LlavaShared => Instance._libraryPathLLava, - _ => throw new ArgumentException($"Unknown library name '{library}'", nameof(library)), - }; - - return new Description( - path, - library, - Instance._useCuda, - Instance._avxLevel, - Instance._allowFallback, - Instance._skipCheck, - Instance._searchDirectories.Concat(new[] { "./" }).ToArray() - ); - } - - internal static string AvxLevelToString(AvxLevel level) - { - return level switch - { - AvxLevel.None => string.Empty, - AvxLevel.Avx => "avx", - AvxLevel.Avx2 => "avx2", - AvxLevel.Avx512 => "avx512", - _ => throw new ArgumentException($"Unknown AvxLevel '{level}'") - }; - } - - /// - /// Private constructor prevents new instances of this class being created - /// - private NativeLibraryConfig() - { - // Automatically detect the highest supported AVX level - if (System.Runtime.Intrinsics.X86.Avx.IsSupported) - _avxLevel = AvxLevel.Avx; - if (System.Runtime.Intrinsics.X86.Avx2.IsSupported) - _avxLevel = AvxLevel.Avx2; - - if (CheckAVX512()) - _avxLevel = AvxLevel.Avx512; - } - - private static bool CheckAVX512() - { - if (!System.Runtime.Intrinsics.X86.X86Base.IsSupported) - return false; - - // ReSharper disable UnusedVariable (ebx is used when < NET8) - var (_, ebx, ecx, _) = System.Runtime.Intrinsics.X86.X86Base.CpuId(7, 0); - // ReSharper restore UnusedVariable - - var vnni = (ecx & 0b_1000_0000_0000) != 0; - -#if NET8_0_OR_GREATER - var f = System.Runtime.Intrinsics.X86.Avx512F.IsSupported; - var bw = System.Runtime.Intrinsics.X86.Avx512BW.IsSupported; - var vbmi = System.Runtime.Intrinsics.X86.Avx512Vbmi.IsSupported; -#else - var f = (ebx & (1 << 16)) != 0; - var bw = (ebx & (1 << 30)) != 0; - var vbmi = (ecx & 0b_0000_0000_0010) != 0; -#endif - - return vnni && vbmi && bw && f; - } - - /// - /// Avx support configuration - /// - public enum AvxLevel - { - /// - /// No AVX - /// - None, - - /// - /// Advanced Vector Extensions (supported by most processors after 2011) - /// - Avx, - - /// - /// AVX2 (supported by most processors after 2013) - /// - Avx2, - - /// - /// AVX512 (supported by some processors after 2016, not widely supported) - /// - Avx512, - } - - internal record Description(string? Path, LibraryName Library, bool UseCuda, AvxLevel AvxLevel, bool AllowFallback, bool SkipCheck, string[] SearchDirectories) - { - public override string ToString() - { - string avxLevelString = AvxLevel switch - { - AvxLevel.None => "NoAVX", - AvxLevel.Avx => "AVX", - AvxLevel.Avx2 => "AVX2", - AvxLevel.Avx512 => "AVX512", - _ => "Unknown" - }; - - string searchDirectoriesString = "{ " + string.Join(", ", SearchDirectories) + " }"; - - return $"NativeLibraryConfig Description:\n" + - $"- LibraryName: {Library}\n" + - $"- Path: '{Path}'\n" + - $"- PreferCuda: {UseCuda}\n" + - $"- PreferredAvxLevel: {avxLevelString}\n" + - $"- AllowFallback: {AllowFallback}\n" + - $"- SkipCheck: {SkipCheck}\n" + - $"- SearchDirectories and Priorities: {searchDirectoriesString}"; - } - } - } -#endif - - public sealed partial class NativeLibraryConfig - { - /// - /// Get the config instance - /// - public static NativeLibraryConfig Instance { get; } = new(); - - /// - /// Check if the native library has already been loaded. Configuration cannot be modified if this is true. - /// - public static bool LibraryHasLoaded { get; internal set; } - - internal NativeLogConfig.LLamaLogCallback? LogCallback; - - private static void ThrowIfLoaded() - { - if (LibraryHasLoaded) - throw new InvalidOperationException("NativeLibraryConfig must be configured before using **any** other LLamaSharp methods!"); - } - - /// - /// Set the log callback that will be used for all llama.cpp log messages - /// - /// - /// - public NativeLibraryConfig WithLogCallback(NativeLogConfig.LLamaLogCallback? callback) - { - ThrowIfLoaded(); - - LogCallback = callback; - return this; - } - - /// - /// Set the log callback that will be used for all llama.cpp log messages - /// - /// - /// - public NativeLibraryConfig WithLogCallback(ILogger? logger) - { - ThrowIfLoaded(); - - // Redirect to llama_log_set. This will wrap the logger in a delegate and bind that as the log callback instead. - NativeLogConfig.llama_log_set(logger); - - return this; - } - } - - internal enum LibraryName - { - Llama, - LlavaShared - } - - internal static class LibraryNameExtensions - { - public static string GetLibraryName(this LibraryName name) - { - switch (name) - { - case LibraryName.Llama: - return NativeApi.libraryName; - case LibraryName.LlavaShared: - return NativeApi.llavaLibraryName; - default: - throw new ArgumentOutOfRangeException(nameof(name), name, null); - } - } - } -} diff --git a/LLama/Native/NativeLogConfig.cs b/LLama/Native/NativeLogConfig.cs index ebcd23d47..82b097fb3 100644 --- a/LLama/Native/NativeLogConfig.cs +++ b/LLama/Native/NativeLogConfig.cs @@ -37,7 +37,7 @@ public static class NativeLogConfig public static void llama_log_set(LLamaLogCallback? logCallback) #pragma warning restore IDE1006 // Naming Styles { - if (NativeLibraryConfig.LibraryHasLoaded) + if (NativeLibraryConfig.LLama.LibraryHasLoaded) { // The library is loaded, just pass the callback directly to llama.cpp native_llama_log_set(logCallback); From 6e28b21da3de91a473328e7ed3b5d4dfc9f727bf Mon Sep 17 00:00:00 2001 From: Rinne Date: Thu, 25 Apr 2024 02:39:25 +0800 Subject: [PATCH 2/7] refactor: remove the auto-download related parts. --- LLama.Examples/Program.cs | 5 +- LLama/Abstractions/INativeLibrary.cs | 3 +- .../INativeLibrarySelectingPolicy.cs | 2 +- LLama/LLamaSharp.csproj | 1 - .../DefaultNativeLibrarySelectingPolicy.cs | 16 +- LLama/Native/Load/NativeLibraryConfig.cs | 97 ++---- .../Load/NativeLibraryDownloadManager.cs | 285 ------------------ LLama/Native/Load/NativeLibraryFromPath.cs | 4 +- LLama/Native/Load/NativeLibraryUtils.cs | 31 +- LLama/Native/Load/NativeLibraryWithAvx.cs | 23 +- LLama/Native/Load/NativeLibraryWithCuda.cs | 26 +- ...c.cs => NativeLibraryWithMacOrFallback.cs} | 25 +- LLama/Native/NativeApi.Load.cs | 12 - 13 files changed, 54 insertions(+), 476 deletions(-) delete mode 100644 LLama/Native/Load/NativeLibraryDownloadManager.cs rename LLama/Native/Load/{NativeLibraryWithCpuOrMac.cs => NativeLibraryWithMacOrFallback.cs} (63%) diff --git a/LLama.Examples/Program.cs b/LLama.Examples/Program.cs index cee9540be..53edfb3be 100644 --- a/LLama.Examples/Program.cs +++ b/LLama.Examples/Program.cs @@ -18,7 +18,7 @@ __ __ ____ __ """); // Configure logging. Change this to `true` to see log messages from llama.cpp -var showLLamaCppLogs = false; +var showLLamaCppLogs = true; NativeLibraryConfig .All .WithLogCallback((level, message) => @@ -31,7 +31,8 @@ __ __ ____ __ NativeLibraryConfig .All .WithCuda() - .WithAutoDownload().DryRun(); + //.WithAutoDownload() // An experimental feature + .DryRun(); // Calling this method forces loading to occur now. NativeApi.llama_empty_call(); diff --git a/LLama/Abstractions/INativeLibrary.cs b/LLama/Abstractions/INativeLibrary.cs index 64ba182e0..a7e00b753 100644 --- a/LLama/Abstractions/INativeLibrary.cs +++ b/LLama/Abstractions/INativeLibrary.cs @@ -20,11 +20,10 @@ public interface INativeLibrary /// If it's a relative path, LLamaSharp will search the path in the search directies you set. /// /// The system information of the current machine. - /// Whether it's allowed to download from remote. /// The log callback. /// /// The relative paths of the library. You could return multiple paths to try them one by one. If no file is available, please return an empty array. /// - IEnumerable Prepare(SystemInfo systemInfo, bool fromRemote = false, NativeLogConfig.LLamaLogCallback? logCallback = null); + IEnumerable Prepare(SystemInfo systemInfo, NativeLogConfig.LLamaLogCallback? logCallback = null); } } diff --git a/LLama/Abstractions/INativeLibrarySelectingPolicy.cs b/LLama/Abstractions/INativeLibrarySelectingPolicy.cs index f57ae6d1a..41335202e 100644 --- a/LLama/Abstractions/INativeLibrarySelectingPolicy.cs +++ b/LLama/Abstractions/INativeLibrarySelectingPolicy.cs @@ -18,7 +18,7 @@ public interface INativeLibrarySelectingPolicy /// The system information of the current machine. /// The log callback. /// The information of the selected native library files, in order by priority from the beginning to the end. - IEnumerable Select(NativeLibraryConfig.Description description, SystemInfo systemInfo, NativeLogConfig.LLamaLogCallback? logCallback = null); + IEnumerable Apply(NativeLibraryConfig.Description description, SystemInfo systemInfo, NativeLogConfig.LLamaLogCallback? logCallback = null); } #endif } diff --git a/LLama/LLamaSharp.csproj b/LLama/LLamaSharp.csproj index 8277ce9eb..2d8bbefd4 100644 --- a/LLama/LLamaSharp.csproj +++ b/LLama/LLamaSharp.csproj @@ -49,7 +49,6 @@ - diff --git a/LLama/Native/Load/DefaultNativeLibrarySelectingPolicy.cs b/LLama/Native/Load/DefaultNativeLibrarySelectingPolicy.cs index 42433cecb..5cb3b0c5a 100644 --- a/LLama/Native/Load/DefaultNativeLibrarySelectingPolicy.cs +++ b/LLama/Native/Load/DefaultNativeLibrarySelectingPolicy.cs @@ -9,7 +9,7 @@ namespace LLama.Native public class DefaultNativeLibrarySelectingPolicy: INativeLibrarySelectingPolicy { /// - public IEnumerable Select(NativeLibraryConfig.Description description, SystemInfo systemInfo, NativeLogConfig.LLamaLogCallback? logCallback) + public IEnumerable Apply(NativeLibraryConfig.Description description, SystemInfo systemInfo, NativeLogConfig.LLamaLogCallback? logCallback) { List results = new(); @@ -25,7 +25,7 @@ public IEnumerable Select(NativeLibraryConfig.Description descri { if (description.UseCuda) { - yield return new NativeLibraryWithCuda(systemInfo.CudaMajorVersion, description.Library, description.SkipCheck, description.DownloadSettings); + yield return new NativeLibraryWithCuda(systemInfo.CudaMajorVersion, description.Library, description.SkipCheck); } if(!description.UseCuda || description.AllowFallback) @@ -34,25 +34,25 @@ public IEnumerable Select(NativeLibraryConfig.Description descri { // Try all of the AVX levels we can support. if (description.AvxLevel >= AvxLevel.Avx512) - yield return new NativeLibraryWithAvx(description.Library, AvxLevel.Avx512, description.SkipCheck, description.DownloadSettings); + yield return new NativeLibraryWithAvx(description.Library, AvxLevel.Avx512, description.SkipCheck); if (description.AvxLevel >= AvxLevel.Avx2) - yield return new NativeLibraryWithAvx(description.Library, AvxLevel.Avx2, description.SkipCheck, description.DownloadSettings); + yield return new NativeLibraryWithAvx(description.Library, AvxLevel.Avx2, description.SkipCheck); if (description.AvxLevel >= AvxLevel.Avx) - yield return new NativeLibraryWithAvx(description.Library, AvxLevel.Avx, description.SkipCheck, description.DownloadSettings); + yield return new NativeLibraryWithAvx(description.Library, AvxLevel.Avx, description.SkipCheck); - yield return new NativeLibraryWithAvx(description.Library, AvxLevel.None, description.SkipCheck, description.DownloadSettings); + yield return new NativeLibraryWithAvx(description.Library, AvxLevel.None, description.SkipCheck); } else { - yield return new NativeLibraryWithAvx(description.Library, description.AvxLevel, description.SkipCheck, description.DownloadSettings); + yield return new NativeLibraryWithAvx(description.Library, description.AvxLevel, description.SkipCheck); } } if(systemInfo.OSPlatform == OSPlatform.OSX || description.AllowFallback) { - yield return new NativeLibraryWithCpuOrMac(description.Library, description.SkipCheck, description.DownloadSettings); + yield return new NativeLibraryWithMacOrFallback(description.Library, description.SkipCheck); } } } diff --git a/LLama/Native/Load/NativeLibraryConfig.cs b/LLama/Native/Load/NativeLibraryConfig.cs index 6f77927d5..69ae7745a 100644 --- a/LLama/Native/Load/NativeLibraryConfig.cs +++ b/LLama/Native/Load/NativeLibraryConfig.cs @@ -19,8 +19,6 @@ public sealed partial class NativeLibraryConfig private AvxLevel _avxLevel; private bool _allowFallback = true; private bool _skipCheck = false; - private bool _allowAutoDownload = false; - private NativeLibraryDownloadSettings _downloadSettings = NativeLibraryDownloadSettings.Create(); /// /// search directory -> priority level, 0 is the lowest. @@ -29,8 +27,6 @@ public sealed partial class NativeLibraryConfig internal INativeLibrarySelectingPolicy SelectingPolicy { get; private set; } = new DefaultNativeLibrarySelectingPolicy(); - internal bool AllowAutoDownload => _allowAutoDownload; - #region configurators /// /// Load a specified native library as backend for LLamaSharp. @@ -135,25 +131,6 @@ public NativeLibraryConfig WithSearchDirectory(string directory) return this; } - /// - /// Set whether to download the best-matched native library file automatically if there's no backend or specified file to load. - /// You could add a setting here to customize the behavior of the download. - /// - /// If auto-download is enabled, please call after you have finished setting your configurations. - /// - /// - /// - /// - public NativeLibraryConfig WithAutoDownload(bool enable = true, NativeLibraryDownloadSettings? settings = null) - { - ThrowIfLoaded(); - - _allowAutoDownload = enable; - if (settings is not null) - _downloadSettings = settings; - return this; - } - /// /// Set the policy which decides how to select the desired native libraries and order them by priority. /// By default we use . @@ -177,13 +154,6 @@ internal Description CheckAndGatherDescription() var path = _libraryPath; - // Don't modify and pass the original object to `Description`, create a new one instead. - // Also, we need to set the default local directory if the user does not. - var defaultLocalDir = NativeLibraryDownloadSettings.GetDefaultLocalDir(GetCommitHash(_downloadSettings.Tag)); - var downloadSettings = NativeLibraryDownloadSettings.Create() - .WithEndpoint(_downloadSettings.Endpoint).WithEndpointFallbacks(_downloadSettings.EndpointFallbacks ?? []) - .WithRepoId(_downloadSettings.RepoId).WithToken(_downloadSettings.Token).WithTag(_downloadSettings.Tag) - .WithTimeout(_downloadSettings.Timeout).WithLocalDir(_downloadSettings.LocalDir ?? defaultLocalDir); return new Description( path, @@ -192,9 +162,7 @@ internal Description CheckAndGatherDescription() _avxLevel, _allowFallback, _skipCheck, - _searchDirectories.Concat(new[] { "./" }).ToArray(), - _allowAutoDownload, - downloadSettings + _searchDirectories.Concat(new[] { "./" }).ToArray() ); } @@ -216,8 +184,6 @@ internal static string AvxLevelToString(AvxLevel level) private NativeLibraryConfig(NativeLibraryName nativeLibraryName) { NativeLibraryName = nativeLibraryName; - // This value should be changed when we're going to publish new release. (any better approach?) - _downloadSettings = new NativeLibraryDownloadSettings().WithTag(GetCommitHash("master")); // Automatically detect the highest supported AVX level if (System.Runtime.Intrinsics.X86.Avx.IsSupported) @@ -263,10 +229,8 @@ private static bool CheckAVX512() /// /// /// - /// - /// public record Description(string? Path, NativeLibraryName Library, bool UseCuda, AvxLevel AvxLevel, bool AllowFallback, bool SkipCheck, - string[] SearchDirectories, bool AllowAutoDownload, NativeLibraryDownloadSettings DownloadSettings) + string[] SearchDirectories) { /// public override string ToString() @@ -289,9 +253,7 @@ public override string ToString() $"- PreferredAvxLevel: {avxLevelString}\n" + $"- AllowFallback: {AllowFallback}\n" + $"- SkipCheck: {SkipCheck}\n" + - $"- SearchDirectories and Priorities: {searchDirectoriesString}" + - $"- AllowAutoDownload: {AllowAutoDownload}\n" + - $"- DownloadSettings: {DownloadSettings}\n"; + $"- SearchDirectories and Priorities: {searchDirectoriesString}"; } } } @@ -331,17 +293,10 @@ public sealed partial class NativeLibraryConfig {"master", "f7001c"} }; - internal static string GetCommitHash(string version) - { - if(VersionMap.TryGetValue(version, out var hash)) - { - return hash; - } - else - { - return version; - } - } + /// + /// The current version. + /// + public static readonly string CurrentVersion = "master"; // This should be changed before publishing new version. TODO: any better approach? static NativeLibraryConfig() { @@ -363,11 +318,6 @@ private NativeLibraryConfig(NativeLibraryName nativeLibraryName) /// public bool LibraryHasLoaded { get; internal set; } - /// - /// Whether has been called. - /// - internal bool HasCalledDryRun { get; private set; } = false; - internal NativeLibraryName NativeLibraryName { get; } internal NativeLogConfig.LLamaLogCallback? LogCallback { get; private set; } = null; @@ -375,7 +325,10 @@ private NativeLibraryConfig(NativeLibraryName nativeLibraryName) private void ThrowIfLoaded() { if (LibraryHasLoaded) - throw new InvalidOperationException("NativeLibraryConfig must be configured before using **any** other LLamaSharp methods!"); + throw new InvalidOperationException("The library has already loaded, you can't change the configurations. " + + "Please finish the configuration setting before any call to LLamaSharp native APIs." + + "Please use NativeLibraryConfig.DryRun if you want to see whether it's loaded " + + "successfully but still have chance to modify the configurations."); } /// @@ -416,7 +369,6 @@ public NativeLibraryConfig WithLogCallback(ILogger? logger) public bool DryRun() { LogCallback?.Invoke(LLamaLogLevel.Debug, $"Beginning dry run for {this.NativeLibraryName.GetLibraryName()}..."); - HasCalledDryRun = true; return NativeLibraryUtils.TryLoadLibrary(this) != IntPtr.Zero; } } @@ -424,10 +376,17 @@ public bool DryRun() /// /// A class to set same configurations to multiple libraries at the same time. /// - public sealed partial class NativeLibraryConfigContainer + public sealed class NativeLibraryConfigContainer { private NativeLibraryConfig[] _configs; + /// + /// All the configurations in this container. + /// Please avoid calling this property explicitly, use + /// and instead. + /// + public NativeLibraryConfig[] Configs => _configs; + internal NativeLibraryConfigContainer(params NativeLibraryConfig[] configs) { _configs = configs; @@ -554,24 +513,6 @@ public NativeLibraryConfigContainer WithSearchDirectory(string directory) return this; } - /// - /// Set whether to download the best-matched native library file automatically if there's no backend or specified file to load. - /// You could add a setting here to customize the behavior of the download. - /// - /// If auto-download is enabled, please call after you have finished setting your configurations. - /// - /// - /// - /// - public NativeLibraryConfigContainer WithAutoDownload(bool enable = true, NativeLibraryDownloadSettings? settings = null) - { - foreach (var config in _configs) - { - config.WithAutoDownload(enable, settings); - } - return this; - } - /// /// Set the policy which decides how to select the desired native libraries and order them by priority. /// By default we use . diff --git a/LLama/Native/Load/NativeLibraryDownloadManager.cs b/LLama/Native/Load/NativeLibraryDownloadManager.cs deleted file mode 100644 index b9a136991..000000000 --- a/LLama/Native/Load/NativeLibraryDownloadManager.cs +++ /dev/null @@ -1,285 +0,0 @@ -using System; -using System.Collections.Generic; -using System.IO; -using System.Linq; -using System.Threading.Tasks; -using Huggingface; - -namespace LLama.Native -{ - internal class NativeLibraryDownloadManager - { - /// - /// Download the library file - /// - /// - /// - /// - /// The local path of the file if successful otherwise null. - public static async Task DownloadLibraryFile(NativeLibraryDownloadSettings settings, string remoteFilePath, NativeLogConfig.LLamaLogCallback? logCallback = null) - { - HFGlobalConfig.DefaultDownloadTimeout = settings.Timeout; - - HashSet endpointSet = new([settings.Endpoint]); - if (settings.EndpointFallbacks is not null) - { - foreach (var endpoint in settings.EndpointFallbacks) - { - endpointSet.Add(endpoint); - } - } - var endpoints = endpointSet.ToArray(); - - Dictionary exceptionMap = new(); - foreach(var endpoint in endpoints) - { - logCallback?.Invoke(LLamaLogLevel.Debug, $"Downloading the native library file '{remoteFilePath}' from {endpoint} with repo = {settings.RepoId}, tag = {settings.Tag}"); - var path = await HFDownloader.DownloadFileAsync(settings.RepoId, remoteFilePath, revision: settings.Tag, cacheDir: settings.CacheDir, - localDir: settings.LocalDir, token: settings.Token, endpoint: endpoint); - if (path is not null) - { - logCallback?.Invoke(LLamaLogLevel.Debug, $"Successfully downloaded the native library file to {path}"); - return path; - } - else - { - logCallback?.Invoke(LLamaLogLevel.Warning, "The download failed without an explicit error, please check your configuration or report an issue to LLamaSharp."); - } - //try - //{ - // logCallback?.Invoke(LLamaLogLevel.Debug, $"Downloading the native library file '{remoteFilePath}' from {endpoint} with repo = {settings.RepoId}, tag = {settings.Tag}"); - // var path = await HFDownloader.DownloadFileAsync(settings.RepoId, remoteFilePath, revision: settings.Tag, cacheDir: settings.CacheDir, - // localDir: settings.LocalDir, token: settings.Token, endpoint: endpoint); - // if(path is not null) - // { - // logCallback?.Invoke(LLamaLogLevel.Debug, $"Successfully downloaded the native library file to {path}"); - // return path; - // } - // else - // { - // logCallback?.Invoke(LLamaLogLevel.Warning, "The download failed without an explicit error, please check your configuration or report an issue to LLamaSharp."); - // } - //} - //catch(Exception ex) - //{ - // logCallback?.Invoke(LLamaLogLevel.Warning, $"An exception was thrown when downloading the native library file from {endpoint}: {ex.Message}"); - //} - } - - // means that the download finally fails. - return null; - } - } - - /// - /// Settings for downloading the native library. - /// - public class NativeLibraryDownloadSettings - { - /// - /// The endpoint to download from, by default the official site of HuggingFace. - /// - public string Endpoint { get; private set; } = "https://huggingface.co"; - - /// - /// Endpoints to fallback to if downloading with the main endpoint fails. - /// - /// Generally this is an option for those countries or regions where the main endpoint is blocked. - /// You should not put too many endpoints here, as it will slow down the downloading process. - /// - public string[]? EndpointFallbacks { get; private set; } = null; - - /// - /// The version of the library to download. Please use LLamaSharp version in format `[major].[minor].[patch]` as tag - /// or go to https://huggingface.co/AsakusaRinne/LLamaSharpNative - /// to see all available tags, or use your own repo and tags. - /// - public string Tag { get; private set; } = string.Empty; - - /// - /// The repo id to download the native library files. - /// - public string RepoId { get; private set; } = "AsakusaRinne/LLamaSharpNative"; - - /// - /// The directory to cache the downloaded files. If you only want to make the downloaded files appear in a directory, - /// regardless of whether the file will have a copy in another place, please set instead. - /// - public string CacheDir { get; private set; } - - /// - /// If provided, the downloaded file will be placed under this directory, - /// either as a symlink (default) or a regular file. - /// - public string? LocalDir { get; private set; } = null; - - /// - /// If you are using your own private repo as remote source, you could set the token to get the access. - /// - public string? Token { get; private set; } = null; - - /// - /// The timeout (second) of the native library file download. - /// - public int Timeout { get; private set; } = 10; - - /// - /// Extra search directories. They will only be used when finding files downloaded from remote. - /// Generally it will be useful when you wnat to replace the downloading process with your custom implementation. - /// If you are not sure how it works, please leave it empty. - /// - public string[]? ExtraSearchDirectories { get;private set; } = null; - - internal NativeLibraryDownloadSettings() - { - var home = Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.UserProfile), ".cache"); - CacheDir = Path.Combine(home, "llama_sharp"); - } - - internal static string GetDefaultLocalDir(string tag) - { - var home = Environment.GetFolderPath(Environment.SpecialFolder.UserProfile); - return Path.Combine(home, ".llama_sharp", tag); - } - - /// - /// Create a with default settings. - /// - /// - public static NativeLibraryDownloadSettings Create() - { - return new NativeLibraryDownloadSettings(); - } - - /// - /// Set the default endpoint to download file from. - /// - /// - /// - public NativeLibraryDownloadSettings WithEndpoint(string endpoint) - { - Endpoint = endpoint; - return this; - } - - /// - /// Set the endpoints to try when the download fails with the default endpoint. - /// - /// - /// - public NativeLibraryDownloadSettings WithEndpointFallbacks(params string[] endpoints) - { - EndpointFallbacks = endpoints; - return this; - } - - /// - /// Set the - /// - /// - /// - public NativeLibraryDownloadSettings WithTag(string tag) - { - Tag = tag; - return this; - } - - /// - /// Set the - /// - /// - /// - public NativeLibraryDownloadSettings WithRepoId(string repoId) - { - RepoId = repoId; - return this; - } - - /// - /// Set the . If you only want to make the downloaded files appear in a directory, - /// regardless of whether the file may have a copy in another place, please use instead. - /// - /// - /// - public NativeLibraryDownloadSettings WithCacheDir(string cacheDir) - { - CacheDir = cacheDir; - return this; - } - - /// - /// Set the - /// - /// - /// - public NativeLibraryDownloadSettings WithLocalDir(string localDir) - { - LocalDir = localDir; - return this; - } - - /// - /// Set the - /// - /// - /// - public NativeLibraryDownloadSettings WithToken(string token) - { - Token = token; - return this; - } - - /// - /// Set the - /// - /// - /// - public NativeLibraryDownloadSettings WithTimeout(int timeout) - { - Timeout = timeout; - return this; - } - - /// - /// Set . They will only be used when finding files downloaded from remote. - /// Generally it will be useful when you wnat to replace the downloading process with your custom implementation. - /// If you are not sure how it works, please ignore this method. - /// - /// - /// - public NativeLibraryDownloadSettings WithExtraSearchDirectories(string[] directories) - { - ExtraSearchDirectories = directories; - return this; - } - - /// - public override string ToString() - { - // Token should be hidden when printing it. - string hiddenToken = ""; - if(Token is not null) - { - if(Token.Length <= 10) - { - hiddenToken = new string('*', Token.Length - 1) + Token.Last(); - } - else - { - hiddenToken += Token.Substring(0, 2); - hiddenToken += new string('*', Token.Length - 3); - hiddenToken += Token.Last(); - } - } - - return $"(Endpoint = {Endpoint}, " + - $"EndpointFallbacks = {string.Join(", ", EndpointFallbacks ?? new string[0])}, " + - $"Tag = {Tag}, " + - $"RepoId = {RepoId}, " + - $"CacheDir = {CacheDir}, " + - $"LocalDir = {LocalDir}, " + - $"Token = {hiddenToken}, " + - $"Timeout = {Timeout}s)"; - } - } -} diff --git a/LLama/Native/Load/NativeLibraryFromPath.cs b/LLama/Native/Load/NativeLibraryFromPath.cs index 45c7e48ec..c3e62a223 100644 --- a/LLama/Native/Load/NativeLibraryFromPath.cs +++ b/LLama/Native/Load/NativeLibraryFromPath.cs @@ -23,9 +23,9 @@ public NativeLibraryFromPath(string path) } /// - public IEnumerable Prepare(SystemInfo systemInfo, bool fromRemote, NativeLogConfig.LLamaLogCallback? logCallback) + public IEnumerable Prepare(SystemInfo systemInfo, NativeLogConfig.LLamaLogCallback? logCallback) { - return fromRemote? [] : [_path]; + return [_path]; } } } diff --git a/LLama/Native/Load/NativeLibraryUtils.cs b/LLama/Native/Load/NativeLibraryUtils.cs index 513f6ccc2..e3ca06e14 100644 --- a/LLama/Native/Load/NativeLibraryUtils.cs +++ b/LLama/Native/Load/NativeLibraryUtils.cs @@ -33,12 +33,12 @@ internal static IntPtr TryLoadLibrary(NativeLibraryConfig config) Log(description.ToString(), LLamaLogLevel.Info, config.LogCallback); // Get the libraries ordered by priority from the selecting policy. - var libraries = config.SelectingPolicy.Select(description, systemInfo, config.LogCallback); + var libraries = config.SelectingPolicy.Apply(description, systemInfo, config.LogCallback); foreach (var library in libraries) { // Prepare the local library file and get the path. - var paths = library.Prepare(systemInfo, false, config.LogCallback); + var paths = library.Prepare(systemInfo, config.LogCallback); foreach (var path in paths) { Log($"Got relative library path '{path}' from local with {library.Metadata}, trying to load it...", LLamaLogLevel.Debug, config.LogCallback); @@ -49,33 +49,6 @@ internal static IntPtr TryLoadLibrary(NativeLibraryConfig config) return result; } } - // If we failed but auto-download is allowed, try to prepare the file from remote. - if (description.AllowAutoDownload) - { - paths = library.Prepare(systemInfo, true, config.LogCallback); - if (description.DownloadSettings.LocalDir is null) - { - // Null local directory is not expected here (it will make things more complex if we want to handle it). - // It should always be set when gathering the description. - throw new RuntimeError("Auto-download is enabled for native library but the `LocalDir` is null. " + - "It's an unexpected behavior and please report an issue to LLamaSharp."); - } - // When using auto-download, this should be the only search directory. - List searchDirectoriesForDownload = [description.DownloadSettings.LocalDir]; - // unless extra search paths are added by the user. - searchDirectoriesForDownload.AddRange(description.DownloadSettings.ExtraSearchDirectories ?? []); - - foreach (var path in paths) - { - Log($"Got relative library path '{path}' from remote with {library.Metadata}, trying to load it...", LLamaLogLevel.Debug, config.LogCallback); - - var result = TryLoad(path, searchDirectoriesForDownload, config.LogCallback); - if (result != IntPtr.Zero) - { - return result; - } - } - } } // If fallback is allowed, we will make the last try (the default system loading) when calling the native api. diff --git a/LLama/Native/Load/NativeLibraryWithAvx.cs b/LLama/Native/Load/NativeLibraryWithAvx.cs index 5b1f65142..7b5421b4d 100644 --- a/LLama/Native/Load/NativeLibraryWithAvx.cs +++ b/LLama/Native/Load/NativeLibraryWithAvx.cs @@ -13,7 +13,6 @@ public class NativeLibraryWithAvx : INativeLibrary private NativeLibraryName _libraryName; private AvxLevel _avxLevel; private bool _skipCheck; - private NativeLibraryDownloadSettings _downloadSettings; /// public NativeLibraryMetadata? Metadata @@ -30,45 +29,33 @@ public NativeLibraryMetadata? Metadata /// /// /// - /// - public NativeLibraryWithAvx(NativeLibraryName libraryName, AvxLevel avxLevel, bool skipCheck, NativeLibraryDownloadSettings downloadSettings) + public NativeLibraryWithAvx(NativeLibraryName libraryName, AvxLevel avxLevel, bool skipCheck) { _libraryName = libraryName; _avxLevel = avxLevel; _skipCheck = skipCheck; - _downloadSettings = downloadSettings; } /// - public IEnumerable Prepare(SystemInfo systemInfo, bool fromRemote, NativeLogConfig.LLamaLogCallback? logCallback) + public IEnumerable Prepare(SystemInfo systemInfo, NativeLogConfig.LLamaLogCallback? logCallback) { if (systemInfo.OSPlatform != OSPlatform.Windows && systemInfo.OSPlatform != OSPlatform.Linux && !_skipCheck) { // Not supported on systems other than Windows and Linux. return []; } - var path = GetAvxPath(systemInfo, _avxLevel, fromRemote, logCallback); + var path = GetAvxPath(systemInfo, _avxLevel, logCallback); return path is null ? [] : [path]; } - private string? GetAvxPath(SystemInfo systemInfo, AvxLevel avxLevel, bool fromRemote, NativeLogConfig.LLamaLogCallback? logCallback) + private string? GetAvxPath(SystemInfo systemInfo, AvxLevel avxLevel, NativeLogConfig.LLamaLogCallback? logCallback) { NativeLibraryUtils.GetPlatformPathParts(systemInfo.OSPlatform, out var os, out var fileExtension, out var libPrefix); var avxStr = NativeLibraryConfig.AvxLevelToString(avxLevel); if (!string.IsNullOrEmpty(avxStr)) avxStr += "/"; var relativePath = $"runtimes/{os}/native/{avxStr}{libPrefix}{_libraryName.GetLibraryName()}{fileExtension}"; - - if (fromRemote) - { - // Download and return the local path. - // We make it sychronize because we c'd better not use async method when loading library later. - return NativeLibraryDownloadManager.DownloadLibraryFile(_downloadSettings, relativePath, logCallback).Result; - } - else - { - return relativePath; - } + return relativePath; } } #endif diff --git a/LLama/Native/Load/NativeLibraryWithCuda.cs b/LLama/Native/Load/NativeLibraryWithCuda.cs index 1e2840d5e..d3b06b864 100644 --- a/LLama/Native/Load/NativeLibraryWithCuda.cs +++ b/LLama/Native/Load/NativeLibraryWithCuda.cs @@ -14,7 +14,6 @@ public class NativeLibraryWithCuda : INativeLibrary private NativeLibraryName _libraryName; private AvxLevel _avxLevel; private bool _skipCheck; - private NativeLibraryDownloadSettings _downloadSettings; /// public NativeLibraryMetadata? Metadata @@ -31,17 +30,15 @@ public NativeLibraryMetadata? Metadata /// /// /// - /// - public NativeLibraryWithCuda(int majorCudaVersion, NativeLibraryName libraryName, bool skipCheck, NativeLibraryDownloadSettings downloadSettings) + public NativeLibraryWithCuda(int majorCudaVersion, NativeLibraryName libraryName, bool skipCheck) { _majorCudaVersion = majorCudaVersion; _libraryName = libraryName; _skipCheck = skipCheck; - _downloadSettings = downloadSettings; } /// - public IEnumerable Prepare(SystemInfo systemInfo, bool fromRemote, NativeLogConfig.LLamaLogCallback? logCallback) + public IEnumerable Prepare(SystemInfo systemInfo, NativeLogConfig.LLamaLogCallback? logCallback) { // TODO: Avx level is ignored now, needs to be implemented in the future. if (systemInfo.OSPlatform == OSPlatform.Windows || systemInfo.OSPlatform == OSPlatform.Linux || _skipCheck) @@ -49,12 +46,12 @@ public IEnumerable Prepare(SystemInfo systemInfo, bool fromRemote, Nativ if (_majorCudaVersion == -1 && _skipCheck) { // Currently only 11 and 12 are supported. - var cuda12LibraryPath = GetCudaPath(systemInfo, 12, fromRemote, logCallback); + var cuda12LibraryPath = GetCudaPath(systemInfo, 12, logCallback); if (cuda12LibraryPath is not null) { yield return cuda12LibraryPath; } - var cuda11LibraryPath = GetCudaPath(systemInfo, 11, fromRemote, logCallback); + var cuda11LibraryPath = GetCudaPath(systemInfo, 11, logCallback); if (cuda11LibraryPath is not null) { yield return cuda11LibraryPath; @@ -62,7 +59,7 @@ public IEnumerable Prepare(SystemInfo systemInfo, bool fromRemote, Nativ } else if (_majorCudaVersion != -1) { - var cudaLibraryPath = GetCudaPath(systemInfo, _majorCudaVersion, fromRemote, logCallback); + var cudaLibraryPath = GetCudaPath(systemInfo, _majorCudaVersion, logCallback); if (cudaLibraryPath is not null) { yield return cudaLibraryPath; @@ -71,20 +68,11 @@ public IEnumerable Prepare(SystemInfo systemInfo, bool fromRemote, Nativ } } - private string? GetCudaPath(SystemInfo systemInfo, int cudaVersion, bool remote, NativeLogConfig.LLamaLogCallback? logCallback) + private string? GetCudaPath(SystemInfo systemInfo, int cudaVersion, NativeLogConfig.LLamaLogCallback? logCallback) { NativeLibraryUtils.GetPlatformPathParts(systemInfo.OSPlatform, out var os, out var fileExtension, out var libPrefix); var relativePath = $"runtimes/{os}/native/cuda{cudaVersion}/{libPrefix}{_libraryName.GetLibraryName()}{fileExtension}"; - if (remote) - { - // Download and return the local path. - // We make it sychronize because we c'd better not use async method when loading library later. - return NativeLibraryDownloadManager.DownloadLibraryFile(_downloadSettings, relativePath, logCallback).Result; - } - else - { - return relativePath; - } + return relativePath; } } #endif diff --git a/LLama/Native/Load/NativeLibraryWithCpuOrMac.cs b/LLama/Native/Load/NativeLibraryWithMacOrFallback.cs similarity index 63% rename from LLama/Native/Load/NativeLibraryWithCpuOrMac.cs rename to LLama/Native/Load/NativeLibraryWithMacOrFallback.cs index 3bc09b13e..5df339307 100644 --- a/LLama/Native/Load/NativeLibraryWithCpuOrMac.cs +++ b/LLama/Native/Load/NativeLibraryWithMacOrFallback.cs @@ -8,12 +8,10 @@ namespace LLama.Native /// /// A native library compiled on Mac, or fallbacks from all other libraries in the selection. /// - public class NativeLibraryWithCpuOrMac - : INativeLibrary + public class NativeLibraryWithMacOrFallback : INativeLibrary { private NativeLibraryName _libraryName; private bool _skipCheck; - private NativeLibraryDownloadSettings _downloadSettings; /// public NativeLibraryMetadata? Metadata @@ -29,22 +27,20 @@ public NativeLibraryMetadata? Metadata /// /// /// - /// - public NativeLibraryWithCpuOrMac(NativeLibraryName libraryName, bool skipCheck, NativeLibraryDownloadSettings downloadSettings) + public NativeLibraryWithMacOrFallback(NativeLibraryName libraryName, bool skipCheck) { _libraryName = libraryName; _skipCheck = skipCheck; - _downloadSettings = downloadSettings; } /// - public IEnumerable Prepare(SystemInfo systemInfo, bool fromRemote, NativeLogConfig.LLamaLogCallback? logCallback) + public IEnumerable Prepare(SystemInfo systemInfo, NativeLogConfig.LLamaLogCallback? logCallback) { - var path = GetPath(systemInfo, AvxLevel.None, fromRemote, logCallback); + var path = GetPath(systemInfo, AvxLevel.None, logCallback); return path is null ?[] : [path]; } - private string? GetPath(SystemInfo systemInfo, AvxLevel avxLevel, bool fromRemote, NativeLogConfig.LLamaLogCallback? logCallback) + private string? GetPath(SystemInfo systemInfo, AvxLevel avxLevel, NativeLogConfig.LLamaLogCallback? logCallback) { NativeLibraryUtils.GetPlatformPathParts(systemInfo.OSPlatform, out var os, out var fileExtension, out var libPrefix); string relativePath; @@ -61,16 +57,7 @@ public IEnumerable Prepare(SystemInfo systemInfo, bool fromRemote, Nativ relativePath = $"runtimes/{os}/native/{avxStr}{libPrefix}{_libraryName.GetLibraryName()}{fileExtension}"; } - if (fromRemote) - { - // Download and return the local path. - // We make it sychronize because we c'd better not use async method when loading library later. - return NativeLibraryDownloadManager.DownloadLibraryFile(_downloadSettings, relativePath, logCallback).Result; - } - else - { - return relativePath; - } + return relativePath; } } #endif diff --git a/LLama/Native/NativeApi.Load.cs b/LLama/Native/NativeApi.Load.cs index b3c9a6ad2..277555e7b 100644 --- a/LLama/Native/NativeApi.Load.cs +++ b/LLama/Native/NativeApi.Load.cs @@ -64,12 +64,6 @@ private static void SetDllImportResolver() if (_loadedLlamaHandle != IntPtr.Zero) return _loadedLlamaHandle; - // We don't allow downloading in static constructor to avoid potentially uncertain behaviors now. - if (NativeLibraryConfig.LLama.AllowAutoDownload && !NativeLibraryConfig.LLama.HasCalledDryRun) - { - throw new RuntimeError("When using auto-download, please call NativeLibraryConfig.DryRun() right after finishing you configurations."); - } - // Try to load a preferred library, based on CPU feature detection _loadedLlamaHandle = NativeLibraryUtils.TryLoadLibrary(NativeLibraryConfig.LLama); return _loadedLlamaHandle; @@ -81,12 +75,6 @@ private static void SetDllImportResolver() if (_loadedLlavaSharedHandle != IntPtr.Zero) return _loadedLlavaSharedHandle; - // We don't allow downloading in static constructor to avoid potentially uncertain behaviors now. - if (NativeLibraryConfig.LLavaShared.AllowAutoDownload && !NativeLibraryConfig.LLavaShared.HasCalledDryRun) - { - throw new RuntimeError("When using auto-download, please call NativeLibraryConfig.DryRun() right after finishing you configurations."); - } - // Try to load a preferred library, based on CPU feature detection _loadedLlavaSharedHandle = NativeLibraryUtils.TryLoadLibrary(NativeLibraryConfig.LLavaShared); return _loadedLlavaSharedHandle; From f0e2a3dc457643ab33c95866302e1e647b7f5df6 Mon Sep 17 00:00:00 2001 From: Rinne Date: Fri, 26 Apr 2024 09:18:52 +0800 Subject: [PATCH 3/7] Update LLama/Native/Load/NativeLibraryFromPath.cs Co-authored-by: Martin Evans --- LLama/Native/Load/NativeLibraryFromPath.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/LLama/Native/Load/NativeLibraryFromPath.cs b/LLama/Native/Load/NativeLibraryFromPath.cs index c3e62a223..8cd99c308 100644 --- a/LLama/Native/Load/NativeLibraryFromPath.cs +++ b/LLama/Native/Load/NativeLibraryFromPath.cs @@ -4,7 +4,7 @@ namespace LLama.Native { /// - /// A native library specified with a lcoal file path. + /// A native library specified with a local file path. /// public class NativeLibraryFromPath: INativeLibrary { From 31ff3636305f15a360cc2a8b96d7a570864bc61d Mon Sep 17 00:00:00 2001 From: Rinne Date: Sat, 27 Apr 2024 02:23:36 +0800 Subject: [PATCH 4/7] fix: resolve comments. --- LLama.Examples/Program.cs | 4 +- LLama/Native/Load/NativeLibraryConfig.cs | 59 ++++++++++++++--------- LLama/Native/Load/NativeLibraryUtils.cs | 9 +++- LLama/Native/Load/UnknownNativeLibrary.cs | 25 ++++++++++ LLama/Native/NativeApi.Load.cs | 4 +- 5 files changed, 72 insertions(+), 29 deletions(-) create mode 100644 LLama/Native/Load/UnknownNativeLibrary.cs diff --git a/LLama.Examples/Program.cs b/LLama.Examples/Program.cs index 53edfb3be..63114120d 100644 --- a/LLama.Examples/Program.cs +++ b/LLama.Examples/Program.cs @@ -18,7 +18,7 @@ __ __ ____ __ """); // Configure logging. Change this to `true` to see log messages from llama.cpp -var showLLamaCppLogs = true; +var showLLamaCppLogs = false; NativeLibraryConfig .All .WithLogCallback((level, message) => @@ -32,7 +32,7 @@ __ __ ____ __ .All .WithCuda() //.WithAutoDownload() // An experimental feature - .DryRun(); + .DryRun(out var loadedllamaLibrary, out var loadedLLavaLibrary); // Calling this method forces loading to occur now. NativeApi.llama_empty_call(); diff --git a/LLama/Native/Load/NativeLibraryConfig.cs b/LLama/Native/Load/NativeLibraryConfig.cs index 69ae7745a..3d9fe69ac 100644 --- a/LLama/Native/Load/NativeLibraryConfig.cs +++ b/LLama/Native/Load/NativeLibraryConfig.cs @@ -265,7 +265,7 @@ public sealed partial class NativeLibraryConfig /// Set configurations for all the native libraries, including LLama and LLava /// [Obsolete("Please use NativeLibraryConfig.All instead, or set configurations for NativeLibraryConfig.LLama and NativeLibraryConfig.LLavaShared respectively.")] - public static NativeLibraryConfigContainer Instance { get; } + public static NativeLibraryConfigContainer Instance => All; /// /// Set configurations for all the native libraries, including LLama and LLava @@ -282,28 +282,26 @@ public sealed partial class NativeLibraryConfig /// public static NativeLibraryConfig LLavaShared { get; } + /// - /// A dictionary mapping from version to corresponding llama.cpp commit hash. - /// The version should be formatted int `[major].[minor].[patch]`. But there's an exceptance that you can - /// use `master` as a version to get the llama.cpp commit hash from the master branch. + /// The current version. /// - public static Dictionary VersionMap { get; } = new Dictionary() - // This value should be changed when we're going to publish new release. (any better approach?) - { - {"master", "f7001c"} - }; + public static string CurrentVersion => VERSION; // This should be changed before publishing new version. TODO: any better approach? + + private const string COMMIT_HASH = "f7001c"; + private const string VERSION = "master"; /// - /// The current version. + /// Get the llama.cpp commit hash of the current version. /// - public static readonly string CurrentVersion = "master"; // This should be changed before publishing new version. TODO: any better approach? + /// + public static string GetNativeLibraryCommitHash() => COMMIT_HASH; static NativeLibraryConfig() { LLama = new(NativeLibraryName.Llama); LLavaShared = new(NativeLibraryName.LlavaShared); All = new(LLama, LLavaShared); - Instance = All; } #if NETSTANDARD2_0 @@ -365,11 +363,15 @@ public NativeLibraryConfig WithLogCallback(ILogger? logger) /// /// You can still modify the configuration after this calling but only before any call from . /// + /// + /// The loaded livrary. When the loading failed, this will be null. + /// However if you are using .NET standard2.0, this will never return null. + /// /// Whether the running is successful. - public bool DryRun() + public bool DryRun(out INativeLibrary? loadedLibrary) { LogCallback?.Invoke(LLamaLogLevel.Debug, $"Beginning dry run for {this.NativeLibraryName.GetLibraryName()}..."); - return NativeLibraryUtils.TryLoadLibrary(this) != IntPtr.Zero; + return NativeLibraryUtils.TryLoadLibrary(this, out loadedLibrary) != IntPtr.Zero; } } @@ -380,13 +382,6 @@ public sealed class NativeLibraryConfigContainer { private NativeLibraryConfig[] _configs; - /// - /// All the configurations in this container. - /// Please avoid calling this property explicitly, use - /// and instead. - /// - public NativeLibraryConfig[] Configs => _configs; - internal NativeLibraryConfigContainer(params NativeLibraryConfig[] configs) { _configs = configs; @@ -566,9 +561,27 @@ public NativeLibraryConfigContainer WithLogCallback(ILogger? logger) /// You can still modify the configuration after this calling but only before any call from . /// /// Whether the running is successful. - public bool DryRun() + public bool DryRun(out INativeLibrary? loadedLLamaNativeLibrary, out INativeLibrary? loadedLLavaNativeLibrary) { - return _configs.All(config => config.DryRun()); + bool success = true; + foreach(var config in _configs) + { + success &= config.DryRun(out var loadedLibrary); + if(config.NativeLibraryName == NativeLibraryName.Llama) + { + loadedLLamaNativeLibrary = loadedLibrary; + } + else if(config.NativeLibraryName == NativeLibraryName.LlavaShared) + { + loadedLLavaNativeLibrary = loadedLibrary; + } + else + { + throw new Exception("Unknown native library config during the dry run."); + } + } + loadedLLamaNativeLibrary = loadedLLavaNativeLibrary = null; + return success; } } diff --git a/LLama/Native/Load/NativeLibraryUtils.cs b/LLama/Native/Load/NativeLibraryUtils.cs index e3ca06e14..9dd7c8af1 100644 --- a/LLama/Native/Load/NativeLibraryUtils.cs +++ b/LLama/Native/Load/NativeLibraryUtils.cs @@ -1,4 +1,5 @@ -using LLama.Exceptions; +using LLama.Abstractions; +using LLama.Exceptions; using System; using System.Collections.Generic; using System.IO; @@ -12,7 +13,7 @@ internal static class NativeLibraryUtils /// Try to load libllama/llava_shared, using CPU feature detection to try and load a more specialised DLL if possible /// /// The library handle to unload later, or IntPtr.Zero if no library was loaded - internal static IntPtr TryLoadLibrary(NativeLibraryConfig config) + internal static IntPtr TryLoadLibrary(NativeLibraryConfig config, out INativeLibrary? loadedLibrary) { #if NET6_0_OR_GREATER var description = config.CheckAndGatherDescription(); @@ -46,6 +47,7 @@ internal static IntPtr TryLoadLibrary(NativeLibraryConfig config) var result = TryLoad(path, description.SearchDirectories, config.LogCallback); if (result != IntPtr.Zero) { + loadedLibrary = library; return result; } } @@ -57,6 +59,9 @@ internal static IntPtr TryLoadLibrary(NativeLibraryConfig config) { throw new RuntimeError("Failed to load the native library. Please check the log for more information."); } + loadedLibrary = null; +#else + loadedLibrary = new UnknownNativeLibrary(); #endif Log($"No library was loaded before calling native apis. " + diff --git a/LLama/Native/Load/UnknownNativeLibrary.cs b/LLama/Native/Load/UnknownNativeLibrary.cs new file mode 100644 index 000000000..823e77a38 --- /dev/null +++ b/LLama/Native/Load/UnknownNativeLibrary.cs @@ -0,0 +1,25 @@ +using LLama.Abstractions; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace LLama.Native +{ + /// + /// When you are using .NET standard2.0, dynamic native library loading is not supported. + /// This class will be returned in . + /// + public class UnknownNativeLibrary: INativeLibrary + { + /// + public NativeLibraryMetadata? Metadata => null; + + /// + public IEnumerable Prepare(SystemInfo systemInfo, NativeLogConfig.LLamaLogCallback? logCallback = null) + { + throw new NotImplementedException("This class is only a placeholder and should not be used to load native library."); + } + } +} diff --git a/LLama/Native/NativeApi.Load.cs b/LLama/Native/NativeApi.Load.cs index 277555e7b..3c3edcb03 100644 --- a/LLama/Native/NativeApi.Load.cs +++ b/LLama/Native/NativeApi.Load.cs @@ -65,7 +65,7 @@ private static void SetDllImportResolver() return _loadedLlamaHandle; // Try to load a preferred library, based on CPU feature detection - _loadedLlamaHandle = NativeLibraryUtils.TryLoadLibrary(NativeLibraryConfig.LLama); + _loadedLlamaHandle = NativeLibraryUtils.TryLoadLibrary(NativeLibraryConfig.LLama, out var _); return _loadedLlamaHandle; } @@ -76,7 +76,7 @@ private static void SetDllImportResolver() return _loadedLlavaSharedHandle; // Try to load a preferred library, based on CPU feature detection - _loadedLlavaSharedHandle = NativeLibraryUtils.TryLoadLibrary(NativeLibraryConfig.LLavaShared); + _loadedLlavaSharedHandle = NativeLibraryUtils.TryLoadLibrary(NativeLibraryConfig.LLavaShared, out var _); return _loadedLlavaSharedHandle; } From 2c19b8b8311d7a0ad158eb2d4c33d74282b22591 Mon Sep 17 00:00:00 2001 From: Rinne Date: Sat, 27 Apr 2024 03:13:24 +0800 Subject: [PATCH 5/7] rename the llava library name. --- LLama/Native/Load/NativeLibraryConfig.cs | 6 +++--- LLama/Native/NativeApi.Load.cs | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/LLama/Native/Load/NativeLibraryConfig.cs b/LLama/Native/Load/NativeLibraryConfig.cs index 3d9fe69ac..720ad772f 100644 --- a/LLama/Native/Load/NativeLibraryConfig.cs +++ b/LLama/Native/Load/NativeLibraryConfig.cs @@ -280,7 +280,7 @@ public sealed partial class NativeLibraryConfig /// /// Configuration for LLava native library /// - public static NativeLibraryConfig LLavaShared { get; } + public static NativeLibraryConfig LLava { get; } /// @@ -300,8 +300,8 @@ public sealed partial class NativeLibraryConfig static NativeLibraryConfig() { LLama = new(NativeLibraryName.Llama); - LLavaShared = new(NativeLibraryName.LlavaShared); - All = new(LLama, LLavaShared); + LLava = new(NativeLibraryName.LlavaShared); + All = new(LLama, LLava); } #if NETSTANDARD2_0 diff --git a/LLama/Native/NativeApi.Load.cs b/LLama/Native/NativeApi.Load.cs index 3c3edcb03..927f63e82 100644 --- a/LLama/Native/NativeApi.Load.cs +++ b/LLama/Native/NativeApi.Load.cs @@ -19,7 +19,7 @@ static NativeApi() // Set flag to indicate that this point has been passed. No native library config can be done after this point. NativeLibraryConfig.LLama.LibraryHasLoaded = true; - NativeLibraryConfig.LLavaShared.LibraryHasLoaded = true; + NativeLibraryConfig.LLava.LibraryHasLoaded = true; // Immediately make a call which requires loading the llama DLL. This method call // can't fail unless the DLL hasn't been loaded. @@ -76,7 +76,7 @@ private static void SetDllImportResolver() return _loadedLlavaSharedHandle; // Try to load a preferred library, based on CPU feature detection - _loadedLlavaSharedHandle = NativeLibraryUtils.TryLoadLibrary(NativeLibraryConfig.LLavaShared, out var _); + _loadedLlavaSharedHandle = NativeLibraryUtils.TryLoadLibrary(NativeLibraryConfig.LLava, out var _); return _loadedLlavaSharedHandle; } From a86f14d1fb6c215bd92094181664c39c8f8c1f20 Mon Sep 17 00:00:00 2001 From: Rinne Date: Thu, 2 May 2024 07:44:25 +0800 Subject: [PATCH 6/7] Add an API to get the loaded native library. --- LLama/Native/Load/NativeLibraryConfig.cs | 20 +++++++++--------- LLama/Native/NativeApi.Load.cs | 27 ++++++++++++++++++++---- 2 files changed, 33 insertions(+), 14 deletions(-) diff --git a/LLama/Native/Load/NativeLibraryConfig.cs b/LLama/Native/Load/NativeLibraryConfig.cs index 720ad772f..4d158ebff 100644 --- a/LLama/Native/Load/NativeLibraryConfig.cs +++ b/LLama/Native/Load/NativeLibraryConfig.cs @@ -299,8 +299,8 @@ public sealed partial class NativeLibraryConfig static NativeLibraryConfig() { - LLama = new(NativeLibraryName.Llama); - LLava = new(NativeLibraryName.LlavaShared); + LLama = new(NativeLibraryName.LLama); + LLava = new(NativeLibraryName.LLava); All = new(LLama, LLava); } @@ -401,11 +401,11 @@ public NativeLibraryConfigContainer WithLibrary(string? llamaPath, string? llava { foreach(var config in _configs) { - if(config.NativeLibraryName == NativeLibraryName.Llama && llamaPath is not null) + if(config.NativeLibraryName == NativeLibraryName.LLama && llamaPath is not null) { config.WithLibrary(llamaPath); } - if(config.NativeLibraryName == NativeLibraryName.LlavaShared && llavaPath is not null) + if(config.NativeLibraryName == NativeLibraryName.LLava && llavaPath is not null) { config.WithLibrary(llavaPath); } @@ -567,11 +567,11 @@ public bool DryRun(out INativeLibrary? loadedLLamaNativeLibrary, out INativeLibr foreach(var config in _configs) { success &= config.DryRun(out var loadedLibrary); - if(config.NativeLibraryName == NativeLibraryName.Llama) + if(config.NativeLibraryName == NativeLibraryName.LLama) { loadedLLamaNativeLibrary = loadedLibrary; } - else if(config.NativeLibraryName == NativeLibraryName.LlavaShared) + else if(config.NativeLibraryName == NativeLibraryName.LLava) { loadedLLavaNativeLibrary = loadedLibrary; } @@ -593,11 +593,11 @@ public enum NativeLibraryName /// /// The native library compiled from llama.cpp. /// - Llama, + LLama, /// /// The native library compiled from the LLaVA example of llama.cpp. /// - LlavaShared + LLava } internal static class LibraryNameExtensions @@ -606,9 +606,9 @@ public static string GetLibraryName(this NativeLibraryName name) { switch (name) { - case NativeLibraryName.Llama: + case NativeLibraryName.LLama: return NativeApi.libraryName; - case NativeLibraryName.LlavaShared: + case NativeLibraryName.LLava: return NativeApi.llavaLibraryName; default: throw new ArgumentOutOfRangeException(nameof(name), name, null); diff --git a/LLama/Native/NativeApi.Load.cs b/LLama/Native/NativeApi.Load.cs index 927f63e82..cdf4c8e97 100644 --- a/LLama/Native/NativeApi.Load.cs +++ b/LLama/Native/NativeApi.Load.cs @@ -4,6 +4,7 @@ using System.Runtime.InteropServices; using System.Text.Json; using System.Collections.Generic; +using LLama.Abstractions; namespace LLama.Native { @@ -65,7 +66,7 @@ private static void SetDllImportResolver() return _loadedLlamaHandle; // Try to load a preferred library, based on CPU feature detection - _loadedLlamaHandle = NativeLibraryUtils.TryLoadLibrary(NativeLibraryConfig.LLama, out var _); + _loadedLlamaHandle = NativeLibraryUtils.TryLoadLibrary(NativeLibraryConfig.LLama, out _loadedLLamaLibrary); return _loadedLlamaHandle; } @@ -76,7 +77,7 @@ private static void SetDllImportResolver() return _loadedLlavaSharedHandle; // Try to load a preferred library, based on CPU feature detection - _loadedLlavaSharedHandle = NativeLibraryUtils.TryLoadLibrary(NativeLibraryConfig.LLava, out var _); + _loadedLlavaSharedHandle = NativeLibraryUtils.TryLoadLibrary(NativeLibraryConfig.LLava, out _loadedLLavaLibrary); return _loadedLlavaSharedHandle; } @@ -86,8 +87,26 @@ private static void SetDllImportResolver() #endif } + /// + /// Get the loaded native library. If you are using netstandard2.0, it will always return null. + /// + /// + /// + /// + public static INativeLibrary? GetLoadedNativeLibrary(NativeLibraryName name) + { + return name switch + { + NativeLibraryName.LLama => _loadedLLamaLibrary, + NativeLibraryName.LLava => _loadedLLavaLibrary, + _ => throw new ArgumentException($"Library name {name} is not found.") + }; + } + internal const string libraryName = "llama"; - internal const string llavaLibraryName = "llava_shared"; - private const string cudaVersionFile = "version.json"; + internal const string llavaLibraryName = "llava_shared"; + + private static INativeLibrary? _loadedLLamaLibrary = null; + private static INativeLibrary? _loadedLLavaLibrary = null; } } From 9d977b6ee322ef6e75a360a606495045e3d710f1 Mon Sep 17 00:00:00 2001 From: Rinne Date: Fri, 3 May 2024 09:13:17 +0800 Subject: [PATCH 7/7] Update LLama/Native/Load/UnknownNativeLibrary.cs Co-authored-by: Martin Evans --- LLama/Native/Load/UnknownNativeLibrary.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/LLama/Native/Load/UnknownNativeLibrary.cs b/LLama/Native/Load/UnknownNativeLibrary.cs index 823e77a38..fa29ac0d4 100644 --- a/LLama/Native/Load/UnknownNativeLibrary.cs +++ b/LLama/Native/Load/UnknownNativeLibrary.cs @@ -19,7 +19,7 @@ public class UnknownNativeLibrary: INativeLibrary /// public IEnumerable Prepare(SystemInfo systemInfo, NativeLogConfig.LLamaLogCallback? logCallback = null) { - throw new NotImplementedException("This class is only a placeholder and should not be used to load native library."); + throw new NotSupportedException("This class is only a placeholder and should not be used to load native library."); } } }