Merge remote-tracking branch 'origin/main' into qingying-model-router…

…_streamlit

.gitignore

@@ -90,4 +90,13 @@ build_*/
 .cache/
 
 # tests
-quantization_test.py
+quantization_test.py
+
+# Swift
+.swiftpm/
+UserInterfaceState.xcuserstate
+xcuserdata/
+*.xcworkspace/xcuserdata/
+*.playground/playground.xcworkspace/xcuserdata/
+*.generated.plist
+.build/

CMakeLists.txt

@@ -123,6 +123,7 @@ if(LLAMA_BUILD)
             -DCMAKE_INSTALL_PREFIX=${CMAKE_CURRENT_BINARY_DIR}/llama_install
             -DCMAKE_POSITION_INDEPENDENT_CODE=ON
             -DCMAKE_CXX_STANDARD=17
+            -DBUILD_SHARED_LIBS=ON
             -DLLAMA_CUDA=${LLAMA_CUDA}
             -DLLAMA_METAL=${LLAMA_METAL}
             -DGGML_AVX=$<IF:$<AND:$<PLATFORM_ID:Darwin>,$<NOT:$<STREQUAL:${CMAKE_SYSTEM_PROCESSOR},arm64>>>,OFF,ON>

Package.swift

@@ -0,0 +1,32 @@
+// swift-tools-version: 6.0
+
+import PackageDescription
+
+let package = Package(
+    name: "NexaSwift",
+    platforms: [
+        .macOS(.v15),
+        .iOS(.v18),
+        .watchOS(.v11),
+        .tvOS(.v18),
+        .visionOS(.v2)
+    ],
+    products: [
+        .library(name: "NexaSwift", targets: ["NexaSwift"]),
+    ],
+    dependencies: [
+        .package(url: "https://github.com/ggerganov/llama.cpp.git", branch: "master")
+    ],
+    targets: [
+        .target(
+            name: "NexaSwift", 
+            dependencies: [
+                .product(name: "llama", package: "llama.cpp")
+            ],
+            path: "swift/Sources/NexaSwift"),
+        .testTarget(
+            name: "NexaSwiftTests", 
+            dependencies: ["NexaSwift"],
+            path: "swift/Tests/NexaSwiftTests"),
+    ]
+)

README.md

@@ -330,6 +330,10 @@ For detailed information on CLI commands and usage, please refer to the [CLI Ref
 To start a local server using models on your local computer, you can use the `nexa server` command.
 For detailed information on server setup, API endpoints, and usage examples, please refer to the [Server Reference](SERVER.md) document.
 
+## Swift Package
+
+**[Swift SDK](https://github.com/NexaAI/nexa-sdk/tree/main/swift):** Provides a Swifty API, allowing Swift developers to easily integrate and use llama.cpp models in their projects.
+
 ## Acknowledgements
 
 We would like to thank the following projects:

examples/swift-test/Shared/ContentView.swift

@@ -0,0 +1,39 @@
+import SwiftUI
+
+struct ContentView: View {
+    @State private var viewModel = ViewModel()
+    @State private var prompt = ""
+
+    var body: some View {
+        VStack {
+            Text("Nexa Swift Demo").font(.title)
+
+            Toggle(isOn: $viewModel.usingStream) {
+                Text("Use Stream")
+            }
+            .padding(.bottom)
+
+            TextField("Enter your message", text: $prompt, axis: .vertical)
+                .textFieldStyle(.roundedBorder)
+                .lineLimit(3...5)
+                .padding(.bottom)
+                .onSubmit {
+                    guard !prompt.isEmpty else { return }
+                    viewModel.run(for: prompt)
+                }
+
+            ScrollView {
+                Text(viewModel.result)
+                    .frame(maxWidth: .infinity, alignment: .leading)
+                    .textSelection(.enabled)
+            }
+
+            Spacer()
+        }
+        .padding()
+    }
+}
+
+#Preview {
+    ContentView()
+}

examples/swift-test/Shared/ViewModel.swift

@@ -0,0 +1,57 @@
+import Foundation
+import NexaSwift
+import SwiftUI
+import Combine
+
+@Observable
+class ViewModel {
+    let nexaSwift: NexaTextInference
+    var result = ""
+    var usingStream = true
+    private var messages: [ChatCompletionRequestMessage] = []
+    private let maxHistory = 1
+    private var cancallable: Set<AnyCancellable> = []
+
+    init() {
+        let configuration = Configuration(maxNewToken: 128, stopTokens: [])
+        let model_path = Bundle.main.path(forResource: "llama3_2_3b_q4_K_M", ofType: "gguf") ?? ""
+        nexaSwift = (try? NexaTextInference(modelPath: model_path, modelConfiguration: configuration))!
+    }
+
+    func run(for userMessage: String) {
+        result = ""
+        let userMessageText = ChatCompletionRequestMessage.user(
+            ChatCompletionRequestUserMessage(content: .text(userMessage))
+        )
+
+        messages.append(userMessageText)
+        if messages.count > maxHistory * 2 {
+            messages.removeFirst(2)
+        }
+
+        Task {
+            switch usingStream {
+            case true:
+                for try await value in await nexaSwift.createChatCompletionStream(for: messages) {
+                    let delta = value.choices[0].delta.content ?? ""
+                    result += delta
+                }
+            case false:
+                if let completionResponse = try? await nexaSwift.createChatCompletion(for: messages) {
+                    let content = completionResponse.choices[0].message.content ?? ""
+                    result += content
+                }
+            }
+
+            // Add assistant's response to history
+            let assistantMessage = ChatCompletionRequestMessage.assistant(
+                ChatCompletionRequestAssistantMessage(
+                    content: result,
+                    toolCalls: nil,
+                    functionCall: nil
+                )
+            )
+            messages.append(assistantMessage)
+        }
+    }
+}

examples/swift-test/TestApp-Commandline/main.swift

@@ -0,0 +1,62 @@
+import Foundation
+import NexaSwift
+
+let configuration = NexaSwift.Configuration(
+    maxNewToken: 128,
+    stopTokens: []
+)
+
+let model_path = "path/to/your/model" // For Commandline, please add the local path here. 
+let nexaSwift = try NexaSwift.NexaTextInference(modelPath: model_path, modelConfiguration: configuration)
+
+var streamMode = false
+print("Do you want to enable stream mode? (yes/y or no/n):", terminator: " ")
+var userInput = readLine()?.lowercased() ?? ""
+if userInput == "yes" || userInput == "y" {
+    streamMode = true
+}
+print("")
+
+var messages:[ChatCompletionRequestMessage] = []
+let maxHistory = 2
+
+while true {
+    print("You:", terminator: " ")
+    userInput = readLine() ?? ""
+    print("Bot:", terminator: " ")
+
+    let userMessageText = ChatCompletionRequestMessage.user(
+        ChatCompletionRequestUserMessage(content: .text(userInput))
+    )
+
+    messages.append(userMessageText)
+    if messages.count > maxHistory * 2 {
+        messages.removeFirst(2)
+    }
+
+    var currentMessage = ""
+    if streamMode{
+        for try await value in await nexaSwift
+            .createChatCompletionStream(for: messages) {
+            print(value.choices[0].delta.content ?? "", terminator: "")
+            currentMessage += value.choices[0].delta.content ?? ""
+        }
+    }else{
+        let response = try await nexaSwift.createChatCompletion(for: messages)
+        print(response.choices[0].message.content ?? "", terminator: "")
+        currentMessage += response.choices[0].message.content ?? ""
+    }
+
+
+    let assistantMessage = ChatCompletionRequestMessage.assistant(
+        ChatCompletionRequestAssistantMessage(
+            content: currentMessage,
+            toolCalls: nil,
+            functionCall: nil
+        )
+    )
+
+    messages.append(assistantMessage)
+
+    print("")
+}

examples/swift-test/TestApp-Macos/Assets.xcassets/AccentColor.colorset/Contents.json

@@ -0,0 +1,11 @@
+{
+  "colors" : [
+    {
+      "idiom" : "universal"
+    }
+  ],
+  "info" : {
+    "author" : "xcode",
+    "version" : 1
+  }
+}

examples/swift-test/TestApp-Macos/Assets.xcassets/AppIcon.appiconset/Contents.json

@@ -0,0 +1,58 @@
+{
+  "images" : [
+    {
+      "idiom" : "mac",
+      "scale" : "1x",
+      "size" : "16x16"
+    },
+    {
+      "idiom" : "mac",
+      "scale" : "2x",
+      "size" : "16x16"
+    },
+    {
+      "idiom" : "mac",
+      "scale" : "1x",
+      "size" : "32x32"
+    },
+    {
+      "idiom" : "mac",
+      "scale" : "2x",
+      "size" : "32x32"
+    },
+    {
+      "idiom" : "mac",
+      "scale" : "1x",
+      "size" : "128x128"
+    },
+    {
+      "idiom" : "mac",
+      "scale" : "2x",
+      "size" : "128x128"
+    },
+    {
+      "idiom" : "mac",
+      "scale" : "1x",
+      "size" : "256x256"
+    },
+    {
+      "idiom" : "mac",
+      "scale" : "2x",
+      "size" : "256x256"
+    },
+    {
+      "idiom" : "mac",
+      "scale" : "1x",
+      "size" : "512x512"
+    },
+    {
+      "idiom" : "mac",
+      "scale" : "2x",
+      "size" : "512x512"
+    }
+  ],
+  "info" : {
+    "author" : "xcode",
+    "version" : 1
+  }
+}

examples/swift-test/TestApp-Macos/Assets.xcassets/Contents.json

@@ -0,0 +1,6 @@
+{
+  "info" : {
+    "author" : "xcode",
+    "version" : 1
+  }
+}

examples/swift-test/TestApp-Macos/Preview Content/Preview Assets.xcassets/Contents.json

@@ -0,0 +1,6 @@
+{
+  "info" : {
+    "author" : "xcode",
+    "version" : 1
+  }
+}

examples/swift-test/TestApp-Macos/TestApp_Macos.entitlements

@@ -0,0 +1,10 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
+<plist version="1.0">
+<dict>
+	<key>com.apple.security.app-sandbox</key>
+	<true/>
+	<key>com.apple.security.files.user-selected.read-only</key>
+	<true/>
+</dict>
+</plist>

examples/swift-test/TestApp-Macos/TestApp_MacosApp.swift

@@ -0,0 +1,10 @@
+import SwiftUI
+
+@main
+struct TestApp_MacosApp: App {
+    var body: some Scene {
+        WindowGroup {
+            ContentView()
+        }
+    }
+}

examples/swift-test/TestApp-iOS/Assets.xcassets/AccentColor.colorset/Contents.json

@@ -0,0 +1,11 @@
+{
+  "colors" : [
+    {
+      "idiom" : "universal"
+    }
+  ],
+  "info" : {
+    "author" : "xcode",
+    "version" : 1
+  }
+}

examples/swift-test/TestApp-iOS/Assets.xcassets/AppIcon.appiconset/Contents.json

@@ -0,0 +1,13 @@
+{
+  "images" : [
+    {
+      "idiom" : "universal",
+      "platform" : "ios",
+      "size" : "1024x1024"
+    }
+  ],
+  "info" : {
+    "author" : "xcode",
+    "version" : 1
+  }
+}

examples/swift-test/TestApp-iOS/Assets.xcassets/Contents.json

@@ -0,0 +1,6 @@
+{
+  "info" : {
+    "author" : "xcode",
+    "version" : 1
+  }
+}

examples/swift-test/TestApp-iOS/Preview Content/Preview Assets.xcassets/Contents.json

@@ -0,0 +1,6 @@
+{
+  "info" : {
+    "author" : "xcode",
+    "version" : 1
+  }
+}

examples/swift-test/TestApp-iOS/TestApp_iOSApp.swift

@@ -0,0 +1,10 @@
+import SwiftUI
+
+@main
+struct TestAppApp: App {
+    var body: some Scene {
+        WindowGroup {
+            ContentView()
+        }
+    }
+}