diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index a7bf1088..0a51bc2b 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -230,6 +230,19 @@ jobs:
cuda: '12.4.0'
method: 'network'
+ - name: Install CUDA on Ubuntu (arm64)
+ if: matrix.config.name == 'Ubuntu (arm64)'
+ env:
+ CUDA_VERSION: '13.4.2'
+ run: |
+ curl -fsSL "https://developer.download.nvidia.com/compute/cuda/${CUDA_VERSION}/local_installers/cuda_${CUDA_VERSION}_linux_sbsa.run" -o "$RUNNER_TEMP/cuda.run"
+ sudo sh "$RUNNER_TEMP/cuda.run" --silent --toolkit --override
+ rm "$RUNNER_TEMP/cuda.run"
+ echo "CUDA_PATH=/usr/local/cuda-${CUDA_VERSION%.*}" >> "$GITHUB_ENV"
+ echo "/usr/local/cuda-${CUDA_VERSION%.*}/bin" >> "$GITHUB_PATH"
+ echo "LD_LIBRARY_PATH=/usr/local/cuda-${CUDA_VERSION%.*}/lib64${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}" >> "$GITHUB_ENV"
+ "/usr/local/cuda-${CUDA_VERSION%.*}/bin/nvcc" --version
+
- name: Install Vulkan SDK on Windows (1)
if: matrix.config.name == 'Windows (1)'
shell: powershell
@@ -347,6 +360,7 @@ jobs:
} else if (process.env.ARTIFACT_NAME === "linux-arm64") {
await buildBinary("arm64", ["--gpu", "false"]);
await buildBinary("armv7l", ["--gpu", "false"]);
+ await buildBinary("arm64", ["--gpu", "cuda"]);
} else if (process.env.ARTIFACT_NAME === "mac-x64") {
await buildBinary("x64", ["--gpu", "false"]);
} else if (process.env.ARTIFACT_NAME === "mac-arm64") {
@@ -383,6 +397,10 @@ jobs:
if (!(await fs.pathExists(path.join(llamaBinsDirectoryPath, "linux-x64-cuda-2", "libggml-cuda.so")))) {
throw new Error("libggml-cuda.so not found in linux-x64-cuda-2");
}
+ } else if (process.env.ARTIFACT_NAME === "linux-arm64") {
+ if (!(await fs.pathExists(path.join(llamaBinsDirectoryPath, "linux-arm64-cuda", "libggml-cuda.so")))) {
+ throw new Error("libggml-cuda.so not found in linux-arm64-cuda");
+ }
}
await $`echo "Built binaries:"`;
@@ -794,7 +812,8 @@ jobs:
npx --no vite-node ./scripts/waitForNpmRegistryToHaveModuleVersion.ts --packageVersion "$DEPLOYED_PACKAGE_VERSION" --additionalWait 60 --timeoutMinutes 10
cd electron-app-example
- npm install
+ npm install --cpu=x64
+ npm install --cpu=arm64
- name: Build electron app
id: build
diff --git a/docs/guide/CUDA.md b/docs/guide/CUDA.md
index 66ba2d14..6b59f3cc 100644
--- a/docs/guide/CUDA.md
+++ b/docs/guide/CUDA.md
@@ -142,10 +142,14 @@ To fix it, uninstall older CUDA versions and restart your machine (important).
Run this command inside of your project:
::: code-group
-```shell [Linux]
+```shell [Linux x64]
ldd ./node_modules/@node-llama-cpp/linux-x64-cuda/bins/linux-x64-cuda/libggml-cuda.so
```
+```shell [Linux ARM64]
+ldd ./node_modules/@node-llama-cpp/linux-arm64-cuda/bins/linux-arm64-cuda/libggml-cuda.so
+```
+
```cmd [Windows (cmd)]
"C:\Program Files\Git\usr\bin\ldd.exe" node_modules\@node-llama-cpp\win-x64-cuda\bins\win-x64-cuda\ggml-cuda.dll
```
diff --git a/docs/guide/awesome.md b/docs/guide/awesome.md
index 0b4bf86c..eaea9f99 100644
--- a/docs/guide/awesome.md
+++ b/docs/guide/awesome.md
@@ -18,6 +18,9 @@ import DataBadge from "../../.vitepress/components/DataBadge/DataBadge.vue";
* [Clippy](https://felixrieseberg.github.io/clippy/) ([GitHub](https://github.com/felixrieseberg/clippy)) - Clippy, resurrected from the 1990s, now with some AI
+* [Semantic Golfer](https://semantic-golfer.giladgd.com) ([GitHub](https://github.com/giladgd/semantic-golfer)) - How little can you write and still get everything right? Play text golfing with `node-llama-cpp`'s structured decisions
+
+
## Proprietary
* [BashBuddy](https://bashbuddy.run) ([GitHub](https://github.com/wosherco/bashbuddy)) - write bash commands with natural language
diff --git a/docs/guide/structured-decisions.md b/docs/guide/structured-decisions.md
index b319b508..5a931648 100644
--- a/docs/guide/structured-decisions.md
+++ b/docs/guide/structured-decisions.md
@@ -90,6 +90,12 @@ There are two places where you can use the structured decisions API:
* [On a decision context](#decision-context) with a given document as the context for the questions
* [On a chat session](#chat-session) with the entire chat history as the context for the questions
+
+
+Play with structured decisions in [Semantic Golfer](https://semantic-golfer.giladgd.com)'s interactive playground
+
+
+
### On a Decision Context {#decision-context}
When using a [`LlamaDecisionContext`](../api/classes/LlamaDecisionContext.md) (via [`.decide()`](../api/classes/LlamaDecisionContext.md#decide)),
diff --git a/package.json b/package.json
index 85812c66..0fcc2424 100644
--- a/package.json
+++ b/package.json
@@ -232,6 +232,7 @@
"@node-llama-cpp/linux-x64": "0.1.0",
"@node-llama-cpp/linux-x64-cuda": "0.1.0",
"@node-llama-cpp/linux-x64-cuda-ext": "0.1.0",
+ "@node-llama-cpp/linux-arm64-cuda": "0.1.0",
"@node-llama-cpp/linux-x64-vulkan": "0.1.0",
"@node-llama-cpp/mac-arm64-metal": "0.1.0",
"@node-llama-cpp/mac-x64": "0.1.0",
diff --git a/packages/@node-llama-cpp/linux-arm64-cuda/.gitignore b/packages/@node-llama-cpp/linux-arm64-cuda/.gitignore
new file mode 100644
index 00000000..8c780bd4
--- /dev/null
+++ b/packages/@node-llama-cpp/linux-arm64-cuda/.gitignore
@@ -0,0 +1,2 @@
+/dist
+/tsconfig.tsbuildinfo
diff --git a/packages/@node-llama-cpp/linux-arm64-cuda/LICENSE b/packages/@node-llama-cpp/linux-arm64-cuda/LICENSE
new file mode 100644
index 00000000..22789ae3
--- /dev/null
+++ b/packages/@node-llama-cpp/linux-arm64-cuda/LICENSE
@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2024 Gilad S.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
diff --git a/packages/@node-llama-cpp/linux-arm64-cuda/README.md b/packages/@node-llama-cpp/linux-arm64-cuda/README.md
new file mode 100644
index 00000000..75b7f9bb
--- /dev/null
+++ b/packages/@node-llama-cpp/linux-arm64-cuda/README.md
@@ -0,0 +1,4 @@
+# [`node-llama-cpp`](https://github.com/withcatai/node-llama-cpp)
+This is a prebuilt binary package for [`node-llama-cpp`](https://github.com/withcatai/node-llama-cpp) for Linux arm64 with CUDA support.
+
+Do not install this package directly.
diff --git a/packages/@node-llama-cpp/linux-arm64-cuda/package-lock.json b/packages/@node-llama-cpp/linux-arm64-cuda/package-lock.json
new file mode 100644
index 00000000..0c407a77
--- /dev/null
+++ b/packages/@node-llama-cpp/linux-arm64-cuda/package-lock.json
@@ -0,0 +1,38 @@
+{
+ "name": "@node-llama-cpp/linux-arm64-cuda",
+ "version": "0.1.0",
+ "lockfileVersion": 3,
+ "requires": true,
+ "packages": {
+ "": {
+ "name": "@node-llama-cpp/linux-arm64-cuda",
+ "version": "0.1.0",
+ "cpu": [
+ "arm64"
+ ],
+ "license": "MIT",
+ "os": [
+ "linux"
+ ],
+ "devDependencies": {
+ "typescript": "^5.2.2"
+ },
+ "engines": {
+ "node": ">=20.0.0"
+ }
+ },
+ "node_modules/typescript": {
+ "version": "5.5.2",
+ "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.5.2.tgz",
+ "integrity": "sha512-NcRtPEOsPFFWjobJEtfihkLCZCXZt/os3zf8nTxjVH3RvTSxjrCamJpbExGvYOF+tFHc3pA65qpdwPbzjohhew==",
+ "dev": true,
+ "bin": {
+ "tsc": "bin/tsc",
+ "tsserver": "bin/tsserver"
+ },
+ "engines": {
+ "node": ">=14.17"
+ }
+ }
+ }
+}
diff --git a/packages/@node-llama-cpp/linux-arm64-cuda/package.json b/packages/@node-llama-cpp/linux-arm64-cuda/package.json
new file mode 100644
index 00000000..9fcc186e
--- /dev/null
+++ b/packages/@node-llama-cpp/linux-arm64-cuda/package.json
@@ -0,0 +1,48 @@
+{
+ "name": "@node-llama-cpp/linux-arm64-cuda",
+ "version": "0.1.0",
+ "description": "Prebuilt binary for node-llama-cpp for Linux arm64 with CUDA support",
+ "main": "dist/index.js",
+ "type": "module",
+ "files": [
+ "dist/",
+ "bins/",
+ "package.json",
+ "README.md",
+ "LICENSE"
+ ],
+ "exports": {
+ ".": {
+ "import": "./dist/index.js",
+ "node": "./dist/index.js",
+ "default": "./dist/index.js"
+ }
+ },
+ "engines": {
+ "node": ">=20.0.0"
+ },
+ "os": ["linux"],
+ "cpu": ["arm64"],
+ "libc": ["glibc"],
+ "scripts": {
+ "prebuild": "rimraf ./dist ./tsconfig.tsbuildinfo",
+ "build": "tsc --build tsconfig.json --force",
+ "prewatch": "rimraf ./dist ./tsconfig.tsbuildinfo",
+ "watch": "tsc --build tsconfig.json --watch --force",
+ "clean": "rm -rf ./node_modules ./dist ./tsconfig.tsbuildinfo"
+ },
+ "repository": {
+ "type": "git",
+ "url": "git+https://github.com/withcatai/node-llama-cpp.git"
+ },
+ "author": "Gilad S.",
+ "license": "MIT",
+ "preferUnplugged": true,
+ "bugs": {
+ "url": "https://github.com/withcatai/node-llama-cpp/issues"
+ },
+ "homepage": "https://node-llama-cpp.withcat.ai",
+ "devDependencies": {
+ "typescript": "^5.2.2"
+ }
+}
diff --git a/packages/@node-llama-cpp/linux-arm64-cuda/src/index.ts b/packages/@node-llama-cpp/linux-arm64-cuda/src/index.ts
new file mode 100644
index 00000000..a4cb56d5
--- /dev/null
+++ b/packages/@node-llama-cpp/linux-arm64-cuda/src/index.ts
@@ -0,0 +1,14 @@
+import path from "path";
+import {fileURLToPath} from "url";
+import fs from "node:fs/promises";
+
+const __dirname = path.dirname(fileURLToPath(import.meta.url));
+const binsDir = path.join(__dirname, "..", "bins");
+const packageVersion: string = (JSON.parse(await fs.readFile(path.join(__dirname, "..", "package.json"), "utf8"))).version;
+
+export function getBinsDir() {
+ return {
+ binsDir,
+ packageVersion
+ };
+}
diff --git a/packages/@node-llama-cpp/linux-arm64-cuda/tsconfig.json b/packages/@node-llama-cpp/linux-arm64-cuda/tsconfig.json
new file mode 100644
index 00000000..c3182e71
--- /dev/null
+++ b/packages/@node-llama-cpp/linux-arm64-cuda/tsconfig.json
@@ -0,0 +1,35 @@
+{
+ "compilerOptions": {
+ "lib": ["es2022"],
+ "module": "node16",
+ "target": "es2022",
+ "esModuleInterop": true,
+ "noImplicitAny": true,
+ "noImplicitReturns": true,
+ "noImplicitThis": true,
+ "noImplicitOverride": true,
+ "removeComments": false,
+ "allowSyntheticDefaultImports": true,
+ "forceConsistentCasingInFileNames": true,
+ "noFallthroughCasesInSwitch": true,
+ "skipLibCheck": true,
+ "moduleResolution": "node16",
+ "resolveJsonModule": false,
+ "strictNullChecks": true,
+ "isolatedModules": true,
+ "noEmit": false,
+ "rootDir": "./src",
+ "outDir": "./dist",
+ "strict": true,
+ "sourceMap": false,
+ "composite": false,
+ "declaration": false,
+ "stripInternal": true
+ },
+ "files": [
+ "./src/index.ts"
+ ],
+ "include": [
+ "./src"
+ ]
+}
diff --git a/scripts/movePrebuiltBinariesToStandaloneModules.ts b/scripts/movePrebuiltBinariesToStandaloneModules.ts
index 5eb1dafb..d6319527 100644
--- a/scripts/movePrebuiltBinariesToStandaloneModules.ts
+++ b/scripts/movePrebuiltBinariesToStandaloneModules.ts
@@ -53,6 +53,7 @@ await moveBinariesFolderToStandaloneModule((folderName) => folderName.startsWith
await moveBinariesFallbackDirToStandaloneExtModule((folderName) => folderName.startsWith("linux-x64-cuda"), "@node-llama-cpp/linux-x64-cuda-ext");
await moveBinariesFolderToStandaloneModule((folderName) => folderName.startsWith("linux-x64-cuda"), "@node-llama-cpp/linux-x64-cuda");
+await moveBinariesFolderToStandaloneModule((folderName) => folderName.startsWith("linux-arm64-cuda"), "@node-llama-cpp/linux-arm64-cuda");
await moveBinariesFolderToStandaloneModule((folderName) => folderName.startsWith("linux-x64-vulkan"), "@node-llama-cpp/linux-x64-vulkan");
await moveBinariesFolderToStandaloneModule((folderName) => folderName.startsWith("linux-x64"), "@node-llama-cpp/linux-x64");
diff --git a/src/bindings/utils/compileLLamaCpp.ts b/src/bindings/utils/compileLLamaCpp.ts
index 1030c408..9a07c1a4 100644
--- a/src/bindings/utils/compileLLamaCpp.ts
+++ b/src/bindings/utils/compileLLamaCpp.ts
@@ -644,10 +644,14 @@ function getPrebuiltBinariesPackageDirectoryForBuildOptions(buildOptions: {
else if (buildOptions.gpu === false)
// @ts-ignore
return getBinariesPathFromModules(() => import("@node-llama-cpp/linux-x64"));
- } else if (buildOptions.arch === "arm64")
+ } else if (buildOptions.arch === "arm64") {
+ if (buildOptions.gpu === "cuda")
+ // @ts-ignore
+ return getBinariesPathFromModules(() => import("@node-llama-cpp/linux-arm64-cuda"));
+
// @ts-ignore
return getBinariesPathFromModules(() => import("@node-llama-cpp/linux-arm64"));
- else if (buildOptions.arch === "arm")
+ } else if (buildOptions.arch === "arm")
// @ts-ignore
return getBinariesPathFromModules(() => import("@node-llama-cpp/linux-armv7l"));
else if (buildOptions.arch === "riscv64")
diff --git a/src/bindings/utils/detectAvailableComputeLayers.ts b/src/bindings/utils/detectAvailableComputeLayers.ts
index 4978eaf2..50f8563d 100644
--- a/src/bindings/utils/detectAvailableComputeLayers.ts
+++ b/src/bindings/utils/detectAvailableComputeLayers.ts
@@ -82,6 +82,8 @@ async function detectCudaSupport({
"/usr/lib64",
"/usr/lib/x86_64-linux-gnu",
"/usr/lib/aarch64-linux-gnu",
+ "/usr/lib/aarch64-linux-gnu/tegra",
+ "/usr/lib/aarch64-linux-gnu/nvidia",
"/usr/lib/armv7l-linux-gnu",
...cudaLibraryPaths
];
diff --git a/test/standalone/chatWrappers/MuseChatWrapper.test.ts b/test/standalone/chatWrappers/MuseChatWrapper.test.ts
index 8eddbadf..eef652b2 100644
--- a/test/standalone/chatWrappers/MuseChatWrapper.test.ts
+++ b/test/standalone/chatWrappers/MuseChatWrapper.test.ts
@@ -36,9 +36,11 @@ describe("MuseChatWrapper", () => {
});
test("should generate the default system message", () => {
+ const todayDate = new Date("2026-08-11T00:00:00Z");
+ todayDate.setDate(11); // ensure the date is 11 regardless of timezone
const chatWrapper = new MuseChatWrapper({
reasoningStrength: "low",
- todayDate: new Date("2026-08-11T00:00:00Z")
+ todayDate
});
const {contextText} = chatWrapper.generateContextState({