From af8804231fc1e6db1215114323160829b22bdf2b Mon Sep 17 00:00:00 2001 From: "Gilad S." Date: Fri, 2 Oct 2026 17:06:35 +0200 Subject: [PATCH 1/5] feat: arm64 CUDA prebuilt binaries --- .github/workflows/build.yml | 15 +++++- docs/guide/CUDA.md | 6 ++- package.json | 1 + .../linux-arm64-cuda/.gitignore | 2 + .../@node-llama-cpp/linux-arm64-cuda/LICENSE | 21 ++++++++ .../linux-arm64-cuda/README.md | 4 ++ .../linux-arm64-cuda/package-lock.json | 39 +++++++++++++++ .../linux-arm64-cuda/package.json | 48 +++++++++++++++++++ .../linux-arm64-cuda/src/index.ts | 14 ++++++ .../linux-arm64-cuda/tsconfig.json | 35 ++++++++++++++ ...movePrebuiltBinariesToStandaloneModules.ts | 1 + src/bindings/utils/compileLLamaCpp.ts | 8 +++- .../utils/detectAvailableComputeLayers.ts | 2 + 13 files changed, 192 insertions(+), 4 deletions(-) create mode 100644 packages/@node-llama-cpp/linux-arm64-cuda/.gitignore create mode 100644 packages/@node-llama-cpp/linux-arm64-cuda/LICENSE create mode 100644 packages/@node-llama-cpp/linux-arm64-cuda/README.md create mode 100644 packages/@node-llama-cpp/linux-arm64-cuda/package-lock.json create mode 100644 packages/@node-llama-cpp/linux-arm64-cuda/package.json create mode 100644 packages/@node-llama-cpp/linux-arm64-cuda/src/index.ts create mode 100644 packages/@node-llama-cpp/linux-arm64-cuda/tsconfig.json diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index a7bf1088..b07f1826 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -230,6 +230,13 @@ jobs: cuda: '12.4.0' method: 'network' + - name: Install Cuda 13.1 on Ubuntu (arm64) + if: matrix.config.name == 'Ubuntu (arm64)' + uses: Jimver/cuda-toolkit@v0.2.36 + with: + cuda: '13.4.2' + method: 'network' + - name: Install Vulkan SDK on Windows (1) if: matrix.config.name == 'Windows (1)' shell: powershell @@ -347,6 +354,7 @@ jobs: } else if (process.env.ARTIFACT_NAME === "linux-arm64") { await buildBinary("arm64", ["--gpu", "false"]); await buildBinary("armv7l", ["--gpu", "false"]); + await buildBinary("arm64", ["--gpu", "cuda"]); } else if (process.env.ARTIFACT_NAME === "mac-x64") { await buildBinary("x64", ["--gpu", "false"]); } else if (process.env.ARTIFACT_NAME === "mac-arm64") { @@ -383,6 +391,10 @@ jobs: if (!(await fs.pathExists(path.join(llamaBinsDirectoryPath, "linux-x64-cuda-2", "libggml-cuda.so")))) { throw new Error("libggml-cuda.so not found in linux-x64-cuda-2"); } + } else if (process.env.ARTIFACT_NAME === "linux-arm64") { + if (!(await fs.pathExists(path.join(llamaBinsDirectoryPath, "linux-arm64-cuda", "libggml-cuda.so")))) { + throw new Error("libggml-cuda.so not found in linux-arm64-cuda"); + } } await $`echo "Built binaries:"`; @@ -794,7 +806,8 @@ jobs: npx --no vite-node ./scripts/waitForNpmRegistryToHaveModuleVersion.ts --packageVersion "$DEPLOYED_PACKAGE_VERSION" --additionalWait 60 --timeoutMinutes 10 cd electron-app-example - npm install + npm install --cpu=x64 + npm install --cpu=arm64 - name: Build electron app id: build diff --git a/docs/guide/CUDA.md b/docs/guide/CUDA.md index 66ba2d14..6b59f3cc 100644 --- a/docs/guide/CUDA.md +++ b/docs/guide/CUDA.md @@ -142,10 +142,14 @@ To fix it, uninstall older CUDA versions and restart your machine (important). Run this command inside of your project: ::: code-group -```shell [Linux] +```shell [Linux x64] ldd ./node_modules/@node-llama-cpp/linux-x64-cuda/bins/linux-x64-cuda/libggml-cuda.so ``` +```shell [Linux ARM64] +ldd ./node_modules/@node-llama-cpp/linux-arm64-cuda/bins/linux-arm64-cuda/libggml-cuda.so +``` + ```cmd [Windows (cmd)] "C:\Program Files\Git\usr\bin\ldd.exe" node_modules\@node-llama-cpp\win-x64-cuda\bins\win-x64-cuda\ggml-cuda.dll ``` diff --git a/package.json b/package.json index 85812c66..0fcc2424 100644 --- a/package.json +++ b/package.json @@ -232,6 +232,7 @@ "@node-llama-cpp/linux-x64": "0.1.0", "@node-llama-cpp/linux-x64-cuda": "0.1.0", "@node-llama-cpp/linux-x64-cuda-ext": "0.1.0", + "@node-llama-cpp/linux-arm64-cuda": "0.1.0", "@node-llama-cpp/linux-x64-vulkan": "0.1.0", "@node-llama-cpp/mac-arm64-metal": "0.1.0", "@node-llama-cpp/mac-x64": "0.1.0", diff --git a/packages/@node-llama-cpp/linux-arm64-cuda/.gitignore b/packages/@node-llama-cpp/linux-arm64-cuda/.gitignore new file mode 100644 index 00000000..8c780bd4 --- /dev/null +++ b/packages/@node-llama-cpp/linux-arm64-cuda/.gitignore @@ -0,0 +1,2 @@ +/dist +/tsconfig.tsbuildinfo diff --git a/packages/@node-llama-cpp/linux-arm64-cuda/LICENSE b/packages/@node-llama-cpp/linux-arm64-cuda/LICENSE new file mode 100644 index 00000000..22789ae3 --- /dev/null +++ b/packages/@node-llama-cpp/linux-arm64-cuda/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2024 Gilad S. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/packages/@node-llama-cpp/linux-arm64-cuda/README.md b/packages/@node-llama-cpp/linux-arm64-cuda/README.md new file mode 100644 index 00000000..75b7f9bb --- /dev/null +++ b/packages/@node-llama-cpp/linux-arm64-cuda/README.md @@ -0,0 +1,4 @@ +# [`node-llama-cpp`](https://github.com/withcatai/node-llama-cpp) +This is a prebuilt binary package for [`node-llama-cpp`](https://github.com/withcatai/node-llama-cpp) for Linux arm64 with CUDA support. + +Do not install this package directly. diff --git a/packages/@node-llama-cpp/linux-arm64-cuda/package-lock.json b/packages/@node-llama-cpp/linux-arm64-cuda/package-lock.json new file mode 100644 index 00000000..cca8419b --- /dev/null +++ b/packages/@node-llama-cpp/linux-arm64-cuda/package-lock.json @@ -0,0 +1,39 @@ +{ + "name": "@node-llama-cpp/linux-arm64-cuda", + "version": "0.1.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "@node-llama-cpp/linux-arm64-cuda", + "version": "0.1.0", + "cpu": [ + "arm64", + "x64" + ], + "license": "MIT", + "os": [ + "linux" + ], + "devDependencies": { + "typescript": "^5.2.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/typescript": { + "version": "5.5.2", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.5.2.tgz", + "integrity": "sha512-NcRtPEOsPFFWjobJEtfihkLCZCXZt/os3zf8nTxjVH3RvTSxjrCamJpbExGvYOF+tFHc3pA65qpdwPbzjohhew==", + "dev": true, + "bin": { + "tsc": "bin/tsc", + "tsserver": "bin/tsserver" + }, + "engines": { + "node": ">=14.17" + } + } + } +} diff --git a/packages/@node-llama-cpp/linux-arm64-cuda/package.json b/packages/@node-llama-cpp/linux-arm64-cuda/package.json new file mode 100644 index 00000000..b52ff85c --- /dev/null +++ b/packages/@node-llama-cpp/linux-arm64-cuda/package.json @@ -0,0 +1,48 @@ +{ + "name": "@node-llama-cpp/linux-arm64-cuda", + "version": "0.1.0", + "description": "Prebuilt binary for node-llama-cpp for Linux arm64 with CUDA support", + "main": "dist/index.js", + "type": "module", + "files": [ + "dist/", + "bins/", + "package.json", + "README.md", + "LICENSE" + ], + "exports": { + ".": { + "import": "./dist/index.js", + "node": "./dist/index.js", + "default": "./dist/index.js" + } + }, + "engines": { + "node": ">=20.0.0" + }, + "os": ["linux"], + "cpu": ["arm64", "x64"], + "libc": ["glibc"], + "scripts": { + "prebuild": "rimraf ./dist ./tsconfig.tsbuildinfo", + "build": "tsc --build tsconfig.json --force", + "prewatch": "rimraf ./dist ./tsconfig.tsbuildinfo", + "watch": "tsc --build tsconfig.json --watch --force", + "clean": "rm -rf ./node_modules ./dist ./tsconfig.tsbuildinfo" + }, + "repository": { + "type": "git", + "url": "git+https://github.com/withcatai/node-llama-cpp.git" + }, + "author": "Gilad S.", + "license": "MIT", + "preferUnplugged": true, + "bugs": { + "url": "https://github.com/withcatai/node-llama-cpp/issues" + }, + "homepage": "https://node-llama-cpp.withcat.ai", + "devDependencies": { + "typescript": "^5.2.2" + } +} diff --git a/packages/@node-llama-cpp/linux-arm64-cuda/src/index.ts b/packages/@node-llama-cpp/linux-arm64-cuda/src/index.ts new file mode 100644 index 00000000..a4cb56d5 --- /dev/null +++ b/packages/@node-llama-cpp/linux-arm64-cuda/src/index.ts @@ -0,0 +1,14 @@ +import path from "path"; +import {fileURLToPath} from "url"; +import fs from "node:fs/promises"; + +const __dirname = path.dirname(fileURLToPath(import.meta.url)); +const binsDir = path.join(__dirname, "..", "bins"); +const packageVersion: string = (JSON.parse(await fs.readFile(path.join(__dirname, "..", "package.json"), "utf8"))).version; + +export function getBinsDir() { + return { + binsDir, + packageVersion + }; +} diff --git a/packages/@node-llama-cpp/linux-arm64-cuda/tsconfig.json b/packages/@node-llama-cpp/linux-arm64-cuda/tsconfig.json new file mode 100644 index 00000000..c3182e71 --- /dev/null +++ b/packages/@node-llama-cpp/linux-arm64-cuda/tsconfig.json @@ -0,0 +1,35 @@ +{ + "compilerOptions": { + "lib": ["es2022"], + "module": "node16", + "target": "es2022", + "esModuleInterop": true, + "noImplicitAny": true, + "noImplicitReturns": true, + "noImplicitThis": true, + "noImplicitOverride": true, + "removeComments": false, + "allowSyntheticDefaultImports": true, + "forceConsistentCasingInFileNames": true, + "noFallthroughCasesInSwitch": true, + "skipLibCheck": true, + "moduleResolution": "node16", + "resolveJsonModule": false, + "strictNullChecks": true, + "isolatedModules": true, + "noEmit": false, + "rootDir": "./src", + "outDir": "./dist", + "strict": true, + "sourceMap": false, + "composite": false, + "declaration": false, + "stripInternal": true + }, + "files": [ + "./src/index.ts" + ], + "include": [ + "./src" + ] +} diff --git a/scripts/movePrebuiltBinariesToStandaloneModules.ts b/scripts/movePrebuiltBinariesToStandaloneModules.ts index 5eb1dafb..d6319527 100644 --- a/scripts/movePrebuiltBinariesToStandaloneModules.ts +++ b/scripts/movePrebuiltBinariesToStandaloneModules.ts @@ -53,6 +53,7 @@ await moveBinariesFolderToStandaloneModule((folderName) => folderName.startsWith await moveBinariesFallbackDirToStandaloneExtModule((folderName) => folderName.startsWith("linux-x64-cuda"), "@node-llama-cpp/linux-x64-cuda-ext"); await moveBinariesFolderToStandaloneModule((folderName) => folderName.startsWith("linux-x64-cuda"), "@node-llama-cpp/linux-x64-cuda"); +await moveBinariesFolderToStandaloneModule((folderName) => folderName.startsWith("linux-arm64-cuda"), "@node-llama-cpp/linux-arm64-cuda"); await moveBinariesFolderToStandaloneModule((folderName) => folderName.startsWith("linux-x64-vulkan"), "@node-llama-cpp/linux-x64-vulkan"); await moveBinariesFolderToStandaloneModule((folderName) => folderName.startsWith("linux-x64"), "@node-llama-cpp/linux-x64"); diff --git a/src/bindings/utils/compileLLamaCpp.ts b/src/bindings/utils/compileLLamaCpp.ts index 1030c408..9a07c1a4 100644 --- a/src/bindings/utils/compileLLamaCpp.ts +++ b/src/bindings/utils/compileLLamaCpp.ts @@ -644,10 +644,14 @@ function getPrebuiltBinariesPackageDirectoryForBuildOptions(buildOptions: { else if (buildOptions.gpu === false) // @ts-ignore return getBinariesPathFromModules(() => import("@node-llama-cpp/linux-x64")); - } else if (buildOptions.arch === "arm64") + } else if (buildOptions.arch === "arm64") { + if (buildOptions.gpu === "cuda") + // @ts-ignore + return getBinariesPathFromModules(() => import("@node-llama-cpp/linux-arm64-cuda")); + // @ts-ignore return getBinariesPathFromModules(() => import("@node-llama-cpp/linux-arm64")); - else if (buildOptions.arch === "arm") + } else if (buildOptions.arch === "arm") // @ts-ignore return getBinariesPathFromModules(() => import("@node-llama-cpp/linux-armv7l")); else if (buildOptions.arch === "riscv64") diff --git a/src/bindings/utils/detectAvailableComputeLayers.ts b/src/bindings/utils/detectAvailableComputeLayers.ts index 4978eaf2..50f8563d 100644 --- a/src/bindings/utils/detectAvailableComputeLayers.ts +++ b/src/bindings/utils/detectAvailableComputeLayers.ts @@ -82,6 +82,8 @@ async function detectCudaSupport({ "/usr/lib64", "/usr/lib/x86_64-linux-gnu", "/usr/lib/aarch64-linux-gnu", + "/usr/lib/aarch64-linux-gnu/tegra", + "/usr/lib/aarch64-linux-gnu/nvidia", "/usr/lib/armv7l-linux-gnu", ...cudaLibraryPaths ]; From 9443c35ed9f5742ab949150d2af507c10bc484dd Mon Sep 17 00:00:00 2001 From: "Gilad S." Date: Fri, 2 Oct 2026 17:07:52 +0200 Subject: [PATCH 2/5] docs: reference Semantic Golfer --- docs/guide/awesome.md | 3 +++ docs/guide/structured-decisions.md | 6 ++++++ 2 files changed, 9 insertions(+) diff --git a/docs/guide/awesome.md b/docs/guide/awesome.md index 0b4bf86c..eaea9f99 100644 --- a/docs/guide/awesome.md +++ b/docs/guide/awesome.md @@ -18,6 +18,9 @@ import DataBadge from "../../.vitepress/components/DataBadge/DataBadge.vue"; * [Clippy](https://felixrieseberg.github.io/clippy/) ([GitHub](https://github.com/felixrieseberg/clippy)) - Clippy, resurrected from the 1990s, now with some AI
+* [Semantic Golfer](https://semantic-golfer.giladgd.com) ([GitHub](https://github.com/giladgd/semantic-golfer)) - How little can you write and still get everything right? Play text golfing with `node-llama-cpp`'s structured decisions +
+ ## Proprietary * [BashBuddy](https://bashbuddy.run) ([GitHub](https://github.com/wosherco/bashbuddy)) - write bash commands with natural language diff --git a/docs/guide/structured-decisions.md b/docs/guide/structured-decisions.md index b319b508..5a931648 100644 --- a/docs/guide/structured-decisions.md +++ b/docs/guide/structured-decisions.md @@ -90,6 +90,12 @@ There are two places where you can use the structured decisions API: * [On a decision context](#decision-context) with a given document as the context for the questions * [On a chat session](#chat-session) with the entire chat history as the context for the questions +
+ +Play with structured decisions in [Semantic Golfer](https://semantic-golfer.giladgd.com)'s interactive playground + +
+ ### On a Decision Context {#decision-context} When using a [`LlamaDecisionContext`](../api/classes/LlamaDecisionContext.md) (via [`.decide()`](../api/classes/LlamaDecisionContext.md#decide)), From eadb243d6024ffe34b05ceb0c3eb3dbc733fba6b Mon Sep 17 00:00:00 2001 From: "Gilad S." Date: Fri, 2 Oct 2026 22:18:58 +0200 Subject: [PATCH 3/5] build: fix arm64 CUDA install --- .github/workflows/build.yml | 16 +++++++++++----- .../linux-arm64-cuda/package-lock.json | 3 +-- .../linux-arm64-cuda/package.json | 2 +- 3 files changed, 13 insertions(+), 8 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index b07f1826..020dabd4 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -230,12 +230,18 @@ jobs: cuda: '12.4.0' method: 'network' - - name: Install Cuda 13.1 on Ubuntu (arm64) + - name: Install CUDA on Ubuntu (arm64) if: matrix.config.name == 'Ubuntu (arm64)' - uses: Jimver/cuda-toolkit@v0.2.36 - with: - cuda: '13.4.2' - method: 'network' + env: + CUDA_VERSION: '13.4.2' + run: | + curl -fsSL "https://developer.download.nvidia.com/compute/cuda/${CUDA_VERSION}/local_installers/cuda_${CUDA_VERSION}_linux_sbsa.run" -o "$RUNNER_TEMP/cuda.run" + sudo sh "$RUNNER_TEMP/cuda.run" --silent --toolkit + rm "$RUNNER_TEMP/cuda.run" + echo "CUDA_PATH=/usr/local/cuda-${CUDA_VERSION%.*}" >> "$GITHUB_ENV" + echo "/usr/local/cuda-${CUDA_VERSION%.*}/bin" >> "$GITHUB_PATH" + echo "LD_LIBRARY_PATH=/usr/local/cuda-${CUDA_VERSION%.*}/lib64${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}" >> "$GITHUB_ENV" + "/usr/local/cuda-${CUDA_VERSION%.*}/bin/nvcc" --version - name: Install Vulkan SDK on Windows (1) if: matrix.config.name == 'Windows (1)' diff --git a/packages/@node-llama-cpp/linux-arm64-cuda/package-lock.json b/packages/@node-llama-cpp/linux-arm64-cuda/package-lock.json index cca8419b..0c407a77 100644 --- a/packages/@node-llama-cpp/linux-arm64-cuda/package-lock.json +++ b/packages/@node-llama-cpp/linux-arm64-cuda/package-lock.json @@ -8,8 +8,7 @@ "name": "@node-llama-cpp/linux-arm64-cuda", "version": "0.1.0", "cpu": [ - "arm64", - "x64" + "arm64" ], "license": "MIT", "os": [ diff --git a/packages/@node-llama-cpp/linux-arm64-cuda/package.json b/packages/@node-llama-cpp/linux-arm64-cuda/package.json index b52ff85c..9fcc186e 100644 --- a/packages/@node-llama-cpp/linux-arm64-cuda/package.json +++ b/packages/@node-llama-cpp/linux-arm64-cuda/package.json @@ -22,7 +22,7 @@ "node": ">=20.0.0" }, "os": ["linux"], - "cpu": ["arm64", "x64"], + "cpu": ["arm64"], "libc": ["glibc"], "scripts": { "prebuild": "rimraf ./dist ./tsconfig.tsbuildinfo", From 7f830cb44361abf26b9dc5c4d805cf86310baf6d Mon Sep 17 00:00:00 2001 From: "Gilad S." Date: Sun, 4 Oct 2026 01:20:09 +0200 Subject: [PATCH 4/5] build: fix arm64 CUDA installation in the CI --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 020dabd4..0a51bc2b 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -236,7 +236,7 @@ jobs: CUDA_VERSION: '13.4.2' run: | curl -fsSL "https://developer.download.nvidia.com/compute/cuda/${CUDA_VERSION}/local_installers/cuda_${CUDA_VERSION}_linux_sbsa.run" -o "$RUNNER_TEMP/cuda.run" - sudo sh "$RUNNER_TEMP/cuda.run" --silent --toolkit + sudo sh "$RUNNER_TEMP/cuda.run" --silent --toolkit --override rm "$RUNNER_TEMP/cuda.run" echo "CUDA_PATH=/usr/local/cuda-${CUDA_VERSION%.*}" >> "$GITHUB_ENV" echo "/usr/local/cuda-${CUDA_VERSION%.*}/bin" >> "$GITHUB_PATH" From 0306be665774b4dfb9a4089c8321b57ee6d9de87 Mon Sep 17 00:00:00 2001 From: "Gilad S." Date: Sun, 4 Oct 2026 01:48:52 +0200 Subject: [PATCH 5/5] fix: Muse test timezone --- test/standalone/chatWrappers/MuseChatWrapper.test.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/test/standalone/chatWrappers/MuseChatWrapper.test.ts b/test/standalone/chatWrappers/MuseChatWrapper.test.ts index 8eddbadf..eef652b2 100644 --- a/test/standalone/chatWrappers/MuseChatWrapper.test.ts +++ b/test/standalone/chatWrappers/MuseChatWrapper.test.ts @@ -36,9 +36,11 @@ describe("MuseChatWrapper", () => { }); test("should generate the default system message", () => { + const todayDate = new Date("2026-08-11T00:00:00Z"); + todayDate.setDate(11); // ensure the date is 11 regardless of timezone const chatWrapper = new MuseChatWrapper({ reasoningStrength: "low", - todayDate: new Date("2026-08-11T00:00:00Z") + todayDate }); const {contextText} = chatWrapper.generateContextState({