declarative natives build script, MSVC for Windows targets + FreeBSD x86 & Windows ARM natives

README.md

@@ -27,7 +27,7 @@ View the [Sable Developer Wiki](https://github.com/ryanhcode/sable/wiki) for doc
 # Building Rust Natives
 
 1. Install Docker from https://www.docker.com/get-started/ or from your relevant package manager
-2. Run `gradlew common:buildImage` (only has to be done once)
+2. Run `gradlew common:buildImages` (only has to be done once)
 3. Run `gradlew common:buildRustNatives`
 
 ### Thanks

common/build.gradle

@@ -1,16 +1,53 @@
 import org.apache.tools.ant.taskdefs.condition.Os
 
+import org.tukaani.xz.LZMA2Options
+import org.tukaani.xz.XZOutputStream
+
+import java.nio.file.Files
+buildscript {
+    dependencies {
+        classpath 'org.tukaani:xz:1.10'
+    }
+}
+
 plugins {
     id 'multiloader-common'
     id 'net.neoforged.moddev'
 }
 
+def mac(arch) {
+    [triple: "${arch}-apple-darwin", arch: arch, os: "macos", ext: "dylib", lib: true]
+}
+def linux(arch) {
+    [triple: "${arch}-unknown-linux-gnu", arch: arch, os: "linux", ext: "so", lib: true]
+}
+def freebsd(arch) {
+    [triple: "${arch}-unknown-freebsd", arch: arch, os: "freebsd", ext: ".so", lib: true]
+}
+def windows(arch) {
+    [triple: "${arch}-pc-windows-msvc", arch: arch, os: "windows", ext: "dll", lib: false]
+}
+
+def supportedTargets = [
+    mac("x86_64"),
+    mac("aarch64"),
+    linux("x86_64"),
+    linux("aarch64"),
+    windows("x86_64"),
+    windows("aarch64"),
+    freebsd("x86_64"),
+    // freebsd("aarch64"),
+]
+def pinnedRustVersion = 'nightly-2026-01-29'
 def rustRootDir = file("src/main/rust")
 def rustProjectDir = file("$rustRootDir/rapier")
 def nativesDir = file("src/main/resources/natives/sable_rapier")
 def cargoCacheDir = project.layout.buildDirectory.file("cargo").get().asFile
 def docker = Os.isFamily(Os.FAMILY_MAC) ? "/usr/local/bin/docker" : "docker"
-def dockerCommand = [
+def xWinImage = 'sable-build-xwin'
+def zigbuildImage = 'sable-build-zigbuild'
+
+def baseDockerCommand = [
         docker,
         'run',
         '--rm',
@@ -22,8 +59,9 @@ def dockerCommand = [
         "$cargoCacheDir/registry:/usr/local/cargo/registry",
         '-w',
         '/io/rapier',
-        'sable-build'
 ]
+project.ext.zigbuildCargo = [baseDockerCommand, zigbuildImage, 'cargo'].flatten()
+project.ext.xWinCargo = [baseDockerCommand, xWinImage, 'cargo'].flatten()
 
 tasks.register('createContainersDirectory') {
     doLast {
@@ -33,116 +71,112 @@ tasks.register('createContainersDirectory') {
     }
 }
 
-tasks.register('buildImage', Exec) {
+
+tasks.register('buildZigbuildImage', Exec) {
     group = 'rust'
     workingDir rustRootDir
-    commandLine docker, 'build', '-t', 'sable-build', '.'
+    commandLine docker, 'build', '-t', zigbuildImage, 'container/zigbuild', '--build-arg', "RUST_VERSION=${pinnedRustVersion}"
     dependsOn createContainersDirectory
 }
-
-tasks.register('cleanRust', Exec) {
+tasks.register('buildXWinImage', Exec) {
     group = 'rust'
-    workingDir rustProjectDir
-    commandLine dockerCommand
-    args 'cargo', 'clean'
+    workingDir rustRootDir
+    commandLine docker, 'build', '-t', xWinImage, 'container/xwin', '--build-arg', "RUST_VERSION=${pinnedRustVersion}"
     dependsOn createContainersDirectory
 }
 
-tasks.register('compileRust', Exec) {
+tasks.register('buildImages') {
     group = 'rust'
-    workingDir rustProjectDir
-    commandLine dockerCommand
-    args 'cargo', 'zigbuild'
-    dependsOn createContainersDirectory
-    finalizedBy copyRustNativesDev
+    dependsOn buildZigbuildImage, buildXWinImage
 }
 
-tasks.register('compileRustMacAArch64', Exec) {
+tasks.register('cleanRust', Exec) {
     group = 'rust'
     workingDir rustProjectDir
-    commandLine dockerCommand
-    args 'cargo', 'zigbuild', '--release', '--target', 'aarch64-apple-darwin'
+    commandLine project.ext.zigbuildCargo // only need to use zigbuild cargo since both containers share the same target folder
+    args 'clean'
     dependsOn createContainersDirectory
 }
 
-tasks.register('compileRustMacX86', Exec) {
-    group = 'rust'
-    workingDir rustProjectDir
-    commandLine dockerCommand
-    args 'cargo', 'zigbuild', '--release', '--target', 'x86_64-apple-darwin'
-    dependsOn createContainersDirectory
+def compileRustTaskName = { target ->
+    "compileRust-${target.os}-${target.arch}"
 }
 
-tasks.register('compileRustLinuxAArch64', Exec) {
-    group = 'rust'
-    workingDir rustProjectDir
-    commandLine dockerCommand
-    args 'cargo', 'zigbuild', '--release', '--target', 'aarch64-unknown-linux-gnu.2.17'
-    dependsOn createContainersDirectory
-}
+def commandLineForTarget(target) {
+    if(target.triple.contains('msvc')) {
+        [project.ext.xWinCargo, 'xwin', 'build', '--release', '--target', target.triple].flatten()
+        // support meme platforms, this does nothing by default because the target is disabled
+    } else if(target.triple == 'aarch64-unknown-freebsd') {
+        [project.ext.zigbuildCargo, 'zigbuild', '-Z', 'build-std', '--release', '--target', target.triple].flatten()
+    } else {
+        [project.ext.zigbuildCargo, 'zigbuild', '--release', '--target', target.triple].flatten()
+    }
 
-tasks.register('compileRustLinux', Exec) {
-    group = 'rust'
-    workingDir rustProjectDir
-    commandLine dockerCommand
-    args 'cargo', 'zigbuild', '--release', '--target', 'x86_64-unknown-linux-gnu.2.17'
-    dependsOn createContainersDirectory
 }
-
-tasks.register('compileRustWindows', Exec) {
-    group = 'rust'
-    workingDir rustProjectDir
-    commandLine dockerCommand
-    args 'cargo', 'zigbuild', '--release', '--target', 'x86_64-pc-windows-gnu'
-    dependsOn createContainersDirectory
+def nativesNameForTarget(target) {
+    "sable_rapier_${target.arch}_${target.os}.${target.ext}"
+}
+supportedTargets.forEach { target ->
+    tasks.register(compileRustTaskName(target), Exec) {
+        group = 'rust'
+        workingDir rustProjectDir
+        description = "Cross-compiles natives for the ${target.triple} target"
+        commandLine = commandLineForTarget(target)
+    }
 }
 
 tasks.register('buildRustNatives') {
     group = 'build'
     description = 'Compiles all Rust natives and moves them to resources.'
 
-    dependsOn compileRustMacAArch64, compileRustMacX86, compileRustLinux, compileRustLinuxAArch64, compileRustWindows
+    dependsOn = supportedTargets.stream().map(compileRustTaskName).collect()
     finalizedBy copyRustNatives
 }
 
 tasks.register('copyRustNatives', Copy) {
     group = 'rust'
-
     into nativesDir
     mustRunAfter(buildRustNatives)
 
-    def moves = [
-            [src: "target/aarch64-apple-darwin/release/libsable_rapier.dylib", dest: "sable_rapier_aarch64_macos.dylib"],
-            [src: "target/x86_64-apple-darwin/release/libsable_rapier.dylib", dest: "sable_rapier_x86_64_macos.dylib"],
-            [src: "target/x86_64-unknown-linux-gnu/release/libsable_rapier.so", dest: "sable_rapier_x86_64_linux.so"],
-            [src: "target/aarch64-unknown-linux-gnu/release/libsable_rapier.so", dest: "sable_rapier_aarch64_linux.so"],
-            [src: "target/x86_64-pc-windows-gnu/release/sable_rapier.dll", dest: "sable_rapier_x86_64_windows.dll"]
-    ]
-
-    moves.forEach { map ->
-        from(file("$rustRootDir/${map.src}")) {
-            rename { map.dest }
+    supportedTargets.forEach { target ->
+        from(file("$rustRootDir/target/${target.triple}/release/${ if(target.lib) {"lib"} else {""}}sable_rapier.${target.ext}")) {
+            rename { nativesNameForTarget(target) }
         }
     }
+    finalizedBy packRustNatives
 }
 
 tasks.register('copyRustNativesDev', Copy) {
     group = 'rust'
+    into nativesDir
 
+    supportedTargets.forEach { target ->
+        from(file("$rustRootDir/target/debug/${ if(target.lib) {"lib"} else {""}}sable_rapier.${target.ext}")) {
+            rename { nativesNameForTarget(target) }
+        }
+    }
+    finalizedBy packRustNatives
+}
+tasks.register('packRustNatives', Tar) {
+    group = 'rust'
+    archiveFile.set file("${nativesDir}/sable_rapier_binaries.tar.xz")
     into nativesDir
-    mustRunAfter(compileRust)
-
-    def moves = [
-            [src: "target/debug/libsable_rapier.dylib", dest: "sable_rapier_aarch64_macos.dylib"],
-            [src: "target/debug/libsable_rapier.dylib", dest: "sable_rapier_x86_64_macos.dylib"],
-            [src: "target/debug/libsable_rapier.so", dest: "sable_rapier_aarch64_linux.so"],
-            [src: "target/debug/libsable_rapier.so", dest: "sable_rapier_x86_64_linux.so"],
-            [src: "target/debug/sable_rapier.dll", dest: "sable_rapier_aarch64_windows.dll"],
-            [src: "target/debug/sable_rapier.dll", dest: "sable_rapier_x86_64_windows.dll"]
-    ]
-    moves.forEach { map ->
-        from(file("$rustRootDir/${map.src}")) {
-            rename { map.dest }
+
+    mustRunAfter copyRustNatives, copyRustNativesDev
+    supportedTargets.forEach { target ->
+        var f = nativesNameForTarget(target);
+        // No, you can't use rename here in case you were wondering.
+        from(file("${nativesDir}/${f}")) { eachFile { setPath f } }
+    }
+    doLast {
+        byte[] bytes = Files.readAllBytes(getArchiveFile().get().asFile.toPath());
+        try (var f = new FileOutputStream(getArchiveFile().get().asFile))  {
+            try (var x = new XZOutputStream(f, new LZMA2Options(LZMA2Options.PRESET_MAX))) {
+                x.write(bytes);
+            }
+        }
+        supportedTargets.forEach { t ->
+            delete(file("${nativesDir}/${nativesNameForTarget(t)}"))
         }
     }
 }

common/src/main/java/dev/ryanhcode/sable/physics/impl/rapier/Rapier3D.java

@@ -9,9 +9,13 @@
 import net.minecraft.Util;
 import net.minecraft.Util.OS;
 import net.minecraft.server.level.ServerLevel;
+
+import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
+import org.apache.commons.compress.archivers.tar.TarArchiveInputStream;
 import org.jetbrains.annotations.ApiStatus;
 import org.joml.Matrix3dc;
 import org.joml.Vector3dc;
+import org.tukaani.xz.XZInputStream;
 
 import java.io.FileNotFoundException;
 import java.io.InputStream;
@@ -57,19 +61,33 @@ private static String getNativeName() {
 
     private static void loadLibrary() {
         final String nativeName = getNativeName();
-        try (final InputStream is = Rapier3D.class.getResourceAsStream("/natives/" + LIB_NAME + "/" + nativeName)) {
+        try (final InputStream is = Rapier3D.class.getResourceAsStream("/natives/" + LIB_NAME + "/sable_rapier_binaries.tar.xz")) {
             if (is == null) {
-                throw new FileNotFoundException(LIB_NAME);
+                throw new FileNotFoundException("sable_rapier_binaries.tar.xz");
+            }
+            try (final XZInputStream is2 = new XZInputStream(is);
+                 final TarArchiveInputStream ti = new TarArchiveInputStream(is2)) {
+
+                TarArchiveEntry entry;
+                while ((entry = ti.getNextEntry()) != null) {
+                    if (entry.getName().equals(nativeName)) {
+                        final String[] split = nativeName.split("\\.");
+                        final Path tempFile = Files.createTempFile(split[0], "." + split[1]);
+                        Files.copy(ti, tempFile, StandardCopyOption.REPLACE_EXISTING);
+                        System.load(tempFile.toAbsolutePath().toString());
+                        ENABLED = true;
+                        return;
+                    }
+                }
+
+                throw new FileNotFoundException(nativeName);
             }
-
-            final Path tempFile = Files.createTempFile(LIB_TMP_DIR_PREFIX, null);
-            Files.copy(is, tempFile, StandardCopyOption.REPLACE_EXISTING);
-            System.load(tempFile.toAbsolutePath().toString());
-            ENABLED = true;
         } catch (final Throwable t) {
             ENABLED = false;
 
-            Sable.LOGGER.error("Sable has failed to load the natives needed for its Rapier pipeline. Native library name {}. Please report with system details and logs to {}", nativeName, Sable.ISSUE_TRACKER_URL, t);
+            Sable.LOGGER.error(
+                    "Sable has failed to load the natives needed for its Rapier pipeline. Native library name {}. Please report with system details and logs to {}",
+                    nativeName, Sable.ISSUE_TRACKER_URL, t);
         }
     }
 

common/src/main/rust/Cargo.toml

@@ -14,3 +14,11 @@ jni = "0.21.1"
 colored = "2.1.0"
 fern = { version = "0.6.2", features = ["colored"] }
 humantime = "2.1.0"
+
+
+[profile.release]
+lto = "thin"
+codegen-units = 1
+
+[profile.bench]
+debug = true

common/src/main/rust/container/xwin/Dockerfile

@@ -0,0 +1,10 @@
+# Build container versions pinned on 04-18-26
+FROM messense/cargo-xwin@sha256:f3e680c963eff35f77908c78d97ac095d4bef1f69e4fe200447576b34cd251cb
+
+ARG RUST_VERSION
+RUN rustup default ${RUST_VERSION}
+RUN rustup target add x86_64-pc-windows-msvc
+RUN rustup target add aarch64-pc-windows-msvc
+RUN cargo xwin cache xwin
+
+WORKDIR /

common/src/main/rust/container/zigbuild/Dockerfile

@@ -0,0 +1,14 @@
+# Build container versions pinned on 04-18-26
+FROM messense/cargo-zigbuild@sha256:4c18634a3d9c7775683b0f1e313584770d32216623a1273a104b634fa294bf4c
+
+ARG RUST_VERSION
+RUN rustup default ${RUST_VERSION}
+RUN rustup target add x86_64-apple-darwin
+RUN rustup target add aarch64-apple-darwin
+RUN rustup target add x86_64-unknown-linux-gnu
+RUN rustup target add aarch64-unknown-linux-gnu
+RUN rustup target add x86_64-unknown-freebsd
+# used for AArch64 FreeBSD
+#RUN rustup component add rust-src
+
+WORKDIR /

common/src/main/rust/rapier/Cargo.toml

@@ -26,16 +26,9 @@ crate-type = ["cdylib", "lib"]
 name = "sable_rapier"
 path = "src/lib.rs"
 
-[profile.bench]
-debug = true
-
 [[bench]]
 name = "collision_benchmark"
 harness = false
 
 [dev-dependencies]
 criterion = "0.8.2"
-
-[profile.release]
-lto = "thin"
-codegen-units = 1