diff --git a/README.md b/README.md index fee69f77..1732385f 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,8 @@ This repository holds Replit's Nix modules. * Each module is located as a folder under `pkgs/modules` -* `pkgs/modules/default.nix` specifies a list of the active modules, some of which are parameterized with the version of the runtime or compiler +* `pkgs/modules/default.nix` registers every shipped implementation, some of which are parameterized with the version of the runtime or compiler +* Docker and Zig 0.11 remain compatibility-only implementations: included in the bundle but excluded from `activeModules` and `activeDeploymentModules`, as before To list all active modules, you can do: @@ -24,12 +25,29 @@ Output might look like: } ``` -To list all modules (including historical modules): +To list all modules (including compatibility implementations and retired-ID aliases): ``` nix eval .#modules --json | jq ``` -This will take a while. +### Supporting and retiring versions + +Every shipped module implementation lives in this checkout. Module definitions +must not import outputs from an older nixmodules flake. Runtime packages can use +explicit, locked upstream inputs: Docker and Zig retain their previous toolchain +pins through `nixpkgs-docker`, `nixpkgs-zig`, and `zig-overlay`. Their definitions +use the current module schema; moving them here does not upgrade their runtimes. + +When adding a version, prefer another registration of a parameterized module. +To keep an older version supported, retain its package pin and registration. +To retire an implementation, remove its registration and unused definition and +package inputs, then optionally add an upgrade alias for its old ID. An alias is +not a supported version of the old runtime. Removing the alias stops accepting +that ID in new bundles; neither operation changes previously published bundles. + +Implementation IDs must be unique. Aliases must resolve to an implementation, +must not form cycles, and must not shadow an implementation. CI checks these +invariants and evaluates both development and deployment module maps. Retired module IDs remain available through `pkgs/upgrade-map`: they resolve to newer modules without keeping the retired implementations in the bundle. @@ -43,7 +61,7 @@ runtime and tools with this bundle and may need code or dependency changes. For each of the following examples, the build result will be found in a file or directory named `result`. -To build all active modules (this will take a while, and a lot of disk space): +To build the full bundle, including compatibility modules (this will take a while, and a lot of disk space): ``` nix build .#bundle diff --git a/flake.lock b/flake.lock index fdf7544d..2586be58 100644 --- a/flake.lock +++ b/flake.lock @@ -18,8 +18,8 @@ }, "crane": { "inputs": { - "flake-compat": "flake-compat", - "flake-utils": "flake-utils_3", + "flake-compat": "flake-compat_2", + "flake-utils": "flake-utils_4", "nixpkgs": [ "ztoc-rs", "nixpkgs" @@ -77,6 +77,22 @@ "type": "github" } }, + "flake-compat_2": { + "flake": false, + "locked": { + "lastModified": 1673956053, + "narHash": "sha256-4gtG9iQuiKITOjNQQeQIpoIB6b16fm+504Ch3sNKLd8=", + "owner": "edolstra", + "repo": "flake-compat", + "rev": "35bb57c0c8d8b62bbfd284272c928ceb64ddbde9", + "type": "github" + }, + "original": { + "owner": "edolstra", + "repo": "flake-compat", + "type": "github" + } + }, "flake-utils": { "inputs": { "systems": "systems" @@ -111,6 +127,21 @@ } }, "flake-utils_3": { + "locked": { + "lastModified": 1659877975, + "narHash": "sha256-zllb8aq3YO3h8B/U0/J1WBgAL8EX5yWf5pMj3G0NAmc=", + "owner": "numtide", + "repo": "flake-utils", + "rev": "c0e246b9b83f637f4681389ecabcb2681b4f3af0", + "type": "github" + }, + "original": { + "owner": "numtide", + "repo": "flake-utils", + "type": "github" + } + }, + "flake-utils_4": { "locked": { "lastModified": 1676283394, "narHash": "sha256-XX2f9c3iySLCw54rJ/CZs+ZK6IQy7GXNY4nSOyu2QG4=", @@ -125,7 +156,7 @@ "type": "github" } }, - "flake-utils_4": { + "flake-utils_5": { "locked": { "lastModified": 1676283394, "narHash": "sha256-XX2f9c3iySLCw54rJ/CZs+ZK6IQy7GXNY4nSOyu2QG4=", @@ -246,6 +277,22 @@ "type": "github" } }, + "nixpkgs-docker": { + "locked": { + "lastModified": 1725503534, + "narHash": "sha256-hd1eRyPtTkRnAPYpnEfzugQwAifVYMmOR3z+MTTSw+g=", + "owner": "nixos", + "repo": "nixpkgs", + "rev": "fcb54ddcc974cff59bdfb7c1ac9e080299763d2d", + "type": "github" + }, + "original": { + "owner": "nixos", + "repo": "nixpkgs", + "rev": "fcb54ddcc974cff59bdfb7c1ac9e080299763d2d", + "type": "github" + } + }, "nixpkgs-master": { "locked": { "lastModified": 1768419155, @@ -326,6 +373,22 @@ "type": "github" } }, + "nixpkgs-zig": { + "locked": { + "lastModified": 1687466461, + "narHash": "sha256-oupXI7g7RPzlpGUfAu1xG4KBK53GrZH8/xeKgKDB4+Q=", + "owner": "nixos", + "repo": "nixpkgs", + "rev": "ecb441f22067ba1d6312f4932a7c64efa8d19a7b", + "type": "github" + }, + "original": { + "owner": "nixos", + "repo": "nixpkgs", + "rev": "ecb441f22067ba1d6312f4932a7c64efa8d19a7b", + "type": "github" + } + }, "prybar": { "inputs": { "flake-utils": "flake-utils_2", @@ -376,13 +439,16 @@ "nixpkgs-23_05": "nixpkgs-23_05", "nixpkgs-24_11": "nixpkgs-24_11", "nixpkgs-25_05": "nixpkgs-25_05", + "nixpkgs-docker": "nixpkgs-docker", "nixpkgs-master": "nixpkgs-master", "nixpkgs-node24": "nixpkgs-node24", "nixpkgs-nodejs": "nixpkgs-nodejs", "nixpkgs-php": "nixpkgs-php", "nixpkgs-staging": "nixpkgs-staging", + "nixpkgs-zig": "nixpkgs-zig", "prybar": "prybar", "replit-rtld-loader": "replit-rtld-loader", + "zig-overlay": "zig-overlay", "ztoc-rs": "ztoc-rs" } }, @@ -466,11 +532,34 @@ "type": "github" } }, + "zig-overlay": { + "inputs": { + "flake-compat": "flake-compat", + "flake-utils": "flake-utils_3", + "nixpkgs": [ + "nixpkgs-zig" + ] + }, + "locked": { + "lastModified": 1697285248, + "narHash": "sha256-CPV/LWV/2nJKasawkshdCX1bLRsKGtSSiDoy14jix8c=", + "owner": "mitchellh", + "repo": "zig-overlay", + "rev": "ee376d4012e3c853123ca7083af852313235407c", + "type": "github" + }, + "original": { + "owner": "mitchellh", + "repo": "zig-overlay", + "rev": "ee376d4012e3c853123ca7083af852313235407c", + "type": "github" + } + }, "ztoc-rs": { "inputs": { "advisory-db": "advisory-db", "crane": "crane", - "flake-utils": "flake-utils_4", + "flake-utils": "flake-utils_5", "nixpkgs": [ "nixpkgs" ] diff --git a/flake.nix b/flake.nix index 581289ed..d8d77d58 100644 --- a/flake.nix +++ b/flake.nix @@ -9,6 +9,10 @@ inputs.nixpkgs-nodejs.url = "github:nixos/nixpkgs/5873d3da531a3a8320c9e639e2147eccbe7f2d0d"; inputs.nixpkgs-node24.url = "github:nixos/nixpkgs/dc0146be65ada0333393ac87b6ff45a4a8e9b722"; inputs.nixpkgs-php.url = "github:nixos/nixpkgs/8c259fc0dd1f3ccc1aaf4f9156ab16b7328b5a4f"; + inputs.nixpkgs-docker.url = "github:nixos/nixpkgs/fcb54ddcc974cff59bdfb7c1ac9e080299763d2d"; + inputs.nixpkgs-zig.url = "github:nixos/nixpkgs/ecb441f22067ba1d6312f4932a7c64efa8d19a7b"; + inputs.zig-overlay.url = "github:mitchellh/zig-overlay/ee376d4012e3c853123ca7083af852313235407c"; + inputs.zig-overlay.inputs.nixpkgs.follows = "nixpkgs-zig"; inputs.fenix.url = "github:nix-community/fenix"; inputs.fenix.inputs.nixpkgs.follows = "nixpkgs"; inputs.nil.url = "github:oxalica/nil"; @@ -22,7 +26,7 @@ inputs.replit-rtld-loader.url = "github:replit/replit_rtld_loader"; inputs.replit-rtld-loader.inputs.nixpkgs.follows = "nixpkgs"; - outputs = { self, nixpkgs, nixpkgs-23_05, nixpkgs-24_11, nixpkgs-25_05, nixpkgs-master, nixpkgs-staging, nixpkgs-nodejs, nixpkgs-node24, nixpkgs-php, prybar, java-language-server, nil, fenix, replit-rtld-loader, ... }: + outputs = { self, nixpkgs, nixpkgs-23_05, nixpkgs-24_11, nixpkgs-25_05, nixpkgs-master, nixpkgs-staging, nixpkgs-nodejs, nixpkgs-node24, nixpkgs-php, nixpkgs-docker, nixpkgs-zig, zig-overlay, prybar, java-language-server, nil, fenix, replit-rtld-loader, ... }: let mkPkgs = nixpkgs-spec: system: import nixpkgs-spec { inherit system; @@ -50,6 +54,18 @@ pkgs-php = import nixpkgs-php { system = "x86_64-linux"; }; pkgs-nodejs = import nixpkgs-nodejs { system = "x86_64-linux"; }; pkgs-node24 = import nixpkgs-node24 { system = "x86_64-linux"; }; + # Keep the shipped toolchains pinned while moving their module definitions in-tree. + pkgs-docker = import + (nixpkgs-docker.legacyPackages.x86_64-linux.applyPatches { + name = "nixpkgs-unstable-patched"; + src = nixpkgs-docker; + patches = [ ]; + }) + { system = "x86_64-linux"; }; + pkgs-zig = import nixpkgs-zig { + system = "x86_64-linux"; + overlays = [ zig-overlay.overlays.default ]; + }; nodejs-22 = pkgs-nodejs.nodejs_22; nodejs-24 = pkgs-node24.nodejs_24; deno-src = pkgs-nodejs.fetchFromGitHub { @@ -136,6 +152,18 @@ packages.x86_64-linux = import ./pkgs { inherit pkgs self; }; + checks.x86_64-linux.module-registry = + assert import ./scripts/module-registry-test.nix; + assert builtins.attrNames self.modules == builtins.attrNames self.deploymentModules; + assert builtins.attrNames self.activeModules == builtins.attrNames self.activeDeploymentModules; + assert builtins.all + (id: + builtins.hasAttr id self.modules + && builtins.hasAttr id self.deploymentModules + && !(builtins.hasAttr id self.activeModules) + && !(builtins.hasAttr id self.activeDeploymentModules)) + [ "docker" "zig-0.11" ]; + pkgs.writeText "module-registry-check" "ok"; devShells.x86_64-linux.default = pkgs.mkShell { packages = with pkgs; [ python310 @@ -158,6 +186,7 @@ inherit deno uv; inherit nodejs-22 nodejs-24; inherit pkgs-php; + inherit pkgs-docker pkgs-zig; } ); } diff --git a/pkgs/historical-modules/default.nix b/pkgs/historical-modules/default.nix deleted file mode 100644 index 116ab89c..00000000 --- a/pkgs/historical-modules/default.nix +++ /dev/null @@ -1,118 +0,0 @@ -{ pkgs }: -with builtins; -let - # This is a list of modules we no longer actively maintain - # but still provide to users via historical versions of this repo. - # It can happen when we upgrade a Nix channel, which changes the - # version of the software, so the older version is no longer in the - # channel. Technically it is feasible to maintain older versions even - # in the latest channel, but it's more work. - historicalModulesList = [ - { - moduleId = "bun-1.0"; - commit = "fdbd39619614e07d1ff44cfe20de2ad56eb8a40f"; - overrides = { - # /nix/store/cvz36f39793v9l361ibfxznjjmk4jpd6-replit-module-bun-1.0 - # .runners["package.json"].displayVersion = "1.0.23"; - displayVersion = "1.0.23"; - }; - } - { - moduleId = "bun-1.1"; - commit = "1113faeb6840ba57bef3679b1f763782f5d3dd30"; - overrides = { - displayVersion = "1.1.45"; - }; - } - { - moduleId = "dart-3.8"; - commit = "ae6ed05c623804560afd1cf3d80d13b6170d5c24"; - overrides = { - displayVersion = "3.8"; - }; - } - { - moduleId = "docker"; - commit = "185d7dc7645178f92fd891f9efa20b3343c0f20d"; - overrides = { - displayVersion = "1"; - internal = true; - }; - } - { - moduleId = "zig-0.11"; - commit = "15426ef79793bf7c424eb40865d507eacfdd44e6"; - overrides = { - # /nix/store/65v1j76svgdi8bj3sxx5hfshbvdyissl-replit-module-zig-0.11 - # .runners["zig-build-run"].displayVersion = "0.11.0"; - displayVersion = "0"; - }; - } - { - moduleId = "go-1.24"; - commit = "ae6ed05c623804560afd1cf3d80d13b6170d5c24"; - overrides = { - displayVersion = "1.24"; - }; - } - { - moduleId = "ruby-3.1"; - commit = "ae6ed05c623804560afd1cf3d80d13b6170d5c24"; - overrides = { - displayVersion = "3.1"; - }; - } - ]; - - moduleFromHistory = - { moduleId - , commit - , deployment ? false - , overrides - , - }: - let - flake = getFlake "github:replit/nixmodules/${commit}"; - module = - if deployment then - (flake.deploymentModules or flake.modules).${moduleId} - else - flake.modules.${moduleId}; - in - pkgs.stdenvNoCC.mkDerivation { - name = "replit-module-${moduleId}"; - buildCommand = '' - set -x - ${pkgs.jq}/bin/jq --argjson overrides '${builtins.toJSON (overrides)}' '. * $overrides' < ${module} > $out - ''; - }; - - modules = foldl' - ( - acc: module: - acc - // ({ - ${module.moduleId} = moduleFromHistory module; - }) - ) - { } - historicalModulesList; - - deploymentModules = foldl' - ( - acc: module: - acc - // ({ - ${module.moduleId} = moduleFromHistory { - inherit (module) moduleId commit overrides; - deployment = true; - }; - }) - ) - { } - historicalModulesList; - -in -{ - inherit modules deploymentModules; -} diff --git a/pkgs/modules/bun/default.nix b/pkgs/modules/bun/default.nix index 397a8c42..008bf722 100644 --- a/pkgs/modules/bun/default.nix +++ b/pkgs/modules/bun/default.nix @@ -1,10 +1,9 @@ # NOTE: Unlike Node.js, nixpkgs doesn't provide versioned bun attributes (e.g., bun_1_3). # This module uses pkgs.bun which tracks the latest version. # -# When nixpkgs updates bun to a new major.minor version: -# 1. Add the previous version to pkgs/historical-modules/default.nix, pinned to the -# commit before the nixpkgs update -# 2. Update pkgs/upgrade-map/default.nix to chain the old version to the new one +# When nixpkgs updates bun to a new major.minor version, either retain the old +# package pin and register both versions, or retire the old implementation and +# map its ID to the new version in pkgs/upgrade-map/default.nix. Do not do both. { bun }: { pkgs, lib, ... }: diff --git a/pkgs/modules/default.nix b/pkgs/modules/default.nix index 95872ccb..78dd6137 100644 --- a/pkgs/modules/default.nix +++ b/pkgs/modules/default.nix @@ -9,6 +9,8 @@ , deno , uv , pkgs-php +, pkgs-docker +, pkgs-zig , ... }: with builtins; @@ -27,7 +29,16 @@ let deployment = true; }; apply-upgrade-map = import ../upgrade-map; - historical = pkgs.callPackage ../historical-modules { }; + moduleSet = import ./module-set.nix; + + # Keep these out of activeModules, as before, without hiding their implementations. + compatibilityModulesList = [ + (import ./docker { pkgs = pkgs-docker; }) + (import ./zig { + zig = pkgs-zig.zigpkgs.default; + zls = pkgs-zig.zls; + }) + ]; modulesList = [ (import ./python { @@ -96,7 +107,6 @@ let (import ./cpp) (import ./dart) (import ./deno { inherit deno; }) - # (import ./docker) (import ./elixir) (import ./gcloud) (import ./clojure) @@ -164,39 +174,21 @@ let (import ./databricks-cli) ]; - activeModules = listToAttrs ( - map - ( - moduleInput: - let - module = mkModule moduleInput; - in - { - name = get-module-id module; - value = module; - } - ) - modulesList - ); - - modules = apply-upgrade-map (activeModules // historical.modules); + mkModuleSet = buildModule: getId: inputs: + moduleSet (map + (input: + let module = buildModule input; + in { name = getId module; value = module; }) + inputs); - activeDeploymentModules = listToAttrs ( - map - ( - moduleInput: - let - module = mkDeploymentModule moduleInput; - in - { - name = get-deployment-module-id module; - value = module; - } - ) - modulesList - ); + developmentModuleSet = mkModuleSet mkModule get-module-id; + deploymentModuleSet = mkModuleSet mkDeploymentModule get-deployment-module-id; + allModulesList = modulesList ++ compatibilityModulesList; - deploymentModules = apply-upgrade-map (activeDeploymentModules // historical.deploymentModules); + activeModules = developmentModuleSet modulesList; + activeDeploymentModules = deploymentModuleSet modulesList; + modules = apply-upgrade-map (developmentModuleSet allModulesList); + deploymentModules = apply-upgrade-map (deploymentModuleSet allModulesList); get-module-id = module: diff --git a/pkgs/modules/docker/default.nix b/pkgs/modules/docker/default.nix index 9ada8fc3..3bf050a0 100644 --- a/pkgs/modules/docker/default.nix +++ b/pkgs/modules/docker/default.nix @@ -4,7 +4,7 @@ let configFiles = pkgs.copyPathToStore ./etc; - replit-runc = pkgs.buildGo123Module { + replit-runc = pkgs.buildGo121Module { pname = "replit-runc"; version = "1.1.9+replit"; @@ -30,7 +30,7 @@ let ''; }; - replit-containerd = pkgs.buildGo123Module { + replit-containerd = pkgs.buildGo121Module { pname = "replit-containerd"; version = "1.7.5+replit"; @@ -65,7 +65,7 @@ let replitShimRunc = replit-containerd; }; - replit-buildkit = pkgs.buildGo123Module { + replit-buildkit = pkgs.buildGo121Module { pname = "replit-buildkit"; version = "v0.13.0-beta1+replit"; @@ -164,6 +164,7 @@ in { id = "docker"; + displayVersion = "1"; name = "Support for Docker containers"; internal = true; description = '' diff --git a/pkgs/modules/module-set.nix b/pkgs/modules/module-set.nix new file mode 100644 index 00000000..c124aa85 --- /dev/null +++ b/pkgs/modules/module-set.nix @@ -0,0 +1,8 @@ +entries: +let + modules = builtins.listToAttrs entries; +in +if builtins.length entries != builtins.length (builtins.attrNames modules) then + throw "Duplicate module implementation ID" +else + modules diff --git a/pkgs/modules/zig/default.nix b/pkgs/modules/zig/default.nix new file mode 100644 index 00000000..7547913f --- /dev/null +++ b/pkgs/modules/zig/default.nix @@ -0,0 +1,36 @@ +{ zig, zls }: +{ lib, ... }: +let + version = lib.versions.majorMinor zig.version; + language = "zig"; + extensions = [ ".zig" ]; +in +{ + id = "zig-${version}"; + name = "zig"; + displayVersion = "0"; + + replit = { + runners.zig-build-run = { + name = "zig build run"; + inherit language extensions; + fileParam = false; + start = "${zig}/bin/zig build run"; + }; + + dev = { + packages = [ zig ]; + formatters.zig-fmt = { + name = "zig fmt"; + inherit language extensions; + start = "${zig}/bin/zig fmt $file"; + stdin = false; + }; + languageServers.zls = { + name = "zls"; + inherit language extensions; + start = "${zls}/bin/zls"; + }; + }; + }; +} diff --git a/pkgs/upgrade-map/apply.nix b/pkgs/upgrade-map/apply.nix new file mode 100644 index 00000000..ff02529d --- /dev/null +++ b/pkgs/upgrade-map/apply.nix @@ -0,0 +1,20 @@ +aliases: modules: +with builtins; +let + resolve = seen: id: + if elem id seen then + throw "Module alias cycle: ${concatStringsSep " -> " (seen ++ [ id ])}" + else if hasAttr id aliases then + resolve (seen ++ [ id ]) aliases.${id} + else if hasAttr id modules then + id + else + throw "Module alias target has no implementation: ${id}"; + + resolved = mapAttrs (id: _: resolve [ ] id) aliases; + collisions = filter (id: hasAttr id modules) (attrNames aliases); +in +if collisions != [ ] then + throw "Module aliases shadow implementations: ${concatStringsSep ", " collisions}" +else + deepSeq resolved (modules // mapAttrs (_: target: modules.${target}) resolved) diff --git a/pkgs/upgrade-map/default.nix b/pkgs/upgrade-map/default.nix index 7a337d83..b7a38248 100644 --- a/pkgs/upgrade-map/default.nix +++ b/pkgs/upgrade-map/default.nix @@ -1,9 +1,3 @@ -/* - This implements auto-upgrading from one module ID to another at build time, based on the - mapping below. This is here for historical purposes, and we don't expect to use it much, although - there are valid use cases. -*/ -with builtins; let upgrade-map = { "bun" = "bun-0.5"; @@ -61,23 +55,5 @@ let "vue-node-18" = "vue-node-20"; }; - upgrade-module = - moduleId: if hasAttr moduleId upgrade-map then upgrade-module upgrade-map.${moduleId} else moduleId; - - apply-upgrade-map = - modules: - foldl' - ( - acc: moduleId: - let - upgraded = upgrade-module moduleId; - in - acc - // { - ${moduleId} = modules.${upgraded}; - } - ) - modules - (attrNames upgrade-map); in -apply-upgrade-map +import ./apply.nix upgrade-map diff --git a/scripts/ci_check.sh b/scripts/ci_check.sh index b964c939..b89b88ab 100755 --- a/scripts/ci_check.sh +++ b/scripts/ci_check.sh @@ -13,7 +13,16 @@ NIX_FLAGS=( ) echo "Evaluate modules derivations" +nix eval "${NIX_FLAGS[@]}" --file scripts/module-registry-test.nix nix eval "${NIX_FLAGS[@]}" .#modules --json +nix eval "${NIX_FLAGS[@]}" .#deploymentModules --json +nix build "${NIX_FLAGS[@]}" .#checks.x86_64-linux.module-registry --no-link + +# The standalone lockfile OCI helper reconstructs old bundles, not current implementations. +if git grep -n -E 'builtins[.]getFlake|(^|[^[:alnum:]_])getFlake[[:space:]]|github:replit/nixmodules/' -- '*.nix' ':!pkgs/mk-phony-ocis/default.nix'; then + echo "Module implementations must be defined in-tree, not imported from old nixmodules flakes" + exit 1 +fi nix develop "${NIX_FLAGS[@]}" --command echo Hello, world diff --git a/scripts/module-registry-test.nix b/scripts/module-registry-test.nix new file mode 100644 index 00000000..1f99a9d5 --- /dev/null +++ b/scripts/module-registry-test.nix @@ -0,0 +1,25 @@ +let + apply = import ../pkgs/upgrade-map/apply.nix; + moduleSet = import ../pkgs/modules/module-set.nix; + fails = value: !(builtins.tryEval (builtins.deepSeq value true)).success; + modules = { current = "implementation"; other = "other implementation"; }; + aliases = apply { old = "older"; older = "current"; } modules; +in +assert aliases.old == modules.current; +assert aliases.older == modules.current; +assert aliases.current == modules.current; +assert aliases.other == modules.other; +assert fails (apply { old = "missing"; } modules); +assert fails (apply { old = "old"; } modules); +assert fails (apply { old = "older"; older = "old"; } modules); +assert fails (apply { current = "other"; } modules); +assert fails + (moduleSet [ + { name = "duplicate"; value = "first"; } + { name = "duplicate"; value = "second"; } + ]); +assert moduleSet [ + { name = "current"; value = "implementation"; } + { name = "other"; value = "other implementation"; } +] == modules; +true