Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .bazelrc.deleted_packages
Original file line number Diff line number Diff line change
Expand Up @@ -54,3 +54,4 @@ common --deleted_packages=tests/modules/other/simple_v1
common --deleted_packages=tests/modules/other/simple_v2
common --deleted_packages=tests/modules/other/with_external_data
common --deleted_packages=tests/modules/rules_pyrefly_stub/pyrefly
common --deleted_packages=tests/modules/rules_rust_stub/rust
1 change: 1 addition & 0 deletions BUILD.bazel
Original file line number Diff line number Diff line change
Expand Up @@ -79,6 +79,7 @@ filegroup(
"internal_dev_setup.bzl",
"version.bzl",
"//command_line_option:distribution",
"//crates:distribution",
"//python:distribution",
"//tools:distribution",
],
Expand Down
14 changes: 14 additions & 0 deletions MODULE.bazel
Original file line number Diff line number Diff line change
Expand Up @@ -98,6 +98,20 @@ bazel_dep(name = "rules_shell", version = "0.3.0", dev_dependency = True)
bazel_dep(name = "rules_multirun", version = "0.9.0", dev_dependency = True)
bazel_dep(name = "bazel_ci_rules", version = "1.0.0", dev_dependency = True)
bazel_dep(name = "rules_pkg", version = "1.2.0", dev_dependency = True)
bazel_dep(name = "rules_rust", version = "0.73.0", dev_dependency = True)

rust_crates = use_extension(
"@rules_rust//crate_universe:extensions.bzl",
"crate",
dev_dependency = True,
)
rust_crates.spec(
package = "sha2",
version = "0.10.8",
)
rust_crates.from_specs()
use_repo(rust_crates, "crates")

bazel_dep(name = "other", version = "0", dev_dependency = True)
bazel_dep(name = "another_module", version = "0", dev_dependency = True)

Expand Down
10 changes: 10 additions & 0 deletions crates/BUILD.bazel
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
package(default_visibility = ["//:__subpackages__"])

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Should we have a convention to keep the rust tools part of //tools?

Copy link
Copy Markdown
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

maybe? idk.

IMHO, tools/ is for things that are run as build tools. That would be most of the rust stuff, but there's a couple cases that wouldn't be:

  1. When we replace the shell bootstrap with a native executable bootstrap
  2. Similar for entry points in the venv bin/ directory (right now its a polyglot shell thing)

There were two other reasons I ended up going with the crates/<project> layout

  1. From what I could tell, crates/<project> seems the common idiom? This was just after some light searching, didn't look universal
  2. We had previously talked about creating a src/ directory for python stuff, e.g. segmenting code by language

I don't have a strong opinion, though I haven't thought about this much, and my brain is mostly mush right now.

The most important thing on my mind is preventing unnecessary dependencies from sneaking into prod code (e.g. a dev/test-only load occurs and then a BULID file can't load)


licenses(["notice"])

filegroup(
name = "distribution",
srcs = glob(["**"]) + [
"//crates/exe_zip_maker:distribution",
],
)
25 changes: 25 additions & 0 deletions crates/exe_zip_maker/BUILD.bazel
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
load("@rules_rust//rust:defs.bzl", "rust_binary", "rust_library")

package(default_visibility = ["//:__subpackages__"])

licenses(["notice"])

rust_library(
name = "exe_zip_maker_lib",
srcs = ["src/lib.rs"],
edition = "2021",
deps = ["@crates//:sha2"],
)

rust_binary(
name = "exe_zip_maker",
srcs = ["src/main.rs"],
edition = "2021",
visibility = ["//visibility:public"],
deps = [":exe_zip_maker_lib"],
)

filegroup(
name = "distribution",
srcs = glob(["**"]),
)
79 changes: 79 additions & 0 deletions crates/exe_zip_maker/src/lib.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,79 @@
//! Library supporting creating self-executable zip files.

use std::fs::{self, File};
use std::io::{self, BufReader, BufWriter, Read, Write};
use std::path::Path;

use sha2::{Digest, Sha256};

pub const BLOCK_SIZE: usize = 256 * 1024;
pub const PLACEHOLDER: &[u8] = b"%ZIP_HASH%";

/// Replaces all occurrences of `from` with `to` in `src`.
pub fn replace_bytes(src: &[u8], from: &[u8], to: &[u8]) -> Vec<u8> {
if from.is_empty() {
return src.to_vec();
}
let mut result = Vec::new();
let mut i = 0;
while i < src.len() {
if src[i..].starts_with(from) {
result.extend_from_slice(to);
i += from.len();
} else {
result.push(src[i]);
i += 1;
}
}
result
}

/// Computes the SHA256 hex digest of the file at `path`.
pub fn compute_file_sha256_hex(path: &Path) -> io::Result<String> {
let mut file = File::open(path)?;
let mut hasher = Sha256::new();
let mut buffer = [0u8; BLOCK_SIZE];
loop {
let n = file.read(&mut buffer)?;
if n == 0 {
break;
}
hasher.update(&buffer[..n]);
}
let digest = hasher.finalize();
Ok(format!("{:x}", digest))
}

/// Creates a self-executable zip archive by prepending a preamble to a zip archive
/// and substituting `%ZIP_HASH%` with the SHA-256 hash of the zip archive.
pub fn create_exe_zip(preamble_path: &Path, zip_path: &Path, output_path: &Path) -> io::Result<()> {
if let Some(parent) = output_path.parent() {
if !parent.as_os_str().is_empty() {
fs::create_dir_all(parent)?;
}
}

let zip_hash = compute_file_sha256_hex(zip_path)?;

let preamble_content = fs::read(preamble_path)?;
let modified_preamble = replace_bytes(&preamble_content, PLACEHOLDER, zip_hash.as_bytes());

let mut out_file = BufWriter::with_capacity(BLOCK_SIZE, File::create(output_path)?);
out_file.write_all(&modified_preamble)?;

let zip_file = File::open(zip_path)?;
let mut zip_reader = BufReader::with_capacity(BLOCK_SIZE, zip_file);
io::copy(&mut zip_reader, &mut out_file)?;
out_file.flush()?;

#[cfg(unix)]
{
use std::os::unix::fs::PermissionsExt;
let metadata = fs::metadata(output_path)?;
let mut perms = metadata.permissions();
perms.set_mode(perms.mode() | 0o111);
fs::set_permissions(output_path, perms)?;
}

Ok(())
}
24 changes: 24 additions & 0 deletions crates/exe_zip_maker/src/main.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
use std::env;
use std::path::Path;
use std::process;

fn main() {
let args: Vec<_> = env::args_os().collect();
if args.len() != 4 {
let prog_name = args
.first()
.map(|s| s.to_string_lossy().into_owned())
.unwrap_or_else(|| "exe_zip_maker".to_string());
eprintln!("Usage: {} <preamble> <zip> <output>", prog_name);
process::exit(1);
}

let preamble_path = Path::new(&args[1]);
let zip_path = Path::new(&args[2]);
let output_path = Path::new(&args[3]);

if let Err(e) = exe_zip_maker_lib::create_exe_zip(preamble_path, zip_path, output_path) {
eprintln!("exe_zip_maker: error: {}", e);
process::exit(1);
}
}
7 changes: 7 additions & 0 deletions internal_dev_deps.bzl
Original file line number Diff line number Diff line change
Expand Up @@ -114,6 +114,13 @@ def rules_python_internal_deps():
path = "tests/modules/rules_pyrefly_stub",
)

# Stub repository for rules_rust in WORKSPACE mode so that load()
# statements for @rules_rust resolve without requiring full rules_rust.
local_repository(
name = "rules_rust",
path = "tests/modules/rules_rust_stub",
)

# The below two deps are required for the integration test with bazel
# gazelle. Maybe the test should be moved to the `gazelle` workspace?
http_archive(
Expand Down
14 changes: 14 additions & 0 deletions tests/exe_zip_maker/BUILD.bazel
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
load("@rules_rust//rust:defs.bzl", "rust_test")

package(default_visibility = ["//:__subpackages__"])

licenses(["notice"])

rust_test(
name = "exe_zip_maker_test",
size = "small",
srcs = ["exe_zip_maker_test.rs"],
deps = [
"//crates/exe_zip_maker:exe_zip_maker_lib",
],
)
155 changes: 155 additions & 0 deletions tests/exe_zip_maker/exe_zip_maker_test.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,155 @@
use std::env;
use std::fs;

use exe_zip_maker_lib::{
compute_file_sha256_hex, create_exe_zip, replace_bytes, PLACEHOLDER,
};

#[test]

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

My thinking is that we should write idiomatic Rust code and what I've seen in the past is:

  • Rust code with unit tests is the same source file.
  • Integration tests is a separate file.

What do you think about this convention?

Copy link
Copy Markdown
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It originally generated code like that and I thought it was just being lazy. Another person confirmed it, too, though. I think that's a wacky convention.

But, if that'd idiomatic, then, well, when in Rust-ome

@aignas aignas Sep 7, 2026

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I personally like that the tests become sort of executable docs.

But I agree, it's a little whacky :D

Though it allows to keep the exposed symbols to minimum and keep the unit tests exercising the implementation details in optimum way.

Kind of no longer need to do "buildifier ignore private" if you follow this convention.

Copy link
Copy Markdown
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Hm, I also saw the rust_test target was outside of //tests. I'm more resistant to that -- I really like being able to run bazel test //tests/... to capture almost all the tests. It also helps keep test-only bzl code from sneaking inside the non-test directories

fn test_replace_bytes_none() {
let src = b"hello world";
assert_eq!(replace_bytes(src, b"foo", b"bar"), b"hello world");
}

#[test]
fn test_replace_bytes_single() {
let src = b"EXPECTED_HASH='%ZIP_HASH%'";
let replaced = replace_bytes(src, PLACEHOLDER, b"12345678");
assert_eq!(replaced, b"EXPECTED_HASH='12345678'");
}

#[test]
fn test_replace_bytes_multiple() {
let src = b"%ZIP_HASH% and %ZIP_HASH%";
let replaced = replace_bytes(src, PLACEHOLDER, b"abc");
assert_eq!(replaced, b"abc and abc");
}

#[test]
fn test_replace_bytes_empty_from() {
let src = b"unchanged";
assert_eq!(replace_bytes(src, b"", b"abc"), b"unchanged");
}

#[test]
fn test_compute_file_sha256_hex() {
let temp_dir = env::temp_dir().join(format!("sha256_test_{}", std::process::id()));
fs::create_dir_all(&temp_dir).unwrap();
let file_path = temp_dir.join("sample.txt");

fs::write(&file_path, b"hello world\n").unwrap();
let hash = compute_file_sha256_hex(&file_path).unwrap();
assert_eq!(
hash,
"a948904f2f0f479b8f8197694b30184b0d2ed1c1cd2a1ec0fb85d299a192a447"
);

let _ = fs::remove_dir_all(&temp_dir);
}

#[test]
fn test_create_exe_zip_successful() {
let temp_dir = env::temp_dir().join(format!("create_exe_zip_test_{}", std::process::id()));
fs::create_dir_all(&temp_dir).unwrap();

let preamble_path = temp_dir.join("preamble.sh");
let zip_path = temp_dir.join("data.zip");
let output_path = temp_dir.join("output.exe");

let zip_content = b"PK\x03\x04dummyzipcontent";
fs::write(&zip_path, zip_content).unwrap();

let preamble_text = b"#!/bin/bash\nEXPECTED_HASH='%ZIP_HASH%'\n# ... logic ...\n";
fs::write(&preamble_path, preamble_text).unwrap();

create_exe_zip(&preamble_path, &zip_path, &output_path).unwrap();

assert!(output_path.exists());

#[cfg(unix)]
{
use std::os::unix::fs::PermissionsExt;
let st = fs::metadata(&output_path).unwrap();
assert_ne!(
st.permissions().mode() & 0o100,
0,
"Expected executable permission on output file"
);
}

let content = fs::read(&output_path).unwrap();
let expected_hash = "65e39989ca91c49484998aa3f0429f6943c029609bfd2f3c18c77bf9ded72c59";
let expected_preamble = replace_bytes(preamble_text, PLACEHOLDER, expected_hash.as_bytes());

assert!(content.starts_with(&expected_preamble));
assert!(content.ends_with(zip_content));
assert_eq!(content.len(), expected_preamble.len() + zip_content.len());

let _ = fs::remove_dir_all(&temp_dir);
}

#[test]
fn test_create_exe_zip_multiple_placeholders() {
let temp_dir = env::temp_dir().join(format!("create_exe_zip_multi_{}", std::process::id()));
fs::create_dir_all(&temp_dir).unwrap();

let preamble_path = temp_dir.join("preamble.sh");
let zip_path = temp_dir.join("data.zip");
let output_path = temp_dir.join("output.exe");

let zip_content = b"PK\x03\x04dummyzipcontent";
fs::write(&zip_path, zip_content).unwrap();

let preamble_text = b"# First: %ZIP_HASH%\n# Second: %ZIP_HASH%\n";
fs::write(&preamble_path, preamble_text).unwrap();

create_exe_zip(&preamble_path, &zip_path, &output_path).unwrap();

let content = fs::read(&output_path).unwrap();
let expected_hash = "65e39989ca91c49484998aa3f0429f6943c029609bfd2f3c18c77bf9ded72c59";
let expected_preamble = replace_bytes(preamble_text, PLACEHOLDER, expected_hash.as_bytes());

assert!(content.starts_with(&expected_preamble));
assert!(content.ends_with(zip_content));

let _ = fs::remove_dir_all(&temp_dir);
}

#[test]
fn test_create_exe_zip_creates_parent_dir() {
let temp_dir = env::temp_dir().join(format!("create_exe_zip_parent_{}", std::process::id()));
fs::create_dir_all(&temp_dir).unwrap();

let preamble_path = temp_dir.join("preamble.sh");
let zip_path = temp_dir.join("data.zip");
let output_path = temp_dir.join("nested").join("sub").join("output.exe");

fs::write(&zip_path, b"content").unwrap();
fs::write(&preamble_path, b"preamble").unwrap();

create_exe_zip(&preamble_path, &zip_path, &output_path).unwrap();
assert!(output_path.exists());

let _ = fs::remove_dir_all(&temp_dir);
}

#[test]
fn test_create_exe_zip_missing_files() {
let temp_dir = env::temp_dir().join(format!("create_exe_zip_err_{}", std::process::id()));
fs::create_dir_all(&temp_dir).unwrap();

let missing_preamble = temp_dir.join("nonexistent_preamble.sh");
let zip_path = temp_dir.join("data.zip");
let output_path = temp_dir.join("output.exe");
fs::write(&zip_path, b"dummy").unwrap();

assert!(create_exe_zip(&missing_preamble, &zip_path, &output_path).is_err());

let preamble_path = temp_dir.join("preamble.sh");
fs::write(&preamble_path, b"preamble").unwrap();
let missing_zip = temp_dir.join("nonexistent_data.zip");

assert!(create_exe_zip(&preamble_path, &missing_zip, &output_path).is_err());

let _ = fs::remove_dir_all(&temp_dir);
}
1 change: 1 addition & 0 deletions tests/modules/rules_rust_stub/WORKSPACE
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
workspace(name = "rules_rust")
3 changes: 3 additions & 0 deletions tests/modules/rules_rust_stub/rust/BUILD.bazel
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
package(default_visibility = ["//visibility:public"])

exports_files(["defs.bzl"])
Loading