diff --git a/sphinxdocs/.bazelignore b/sphinxdocs/.bazelignore new file mode 100644 index 0000000000..80f2d3514f --- /dev/null +++ b/sphinxdocs/.bazelignore @@ -0,0 +1,4 @@ +bazel-bin +bazel-out +bazel-testlogs +bazel-sphinxdocs diff --git a/sphinxdocs/.bazelrc b/sphinxdocs/.bazelrc index 65c996c678..ce4d782113 100644 --- a/sphinxdocs/.bazelrc +++ b/sphinxdocs/.bazelrc @@ -21,5 +21,13 @@ common --experimental_repository_downloader_retries=10 common --incompatible_python_disallow_native_rules common --incompatible_no_implicit_file_export +# Pass host PATH to build and test actions so bazel_from_env (used by rules_bazel_integration_test +# for local 'self' testing) can locate the user's bazel binary without disabling strict action env. +common --action_env=PATH +common --test_env=PATH build --lockfile_mode=update + +common:fast-tests --build_tests_only=true +common:fast-tests --build_tag_filters=-large,-enormous,-integration-test +common:fast-tests --test_tag_filters=-large,-enormous,-integration-test diff --git a/sphinxdocs/.bazelrc.deleted_packages b/sphinxdocs/.bazelrc.deleted_packages index 442c80e960..9a90ef21be 100644 --- a/sphinxdocs/.bazelrc.deleted_packages +++ b/sphinxdocs/.bazelrc.deleted_packages @@ -1 +1,2 @@ common --deleted_packages=integration_tests/bcr +common --deleted_packages=integration_tests/persistent_worker/workspace diff --git a/sphinxdocs/BUILD.bazel b/sphinxdocs/BUILD.bazel new file mode 100644 index 0000000000..8ff9bc956f --- /dev/null +++ b/sphinxdocs/BUILD.bazel @@ -0,0 +1,16 @@ +package(default_visibility = ["//visibility:public"]) + +filegroup( + name = "distribution", + srcs = glob( + ["**/*"], + exclude = [ + "bazel-*/**", + "integration_tests/**", + "tests/**", + ], + ) + [ + "//sphinxdocs:distribution", + ], + visibility = ["//visibility:public"], +) diff --git a/sphinxdocs/MODULE.bazel b/sphinxdocs/MODULE.bazel index 30fb2196dc..3d4caf2bb2 100644 --- a/sphinxdocs/MODULE.bazel +++ b/sphinxdocs/MODULE.bazel @@ -21,3 +21,21 @@ dev_pip.parse( requirements_lock = "@rules_python//docs:requirements.txt", ) use_repo(dev_pip, "dev_pip") + +bazel_dep(name = "rules_bazel_integration_test", version = "0.37.1", dev_dependency = True) + +bazel_binaries = use_extension( + "@rules_bazel_integration_test//:extensions.bzl", + "bazel_binaries", + dev_dependency = True, +) +bazel_binaries.local( + name = "self", + path = "integration_tests/bazel_from_env", +) +use_repo( + bazel_binaries, + "bazel_binaries", + "bazel_binaries_bazelisk", + "build_bazel_bazel_self", +) diff --git a/sphinxdocs/integration_tests/BUILD.bazel b/sphinxdocs/integration_tests/BUILD.bazel new file mode 100644 index 0000000000..45e894fc2e --- /dev/null +++ b/sphinxdocs/integration_tests/BUILD.bazel @@ -0,0 +1,8 @@ +load("@rules_python//python:py_library.bzl", "py_library") + +package(default_visibility = ["//visibility:public"]) + +py_library( + name = "runner_lib", + srcs = ["runner.py"], +) diff --git a/sphinxdocs/integration_tests/bazel_from_env b/sphinxdocs/integration_tests/bazel_from_env new file mode 100755 index 0000000000..a372736f32 --- /dev/null +++ b/sphinxdocs/integration_tests/bazel_from_env @@ -0,0 +1,6 @@ +#!/usr/bin/env bash +# +# A simple wrapper so rules_bazel_integration_test can use the +# bazel version inherited from the environment. + +bazel "$@" diff --git a/sphinxdocs/integration_tests/integration_test.bzl b/sphinxdocs/integration_tests/integration_test.bzl new file mode 100644 index 0000000000..b6786b9c6b --- /dev/null +++ b/sphinxdocs/integration_tests/integration_test.bzl @@ -0,0 +1,68 @@ +"""Helpers for running bazel-in-bazel integration tests for sphinxdocs.""" + +load( + "@rules_bazel_integration_test//bazel_integration_test:defs.bzl", + "bazel_integration_test", + "integration_test_utils", +) +load("@rules_python//python:py_test.bzl", "py_test") + +def _test_runner(*, name, bazel_version, py_main, py_deps): + test_runner = "{}_bazel_{}_py_runner".format(name, bazel_version) + py_test( + name = test_runner, + srcs = [py_main], + main = py_main, + deps = ["//integration_tests:runner_lib"] + py_deps, + tags = ["manual"], + ) + return test_runner + +def sphinxdocs_integration_test( + name, + workspace_path = "workspace", + tags = None, + py_main = None, + py_deps = None, + bazel_versions = None, + **kwargs): + """Runs a bazel-in-bazel integration test for sphinxdocs. + + Args: + name: Name of the test. + workspace_path: The directory name of the sub-workspace. + tags: Test tags. + py_main: Main Python test runner script. + py_deps: Dependencies for py_main. + bazel_versions: List of bazel versions to test. + **kwargs: Passed to bazel_integration_test. + """ + workspace_files = integration_test_utils.glob_workspace_files(workspace_path) + native.filegroup( + name = name + "_workspace_files", + srcs = workspace_files + [ + "//:distribution", + ], + ) + kwargs.setdefault("size", "large") + for bazel_version in bazel_versions or ["self"]: + test_runner = _test_runner( + name = name, + bazel_version = bazel_version, + py_main = py_main, + py_deps = py_deps or [], + ) + bazel_integration_test( + name = "{}_bazel_{}".format(name, bazel_version), + workspace_path = workspace_path, + test_runner = test_runner, + bazel_version = bazel_version, + workspace_files = [name + "_workspace_files"], + tags = (tags or []) + [ + "exclusive", + "no-sandbox", + "no-remote-exec", + "integration-test", + ], + **kwargs + ) diff --git a/sphinxdocs/integration_tests/persistent_worker/BUILD.bazel b/sphinxdocs/integration_tests/persistent_worker/BUILD.bazel new file mode 100644 index 0000000000..60fb4cbc6c --- /dev/null +++ b/sphinxdocs/integration_tests/persistent_worker/BUILD.bazel @@ -0,0 +1,8 @@ +load("//integration_tests:integration_test.bzl", "sphinxdocs_integration_test") + +package(default_visibility = ["//visibility:public"]) + +sphinxdocs_integration_test( + name = "persistent_worker_test", + py_main = "persistent_worker_test.py", +) diff --git a/sphinxdocs/integration_tests/persistent_worker/persistent_worker_test.py b/sphinxdocs/integration_tests/persistent_worker/persistent_worker_test.py new file mode 100644 index 0000000000..30a8b46c00 --- /dev/null +++ b/sphinxdocs/integration_tests/persistent_worker/persistent_worker_test.py @@ -0,0 +1,54 @@ +import unittest + +from integration_tests import runner + + +class PersistentWorkerTest(runner.TestCase): + def _check_index_html(self, text: str, should_exist: bool): + index_html = ( + self.repo_root / "bazel-bin" / "docs" / "_build" / "html" / "index.html" + ) + if not index_html.exists(): + self.fail(f"Could not find index.html at {index_html}") + content = index_html.read_text() + if should_exist: + self.assertIn( + text, + content, + f"Expected '{text}' in index.html after build, but not found.", + ) + else: + self.assertNotIn( + text, + content, + f"Expected '{text}' NOT to be in index.html after build, but found it.", + ) + + def test_incremental_add_and_remove_files(self): + # 1. Initial build + result = self.run_bazel("build", "//:docs") + self.assert_result_matches(result, "bazel-bin") + index_html = ( + self.repo_root / "bazel-bin" / "docs" / "_build" / "html" / "index.html" + ) + self.assertTrue( + index_html.exists(), "index.html should exist after initial build" + ) + + # 2. Add a new markdown file and verify it is included across incremental build + page2_md = self.repo_root / "page2.md" + page2_md.write_text("# Page 2\n\nThis is a newly added page.\n") + result = self.run_bazel("build", "//:docs") + self.assert_result_matches(result, "bazel-bin") + self._check_index_html("page2.html", should_exist=True) + + # 3. Remove the added markdown file and verify the persistent worker cleans up + # stale source files and invalidates toctrees without errors or warnings. + page2_md.unlink() + result = self.run_bazel("build", "//:docs") + self.assert_result_matches(result, "bazel-bin") + self._check_index_html("page2.html", should_exist=False) + + +if __name__ == "__main__": + unittest.main() diff --git a/sphinxdocs/integration_tests/persistent_worker/workspace/.bazelrc b/sphinxdocs/integration_tests/persistent_worker/workspace/.bazelrc new file mode 100644 index 0000000000..f3162b9547 --- /dev/null +++ b/sphinxdocs/integration_tests/persistent_worker/workspace/.bazelrc @@ -0,0 +1,8 @@ +# Disable disk caching and remote action caching so Bazel is forced to dispatch builds +# directly to the running persistent worker instead of restoring pre-built HTML cache results across test steps. +common --disk_cache= +build --noremote_accept_cached +common --http_timeout_scaling=10.0 +common --experimental_repository_downloader_retries=10 +common --experimental_downloader_config=downloader_config.cfg +common --lockfile_mode=off diff --git a/sphinxdocs/integration_tests/persistent_worker/workspace/BUILD.bazel b/sphinxdocs/integration_tests/persistent_worker/workspace/BUILD.bazel new file mode 100644 index 0000000000..60e66004fd --- /dev/null +++ b/sphinxdocs/integration_tests/persistent_worker/workspace/BUILD.bazel @@ -0,0 +1,17 @@ +load("@sphinxdocs//sphinxdocs:sphinx.bzl", "sphinx_build_binary", "sphinx_docs") + +sphinx_docs( + name = "docs", + srcs = glob(["*.md"]), + config = "conf.py", + formats = ["html"], + sphinx = ":sphinx-build", +) + +sphinx_build_binary( + name = "sphinx-build", + deps = [ + "@dev_pip//myst_parser", + "@dev_pip//sphinx", + ], +) diff --git a/sphinxdocs/integration_tests/persistent_worker/workspace/MODULE.bazel b/sphinxdocs/integration_tests/persistent_worker/workspace/MODULE.bazel new file mode 100644 index 0000000000..4e2c25e8e4 --- /dev/null +++ b/sphinxdocs/integration_tests/persistent_worker/workspace/MODULE.bazel @@ -0,0 +1,23 @@ +module(version = "0.0.0") + +bazel_dep(name = "sphinxdocs", version = "0.0.0") +local_path_override( + module_name = "sphinxdocs", + path = "../../..", +) + +bazel_dep(name = "rules_python", version = "1.8.5") + +dev_pip = use_extension( + "@rules_python//python/extensions:pip.bzl", + "pip", + dev_dependency = True, +) +dev_pip.parse( + hub_name = "dev_pip", + python_version = "3.11", + requirements_lock = "@rules_python//docs:requirements.txt", +) +use_repo(dev_pip, "dev_pip") + +bazel_dep(name = "bazel_skylib", version = "1.8.2") diff --git a/sphinxdocs/integration_tests/persistent_worker/workspace/WORKSPACE b/sphinxdocs/integration_tests/persistent_worker/workspace/WORKSPACE new file mode 100644 index 0000000000..4be37fab94 --- /dev/null +++ b/sphinxdocs/integration_tests/persistent_worker/workspace/WORKSPACE @@ -0,0 +1 @@ +# Bzlmod-enabled workspace; this empty WORKSPACE file satisfies rules_bazel_integration_test checks. diff --git a/sphinxdocs/integration_tests/persistent_worker/workspace/conf.py b/sphinxdocs/integration_tests/persistent_worker/workspace/conf.py new file mode 100644 index 0000000000..de33bde101 --- /dev/null +++ b/sphinxdocs/integration_tests/persistent_worker/workspace/conf.py @@ -0,0 +1 @@ +extensions = ["myst_parser"] diff --git a/sphinxdocs/integration_tests/persistent_worker/workspace/downloader_config.cfg b/sphinxdocs/integration_tests/persistent_worker/workspace/downloader_config.cfg new file mode 100644 index 0000000000..3fa6264eda --- /dev/null +++ b/sphinxdocs/integration_tests/persistent_worker/workspace/downloader_config.cfg @@ -0,0 +1,21 @@ +# Try GitHub first (primary) +rewrite ^github\.com/bazel-contrib/bazel_features/(.*) github.com/bazel-contrib/bazel_features/$1 +rewrite ^github\.com/bazel-contrib/rules_go/(.*) github.com/bazel-contrib/rules_go/$1 +rewrite ^github\.com/bazelbuild/bazel-skylib/(.*) github.com/bazelbuild/bazel-skylib/$1 +rewrite ^github\.com/bazelbuild/platforms/(.*) github.com/bazelbuild/platforms/$1 +rewrite ^github\.com/bazelbuild/rules_kotlin/(.*) github.com/bazelbuild/rules_kotlin/$1 +rewrite ^github\.com/bazelbuild/rules_shell/(.*) github.com/bazelbuild/rules_shell/$1 +rewrite ^github\.com/bazelbuild/rules_java/(.*) github.com/bazelbuild/rules_java/$1 +rewrite ^github\.com/bazelbuild/stardoc/(.*) github.com/bazelbuild/stardoc/$1 + + +# Fall back to mirror (secondary) +# Tracking upstream BCR mirror addition: https://github.com/bazelbuild/platforms/issues/139 +rewrite ^github\.com/bazel-contrib/bazel_features/(.*) mirror.bazel.build/github.com/bazel-contrib/bazel_features/$1 +rewrite ^github\.com/bazel-contrib/rules_go/(.*) mirror.bazel.build/github.com/bazel-contrib/rules_go/$1 +rewrite ^github\.com/bazelbuild/bazel-skylib/(.*) mirror.bazel.build/github.com/bazelbuild/bazel-skylib/$1 +rewrite ^github\.com/bazelbuild/platforms/(.*) mirror.bazel.build/github.com/bazelbuild/platforms/$1 +rewrite ^github\.com/bazelbuild/rules_kotlin/(.*) mirror.bazel.build/github.com/bazelbuild/rules_kotlin/$1 +rewrite ^github\.com/bazelbuild/rules_shell/(.*) mirror.bazel.build/github.com/bazelbuild/rules_shell/$1 +rewrite ^github\.com/bazelbuild/rules_java/(.*) mirror.bazel.build/github.com/bazelbuild/rules_java/$1 +rewrite ^github\.com/bazelbuild/stardoc/(.*) mirror.bazel.build/github.com/bazelbuild/stardoc/$1 diff --git a/sphinxdocs/integration_tests/persistent_worker/workspace/index.md b/sphinxdocs/integration_tests/persistent_worker/workspace/index.md new file mode 100644 index 0000000000..d1dca92350 --- /dev/null +++ b/sphinxdocs/integration_tests/persistent_worker/workspace/index.md @@ -0,0 +1,7 @@ +# Test Documentation + +```{toctree} +:glob: true + +* +``` diff --git a/sphinxdocs/integration_tests/runner.py b/sphinxdocs/integration_tests/runner.py new file mode 100644 index 0000000000..cab9730bb8 --- /dev/null +++ b/sphinxdocs/integration_tests/runner.py @@ -0,0 +1,99 @@ +import logging +import os +import os.path +import pathlib +import re +import shlex +import subprocess +import unittest + +_logger = logging.getLogger(__name__) + + +class ExecuteError(Exception): + def __init__(self, result): + self.result = result + + def __str__(self): + return self.result.describe() + + +class ExecuteResult: + def __init__( + self, + args: list[str], + env: dict[str, str], + cwd: pathlib.Path, + proc_result: subprocess.CompletedProcess, + ): + self.args = args + self.env = env + self.cwd = cwd + self.exit_code = proc_result.returncode + self.stdout = proc_result.stdout + self.stderr = proc_result.stderr + + def describe(self) -> str: + env_lines = [ + " " + shlex.quote(f"{key}={value}") + for key, value in sorted(self.env.items()) + ] + env = " \\\n".join(env_lines) + args = shlex.join(self.args) + maybe_stdout_nl = "" if self.stdout.endswith("\n") else "\n" + maybe_stderr_nl = "" if self.stderr.endswith("\n") else "\n" + return f"""\ +COMMAND: +cd {self.cwd} && \\ +env \\ +{env} \\ + {args} +RESULT: exit_code: {self.exit_code} +===== STDOUT START ===== +{self.stdout}{maybe_stdout_nl}===== STDOUT END ===== +===== STDERR START ===== +{self.stderr}{maybe_stderr_nl}===== STDERR END ===== +""" + + +class TestCase(unittest.TestCase): + def setUp(self): + super().setUp() + self.repo_root = pathlib.Path(os.environ["BIT_WORKSPACE_DIR"]) + self.bazel = pathlib.Path(os.environ["BIT_BAZEL_BINARY"]) + outer_test_tmpdir = pathlib.Path(os.environ["TEST_TMPDIR"]) + self.test_tmp_dir = outer_test_tmpdir / "bit_test_tmp" + self.tmp_dir = outer_test_tmpdir / "bit_tmp" + self.bazel_env = { + "PATH": os.environ["PATH"], + "TEST_TMPDIR": str(self.test_tmp_dir), + "TMP": str(self.tmp_dir), + "RUNFILES_DIR": os.environ["TEST_SRCDIR"], + } + + def run_bazel(self, *args: str, check: bool = True) -> ExecuteResult: + args = [str(self.bazel), *args] + env = self.bazel_env + _logger.info("executing: %s", shlex.join(args)) + cwd = self.repo_root + proc_result = subprocess.run( + args=args, + text=True, + capture_output=True, + cwd=cwd, + env=env, + check=False, + ) + exec_result = ExecuteResult(args, env, cwd, proc_result) + if check and exec_result.exit_code: + raise ExecuteError(exec_result) + else: + return exec_result + + def assert_result_matches(self, result: ExecuteResult, regex: str) -> None: + if not re.search(regex, result.stdout + result.stderr): + self.fail( + "Bazel output did not match expected pattern\n" + + f"expected pattern: {regex}\n" + + f"invocation details:\n{result.describe()}" + ) diff --git a/sphinxdocs/sphinxdocs/BUILD.bazel b/sphinxdocs/sphinxdocs/BUILD.bazel index fbdd633a32..9a871f5268 100644 --- a/sphinxdocs/sphinxdocs/BUILD.bazel +++ b/sphinxdocs/sphinxdocs/BUILD.bazel @@ -20,6 +20,14 @@ package( default_visibility = ["//:__subpackages__"], ) +filegroup( + name = "distribution", + srcs = glob(["**/*"]) + [ + "//sphinxdocs/private:distribution", + ], + visibility = ["//:__pkg__"], +) + # Additional -D values to add to every Sphinx build. # This is usually used to override the version when building repeated_string_list_flag( diff --git a/sphinxdocs/sphinxdocs/private/BUILD.bazel b/sphinxdocs/sphinxdocs/private/BUILD.bazel index b054ed2611..0fe0ba4152 100644 --- a/sphinxdocs/sphinxdocs/private/BUILD.bazel +++ b/sphinxdocs/sphinxdocs/private/BUILD.bazel @@ -34,6 +34,12 @@ exports_files( visibility = ["//visibility:public"], ) +filegroup( + name = "distribution", + srcs = glob(["**/*"]), + visibility = ["//sphinxdocs:__pkg__"], +) + bzl_library( name = "util", srcs = ["util.bzl"], diff --git a/sphinxdocs/sphinxdocs/private/sphinx_build.py b/sphinxdocs/sphinxdocs/private/sphinx_build.py index 3605b4579d..07a123b496 100644 --- a/sphinxdocs/sphinxdocs/private/sphinx_build.py +++ b/sphinxdocs/sphinxdocs/private/sphinx_build.py @@ -144,6 +144,30 @@ def _prepare_sphinx(self, request): logger.info("path %s changed", path) changed_paths.append(path) + # Remove any source files that were tracked in the previous request (`current_digests`) + # but are missing from the current `request["inputs"]` (`incoming_digests`). + # Across incremental branch switches or file removals, if these stale symlinks + # remain in `srcdir` on disk, Sphinx will discover broken/unreadable files during + # `find_files()` and abort with "WARNING: Ignored unreadable document" (fatal with -W). + for path in set(current_digests) - set(incoming_digests): + removed_path = os.path.join(srcdir, path) + if os.path.exists(removed_path) or os.path.islink(removed_path): + logger.info("removing stale source file %s", removed_path) + try: + if os.path.islink(removed_path): + try: + os.remove(removed_path) + except OSError: + os.rmdir(removed_path) + elif os.path.isdir(removed_path): + shutil.rmtree(removed_path) + else: + os.remove(removed_path) + except OSError as e: + logger.warning( + "failed to remove stale source %s: %s", removed_path, e + ) + self._digests[srcdir] = incoming_digests self._extension.changed_paths = changed_paths request_info["changed_sources"] = changed_paths @@ -188,6 +212,12 @@ def _process_request(self, request: "WorkRequest") -> "WorkResponse | None": stdout.truncate(0) stderr.seek(0) stderr.truncate(0) + # If Sphinx cache (`--doctree-dir`) becomes corrupted across incremental + # updates or branch checkouts, exit code 2 is returned. Wiping out the cached + # doctrees before retrying allows Sphinx to recover cleanly from scratch. + for arg in sphinx_args: + if arg.startswith("--doctree-dir="): + shutil.rmtree(arg.split("=", 1)[1], ignore_errors=True) exit_code = main(sphinx_args) if exit_code: @@ -247,14 +277,31 @@ def setup(self, app): return {"parallel_read_safe": True, "parallel_write_safe": True} def _handle_env_get_outdated(self, app, env, added, changed, removed): - changed = { - # NOTE: path2doc returns None if it's not a doc path - env.path2doc(p) - for p in self.changed_paths - } - - logger.info("changed docs: %s", changed) - return changed + changed_docs = set() + for p in self.changed_paths: + # Try multiple path resolutions because depending on how Sphinx and Bazel + # represent inputs (`p`), `env.path2doc` may require relative, srcdir-joined, + # or absolute paths to successfully resolve the document name. + doc = ( + env.path2doc(p) + or env.path2doc(os.path.join(env.srcdir, p)) + or env.path2doc(os.path.abspath(os.path.join(env.srcdir, p))) + ) + if doc: + changed_docs.add(doc) + + # When documents are added or removed across incremental builds or branch checkouts, + # parent documents whose `toctree` includes them (especially via glob patterns or + # explicit references to removed docs) must be invalidated and re-read. Otherwise, + # Sphinx retains stale table of contents entries or throws unresolvable reference errors. + if added or removed: + glob_toctrees = getattr(env, "glob_toctrees", set()) + for doc, includes in getattr(env, "toctree_includes", {}).items(): + if doc in glob_toctrees or not removed.isdisjoint(includes): + changed_docs.add(doc) + + logger.info("changed docs: %s", changed_docs) + return changed_docs def _worker_main(stdin, stdout, exec_root):