Add replay to the edg-docker-test tool []

dev_tools/bin/edg-docker-test

@@ -17,14 +17,17 @@ import edgdocker
 import edgtest
 import edgutil
 
-from datetime import datetime
 from typing import Any, Dict, List, Optional
-from pathlib import Path, PurePosixPath
+from pathlib import Path, PurePath, PurePosixPath
 
 from edgutil import eprint
 
 TOOL_PATH_EDG_EXEC = edgutil.RequiredToolPath('edg-exec')
 
+# Name of the file (within the temporary directory) edgy writes its run
+# directory to.
+RUN_DIR_FILE_PATH = PurePath('edgy-run-dir.txt')
+
 BUILTIN_POSTURES = [
   'day',
   'night',
@@ -67,6 +70,13 @@ def parse_args() -> argparse.Namespace:
     action = 'store_true',
     help = 'record tests'
   )
+  parser.add_argument(
+    '--replay',
+    dest = 'replay',
+    choices = ['all-failing', 'output-mismatch', 'regress', 'unexpected'],
+    default = None,
+    help = 'rerun tests from the last run (see edgy --help)'
+  )
   parser.add_argument(
     '--discard-test-results',
     dest = 'discard_test_results',
@@ -115,7 +125,8 @@ class TestRunPosture:
                record_tests: bool, slow_tests: bool, meticulous: bool,
                test_filters: str,
                acknowledg_ui_host: Optional[str],
-               acknowledg_api_host: Optional[str]) -> None:
+               acknowledg_api_host: Optional[str],
+               replay_mode: Optional[str] = None) -> None:
     self.builds = builds
     self.keep_test_results = keep_test_results
     self.test_timeout_seconds = test_timeout_seconds
@@ -125,6 +136,7 @@ class TestRunPosture:
     self.test_filters = test_filters
     self.acknowledg_ui_host = acknowledg_ui_host
     self.acknowledg_api_host = acknowledg_api_host
+    self.replay_mode = replay_mode
 
 def _configure_day_posture(args: argparse.Namespace) -> TestRunPosture:
   return TestRunPosture(
@@ -256,6 +268,10 @@ def load_posture_file(file_path: Path,
       acknowledg_api_host = config_file_dict.get(
         'acknowledg_api_host',
         default_posture.acknowledg_api_host
+      ),
+      replay_mode = config_file_dict.get(
+        'replay_mode',
+        default_posture.replay_mode
       )
     )
 
@@ -292,6 +308,8 @@ def _configure_posture(args: argparse.Namespace) -> TestRunPosture:
     posture.test_timeout_seconds = args.timeout_seconds
   if args.discard_test_results:
     posture.keep_test_results = False
+  if args.replay is not None:
+    posture.replay_mode = args.replay
   if args.non_interactive:
     posture.acknowledg_api_host = None
     posture.acknowledg_ui_host = None
@@ -374,9 +392,6 @@ def update_build(src_dir: Path, posture: TestRunPosture, *,
       if build_process.returncode != 0:
         sys.exit(11)
 
-def generate_run_timestamp() -> str:
-  return datetime.now().strftime(edgutil.RUN_TIMESTAMP_FORMAT)
-
 def _build_edgy_test_config_arg(posture: TestRunPosture) -> str:
   all_configs = set()
 
@@ -385,12 +400,8 @@ def _build_edgy_test_config_arg(posture: TestRunPosture) -> str:
 
   return ','.join(all_configs)
 
-def _build_edgy_args(posture: TestRunPosture, *,
-                     run_timestamp: str, verbose: bool) -> str:
-  arg_str = (
-    f"--run-timestamp={run_timestamp} "
-    f"--config=\"{_build_edgy_test_config_arg(posture)}\""
-  )
+def _build_edgy_args(posture: TestRunPosture, *, verbose: bool) -> str:
+  arg_str = f"--config=\"{_build_edgy_test_config_arg(posture)}\""
 
   if posture.test_timeout_seconds != 0:
     arg_str += f" --timeout={posture.test_timeout_seconds}"
@@ -400,17 +411,29 @@ def _build_edgy_args(posture: TestRunPosture, *,
     # This works because each run is in its own container, so we can safely
     # "claim" /tmp/edgy-run within the container.  It will be both unique and
     # auto-deleted upon the contianer terminating.
-    arg_str += ' --runs-dir=/tmp/edgy-run'
+    arg_str += ' --output-directory=/tmp/edgy-run'
   if posture.slow_tests:
     arg_str += ' -A'
   if posture.meticulous:
     arg_str += ' --meticulous'
   if verbose:
     arg_str += ' --verbose'
+  if posture.replay_mode is not None:
+    arg_str += f" --replay={posture.replay_mode}"
   if len(posture.test_filters) != 0:
     arg_str += f" {posture.test_filters}"
   return arg_str
 
+def as_docker_tmp_dir(tmp_dir: Path, src_dir: Path,
+                      target_path: PurePath) -> PurePosixPath:
+  '''Map ``tmp_dir / target_path`` on the host to its absolute path inside
+  the container, where ``src_dir`` is mounted at /edg/workspace.
+  '''
+  host_path = tmp_dir / target_path
+  return PurePosixPath(
+    '/', 'edg', 'workspace', host_path.relative_to(src_dir).as_posix()
+  )
+
 def _write_prefixed_env_context(tmp_dir: Path, src_dir: Path,
                                 posture: TestRunPosture) -> Path:
   '''Write prefix_source directives for every build/test config.
@@ -426,12 +449,15 @@ def _write_prefixed_env_context(tmp_dir: Path, src_dir: Path,
       env_file = str(PurePosixPath(build.build_path, 'environment.sh'))
       # The EDG_BASE environment variable is prefixed through this system
       # by a file generated by edg-docker-test itself.
-      generated_env_file = tmp_dir / f"build-{idx}.sh"
+      generated_env_name = PurePath(f"build-{idx}.sh")
+      generated_env_file = tmp_dir / generated_env_name
       with open(generated_env_file, 'w') as gen_file_handle:
         base_path = PurePosixPath('/', 'edg', 'workspace', build.edg_base)
         gen_file_handle.write(f"export EDG_BASE=\"{base_path}\"\n")
-      docker_test_env_file = (
-        generated_env_file.relative_to(src_dir).as_posix()
+      docker_test_env_file = as_docker_tmp_dir(
+        tmp_dir,
+        src_dir,
+        generated_env_name
       )
 
       for test_config in build.test_configs:
@@ -444,18 +470,38 @@ def _write_prefixed_env_context(tmp_dir: Path, src_dir: Path,
 def _write_loader_invocation(tmp_dir: Path, src_dir: Path,
                              context_file: Path, invocation_name: str,
                              exec_cmd: str, *, dry_run: bool,
-                             verbose: bool) -> str:
+                             verbose: bool,
+                             extra_env: Optional[Dict[str, str]] = None
+                             ) -> str:
   '''Combine the shared prefix context with an exec directive.
 
-  Returns the path to the invocation file relative to ``src_dir`` (as used
-  inside the container under /edg/workspace).
+  Any ``extra_env`` variables are written to a generated environment file
+  that the invocation sources before the exec directive.
+
+  Returns the path to the invocation file as seen inside the container.
   '''
   invocation_file = tmp_dir / invocation_name
   shutil.copyfile(context_file, invocation_file)
   with open(invocation_file, 'a') as file_handle:
+    if extra_env:
+      extra_env_name = PurePath(
+        f"extra-{PurePath(invocation_name).stem}-env.sh"
+      )
+      extra_env_file = tmp_dir / extra_env_name
+      with open(extra_env_file, 'w') as env_file_handle:
+        for key, value in extra_env.items():
+          env_file_handle.write(f"export {key}=\"{value}\"\n")
+      docker_extra_env_file = as_docker_tmp_dir(
+        tmp_dir,
+        src_dir,
+        extra_env_name
+      )
+      file_handle.write(f"source {docker_extra_env_file}\n")
     file_handle.write(f"exec {exec_cmd}\n")
 
-  docker_invocation_file = invocation_file.relative_to(src_dir).as_posix()
+  docker_invocation_file = str(
+    as_docker_tmp_dir(tmp_dir, src_dir, PurePath(invocation_name))
+  )
   if dry_run or verbose:
     print(f"Wrote executable directives file ({docker_invocation_file}):")
     with open(invocation_file) as file_handle:
@@ -465,23 +511,24 @@ def _write_loader_invocation(tmp_dir: Path, src_dir: Path,
   return docker_invocation_file
 
 def run_tests(tmp_dir: Path, src_dir: Path, posture: TestRunPosture, *,
-              context_file: Path, run_timestamp: str, dry_run: bool,
+              context_file: Path, dry_run: bool,
               verbose: bool) -> int:
   print('Running tests...', flush = True)
 
-  edgy_args = _build_edgy_args(
-    posture,
-    run_timestamp = run_timestamp,
-    verbose = verbose
-  )
+  edgy_args = _build_edgy_args(posture, verbose = verbose)
+  # edgy writes the directory it chose (a new run or a replay) to this file.
+  docker_run_dir_file = as_docker_tmp_dir(tmp_dir, src_dir, RUN_DIR_FILE_PATH)
   docker_invocation_file = _write_loader_invocation(
     tmp_dir,
     src_dir,
     context_file,
     'edgy.edgivk',
     f"edgy {edgy_args}",
     dry_run = dry_run,
-    verbose = verbose
+    verbose = verbose,
+    extra_env = {
+      'EDGY_RUN_DIR_FILE': str(docker_run_dir_file)
+    }
   )
 
   cmd_args = [
@@ -511,7 +558,7 @@ def _is_posture_acknowledg_enabled(posture: TestRunPosture) -> bool:
               posture.keep_test_results)
 
 def run_acknowledg(tmp_dir: Path, src_dir: Path, posture: TestRunPosture, *,
-                    context_file: Path, run_timestamp: str, dry_run: bool,
+                    context_file: Path, run_dir: str, dry_run: bool,
                     verbose: bool) -> None:
   print('Opening AcknowlEDG...', flush = True)
 
@@ -520,7 +567,7 @@ def run_acknowledg(tmp_dir: Path, src_dir: Path, posture: TestRunPosture, *,
     src_dir,
     context_file,
     'edgy-review.edgivk',
-    f"edgy-review --root='/edg/workspace/tests/runs/{run_timestamp}'",
+    f"edgy-review --root='{run_dir}'",
     dry_run = dry_run,
     verbose = verbose
   )
@@ -598,13 +645,11 @@ def main() -> None:
     tmp_dir = Path(tmp_dir_str)
     context_file = _write_prefixed_env_context(tmp_dir, src_dir, run_posture)
 
-    test_run_timestamp = generate_run_timestamp()
     test_exit_code = run_tests(
       tmp_dir,
       src_dir,
       run_posture,
       context_file = context_file,
-      run_timestamp = test_run_timestamp,
       dry_run = args.dry_run,
       verbose = args.verbose
     )
@@ -617,12 +662,20 @@ def main() -> None:
       (test_exit_code == 0 or test_exit_code == 5) and
       _is_posture_acknowledg_enabled(run_posture)
     ):
+      if args.dry_run:
+        test_run_dir = '<run directory reported by edgy>'
+      elif (tmp_dir / RUN_DIR_FILE_PATH).exists():
+        test_run_dir = (tmp_dir / RUN_DIR_FILE_PATH).read_text().strip()
+      else:
+        # edgy ran no tests (e.g. nothing to replay), so there's no output.
+        eprint('edgy did not report a run directory, skipping AcknowlEDG')
+        sys.exit(0 if test_exit_code == 0 else 20)
       run_acknowledg(
         tmp_dir,
         src_dir,
         run_posture,
         context_file = context_file,
-        run_timestamp = test_run_timestamp,
+        run_dir = test_run_dir,
         dry_run = args.dry_run,
         verbose = args.verbose
       )

dev_tools/bin/edgy

@@ -902,7 +902,9 @@ def get_latest_replay_number(replay_dir: Path) -> int:
 
 def get_run_dir(test_runs_dir: Path, args: argparse.Namespace,
                 test_suites: List[TestSuiteConfig]) -> Path:
-  if is_replay_run(args):
+  if args.output_directory is not None:
+    return args.output_directory
+  elif is_replay_run(args):
     # This is guaranteed to exist as the checks on test counts prior to calling
     # this function ensure that at least one test from the last run is being
     # replay (so there must be a last run).
@@ -933,6 +935,12 @@ def prepare_for_test_run(test_runs_dir: Path, args: argparse.Namespace,
 
     run_dir.mkdir(parents = True)
 
+    # Let a wrapping tool (e.g. edg-docker-test) learn exactly which directory
+    # this invocation is using, even for replays, without racing other runs.
+    run_dir_file = os.environ.get('EDGY_RUN_DIR_FILE')
+    if run_dir_file:
+      Path(run_dir_file).write_text(f"{run_dir}\n")
+
     setup_cache = RunSetupCache(run_dir)
 
     for test_suite in test_suites:
@@ -949,7 +957,12 @@ def prepare_for_test_run(test_runs_dir: Path, args: argparse.Namespace,
 
       # Set the test suite run as the current run
       test_suite.set_current_run(test_suite_run)
-  except FileExistsError:
+  except FileExistsError as ex:
+    if args.output_directory is not None:
+      # Retrying can't help, as an explicit output directory is never bumped.
+      edgutil.eprint(f"Path already exists: {ex.filename}")
+      sys.exit(1)
+
     # This can happen if edgy is rapidly invoked, just wait a bit and retry (to
     # bump the directory name).
     time.sleep(.5)
@@ -2176,6 +2189,16 @@ def parse_args() -> argparse.Namespace:
     type = Path,
     help = 'override the directory where tests runs are stored'
   )
+  parser.add_argument(
+    '--output-directory',
+    dest = 'output_directory',
+    type = Path,
+    help = (
+      'store the output of this run in exactly this directory, in place of a '
+      'new run or replay directory (replays still read the previous run from '
+      'the runs directory, and no old runs are cleaned up)'
+    )
+  )
   parser.add_argument(
     '--run-timestamp',
     dest = 'test_run_timestamp',
@@ -2262,7 +2285,11 @@ def parse_args() -> argparse.Namespace:
     '--disp-run-dir',
     dest = 'disp_run_dir',
     action = 'store_true',
-    help = 'print the directory where the run output is being stored'
+    help = (
+      'print the directory where the run output is being stored (if '
+      'scripting, prefer setting EDGY_RUN_DIR_FILE to have the run directory '
+      'path written there)'
+    )
   )
   parser.add_argument(
     '-W',
@@ -2432,7 +2459,8 @@ def main() -> None:
 
   generate_logs(args, test_suites, tests, skipped_tests)
   maybe_dump_performance_info(args, tests, multi_config_run = multi_config_run)
-  try_cleanup_test_runs(test_runs_dir, args)
+  if args.output_directory is None:
+    try_cleanup_test_runs(test_runs_dir, args)
   sys.exit(exit_code)
 
 main()