diff --git a/alts/shared/models.py b/alts/shared/models.py index 20ea7e4..9418d16 100644 --- a/alts/shared/models.py +++ b/alts/shared/models.py @@ -243,6 +243,13 @@ class ThirdPartyRepoSshHost(BaseModel): user: Optional[str] = None +class CachedTestRepo(BaseModel): + # Baked-in repo cache (`src`) on the VM image, re-seeded into the + # host-aware path (`dest`) during provisioning to avoid a full clone. + src: str + dest: str + + class CeleryConfig(BaseModel): def __init__(self, **data): super().__init__(**data) @@ -353,6 +360,10 @@ def __init__(self, **data): # default (current SSH user). Configured per-deployment; empty by # default so no `~/.ssh/config` is written unless explicitly set. third_party_repo_ssh_hosts: List[ThirdPartyRepoSshHost] = [] + # Baked-in QA repo caches to re-seed into the host-aware layout during + # provisioning (avoids a full network clone on every VM). Empty by + # default; configured per-deployment to match the VM image's caches. + cached_test_repos: List[CachedTestRepo] = [] tests_base_dir: str = '/tests' package_proxy: str = '' disabled_al_repos: List[str] = [] diff --git a/alts/worker/runners/base.py b/alts/worker/runners/base.py index 2fdefa5..b159c51 100644 --- a/alts/worker/runners/base.py +++ b/alts/worker/runners/base.py @@ -751,6 +751,10 @@ def initial_provision(self, verbose=False): entry.model_dump(exclude_none=True) for entry in CONFIG.third_party_repo_ssh_hosts ], + 'cached_test_repos': [ + entry.model_dump() + for entry in CONFIG.cached_test_repos + ], } dist_major_version = self.dist_version[0] if self.dist_name in CONFIG.rhel_flavors and dist_major_version in ('6', '7'): diff --git a/resources/roles/preparation/defaults/main.yml b/resources/roles/preparation/defaults/main.yml index b3ae8f4..769f892 100644 --- a/resources/roles/preparation/defaults/main.yml +++ b/resources/roles/preparation/defaults/main.yml @@ -6,3 +6,7 @@ centos_repo_baseurl: "https://vault.centos.org" # extra-var by the runner. Empty list here is a no-op fallback so the # role still works if invoked standalone (e.g. ansible-lint). third_party_repo_ssh_hosts: [] +# Source of truth lives in the alts config (`CelerConfig.cached_test_repos`) +# and is passed in as an extra-var by the runner. Empty list here is a +# no-op fallback so the role still works if invoked standalone. +cached_test_repos: [] diff --git a/resources/roles/preparation/tasks/main.yml b/resources/roles/preparation/tasks/main.yml index da73b4b..fa95094 100644 --- a/resources/roles/preparation/tasks/main.yml +++ b/resources/roles/preparation/tasks/main.yml @@ -62,6 +62,49 @@ tags: - initial_provision +# Re-seed baked-in QA caches into the host-aware layout so the test +# runner reuses them (light `git pull`) instead of a full network clone. +# Docker envs receive repos via `docker cp` from the worker, so skip them. +- name: Check which baked test repo caches exist + ansible.builtin.stat: + path: "{{ item.src }}" + register: cached_test_repo_stats + loop: "{{ cached_test_repos }}" + when: + - connection_type != 'docker' + - cached_test_repos | length > 0 + tags: + - initial_provision + +- name: Ensure parent directories for seeded test repos exist + ansible.builtin.file: + path: "{{ item.item.dest | dirname }}" + state: directory + mode: '0755' + loop: "{{ cached_test_repo_stats.results | default([]) }}" + loop_control: + label: "{{ item.item.dest }}" + when: + - not (item.skipped | default(false)) + - item.stat.exists + tags: + - initial_provision + +- name: Seed host-aware test repo paths from baked caches + ansible.builtin.copy: + src: "{{ item.item.src }}/" + dest: "{{ item.item.dest }}" + remote_src: true + mode: preserve + loop: "{{ cached_test_repo_stats.results | default([]) }}" + loop_control: + label: "{{ item.item.dest }}" + when: + - not (item.skipped | default(false)) + - item.stat.exists + tags: + - initial_provision + - name: Copy tests to test environment copy: src: "{{ integrity_tests_dir }}" diff --git a/tests/runners/test_provision_ssh_hosts.py b/tests/runners/test_provision_ssh_hosts.py index 877e306..3680f7c 100644 --- a/tests/runners/test_provision_ssh_hosts.py +++ b/tests/runners/test_provision_ssh_hosts.py @@ -14,7 +14,7 @@ from alts.worker import runners from alts.worker.runners.base import GenericVMRunner -from alts.shared.models import ThirdPartyRepoSshHost +from alts.shared.models import CachedTestRepo, ThirdPartyRepoSshHost def _make_runner(tmp_path): @@ -97,3 +97,58 @@ def fake_run(cmd_args, timeout=None): extra_vars = _extract_extra_vars(captured['args']) assert extra_vars['third_party_repo_ssh_hosts'] == [] + + +class TestInitialProvisionCachedTestRepos: + def test_extra_vars_include_configured_caches( + self, tmp_path, monkeypatch, + ): + monkeypatch.setattr( + runners.base.CONFIG, + 'cached_test_repos', + [ + CachedTestRepo( + src='/opt/QA', + dest='/opt/gerrit.cloudlinux.com/QA', + ), + ], + raising=False, + ) + captured = {} + + def fake_run(cmd_args, timeout=None): + captured['args'] = cmd_args + return 0, '', '' + + runner = _make_runner(tmp_path) + runner.run_ansible_command = fake_run + + runner.initial_provision() + + extra_vars = _extract_extra_vars(captured['args']) + assert extra_vars['cached_test_repos'] == [ + {'src': '/opt/QA', 'dest': '/opt/gerrit.cloudlinux.com/QA'}, + ] + + def test_extra_vars_empty_when_unconfigured( + self, tmp_path, monkeypatch, + ): + monkeypatch.setattr( + runners.base.CONFIG, + 'cached_test_repos', + [], + raising=False, + ) + captured = {} + + def fake_run(cmd_args, timeout=None): + captured['args'] = cmd_args + return 0, '', '' + + runner = _make_runner(tmp_path) + runner.run_ansible_command = fake_run + + runner.initial_provision() + + extra_vars = _extract_extra_vars(captured['args']) + assert extra_vars['cached_test_repos'] == [] diff --git a/tests/shared/test_third_party_ssh_hosts.py b/tests/shared/test_third_party_ssh_hosts.py index 839b845..6847592 100644 --- a/tests/shared/test_third_party_ssh_hosts.py +++ b/tests/shared/test_third_party_ssh_hosts.py @@ -7,7 +7,11 @@ """ import pytest -from alts.shared.models import CeleryConfig, ThirdPartyRepoSshHost +from alts.shared.models import ( + CachedTestRepo, + CeleryConfig, + ThirdPartyRepoSshHost, +) class TestThirdPartyRepoSshHost: @@ -35,12 +39,31 @@ def test_host_is_required(self): ThirdPartyRepoSshHost() +class TestCachedTestRepo: + def test_requires_src_and_dest(self): + entry = CachedTestRepo( + src='/opt/QA', + dest='/opt/gerrit.cloudlinux.com/QA', + ) + assert entry.model_dump() == { + 'src': '/opt/QA', + 'dest': '/opt/gerrit.cloudlinux.com/QA', + } + + @pytest.mark.parametrize('kwargs', [{}, {'src': '/opt/QA'}]) + def test_missing_field_raises(self, kwargs): + with pytest.raises(Exception): + CachedTestRepo(**kwargs) + + class TestCeleryConfigDefault: - def test_third_party_repo_ssh_hosts_defaults_to_empty(self): - # No built-in hosts: an unconfigured deployment writes no - # ~/.ssh/config (the role task is gated on a non-empty list). - config = CeleryConfig.__new__(CeleryConfig) - field = CeleryConfig.model_fields['third_party_repo_ssh_hosts'] - # Pydantic stores the default factory / default value. + @pytest.mark.parametrize( + 'field_name', + ['third_party_repo_ssh_hosts', 'cached_test_repos'], + ) + def test_field_defaults_to_empty(self, field_name): + # No built-in entries: an unconfigured deployment is a no-op + # (the role tasks are gated on non-empty lists). + field = CeleryConfig.model_fields[field_name] default = field.get_default(call_default_factory=True) assert default == []