diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 775ee06a82..888aa97a0e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -37,15 +37,15 @@ jobs: timeout-minutes: 5 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v7 - name: Set up Python - uses: actions/setup-python@v5 + uses: actions/setup-python@v6 with: python-version: 3.11 - name: Lint with flake8 run: | pip install flake8 - flake8 src scripts conftest.py + flake8 src scripts pytest: @@ -79,22 +79,21 @@ jobs: timeout-minutes: 360 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v7 with: # just fetching 1 commit is not enough for setuptools-scm, so we fetch all fetch-depth: 0 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v5 + uses: actions/setup-python@v6 with: python-version: ${{ matrix.python-version }} - name: Cache pip - uses: actions/cache@v4 + uses: actions/cache@v6 with: path: ~/.cache/pip - key: ${{ runner.os }}-pip-${{ hashFiles('requirements.d/development.txt') }} + key: ${{ runner.os }}-${{ runner.arch }}-pip-${{ hashFiles('requirements.d/development.txt') }} restore-keys: | - ${{ runner.os }}-pip- - ${{ runner.os }}- + ${{ runner.os }}-${{ runner.arch }}-pip- - name: Install Linux packages if: ${{ runner.os == 'Linux' }} @@ -136,7 +135,7 @@ jobs: #sudo -E bash -c "tox -e py" tox --skip-missing-interpreters - name: Upload coverage to Codecov - uses: codecov/codecov-action@v4 + uses: codecov/codecov-action@v7 env: OS: ${{ runner.os }} python: ${{ matrix.python-version }} diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index c96dc6b0cf..c31cb6fd9b 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -29,16 +29,16 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v4 + uses: actions/checkout@v7 with: # just fetching 1 commit is not enough for setuptools-scm, so we fetch all fetch-depth: 0 - name: Set up Python - uses: actions/setup-python@v5 + uses: actions/setup-python@v6 with: python-version: 3.11 - name: Cache pip - uses: actions/cache@v4 + uses: actions/cache@v6 with: path: ~/.cache/pip key: ${{ runner.os }}-pip-${{ hashFiles('requirements.d/development.txt') }} @@ -52,7 +52,7 @@ jobs: sudo apt-get install -y libssl-dev libacl1-dev libxxhash-dev liblz4-dev libzstd-dev # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL - uses: github/codeql-action/init@v2 + uses: github/codeql-action/init@v4 with: languages: ${{ matrix.language }} # If you wish to specify custom queries, you can do so here or in a config file. @@ -66,4 +66,4 @@ jobs: pip3 install -r requirements.d/development.txt pip3 install -ve . - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v2 + uses: github/codeql-action/analyze@v4 diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml index 4d901fa292..dd2b667a78 100644 --- a/.github/workflows/windows.yml +++ b/.github/workflows/windows.yml @@ -8,7 +8,7 @@ jobs: run: shell: msys2 {0} steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v7 with: fetch-depth: 0 - uses: msys2/setup-msys2@v2 @@ -21,7 +21,7 @@ jobs: run: ./scripts/msys2-build - name: Test run: ./dist/borg.exe -V - - uses: actions/upload-artifact@v4 + - uses: actions/upload-artifact@v7 with: name: borg-windows path: dist/borg.exe diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 0c1a27a78b..f6df9b4856 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -3,4 +3,4 @@ repos: rev: 6.1.0 hooks: - id: flake8 - files: '(src|scripts|conftest.py)' + files: '(src|scripts)' diff --git a/src/borg/testsuite/__init__.py b/src/borg/testsuite/__init__.py index 7a3aaef37d..1e9660062c 100644 --- a/src/borg/testsuite/__init__.py +++ b/src/borg/testsuite/__init__.py @@ -156,13 +156,14 @@ def is_birthtime_fully_supported(): return False -def no_selinux(x): - # selinux fails our FUSE tests, thus ignore selinux xattrs - SELINUX_KEY = b'security.selinux' +def filter_xattrs(x): + # selinux and com.apple.provenance fail our FUSE tests, thus ignore them + UNWANTED_KEYS = {b"security.selinux", b"com.apple.provenance"} if isinstance(x, dict): - return {k: v for k, v in x.items() if k != SELINUX_KEY} + return {k: v for k, v in x.items() if k not in UNWANTED_KEYS} if isinstance(x, list): - return [k for k in x if k != SELINUX_KEY] + return [k for k in x if k not in UNWANTED_KEYS] + raise ValueError("Unsupported type: %s" % type(x)) class BaseTestCase(unittest.TestCase): @@ -233,8 +234,8 @@ def _assert_dirs_equal_cmp(self, diff, ignore_flags=False, ignore_xattrs=False, d1.append(round(s1.st_mtime_ns, st_mtime_ns_round)) d2.append(round(s2.st_mtime_ns, st_mtime_ns_round)) if not ignore_xattrs: - d1.append(no_selinux(get_all(path1, follow_symlinks=False))) - d2.append(no_selinux(get_all(path2, follow_symlinks=False))) + d1.append(filter_xattrs(get_all(path1, follow_symlinks=False))) + d2.append(filter_xattrs(get_all(path2, follow_symlinks=False))) self.assert_equal(d1, d2) for sub_diff in diff.subdirs.values(): self._assert_dirs_equal_cmp(sub_diff, ignore_flags=ignore_flags, ignore_xattrs=ignore_xattrs, ignore_ns=ignore_ns) diff --git a/src/borg/testsuite/archiver.py b/src/borg/testsuite/archiver.py index 55e1c3dcfa..6af41a967f 100644 --- a/src/borg/testsuite/archiver.py +++ b/src/borg/testsuite/archiver.py @@ -55,7 +55,7 @@ from ..remote import RemoteRepository, PathNotAllowed from ..repository import Repository from . import has_lchflags, llfuse -from . import BaseTestCase, changedir, environment_variable, no_selinux, same_ts_ns +from . import BaseTestCase, changedir, environment_variable, filter_xattrs, same_ts_ns from . import are_symlinks_supported, are_hardlinks_supported, are_fifos_supported, is_utime_fully_supported, is_birthtime_fully_supported from .platform import fakeroot_detected, is_darwin, is_freebsd, is_win32 from .upgrader import make_attic_repo @@ -2166,6 +2166,15 @@ def fifo_feeder(fifo_fn, data): try: self.cmd('create', '--read-special', archive, 'input/link_fifo') finally: + # In case `borg create` failed to open FIFO, read all data to avoid join() hanging. + fd = os.open(fifo_fn, os.O_RDONLY | os.O_NONBLOCK) + try: + os.read(fd, len(data)) + except OSError: + # fails on FreeBSD 13 with BlockingIOError + pass + finally: + os.close(fd) t.join() with changedir('output'): self.cmd('extract', archive) @@ -2612,11 +2621,14 @@ def test_compression_auto_uncompressible(self): def test_change_passphrase(self): self.cmd('init', '--encryption=repokey', self.repository_location) - os.environ['BORG_NEW_PASSPHRASE'] = 'newpassphrase' - # here we have both BORG_PASSPHRASE and BORG_NEW_PASSPHRASE set: - self.cmd('key', 'change-passphrase', self.repository_location) - os.environ['BORG_PASSPHRASE'] = 'newpassphrase' - self.cmd('list', self.repository_location) + # use the context manager so BORG_NEW_PASSPHRASE / BORG_PASSPHRASE are restored + # afterwards and cannot leak into later tests (a leaked BORG_NEW_PASSPHRASE would + # make a later repokey `init` encrypt with the wrong passphrase). + with environment_variable(BORG_NEW_PASSPHRASE='newpassphrase'): + # here we have both BORG_PASSPHRASE and BORG_NEW_PASSPHRASE set: + self.cmd('key', 'change-passphrase', self.repository_location) + with environment_variable(BORG_PASSPHRASE='newpassphrase'): + self.cmd('list', self.repository_location) def test_break_lock(self): self.cmd('init', '--encryption=repokey', self.repository_location) @@ -2716,11 +2728,11 @@ def has_noatime(some_file): in_fn = 'input/fusexattr' out_fn = os.fsencode(os.path.join(mountpoint, 'input', 'fusexattr')) if not xattr.XATTR_FAKEROOT and xattr.is_enabled(self.input_path): - assert sorted(no_selinux(xattr.listxattr(out_fn))) == [b'user.empty', b'user.foo', ] + assert sorted(filter_xattrs(xattr.listxattr(out_fn))) == [b'user.empty', b'user.foo', ] assert xattr.getxattr(out_fn, b'user.foo') == b'bar' assert xattr.getxattr(out_fn, b'user.empty') == b'' else: - assert no_selinux(xattr.listxattr(out_fn)) == [] + assert filter_xattrs(xattr.listxattr(out_fn)) == [] try: xattr.getxattr(out_fn, b'user.foo') except OSError as e: @@ -3878,8 +3890,6 @@ def test_check_usage(self): self.assert_in('Starting repository check', output) self.assert_in('Starting archive consistency check', output) self.assert_in('Checking segments', output) - # reset logging to new process default to avoid need for fork=True on next check - logging.getLogger('borg.output.progress').setLevel(logging.NOTSET) output = self.cmd('check', '-v', '--repository-only', self.repository_location, exit_code=0) self.assert_in('Starting repository check', output) self.assert_not_in('Starting archive consistency check', output) diff --git a/conftest.py b/src/borg/testsuite/conftest.py similarity index 78% rename from conftest.py rename to src/borg/testsuite/conftest.py index bc40f33b7e..12070964ca 100644 --- a/conftest.py +++ b/src/borg/testsuite/conftest.py @@ -1,3 +1,4 @@ +import logging import os import pytest @@ -26,6 +27,23 @@ from borg.testsuite.platform import fakeroot_detected # noqa: E402 +@pytest.fixture(autouse=True) +def reset_progress_logger(): + # ProgressIndicatorBase installs a handler on the process-global + # 'borg.output.progress' logger and only removes it in __del__, and in-process + # (fork=False) command execution leaves this logger at level INFO / propagate=False. + # This state leaks across tests and makes the progress tests flaky (the progress + # output ends up in the logging system instead of on the stderr captured by capfd). + # Reset it after each test to keep tests isolated. + yield + logger = logging.getLogger('borg.output.progress') + for handler in logger.handlers[:]: + logger.removeHandler(handler) + handler.close() + logger.setLevel(logging.NOTSET) + logger.propagate = True + + @pytest.fixture(autouse=True) def clean_env(tmpdir_factory, monkeypatch): # avoid that we access / modify the user's normal .config / .cache directory: diff --git a/src/borg/testsuite/repository.py b/src/borg/testsuite/repository.py index b537ccc811..ca0f9d9c07 100644 --- a/src/borg/testsuite/repository.py +++ b/src/borg/testsuite/repository.py @@ -1062,6 +1062,13 @@ class RemoteLoggerTestCase(BaseTestCase): def setUp(self): self.stream = io.StringIO() self.handler = logging.StreamHandler(self.stream) + + # Save old logging state + self.old_root_handlers = logging.getLogger().handlers[:] + self.old_root_level = logging.getLogger().level + self.old_repo_handlers = logging.getLogger('borg.repository').handlers[:] + self.old_repo_level = logging.getLogger('borg.repository').level + logging.getLogger().handlers[:] = [self.handler] logging.getLogger('borg.repository').handlers[:] = [] logging.getLogger('borg.repository.foo').handlers[:] = [] @@ -1073,6 +1080,12 @@ def setUp(self): def tearDown(self): sys.stderr = self.old_stderr + # Restore old logging state + logging.getLogger().handlers[:] = self.old_root_handlers + logging.getLogger().setLevel(self.old_root_level) + logging.getLogger('borg.repository').handlers[:] = self.old_repo_handlers + logging.getLogger('borg.repository').setLevel(self.old_repo_level) + def test_stderr_messages(self): handle_remote_line("unstructured stderr message\n") self.assert_equal(self.stream.getvalue(), '') diff --git a/tox.ini b/tox.ini index 425aba6ab2..3ae0d53e6e 100644 --- a/tox.ini +++ b/tox.ini @@ -32,4 +32,4 @@ skip_install=true changedir = deps = flake8 -commands = flake8 src scripts conftest.py +commands = flake8 src scripts