1.2-maint: fix the testsuite

.github/workflows/ci.yml

@@ -37,15 +37,15 @@ jobs:
     timeout-minutes: 5
 
     steps:
-    - uses: actions/checkout@v4
+    - uses: actions/checkout@v7
     - name: Set up Python
-      uses: actions/setup-python@v5
+      uses: actions/setup-python@v6
       with:
         python-version: 3.11
     - name: Lint with flake8
       run: |
         pip install flake8
-        flake8 src scripts conftest.py
+        flake8 src scripts
 
   pytest:
 
@@ -79,22 +79,21 @@ jobs:
     timeout-minutes: 360
 
     steps:
-    - uses: actions/checkout@v4
+    - uses: actions/checkout@v7
       with:
         # just fetching 1 commit is not enough for setuptools-scm, so we fetch all
         fetch-depth: 0
     - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v5
+      uses: actions/setup-python@v6
       with:
         python-version: ${{ matrix.python-version }}
     - name: Cache pip
-      uses: actions/cache@v4
+      uses: actions/cache@v6
       with:
         path: ~/.cache/pip
-        key: ${{ runner.os }}-pip-${{ hashFiles('requirements.d/development.txt') }}
+        key: ${{ runner.os }}-${{ runner.arch }}-pip-${{ hashFiles('requirements.d/development.txt') }}
         restore-keys: |
-            ${{ runner.os }}-pip-
-            ${{ runner.os }}-
+            ${{ runner.os }}-${{ runner.arch }}-pip-
 
     - name: Install Linux packages
       if: ${{ runner.os == 'Linux' }}
@@ -136,7 +135,7 @@ jobs:
         #sudo -E bash -c "tox -e py"
         tox --skip-missing-interpreters
     - name: Upload coverage to Codecov
-      uses: codecov/codecov-action@v4
+      uses: codecov/codecov-action@v7
       env:
         OS: ${{ runner.os }}
         python: ${{ matrix.python-version }}

.github/workflows/codeql-analysis.yml

@@ -29,16 +29,16 @@ jobs:
 
     steps:
     - name: Checkout repository
-      uses: actions/checkout@v4
+      uses: actions/checkout@v7
       with:
         # just fetching 1 commit is not enough for setuptools-scm, so we fetch all
         fetch-depth: 0
     - name: Set up Python
-      uses: actions/setup-python@v5
+      uses: actions/setup-python@v6
       with:
         python-version: 3.11
     - name: Cache pip
-      uses: actions/cache@v4
+      uses: actions/cache@v6
       with:
         path: ~/.cache/pip
         key: ${{ runner.os }}-pip-${{ hashFiles('requirements.d/development.txt') }}
@@ -52,7 +52,7 @@ jobs:
         sudo apt-get install -y libssl-dev libacl1-dev libxxhash-dev liblz4-dev libzstd-dev
     # Initializes the CodeQL tools for scanning.
     - name: Initialize CodeQL
-      uses: github/codeql-action/init@v2
+      uses: github/codeql-action/init@v4
       with:
         languages: ${{ matrix.language }}
         # If you wish to specify custom queries, you can do so here or in a config file.
@@ -66,4 +66,4 @@ jobs:
         pip3 install -r requirements.d/development.txt
         pip3 install -ve .
     - name: Perform CodeQL Analysis
-      uses: github/codeql-action/analyze@v2
+      uses: github/codeql-action/analyze@v4

.github/workflows/windows.yml

@@ -8,7 +8,7 @@ jobs:
       run:
         shell: msys2 {0}
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v7
         with:
           fetch-depth: 0
       - uses: msys2/setup-msys2@v2
@@ -21,7 +21,7 @@ jobs:
         run: ./scripts/msys2-build
       - name: Test
         run: ./dist/borg.exe -V
-      - uses: actions/upload-artifact@v4
+      - uses: actions/upload-artifact@v7
         with:
           name: borg-windows
           path: dist/borg.exe

.pre-commit-config.yaml

@@ -3,4 +3,4 @@ repos:
     rev: 6.1.0
     hooks:
     -   id: flake8
-        files: '(src|scripts|conftest.py)'
+        files: '(src|scripts)'

src/borg/testsuite/__init__.py

@@ -156,13 +156,14 @@ def is_birthtime_fully_supported():
         return False
 
 
-def no_selinux(x):
-    # selinux fails our FUSE tests, thus ignore selinux xattrs
-    SELINUX_KEY = b'security.selinux'
+def filter_xattrs(x):
+    # selinux and com.apple.provenance fail our FUSE tests, thus ignore them
+    UNWANTED_KEYS = {b"security.selinux", b"com.apple.provenance"}
     if isinstance(x, dict):
-        return {k: v for k, v in x.items() if k != SELINUX_KEY}
+        return {k: v for k, v in x.items() if k not in UNWANTED_KEYS}
     if isinstance(x, list):
-        return [k for k in x if k != SELINUX_KEY]
+        return [k for k in x if k not in UNWANTED_KEYS]
+    raise ValueError("Unsupported type: %s" % type(x))
 
 
 class BaseTestCase(unittest.TestCase):
@@ -233,8 +234,8 @@ def _assert_dirs_equal_cmp(self, diff, ignore_flags=False, ignore_xattrs=False,
                     d1.append(round(s1.st_mtime_ns, st_mtime_ns_round))
                     d2.append(round(s2.st_mtime_ns, st_mtime_ns_round))
             if not ignore_xattrs:
-                d1.append(no_selinux(get_all(path1, follow_symlinks=False)))
-                d2.append(no_selinux(get_all(path2, follow_symlinks=False)))
+                d1.append(filter_xattrs(get_all(path1, follow_symlinks=False)))
+                d2.append(filter_xattrs(get_all(path2, follow_symlinks=False)))
             self.assert_equal(d1, d2)
         for sub_diff in diff.subdirs.values():
             self._assert_dirs_equal_cmp(sub_diff, ignore_flags=ignore_flags, ignore_xattrs=ignore_xattrs, ignore_ns=ignore_ns)

src/borg/testsuite/archiver.py

@@ -55,7 +55,7 @@
 from ..remote import RemoteRepository, PathNotAllowed
 from ..repository import Repository
 from . import has_lchflags, llfuse
-from . import BaseTestCase, changedir, environment_variable, no_selinux, same_ts_ns
+from . import BaseTestCase, changedir, environment_variable, filter_xattrs, same_ts_ns
 from . import are_symlinks_supported, are_hardlinks_supported, are_fifos_supported, is_utime_fully_supported, is_birthtime_fully_supported
 from .platform import fakeroot_detected, is_darwin, is_freebsd, is_win32
 from .upgrader import make_attic_repo
@@ -2166,6 +2166,15 @@ def fifo_feeder(fifo_fn, data):
         try:
             self.cmd('create', '--read-special', archive, 'input/link_fifo')
         finally:
+            # In case `borg create` failed to open FIFO, read all data to avoid join() hanging.
+            fd = os.open(fifo_fn, os.O_RDONLY | os.O_NONBLOCK)
+            try:
+                os.read(fd, len(data))
+            except OSError:
+                # fails on FreeBSD 13 with BlockingIOError
+                pass
+            finally:
+                os.close(fd)
             t.join()
         with changedir('output'):
             self.cmd('extract', archive)
@@ -2612,11 +2621,14 @@ def test_compression_auto_uncompressible(self):
 
     def test_change_passphrase(self):
         self.cmd('init', '--encryption=repokey', self.repository_location)
-        os.environ['BORG_NEW_PASSPHRASE'] = 'newpassphrase'
-        # here we have both BORG_PASSPHRASE and BORG_NEW_PASSPHRASE set:
-        self.cmd('key', 'change-passphrase', self.repository_location)
-        os.environ['BORG_PASSPHRASE'] = 'newpassphrase'
-        self.cmd('list', self.repository_location)
+        # use the context manager so BORG_NEW_PASSPHRASE / BORG_PASSPHRASE are restored
+        # afterwards and cannot leak into later tests (a leaked BORG_NEW_PASSPHRASE would
+        # make a later repokey `init` encrypt with the wrong passphrase).
+        with environment_variable(BORG_NEW_PASSPHRASE='newpassphrase'):
+            # here we have both BORG_PASSPHRASE and BORG_NEW_PASSPHRASE set:
+            self.cmd('key', 'change-passphrase', self.repository_location)
+            with environment_variable(BORG_PASSPHRASE='newpassphrase'):
+                self.cmd('list', self.repository_location)
 
     def test_break_lock(self):
         self.cmd('init', '--encryption=repokey', self.repository_location)
@@ -2716,11 +2728,11 @@ def has_noatime(some_file):
                 in_fn = 'input/fusexattr'
                 out_fn = os.fsencode(os.path.join(mountpoint, 'input', 'fusexattr'))
                 if not xattr.XATTR_FAKEROOT and xattr.is_enabled(self.input_path):
-                    assert sorted(no_selinux(xattr.listxattr(out_fn))) == [b'user.empty', b'user.foo', ]
+                    assert sorted(filter_xattrs(xattr.listxattr(out_fn))) == [b'user.empty', b'user.foo', ]
                     assert xattr.getxattr(out_fn, b'user.foo') == b'bar'
                     assert xattr.getxattr(out_fn, b'user.empty') == b''
                 else:
-                    assert no_selinux(xattr.listxattr(out_fn)) == []
+                    assert filter_xattrs(xattr.listxattr(out_fn)) == []
                     try:
                         xattr.getxattr(out_fn, b'user.foo')
                     except OSError as e:
@@ -3878,8 +3890,6 @@ def test_check_usage(self):
         self.assert_in('Starting repository check', output)
         self.assert_in('Starting archive consistency check', output)
         self.assert_in('Checking segments', output)
-        # reset logging to new process default to avoid need for fork=True on next check
-        logging.getLogger('borg.output.progress').setLevel(logging.NOTSET)
         output = self.cmd('check', '-v', '--repository-only', self.repository_location, exit_code=0)
         self.assert_in('Starting repository check', output)
         self.assert_not_in('Starting archive consistency check', output)

conftest.py → src/borg/testsuite/conftest.py

@@ -1,3 +1,4 @@
+import logging
 import os
 
 import pytest
@@ -26,6 +27,23 @@
 from borg.testsuite.platform import fakeroot_detected  # noqa: E402
 
 
+@pytest.fixture(autouse=True)
+def reset_progress_logger():
+    # ProgressIndicatorBase installs a handler on the process-global
+    # 'borg.output.progress' logger and only removes it in __del__, and in-process
+    # (fork=False) command execution leaves this logger at level INFO / propagate=False.
+    # This state leaks across tests and makes the progress tests flaky (the progress
+    # output ends up in the logging system instead of on the stderr captured by capfd).
+    # Reset it after each test to keep tests isolated.
+    yield
+    logger = logging.getLogger('borg.output.progress')
+    for handler in logger.handlers[:]:
+        logger.removeHandler(handler)
+        handler.close()
+    logger.setLevel(logging.NOTSET)
+    logger.propagate = True
+
+
 @pytest.fixture(autouse=True)
 def clean_env(tmpdir_factory, monkeypatch):
     # avoid that we access / modify the user's normal .config / .cache directory:

src/borg/testsuite/repository.py

@@ -1062,6 +1062,13 @@ class RemoteLoggerTestCase(BaseTestCase):
     def setUp(self):
         self.stream = io.StringIO()
         self.handler = logging.StreamHandler(self.stream)
+
+        # Save old logging state
+        self.old_root_handlers = logging.getLogger().handlers[:]
+        self.old_root_level = logging.getLogger().level
+        self.old_repo_handlers = logging.getLogger('borg.repository').handlers[:]
+        self.old_repo_level = logging.getLogger('borg.repository').level
+
         logging.getLogger().handlers[:] = [self.handler]
         logging.getLogger('borg.repository').handlers[:] = []
         logging.getLogger('borg.repository.foo').handlers[:] = []
@@ -1073,6 +1080,12 @@ def setUp(self):
     def tearDown(self):
         sys.stderr = self.old_stderr
 
+        # Restore old logging state
+        logging.getLogger().handlers[:] = self.old_root_handlers
+        logging.getLogger().setLevel(self.old_root_level)
+        logging.getLogger('borg.repository').handlers[:] = self.old_repo_handlers
+        logging.getLogger('borg.repository').setLevel(self.old_repo_level)
+
     def test_stderr_messages(self):
         handle_remote_line("unstructured stderr message\n")
         self.assert_equal(self.stream.getvalue(), '')

tox.ini

@@ -32,4 +32,4 @@ skip_install=true
 changedir =
 deps =
     flake8
-commands = flake8 src scripts conftest.py
+commands = flake8 src scripts