diff --git a/packaging/leapp-repository-changelog.txt b/packaging/leapp-repository-changelog.txt index 4bf0cd26f7..1a70c3a817 100644 --- a/packaging/leapp-repository-changelog.txt +++ b/packaging/leapp-repository-changelog.txt @@ -1,3 +1,7 @@ +* Tue Aug 11 2026 Roman Prilipskii 0.20.0-11.cloudlinux +- CLOS-6809: Fix the upgrade being blocked on servers running MySQL Governor with MariaDB 11.4 or 11.8, where the installed database version was reported in a form that the cl-mysql repository check did not recognise +- Determine the database module to enable from the configured cl-mysql repository, so that newly released CloudLinux MySQL and MariaDB versions are upgraded correctly without waiting for a Leapp update + * Thu Aug 06 2026 Roman Prilipskii 0.20.0-10.cloudlinux - CLOS-2132: Warn before the upgrade when the PostgreSQL configuration would keep the database from starting on the new system, and spell out the migration steps to run afterwards - CLOS-3716: Stop the upgrade when the target repositories offer a kernel newer than the rest of the target system, which could leave LVE unusable after the reboot diff --git a/packaging/leapp-repository.spec b/packaging/leapp-repository.spec index 22e153e507..8f21f86c37 100644 --- a/packaging/leapp-repository.spec +++ b/packaging/leapp-repository.spec @@ -42,7 +42,7 @@ py2_byte_compile "%1" "%2"} Name: leapp-repository Version: 0.20.0 -Release: 10%{?dist}.cloudlinux +Release: 11%{?dist}.cloudlinux Summary: Repositories for leapp License: ASL 2.0 diff --git a/repos/system_upgrade/cloudlinux/actors/clmysqlrepositorysetup/libraries/clmysql_cloudlinux.py b/repos/system_upgrade/cloudlinux/actors/clmysqlrepositorysetup/libraries/clmysql_cloudlinux.py index 14a4760609..463a58870b 100644 --- a/repos/system_upgrade/cloudlinux/actors/clmysqlrepositorysetup/libraries/clmysql_cloudlinux.py +++ b/repos/system_upgrade/cloudlinux/actors/clmysqlrepositorysetup/libraries/clmysql_cloudlinux.py @@ -9,7 +9,8 @@ ClMysqlTypeStatus, construct_repomap_data, get_clmysql_type, - get_expected_repo_url_fragment, + parse_clmysql_repo_url, + parse_clmysql_type, ) from leapp.libraries.common.config.version import get_target_major_version from leapp.libraries.stdlib import api @@ -82,64 +83,99 @@ def clmysql_process(lib, repofile_name, repofile_data): api.current_logger().debug("repoids from CloudLinux repofile {}: {}".format(repofile_name, data_to_log)) - # Validate that cl-mysql-meta repo's baseurl matches the detected DB type. - # Governor configures the URL based on the selected DB version (e.g. cl-mariadb-10.6 for mariadb106). - # If the user switched DB versions without re-running Governor --install, - # the repo may point to the wrong package set. + # Validate that the cl-mysql-meta repo's baseurl matches the detected DB type. + # Governor picks the URL from the selected DB version (e.g. cl-mariadb-10.6 for + # mariadb106). If the user switched DB versions without re-running Governor + # --install, the repo may point at the wrong package set. + # + # Compare the parsed versions rather than a predicted directory name. Governor's + # spelling is not derivable - 11.4 is published as cl-mariadb-11.04 but 10.6 as + # cl-mariadb-10.6 - and predicting it produced a phantom cl-mariadb-11.4, which + # inhibited every MariaDB 11.x upgrade and printed a 404 URL as the remedy + # (CLOS-6809). + # # We inhibit rather than auto-correct because Governor is the authoritative source # for this repo file and has the real download-and-write logic; # re-running --install is the proper fix. - expected_fragment = get_expected_repo_url_fragment(lib.clmysql_type) - if expected_fragment: + detected_version = parse_clmysql_type(lib.clmysql_type) + if detected_version: for repo_data in repofile_data.data: - if repo_data.repoid == "cl-mysql-meta" and "/{}/".format(expected_fragment) not in repo_data.baseurl: + if repo_data.repoid != "cl-mysql-meta": + continue + repo_version = parse_clmysql_repo_url(repo_data.baseurl) + if repo_version == detected_version: + # Confirmed to describe the installed DB, so it is a trustworthy source + # for the DNF module stream, which the type token cannot spell reliably. + lib.clmysql_meta_baseurl = repo_data.baseurl + continue + # Either the repository describes a different version, or its URL cannot be + # read at all. Both mean the repository cannot be confirmed to match the + # installed database, and an unconfirmable repository is exactly what this + # check exists to stop - letting it through would hand the upgrade a + # repository nobody has validated. + if repo_version is None: + api.current_logger().warning( + "cl-mysql-meta repo baseurl '{}' does not name a recognisable database " + "version; cannot confirm it matches the installed type '{}'." + .format(repo_data.baseurl, lib.clmysql_type) + ) + detail = ( + "the cl-mysql-meta repository URL does not name a database version that " + "Leapp can read: '{0}'. Leapp cannot confirm that this repository provides " + "the installed database, and will not upgrade against a repository it " + "cannot verify.".format(repo_data.baseurl) + ) + else: api.current_logger().warning( "cl-mysql-meta repo baseurl '{}' does not match detected DB type '{}' " - "(expected '{}' in URL)." - .format(repo_data.baseurl, lib.clmysql_type, expected_fragment) + "(repo describes {}.{}, installed is {}.{})." + .format( + repo_data.baseurl, lib.clmysql_type, + repo_version[1], repo_version[2], + detected_version[1], detected_version[2], + ) ) - reporting.create_report( - [ - reporting.Title( - "cl-mysql.repo does not match the installed database type" - ), - reporting.Summary( - "The cl-mysql-meta repository is configured for a different " - "database type than what is actually installed. " - "The detected database type is '{}', but the cl-mysql-meta " - "repo URL points to '{}'. " - "This may happen when the database version was changed " - "without a follow-up '/usr/share/lve/dbgovernor/mysqlgovernor.py --install', or the " - "cl-mysql.repo file was manually edited. " - "Proceeding with the wrong repository would result in " - "an incorrect upgrade operation." - .format(lib.clmysql_type, repo_data.baseurl) - ), - reporting.Severity(reporting.Severity.HIGH), - reporting.Groups( - [ - reporting.Groups.REPOSITORY, - reporting.Groups.OS_FACTS - ] - ), - reporting.Groups([reporting.Groups.INHIBITOR]), - reporting.Remediation( - hint=( - "Download the correct repository file for the installed " - "database type: " - "curl -o /etc/yum.repos.d/cl-mysql.repo " - "http://repo.cloudlinux.com/other/" - "cl${{releasever}}/mysqlmeta/{expected}-common.repo\n" - "Or re-run MySQL Governor to regenerate it " - "(this reinstalls the full DB stack): " - "/usr/share/lve/dbgovernor/mysqlgovernor.py --install --yes\n" - "Then restart the upgrade process." - .format(expected=expected_fragment) - ) - ), - ] + detail = ( + "the cl-mysql-meta repository is configured for {0} {1}.{2} while the " + "installed database is {3} {4}.{5}: '{6}'.".format( + repo_version[0], repo_version[1], repo_version[2], + detected_version[0], detected_version[1], detected_version[2], + repo_data.baseurl, + ) ) - return + reporting.create_report( + [ + reporting.Title( + "cl-mysql.repo does not match the installed database type" + ), + reporting.Summary( + "The database packages that the upgrade installs come from the " + "cl-mysql-meta repository, so it has to describe the database that is " + "actually installed. Here {0} " + "This may happen when the database version was changed without a " + "follow-up '/usr/share/lve/dbgovernor/mysqlgovernor.py --install', or " + "when the cl-mysql.repo file was edited by hand. Proceeding would " + "result in an incorrect upgrade operation.".format(detail) + ), + reporting.Severity(reporting.Severity.HIGH), + reporting.Groups( + [ + reporting.Groups.REPOSITORY, + reporting.Groups.OS_FACTS + ] + ), + reporting.Groups([reporting.Groups.INHIBITOR]), + reporting.Remediation( + hint=( + "Re-run MySQL Governor to regenerate the repository file for the " + "installed database (this reinstalls the full DB stack): " + "/usr/share/lve/dbgovernor/mysqlgovernor.py --install --yes\n" + "Then restart the upgrade process." + ) + ), + ] + ) + return cl_target_repofile_list = [] target_major = get_target_major_version() diff --git a/repos/system_upgrade/cloudlinux/actors/clmysqlrepositorysetup/libraries/clmysqlrepositorysetup.py b/repos/system_upgrade/cloudlinux/actors/clmysqlrepositorysetup/libraries/clmysqlrepositorysetup.py index e922798984..fc0759d058 100644 --- a/repos/system_upgrade/cloudlinux/actors/clmysqlrepositorysetup/libraries/clmysqlrepositorysetup.py +++ b/repos/system_upgrade/cloudlinux/actors/clmysqlrepositorysetup/libraries/clmysqlrepositorysetup.py @@ -17,6 +17,8 @@ ) from leapp.libraries.common.clmysql import ( MODULE_STREAMS, + canonical_clmysql_type, + clmysql_module_stream_from_url, construct_repomap_data, get_pkg_prefix, resolve_clmysql_module_stream, @@ -67,6 +69,8 @@ class MySqlRepositorySetupLibrary(object): def __init__(self): self.mysql_types = set() self.clmysql_type = None + # baseurl of the cl-mysql-meta repo, once confirmed to match the installed DB. + self.clmysql_meta_baseurl = None # Messages to send about custom generated package repositories. self.custom_repo_msgs = [] self.mapping_msgs = [] @@ -116,23 +120,36 @@ def finalize(self): ) if "cloudlinux" in self.mysql_types and self.clmysql_type: - mod_name, mod_stream = resolve_clmysql_module_stream(self.clmysql_type) + mod_name, mod_stream = resolve_clmysql_module_stream( + self.clmysql_type, baseurl=self.clmysql_meta_baseurl + ) if mod_name and mod_stream: - if self.clmysql_type not in MODULE_STREAMS: + # Ask where the stream actually came from. Governor caches a spelling that + # MODULE_STREAMS does not list ("mariadb114" for MariaDB 11.4), so testing + # the raw token would report a verified stream as though it had been guessed. + if canonical_clmysql_type(self.clmysql_type) in MODULE_STREAMS: + stream_source = None + elif clmysql_module_stream_from_url(self.clmysql_meta_baseurl)[1] == mod_stream: + stream_source = "the configured cl-mysql repository" + else: + stream_source = "the detected database type" + + if stream_source: api.current_logger().warning( - "CL database type {} is not in MODULE_STREAMS; using derived DNF module {}:{}. " - "Add an explicit MODULE_STREAMS entry when this stream is product-supported." - .format(self.clmysql_type, mod_name, mod_stream) + "CL database type {} is not in MODULE_STREAMS; using DNF module {}:{} derived " + "from {}. Add an explicit MODULE_STREAMS entry when this stream is " + "product-supported." + .format(self.clmysql_type, mod_name, mod_stream, stream_source) ) reporting.create_report( [ reporting.Title("CloudLinux database module stream was derived automatically"), reporting.Summary( "The active CloudLinux MySQL/MariaDB/Percona type ({0}) has no explicit Leapp " - "MODULE_STREAMS entry. Leapp will enable DNF module {1}:{2} derived from the " - "detected type string. If the upgrade fails, confirm this module exists for the " - "target OS and add MODULE_STREAMS in Leapp if the product stream name differs." - .format(self.clmysql_type, mod_name, mod_stream) + "MODULE_STREAMS entry. Leapp will enable DNF module {1}:{2}, derived from " + "{3}. If the upgrade fails, confirm this module exists for the target OS and " + "add MODULE_STREAMS in Leapp if the product stream name differs." + .format(self.clmysql_type, mod_name, mod_stream, stream_source) ), reporting.Severity(reporting.Severity.MEDIUM), reporting.Groups([reporting.Groups.REPOSITORY, reporting.Groups.OS_FACTS]), diff --git a/repos/system_upgrade/cloudlinux/actors/clmysqlrepositorysetup/tests/test_clmysql_governor_token.py b/repos/system_upgrade/cloudlinux/actors/clmysqlrepositorysetup/tests/test_clmysql_governor_token.py new file mode 100644 index 0000000000..515df85400 --- /dev/null +++ b/repos/system_upgrade/cloudlinux/actors/clmysqlrepositorysetup/tests/test_clmysql_governor_token.py @@ -0,0 +1,133 @@ +""" +CLOS-6809 regression: a Governor-managed MariaDB 11.4 (or 11.8) system was +inhibited even though nothing on it was wrong. + +Governor derives the token it caches in mysql.type.installed by concatenating the +RPM major and minor version, so an 11.4.12 install is recorded as 'mariadb114'. +The canonical Governor token is 'mariadb1104' and the published repository +directory is cl-mariadb-11.04, so predicting the URL fragment from the cached +token produced 'cl-mariadb-11.4' - a path that does not exist - and the handler +inhibited the upgrade with a remediation hint pointing at a 404. + +The handler now compares parsed versions instead of predicting the spelling, so +either spelling of the token agrees with either spelling of the URL. These tests +drive it with the cached token exactly as it appears on disk, against the +repofile Governor actually writes. +""" + +from leapp import reporting +from leapp.libraries.actor.clmysql_cloudlinux import clmysql_process +from leapp.libraries.actor.clmysqlrepositorysetup import MySqlRepositorySetupLibrary + +# Verbatim from http://repo.cloudlinux.com/other/cl8/mysqlmeta/cl-mariadb-11.04-common.repo +_BASEURL_1104 = "http://repo.cloudlinux.com/other/cl$releasever/mysqlmeta/cl-mariadb-11.04/$basearch/" +_BASEURL_1108 = "http://repo.cloudlinux.com/other/cl$releasever/mysqlmeta/cl-mariadb-11.08/$basearch/" +# A URL whose version component Leapp cannot read at all. +_BASEURL_UNREADABLE = "http://repo.cloudlinux.com/other/cl$releasever/mysqlmeta/current/$basearch/" + + +class TestGovernorTokenSpelling: + + def test_mariadb_11_4_is_not_inhibited(self, patch_env, make_cl_mysql_repofile): + patch_env(clmysql_type="mariadb114") + + lib = MySqlRepositorySetupLibrary() + clmysql_process( + lib, + "cl-mysql", + make_cl_mysql_repofile(cl_mysql_meta_baseurl=_BASEURL_1104), + ) + + assert "cloudlinux" in lib.mysql_types + assert lib.custom_repo_msgs != [] + assert reporting.create_report.called == 0 + + def test_mariadb_11_8_is_not_inhibited(self, patch_env, make_cl_mysql_repofile): + patch_env(clmysql_type="mariadb118") + + lib = MySqlRepositorySetupLibrary() + clmysql_process( + lib, + "cl-mysql", + make_cl_mysql_repofile(cl_mysql_meta_baseurl=_BASEURL_1108), + ) + + assert "cloudlinux" in lib.mysql_types + assert lib.custom_repo_msgs != [] + assert reporting.create_report.called == 0 + + def test_canonical_token_against_padded_url_is_not_inhibited(self, patch_env, make_cl_mysql_repofile): + """The other spelling pairing: canonical token, padded URL.""" + patch_env(clmysql_type="mariadb1104") + + lib = MySqlRepositorySetupLibrary() + clmysql_process( + lib, + "cl-mysql", + make_cl_mysql_repofile(cl_mysql_meta_baseurl=_BASEURL_1104), + ) + + assert "cloudlinux" in lib.mysql_types + assert reporting.create_report.called == 0 + + def test_genuinely_wrong_repofile_still_inhibits(self, patch_env, make_cl_mysql_repofile): + """An 11.4 install pointed at the 10.6 repository is still a real problem.""" + patch_env(clmysql_type="mariadb114") + + lib = MySqlRepositorySetupLibrary() + clmysql_process(lib, "cl-mysql", make_cl_mysql_repofile()) + + assert "cloudlinux" not in lib.mysql_types + assert lib.custom_repo_msgs == [] + assert reporting.create_report.called == 1 + + def test_remediation_does_not_point_at_a_guessed_url(self, patch_env, make_cl_mysql_repofile): + """ + The old hint told customers to curl a predicted -common.repo file, which 404s + for MariaDB 11.x. Re-running Governor regenerates the file by definition. + """ + patch_env(clmysql_type="mariadb114") + + lib = MySqlRepositorySetupLibrary() + clmysql_process(lib, "cl-mysql", make_cl_mysql_repofile()) + + hint = reporting.create_report.report_fields["detail"]["remediations"][0]["context"] + assert "-common.repo" not in hint + assert "mysqlgovernor.py --install --yes" in hint + + +class TestUnreadableRepoUrl: + """ + A cl-mysql-meta URL whose version cannot be read must inhibit. Comparing parsed + versions replaced a substring test that inhibited on anything unrecognised, and + for a while an unreadable URL slipped through: the repository was copied, + force-enabled and handed to the upgrade without ever being checked. + """ + + def test_unreadable_url_inhibits(self, patch_env, make_cl_mysql_repofile): + patch_env(clmysql_type="mariadb114") + + lib = MySqlRepositorySetupLibrary() + clmysql_process( + lib, + "cl-mysql", + make_cl_mysql_repofile(cl_mysql_meta_baseurl=_BASEURL_UNREADABLE), + ) + + assert reporting.create_report.called == 1 + assert "cloudlinux" not in lib.mysql_types + assert lib.custom_repo_msgs == [] + + def test_unreadable_url_says_why(self, patch_env, make_cl_mysql_repofile): + patch_env(clmysql_type="mariadb114") + + lib = MySqlRepositorySetupLibrary() + clmysql_process( + lib, + "cl-mysql", + make_cl_mysql_repofile(cl_mysql_meta_baseurl=_BASEURL_UNREADABLE), + ) + + summary = reporting.create_report.report_fields["summary"] + assert "does not name a database version" in summary + assert _BASEURL_UNREADABLE in summary diff --git a/repos/system_upgrade/cloudlinux/libraries/clmysql.py b/repos/system_upgrade/cloudlinux/libraries/clmysql.py index e6a9d397ff..2f9bf2a86c 100644 --- a/repos/system_upgrade/cloudlinux/libraries/clmysql.py +++ b/repos/system_upgrade/cloudlinux/libraries/clmysql.py @@ -28,6 +28,13 @@ class ClMysqlTypeStatus(Enum): # Both files are present on CL7 and CL8+ when governor-mysql is installed. GOVERNOR_INSTALLED_TYPE_FILE = "/usr/share/lve/dbgovernor/mysql.type.installed" +# Matches the version directory Governor puts in the cl-mysql-meta repository URL, e.g. +# ".../mysqlmeta/cl-mariadb-11.04/$basearch/" -> ("mariadb", "11", "04"). +CLMYSQL_REPO_URL_RE = re.compile(r"cl-(mariadb|mysql|percona)-(\d+)\.(\d+)", re.IGNORECASE) + +# Capitalisation Governor uses for the DNF module stream of each family. +CLMYSQL_FAMILY_CAMEL = {"mariadb": "MariaDB", "mysql": "MySQL", "percona": "Percona"} + # This dict matches the MySQL type strings with DNF module and stream IDs. MODULE_STREAMS = { "mysql55": "mysql:cl-MySQL55", @@ -45,25 +52,145 @@ class ClMysqlTypeStatus(Enum): "mariadb106": "mariadb:cl-MariaDB106", "mariadb1011": "mariadb:cl-MariaDB1011", "mariadb1104": "mariadb:cl-MariaDB1104", + # No mariadb1108 entry on purpose. Governor declares mariadb:cl-MariaDB1108, but the + # published cl-mariadb-11.08 repository carries no modules.yaml on cl8 or cl9 (unlike + # cl-mariadb-11.04), so that stream cannot be confirmed to exist. Leaving it out keeps + # the "module stream was derived automatically" warning, which is the signal we want + # until the 11.08 content question is settled. "percona56": "percona:cl-Percona56", } -def resolve_clmysql_module_stream(clmysql_type): +def parse_clmysql_type(clmysql_type): + """ + Return ``(family, major, minor)`` for a Governor DB type token, or None. + + Governor spells the token by concatenating the major and minor version, and the + spelling is not consistent across series: MariaDB 11.4 is ``mariadb1104`` (minor + padded to two digits) while 10.6 is ``mariadb106`` (not padded). Governor also + re-derives the token from the RPM version when it caches it in mysql.type.installed + (``mysql_version()`` in install/utilities.py), which drops that padding again and + yields ``mariadb114`` for the very same installation. + + Comparing the numeric triple instead of the spelling makes the two forms equal - + ``mariadb114`` and ``mariadb1104`` both parse to ``("mariadb", 11, 4)`` - so no + table of Governor's spellings has to be kept in sync here (CLOS-6809). + + :param clmysql_type: type string such as ``mariadb114``, ``mariadb1104``, ``mysql80`` + :returns: ``(family, major, minor)``, or None when the token is not recognised + """ + match = re.match(r"^(mariadb|mysql|percona)(\d+)$", clmysql_type or "") + if not match: + return None + + family, digits = match.group(1), match.group(2) + # Two-digit tokens are a one-digit major ("mysql57" -> 5.7); longer ones are a + # two-digit major ("mariadb106" -> 10.6, "mariadb1104" -> 11.04). + split = 1 if len(digits) <= 2 else 2 + return family, int(digits[:split]), int(digits[split:]) + + +def parse_clmysql_repo_url(baseurl): + """ + Return ``(family, major, minor)`` for a cl-mysql-meta baseurl, or None. + + Parsing the version out of the URL and comparing it numerically avoids having to + predict how Governor spelled the directory: ``cl-mariadb-11.04`` and a hypothetical + ``cl-mariadb-11.4`` both parse to ``("mariadb", 11, 4)``. + """ + match = CLMYSQL_REPO_URL_RE.search(baseurl or "") + if not match: + return None + return match.group(1).lower(), int(match.group(2)), int(match.group(3)) + + +def canonical_clmysql_type(clmysql_type): + """ + Return the spelling of *clmysql_type* that MODULE_STREAMS knows, if there is one. + + Only the DNF module lookup still needs Governor's exact spelling, because the stream + name is built from it (``mariadb1104`` -> ``cl-MariaDB1104``). MODULE_STREAMS is the + registry of streams we have confirmed, so try the token as given and then with the + minor version padded back to two digits, and use whichever it lists. + + Unknown tokens are returned unchanged; the caller then derives a stream name and + reports that it did so, which is the existing signal for a series nobody has + verified yet. + """ + if not clmysql_type or clmysql_type in MODULE_STREAMS: + return clmysql_type + + parsed = parse_clmysql_type(clmysql_type) + if not parsed: + return clmysql_type + + family, major, minor = parsed + candidate = "{}{}{:02d}".format(family, major, minor) + if candidate in MODULE_STREAMS: + api.current_logger().debug( + "CL-MySQL type '{}' matches known module stream entry '{}'".format(clmysql_type, candidate) + ) + return candidate + + return clmysql_type + + +def clmysql_module_stream_from_url(baseurl): + """ + Derive (dnf_module_name, stream) from the cl-mysql-meta repository directory. + + The directory carries the same digits as the module stream, padding included - + cl-mariadb-11.04 goes with cl-MariaDB1104, cl-mariadb-10.6 with cl-MariaDB106 - + so the stream can be read off the repository the system is actually configured for + instead of being guessed from the cached type token, which has lost that padding. + That guess produced cl-MariaDB114 for MariaDB 11.4, and would produce + cl-MariaDB124 for a future 12.4, neither of which exists. + + Only meaningful once the repository has been confirmed to describe the installed + database: clmysql_process() inhibits the upgrade before reaching this point when + the two disagree. + + :returns: ``(module_name, stream)`` or ``(None, None)`` if the URL is not recognised + """ + match = CLMYSQL_REPO_URL_RE.search(baseurl or "") + if not match: + return None, None + family = match.group(1).lower() + # group(2)/group(3) are the raw digits, so "11.04" stays "1104" and not "114". + return family, "cl-{}{}{}".format(CLMYSQL_FAMILY_CAMEL[family], match.group(2), match.group(3)) + + +def resolve_clmysql_module_stream(clmysql_type, baseurl=None): """ Return (dnf_module_name, stream) for CloudLinux Governor MySQL/MariaDB/Percona types. - Prefer MODULE_STREAMS; if missing, derive stream from the type string (e.g. mariadb1012 -> - mariadb:cl-MariaDB1012) so newer CL releases work before this table is updated. + Prefer MODULE_STREAMS, which lists the streams we have confirmed exist. Otherwise + read the stream off the configured cl-mysql-meta repository, which spells the + version the same way the stream does. Only when no repository is available does + this fall back to deriving from the type token, which cannot recover a minor + version whose leading zero Governor dropped. + + :param clmysql_type: Governor DB type token, e.g. ``mariadb1104`` or ``mariadb114`` + :param baseurl: cl-mysql-meta baseurl, when one has been matched to the installed DB """ if not clmysql_type: return None, None + clmysql_type = canonical_clmysql_type(clmysql_type) + entry = MODULE_STREAMS.get(clmysql_type) if entry: mod_name, mod_stream = entry.split(":", 1) return mod_name, mod_stream + mod_name, mod_stream = clmysql_module_stream_from_url(baseurl) + if mod_name and mod_stream: + api.current_logger().debug( + "Derived DNF module {}:{} for CL-MySQL type '{}' from the configured repository." + .format(mod_name, mod_stream, clmysql_type) + ) + return mod_name, mod_stream + match = re.match(r"^(mariadb)(\d+)$", clmysql_type) if match: return "mariadb", "cl-MariaDB{}".format(match.group(2)) @@ -166,6 +293,9 @@ def get_clmysql_version_from_pkg(): else: return None + # The same concatenation Governor uses, so this agrees with mysql.type.installed + # by construction. It is lossy ("11.4.12" -> "mariadb114"), which is why callers + # compare parse_clmysql_type() triples rather than these strings. return "%s%s" % (name, "".join(version.split(".")[:2])) @@ -183,40 +313,6 @@ def get_pkg_prefix(clmysql_type): return None -def get_expected_repo_url_fragment(clmysql_type): - """ - Derive the expected cl-mysql-meta repo URL path fragment from the detected DB type. - - Governor writes cl-mysql.repo with a baseurl like:: - - http://repo.cloudlinux.com/other/cl$releasever/mysqlmeta/cl-mariadb-10.6/$basearch/ - - The path component (``cl-mariadb-10.6``) is determined by the DB type. This function - returns that expected fragment so callers can validate the repo URL. - - :param clmysql_type: type string like ``mariadb106``, ``mysql57``, ``percona56`` - :returns: expected path fragment like ``cl-mariadb-10.6``, or None if the type is - not recognised. - """ - if not clmysql_type: - return None - # Match family + version digits: mariadb106 -> ("mariadb", "106") - m = re.match(r"^(mariadb|mysql|percona)(\d+)$", clmysql_type) - if not m: - return None - family, digits = m.group(1), m.group(2) - # Split digits into major.minor. Governor concatenates major and minor: - # "57" -> "5.7", "80" -> "8.0" (2-digit: single-digit major) - # "106" -> "10.6", "100" -> "10.0" (3-digit: two-digit major) - # "1011" -> "10.11", "1104" -> "11.04" (4-digit: two-digit major) - # All known DB majors <= 2 digits; the split point is 1 for short, 2 otherwise. - if len(digits) <= 2: - major, minor = digits[:1], digits[1:] - else: - major, minor = digits[:2], digits[2:] - return "cl-{}-{}.{}".format(family, major, minor) - - def _get_clmysql_type_from_governor(): """ Read the actually installed DB type from the MySQL Governor cache file. @@ -226,6 +322,13 @@ def _get_clmysql_type_from_governor(): successful `--install`). We read the installed file so that a pending `--mysql-version` that was never followed by `--install` does not mislead Leapp. + The cached value is not necessarily the token the administrator passed to + `--mysql-version`: Governor re-derives it from the mysqld owner RPM + (`mysql_version()` in install/utilities.py), which loses a leading zero in the + minor version, so a MariaDB 11.4 system caches `mariadb114` rather than + `mariadb1104`. Both spellings are returned as-is; callers compare + parse_clmysql_type() triples, which treats them as the same version (CLOS-6809). + Returns a type string like `mariadb106`, or None when Governor is absent, the file is missing/empty, or the value is `auto`. """ @@ -254,12 +357,26 @@ def get_clmysql_type(): When both sources are available, cross-check them. On mismatch, return a result with :attr:`ClMysqlTypeStatus.MISMATCH` so the caller can raise an inhibitor. + The two sources are compared as parsed versions rather than as strings: Governor may + have cached ``mariadb114`` for the same installation whose canonical token is + ``mariadb1104``, and treating that spelling difference as a mismatch inhibited the + upgrade on every MariaDB 11.x system (CLOS-6809). + :returns: :class:`ClMysqlTypeResult` with status, resolved type, and raw detection values. """ governor_type = _get_clmysql_type_from_governor() pkg_type = get_clmysql_version_from_pkg() - if governor_type and pkg_type and governor_type != pkg_type: + # Fall back to comparing the raw strings when either side is unparseable, so two + # different unrecognised values are still reported rather than both becoming None. + governor_version = parse_clmysql_type(governor_type) + pkg_version = parse_clmysql_type(pkg_type) + if governor_version and pkg_version: + differ = governor_version != pkg_version + else: + differ = governor_type != pkg_type + + if governor_type and pkg_type and differ: api.current_logger().warning( "Governor mysql.type.installed says '{}' but RPM-based detection says '{}'." .format(governor_type, pkg_type) diff --git a/repos/system_upgrade/cloudlinux/libraries/tests/test_clmysql.py b/repos/system_upgrade/cloudlinux/libraries/tests/test_clmysql.py index ce3097f8d9..31e811d27c 100644 --- a/repos/system_upgrade/cloudlinux/libraries/tests/test_clmysql.py +++ b/repos/system_upgrade/cloudlinux/libraries/tests/test_clmysql.py @@ -3,17 +3,113 @@ import pytest from leapp.libraries.common.clmysql import ( + MODULE_STREAMS, ClMysqlTypeStatus, + canonical_clmysql_type, _clmysql_name_version_from_rpm, _get_clmysql_type_from_governor, _resolve_mysqld_path, get_clmysql_type, - get_expected_repo_url_fragment, + get_clmysql_version_from_pkg, + clmysql_module_stream_from_url, get_pkg_prefix, + parse_clmysql_repo_url, + parse_clmysql_type, resolve_clmysql_module_stream, ) +# --------------------------------------------------------------------------- +# normalize_clmysql_type +# --------------------------------------------------------------------------- + +class TestParseClmysqlType(object): + """ + CLOS-6809: Governor spells the same version two ways - it accepts and publishes + mariadb1104, but caches mariadb114 in mysql.type.installed after re-deriving the + token from the RPM version. Parsing to a numeric triple has to make them equal, + without a table of Governor's spellings to keep in sync. + """ + + @pytest.mark.parametrize( + "clmysql_type,expected", + [ + ("mysql56", ("mysql", 5, 6)), + ("mysql80", ("mysql", 8, 0)), + ("mysql84", ("mysql", 8, 4)), + ("mariadb102", ("mariadb", 10, 2)), + ("mariadb106", ("mariadb", 10, 6)), + ("mariadb1011", ("mariadb", 10, 11)), + ("mariadb1104", ("mariadb", 11, 4)), + ("mariadb1108", ("mariadb", 11, 8)), + ("percona56", ("percona", 5, 6)), + ], + ) + def test_canonical_tokens(self, clmysql_type, expected): + assert parse_clmysql_type(clmysql_type) == expected + + @pytest.mark.parametrize( + "lossy,canonical", + [ + ("mariadb114", "mariadb1104"), + ("mariadb118", "mariadb1108"), + ], + ) + def test_both_spellings_are_the_same_version(self, lossy, canonical): + assert parse_clmysql_type(lossy) == parse_clmysql_type(canonical) + + def test_future_series_needs_no_table_entry(self): + """The 11.x padding trap must not come back for a series nobody listed yet.""" + assert parse_clmysql_type("mariadb124") == parse_clmysql_type("mariadb1204") + assert parse_clmysql_type("mariadb124") == ("mariadb", 12, 4) + + @pytest.mark.parametrize( + "clmysql_type", + [None, "", "auto", "postgres15", "maria-db103"], + ) + def test_unrecognised(self, clmysql_type): + assert parse_clmysql_type(clmysql_type) is None + + +# --------------------------------------------------------------------------- +# parse_clmysql_repo_url +# --------------------------------------------------------------------------- + +class TestParseClmysqlRepoUrl(object): + + @pytest.mark.parametrize( + "baseurl,expected", + [ + ("http://repo.cloudlinux.com/other/cl$releasever/mysqlmeta/cl-mariadb-11.04/$basearch/", + ("mariadb", 11, 4)), + ("http://repo.cloudlinux.com/other/cl$releasever/mysqlmeta/cl-mariadb-11.08/$basearch/", + ("mariadb", 11, 8)), + ("http://repo.cloudlinux.com/other/cl$releasever/mysqlmeta/cl-mariadb-10.6/$basearch/", + ("mariadb", 10, 6)), + ("http://repo.cloudlinux.com/other/cl$releasever/mysqlmeta/cl-mariadb-10.11/$basearch/", + ("mariadb", 10, 11)), + ("http://repo.cloudlinux.com/other/cl$releasever/mysqlmeta/cl-mysql-8.0/$basearch/", + ("mysql", 8, 0)), + ("http://repo.cloudlinux.com/other/cl$releasever/mysqlmeta/cl-percona-5.6/$basearch/", + ("percona", 5, 6)), + ], + ) + def test_known_urls(self, baseurl, expected): + assert parse_clmysql_repo_url(baseurl) == expected + + def test_padding_is_irrelevant(self): + """11.04 and 11.4 in a URL describe the same version.""" + base = "http://repo.cloudlinux.com/other/cl8/mysqlmeta/cl-mariadb-{}/x86_64/" + assert parse_clmysql_repo_url(base.format("11.04")) == parse_clmysql_repo_url(base.format("11.4")) + + @pytest.mark.parametrize( + "baseurl", + [None, "", "http://repo.cloudlinux.com/other/cl8/mysqlmeta/mysqlclient/x86_64/"], + ) + def test_unrecognised(self, baseurl): + assert parse_clmysql_repo_url(baseurl) is None + + # --------------------------------------------------------------------------- # resolve_clmysql_module_stream # --------------------------------------------------------------------------- @@ -48,18 +144,102 @@ def test_known_streams(self, clmysql_type, expected): def test_fallback_derivation(self, clmysql_type, expected): assert resolve_clmysql_module_stream(clmysql_type) == expected + def test_repo_url_supplies_the_stream_for_an_unlisted_series(self): + """ + A series with no MODULE_STREAMS entry takes its stream from the repository + rather than from the token, which cannot spell a padded minor version. + """ + url = "http://repo.cloudlinux.com/other/cl8/mysqlmeta/cl-mariadb-12.04/$basearch/" + assert resolve_clmysql_module_stream("mariadb124", baseurl=url) == ("mariadb", "cl-MariaDB1204") + # Without the repository we can only fall back to the token, unpadded. + assert resolve_clmysql_module_stream("mariadb124") == ("mariadb", "cl-MariaDB124") + + def test_confirmed_entry_wins_over_the_repo_url(self): + """MODULE_STREAMS lists streams we verified, so it takes precedence.""" + url = "http://repo.cloudlinux.com/other/cl8/mysqlmeta/cl-mariadb-99.99/$basearch/" + assert resolve_clmysql_module_stream("mariadb1104", baseurl=url) == ("mariadb", "cl-MariaDB1104") + + def test_mariadb_11_8_gets_its_real_stream_name_from_the_repo(self): + """ + 11.08 is deliberately absent from MODULE_STREAMS (its module is unconfirmed), + but the repository still spells the name correctly: cl-MariaDB1108, not 118. + """ + url = "http://repo.cloudlinux.com/other/cl8/mysqlmeta/cl-mariadb-11.08/$basearch/" + assert resolve_clmysql_module_stream("mariadb118", baseurl=url) == ("mariadb", "cl-MariaDB1108") + + def test_confirmed_entry_is_reachable_from_the_cached_spelling(self): + """ + Governor caches "mariadb114"; the confirmed table entry is "mariadb1104". A caller + deciding whether the stream was verified has to canonicalise first, or it will + report a table-backed stream as though it had been guessed. + """ + assert canonical_clmysql_type("mariadb114") in MODULE_STREAMS + assert canonical_clmysql_type("mariadb124") not in MODULE_STREAMS + + def test_rpm_derived_token_resolves_to_confirmed_stream(self): + """ + CLOS-6809: cl-MariaDB1104 is the stream that exists in the target repo. + Deriving cl-MariaDB114 from the lossy token would enable a module that + the cl-mysql-meta repository does not carry. + """ + assert resolve_clmysql_module_stream("mariadb114") == ("mariadb", "cl-MariaDB1104") + + +# --------------------------------------------------------------------------- +# clmysql_module_stream_from_url +# --------------------------------------------------------------------------- + +class TestClmysqlModuleStreamFromUrl(object): + """ + The repository directory keeps the digits the module stream uses, padding and + all, so the stream can be read off it instead of guessed from the type token. + Checked against every (repo dir, stream) pair in governor-mysql 1.2-147. + """ + @pytest.mark.parametrize( - "clmysql_type", + "repo_dir,expected", [ - None, - "", - "postgres15", - "unknown", - "maria-db103", # hyphen breaks the pattern + ("mysql-5.5", ("mysql", "cl-MySQL55")), + ("mysql-5.6", ("mysql", "cl-MySQL56")), + ("mysql-5.7", ("mysql", "cl-MySQL57")), + ("mysql-8.0", ("mysql", "cl-MySQL80")), + ("mysql-8.4", ("mysql", "cl-MySQL84")), + ("mariadb-5.5", ("mariadb", "cl-MariaDB55")), + ("mariadb-10.0", ("mariadb", "cl-MariaDB100")), + ("mariadb-10.1", ("mariadb", "cl-MariaDB101")), + ("mariadb-10.2", ("mariadb", "cl-MariaDB102")), + ("mariadb-10.3", ("mariadb", "cl-MariaDB103")), + ("mariadb-10.4", ("mariadb", "cl-MariaDB104")), + ("mariadb-10.5", ("mariadb", "cl-MariaDB105")), + ("mariadb-10.6", ("mariadb", "cl-MariaDB106")), + ("mariadb-10.11", ("mariadb", "cl-MariaDB1011")), + ("mariadb-11.04", ("mariadb", "cl-MariaDB1104")), + ("mariadb-11.08", ("mariadb", "cl-MariaDB1108")), + ("percona-5.6", ("percona", "cl-Percona56")), ], ) - def test_unresolvable(self, clmysql_type): - assert resolve_clmysql_module_stream(clmysql_type) == (None, None) + def test_matches_governor_tables(self, repo_dir, expected): + url = "http://repo.cloudlinux.com/other/cl$releasever/mysqlmeta/cl-{}/$basearch/".format(repo_dir) + assert clmysql_module_stream_from_url(url) == expected + + @pytest.mark.parametrize( + "repo_dir,expected", + [ + ("mariadb-12.04", ("mariadb", "cl-MariaDB1204")), + ("mariadb-12.10", ("mariadb", "cl-MariaDB1210")), + ], + ) + def test_future_series_needs_no_table_entry(self, repo_dir, expected): + """A padded minor is exactly what the token-based guess got wrong.""" + url = "http://repo.cloudlinux.com/other/cl$releasever/mysqlmeta/cl-{}/$basearch/".format(repo_dir) + assert clmysql_module_stream_from_url(url) == expected + + @pytest.mark.parametrize( + "baseurl", + [None, "", "http://repo.cloudlinux.com/other/cl8/mysqlmeta/mysqlclient/x86_64/"], + ) + def test_unrecognised(self, baseurl): + assert clmysql_module_stream_from_url(baseurl) == (None, None) # --------------------------------------------------------------------------- @@ -181,6 +361,18 @@ def test_file_missing(self, monkeypatch): ) assert _get_clmysql_type_from_governor() is None + def test_lossy_governor_value_returned_verbatim(self, monkeypatch, tmpdir): + """ + CLOS-6809: this is what Governor actually writes on a MariaDB 11.4 system. + It is reported as-is; callers compare parsed versions, not spellings. + """ + f = tmpdir.join("mysql.type.installed") + f.write("mariadb114\n") + monkeypatch.setattr( + "leapp.libraries.common.clmysql.GOVERNOR_INSTALLED_TYPE_FILE", str(f) + ) + assert _get_clmysql_type_from_governor() == "mariadb114" + def test_file_empty(self, monkeypatch, tmpdir): f = tmpdir.join("mysql.type.installed") f.write("") @@ -199,37 +391,44 @@ def test_auto_value_ignored(self, monkeypatch, tmpdir): # --------------------------------------------------------------------------- -# get_expected_repo_url_fragment +# get_clmysql_version_from_pkg # --------------------------------------------------------------------------- -class TestGetExpectedRepoUrlFragment(object): - """Validate that the type-to-URL-fragment mapping matches Governor's REPO_NAMES.""" +class TestGetClmysqlVersionFromPkg(object): + """The token derived from the mysqld owner RPM, matching Governor's own derivation.""" + + def _patch_rpm(self, monkeypatch, name, version): + monkeypatch.setattr( + "leapp.libraries.common.clmysql._resolve_mysqld_path", + lambda: "/usr/sbin/mysqld", + ) + monkeypatch.setattr( + "leapp.libraries.common.clmysql._clmysql_name_version_from_rpm", + lambda _path: (name, version), + ) @pytest.mark.parametrize( - "clmysql_type,expected", + "name,version,expected", [ - ("mysql51", "cl-mysql-5.1"), - ("mysql55", "cl-mysql-5.5"), - ("mysql57", "cl-mysql-5.7"), - ("mysql80", "cl-mysql-8.0"), - ("mysql84", "cl-mysql-8.4"), - ("mariadb55", "cl-mariadb-5.5"), - ("mariadb100", "cl-mariadb-10.0"), - ("mariadb106", "cl-mariadb-10.6"), - ("mariadb1011", "cl-mariadb-10.11"), - ("mariadb1104", "cl-mariadb-11.04"), - ("percona56", "cl-percona-5.6"), + # CLOS-6809: the reporter's exact package version. Governor derives the + # same lossy spelling, so the two sources agree by construction. + ("cl-mariadb1104-server", "11.4.12", "mariadb114"), + ("cl-mariadb1108-server", "11.8.3", "mariadb118"), + ("cl-mariadb1011-server", "10.11.6", "mariadb1011"), + ("cl-mariadb106-server", "10.6.16", "mariadb106"), + ("cl-mysql80-server", "8.0.35", "mysql80"), + ("cl-percona56-server", "5.6.51", "percona56"), ], ) - def test_known_types(self, clmysql_type, expected): - assert get_expected_repo_url_fragment(clmysql_type) == expected + def test_canonical_token(self, monkeypatch, name, version, expected): + self._patch_rpm(monkeypatch, name, version) + assert get_clmysql_version_from_pkg() == expected - @pytest.mark.parametrize( - "clmysql_type", - [None, "", "unknown", "postgres15"], - ) - def test_unrecognised(self, clmysql_type): - assert get_expected_repo_url_fragment(clmysql_type) is None + def test_no_mysqld(self, monkeypatch): + monkeypatch.setattr( + "leapp.libraries.common.clmysql._resolve_mysqld_path", lambda: None + ) + assert get_clmysql_version_from_pkg() is None # --------------------------------------------------------------------------- @@ -299,6 +498,60 @@ def test_mismatch(self, monkeypatch): assert result.governor_type == "mariadb1011" assert result.pkg_type == "mariadb106" + def test_mariadb114_untouched_governor_file_is_not_a_mismatch(self, monkeypatch, tmpdir): + """ + CLOS-6809 inhibitor #2: on an untouched MariaDB 11.4 system Governor writes + 'mariadb114' and the mysqld owner RPM is 11.4.12. Both sides describe the + same installation, so no mismatch may be reported. + """ + f = tmpdir.join("mysql.type.installed") + f.write("mariadb114") + monkeypatch.setattr( + "leapp.libraries.common.clmysql.GOVERNOR_INSTALLED_TYPE_FILE", str(f) + ) + monkeypatch.setattr( + "leapp.libraries.common.clmysql._resolve_mysqld_path", + lambda: "/usr/sbin/mysqld", + ) + monkeypatch.setattr( + "leapp.libraries.common.clmysql._clmysql_name_version_from_rpm", + lambda _path: ("cl-mariadb1104-server", "11.4.12"), + ) + result = get_clmysql_type() + assert result.status == ClMysqlTypeStatus.OK + assert result.governor_type == "mariadb114" + assert result.pkg_type == "mariadb114" + + def test_hand_corrected_governor_file_is_not_a_mismatch(self, monkeypatch, tmpdir): + """ + CLOS-6809: KCS 27757797511068 tells customers to write the canonical token + into mysql.type.installed by hand. That must not turn into a mismatch + against the RPM-derived token either. + """ + f = tmpdir.join("mysql.type.installed") + f.write("mariadb1104") + monkeypatch.setattr( + "leapp.libraries.common.clmysql.GOVERNOR_INSTALLED_TYPE_FILE", str(f) + ) + monkeypatch.setattr( + "leapp.libraries.common.clmysql._resolve_mysqld_path", + lambda: "/usr/sbin/mysqld", + ) + monkeypatch.setattr( + "leapp.libraries.common.clmysql._clmysql_name_version_from_rpm", + lambda _path: ("cl-mariadb1104-server", "11.4.12"), + ) + result = get_clmysql_type() + assert result.status == ClMysqlTypeStatus.OK + assert result.governor_type == "mariadb1104" + assert result.pkg_type == "mariadb114" + + def test_real_mismatch_still_detected(self, monkeypatch): + """Normalization must not paper over a genuine version disagreement.""" + self._patch_sources(monkeypatch, "mariadb1104", "mariadb1011") + result = get_clmysql_type() + assert result.status == ClMysqlTypeStatus.MISMATCH + def test_governor_only(self, monkeypatch): """Governor file present but mysqld not found (no RPM detection).""" self._patch_sources(monkeypatch, "percona56", None)