From a550f9a5ac0843f9f7277426eb7b7ceafc3fb929 Mon Sep 17 00:00:00 2001 From: Blake Li Date: Mon, 24 Aug 2026 21:51:39 +0000 Subject: [PATCH 1/2] chore: migrate xmllint to Python xml.etree.ElementTree in CI scripts and workflows Migrate xmllint (libxml2-utils) usages across Kokoro scripts and generation tools to Python 3's built-in xml.etree.ElementTree. This eliminates the need for installing libxml2-utils in GitHub Actions workflows, resolving CI timeouts caused by transient mirror stalls (fixes #14132). --- ...shared-config-downstream-dependencies.yaml | 11 --- ...hared-config-downstream-maven-plugins.yaml | 30 ------ .../sdk-platform-java-downstream.yaml | 20 ---- .github/workflows/versions.yaml | 10 -- .kokoro/client-library-check-doclet.sh | 73 +++++++++----- .kokoro/client-library-check.sh | 73 +++++++++----- .kokoro/common.sh | 56 +++++++++-- .kokoro/common_test.sh | 19 ++-- .kokoro/presubmit/downstream-build.sh | 27 ++++-- generation/check_existing_release_versions.sh | 20 ++-- sdk-platform-java/.kokoro/presubmit/common.sh | 56 +++++++++-- .../scripts/create_native_image_test_env.sh | 96 +++++++++++++------ 12 files changed, 301 insertions(+), 190 deletions(-) diff --git a/.github/workflows/java-shared-config-downstream-dependencies.yaml b/.github/workflows/java-shared-config-downstream-dependencies.yaml index 66cc75201fcb..689eaadea62a 100644 --- a/.github/workflows/java-shared-config-downstream-dependencies.yaml +++ b/.github/workflows/java-shared-config-downstream-dependencies.yaml @@ -48,16 +48,6 @@ jobs: java-version: ${{matrix.java}} cache: maven - run: java -version - - name: Install xmllint - timeout-minutes: 5 - run: | - # Retry apt-get update and install up to 3 times with a 15-second per-request timeout. - # Retries protect against transient network glitches, mirror timeouts, and apt lock contention on runner startup. - for i in {1..3}; do - sudo apt-get -o Acquire::http::Timeout="15" -o Acquire::Retries=3 update && \ - sudo apt-get -o Acquire::http::Timeout="15" -o Acquire::Retries=3 -y install libxml2-utils && \ - break || sleep 5 - done - name: Install java-shared-dependencies and common dependencies run: | .kokoro/build.sh @@ -81,4 +71,3 @@ jobs: fi mvn install -pl ${TARGET_PATH} -am -DskipTests=true -Dmaven.javadoc.skip=true -Dgcloud.download.skip=true -B -V -q - run: .kokoro/client-library-check.sh ${{matrix.repo}} dependencies - diff --git a/.github/workflows/java-shared-config-downstream-maven-plugins.yaml b/.github/workflows/java-shared-config-downstream-maven-plugins.yaml index c02274b973f0..fc1b83a06907 100644 --- a/.github/workflows/java-shared-config-downstream-maven-plugins.yaml +++ b/.github/workflows/java-shared-config-downstream-maven-plugins.yaml @@ -69,16 +69,6 @@ jobs: java-version: ${{matrix.java}} cache: maven - run: java -version - - name: Install xmllint - timeout-minutes: 5 - run: | - # Retry apt-get update and install up to 3 times with a 15-second per-request timeout. - # Retries protect against transient network glitches, mirror timeouts, and apt lock contention on runner startup. - for i in {1..3}; do - sudo apt-get -o Acquire::http::Timeout="15" -o Acquire::Retries=3 update && \ - sudo apt-get -o Acquire::http::Timeout="15" -o Acquire::Retries=3 -y install libxml2-utils && \ - break || sleep 5 - done - run: .kokoro/client-library-check.sh ${{matrix.repo}} ${{matrix.job-type}} lint: needs: filter @@ -104,16 +94,6 @@ jobs: java-version: ${{matrix.java}} cache: maven - run: java -version - - name: Install xmllint - timeout-minutes: 5 - run: | - # Retry apt-get update and install up to 3 times with a 15-second per-request timeout. - # Retries protect against transient network glitches, mirror timeouts, and apt lock contention on runner startup. - for i in {1..3}; do - sudo apt-get -o Acquire::http::Timeout="15" -o Acquire::Retries=3 update && \ - sudo apt-get -o Acquire::http::Timeout="15" -o Acquire::Retries=3 -y install libxml2-utils && \ - break || sleep 5 - done - name: Install java-shared-dependencies and common dependencies run: | .kokoro/build.sh @@ -167,16 +147,6 @@ jobs: java-version: 17 cache: maven - run: java -version - - name: Install xmllint - timeout-minutes: 5 - run: | - # Retry apt-get update and install up to 3 times with a 15-second per-request timeout. - # Retries protect against transient network glitches, mirror timeouts, and apt lock contention on runner startup. - for i in {1..3}; do - sudo apt-get -o Acquire::http::Timeout="15" -o Acquire::Retries=3 update && \ - sudo apt-get -o Acquire::http::Timeout="15" -o Acquire::Retries=3 -y install libxml2-utils && \ - break || sleep 5 - done - name: Install java-shared-dependencies and common dependencies run: | .kokoro/build.sh diff --git a/.github/workflows/sdk-platform-java-downstream.yaml b/.github/workflows/sdk-platform-java-downstream.yaml index 29daf10927b5..d187bfdbb2aa 100644 --- a/.github/workflows/sdk-platform-java-downstream.yaml +++ b/.github/workflows/sdk-platform-java-downstream.yaml @@ -51,16 +51,6 @@ jobs: distribution: temurin cache: maven - run: mvn -version - - name: Install xmllint - timeout-minutes: 5 - run: | - # Retry apt-get update and install up to 3 times with a 15-second per-request timeout. - # Retries protect against transient network glitches, mirror timeouts, and apt lock contention on runner startup. - for i in {1..3}; do - sudo apt-get -o Acquire::http::Timeout="15" -o Acquire::Retries=3 update && \ - sudo apt-get -o Acquire::http::Timeout="15" -o Acquire::Retries=3 -y install libxml2-utils && \ - break || sleep 5 - done - name: Test helper scripts run: .kokoro/common_test.sh - name: Perform downstream compatibility testing @@ -82,16 +72,6 @@ jobs: distribution: temurin cache: maven - run: mvn -version - - name: Install xmllint - timeout-minutes: 5 - run: | - # Retry apt-get update and install up to 3 times with a 15-second per-request timeout. - # Retries protect against transient network glitches, mirror timeouts, and apt lock contention on runner startup. - for i in {1..3}; do - sudo apt-get -o Acquire::http::Timeout="15" -o Acquire::Retries=3 update && \ - sudo apt-get -o Acquire::http::Timeout="15" -o Acquire::Retries=3 -y install libxml2-utils && \ - break || sleep 5 - done - name: Perform downstream compatibility testing run: .kokoro/downstream-compatibility-spring.sh required: diff --git a/.github/workflows/versions.yaml b/.github/workflows/versions.yaml index d249ed7956ce..75da02ca5064 100644 --- a/.github/workflows/versions.yaml +++ b/.github/workflows/versions.yaml @@ -42,16 +42,6 @@ jobs: github.repository_owner == 'googleapis' && github.head_ref == 'release-please--branches--main' && !endsWith(github.event.pull_request.title, 'SNAPSHOT') steps: - - name: Install xmllint - timeout-minutes: 5 - run: | - # Retry apt-get update and install up to 3 times with a 15-second per-request timeout. - # Retries protect against transient network glitches, mirror timeouts, and apt lock contention on runner startup. - for i in {1..3}; do - sudo apt-get -o Acquire::http::Timeout="15" -o Acquire::Retries=3 update && \ - sudo apt-get -o Acquire::http::Timeout="15" -o Acquire::Retries=3 -y install libxml2-utils && \ - break || sleep 5 - done - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0 with: persist-credentials: false diff --git a/.kokoro/client-library-check-doclet.sh b/.kokoro/client-library-check-doclet.sh index 3c774c62582f..45a28927f3ba 100755 --- a/.kokoro/client-library-check-doclet.sh +++ b/.kokoro/client-library-check-doclet.sh @@ -38,36 +38,54 @@ function get_released_version_from_versions_txt() { function replace_java_shared_config_version() { version=$1 # replace version - xmllint --shell <(cat pom.xml) << EOF - setns x=http://maven.apache.org/POM/4.0.0 - cd .//x:artifactId[text()="google-cloud-shared-config"] - cd ../x:version - set ${version} - save pom.xml -EOF + python3 -c " +import xml.etree.ElementTree as ET, sys +file = 'pom.xml' +ET.register_namespace('', 'http://maven.apache.org/POM/4.0.0') +tree = ET.parse(file) +root = tree.getroot() +for elem in root.iter(): + art = elem.find('{*}artifactId') + ver = elem.find('{*}version') + if art is not None and art.text == 'google-cloud-shared-config' and ver is not None: + ver.text = sys.argv[1] +tree.write(file, encoding='utf-8', xml_declaration=True) +" "${version}" } function replace_java_shared_dependencies_version() { version=$1 # replace version - xmllint --shell <(cat pom.xml) << EOF - setns x=http://maven.apache.org/POM/4.0.0 - cd .//x:properties/x:google-cloud-shared-dependencies.version - set ${version} - save pom.xml -EOF + python3 -c " +import xml.etree.ElementTree as ET, sys +file = 'pom.xml' +ET.register_namespace('', 'http://maven.apache.org/POM/4.0.0') +tree = ET.parse(file) +root = tree.getroot() +for prop in root.findall('.//{*}properties'): + target = prop.find('{*}google-cloud-shared-dependencies.version') + if target is not None: + target.text = sys.argv[1] +tree.write(file, encoding='utf-8', xml_declaration=True) +" "${version}" } function replace_sdk_platform_java_config_version() { version=$1 # replace version in the shared parent POM - xmllint --shell <(cat ../google-cloud-pom-parent/pom.xml) << EOF - setns x=http://maven.apache.org/POM/4.0.0 - cd .//x:artifactId[text()="sdk-platform-java-config"] - cd ../x:version - set ${version} - save ../google-cloud-pom-parent/pom.xml -EOF + python3 -c " +import xml.etree.ElementTree as ET, sys +file = '../google-cloud-pom-parent/pom.xml' +ET.register_namespace('', 'http://maven.apache.org/POM/4.0.0') +tree = ET.parse(file) +root = tree.getroot() +for elem in root.iter(): + art = elem.find('{*}artifactId') + ver = elem.find('{*}version') + if art is not None and art.text == 'sdk-platform-java-config' and ver is not None: + ver.text = sys.argv[1] +tree.write(file, encoding='utf-8', xml_declaration=True) +" "${version}" } REPO=$1 # Get the directory of the build script @@ -94,8 +112,12 @@ echo "This is the doclet path: ${docletPath}" # Read the current version of this BOM in the POM. Example version: '0.116.1-alpha-SNAPSHOT' VERSION_POM=java-shared-config/java-shared-config/pom.xml -# Namespace (xmlns) prevents xmllint from specifying tag names in XPath -JAVA_SHARED_CONFIG_VERSION=`sed -e 's/xmlns=".*"//' ${VERSION_POM} | xmllint --xpath '/project/version/text()' -` +JAVA_SHARED_CONFIG_VERSION=$(python3 -c " +import xml.etree.ElementTree as ET +root = ET.parse('${VERSION_POM}').getroot() +v = root.find('{*}version') or root.find('{*}parent/{*}version') +print(v.text if v is not None and v.text else '') +") if [ -z "${JAVA_SHARED_CONFIG_VERSION}" ]; then echo "Version is not found in ${VERSION_POM}" @@ -110,7 +132,12 @@ LATEST_TAG=$(git ls-remote --tags https://github.com/googleapis/google-cloud-jav echo "Cloning google-cloud-java at tag: ${LATEST_TAG}" git clone "https://github.com/googleapis/google-cloud-java.git" -b "${LATEST_TAG}" --depth=1 pushd google-cloud-java/sdk-platform-java -SDK_PLATFORM_JAVA_CONFIG_VERSION=$(sed -e 's/xmlns=".*"//' sdk-platform-java-config/pom.xml | xmllint --xpath '/project/version/text()' -) +SDK_PLATFORM_JAVA_CONFIG_VERSION=$(python3 -c " +import xml.etree.ElementTree as ET +root = ET.parse('sdk-platform-java-config/pom.xml').getroot() +v = root.find('{*}version') or root.find('{*}parent/{*}version') +print(v.text if v is not None and v.text else '') +") pushd sdk-platform-java-config # Use released version of google-cloud-shared-dependencies to avoid verifying SNAPSHOT changes. diff --git a/.kokoro/client-library-check.sh b/.kokoro/client-library-check.sh index bfa12690ede1..c523917f7206 100755 --- a/.kokoro/client-library-check.sh +++ b/.kokoro/client-library-check.sh @@ -38,36 +38,54 @@ function get_released_version_from_versions_txt() { function replace_java_shared_config_version() { version=$1 # replace version - xmllint --shell <(cat pom.xml) << EOF - setns x=http://maven.apache.org/POM/4.0.0 - cd .//x:artifactId[text()="google-cloud-shared-config"] - cd ../x:version - set ${version} - save pom.xml -EOF + python3 -c " +import xml.etree.ElementTree as ET, sys +file = 'pom.xml' +ET.register_namespace('', 'http://maven.apache.org/POM/4.0.0') +tree = ET.parse(file) +root = tree.getroot() +for elem in root.iter(): + art = elem.find('{*}artifactId') + ver = elem.find('{*}version') + if art is not None and art.text == 'google-cloud-shared-config' and ver is not None: + ver.text = sys.argv[1] +tree.write(file, encoding='utf-8', xml_declaration=True) +" "${version}" } function replace_java_shared_dependencies_version() { version=$1 # replace version - xmllint --shell <(cat pom.xml) << EOF - setns x=http://maven.apache.org/POM/4.0.0 - cd .//x:properties/x:google-cloud-shared-dependencies.version - set ${version} - save pom.xml -EOF + python3 -c " +import xml.etree.ElementTree as ET, sys +file = 'pom.xml' +ET.register_namespace('', 'http://maven.apache.org/POM/4.0.0') +tree = ET.parse(file) +root = tree.getroot() +for prop in root.findall('.//{*}properties'): + target = prop.find('{*}google-cloud-shared-dependencies.version') + if target is not None: + target.text = sys.argv[1] +tree.write(file, encoding='utf-8', xml_declaration=True) +" "${version}" } function replace_sdk_platform_java_config_version() { version=$1 # replace version in the shared parent POM - xmllint --shell <(cat ../google-cloud-pom-parent/pom.xml) << EOF - setns x=http://maven.apache.org/POM/4.0.0 - cd .//x:artifactId[text()="sdk-platform-java-config"] - cd ../x:version - set ${version} - save ../google-cloud-pom-parent/pom.xml -EOF + python3 -c " +import xml.etree.ElementTree as ET, sys +file = '../google-cloud-pom-parent/pom.xml' +ET.register_namespace('', 'http://maven.apache.org/POM/4.0.0') +tree = ET.parse(file) +root = tree.getroot() +for elem in root.iter(): + art = elem.find('{*}artifactId') + ver = elem.find('{*}version') + if art is not None and art.text == 'sdk-platform-java-config' and ver is not None: + ver.text = sys.argv[1] +tree.write(file, encoding='utf-8', xml_declaration=True) +" "${version}" } if [[ $# -ne 2 ]]; @@ -93,8 +111,12 @@ popd # Read the current version of this BOM in the POM. Example version: '0.116.1-alpha-SNAPSHOT' VERSION_POM=java-shared-config/java-shared-config/pom.xml -# Namespace (xmlns) prevents xmllint from specifying tag names in XPath -JAVA_SHARED_CONFIG_VERSION=`sed -e 's/xmlns=".*"//' ${VERSION_POM} | xmllint --xpath '/project/version/text()' -` +JAVA_SHARED_CONFIG_VERSION=$(python3 -c " +import xml.etree.ElementTree as ET +root = ET.parse('${VERSION_POM}').getroot() +v = root.find('{*}version') or root.find('{*}parent/{*}version') +print(v.text if v is not None and v.text else '') +") if [ -z "${JAVA_SHARED_CONFIG_VERSION}" ]; then echo "Version is not found in ${VERSION_POM}" @@ -109,7 +131,12 @@ LATEST_TAG=$(git ls-remote --tags https://github.com/googleapis/google-cloud-jav echo "Cloning google-cloud-java at tag: ${LATEST_TAG}" git clone "https://github.com/googleapis/google-cloud-java.git" -b "${LATEST_TAG}" --depth=1 pushd google-cloud-java/sdk-platform-java -SDK_PLATFORM_JAVA_CONFIG_VERSION=$(sed -e 's/xmlns=".*"//' sdk-platform-java-config/pom.xml | xmllint --xpath '/project/version/text()' -) +SDK_PLATFORM_JAVA_CONFIG_VERSION=$(python3 -c " +import xml.etree.ElementTree as ET +root = ET.parse('sdk-platform-java-config/pom.xml').getroot() +v = root.find('{*}version') or root.find('{*}parent/{*}version') +print(v.text if v is not None and v.text else '') +") pushd sdk-platform-java-config # Use released version of google-cloud-shared-dependencies to avoid verifying SNAPSHOT changes. diff --git a/.kokoro/common.sh b/.kokoro/common.sh index 9a56b378131a..51f2ca5a768c 100644 --- a/.kokoro/common.sh +++ b/.kokoro/common.sh @@ -492,21 +492,45 @@ function install_modules() { # ex: update_dependency google-cloud-java/google-cloud-jar-parent google-cloud-shared-dependencies 1.2.3 function update_pom_dependency { pushd "$1" || exit 1 - xmllint --shell pom.xml &>/dev/null </dev/null; then + if python3 -c " +import xml.etree.ElementTree as ET, sys +try: + root = ET.parse(sys.argv[1]).getroot() + found = False + for elem in root.iter(): + art = elem.find('{*}artifactId') + ver = elem.find('{*}version') + if art is not None and art.text == sys.argv[2] and ver is not None and ver.text: + found = True + break + sys.exit(0 if found else 1) +except Exception: + sys.exit(1) +" "$pom" "$1" &>/dev/null; then found+=("$pom") fi done @@ -531,8 +555,20 @@ function update_all_poms_dependency { # Parse the version of the pom.xml file in the given directory ($1) # ex: VERSION=$(parse_pom_version java-shared-dependencies) function parse_pom_version { - # Namespace (xmlns) prevents xmllint from specifying tag names in XPath - result=$(sed -e 's/xmlns=".*"//' "$1/pom.xml" | xmllint --xpath '/project/version/text()' -) + result=$(python3 -c " +import xml.etree.ElementTree as ET, sys +try: + root = ET.parse('$1/pom.xml').getroot() + v = root.find('{*}version') + if v is None: + v = root.find('{*}parent/{*}version') + if v is not None and v.text: + print(v.text.strip()) + else: + sys.exit(1) +except Exception: + sys.exit(1) +") if [ -z "${result}" ]; then echo "Version is not found in $1" diff --git a/.kokoro/common_test.sh b/.kokoro/common_test.sh index e78e4d36131b..77805f77dd60 100755 --- a/.kokoro/common_test.sh +++ b/.kokoro/common_test.sh @@ -49,17 +49,20 @@ function test_update_pom_dependency { update_pom_dependency . truth "99.88.77" - xmllint --shell pom.xml &>/dev/null </dev/null; then + version=$(python3 -c " +import xml.etree.ElementTree as ET +root = ET.parse('pom.xml').getroot() +for elem in root.iter(): + art = elem.find('{*}artifactId') + ver = elem.find('{*}version') + if art is not None and art.text == 'truth' and ver is not None: + print(ver.text) + break +") + if [ "$version" != "99.88.77" ]; then echo "update_pom_dependency failed to change version to expected value." exit 1 fi - rm found-version.txt popd } diff --git a/.kokoro/presubmit/downstream-build.sh b/.kokoro/presubmit/downstream-build.sh index aa6781b1ffee..f459b5a79dfa 100755 --- a/.kokoro/presubmit/downstream-build.sh +++ b/.kokoro/presubmit/downstream-build.sh @@ -17,13 +17,19 @@ set -eo pipefail set -x function modify_shared_config() { - xmllint --shell pom.xml </dev/null </dev/null; then + if python3 -c " +import xml.etree.ElementTree as ET, sys +try: + root = ET.parse(sys.argv[1]).getroot() + found = False + for elem in root.iter(): + art = elem.find('{*}artifactId') + ver = elem.find('{*}version') + if art is not None and art.text == sys.argv[2] and ver is not None and ver.text: + found = True + break + sys.exit(0 if found else 1) +except Exception: + sys.exit(1) +" "$pom" "$1" &>/dev/null; then found+=("$pom") fi done @@ -59,8 +83,20 @@ function update_all_poms_dependency { # Parse the version of the pom.xml file in the given directory ($1) # ex: VERSION=$(parse_pom_version java-shared-dependencies) function parse_pom_version { - # Namespace (xmlns) prevents xmllint from specifying tag names in XPath - result=$(sed -e 's/xmlns=".*"//' "$1/pom.xml" | xmllint --xpath '/project/version/text()' -) + result=$(python3 -c " +import xml.etree.ElementTree as ET, sys +try: + root = ET.parse('$1/pom.xml').getroot() + v = root.find('{*}version') + if v is None: + v = root.find('{*}parent/{*}version') + if v is not None and v.text: + print(v.text.strip()) + else: + sys.exit(1) +except Exception: + sys.exit(1) +") if [ -z "${result}" ]; then echo "Version is not found in $1" diff --git a/sdk-platform-java/scripts/create_native_image_test_env.sh b/sdk-platform-java/scripts/create_native_image_test_env.sh index 4e3689ec30fe..5fdf2e74ee12 100644 --- a/sdk-platform-java/scripts/create_native_image_test_env.sh +++ b/sdk-platform-java/scripts/create_native_image_test_env.sh @@ -16,23 +16,35 @@ set -eo pipefail function modify_shared_config() { - xmllint --shell pom.xml < Date: Mon, 24 Aug 2026 21:58:14 +0000 Subject: [PATCH 2/2] refactor: extract Maven POM XML operations to standalone pom_utils.py script Extract XML parsing and manipulation logic into standalone pom_utils.py scripts with dedicated CLI commands (get-version, has-versioned-dep, get-dep-version, update-dep-version, update-property, get-coordinates) for cleaner readability and reusability. --- .kokoro/client-library-check-doclet.sh | 66 +---- .kokoro/client-library-check.sh | 66 +---- .kokoro/common.sh | 52 +--- .kokoro/common_test.sh | 11 +- .kokoro/pom_utils.py | 238 ++++++++++++++++++ .kokoro/presubmit/downstream-build.sh | 28 +-- generation/check_existing_release_versions.sh | 15 +- sdk-platform-java/.kokoro/presubmit/common.sh | 52 +--- .../.kokoro/presubmit/pom_utils.py | 238 ++++++++++++++++++ .../scripts/create_native_image_test_env.sh | 75 +----- 10 files changed, 531 insertions(+), 310 deletions(-) create mode 100644 .kokoro/pom_utils.py create mode 100644 sdk-platform-java/.kokoro/presubmit/pom_utils.py diff --git a/.kokoro/client-library-check-doclet.sh b/.kokoro/client-library-check-doclet.sh index 45a28927f3ba..6f033999a19f 100755 --- a/.kokoro/client-library-check-doclet.sh +++ b/.kokoro/client-library-check-doclet.sh @@ -21,6 +21,10 @@ set -eo pipefail # Display commands being run. set -x +## Get the directory of the build script +scriptDir=$(realpath "$(dirname "${BASH_SOURCE[0]}")") +POM_UTILS="${scriptDir}/pom_utils.py" + function get_current_version_from_versions_txt() { versions=$1 key=$2 @@ -37,61 +41,21 @@ function get_released_version_from_versions_txt() { function replace_java_shared_config_version() { version=$1 - # replace version - python3 -c " -import xml.etree.ElementTree as ET, sys -file = 'pom.xml' -ET.register_namespace('', 'http://maven.apache.org/POM/4.0.0') -tree = ET.parse(file) -root = tree.getroot() -for elem in root.iter(): - art = elem.find('{*}artifactId') - ver = elem.find('{*}version') - if art is not None and art.text == 'google-cloud-shared-config' and ver is not None: - ver.text = sys.argv[1] -tree.write(file, encoding='utf-8', xml_declaration=True) -" "${version}" + python3 "${POM_UTILS}" update-dep-version pom.xml google-cloud-shared-config "${version}" } function replace_java_shared_dependencies_version() { version=$1 - # replace version - python3 -c " -import xml.etree.ElementTree as ET, sys -file = 'pom.xml' -ET.register_namespace('', 'http://maven.apache.org/POM/4.0.0') -tree = ET.parse(file) -root = tree.getroot() -for prop in root.findall('.//{*}properties'): - target = prop.find('{*}google-cloud-shared-dependencies.version') - if target is not None: - target.text = sys.argv[1] -tree.write(file, encoding='utf-8', xml_declaration=True) -" "${version}" + python3 "${POM_UTILS}" update-property pom.xml google-cloud-shared-dependencies.version "${version}" } function replace_sdk_platform_java_config_version() { version=$1 - # replace version in the shared parent POM - python3 -c " -import xml.etree.ElementTree as ET, sys -file = '../google-cloud-pom-parent/pom.xml' -ET.register_namespace('', 'http://maven.apache.org/POM/4.0.0') -tree = ET.parse(file) -root = tree.getroot() -for elem in root.iter(): - art = elem.find('{*}artifactId') - ver = elem.find('{*}version') - if art is not None and art.text == 'sdk-platform-java-config' and ver is not None: - ver.text = sys.argv[1] -tree.write(file, encoding='utf-8', xml_declaration=True) -" "${version}" + python3 "${POM_UTILS}" update-dep-version ../google-cloud-pom-parent/pom.xml sdk-platform-java-config "${version}" } REPO=$1 -# Get the directory of the build script -scriptDir=$(realpath $(dirname "${BASH_SOURCE[0]}")) ## cd to the parent directory, i.e. the root of the git repo -cd ${scriptDir}/.. +cd "${scriptDir}/.." # Make artifacts available for 'mvn validate' at the bottom pushd java-shared-config @@ -112,12 +76,7 @@ echo "This is the doclet path: ${docletPath}" # Read the current version of this BOM in the POM. Example version: '0.116.1-alpha-SNAPSHOT' VERSION_POM=java-shared-config/java-shared-config/pom.xml -JAVA_SHARED_CONFIG_VERSION=$(python3 -c " -import xml.etree.ElementTree as ET -root = ET.parse('${VERSION_POM}').getroot() -v = root.find('{*}version') or root.find('{*}parent/{*}version') -print(v.text if v is not None and v.text else '') -") +JAVA_SHARED_CONFIG_VERSION=$(python3 "${POM_UTILS}" get-version "${VERSION_POM}") if [ -z "${JAVA_SHARED_CONFIG_VERSION}" ]; then echo "Version is not found in ${VERSION_POM}" @@ -132,12 +91,7 @@ LATEST_TAG=$(git ls-remote --tags https://github.com/googleapis/google-cloud-jav echo "Cloning google-cloud-java at tag: ${LATEST_TAG}" git clone "https://github.com/googleapis/google-cloud-java.git" -b "${LATEST_TAG}" --depth=1 pushd google-cloud-java/sdk-platform-java -SDK_PLATFORM_JAVA_CONFIG_VERSION=$(python3 -c " -import xml.etree.ElementTree as ET -root = ET.parse('sdk-platform-java-config/pom.xml').getroot() -v = root.find('{*}version') or root.find('{*}parent/{*}version') -print(v.text if v is not None and v.text else '') -") +SDK_PLATFORM_JAVA_CONFIG_VERSION=$(python3 "${POM_UTILS}" get-version sdk-platform-java-config/pom.xml) pushd sdk-platform-java-config # Use released version of google-cloud-shared-dependencies to avoid verifying SNAPSHOT changes. diff --git a/.kokoro/client-library-check.sh b/.kokoro/client-library-check.sh index c523917f7206..5d7a70c62934 100755 --- a/.kokoro/client-library-check.sh +++ b/.kokoro/client-library-check.sh @@ -21,6 +21,10 @@ set -eo pipefail # Display commands being run. set -x +## Get the directory of the build script +scriptDir=$(realpath "$(dirname "${BASH_SOURCE[0]}")") +POM_UTILS="${scriptDir}/pom_utils.py" + function get_current_version_from_versions_txt() { versions=$1 key=$2 @@ -37,55 +41,17 @@ function get_released_version_from_versions_txt() { function replace_java_shared_config_version() { version=$1 - # replace version - python3 -c " -import xml.etree.ElementTree as ET, sys -file = 'pom.xml' -ET.register_namespace('', 'http://maven.apache.org/POM/4.0.0') -tree = ET.parse(file) -root = tree.getroot() -for elem in root.iter(): - art = elem.find('{*}artifactId') - ver = elem.find('{*}version') - if art is not None and art.text == 'google-cloud-shared-config' and ver is not None: - ver.text = sys.argv[1] -tree.write(file, encoding='utf-8', xml_declaration=True) -" "${version}" + python3 "${POM_UTILS}" update-dep-version pom.xml google-cloud-shared-config "${version}" } function replace_java_shared_dependencies_version() { version=$1 - # replace version - python3 -c " -import xml.etree.ElementTree as ET, sys -file = 'pom.xml' -ET.register_namespace('', 'http://maven.apache.org/POM/4.0.0') -tree = ET.parse(file) -root = tree.getroot() -for prop in root.findall('.//{*}properties'): - target = prop.find('{*}google-cloud-shared-dependencies.version') - if target is not None: - target.text = sys.argv[1] -tree.write(file, encoding='utf-8', xml_declaration=True) -" "${version}" + python3 "${POM_UTILS}" update-property pom.xml google-cloud-shared-dependencies.version "${version}" } function replace_sdk_platform_java_config_version() { version=$1 - # replace version in the shared parent POM - python3 -c " -import xml.etree.ElementTree as ET, sys -file = '../google-cloud-pom-parent/pom.xml' -ET.register_namespace('', 'http://maven.apache.org/POM/4.0.0') -tree = ET.parse(file) -root = tree.getroot() -for elem in root.iter(): - art = elem.find('{*}artifactId') - ver = elem.find('{*}version') - if art is not None and art.text == 'sdk-platform-java-config' and ver is not None: - ver.text = sys.argv[1] -tree.write(file, encoding='utf-8', xml_declaration=True) -" "${version}" + python3 "${POM_UTILS}" update-dep-version ../google-cloud-pom-parent/pom.xml sdk-platform-java-config "${version}" } if [[ $# -ne 2 ]]; @@ -99,10 +65,8 @@ LIBRARY_NAME="google-cloud-${REPO#java-}" # build.sh uses this environment variable export JOB_TYPE=$2 -## Get the directory of the build script -scriptDir=$(realpath $(dirname "${BASH_SOURCE[0]}")) ## cd to the parent directory, i.e. the root of the git repo -cd ${scriptDir}/.. +cd "${scriptDir}/.." # Make artifacts available for 'mvn validate' at the bottom pushd java-shared-config @@ -111,12 +75,7 @@ popd # Read the current version of this BOM in the POM. Example version: '0.116.1-alpha-SNAPSHOT' VERSION_POM=java-shared-config/java-shared-config/pom.xml -JAVA_SHARED_CONFIG_VERSION=$(python3 -c " -import xml.etree.ElementTree as ET -root = ET.parse('${VERSION_POM}').getroot() -v = root.find('{*}version') or root.find('{*}parent/{*}version') -print(v.text if v is not None and v.text else '') -") +JAVA_SHARED_CONFIG_VERSION=$(python3 "${POM_UTILS}" get-version "${VERSION_POM}") if [ -z "${JAVA_SHARED_CONFIG_VERSION}" ]; then echo "Version is not found in ${VERSION_POM}" @@ -131,12 +90,7 @@ LATEST_TAG=$(git ls-remote --tags https://github.com/googleapis/google-cloud-jav echo "Cloning google-cloud-java at tag: ${LATEST_TAG}" git clone "https://github.com/googleapis/google-cloud-java.git" -b "${LATEST_TAG}" --depth=1 pushd google-cloud-java/sdk-platform-java -SDK_PLATFORM_JAVA_CONFIG_VERSION=$(python3 -c " -import xml.etree.ElementTree as ET -root = ET.parse('sdk-platform-java-config/pom.xml').getroot() -v = root.find('{*}version') or root.find('{*}parent/{*}version') -print(v.text if v is not None and v.text else '') -") +SDK_PLATFORM_JAVA_CONFIG_VERSION=$(python3 "${POM_UTILS}" get-version sdk-platform-java-config/pom.xml) pushd sdk-platform-java-config # Use released version of google-cloud-shared-dependencies to avoid verifying SNAPSHOT changes. diff --git a/.kokoro/common.sh b/.kokoro/common.sh index 51f2ca5a768c..0383462c093c 100644 --- a/.kokoro/common.sh +++ b/.kokoro/common.sh @@ -492,22 +492,7 @@ function install_modules() { # ex: update_dependency google-cloud-java/google-cloud-jar-parent google-cloud-shared-dependencies 1.2.3 function update_pom_dependency { pushd "$1" || exit 1 - python3 -c " -import xml.etree.ElementTree as ET, sys -file, artifact_id, new_ver = 'pom.xml', sys.argv[1], sys.argv[2] -ET.register_namespace('', 'http://maven.apache.org/POM/4.0.0') -tree = ET.parse(file) -root = tree.getroot() -updated = False -for elem in root.iter(): - art = elem.find('{*}artifactId') - ver = elem.find('{*}version') - if art is not None and art.text == artifact_id and ver is not None: - ver.text = new_ver - updated = True -if updated: - tree.write(file, encoding='utf-8', xml_declaration=True) -" "$2" "$3" + python3 "${commonScriptDir}/pom_utils.py" update-dep-version pom.xml "$2" "$3" popd || exit 1 } @@ -516,21 +501,7 @@ function find_all_poms_with_versioned_dependency { poms=($(find . -name pom.xml)) found=() for pom in "${poms[@]}"; do - if python3 -c " -import xml.etree.ElementTree as ET, sys -try: - root = ET.parse(sys.argv[1]).getroot() - found = False - for elem in root.iter(): - art = elem.find('{*}artifactId') - ver = elem.find('{*}version') - if art is not None and art.text == sys.argv[2] and ver is not None and ver.text: - found = True - break - sys.exit(0 if found else 1) -except Exception: - sys.exit(1) -" "$pom" "$1" &>/dev/null; then + if python3 "${commonScriptDir}/pom_utils.py" has-versioned-dep "$pom" "$1"; then found+=("$pom") fi done @@ -555,20 +526,11 @@ function update_all_poms_dependency { # Parse the version of the pom.xml file in the given directory ($1) # ex: VERSION=$(parse_pom_version java-shared-dependencies) function parse_pom_version { - result=$(python3 -c " -import xml.etree.ElementTree as ET, sys -try: - root = ET.parse('$1/pom.xml').getroot() - v = root.find('{*}version') - if v is None: - v = root.find('{*}parent/{*}version') - if v is not None and v.text: - print(v.text.strip()) - else: - sys.exit(1) -except Exception: - sys.exit(1) -") + local pom_file="$1/pom.xml" + if [ ! -f "${pom_file}" ]; then + pom_file="$1" + fi + result=$(python3 "${commonScriptDir}/pom_utils.py" get-version "${pom_file}") if [ -z "${result}" ]; then echo "Version is not found in $1" diff --git a/.kokoro/common_test.sh b/.kokoro/common_test.sh index 77805f77dd60..93601a307b31 100755 --- a/.kokoro/common_test.sh +++ b/.kokoro/common_test.sh @@ -49,16 +49,7 @@ function test_update_pom_dependency { update_pom_dependency . truth "99.88.77" - version=$(python3 -c " -import xml.etree.ElementTree as ET -root = ET.parse('pom.xml').getroot() -for elem in root.iter(): - art = elem.find('{*}artifactId') - ver = elem.find('{*}version') - if art is not None and art.text == 'truth' and ver is not None: - print(ver.text) - break -") + version=$(python3 "${scriptDir}/pom_utils.py" get-dep-version pom.xml truth) if [ "$version" != "99.88.77" ]; then echo "update_pom_dependency failed to change version to expected value." exit 1 diff --git a/.kokoro/pom_utils.py b/.kokoro/pom_utils.py new file mode 100644 index 000000000000..8ed892be2d9b --- /dev/null +++ b/.kokoro/pom_utils.py @@ -0,0 +1,238 @@ +#!/usr/bin/env python3 +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Utility script for reading and manipulating Maven pom.xml files.""" + +import argparse +import sys +import xml.etree.ElementTree as ET + +MAVEN_NAMESPACE = "http://maven.apache.org/POM/4.0.0" + + +def parse_pom_version(pom_path: str) -> str | None: + """Extracts project version or parent version from pom.xml.""" + try: + tree = ET.parse(pom_path) + root = tree.getroot() + version_elem = root.find("{*}version") + if version_elem is None: + version_elem = root.find("{*}parent/{*}version") + if version_elem is not None and version_elem.text: + return version_elem.text.strip() + except Exception as e: + sys.stderr.write(f"Error parsing version from {pom_path}: {e}\n") + return None + + +def has_versioned_dependency(pom_path: str, artifact_id: str) -> bool: + """Checks if a pom.xml defines a tag for the given artifactId.""" + try: + tree = ET.parse(pom_path) + root = tree.getroot() + for elem in root.iter(): + art = elem.find("{*}artifactId") + ver = elem.find("{*}version") + if ( + art is not None + and art.text == artifact_id + and ver is not None + and ver.text + ): + return True + except Exception as e: + sys.stderr.write(f"Error checking dependency in {pom_path}: {e}\n") + return False + + +def get_dependency_version(pom_path: str, artifact_id: str) -> str | None: + """Returns the version for the given artifactId in pom.xml.""" + try: + tree = ET.parse(pom_path) + root = tree.getroot() + for elem in root.iter(): + art = elem.find("{*}artifactId") + ver = elem.find("{*}version") + if ( + art is not None + and art.text == artifact_id + and ver is not None + and ver.text + ): + return ver.text.strip() + except Exception as e: + sys.stderr.write(f"Error reading dependency version from {pom_path}: {e}\n") + return None + + +def update_dependency_version( + pom_path: str, artifact_id: str, new_version: str +) -> bool: + """Updates the version for the given artifactId in pom.xml.""" + try: + ET.register_namespace("", MAVEN_NAMESPACE) + tree = ET.parse(pom_path) + root = tree.getroot() + updated = False + for elem in root.iter(): + art = elem.find("{*}artifactId") + ver = elem.find("{*}version") + if art is not None and art.text == artifact_id and ver is not None: + ver.text = new_version + updated = True + if updated: + tree.write(pom_path, encoding="utf-8", xml_declaration=True) + return True + except Exception as e: + sys.stderr.write(f"Error updating dependency in {pom_path}: {e}\n") + return False + + +def update_property_value( + pom_path: str, property_name: str, new_value: str +) -> bool: + """Updates the value of a named property in of pom.xml.""" + try: + ET.register_namespace("", MAVEN_NAMESPACE) + tree = ET.parse(pom_path) + root = tree.getroot() + updated = False + for prop in root.findall(".//{*}properties"): + target = prop.find(f"{{*}}{property_name}") + if target is not None: + target.text = new_value + updated = True + if updated: + tree.write(pom_path, encoding="utf-8", xml_declaration=True) + return True + except Exception as e: + sys.stderr.write(f"Error updating property in {pom_path}: {e}\n") + return False + + +def get_coordinates(pom_path: str) -> tuple[str, str, str]: + """Returns (groupId, artifactId, version) from pom.xml.""" + try: + tree = ET.parse(pom_path) + root = tree.getroot() + gid = root.find("{*}groupId") + aid = root.find("{*}artifactId") + ver = root.find("{*}version") + if gid is None: + gid = root.find("{*}parent/{*}groupId") + if ver is None: + ver = root.find("{*}parent/{*}version") + return ( + gid.text.strip() if gid is not None and gid.text else "", + aid.text.strip() if aid is not None and aid.text else "", + ver.text.strip() if ver is not None and ver.text else "", + ) + except Exception as e: + sys.stderr.write(f"Error reading coordinates from {pom_path}: {e}\n") + return ("", "", "") + + +def main(): + parser = argparse.ArgumentParser(description="Maven POM XML utility CLI") + subparsers = parser.add_subparsers(dest="command", required=True) + + # get-version + get_version_parser = subparsers.add_parser( + "get-version", help="Get project version from pom.xml" + ) + get_version_parser.add_argument("pom_file", help="Path to pom.xml") + + # has-versioned-dep + has_dep_parser = subparsers.add_parser( + "has-versioned-dep", + help="Check if pom.xml declares a versioned dependency for an artifact", + ) + has_dep_parser.add_argument("pom_file", help="Path to pom.xml") + has_dep_parser.add_argument("artifact_id", help="Artifact ID to search for") + + # get-dep-version + get_dep_ver_parser = subparsers.add_parser( + "get-dep-version", + help="Get version for an artifactId in pom.xml", + ) + get_dep_ver_parser.add_argument("pom_file", help="Path to pom.xml") + get_dep_ver_parser.add_argument("artifact_id", help="Artifact ID to search for") + + # update-dep-version + update_dep_parser = subparsers.add_parser( + "update-dep-version", + help="Update version for an artifactId in pom.xml", + ) + update_dep_parser.add_argument("pom_file", help="Path to pom.xml") + update_dep_parser.add_argument("artifact_id", help="Artifact ID to update") + update_dep_parser.add_argument("new_version", help="New version to set") + + # update-property + update_prop_parser = subparsers.add_parser( + "update-property", + help="Update property value in pom.xml", + ) + update_prop_parser.add_argument("pom_file", help="Path to pom.xml") + update_prop_parser.add_argument("property_name", help="Property tag name to update") + update_prop_parser.add_argument("new_value", help="New value to set") + + # get-coordinates + coord_parser = subparsers.add_parser( + "get-coordinates", help="Get groupId, artifactId, and version" + ) + coord_parser.add_argument("pom_file", help="Path to pom.xml") + + args = parser.parse_args() + + if args.command == "get-version": + version = parse_pom_version(args.pom_file) + if version: + print(version) + sys.exit(0) + else: + sys.exit(1) + + elif args.command == "has-versioned-dep": + found = has_versioned_dependency(args.pom_file, args.artifact_id) + sys.exit(0 if found else 1) + + elif args.command == "get-dep-version": + ver = get_dependency_version(args.pom_file, args.artifact_id) + if ver: + print(ver) + sys.exit(0) + else: + sys.exit(1) + + elif args.command == "update-dep-version": + success = update_dependency_version( + args.pom_file, args.artifact_id, args.new_version + ) + sys.exit(0 if success else 1) + + elif args.command == "update-property": + success = update_property_value( + args.pom_file, args.property_name, args.new_value + ) + sys.exit(0 if success else 1) + + elif args.command == "get-coordinates": + gid, aid, ver = get_coordinates(args.pom_file) + print(f"{gid} {aid} {ver}") + sys.exit(0 if aid else 1) + + +if __name__ == "__main__": + main() diff --git a/.kokoro/presubmit/downstream-build.sh b/.kokoro/presubmit/downstream-build.sh index f459b5a79dfa..fc49f76778f0 100755 --- a/.kokoro/presubmit/downstream-build.sh +++ b/.kokoro/presubmit/downstream-build.sh @@ -16,38 +16,22 @@ set -eo pipefail set -x -function modify_shared_config() { - python3 -c " -import xml.etree.ElementTree as ET, sys -file = 'pom.xml' -ET.register_namespace('', 'http://maven.apache.org/POM/4.0.0') -tree = ET.parse(file) -root = tree.getroot() -for elem in root.iter(): - art = elem.find('{*}artifactId') - ver = elem.find('{*}version') - if art is not None and art.text == 'google-cloud-shared-config' and ver is not None: - ver.text = sys.argv[1] -tree.write(file, encoding='utf-8', xml_declaration=True) -" "${SHARED_CONFIG_VERSION}" -} - ## Get the directory of the build script and install all modules in the monorepo scriptDir=$(realpath "$(dirname "${BASH_SOURCE[0]}")") +POM_UTILS="${scriptDir}/../pom_utils.py" ## cd to the parent directory, i.e. the root of the git repo cd "${scriptDir}/../.." +function modify_shared_config() { + python3 "${POM_UTILS}" update-dep-version pom.xml google-cloud-shared-config "${SHARED_CONFIG_VERSION}" +} + # Build and install the entire monorepo to local cache (including the under-test java-shared-config) mvn -B -ntp install -Dcheckstyle.skip -Dfmt.skip -DskipTests # Get the version of java-shared-config under test VERSION_POM=java-shared-config/java-shared-config/pom.xml -SHARED_CONFIG_VERSION=$(python3 -c " -import xml.etree.ElementTree as ET -root = ET.parse('${VERSION_POM}').getroot() -v = root.find('{*}version') or root.find('{*}parent/{*}version') -print(v.text if v is not None and v.text else '') -") +SHARED_CONFIG_VERSION=$(python3 "${POM_UTILS}" get-version "${VERSION_POM}") # Use GCP Maven Mirror (as in original script) mkdir -p "${HOME}/.m2" diff --git a/generation/check_existing_release_versions.sh b/generation/check_existing_release_versions.sh index 80ce9ec6413c..b7680e7a5c14 100755 --- a/generation/check_existing_release_versions.sh +++ b/generation/check_existing_release_versions.sh @@ -5,6 +5,9 @@ MAVEN_SITE=https://maven-central.storage-download.googleapis.com/maven2 set -e +scriptDir=$(realpath "$(dirname "${BASH_SOURCE[0]}")") +POM_UTILS="${scriptDir}/../.kokoro/pom_utils.py" + function find_existing_version_pom() { local pom_file=$1 if [ -z "${pom_file}" ]; then @@ -14,17 +17,7 @@ function find_existing_version_pom() { local group_id local artifact_id local version - read -r group_id artifact_id version < <(python3 -c " -import xml.etree.ElementTree as ET, sys -try: - root = ET.parse(sys.argv[1]).getroot() - gid = root.find('{*}groupId') - aid = root.find('{*}artifactId') - ver = root.find('{*}version') - print(f\"{gid.text if gid is not None and gid.text else ''} {aid.text if aid is not None and aid.text else ''} {ver.text if ver is not None and ver.text else ''}\") -except Exception: - print(' ') -" "${pom_file}") + read -r group_id artifact_id version < <(python3 "${POM_UTILS}" get-coordinates "${pom_file}") echo -n "Checking ${group_id}:${artifact_id}:${version}:" if [ -z "${artifact_id}" ]; then echo "Couldn't parse artifact_id in the pom file: $pom_file" diff --git a/sdk-platform-java/.kokoro/presubmit/common.sh b/sdk-platform-java/.kokoro/presubmit/common.sh index 6c143bdd25d7..97ee180e9664 100644 --- a/sdk-platform-java/.kokoro/presubmit/common.sh +++ b/sdk-platform-java/.kokoro/presubmit/common.sh @@ -20,22 +20,7 @@ commonScriptDir=$(realpath "$(dirname "${BASH_SOURCE[0]}")") # ex: update_dependency google-cloud-java/google-cloud-jar-parent google-cloud-shared-dependencies 1.2.3 function update_pom_dependency { pushd "$1" || exit 1 - python3 -c " -import xml.etree.ElementTree as ET, sys -file, artifact_id, new_ver = 'pom.xml', sys.argv[1], sys.argv[2] -ET.register_namespace('', 'http://maven.apache.org/POM/4.0.0') -tree = ET.parse(file) -root = tree.getroot() -updated = False -for elem in root.iter(): - art = elem.find('{*}artifactId') - ver = elem.find('{*}version') - if art is not None and art.text == artifact_id and ver is not None: - ver.text = new_ver - updated = True -if updated: - tree.write(file, encoding='utf-8', xml_declaration=True) -" "$2" "$3" + python3 "${commonScriptDir}/pom_utils.py" update-dep-version pom.xml "$2" "$3" popd || exit 1 } @@ -44,21 +29,7 @@ function find_all_poms_with_versioned_dependency { poms=($(find . -name pom.xml)) found=() for pom in "${poms[@]}"; do - if python3 -c " -import xml.etree.ElementTree as ET, sys -try: - root = ET.parse(sys.argv[1]).getroot() - found = False - for elem in root.iter(): - art = elem.find('{*}artifactId') - ver = elem.find('{*}version') - if art is not None and art.text == sys.argv[2] and ver is not None and ver.text: - found = True - break - sys.exit(0 if found else 1) -except Exception: - sys.exit(1) -" "$pom" "$1" &>/dev/null; then + if python3 "${commonScriptDir}/pom_utils.py" has-versioned-dep "$pom" "$1"; then found+=("$pom") fi done @@ -83,20 +54,11 @@ function update_all_poms_dependency { # Parse the version of the pom.xml file in the given directory ($1) # ex: VERSION=$(parse_pom_version java-shared-dependencies) function parse_pom_version { - result=$(python3 -c " -import xml.etree.ElementTree as ET, sys -try: - root = ET.parse('$1/pom.xml').getroot() - v = root.find('{*}version') - if v is None: - v = root.find('{*}parent/{*}version') - if v is not None and v.text: - print(v.text.strip()) - else: - sys.exit(1) -except Exception: - sys.exit(1) -") + local pom_file="$1/pom.xml" + if [ ! -f "${pom_file}" ]; then + pom_file="$1" + fi + result=$(python3 "${commonScriptDir}/pom_utils.py" get-version "${pom_file}") if [ -z "${result}" ]; then echo "Version is not found in $1" diff --git a/sdk-platform-java/.kokoro/presubmit/pom_utils.py b/sdk-platform-java/.kokoro/presubmit/pom_utils.py new file mode 100644 index 000000000000..8ed892be2d9b --- /dev/null +++ b/sdk-platform-java/.kokoro/presubmit/pom_utils.py @@ -0,0 +1,238 @@ +#!/usr/bin/env python3 +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Utility script for reading and manipulating Maven pom.xml files.""" + +import argparse +import sys +import xml.etree.ElementTree as ET + +MAVEN_NAMESPACE = "http://maven.apache.org/POM/4.0.0" + + +def parse_pom_version(pom_path: str) -> str | None: + """Extracts project version or parent version from pom.xml.""" + try: + tree = ET.parse(pom_path) + root = tree.getroot() + version_elem = root.find("{*}version") + if version_elem is None: + version_elem = root.find("{*}parent/{*}version") + if version_elem is not None and version_elem.text: + return version_elem.text.strip() + except Exception as e: + sys.stderr.write(f"Error parsing version from {pom_path}: {e}\n") + return None + + +def has_versioned_dependency(pom_path: str, artifact_id: str) -> bool: + """Checks if a pom.xml defines a tag for the given artifactId.""" + try: + tree = ET.parse(pom_path) + root = tree.getroot() + for elem in root.iter(): + art = elem.find("{*}artifactId") + ver = elem.find("{*}version") + if ( + art is not None + and art.text == artifact_id + and ver is not None + and ver.text + ): + return True + except Exception as e: + sys.stderr.write(f"Error checking dependency in {pom_path}: {e}\n") + return False + + +def get_dependency_version(pom_path: str, artifact_id: str) -> str | None: + """Returns the version for the given artifactId in pom.xml.""" + try: + tree = ET.parse(pom_path) + root = tree.getroot() + for elem in root.iter(): + art = elem.find("{*}artifactId") + ver = elem.find("{*}version") + if ( + art is not None + and art.text == artifact_id + and ver is not None + and ver.text + ): + return ver.text.strip() + except Exception as e: + sys.stderr.write(f"Error reading dependency version from {pom_path}: {e}\n") + return None + + +def update_dependency_version( + pom_path: str, artifact_id: str, new_version: str +) -> bool: + """Updates the version for the given artifactId in pom.xml.""" + try: + ET.register_namespace("", MAVEN_NAMESPACE) + tree = ET.parse(pom_path) + root = tree.getroot() + updated = False + for elem in root.iter(): + art = elem.find("{*}artifactId") + ver = elem.find("{*}version") + if art is not None and art.text == artifact_id and ver is not None: + ver.text = new_version + updated = True + if updated: + tree.write(pom_path, encoding="utf-8", xml_declaration=True) + return True + except Exception as e: + sys.stderr.write(f"Error updating dependency in {pom_path}: {e}\n") + return False + + +def update_property_value( + pom_path: str, property_name: str, new_value: str +) -> bool: + """Updates the value of a named property in of pom.xml.""" + try: + ET.register_namespace("", MAVEN_NAMESPACE) + tree = ET.parse(pom_path) + root = tree.getroot() + updated = False + for prop in root.findall(".//{*}properties"): + target = prop.find(f"{{*}}{property_name}") + if target is not None: + target.text = new_value + updated = True + if updated: + tree.write(pom_path, encoding="utf-8", xml_declaration=True) + return True + except Exception as e: + sys.stderr.write(f"Error updating property in {pom_path}: {e}\n") + return False + + +def get_coordinates(pom_path: str) -> tuple[str, str, str]: + """Returns (groupId, artifactId, version) from pom.xml.""" + try: + tree = ET.parse(pom_path) + root = tree.getroot() + gid = root.find("{*}groupId") + aid = root.find("{*}artifactId") + ver = root.find("{*}version") + if gid is None: + gid = root.find("{*}parent/{*}groupId") + if ver is None: + ver = root.find("{*}parent/{*}version") + return ( + gid.text.strip() if gid is not None and gid.text else "", + aid.text.strip() if aid is not None and aid.text else "", + ver.text.strip() if ver is not None and ver.text else "", + ) + except Exception as e: + sys.stderr.write(f"Error reading coordinates from {pom_path}: {e}\n") + return ("", "", "") + + +def main(): + parser = argparse.ArgumentParser(description="Maven POM XML utility CLI") + subparsers = parser.add_subparsers(dest="command", required=True) + + # get-version + get_version_parser = subparsers.add_parser( + "get-version", help="Get project version from pom.xml" + ) + get_version_parser.add_argument("pom_file", help="Path to pom.xml") + + # has-versioned-dep + has_dep_parser = subparsers.add_parser( + "has-versioned-dep", + help="Check if pom.xml declares a versioned dependency for an artifact", + ) + has_dep_parser.add_argument("pom_file", help="Path to pom.xml") + has_dep_parser.add_argument("artifact_id", help="Artifact ID to search for") + + # get-dep-version + get_dep_ver_parser = subparsers.add_parser( + "get-dep-version", + help="Get version for an artifactId in pom.xml", + ) + get_dep_ver_parser.add_argument("pom_file", help="Path to pom.xml") + get_dep_ver_parser.add_argument("artifact_id", help="Artifact ID to search for") + + # update-dep-version + update_dep_parser = subparsers.add_parser( + "update-dep-version", + help="Update version for an artifactId in pom.xml", + ) + update_dep_parser.add_argument("pom_file", help="Path to pom.xml") + update_dep_parser.add_argument("artifact_id", help="Artifact ID to update") + update_dep_parser.add_argument("new_version", help="New version to set") + + # update-property + update_prop_parser = subparsers.add_parser( + "update-property", + help="Update property value in pom.xml", + ) + update_prop_parser.add_argument("pom_file", help="Path to pom.xml") + update_prop_parser.add_argument("property_name", help="Property tag name to update") + update_prop_parser.add_argument("new_value", help="New value to set") + + # get-coordinates + coord_parser = subparsers.add_parser( + "get-coordinates", help="Get groupId, artifactId, and version" + ) + coord_parser.add_argument("pom_file", help="Path to pom.xml") + + args = parser.parse_args() + + if args.command == "get-version": + version = parse_pom_version(args.pom_file) + if version: + print(version) + sys.exit(0) + else: + sys.exit(1) + + elif args.command == "has-versioned-dep": + found = has_versioned_dependency(args.pom_file, args.artifact_id) + sys.exit(0 if found else 1) + + elif args.command == "get-dep-version": + ver = get_dependency_version(args.pom_file, args.artifact_id) + if ver: + print(ver) + sys.exit(0) + else: + sys.exit(1) + + elif args.command == "update-dep-version": + success = update_dependency_version( + args.pom_file, args.artifact_id, args.new_version + ) + sys.exit(0 if success else 1) + + elif args.command == "update-property": + success = update_property_value( + args.pom_file, args.property_name, args.new_value + ) + sys.exit(0 if success else 1) + + elif args.command == "get-coordinates": + gid, aid, ver = get_coordinates(args.pom_file) + print(f"{gid} {aid} {ver}") + sys.exit(0 if aid else 1) + + +if __name__ == "__main__": + main() diff --git a/sdk-platform-java/scripts/create_native_image_test_env.sh b/sdk-platform-java/scripts/create_native_image_test_env.sh index 5fdf2e74ee12..0a2a006dbcdf 100644 --- a/sdk-platform-java/scripts/create_native_image_test_env.sh +++ b/sdk-platform-java/scripts/create_native_image_test_env.sh @@ -15,36 +15,15 @@ # with the associated changes, to the submodule project. set -eo pipefail +scriptDir=$(realpath "$(dirname "${BASH_SOURCE[0]}")") +POM_UTILS="${scriptDir}/../.kokoro/presubmit/pom_utils.py" + function modify_shared_config() { - python3 -c " -import xml.etree.ElementTree as ET, sys -file = 'pom.xml' -ET.register_namespace('', 'http://maven.apache.org/POM/4.0.0') -tree = ET.parse(file) -root = tree.getroot() -for elem in root.iter(): - art = elem.find('{*}artifactId') - ver = elem.find('{*}version') - if art is not None and art.text == 'google-cloud-shared-config' and ver is not None: - ver.text = sys.argv[1] -tree.write(file, encoding='utf-8', xml_declaration=True) -" "${SHARED_CONFIG_VERSION}" + python3 "${POM_UTILS}" update-dep-version pom.xml google-cloud-shared-config "${SHARED_CONFIG_VERSION}" } function modify_shared_dependencies() { - python3 -c " -import xml.etree.ElementTree as ET, sys -file = 'pom.xml' -ET.register_namespace('', 'http://maven.apache.org/POM/4.0.0') -tree = ET.parse(file) -root = tree.getroot() -for elem in root.iter(): - art = elem.find('{*}artifactId') - ver = elem.find('{*}version') - if art is not None and art.text == 'google-cloud-shared-dependencies' and ver is not None: - ver.text = sys.argv[1] -tree.write(file, encoding='utf-8', xml_declaration=True) -" "${SHARED_DEPS_VERSION}" + python3 "${POM_UTILS}" update-dep-version pom.xml google-cloud-shared-dependencies "${SHARED_DEPS_VERSION}" } if [ -z "$GRAALVM_VERSION" ]; then @@ -78,29 +57,12 @@ fi # Modify graal-sdk version in GAX pushd gapic-generator-java/gax-java -python3 -c " -import xml.etree.ElementTree as ET, sys -file = 'pom.xml' -ET.register_namespace('', 'http://maven.apache.org/POM/4.0.0') -tree = ET.parse(file) -root = tree.getroot() -for elem in root.iter(): - art = elem.find('{*}artifactId') - ver = elem.find('{*}version') - if art is not None and art.text == 'graal-sdk' and ver is not None: - ver.text = sys.argv[1] -tree.write(file, encoding='utf-8', xml_declaration=True) -" "${GRAALVM_VERSION}" +python3 "${POM_UTILS}" update-dep-version pom.xml graal-sdk "${GRAALVM_VERSION}" # Get java-shared-dependencies version popd pushd gapic-generator-java -SHARED_DEPS_VERSION=$(python3 -c " -import xml.etree.ElementTree as ET -root = ET.parse('java-shared-dependencies/pom.xml').getroot() -v = root.find('{*}version') or root.find('{*}parent/{*}version') -print(v.text if v is not None and v.text else '') -") +SHARED_DEPS_VERSION=$(python3 "${POM_UTILS}" get-version java-shared-dependencies/pom.xml) echo $SHARED_DEPS_VERSION if [ ! "$(git branch --list "$GRAALVM_BRANCH")" ] @@ -122,26 +84,9 @@ fi # Modify junit-platform-native and native-maven-plugin pushd java-shared-config -SHARED_CONFIG_VERSION=$(python3 -c " -import xml.etree.ElementTree as ET -root = ET.parse('pom.xml').getroot() -v = root.find('{*}version') or root.find('{*}parent/{*}version') -print(v.text if v is not None and v.text else '') -") - -python3 -c " -import xml.etree.ElementTree as ET, sys -file = 'pom.xml' -ET.register_namespace('', 'http://maven.apache.org/POM/4.0.0') -tree = ET.parse(file) -root = tree.getroot() -for elem in root.iter(): - art = elem.find('{*}artifactId') - ver = elem.find('{*}version') - if art is not None and art.text == 'native-maven-plugin' and ver is not None: - ver.text = sys.argv[1] -tree.write(file, encoding='utf-8', xml_declaration=True) -" "${NATIVE_MAVEN_PLUGIN}" +SHARED_CONFIG_VERSION=$(python3 "${POM_UTILS}" get-version pom.xml) + +python3 "${POM_UTILS}" update-dep-version pom.xml native-maven-plugin "${NATIVE_MAVEN_PLUGIN}" echo "Modified native-maven-plugin in shared-config" git diff