chore: migrate xmllint to Python xml.etree.ElementTree in CI scripts and workflows

.github/workflows/java-shared-config-downstream-dependencies.yaml

@@ -48,16 +48,6 @@ jobs:
         java-version: ${{matrix.java}}
         cache: maven
     - run: java -version
-    - name: Install xmllint
-      timeout-minutes: 5
-      run: |
-        # Retry apt-get update and install up to 3 times with a 15-second per-request timeout.
-        # Retries protect against transient network glitches, mirror timeouts, and apt lock contention on runner startup.
-        for i in {1..3}; do
-          sudo apt-get -o Acquire::http::Timeout="15" -o Acquire::Retries=3 update && \
-          sudo apt-get -o Acquire::http::Timeout="15" -o Acquire::Retries=3 -y install libxml2-utils && \
-          break || sleep 5
-        done
     - name: Install java-shared-dependencies and common dependencies
       run: |
         .kokoro/build.sh
@@ -81,4 +71,3 @@ jobs:
         fi
         mvn install -pl ${TARGET_PATH} -am -DskipTests=true -Dmaven.javadoc.skip=true -Dgcloud.download.skip=true -B -V -q
     - run: .kokoro/client-library-check.sh ${{matrix.repo}} dependencies
-

.github/workflows/java-shared-config-downstream-maven-plugins.yaml

@@ -69,16 +69,6 @@ jobs:
         java-version: ${{matrix.java}}
         cache: maven
     - run: java -version
-    - name: Install xmllint
-      timeout-minutes: 5
-      run: |
-        # Retry apt-get update and install up to 3 times with a 15-second per-request timeout.
-        # Retries protect against transient network glitches, mirror timeouts, and apt lock contention on runner startup.
-        for i in {1..3}; do
-          sudo apt-get -o Acquire::http::Timeout="15" -o Acquire::Retries=3 update && \
-          sudo apt-get -o Acquire::http::Timeout="15" -o Acquire::Retries=3 -y install libxml2-utils && \
-          break || sleep 5
-        done
     - run: .kokoro/client-library-check.sh ${{matrix.repo}} ${{matrix.job-type}}
   lint:
     needs: filter
@@ -104,16 +94,6 @@ jobs:
         java-version: ${{matrix.java}}
         cache: maven
     - run: java -version
-    - name: Install xmllint
-      timeout-minutes: 5
-      run: |
-        # Retry apt-get update and install up to 3 times with a 15-second per-request timeout.
-        # Retries protect against transient network glitches, mirror timeouts, and apt lock contention on runner startup.
-        for i in {1..3}; do
-          sudo apt-get -o Acquire::http::Timeout="15" -o Acquire::Retries=3 update && \
-          sudo apt-get -o Acquire::http::Timeout="15" -o Acquire::Retries=3 -y install libxml2-utils && \
-          break || sleep 5
-        done
     - name: Install java-shared-dependencies and common dependencies
       run: |
         .kokoro/build.sh
@@ -167,16 +147,6 @@ jobs:
         java-version: 17
         cache: maven
     - run: java -version
-    - name: Install xmllint
-      timeout-minutes: 5
-      run: |
-        # Retry apt-get update and install up to 3 times with a 15-second per-request timeout.
-        # Retries protect against transient network glitches, mirror timeouts, and apt lock contention on runner startup.
-        for i in {1..3}; do
-          sudo apt-get -o Acquire::http::Timeout="15" -o Acquire::Retries=3 update && \
-          sudo apt-get -o Acquire::http::Timeout="15" -o Acquire::Retries=3 -y install libxml2-utils && \
-          break || sleep 5
-        done
     - name: Install java-shared-dependencies and common dependencies
       run: |
         .kokoro/build.sh

.github/workflows/sdk-platform-java-downstream.yaml

@@ -51,16 +51,6 @@ jobs:
           distribution: temurin
           cache: maven
       - run: mvn -version
-      - name: Install xmllint
-        timeout-minutes: 5
-        run: |
-          # Retry apt-get update and install up to 3 times with a 15-second per-request timeout.
-          # Retries protect against transient network glitches, mirror timeouts, and apt lock contention on runner startup.
-          for i in {1..3}; do
-            sudo apt-get -o Acquire::http::Timeout="15" -o Acquire::Retries=3 update && \
-            sudo apt-get -o Acquire::http::Timeout="15" -o Acquire::Retries=3 -y install libxml2-utils && \
-            break || sleep 5
-          done
       - name: Test helper scripts
         run: .kokoro/common_test.sh
       - name: Perform downstream compatibility testing
@@ -82,16 +72,6 @@ jobs:
           distribution: temurin
           cache: maven
       - run: mvn -version
-      - name: Install xmllint
-        timeout-minutes: 5
-        run: |
-          # Retry apt-get update and install up to 3 times with a 15-second per-request timeout.
-          # Retries protect against transient network glitches, mirror timeouts, and apt lock contention on runner startup.
-          for i in {1..3}; do
-            sudo apt-get -o Acquire::http::Timeout="15" -o Acquire::Retries=3 update && \
-            sudo apt-get -o Acquire::http::Timeout="15" -o Acquire::Retries=3 -y install libxml2-utils && \
-            break || sleep 5
-          done
       - name: Perform downstream compatibility testing
         run: .kokoro/downstream-compatibility-spring.sh
   required:

.github/workflows/versions.yaml

@@ -42,16 +42,6 @@ jobs:
       github.repository_owner == 'googleapis' && github.head_ref == 'release-please--branches--main' &&
       !endsWith(github.event.pull_request.title, 'SNAPSHOT')
     steps:
-    - name: Install xmllint
-      timeout-minutes: 5
-      run: |
-        # Retry apt-get update and install up to 3 times with a 15-second per-request timeout.
-        # Retries protect against transient network glitches, mirror timeouts, and apt lock contention on runner startup.
-        for i in {1..3}; do
-          sudo apt-get -o Acquire::http::Timeout="15" -o Acquire::Retries=3 update && \
-          sudo apt-get -o Acquire::http::Timeout="15" -o Acquire::Retries=3 -y install libxml2-utils && \
-          break || sleep 5
-        done
     - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0
       with:
         persist-credentials: false

.kokoro/client-library-check-doclet.sh

@@ -21,6 +21,10 @@ set -eo pipefail
 # Display commands being run.
 set -x
 
+## Get the directory of the build script
+scriptDir=$(realpath "$(dirname "${BASH_SOURCE[0]}")")
+POM_UTILS="${scriptDir}/pom_utils.py"
+
 function get_current_version_from_versions_txt() {
   versions=$1
   key=$2
@@ -37,43 +41,21 @@ function get_released_version_from_versions_txt() {
 
 function replace_java_shared_config_version() {
   version=$1
-  # replace version
-  xmllint --shell <(cat pom.xml) << EOF
-  setns x=http://maven.apache.org/POM/4.0.0
-  cd .//x:artifactId[text()="google-cloud-shared-config"]
-  cd ../x:version
-  set ${version}
-  save pom.xml
-EOF
+  python3 "${POM_UTILS}" update-dep-version pom.xml google-cloud-shared-config "${version}"
 }
 
 function replace_java_shared_dependencies_version() {
   version=$1
-  # replace version
-  xmllint --shell <(cat pom.xml) << EOF
-  setns x=http://maven.apache.org/POM/4.0.0
-  cd .//x:properties/x:google-cloud-shared-dependencies.version
-  set ${version}
-  save pom.xml
-EOF
+  python3 "${POM_UTILS}" update-property pom.xml google-cloud-shared-dependencies.version "${version}"
 }
 
 function replace_sdk_platform_java_config_version() {
   version=$1
-  # replace version in the shared parent POM
-  xmllint --shell <(cat ../google-cloud-pom-parent/pom.xml) << EOF
-  setns x=http://maven.apache.org/POM/4.0.0
-  cd .//x:artifactId[text()="sdk-platform-java-config"]
-  cd ../x:version
-  set ${version}
-  save ../google-cloud-pom-parent/pom.xml
-EOF
+  python3 "${POM_UTILS}" update-dep-version ../google-cloud-pom-parent/pom.xml sdk-platform-java-config "${version}"
 }
 REPO=$1
-# Get the directory of the build script
-scriptDir=$(realpath $(dirname "${BASH_SOURCE[0]}"))
 ## cd to the parent directory, i.e. the root of the git repo
-cd ${scriptDir}/..
+cd "${scriptDir}/.."
 
 # Make artifacts available for 'mvn validate' at the bottom
 pushd java-shared-config
@@ -94,8 +76,7 @@ echo "This is the doclet path: ${docletPath}"
 
 # Read the current version of this BOM in the POM. Example version: '0.116.1-alpha-SNAPSHOT'
 VERSION_POM=java-shared-config/java-shared-config/pom.xml
-# Namespace (xmlns) prevents xmllint from specifying tag names in XPath
-JAVA_SHARED_CONFIG_VERSION=`sed -e 's/xmlns=".*"//' ${VERSION_POM} | xmllint --xpath '/project/version/text()' -`
+JAVA_SHARED_CONFIG_VERSION=$(python3 "${POM_UTILS}" get-version "${VERSION_POM}")
 
 if [ -z "${JAVA_SHARED_CONFIG_VERSION}" ]; then
   echo "Version is not found in ${VERSION_POM}"
@@ -110,7 +91,7 @@ LATEST_TAG=$(git ls-remote --tags https://github.com/googleapis/google-cloud-jav
 echo "Cloning google-cloud-java at tag: ${LATEST_TAG}"
 git clone "https://github.com/googleapis/google-cloud-java.git" -b "${LATEST_TAG}" --depth=1
 pushd google-cloud-java/sdk-platform-java
-SDK_PLATFORM_JAVA_CONFIG_VERSION=$(sed -e 's/xmlns=".*"//' sdk-platform-java-config/pom.xml | xmllint --xpath '/project/version/text()' -)
+SDK_PLATFORM_JAVA_CONFIG_VERSION=$(python3 "${POM_UTILS}" get-version sdk-platform-java-config/pom.xml)
 
 pushd sdk-platform-java-config
 # Use released version of google-cloud-shared-dependencies to avoid verifying SNAPSHOT changes.

.kokoro/client-library-check.sh

@@ -21,6 +21,10 @@ set -eo pipefail
 # Display commands being run.
 set -x
 
+## Get the directory of the build script
+scriptDir=$(realpath "$(dirname "${BASH_SOURCE[0]}")")
+POM_UTILS="${scriptDir}/pom_utils.py"
+
 function get_current_version_from_versions_txt() {
   versions=$1
   key=$2
@@ -37,37 +41,17 @@ function get_released_version_from_versions_txt() {
 
 function replace_java_shared_config_version() {
   version=$1
-  # replace version
-  xmllint --shell <(cat pom.xml) << EOF
-  setns x=http://maven.apache.org/POM/4.0.0
-  cd .//x:artifactId[text()="google-cloud-shared-config"]
-  cd ../x:version
-  set ${version}
-  save pom.xml
-EOF
+  python3 "${POM_UTILS}" update-dep-version pom.xml google-cloud-shared-config "${version}"
 }
 
 function replace_java_shared_dependencies_version() {
   version=$1
-  # replace version
-  xmllint --shell <(cat pom.xml) << EOF
-  setns x=http://maven.apache.org/POM/4.0.0
-  cd .//x:properties/x:google-cloud-shared-dependencies.version
-  set ${version}
-  save pom.xml
-EOF
+  python3 "${POM_UTILS}" update-property pom.xml google-cloud-shared-dependencies.version "${version}"
 }
 
 function replace_sdk_platform_java_config_version() {
   version=$1
-  # replace version in the shared parent POM
-  xmllint --shell <(cat ../google-cloud-pom-parent/pom.xml) << EOF
-  setns x=http://maven.apache.org/POM/4.0.0
-  cd .//x:artifactId[text()="sdk-platform-java-config"]
-  cd ../x:version
-  set ${version}
-  save ../google-cloud-pom-parent/pom.xml
-EOF
+  python3 "${POM_UTILS}" update-dep-version ../google-cloud-pom-parent/pom.xml sdk-platform-java-config "${version}"
 }
 
 if [[ $# -ne 2 ]];
@@ -81,10 +65,8 @@ LIBRARY_NAME="google-cloud-${REPO#java-}"
 # build.sh uses this environment variable
 export JOB_TYPE=$2
 
-## Get the directory of the build script
-scriptDir=$(realpath $(dirname "${BASH_SOURCE[0]}"))
 ## cd to the parent directory, i.e. the root of the git repo
-cd ${scriptDir}/..
+cd "${scriptDir}/.."
 
 # Make artifacts available for 'mvn validate' at the bottom
 pushd java-shared-config
@@ -93,8 +75,7 @@ popd
 
 # Read the current version of this BOM in the POM. Example version: '0.116.1-alpha-SNAPSHOT'
 VERSION_POM=java-shared-config/java-shared-config/pom.xml
-# Namespace (xmlns) prevents xmllint from specifying tag names in XPath
-JAVA_SHARED_CONFIG_VERSION=`sed -e 's/xmlns=".*"//' ${VERSION_POM} | xmllint --xpath '/project/version/text()' -`
+JAVA_SHARED_CONFIG_VERSION=$(python3 "${POM_UTILS}" get-version "${VERSION_POM}")
 
 if [ -z "${JAVA_SHARED_CONFIG_VERSION}" ]; then
   echo "Version is not found in ${VERSION_POM}"
@@ -109,7 +90,7 @@ LATEST_TAG=$(git ls-remote --tags https://github.com/googleapis/google-cloud-jav
 echo "Cloning google-cloud-java at tag: ${LATEST_TAG}"
 git clone "https://github.com/googleapis/google-cloud-java.git" -b "${LATEST_TAG}" --depth=1
 pushd google-cloud-java/sdk-platform-java
-SDK_PLATFORM_JAVA_CONFIG_VERSION=$(sed -e 's/xmlns=".*"//' sdk-platform-java-config/pom.xml | xmllint --xpath '/project/version/text()' -)
+SDK_PLATFORM_JAVA_CONFIG_VERSION=$(python3 "${POM_UTILS}" get-version sdk-platform-java-config/pom.xml)
 
 pushd sdk-platform-java-config
 # Use released version of google-cloud-shared-dependencies to avoid verifying SNAPSHOT changes.

.kokoro/common.sh

@@ -492,21 +492,16 @@ function install_modules() {
 # ex: update_dependency google-cloud-java/google-cloud-jar-parent google-cloud-shared-dependencies 1.2.3
 function update_pom_dependency {
   pushd "$1" || exit 1
-  xmllint --shell pom.xml &>/dev/null <<EOF
-setns x=http://maven.apache.org/POM/4.0.0
-cd .//x:artifactId[text()="$2"]
-cd ../x:version
-set $3
-save pom.xml
-EOF
+  python3 "${commonScriptDir}/pom_utils.py" update-dep-version pom.xml "$2" "$3"
   popd || exit 1
 }
 
 # Find all pom.xml files that declare a specific version for the given artifact ($1)
 function find_all_poms_with_versioned_dependency {
   poms=($(find . -name pom.xml))
+  found=()
   for pom in "${poms[@]}"; do
-    if xmllint --xpath "//*[local-name()='artifactId' and text()='$1']/following-sibling::*[local-name()='version']" "$pom" &>/dev/null; then
+    if python3 "${commonScriptDir}/pom_utils.py" has-versioned-dep "$pom" "$1"; then
       found+=("$pom")
     fi
   done
@@ -531,8 +526,11 @@ function update_all_poms_dependency {
 # Parse the version of the pom.xml file in the given directory ($1)
 # ex: VERSION=$(parse_pom_version java-shared-dependencies)
 function parse_pom_version {
-  # Namespace (xmlns) prevents xmllint from specifying tag names in XPath
-  result=$(sed -e 's/xmlns=".*"//' "$1/pom.xml" | xmllint --xpath '/project/version/text()' -)
+  local pom_file="$1/pom.xml"
+  if [ ! -f "${pom_file}" ]; then
+    pom_file="$1"
+  fi
+  result=$(python3 "${commonScriptDir}/pom_utils.py" get-version "${pom_file}")
 
   if [ -z "${result}" ]; then
     echo "Version is not found in $1"

.kokoro/common_test.sh

@@ -49,17 +49,11 @@ function test_update_pom_dependency {
 
   update_pom_dependency . truth "99.88.77"
 
-  xmllint --shell pom.xml &>/dev/null <<EOF
-setns x=http://maven.apache.org/POM/4.0.0
-cd .//x:artifactId[text()="truth"]
-cd ../x:version
-write found-version.txt
-EOF
-  if ! grep 99.88.77 found-version.txt &>/dev/null; then
+  version=$(python3 "${scriptDir}/pom_utils.py" get-dep-version pom.xml truth)
+  if [ "$version" != "99.88.77" ]; then
     echo "update_pom_dependency failed to change version to expected value."
     exit 1
   fi
-  rm found-version.txt
   popd
 }