Publish DBR Scala 2.13 image when a runtime ships two Scala variants

.github/scripts/envgen.py

@@ -78,6 +78,11 @@
     "16.4.x-scala2.12": {"pandas": "1.5."},
     "16.4.x-cpu-ml-scala2.12": {"pandas": "1.5."},
     "16.4.x-gpu-ml-scala2.12": {"pandas": "1.5."},
+    # DBR 16.4 also ships a Scala 2.13 image (same Python packages, so same pandas
+    # 1.5.x-on-cp312 problem). Drop it for those envs too, keyed by their own names.
+    "16.4.x-scala2.13": {"pandas": "1.5."},
+    "16.4.x-cpu-ml-scala2.13": {"pandas": "1.5."},
+    "16.4.x-gpu-ml-scala2.13": {"pandas": "1.5."},
     "serverless-v3": {"pandas": "1.5."},
 }
 

.github/scripts/sync.py

@@ -24,6 +24,7 @@
 """
 import argparse
 import hashlib
+import html as html_lib          # aliased: functions here take an ``html`` parameter
 import os
 import re
 import subprocess
@@ -191,30 +192,71 @@ def is_eos(html):
     return "(EoS)" in dbr_title(html)
 
 
+def dbr_scalas(html):
+    """The Scala versions ('2.12', '2.13', ...) a runtime page's System environment
+    lists, in page order and de-duplicated.
+
+    Most runtimes ship one image ('Scala</strong>: 2.13.16' -> ['2.13']). A runtime in
+    the 2.12->2.13 migration window publishes two images from a single page, rendered as
+    'Scala</strong>: 2.12.15 <strong>or</strong> 2.13.10' -> ['2.12', '2.13']. Both
+    images share one 'Installed Python libraries' table (only the Java/Scala libraries
+    differ, which this repo doesn't consume), so the caller writes one folder per Scala
+    version off the same package set.
+
+    Capture the field value up to the list item's close, strip inline tags (mirroring
+    ``table_pkgs``, so a digit inside an attribute/href -- e.g. a linked Spark version
+    -- can't leak in) and unescape entities (so a ``&nbsp;`` delimiter becomes real
+    whitespace). Then read only the leading enumeration: a version, or several joined by
+    delimiters. Matching just that run stops a trailing annotation ('(Apache Spark 3.5)')
+    from contributing a bogus version, while the delimiter class stays deliberately wide
+    -- whitespace, ',', '/', or the words 'or'/'and' -- because narrowing it to the exact
+    ' or ' the page happens to use today would silently drop the second variant (and
+    resurrect the 404) the day a copy-edit changes the separator. Each MAJOR.MINOR is
+    taken with the patch consumed by '(?:\\.\\d+)?' ('2.12.15' -> '2.12', not '2.12' plus
+    a stray '.15'). Fall back to a single match if the item isn't delimited as expected,
+    so a layout change degrades to today's behaviour rather than to nothing."""
+    field = re.search(r"Scala</strong>\s*:(.*?)</li>", html, re.S)
+    if not field:
+        m = re.search(r"Scala</strong>\s*:\s*(\d+\.\d+)", html)
+        return [m.group(1)] if m else []
+    text = html_lib.unescape(re.sub(r"<[^>]+>", " ", field.group(1)))
+    ver = r"\d+\.\d+(?:\.\d+)?"
+    sep = r"(?:[\s,/]|\bor\b|\band\b)+"          # delimiter, never itself a version
+    enum = re.match(rf"\s*({ver}(?:{sep}{ver})*)", text)
+    if not enum:
+        return []
+    scalas = []
+    for v in re.findall(r"(\d+\.\d+)(?:\.\d+)?", enum.group(1)):
+        if v not in scalas:
+            scalas.append(v)
+    return scalas
+
+
 def dbr_meta(html):
-    """Return (key_ver, dbconnect_ver, scala, python_version) from a standard runtime
-    page, or None if any piece is missing.
+    """Return (key_ver, dbconnect_ver, scalas, python_version) from a standard runtime
+    page, or None if any piece is missing. ``scalas`` is a non-empty list (see
+    ``dbr_scalas``); the caller emits one '<key_ver>.x-scala<scala>' folder per entry.
 
     A point-release page carries the minor in its title ('Databricks Runtime 18.2'), so
-    both versions are the real minor: ('18.2', '18.2', '2.13', '3.12.3') -> folder
+    both versions are the real minor: ('18.2', '18.2', ['2.13'], '3.12.3') -> folder
     '18.2.x-scala2.13', databricks-connect~=18.2.0.
 
     An umbrella LTS page has no minor in its title ('Databricks Runtime 19 LTS'). Such a
     line is addressed by its bare major in a cluster's ``spark_version`` ('19.x-scala2.13'),
     so ``key_ver`` drops the minor to match that naming, while ``dbconnect_ver`` still
     defaults the minor to '.0' (databricks-connect is published per point release, so the
-    pin needs a concrete minor): ('19', '19.0', '2.13', '3.12.3') -> folder '19.x-scala2.13',
-    databricks-connect~=19.0.0. Feed point-release pages here (see ``dbr_point_releases``)
-    so a real minor is used whenever one is published."""
+    pin needs a concrete minor): ('19', '19.0', ['2.13'], '3.12.3') -> folder
+    '19.x-scala2.13', databricks-connect~=19.0.0. Feed point-release pages here (see
+    ``dbr_point_releases``) so a real minor is used whenever one is published."""
     ver = re.search(r"<title[^>]*>Databricks Runtime\s+(\d+)(?:\.(\d+))?", html)
-    sc = re.search(r"Scala</strong>\s*:\s*(\d+\.\d+)", html)
+    scalas = dbr_scalas(html)
     pv = re.search(r"Python</strong>\s*:\s*(\d+\.\d+\.\d+)", html)
-    if not (ver and sc and pv):
+    if not (ver and scalas and pv):
         return None
     major, minor = ver.group(1), ver.group(2)
     key_ver = f"{major}.{minor}" if minor else major
     dbconnect_ver = f"{major}.{minor or '0'}"
-    return key_ver, dbconnect_ver, sc.group(1), pv.group(1)
+    return key_ver, dbconnect_ver, scalas, pv.group(1)
 
 
 def parse_dbr_page(html):
@@ -326,8 +368,9 @@ def sync_dbr():
             if not meta or not pkgs:
                 print(f"  ! dbr [{slug}]: no meta / Python table; skipping")
                 continue
-            key_ver, dbconnect_ver, scala, python_version = meta
-            _write_env(f"{key_ver}.x-scala{scala}", pkgs, python_version, dbconnect_ver)
+            key_ver, dbconnect_ver, scalas, python_version = meta
+            for scala in scalas:
+                _write_env(f"{key_ver}.x-scala{scala}", pkgs, python_version, dbconnect_ver)
 
 
 def ml_variant_pkgs(ml_html, variant):
@@ -360,13 +403,14 @@ def _sync_dbr_ml_page(slug):
     if not meta:
         print(f"  ! dbr-ml [{slug}]: no base meta from {base}; skipping")
         return
-    key_ver, dbconnect_ver, scala, python_version = meta
+    key_ver, dbconnect_ver, scalas, python_version = meta
     for variant in ("cpu", "gpu"):
         pkgs = ml_variant_pkgs(ml_html, variant)
         if not pkgs:
             print(f"  ! dbr-ml [{slug}] {variant}: no packages found; skipping")
             continue
-        _write_env(f"{key_ver}.x-{variant}-ml-scala{scala}", pkgs, python_version, dbconnect_ver)
+        for scala in scalas:
+            _write_env(f"{key_ver}.x-{variant}-ml-scala{scala}", pkgs, python_version, dbconnect_ver)
 
 
 def git(*args):

.github/scripts/test_envgen.py

@@ -44,12 +44,17 @@ def test_keeps_installable_pins(self):
 class EnvScopedDropTest(unittest.TestCase):
     def test_pandas_dropped_on_py312_1_5_runtimes(self):
         # DBR 16.4 + serverless-v3 are the only Python-3.12 runtimes still on
-        # pandas 1.5.x, which has no cp312 wheel — dropped for just these envs.
+        # pandas 1.5.x, which has no cp312 wheel — dropped for just these envs. DBR 16.4
+        # ships both Scala images off one page (same Python packages), so the scala2.13
+        # variants carry the identical pandas problem and are dropped the same way.
         pkgs = {"pandas": "1.5.3", "numpy": "2.1.3"}
         for env in (
             "16.4.x-scala2.12",
             "16.4.x-cpu-ml-scala2.12",
             "16.4.x-gpu-ml-scala2.12",
+            "16.4.x-scala2.13",
+            "16.4.x-cpu-ml-scala2.13",
+            "16.4.x-gpu-ml-scala2.13",
             "serverless-v3",
         ):
             self.assertEqual(_filtered(pkgs, env), {"numpy": "2.1.3"}, env)

.github/scripts/test_sync.py

@@ -0,0 +1,87 @@
+"""Unit tests for sync.py page-parsing (run: python -m unittest test_sync)."""
+import unittest
+
+from sync import dbr_meta, dbr_scalas
+
+# Minimal fragments mirroring the runtime pages' System environment list. The real
+# pages are ~200 KB; only the shape dbr_scalas keys off is reproduced here.
+SINGLE = (
+    "<title>Databricks Runtime 17.3 LTS | Databricks on AWS</title>"
+    "<li><strong>Java</strong>: Zulu17.54+21-CA</li>"
+    "<li><strong>Scala</strong>: 2.13.16</li>"
+    "<li><strong>Python</strong>: 3.12.3</li>"
+)
+# A dual-image release renders the two Scala versions in one field, joined by
+# '<strong>or</strong>', each with a patch segment.
+DUAL = (
+    "<title>Databricks Runtime 16.4 LTS | Databricks on AWS</title>"
+    "<li><strong>Java</strong>: Zulu17.54+21-CA</li>"
+    "<li><strong>Scala</strong>: 2.12.15 <strong>or</strong> 2.13.10</li>"
+    "<li><strong>Python</strong>: 3.12.3</li>"
+)
+
+
+class DbrScalasTest(unittest.TestCase):
+    def test_single_scala(self):
+        self.assertEqual(dbr_scalas(SINGLE), ["2.13"])
+
+    def test_dual_scala_in_page_order(self):
+        # Both images off one page; patch segments (.15/.10) must not leak in as
+        # extra entries, and order follows the page.
+        self.assertEqual(dbr_scalas(DUAL), ["2.12", "2.13"])
+
+    def test_no_scala_field(self):
+        self.assertEqual(dbr_scalas("<title>Databricks Runtime 19</title>"), [])
+
+    def test_falls_back_when_list_item_not_closed(self):
+        # If the field isn't delimited by </li> as expected, degrade to a single
+        # match rather than to nothing.
+        self.assertEqual(dbr_scalas("<strong>Scala</strong>: 2.13.16"), ["2.13"])
+
+    def test_ignores_version_like_numbers_inside_tags(self):
+        # A version-like number that lives only inside a tag (an href/attribute, not
+        # the visible text) must not be read as a Scala version and spawn a bogus
+        # environment. Here the visible value is 2.13.16; the '3.5' is only in a link.
+        html = (
+            "<li><strong>Scala</strong>: 2.13.16 "
+            '<a href="https://spark.apache.org/docs/3.5/">docs</a></li>'
+        )
+        self.assertEqual(dbr_scalas(html), ["2.13"])
+
+    def test_ignores_trailing_annotation(self):
+        # Only the leading 'X or Y' enumeration is the Scala version list; a trailing
+        # visible annotation (e.g. the Spark version) must not contribute a version.
+        html = "<li><strong>Scala</strong>: 2.12.15 or 2.13.10 (Apache Spark 3.5)</li>"
+        self.assertEqual(dbr_scalas(html), ["2.12", "2.13"])
+
+    def test_dual_scala_alternate_separators(self):
+        # The two versions must be picked up regardless of the delimiter the page uses
+        # -- not only ' or '. A narrower match silently drops the second variant and
+        # resurrects the 404 this fix exists to prevent.
+        for sep in (" or ", " and ", ", ", ",", " / ", "/", "&nbsp;or&nbsp;"):
+            html = f"<li><strong>Scala</strong>: 2.12.15{sep}2.13.10</li>"
+            self.assertEqual(dbr_scalas(html), ["2.12", "2.13"], repr(sep))
+
+    def test_alternate_separator_still_stops_at_annotation(self):
+        # Widening the separator must not start swallowing trailing annotations.
+        html = "<li><strong>Scala</strong>: 2.12.15, 2.13.10, see Spark 3.5 notes</li>"
+        self.assertEqual(dbr_scalas(html), ["2.12", "2.13"])
+
+
+class DbrMetaTest(unittest.TestCase):
+    def test_dual_variant_yields_both_scalas(self):
+        key_ver, dbconnect_ver, scalas, python_version = dbr_meta(DUAL)
+        self.assertEqual(key_ver, "16.4")
+        self.assertEqual(dbconnect_ver, "16.4")
+        self.assertEqual(scalas, ["2.12", "2.13"])
+        self.assertEqual(python_version, "3.12.3")
+
+    def test_single_variant(self):
+        self.assertEqual(dbr_meta(SINGLE)[2], ["2.13"])
+
+    def test_none_when_scala_missing(self):
+        self.assertIsNone(dbr_meta("<title>Databricks Runtime 19</title>"))
+
+
+if __name__ == "__main__":
+    unittest.main()

README.md

@@ -110,8 +110,11 @@ is best-effort. Nobody hand-edits the `python/` artifacts.
   [runtime release-notes index](https://docs.databricks.com/aws/en/release-notes/runtime/),
   then for each fetches the page and parses the "Installed Python libraries" HTML
   table. The repo key (`<ver>.x-scala<scala>`) is built from the page's title and the
-  Scala version in its System environment. DBR pages don't list `databricks-connect`,
-  so its dev pin is derived from the runtime version.
+  Scala version in its System environment. A release that ships two Scala images from
+  one page (e.g. DBR 16.4 LTS — `Scala: 2.12.15 or 2.13.10`) yields one environment per
+  Scala version (`…-scala2.12` and `…-scala2.13`), both off the page's single Python
+  library table. DBR pages don't list `databricks-connect`, so its dev pin is derived
+  from the runtime version.
 - **DBR ML (CPU + GPU)** — for each `*-ml` runtime, a separate environment is produced
   per cluster type: `<ver>.x-cpu-ml-…` and `<ver>.x-gpu-ml-…`. Newer ML pages link
   downloadable `requirements-{cpu,gpu}-*.txt`; older ones render inline tables under