Harden DBR umbrella: bound probe cost and pin connect to the major line

.github/scripts/sync.py

@@ -300,7 +300,7 @@ def _umbrella_major(index_slug):
     return core if re.fullmatch(r"\d+", core) else None
 
 
-def dbr_point_releases(index_slug, ml=False, max_minor=50):
+def dbr_point_releases(index_slug, ml=False, max_minor=50, max_leading_misses=5):
     """Return ``(pointrelease_slugs, umbrella_slug)`` for a runtime line: the page slugs to
     generate individual '<minor>.x' folders from, and the page the bare-major '<major>.x'
     umbrella folder is generated from (or None when there is none, or it isn't safe this run).
@@ -316,8 +316,10 @@ def dbr_point_releases(index_slug, ml=False, max_minor=50):
 
     We probe '<major>.<minor>[ml]' from minor 0, stopping after two consecutive 404s once a
     page has been seen (leading 404s — e.g. a removed old page — don't end the list before a
-    later release is found); ``max_minor`` is only a runaway backstop set well beyond any
-    real line's point-release count, not the normal terminator. An EoS point release exists
+    later release is found). Before any page is seen, a longer run of ``max_leading_misses``
+    404s ends the probe instead, so a bare major with no point-release pages at all (e.g. DBR
+    19 today) doesn't scan the full range; ``max_minor`` is only a runaway backstop set well
+    beyond any real line's point-release count, not the normal terminator. An EoS point release exists
     (retired, not end-of-list), so it is skipped without ending the probe and is not
     published. The umbrella comes from the newest live point release, EXCEPT when a probe
     newer than that was left indeterminate by a transient error — then umbrella_slug is None
@@ -330,33 +332,34 @@ def dbr_point_releases(index_slug, ml=False, max_minor=50):
     suffix = "ml" if ml else ""
 
     def live_page(slug):
-        """Probe one page, returning one of:
-          'live'   (with html) — a live, non-EoS runtime page.
+        """Classify one probe, returning one of:
+          'live'   — a live, non-EoS runtime page.
           'eos'    — the page exists but the runtime is end-of-support: skip it, but it is
                      NOT end-of-list, so keep probing higher minors.
-          'absent' — a genuine 404: counts toward the two-consecutive end-of-list break.
+          'absent' — a genuine 404: counts toward the end-of-list break.
           'error'  — a transient fetch error (timeout, 5xx): logged and never mistaken for
                      a 404, mirroring discover_serverless — a flaky run must not abort the
-                     sync or be read as end-of-list."""
+                     sync or be read as end-of-list.
+        The page body isn't returned — sync_dbr / sync_dbr_ml refetch the pages they
+        generate from, so a probe only needs the classification."""
         try:
             html = fetch_opt(DBR_PAGE.format(slug=slug))
         except Exception as e:
             print(f"  ! dbr [{slug}]: transient fetch error ({e}); skipping (not end-of-list)")
-            return "error", None
+            return "error"
         if html is None:
-            return "absent", None
+            return "absent"
         if is_eos(html):
-            return "eos", None
-        return "live", html
+            return "eos"
+        return "live"
 
     if not major:
         # Pre-18 scheme: the index slug is the runtime page; no umbrella form.
-        kind, _ = live_page(index_slug)
-        return ([index_slug], None) if kind == "live" else ([], None)
+        return ([index_slug], None) if live_page(index_slug) == "live" else ([], None)
 
     live, transient_minors, saw_page, misses = [], [], False, 0
     for minor in range(0, max_minor + 1):
-        kind, _ = live_page(f"{major}.{minor}{suffix}")
+        kind = live_page(f"{major}.{minor}{suffix}")
         if kind == "error":
             # Record the minor, but don't advance the end-of-list counter (a flaky probe
             # is not a confirmed 404).
@@ -369,11 +372,22 @@ def live_page(slug):
             continue
         if kind == "absent":
             misses += 1
-            # Only trailing 404s end the list. Don't honor the break until at least one page
-            # (live or EoS) has been seen, so leading 404s — e.g. an old point-release page
-            # that was removed — can't stop the probe before a later live release. Mirrors
-            # the "require at least one found" guard in discover_serverless.
-            if saw_page and misses >= 2:
+            # Two kinds of 404 run end the list. Once a page (live or EoS) has been seen, two
+            # consecutive 404s are the trailing end-of-list — mirrors the "require at least
+            # one found" guard in discover_serverless, so a removed early page can't stop the
+            # probe before a later live release. Before any page is seen, a longer run of
+            # leading 404s (max_leading_misses) is the terminator instead: a bare major with
+            # no point-release pages at all (e.g. DBR 19 today) would otherwise probe the full
+            # 0..max_minor range every run. Only genuine 404s count here — an EoS page is a
+            # hit (kind 'eos') that resets misses below. A line's point releases are numbered
+            # contiguously from .0, so five consecutive leading 404s mean none were published
+            # (the empty case), not a gap before a later release. Five sits comfortably above
+            # the trailing slack of 2 while bounding that empty case; it could only skip a real
+            # release if five early minors that once existed were later removed while a higher
+            # one stayed live — but upstream retires a page by marking it '(EoS)' (a hit), not
+            # by deleting it, so that doesn't arise.
+            cap = 2 if saw_page else max_leading_misses
+            if misses >= cap:
                 break
             continue
         saw_page = True
@@ -396,9 +410,13 @@ def live_page(slug):
         return pointrelease_slugs, f"{major}.{latest_minor}{suffix}"
 
     if transient_minors:
-        # Nothing was confirmed live, but a probe was indeterminate — we can't be sure the
-        # line has no point release, so don't fall back to the umbrella page (which could
-        # generate the folder from the wrong metadata). Skip; a clean run resolves it.
+        # Nothing was confirmed live, but a probe was indeterminate. A transient could be
+        # masking a real point release whose page is the correct umbrella source; falling
+        # back to the umbrella index page instead could generate the folder from different
+        # (wrong) metadata. So skip conservatively — the existing folder is left untouched
+        # and a clean run resolves it. Trade-off: on a line that genuinely has no point
+        # release (the umbrella-only case), a single flaky probe still no-ops the line for a
+        # whole cycle; the leading-miss cap keeps that window small (a handful of probes).
         print(f"  ! dbr [{major}.x]: no live point release confirmed and a probe was "
               f"indeterminate (transient error); skipping this run")
         return [], None
@@ -409,8 +427,7 @@ def live_page(slug):
         # are left as-is; pruning retired lines is out of scope (sync only writes).
         return [], None
     # No point-release page exists at all (e.g. DBR 19 today) — fall back to the umbrella page.
-    kind, _ = live_page(index_slug)
-    return ([], index_slug) if kind == "live" else ([], None)
+    return ([], index_slug) if live_page(index_slug) == "live" else ([], None)
 
 
 def _write_env(key, pkgs, python_version, dbconnect):
@@ -449,8 +466,14 @@ def sync_dbr():
             if major and slug == umbrella_slug:
                 folder_vers.append(major)
             for folder_ver in folder_vers:
+                # A point-release folder pins its exact minor (18.2 -> ~=18.2.0). The bare-
+                # major umbrella tracks the whole major line, like a serverless major, so it
+                # pins ~=MAJOR.0 (18 -> ~=18.0): a cluster addressing the line by its bare
+                # major resolves the latest databricks-connect in the major, and a new point
+                # release doesn't need a regen to be covered.
+                dbconnect = major if folder_ver == major else dbconnect_ver
                 for scala in scalas:
-                    _write_env(f"{folder_ver}.x-scala{scala}", pkgs, python_version, dbconnect_ver)
+                    _write_env(f"{folder_ver}.x-scala{scala}", pkgs, python_version, dbconnect)
 
 
 def ml_variant_pkgs(ml_html, variant):
@@ -499,8 +522,11 @@ def _sync_dbr_ml_page(slug, point_release, umbrella_major):
             print(f"  ! dbr-ml [{slug}] {variant}: no packages found; skipping")
             continue
         for folder_ver in folder_vers:
+            # Bare-major umbrella tracks the whole major line (18 -> ~=18.0); point-release
+            # folders pin their exact minor. See sync_dbr for the rationale.
+            dbconnect = umbrella_major if folder_ver == umbrella_major else dbconnect_ver
             for scala in scalas:
-                _write_env(f"{folder_ver}.x-{variant}-ml-scala{scala}", pkgs, python_version, dbconnect_ver)
+                _write_env(f"{folder_ver}.x-{variant}-ml-scala{scala}", pkgs, python_version, dbconnect)
 
 
 def git(*args):

.github/scripts/test_sync.py

@@ -276,6 +276,54 @@ def fetch_opt(url):
         with mock.patch.object(sync, "fetch_opt", fetch_opt):
             self.assertEqual(sync.dbr_point_releases("18"), (["18.0", "18.2"], "18.2"))
 
+    def test_umbrella_only_line_bounds_probes_to_a_leading_miss_cap(self):
+        # A bare major with NO point-release pages (DBR 19 today) must not probe the whole
+        # minor range hunting for pages that don't exist. The trailing-404 break is gated on
+        # having seen a page, so a line where none is ever seen relies on a separate cap on
+        # consecutive LEADING 404s: after that many, give up and fall back to the umbrella
+        # page. Bounds HTTP cost to a handful of requests rather than ~50 per bare major.
+        # Regression guard for the saw_page-gated break running the full 0..max_minor range.
+        pages = {"19": _titled_page("Databricks Runtime 19 LTS")}
+        calls = []
+
+        def counting(url):
+            calls.append(_slug_of(url))
+            return pages.get(_slug_of(url))
+
+        with mock.patch.object(sync, "fetch_opt", counting):
+            self.assertEqual(sync.dbr_point_releases("19"), ([], "19"))
+        # Five consecutive leading 404s (19.0..19.4) trip the cap; 19.5 is never probed, and
+        # the only further request is the umbrella page itself.
+        self.assertIn("19.4", calls)
+        self.assertNotIn("19.5", calls)
+        self.assertIn("19", calls)
+
+    def test_umbrella_leading_miss_cap_sits_above_the_short_gap_slack(self):
+        # The leading cap must sit above the slack that lets a removed early page not hide a
+        # later live release: four leading 404s then a live release is still discovered.
+        pages = {"18.4": _titled_page("Databricks Runtime 18.4")}
+        with mock.patch.object(sync, "fetch_opt", _fake_fetch_opt(pages)):
+            self.assertEqual(sync.dbr_point_releases("18"), (["18.4"], "18.4"))
+
+    def test_umbrella_leading_miss_cap_gives_up_past_its_bound(self):
+        # The accepted trade-off boundary: a live release sitting past a full run of leading
+        # 404s (18.0..18.4 all genuine 404s, 18.5 live) is NOT discovered — the cap fires at
+        # 18.4 and 18.5 is never probed, so the line falls back to the umbrella page. Since
+        # point releases are numbered contiguously from .0, this can only arise if early pages
+        # that once existed were removed (EoS pages return 'eos' and reset the counter), which
+        # upstream doesn't do. This test locks the cap value so a change to it is deliberate.
+        pages = {"18.5": _titled_page("Databricks Runtime 18.5"),
+                 "18": _titled_page("Databricks Runtime 18 LTS")}
+        calls = []
+
+        def counting(url):
+            calls.append(_slug_of(url))
+            return pages.get(_slug_of(url))
+
+        with mock.patch.object(sync, "fetch_opt", counting):
+            self.assertEqual(sync.dbr_point_releases("18"), ([], "18"))
+        self.assertNotIn("18.5", calls)
+
 
 class SyncDbrFolderKeyTest(unittest.TestCase):
     def test_umbrella_line_publishes_point_releases_and_the_bare_major_umbrella(self):
@@ -300,11 +348,14 @@ def counting_fetch(url):
                 mock.patch.object(sync, "_write_env",
                                   lambda key, pkgs, pv, dbconnect: writes.append((key, dbconnect))):
             sync.sync_dbr()
+        # Point-release folders pin their exact minor (18.1 -> ~=18.1.0); the bare-major
+        # umbrella tracks the whole major line like a serverless major (18 -> ~=18.0), so its
+        # dbconnect is the bare major, not the latest point release's minor.
         self.assertEqual(
             sorted(writes),
             [("18.1.x-scala2.13", "18.1"),
              ("18.2.x-scala2.13", "18.2"),
-             ("18.x-scala2.13", "18.2")],
+             ("18.x-scala2.13", "18")],
         )
         # 18.2 is both a point release and the umbrella source, but is fetched only once.
         self.assertEqual(sorted(fetched), ["18.1", "18.2"])
@@ -322,6 +373,23 @@ def test_pre18_line_publishes_only_its_minor_folder(self):
             sync.sync_dbr()
         self.assertEqual(writes, [("17.3.x-scala2.13", "17.3")])
 
+    def test_umbrella_only_line_publishes_bare_major_folder_end_to_end(self):
+        # DBR 19 today: no point-release pages, so dbr_point_releases falls back to the
+        # umbrella page. Driven end-to-end through sync_dbr, dbr_meta reads the bare major
+        # from the title ('Databricks Runtime 19 LTS' -> key_ver '19'), and the single
+        # bare-major umbrella folder is pinned to the whole major line (dbconnect '19' ->
+        # ~=19.0), not to a '.0' point release. This is the path where dbr_meta defaults the
+        # minor, so it's the one most worth an end-to-end assertion.
+        pages = {"19": _runtime_page("Databricks Runtime 19 LTS")}
+        writes = []
+        with mock.patch.object(sync, "discover_dbr", return_value=["19"]), \
+                mock.patch.object(sync, "dbr_point_releases", return_value=([], "19")), \
+                mock.patch.object(sync, "fetch", _fake_fetch(pages)), \
+                mock.patch.object(sync, "_write_env",
+                                  lambda key, pkgs, pv, dbconnect: writes.append((key, dbconnect))):
+            sync.sync_dbr()
+        self.assertEqual(writes, [("19.x-scala2.13", "19")])
+
 
 class SyncDbrMlFolderKeyTest(unittest.TestCase):
     def test_umbrella_ml_line_publishes_point_releases_and_umbrella(self):
@@ -339,14 +407,16 @@ def test_umbrella_ml_line_publishes_point_releases_and_umbrella(self):
                 mock.patch.object(sync, "_write_env",
                                   lambda key, pkgs, pv, dbconnect: writes.append((key, dbconnect))):
             sync.sync_dbr_ml()
+        # The bare-major ML umbrella folders track the whole major line (dbconnect "18" ->
+        # ~=18.0), while the point-release ML folders keep their exact minor.
         self.assertEqual(
             sorted(writes),
             [("18.1.x-cpu-ml-scala2.13", "18.1"),
              ("18.1.x-gpu-ml-scala2.13", "18.1"),
              ("18.2.x-cpu-ml-scala2.13", "18.2"),
              ("18.2.x-gpu-ml-scala2.13", "18.2"),
-             ("18.x-cpu-ml-scala2.13", "18.2"),
-             ("18.x-gpu-ml-scala2.13", "18.2")],
+             ("18.x-cpu-ml-scala2.13", "18"),
+             ("18.x-gpu-ml-scala2.13", "18")],
         )
 
 

README.md

@@ -115,8 +115,11 @@ is best-effort. Nobody hand-edits the `python/` artifacts.
   pages (`18.0`, `18.1`, …) — and a cluster addresses the line either by a specific point
   release (`18.2.x-scala2.13`) or by the bare major (`18.x-scala2.13`) depending on client
   version, so both forms are published: one folder per live point release, plus a
-  bare-major umbrella folder taken from the latest live point release (whose minor sets the
-  `databricks-connect` pin). EoS point releases are skipped. Pre-18 lines keep the old
+  bare-major umbrella folder whose packages come from the latest live point release. A
+  point-release folder pins `databricks-connect` to its exact minor (`18.2.x` → `~=18.2.0`),
+  but the umbrella tracks the whole major line like a serverless major (`18.x` → `~=18.0`),
+  so a cluster addressing the line by its bare major resolves the newest `databricks-connect`
+  in the major. EoS point releases are skipped. Pre-18 lines keep the old
   scheme — one folder, keyed by the minor read from the page title. A release that
   ships two Scala images from
   one page (e.g. DBR 16.4 LTS — `Scala: 2.12.15 or 2.13.10`) yields one environment per

python/dbr/18.x-cpu-ml-scala2.13/pyproject.toml

@@ -7,7 +7,7 @@ requires-python = "==3.12.*"
 
 [dependency-groups]
 dev = [
-    "databricks-connect~=18.2.0",
+    "databricks-connect~=18.0",
 ]
 
 [tool.uv]

python/dbr/18.x-gpu-ml-scala2.13/pyproject.toml

@@ -7,7 +7,7 @@ requires-python = "==3.12.*"
 
 [dependency-groups]
 dev = [
-    "databricks-connect~=18.2.0",
+    "databricks-connect~=18.0",
 ]
 
 [tool.uv]

python/dbr/18.x-scala2.13/pyproject.toml

@@ -7,7 +7,7 @@ requires-python = "==3.12.*"
 
 [dependency-groups]
 dev = [
-    "databricks-connect~=18.2.0",
+    "databricks-connect~=18.0",
 ]
 
 [tool.uv]

python/dbr/19.x-cpu-ml-scala2.13/pyproject.toml

@@ -7,7 +7,7 @@ requires-python = "==3.12.*"
 
 [dependency-groups]
 dev = [
-    "databricks-connect~=19.0.0",
+    "databricks-connect~=19.0",
 ]
 
 [tool.uv]

python/dbr/19.x-gpu-ml-scala2.13/pyproject.toml

@@ -7,7 +7,7 @@ requires-python = "==3.12.*"
 
 [dependency-groups]
 dev = [
-    "databricks-connect~=19.0.0",
+    "databricks-connect~=19.0",
 ]
 
 [tool.uv]

python/dbr/19.x-scala2.13/pyproject.toml

@@ -7,7 +7,7 @@ requires-python = "==3.12.*"
 
 [dependency-groups]
 dev = [
-    "databricks-connect~=19.0.0",
+    "databricks-connect~=19.0",
 ]
 
 [tool.uv]