diff --git a/.github/scripts/sync.py b/.github/scripts/sync.py index 96c4d42..8899301 100755 --- a/.github/scripts/sync.py +++ b/.github/scripts/sync.py @@ -300,7 +300,7 @@ def _umbrella_major(index_slug): return core if re.fullmatch(r"\d+", core) else None -def dbr_point_releases(index_slug, ml=False, max_minor=50): +def dbr_point_releases(index_slug, ml=False, max_minor=50, max_leading_misses=5): """Return ``(pointrelease_slugs, umbrella_slug)`` for a runtime line: the page slugs to generate individual '.x' folders from, and the page the bare-major '.x' umbrella folder is generated from (or None when there is none, or it isn't safe this run). @@ -316,8 +316,10 @@ def dbr_point_releases(index_slug, ml=False, max_minor=50): We probe '.[ml]' from minor 0, stopping after two consecutive 404s once a page has been seen (leading 404s — e.g. a removed old page — don't end the list before a - later release is found); ``max_minor`` is only a runaway backstop set well beyond any - real line's point-release count, not the normal terminator. An EoS point release exists + later release is found). Before any page is seen, a longer run of ``max_leading_misses`` + 404s ends the probe instead, so a bare major with no point-release pages at all (e.g. DBR + 19 today) doesn't scan the full range; ``max_minor`` is only a runaway backstop set well + beyond any real line's point-release count, not the normal terminator. An EoS point release exists (retired, not end-of-list), so it is skipped without ending the probe and is not published. The umbrella comes from the newest live point release, EXCEPT when a probe newer than that was left indeterminate by a transient error — then umbrella_slug is None @@ -330,33 +332,34 @@ def dbr_point_releases(index_slug, ml=False, max_minor=50): suffix = "ml" if ml else "" def live_page(slug): - """Probe one page, returning one of: - 'live' (with html) — a live, non-EoS runtime page. + """Classify one probe, returning one of: + 'live' — a live, non-EoS runtime page. 'eos' — the page exists but the runtime is end-of-support: skip it, but it is NOT end-of-list, so keep probing higher minors. - 'absent' — a genuine 404: counts toward the two-consecutive end-of-list break. + 'absent' — a genuine 404: counts toward the end-of-list break. 'error' — a transient fetch error (timeout, 5xx): logged and never mistaken for a 404, mirroring discover_serverless — a flaky run must not abort the - sync or be read as end-of-list.""" + sync or be read as end-of-list. + The page body isn't returned — sync_dbr / sync_dbr_ml refetch the pages they + generate from, so a probe only needs the classification.""" try: html = fetch_opt(DBR_PAGE.format(slug=slug)) except Exception as e: print(f" ! dbr [{slug}]: transient fetch error ({e}); skipping (not end-of-list)") - return "error", None + return "error" if html is None: - return "absent", None + return "absent" if is_eos(html): - return "eos", None - return "live", html + return "eos" + return "live" if not major: # Pre-18 scheme: the index slug is the runtime page; no umbrella form. - kind, _ = live_page(index_slug) - return ([index_slug], None) if kind == "live" else ([], None) + return ([index_slug], None) if live_page(index_slug) == "live" else ([], None) live, transient_minors, saw_page, misses = [], [], False, 0 for minor in range(0, max_minor + 1): - kind, _ = live_page(f"{major}.{minor}{suffix}") + kind = live_page(f"{major}.{minor}{suffix}") if kind == "error": # Record the minor, but don't advance the end-of-list counter (a flaky probe # is not a confirmed 404). @@ -369,11 +372,22 @@ def live_page(slug): continue if kind == "absent": misses += 1 - # Only trailing 404s end the list. Don't honor the break until at least one page - # (live or EoS) has been seen, so leading 404s — e.g. an old point-release page - # that was removed — can't stop the probe before a later live release. Mirrors - # the "require at least one found" guard in discover_serverless. - if saw_page and misses >= 2: + # Two kinds of 404 run end the list. Once a page (live or EoS) has been seen, two + # consecutive 404s are the trailing end-of-list — mirrors the "require at least + # one found" guard in discover_serverless, so a removed early page can't stop the + # probe before a later live release. Before any page is seen, a longer run of + # leading 404s (max_leading_misses) is the terminator instead: a bare major with + # no point-release pages at all (e.g. DBR 19 today) would otherwise probe the full + # 0..max_minor range every run. Only genuine 404s count here — an EoS page is a + # hit (kind 'eos') that resets misses below. A line's point releases are numbered + # contiguously from .0, so five consecutive leading 404s mean none were published + # (the empty case), not a gap before a later release. Five sits comfortably above + # the trailing slack of 2 while bounding that empty case; it could only skip a real + # release if five early minors that once existed were later removed while a higher + # one stayed live — but upstream retires a page by marking it '(EoS)' (a hit), not + # by deleting it, so that doesn't arise. + cap = 2 if saw_page else max_leading_misses + if misses >= cap: break continue saw_page = True @@ -396,9 +410,13 @@ def live_page(slug): return pointrelease_slugs, f"{major}.{latest_minor}{suffix}" if transient_minors: - # Nothing was confirmed live, but a probe was indeterminate — we can't be sure the - # line has no point release, so don't fall back to the umbrella page (which could - # generate the folder from the wrong metadata). Skip; a clean run resolves it. + # Nothing was confirmed live, but a probe was indeterminate. A transient could be + # masking a real point release whose page is the correct umbrella source; falling + # back to the umbrella index page instead could generate the folder from different + # (wrong) metadata. So skip conservatively — the existing folder is left untouched + # and a clean run resolves it. Trade-off: on a line that genuinely has no point + # release (the umbrella-only case), a single flaky probe still no-ops the line for a + # whole cycle; the leading-miss cap keeps that window small (a handful of probes). print(f" ! dbr [{major}.x]: no live point release confirmed and a probe was " f"indeterminate (transient error); skipping this run") return [], None @@ -409,8 +427,7 @@ def live_page(slug): # are left as-is; pruning retired lines is out of scope (sync only writes). return [], None # No point-release page exists at all (e.g. DBR 19 today) — fall back to the umbrella page. - kind, _ = live_page(index_slug) - return ([], index_slug) if kind == "live" else ([], None) + return ([], index_slug) if live_page(index_slug) == "live" else ([], None) def _write_env(key, pkgs, python_version, dbconnect): @@ -449,8 +466,14 @@ def sync_dbr(): if major and slug == umbrella_slug: folder_vers.append(major) for folder_ver in folder_vers: + # A point-release folder pins its exact minor (18.2 -> ~=18.2.0). The bare- + # major umbrella tracks the whole major line, like a serverless major, so it + # pins ~=MAJOR.0 (18 -> ~=18.0): a cluster addressing the line by its bare + # major resolves the latest databricks-connect in the major, and a new point + # release doesn't need a regen to be covered. + dbconnect = major if folder_ver == major else dbconnect_ver for scala in scalas: - _write_env(f"{folder_ver}.x-scala{scala}", pkgs, python_version, dbconnect_ver) + _write_env(f"{folder_ver}.x-scala{scala}", pkgs, python_version, dbconnect) def ml_variant_pkgs(ml_html, variant): @@ -499,8 +522,11 @@ def _sync_dbr_ml_page(slug, point_release, umbrella_major): print(f" ! dbr-ml [{slug}] {variant}: no packages found; skipping") continue for folder_ver in folder_vers: + # Bare-major umbrella tracks the whole major line (18 -> ~=18.0); point-release + # folders pin their exact minor. See sync_dbr for the rationale. + dbconnect = umbrella_major if folder_ver == umbrella_major else dbconnect_ver for scala in scalas: - _write_env(f"{folder_ver}.x-{variant}-ml-scala{scala}", pkgs, python_version, dbconnect_ver) + _write_env(f"{folder_ver}.x-{variant}-ml-scala{scala}", pkgs, python_version, dbconnect) def git(*args): diff --git a/.github/scripts/test_sync.py b/.github/scripts/test_sync.py index ab80aec..30ee16f 100644 --- a/.github/scripts/test_sync.py +++ b/.github/scripts/test_sync.py @@ -276,6 +276,54 @@ def fetch_opt(url): with mock.patch.object(sync, "fetch_opt", fetch_opt): self.assertEqual(sync.dbr_point_releases("18"), (["18.0", "18.2"], "18.2")) + def test_umbrella_only_line_bounds_probes_to_a_leading_miss_cap(self): + # A bare major with NO point-release pages (DBR 19 today) must not probe the whole + # minor range hunting for pages that don't exist. The trailing-404 break is gated on + # having seen a page, so a line where none is ever seen relies on a separate cap on + # consecutive LEADING 404s: after that many, give up and fall back to the umbrella + # page. Bounds HTTP cost to a handful of requests rather than ~50 per bare major. + # Regression guard for the saw_page-gated break running the full 0..max_minor range. + pages = {"19": _titled_page("Databricks Runtime 19 LTS")} + calls = [] + + def counting(url): + calls.append(_slug_of(url)) + return pages.get(_slug_of(url)) + + with mock.patch.object(sync, "fetch_opt", counting): + self.assertEqual(sync.dbr_point_releases("19"), ([], "19")) + # Five consecutive leading 404s (19.0..19.4) trip the cap; 19.5 is never probed, and + # the only further request is the umbrella page itself. + self.assertIn("19.4", calls) + self.assertNotIn("19.5", calls) + self.assertIn("19", calls) + + def test_umbrella_leading_miss_cap_sits_above_the_short_gap_slack(self): + # The leading cap must sit above the slack that lets a removed early page not hide a + # later live release: four leading 404s then a live release is still discovered. + pages = {"18.4": _titled_page("Databricks Runtime 18.4")} + with mock.patch.object(sync, "fetch_opt", _fake_fetch_opt(pages)): + self.assertEqual(sync.dbr_point_releases("18"), (["18.4"], "18.4")) + + def test_umbrella_leading_miss_cap_gives_up_past_its_bound(self): + # The accepted trade-off boundary: a live release sitting past a full run of leading + # 404s (18.0..18.4 all genuine 404s, 18.5 live) is NOT discovered — the cap fires at + # 18.4 and 18.5 is never probed, so the line falls back to the umbrella page. Since + # point releases are numbered contiguously from .0, this can only arise if early pages + # that once existed were removed (EoS pages return 'eos' and reset the counter), which + # upstream doesn't do. This test locks the cap value so a change to it is deliberate. + pages = {"18.5": _titled_page("Databricks Runtime 18.5"), + "18": _titled_page("Databricks Runtime 18 LTS")} + calls = [] + + def counting(url): + calls.append(_slug_of(url)) + return pages.get(_slug_of(url)) + + with mock.patch.object(sync, "fetch_opt", counting): + self.assertEqual(sync.dbr_point_releases("18"), ([], "18")) + self.assertNotIn("18.5", calls) + class SyncDbrFolderKeyTest(unittest.TestCase): def test_umbrella_line_publishes_point_releases_and_the_bare_major_umbrella(self): @@ -300,11 +348,14 @@ def counting_fetch(url): mock.patch.object(sync, "_write_env", lambda key, pkgs, pv, dbconnect: writes.append((key, dbconnect))): sync.sync_dbr() + # Point-release folders pin their exact minor (18.1 -> ~=18.1.0); the bare-major + # umbrella tracks the whole major line like a serverless major (18 -> ~=18.0), so its + # dbconnect is the bare major, not the latest point release's minor. self.assertEqual( sorted(writes), [("18.1.x-scala2.13", "18.1"), ("18.2.x-scala2.13", "18.2"), - ("18.x-scala2.13", "18.2")], + ("18.x-scala2.13", "18")], ) # 18.2 is both a point release and the umbrella source, but is fetched only once. self.assertEqual(sorted(fetched), ["18.1", "18.2"]) @@ -322,6 +373,23 @@ def test_pre18_line_publishes_only_its_minor_folder(self): sync.sync_dbr() self.assertEqual(writes, [("17.3.x-scala2.13", "17.3")]) + def test_umbrella_only_line_publishes_bare_major_folder_end_to_end(self): + # DBR 19 today: no point-release pages, so dbr_point_releases falls back to the + # umbrella page. Driven end-to-end through sync_dbr, dbr_meta reads the bare major + # from the title ('Databricks Runtime 19 LTS' -> key_ver '19'), and the single + # bare-major umbrella folder is pinned to the whole major line (dbconnect '19' -> + # ~=19.0), not to a '.0' point release. This is the path where dbr_meta defaults the + # minor, so it's the one most worth an end-to-end assertion. + pages = {"19": _runtime_page("Databricks Runtime 19 LTS")} + writes = [] + with mock.patch.object(sync, "discover_dbr", return_value=["19"]), \ + mock.patch.object(sync, "dbr_point_releases", return_value=([], "19")), \ + mock.patch.object(sync, "fetch", _fake_fetch(pages)), \ + mock.patch.object(sync, "_write_env", + lambda key, pkgs, pv, dbconnect: writes.append((key, dbconnect))): + sync.sync_dbr() + self.assertEqual(writes, [("19.x-scala2.13", "19")]) + class SyncDbrMlFolderKeyTest(unittest.TestCase): def test_umbrella_ml_line_publishes_point_releases_and_umbrella(self): @@ -339,14 +407,16 @@ def test_umbrella_ml_line_publishes_point_releases_and_umbrella(self): mock.patch.object(sync, "_write_env", lambda key, pkgs, pv, dbconnect: writes.append((key, dbconnect))): sync.sync_dbr_ml() + # The bare-major ML umbrella folders track the whole major line (dbconnect "18" -> + # ~=18.0), while the point-release ML folders keep their exact minor. self.assertEqual( sorted(writes), [("18.1.x-cpu-ml-scala2.13", "18.1"), ("18.1.x-gpu-ml-scala2.13", "18.1"), ("18.2.x-cpu-ml-scala2.13", "18.2"), ("18.2.x-gpu-ml-scala2.13", "18.2"), - ("18.x-cpu-ml-scala2.13", "18.2"), - ("18.x-gpu-ml-scala2.13", "18.2")], + ("18.x-cpu-ml-scala2.13", "18"), + ("18.x-gpu-ml-scala2.13", "18")], ) diff --git a/README.md b/README.md index 9f0b8fb..204040d 100644 --- a/README.md +++ b/README.md @@ -115,8 +115,11 @@ is best-effort. Nobody hand-edits the `python/` artifacts. pages (`18.0`, `18.1`, …) — and a cluster addresses the line either by a specific point release (`18.2.x-scala2.13`) or by the bare major (`18.x-scala2.13`) depending on client version, so both forms are published: one folder per live point release, plus a - bare-major umbrella folder taken from the latest live point release (whose minor sets the - `databricks-connect` pin). EoS point releases are skipped. Pre-18 lines keep the old + bare-major umbrella folder whose packages come from the latest live point release. A + point-release folder pins `databricks-connect` to its exact minor (`18.2.x` → `~=18.2.0`), + but the umbrella tracks the whole major line like a serverless major (`18.x` → `~=18.0`), + so a cluster addressing the line by its bare major resolves the newest `databricks-connect` + in the major. EoS point releases are skipped. Pre-18 lines keep the old scheme — one folder, keyed by the minor read from the page title. A release that ships two Scala images from one page (e.g. DBR 16.4 LTS — `Scala: 2.12.15 or 2.13.10`) yields one environment per diff --git a/python/dbr/18.x-cpu-ml-scala2.13/pyproject.toml b/python/dbr/18.x-cpu-ml-scala2.13/pyproject.toml index 02d0379..6445344 100644 --- a/python/dbr/18.x-cpu-ml-scala2.13/pyproject.toml +++ b/python/dbr/18.x-cpu-ml-scala2.13/pyproject.toml @@ -7,7 +7,7 @@ requires-python = "==3.12.*" [dependency-groups] dev = [ - "databricks-connect~=18.2.0", + "databricks-connect~=18.0", ] [tool.uv] diff --git a/python/dbr/18.x-gpu-ml-scala2.13/pyproject.toml b/python/dbr/18.x-gpu-ml-scala2.13/pyproject.toml index 30d6c75..f4ad387 100644 --- a/python/dbr/18.x-gpu-ml-scala2.13/pyproject.toml +++ b/python/dbr/18.x-gpu-ml-scala2.13/pyproject.toml @@ -7,7 +7,7 @@ requires-python = "==3.12.*" [dependency-groups] dev = [ - "databricks-connect~=18.2.0", + "databricks-connect~=18.0", ] [tool.uv] diff --git a/python/dbr/18.x-scala2.13/pyproject.toml b/python/dbr/18.x-scala2.13/pyproject.toml index 9818485..de7e20e 100644 --- a/python/dbr/18.x-scala2.13/pyproject.toml +++ b/python/dbr/18.x-scala2.13/pyproject.toml @@ -7,7 +7,7 @@ requires-python = "==3.12.*" [dependency-groups] dev = [ - "databricks-connect~=18.2.0", + "databricks-connect~=18.0", ] [tool.uv] diff --git a/python/dbr/19.x-cpu-ml-scala2.13/pyproject.toml b/python/dbr/19.x-cpu-ml-scala2.13/pyproject.toml index 4710bd3..1480a75 100644 --- a/python/dbr/19.x-cpu-ml-scala2.13/pyproject.toml +++ b/python/dbr/19.x-cpu-ml-scala2.13/pyproject.toml @@ -7,7 +7,7 @@ requires-python = "==3.12.*" [dependency-groups] dev = [ - "databricks-connect~=19.0.0", + "databricks-connect~=19.0", ] [tool.uv] diff --git a/python/dbr/19.x-gpu-ml-scala2.13/pyproject.toml b/python/dbr/19.x-gpu-ml-scala2.13/pyproject.toml index 197ab88..3da43a8 100644 --- a/python/dbr/19.x-gpu-ml-scala2.13/pyproject.toml +++ b/python/dbr/19.x-gpu-ml-scala2.13/pyproject.toml @@ -7,7 +7,7 @@ requires-python = "==3.12.*" [dependency-groups] dev = [ - "databricks-connect~=19.0.0", + "databricks-connect~=19.0", ] [tool.uv] diff --git a/python/dbr/19.x-scala2.13/pyproject.toml b/python/dbr/19.x-scala2.13/pyproject.toml index 9163a89..f0e573a 100644 --- a/python/dbr/19.x-scala2.13/pyproject.toml +++ b/python/dbr/19.x-scala2.13/pyproject.toml @@ -7,7 +7,7 @@ requires-python = "==3.12.*" [dependency-groups] dev = [ - "databricks-connect~=19.0.0", + "databricks-connect~=19.0", ] [tool.uv]