diff --git a/python/Taskfile.yml b/python/Taskfile.yml index 30e2db62ae5..602e92028a5 100644 --- a/python/Taskfile.yml +++ b/python/Taskfile.yml @@ -76,6 +76,8 @@ tasks: ! -path databricks/bundles/core \ ! -path databricks/bundles/resources \ -exec rm -rf {} \; + # core/ is hand-written except for the generated wiring under _generated/. + - rm -rf databricks/bundles/core/_generated - cd codegen && uv run -m pytest codegen_tests - cd codegen && uv run -m codegen.main --output .. # Generated code is fixed and formatted by the global ruff (see ../ruff.toml). diff --git a/python/codegen/codegen/.gitattributes b/python/codegen/codegen/.gitattributes new file mode 100644 index 00000000000..b656adbf2b0 --- /dev/null +++ b/python/codegen/codegen/.gitattributes @@ -0,0 +1,2 @@ +# Render the wiring templates as Python on GitHub. +*.py.tmpl linguist-language=Python diff --git a/python/codegen/codegen/core_init.py.tmpl b/python/codegen/codegen/core_init.py.tmpl new file mode 100644 index 00000000000..dcc012115c5 --- /dev/null +++ b/python/codegen/codegen/core_init.py.tmpl @@ -0,0 +1,31 @@ +__all__ = [ +${all_block} +] + +from databricks.bundles.core._bundle import Bundle +from databricks.bundles.core._diagnostics import ( + Diagnostic, + Diagnostics, + Severity, +) +from databricks.bundles.core._load import ( + load_resources_from_current_package_module, + load_resources_from_module, + load_resources_from_modules, + load_resources_from_package_module, +) +from databricks.bundles.core._location import Location +from databricks.bundles.core._resource import Resource +from databricks.bundles.core._resource_mutator import ResourceMutator +from databricks.bundles.core._resources import Resources +from databricks.bundles.core._variable import ( + Variable, + VariableOr, + VariableOrDict, + VariableOrList, + VariableOrOptional, + variables, +) +from databricks.bundles.core._generated import ( +${mutator_imports}, +) diff --git a/python/codegen/codegen/generated_init.py.tmpl b/python/codegen/codegen/generated_init.py.tmpl new file mode 100644 index 00000000000..ee955fe7a95 --- /dev/null +++ b/python/codegen/codegen/generated_init.py.tmpl @@ -0,0 +1,26 @@ +from typing import TYPE_CHECKING + +${imports} + +if TYPE_CHECKING: + from databricks.bundles.core._resource_type import _ResourceType + +__all__ = [ +${all_block} +] + + +class _GeneratedResources( +${mixins} +): + pass + + +def _all_resource_types() -> "tuple[_ResourceType, ...]": + from databricks.bundles.core._generated import ( +${module_imports} + ) + + return ( +${type_entries} + ) diff --git a/python/codegen/codegen/generated_wiring.py b/python/codegen/codegen/generated_wiring.py new file mode 100644 index 00000000000..b9ae99015d0 --- /dev/null +++ b/python/codegen/codegen/generated_wiring.py @@ -0,0 +1,160 @@ +""" +Generates the per-resource wiring in databricks.bundles.core that used to be +hand-written. All output is rendered from the *.py.tmpl templates in this +directory. + +Each wired resource gets its own file _generated/.py (rendered from +wiring_resource.py.tmpl): the add_* method + collection property mixin, the +*_mutator decorator, and the _ResourceType entry. The generated +_generated/__init__.py (generated_init.py.tmpl) collects them into +_GeneratedResources (mixed into Resources), _all_resource_types(), and the +mutator re-exports. The core package __init__ (core_init.py.tmpl) is generated +too (static exports plus the generated mutators). +""" + +from dataclasses import dataclass +from pathlib import Path +from string import Template + +import codegen.packages as packages + +HEADER = "# Code generated by pydabs-codegen. DO NOT EDIT.\n\n" + + +def _load_template(name: str) -> Template: + return Template((Path(__file__).parent / name).read_text()) + + +_RESOURCE_TEMPLATE = _load_template("wiring_resource.py.tmpl") +_GENERATED_INIT_TEMPLATE = _load_template("generated_init.py.tmpl") +_CORE_INIT_TEMPLATE = _load_template("core_init.py.tmpl") + + +@dataclass(frozen=True) +class _WiredResource: + class_name: str + """Resource dataclass name, e.g. "Job".""" + + singular_name: str + """Singular name used in methods and messages, e.g. "job".""" + + plural_name: str + """Plural name, the same as the "resources" bundle section, e.g. "jobs".""" + + model_module: str + """Module the resource dataclass lives in, e.g. "databricks.bundles.jobs._models.job".""" + + +def _wired_resources() -> list[_WiredResource]: + # Every namespaced resource is wired. RESOURCE_NAMESPACE is the single source + # of truth for both model generation and wiring. + resources = [] + + for ref, namespace in packages.RESOURCE_NAMESPACE.items(): + class_name = packages.get_class_name(ref) + + resources.append( + _WiredResource( + class_name=class_name, + singular_name=class_name.lower(), + plural_name=namespace, + model_module=packages.get_package(namespace, ref), + ) + ) + + resources.sort(key=lambda r: r.plural_name) + + return resources + + +def write_wiring(output: str): + resources = _wired_resources() + + core_path = Path(output) / "databricks" / "bundles" / "core" + generated_path = core_path / "_generated" + generated_path.mkdir(parents=True, exist_ok=True) + + for r in resources: + code = _RESOURCE_TEMPLATE.substitute( + { + "class": r.class_name, + "singular": r.singular_name, + "plural": r.plural_name, + "model_module": r.model_module, + } + ) + (generated_path / f"{r.plural_name}.py").write_text(HEADER + code) + + (generated_path / "__init__.py").write_text(HEADER + _collector_code(resources)) + (core_path / "__init__.py").write_text(HEADER + _core_init_code(resources)) + + print(f"Writing wiring into {generated_path}") + + +def _collector_code(resources: list[_WiredResource]) -> str: + imports = "\n".join( + f"from databricks.bundles.core._generated.{r.plural_name} import " + f"_{r.class_name}Resources, {r.singular_name}_mutator" + for r in resources + ) + + mutator_names = [f"{r.singular_name}_mutator" for r in resources] + exports = sorted(["_GeneratedResources", "_all_resource_types", *mutator_names]) + all_block = "\n".join(f' "{name}",' for name in exports) + + mixins = "\n".join(f" _{r.class_name}Resources," for r in resources) + module_imports = "\n".join(f" {r.plural_name}," for r in resources) + type_entries = "\n".join( + f" {r.plural_name}._resource_type()," for r in resources + ) + + return _GENERATED_INIT_TEMPLATE.substitute( + { + "imports": imports, + "all_block": all_block, + "mixins": mixins, + "module_imports": module_imports, + "type_entries": type_entries, + } + ) + + +# Static, non-resource exports of databricks.bundles.core, used to build the +# sorted __all__ (the resource mutators are appended). Must stay in sync with the +# import statements in core_init.py.tmpl. +_CORE_INIT_STATIC_EXPORTS = [ + "Bundle", + "Diagnostic", + "Diagnostics", + "Location", + "Resource", + "ResourceMutator", + "Resources", + "Severity", + "Variable", + "VariableOr", + "VariableOrDict", + "VariableOrList", + "VariableOrOptional", + "load_resources_from_current_package_module", + "load_resources_from_module", + "load_resources_from_modules", + "load_resources_from_package_module", + "variables", +] + + +def _core_init_code(resources: list[_WiredResource]) -> str: + mutator_names = [f"{r.singular_name}_mutator" for r in resources] + + all_exports = sorted(_CORE_INIT_STATIC_EXPORTS + mutator_names) + all_block = "\n".join(f' "{name}",' for name in all_exports) + + mutator_imports = ",\n".join(f" {name}" for name in sorted(mutator_names)) + + return _CORE_INIT_TEMPLATE.substitute( + { + "all_block": all_block, + "mutator_imports": mutator_imports, + } + ) diff --git a/python/codegen/codegen/main.py b/python/codegen/codegen/main.py index 2cd3fcd91a5..7927da85961 100644 --- a/python/codegen/codegen/main.py +++ b/python/codegen/codegen/main.py @@ -8,6 +8,7 @@ import codegen.generated_dataclass_patch as generated_dataclass_patch import codegen.generated_enum as generated_enum import codegen.generated_imports as generated_imports +import codegen.generated_wiring as generated_wiring import codegen.jsonschema as openapi import codegen.jsonschema_patch as openapi_patch import codegen.packages as packages @@ -46,6 +47,11 @@ def main(output: str): _write_exports(namespace, dataclasses, enums, output) + # Generate the per-resource wiring in databricks.bundles.core (the + # _ResourceType registry, Resources add_*/property methods, *_mutator + # decorators, and the core package __init__). + generated_wiring.write_wiring(output) + def _transitively_mark_deprecated_and_private( roots: list[str], diff --git a/python/codegen/codegen/wiring_resource.py.tmpl b/python/codegen/codegen/wiring_resource.py.tmpl new file mode 100644 index 00000000000..90a2ea0f157 --- /dev/null +++ b/python/codegen/codegen/wiring_resource.py.tmpl @@ -0,0 +1,113 @@ +from collections.abc import Callable +from typing import TYPE_CHECKING, Optional, overload + +from databricks.bundles.core._bundle import Bundle +from databricks.bundles.core._location import Location +from databricks.bundles.core._resource_mutator import ResourceMutator +from databricks.bundles.core._transform import _transform + +if TYPE_CHECKING: + from databricks.bundles.core._resource_type import _ResourceType + from ${model_module} import ${class}, ${class}Param + + +def _resource_type() -> "_ResourceType": + from databricks.bundles.core._resource_type import _ResourceType + from ${model_module} import ${class} + + return _ResourceType( + resource_type=${class}, + singular_name="${singular}", + plural_name="${plural}", + ) + + +class _${class}Resources: + """ + Generated ${singular} accessors, mixed into Resources. + """ + + # Provided by the Resources subclass; declared here so the generated methods + # below type-check. + _resources: dict[str, dict] + + if TYPE_CHECKING: + + def add_location(self, path: tuple[str, ...], location: Location) -> None: ... + + def add_diagnostic_error( + self, + msg: str, + *, + detail: Optional[str] = None, + path: Optional[tuple[str, ...]] = None, + location: Optional[Location] = None, + ) -> None: ... + + @property + def ${plural}(self) -> dict[str, "${class}"]: + return self._resources["${plural}"] + + def add_${singular}( + self, + resource_name: str, + ${singular}: "${class}Param", + *, + location: Optional[Location] = None, + ) -> None: + """ + Adds the resource ${singular} to the collection of resources. Resource name must be unique across all ${plural}. + + :param resource_name: unique identifier for the ${singular} + :param ${singular}: the ${singular} to add, can be ${class} or dict + :param location: optional location of the ${singular} in the source code + """ + from ${model_module} import ${class} + + ${singular} = _transform(${class}, ${singular}) + path = ("resources", "${plural}", resource_name) + location = location or Location.from_stack_frame(depth=1) + + if self._resources["${plural}"].get(resource_name): + self.add_diagnostic_error( + msg=f"Duplicate resource name '{resource_name}' for resource '${singular}'. Resource names must be unique.", + location=location, + path=path, + ) + else: + if location: + self.add_location(path, location) + + self._resources["${plural}"][resource_name] = ${singular} + + +@overload +def ${singular}_mutator( + function: Callable[[Bundle, "${class}"], "${class}"], +) -> ResourceMutator["${class}"]: ... + + +@overload +def ${singular}_mutator( + function: Callable[["${class}"], "${class}"], +) -> ResourceMutator["${class}"]: ... + + +def ${singular}_mutator(function: Callable) -> ResourceMutator["${class}"]: + """ + Decorator for defining mutator for ${plural}. Function should return a new instance of the ${singular} + with the desired changes, instead of mutating the input ${singular}. + + Example: + + .. code-block:: python + + @${singular}_mutator + def my_${singular}_mutator(bundle: Bundle, ${singular}: ${class}) -> ${class}: + return replace(${singular}, ...) + + :param function: Function that mutates ${plural}. + """ + from ${model_module} import ${class} + + return ResourceMutator(resource_type=${class}, function=function) diff --git a/python/databricks/bundles/.gitattributes b/python/databricks/bundles/.gitattributes index 747d44f1640..658274265f1 100644 --- a/python/databricks/bundles/.gitattributes +++ b/python/databricks/bundles/.gitattributes @@ -1,5 +1,6 @@ # Generated by pydabs-codegen (see python/codegen). Each generated namespace -# has a _models/ tree and an __init__.py; core/ is hand-written, so unset it. +# has a _models/ tree and an __init__.py. In core/, only the _generated/ tree +# and the package __init__.py are generated; the rest is hand-written. */_models/** linguist-generated=true */__init__.py linguist-generated=true -core/__init__.py linguist-generated=false +core/_generated/** linguist-generated=true diff --git a/python/databricks/bundles/core/__init__.py b/python/databricks/bundles/core/__init__.py index 98abbaaf458..cbf4661aed8 100644 --- a/python/databricks/bundles/core/__init__.py +++ b/python/databricks/bundles/core/__init__.py @@ -1,3 +1,5 @@ +# Code generated by pydabs-codegen. DO NOT EDIT. + __all__ = [ "Bundle", "Diagnostic", @@ -31,6 +33,14 @@ Diagnostics, Severity, ) +from databricks.bundles.core._generated import ( + alert_mutator, + catalog_mutator, + job_mutator, + pipeline_mutator, + schema_mutator, + volume_mutator, +) from databricks.bundles.core._load import ( load_resources_from_current_package_module, load_resources_from_module, @@ -39,15 +49,7 @@ ) from databricks.bundles.core._location import Location from databricks.bundles.core._resource import Resource -from databricks.bundles.core._resource_mutator import ( - ResourceMutator, - alert_mutator, - catalog_mutator, - job_mutator, - pipeline_mutator, - schema_mutator, - volume_mutator, -) +from databricks.bundles.core._resource_mutator import ResourceMutator from databricks.bundles.core._resources import Resources from databricks.bundles.core._variable import ( Variable, diff --git a/python/databricks/bundles/core/_generated/__init__.py b/python/databricks/bundles/core/_generated/__init__.py new file mode 100644 index 00000000000..ade9313238c --- /dev/null +++ b/python/databricks/bundles/core/_generated/__init__.py @@ -0,0 +1,61 @@ +# Code generated by pydabs-codegen. DO NOT EDIT. + +from typing import TYPE_CHECKING + +from databricks.bundles.core._generated.alerts import _AlertResources, alert_mutator +from databricks.bundles.core._generated.catalogs import ( + _CatalogResources, + catalog_mutator, +) +from databricks.bundles.core._generated.jobs import _JobResources, job_mutator +from databricks.bundles.core._generated.pipelines import ( + _PipelineResources, + pipeline_mutator, +) +from databricks.bundles.core._generated.schemas import _SchemaResources, schema_mutator +from databricks.bundles.core._generated.volumes import _VolumeResources, volume_mutator + +if TYPE_CHECKING: + from databricks.bundles.core._resource_type import _ResourceType + +__all__ = [ + "_GeneratedResources", + "_all_resource_types", + "alert_mutator", + "catalog_mutator", + "job_mutator", + "pipeline_mutator", + "schema_mutator", + "volume_mutator", +] + + +class _GeneratedResources( + _AlertResources, + _CatalogResources, + _JobResources, + _PipelineResources, + _SchemaResources, + _VolumeResources, +): + pass + + +def _all_resource_types() -> "tuple[_ResourceType, ...]": + from databricks.bundles.core._generated import ( + alerts, + catalogs, + jobs, + pipelines, + schemas, + volumes, + ) + + return ( + alerts._resource_type(), + catalogs._resource_type(), + jobs._resource_type(), + pipelines._resource_type(), + schemas._resource_type(), + volumes._resource_type(), + ) diff --git a/python/databricks/bundles/core/_generated/alerts.py b/python/databricks/bundles/core/_generated/alerts.py new file mode 100644 index 00000000000..94e4af9a136 --- /dev/null +++ b/python/databricks/bundles/core/_generated/alerts.py @@ -0,0 +1,115 @@ +# Code generated by pydabs-codegen. DO NOT EDIT. + +from collections.abc import Callable +from typing import TYPE_CHECKING, Optional, overload + +from databricks.bundles.core._bundle import Bundle +from databricks.bundles.core._location import Location +from databricks.bundles.core._resource_mutator import ResourceMutator +from databricks.bundles.core._transform import _transform + +if TYPE_CHECKING: + from databricks.bundles.alerts._models.alert import Alert, AlertParam + from databricks.bundles.core._resource_type import _ResourceType + + +def _resource_type() -> "_ResourceType": + from databricks.bundles.alerts._models.alert import Alert + from databricks.bundles.core._resource_type import _ResourceType + + return _ResourceType( + resource_type=Alert, + singular_name="alert", + plural_name="alerts", + ) + + +class _AlertResources: + """ + Generated alert accessors, mixed into Resources. + """ + + # Provided by the Resources subclass; declared here so the generated methods + # below type-check. + _resources: dict[str, dict] + + if TYPE_CHECKING: + + def add_location(self, path: tuple[str, ...], location: Location) -> None: ... + + def add_diagnostic_error( + self, + msg: str, + *, + detail: Optional[str] = None, + path: Optional[tuple[str, ...]] = None, + location: Optional[Location] = None, + ) -> None: ... + + @property + def alerts(self) -> dict[str, "Alert"]: + return self._resources["alerts"] + + def add_alert( + self, + resource_name: str, + alert: "AlertParam", + *, + location: Optional[Location] = None, + ) -> None: + """ + Adds the resource alert to the collection of resources. Resource name must be unique across all alerts. + + :param resource_name: unique identifier for the alert + :param alert: the alert to add, can be Alert or dict + :param location: optional location of the alert in the source code + """ + from databricks.bundles.alerts._models.alert import Alert + + alert = _transform(Alert, alert) + path = ("resources", "alerts", resource_name) + location = location or Location.from_stack_frame(depth=1) + + if self._resources["alerts"].get(resource_name): + self.add_diagnostic_error( + msg=f"Duplicate resource name '{resource_name}' for resource 'alert'. Resource names must be unique.", + location=location, + path=path, + ) + else: + if location: + self.add_location(path, location) + + self._resources["alerts"][resource_name] = alert + + +@overload +def alert_mutator( + function: Callable[[Bundle, "Alert"], "Alert"], +) -> ResourceMutator["Alert"]: ... + + +@overload +def alert_mutator( + function: Callable[["Alert"], "Alert"], +) -> ResourceMutator["Alert"]: ... + + +def alert_mutator(function: Callable) -> ResourceMutator["Alert"]: + """ + Decorator for defining mutator for alerts. Function should return a new instance of the alert + with the desired changes, instead of mutating the input alert. + + Example: + + .. code-block:: python + + @alert_mutator + def my_alert_mutator(bundle: Bundle, alert: Alert) -> Alert: + return replace(alert, ...) + + :param function: Function that mutates alerts. + """ + from databricks.bundles.alerts._models.alert import Alert + + return ResourceMutator(resource_type=Alert, function=function) diff --git a/python/databricks/bundles/core/_generated/catalogs.py b/python/databricks/bundles/core/_generated/catalogs.py new file mode 100644 index 00000000000..55d6bbcd310 --- /dev/null +++ b/python/databricks/bundles/core/_generated/catalogs.py @@ -0,0 +1,115 @@ +# Code generated by pydabs-codegen. DO NOT EDIT. + +from collections.abc import Callable +from typing import TYPE_CHECKING, Optional, overload + +from databricks.bundles.core._bundle import Bundle +from databricks.bundles.core._location import Location +from databricks.bundles.core._resource_mutator import ResourceMutator +from databricks.bundles.core._transform import _transform + +if TYPE_CHECKING: + from databricks.bundles.catalogs._models.catalog import Catalog, CatalogParam + from databricks.bundles.core._resource_type import _ResourceType + + +def _resource_type() -> "_ResourceType": + from databricks.bundles.catalogs._models.catalog import Catalog + from databricks.bundles.core._resource_type import _ResourceType + + return _ResourceType( + resource_type=Catalog, + singular_name="catalog", + plural_name="catalogs", + ) + + +class _CatalogResources: + """ + Generated catalog accessors, mixed into Resources. + """ + + # Provided by the Resources subclass; declared here so the generated methods + # below type-check. + _resources: dict[str, dict] + + if TYPE_CHECKING: + + def add_location(self, path: tuple[str, ...], location: Location) -> None: ... + + def add_diagnostic_error( + self, + msg: str, + *, + detail: Optional[str] = None, + path: Optional[tuple[str, ...]] = None, + location: Optional[Location] = None, + ) -> None: ... + + @property + def catalogs(self) -> dict[str, "Catalog"]: + return self._resources["catalogs"] + + def add_catalog( + self, + resource_name: str, + catalog: "CatalogParam", + *, + location: Optional[Location] = None, + ) -> None: + """ + Adds the resource catalog to the collection of resources. Resource name must be unique across all catalogs. + + :param resource_name: unique identifier for the catalog + :param catalog: the catalog to add, can be Catalog or dict + :param location: optional location of the catalog in the source code + """ + from databricks.bundles.catalogs._models.catalog import Catalog + + catalog = _transform(Catalog, catalog) + path = ("resources", "catalogs", resource_name) + location = location or Location.from_stack_frame(depth=1) + + if self._resources["catalogs"].get(resource_name): + self.add_diagnostic_error( + msg=f"Duplicate resource name '{resource_name}' for resource 'catalog'. Resource names must be unique.", + location=location, + path=path, + ) + else: + if location: + self.add_location(path, location) + + self._resources["catalogs"][resource_name] = catalog + + +@overload +def catalog_mutator( + function: Callable[[Bundle, "Catalog"], "Catalog"], +) -> ResourceMutator["Catalog"]: ... + + +@overload +def catalog_mutator( + function: Callable[["Catalog"], "Catalog"], +) -> ResourceMutator["Catalog"]: ... + + +def catalog_mutator(function: Callable) -> ResourceMutator["Catalog"]: + """ + Decorator for defining mutator for catalogs. Function should return a new instance of the catalog + with the desired changes, instead of mutating the input catalog. + + Example: + + .. code-block:: python + + @catalog_mutator + def my_catalog_mutator(bundle: Bundle, catalog: Catalog) -> Catalog: + return replace(catalog, ...) + + :param function: Function that mutates catalogs. + """ + from databricks.bundles.catalogs._models.catalog import Catalog + + return ResourceMutator(resource_type=Catalog, function=function) diff --git a/python/databricks/bundles/core/_generated/jobs.py b/python/databricks/bundles/core/_generated/jobs.py new file mode 100644 index 00000000000..8653c55055d --- /dev/null +++ b/python/databricks/bundles/core/_generated/jobs.py @@ -0,0 +1,115 @@ +# Code generated by pydabs-codegen. DO NOT EDIT. + +from collections.abc import Callable +from typing import TYPE_CHECKING, Optional, overload + +from databricks.bundles.core._bundle import Bundle +from databricks.bundles.core._location import Location +from databricks.bundles.core._resource_mutator import ResourceMutator +from databricks.bundles.core._transform import _transform + +if TYPE_CHECKING: + from databricks.bundles.core._resource_type import _ResourceType + from databricks.bundles.jobs._models.job import Job, JobParam + + +def _resource_type() -> "_ResourceType": + from databricks.bundles.core._resource_type import _ResourceType + from databricks.bundles.jobs._models.job import Job + + return _ResourceType( + resource_type=Job, + singular_name="job", + plural_name="jobs", + ) + + +class _JobResources: + """ + Generated job accessors, mixed into Resources. + """ + + # Provided by the Resources subclass; declared here so the generated methods + # below type-check. + _resources: dict[str, dict] + + if TYPE_CHECKING: + + def add_location(self, path: tuple[str, ...], location: Location) -> None: ... + + def add_diagnostic_error( + self, + msg: str, + *, + detail: Optional[str] = None, + path: Optional[tuple[str, ...]] = None, + location: Optional[Location] = None, + ) -> None: ... + + @property + def jobs(self) -> dict[str, "Job"]: + return self._resources["jobs"] + + def add_job( + self, + resource_name: str, + job: "JobParam", + *, + location: Optional[Location] = None, + ) -> None: + """ + Adds the resource job to the collection of resources. Resource name must be unique across all jobs. + + :param resource_name: unique identifier for the job + :param job: the job to add, can be Job or dict + :param location: optional location of the job in the source code + """ + from databricks.bundles.jobs._models.job import Job + + job = _transform(Job, job) + path = ("resources", "jobs", resource_name) + location = location or Location.from_stack_frame(depth=1) + + if self._resources["jobs"].get(resource_name): + self.add_diagnostic_error( + msg=f"Duplicate resource name '{resource_name}' for resource 'job'. Resource names must be unique.", + location=location, + path=path, + ) + else: + if location: + self.add_location(path, location) + + self._resources["jobs"][resource_name] = job + + +@overload +def job_mutator( + function: Callable[[Bundle, "Job"], "Job"], +) -> ResourceMutator["Job"]: ... + + +@overload +def job_mutator( + function: Callable[["Job"], "Job"], +) -> ResourceMutator["Job"]: ... + + +def job_mutator(function: Callable) -> ResourceMutator["Job"]: + """ + Decorator for defining mutator for jobs. Function should return a new instance of the job + with the desired changes, instead of mutating the input job. + + Example: + + .. code-block:: python + + @job_mutator + def my_job_mutator(bundle: Bundle, job: Job) -> Job: + return replace(job, ...) + + :param function: Function that mutates jobs. + """ + from databricks.bundles.jobs._models.job import Job + + return ResourceMutator(resource_type=Job, function=function) diff --git a/python/databricks/bundles/core/_generated/pipelines.py b/python/databricks/bundles/core/_generated/pipelines.py new file mode 100644 index 00000000000..961070a6e4b --- /dev/null +++ b/python/databricks/bundles/core/_generated/pipelines.py @@ -0,0 +1,115 @@ +# Code generated by pydabs-codegen. DO NOT EDIT. + +from collections.abc import Callable +from typing import TYPE_CHECKING, Optional, overload + +from databricks.bundles.core._bundle import Bundle +from databricks.bundles.core._location import Location +from databricks.bundles.core._resource_mutator import ResourceMutator +from databricks.bundles.core._transform import _transform + +if TYPE_CHECKING: + from databricks.bundles.core._resource_type import _ResourceType + from databricks.bundles.pipelines._models.pipeline import Pipeline, PipelineParam + + +def _resource_type() -> "_ResourceType": + from databricks.bundles.core._resource_type import _ResourceType + from databricks.bundles.pipelines._models.pipeline import Pipeline + + return _ResourceType( + resource_type=Pipeline, + singular_name="pipeline", + plural_name="pipelines", + ) + + +class _PipelineResources: + """ + Generated pipeline accessors, mixed into Resources. + """ + + # Provided by the Resources subclass; declared here so the generated methods + # below type-check. + _resources: dict[str, dict] + + if TYPE_CHECKING: + + def add_location(self, path: tuple[str, ...], location: Location) -> None: ... + + def add_diagnostic_error( + self, + msg: str, + *, + detail: Optional[str] = None, + path: Optional[tuple[str, ...]] = None, + location: Optional[Location] = None, + ) -> None: ... + + @property + def pipelines(self) -> dict[str, "Pipeline"]: + return self._resources["pipelines"] + + def add_pipeline( + self, + resource_name: str, + pipeline: "PipelineParam", + *, + location: Optional[Location] = None, + ) -> None: + """ + Adds the resource pipeline to the collection of resources. Resource name must be unique across all pipelines. + + :param resource_name: unique identifier for the pipeline + :param pipeline: the pipeline to add, can be Pipeline or dict + :param location: optional location of the pipeline in the source code + """ + from databricks.bundles.pipelines._models.pipeline import Pipeline + + pipeline = _transform(Pipeline, pipeline) + path = ("resources", "pipelines", resource_name) + location = location or Location.from_stack_frame(depth=1) + + if self._resources["pipelines"].get(resource_name): + self.add_diagnostic_error( + msg=f"Duplicate resource name '{resource_name}' for resource 'pipeline'. Resource names must be unique.", + location=location, + path=path, + ) + else: + if location: + self.add_location(path, location) + + self._resources["pipelines"][resource_name] = pipeline + + +@overload +def pipeline_mutator( + function: Callable[[Bundle, "Pipeline"], "Pipeline"], +) -> ResourceMutator["Pipeline"]: ... + + +@overload +def pipeline_mutator( + function: Callable[["Pipeline"], "Pipeline"], +) -> ResourceMutator["Pipeline"]: ... + + +def pipeline_mutator(function: Callable) -> ResourceMutator["Pipeline"]: + """ + Decorator for defining mutator for pipelines. Function should return a new instance of the pipeline + with the desired changes, instead of mutating the input pipeline. + + Example: + + .. code-block:: python + + @pipeline_mutator + def my_pipeline_mutator(bundle: Bundle, pipeline: Pipeline) -> Pipeline: + return replace(pipeline, ...) + + :param function: Function that mutates pipelines. + """ + from databricks.bundles.pipelines._models.pipeline import Pipeline + + return ResourceMutator(resource_type=Pipeline, function=function) diff --git a/python/databricks/bundles/core/_generated/schemas.py b/python/databricks/bundles/core/_generated/schemas.py new file mode 100644 index 00000000000..e83fe02c415 --- /dev/null +++ b/python/databricks/bundles/core/_generated/schemas.py @@ -0,0 +1,115 @@ +# Code generated by pydabs-codegen. DO NOT EDIT. + +from collections.abc import Callable +from typing import TYPE_CHECKING, Optional, overload + +from databricks.bundles.core._bundle import Bundle +from databricks.bundles.core._location import Location +from databricks.bundles.core._resource_mutator import ResourceMutator +from databricks.bundles.core._transform import _transform + +if TYPE_CHECKING: + from databricks.bundles.core._resource_type import _ResourceType + from databricks.bundles.schemas._models.schema import Schema, SchemaParam + + +def _resource_type() -> "_ResourceType": + from databricks.bundles.core._resource_type import _ResourceType + from databricks.bundles.schemas._models.schema import Schema + + return _ResourceType( + resource_type=Schema, + singular_name="schema", + plural_name="schemas", + ) + + +class _SchemaResources: + """ + Generated schema accessors, mixed into Resources. + """ + + # Provided by the Resources subclass; declared here so the generated methods + # below type-check. + _resources: dict[str, dict] + + if TYPE_CHECKING: + + def add_location(self, path: tuple[str, ...], location: Location) -> None: ... + + def add_diagnostic_error( + self, + msg: str, + *, + detail: Optional[str] = None, + path: Optional[tuple[str, ...]] = None, + location: Optional[Location] = None, + ) -> None: ... + + @property + def schemas(self) -> dict[str, "Schema"]: + return self._resources["schemas"] + + def add_schema( + self, + resource_name: str, + schema: "SchemaParam", + *, + location: Optional[Location] = None, + ) -> None: + """ + Adds the resource schema to the collection of resources. Resource name must be unique across all schemas. + + :param resource_name: unique identifier for the schema + :param schema: the schema to add, can be Schema or dict + :param location: optional location of the schema in the source code + """ + from databricks.bundles.schemas._models.schema import Schema + + schema = _transform(Schema, schema) + path = ("resources", "schemas", resource_name) + location = location or Location.from_stack_frame(depth=1) + + if self._resources["schemas"].get(resource_name): + self.add_diagnostic_error( + msg=f"Duplicate resource name '{resource_name}' for resource 'schema'. Resource names must be unique.", + location=location, + path=path, + ) + else: + if location: + self.add_location(path, location) + + self._resources["schemas"][resource_name] = schema + + +@overload +def schema_mutator( + function: Callable[[Bundle, "Schema"], "Schema"], +) -> ResourceMutator["Schema"]: ... + + +@overload +def schema_mutator( + function: Callable[["Schema"], "Schema"], +) -> ResourceMutator["Schema"]: ... + + +def schema_mutator(function: Callable) -> ResourceMutator["Schema"]: + """ + Decorator for defining mutator for schemas. Function should return a new instance of the schema + with the desired changes, instead of mutating the input schema. + + Example: + + .. code-block:: python + + @schema_mutator + def my_schema_mutator(bundle: Bundle, schema: Schema) -> Schema: + return replace(schema, ...) + + :param function: Function that mutates schemas. + """ + from databricks.bundles.schemas._models.schema import Schema + + return ResourceMutator(resource_type=Schema, function=function) diff --git a/python/databricks/bundles/core/_generated/volumes.py b/python/databricks/bundles/core/_generated/volumes.py new file mode 100644 index 00000000000..fe56e45ce30 --- /dev/null +++ b/python/databricks/bundles/core/_generated/volumes.py @@ -0,0 +1,115 @@ +# Code generated by pydabs-codegen. DO NOT EDIT. + +from collections.abc import Callable +from typing import TYPE_CHECKING, Optional, overload + +from databricks.bundles.core._bundle import Bundle +from databricks.bundles.core._location import Location +from databricks.bundles.core._resource_mutator import ResourceMutator +from databricks.bundles.core._transform import _transform + +if TYPE_CHECKING: + from databricks.bundles.core._resource_type import _ResourceType + from databricks.bundles.volumes._models.volume import Volume, VolumeParam + + +def _resource_type() -> "_ResourceType": + from databricks.bundles.core._resource_type import _ResourceType + from databricks.bundles.volumes._models.volume import Volume + + return _ResourceType( + resource_type=Volume, + singular_name="volume", + plural_name="volumes", + ) + + +class _VolumeResources: + """ + Generated volume accessors, mixed into Resources. + """ + + # Provided by the Resources subclass; declared here so the generated methods + # below type-check. + _resources: dict[str, dict] + + if TYPE_CHECKING: + + def add_location(self, path: tuple[str, ...], location: Location) -> None: ... + + def add_diagnostic_error( + self, + msg: str, + *, + detail: Optional[str] = None, + path: Optional[tuple[str, ...]] = None, + location: Optional[Location] = None, + ) -> None: ... + + @property + def volumes(self) -> dict[str, "Volume"]: + return self._resources["volumes"] + + def add_volume( + self, + resource_name: str, + volume: "VolumeParam", + *, + location: Optional[Location] = None, + ) -> None: + """ + Adds the resource volume to the collection of resources. Resource name must be unique across all volumes. + + :param resource_name: unique identifier for the volume + :param volume: the volume to add, can be Volume or dict + :param location: optional location of the volume in the source code + """ + from databricks.bundles.volumes._models.volume import Volume + + volume = _transform(Volume, volume) + path = ("resources", "volumes", resource_name) + location = location or Location.from_stack_frame(depth=1) + + if self._resources["volumes"].get(resource_name): + self.add_diagnostic_error( + msg=f"Duplicate resource name '{resource_name}' for resource 'volume'. Resource names must be unique.", + location=location, + path=path, + ) + else: + if location: + self.add_location(path, location) + + self._resources["volumes"][resource_name] = volume + + +@overload +def volume_mutator( + function: Callable[[Bundle, "Volume"], "Volume"], +) -> ResourceMutator["Volume"]: ... + + +@overload +def volume_mutator( + function: Callable[["Volume"], "Volume"], +) -> ResourceMutator["Volume"]: ... + + +def volume_mutator(function: Callable) -> ResourceMutator["Volume"]: + """ + Decorator for defining mutator for volumes. Function should return a new instance of the volume + with the desired changes, instead of mutating the input volume. + + Example: + + .. code-block:: python + + @volume_mutator + def my_volume_mutator(bundle: Bundle, volume: Volume) -> Volume: + return replace(volume, ...) + + :param function: Function that mutates volumes. + """ + from databricks.bundles.volumes._models.volume import Volume + + return ResourceMutator(resource_type=Volume, function=function) diff --git a/python/databricks/bundles/core/_resource_mutator.py b/python/databricks/bundles/core/_resource_mutator.py index fafdcdc5efb..22ea17cfc51 100644 --- a/python/databricks/bundles/core/_resource_mutator.py +++ b/python/databricks/bundles/core/_resource_mutator.py @@ -1,18 +1,9 @@ from collections.abc import Callable from dataclasses import dataclass -from typing import TYPE_CHECKING, Generic, Type, TypeVar, overload +from typing import Generic, Type, TypeVar -from databricks.bundles.core._bundle import Bundle from databricks.bundles.core._resource import Resource -if TYPE_CHECKING: - from databricks.bundles.alerts._models.alert import Alert - from databricks.bundles.catalogs._models.catalog import Catalog - from databricks.bundles.jobs._models.job import Job - from databricks.bundles.pipelines._models.pipeline import Pipeline - from databricks.bundles.schemas._models.schema import Schema - from databricks.bundles.volumes._models.volume import Volume - _T = TypeVar("_T", bound=Resource) @@ -57,8 +48,9 @@ def my_job_mutator(bundle: Bundle, job: Job) -> Job: """ -# Below, we define decorators for each resource type. This approach allows us -# to implement mutators that are only applied for specific resource types. +# A decorator is generated for each resource type (see +# _generated/_resource_mutators.py). This approach allows us to implement +# mutators that are only applied for specific resource types. # # Alternative approaches considered and rejected during design: # @@ -69,193 +61,3 @@ def my_job_mutator(bundle: Bundle, job: Job) -> Job: # - Using a universal @mutator decorator. # Rationale: Determining whether a mutator is invoked based solely on type annotations # was deemed overly implicit and potentially confusing. - - -@overload -def alert_mutator( - function: Callable[[Bundle, "Alert"], "Alert"], -) -> ResourceMutator["Alert"]: ... - - -@overload -def alert_mutator( - function: Callable[["Alert"], "Alert"], -) -> ResourceMutator["Alert"]: ... - - -def alert_mutator(function: Callable) -> ResourceMutator["Alert"]: - """ - Decorator for defining an alert mutator. Function should return a new instance of the alert with the desired changes, - instead of mutating the input alert. - - Example: - - .. code-block:: python - - @alert_mutator - def my_alert_mutator(bundle: Bundle, alert: Alert) -> Alert: - return replace(alert, display_name="my_alert") - - :param function: Function that mutates an alert. - """ - from databricks.bundles.alerts._models.alert import Alert - - return ResourceMutator(resource_type=Alert, function=function) - - -@overload -def catalog_mutator( - function: Callable[[Bundle, "Catalog"], "Catalog"], -) -> ResourceMutator["Catalog"]: ... - - -@overload -def catalog_mutator( - function: Callable[["Catalog"], "Catalog"], -) -> ResourceMutator["Catalog"]: ... - - -def catalog_mutator(function: Callable) -> ResourceMutator["Catalog"]: - """ - Decorator for defining a catalog mutator. Function should return a new instance of the catalog with the desired changes, - instead of mutating the input catalog. - - Example: - - .. code-block:: python - - @catalog_mutator - def my_catalog_mutator(bundle: Bundle, catalog: Catalog) -> Catalog: - return replace(catalog, name="my_catalog") - - :param function: Function that mutates a catalog. - """ - from databricks.bundles.catalogs._models.catalog import Catalog - - return ResourceMutator(resource_type=Catalog, function=function) - - -@overload -def job_mutator( - function: Callable[[Bundle, "Job"], "Job"], -) -> ResourceMutator["Job"]: ... - - -@overload -def job_mutator(function: Callable[["Job"], "Job"]) -> ResourceMutator["Job"]: ... - - -def job_mutator(function: Callable) -> ResourceMutator["Job"]: - """ - Decorator for defining a job mutator. Function should return a new instance of the job with the desired changes, - instead of mutating the input job. - - Example: - - .. code-block:: python - - @job_mutator - def my_job_mutator(bundle: Bundle, job: Job) -> Job: - return replace(job, name="my_job") - - :param function: Function that mutates a job. - """ - from databricks.bundles.jobs._models.job import Job - - return ResourceMutator(resource_type=Job, function=function) - - -@overload -def pipeline_mutator( - function: Callable[[Bundle, "Pipeline"], "Pipeline"], -) -> ResourceMutator["Pipeline"]: ... - - -@overload -def pipeline_mutator( - function: Callable[["Pipeline"], "Pipeline"], -) -> ResourceMutator["Pipeline"]: ... - - -def pipeline_mutator(function: Callable) -> ResourceMutator["Pipeline"]: - """ - Decorator for defining a pipeline mutator. Function should return a new instance of the pipeline with the desired changes, - instead of mutating the input pipeline. - - Example: - - .. code-block:: python - - @pipeline_mutator - def my_pipeline_mutator(bundle: Bundle, pipeline: Pipeline) -> Pipeline: - return replace(pipeline, name="my_job") - - :param function: Function that mutates a pipeline. - """ - from databricks.bundles.pipelines._models.pipeline import Pipeline - - return ResourceMutator(resource_type=Pipeline, function=function) - - -@overload -def schema_mutator( - function: Callable[[Bundle, "Schema"], "Schema"], -) -> ResourceMutator["Schema"]: ... - - -@overload -def schema_mutator( - function: Callable[["Schema"], "Schema"], -) -> ResourceMutator["Schema"]: ... - - -def schema_mutator(function: Callable) -> ResourceMutator["Schema"]: - """ - Decorator for defining a schema mutator. Function should return a new instance of the schema with the desired changes, - instead of mutating the input schema. - - Example: - - .. code-block:: python - - @schema_mutator - def my_schema_mutator(bundle: Bundle, schema: Schema) -> Schema: - return replace(schema, name="my_schema") - - :param function: Function that mutates a schema. - """ - from databricks.bundles.schemas._models.schema import Schema - - return ResourceMutator(resource_type=Schema, function=function) - - -@overload -def volume_mutator( - function: Callable[[Bundle, "Volume"], "Volume"], -) -> ResourceMutator["Volume"]: ... - - -@overload -def volume_mutator( - function: Callable[["Volume"], "Volume"], -) -> ResourceMutator["Volume"]: ... - - -def volume_mutator(function: Callable) -> ResourceMutator["Volume"]: - """ - Decorator for defining a volume mutator. Function should return a new instance of the volume with the desired changes, - instead of mutating the input volume. - - Example: - - .. code-block:: python - - @volume_mutator - def my_volume_mutator(bundle: Bundle, volume: Volume) -> Volume: - return replace(volume, name="my_volume") - - :param function: Function that mutates a volume. - """ - from databricks.bundles.volumes._models.volume import Volume - - return ResourceMutator(resource_type=Volume, function=function) diff --git a/python/databricks/bundles/core/_resource_type.py b/python/databricks/bundles/core/_resource_type.py index 73d30a92072..53fd17c169b 100644 --- a/python/databricks/bundles/core/_resource_type.py +++ b/python/databricks/bundles/core/_resource_type.py @@ -27,46 +27,6 @@ def all(cls) -> tuple["_ResourceType", ...]: """ Returns all supported resource types. """ + from databricks.bundles.core._generated import _all_resource_types - # intentionally lazily load all resource types to avoid imports from databricks.bundles.core to - # be imported in databricks.bundles. - - from databricks.bundles.alerts._models.alert import Alert - from databricks.bundles.catalogs._models.catalog import Catalog - from databricks.bundles.jobs._models.job import Job - from databricks.bundles.pipelines._models.pipeline import Pipeline - from databricks.bundles.schemas._models.schema import Schema - from databricks.bundles.volumes._models.volume import Volume - - return ( - _ResourceType( - resource_type=Job, - singular_name="job", - plural_name="jobs", - ), - _ResourceType( - resource_type=Pipeline, - plural_name="pipelines", - singular_name="pipeline", - ), - _ResourceType( - resource_type=Volume, - plural_name="volumes", - singular_name="volume", - ), - _ResourceType( - resource_type=Schema, - plural_name="schemas", - singular_name="schema", - ), - _ResourceType( - resource_type=Alert, - plural_name="alerts", - singular_name="alert", - ), - _ResourceType( - resource_type=Catalog, - plural_name="catalogs", - singular_name="catalog", - ), - ) + return _all_resource_types() diff --git a/python/databricks/bundles/core/_resources.py b/python/databricks/bundles/core/_resources.py index b926f14b750..6818031596c 100644 --- a/python/databricks/bundles/core/_resources.py +++ b/python/databricks/bundles/core/_resources.py @@ -1,22 +1,15 @@ -from typing import TYPE_CHECKING, Optional +from typing import Optional from databricks.bundles.core._diagnostics import Diagnostics +from databricks.bundles.core._generated import _GeneratedResources from databricks.bundles.core._location import Location from databricks.bundles.core._resource import Resource -from databricks.bundles.core._transform import _transform - -if TYPE_CHECKING: - from databricks.bundles.alerts._models.alert import Alert, AlertParam - from databricks.bundles.catalogs._models.catalog import Catalog, CatalogParam - from databricks.bundles.jobs._models.job import Job, JobParam - from databricks.bundles.pipelines._models.pipeline import Pipeline, PipelineParam - from databricks.bundles.schemas._models.schema import Schema, SchemaParam - from databricks.bundles.volumes._models.volume import Volume, VolumeParam +from databricks.bundles.core._resource_type import _ResourceType __all__ = ["Resources"] -class Resources: +class Resources(_GeneratedResources): """ Resources is a collection of resources in a bundle. @@ -58,31 +51,12 @@ def load_resources(bundle: Bundle) -> Resources: """ def __init__(self): - self._jobs = dict[str, "Job"]() - self._pipelines = dict[str, "Pipeline"]() - self._schemas = dict[str, "Schema"]() - self._volumes = dict[str, "Volume"]() - self._alerts = dict[str, "Alert"]() - self._catalogs = dict[str, "Catalog"]() + self._resources: dict[str, dict] = { + resource_type.plural_name: {} for resource_type in _ResourceType.all() + } self._locations = dict[tuple[str, ...], Location]() self._diagnostics = Diagnostics() - @property - def jobs(self) -> dict[str, "Job"]: - return self._jobs - - @property - def pipelines(self) -> dict[str, "Pipeline"]: - return self._pipelines - - @property - def schemas(self) -> dict[str, "Schema"]: - return self._schemas - - @property - def volumes(self) -> dict[str, "Volume"]: - return self._volumes - @property def diagnostics(self) -> Diagnostics: """ @@ -90,14 +64,6 @@ def diagnostics(self) -> Diagnostics: """ return self._diagnostics - @property - def alerts(self) -> dict[str, "Alert"]: - return self._alerts - - @property - def catalogs(self) -> dict[str, "Catalog"]: - return self._catalogs - def add_resource( self, resource_name: str, @@ -114,218 +80,15 @@ def add_resource( :param location: optional location of the resource in the source code """ - from databricks.bundles.alerts import Alert - from databricks.bundles.catalogs import Catalog - from databricks.bundles.jobs import Job - from databricks.bundles.pipelines import Pipeline - from databricks.bundles.schemas import Schema - from databricks.bundles.volumes import Volume - - location = location or Location.from_stack_frame(depth=1) - - match resource: - case Job(): - self.add_job(resource_name, resource, location=location) - case Pipeline(): - self.add_pipeline(resource_name, resource, location=location) - case Schema(): - self.add_schema(resource_name, resource, location=location) - case Volume(): - self.add_volume(resource_name, resource, location=location) - case Alert(): - self.add_alert(resource_name, resource, location=location) - case Catalog(): - self.add_catalog(resource_name, resource, location=location) - case _: - raise ValueError(f"Unsupported resource type: {type(resource)}") - - def add_job( - self, - resource_name: str, - job: "JobParam", - *, - location: Optional[Location] = None, - ) -> None: - """ - Adds a job to the collection of resources. Resource name must be unique across all jobs. - - :param resource_name: unique identifier for the job - :param job: the job to add, can be Job or dict - :param location: optional location of the job in the source code - """ - from databricks.bundles.jobs import Job - - job = _transform(Job, job) - path = ("resources", "jobs", resource_name) - location = location or Location.from_stack_frame(depth=1) - - if self._jobs.get(resource_name): - self.add_diagnostic_error( - msg=f"Duplicate resource name '{resource_name}' for a job. Resource names must be unique.", - location=location, - path=path, - ) - else: - if location: - self.add_location(path, location) - - self._jobs[resource_name] = job - - def add_pipeline( - self, - resource_name: str, - pipeline: "PipelineParam", - *, - location: Optional[Location] = None, - ) -> None: - """ - Adds a pipeline to the collection of resources. Resource name must be unique across all pipelines. - - :param resource_name: unique identifier for the pipeline - :param pipeline: the pipeline to add, can be Pipeline or dict - :param location: optional location of the pipeline in the source code - """ - from databricks.bundles.pipelines import Pipeline - - pipeline = _transform(Pipeline, pipeline) - path = ("resources", "pipelines", resource_name) - location = location or Location.from_stack_frame(depth=1) - - if self._pipelines.get(resource_name): - self.add_diagnostic_error( - msg=f"Duplicate resource name '{resource_name}' for a pipeline. Resource names must be unique.", - location=location, - path=path, - ) - else: - if location: - self.add_location(path, location) - - self._pipelines[resource_name] = pipeline - - def add_schema( - self, - resource_name: str, - schema: "SchemaParam", - *, - location: Optional[Location] = None, - ) -> None: - """ - Adds a schema to the collection of resources. Resource name must be unique across all schemas. - - :param resource_name: unique identifier for the schema - :param schema: the schema to add, can be Schema or dict - :param location: optional location of the schema in the source code - """ - from databricks.bundles.schemas import Schema - - schema = _transform(Schema, schema) - path = ("resources", "schemas", resource_name) - location = location or Location.from_stack_frame(depth=1) - - if self._schemas.get(resource_name): - self.add_diagnostic_error( - msg=f"Duplicate resource name '{resource_name}' for a schema. Resource names must be unique.", - location=location, - path=path, - ) - else: - if location: - self.add_location(path, location) - - self._schemas[resource_name] = schema - - def add_volume( - self, - resource_name: str, - volume: "VolumeParam", - *, - location: Optional[Location] = None, - ) -> None: - """ - Adds a volume to the collection of resources. Resource name must be unique across all volumes. - - :param resource_name: unique identifier for the volume - :param volume: the volume to add, can be Volume or dict - :param location: optional location of the volume in the source code - """ - from databricks.bundles.volumes import Volume - - volume = _transform(Volume, volume) - path = ("resources", "volumes", resource_name) - location = location or Location.from_stack_frame(depth=1) - - if self._volumes.get(resource_name): - self.add_diagnostic_error( - msg=f"Duplicate resource name '{resource_name}' for a volume. Resource names must be unique.", - location=location, - path=path, - ) - else: - if location: - self.add_location(path, location) - - self._volumes[resource_name] = volume - - def add_alert( - self, - resource_name: str, - alert: "AlertParam", - *, - location: Optional[Location] = None, - ) -> None: - """ - Adds an alert to the collection of resources. Resource name must be unique across all alerts. - """ - from databricks.bundles.alerts import Alert - - alert = _transform(Alert, alert) - path = ("resources", "alerts", resource_name) - location = location or Location.from_stack_frame(depth=1) - - if self._alerts.get(resource_name): - self.add_diagnostic_error( - msg=f"Duplicate resource name '{resource_name}' for an alert. Resource names must be unique.", - location=location, - path=path, - ) - else: - if location: - self.add_location(path, location) - - self._alerts[resource_name] = alert - - def add_catalog( - self, - resource_name: str, - catalog: "CatalogParam", - *, - location: Optional[Location] = None, - ) -> None: - """ - Adds a catalog to the collection of resources. Resource name must be unique across all catalogs. - - :param resource_name: unique identifier for the catalog - :param catalog: the catalog to add, can be Catalog or dict - :param location: optional location of the catalog in the source code - """ - from databricks.bundles.catalogs import Catalog - - catalog = _transform(Catalog, catalog) - path = ("resources", "catalogs", resource_name) location = location or Location.from_stack_frame(depth=1) - if self._catalogs.get(resource_name): - self.add_diagnostic_error( - msg=f"Duplicate resource name '{resource_name}' for a catalog. Resource names must be unique.", - location=location, - path=path, - ) - else: - if location: - self.add_location(path, location) + for resource_type in _ResourceType.all(): + if isinstance(resource, resource_type.resource_type): + add_method = getattr(self, f"add_{resource_type.singular_name}") + add_method(resource_name, resource, location=location) + return - self._catalogs[resource_name] = catalog + raise ValueError(f"Unsupported resource type: {type(resource)}") def add_location(self, path: tuple[str, ...], location: Location) -> None: """ @@ -397,23 +160,10 @@ def add_resources(self, other: "Resources") -> None: Adds error to diagnostics if there are duplicate resource names. """ - for name, job in other.jobs.items(): - self.add_job(name, job) - - for name, pipeline in other.pipelines.items(): - self.add_pipeline(name, pipeline) - - for name, schema in other.schemas.items(): - self.add_schema(name, schema) - - for name, volume in other.volumes.items(): - self.add_volume(name, volume) - - for name, alert in other.alerts.items(): - self.add_alert(name, alert) - - for name, catalog in other.catalogs.items(): - self.add_catalog(name, catalog) + for resource_type in _ResourceType.all(): + add_method = getattr(self, f"add_{resource_type.singular_name}") + for name, resource in getattr(other, resource_type.plural_name).items(): + add_method(name, resource) for path, location in other._locations.items(): self.add_location(path, location) diff --git a/python/databricks_tests/core/public_api.txt b/python/databricks_tests/core/public_api.txt new file mode 100644 index 00000000000..222665796df --- /dev/null +++ b/python/databricks_tests/core/public_api.txt @@ -0,0 +1,145 @@ +== module databricks.bundles.core == +__all__ = [ + Bundle, + Diagnostic, + Diagnostics, + Location, + Resource, + ResourceMutator, + Resources, + Severity, + Variable, + VariableOr, + VariableOrDict, + VariableOrList, + VariableOrOptional, + alert_mutator, + catalog_mutator, + job_mutator, + load_resources_from_current_package_module, + load_resources_from_module, + load_resources_from_modules, + load_resources_from_package_module, + pipeline_mutator, + schema_mutator, + variables, + volume_mutator, +] + +class Bundle: + target: str + variables: dict[str, Any] = + def resolve_variable(self, variable: Union[Variable[_T], _T]) -> _T + def resolve_variable_list(self, variable: Union[Variable[list[Union[Variable[_T], _T]]], list[Union[Variable[_T], _T]]]) -> list[_T] + +class Diagnostic: + severity: Severity + summary: str + detail: Union[str, None] = None + path: Union[tuple[str, ...], None] = None + location: Union[Location, None] = None + def as_dict(self) -> dict + +class Diagnostics: + items: tuple[Diagnostic, ...] = + @classmethod def create_error(msg: str, *, detail: Union[str, None] = None, location: Union[Location, None] = None, path: Union[tuple[str, ...], None] = None) -> Self + @classmethod def create_warning(msg: str, *, detail: Union[str, None] = None, location: Union[Location, None] = None, path: Union[tuple[str, ...], None] = None) -> Self + @classmethod def from_exception(exc: Exception, *, summary: str, location: Union[Location, None] = None, path: Union[tuple[str, ...], None] = None, explanation: Union[str, None] = None) -> Self + def extend(self, diagnostics: Self) -> Self + def extend_tuple(self, pair: tuple[_T, Self]) -> tuple[_T, Self] + def has_error(self) -> bool + def has_warning(self) -> bool + +class Location: + file: str + line: Union[int, None] = None + column: Union[int, None] = None + def as_dict(self) -> dict + def from_callable(fn: Callable) -> Union[Location, None] + def from_stack_frame(depth: int = 0) -> Location + +class Resource: + +class ResourceMutator(Generic): + resource_type: type[_T] + function: Callable + +class Resources: + def add_alert(self, resource_name: str, alert: AlertParam, *, location: Union[Location, None] = None) -> None + def add_catalog(self, resource_name: str, catalog: CatalogParam, *, location: Union[Location, None] = None) -> None + def add_diagnostic_error(self, msg: str, *, detail: Union[str, None] = None, path: Union[tuple[str, ...], None] = None, location: Union[Location, None] = None) -> None + def add_diagnostic_warning(self, msg: str, *, detail: Union[str, None] = None, path: Union[tuple[str, ...], None] = None, location: Union[Location, None] = None) -> None + def add_diagnostics(self, other: Diagnostics) -> None + def add_job(self, resource_name: str, job: JobParam, *, location: Union[Location, None] = None) -> None + def add_location(self, path: tuple[str, ...], location: Location) -> None + def add_pipeline(self, resource_name: str, pipeline: PipelineParam, *, location: Union[Location, None] = None) -> None + def add_resource(self, resource_name: str, resource: Resource, *, location: Union[Location, None] = None) -> None + def add_resources(self, other: Resources) -> None + def add_schema(self, resource_name: str, schema: SchemaParam, *, location: Union[Location, None] = None) -> None + def add_volume(self, resource_name: str, volume: VolumeParam, *, location: Union[Location, None] = None) -> None + @property alerts -> dict[str, Alert] + @property catalogs -> dict[str, Catalog] + @property diagnostics -> Diagnostics + @property jobs -> dict[str, Job] + @property pipelines -> dict[str, Pipeline] + @property schemas -> dict[str, Schema] + @property volumes -> dict[str, Volume] + +class Severity(Enum): + WARNING = 'warning' + ERROR = 'error' + +class Variable(Generic): + path: str + type: type[_T] + @property value -> str + +VariableOr = Union[Variable[_T], _T] + +VariableOrDict = Union[Variable[dict[str, Union[Variable[_T], _T]]], dict[str, Union[Variable[_T], _T]]] + +VariableOrList = Union[Variable[list[Union[Variable[_T], _T]]], list[Union[Variable[_T], _T]]] + +VariableOrOptional = Union[Variable[_T], _T, None] + +@overload def alert_mutator(function: Callable[[Bundle, Alert], Alert]) -> ResourceMutator[Alert] +@overload def alert_mutator(function: Callable[[Alert], Alert]) -> ResourceMutator[Alert] +def alert_mutator(function: Callable) -> ResourceMutator[Alert] + +@overload def catalog_mutator(function: Callable[[Bundle, Catalog], Catalog]) -> ResourceMutator[Catalog] +@overload def catalog_mutator(function: Callable[[Catalog], Catalog]) -> ResourceMutator[Catalog] +def catalog_mutator(function: Callable) -> ResourceMutator[Catalog] + +@overload def job_mutator(function: Callable[[Bundle, Job], Job]) -> ResourceMutator[Job] +@overload def job_mutator(function: Callable[[Job], Job]) -> ResourceMutator[Job] +def job_mutator(function: Callable) -> ResourceMutator[Job] + +def load_resources_from_current_package_module() -> Resources + +def load_resources_from_module(module: module) -> Resources + +def load_resources_from_modules(modules: Iterable[module]) -> Resources + +def load_resources_from_package_module(package_module: module) -> Resources + +@overload def pipeline_mutator(function: Callable[[Bundle, Pipeline], Pipeline]) -> ResourceMutator[Pipeline] +@overload def pipeline_mutator(function: Callable[[Pipeline], Pipeline]) -> ResourceMutator[Pipeline] +def pipeline_mutator(function: Callable) -> ResourceMutator[Pipeline] + +@overload def schema_mutator(function: Callable[[Bundle, Schema], Schema]) -> ResourceMutator[Schema] +@overload def schema_mutator(function: Callable[[Schema], Schema]) -> ResourceMutator[Schema] +def schema_mutator(function: Callable) -> ResourceMutator[Schema] + +def variables(cls: type[_T]) -> type[_T] + +@overload def volume_mutator(function: Callable[[Bundle, Volume], Volume]) -> ResourceMutator[Volume] +@overload def volume_mutator(function: Callable[[Volume], Volume]) -> ResourceMutator[Volume] +def volume_mutator(function: Callable) -> ResourceMutator[Volume] + +== _ResourceType.all() registry == +singular_name=alert plural_name=alerts resource_type=Alert +singular_name=catalog plural_name=catalogs resource_type=Catalog +singular_name=job plural_name=jobs resource_type=Job +singular_name=pipeline plural_name=pipelines resource_type=Pipeline +singular_name=schema plural_name=schemas resource_type=Schema +singular_name=volume plural_name=volumes resource_type=Volume diff --git a/python/databricks_tests/core/test_public_api.py b/python/databricks_tests/core/test_public_api.py new file mode 100644 index 00000000000..3ef35c24be4 --- /dev/null +++ b/python/databricks_tests/core/test_public_api.py @@ -0,0 +1,233 @@ +"""Regression guard for the typed public API surface of databricks.bundles.core. + +The core wiring — Resources, the *_mutator functions, the _ResourceType registry, +__all__ — is hand-written (the resource namespaces are pydabs-codegen output, already +guarded by generate-check). This snapshots the core public surface to a golden file so a +refactor can't silently drop a type hint, move a `*` marker, rename a method, or change +the export set. + +Regenerate the golden after an intended public-API change: + + UPDATE_SNAPSHOTS=1 uv run pytest databricks_tests/core/test_public_api.py + +Determinism / version notes: + * Types are rendered by their PUBLIC SHORT NAME (`Variable[str]`, `Location`, `None`) + rather than repr's fully-qualified internal module path — so moving an internal + `_`-module doesn't perturb the golden; only a real public-API change does. + * Signatures are reconstructed from inspect.Signature so `/`, `*`, `*args`, `**kwargs` + markers render explicitly and stably. + * Requires Python >= 3.11 for typing.get_overloads (the *_mutator overloads). Output is + identical on 3.11/3.12/3.13, so the single golden holds across those versions. +""" + +import collections.abc +import dataclasses +import enum +import inspect +import os +import sys +import types +import typing +from pathlib import Path + +import pytest + +import databricks.bundles.core as core + +_GOLDEN = Path(__file__).parent / "public_api.txt" + + +def _short_name(t) -> str: + return getattr(t, "__name__", None) or getattr(t, "_name", None) or str(t) + + +def render_type(t) -> str: + """Render a type annotation by public short name, module-location independent.""" + if t is None or t is type(None): + return "None" + if t is Ellipsis: + return "..." + if isinstance(t, str): + # A forward-ref written as a string literal in the source (e.g. "JobParam"). + return t + if isinstance(t, typing.ForwardRef): + return t.__forward_arg__ + if isinstance(t, typing.TypeVar): + return t.__name__ + + origin = typing.get_origin(t) + args = typing.get_args(t) + + if origin is not None: + if origin is typing.Union or origin is types.UnionType: + return "Union[" + ", ".join(render_type(a) for a in args) + "]" + if origin is typing.Literal: + return "Literal[" + ", ".join(repr(a) for a in args) + "]" + if origin is collections.abc.Callable: + if not args: + return "Callable" + # get_args(Callable[[int], str]) == ([int], str); [0] is the arg list. + params, ret = args[0], args[-1] + params_str = ( + "..." + if params is Ellipsis + else "[" + ", ".join(render_type(a) for a in params) + "]" + ) + return "Callable[" + params_str + ", " + render_type(ret) + "]" + name = _short_name(origin) + if args: + return name + "[" + ", ".join(render_type(a) for a in args) + "]" + return name + + return _short_name(t) + + +def render_signature(func) -> str: + """Reconstruct a signature string with explicit / * ** markers and short types.""" + sig = inspect.signature(func) + parts = [] + last_kind = None + emitted_star = False + for p in sig.parameters.values(): + if ( + last_kind == inspect.Parameter.POSITIONAL_ONLY + and p.kind != inspect.Parameter.POSITIONAL_ONLY + ): + parts.append("/") + if p.kind == inspect.Parameter.KEYWORD_ONLY and not emitted_star: + parts.append("*") + emitted_star = True + + s = p.name + if p.kind == inspect.Parameter.VAR_POSITIONAL: + s = "*" + s + emitted_star = True + elif p.kind == inspect.Parameter.VAR_KEYWORD: + s = "**" + s + + if p.annotation is not inspect.Parameter.empty: + s += ": " + render_type(p.annotation) + if p.default is not inspect.Parameter.empty: + sep = " = " if p.annotation is not inspect.Parameter.empty else "=" + s += sep + repr(p.default) + parts.append(s) + last_kind = p.kind + + if last_kind == inspect.Parameter.POSITIONAL_ONLY: + parts.append("/") + + ret = "" + if sig.return_annotation is not inspect.Signature.empty: + ret = " -> " + render_type(sig.return_annotation) + return "(" + ", ".join(parts) + ")" + ret + + +def _bases(cls) -> str: + # Skip object and private (underscore) bases, mirroring _members(): a generated private + # base like _GeneratedResources is an implementation detail, not the public contract. + names = [ + b.__name__ + for b in cls.__bases__ + if b is not object and not b.__name__.startswith("_") + ] + return "(" + ", ".join(names) + ")" if names else "" + + +def _members(cls, predicate): + return sorted( + (name, obj) + for name, obj in inspect.getmembers(cls, predicate) + if not name.startswith("_") + ) + + +def render_class(name, cls, out: list[str]) -> None: + if isinstance(cls, type) and issubclass(cls, enum.Enum): + out.append(f"class {name}(Enum):") + for member in cls: + out.append(f" {member.name} = {member.value!r}") + out.append("") + return + + out.append(f"class {name}{_bases(cls)}:") + if dataclasses.is_dataclass(cls): + for f in dataclasses.fields(cls): + line = f" {f.name}: {render_type(f.type)}" + if f.default is not dataclasses.MISSING: + line += f" = {f.default!r}" + elif f.default_factory is not dataclasses.MISSING: + line += " = " + out.append(line) + + # classmethods (e.g. create_error) surface as bound methods, not plain functions. + for m_name, m in _members(cls, inspect.ismethod): + out.append(f" @classmethod def {m_name}{render_signature(m)}") + for m_name, m in _members(cls, inspect.isfunction): + out.append(f" def {m_name}{render_signature(m)}") + for p_name, prop in _members(cls, lambda x: isinstance(x, property)): + ret = "" + if prop.fget is not None: + r = inspect.signature(prop.fget).return_annotation + if r is not inspect.Signature.empty: + ret = " -> " + render_type(r) + out.append(f" @property {p_name}{ret}") + out.append("") + + +def render_symbol(name, obj, out: list[str]) -> None: + if inspect.isclass(obj): + render_class(name, obj, out) + elif inspect.isfunction(obj): + # typing.get_overloads is 3.11+; the test is skipped below on older versions. + overloads = typing.get_overloads(obj) if sys.version_info >= (3, 11) else [] + for ov in overloads: + out.append(f"@overload def {name}{render_signature(ov)}") + out.append(f"def {name}{render_signature(obj)}") + out.append("") + else: + # Type aliases (VariableOr*), rendered by structure. + out.append(f"{name} = {render_type(obj)}") + out.append("") + + +def render_registry(out: list[str]) -> None: + # _ResourceType is intentionally not exported from core, but the registry it builds is + # part of the wiring a refactor regenerates, so snapshot it too. + from databricks.bundles.core._resource_type import _ResourceType + + out.append("== _ResourceType.all() registry ==") + for rt in sorted(_ResourceType.all(), key=lambda rt: rt.singular_name): + out.append( + f"singular_name={rt.singular_name} plural_name={rt.plural_name} resource_type={rt.resource_type.__name__}" + ) + out.append("") + + +def dump_core_public_api() -> str: + out = ["== module databricks.bundles.core =="] + out.append("__all__ = [") + for name in sorted(core.__all__): + out.append(f" {name},") + out.append("]") + out.append("") + + for name in sorted(core.__all__): + render_symbol(name, getattr(core, name), out) + + render_registry(out) + + # Single trailing newline, no blank last line (the whitespace linter strips it). + return "\n".join(out).rstrip("\n") + "\n" + + +@pytest.mark.skipif( + sys.version_info < (3, 11), reason="typing.get_overloads requires Python 3.11+" +) +def test_core_public_api(): + actual = dump_core_public_api() + if os.environ.get("UPDATE_SNAPSHOTS"): + _GOLDEN.write_text(actual) + assert actual == _GOLDEN.read_text(), ( + "databricks.bundles.core public API changed. If intended, regenerate with " + "UPDATE_SNAPSHOTS=1 uv run pytest databricks_tests/core/test_public_api.py" + ) diff --git a/python/databricks_tests/core/test_resources.py b/python/databricks_tests/core/test_resources.py index d58650d8936..ee2ab7ec405 100644 --- a/python/databricks_tests/core/test_resources.py +++ b/python/databricks_tests/core/test_resources.py @@ -11,11 +11,10 @@ from databricks.bundles.alerts._models.comparison_operator import ComparisonOperator from databricks.bundles.alerts._models.cron_schedule import CronSchedule from databricks.bundles.catalogs._models.catalog import Catalog -from databricks.bundles.core import Location, Resources, Severity -from databricks.bundles.core._bundle import Bundle -from databricks.bundles.core._resource import Resource -from databricks.bundles.core._resource_mutator import ( - ResourceMutator, +from databricks.bundles.core import ( + Location, + Resources, + Severity, alert_mutator, catalog_mutator, job_mutator, @@ -23,6 +22,9 @@ schema_mutator, volume_mutator, ) +from databricks.bundles.core._bundle import Bundle +from databricks.bundles.core._resource import Resource +from databricks.bundles.core._resource_mutator import ResourceMutator from databricks.bundles.core._resource_type import _ResourceType from databricks.bundles.jobs._models.job import Job from databricks.bundles.pipelines._models.pipeline import Pipeline @@ -36,7 +38,6 @@ class TestCase: dict_example: dict dataclass_example: Resource mutator: Callable - article: str = "a" # grammatical article in the duplicate-resource error message resource_types = {tpe.resource_type: tpe for tpe in _ResourceType.all()} @@ -115,7 +116,6 @@ class TestCase: ), ), mutator=alert_mutator, - article="an", ), resource_types[Alert], ), @@ -324,7 +324,7 @@ def test_add_duplicate_resource(tc: TestCase, tpe: _ResourceType): assert item.severity == Severity.ERROR assert ( item.summary - == f"Duplicate resource name 'my_resource' for {tc.article} {tpe.singular_name}. Resource names must be unique." + == f"Duplicate resource name 'my_resource' for resource '{tpe.singular_name}'. Resource names must be unique." ) diff --git a/python/databricks_tests/test_build.py b/python/databricks_tests/test_build.py index 65d9683e922..14fce619757 100644 --- a/python/databricks_tests/test_build.py +++ b/python/databricks_tests/test_build.py @@ -28,8 +28,8 @@ Resources, Severity, job_mutator, + pipeline_mutator, ) -from databricks.bundles.core._resource_mutator import pipeline_mutator from databricks.bundles.jobs import Job from databricks.bundles.pipelines._models.pipeline import Pipeline