From dd8787f2f63ab516f908c8da92768bf255dcdb5c Mon Sep 17 00:00:00 2001 From: Jack Stein Date: Fri, 12 Jun 2026 07:53:38 -0600 Subject: [PATCH 1/2] chore: expand .gitignore with standard development artifacts Add entries for Python build artifacts, Hypothesis test database, IDE files, OS junk, and test coverage output. --- .gitignore | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/.gitignore b/.gitignore index 63e36ff..d5eec31 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,22 @@ **/__pycache__/ **/.venv/ +*.egg-info/ +dist/ +build/ + +# Hypothesis property-based testing database +**/.hypothesis/ + +# IDE +.idea/ +.vscode/ +*.swp + +# OS +.DS_Store +Thumbs.db + +# Test / coverage artifacts +.coverage +htmlcov/ +.pytest_cache/ From 41b4cee8ba725603d1330abd2b1e51d76607e6a3 Mon Sep 17 00:00:00 2001 From: Jack Stein Date: Fri, 12 Jun 2026 07:54:25 -0600 Subject: [PATCH 2/2] feat(converters): add Databricks Unity Catalog Metric View converter MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add bidirectional converter between OSI semantic model YAML and Databricks Unity Catalog Metric View YAML (v1.1). Import (Metric View → OSI): - Maps fields to OSI fields with DATABRICKS dialect expressions - Maps measures to OSI metrics - Parses JOIN ON/USING clauses into OSI relationships - Preserves synonyms, display_name, and comments as ai_context - Stores Databricks-specific metadata (filter, materialization, window, format) in custom_extensions Export (OSI → Metric View): - Generates valid Metric View YAML per dataset - Selects DATABRICKS dialect with ANSI_SQL fallback - Reconstructs JOIN clauses from OSI relationships - Restores filter and materialization from custom_extensions Also includes: - CLI entry point (osi-databricks import/export) - Comprehensive test suite (unit + Hypothesis property tests) - TPC-DS-based test fixtures - README with mapping table and usage examples --- converters/databricks/README.md | 139 +++ converters/databricks/pyproject.toml | 47 + .../databricks/src/osi_databricks/__init__.py | 1 + .../databricks/src/osi_databricks/cli.py | 87 ++ .../src/osi_databricks/dialect_utils.py | 59 ++ .../src/osi_databricks/metric_view_to_osi.py | 460 +++++++++ .../databricks/src/osi_databricks/models.py | 251 +++++ .../src/osi_databricks/osi_to_metric_view.py | 462 +++++++++ converters/databricks/tests/__init__.py | 1 + converters/databricks/tests/conftest.py | 532 +++++++++++ converters/databricks/tests/fixtures/.gitkeep | 1 + .../tests/fixtures/metric_view_tpcds.yaml | 96 ++ .../databricks/tests/fixtures/osi_tpcds.yaml | 174 ++++ converters/databricks/tests/test_cli.py | 362 +++++++ .../databricks/tests/test_dialect_utils.py | 104 ++ .../tests/test_metric_view_to_osi.py | 821 ++++++++++++++++ converters/databricks/tests/test_models.py | 422 +++++++++ .../tests/test_osi_to_metric_view.py | 824 ++++++++++++++++ converters/databricks/tests/test_roundtrip.py | 392 ++++++++ .../tests/test_roundtrip_properties.py | 896 ++++++++++++++++++ 20 files changed, 6131 insertions(+) create mode 100644 converters/databricks/README.md create mode 100644 converters/databricks/pyproject.toml create mode 100644 converters/databricks/src/osi_databricks/__init__.py create mode 100644 converters/databricks/src/osi_databricks/cli.py create mode 100644 converters/databricks/src/osi_databricks/dialect_utils.py create mode 100644 converters/databricks/src/osi_databricks/metric_view_to_osi.py create mode 100644 converters/databricks/src/osi_databricks/models.py create mode 100644 converters/databricks/src/osi_databricks/osi_to_metric_view.py create mode 100644 converters/databricks/tests/__init__.py create mode 100644 converters/databricks/tests/conftest.py create mode 100644 converters/databricks/tests/fixtures/.gitkeep create mode 100644 converters/databricks/tests/fixtures/metric_view_tpcds.yaml create mode 100644 converters/databricks/tests/fixtures/osi_tpcds.yaml create mode 100644 converters/databricks/tests/test_cli.py create mode 100644 converters/databricks/tests/test_dialect_utils.py create mode 100644 converters/databricks/tests/test_metric_view_to_osi.py create mode 100644 converters/databricks/tests/test_models.py create mode 100644 converters/databricks/tests/test_osi_to_metric_view.py create mode 100644 converters/databricks/tests/test_roundtrip.py create mode 100644 converters/databricks/tests/test_roundtrip_properties.py diff --git a/converters/databricks/README.md b/converters/databricks/README.md new file mode 100644 index 0000000..c6ad250 --- /dev/null +++ b/converters/databricks/README.md @@ -0,0 +1,139 @@ +# osi-databricks + +Bidirectional converter between [Databricks Unity Catalog Metric View YAML](https://docs.databricks.com/aws/metric-views/yaml-ref) and the [Open Semantic Interchange (OSI)](../../core-spec/spec.md) format. + +Part of the OSI hub-and-spoke converter architecture — see the [converter guide](../index.md) for background. + +## Installation + +**With uv (recommended):** + +```bash +cd converters/databricks +uv sync +``` + +**With pip:** + +```bash +cd converters/databricks +pip install -e . +``` + +### Dependencies + +- Python ≥ 3.11 +- `osi-python >= 0.2.0.dev0` +- `pydantic >= 2.0` +- `PyYAML >= 6.0` + +## Usage + +The package provides a CLI entry point `osi-databricks` with two subcommands. + +### Import: Metric View → OSI + +Convert a Databricks Metric View YAML file to OSI format: + +```bash +osi-databricks import -i metric_view.yaml -o output_osi.yaml + +# Specify a custom model name +osi-databricks import -i metric_view.yaml -o output_osi.yaml --model-name my_model +``` + +### Export: OSI → Metric View + +Convert an OSI YAML file to one or more Metric View YAML files (one per dataset): + +```bash +osi-databricks export -i osi_model.yaml -o ./output_dir/ +``` + +Output files are named `{dataset_name}.yaml`. The output directory is created automatically if it doesn't exist. + +### Programmatic Usage + +```python +from osi_databricks.models import MetricViewModel +from osi_databricks.metric_view_to_osi import metric_view_to_osi +from osi_databricks.osi_to_metric_view import osi_to_metric_view + +# Import +mv = MetricViewModel.from_yaml(open("metric_view.yaml").read()) +osi_doc = metric_view_to_osi(mv, model_name="my_model") +print(osi_doc.to_osi_yaml()) + +# Export +from osi import OSIDocument +import yaml + +raw = yaml.safe_load(open("osi_model.yaml").read()) +doc = OSIDocument.model_validate(raw) +results = osi_to_metric_view(doc) +for name, model in results: + open(f"{name}.yaml", "w").write(model.to_yaml()) +``` + +## Mapping Reference + +### Core Constructs + +| Metric View YAML | OSI Equivalent | Direction | Notes | +|---|---|---|---| +| `source` (three-part name) | `dataset.source` | ↔ | Direct pass-through | +| `source` (SQL query) | `custom_extension` (DATABRICKS, `source_query`) | ↔ | Detected by SQL keywords | +| `fields[].name` | `field.name` | ↔ | Direct | +| `fields[].expr` | `field.expression.dialects[DATABRICKS]` | ↔ | Always stored as DATABRICKS dialect | +| `measures[].name` | `metric.name` | ↔ | Direct | +| `measures[].expr` | `metric.expression.dialects[DATABRICKS]` | ↔ | Same dialect logic as fields | +| `joins[].name` | `relationship.name` | ↔ | Direct | +| `joins[].source` | `relationship.to` | ↔ | Dataset name extracted from source | +| `joins[].on` | `relationship.from_columns` / `to_columns` | ↔ | Parsed from `a.col = b.col` pattern | +| `joins[].using` | `from_columns = to_columns = using` | → OSI | Columns same on both sides | +| `comment` (top-level) | `semantic_model.description` | ↔ | Direct | + +### Metadata Mapping + +| Metric View YAML | OSI Equivalent | Direction | Notes | +|---|---|---|---| +| `fields[].comment` | `field.description` | ↔ | Direct | +| `fields[].display_name` | `field.ai_context.synonyms[0]` | ↔ | First synonym = display_name | +| `fields[].synonyms` | `field.ai_context.synonyms[1:]` | ↔ | Remaining synonyms | +| `fields[].format` | `field.custom_extension` (DATABRICKS) | ↔ | No OSI core equivalent | +| `measures[].window` | `metric.custom_extension` (DATABRICKS) | ↔ | No OSI core equivalent | +| `filter` | `dataset.custom_extension` (DATABRICKS) | ↔ | No OSI core equivalent | +| `materialization` | `semantic_model.custom_extension` (DATABRICKS) | ↔ | No OSI core equivalent | + +### Dialect Handling + +On **import** (MV → OSI): +- Expressions are stored as `DATABRICKS` dialect +- If the expression uses only standard SQL (no `FILTER()`, `MEASURE()`, `QUALIFY`, `::`), an `ANSI_SQL` entry is also generated + +On **export** (OSI → MV): +- Prefers `DATABRICKS` dialect expression +- Falls back to `ANSI_SQL` if `DATABRICKS` is unavailable +- Skips the field/metric with a warning if neither dialect is present + +## Development + +```bash +cd converters/databricks +uv sync + +# Run tests +uv run pytest + +# Run linter +uv run ruff check src/ tests/ +``` + +## Known Limitations + +- **Time dimension inference is heuristic.** The importer infers `is_time` from expression content (looking for DATE, TIME, TIMESTAMP, etc.). This may produce false positives or miss custom time expressions. +- **ON clause parsing is pattern-based.** Only `alias.column = alias.column` patterns are parsed. Complex ON conditions (functions, nested expressions) are stored in a custom extension with a warning. +- **Nested joins (snowflake schema) are flattened.** Metric View supports nested join definitions; during import these are flattened into a list of OSI relationships. The nesting structure is not preserved during round-trip. +- **USING clause becomes ON on round-trip.** Joins defined with `USING` are converted to explicit column pairs in OSI. On export, they are reconstructed as `ON` clauses rather than `USING`. +- **No model-level description in Metric View.** The top-level `comment` is the closest equivalent. Model-level `ai_context.instructions` is stored in a custom extension. +- **Custom extensions for other vendors are preserved but not applied.** Extensions for SNOWFLAKE, DBT, etc. pass through without modification. diff --git a/converters/databricks/pyproject.toml b/converters/databricks/pyproject.toml new file mode 100644 index 0000000..cf7d1d6 --- /dev/null +++ b/converters/databricks/pyproject.toml @@ -0,0 +1,47 @@ +[project] +name = "osi-databricks" +version = "0.2.0.dev0" +description = "Bidirectional converter between Databricks Unity Catalog Metric View YAML and OSI semantic model" +readme = "README.md" +requires-python = ">=3.11" +dependencies = [ + "osi-python>=0.2.0.dev0", + "pydantic>=2.0", + "pyyaml>=6.0", +] + +[project.scripts] +osi-databricks = "osi_databricks.cli:main" + +[dependency-groups] +dev = [ + "pytest>=8.0", + "hypothesis>=6.0", + "ruff>=0.11.5", +] + +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[tool.hatch.build] +include = ["src/osi_databricks/**/*"] +ignore-vcs = true + +[tool.hatch.build.targets.wheel] +packages = ["src/osi_databricks"] + +[tool.ruff] +line-length = 120 +target-version = "py311" + +[tool.ruff.lint] +select = ["E", "F", "I", "N", "UP", "W", "C4", "PIE", "SIM"] +ignore = ["E501"] + +[tool.uv.sources] +osi-python = { path = "../../python", editable = true } + +[tool.pytest.ini_options] +testpaths = ["tests"] +pythonpath = ["src"] diff --git a/converters/databricks/src/osi_databricks/__init__.py b/converters/databricks/src/osi_databricks/__init__.py new file mode 100644 index 0000000..d60392f --- /dev/null +++ b/converters/databricks/src/osi_databricks/__init__.py @@ -0,0 +1 @@ +"""osi-databricks: Bidirectional converter between Databricks Metric View YAML and OSI.""" diff --git a/converters/databricks/src/osi_databricks/cli.py b/converters/databricks/src/osi_databricks/cli.py new file mode 100644 index 0000000..e513afc --- /dev/null +++ b/converters/databricks/src/osi_databricks/cli.py @@ -0,0 +1,87 @@ +"""CLI entry point for the osi-databricks converter. + +Usage: + osi-databricks import -i metric_view.yaml -o output_osi.yaml + osi-databricks export -i osi_model.yaml -o ./output_dir/ +""" + +import argparse +import sys +from pathlib import Path + +import yaml +from osi import OSIDocument + +from osi_databricks.metric_view_to_osi import metric_view_to_osi +from osi_databricks.models import MetricViewModel +from osi_databricks.osi_to_metric_view import osi_to_metric_view + + +def _cmd_import(args: argparse.Namespace) -> None: + """Import: Metric View YAML → OSI YAML.""" + input_path = Path(args.input) + output_path = Path(args.output) + + try: + mv_model = MetricViewModel.from_yaml(input_path.read_text()) + except Exception as e: + print(f"Error parsing {input_path}: {e}", file=sys.stderr) + sys.exit(1) + + osi_doc = metric_view_to_osi(mv_model, model_name=args.model_name) + output_path.parent.mkdir(parents=True, exist_ok=True) + output_path.write_text(osi_doc.to_osi_yaml()) + print(f"Written to {output_path}", file=sys.stderr) + + +def _cmd_export(args: argparse.Namespace) -> None: + """Export: OSI YAML → Metric View YAML file(s).""" + input_path = Path(args.input) + output_dir = Path(args.output) + output_dir.mkdir(parents=True, exist_ok=True) + + try: + raw = yaml.safe_load(input_path.read_text()) + document = OSIDocument.model_validate(raw) + except Exception as e: + print(f"Error parsing {input_path}: {e}", file=sys.stderr) + sys.exit(1) + + results = osi_to_metric_view(document) + for dataset_name, mv_model in results: + out_file = output_dir / f"{dataset_name}.yaml" + out_file.write_text(mv_model.to_yaml()) + print(f"Written {out_file}", file=sys.stderr) + + +def main() -> None: + """Entry point for the osi-databricks CLI.""" + parser = argparse.ArgumentParser( + prog="osi-databricks", + description="Convert between Databricks Metric View YAML and OSI YAML.", + ) + subparsers = parser.add_subparsers(dest="command", required=True) + + # Import subcommand + import_cmd = subparsers.add_parser("import", help="Convert Metric View YAML → OSI YAML") + import_cmd.add_argument("-i", "--input", required=True, metavar="FILE", help="Path to Metric View YAML") + import_cmd.add_argument("-o", "--output", required=True, metavar="FILE", help="Path for output OSI YAML") + import_cmd.add_argument( + "--model-name", default="metric_view_model", metavar="NAME", + help="OSI semantic model name (default: metric_view_model)", + ) + + # Export subcommand + export_cmd = subparsers.add_parser("export", help="Convert OSI YAML → Metric View YAML") + export_cmd.add_argument("-i", "--input", required=True, metavar="FILE", help="Path to OSI YAML") + export_cmd.add_argument("-o", "--output", required=True, metavar="DIR", help="Output directory for Metric View files") + + args = parser.parse_args() + if args.command == "import": + _cmd_import(args) + elif args.command == "export": + _cmd_export(args) + + +if __name__ == "__main__": + main() diff --git a/converters/databricks/src/osi_databricks/dialect_utils.py b/converters/databricks/src/osi_databricks/dialect_utils.py new file mode 100644 index 0000000..4322ce9 --- /dev/null +++ b/converters/databricks/src/osi_databricks/dialect_utils.py @@ -0,0 +1,59 @@ +"""Dialect selection and ANSI SQL detection utilities. + +Provides shared logic for both import (Metric View → OSI) and export (OSI → Metric View) +directions, including detecting Databricks-specific SQL patterns and selecting the +best available dialect expression. +""" + +from __future__ import annotations + +import re + +from osi.models import OSIDialect, OSIDialectExpression + +# Databricks-specific SQL patterns not found in ANSI SQL +_DATABRICKS_ONLY_PATTERNS = [ + r"\bFILTER\s*\(", + r"\bMEASURE\s*\(", + r"\bQUALIFY\b", + r"::", # Type casting syntax +] + + +def is_standard_sql(expr: str) -> bool: + """Determine if an expression uses only ANSI-standard SQL syntax. + + Args: + expr: SQL expression string to check. + + Returns: + True if the expression appears to use only standard SQL constructs. + """ + for pattern in _DATABRICKS_ONLY_PATTERNS: + if re.search(pattern, expr, re.IGNORECASE): + return False + return True + + +def select_dialect_expression( + dialects: list[OSIDialectExpression], + preferred: OSIDialect = OSIDialect.DATABRICKS, + fallback: OSIDialect = OSIDialect.ANSI_SQL, +) -> str | None: + """Select expression string with dialect preference chain. + + Args: + dialects: List of dialect expressions to choose from. + preferred: First-choice dialect (default: DATABRICKS). + fallback: Second-choice dialect (default: ANSI_SQL). + + Returns: + The expression string for the best available dialect, or None if + neither preferred nor fallback dialect is available. + """ + by_dialect = {d.dialect: d.expression for d in dialects} + if preferred in by_dialect: + return by_dialect[preferred] + if fallback in by_dialect: + return by_dialect[fallback] + return None diff --git a/converters/databricks/src/osi_databricks/metric_view_to_osi.py b/converters/databricks/src/osi_databricks/metric_view_to_osi.py new file mode 100644 index 0000000..d04cd25 --- /dev/null +++ b/converters/databricks/src/osi_databricks/metric_view_to_osi.py @@ -0,0 +1,460 @@ +"""Import: Convert Databricks Metric View YAML to OSI semantic model. + +Maps a MetricViewModel to an OSIDocument following the hub-and-spoke architecture. +Each Metric View definition becomes one OSI semantic model with a single dataset, +associated fields, metrics, and relationships. +""" + +from __future__ import annotations + +import json +import re +import warnings + +from osi.models import ( + OSIAIContextObject, + OSICustomExtension, + OSIDataset, + OSIDialect, + OSIDialectExpression, + OSIDimension, + OSIDocument, + OSIExpression, + OSIField, + OSIMetric, + OSIRelationship, + OSISemanticModel, + OSIVendor, +) + +from osi_databricks.dialect_utils import is_standard_sql +from osi_databricks.models import ( + MetricViewJoin, + MetricViewModel, +) + +# Patterns that indicate a time-related dimension +_TIME_PATTERNS = {"DATE", "TIME", "TIMESTAMP", "DATE_TRUNC", "YEAR", "MONTH", "QUARTER", "DAY", "HOUR"} + + +def metric_view_to_osi( + model: MetricViewModel, + model_name: str = "metric_view_model", + model_description: str = "", +) -> OSIDocument: + """Convert a Metric View definition to an OSI document. + + Args: + model: Parsed Metric View model. + model_name: Name for the OSI semantic model. + model_description: Optional description. + + Returns: + A fully populated OSIDocument. + """ + # Determine dataset name from source + dataset_name = _extract_dataset_name(model.source) + + # Build dataset + dataset = _build_dataset(model, dataset_name) + + # Build relationships from joins + relationships = _build_relationships(model.joins, dataset_name) if model.joins else None + + # Build metrics from measures + metrics = _build_metrics(model.measures) if model.measures else None + + # Build semantic model custom extensions (materialization) + sm_extensions = _build_semantic_model_extensions(model) + + # Use top-level comment as description + description = model.comment if model.comment else (model_description or None) + + semantic_model = OSISemanticModel( + name=model_name, + description=description, + datasets=[dataset], + relationships=relationships, + metrics=metrics, + custom_extensions=sm_extensions, + ) + + # Determine which dialects are used + dialects = [OSIDialect.DATABRICKS] + if _has_ansi_sql_entries(model): + dialects.append(OSIDialect.ANSI_SQL) + + return OSIDocument( + version="0.2.0.dev0", + dialects=dialects, + vendors=[OSIVendor.DATABRICKS], + semantic_model=[semantic_model], + ) + + +def _is_sql_query(source: str) -> bool: + """Detect if source is a SQL query vs a table reference. + + Args: + source: The source string from a Metric View definition. + + Returns: + True if the source appears to be a SQL query. + """ + upper = source.strip().upper() + return upper.startswith("SELECT") or upper.startswith("WITH") or upper.startswith("(SELECT") or upper.startswith("(WITH") + + +def _extract_dataset_name(source: str) -> str: + """Extract a dataset name from the source string. + + For three-part names (catalog.schema.table), returns the table name. + For SQL queries, returns a generic name. + + Args: + source: The source string from a Metric View definition. + + Returns: + Dataset name string. + """ + if _is_sql_query(source): + return "source" + parts = source.split(".") + return parts[-1] if parts else source + + +def _infer_is_time(expr: str) -> bool: + """Heuristically determine if a field expression represents a time dimension. + + Args: + expr: SQL expression string for the field. + + Returns: + True if the expression likely represents a time dimension. + """ + upper = expr.upper() + return any(pat in upper for pat in _TIME_PATTERNS) + + +def _build_dialect_expressions(expr: str) -> list[OSIDialectExpression]: + """Build dialect expression list with DATABRICKS and optionally ANSI_SQL. + + Args: + expr: The SQL expression string. + + Returns: + List of dialect expressions (always includes DATABRICKS, adds ANSI_SQL + if the expression uses only standard SQL). + """ + dialects = [OSIDialectExpression(dialect=OSIDialect.DATABRICKS, expression=expr)] + if is_standard_sql(expr): + dialects.append(OSIDialectExpression(dialect=OSIDialect.ANSI_SQL, expression=expr)) + return dialects + + +def _build_ai_context( + display_name: str | None, + synonyms: list[str] | None, +) -> OSIAIContextObject | None: + """Build AI context from display_name and synonyms. + + display_name becomes the first synonym, followed by any additional synonyms. + + Args: + display_name: Human-readable label for the field/measure. + synonyms: Additional synonym strings. + + Returns: + OSIAIContextObject or None if no metadata is present. + """ + all_synonyms: list[str] = [] + if display_name: + all_synonyms.append(display_name) + if synonyms: + all_synonyms.extend(synonyms) + if not all_synonyms: + return None + return OSIAIContextObject(synonyms=tuple(all_synonyms)) + + +def _build_field(field) -> OSIField: + """Convert a MetricViewField to an OSIField. + + Args: + field: A MetricViewField instance. + + Returns: + An OSIField with dialect expressions, dimension, and metadata. + """ + expression = OSIExpression(dialects=_build_dialect_expressions(field.expr)) + dimension = OSIDimension(is_time=_infer_is_time(field.expr)) + ai_context = _build_ai_context(field.display_name, field.synonyms) + + # Build custom extensions for format if present + custom_extensions = None + if field.format is not None: + format_data = field.format.model_dump(exclude_none=True) + custom_extensions = [ + OSICustomExtension( + vendor_name=OSIVendor.DATABRICKS, + data=json.dumps({"format": format_data}), + ) + ] + + return OSIField( + name=field.name, + expression=expression, + dimension=dimension, + description=field.comment, + ai_context=ai_context, + custom_extensions=custom_extensions, + ) + + +def _build_metrics(measures) -> list[OSIMetric] | None: + """Convert Metric View measures to OSI metrics. + + Args: + measures: List of MetricViewMeasure instances. + + Returns: + List of OSIMetric instances, or None if input is empty/None. + """ + if not measures: + return None + + metrics = [] + for measure in measures: + expression = OSIExpression(dialects=_build_dialect_expressions(measure.expr)) + ai_context = _build_ai_context(measure.display_name, measure.synonyms) + + # Build custom extensions for window if present + custom_extensions = None + if measure.window is not None: + window_data = [w.model_dump(exclude_none=True) for w in measure.window] + custom_extensions = [ + OSICustomExtension( + vendor_name=OSIVendor.DATABRICKS, + data=json.dumps({"window": window_data}), + ) + ] + + # Also include format in custom extensions if present + if measure.format is not None: + format_ext = OSICustomExtension( + vendor_name=OSIVendor.DATABRICKS, + data=json.dumps({"format": measure.format.model_dump(exclude_none=True)}), + ) + if custom_extensions: + custom_extensions.append(format_ext) + else: + custom_extensions = [format_ext] + + metrics.append( + OSIMetric( + name=measure.name, + expression=expression, + description=measure.comment, + ai_context=ai_context, + custom_extensions=custom_extensions, + ) + ) + return metrics + + +def _parse_on_clause(on_clause: str) -> tuple[list[str], list[str]]: + """Parse a JOIN ON clause into from_columns and to_columns. + + Supports patterns like: + source.col1 = target.col2 + source.col1 = target.col2 AND source.col3 = target.col4 + + Args: + on_clause: The ON clause string. + + Returns: + Tuple of (from_columns, to_columns). + """ + from_columns: list[str] = [] + to_columns: list[str] = [] + + # Split on AND (case-insensitive) + conditions = re.split(r"\s+AND\s+", on_clause, flags=re.IGNORECASE) + + for condition in conditions: + condition = condition.strip() + # Match pattern: alias.column = alias.column + match = re.match( + r"(\w+)\.(\w+)\s*=\s*(\w+)\.(\w+)", + condition, + ) + if match: + from_columns.append(match.group(2)) + to_columns.append(match.group(4)) + else: + # If we can't parse it, store the raw condition and warn + warnings.warn( + f"Could not parse ON clause condition: {condition}", + stacklevel=2, + ) + from_columns.append(condition) + to_columns.append(condition) + + return from_columns, to_columns + + +def _build_relationships( + joins: list[MetricViewJoin] | None, + from_dataset: str, +) -> list[OSIRelationship] | None: + """Convert Metric View joins to OSI relationships. + + Handles both ON clause parsing and USING clause mapping. + Recursively processes nested joins. + + Args: + joins: List of MetricViewJoin instances. + from_dataset: Name of the source dataset (used as 'from' in relationship). + + Returns: + List of OSIRelationship instances, or None if no joins. + """ + if not joins: + return None + + relationships: list[OSIRelationship] = [] + + for join in joins: + to_dataset = _extract_dataset_name(join.source) + + # Parse columns from ON or USING clause + if join.using: + from_columns = list(join.using) + to_columns = list(join.using) + elif join.on: + from_columns, to_columns = _parse_on_clause(join.on) + else: + from_columns = [] + to_columns = [] + + # Build custom extensions for cardinality and rely + custom_ext_data: dict = {} + if join.cardinality: + custom_ext_data["cardinality"] = join.cardinality + if join.rely: + custom_ext_data["rely"] = join.rely.model_dump(exclude_none=True) + + custom_extensions = None + if custom_ext_data: + custom_extensions = [ + OSICustomExtension( + vendor_name=OSIVendor.DATABRICKS, + data=json.dumps(custom_ext_data), + ) + ] + + relationships.append( + OSIRelationship( + name=join.name, + **{"from": from_dataset}, + to=to_dataset, + from_columns=from_columns, + to_columns=to_columns, + custom_extensions=custom_extensions, + ) + ) + + # Recursively process nested joins + if join.joins: + nested_rels = _build_relationships(join.joins, to_dataset) + if nested_rels: + relationships.extend(nested_rels) + + return relationships if relationships else None + + +def _build_dataset(model: MetricViewModel, dataset_name: str) -> OSIDataset: + """Build an OSI dataset from the Metric View model. + + Args: + model: The MetricViewModel instance. + dataset_name: Name for the dataset. + + Returns: + An OSIDataset with fields and custom extensions. + """ + # Build fields + fields = None + if model.fields: + fields = [_build_field(f) for f in model.fields] + + # Build dataset custom extensions (filter, source_query) + ds_ext_data: dict = {} + if model.filter: + ds_ext_data["filter"] = model.filter + if _is_sql_query(model.source): + ds_ext_data["source_query"] = model.source + + custom_extensions = None + if ds_ext_data: + custom_extensions = [ + OSICustomExtension( + vendor_name=OSIVendor.DATABRICKS, + data=json.dumps(ds_ext_data), + ) + ] + + # Determine source — use as-is for three-part names, use dataset_name for SQL queries + source = model.source if not _is_sql_query(model.source) else dataset_name + + return OSIDataset( + name=dataset_name, + source=source, + fields=fields, + custom_extensions=custom_extensions, + ) + + +def _build_semantic_model_extensions(model: MetricViewModel) -> list[OSICustomExtension] | None: + """Build semantic model-level custom extensions. + + Stores materialization config in a DATABRICKS custom extension. + + Args: + model: The MetricViewModel instance. + + Returns: + List of custom extensions or None. + """ + if model.materialization is None: + return None + + mat_data = model.materialization.model_dump(exclude_none=True) + return [ + OSICustomExtension( + vendor_name=OSIVendor.DATABRICKS, + data=json.dumps({"materialization": mat_data}), + ) + ] + + +def _has_ansi_sql_entries(model: MetricViewModel) -> bool: + """Check if any field or measure expression qualifies as standard SQL. + + Used to determine whether ANSI_SQL should be listed in the document dialects. + + Args: + model: The MetricViewModel instance. + + Returns: + True if at least one expression is standard SQL. + """ + if model.fields: + for f in model.fields: + if is_standard_sql(f.expr): + return True + if model.measures: + for m in model.measures: + if is_standard_sql(m.expr): + return True + return False diff --git a/converters/databricks/src/osi_databricks/models.py b/converters/databricks/src/osi_databricks/models.py new file mode 100644 index 0000000..f56a2c0 --- /dev/null +++ b/converters/databricks/src/osi_databricks/models.py @@ -0,0 +1,251 @@ +"""Pydantic models for Databricks Unity Catalog Metric View YAML (v1.1). + +These models represent the parsed Metric View YAML structure and provide +methods for parsing from and serializing to YAML with correct field ordering. + +Metric View YAML structure: + version, source, comment, filter, joins[], fields[], measures[], materialization +""" + +from __future__ import annotations + +import yaml +from pydantic import BaseModel, ConfigDict + + +class MetricViewFormat(BaseModel): + """Format specification for display in visualization tools.""" + + model_config = ConfigDict(frozen=True) + + type: str + currency_code: str | None = None + decimal_places: dict | None = None + hide_group_separator: bool | None = None + abbreviation: str | None = None + date_format: str | None = None + time_format: str | None = None + leading_zeros: bool | None = None + + +class MetricViewField(BaseModel): + """A field (dimension) in a Metric View definition.""" + + model_config = ConfigDict(frozen=True) + + name: str + expr: str + comment: str | None = None + display_name: str | None = None + synonyms: list[str] | None = None + format: MetricViewFormat | None = None + + +class MetricViewWindow(BaseModel): + """Window specification for a windowed measure.""" + + model_config = ConfigDict(frozen=True) + + order: str + range: str + semiadditive: str | None = None + + +class MetricViewMeasure(BaseModel): + """A measure in a Metric View definition.""" + + model_config = ConfigDict(frozen=True) + + name: str + expr: str + comment: str | None = None + display_name: str | None = None + synonyms: list[str] | None = None + format: MetricViewFormat | None = None + window: list[MetricViewWindow] | None = None + + +class MetricViewRely(BaseModel): + """Join optimization hints.""" + + model_config = ConfigDict(frozen=True) + + at_most_one_match: bool | None = None + + +class MetricViewJoin(BaseModel): + """A join definition in a Metric View.""" + + model_config = ConfigDict(frozen=True) + + name: str + source: str + on: str | None = None + using: list[str] | None = None + cardinality: str | None = None + rely: MetricViewRely | None = None + joins: list[MetricViewJoin] | None = None + + +class MetricViewMaterializedView(BaseModel): + """A materialized view definition within materialization config.""" + + model_config = ConfigDict(frozen=True) + + name: str + type: str + dimensions: list[str] | None = None + measures: list[str] | None = None + + +class MetricViewMaterialization(BaseModel): + """Materialization configuration for query acceleration.""" + + model_config = ConfigDict(frozen=True) + + schedule: str | None = None + mode: str | None = None + materialized_views: list[MetricViewMaterializedView] | None = None + + +# Key ordering for YAML serialization +_YAML_KEY_ORDER = [ + "version", + "source", + "comment", + "filter", + "joins", + "fields", + "measures", + "materialization", +] + +# Strings that PyYAML's safe_load interprets as non-string types. +# We must force-quote these when serializing to ensure round-trip fidelity. +_YAML_BOOL_STRINGS = frozenset({ + "true", "false", "yes", "no", "on", "off", + "True", "False", "Yes", "No", "On", "Off", + "TRUE", "FALSE", "YES", "NO", "ON", "OFF", +}) +_YAML_NULL_STRINGS = frozenset({"null", "Null", "NULL", "~", ""}) + +# Characters that can cause YAML parsing ambiguity if unquoted +_YAML_SPECIAL_CHARS = set(":{}[]|>&*!#%@`,'\"?") + + +def _needs_quoting(value: str) -> bool: + """Determine if a string value needs explicit quoting for safe YAML round-trip.""" + if not value: + return True + if value in _YAML_BOOL_STRINGS or value in _YAML_NULL_STRINGS: + return True + # Strings that look like numbers or dates + try: + float(value) + return True + except (ValueError, OverflowError): + pass + # Contains characters that could confuse YAML parsers + if any(c in _YAML_SPECIAL_CHARS for c in value): + return True + # Starts or ends with whitespace + if value != value.strip(): + return True + # Contains newlines or other control characters + if any(ord(c) < 32 or ord(c) == 127 for c in value): + return True + # Contains non-ASCII that might not round-trip + if any(ord(c) > 126 for c in value): + return True + return False + + +class _SafeStrDumper(yaml.SafeDumper): + """Custom YAML dumper that quotes strings when needed for round-trip safety.""" + + +def _safe_str_representer(dumper: _SafeStrDumper, data: str) -> yaml.ScalarNode: + """Represent strings with quoting when they could be misinterpreted.""" + if _needs_quoting(data): + return dumper.represent_scalar("tag:yaml.org,2002:str", data, style="'") + return dumper.represent_scalar("tag:yaml.org,2002:str", data) + + +_SafeStrDumper.add_representer(str, _safe_str_representer) + + +def _ordered_dump(data: dict, **kwargs) -> str: + """Dump a dict to YAML with keys in the canonical Metric View order. + + Uses a custom dumper that forces quoting on strings that could be + misinterpreted by YAML parsers (booleans, nulls, numbers, special chars). + """ + ordered = {} + for key in _YAML_KEY_ORDER: + if key in data: + ordered[key] = data[key] + # Include any remaining keys not in the predefined order + for key in data: + if key not in ordered: + ordered[key] = data[key] + return yaml.dump( + ordered, + Dumper=_SafeStrDumper, + default_flow_style=False, + sort_keys=False, + allow_unicode=True, + **kwargs, + ) + + +class MetricViewModel(BaseModel): + """Root model for a Databricks Metric View YAML definition (v1.1). + + Args: + version: Metric View spec version. + source: Three-part table name or SQL query. + comment: Top-level description of the metric view. + filter: Boolean filter expression applied to all queries. + joins: Join definitions for related tables. + fields: Dimension fields. + measures: Aggregation measures. + materialization: Materialization configuration for query acceleration. + """ + + model_config = ConfigDict(frozen=True) + + version: str = "1.1" + source: str + comment: str | None = None + filter: str | None = None + joins: list[MetricViewJoin] | None = None + fields: list[MetricViewField] | None = None + measures: list[MetricViewMeasure] | None = None + materialization: MetricViewMaterialization | None = None + + def to_yaml(self) -> str: + """Serialize to Metric View YAML with correct field ordering. + + Returns: + YAML string with keys ordered per Metric View convention, + excluding None-valued optional fields. + """ + data = self.model_dump(exclude_none=True) + return _ordered_dump(data) + + @classmethod + def from_yaml(cls, yaml_str: str) -> MetricViewModel: + """Parse a Metric View YAML string into a validated model. + + Args: + yaml_str: Raw YAML content. + + Returns: + A validated MetricViewModel instance. + + Raises: + pydantic.ValidationError: If required fields are missing or invalid. + yaml.YAMLError: If the YAML syntax is invalid. + """ + raw = yaml.safe_load(yaml_str) + return cls.model_validate(raw) diff --git a/converters/databricks/src/osi_databricks/osi_to_metric_view.py b/converters/databricks/src/osi_databricks/osi_to_metric_view.py new file mode 100644 index 0000000..576e65b --- /dev/null +++ b/converters/databricks/src/osi_databricks/osi_to_metric_view.py @@ -0,0 +1,462 @@ +"""Export: Convert OSI semantic model to Databricks Metric View YAML. + +Maps an OSIDocument to one or more MetricViewModel instances following the +hub-and-spoke architecture. Each OSI dataset that contains fields or associated +metrics produces one MetricViewModel. +""" + +from __future__ import annotations + +import json +import warnings + +from osi.models import ( + OSIAIContextObject, + OSICustomExtension, + OSIDataset, + OSIDocument, + OSIField, + OSIMetric, + OSIRelationship, + OSISemanticModel, + OSIVendor, +) + +from osi_databricks.dialect_utils import select_dialect_expression +from osi_databricks.models import ( + MetricViewField, + MetricViewFormat, + MetricViewJoin, + MetricViewMaterialization, + MetricViewMeasure, + MetricViewModel, + MetricViewRely, + MetricViewWindow, +) + + +def osi_to_metric_view( + document: OSIDocument, +) -> list[tuple[str, MetricViewModel]]: + """Convert an OSI document to Metric View definitions. + + Produces one MetricViewModel per OSI dataset that has fields or associated + metrics. Custom extensions for other vendors (SNOWFLAKE, DBT, etc.) are + ignored during export without being discarded from the source model. + + Args: + document: A validated OSI document. + + Returns: + List of (dataset_name, MetricViewModel) tuples, one per dataset that + has fields or associated metrics. + """ + results: list[tuple[str, MetricViewModel]] = [] + + for semantic_model in document.semantic_model: + # Build a lookup of metrics by name for association + metrics = semantic_model.metrics or [] + + # Build a lookup of relationships by from_dataset + relationships = semantic_model.relationships or [] + rel_by_from: dict[str, list[OSIRelationship]] = {} + for rel in relationships: + rel_by_from.setdefault(rel.from_dataset, []).append(rel) + + # Get semantic model-level custom extensions (materialization, instructions) + sm_materialization = _extract_materialization(semantic_model) + + for dataset in semantic_model.datasets: + # Check if dataset has fields or associated metrics + has_fields = dataset.fields is not None and len(dataset.fields) > 0 + has_metrics = len(metrics) > 0 + + if not has_fields and not has_metrics: + continue + + mv_model = _build_metric_view_model( + dataset=dataset, + metrics=metrics, + relationships=rel_by_from.get(dataset.name, []), + materialization=sm_materialization, + semantic_model=semantic_model, + ) + results.append((dataset.name, mv_model)) + + return results + + +def _build_metric_view_model( + dataset: OSIDataset, + metrics: list[OSIMetric], + relationships: list[OSIRelationship], + materialization: MetricViewMaterialization | None, + semantic_model: OSISemanticModel, +) -> MetricViewModel: + """Build a MetricViewModel from an OSI dataset and its associated data. + + Args: + dataset: The OSI dataset to convert. + metrics: All metrics in the semantic model. + relationships: Relationships originating from this dataset. + materialization: Materialization config extracted from semantic model extensions. + semantic_model: The parent semantic model (for model-level metadata). + + Returns: + A fully populated MetricViewModel. + """ + # Determine source + source = _resolve_source(dataset) + + # Build fields + fields = _build_fields(dataset.fields) if dataset.fields else None + + # Build measures from metrics + measures = _build_measures(metrics) if metrics else None + + # Build joins from relationships + joins = _build_joins(relationships, dataset.name) if relationships else None + + # Extract filter from dataset custom extensions + ds_filter = _extract_filter(dataset) + + # Extract comment from semantic model description + comment = semantic_model.description + + return MetricViewModel( + version="1.1", + source=source, + comment=comment, + filter=ds_filter, + joins=joins, + fields=fields, + measures=measures, + materialization=materialization, + ) + + +def _resolve_source(dataset: OSIDataset) -> str: + """Determine the Metric View source from an OSI dataset. + + Checks for a source_query in DATABRICKS custom extensions first, + falling back to the dataset.source field. + + Args: + dataset: The OSI dataset. + + Returns: + Source string (either SQL query or three-part table name). + """ + # Check for source_query in custom extensions + source_query = _get_databricks_extension_value(dataset.custom_extensions, "source_query") + if source_query is not None: + return source_query + return dataset.source + + +def _build_fields(osi_fields: list[OSIField]) -> list[MetricViewField] | None: + """Convert OSI fields to Metric View fields with dialect selection. + + Uses DATABRICKS dialect preferred, ANSI_SQL as fallback. Fields with no + usable dialect are skipped with a warning. + + Args: + osi_fields: List of OSI field instances. + + Returns: + List of MetricViewField instances, or None if all fields were skipped. + """ + fields: list[MetricViewField] = [] + + for osi_field in osi_fields: + expr = select_dialect_expression(osi_field.expression.dialects) + if expr is None: + warnings.warn( + f"Skipping field '{osi_field.name}': no DATABRICKS or ANSI_SQL dialect available", + stacklevel=2, + ) + continue + + display_name, synonyms = _extract_ai_context_metadata(osi_field.ai_context) + + # Extract format from custom extensions + fmt = _extract_format(osi_field.custom_extensions) + + fields.append( + MetricViewField( + name=osi_field.name, + expr=expr, + comment=osi_field.description, + display_name=display_name, + synonyms=synonyms, + format=fmt, + ) + ) + + return fields if fields else None + + +def _build_measures(osi_metrics: list[OSIMetric]) -> list[MetricViewMeasure] | None: + """Convert OSI metrics to Metric View measures with dialect selection. + + Uses DATABRICKS dialect preferred, ANSI_SQL as fallback. Metrics with no + usable dialect are skipped with a warning. + + Args: + osi_metrics: List of OSI metric instances. + + Returns: + List of MetricViewMeasure instances, or None if all metrics were skipped. + """ + measures: list[MetricViewMeasure] = [] + + for osi_metric in osi_metrics: + expr = select_dialect_expression(osi_metric.expression.dialects) + if expr is None: + warnings.warn( + f"Skipping metric '{osi_metric.name}': no DATABRICKS or ANSI_SQL dialect available", + stacklevel=2, + ) + continue + + display_name, synonyms = _extract_ai_context_metadata(osi_metric.ai_context) + + # Extract window and format from custom extensions + window = _extract_window(osi_metric.custom_extensions) + fmt = _extract_format(osi_metric.custom_extensions) + + measures.append( + MetricViewMeasure( + name=osi_metric.name, + expr=expr, + comment=osi_metric.description, + display_name=display_name, + synonyms=synonyms, + format=fmt, + window=window, + ) + ) + + return measures if measures else None + + +def _build_joins( + relationships: list[OSIRelationship], + from_dataset: str, +) -> list[MetricViewJoin] | None: + """Convert OSI relationships to Metric View joins. + + Reconstructs ON clauses from from_columns/to_columns pairs. + + Args: + relationships: List of OSI relationships originating from the dataset. + from_dataset: Name of the source dataset. + + Returns: + List of MetricViewJoin instances, or None if no relationships. + """ + if not relationships: + return None + + joins: list[MetricViewJoin] = [] + + for rel in relationships: + on_clause = _build_on_clause(rel, from_dataset) + + # Extract cardinality and rely from custom extensions + cardinality = None + rely = None + rel_ext = _get_databricks_extension_data(rel.custom_extensions) + if rel_ext: + cardinality = rel_ext.get("cardinality") + rely_data = rel_ext.get("rely") + if rely_data: + rely = MetricViewRely(**rely_data) + + joins.append( + MetricViewJoin( + name=rel.name, + source=rel.to, + on=on_clause, + cardinality=cardinality, + rely=rely, + ) + ) + + return joins if joins else None + + +def _build_on_clause(rel: OSIRelationship, from_dataset: str) -> str: + """Build an ON clause from OSI relationship columns. + + Produces: source.from_col = target.to_col [AND ...] + + Args: + rel: OSI relationship with from_columns and to_columns. + from_dataset: Name of the source dataset (used as left-side alias). + + Returns: + ON clause string. + """ + parts = [] + for from_col, to_col in zip(rel.from_columns, rel.to_columns): + parts.append(f"source.{from_col} = {rel.name}.{to_col}") + return " AND ".join(parts) + + +def _extract_ai_context_metadata( + ai_context, +) -> tuple[str | None, list[str] | None]: + """Extract display_name and synonyms from an OSI AI context. + + The first synonym becomes display_name, remaining become the synonyms list. + + Args: + ai_context: OSI AI context (string or OSIAIContextObject), or None. + + Returns: + Tuple of (display_name, synonyms) where either may be None. + """ + if ai_context is None: + return None, None + + if isinstance(ai_context, str): + return None, None + + if not isinstance(ai_context, OSIAIContextObject): + return None, None + + if ai_context.synonyms is None or len(ai_context.synonyms) == 0: + return None, None + + synonyms_list = list(ai_context.synonyms) + display_name = synonyms_list[0] + remaining = synonyms_list[1:] if len(synonyms_list) > 1 else None + + return display_name, remaining if remaining else None + + +def _extract_filter(dataset: OSIDataset) -> str | None: + """Extract filter expression from dataset DATABRICKS custom extensions. + + Args: + dataset: OSI dataset to check. + + Returns: + Filter expression string or None. + """ + return _get_databricks_extension_value(dataset.custom_extensions, "filter") + + +def _extract_materialization( + semantic_model: OSISemanticModel, +) -> MetricViewMaterialization | None: + """Extract materialization config from semantic model DATABRICKS custom extensions. + + Args: + semantic_model: OSI semantic model to check. + + Returns: + MetricViewMaterialization instance or None. + """ + mat_data = _get_databricks_extension_value(semantic_model.custom_extensions, "materialization") + if mat_data is None: + return None + + if isinstance(mat_data, dict): + return MetricViewMaterialization.model_validate(mat_data) + return None + + +def _extract_format( + custom_extensions: list[OSICustomExtension] | None, +) -> MetricViewFormat | None: + """Extract format config from DATABRICKS custom extensions. + + Args: + custom_extensions: List of custom extensions to search. + + Returns: + MetricViewFormat instance or None. + """ + format_data = _get_databricks_extension_value(custom_extensions, "format") + if format_data is None: + return None + if isinstance(format_data, dict): + return MetricViewFormat.model_validate(format_data) + return None + + +def _extract_window( + custom_extensions: list[OSICustomExtension] | None, +) -> list[MetricViewWindow] | None: + """Extract window config from DATABRICKS custom extensions. + + Args: + custom_extensions: List of custom extensions to search. + + Returns: + List of MetricViewWindow instances or None. + """ + window_data = _get_databricks_extension_value(custom_extensions, "window") + if window_data is None: + return None + if isinstance(window_data, list): + return [MetricViewWindow.model_validate(w) for w in window_data] + return None + + +def _get_databricks_extension_value( + custom_extensions: list[OSICustomExtension] | None, + key: str, +): + """Get a specific value from DATABRICKS custom extensions. + + Searches through all DATABRICKS vendor extensions for the given key. + + Args: + custom_extensions: List of custom extensions to search. + key: The key to look for in the extension data JSON. + + Returns: + The value associated with the key, or None if not found. + """ + if not custom_extensions: + return None + + for ext in custom_extensions: + if ext.vendor_name == OSIVendor.DATABRICKS: + try: + data = json.loads(ext.data) + if key in data: + return data[key] + except (json.JSONDecodeError, TypeError): + continue + + return None + + +def _get_databricks_extension_data( + custom_extensions: list[OSICustomExtension] | None, +) -> dict | None: + """Get the full merged data dict from all DATABRICKS custom extensions. + + Args: + custom_extensions: List of custom extensions to search. + + Returns: + Merged dictionary of all DATABRICKS extension data, or None. + """ + if not custom_extensions: + return None + + merged: dict = {} + for ext in custom_extensions: + if ext.vendor_name == OSIVendor.DATABRICKS: + try: + data = json.loads(ext.data) + merged.update(data) + except (json.JSONDecodeError, TypeError): + continue + + return merged if merged else None diff --git a/converters/databricks/tests/__init__.py b/converters/databricks/tests/__init__.py new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/converters/databricks/tests/__init__.py @@ -0,0 +1 @@ + diff --git a/converters/databricks/tests/conftest.py b/converters/databricks/tests/conftest.py new file mode 100644 index 0000000..9f419c0 --- /dev/null +++ b/converters/databricks/tests/conftest.py @@ -0,0 +1,532 @@ +"""Shared pytest fixtures and Hypothesis strategies for osi-databricks tests. + +This module provides: +- Hypothesis strategies for generating valid MetricViewModel instances +- Hypothesis strategies for generating valid OSIDocument instances with DATABRICKS dialects +- Pytest fixtures that load TPC-DS YAML fixture files from disk +""" + +from __future__ import annotations + +import json +from pathlib import Path + +import pytest +import yaml +from hypothesis import strategies as st +from osi.models import ( + OSIAIContextObject, + OSICustomExtension, + OSIDataset, + OSIDialect, + OSIDialectExpression, + OSIDimension, + OSIDocument, + OSIExpression, + OSIField, + OSIMetric, + OSIRelationship, + OSISemanticModel, + OSIVendor, +) + +from osi_databricks.models import ( + MetricViewField, + MetricViewFormat, + MetricViewJoin, + MetricViewMaterialization, + MetricViewMaterializedView, + MetricViewMeasure, + MetricViewModel, + MetricViewRely, + MetricViewWindow, +) + +# --------------------------------------------------------------------------- +# Paths +# --------------------------------------------------------------------------- + +FIXTURES_DIR = Path(__file__).parent / "fixtures" + + +# --------------------------------------------------------------------------- +# Pytest Fixtures — TPC-DS Fixture Files +# --------------------------------------------------------------------------- + + +@pytest.fixture +def metric_view_tpcds_yaml() -> str: + """Raw YAML string of the TPC-DS Metric View fixture.""" + return (FIXTURES_DIR / "metric_view_tpcds.yaml").read_text() + + +@pytest.fixture +def metric_view_tpcds_model(metric_view_tpcds_yaml: str) -> MetricViewModel: + """Parsed MetricViewModel from the TPC-DS fixture.""" + return MetricViewModel.from_yaml(metric_view_tpcds_yaml) + + +@pytest.fixture +def osi_tpcds_yaml() -> str: + """Raw YAML string of the TPC-DS OSI fixture.""" + return (FIXTURES_DIR / "osi_tpcds.yaml").read_text() + + +@pytest.fixture +def osi_tpcds_document(osi_tpcds_yaml: str) -> OSIDocument: + """Parsed OSIDocument from the TPC-DS fixture.""" + raw = yaml.safe_load(osi_tpcds_yaml) + return OSIDocument.model_validate(raw) + + +# --------------------------------------------------------------------------- +# Base Hypothesis Strategies — Building Blocks +# --------------------------------------------------------------------------- + +# Simple SQL-safe identifiers (lowercase, starts with letter) +identifier = st.from_regex(r"[a-z][a-z0-9_]{0,19}", fullmatch=True) + +# Three-part catalog.schema.table names +three_part_name = st.builds( + lambda a, b, c: f"{a}.{b}.{c}", + identifier, + identifier, + identifier, +) + +# Simple column-like expressions (standard SQL, no Databricks-specific syntax) +standard_expr = st.from_regex(r"[A-Za-z_][A-Za-z0-9_]{0,19}", fullmatch=True) + +# Safe text for metadata fields (avoids JSON/YAML issues) +safe_text = st.text( + alphabet=st.characters( + min_codepoint=32, + max_codepoint=126, + blacklist_characters='"\\', + ), + min_size=1, + max_size=30, +) + +# Aggregate function expressions +aggregate_expr = st.builds( + lambda func, col: f"{func}({col})", + st.sampled_from(["SUM", "COUNT", "AVG", "MAX", "MIN"]), + standard_expr, +) + + +# --------------------------------------------------------------------------- +# Hypothesis Strategies — MetricViewModel Components +# --------------------------------------------------------------------------- + + +@st.composite +def mv_formats(draw): + """Generate valid MetricViewFormat instances.""" + fmt_type = draw(st.sampled_from(["number", "currency", "date", "percentage"])) + kwargs = {"type": fmt_type} + + if fmt_type == "currency": + kwargs["currency_code"] = draw(st.sampled_from(["USD", "EUR", "GBP"])) + elif fmt_type == "date": + kwargs["date_format"] = draw(st.sampled_from(["year_month_day", "month_day_year"])) + elif fmt_type == "number": + kwargs["decimal_places"] = {"min": 2, "max": 2} + + return MetricViewFormat(**kwargs) + + +@st.composite +def mv_fields(draw, *, with_metadata: bool = True): + """Generate valid MetricViewField instances. + + Args: + with_metadata: If True, may include comment, display_name, synonyms, format. + """ + name = draw(identifier) + expr = draw(standard_expr) + + kwargs = {"name": name, "expr": expr} + + if with_metadata: + kwargs["comment"] = draw(st.none() | safe_text) + kwargs["display_name"] = draw(st.none() | safe_text) + kwargs["synonyms"] = draw(st.none() | st.lists(safe_text, min_size=1, max_size=3)) + kwargs["format"] = draw(st.none() | mv_formats()) + + return MetricViewField(**kwargs) + + +@st.composite +def mv_windows(draw): + """Generate valid MetricViewWindow instances.""" + order = draw(identifier) + range_val = draw(st.sampled_from([ + "trailing 7 day", + "trailing 30 day", + "trailing 1 month", + "unbounded", + ])) + semiadditive = draw(st.none() | st.sampled_from(["last", "first"])) + return MetricViewWindow(order=order, range=range_val, semiadditive=semiadditive) + + +@st.composite +def mv_measures(draw, *, with_metadata: bool = True): + """Generate valid MetricViewMeasure instances. + + Args: + with_metadata: If True, may include comment, display_name, synonyms, format, window. + """ + name = draw(identifier) + expr = draw(aggregate_expr) + + kwargs = {"name": name, "expr": expr} + + if with_metadata: + kwargs["comment"] = draw(st.none() | safe_text) + kwargs["display_name"] = draw(st.none() | safe_text) + kwargs["synonyms"] = draw(st.none() | st.lists(safe_text, min_size=1, max_size=3)) + kwargs["format"] = draw(st.none() | mv_formats()) + kwargs["window"] = draw(st.none() | st.lists(mv_windows(), min_size=1, max_size=2)) + + return MetricViewMeasure(**kwargs) + + +@st.composite +def mv_joins(draw, *, allow_nested: bool = False): + """Generate valid MetricViewJoin instances. + + The ON clause uses the format: source.from_col = join_name.to_col + which matches what the converter produces on export. + + Args: + allow_nested: If True, may include nested joins (one level deep only). + """ + name = draw(identifier) + source = draw(three_part_name) + from_col = draw(identifier) + to_col = draw(identifier) + on_clause = f"source.{from_col} = {name}.{to_col}" + + cardinality = draw(st.none() | st.sampled_from(["many_to_one", "one_to_many"])) + rely = draw(st.none() | st.just(MetricViewRely(at_most_one_match=True))) + + nested_joins = None + if allow_nested: + nested_joins = draw(st.none() | st.lists(mv_joins(allow_nested=False), min_size=1, max_size=1)) + + return MetricViewJoin( + name=name, + source=source, + on=on_clause, + cardinality=cardinality, + rely=rely, + joins=nested_joins, + ) + + +@st.composite +def mv_materializations(draw): + """Generate valid MetricViewMaterialization instances.""" + schedule = draw(st.none() | st.sampled_from(["every 6 hours", "every 1 hour", "daily"])) + mode = draw(st.none() | st.sampled_from(["relaxed", "strict"])) + + # Optionally generate materialized views + mat_views = draw(st.none() | st.lists( + st.builds( + MetricViewMaterializedView, + name=identifier, + type=st.sampled_from(["aggregated", "unaggregated"]), + dimensions=st.none() | st.lists(identifier, min_size=1, max_size=3), + measures=st.none() | st.lists(identifier, min_size=1, max_size=2), + ), + min_size=1, + max_size=2, + )) + + return MetricViewMaterialization( + schedule=schedule, + mode=mode, + materialized_views=mat_views, + ) + + +@st.composite +def mv_models(draw, *, with_joins: bool = True, with_materialization: bool = True): + """Generate valid MetricViewModel instances. + + Produces models with three-part name sources, fields, optional measures, + optional joins, and optional materialization. + + Args: + with_joins: If True, may include joins. + with_materialization: If True, may include materialization config. + """ + source = draw(three_part_name) + comment = draw(st.none() | safe_text) + filter_expr = draw(st.none() | standard_expr) + fields = draw(st.lists(mv_fields(), min_size=1, max_size=5)) + measures = draw(st.none() | st.lists(mv_measures(), min_size=1, max_size=4)) + + joins = None + if with_joins: + joins = draw(st.none() | st.lists(mv_joins(allow_nested=True), min_size=1, max_size=3)) + + materialization = None + if with_materialization: + materialization = draw(st.none() | mv_materializations()) + + return MetricViewModel( + version="1.1", + source=source, + comment=comment, + filter=filter_expr, + fields=fields, + measures=measures, + joins=joins, + materialization=materialization, + ) + + +# --------------------------------------------------------------------------- +# Hypothesis Strategies — OSI Document Components +# --------------------------------------------------------------------------- + + +@st.composite +def osi_fields(draw, *, with_ansi: bool = False, with_metadata: bool = True): + """Generate valid OSIField instances with DATABRICKS dialect. + + Args: + with_ansi: If True, also includes an ANSI_SQL dialect expression. + with_metadata: If True, may include description and ai_context. + """ + name = draw(identifier) + expr_str = draw(standard_expr) + + dialects = [OSIDialectExpression(dialect=OSIDialect.DATABRICKS, expression=expr_str)] + if with_ansi: + dialects.append(OSIDialectExpression(dialect=OSIDialect.ANSI_SQL, expression=expr_str)) + + kwargs: dict = { + "name": name, + "expression": OSIExpression(dialects=dialects), + "dimension": OSIDimension(is_time=draw(st.booleans())), + } + + if with_metadata: + kwargs["description"] = draw(st.none() | safe_text) + synonyms = draw(st.none() | st.lists(safe_text, min_size=1, max_size=4)) + if synonyms: + kwargs["ai_context"] = OSIAIContextObject(synonyms=tuple(synonyms)) + + return OSIField(**kwargs) + + +@st.composite +def osi_metrics(draw, *, with_ansi: bool = False, with_metadata: bool = True): + """Generate valid OSIMetric instances with DATABRICKS dialect. + + Args: + with_ansi: If True, also includes an ANSI_SQL dialect expression. + with_metadata: If True, may include description and ai_context. + """ + name = draw(identifier) + expr_str = draw(aggregate_expr) + + dialects = [OSIDialectExpression(dialect=OSIDialect.DATABRICKS, expression=expr_str)] + if with_ansi: + dialects.append(OSIDialectExpression(dialect=OSIDialect.ANSI_SQL, expression=expr_str)) + + kwargs: dict = { + "name": name, + "expression": OSIExpression(dialects=dialects), + } + + if with_metadata: + kwargs["description"] = draw(st.none() | safe_text) + synonyms = draw(st.none() | st.lists(safe_text, min_size=1, max_size=4)) + if synonyms: + kwargs["ai_context"] = OSIAIContextObject(synonyms=tuple(synonyms)) + + return OSIMetric(**kwargs) + + +@st.composite +def osi_relationships(draw, dataset_name: str): + """Generate valid OSIRelationship instances. + + Args: + dataset_name: The name of the 'from' dataset for the relationship. + """ + name = draw(identifier) + from_col = draw(identifier) + to_col = draw(identifier) + + cardinality = draw(st.none() | st.sampled_from(["many_to_one", "one_to_many"])) + custom_extensions = None + if cardinality: + custom_extensions = [ + OSICustomExtension( + vendor_name=OSIVendor.DATABRICKS, + data=json.dumps({"cardinality": cardinality}), + ) + ] + + return OSIRelationship( + name=name, + **{"from": dataset_name}, + to=name, # target dataset = relationship name for simplicity + from_columns=[from_col], + to_columns=[to_col], + custom_extensions=custom_extensions, + ) + + +@st.composite +def osi_documents( + draw, + *, + with_metrics: bool = True, + with_relationships: bool = True, + with_ansi: bool = False, +): + """Generate valid OSIDocument instances with DATABRICKS dialect expressions. + + Produces documents with a single semantic model containing one dataset, + optional metrics, and optional relationships. + + Args: + with_metrics: If True, may include metrics. + with_relationships: If True, may include relationships. + with_ansi: If True, fields/metrics also get ANSI_SQL dialect entries. + """ + dataset_name = draw(identifier) + source = f"catalog.schema.{dataset_name}" + fields = draw(st.lists(osi_fields(with_ansi=with_ansi), min_size=1, max_size=4)) + + metrics = None + if with_metrics: + metrics = draw(st.none() | st.lists(osi_metrics(with_ansi=with_ansi), min_size=1, max_size=3)) + + relationships = None + if with_relationships: + relationships = draw( + st.none() | st.lists(osi_relationships(dataset_name), min_size=1, max_size=2) + ) + + description = draw(st.none() | safe_text) + + dataset = OSIDataset( + name=dataset_name, + source=source, + fields=fields, + ) + semantic_model = OSISemanticModel( + name="test_model", + description=description, + datasets=[dataset], + relationships=relationships, + metrics=metrics, + ) + + dialects = [OSIDialect.DATABRICKS] + if with_ansi: + dialects.append(OSIDialect.ANSI_SQL) + + return OSIDocument( + version="0.2.0.dev0", + dialects=dialects, + vendors=[OSIVendor.DATABRICKS], + semantic_model=[semantic_model], + ) + + +@st.composite +def osi_documents_with_extensions(draw): + """Generate OSIDocument instances with DATABRICKS and other vendor custom extensions. + + Useful for testing that round-trips preserve custom extensions for all vendors. + """ + dataset_name = draw(identifier) + source = f"catalog.schema.{dataset_name}" + expr_str = draw(standard_expr) + + # DATABRICKS dataset extension (filter) + filter_expr = draw(st.none() | standard_expr) + ds_extensions = [] + if filter_expr: + ds_extensions.append( + OSICustomExtension( + vendor_name=OSIVendor.DATABRICKS, + data=json.dumps({"filter": filter_expr}), + ) + ) + + # Other vendor extension that should survive round-trips + other_vendor = draw(st.sampled_from([OSIVendor.SNOWFLAKE, OSIVendor.DBT, OSIVendor.GOODDATA])) + other_key = draw(identifier) + other_value = draw(safe_text) + ds_extensions.append( + OSICustomExtension( + vendor_name=other_vendor, + data=json.dumps({other_key: other_value}), + ) + ) + + # Field with format extension + field = OSIField( + name=draw(identifier), + expression=OSIExpression( + dialects=[OSIDialectExpression(dialect=OSIDialect.DATABRICKS, expression=expr_str)] + ), + dimension=OSIDimension(is_time=False), + custom_extensions=[ + OSICustomExtension( + vendor_name=OSIVendor.DATABRICKS, + data=json.dumps({"format": {"type": "number"}}), + ) + ], + ) + + # Metric with window extension + metric = OSIMetric( + name=draw(identifier), + expression=OSIExpression( + dialects=[OSIDialectExpression(dialect=OSIDialect.DATABRICKS, expression=f"SUM({expr_str})")] + ), + custom_extensions=[ + OSICustomExtension( + vendor_name=OSIVendor.DATABRICKS, + data=json.dumps({"window": [{"order": "d_date", "range": "trailing 7 day"}]}), + ) + ], + ) + + # Semantic model with materialization extension + sm_ext = OSICustomExtension( + vendor_name=OSIVendor.DATABRICKS, + data=json.dumps({"materialization": {"schedule": "every 6 hours", "mode": "relaxed"}}), + ) + + dataset = OSIDataset( + name=dataset_name, + source=source, + fields=[field], + custom_extensions=ds_extensions if ds_extensions else None, + ) + semantic_model = OSISemanticModel( + name="test_model", + datasets=[dataset], + metrics=[metric], + custom_extensions=[sm_ext], + ) + + return OSIDocument( + version="0.2.0.dev0", + dialects=[OSIDialect.DATABRICKS], + vendors=[OSIVendor.DATABRICKS], + semantic_model=[semantic_model], + ) diff --git a/converters/databricks/tests/fixtures/.gitkeep b/converters/databricks/tests/fixtures/.gitkeep new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/converters/databricks/tests/fixtures/.gitkeep @@ -0,0 +1 @@ + diff --git a/converters/databricks/tests/fixtures/metric_view_tpcds.yaml b/converters/databricks/tests/fixtures/metric_view_tpcds.yaml new file mode 100644 index 0000000..bac92ba --- /dev/null +++ b/converters/databricks/tests/fixtures/metric_view_tpcds.yaml @@ -0,0 +1,96 @@ +version: '1.1' +source: tpcds.analytics.store_sales +comment: Store sales fact table with date and item dimensions from TPC-DS benchmark +filter: ss_net_profit > 0 +joins: + - name: date_dim + source: tpcds.analytics.date_dim + on: source.ss_sold_date_sk = date_dim.d_date_sk + cardinality: many_to_one + rely: + at_most_one_match: true + - name: item + source: tpcds.analytics.item + on: source.ss_item_sk = item.i_item_sk + cardinality: many_to_one +fields: + - name: sold_date + expr: DATE_TRUNC('DAY', d_date) + comment: Date the sale occurred + display_name: Sale Date + synonyms: + - transaction date + - order date + format: + type: date + date_format: year_month_day + - name: sold_year + expr: YEAR(d_date) + comment: Year the sale occurred + display_name: Sale Year + - name: item_id + expr: i_item_id + comment: Unique item identifier + - name: item_category + expr: i_category + comment: Product category + display_name: Category + synonyms: + - product category + - item type + - name: store_id + expr: ss_store_sk + comment: Store identifier +measures: + - name: total_sales + expr: SUM(ss_sales_price) + comment: Total sales revenue + display_name: Total Sales + synonyms: + - revenue + - gross sales + format: + type: currency + currency_code: USD + - name: total_quantity + expr: SUM(ss_quantity) + comment: Total units sold + display_name: Units Sold + - name: avg_discount + expr: AVG(ss_ext_discount_amt) + comment: Average discount amount per transaction + display_name: Avg Discount + format: + type: number + decimal_places: + min: 2 + max: 2 + - name: net_profit + expr: SUM(ss_net_profit) + comment: Total net profit + display_name: Net Profit + synonyms: + - profit + - margin + format: + type: currency + currency_code: USD + - name: trailing_7d_sales + expr: SUM(ss_sales_price) + comment: Trailing 7-day rolling sales + display_name: 7-Day Sales + window: + - order: sold_date + range: trailing 7 day +materialization: + schedule: every 6 hours + mode: relaxed + materialized_views: + - name: sales_by_date + type: aggregated + dimensions: + - sold_date + - item_category + measures: + - total_sales + - total_quantity diff --git a/converters/databricks/tests/fixtures/osi_tpcds.yaml b/converters/databricks/tests/fixtures/osi_tpcds.yaml new file mode 100644 index 0000000..8670208 --- /dev/null +++ b/converters/databricks/tests/fixtures/osi_tpcds.yaml @@ -0,0 +1,174 @@ +version: 0.2.0.dev0 +dialects: + - DATABRICKS + - ANSI_SQL +vendors: + - DATABRICKS +semantic_model: + - name: tpcds_store_sales + description: Store sales fact table with date and item dimensions from TPC-DS benchmark + datasets: + - name: store_sales + source: tpcds.analytics.store_sales + fields: + - name: sold_date + expression: + dialects: + - dialect: DATABRICKS + expression: DATE_TRUNC('DAY', d_date) + dimension: + is_time: true + description: Date the sale occurred + ai_context: + synonyms: + - Sale Date + - transaction date + - order date + custom_extensions: + - vendor_name: DATABRICKS + data: '{"format": {"type": "date", "date_format": "year_month_day"}}' + - name: sold_year + expression: + dialects: + - dialect: DATABRICKS + expression: YEAR(d_date) + dimension: + is_time: true + description: Year the sale occurred + ai_context: + synonyms: + - Sale Year + - name: item_id + expression: + dialects: + - dialect: DATABRICKS + expression: i_item_id + - dialect: ANSI_SQL + expression: i_item_id + dimension: + is_time: false + description: Unique item identifier + - name: item_category + expression: + dialects: + - dialect: DATABRICKS + expression: i_category + - dialect: ANSI_SQL + expression: i_category + dimension: + is_time: false + description: Product category + ai_context: + synonyms: + - Category + - product category + - item type + - name: store_id + expression: + dialects: + - dialect: DATABRICKS + expression: ss_store_sk + - dialect: ANSI_SQL + expression: ss_store_sk + dimension: + is_time: false + description: Store identifier + custom_extensions: + - vendor_name: DATABRICKS + data: '{"filter": "ss_net_profit > 0"}' + relationships: + - name: date_dim + from: store_sales + to: date_dim + from_columns: + - ss_sold_date_sk + to_columns: + - d_date_sk + custom_extensions: + - vendor_name: DATABRICKS + data: '{"cardinality": "many_to_one", "rely": {"at_most_one_match": true}}' + - name: item + from: store_sales + to: item + from_columns: + - ss_item_sk + to_columns: + - i_item_sk + custom_extensions: + - vendor_name: DATABRICKS + data: '{"cardinality": "many_to_one"}' + metrics: + - name: total_sales + expression: + dialects: + - dialect: DATABRICKS + expression: SUM(ss_sales_price) + - dialect: ANSI_SQL + expression: SUM(ss_sales_price) + description: Total sales revenue + ai_context: + synonyms: + - Total Sales + - revenue + - gross sales + custom_extensions: + - vendor_name: DATABRICKS + data: '{"format": {"type": "currency", "currency_code": "USD"}}' + - name: total_quantity + expression: + dialects: + - dialect: DATABRICKS + expression: SUM(ss_quantity) + - dialect: ANSI_SQL + expression: SUM(ss_quantity) + description: Total units sold + ai_context: + synonyms: + - Units Sold + - name: avg_discount + expression: + dialects: + - dialect: DATABRICKS + expression: AVG(ss_ext_discount_amt) + - dialect: ANSI_SQL + expression: AVG(ss_ext_discount_amt) + description: Average discount amount per transaction + ai_context: + synonyms: + - Avg Discount + custom_extensions: + - vendor_name: DATABRICKS + data: '{"format": {"type": "number", "decimal_places": {"min": 2, "max": 2}}}' + - name: net_profit + expression: + dialects: + - dialect: DATABRICKS + expression: SUM(ss_net_profit) + - dialect: ANSI_SQL + expression: SUM(ss_net_profit) + description: Total net profit + ai_context: + synonyms: + - Net Profit + - profit + - margin + custom_extensions: + - vendor_name: DATABRICKS + data: '{"format": {"type": "currency", "currency_code": "USD"}}' + - name: trailing_7d_sales + expression: + dialects: + - dialect: DATABRICKS + expression: SUM(ss_sales_price) + - dialect: ANSI_SQL + expression: SUM(ss_sales_price) + description: Trailing 7-day rolling sales + ai_context: + synonyms: + - 7-Day Sales + custom_extensions: + - vendor_name: DATABRICKS + data: '{"window": [{"order": "sold_date", "range": "trailing 7 day"}]}' + custom_extensions: + - vendor_name: DATABRICKS + data: '{"materialization": {"schedule": "every 6 hours", "mode": "relaxed", "materialized_views": [{"name": "sales_by_date", "type": "aggregated", "dimensions": ["sold_date", "item_category"], "measures": ["total_sales", "total_quantity"]}]}}' diff --git a/converters/databricks/tests/test_cli.py b/converters/databricks/tests/test_cli.py new file mode 100644 index 0000000..d871a31 --- /dev/null +++ b/converters/databricks/tests/test_cli.py @@ -0,0 +1,362 @@ +"""Tests for the osi-databricks CLI. + +Validates: +- Import subcommand produces valid OSI YAML output +- Export subcommand produces one file per dataset named {dataset}.yaml +- Error handling: invalid input exits non-zero with stderr message +- Missing output directory is created automatically +- Requirements: 8.1, 8.2, 8.3, 8.4, 8.5, 8.6 +""" + +import subprocess +import sys +from pathlib import Path + +import pytest +import yaml +from osi.models import OSIDocument + +from osi_databricks.models import MetricViewModel + +# --- Fixtures --- + + +@pytest.fixture +def sample_metric_view_yaml(tmp_path: Path) -> Path: + """Create a sample Metric View YAML file for testing import.""" + content = """\ +version: '1.1' +source: catalog.schema.store_sales +comment: Store sales metric view +filter: ss_quantity > 0 +joins: + - name: date_dim + source: catalog.schema.date_dim + on: source.ss_sold_date_sk = date_dim.d_date_sk + cardinality: many_to_one +fields: + - name: sold_date + expr: date_dim.d_date + comment: Sale date + display_name: Date of Sale + synonyms: + - sale date + - transaction date +measures: + - name: total_sales + expr: SUM(ss_net_paid) + comment: Total net sales +""" + p = tmp_path / "metric_view.yaml" + p.write_text(content) + return p + + +@pytest.fixture +def sample_osi_yaml(tmp_path: Path) -> Path: + """Create a sample OSI YAML file for testing export.""" + content = """\ +version: 0.2.0.dev0 +dialects: + - DATABRICKS + - ANSI_SQL +vendors: + - DATABRICKS +semantic_model: + - name: my_model + description: Test model + datasets: + - name: store_sales + source: catalog.schema.store_sales + fields: + - name: sold_date + expression: + dialects: + - dialect: DATABRICKS + expression: d_date + - dialect: ANSI_SQL + expression: d_date + dimension: + is_time: true + description: Sale date + - name: quantity + expression: + dialects: + - dialect: DATABRICKS + expression: ss_quantity + - dialect: ANSI_SQL + expression: ss_quantity + dimension: + is_time: false + metrics: + - name: total_sales + expression: + dialects: + - dialect: DATABRICKS + expression: SUM(ss_net_paid) + - dialect: ANSI_SQL + expression: SUM(ss_net_paid) + description: Total net sales +""" + p = tmp_path / "osi_model.yaml" + p.write_text(content) + return p + + +@pytest.fixture +def multi_dataset_osi_yaml(tmp_path: Path) -> Path: + """Create an OSI YAML file with multiple datasets for testing multi-file export.""" + content = """\ +version: 0.2.0.dev0 +dialects: + - DATABRICKS +vendors: + - DATABRICKS +semantic_model: + - name: multi_model + datasets: + - name: orders + source: catalog.schema.orders + fields: + - name: order_id + expression: + dialects: + - dialect: DATABRICKS + expression: order_id + dimension: + is_time: false + - name: customers + source: catalog.schema.customers + fields: + - name: customer_id + expression: + dialects: + - dialect: DATABRICKS + expression: customer_id + dimension: + is_time: false + metrics: + - name: order_count + expression: + dialects: + - dialect: DATABRICKS + expression: COUNT(order_id) +""" + p = tmp_path / "multi_osi.yaml" + p.write_text(content) + return p + + +# --- Helper --- + + +def run_cli(*args: str, cwd: Path | None = None) -> subprocess.CompletedProcess: + """Run the osi-databricks CLI as a subprocess.""" + return subprocess.run( + [sys.executable, "-m", "osi_databricks.cli", *args], + capture_output=True, + text=True, + cwd=cwd, + ) + + +# --- Import Subcommand Tests --- + + +class TestImportSubcommand: + """Tests for the 'import' subcommand (Metric View → OSI).""" + + def test_import_produces_valid_osi_yaml(self, sample_metric_view_yaml: Path, tmp_path: Path): + """Import subcommand produces valid OSI YAML output.""" + output_path = tmp_path / "output_osi.yaml" + result = run_cli("import", "-i", str(sample_metric_view_yaml), "-o", str(output_path)) + + assert result.returncode == 0 + assert output_path.exists() + + # Validate the output is parseable OSI + raw = yaml.safe_load(output_path.read_text()) + doc = OSIDocument.model_validate(raw) + assert doc.version == "0.2.0.dev0" + assert len(doc.semantic_model) == 1 + sm = doc.semantic_model[0] + assert sm.name == "metric_view_model" + assert len(sm.datasets) == 1 + assert sm.datasets[0].name == "store_sales" + + def test_import_with_custom_model_name(self, sample_metric_view_yaml: Path, tmp_path: Path): + """Import subcommand respects --model-name argument.""" + output_path = tmp_path / "output.yaml" + result = run_cli( + "import", "-i", str(sample_metric_view_yaml), + "-o", str(output_path), + "--model-name", "custom_name", + ) + + assert result.returncode == 0 + raw = yaml.safe_load(output_path.read_text()) + doc = OSIDocument.model_validate(raw) + assert doc.semantic_model[0].name == "custom_name" + + def test_import_preserves_fields_and_measures(self, sample_metric_view_yaml: Path, tmp_path: Path): + """Import preserves field and measure data in OSI output.""" + output_path = tmp_path / "output.yaml" + result = run_cli("import", "-i", str(sample_metric_view_yaml), "-o", str(output_path)) + + assert result.returncode == 0 + raw = yaml.safe_load(output_path.read_text()) + doc = OSIDocument.model_validate(raw) + sm = doc.semantic_model[0] + + # Check fields + fields = sm.datasets[0].fields + assert any(f.name == "sold_date" for f in fields) + + # Check metrics + assert sm.metrics is not None + assert any(m.name == "total_sales" for m in sm.metrics) + + def test_import_creates_output_parent_directory(self, sample_metric_view_yaml: Path, tmp_path: Path): + """Import creates missing parent directories for output file.""" + output_path = tmp_path / "nested" / "deep" / "output.yaml" + result = run_cli("import", "-i", str(sample_metric_view_yaml), "-o", str(output_path)) + + assert result.returncode == 0 + assert output_path.exists() + + def test_import_writes_status_to_stderr(self, sample_metric_view_yaml: Path, tmp_path: Path): + """Import writes a status message to stderr.""" + output_path = tmp_path / "output.yaml" + result = run_cli("import", "-i", str(sample_metric_view_yaml), "-o", str(output_path)) + + assert result.returncode == 0 + assert "Written to" in result.stderr + + +# --- Export Subcommand Tests --- + + +class TestExportSubcommand: + """Tests for the 'export' subcommand (OSI → Metric View).""" + + def test_export_produces_valid_metric_view_yaml(self, sample_osi_yaml: Path, tmp_path: Path): + """Export subcommand produces valid Metric View YAML output.""" + output_dir = tmp_path / "export_output" + result = run_cli("export", "-i", str(sample_osi_yaml), "-o", str(output_dir)) + + assert result.returncode == 0 + assert output_dir.exists() + + # Should produce a file named after the dataset + output_file = output_dir / "store_sales.yaml" + assert output_file.exists() + + # Validate the output is parseable Metric View + mv_model = MetricViewModel.from_yaml(output_file.read_text()) + assert mv_model.source == "catalog.schema.store_sales" + assert mv_model.fields is not None + assert any(f.name == "sold_date" for f in mv_model.fields) + + def test_export_one_file_per_dataset(self, multi_dataset_osi_yaml: Path, tmp_path: Path): + """Export produces one file per dataset named {dataset_name}.yaml.""" + output_dir = tmp_path / "multi_export" + result = run_cli("export", "-i", str(multi_dataset_osi_yaml), "-o", str(output_dir)) + + assert result.returncode == 0 + + # Should have files for both datasets + orders_file = output_dir / "orders.yaml" + customers_file = output_dir / "customers.yaml" + assert orders_file.exists() + assert customers_file.exists() + + # Validate each + orders_model = MetricViewModel.from_yaml(orders_file.read_text()) + assert orders_model.source == "catalog.schema.orders" + + customers_model = MetricViewModel.from_yaml(customers_file.read_text()) + assert customers_model.source == "catalog.schema.customers" + + def test_export_creates_output_directory(self, sample_osi_yaml: Path, tmp_path: Path): + """Export creates missing output directory automatically.""" + output_dir = tmp_path / "new" / "nested" / "dir" + assert not output_dir.exists() + + result = run_cli("export", "-i", str(sample_osi_yaml), "-o", str(output_dir)) + + assert result.returncode == 0 + assert output_dir.exists() + # Files should have been written + assert any(output_dir.iterdir()) + + def test_export_writes_status_to_stderr(self, sample_osi_yaml: Path, tmp_path: Path): + """Export writes status messages to stderr for each file written.""" + output_dir = tmp_path / "export_out" + result = run_cli("export", "-i", str(sample_osi_yaml), "-o", str(output_dir)) + + assert result.returncode == 0 + assert "Written" in result.stderr + + +# --- Error Handling Tests --- + + +class TestErrorHandling: + """Tests for CLI error handling.""" + + def test_import_invalid_yaml_exits_nonzero(self, tmp_path: Path): + """Import exits non-zero with stderr message on invalid YAML input.""" + bad_file = tmp_path / "bad.yaml" + bad_file.write_text("this: is: not: valid: {{yaml") + output_path = tmp_path / "output.yaml" + + result = run_cli("import", "-i", str(bad_file), "-o", str(output_path)) + + assert result.returncode != 0 + assert "Error parsing" in result.stderr + + def test_import_missing_required_field_exits_nonzero(self, tmp_path: Path): + """Import exits non-zero when required fields are missing.""" + bad_file = tmp_path / "incomplete.yaml" + bad_file.write_text("version: '1.1'\nfields:\n - name: f1\n expr: col1\n") + output_path = tmp_path / "output.yaml" + + result = run_cli("import", "-i", str(bad_file), "-o", str(output_path)) + + assert result.returncode != 0 + assert "Error parsing" in result.stderr + + def test_import_nonexistent_file_exits_nonzero(self, tmp_path: Path): + """Import exits non-zero when input file does not exist.""" + result = run_cli("import", "-i", str(tmp_path / "does_not_exist.yaml"), "-o", str(tmp_path / "out.yaml")) + + assert result.returncode != 0 + assert "Error parsing" in result.stderr + + def test_export_invalid_osi_yaml_exits_nonzero(self, tmp_path: Path): + """Export exits non-zero with stderr message on invalid OSI input.""" + bad_file = tmp_path / "bad_osi.yaml" + bad_file.write_text("not_a_valid: osi_document") + output_dir = tmp_path / "out" + + result = run_cli("export", "-i", str(bad_file), "-o", str(output_dir)) + + assert result.returncode != 0 + assert "Error parsing" in result.stderr + + def test_export_nonexistent_file_exits_nonzero(self, tmp_path: Path): + """Export exits non-zero when input file does not exist.""" + result = run_cli("export", "-i", str(tmp_path / "missing.yaml"), "-o", str(tmp_path / "out")) + + assert result.returncode != 0 + assert "Error parsing" in result.stderr + + def test_no_command_shows_help(self): + """Running without a subcommand exits non-zero.""" + result = run_cli() + assert result.returncode != 0 + + def test_unknown_command_exits_nonzero(self): + """Running with an unknown subcommand exits non-zero.""" + result = run_cli("unknown") + assert result.returncode != 0 diff --git a/converters/databricks/tests/test_dialect_utils.py b/converters/databricks/tests/test_dialect_utils.py new file mode 100644 index 0000000..630955f --- /dev/null +++ b/converters/databricks/tests/test_dialect_utils.py @@ -0,0 +1,104 @@ +"""Tests for dialect_utils module. + +Property 5: Dialect selection prefers DATABRICKS with ANSI_SQL fallback. +""" + +from hypothesis import given, settings +from hypothesis import strategies as st +from osi.models import OSIDialect, OSIDialectExpression + +from osi_databricks.dialect_utils import is_standard_sql, select_dialect_expression + +# --- Hypothesis Strategies --- + +_expr_text = st.text( + alphabet=st.characters(whitelist_categories=("L", "N", "P", "Z"), whitelist_characters="_"), + min_size=1, + max_size=50, +) + + +@st.composite +def dialect_expression_lists(draw): + """Generate lists of OSIDialectExpression with various dialect combos.""" + available_dialects = draw( + st.lists( + st.sampled_from([OSIDialect.DATABRICKS, OSIDialect.ANSI_SQL, OSIDialect.SNOWFLAKE]), + min_size=1, + max_size=3, + unique=True, + ) + ) + return [ + OSIDialectExpression(dialect=d, expression=draw(_expr_text)) + for d in available_dialects + ] + + +# --- Property Tests --- + + +class TestDialectSelectionProperty: + """Property 5: Dialect selection prefers DATABRICKS with ANSI_SQL fallback.""" + + @given(dialects=dialect_expression_lists()) + @settings(max_examples=100) + def test_prefers_databricks_over_ansi(self, dialects: list[OSIDialectExpression]): + """When DATABRICKS is present, it is always selected.""" + result = select_dialect_expression(dialects) + by_dialect = {d.dialect: d.expression for d in dialects} + + if OSIDialect.DATABRICKS in by_dialect: + assert result == by_dialect[OSIDialect.DATABRICKS] + elif OSIDialect.ANSI_SQL in by_dialect: + assert result == by_dialect[OSIDialect.ANSI_SQL] + else: + assert result is None + + @given(expr=_expr_text) + @settings(max_examples=100) + def test_databricks_only_returns_databricks(self, expr: str): + """Single DATABRICKS dialect always returns that expression.""" + dialects = [OSIDialectExpression(dialect=OSIDialect.DATABRICKS, expression=expr)] + assert select_dialect_expression(dialects) == expr + + @given(expr=_expr_text) + @settings(max_examples=100) + def test_ansi_only_returns_ansi(self, expr: str): + """Single ANSI_SQL dialect is used as fallback.""" + dialects = [OSIDialectExpression(dialect=OSIDialect.ANSI_SQL, expression=expr)] + assert select_dialect_expression(dialects) == expr + + def test_no_usable_dialect_returns_none(self): + """When neither DATABRICKS nor ANSI_SQL is present, returns None.""" + dialects = [OSIDialectExpression(dialect=OSIDialect.SNOWFLAKE, expression="x")] + assert select_dialect_expression(dialects) is None + + +# --- Unit Tests for is_standard_sql --- + + +class TestIsStandardSql: + """Unit tests for ANSI SQL detection.""" + + def test_simple_column_is_standard(self): + assert is_standard_sql("col1") is True + + def test_sum_is_standard(self): + assert is_standard_sql("SUM(amount)") is True + + def test_filter_clause_is_not_standard(self): + assert is_standard_sql("COUNT(*) FILTER (WHERE active)") is False + + def test_measure_is_not_standard(self): + assert is_standard_sql("MEASURE(total_sales)") is False + + def test_qualify_is_not_standard(self): + assert is_standard_sql("ROW_NUMBER() OVER() QUALIFY rn = 1") is False + + def test_cast_operator_is_not_standard(self): + assert is_standard_sql("col::INT") is False + + def test_date_trunc_is_standard(self): + # DATE_TRUNC itself is widely supported, not Databricks-only + assert is_standard_sql("DATE_TRUNC('month', d_date)") is True diff --git a/converters/databricks/tests/test_metric_view_to_osi.py b/converters/databricks/tests/test_metric_view_to_osi.py new file mode 100644 index 0000000..9b6e50a --- /dev/null +++ b/converters/databricks/tests/test_metric_view_to_osi.py @@ -0,0 +1,821 @@ +"""Tests for metric_view_to_osi import logic. + +Includes property-based tests (Hypothesis) for correctness properties 2, 3, and 4, +plus example-based unit tests for specific mapping behaviors. +""" + +from __future__ import annotations + +import json + +from hypothesis import HealthCheck, given, settings +from hypothesis import strategies as st +from osi.models import OSIDialect, OSIVendor + +from osi_databricks.metric_view_to_osi import ( + metric_view_to_osi, +) +from osi_databricks.models import ( + MetricViewField, + MetricViewFormat, + MetricViewJoin, + MetricViewMaterialization, + MetricViewMaterializedView, + MetricViewMeasure, + MetricViewModel, + MetricViewRely, + MetricViewWindow, +) + +# --- Hypothesis Strategies --- + +_identifier = st.from_regex(r"[a-z][a-z0-9_]{0,29}", fullmatch=True) +_three_part_name = st.builds( + lambda a, b, c: f"{a}.{b}.{c}", + _identifier, + _identifier, + _identifier, +) +_expr = st.from_regex(r"[A-Za-z_][A-Za-z0-9_. ()]{0,49}", fullmatch=True) +_safe_text = st.text( + alphabet=st.characters(min_codepoint=32, max_codepoint=126), + min_size=1, + max_size=50, +) + +# Adversarial strings for metadata fields (synonyms, comments, display_name) +# These test JSON serialization boundaries in custom extensions and YAML-unsafe values +_yaml_dangerous_literals = st.sampled_from([ + "true", "false", "null", "~", "1.0", "0", "-1", + "2024-01-01", "key: value", "has # comment", + "{curly}", "[bracket]", "pipe | here", + "star * wild", 'double"quote', "single'quote", + "backslash\\here", "newline\\n", "tab\\t", + "ratio: sales/cost", "a > b", "x & y", + " leading spaces", "trailing spaces ", +]) +_adversarial_text = st.one_of(_safe_text, _yaml_dangerous_literals) + + +@st.composite +def metric_view_fields_strategy(draw): + """Generate valid MetricViewField instances with adversarial metadata.""" + return MetricViewField( + name=draw(_identifier), + expr=draw(_expr), + comment=draw(st.none() | _adversarial_text), + display_name=draw(st.none() | _adversarial_text), + synonyms=draw(st.none() | st.lists(_adversarial_text, min_size=1, max_size=3)), + format=draw(st.none() | st.just(MetricViewFormat(type="number"))), + ) + + +@st.composite +def metric_view_measures_strategy(draw): + """Generate valid MetricViewMeasure instances with adversarial metadata.""" + return MetricViewMeasure( + name=draw(_identifier), + expr=draw(_expr), + comment=draw(st.none() | _adversarial_text), + display_name=draw(st.none() | _adversarial_text), + synonyms=draw(st.none() | st.lists(_adversarial_text, min_size=1, max_size=3)), + format=draw(st.none() | st.just(MetricViewFormat(type="currency", currency_code="USD"))), + window=draw(st.none() | st.just([MetricViewWindow(order="d_date", range="trailing 7 day")])), + ) + + +@st.composite +def metric_view_models_with_fields_and_measures(draw): + """Generate models that always have at least one field and one measure.""" + return MetricViewModel( + version="1.1", + source=draw(_three_part_name), + fields=draw(st.lists(metric_view_fields_strategy(), min_size=1, max_size=5)), + measures=draw(st.lists(metric_view_measures_strategy(), min_size=1, max_size=5)), + ) + + +# --- Property 2: Import Maps Fields and Measures Correctly --- + + +class TestImportFieldAndMeasureMapping: + """Property 2: Import maps fields and measures correctly. + + For any valid Metric View definition containing fields and measures, + importing to OSI SHALL produce one OSI field per Metric View field + (each with a DATABRICKS dialect expression and dimension metadata) + and one OSI metric per Metric View measure (each with a DATABRICKS + dialect expression). + """ + + @given(model=metric_view_models_with_fields_and_measures()) + @settings(max_examples=100, suppress_health_check=[HealthCheck.too_slow]) + def test_one_osi_field_per_metric_view_field(self, model: MetricViewModel): + """Each Metric View field maps to exactly one OSI field.""" + doc = metric_view_to_osi(model) + dataset = doc.semantic_model[0].datasets[0] + assert dataset.fields is not None + assert len(dataset.fields) == len(model.fields) + + @given(model=metric_view_models_with_fields_and_measures()) + @settings(max_examples=100, suppress_health_check=[HealthCheck.too_slow]) + def test_each_field_has_databricks_dialect(self, model: MetricViewModel): + """Every OSI field has at least a DATABRICKS dialect expression.""" + doc = metric_view_to_osi(model) + dataset = doc.semantic_model[0].datasets[0] + for osi_field in dataset.fields: + dialect_names = [d.dialect for d in osi_field.expression.dialects] + assert OSIDialect.DATABRICKS in dialect_names + + @given(model=metric_view_models_with_fields_and_measures()) + @settings(max_examples=100, suppress_health_check=[HealthCheck.too_slow]) + def test_each_field_has_dimension_metadata(self, model: MetricViewModel): + """Every OSI field has dimension metadata set.""" + doc = metric_view_to_osi(model) + dataset = doc.semantic_model[0].datasets[0] + for osi_field in dataset.fields: + assert osi_field.dimension is not None + assert isinstance(osi_field.dimension.is_time, bool) + + @given(model=metric_view_models_with_fields_and_measures()) + @settings(max_examples=100, suppress_health_check=[HealthCheck.too_slow]) + def test_field_names_preserved(self, model: MetricViewModel): + """Field names are preserved during import.""" + doc = metric_view_to_osi(model) + dataset = doc.semantic_model[0].datasets[0] + mv_names = [f.name for f in model.fields] + osi_names = [f.name for f in dataset.fields] + assert mv_names == osi_names + + @given(model=metric_view_models_with_fields_and_measures()) + @settings(max_examples=100, suppress_health_check=[HealthCheck.too_slow]) + def test_field_expressions_stored_as_databricks(self, model: MetricViewModel): + """Field expressions are stored in the DATABRICKS dialect.""" + doc = metric_view_to_osi(model) + dataset = doc.semantic_model[0].datasets[0] + for mv_field, osi_field in zip(model.fields, dataset.fields): + databricks_exprs = [ + d.expression for d in osi_field.expression.dialects + if d.dialect == OSIDialect.DATABRICKS + ] + assert len(databricks_exprs) == 1 + assert databricks_exprs[0] == mv_field.expr + + @given(model=metric_view_models_with_fields_and_measures()) + @settings(max_examples=100, suppress_health_check=[HealthCheck.too_slow]) + def test_one_osi_metric_per_metric_view_measure(self, model: MetricViewModel): + """Each Metric View measure maps to exactly one OSI metric.""" + doc = metric_view_to_osi(model) + metrics = doc.semantic_model[0].metrics + assert metrics is not None + assert len(metrics) == len(model.measures) + + @given(model=metric_view_models_with_fields_and_measures()) + @settings(max_examples=100, suppress_health_check=[HealthCheck.too_slow]) + def test_each_metric_has_databricks_dialect(self, model: MetricViewModel): + """Every OSI metric has at least a DATABRICKS dialect expression.""" + doc = metric_view_to_osi(model) + for metric in doc.semantic_model[0].metrics: + dialect_names = [d.dialect for d in metric.expression.dialects] + assert OSIDialect.DATABRICKS in dialect_names + + @given(model=metric_view_models_with_fields_and_measures()) + @settings(max_examples=100, suppress_health_check=[HealthCheck.too_slow]) + def test_measure_names_and_expressions_preserved(self, model: MetricViewModel): + """Measure names and expressions are preserved during import.""" + doc = metric_view_to_osi(model) + metrics = doc.semantic_model[0].metrics + for mv_measure, osi_metric in zip(model.measures, metrics): + assert osi_metric.name == mv_measure.name + databricks_exprs = [ + d.expression for d in osi_metric.expression.dialects + if d.dialect == OSIDialect.DATABRICKS + ] + assert databricks_exprs[0] == mv_measure.expr + + +# --- Property 3: Import Maps Source Correctly --- + + +class TestImportSourceMapping: + """Property 3: Import maps source correctly. + + For any Metric View source string, if it matches a three-part name pattern + (X.Y.Z) the importer SHALL map it directly to the OSI dataset source field; + if it is a SQL query the importer SHALL store it in a DATABRICKS + custom_extension with key 'source_query'. + """ + + @given(source=_three_part_name) + @settings(max_examples=100) + def test_three_part_name_maps_to_source(self, source: str): + """Three-part name source maps directly to dataset.source.""" + model = MetricViewModel( + source=source, + fields=[MetricViewField(name="f1", expr="col1")], + ) + doc = metric_view_to_osi(model) + dataset = doc.semantic_model[0].datasets[0] + assert dataset.source == source + + @given(source=_three_part_name) + @settings(max_examples=100) + def test_three_part_name_no_source_query_extension(self, source: str): + """Three-part name source does not generate a source_query custom extension.""" + model = MetricViewModel( + source=source, + fields=[MetricViewField(name="f1", expr="col1")], + ) + doc = metric_view_to_osi(model) + dataset = doc.semantic_model[0].datasets[0] + if dataset.custom_extensions: + for ext in dataset.custom_extensions: + data = json.loads(ext.data) + assert "source_query" not in data + + @given( + table=_identifier, + cols=st.lists(_identifier, min_size=1, max_size=3), + ) + @settings(max_examples=100) + def test_sql_query_stored_in_custom_extension(self, table: str, cols: list[str]): + """SQL query source is stored in a DATABRICKS custom extension.""" + query = f"SELECT {', '.join(cols)} FROM {table}" + model = MetricViewModel( + source=query, + fields=[MetricViewField(name="f1", expr="col1")], + ) + doc = metric_view_to_osi(model) + dataset = doc.semantic_model[0].datasets[0] + + # Should have custom extension with source_query + assert dataset.custom_extensions is not None + ext_data_list = [json.loads(e.data) for e in dataset.custom_extensions] + source_queries = [d["source_query"] for d in ext_data_list if "source_query" in d] + assert len(source_queries) == 1 + assert source_queries[0] == query + + @given( + table=_identifier, + cols=st.lists(_identifier, min_size=1, max_size=3), + ) + @settings(max_examples=100) + def test_sql_query_dataset_source_is_generic(self, table: str, cols: list[str]): + """When source is a SQL query, dataset.source is a generic name (not the query).""" + query = f"SELECT {', '.join(cols)} FROM {table}" + model = MetricViewModel( + source=query, + fields=[MetricViewField(name="f1", expr="col1")], + ) + doc = metric_view_to_osi(model) + dataset = doc.semantic_model[0].datasets[0] + # Source should not be the SQL query itself + assert not dataset.source.upper().startswith("SELECT") + + +# --- Property 4: Import Preserves Semantic Metadata --- + + +class TestImportSemanticMetadata: + """Property 4: Import preserves semantic metadata. + + For any Metric View field or measure with a display_name, synonyms, or comment, + the importer SHALL map display_name as the first element of ai_context.synonyms, + append synonyms to the same list, and map comment to the OSI description field. + """ + + @given( + display_name=_adversarial_text, + synonyms=st.lists(_adversarial_text, min_size=1, max_size=3), + comment=_adversarial_text, + ) + @settings(max_examples=100) + def test_field_display_name_is_first_synonym( + self, display_name: str, synonyms: list[str], comment: str + ): + """display_name becomes the first element of ai_context.synonyms.""" + model = MetricViewModel( + source="a.b.c", + fields=[ + MetricViewField( + name="f1", + expr="col1", + display_name=display_name, + synonyms=synonyms, + comment=comment, + ) + ], + ) + doc = metric_view_to_osi(model) + field = doc.semantic_model[0].datasets[0].fields[0] + assert field.ai_context is not None + assert field.ai_context.synonyms[0] == display_name + + @given( + display_name=_adversarial_text, + synonyms=st.lists(_adversarial_text, min_size=1, max_size=3), + ) + @settings(max_examples=100) + def test_field_synonyms_appended_after_display_name( + self, display_name: str, synonyms: list[str] + ): + """synonyms list is appended after display_name in ai_context.synonyms.""" + model = MetricViewModel( + source="a.b.c", + fields=[ + MetricViewField( + name="f1", + expr="col1", + display_name=display_name, + synonyms=synonyms, + ) + ], + ) + doc = metric_view_to_osi(model) + field = doc.semantic_model[0].datasets[0].fields[0] + expected = tuple([display_name] + synonyms) + assert field.ai_context.synonyms == expected + + @given(comment=_adversarial_text) + @settings(max_examples=100) + def test_field_comment_maps_to_description(self, comment: str): + """comment maps to the OSI field description.""" + model = MetricViewModel( + source="a.b.c", + fields=[MetricViewField(name="f1", expr="col1", comment=comment)], + ) + doc = metric_view_to_osi(model) + field = doc.semantic_model[0].datasets[0].fields[0] + assert field.description == comment + + @given( + display_name=_adversarial_text, + synonyms=st.lists(_adversarial_text, min_size=1, max_size=3), + comment=_adversarial_text, + ) + @settings(max_examples=100) + def test_measure_metadata_preserved( + self, display_name: str, synonyms: list[str], comment: str + ): + """Measure display_name, synonyms, and comment are preserved in OSI metric.""" + model = MetricViewModel( + source="a.b.c", + measures=[ + MetricViewMeasure( + name="m1", + expr="SUM(col1)", + display_name=display_name, + synonyms=synonyms, + comment=comment, + ) + ], + ) + doc = metric_view_to_osi(model) + metric = doc.semantic_model[0].metrics[0] + assert metric.description == comment + assert metric.ai_context is not None + assert metric.ai_context.synonyms[0] == display_name + expected_synonyms = tuple([display_name] + synonyms) + assert metric.ai_context.synonyms == expected_synonyms + + @given(synonyms=st.lists(_adversarial_text, min_size=1, max_size=5)) + @settings(max_examples=100) + def test_field_synonyms_only_no_display_name(self, synonyms: list[str]): + """When only synonyms are present (no display_name), they map directly.""" + model = MetricViewModel( + source="a.b.c", + fields=[ + MetricViewField( + name="f1", + expr="col1", + synonyms=synonyms, + ) + ], + ) + doc = metric_view_to_osi(model) + field = doc.semantic_model[0].datasets[0].fields[0] + assert field.ai_context is not None + assert field.ai_context.synonyms == tuple(synonyms) + + def test_no_metadata_produces_no_ai_context(self): + """When no display_name, synonyms, or comment, ai_context and description are None.""" + model = MetricViewModel( + source="a.b.c", + fields=[MetricViewField(name="f1", expr="col1")], + ) + doc = metric_view_to_osi(model) + field = doc.semantic_model[0].datasets[0].fields[0] + assert field.ai_context is None + assert field.description is None + + +# --- Unit Tests --- + + +class TestImportComplete: + """Unit tests for complete Metric View import scenarios.""" + + def test_import_complete_metric_view(self): + """Test import of a complete Metric View YAML (all sections).""" + model = MetricViewModel( + version="1.1", + source="catalog.schema.store_sales", + comment="Store sales metric view", + filter="ss_quantity > 0", + joins=[ + MetricViewJoin( + name="date_dim", + source="catalog.schema.date_dim", + on="source.ss_sold_date_sk = date_dim.d_date_sk", + cardinality="many_to_one", + rely=MetricViewRely(at_most_one_match=True), + ) + ], + fields=[ + MetricViewField( + name="sold_date", + expr="date_dim.d_date", + comment="Sale date", + display_name="Date of Sale", + synonyms=["sale date", "transaction date"], + ), + MetricViewField(name="store_id", expr="ss_store_sk"), + ], + measures=[ + MetricViewMeasure( + name="total_sales", + expr="SUM(ss_net_paid)", + comment="Total net sales", + display_name="Total Sales", + ) + ], + materialization=MetricViewMaterialization( + schedule="every 6 hours", + mode="relaxed", + materialized_views=[ + MetricViewMaterializedView( + name="mv_daily", + type="aggregated", + dimensions=["sold_date"], + measures=["total_sales"], + ) + ], + ), + ) + + doc = metric_view_to_osi(model, model_name="store_sales_model") + + # Check document-level + assert doc.version == "0.2.0.dev0" + assert OSIDialect.DATABRICKS in doc.dialects + assert OSIVendor.DATABRICKS in doc.vendors + + # Check semantic model + sm = doc.semantic_model[0] + assert sm.name == "store_sales_model" + assert sm.description == "Store sales metric view" + + # Check dataset + ds = sm.datasets[0] + assert ds.name == "store_sales" + assert ds.source == "catalog.schema.store_sales" + assert len(ds.fields) == 2 + + # Check field mapping + sold_date_field = ds.fields[0] + assert sold_date_field.name == "sold_date" + assert sold_date_field.description == "Sale date" + assert sold_date_field.dimension.is_time is True # "d_date" contains DATE + assert sold_date_field.ai_context.synonyms == ("Date of Sale", "sale date", "transaction date") + + store_id_field = ds.fields[1] + assert store_id_field.name == "store_id" + assert store_id_field.dimension.is_time is False + assert store_id_field.ai_context is None + + # Check filter in custom extension + assert ds.custom_extensions is not None + ds_ext = json.loads(ds.custom_extensions[0].data) + assert ds_ext["filter"] == "ss_quantity > 0" + + # Check metrics + assert len(sm.metrics) == 1 + metric = sm.metrics[0] + assert metric.name == "total_sales" + assert metric.description == "Total net sales" + databricks_expr = [d for d in metric.expression.dialects if d.dialect == OSIDialect.DATABRICKS] + assert databricks_expr[0].expression == "SUM(ss_net_paid)" + + # Check relationships + assert len(sm.relationships) == 1 + rel = sm.relationships[0] + assert rel.name == "date_dim" + assert rel.from_dataset == "store_sales" + assert rel.to == "date_dim" + assert rel.from_columns == ["ss_sold_date_sk"] + assert rel.to_columns == ["d_date_sk"] + + # Check relationship custom extensions (cardinality, rely) + rel_ext = json.loads(rel.custom_extensions[0].data) + assert rel_ext["cardinality"] == "many_to_one" + assert rel_ext["rely"]["at_most_one_match"] is True + + # Check materialization custom extension + assert sm.custom_extensions is not None + mat_ext = json.loads(sm.custom_extensions[0].data) + assert "materialization" in mat_ext + assert mat_ext["materialization"]["schedule"] == "every 6 hours" + + +class TestImportSourceDetection: + """Unit tests for source detection and mapping.""" + + def test_three_part_name_maps_to_source(self): + """Three-part name source maps to dataset.source directly.""" + model = MetricViewModel( + source="my_catalog.my_schema.my_table", + fields=[MetricViewField(name="f1", expr="col1")], + ) + doc = metric_view_to_osi(model) + ds = doc.semantic_model[0].datasets[0] + assert ds.source == "my_catalog.my_schema.my_table" + assert ds.name == "my_table" + + def test_sql_query_stored_in_extension(self): + """SQL query source stored in custom_extension.""" + query = "SELECT a, b FROM my_table WHERE active = true" + model = MetricViewModel( + source=query, + fields=[MetricViewField(name="f1", expr="a")], + ) + doc = metric_view_to_osi(model) + ds = doc.semantic_model[0].datasets[0] + assert ds.name == "source" + assert ds.source == "source" + ext_data = json.loads(ds.custom_extensions[0].data) + assert ext_data["source_query"] == query + + def test_with_clause_detected_as_query(self): + """WITH clause is detected as SQL query.""" + query = "WITH cte AS (SELECT 1) SELECT * FROM cte" + model = MetricViewModel( + source=query, + fields=[MetricViewField(name="f1", expr="col1")], + ) + doc = metric_view_to_osi(model) + ds = doc.semantic_model[0].datasets[0] + ext_data = json.loads(ds.custom_extensions[0].data) + assert ext_data["source_query"] == query + + +class TestImportTimeDimension: + """Unit tests for time dimension inference.""" + + def test_date_expression_inferred_as_time(self): + """Expression containing 'date' is inferred as time dimension.""" + model = MetricViewModel( + source="a.b.c", + fields=[MetricViewField(name="sale_date", expr="d_date")], + ) + doc = metric_view_to_osi(model) + field = doc.semantic_model[0].datasets[0].fields[0] + assert field.dimension.is_time is True + + def test_timestamp_expression_inferred_as_time(self): + """Expression containing 'timestamp' is inferred as time dimension.""" + model = MetricViewModel( + source="a.b.c", + fields=[MetricViewField(name="ts", expr="event_timestamp")], + ) + doc = metric_view_to_osi(model) + field = doc.semantic_model[0].datasets[0].fields[0] + assert field.dimension.is_time is True + + def test_date_trunc_inferred_as_time(self): + """DATE_TRUNC function is inferred as time dimension.""" + model = MetricViewModel( + source="a.b.c", + fields=[MetricViewField(name="month", expr="DATE_TRUNC('month', d_date)")], + ) + doc = metric_view_to_osi(model) + field = doc.semantic_model[0].datasets[0].fields[0] + assert field.dimension.is_time is True + + def test_plain_column_not_time(self): + """Simple column reference without time patterns is not a time dimension.""" + model = MetricViewModel( + source="a.b.c", + fields=[MetricViewField(name="store_id", expr="ss_store_sk")], + ) + doc = metric_view_to_osi(model) + field = doc.semantic_model[0].datasets[0].fields[0] + assert field.dimension.is_time is False + + +class TestImportJoins: + """Unit tests for join mapping.""" + + def test_on_clause_single_key(self): + """Single-key ON clause parsed into from_columns/to_columns.""" + model = MetricViewModel( + source="cat.sch.fact_sales", + joins=[ + MetricViewJoin( + name="date_dim", + source="cat.sch.date_dim", + on="source.date_sk = date_dim.d_date_sk", + ) + ], + fields=[MetricViewField(name="f1", expr="col1")], + ) + doc = metric_view_to_osi(model) + rel = doc.semantic_model[0].relationships[0] + assert rel.from_columns == ["date_sk"] + assert rel.to_columns == ["d_date_sk"] + + def test_on_clause_composite_keys(self): + """Composite ON clause (AND) parsed correctly.""" + model = MetricViewModel( + source="cat.sch.orders", + joins=[ + MetricViewJoin( + name="items", + source="cat.sch.order_items", + on="source.order_id = items.order_id AND source.item_id = items.item_id", + ) + ], + fields=[MetricViewField(name="f1", expr="col1")], + ) + doc = metric_view_to_osi(model) + rel = doc.semantic_model[0].relationships[0] + assert rel.from_columns == ["order_id", "item_id"] + assert rel.to_columns == ["order_id", "item_id"] + + def test_using_clause_mapping(self): + """USING clause maps to same from_columns and to_columns.""" + model = MetricViewModel( + source="cat.sch.fact_sales", + joins=[ + MetricViewJoin( + name="date_dim", + source="cat.sch.date_dim", + using=["date_sk", "store_sk"], + ) + ], + fields=[MetricViewField(name="f1", expr="col1")], + ) + doc = metric_view_to_osi(model) + rel = doc.semantic_model[0].relationships[0] + assert rel.from_columns == ["date_sk", "store_sk"] + assert rel.to_columns == ["date_sk", "store_sk"] + + def test_nested_joins_produce_multiple_relationships(self): + """Nested joins (snowflake schema) produce one relationship per join.""" + model = MetricViewModel( + source="cat.sch.fact_sales", + joins=[ + MetricViewJoin( + name="date_dim", + source="cat.sch.date_dim", + on="source.date_sk = date_dim.d_date_sk", + joins=[ + MetricViewJoin( + name="fiscal_cal", + source="cat.sch.fiscal_calendar", + on="date_dim.cal_id = fiscal_cal.id", + ) + ], + ) + ], + fields=[MetricViewField(name="f1", expr="col1")], + ) + doc = metric_view_to_osi(model) + rels = doc.semantic_model[0].relationships + assert len(rels) == 2 + # First relationship: fact_sales -> date_dim + assert rels[0].name == "date_dim" + assert rels[0].from_dataset == "fact_sales" + assert rels[0].to == "date_dim" + # Second relationship: date_dim -> fiscal_calendar + assert rels[1].name == "fiscal_cal" + assert rels[1].from_dataset == "date_dim" + assert rels[1].to == "fiscal_calendar" + + +class TestImportDialectGeneration: + """Unit tests for ANSI_SQL dialect generation.""" + + def test_standard_sql_generates_both_dialects(self): + """Standard SQL expression generates both DATABRICKS and ANSI_SQL dialects.""" + model = MetricViewModel( + source="a.b.c", + fields=[MetricViewField(name="f1", expr="SUM(amount)")], + ) + doc = metric_view_to_osi(model) + field = doc.semantic_model[0].datasets[0].fields[0] + dialect_names = [d.dialect for d in field.expression.dialects] + assert OSIDialect.DATABRICKS in dialect_names + assert OSIDialect.ANSI_SQL in dialect_names + + def test_databricks_specific_generates_only_databricks(self): + """Databricks-specific expression generates only DATABRICKS dialect.""" + model = MetricViewModel( + source="a.b.c", + measures=[MetricViewMeasure(name="m1", expr="COUNT(*) FILTER (WHERE active)")], + ) + doc = metric_view_to_osi(model) + metric = doc.semantic_model[0].metrics[0] + dialect_names = [d.dialect for d in metric.expression.dialects] + assert OSIDialect.DATABRICKS in dialect_names + assert OSIDialect.ANSI_SQL not in dialect_names + + def test_document_dialects_include_ansi_when_present(self): + """Document-level dialects includes ANSI_SQL when at least one expr is standard.""" + model = MetricViewModel( + source="a.b.c", + fields=[MetricViewField(name="f1", expr="col1")], + ) + doc = metric_view_to_osi(model) + assert OSIDialect.ANSI_SQL in doc.dialects + + def test_document_dialects_no_ansi_when_all_databricks_specific(self): + """Document-level dialects excludes ANSI_SQL when all exprs are Databricks-specific.""" + model = MetricViewModel( + source="a.b.c", + fields=[MetricViewField(name="f1", expr="col1::INT")], + measures=[MetricViewMeasure(name="m1", expr="MEASURE(total)")], + ) + doc = metric_view_to_osi(model) + assert OSIDialect.ANSI_SQL not in doc.dialects + + +class TestImportCustomExtensions: + """Unit tests for custom extension handling.""" + + def test_filter_stored_in_dataset_extension(self): + """Filter is stored in dataset custom extension.""" + model = MetricViewModel( + source="a.b.c", + filter="status = 'active'", + fields=[MetricViewField(name="f1", expr="col1")], + ) + doc = metric_view_to_osi(model) + ds = doc.semantic_model[0].datasets[0] + assert ds.custom_extensions is not None + ext_data = json.loads(ds.custom_extensions[0].data) + assert ext_data["filter"] == "status = 'active'" + + def test_materialization_stored_in_model_extension(self): + """Materialization is stored in semantic model custom extension.""" + model = MetricViewModel( + source="a.b.c", + fields=[MetricViewField(name="f1", expr="col1")], + materialization=MetricViewMaterialization( + schedule="every 6 hours", + mode="relaxed", + ), + ) + doc = metric_view_to_osi(model) + sm = doc.semantic_model[0] + assert sm.custom_extensions is not None + ext_data = json.loads(sm.custom_extensions[0].data) + assert ext_data["materialization"]["schedule"] == "every 6 hours" + assert ext_data["materialization"]["mode"] == "relaxed" + + def test_field_format_stored_in_field_extension(self): + """Field format is stored in field custom extension.""" + model = MetricViewModel( + source="a.b.c", + fields=[ + MetricViewField( + name="f1", + expr="col1", + format=MetricViewFormat(type="currency", currency_code="USD"), + ) + ], + ) + doc = metric_view_to_osi(model) + field = doc.semantic_model[0].datasets[0].fields[0] + assert field.custom_extensions is not None + ext_data = json.loads(field.custom_extensions[0].data) + assert ext_data["format"]["type"] == "currency" + assert ext_data["format"]["currency_code"] == "USD" + + def test_measure_window_stored_in_metric_extension(self): + """Measure window is stored in metric custom extension.""" + model = MetricViewModel( + source="a.b.c", + measures=[ + MetricViewMeasure( + name="m1", + expr="SUM(amount)", + window=[MetricViewWindow(order="d_date", range="trailing 7 day")], + ) + ], + ) + doc = metric_view_to_osi(model) + metric = doc.semantic_model[0].metrics[0] + assert metric.custom_extensions is not None + ext_data = json.loads(metric.custom_extensions[0].data) + assert ext_data["window"][0]["order"] == "d_date" + assert ext_data["window"][0]["range"] == "trailing 7 day" diff --git a/converters/databricks/tests/test_models.py b/converters/databricks/tests/test_models.py new file mode 100644 index 0000000..6991581 --- /dev/null +++ b/converters/databricks/tests/test_models.py @@ -0,0 +1,422 @@ +"""Tests for Metric View Pydantic models. + +Includes property-based tests (Hypothesis) and example-based unit tests. + +The Hypothesis strategies deliberately generate adversarial inputs: +- YAML-special characters (: # { } [ ] * & ! etc.) +- Strings that YAML interprets as booleans (true, yes, on) +- Strings that YAML interprets as null (null, ~) +- Strings that look like numbers or dates +- Nested joins (snowflake schema, depth 2) +- Materialization config +- Empty vs None distinctions +""" + +import pytest +from hypothesis import HealthCheck, given, settings +from hypothesis import strategies as st +from pydantic import ValidationError + +from osi_databricks.models import ( + MetricViewField, + MetricViewFormat, + MetricViewJoin, + MetricViewMaterialization, + MetricViewMaterializedView, + MetricViewMeasure, + MetricViewModel, + MetricViewRely, + MetricViewWindow, +) + +# --- Hypothesis Strategies --- + +_identifier = st.from_regex(r"[a-z][a-z0-9_]{0,29}", fullmatch=True) +_three_part_name = st.builds( + lambda a, b, c: f"{a}.{b}.{c}", + _identifier, + _identifier, + _identifier, +) +_expr = st.from_regex(r"[A-Za-z_][A-Za-z0-9_. ()]{0,49}", fullmatch=True) + +# Adversarial strings that exercise YAML quoting edge cases +_yaml_dangerous_literals = st.sampled_from([ + "true", "false", "yes", "no", "on", "off", + "True", "False", "Yes", "No", "On", "Off", + "null", "Null", "NULL", "~", + "1.0", "0", "3.14", "1e10", "-1", + "2024-01-01", "12:30:00", + "key: value", "has # comment", "with: colon", + "{curly}", "[bracket]", "pipe | here", + "star * wild", "ampersand & ref", "exclaim!", + "percent%", "at@sign", "backtick`", + "single'quote", 'double"quote', + " leading spaces", "trailing spaces ", + "ratio: sales/cost", "a > b", "x & y", +]) + +# Mix safe text with adversarial strings for broad coverage +_safe_text = st.text( + alphabet=st.characters(min_codepoint=32, max_codepoint=126), + min_size=1, + max_size=50, +) +_adversarial_text = st.one_of(_safe_text, _yaml_dangerous_literals) + + +@st.composite +def metric_view_formats(draw): + return MetricViewFormat( + type=draw(st.sampled_from(["number", "currency", "date", "date_time", "percentage", "byte"])), + currency_code=draw(st.none() | st.sampled_from(["USD", "EUR", "GBP"])), + decimal_places=draw(st.none() | st.just({"min": 0, "max": 2})), + hide_group_separator=draw(st.none() | st.booleans()), + abbreviation=draw(st.none() | st.sampled_from(["K", "M", "B"])), + ) + + +@st.composite +def metric_view_fields(draw): + return MetricViewField( + name=draw(_identifier), + expr=draw(_expr), + comment=draw(st.none() | _adversarial_text), + display_name=draw(st.none() | _adversarial_text), + synonyms=draw(st.none() | st.lists(_adversarial_text, min_size=1, max_size=3)), + format=draw(st.none() | metric_view_formats()), + ) + + +@st.composite +def metric_view_windows(draw): + return MetricViewWindow( + order=draw(_identifier), + range=draw(st.sampled_from(["trailing 7 day", "trailing 30 day", "trailing 1 hour", "unbounded"])), + semiadditive=draw(st.none() | st.sampled_from(["last", "first"])), + ) + + +@st.composite +def metric_view_measures(draw): + return MetricViewMeasure( + name=draw(_identifier), + expr=draw(_expr), + comment=draw(st.none() | _adversarial_text), + display_name=draw(st.none() | _adversarial_text), + synonyms=draw(st.none() | st.lists(_adversarial_text, min_size=1, max_size=3)), + format=draw(st.none() | metric_view_formats()), + window=draw(st.none() | st.lists(metric_view_windows(), min_size=1, max_size=2)), + ) + + +@st.composite +def metric_view_joins_leaf(draw): + """Generate a leaf join (no nested joins).""" + return MetricViewJoin( + name=draw(_identifier), + source=draw(_three_part_name), + on=draw(st.none() | _expr), + using=draw(st.none() | st.lists(_identifier, min_size=1, max_size=3)), + cardinality=draw(st.none() | st.sampled_from(["many_to_one", "one_to_many"])), + rely=draw(st.none() | st.builds(MetricViewRely, at_most_one_match=st.booleans())), + joins=None, + ) + + +@st.composite +def metric_view_joins_nested(draw): + """Generate a join with one level of nested joins (snowflake schema depth 2).""" + children = draw(st.lists(metric_view_joins_leaf(), min_size=1, max_size=2)) + return MetricViewJoin( + name=draw(_identifier), + source=draw(_three_part_name), + on=draw(st.none() | _expr), + using=draw(st.none() | st.lists(_identifier, min_size=1, max_size=3)), + cardinality=draw(st.none() | st.sampled_from(["many_to_one", "one_to_many"])), + rely=draw(st.none() | st.builds(MetricViewRely, at_most_one_match=st.booleans())), + joins=children, + ) + + +# Mix leaf and nested joins +_joins_strategy = st.one_of(metric_view_joins_leaf(), metric_view_joins_nested()) + + +@st.composite +def metric_view_materializations(draw): + mvs = draw(st.none() | st.lists( + st.builds( + MetricViewMaterializedView, + name=_identifier, + type=st.sampled_from(["aggregated", "unaggregated"]), + dimensions=st.none() | st.lists(_identifier, min_size=1, max_size=3), + measures=st.none() | st.lists(_identifier, min_size=1, max_size=3), + ), + min_size=1, + max_size=2, + )) + return MetricViewMaterialization( + schedule=draw(st.none() | st.sampled_from(["every 6 hours", "every 1 hour", "daily"])), + mode=draw(st.none() | st.sampled_from(["relaxed", "strict"])), + materialized_views=mvs, + ) + + +@st.composite +def metric_view_models(draw): + return MetricViewModel( + version="1.1", + source=draw(_three_part_name), + comment=draw(st.none() | _adversarial_text), + filter=draw(st.none() | _expr), + joins=draw(st.none() | st.lists(_joins_strategy, min_size=1, max_size=3)), + fields=draw(st.none() | st.lists(metric_view_fields(), min_size=1, max_size=5)), + measures=draw(st.none() | st.lists(metric_view_measures(), min_size=1, max_size=5)), + materialization=draw(st.none() | metric_view_materializations()), + ) + + +# --- Property Tests --- + + +class TestMetricViewModelRoundTrip: + """Property 1: Metric View Model parse-serialize round-trip.""" + + @given(model=metric_view_models()) + @settings(max_examples=200, suppress_health_check=[HealthCheck.too_slow]) + def test_parse_serialize_roundtrip(self, model: MetricViewModel): + """For any valid MetricViewModel, serializing then parsing produces an equivalent model.""" + yaml_str = model.to_yaml() + parsed = MetricViewModel.from_yaml(yaml_str) + assert parsed == model + + @given(model=metric_view_models()) + @settings(max_examples=50, suppress_health_check=[HealthCheck.too_slow]) + def test_double_roundtrip_stable(self, model: MetricViewModel): + """Double round-trip (serialize→parse→serialize→parse) is stable.""" + yaml1 = model.to_yaml() + parsed1 = MetricViewModel.from_yaml(yaml1) + yaml2 = parsed1.to_yaml() + parsed2 = MetricViewModel.from_yaml(yaml2) + assert parsed1 == parsed2 + assert yaml1 == yaml2 + + +# --- Regression Tests for Specific Edge Cases --- + + +class TestYamlEdgeCases: + """Pinned regression tests for YAML serialization edge cases.""" + + def test_boolean_like_comment(self): + """Comments that look like YAML booleans must round-trip.""" + model = MetricViewModel( + source="a.b.c", + fields=[MetricViewField(name="f1", expr="col1", comment="true")], + ) + parsed = MetricViewModel.from_yaml(model.to_yaml()) + assert parsed.fields[0].comment == "true" + + def test_null_like_synonym(self): + """Synonyms that look like YAML null must round-trip.""" + model = MetricViewModel( + source="a.b.c", + fields=[MetricViewField(name="f1", expr="col1", synonyms=["null", "~"])], + ) + parsed = MetricViewModel.from_yaml(model.to_yaml()) + assert parsed.fields[0].synonyms == ["null", "~"] + + def test_numeric_like_display_name(self): + """Display names that look like numbers must round-trip as strings.""" + model = MetricViewModel( + source="a.b.c", + measures=[MetricViewMeasure(name="m1", expr="SUM(x)", display_name="1.0")], + ) + parsed = MetricViewModel.from_yaml(model.to_yaml()) + assert parsed.measures[0].display_name == "1.0" + + def test_colon_in_comment(self): + """Colons in comments must not break YAML parsing.""" + model = MetricViewModel( + source="a.b.c", + fields=[MetricViewField(name="f1", expr="col1", comment="ratio: sales/cost")], + ) + parsed = MetricViewModel.from_yaml(model.to_yaml()) + assert parsed.fields[0].comment == "ratio: sales/cost" + + def test_special_chars_in_synonyms(self): + """Special characters in synonyms must round-trip.""" + syns = ["star * wild", "hash # tag", "{curly}", "[bracket]"] + model = MetricViewModel( + source="a.b.c", + fields=[MetricViewField(name="f1", expr="col1", synonyms=syns)], + ) + parsed = MetricViewModel.from_yaml(model.to_yaml()) + assert parsed.fields[0].synonyms == syns + + def test_leading_trailing_whitespace(self): + """Strings with leading/trailing whitespace must round-trip.""" + model = MetricViewModel( + source="a.b.c", + fields=[MetricViewField(name="f1", expr="col1", comment=" padded ")], + ) + parsed = MetricViewModel.from_yaml(model.to_yaml()) + assert parsed.fields[0].comment == " padded " + + def test_nested_joins_roundtrip(self): + """Nested joins (snowflake schema) must round-trip.""" + model = MetricViewModel( + source="cat.sch.fact_sales", + joins=[ + MetricViewJoin( + name="date_dim", + source="cat.sch.date_dim", + on="source.date_sk = date_dim.d_date_sk", + cardinality="many_to_one", + joins=[ + MetricViewJoin( + name="fiscal_cal", + source="cat.sch.fiscal_calendar", + on="date_dim.cal_id = fiscal_cal.id", + ), + ], + ), + ], + ) + parsed = MetricViewModel.from_yaml(model.to_yaml()) + assert parsed == model + assert parsed.joins[0].joins[0].name == "fiscal_cal" + + def test_materialization_roundtrip(self): + """Full materialization config round-trips correctly.""" + model = MetricViewModel( + source="a.b.c", + fields=[MetricViewField(name="f1", expr="col1")], + materialization=MetricViewMaterialization( + schedule="every 6 hours", + mode="relaxed", + materialized_views=[ + MetricViewMaterializedView( + name="mv1", + type="aggregated", + dimensions=["f1"], + measures=["m1"], + ), + ], + ), + ) + parsed = MetricViewModel.from_yaml(model.to_yaml()) + assert parsed == model + + def test_date_like_string_source(self): + """A source that could look like a date still round-trips as string.""" + # This is an edge case — three-part names won't look like dates, + # but expressions in comments could + model = MetricViewModel( + source="a.b.c", + fields=[MetricViewField(name="f1", expr="col1", comment="2024-01-01")], + ) + parsed = MetricViewModel.from_yaml(model.to_yaml()) + assert parsed.fields[0].comment == "2024-01-01" + + +# --- Unit Tests --- + + +class TestMetricViewModelParsing: + """Unit tests for parsing Metric View YAML.""" + + def test_parse_complete_metric_view(self): + """Test parsing valid Metric View YAML with all sections populated.""" + yaml_str = """ +version: "1.1" +source: catalog.schema.store_sales +comment: "Store sales metric view" +filter: "ss_quantity > 0" +joins: + - name: date_dim + source: catalog.schema.date_dim + on: "source.ss_sold_date_sk = date_dim.d_date_sk" + cardinality: many_to_one +fields: + - name: sold_date + expr: date_dim.d_date + comment: "Sale date" + display_name: "Date of Sale" + synonyms: ["sale date", "transaction date"] +measures: + - name: total_sales + expr: "SUM(ss_net_paid)" + comment: "Total net sales" +materialization: + schedule: "every 6 hours" + mode: relaxed + materialized_views: + - name: mv_daily_sales + type: aggregated + dimensions: [sold_date] + measures: [total_sales] +""" + model = MetricViewModel.from_yaml(yaml_str) + assert model.version == "1.1" + assert model.source == "catalog.schema.store_sales" + assert model.comment == "Store sales metric view" + assert model.filter == "ss_quantity > 0" + assert len(model.joins) == 1 + assert model.joins[0].name == "date_dim" + assert model.joins[0].cardinality == "many_to_one" + assert len(model.fields) == 1 + assert model.fields[0].name == "sold_date" + assert model.fields[0].display_name == "Date of Sale" + assert model.fields[0].synonyms == ["sale date", "transaction date"] + assert len(model.measures) == 1 + assert model.measures[0].name == "total_sales" + assert model.materialization.schedule == "every 6 hours" + assert model.materialization.materialized_views[0].name == "mv_daily_sales" + + def test_parse_minimal_metric_view(self): + """Test parsing YAML with optional sections absent.""" + yaml_str = """ +version: "1.1" +source: catalog.schema.my_table +fields: + - name: col1 + expr: col1 +""" + model = MetricViewModel.from_yaml(yaml_str) + assert model.source == "catalog.schema.my_table" + assert model.joins is None + assert model.filter is None + assert model.materialization is None + assert model.comment is None + assert len(model.fields) == 1 + + def test_validation_error_missing_source(self): + """Test validation error on missing required field (source).""" + yaml_str = """ +version: "1.1" +fields: + - name: col1 + expr: col1 +""" + with pytest.raises(ValidationError) as exc_info: + MetricViewModel.from_yaml(yaml_str) + assert "source" in str(exc_info.value) + + def test_serialization_excludes_none_fields(self): + """Test serialization excludes None fields and preserves key ordering.""" + model = MetricViewModel( + source="catalog.schema.table", + fields=[MetricViewField(name="f1", expr="col1")], + ) + yaml_str = model.to_yaml() + assert "filter" not in yaml_str + assert "joins" not in yaml_str + assert "materialization" not in yaml_str + assert "comment" not in yaml_str + # Check key ordering: version before source, source before fields + version_pos = yaml_str.index("version") + source_pos = yaml_str.index("source") + fields_pos = yaml_str.index("fields") + assert version_pos < source_pos < fields_pos diff --git a/converters/databricks/tests/test_osi_to_metric_view.py b/converters/databricks/tests/test_osi_to_metric_view.py new file mode 100644 index 0000000..bf4b825 --- /dev/null +++ b/converters/databricks/tests/test_osi_to_metric_view.py @@ -0,0 +1,824 @@ +"""Tests for osi_to_metric_view export logic. + +Includes property-based tests (Hypothesis) for correctness property 6, +plus example-based unit tests for specific export mapping behaviors. +""" + +from __future__ import annotations + +import json +import warnings + +from hypothesis import HealthCheck, given, settings +from hypothesis import strategies as st +from osi.models import ( + OSIAIContextObject, + OSICustomExtension, + OSIDataset, + OSIDialect, + OSIDialectExpression, + OSIDimension, + OSIDocument, + OSIExpression, + OSIField, + OSIMetric, + OSIRelationship, + OSISemanticModel, + OSIVendor, +) + +from osi_databricks.osi_to_metric_view import ( + osi_to_metric_view, +) + +# --- Hypothesis Strategies --- + +_identifier = st.from_regex(r"[a-z][a-z0-9_]{0,29}", fullmatch=True) +_safe_text = st.text( + alphabet=st.characters(min_codepoint=32, max_codepoint=126), + min_size=1, + max_size=50, +) +_expr = st.from_regex(r"[A-Za-z_][A-Za-z0-9_. ()]{0,49}", fullmatch=True) + +# Adversarial strings that test YAML quoting and JSON serialization edge cases +_yaml_dangerous_literals = st.sampled_from([ + "true", "false", "null", "~", "1.0", "0", "-1", + "2024-01-01", "key: value", "has # comment", + "{curly}", "[bracket]", "pipe | here", + "star * wild", 'double"quote', "single'quote", + "backslash\\here", "newline\\n", "tab\\t", + "ratio: sales/cost", "a > b", "x & y", + " leading spaces", "trailing spaces ", +]) +_adversarial_text = st.one_of(_safe_text, _yaml_dangerous_literals) + + +@st.composite +def osi_field_with_ai_context(draw): + """Generate an OSI field with ai_context.synonyms and description.""" + synonyms = draw(st.lists(_adversarial_text, min_size=1, max_size=5)) + description = draw(st.none() | _adversarial_text) + expr_str = draw(_expr) + name = draw(_identifier) + + return OSIField( + name=name, + expression=OSIExpression( + dialects=[OSIDialectExpression(dialect=OSIDialect.DATABRICKS, expression=expr_str)] + ), + dimension=OSIDimension(is_time=False), + description=description, + ai_context=OSIAIContextObject(synonyms=tuple(synonyms)), + ) + + +@st.composite +def osi_metric_with_ai_context(draw): + """Generate an OSI metric with ai_context.synonyms and description.""" + synonyms = draw(st.lists(_adversarial_text, min_size=1, max_size=5)) + description = draw(st.none() | _adversarial_text) + expr_str = draw(_expr) + name = draw(_identifier) + + return OSIMetric( + name=name, + expression=OSIExpression( + dialects=[OSIDialectExpression(dialect=OSIDialect.DATABRICKS, expression=expr_str)] + ), + description=description, + ai_context=OSIAIContextObject(synonyms=tuple(synonyms)), + ) + + +@st.composite +def osi_document_with_ai_context(draw): + """Generate an OSI document with fields and metrics that have AI context.""" + fields = draw(st.lists(osi_field_with_ai_context(), min_size=1, max_size=3)) + metrics = draw(st.lists(osi_metric_with_ai_context(), min_size=1, max_size=3)) + dataset_name = draw(_identifier) + + dataset = OSIDataset( + name=dataset_name, + source=f"catalog.schema.{dataset_name}", + fields=fields, + ) + semantic_model = OSISemanticModel( + name="test_model", + datasets=[dataset], + metrics=metrics, + ) + return OSIDocument( + version="0.2.0.dev0", + dialects=[OSIDialect.DATABRICKS], + vendors=[OSIVendor.DATABRICKS], + semantic_model=[semantic_model], + ) + + +# --- Property 6: Export Maps AI Context to Synonyms and Comment --- + + +class TestExportAIContextMapping: + """Property 6: Export maps AI context to synonyms and comment. + + For any OSI field or metric with ai_context.synonyms and/or description, + the exporter SHALL map the first synonym to display_name, remaining synonyms + to the synonyms list, and description to comment. + """ + + @given(doc=osi_document_with_ai_context()) + @settings(max_examples=100, suppress_health_check=[HealthCheck.too_slow]) + def test_first_synonym_becomes_display_name(self, doc: OSIDocument): + """First ai_context.synonym maps to display_name on exported fields.""" + results = osi_to_metric_view(doc) + assert len(results) > 0 + _, mv_model = results[0] + + dataset = doc.semantic_model[0].datasets[0] + if mv_model.fields and dataset.fields: + for osi_field, mv_field in zip(dataset.fields, mv_model.fields): + if osi_field.ai_context and osi_field.ai_context.synonyms: + assert mv_field.display_name == osi_field.ai_context.synonyms[0] + + @given(doc=osi_document_with_ai_context()) + @settings(max_examples=100, suppress_health_check=[HealthCheck.too_slow]) + def test_remaining_synonyms_become_synonyms_list(self, doc: OSIDocument): + """Remaining ai_context.synonyms map to the synonyms list on fields.""" + results = osi_to_metric_view(doc) + assert len(results) > 0 + _, mv_model = results[0] + + dataset = doc.semantic_model[0].datasets[0] + if mv_model.fields and dataset.fields: + for osi_field, mv_field in zip(dataset.fields, mv_model.fields): + if osi_field.ai_context and osi_field.ai_context.synonyms: + remaining = list(osi_field.ai_context.synonyms[1:]) + if remaining: + assert mv_field.synonyms == remaining + else: + assert mv_field.synonyms is None + + @given(doc=osi_document_with_ai_context()) + @settings(max_examples=100, suppress_health_check=[HealthCheck.too_slow]) + def test_description_becomes_comment(self, doc: OSIDocument): + """OSI field description maps to Metric View field comment.""" + results = osi_to_metric_view(doc) + assert len(results) > 0 + _, mv_model = results[0] + + dataset = doc.semantic_model[0].datasets[0] + if mv_model.fields and dataset.fields: + for osi_field, mv_field in zip(dataset.fields, mv_model.fields): + assert mv_field.comment == osi_field.description + + @given(doc=osi_document_with_ai_context()) + @settings(max_examples=100, suppress_health_check=[HealthCheck.too_slow]) + def test_metric_first_synonym_becomes_display_name(self, doc: OSIDocument): + """First ai_context.synonym maps to display_name on exported measures.""" + results = osi_to_metric_view(doc) + assert len(results) > 0 + _, mv_model = results[0] + + metrics = doc.semantic_model[0].metrics + if mv_model.measures and metrics: + for osi_metric, mv_measure in zip(metrics, mv_model.measures): + if osi_metric.ai_context and osi_metric.ai_context.synonyms: + assert mv_measure.display_name == osi_metric.ai_context.synonyms[0] + + @given(doc=osi_document_with_ai_context()) + @settings(max_examples=100, suppress_health_check=[HealthCheck.too_slow]) + def test_metric_remaining_synonyms_become_synonyms_list(self, doc: OSIDocument): + """Remaining ai_context.synonyms map to the synonyms list on measures.""" + results = osi_to_metric_view(doc) + assert len(results) > 0 + _, mv_model = results[0] + + metrics = doc.semantic_model[0].metrics + if mv_model.measures and metrics: + for osi_metric, mv_measure in zip(metrics, mv_model.measures): + if osi_metric.ai_context and osi_metric.ai_context.synonyms: + remaining = list(osi_metric.ai_context.synonyms[1:]) + if remaining: + assert mv_measure.synonyms == remaining + else: + assert mv_measure.synonyms is None + + @given(doc=osi_document_with_ai_context()) + @settings(max_examples=100, suppress_health_check=[HealthCheck.too_slow]) + def test_metric_description_becomes_comment(self, doc: OSIDocument): + """OSI metric description maps to Metric View measure comment.""" + results = osi_to_metric_view(doc) + assert len(results) > 0 + _, mv_model = results[0] + + metrics = doc.semantic_model[0].metrics + if mv_model.measures and metrics: + for osi_metric, mv_measure in zip(metrics, mv_model.measures): + assert mv_measure.comment == osi_metric.description + + +# --- Unit Tests --- + + +class TestExportComplete: + """Unit tests for complete OSI → Metric View export scenarios.""" + + def test_export_with_databricks_dialect(self): + """Export uses DATABRICKS dialect expressions.""" + doc = _make_doc( + fields=[ + OSIField( + name="amount", + expression=OSIExpression( + dialects=[ + OSIDialectExpression(dialect=OSIDialect.DATABRICKS, expression="ss_net_paid"), + OSIDialectExpression(dialect=OSIDialect.ANSI_SQL, expression="ss_net_paid"), + ] + ), + ) + ], + metrics=[ + OSIMetric( + name="total_sales", + expression=OSIExpression( + dialects=[ + OSIDialectExpression(dialect=OSIDialect.DATABRICKS, expression="SUM(ss_net_paid)"), + ] + ), + ) + ], + ) + + results = osi_to_metric_view(doc) + assert len(results) == 1 + name, mv = results[0] + assert name == "store_sales" + assert mv.fields[0].expr == "ss_net_paid" + assert mv.measures[0].expr == "SUM(ss_net_paid)" + + def test_fallback_to_ansi_sql(self): + """Export falls back to ANSI_SQL when DATABRICKS dialect absent.""" + doc = _make_doc( + fields=[ + OSIField( + name="amount", + expression=OSIExpression( + dialects=[ + OSIDialectExpression(dialect=OSIDialect.ANSI_SQL, expression="net_paid"), + ] + ), + ) + ], + ) + + results = osi_to_metric_view(doc) + assert len(results) == 1 + _, mv = results[0] + assert mv.fields[0].expr == "net_paid" + + def test_skip_field_no_usable_dialect(self): + """Skip field when no DATABRICKS or ANSI_SQL dialect available.""" + doc = _make_doc( + fields=[ + OSIField( + name="good_field", + expression=OSIExpression( + dialects=[ + OSIDialectExpression(dialect=OSIDialect.DATABRICKS, expression="col1"), + ] + ), + ), + OSIField( + name="bad_field", + expression=OSIExpression( + dialects=[ + OSIDialectExpression(dialect=OSIDialect.SNOWFLAKE, expression="col2"), + ] + ), + ), + ], + ) + + with warnings.catch_warnings(record=True) as w: + warnings.simplefilter("always") + results = osi_to_metric_view(doc) + + _, mv = results[0] + # Only the good field should be exported + assert len(mv.fields) == 1 + assert mv.fields[0].name == "good_field" + # Warning should be emitted for skipped field + assert any("bad_field" in str(warning.message) for warning in w) + + def test_skip_metric_no_usable_dialect(self): + """Skip metric when no DATABRICKS or ANSI_SQL dialect available.""" + doc = _make_doc( + fields=[ + OSIField( + name="f1", + expression=OSIExpression( + dialects=[OSIDialectExpression(dialect=OSIDialect.DATABRICKS, expression="col1")] + ), + ) + ], + metrics=[ + OSIMetric( + name="bad_metric", + expression=OSIExpression( + dialects=[ + OSIDialectExpression(dialect=OSIDialect.SNOWFLAKE, expression="SUM(x)"), + ] + ), + ), + ], + ) + + with warnings.catch_warnings(record=True) as w: + warnings.simplefilter("always") + results = osi_to_metric_view(doc) + + _, mv = results[0] + assert mv.measures is None + assert any("bad_metric" in str(warning.message) for warning in w) + + +class TestExportJoinReconstruction: + """Unit tests for relationship → join ON clause reconstruction.""" + + def test_single_key_join(self): + """Single-key relationship reconstructed as ON clause.""" + doc = _make_doc( + fields=[ + OSIField( + name="f1", + expression=OSIExpression( + dialects=[OSIDialectExpression(dialect=OSIDialect.DATABRICKS, expression="col1")] + ), + ) + ], + relationships=[ + OSIRelationship( + name="date_dim", + **{"from": "store_sales"}, + to="date_dim", + from_columns=["ss_sold_date_sk"], + to_columns=["d_date_sk"], + ) + ], + ) + + results = osi_to_metric_view(doc) + _, mv = results[0] + assert mv.joins is not None + assert len(mv.joins) == 1 + join = mv.joins[0] + assert join.name == "date_dim" + assert join.source == "date_dim" + assert join.on == "source.ss_sold_date_sk = date_dim.d_date_sk" + + def test_composite_key_join(self): + """Multi-key relationship reconstructed with AND in ON clause.""" + doc = _make_doc( + fields=[ + OSIField( + name="f1", + expression=OSIExpression( + dialects=[OSIDialectExpression(dialect=OSIDialect.DATABRICKS, expression="col1")] + ), + ) + ], + relationships=[ + OSIRelationship( + name="items", + **{"from": "store_sales"}, + to="order_items", + from_columns=["order_id", "item_id"], + to_columns=["order_id", "item_id"], + ) + ], + ) + + results = osi_to_metric_view(doc) + _, mv = results[0] + join = mv.joins[0] + assert join.on == "source.order_id = items.order_id AND source.item_id = items.item_id" + + def test_join_with_cardinality_and_rely(self): + """Cardinality and rely restored from custom extensions.""" + doc = _make_doc( + fields=[ + OSIField( + name="f1", + expression=OSIExpression( + dialects=[OSIDialectExpression(dialect=OSIDialect.DATABRICKS, expression="col1")] + ), + ) + ], + relationships=[ + OSIRelationship( + name="date_dim", + **{"from": "store_sales"}, + to="date_dim", + from_columns=["date_sk"], + to_columns=["d_date_sk"], + custom_extensions=[ + OSICustomExtension( + vendor_name=OSIVendor.DATABRICKS, + data=json.dumps({"cardinality": "many_to_one", "rely": {"at_most_one_match": True}}), + ) + ], + ) + ], + ) + + results = osi_to_metric_view(doc) + _, mv = results[0] + join = mv.joins[0] + assert join.cardinality == "many_to_one" + assert join.rely is not None + assert join.rely.at_most_one_match is True + + +class TestExportAIContextUnit: + """Unit tests for ai_context → display_name + synonyms mapping.""" + + def test_synonyms_split_to_display_name_and_list(self): + """First synonym → display_name, rest → synonyms.""" + doc = _make_doc( + fields=[ + OSIField( + name="amount", + expression=OSIExpression( + dialects=[OSIDialectExpression(dialect=OSIDialect.DATABRICKS, expression="net_paid")] + ), + ai_context=OSIAIContextObject(synonyms=("Net Amount", "total paid", "payment")), + ) + ], + ) + + results = osi_to_metric_view(doc) + _, mv = results[0] + field = mv.fields[0] + assert field.display_name == "Net Amount" + assert field.synonyms == ["total paid", "payment"] + + def test_single_synonym_becomes_display_name_only(self): + """Single synonym → display_name with no synonyms list.""" + doc = _make_doc( + fields=[ + OSIField( + name="amount", + expression=OSIExpression( + dialects=[OSIDialectExpression(dialect=OSIDialect.DATABRICKS, expression="net_paid")] + ), + ai_context=OSIAIContextObject(synonyms=("Net Amount",)), + ) + ], + ) + + results = osi_to_metric_view(doc) + _, mv = results[0] + field = mv.fields[0] + assert field.display_name == "Net Amount" + assert field.synonyms is None + + def test_no_ai_context_produces_no_display_name(self): + """No ai_context → no display_name or synonyms.""" + doc = _make_doc( + fields=[ + OSIField( + name="amount", + expression=OSIExpression( + dialects=[OSIDialectExpression(dialect=OSIDialect.DATABRICKS, expression="net_paid")] + ), + ) + ], + ) + + results = osi_to_metric_view(doc) + _, mv = results[0] + field = mv.fields[0] + assert field.display_name is None + assert field.synonyms is None + + def test_description_maps_to_comment(self): + """OSI description → Metric View comment.""" + doc = _make_doc( + fields=[ + OSIField( + name="amount", + expression=OSIExpression( + dialects=[OSIDialectExpression(dialect=OSIDialect.DATABRICKS, expression="net_paid")] + ), + description="Total net payment amount", + ) + ], + ) + + results = osi_to_metric_view(doc) + _, mv = results[0] + field = mv.fields[0] + assert field.comment == "Total net payment amount" + + +class TestExportCustomExtensions: + """Unit tests for materialization and filter restored from custom_extensions.""" + + def test_filter_restored_from_extension(self): + """Filter in dataset custom_extension → Metric View filter.""" + doc = _make_doc( + fields=[ + OSIField( + name="f1", + expression=OSIExpression( + dialects=[OSIDialectExpression(dialect=OSIDialect.DATABRICKS, expression="col1")] + ), + ) + ], + dataset_extensions=[ + OSICustomExtension( + vendor_name=OSIVendor.DATABRICKS, + data=json.dumps({"filter": "quantity > 0"}), + ) + ], + ) + + results = osi_to_metric_view(doc) + _, mv = results[0] + assert mv.filter == "quantity > 0" + + def test_materialization_restored_from_extension(self): + """Materialization in semantic model extension → Metric View materialization.""" + mat_data = { + "schedule": "every 6 hours", + "mode": "relaxed", + "materialized_views": [ + {"name": "mv_daily", "type": "aggregated", "dimensions": ["date"], "measures": ["total"]} + ], + } + doc = _make_doc( + fields=[ + OSIField( + name="f1", + expression=OSIExpression( + dialects=[OSIDialectExpression(dialect=OSIDialect.DATABRICKS, expression="col1")] + ), + ) + ], + sm_extensions=[ + OSICustomExtension( + vendor_name=OSIVendor.DATABRICKS, + data=json.dumps({"materialization": mat_data}), + ) + ], + ) + + results = osi_to_metric_view(doc) + _, mv = results[0] + assert mv.materialization is not None + assert mv.materialization.schedule == "every 6 hours" + assert mv.materialization.mode == "relaxed" + assert len(mv.materialization.materialized_views) == 1 + assert mv.materialization.materialized_views[0].name == "mv_daily" + + def test_source_query_restored_from_extension(self): + """Source query in dataset custom_extension → Metric View source.""" + query = "SELECT a, b FROM my_table WHERE active" + doc = _make_doc( + fields=[ + OSIField( + name="f1", + expression=OSIExpression( + dialects=[OSIDialectExpression(dialect=OSIDialect.DATABRICKS, expression="col1")] + ), + ) + ], + dataset_extensions=[ + OSICustomExtension( + vendor_name=OSIVendor.DATABRICKS, + data=json.dumps({"source_query": query}), + ) + ], + ) + + results = osi_to_metric_view(doc) + _, mv = results[0] + assert mv.source == query + + def test_other_vendor_extensions_ignored(self): + """Extensions for other vendors (SNOWFLAKE, DBT) are ignored during export.""" + doc = _make_doc( + fields=[ + OSIField( + name="f1", + expression=OSIExpression( + dialects=[OSIDialectExpression(dialect=OSIDialect.DATABRICKS, expression="col1")] + ), + custom_extensions=[ + OSICustomExtension( + vendor_name=OSIVendor.SNOWFLAKE, + data=json.dumps({"snowflake_specific": "value"}), + ) + ], + ) + ], + dataset_extensions=[ + OSICustomExtension( + vendor_name=OSIVendor.DBT, + data=json.dumps({"dbt_specific": "value"}), + ) + ], + ) + + # Should not error and should produce valid output + results = osi_to_metric_view(doc) + assert len(results) == 1 + _, mv = results[0] + assert mv.fields[0].name == "f1" + assert mv.filter is None # No DATABRICKS filter extension + + def test_field_format_restored_from_extension(self): + """Field format in custom_extension → Metric View field format.""" + doc = _make_doc( + fields=[ + OSIField( + name="amount", + expression=OSIExpression( + dialects=[OSIDialectExpression(dialect=OSIDialect.DATABRICKS, expression="net_paid")] + ), + custom_extensions=[ + OSICustomExtension( + vendor_name=OSIVendor.DATABRICKS, + data=json.dumps({"format": {"type": "currency", "currency_code": "USD"}}), + ) + ], + ) + ], + ) + + results = osi_to_metric_view(doc) + _, mv = results[0] + assert mv.fields[0].format is not None + assert mv.fields[0].format.type == "currency" + assert mv.fields[0].format.currency_code == "USD" + + def test_measure_window_restored_from_extension(self): + """Measure window in custom_extension → Metric View measure window.""" + doc = _make_doc( + fields=[ + OSIField( + name="f1", + expression=OSIExpression( + dialects=[OSIDialectExpression(dialect=OSIDialect.DATABRICKS, expression="col1")] + ), + ) + ], + metrics=[ + OSIMetric( + name="rolling_sum", + expression=OSIExpression( + dialects=[OSIDialectExpression(dialect=OSIDialect.DATABRICKS, expression="SUM(amount)")] + ), + custom_extensions=[ + OSICustomExtension( + vendor_name=OSIVendor.DATABRICKS, + data=json.dumps({"window": [{"order": "d_date", "range": "trailing 7 day"}]}), + ) + ], + ) + ], + ) + + results = osi_to_metric_view(doc) + _, mv = results[0] + assert mv.measures[0].window is not None + assert len(mv.measures[0].window) == 1 + assert mv.measures[0].window[0].order == "d_date" + assert mv.measures[0].window[0].range == "trailing 7 day" + + +class TestExportDatasetSelection: + """Unit tests for dataset selection logic.""" + + def test_dataset_without_fields_or_metrics_skipped(self): + """Datasets with no fields and no metrics are not exported.""" + doc = OSIDocument( + version="0.2.0.dev0", + semantic_model=[ + OSISemanticModel( + name="model", + datasets=[ + OSIDataset(name="empty_ds", source="cat.sch.empty"), + ], + ) + ], + ) + + results = osi_to_metric_view(doc) + assert len(results) == 0 + + def test_multiple_datasets_produce_multiple_results(self): + """Each dataset with content produces one (name, model) tuple.""" + doc = OSIDocument( + version="0.2.0.dev0", + semantic_model=[ + OSISemanticModel( + name="model", + datasets=[ + OSIDataset( + name="ds1", + source="cat.sch.ds1", + fields=[ + OSIField( + name="f1", + expression=OSIExpression( + dialects=[OSIDialectExpression(dialect=OSIDialect.DATABRICKS, expression="c1")] + ), + ) + ], + ), + OSIDataset( + name="ds2", + source="cat.sch.ds2", + fields=[ + OSIField( + name="f2", + expression=OSIExpression( + dialects=[OSIDialectExpression(dialect=OSIDialect.DATABRICKS, expression="c2")] + ), + ) + ], + ), + ], + metrics=[ + OSIMetric( + name="m1", + expression=OSIExpression( + dialects=[OSIDialectExpression(dialect=OSIDialect.DATABRICKS, expression="SUM(c1)")] + ), + ) + ], + ) + ], + ) + + results = osi_to_metric_view(doc) + assert len(results) == 2 + names = [r[0] for r in results] + assert "ds1" in names + assert "ds2" in names + + def test_semantic_model_description_becomes_comment(self): + """Semantic model description maps to Metric View top-level comment.""" + doc = _make_doc( + fields=[ + OSIField( + name="f1", + expression=OSIExpression( + dialects=[OSIDialectExpression(dialect=OSIDialect.DATABRICKS, expression="col1")] + ), + ) + ], + description="Store sales model", + ) + + results = osi_to_metric_view(doc) + _, mv = results[0] + assert mv.comment == "Store sales model" + + +# --- Test Helpers --- + + +def _make_doc( + fields: list[OSIField] | None = None, + metrics: list[OSIMetric] | None = None, + relationships: list[OSIRelationship] | None = None, + dataset_extensions: list[OSICustomExtension] | None = None, + sm_extensions: list[OSICustomExtension] | None = None, + description: str | None = None, + dataset_name: str = "store_sales", + dataset_source: str = "catalog.schema.store_sales", +) -> OSIDocument: + """Helper to build a minimal OSI document for testing.""" + dataset = OSIDataset( + name=dataset_name, + source=dataset_source, + fields=fields, + custom_extensions=dataset_extensions, + ) + semantic_model = OSISemanticModel( + name="test_model", + description=description, + datasets=[dataset], + relationships=relationships, + metrics=metrics, + custom_extensions=sm_extensions, + ) + return OSIDocument( + version="0.2.0.dev0", + dialects=[OSIDialect.DATABRICKS], + vendors=[OSIVendor.DATABRICKS], + semantic_model=[semantic_model], + ) diff --git a/converters/databricks/tests/test_roundtrip.py b/converters/databricks/tests/test_roundtrip.py new file mode 100644 index 0000000..94cb6c5 --- /dev/null +++ b/converters/databricks/tests/test_roundtrip.py @@ -0,0 +1,392 @@ +"""Integration round-trip tests using TPC-DS fixtures. + +These tests verify that converting between Metric View YAML and OSI YAML +preserves all essential information when using realistic, fixture-based inputs. + +Test coverage: +- MV→OSI→MV: Import the TPC-DS Metric View fixture to OSI, export back, + and verify fields, measures, joins, filter, and metadata are preserved. +- OSI→MV→OSI: Export the TPC-DS OSI fixture to Metric View, import back, + and verify datasets, fields, metrics, relationships, and metadata are preserved. +""" + +from __future__ import annotations + +import json + +from osi.models import OSIDialect, OSIDocument, OSIVendor + +from osi_databricks.metric_view_to_osi import metric_view_to_osi +from osi_databricks.models import MetricViewModel +from osi_databricks.osi_to_metric_view import osi_to_metric_view + + +class TestMVtoOSItoMVFixtureRoundTrip: + """MV→OSI→MV round-trip with the TPC-DS Metric View fixture. + + Verifies that importing to OSI and exporting back preserves all fields, + measures, joins, filter, and semantic metadata. + """ + + def test_source_preserved(self, metric_view_tpcds_model: MetricViewModel): + """Three-part source name survives the round-trip.""" + osi_doc = metric_view_to_osi(metric_view_tpcds_model, model_name="tpcds_store_sales") + results = osi_to_metric_view(osi_doc) + assert len(results) == 1 + _, mv_out = results[0] + assert mv_out.source == "tpcds.analytics.store_sales" + + def test_all_field_names_preserved(self, metric_view_tpcds_model: MetricViewModel): + """All 5 field names are preserved through the round-trip.""" + osi_doc = metric_view_to_osi(metric_view_tpcds_model, model_name="tpcds_store_sales") + results = osi_to_metric_view(osi_doc) + _, mv_out = results[0] + + expected_names = ["sold_date", "sold_year", "item_id", "item_category", "store_id"] + actual_names = [f.name for f in mv_out.fields] + assert actual_names == expected_names + + def test_all_field_expressions_preserved(self, metric_view_tpcds_model: MetricViewModel): + """All field expressions are preserved through the round-trip.""" + osi_doc = metric_view_to_osi(metric_view_tpcds_model, model_name="tpcds_store_sales") + results = osi_to_metric_view(osi_doc) + _, mv_out = results[0] + + expected_exprs = [ + "DATE_TRUNC('DAY', d_date)", + "YEAR(d_date)", + "i_item_id", + "i_category", + "ss_store_sk", + ] + actual_exprs = [f.expr for f in mv_out.fields] + assert actual_exprs == expected_exprs + + def test_all_measure_names_preserved(self, metric_view_tpcds_model: MetricViewModel): + """All 5 measure names are preserved through the round-trip.""" + osi_doc = metric_view_to_osi(metric_view_tpcds_model, model_name="tpcds_store_sales") + results = osi_to_metric_view(osi_doc) + _, mv_out = results[0] + + expected_names = [ + "total_sales", + "total_quantity", + "avg_discount", + "net_profit", + "trailing_7d_sales", + ] + actual_names = [m.name for m in mv_out.measures] + assert actual_names == expected_names + + def test_all_measure_expressions_preserved(self, metric_view_tpcds_model: MetricViewModel): + """All measure expressions are preserved through the round-trip.""" + osi_doc = metric_view_to_osi(metric_view_tpcds_model, model_name="tpcds_store_sales") + results = osi_to_metric_view(osi_doc) + _, mv_out = results[0] + + expected_exprs = [ + "SUM(ss_sales_price)", + "SUM(ss_quantity)", + "AVG(ss_ext_discount_amt)", + "SUM(ss_net_profit)", + "SUM(ss_sales_price)", + ] + actual_exprs = [m.expr for m in mv_out.measures] + assert actual_exprs == expected_exprs + + def test_join_names_preserved(self, metric_view_tpcds_model: MetricViewModel): + """Both join names are preserved through the round-trip.""" + osi_doc = metric_view_to_osi(metric_view_tpcds_model, model_name="tpcds_store_sales") + results = osi_to_metric_view(osi_doc) + _, mv_out = results[0] + + assert mv_out.joins is not None + expected_names = ["date_dim", "item"] + actual_names = [j.name for j in mv_out.joins] + assert actual_names == expected_names + + def test_join_sources_preserved(self, metric_view_tpcds_model: MetricViewModel): + """Join target datasets are preserved (derived from source three-part name).""" + osi_doc = metric_view_to_osi(metric_view_tpcds_model, model_name="tpcds_store_sales") + results = osi_to_metric_view(osi_doc) + _, mv_out = results[0] + + # The exporter stores relationship.to as the join source, which is the + # table name extracted from the original three-part name + assert mv_out.joins is not None + actual_sources = [j.source for j in mv_out.joins] + assert actual_sources == ["date_dim", "item"] + + def test_filter_preserved(self, metric_view_tpcds_model: MetricViewModel): + """Filter expression is preserved through the round-trip.""" + osi_doc = metric_view_to_osi(metric_view_tpcds_model, model_name="tpcds_store_sales") + results = osi_to_metric_view(osi_doc) + _, mv_out = results[0] + assert mv_out.filter == "ss_net_profit > 0" + + def test_comment_preserved(self, metric_view_tpcds_model: MetricViewModel): + """Top-level comment is preserved through the round-trip.""" + osi_doc = metric_view_to_osi(metric_view_tpcds_model, model_name="tpcds_store_sales") + results = osi_to_metric_view(osi_doc) + _, mv_out = results[0] + assert mv_out.comment == "Store sales fact table with date and item dimensions from TPC-DS benchmark" + + def test_field_synonyms_and_display_name_preserved( + self, metric_view_tpcds_model: MetricViewModel + ): + """Field display_name and synonyms content is preserved through the round-trip. + + The round-trip merges display_name + synonyms into a single list in OSI, + then splits the first element back to display_name on export. + """ + osi_doc = metric_view_to_osi(metric_view_tpcds_model, model_name="tpcds_store_sales") + results = osi_to_metric_view(osi_doc) + _, mv_out = results[0] + + # sold_date: display_name="Sale Date", synonyms=["transaction date", "order date"] + sold_date = mv_out.fields[0] + assert sold_date.display_name == "Sale Date" + assert sold_date.synonyms == ["transaction date", "order date"] + + # item_category: display_name="Category", synonyms=["product category", "item type"] + item_category = mv_out.fields[3] + assert item_category.display_name == "Category" + assert item_category.synonyms == ["product category", "item type"] + + def test_measure_synonyms_preserved(self, metric_view_tpcds_model: MetricViewModel): + """Measure display_name and synonyms are preserved through the round-trip.""" + osi_doc = metric_view_to_osi(metric_view_tpcds_model, model_name="tpcds_store_sales") + results = osi_to_metric_view(osi_doc) + _, mv_out = results[0] + + # total_sales: display_name="Total Sales", synonyms=["revenue", "gross sales"] + total_sales = mv_out.measures[0] + assert total_sales.display_name == "Total Sales" + assert total_sales.synonyms == ["revenue", "gross sales"] + + # net_profit: display_name="Net Profit", synonyms=["profit", "margin"] + net_profit = mv_out.measures[3] + assert net_profit.display_name == "Net Profit" + assert net_profit.synonyms == ["profit", "margin"] + + def test_materialization_preserved(self, metric_view_tpcds_model: MetricViewModel): + """Materialization config survives the round-trip via custom extensions.""" + osi_doc = metric_view_to_osi(metric_view_tpcds_model, model_name="tpcds_store_sales") + results = osi_to_metric_view(osi_doc) + _, mv_out = results[0] + + assert mv_out.materialization is not None + assert mv_out.materialization.schedule == "every 6 hours" + assert mv_out.materialization.mode == "relaxed" + assert mv_out.materialization.materialized_views is not None + assert len(mv_out.materialization.materialized_views) == 1 + mat_view = mv_out.materialization.materialized_views[0] + assert mat_view.name == "sales_by_date" + assert mat_view.type == "aggregated" + assert mat_view.dimensions == ["sold_date", "item_category"] + assert mat_view.measures == ["total_sales", "total_quantity"] + + def test_window_measure_preserved(self, metric_view_tpcds_model: MetricViewModel): + """Window specification on trailing_7d_sales measure is preserved.""" + osi_doc = metric_view_to_osi(metric_view_tpcds_model, model_name="tpcds_store_sales") + results = osi_to_metric_view(osi_doc) + _, mv_out = results[0] + + trailing = mv_out.measures[4] + assert trailing.name == "trailing_7d_sales" + assert trailing.window is not None + assert len(trailing.window) == 1 + assert trailing.window[0].order == "sold_date" + assert trailing.window[0].range == "trailing 7 day" + + def test_format_preserved_on_fields(self, metric_view_tpcds_model: MetricViewModel): + """Field format metadata is preserved through the round-trip.""" + osi_doc = metric_view_to_osi(metric_view_tpcds_model, model_name="tpcds_store_sales") + results = osi_to_metric_view(osi_doc) + _, mv_out = results[0] + + # sold_date has format: type=date, date_format=year_month_day + sold_date = mv_out.fields[0] + assert sold_date.format is not None + assert sold_date.format.type == "date" + assert sold_date.format.date_format == "year_month_day" + + def test_format_preserved_on_measures(self, metric_view_tpcds_model: MetricViewModel): + """Measure format metadata is preserved through the round-trip.""" + osi_doc = metric_view_to_osi(metric_view_tpcds_model, model_name="tpcds_store_sales") + results = osi_to_metric_view(osi_doc) + _, mv_out = results[0] + + # total_sales has format: type=currency, currency_code=USD + total_sales = mv_out.measures[0] + assert total_sales.format is not None + assert total_sales.format.type == "currency" + assert total_sales.format.currency_code == "USD" + + +class TestOSItoMVtoOSIFixtureRoundTrip: + """OSI→MV→OSI round-trip with the TPC-DS OSI fixture. + + Verifies that exporting to Metric View and importing back preserves + datasets, fields, metrics, relationships, and metadata. + """ + + def test_dataset_name_preserved(self, osi_tpcds_document: OSIDocument): + """Dataset name is preserved through OSI→MV→OSI.""" + results = osi_to_metric_view(osi_tpcds_document) + assert len(results) == 1 + dataset_name, mv_model = results[0] + assert dataset_name == "store_sales" + + # Import back + osi_back = metric_view_to_osi(mv_model, model_name="tpcds_store_sales") + ds_back = osi_back.semantic_model[0].datasets[0] + assert ds_back.name == "store_sales" + + def test_field_names_preserved(self, osi_tpcds_document: OSIDocument): + """All field names are preserved through OSI→MV→OSI.""" + original_ds = osi_tpcds_document.semantic_model[0].datasets[0] + original_names = [f.name for f in original_ds.fields] + + results = osi_to_metric_view(osi_tpcds_document) + _, mv_model = results[0] + osi_back = metric_view_to_osi(mv_model, model_name="tpcds_store_sales") + ds_back = osi_back.semantic_model[0].datasets[0] + + roundtrip_names = [f.name for f in ds_back.fields] + assert roundtrip_names == original_names + + def test_field_databricks_expressions_preserved(self, osi_tpcds_document: OSIDocument): + """Field DATABRICKS dialect expressions are preserved through OSI→MV→OSI.""" + original_ds = osi_tpcds_document.semantic_model[0].datasets[0] + original_exprs = [] + for f in original_ds.fields: + for d in f.expression.dialects: + if d.dialect == OSIDialect.DATABRICKS: + original_exprs.append(d.expression) + + results = osi_to_metric_view(osi_tpcds_document) + _, mv_model = results[0] + osi_back = metric_view_to_osi(mv_model, model_name="tpcds_store_sales") + ds_back = osi_back.semantic_model[0].datasets[0] + + roundtrip_exprs = [] + for f in ds_back.fields: + for d in f.expression.dialects: + if d.dialect == OSIDialect.DATABRICKS: + roundtrip_exprs.append(d.expression) + + assert roundtrip_exprs == original_exprs + + def test_metric_names_preserved(self, osi_tpcds_document: OSIDocument): + """All metric names are preserved through OSI→MV→OSI.""" + original_metrics = osi_tpcds_document.semantic_model[0].metrics + original_names = [m.name for m in original_metrics] + + results = osi_to_metric_view(osi_tpcds_document) + _, mv_model = results[0] + osi_back = metric_view_to_osi(mv_model, model_name="tpcds_store_sales") + back_metrics = osi_back.semantic_model[0].metrics + + assert back_metrics is not None + roundtrip_names = [m.name for m in back_metrics] + assert roundtrip_names == original_names + + def test_metric_databricks_expressions_preserved(self, osi_tpcds_document: OSIDocument): + """Metric DATABRICKS dialect expressions are preserved through OSI→MV→OSI.""" + original_metrics = osi_tpcds_document.semantic_model[0].metrics + original_exprs = [] + for m in original_metrics: + for d in m.expression.dialects: + if d.dialect == OSIDialect.DATABRICKS: + original_exprs.append(d.expression) + + results = osi_to_metric_view(osi_tpcds_document) + _, mv_model = results[0] + osi_back = metric_view_to_osi(mv_model, model_name="tpcds_store_sales") + back_metrics = osi_back.semantic_model[0].metrics + + roundtrip_exprs = [] + for m in back_metrics: + for d in m.expression.dialects: + if d.dialect == OSIDialect.DATABRICKS: + roundtrip_exprs.append(d.expression) + + assert roundtrip_exprs == original_exprs + + def test_relationship_names_preserved(self, osi_tpcds_document: OSIDocument): + """Relationship names are preserved through OSI→MV→OSI.""" + original_rels = osi_tpcds_document.semantic_model[0].relationships + original_names = [r.name for r in original_rels] + + results = osi_to_metric_view(osi_tpcds_document) + _, mv_model = results[0] + osi_back = metric_view_to_osi(mv_model, model_name="tpcds_store_sales") + back_rels = osi_back.semantic_model[0].relationships + + assert back_rels is not None + roundtrip_names = [r.name for r in back_rels] + assert roundtrip_names == original_names + + def test_ai_context_synonyms_preserved(self, osi_tpcds_document: OSIDocument): + """ai_context.synonyms on fields are preserved through OSI→MV→OSI.""" + original_ds = osi_tpcds_document.semantic_model[0].datasets[0] + + results = osi_to_metric_view(osi_tpcds_document) + _, mv_model = results[0] + osi_back = metric_view_to_osi(mv_model, model_name="tpcds_store_sales") + ds_back = osi_back.semantic_model[0].datasets[0] + + for orig_field, rt_field in zip(original_ds.fields, ds_back.fields): + if orig_field.ai_context and hasattr(orig_field.ai_context, "synonyms"): + if orig_field.ai_context.synonyms: + assert rt_field.ai_context is not None + assert rt_field.ai_context.synonyms == orig_field.ai_context.synonyms + + def test_filter_extension_preserved(self, osi_tpcds_document: OSIDocument): + """DATABRICKS filter custom extension is preserved through OSI→MV→OSI.""" + results = osi_to_metric_view(osi_tpcds_document) + _, mv_model = results[0] + + # Verify filter was extracted + assert mv_model.filter == "ss_net_profit > 0" + + # Import back and verify extension is reconstructed + osi_back = metric_view_to_osi(mv_model, model_name="tpcds_store_sales") + ds_back = osi_back.semantic_model[0].datasets[0] + assert ds_back.custom_extensions is not None + + # Find the DATABRICKS extension with filter + filter_found = False + for ext in ds_back.custom_extensions: + if ext.vendor_name == OSIVendor.DATABRICKS: + data = json.loads(ext.data) + if "filter" in data: + assert data["filter"] == "ss_net_profit > 0" + filter_found = True + assert filter_found + + def test_materialization_extension_preserved(self, osi_tpcds_document: OSIDocument): + """DATABRICKS materialization custom extension is preserved through OSI→MV→OSI.""" + results = osi_to_metric_view(osi_tpcds_document) + _, mv_model = results[0] + + # Verify materialization was extracted + assert mv_model.materialization is not None + assert mv_model.materialization.schedule == "every 6 hours" + + # Import back and verify extension is reconstructed + osi_back = metric_view_to_osi(mv_model, model_name="tpcds_store_sales") + sm_back = osi_back.semantic_model[0] + assert sm_back.custom_extensions is not None + + mat_found = False + for ext in sm_back.custom_extensions: + if ext.vendor_name == OSIVendor.DATABRICKS: + data = json.loads(ext.data) + if "materialization" in data: + mat = data["materialization"] + assert mat["schedule"] == "every 6 hours" + assert mat["mode"] == "relaxed" + mat_found = True + assert mat_found diff --git a/converters/databricks/tests/test_roundtrip_properties.py b/converters/databricks/tests/test_roundtrip_properties.py new file mode 100644 index 0000000..dd81b5b --- /dev/null +++ b/converters/databricks/tests/test_roundtrip_properties.py @@ -0,0 +1,896 @@ +"""Round-trip property tests for the osi-databricks converter. + +Property 7: MV→OSI→MV round-trip preservation +Property 8: OSI→MV→OSI round-trip preservation +Property 9: Custom extensions preserved during round-trip + +These tests verify that converting in one direction and back preserves the +essential information — field names, expressions, measure names, join sources, +filter expressions, synonyms, and custom extensions for all vendors. +""" + +from __future__ import annotations + +import json + +from hypothesis import HealthCheck, given, settings +from hypothesis import strategies as st +from osi.models import ( + OSIAIContextObject, + OSICustomExtension, + OSIDataset, + OSIDialect, + OSIDialectExpression, + OSIDimension, + OSIDocument, + OSIExpression, + OSIField, + OSIMetric, + OSIRelationship, + OSISemanticModel, + OSIVendor, +) + +from osi_databricks.metric_view_to_osi import metric_view_to_osi +from osi_databricks.models import ( + MetricViewField, + MetricViewJoin, + MetricViewMeasure, + MetricViewModel, + MetricViewRely, +) +from osi_databricks.osi_to_metric_view import osi_to_metric_view + +# --- Hypothesis Strategies --- + +_identifier = st.from_regex(r"[a-z][a-z0-9_]{0,19}", fullmatch=True) +_three_part_name = st.builds( + lambda a, b, c: f"{a}.{b}.{c}", + _identifier, + _identifier, + _identifier, +) + +# Expressions that are valid column-like expressions (no Databricks-specific syntax +# to ensure ANSI_SQL dialect is also generated, making round-trips more predictable) +_standard_expr = st.from_regex(r"[A-Za-z_][A-Za-z0-9_]{0,19}", fullmatch=True) + +# Safe text that avoids JSON and YAML issues for metadata +_safe_text = st.text( + alphabet=st.characters( + min_codepoint=32, + max_codepoint=126, + blacklist_characters='"\\', + ), + min_size=1, + max_size=30, +) + + +# --- Strategies for Property 7 (MV→OSI→MV) --- + + +@st.composite +def roundtrip_mv_fields(draw): + """Generate fields suitable for MV→OSI→MV round-trip testing. + + Uses simple column-like expressions that produce parseable ON clauses + and avoid time-inference ambiguity. + """ + name = draw(_identifier) + # Use expressions that won't trigger time inference (avoid DATE, TIME, etc.) + expr = draw(_standard_expr) + comment = draw(st.none() | _safe_text) + display_name = draw(st.none() | _safe_text) + synonyms = draw(st.none() | st.lists(_safe_text, min_size=1, max_size=3)) + + return MetricViewField( + name=name, + expr=expr, + comment=comment, + display_name=display_name, + synonyms=synonyms, + ) + + +@st.composite +def roundtrip_mv_measures(draw): + """Generate measures suitable for round-trip testing.""" + name = draw(_identifier) + # Use SUM/COUNT-like expressions that are standard SQL + col = draw(_standard_expr) + expr = draw(st.sampled_from([ + f"SUM({col})", + f"COUNT({col})", + f"AVG({col})", + f"MAX({col})", + f"MIN({col})", + ])) + comment = draw(st.none() | _safe_text) + display_name = draw(st.none() | _safe_text) + synonyms = draw(st.none() | st.lists(_safe_text, min_size=1, max_size=3)) + + return MetricViewMeasure( + name=name, + expr=expr, + comment=comment, + display_name=display_name, + synonyms=synonyms, + ) + + +@st.composite +def roundtrip_mv_joins(draw): + """Generate joins with parseable ON clauses for round-trip testing. + + The ON clause format must match what _build_on_clause produces: + source.from_col = join_name.to_col + """ + name = draw(_identifier) + source = draw(_three_part_name) + # Generate a parseable ON clause: source.col1 = name.col2 + from_col = draw(_identifier) + to_col = draw(_identifier) + on_clause = f"source.{from_col} = {name}.{to_col}" + + cardinality = draw(st.none() | st.sampled_from(["many_to_one", "one_to_many"])) + rely = draw(st.none() | st.just(MetricViewRely(at_most_one_match=True))) + + return MetricViewJoin( + name=name, + source=source, + on=on_clause, + cardinality=cardinality, + rely=rely, + ) + + +@st.composite +def roundtrip_mv_models(draw): + """Generate MetricViewModel instances designed for round-trip fidelity testing. + + Constraints: + - Uses three-part name sources (SQL queries change dataset naming) + - Uses parseable ON clause format in joins + - Avoids expressions that trigger time inference ambiguity + """ + source = draw(_three_part_name) + comment = draw(st.none() | _safe_text) + filter_expr = draw(st.none() | _standard_expr) + fields = draw(st.lists(roundtrip_mv_fields(), min_size=1, max_size=4)) + measures = draw(st.none() | st.lists(roundtrip_mv_measures(), min_size=1, max_size=3)) + joins = draw(st.none() | st.lists(roundtrip_mv_joins(), min_size=1, max_size=2)) + + return MetricViewModel( + version="1.1", + source=source, + comment=comment, + filter=filter_expr, + fields=fields, + measures=measures, + joins=joins, + ) + + +# --- Strategies for Property 8 (OSI→MV→OSI) --- + + +@st.composite +def roundtrip_osi_fields(draw): + """Generate OSI fields with DATABRICKS dialect for round-trip testing.""" + name = draw(_identifier) + expr_str = draw(_standard_expr) + description = draw(st.none() | _safe_text) + synonyms = draw(st.none() | st.lists(_safe_text, min_size=1, max_size=4)) + + ai_context = None + if synonyms: + ai_context = OSIAIContextObject(synonyms=tuple(synonyms)) + + return OSIField( + name=name, + expression=OSIExpression( + dialects=[ + OSIDialectExpression(dialect=OSIDialect.DATABRICKS, expression=expr_str), + ] + ), + dimension=OSIDimension(is_time=False), + description=description, + ai_context=ai_context, + ) + + +@st.composite +def roundtrip_osi_metrics(draw): + """Generate OSI metrics with DATABRICKS dialect for round-trip testing.""" + name = draw(_identifier) + col = draw(_standard_expr) + expr_str = draw(st.sampled_from([ + f"SUM({col})", + f"COUNT({col})", + f"AVG({col})", + f"MAX({col})", + f"MIN({col})", + ])) + description = draw(st.none() | _safe_text) + synonyms = draw(st.none() | st.lists(_safe_text, min_size=1, max_size=4)) + + ai_context = None + if synonyms: + ai_context = OSIAIContextObject(synonyms=tuple(synonyms)) + + return OSIMetric( + name=name, + expression=OSIExpression( + dialects=[ + OSIDialectExpression(dialect=OSIDialect.DATABRICKS, expression=expr_str), + ] + ), + description=description, + ai_context=ai_context, + ) + + +@st.composite +def roundtrip_osi_relationships(draw, dataset_name): + """Generate OSI relationships for round-trip testing.""" + name = draw(_identifier) + from_col = draw(_identifier) + to_col = draw(_identifier) + + return OSIRelationship( + name=name, + **{"from": dataset_name}, + to=name, # target dataset name = relationship name for simplicity + from_columns=[from_col], + to_columns=[to_col], + ) + + +@st.composite +def roundtrip_osi_documents(draw): + """Generate OSI documents suitable for OSI→MV→OSI round-trip testing. + + Constraints: + - Single semantic model with a single dataset + - All fields/metrics use DATABRICKS dialect + - Source is a three-part name + - Relationships reference the dataset correctly + """ + dataset_name = draw(_identifier) + source = f"catalog.schema.{dataset_name}" + fields = draw(st.lists(roundtrip_osi_fields(), min_size=1, max_size=4)) + metrics = draw(st.none() | st.lists(roundtrip_osi_metrics(), min_size=1, max_size=3)) + relationships = draw( + st.none() | st.lists(roundtrip_osi_relationships(dataset_name), min_size=1, max_size=2) + ) + description = draw(st.none() | _safe_text) + + dataset = OSIDataset( + name=dataset_name, + source=source, + fields=fields, + ) + semantic_model = OSISemanticModel( + name="test_model", + description=description, + datasets=[dataset], + relationships=relationships, + metrics=metrics, + ) + return OSIDocument( + version="0.2.0.dev0", + dialects=[OSIDialect.DATABRICKS], + vendors=[OSIVendor.DATABRICKS], + semantic_model=[semantic_model], + ) + + +# --- Strategies for Property 9 (Custom Extensions Round-Trip) --- + + +@st.composite +def roundtrip_osi_documents_with_multi_vendor_extensions(draw): + """Generate OSI documents with custom extensions for multiple vendors. + + Tests that DATABRICKS extensions are preserved through round-trip and + that other vendor extensions (SNOWFLAKE, DBT) survive the trip unharmed. + """ + dataset_name = draw(_identifier) + source = f"catalog.schema.{dataset_name}" + expr_str = draw(_standard_expr) + + # Generate DATABRICKS-specific extensions + filter_expr = draw(st.none() | _standard_expr) + ds_ext_data: dict = {} + if filter_expr: + ds_ext_data["filter"] = filter_expr + + dataset_extensions = [] + if ds_ext_data: + dataset_extensions.append( + OSICustomExtension( + vendor_name=OSIVendor.DATABRICKS, + data=json.dumps(ds_ext_data), + ) + ) + + # Add other vendor extensions that should survive + other_vendor = draw(st.sampled_from([OSIVendor.SNOWFLAKE, OSIVendor.DBT, OSIVendor.GOODDATA])) + other_ext_key = draw(_identifier) + other_ext_value = draw(_safe_text) + dataset_extensions.append( + OSICustomExtension( + vendor_name=other_vendor, + data=json.dumps({other_ext_key: other_ext_value}), + ) + ) + + # Field with DATABRICKS custom extension (format) + field_ext = OSICustomExtension( + vendor_name=OSIVendor.DATABRICKS, + data=json.dumps({"format": {"type": "number"}}), + ) + # Field with another vendor's extension + other_field_ext = OSICustomExtension( + vendor_name=other_vendor, + data=json.dumps({"other_field_meta": "preserved"}), + ) + + field = OSIField( + name=draw(_identifier), + expression=OSIExpression( + dialects=[OSIDialectExpression(dialect=OSIDialect.DATABRICKS, expression=expr_str)] + ), + dimension=OSIDimension(is_time=False), + custom_extensions=[field_ext, other_field_ext], + ) + + # Metric with DATABRICKS window extension + metric_name = draw(_identifier) + metric_ext = OSICustomExtension( + vendor_name=OSIVendor.DATABRICKS, + data=json.dumps({"window": [{"order": "d_date", "range": "trailing 7 day"}]}), + ) + metric = OSIMetric( + name=metric_name, + expression=OSIExpression( + dialects=[OSIDialectExpression(dialect=OSIDialect.DATABRICKS, expression=f"SUM({expr_str})")] + ), + custom_extensions=[metric_ext], + ) + + # Semantic model with materialization extension + mat_data = {"schedule": "every 6 hours", "mode": "relaxed"} + sm_ext = OSICustomExtension( + vendor_name=OSIVendor.DATABRICKS, + data=json.dumps({"materialization": mat_data}), + ) + + dataset = OSIDataset( + name=dataset_name, + source=source, + fields=[field], + custom_extensions=dataset_extensions if dataset_extensions else None, + ) + semantic_model = OSISemanticModel( + name="test_model", + datasets=[dataset], + metrics=[metric], + custom_extensions=[sm_ext], + ) + return OSIDocument( + version="0.2.0.dev0", + dialects=[OSIDialect.DATABRICKS], + vendors=[OSIVendor.DATABRICKS], + semantic_model=[semantic_model], + ) + + +# --- Property 7: MV→OSI→MV Round-Trip Preservation --- + + +class TestMVtoOSItoMVRoundTrip: + """Property 7: MV→OSI→MV round-trip preservation. + + For any valid Metric View YAML input, importing to OSI then exporting + back to Metric View YAML SHALL preserve: the source, all field names and + expressions, all measure names and expressions, all join names and sources, + the filter expression, and all synonyms/display_name metadata. + """ + + @given(model=roundtrip_mv_models()) + @settings(max_examples=100, suppress_health_check=[HealthCheck.too_slow]) + def test_source_preserved(self, model: MetricViewModel): + """Source (three-part name) is preserved through MV→OSI→MV.""" + osi_doc = metric_view_to_osi(model, model_name="rt_model") + results = osi_to_metric_view(osi_doc) + assert len(results) > 0 + _, mv_out = results[0] + assert mv_out.source == model.source + + @given(model=roundtrip_mv_models()) + @settings(max_examples=100, suppress_health_check=[HealthCheck.too_slow]) + def test_field_names_preserved(self, model: MetricViewModel): + """All field names are preserved through round-trip.""" + osi_doc = metric_view_to_osi(model, model_name="rt_model") + results = osi_to_metric_view(osi_doc) + assert len(results) > 0 + _, mv_out = results[0] + assert mv_out.fields is not None + original_names = [f.name for f in model.fields] + roundtrip_names = [f.name for f in mv_out.fields] + assert original_names == roundtrip_names + + @given(model=roundtrip_mv_models()) + @settings(max_examples=100, suppress_health_check=[HealthCheck.too_slow]) + def test_field_expressions_preserved(self, model: MetricViewModel): + """All field expressions are preserved through round-trip.""" + osi_doc = metric_view_to_osi(model, model_name="rt_model") + results = osi_to_metric_view(osi_doc) + assert len(results) > 0 + _, mv_out = results[0] + assert mv_out.fields is not None + original_exprs = [f.expr for f in model.fields] + roundtrip_exprs = [f.expr for f in mv_out.fields] + assert original_exprs == roundtrip_exprs + + @given(model=roundtrip_mv_models()) + @settings(max_examples=100, suppress_health_check=[HealthCheck.too_slow]) + def test_measure_names_preserved(self, model: MetricViewModel): + """All measure names are preserved through round-trip.""" + if model.measures is None: + return + osi_doc = metric_view_to_osi(model, model_name="rt_model") + results = osi_to_metric_view(osi_doc) + _, mv_out = results[0] + assert mv_out.measures is not None + original_names = [m.name for m in model.measures] + roundtrip_names = [m.name for m in mv_out.measures] + assert original_names == roundtrip_names + + @given(model=roundtrip_mv_models()) + @settings(max_examples=100, suppress_health_check=[HealthCheck.too_slow]) + def test_measure_expressions_preserved(self, model: MetricViewModel): + """All measure expressions are preserved through round-trip.""" + if model.measures is None: + return + osi_doc = metric_view_to_osi(model, model_name="rt_model") + results = osi_to_metric_view(osi_doc) + _, mv_out = results[0] + assert mv_out.measures is not None + original_exprs = [m.expr for m in model.measures] + roundtrip_exprs = [m.expr for m in mv_out.measures] + assert original_exprs == roundtrip_exprs + + @given(model=roundtrip_mv_models()) + @settings(max_examples=100, suppress_health_check=[HealthCheck.too_slow]) + def test_join_names_and_sources_preserved(self, model: MetricViewModel): + """All join names and sources are preserved through round-trip.""" + if model.joins is None: + return + osi_doc = metric_view_to_osi(model, model_name="rt_model") + results = osi_to_metric_view(osi_doc) + _, mv_out = results[0] + assert mv_out.joins is not None + original_names = [j.name for j in model.joins] + roundtrip_names = [j.name for j in mv_out.joins] + assert original_names == roundtrip_names + + @given(model=roundtrip_mv_models()) + @settings(max_examples=100, suppress_health_check=[HealthCheck.too_slow]) + def test_filter_preserved(self, model: MetricViewModel): + """Filter expression is preserved through round-trip.""" + osi_doc = metric_view_to_osi(model, model_name="rt_model") + results = osi_to_metric_view(osi_doc) + _, mv_out = results[0] + assert mv_out.filter == model.filter + + @given(model=roundtrip_mv_models()) + @settings(max_examples=100, suppress_health_check=[HealthCheck.too_slow]) + def test_synonyms_and_display_name_preserved(self, model: MetricViewModel): + """Synonyms and display_name metadata content is preserved through round-trip. + + The round-trip merges display_name and synonyms into a single ordered list + (ai_context.synonyms). On export, the first element becomes display_name + and the rest become synonyms. The total content is preserved, though the + split point may shift when display_name was originally None. + """ + osi_doc = metric_view_to_osi(model, model_name="rt_model") + results = osi_to_metric_view(osi_doc) + _, mv_out = results[0] + + for orig_field, rt_field in zip(model.fields, mv_out.fields): + # Reconstruct the combined synonym list as the importer sees it + original_all: list[str] = [] + if orig_field.display_name: + original_all.append(orig_field.display_name) + if orig_field.synonyms: + original_all.extend(orig_field.synonyms) + + # Reconstruct the combined list from the round-tripped field + roundtrip_all: list[str] = [] + if rt_field.display_name: + roundtrip_all.append(rt_field.display_name) + if rt_field.synonyms: + roundtrip_all.extend(rt_field.synonyms) + + assert original_all == roundtrip_all + + @given(model=roundtrip_mv_models()) + @settings(max_examples=100, suppress_health_check=[HealthCheck.too_slow]) + def test_comment_preserved(self, model: MetricViewModel): + """Top-level comment is preserved through round-trip (as semantic model description).""" + osi_doc = metric_view_to_osi(model, model_name="rt_model") + results = osi_to_metric_view(osi_doc) + _, mv_out = results[0] + assert mv_out.comment == model.comment + + +# --- Property 8: OSI→MV→OSI Round-Trip Preservation --- + + +class TestOSItoMVtoOSIRoundTrip: + """Property 8: OSI→MV→OSI round-trip preservation. + + For any valid OSI document containing datasets with DATABRICKS dialect + expressions, exporting to Metric View YAML then importing back to OSI + SHALL preserve: all dataset names, field names, field DATABRICKS expressions, + metric names, metric DATABRICKS expressions, relationship names, and + ai_context.synonyms. + """ + + @given(doc=roundtrip_osi_documents()) + @settings(max_examples=100, suppress_health_check=[HealthCheck.too_slow]) + def test_dataset_name_preserved(self, doc: OSIDocument): + """Dataset name is preserved through OSI→MV→OSI.""" + results = osi_to_metric_view(doc) + assert len(results) > 0 + dataset_name, mv_model = results[0] + + # Import back to OSI + osi_back = metric_view_to_osi(mv_model, model_name="rt_model") + ds_back = osi_back.semantic_model[0].datasets[0] + assert ds_back.name == dataset_name + + @given(doc=roundtrip_osi_documents()) + @settings(max_examples=100, suppress_health_check=[HealthCheck.too_slow]) + def test_field_names_preserved(self, doc: OSIDocument): + """Field names are preserved through OSI→MV→OSI.""" + original_ds = doc.semantic_model[0].datasets[0] + original_names = [f.name for f in original_ds.fields] + + results = osi_to_metric_view(doc) + _, mv_model = results[0] + osi_back = metric_view_to_osi(mv_model, model_name="rt_model") + ds_back = osi_back.semantic_model[0].datasets[0] + + roundtrip_names = [f.name for f in ds_back.fields] + assert original_names == roundtrip_names + + @given(doc=roundtrip_osi_documents()) + @settings(max_examples=100, suppress_health_check=[HealthCheck.too_slow]) + def test_field_databricks_expressions_preserved(self, doc: OSIDocument): + """Field DATABRICKS dialect expressions are preserved through round-trip.""" + original_ds = doc.semantic_model[0].datasets[0] + original_exprs = [] + for f in original_ds.fields: + for d in f.expression.dialects: + if d.dialect == OSIDialect.DATABRICKS: + original_exprs.append(d.expression) + + results = osi_to_metric_view(doc) + _, mv_model = results[0] + osi_back = metric_view_to_osi(mv_model, model_name="rt_model") + ds_back = osi_back.semantic_model[0].datasets[0] + + roundtrip_exprs = [] + for f in ds_back.fields: + for d in f.expression.dialects: + if d.dialect == OSIDialect.DATABRICKS: + roundtrip_exprs.append(d.expression) + + assert original_exprs == roundtrip_exprs + + @given(doc=roundtrip_osi_documents()) + @settings(max_examples=100, suppress_health_check=[HealthCheck.too_slow]) + def test_metric_names_preserved(self, doc: OSIDocument): + """Metric names are preserved through OSI→MV→OSI.""" + original_metrics = doc.semantic_model[0].metrics + if not original_metrics: + return + + original_names = [m.name for m in original_metrics] + + results = osi_to_metric_view(doc) + _, mv_model = results[0] + osi_back = metric_view_to_osi(mv_model, model_name="rt_model") + back_metrics = osi_back.semantic_model[0].metrics + + assert back_metrics is not None + roundtrip_names = [m.name for m in back_metrics] + assert original_names == roundtrip_names + + @given(doc=roundtrip_osi_documents()) + @settings(max_examples=100, suppress_health_check=[HealthCheck.too_slow]) + def test_metric_databricks_expressions_preserved(self, doc: OSIDocument): + """Metric DATABRICKS dialect expressions are preserved through round-trip.""" + original_metrics = doc.semantic_model[0].metrics + if not original_metrics: + return + + original_exprs = [] + for m in original_metrics: + for d in m.expression.dialects: + if d.dialect == OSIDialect.DATABRICKS: + original_exprs.append(d.expression) + + results = osi_to_metric_view(doc) + _, mv_model = results[0] + osi_back = metric_view_to_osi(mv_model, model_name="rt_model") + back_metrics = osi_back.semantic_model[0].metrics + + roundtrip_exprs = [] + for m in back_metrics: + for d in m.expression.dialects: + if d.dialect == OSIDialect.DATABRICKS: + roundtrip_exprs.append(d.expression) + + assert original_exprs == roundtrip_exprs + + @given(doc=roundtrip_osi_documents()) + @settings(max_examples=100, suppress_health_check=[HealthCheck.too_slow]) + def test_relationship_names_preserved(self, doc: OSIDocument): + """Relationship names are preserved through OSI→MV→OSI.""" + original_rels = doc.semantic_model[0].relationships + if not original_rels: + return + + original_names = [r.name for r in original_rels] + + results = osi_to_metric_view(doc) + _, mv_model = results[0] + osi_back = metric_view_to_osi(mv_model, model_name="rt_model") + back_rels = osi_back.semantic_model[0].relationships + + assert back_rels is not None + roundtrip_names = [r.name for r in back_rels] + assert original_names == roundtrip_names + + @given(doc=roundtrip_osi_documents()) + @settings(max_examples=100, suppress_health_check=[HealthCheck.too_slow]) + def test_ai_context_synonyms_preserved(self, doc: OSIDocument): + """ai_context.synonyms on fields are preserved through round-trip.""" + original_ds = doc.semantic_model[0].datasets[0] + + results = osi_to_metric_view(doc) + _, mv_model = results[0] + osi_back = metric_view_to_osi(mv_model, model_name="rt_model") + ds_back = osi_back.semantic_model[0].datasets[0] + + for orig_field, rt_field in zip(original_ds.fields, ds_back.fields): + if orig_field.ai_context and hasattr(orig_field.ai_context, "synonyms") and orig_field.ai_context.synonyms: + assert rt_field.ai_context is not None + assert rt_field.ai_context.synonyms == orig_field.ai_context.synonyms + else: + # No synonyms in → no synonyms out + if rt_field.ai_context: + assert rt_field.ai_context.synonyms is None or len(rt_field.ai_context.synonyms) == 0 + + +# --- Property 9: Custom Extensions Preserved During Round-Trip --- + + +class TestCustomExtensionsRoundTrip: + """Property 9: Custom extensions preserved during round-trip. + + For any OSI document containing custom_extensions for multiple vendors + (DATABRICKS, SNOWFLAKE, DBT, etc.), exporting to Metric View and then + importing back SHALL preserve all custom_extensions for all vendors. + + Note: The MV format only understands DATABRICKS extensions natively. + Other vendor extensions on the OSI document are NOT written into the MV YAML + (since MV has no concept of foreign vendor metadata). Therefore, full + round-trip preservation of non-DATABRICKS extensions is only achievable at + the OSI document level if the source document is retained. + + This test verifies that: + 1. DATABRICKS extensions (filter, materialization, format, window) survive + the OSI→MV→OSI round-trip + 2. The export step does not corrupt or discard DATABRICKS extensions + 3. The import step correctly reconstructs DATABRICKS extensions from MV data + """ + + @given(doc=roundtrip_osi_documents_with_multi_vendor_extensions()) + @settings(max_examples=100, suppress_health_check=[HealthCheck.too_slow]) + def test_databricks_filter_extension_preserved(self, doc: OSIDocument): + """DATABRICKS filter extension survives OSI→MV→OSI round-trip.""" + original_ds = doc.semantic_model[0].datasets[0] + + # Extract original DATABRICKS filter + original_filter = None + if original_ds.custom_extensions: + for ext in original_ds.custom_extensions: + if ext.vendor_name == OSIVendor.DATABRICKS: + data = json.loads(ext.data) + if "filter" in data: + original_filter = data["filter"] + + results = osi_to_metric_view(doc) + assert len(results) > 0 + _, mv_model = results[0] + + # Verify filter is in the MV model + if original_filter: + assert mv_model.filter == original_filter + + # Import back to OSI + osi_back = metric_view_to_osi(mv_model, model_name="rt_model") + ds_back = osi_back.semantic_model[0].datasets[0] + + # Check filter extension is restored + if original_filter: + roundtrip_filter = None + if ds_back.custom_extensions: + for ext in ds_back.custom_extensions: + if ext.vendor_name == OSIVendor.DATABRICKS: + data = json.loads(ext.data) + if "filter" in data: + roundtrip_filter = data["filter"] + assert roundtrip_filter == original_filter + + @given(doc=roundtrip_osi_documents_with_multi_vendor_extensions()) + @settings(max_examples=100, suppress_health_check=[HealthCheck.too_slow]) + def test_databricks_materialization_extension_preserved(self, doc: OSIDocument): + """DATABRICKS materialization extension survives OSI→MV→OSI round-trip.""" + original_sm = doc.semantic_model[0] + + # Extract original materialization + original_mat = None + if original_sm.custom_extensions: + for ext in original_sm.custom_extensions: + if ext.vendor_name == OSIVendor.DATABRICKS: + data = json.loads(ext.data) + if "materialization" in data: + original_mat = data["materialization"] + + results = osi_to_metric_view(doc) + _, mv_model = results[0] + + # Verify materialization is in the MV model + if original_mat: + assert mv_model.materialization is not None + assert mv_model.materialization.schedule == original_mat.get("schedule") + assert mv_model.materialization.mode == original_mat.get("mode") + + # Import back to OSI + osi_back = metric_view_to_osi(mv_model, model_name="rt_model") + sm_back = osi_back.semantic_model[0] + + # Check materialization extension is restored + if original_mat: + roundtrip_mat = None + if sm_back.custom_extensions: + for ext in sm_back.custom_extensions: + if ext.vendor_name == OSIVendor.DATABRICKS: + data = json.loads(ext.data) + if "materialization" in data: + roundtrip_mat = data["materialization"] + assert roundtrip_mat is not None + assert roundtrip_mat["schedule"] == original_mat["schedule"] + assert roundtrip_mat["mode"] == original_mat["mode"] + + @given(doc=roundtrip_osi_documents_with_multi_vendor_extensions()) + @settings(max_examples=100, suppress_health_check=[HealthCheck.too_slow]) + def test_databricks_format_extension_preserved(self, doc: OSIDocument): + """DATABRICKS format extension on fields survives OSI→MV→OSI round-trip.""" + original_ds = doc.semantic_model[0].datasets[0] + original_field = original_ds.fields[0] + + # Extract original DATABRICKS format + original_format = None + if original_field.custom_extensions: + for ext in original_field.custom_extensions: + if ext.vendor_name == OSIVendor.DATABRICKS: + data = json.loads(ext.data) + if "format" in data: + original_format = data["format"] + + results = osi_to_metric_view(doc) + _, mv_model = results[0] + + # Verify format is on the MV field + if original_format and mv_model.fields: + assert mv_model.fields[0].format is not None + assert mv_model.fields[0].format.type == original_format["type"] + + # Import back to OSI + osi_back = metric_view_to_osi(mv_model, model_name="rt_model") + ds_back = osi_back.semantic_model[0].datasets[0] + field_back = ds_back.fields[0] + + # Check format extension is restored + if original_format: + roundtrip_format = None + if field_back.custom_extensions: + for ext in field_back.custom_extensions: + if ext.vendor_name == OSIVendor.DATABRICKS: + data = json.loads(ext.data) + if "format" in data: + roundtrip_format = data["format"] + assert roundtrip_format is not None + assert roundtrip_format["type"] == original_format["type"] + + @given(doc=roundtrip_osi_documents_with_multi_vendor_extensions()) + @settings(max_examples=100, suppress_health_check=[HealthCheck.too_slow]) + def test_databricks_window_extension_preserved(self, doc: OSIDocument): + """DATABRICKS window extension on metrics survives OSI→MV→OSI round-trip.""" + original_metrics = doc.semantic_model[0].metrics + if not original_metrics: + return + + original_metric = original_metrics[0] + + # Extract original DATABRICKS window + original_window = None + if original_metric.custom_extensions: + for ext in original_metric.custom_extensions: + if ext.vendor_name == OSIVendor.DATABRICKS: + data = json.loads(ext.data) + if "window" in data: + original_window = data["window"] + + results = osi_to_metric_view(doc) + _, mv_model = results[0] + + # Verify window is on the MV measure + if original_window and mv_model.measures: + assert mv_model.measures[0].window is not None + assert mv_model.measures[0].window[0].order == original_window[0]["order"] + assert mv_model.measures[0].window[0].range == original_window[0]["range"] + + # Import back to OSI + osi_back = metric_view_to_osi(mv_model, model_name="rt_model") + back_metrics = osi_back.semantic_model[0].metrics + + # Check window extension is restored + if original_window and back_metrics: + roundtrip_window = None + if back_metrics[0].custom_extensions: + for ext in back_metrics[0].custom_extensions: + if ext.vendor_name == OSIVendor.DATABRICKS: + data = json.loads(ext.data) + if "window" in data: + roundtrip_window = data["window"] + assert roundtrip_window is not None + assert roundtrip_window[0]["order"] == original_window[0]["order"] + assert roundtrip_window[0]["range"] == original_window[0]["range"] + + @given(doc=roundtrip_osi_documents_with_multi_vendor_extensions()) + @settings(max_examples=100, suppress_health_check=[HealthCheck.too_slow]) + def test_export_does_not_corrupt_other_vendor_extensions(self, doc: OSIDocument): + """Exporting to MV does not corrupt the source OSI document's non-DATABRICKS extensions. + + The export function should not modify the input document. Other vendor + extensions (SNOWFLAKE, DBT, GOODDATA) on the source remain intact. + """ + original_ds = doc.semantic_model[0].datasets[0] + + # Collect non-DATABRICKS extensions before export + other_exts_before = [] + if original_ds.custom_extensions: + for ext in original_ds.custom_extensions: + if ext.vendor_name != OSIVendor.DATABRICKS: + other_exts_before.append((ext.vendor_name, ext.data)) + + # Perform export (should not mutate source) + _ = osi_to_metric_view(doc) + + # Verify non-DATABRICKS extensions are unchanged + other_exts_after = [] + if original_ds.custom_extensions: + for ext in original_ds.custom_extensions: + if ext.vendor_name != OSIVendor.DATABRICKS: + other_exts_after.append((ext.vendor_name, ext.data)) + + assert other_exts_before == other_exts_after