feat: add DUCKDB dialect and bidirectional DuckDB converter

converters/duckdb/README.md

@@ -0,0 +1,98 @@
+<!--
+  Licensed to the Apache Software Foundation (ASF) under one
+  or more contributor license agreements.  See the NOTICE file
+  distributed with this work for additional information
+  regarding copyright ownership.  The ASF licenses this file
+  to you under the Apache License, Version 2.0 (the
+  "License"); you may not use this file except in compliance
+  with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing,
+  software distributed under the License is distributed on an
+  "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+  KIND, either express or implied.  See the License for the
+  specific language governing permissions and limitations
+  under the License.
+-->
+
+# Apache Ossie ↔ DuckDB Converter
+
+Bidirectional converter between DuckDB databases and the Apache Ossie
+semantic model.
+
+- **Export** (Ossie → DuckDB): renders a semantic model as a DuckDB SQL
+  script — one view per dataset, one `metric_<name>` view per metric (with
+  joins derived from the declared relationships), and `COMMENT ON`
+  statements carrying the descriptions into the DuckDB catalog.
+- **Import** (DuckDB → Ossie): reads a database's tables, views, columns,
+  constraints, and comments and emits a valid Ossie YAML document.
+
+## Usage
+
+```bash
+# Ossie model -> DuckDB SQL script
+ossie-duckdb export -i model.yaml -o model.sql
+ossie-duckdb export -i model.yaml --view-schema semantic   # views in their own schema
+
+# DuckDB database -> Ossie model
+ossie-duckdb import -i analytics.duckdb -o model.yaml
+ossie-duckdb import -i md:my_database --schema gold --name gold_model
+```
+
+Or from Python:
+
+```python
+from ossie_duckdb import convert_osi_to_duckdb, convert_duckdb_to_osi_yaml
+
+sql = convert_osi_to_duckdb(open("model.yaml").read(), view_schema="semantic")
+
+import duckdb
+conn = duckdb.connect("analytics.duckdb")
+osi_yaml = convert_duckdb_to_osi_yaml(conn, schema="main")
+```
+
+## Mapping
+
+| Ossie construct | DuckDB construct |
+|-----------------|------------------|
+| Dataset | `CREATE OR REPLACE VIEW <name>` over the dataset `source` |
+| Field | View column (`<expression> AS <name>`) |
+| Field / dataset / metric `description` | `COMMENT ON COLUMN` / `COMMENT ON VIEW` |
+| Metric | `CREATE OR REPLACE VIEW metric_<name>`, joins derived from relationships |
+| Relationship | `LEFT JOIN` in metric views (export); `FOREIGN KEY` constraint (import) |
+| `primary_key` / `unique_keys` | `PRIMARY KEY` / `UNIQUE` constraints (import) |
+
+Expression dialect selection on export prefers `DUCKDB`, falls back to
+`ANSI_SQL` with a warning, and fails when neither is present.
+
+## MotherDuck
+
+The import side accepts anything `duckdb.connect` accepts, including
+MotherDuck `md:` connection strings and already-open connections, so
+hosted databases work the same way local files do.
+
+## Limitations
+
+- Metric dataset references are detected as unquoted `dataset.field`
+  identifiers; quoted dataset names in metric expressions are not resolved.
+- Metric joins require the referenced datasets to be connected through the
+  model's declared relationships, and the join columns must be exposed as
+  fields on datasets that declare an explicit field list.
+- Join direction is `LEFT JOIN` from the many (`from`) side; Ossie does not
+  yet declare cardinality or join type, so other join semantics are not
+  representable.
+- `ai_context` and `custom_extensions` have no DuckDB catalog equivalent and
+  are not carried into the SQL output.
+- Import emits datasets, fields, keys, and relationships; it does not infer
+  metrics.
+
+## Development
+
+```bash
+cd converters/duckdb
+uv sync
+uv run pytest
+uv run ruff check
+```

converters/duckdb/pyproject.toml

@@ -0,0 +1,69 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+[project]
+name = "apache-ossie-duckdb"
+version = "0.2.0.dev0"
+description = "Bidirectional converter between DuckDB databases and the Apache Ossie semantic model"
+authors = [{name = "Michelle Pellon", email = "mgracepellon+github@gmail.com"}]
+readme = "README.md"
+requires-python = ">=3.12"
+classifiers = [
+    "License :: OSI Approved :: Apache Software License",
+    "Programming Language :: Python :: 3",
+]
+dependencies = [
+    "pyyaml>=6.0.1",
+    "duckdb>=1.1.0",
+]
+
+[project.license]
+text = "Apache-2.0"
+
+[project.scripts]
+ossie-duckdb = "ossie_duckdb.cli:main"
+
+[dependency-groups]
+dev = [
+    "ruff>=0.11.5",
+    "pytest>=8.3.5",
+    "pytest-cov>=6.0.0",
+    "jsonschema>=4.21.0",
+]
+
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[tool.hatch.build]
+include = ["src/ossie_duckdb/**/*"]
+ignore-vcs = true
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/ossie_duckdb"]
+
+[tool.ruff]
+line-length = 120
+target-version = "py312"
+
+[tool.ruff.lint]
+select = ["E", "F", "I", "N", "UP", "W", "C4", "PIE", "SIM"]
+ignore = ["E501"]
+
+[tool.pytest.ini_options]
+testpaths = ["tests"]
+pythonpath = ["src"]

converters/duckdb/src/ossie_duckdb/__init__.py

@@ -0,0 +1,29 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""Bidirectional converter between DuckDB databases and the Ossie semantic model."""
+
+from ossie_duckdb._common import ConversionError
+from ossie_duckdb.duckdb_to_osi import convert_duckdb_to_osi, convert_duckdb_to_osi_yaml
+from ossie_duckdb.osi_to_duckdb import convert_osi_to_duckdb
+
+__all__ = [
+    "ConversionError",
+    "convert_duckdb_to_osi",
+    "convert_duckdb_to_osi_yaml",
+    "convert_osi_to_duckdb",
+]

converters/duckdb/src/ossie_duckdb/_common.py

@@ -0,0 +1,126 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""Shared helpers for the DuckDB <-> Ossie converter."""
+
+import re
+import warnings
+
+SPEC_VERSION = "0.2.0.dev0"
+
+# Dialects the export accepts, in preference order.
+PREFERRED_DIALECTS = ("DUCKDB", "ANSI_SQL")
+
+_SIMPLE_IDENTIFIER = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$")
+
+# Keywords that must be quoted even though they look like simple identifiers.
+# Deliberately small: DuckDB accepts most keywords as identifiers unquoted.
+_RESERVED = {
+    "all",
+    "and",
+    "any",
+    "as",
+    "asc",
+    "between",
+    "by",
+    "case",
+    "cast",
+    "create",
+    "cross",
+    "current_date",
+    "current_time",
+    "current_timestamp",
+    "default",
+    "desc",
+    "distinct",
+    "drop",
+    "else",
+    "end",
+    "except",
+    "exists",
+    "false",
+    "from",
+    "full",
+    "group",
+    "having",
+    "in",
+    "inner",
+    "intersect",
+    "into",
+    "is",
+    "join",
+    "left",
+    "like",
+    "limit",
+    "natural",
+    "not",
+    "null",
+    "offset",
+    "on",
+    "or",
+    "order",
+    "outer",
+    "right",
+    "select",
+    "table",
+    "then",
+    "true",
+    "union",
+    "unique",
+    "using",
+    "view",
+    "when",
+    "where",
+    "with",
+}
+
+
+class ConversionError(Exception):
+    """Raised when a model cannot be converted."""
+
+
+def quote_identifier(name: str) -> str:
+    """Quote an identifier only when necessary (non-simple or reserved)."""
+    if _SIMPLE_IDENTIFIER.match(name) and name.lower() not in _RESERVED:
+        return name
+    escaped = name.replace('"', '""')
+    return f'"{escaped}"'
+
+
+def sql_string_literal(text: str) -> str:
+    """Render text as a single-quoted SQL string literal."""
+    return "'" + text.replace("'", "''") + "'"
+
+
+def select_expression(expression: dict | None, context: str) -> str:
+    """Pick the best expression for DuckDB from a multi-dialect expression object.
+
+    Prefers the DUCKDB dialect, falls back to ANSI_SQL with a warning, and
+    raises ConversionError when neither is available.
+    """
+    dialects = (expression or {}).get("dialects") or []
+    by_dialect = {d.get("dialect"): d.get("expression") for d in dialects if d.get("expression")}
+    for dialect in PREFERRED_DIALECTS:
+        if dialect in by_dialect:
+            if dialect != "DUCKDB":
+                warnings.warn(
+                    f"{context}: no DUCKDB dialect expression; falling back to {dialect}",
+                    stacklevel=2,
+                )
+            return by_dialect[dialect]
+    available = ", ".join(sorted(by_dialect)) or "none"
+    raise ConversionError(f"{context}: no DUCKDB or ANSI_SQL expression available (has: {available})")