BED-8194: Restore dashboard/notebook and add generic notebook support

.github/workflows/test.yml

@@ -38,7 +38,7 @@ jobs:
 
       - name: Install dependencies
         run: |
-          uv pip install --python .venv -e ".[all]" --group dev
+          uv pip install --python .venv -e ".[all,notebooks]" --group dev
 
       - name: Run log handling tests
         run: |
@@ -59,3 +59,7 @@ jobs:
       - name: Run BHE job scheduling test
         run: |
           .venv/bin/pytest tests/test_bhe_job_scheduling.py -v
+
+      - name: Run notebook execution test
+        run: |
+          .venv/bin/pytest tests/test_notebooks.py -v

.gitignore

@@ -5,8 +5,6 @@ site/
 addons/
 collectors/
 
-notebooks
-
 output
 graph
 logs

pyproject.toml

@@ -6,7 +6,7 @@ readme = "README.md"
 requires-python = ">=3.13"
 dependencies = [
     "alive-progress>=3.3.0",
-    "dlt==1.22.2",
+    "dlt==1.26.0",
     "duckdb==1.5.0",
     "griffe>=1.15.0",
     "griffe-fieldz>=0.5.0",
@@ -37,6 +37,13 @@ okta = [
     "openhound-okta==0.1.1",
 ]
 
+notebooks = [
+    "marimo~=0.23.4",
+    "altair>=6.0.0",
+    "polars>=1.40.1",
+    "pyarrow>=24.0.0",
+]
+
 [project.scripts]
 openhound = "openhound.main:app"
 
@@ -65,12 +72,9 @@ local_scheme = "no-local-version"
 
 [dependency-groups]
 dev = [
-    "openhound-faker==0.0.4",
-    "ipython>=9.12.0",
+    "openhound-faker==0.0.6",
     "pre-commit>=4.5.1",
     "pytest>=9.0.1",
-    "marimo>=0.23.0",
-    "altair>=6.0.0",
     "fastapi>=0.129.0",
     "zensical>=0.0.23",
     "ruff>=0.15.4",

src/openhound/cli/notebooks.py

@@ -0,0 +1,74 @@
+import secrets
+import string
+from pathlib import Path
+from typing import Annotated, Literal
+
+import typer
+
+BASE_PATH = Path(__file__).resolve().parents[1] / "notebooks"
+TOKEN_LENGTH = 32
+
+NOTEBOOKS = {
+    "pipeline": BASE_PATH / "pipeline.py",
+}
+
+notebooks_app = typer.Typer(help="Start OpenHound Marimo notebooks")
+
+
+def _generate_token(length: int = TOKEN_LENGTH) -> str:
+    alphabet = string.ascii_letters + string.digits
+    return "".join(secrets.choice(alphabet) for _ in range(length))
+
+
+@notebooks_app.command()
+def start(
+    notebook: Annotated[
+        Literal["pipeline"], typer.Argument(help="Notebook to start")
+    ] = "pipeline",
+    host: Annotated[
+        str,
+        typer.Option("--host", "-h", help="Host for the Marimo server"),
+    ] = "127.0.0.1",
+    port: Annotated[
+        int,
+        typer.Option("--port", "-p", help="Port for the Marimo server"),
+    ] = 2718,
+):
+    """Start one of the bundled OpenHound Marimo notebooks."""
+    from rich.console import Console
+
+    console = Console()
+    try:
+        from marimo._server.file_router import AppFileRouter
+        from marimo._server.start import start
+        from marimo._server.tokens import AuthToken
+        from marimo._session.model import SessionMode
+        from marimo._utils.marimo_path import MarimoPath
+
+    except ImportError:
+        console.print(
+            "[red]Error:[/red] Marimo is not installed. Install OpenHound with Marimo extras using openhound\\[notebooks] [red]"
+        )
+        raise typer.Exit(1)
+
+    notebook_path = NOTEBOOKS[notebook]
+    start(
+        file_router=AppFileRouter.from_filename(MarimoPath(str(notebook_path))),
+        mode=SessionMode.RUN,
+        development_mode=False,
+        quiet=False,
+        include_code=False,
+        ttl_seconds=120,
+        headless=False,
+        port=port,
+        host=host,
+        proxy=None,
+        watch=False,
+        cli_args={},
+        argv=[],
+        base_url="",
+        allow_origins=None,
+        auth_token=AuthToken(_generate_token()),
+        redirect_console_to_browser=False,
+        skew_protection=True,
+    )

src/openhound/core/app.py

@@ -62,6 +62,7 @@ def __init__(self, name: str, source_kind: str, help: str = "OpenGraph collector
         self.collector: Callable | None = None
         self.converter: Callable | None = None
         self.preprocessor: Callable | None = None
+        self.lookup_factory: Callable | None = None
 
         # Store DLT resources/transformers for this source to be used when building the DLT pipeline
         self.dlt_source: DltSource | None = None
@@ -147,6 +148,8 @@ def convert(
             progress (Literal["tqdm", "log", "alive_progress"], optional): Progress backend. Log is preferred for producteion use and alive_progress for interactive use.
         """
 
+        self.lookup_factory = lookup
+
         def decorator(func: Callable):
             def run_convert(
                 input_path: InputPath,

src/openhound/core/manager.py

@@ -137,7 +137,11 @@ def validate_metadata(extension: EntryPoint) -> tuple[bool, Extension | None]:
             return False, None
 
     @classmethod
-    def from_entrypoint(cls, group: str = "openhound.sources") -> "CollectorManager":
+    def from_entrypoint(
+        cls,
+        group: str = "openhound.sources",
+        load_sources: bool = False,
+    ) -> "CollectorManager":
         discover_extension = entry_points(group=group)
         extensions: list[OpenHound] = []
         for extension in discover_extension:
@@ -149,6 +153,9 @@ def from_entrypoint(cls, group: str = "openhound.sources") -> "CollectorManager"
                 )
 
             load_extension: OpenHound = extension.load()
+            if load_sources:
+                cls._load_extension_source(extension)
+
             is_valid_extension = cls.validate_extension(load_extension, extension.name)
             if is_valid_extension:
                 load_extension.metadata = metadata
@@ -163,3 +170,17 @@ def from_entrypoint(cls, group: str = "openhound.sources") -> "CollectorManager"
                     extra={"extension": extension.name, "phase": "extension_loading"},
                 )
         return cls(collectors=extensions)
+
+    @staticmethod
+    def _load_extension_source(extension: EntryPoint) -> None:
+        parent_module_name = extension.module.rsplit(".", 1)[0]
+        source_module_name = f"{parent_module_name}.source"
+        try:
+            import_module(source_module_name)
+        except ModuleNotFoundError as err:
+            if err.name != source_module_name:
+                raise
+            logger.warning(
+                f"Extension '{extension.name}' does not have a source module '{source_module_name}'",
+                extra={"extension": extension.name, "phase": "extension_loading"},
+            )

src/openhound/main.py

@@ -6,6 +6,7 @@
 from openhound.cli.collect import collect
 from openhound.cli.convert import convert
 from openhound.cli.create import create_app
+from openhound.cli.notebooks import notebooks_app
 from openhound.cli.override import TyperOverride
 from openhound.cli.preproc import preprocess
 from openhound.cli.privilege_zone import privilege_zone
@@ -22,6 +23,7 @@
 app.add_typer(create_app, name="create")
 app.add_typer(saved_searches, name="searches")
 app.add_typer(privilege_zone, name="rules")
+app.add_typer(notebooks_app, name="notebooks")
 
 if __name__ == "__main__":
     app()