fix(extract): preserve existing semantic layer on --code-only --force

graphify/cli.py

@@ -3129,6 +3129,16 @@ def _parse_float(name: str, raw: str) -> float:
         # --force: full scan, not the manifest-gated incremental diff — a warm
         # unchanged tree would otherwise dispatch zero files (#1894).
         incremental_mode = incremental_mode and not force
+        # #2923: --force --code-only must NOT drop the existing semantic layer.
+        # It still performs a full AST re-scan (so --force bypasses the
+        # manifest cache), but the semantic pass is itself skipped entirely, so
+        # the existing graph is merged instead of replaced.
+        preserve_semantic = force and code_only and existing_graph_path.exists()
+        if preserve_semantic:
+            print(
+                "[graphify extract] --force --code-only: full AST re-scan, "
+                "existing semantic layer preserved (no semantic pass this run)"
+            )
         if force:
             print("[graphify extract] --force: full re-scan, semantic cache reads skipped")
         elif incremental_mode and not manifest_path.exists():
@@ -3213,6 +3223,16 @@ def _parse_float(name: str, raw: str) -> float:
             excluded_files = []
             graph_stale_sources = []
             unchanged_total = 0
+            if preserve_semantic:
+                # A full scan re-extracts every code file, but doc/paper/image
+                # nodes are not re-dispatched, so the existing graph must still
+                # be merged. Compute stale sources (deleted/excluded files) so
+                # the merge can prune them while carrying the rest forward.
+                _seen_files = {f for _fl in files_by_type.values() for f in _fl}
+                _seen_files.update(detection.get("unclassified", []))
+                graph_stale_sources = _stale_graph_sources(
+                    existing_graph_path, target, _seen_files, detection=detection
+                )
 
         semantic_files = doc_files + paper_files + image_files
         # --code-only: index code (pure local AST, no key) and skip the semantic
@@ -3794,7 +3814,7 @@ def _invalidate_file_manifest_for_db_graph() -> None:
                 existing_graph_node_count as _existing_graph_node_count,
             )
             if (
-                incremental_mode
+                (incremental_mode or preserve_semantic)
                 and not code_files
                 and not semantic_files
                 and not deleted_files
@@ -3828,7 +3848,7 @@ def _invalidate_file_manifest_for_db_graph() -> None:
                 stages.total()
                 sys.exit(0)
 
-            if incremental_mode:
+            if incremental_mode or preserve_semantic:
                 # #2169: this raw path used to write ONLY this run's extraction
                 # over graph.json — on an incremental run that is just the
                 # changed files, silently dropping every node/edge owned by an
@@ -3956,7 +3976,7 @@ def _invalidate_file_manifest_for_db_graph() -> None:
         from graphify.export import to_json as _to_json
         from graphify.analyze import god_nodes as _god_nodes, surprising_connections as _surprising
         dedup_backend = backend if dedup_llm else None
-        if incremental_mode:
+        if incremental_mode or preserve_semantic:
             # Prune everything the current scan no longer covers: genuinely
             # deleted manifest rows, excluded-but-alive manifest rows (#1908),
             # and the graph's own stale sources — which catches files that

tests/test_extract_code_only_cli.py

@@ -258,3 +258,169 @@ def test_extract_names_skipped_sensitive_files(tmp_path):
     out = r.stdout + r.stderr
     assert "skipped as potentially sensitive" in out
     assert "github_token.txt" in out, "the skipped filename must be surfaced (#2106)"
+
+
+def test_code_only_force_preserves_existing_semantic_layer(tmp_path):
+    """#2923 regression: --code-only --force must not drop the existing semantic
+    layer. The AST pass is fully replaced (full re-scan, semantic cache reads
+    skipped) but the semantic pass is itself skipped, so doc/paper/image nodes
+    from graph.json must be carried forward. Before the fix this combination
+    silently rewrote graph.json with only the AST tier, losing every semantic
+    node and every hyperedge connected to one.
+    """
+    repo = _mixed_repo(tmp_path)
+    out = repo / "graphify-out"
+    out.mkdir()
+    graph = out / "graph.json"
+    # Seed a graph.json as if a prior full extract with an LLM backend had run:
+    # 2 AST nodes from app.py + 4 SEMANTIC nodes from README.md/NOTES.txt.
+    graph.write_text(json.dumps({
+        "nodes": [
+            {"id": "app_py", "label": "app.py", "type": "file",
+             "source_file": "app.py", "origin": "AST"},
+            {"id": "app_hello", "label": "hello()", "type": "function",
+             "source_file": "app.py", "origin": "AST"},
+            {"id": "readme_md", "label": "readme.md", "type": "file",
+             "source_file": "README.md", "origin": "SEMANTIC"},
+            {"id": "readme_design", "label": "Design", "type": "concept",
+             "source_file": "README.md", "origin": "SEMANTIC"},
+            {"id": "notes_txt", "label": "NOTES.txt", "type": "file",
+             "source_file": "NOTES.txt", "origin": "SEMANTIC"},
+            {"id": "notes_architecture", "label": "Architecture", "type": "concept",
+             "source_file": "NOTES.txt", "origin": "SEMANTIC"},
+        ],
+        "edges": [
+            {"id": "e1", "source": "app_py", "target": "app_hello",
+             "relation": "contains", "source_file": "app.py"},
+            {"id": "e2", "source": "readme_md", "target": "readme_design",
+             "relation": "concept_about", "source_file": "README.md"},
+            {"id": "e3", "source": "notes_txt", "target": "notes_architecture",
+             "relation": "concept_about", "source_file": "NOTES.txt"},
+        ],
+        "hyperedges": [],
+        "input_tokens": 0,
+        "output_tokens": 0,
+    }))
+
+    r = _run(repo, "--code-only", "--force", "--no-cluster")
+    assert r.returncode == 0, r.stderr
+
+    out_graph = json.loads(graph.read_text())
+    semantic_labels = {n["label"] for n in out_graph["nodes"]
+                       if n.get("origin") == "SEMANTIC"}
+    semantic_source_files = {
+        Path(str(n["source_file"])).name.lower()
+        for n in out_graph["nodes"]
+        if n.get("origin") == "SEMANTIC"
+    }
+    # Every seeded semantic node must survive. The AST pass may add new nodes
+    # (or relabel existing ones — e.g. hello vs hello()) but it must not
+    # silently drop the semantic tier.
+    assert {"readme.md", "notes.txt"}.issubset(semantic_source_files), (
+        "code-only --force erased the existing semantic layer (#2923); "
+        f"semantic nodes remaining: {semantic_labels}"
+    )
+    # Hyperedges are also semantic tier; the seeded graph had none but the
+    # AST re-extract must not have invented any non-semantic work, and the
+    # surviving edges list must not have been wholesale replaced.
+    assert "edges" in out_graph, "graph.json must still have an edges key"
+    # And the user-visible console line must explain why a semantic-layer-
+    # preserving branch fired.
+    assert "existing semantic layer preserved" in r.stdout + r.stderr, (
+        "the --force --code-only print must announce the semantic-preserving branch"
+    )
+
+
+def test_code_only_force_prunes_removed_semantic_files(tmp_path):
+    """#2923 follow-up: --code-only --force preserves surviving semantic nodes
+    but must still prune semantic nodes for files that have been removed from
+    disk (the doc/paper/image tier cannot outlive the corpus it indexes).
+    """
+    repo = _mixed_repo(tmp_path)
+    out = repo / "graphify-out"
+    out.mkdir()
+    graph = out / "graph.json"
+    graph.write_text(json.dumps({
+        "nodes": [
+            {"id": "app_py", "label": "app.py", "type": "file",
+             "source_file": "app.py", "origin": "AST"},
+            {"id": "app_hello", "label": "hello()", "type": "function",
+             "source_file": "app.py", "origin": "AST"},
+            {"id": "notes_txt", "label": "NOTES.txt", "type": "file",
+             "source_file": "NOTES.txt", "origin": "SEMANTIC"},
+            {"id": "notes_architecture", "label": "Architecture", "type": "concept",
+             "source_file": "NOTES.txt", "origin": "SEMANTIC"},
+        ],
+        "edges": [],
+        "hyperedges": [],
+        "input_tokens": 0,
+        "output_tokens": 0,
+    }))
+
+    # Delete NOTES.txt between seed and re-run. The merge's graph_stale_sources
+    # path must drop its semantic nodes because the file no longer exists.
+    (repo / "NOTES.txt").unlink()
+
+    r = _run(repo, "--code-only", "--force", "--no-cluster")
+    assert r.returncode == 0, r.stderr
+    out_graph = json.loads(graph.read_text())
+    remaining_sources = {
+        Path(n["source_file"]).name
+        for n in out_graph["nodes"]
+        if n.get("origin") == "SEMANTIC"
+    }
+    assert "NOTES.txt" not in remaining_sources, (
+        "NOTES.txt was deleted from disk; its semantic nodes must be pruned "
+        "(#2923 follow-up)"
+    )
+
+
+def test_code_only_force_rescans_unchanged_code_with_manifest_and_preserves_seeded_semantic_nodes(tmp_path):
+    """#2923: --force --code-only must perform a full AST re-scan even when the
+    manifest reports no code changes. Without this, a warm unchanged tree causes
+    the old fix to dispatch zero code files, so the AST tier is not rebuilt and
+    the existing graph is only merged unchanged.
+    """
+    repo = _mixed_repo(tmp_path)
+    out = repo / "graphify-out"
+    out.mkdir()
+
+    # Initial code-only extract writes a manifest and AST-only graph.
+    r1 = _run(repo, "--code-only", "--no-cluster")
+    assert r1.returncode == 0, r1.stderr
+    graph_path = out / "graph.json"
+    manifest_path = out / "manifest.json"
+    assert manifest_path.exists(), "code-only run must write a manifest"
+    g = json.loads(graph_path.read_text())
+    assert any(n.get("label") == "hello()" for n in g["nodes"])
+
+    # Seed a semantic layer as if a prior full extract had produced it, then
+    # remove an unchanged AST node to verify the full re-scan restores it.
+    g["nodes"].extend([
+        {"id": "readme_md", "label": "README.md", "type": "file",
+         "source_file": "README.md", "origin": "SEMANTIC"},
+        {"id": "readme_design", "label": "Design", "type": "concept",
+         "source_file": "README.md", "origin": "SEMANTIC"},
+        {"id": "notes_txt", "label": "NOTES.txt", "type": "file",
+         "source_file": "NOTES.txt", "origin": "SEMANTIC"},
+        {"id": "notes_architecture", "label": "Architecture", "type": "concept",
+         "source_file": "NOTES.txt", "origin": "SEMANTIC"},
+    ])
+    g["nodes"] = [n for n in g["nodes"] if n.get("label") != "hello()"]
+    graph_path.write_text(json.dumps(g))
+
+    r2 = _run(repo, "--code-only", "--force", "--no-cluster")
+    assert r2.returncode == 0, r2.stderr
+    out_graph = json.loads(graph_path.read_text())
+    assert any(n.get("label") == "hello()" for n in out_graph["nodes"]), (
+        "--force --code-only must re-extract unchanged code and restore the AST node"
+    )
+    semantic_labels = {n["label"] for n in out_graph["nodes"]
+                       if n.get("origin") == "SEMANTIC"}
+    assert semantic_labels >= {"README.md", "Design", "NOTES.txt", "Architecture"}, (
+        "seeded semantic layer must survive the full AST re-scan: "
+        f"{semantic_labels}"
+    )
+    assert "existing semantic layer preserved" in r2.stdout + r2.stderr, (
+        "the --force --code-only print must announce the semantic-preserving branch"
+    )