fix(temporal): deduplicate tracing and skip replay logs (#524)

Temporal workers registered tracing twice, and workflow replay emitted
duplicate business logs. Keep tracing on the client and add replay-aware
logging with workflow and run IDs.

Changes:
- Preserve inherited tracing and business-interceptor order.
- Suppress replay logs without changing startup or activity logging.
- Adopt the logger in SDK workflow sites and all Temporal templates.
- Cover tracing composition, replay, and logging fields with tests.
- Document adoption for existing agents.

Linear: AGX1-1113

Author: harvhan

adk/README.md

@@ -27,6 +27,26 @@ This automatically pulls in [`agentex-client`](../) (the slim Stainless-generate
 
 The two packages contribute disjoint files to the `agentex.*` namespace — `agentex/lib/*` ships only from `agentex-sdk`.
 
+## Workflow logging
+
+Use the workflow logger in Temporal workflow code:
+
+```python
+from agentex.lib.core.temporal.logging import make_workflow_logger
+
+logger = make_workflow_logger(__name__)
+```
+
+It suppresses logs while Temporal replays recorded history and adds top-level
+`workflow_id` and `run_id` fields during workflow execution. It preserves the
+message, caller fields, and exception details. Outside workflows, including in
+activities, it behaves like the ordinary SDK logger.
+
+New Temporal templates use this helper. Existing agents must replace their own
+workflow loggers to get the same behavior. This does not create trace context or
+add trace IDs to workflows that lack it. Temporal's worker diagnostics still report
+replay failures.
+
 ## Repo layout
 
 This package is hand-authored and lives at `adk/` inside [scaleapi/scale-agentex-python](https://github.com/scaleapi/scale-agentex-python). Stainless codegen never touches `adk/**` — it's outside the generated surface. The sibling `agentex-client` package lives at the repo root and IS Stainless-generated.

src/agentex/lib/cli/templates/temporal-claude-code/project/workflow.py.j2

@@ -27,7 +27,7 @@ from temporalio import workflow
 from agentex.lib import adk
 from agentex.lib.types.acp import SendEventParams, CreateTaskParams
 from agentex.lib.types.tracing import SGPTracingProcessorConfig
-from agentex.lib.utils.logging import make_logger
+from agentex.lib.core.temporal.logging import make_workflow_logger
 from agentex.types.text_content import TextContent
 from agentex.lib.environment_variables import EnvironmentVariables
 from agentex.lib.core.temporal.types.workflow import SignalName
@@ -52,7 +52,7 @@ if environment_variables.WORKFLOW_NAME is None:
 if environment_variables.AGENT_NAME is None:
     raise ValueError("Environment variable AGENT_NAME is not set")
 
-logger = make_logger(__name__)
+logger = make_workflow_logger(__name__)
 
 
 @workflow.defn(name=environment_variables.WORKFLOW_NAME)

src/agentex/lib/cli/templates/temporal-codex/project/workflow.py.j2

@@ -29,7 +29,7 @@ from temporalio import workflow
 from agentex.lib import adk
 from agentex.lib.types.acp import SendEventParams, CreateTaskParams
 from agentex.lib.types.tracing import SGPTracingProcessorConfig
-from agentex.lib.utils.logging import make_logger
+from agentex.lib.core.temporal.logging import make_workflow_logger
 from agentex.types.text_content import TextContent
 from agentex.lib.environment_variables import EnvironmentVariables
 from agentex.lib.core.temporal.types.workflow import SignalName
@@ -54,7 +54,7 @@ if environment_variables.WORKFLOW_NAME is None:
 if environment_variables.AGENT_NAME is None:
     raise ValueError("Environment variable AGENT_NAME is not set")
 
-logger = make_logger(__name__)
+logger = make_workflow_logger(__name__)
 
 MODEL = os.environ.get("CODEX_MODEL", "o4-mini")
 

src/agentex/lib/cli/templates/temporal-langgraph/project/workflow.py.j2

@@ -37,7 +37,7 @@ from project.graph import GRAPH_NAME, build_graph
 from agentex.lib.adk import emit_langgraph_messages
 from agentex.protocol.acp import SendEventParams, CreateTaskParams
 from agentex.lib.types.tracing import SGPTracingProcessorConfig
-from agentex.lib.utils.logging import make_logger
+from agentex.lib.core.temporal.logging import make_workflow_logger
 from agentex.types.text_content import TextContent
 from agentex.lib.environment_variables import EnvironmentVariables
 from agentex.lib.core.temporal.types.workflow import SignalName
@@ -64,7 +64,7 @@ if environment_variables.WORKFLOW_NAME is None:
 if environment_variables.AGENT_NAME is None:
     raise ValueError("Environment variable AGENT_NAME is not set")
 
-logger = make_logger(__name__)
+logger = make_workflow_logger(__name__)
 
 
 @workflow.defn(name=environment_variables.WORKFLOW_NAME)

src/agentex/lib/cli/templates/temporal-openai-agents/project/workflow.py.j2

@@ -7,7 +7,7 @@ from agentex.lib import adk
 from agentex.protocol.acp import CreateTaskParams, SendEventParams
 from agentex.lib.core.temporal.workflows.workflow import BaseWorkflow
 from agentex.lib.core.temporal.types.workflow import SignalName
-from agentex.lib.utils.logging import make_logger
+from agentex.lib.core.temporal.logging import make_workflow_logger
 from agentex.types.text_content import TextContent
 from agentex.lib.environment_variables import EnvironmentVariables
 from agents import Agent, Runner, set_trace_processors
@@ -44,7 +44,7 @@ if environment_variables.WORKFLOW_NAME is None:
 if environment_variables.AGENT_NAME is None:
     raise ValueError("Environment variable AGENT_NAME is not set")
 
-logger = make_logger(__name__)
+logger = make_workflow_logger(__name__)
 
 # Setup tracing for SGP (Scale GenAI Platform)
 # This enables visibility into your agent's execution in the SGP dashboard

src/agentex/lib/cli/templates/temporal-pydantic-ai/project/workflow.py.j2

@@ -25,7 +25,7 @@ from project.agent import TaskDeps, temporal_agent
 from agentex.lib import adk
 from agentex.protocol.acp import SendEventParams, CreateTaskParams
 from agentex.lib.types.tracing import SGPTracingProcessorConfig
-from agentex.lib.utils.logging import make_logger
+from agentex.lib.core.temporal.logging import make_workflow_logger
 from agentex.types.text_content import TextContent
 from agentex.lib.environment_variables import EnvironmentVariables
 from agentex.lib.core.temporal.types.workflow import SignalName
@@ -55,7 +55,7 @@ if environment_variables.WORKFLOW_NAME is None:
 if environment_variables.AGENT_NAME is None:
     raise ValueError("Environment variable AGENT_NAME is not set")
 
-logger = make_logger(__name__)
+logger = make_workflow_logger(__name__)
 
 
 @workflow.defn(name=environment_variables.WORKFLOW_NAME)

src/agentex/lib/cli/templates/temporal/project/workflow.py.j2

@@ -6,7 +6,7 @@ from agentex.lib import adk
 from agentex.protocol.acp import CreateTaskParams, SendEventParams
 from agentex.lib.core.temporal.workflows.workflow import BaseWorkflow
 from agentex.lib.core.temporal.types.workflow import SignalName
-from agentex.lib.utils.logging import make_logger
+from agentex.lib.core.temporal.logging import make_workflow_logger
 from agentex.types.text_content import TextContent
 from agentex.lib.environment_variables import EnvironmentVariables
 
@@ -18,7 +18,7 @@ if environment_variables.WORKFLOW_NAME is None:
 if environment_variables.AGENT_NAME is None:
     raise ValueError("Environment variable AGENT_NAME is not set")
 
-logger = make_logger(__name__)
+logger = make_workflow_logger(__name__)
 
 @workflow.defn(name=environment_variables.WORKFLOW_NAME)
 class {{ workflow_class }}(BaseWorkflow):

src/agentex/lib/core/temporal/logging.py

@@ -0,0 +1,34 @@
+from __future__ import annotations
+
+from typing import Any, override
+from collections.abc import MutableMapping
+
+from temporalio import workflow
+
+from agentex.lib.utils.logging import make_logger
+
+
+class WorkflowLoggerAdapter(workflow.LoggerAdapter):
+    """Skip workflow replay logs and add IDs without changing non-workflow logs."""
+
+    @override
+    def isEnabledFor(self, level: int) -> bool:
+        if not workflow.in_workflow():
+            return self.logger.isEnabledFor(level)
+        return super().isEnabledFor(level)
+
+    @override
+    def process(self, msg: Any, kwargs: MutableMapping[str, Any]) -> tuple[Any, MutableMapping[str, Any]]:
+        if workflow.in_workflow():
+            info = workflow.info()
+            kwargs["extra"] = {
+                "workflow_id": info.workflow_id,
+                "run_id": info.run_id,
+                **(kwargs.get("extra") or {}),
+            }
+        return msg, kwargs
+
+
+def make_workflow_logger(name: str) -> WorkflowLoggerAdapter:
+    """Create an SDK logger that suppresses replay and adds workflow/run IDs."""
+    return WorkflowLoggerAdapter(make_logger(name), {})

src/agentex/lib/core/temporal/plugins/openai_agents/interceptors/context_interceptor.py

@@ -22,8 +22,10 @@
 )
 from temporalio.converter import default
 
+from agentex.lib.core.temporal.logging import WorkflowLoggerAdapter
+
 # Set up logging
-logger = logging.getLogger("context.interceptor")
+logger = WorkflowLoggerAdapter(logging.getLogger("context.interceptor"), {})
 
 # Global context variables that models can read
 # These are thread-safe and work across async boundaries

src/agentex/lib/core/temporal/workers/worker.py

@@ -273,9 +273,8 @@ async def run(
             max_concurrent_activities=self.max_concurrent_activities,
             build_id=str(uuid.uuid4()),
             debug_mode=debug_enabled,  # Disable deadlock detection in debug mode
-            # Tracing interceptor OUTERMOST so business interceptors (and the spans
-            # they create) nest under the propagated workflow/activity span.
-            interceptors=[*temporal_tracing_interceptors(), *self.interceptors],
+            # Temporal inherits client tracing before these business interceptors.
+            interceptors=self.interceptors,
         )
 
         logger.info(f"Starting workers for task queue: {self.task_queue}")