From 00cdab5bdfc9008da5c32b977509f8050082bb92 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 29 Jul 2026 19:46:15 +0000 Subject: [PATCH 1/5] Persist tool interactions and cache tool results Conversations persisted through HasContext only recorded the final assistant message; the tool/MCP roundtrips in the response's message stack were dropped, so downstream views (like the ActiveAgents Interactions dashboard) couldn't show what an agent actually did. - HasContext now persists each tool-result message to the context via add_tool_message (already part of the install generator's AgentContext), deduped by tool_call_id so shared message stacks in multi-turn conversations don't duplicate rows. Contexts without add_tool_message are skipped, keeping the change drop-in for existing apps. - New SolidAgent::ToolCache caches tool/MCP/service results by (tool, normalized args) with a TTL, backed by Rails.cache out of the box and by any read/write store elsewhere. Error-shaped results ({ error: ... }) are never cached, and replayed hashes are tagged cached: true so callers can tell a replay from a fresh call. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01B5PaDq3uaUWuA8UW4K7siP --- lib/solid_agent.rb | 1 + lib/solid_agent/has_context.rb | 37 ++++++++++++ lib/solid_agent/tool_cache.rb | 91 +++++++++++++++++++++++++++++ test/solid_agent/tool_cache_test.rb | 87 +++++++++++++++++++++++++++ 4 files changed, 216 insertions(+) create mode 100644 lib/solid_agent/tool_cache.rb create mode 100644 test/solid_agent/tool_cache_test.rb diff --git a/lib/solid_agent.rb b/lib/solid_agent.rb index 6227845..287a5aa 100644 --- a/lib/solid_agent.rb +++ b/lib/solid_agent.rb @@ -1,6 +1,7 @@ # frozen_string_literal: true require_relative "solid_agent/version" +require_relative "solid_agent/tool_cache" module SolidAgent class Error < StandardError; end diff --git a/lib/solid_agent/has_context.rb b/lib/solid_agent/has_context.rb index 881a378..f036150 100644 --- a/lib/solid_agent/has_context.rb +++ b/lib/solid_agent/has_context.rb @@ -527,6 +527,8 @@ def capture_and_persist_generation def persist_generation_to_context return unless context && generation_response + persist_tool_messages_to_context + begin if generation_response.respond_to?(:message) && generation_response.message&.content.present? # Include provenance if the context supports it @@ -544,5 +546,40 @@ def persist_generation_to_context Rails.logger.error e.backtrace.first(5).join("\n") end end + + # Persists the tool/MCP interaction stream (tool result messages from + # the response's message stack) to the context, so conversations show + # the full agent <-> tool exchange, not just the final assistant text. + # + # Requires the context model to expose add_tool_message (the install + # generator's AgentContext does); contexts without it are skipped. + # Messages are deduped by tool_call_id so re-persisting a shared + # message stack (multi-turn conversations) doesn't duplicate rows. + def persist_tool_messages_to_context + return unless context.respond_to?(:add_tool_message) + return unless generation_response.respond_to?(:messages) + + Array(generation_response.messages).each do |message| + next unless message.respond_to?(:role) && message.role.to_s == "tool" + + tool_call_id = message.respond_to?(:tool_call_id) ? message.tool_call_id : nil + next if tool_call_id.present? && tool_message_persisted?(tool_call_id) + + context.add_tool_message( + tool_call_id: tool_call_id, + tool_name: (message.name if message.respond_to?(:name)), + result: (message.content if message.respond_to?(:content)) + ) + end + rescue => e + Rails.logger.error "[SolidAgent] Failed to persist tool messages: #{e.message}" + end + + def tool_message_persisted?(tool_call_id) + return false unless context.respond_to?(:messages) + + scope = context.messages + scope.respond_to?(:exists?) && scope.exists?(role: "tool", tool_call_id: tool_call_id) + end end end diff --git a/lib/solid_agent/tool_cache.rb b/lib/solid_agent/tool_cache.rb new file mode 100644 index 0000000..62fc3f4 --- /dev/null +++ b/lib/solid_agent/tool_cache.rb @@ -0,0 +1,91 @@ +# frozen_string_literal: true + +require "digest" +require "json" + +# Caches the results of tool / MCP / service interactions so repeated calls +# with the same arguments reuse a persisted result instead of re-running the +# side effect (HTTP fetch, search, remote MCP call, ...). +# +# Works out of the box in Rails apps (backed by Rails.cache); any object +# responding to read/write can be substituted, so tests and non-Rails +# runtimes can inject their own store. +# +# @example Caching a tool implementation +# def fetch_url(url:) +# SolidAgent::ToolCache.fetch(tool: "fetch_url", args: { url: url }) do +# Net::HTTP.get_response(URI(url)).body +# end +# end +# +# @example Disabling globally (e.g. in tests) +# SolidAgent::ToolCache.enabled = false +module SolidAgent + module ToolCache + DEFAULT_TTL = 300 # seconds + + class << self + attr_writer :enabled, :default_ttl, :store + + def enabled + defined?(@enabled) ? @enabled : true + end + + def default_ttl + @default_ttl || DEFAULT_TTL + end + + def store + @store || (defined?(Rails) && Rails.respond_to?(:cache) ? Rails.cache : nil) + end + + # Returns the cached result for (tool, args) or yields, caching the + # fresh result. Results that look like errors ({ error: ... }) are + # never cached, so transient failures don't stick. + # + # The returned hash is tagged with cached: true on cache hits so + # callers (and the model) can tell a replayed result from a fresh one. + def fetch(tool:, args: {}, ttl: nil, cache: store) + return yield unless enabled && cache + + key = cache_key(tool, args) + cached = cache.read(key) + return tag_cached(cached) unless cached.nil? + + result = yield + cache.write(key, result, expires_in: ttl || default_ttl) if cacheable?(result) + result + end + + def cache_key(tool, args) + digest = Digest::SHA256.hexdigest(normalize_args(args).to_json) + "solid_agent:tool_cache:#{tool}:#{digest}" + end + + private + + def cacheable?(result) + return false if result.nil? + return !(result.key?(:error) || result.key?("error")) if result.respond_to?(:key?) + + true + end + + def tag_cached(result) + result.respond_to?(:merge) ? result.merge(cached: true) : result + end + + # Stable key material regardless of hash ordering or string/symbol keys. + def normalize_args(args) + case args + when Hash + args.map { |k, v| [ k.to_s, normalize_args(v) ] }.sort_by(&:first) + when Array + args.map { |v| normalize_args(v) } + else + args + end + end + end + end +end diff --git a/test/solid_agent/tool_cache_test.rb b/test/solid_agent/tool_cache_test.rb new file mode 100644 index 0000000..a434925 --- /dev/null +++ b/test/solid_agent/tool_cache_test.rb @@ -0,0 +1,87 @@ +# frozen_string_literal: true + +require "test_helper" +require "solid_agent/tool_cache" + +class SolidAgent::ToolCacheTest < Minitest::Test + # Minimal store with the read/write contract ToolCache needs. + class MemoryStore + attr_reader :writes + + def initialize + @data = {} + @writes = [] + end + + def read(key) + @data[key] + end + + def write(key, value, **options) + @writes << [ key, value, options ] + @data[key] = value + end + end + + def setup + @store = MemoryStore.new + end + + def test_caches_and_replays_results + calls = 0 + 2.times do + @result = SolidAgent::ToolCache.fetch(tool: "web_search", args: { query: "rails" }, cache: @store) do + calls += 1 + { answer: 42 } + end + end + + assert_equal 1, calls + assert_equal 42, @result[:answer] + assert_equal true, @result[:cached], "replayed results should be tagged cached: true" + end + + def test_key_is_stable_across_argument_ordering_and_key_types + key_a = SolidAgent::ToolCache.cache_key("t", { a: 1, b: 2 }) + key_b = SolidAgent::ToolCache.cache_key("t", { "b" => 2, "a" => 1 }) + key_c = SolidAgent::ToolCache.cache_key("t", { a: 1, b: 3 }) + + assert_equal key_a, key_b + refute_equal key_a, key_c + end + + def test_does_not_cache_error_results + 2.times do + SolidAgent::ToolCache.fetch(tool: "fetch_url", args: { url: "x" }, cache: @store) do + { error: "boom" } + end + end + + assert_empty @store.writes + end + + def test_respects_ttl_option + SolidAgent::ToolCache.fetch(tool: "t", args: {}, ttl: 60, cache: @store) { { ok: true } } + + assert_equal 60, @store.writes.first.last[:expires_in] + end + + def test_disabled_bypasses_cache + SolidAgent::ToolCache.enabled = false + calls = 0 + 2.times do + SolidAgent::ToolCache.fetch(tool: "t", args: {}, cache: @store) { calls += 1; { ok: true } } + end + + assert_equal 2, calls + assert_empty @store.writes + ensure + SolidAgent::ToolCache.enabled = true + end + + def test_yields_without_a_store + result = SolidAgent::ToolCache.fetch(tool: "t", args: {}, cache: nil) { { ok: true } } + + assert_equal({ ok: true }, result) + end +end From 7d241a31b76c6c26eb629b1e5d7f87c8dd365b68 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 29 Jul 2026 20:36:40 +0000 Subject: [PATCH 2/5] Add HasMemory: agent-curated memory with handoff support MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bakes a memory concern into solid_agent: agents persist a summary list they decide when to read/write while interacting with tools, other agents, and users. - SolidAgent::HasMemory exposes save_memory/recall_memory as function-calling tool definitions plus matching instance methods, so a provider's tool calls route straight to persistent memory. The memory subject defaults to params[:memorable], falling back to the HasContext contextable. - Memory is scoped to (memorable subject, scope) rather than the agent class, so any agent operating on the same subject shares it — a handoff channel between agents, with source_agent provenance on every entry and AgentMemory#to_prompt for instruction injection. - The install generator now creates agent_memories / agent_memory_entries tables and AgentMemory/AgentMemoryEntry models implementing the duck-typed contract, so existing Rails apps get memory out of the box alongside contexts. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01B5PaDq3uaUWuA8UW4K7siP --- .../solid_agent/install/install_generator.rb | 5 + .../install/templates/agent_memory.rb.erb | 51 +++++++ .../templates/agent_memory_entry.rb.erb | 12 ++ .../templates/create_agent_memories.rb.erb | 35 +++++ lib/solid_agent.rb | 1 + lib/solid_agent/has_memory.rb | 136 ++++++++++++++++++ test/solid_agent/has_memory_test.rb | 109 ++++++++++++++ 7 files changed, 349 insertions(+) create mode 100644 lib/generators/solid_agent/install/templates/agent_memory.rb.erb create mode 100644 lib/generators/solid_agent/install/templates/agent_memory_entry.rb.erb create mode 100644 lib/generators/solid_agent/install/templates/create_agent_memories.rb.erb create mode 100644 lib/solid_agent/has_memory.rb create mode 100644 test/solid_agent/has_memory_test.rb diff --git a/lib/generators/solid_agent/install/install_generator.rb b/lib/generators/solid_agent/install/install_generator.rb index 74f1ecb..31e7381 100644 --- a/lib/generators/solid_agent/install/install_generator.rb +++ b/lib/generators/solid_agent/install/install_generator.rb @@ -29,6 +29,9 @@ def create_migrations migration_template "create_agent_generations.rb.erb", "db/migrate/create_agent_generations.rb" + + migration_template "create_agent_memories.rb.erb", + "db/migrate/create_agent_memories.rb" end def create_models @@ -37,6 +40,8 @@ def create_models template "agent_context.rb.erb", "app/models/agent_context.rb" template "agent_message.rb.erb", "app/models/agent_message.rb" template "agent_generation.rb.erb", "app/models/agent_generation.rb" + template "agent_memory.rb.erb", "app/models/agent_memory.rb" + template "agent_memory_entry.rb.erb", "app/models/agent_memory_entry.rb" end def create_initializer diff --git a/lib/generators/solid_agent/install/templates/agent_memory.rb.erb b/lib/generators/solid_agent/install/templates/agent_memory.rb.erb new file mode 100644 index 0000000..196ec77 --- /dev/null +++ b/lib/generators/solid_agent/install/templates/agent_memory.rb.erb @@ -0,0 +1,51 @@ +# frozen_string_literal: true + +# Agent-curated long-term memory for a subject record, written and read by +# agents through SolidAgent::HasMemory's save_memory/recall_memory tools. +# +# Memory is scoped to (memorable, scope) — not to an agent class — so any +# agent operating on the same subject shares it, making it a handoff +# channel between agents. Entry source_agent records who wrote each note. +class AgentMemory < ApplicationRecord + belongs_to :memorable, polymorphic: true, optional: true + has_many :entries, class_name: "AgentMemoryEntry", dependent: :destroy + + validates :scope, presence: true + + # Finds or creates the memory for a subject. + def self.for(memorable, scope: SolidAgent::HasMemory::DEFAULT_SCOPE) + find_or_create_by!(memorable: memorable, scope: scope.to_s) + end + + # Appends a summary note. + def remember(content, source_agent: nil, category: nil) + entries.create!(content: content, source_agent: source_agent, category: category) + end + + # Most recent notes first. + def recall(limit: 20, category: nil) + scope = entries.order(created_at: :desc) + scope = scope.where(category: category) if category.present? + scope.limit(limit || 20).to_a + end + + def forget(entry_id) + entries.find(entry_id).destroy! + end + + def summary_list + entries.order(:created_at).pluck(:content) + end + + # Formatted block suitable for injecting into another agent's + # instructions when handing a subject off. + def to_prompt + notes = entries.order(:created_at).map do |entry| + source = entry.source_agent.present? ? " (#{entry.source_agent})" : "" + "- #{entry.content}#{source}" + end + return "" if notes.empty? + + "Memory notes for this subject:\n#{notes.join("\n")}" + end +end diff --git a/lib/generators/solid_agent/install/templates/agent_memory_entry.rb.erb b/lib/generators/solid_agent/install/templates/agent_memory_entry.rb.erb new file mode 100644 index 0000000..62434c8 --- /dev/null +++ b/lib/generators/solid_agent/install/templates/agent_memory_entry.rb.erb @@ -0,0 +1,12 @@ +# frozen_string_literal: true + +# One agent-authored summary note in an AgentMemory. +class AgentMemoryEntry < ApplicationRecord + belongs_to :agent_memory + + validates :content, presence: true + + scope :chronological, -> { order(:created_at) } + scope :by_category, ->(category) { where(category: category) } + scope :from_agent, ->(agent_name) { where(source_agent: agent_name) } +end diff --git a/lib/generators/solid_agent/install/templates/create_agent_memories.rb.erb b/lib/generators/solid_agent/install/templates/create_agent_memories.rb.erb new file mode 100644 index 0000000..e0d098e --- /dev/null +++ b/lib/generators/solid_agent/install/templates/create_agent_memories.rb.erb @@ -0,0 +1,35 @@ +# frozen_string_literal: true + +class CreateAgentMemories < ActiveRecord::Migration<%= migration_version %> + def change + create_table :agent_memories do |t| + # The subject the memory is about (an Agent record, User, Project...). + # Any agent operating on the same subject + scope shares the memory, + # which is what makes it a handoff channel between agents. + t.references :memorable, polymorphic: true, index: true + + # Namespace so a subject can carry independent memory streams. + t.string :scope, null: false, default: "default" + + t.timestamps + end + add_index :agent_memories, [ :memorable_type, :memorable_id, :scope ], unique: true + + create_table :agent_memory_entries do |t| + t.references :agent_memory, null: false, foreign_key: true + + # The summary note the agent chose to persist. + t.text :content, null: false + + # Which agent class wrote it (handoff provenance). + t.string :source_agent + + # Optional label: fact, task, handoff, ... + t.string :category + + t.timestamps + end + add_index :agent_memory_entries, [ :agent_memory_id, :created_at ] + add_index :agent_memory_entries, :category + end +end diff --git a/lib/solid_agent.rb b/lib/solid_agent.rb index 287a5aa..ff92eb3 100644 --- a/lib/solid_agent.rb +++ b/lib/solid_agent.rb @@ -60,6 +60,7 @@ def agents_from(directory, pattern: "**/*.agent.md", **options) end require_relative "solid_agent/has_context" +require_relative "solid_agent/has_memory" require_relative "solid_agent/has_tools" require_relative "solid_agent/streams_tool_updates" require_relative "solid_agent/reasonable" diff --git a/lib/solid_agent/has_memory.rb b/lib/solid_agent/has_memory.rb new file mode 100644 index 0000000..2cefdbe --- /dev/null +++ b/lib/solid_agent/has_memory.rb @@ -0,0 +1,136 @@ +# frozen_string_literal: true + +# HasMemory gives an agent a persistent, agent-curated summary list — the +# model decides when to read and write it while interacting with tools, +# other agents, and users. +# +# Memory is scoped to a subject record (any ActiveRecord model) plus a +# scope name, NOT to the agent class — so a memory written by one agent can +# be recalled by another operating on the same subject. That makes it a +# handoff channel: agent A records what it learned/did, agent B picks the +# subject up and recalls the summary before continuing. +# +# The concern is duck-typed against a memory model exposing: +# Model.for(memorable, scope:) -> memory record +# memory.remember(content, source_agent:, category:) -> entry +# memory.recall(limit:, category:) -> entries (responding to #content) +# The install generator's AgentMemory implements this contract. +# +# @example Give an agent memory tools the model can call +# class SupportAgent < ApplicationAgent +# include SolidAgent::HasMemory +# has_memory +# +# def handle +# prompt(message: params[:message], tools: memory_tool_definitions) +# end +# end +# +# @example Handoff between agents sharing a subject +# ResearchAgent.with(memorable: project).research.generate_now +# # later, a different agent class: +# WriterAgent.with(memorable: project).draft.generate_now +# # WriterAgent's recall_memory returns ResearchAgent's entries too. +module SolidAgent + module HasMemory + extend ActiveSupport::Concern + + DEFAULT_SCOPE = "default" + + # Function-calling schemas (common format) for the two memory tools. + # Exposed as a module method so non-agent callers (platform executors, + # MCP servers) can reuse the exact same contract. + def self.tool_definitions + [ + { + name: "save_memory", + description: "Persist a short summary note to long-term memory. Use for facts, decisions, task outcomes, or anything a future agent or session should know. Keep each note self-contained.", + parameters: { + type: "object", + properties: { + content: { type: "string", description: "The summary note to remember" }, + category: { type: "string", description: "Optional label, e.g. fact, task, handoff" } + }, + required: [ "content" ] + } + }, + { + name: "recall_memory", + description: "Read back previously saved memory notes for the current subject, most recent first. Use before starting work to pick up prior context or another agent's handoff.", + parameters: { + type: "object", + properties: { + category: { type: "string", description: "Only return notes with this label" }, + limit: { type: "integer", description: "Maximum notes to return (default 20)" } + }, + required: [] + } + } + ] + end + + included do + class_attribute :_memory_config, default: nil + end + + class_methods do + # Configures memory for this agent. + # + # @param scope [String, Symbol] memory namespace (default "default") + # @param class_name [String] memory model (default "AgentMemory") + def has_memory(scope: DEFAULT_SCOPE, class_name: "AgentMemory") + self._memory_config = { scope: scope.to_s, class_name: class_name } + end + end + + # The memory record for the current subject (or nil without a subject). + def memory + config = self.class._memory_config || { scope: DEFAULT_SCOPE, class_name: "AgentMemory" } + subject = memory_subject + return nil unless subject + + @memory ||= config[:class_name].constantize.for(subject, scope: config[:scope]) + end + + # The record memory is attached to. Defaults to params[:memorable], + # falling back to the HasContext contextable when present. Override for + # custom subjects. + def memory_subject + return params[:memorable] if respond_to?(:params) && params.is_a?(Hash) && params[:memorable] + + context.contextable if respond_to?(:context) && context.respond_to?(:contextable) + rescue StandardError + nil + end + + def memory_tool_definitions + SolidAgent::HasMemory.tool_definitions + end + + # Tool implementations — routed here by the provider's tool calls. + + def save_memory(content:, category: nil) + return { error: "No memory subject available" } unless memory + + entry = memory.remember(content, source_agent: self.class.name, category: category) + { saved: true, id: entry.respond_to?(:id) ? entry.id : nil, content: content } + end + + def recall_memory(category: nil, limit: 20) + return { error: "No memory subject available" } unless memory + + entries = memory.recall(limit: limit, category: category) + { + count: entries.size, + entries: entries.map do |entry| + { + content: entry.content, + category: (entry.category if entry.respond_to?(:category)), + source_agent: (entry.source_agent if entry.respond_to?(:source_agent)), + created_at: (entry.created_at.iso8601 if entry.respond_to?(:created_at) && entry.created_at) + }.compact + end + } + end + end +end diff --git a/test/solid_agent/has_memory_test.rb b/test/solid_agent/has_memory_test.rb new file mode 100644 index 0000000..4822408 --- /dev/null +++ b/test/solid_agent/has_memory_test.rb @@ -0,0 +1,109 @@ +# frozen_string_literal: true + +# test_helper stubs ActiveSupport::Concern and loads the full gem against +# it — do NOT require the real active_support here: its Concern would +# override the stub's append_features mid-suite and break every include. +require "test_helper" + +class SolidAgent::HasMemoryTest < Minitest::Test + FakeEntry = Struct.new(:id, :content, :category, :source_agent, :created_at) + + # Implements the memory-model contract HasMemory duck-types against. + class FakeMemory + class << self + attr_accessor :instances + + def for(memorable, scope:) + self.instances ||= {} + self.instances[[ memorable, scope ]] ||= new(memorable, scope) + end + end + + attr_reader :memorable, :scope, :entries + + def initialize(memorable, scope) + @memorable = memorable + @scope = scope + @entries = [] + end + + def remember(content, source_agent: nil, category: nil) + entry = FakeEntry.new(@entries.size + 1, content, category, source_agent, Time.now) + @entries << entry + entry + end + + def recall(limit: 20, category: nil) + list = @entries.reverse + list = list.select { |e| e.category == category } if category + list.first(limit || 20) + end + end + + class FakeAgent + include SolidAgent::HasMemory + has_memory class_name: "SolidAgent::HasMemoryTest::FakeMemory" + + attr_reader :params + + def initialize(params = {}) + @params = params + end + end + + def setup + FakeMemory.instances = {} + end + + def test_tool_definitions_expose_save_and_recall + names = SolidAgent::HasMemory.tool_definitions.map { |d| d[:name] } + assert_equal %w[save_memory recall_memory], names + + agent = FakeAgent.new(memorable: "subject-1") + assert_equal names, agent.memory_tool_definitions.map { |d| d[:name] } + end + + def test_save_and_recall_roundtrip + agent = FakeAgent.new(memorable: "subject-1") + + saved = agent.save_memory(content: "User prefers terse answers", category: "fact") + assert saved[:saved] + + recalled = agent.recall_memory + assert_equal 1, recalled[:count] + assert_equal "User prefers terse answers", recalled[:entries].first[:content] + assert_equal "SolidAgent::HasMemoryTest::FakeAgent", recalled[:entries].first[:source_agent] + end + + def test_memory_is_shared_across_agent_classes_for_handoff + other_agent_class = Class.new do + include SolidAgent::HasMemory + has_memory class_name: "SolidAgent::HasMemoryTest::FakeMemory" + def self.name = "SecondAgent" + attr_reader :params + def initialize(params) = @params = params + end + + FakeAgent.new(memorable: "shared-subject").save_memory(content: "Research done: use approach B") + recalled = other_agent_class.new({ memorable: "shared-subject" }).recall_memory + + assert_equal 1, recalled[:count] + assert_equal "Research done: use approach B", recalled[:entries].first[:content] + end + + def test_category_filter_passes_through + agent = FakeAgent.new(memorable: "s") + agent.save_memory(content: "a fact", category: "fact") + agent.save_memory(content: "a task", category: "task") + + recalled = agent.recall_memory(category: "task") + assert_equal [ "a task" ], recalled[:entries].map { |e| e[:content] } + end + + def test_errors_without_a_subject + agent = FakeAgent.new({}) + + assert_equal({ error: "No memory subject available" }, agent.save_memory(content: "x")) + assert_equal({ error: "No memory subject available" }, agent.recall_memory) + end +end From fa89a8fddd668f3a4dea3bd1236a920e1e977a16 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 1 Aug 2026 15:39:58 +0000 Subject: [PATCH 3/5] Enrich tool persistence, add ModelPricing, document the full surface - HasContext#persist_tool_messages_to_context now consults an overridable tool_invocations hook: executors that run tools server-side return invocation records ({tool_call_id:, name:, arguments:, duration_ms:}) which are matched to response tool messages by tool_call_id, else by position (Ollama's tool messages carry neither name nor id). Persisted rows then carry the call's arguments and timing. Enrichment keywords are only passed when the context's add_tool_message accepts them, so models generated before this change keep working. - Install template AgentContext#add_tool_message gains arguments:/ duration_ms: (writes tool_arguments + metadata duration_ms), matching what the enrichment path passes. - SolidAgent::ModelPricing: token counts -> estimated USD. RubyLLM registry rates when available, static pattern table fallback, blended default. Mock models price free (pattern ordered before real models so mock-gpt-4o-mini doesn't bill at gpt-4o rates). The template's AgentGeneration#estimated_cost uses it when no explicit rates given. - README: document all seven concerns + ToolCache/ModelPricing/ AgentManifest and the solid_agent:install generator (it previously described three concerns and omitted install entirely). 215 runs, 496 assertions, 0 failures. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01B5PaDq3uaUWuA8UW4K7siP --- README.md | 77 +++++++- .../install/templates/agent_context.rb.erb | 4 +- .../install/templates/agent_generation.rb.erb | 18 +- lib/solid_agent.rb | 1 + lib/solid_agent/has_context.rb | 54 +++++- lib/solid_agent/model_pricing.rb | 93 ++++++++++ .../has_context_tool_messages_test.rb | 174 ++++++++++++++++++ test/solid_agent/model_pricing_test.rb | 50 +++++ 8 files changed, 451 insertions(+), 20 deletions(-) create mode 100644 lib/solid_agent/model_pricing.rb create mode 100644 test/solid_agent/has_context_tool_messages_test.rb create mode 100644 test/solid_agent/model_pricing_test.rb diff --git a/README.md b/README.md index 3173fd7..2244ba9 100644 --- a/README.md +++ b/README.md @@ -4,14 +4,25 @@ # SolidAgent -SolidAgent extends the [ActiveAgent](https://github.com/activeagents/activeagent) framework with enterprise-grade features for building robust AI agents in Rails applications. It provides three core concerns that add database-backed persistence, declarative tool schemas, and real-time streaming capabilities to your agents. +SolidAgent extends the [ActiveAgent](https://github.com/activeagents/activeagent) framework with database-backed persistence for everything an agent does in a Rails application: conversations, generations, tool/MCP interactions, reasoning, and long-term memory. ## Features -- **HasContext** - Database-backed prompt context management for maintaining conversation history and agent state +Agent-side concerns: + +- **HasContext** - Database-backed prompt context management for maintaining conversation history and agent state, including the full tool/MCP interaction stream +- **HasMemory** - An agent-curated summary list the model reads/writes via `save_memory`/`recall_memory` function-calling tools; scoped to a subject record so agents hand off to each other through shared memory - **HasTools** - Declarative, schema-based tool definitions compatible with LLM function-calling APIs +- **HasReasons** - Capture and inspect extended-thinking/reasoning output across a generation - **StreamsToolUpdates** - Real-time UI feedback during tool execution via ActionCable +Model-side and standalone: + +- **Reasonable** - Persist reasoning content/tokens/metadata on your generation records +- **ToolCache** - Cache tool/MCP/service results by `(tool, normalized args)` with TTL, backed by `Rails.cache`; error results are never cached and replays are tagged `cached: true` +- **ModelPricing** - Token-count → estimated USD cost, using RubyLLM's model registry when available with a static pattern-table fallback +- **AgentManifest** - Load, validate, export, and build agent classes from portable manifests (`.agent.md`, dotprompt, CrewAI) + ## Installation Add this line to your application's Gemfile: @@ -26,19 +37,15 @@ And then execute: $ bundle install ``` -Or install it yourself as: - -```bash -$ gem install solid_agent -``` - ## Usage ### Quick Start -Generate a new agent with context support: +Install the persistence tables and models (`AgentContext`, `AgentMessage`, `AgentGeneration`, `AgentMemory`, `AgentMemoryEntry`), then generate an agent with context support: ```bash +$ rails generate solid_agent:install +$ rails db:migrate $ rails generate solid_agent:agent WritingAssistant --context --context_name conversation --contextual user ``` @@ -130,9 +137,53 @@ class BrowserAgent < ApplicationAgent end ``` +### HasMemory - Agent-Curated Long-Term Memory + +Give an agent a durable summary list it decides when to read and write, scoped to a subject record rather than the agent class — so different agents operating on the same subject share memory, with `source_agent` provenance on every entry: + +```ruby +class SupportAgent < ApplicationAgent + include SolidAgent::HasContext + include SolidAgent::HasMemory + + has_context contextual: :user + has_memory # scope: "default", class_name: "AgentMemory" + + def assist + load_context(contextable: params[:user]) + prompt messages: context_messages, tools: memory_tool_definitions + end +end +``` + +The model calls `save_memory(content:, category:)` and `recall_memory(category:, limit:)` as ordinary function-calling tools. `SolidAgent::HasMemory.tool_definitions` exposes the same schemas module-level for non-agent executors (platform services, MCP servers). Inject `agent.memory.to_prompt` into instructions to prime a handoff. + +### ToolCache - Cached Tool Results + +```ruby +result = SolidAgent::ToolCache.fetch(tool: "fetch_url", args: { url: url }, ttl: 300) do + expensive_call(url) +end +result[:cached] # => true on a replay +``` + +Error-shaped results (`{ error: ... }`) are never cached, so transient failures don't stick; cache keys are stable across argument ordering and symbol/string keys. + +### ModelPricing - Estimated Spend + +```ruby +SolidAgent::ModelPricing.estimate(model: "claude-sonnet-5", input_tokens: 12_000, output_tokens: 800) +# => 0.048 (USD, estimated) +``` + +The generated `AgentGeneration#estimated_cost` uses this automatically. Rates come from RubyLLM's registry when that gem is present, else a static pattern table. + ### Generators ```bash +# Install persistence tables + models (contexts, messages, generations, memories) +$ rails generate solid_agent:install + # Generate a new agent $ rails generate solid_agent:agent MyAgent @@ -142,8 +193,14 @@ $ rails generate solid_agent:agent MyAgent --context --context_name session # Generate a tool template $ rails generate solid_agent:tool search MyAgent --parameters query:string:required -# Generate context models +# Generate custom-named context models $ rails generate solid_agent:context conversation + +# Add reasoning columns to a generation model +$ rails generate solid_agent:reasons AgentGeneration + +# Scaffold an agent manifest (.agent.md) +$ rails generate solid_agent:manifest research ``` ## Example Apps diff --git a/lib/generators/solid_agent/install/templates/agent_context.rb.erb b/lib/generators/solid_agent/install/templates/agent_context.rb.erb index cff4a0a..02167be 100644 --- a/lib/generators/solid_agent/install/templates/agent_context.rb.erb +++ b/lib/generators/solid_agent/install/templates/agent_context.rb.erb @@ -126,12 +126,14 @@ class AgentContext < ApplicationRecord # @param tool_name [String] the name of the tool # @param result [Hash, String] the tool result # @return [AgentMessage] the created message - def add_tool_message(tool_call_id:, tool_name:, result:) + def add_tool_message(tool_call_id:, tool_name:, result:, arguments: nil, duration_ms: nil) messages.create!( role: "tool", tool_call_id: tool_call_id, tool_name: tool_name, tool_result: result, + tool_arguments: arguments.presence || {}, + metadata: duration_ms ? { "duration_ms" => duration_ms } : {}, content: result.is_a?(String) ? result : result.to_json ) end diff --git a/lib/generators/solid_agent/install/templates/agent_generation.rb.erb b/lib/generators/solid_agent/install/templates/agent_generation.rb.erb index 9aec815..1ccd07e 100644 --- a/lib/generators/solid_agent/install/templates/agent_generation.rb.erb +++ b/lib/generators/solid_agent/install/templates/agent_generation.rb.erb @@ -60,11 +60,17 @@ class AgentGeneration < ApplicationRecord # Returns the cost estimate based on token usage (override with your pricing) # # @param input_price_per_million [Float] price per million input tokens - # @param output_price_per_million [Float] price per million output tokens - # @return [Float] estimated cost in dollars - def estimated_cost(input_price_per_million: 3.0, output_price_per_million: 15.0) - input_cost = (input_tokens / 1_000_000.0) * input_price_per_million - output_cost = (output_tokens / 1_000_000.0) * output_price_per_million - input_cost + output_cost + # @param output_price_per_million [Float, nil] price per million output tokens + # @return [Float, nil] estimated cost in dollars, nil when nothing to price + def estimated_cost(input_price_per_million: nil, output_price_per_million: nil) + if input_price_per_million && output_price_per_million + input_cost = (input_tokens / 1_000_000.0) * input_price_per_million + output_cost = (output_tokens / 1_000_000.0) * output_price_per_million + input_cost + output_cost + else + # Per-model rates from SolidAgent::ModelPricing (RubyLLM registry + # when available, static table otherwise) + SolidAgent::ModelPricing.estimate(model: model, input_tokens: input_tokens, output_tokens: output_tokens) + end end end diff --git a/lib/solid_agent.rb b/lib/solid_agent.rb index ff92eb3..5f9c657 100644 --- a/lib/solid_agent.rb +++ b/lib/solid_agent.rb @@ -2,6 +2,7 @@ require_relative "solid_agent/version" require_relative "solid_agent/tool_cache" +require_relative "solid_agent/model_pricing" module SolidAgent class Error < StandardError; end diff --git a/lib/solid_agent/has_context.rb b/lib/solid_agent/has_context.rb index f036150..e7b5173 100644 --- a/lib/solid_agent/has_context.rb +++ b/lib/solid_agent/has_context.rb @@ -547,6 +547,18 @@ def persist_generation_to_context end end + # Overridable enrichment hook for tool persistence. Executors that run + # tools server-side (a platform's execution service, a job) can + # override this to return their own invocation records — an array of + # hashes with symbol keys :tool_call_id, :name, :arguments and + # :duration_ms (all optional) — so persisted tool messages carry the + # call's arguments and timing, which provider response messages don't + # include. Records are matched to response tool messages by + # tool_call_id when both sides have one, otherwise by position. + def tool_invocations + [] + end + # Persists the tool/MCP interaction stream (tool result messages from # the response's message stack) to the context, so conversations show # the full agent <-> tool exchange, not just the final assistant text. @@ -559,17 +571,30 @@ def persist_tool_messages_to_context return unless context.respond_to?(:add_tool_message) return unless generation_response.respond_to?(:messages) + tool_index = -1 Array(generation_response.messages).each do |message| next unless message.respond_to?(:role) && message.role.to_s == "tool" + tool_index += 1 tool_call_id = message.respond_to?(:tool_call_id) ? message.tool_call_id : nil next if tool_call_id.present? && tool_message_persisted?(tool_call_id) - context.add_tool_message( + invocation = tool_invocation_for(tool_call_id, tool_index) + # Provider tool messages often carry no name (Ollama's don't); the + # executor's invocation record is the fallback. + name = (message.name if message.respond_to?(:name)) + name = invocation[:name] if !name.present? && invocation + + attributes = { tool_call_id: tool_call_id, - tool_name: (message.name if message.respond_to?(:name)), + tool_name: name, result: (message.content if message.respond_to?(:content)) - ) + } + if invocation && tool_message_details_supported? + attributes[:arguments] = invocation[:arguments] + attributes[:duration_ms] = invocation[:duration_ms] + end + context.add_tool_message(**attributes) end rescue => e Rails.logger.error "[SolidAgent] Failed to persist tool messages: #{e.message}" @@ -581,5 +606,28 @@ def tool_message_persisted?(tool_call_id) scope = context.messages scope.respond_to?(:exists?) && scope.exists?(role: "tool", tool_call_id: tool_call_id) end + + # Finds the executor invocation record for a response tool message — + # by tool_call_id when the record carries one, else by position among + # the response's tool messages. + def tool_invocation_for(tool_call_id, index) + invocations = Array(tool_invocations) + return nil if invocations.empty? + + if tool_call_id.present? + match = invocations.find { |inv| inv[:tool_call_id] && inv[:tool_call_id].to_s == tool_call_id.to_s } + return match if match + end + invocations[index] + end + + # Whether the context's add_tool_message accepts the arguments:/ + # duration_ms: enrichment keywords (older generated models don't). + def tool_message_details_supported? + parameters = context.method(:add_tool_message).parameters + parameters.any? { |type, param_name| type == :keyrest || ([ :key, :keyreq ].include?(type) && param_name == :arguments) } + rescue ::NameError + false + end end end diff --git a/lib/solid_agent/model_pricing.rb b/lib/solid_agent/model_pricing.rb new file mode 100644 index 0000000..f6fda2f --- /dev/null +++ b/lib/solid_agent/model_pricing.rb @@ -0,0 +1,93 @@ +# frozen_string_literal: true + +module SolidAgent + # Estimates LLM spend from token counts (USD). Generation records store + # tokens only; pricing is layered on top for cost reporting, so figures + # are always estimates. + # + # Rates come from RubyLLM's model registry (USD per million tokens, + # maintained upstream per model) when that gem is available and knows + # the model; the static pattern table below is the fallback for aliases + # and self-hosted models, and a conservative blended rate covers + # everything else so totals stay meaningful. + # + # @example Estimate a generation's cost + # SolidAgent::ModelPricing.estimate( + # model: "claude-sonnet-5", input_tokens: 12_000, output_tokens: 800 + # ) # => 0.048 + # + # @example Look up a model's rates ($/1M input, $/1M output) + # SolidAgent::ModelPricing.rate_for("gpt-4o-mini") # => [0.15, 0.60] + class ModelPricing + PRICES = [ + # [pattern, input $/1M, output $/1M] — first match wins, so mock + # models ("mock-gpt-4o-mini") price free before real-model patterns. + [ /mock/i, 0.0, 0.0 ], + [ /gpt-4o-mini/i, 0.15, 0.60 ], + [ /gpt-4o/i, 2.50, 10.00 ], + [ /gpt-4\.1-nano/i, 0.10, 0.40 ], + [ /gpt-4\.1-mini/i, 0.40, 1.60 ], + [ /gpt-4\.1/i, 2.00, 8.00 ], + [ /o3-mini|o4-mini/i, 1.10, 4.40 ], + [ /claude.*(fable|mythos)/i, 10.00, 50.00 ], + [ /claude.*haiku-?4/i, 1.00, 5.00 ], + [ /claude.*(haiku)/i, 0.80, 4.00 ], + [ /claude.*(sonnet)/i, 3.00, 15.00 ], + [ /claude.*opus-(5|4-[5-9])/i, 5.00, 25.00 ], + [ /claude.*(opus)/i, 15.00, 75.00 ], + [ /gemini.*flash/i, 0.10, 0.40 ], + [ /gemini.*pro/i, 1.25, 10.00 ], + [ /llama|mistral|mixtral|qwen|deepseek/i, 0.20, 0.60 ] + ].freeze + + # Fallback blended rate for unknown models ($/1M input, $/1M output) + DEFAULT_RATE = [ 1.00, 4.00 ].freeze + + class << self + # @return [Float, nil] estimated USD cost, nil when there is nothing + # to price + def estimate(model:, input_tokens:, output_tokens:) + input = input_tokens.to_i + output = output_tokens.to_i + return nil if input.zero? && output.zero? + + input_rate, output_rate = rate_for(model) + ((input * input_rate) + (output * output_rate)) / 1_000_000.0 + end + + # @return [Array(Float, Float)] $/1M input and output token rates + def rate_for(model) + return DEFAULT_RATE if model.blank? + + registry_rate(model) || static_rate(model) + end + + # Exact per-model rates from RubyLLM's registry when that gem is + # loaded. Lookups are memoized — the registry scan is not free and + # cost reporting calls this per row. + def registry_rate(model) + return nil unless defined?(::RubyLLM) + + @registry_rates ||= {} + return @registry_rates[model] if @registry_rates.key?(model) + + @registry_rates[model] = begin + info = ::RubyLLM.models.find(model.to_s) + tokens = info&.pricing&.text_tokens + if tokens&.input && tokens&.output + [ tokens.input, tokens.output ] + end + rescue ::StandardError + nil + end + end + + def static_rate(model) + PRICES.each do |pattern, input_rate, output_rate| + return [ input_rate, output_rate ] if model.to_s.match?(pattern) + end + DEFAULT_RATE + end + end + end +end diff --git a/test/solid_agent/has_context_tool_messages_test.rb b/test/solid_agent/has_context_tool_messages_test.rb new file mode 100644 index 0000000..a2fa377 --- /dev/null +++ b/test/solid_agent/has_context_tool_messages_test.rb @@ -0,0 +1,174 @@ +# frozen_string_literal: true + +require "test_helper" + +# Tool-message persistence (HasContext#persist_tool_messages_to_context): +# tool results from the response's message stack land on the context, +# deduped by tool_call_id, enriched with the executor's own invocation +# records (arguments/timing) via the overridable tool_invocations hook. +# +# NOTE: test_helper stubs ActiveSupport::Concern and loads the full gem +# against it — do NOT require "active_support" here. +class HasContextToolMessagesTest < Minitest::Test + FakeToolMessage = Struct.new(:role, :name, :tool_call_id, :content) + + class FakeResponse + attr_reader :messages + + def initialize(messages) + @messages = messages + end + end + + # Modern context: add_tool_message accepts the enrichment keywords. + class FakeContext + attr_reader :tool_messages + + def initialize(existing_tool_call_ids: []) + @tool_messages = [] + @existing = existing_tool_call_ids + end + + def add_tool_message(tool_call_id:, tool_name:, result:, arguments: nil, duration_ms: nil) + @tool_messages << { + tool_call_id: tool_call_id, tool_name: tool_name, result: result, + arguments: arguments, duration_ms: duration_ms + } + end + + def messages + existing = @existing + scope = Object.new + scope.define_singleton_method(:exists?) do |role:, tool_call_id:| + existing.include?(tool_call_id) + end + scope + end + end + + # Legacy context generated before the enrichment keywords existed. + class LegacyContext + attr_reader :tool_messages + + def initialize + @tool_messages = [] + end + + def add_tool_message(tool_call_id:, tool_name:, result:) + @tool_messages << { tool_call_id: tool_call_id, tool_name: tool_name, result: result } + end + + def messages + scope = Object.new + scope.define_singleton_method(:exists?) { |role:, tool_call_id:| false } + scope + end + end + + def setup + Object.const_set(:AgentContext, SolidAgentTestHelpers::MockAgentContext) unless defined?(AgentContext) + Object.const_set(:AgentMessage, SolidAgentTestHelpers::MockAgentMessage) unless defined?(AgentMessage) + Object.const_set(:AgentGeneration, SolidAgentTestHelpers::MockAgentGeneration) unless defined?(AgentGeneration) + + @agent_class = Class.new(SolidAgentTestHelpers::MockBaseAgent) do + include SolidAgent::HasContext + has_context + end + end + + def teardown + Object.send(:remove_const, :AgentContext) if defined?(AgentContext) + Object.send(:remove_const, :AgentMessage) if defined?(AgentMessage) + Object.send(:remove_const, :AgentGeneration) if defined?(AgentGeneration) + end + + def build_agent(context, response, invocations: nil) + agent = @agent_class.new + agent.context = context + agent.generation_response = response + agent.define_singleton_method(:tool_invocations) { invocations } if invocations + agent + end + + def test_persists_tool_messages_from_response_stack + context = FakeContext.new + response = FakeResponse.new([ + FakeToolMessage.new("assistant", nil, nil, "Let me check."), + FakeToolMessage.new("tool", "calculate", "call_1", "42") + ]) + + build_agent(context, response).send(:persist_tool_messages_to_context) + + assert_equal 1, context.tool_messages.length + message = context.tool_messages.first + assert_equal "call_1", message[:tool_call_id] + assert_equal "calculate", message[:tool_name] + assert_equal "42", message[:result] + end + + def test_enriches_from_tool_invocations_matched_by_tool_call_id + context = FakeContext.new + response = FakeResponse.new([ + FakeToolMessage.new("tool", "fetch_url", "call_9", "{\"body\":\"...\"}") + ]) + invocations = [ + { tool_call_id: "call_9", name: "fetch_url", arguments: { "url" => "https://example.com" }, duration_ms: 120 } + ] + + build_agent(context, response, invocations: invocations).send(:persist_tool_messages_to_context) + + message = context.tool_messages.first + assert_equal({ "url" => "https://example.com" }, message[:arguments]) + assert_equal 120, message[:duration_ms] + end + + def test_enriches_positionally_and_names_nameless_tool_messages + # Ollama-style: tool messages carry no name and no tool_call_id. + context = FakeContext.new + response = FakeResponse.new([ + FakeToolMessage.new("tool", nil, nil, "42"), + FakeToolMessage.new("tool", nil, nil, "cached") + ]) + invocations = [ + { name: "calculate", arguments: { "expression" => "6*7" }, duration_ms: 3 }, + { name: "recall_memory", arguments: {}, duration_ms: 1 } + ] + + build_agent(context, response, invocations: invocations).send(:persist_tool_messages_to_context) + + assert_equal %w[calculate recall_memory], context.tool_messages.map { |m| m[:tool_name] } + assert_equal({ "expression" => "6*7" }, context.tool_messages.first[:arguments]) + end + + def test_skips_enrichment_keywords_for_legacy_contexts + context = LegacyContext.new + response = FakeResponse.new([ FakeToolMessage.new("tool", "calculate", "call_1", "42") ]) + invocations = [ { tool_call_id: "call_1", name: "calculate", arguments: { "expression" => "6*7" } } ] + + build_agent(context, response, invocations: invocations).send(:persist_tool_messages_to_context) + + assert_equal 1, context.tool_messages.length + refute context.tool_messages.first.key?(:arguments) + end + + def test_dedupes_by_tool_call_id + context = FakeContext.new(existing_tool_call_ids: [ "call_1" ]) + response = FakeResponse.new([ + FakeToolMessage.new("tool", "calculate", "call_1", "42"), + FakeToolMessage.new("tool", "fetch_url", "call_2", "ok") + ]) + + build_agent(context, response).send(:persist_tool_messages_to_context) + + assert_equal [ "call_2" ], context.tool_messages.map { |m| m[:tool_call_id] } + end + + def test_skips_contexts_without_add_tool_message + bare_context = Object.new + response = FakeResponse.new([ FakeToolMessage.new("tool", "calculate", "call_1", "42") ]) + + agent = build_agent(bare_context, response) + agent.send(:persist_tool_messages_to_context) + # Nothing raised, nothing persisted — the feature degrades silently. + end +end diff --git a/test/solid_agent/model_pricing_test.rb b/test/solid_agent/model_pricing_test.rb new file mode 100644 index 0000000..517b865 --- /dev/null +++ b/test/solid_agent/model_pricing_test.rb @@ -0,0 +1,50 @@ +# frozen_string_literal: true + +require "test_helper" + +# SolidAgent::ModelPricing — token counts to estimated USD. The RubyLLM +# registry is absent in the gem test environment, so these exercise the +# static-table and default-rate paths (registry_rate returns nil without +# the gem loaded). +class ModelPricingTest < Minitest::Test + def test_static_table_covers_current_models + assert_equal [ 0.15, 0.60 ], SolidAgent::ModelPricing.static_rate("gpt-4o-mini") + assert_equal [ 3.00, 15.00 ], SolidAgent::ModelPricing.static_rate("claude-sonnet-5") + assert_equal [ 5.00, 25.00 ], SolidAgent::ModelPricing.static_rate("claude-opus-5") + assert_equal [ 10.00, 50.00 ], SolidAgent::ModelPricing.static_rate("claude-fable-5") + assert_equal [ 1.00, 5.00 ], SolidAgent::ModelPricing.static_rate("claude-haiku-4-5") + end + + def test_older_generations_keep_legacy_rates + assert_equal [ 0.80, 4.00 ], SolidAgent::ModelPricing.static_rate("claude-3-5-haiku-latest") + assert_equal [ 15.00, 75.00 ], SolidAgent::ModelPricing.static_rate("claude-3-opus-20240229") + assert_equal [ 5.00, 25.00 ], SolidAgent::ModelPricing.static_rate("claude-opus-4-5-20251101") + end + + def test_self_hosted_models_price_at_open_weight_rate + assert_equal [ 0.20, 0.60 ], SolidAgent::ModelPricing.static_rate("qwen3:8b") + assert_equal [ 0.20, 0.60 ], SolidAgent::ModelPricing.static_rate("llama3.1:8b") + end + + def test_mock_models_are_free + assert_equal [ 0.0, 0.0 ], SolidAgent::ModelPricing.static_rate("mock-gpt-4o-mini") + end + + def test_unknown_models_fall_back_to_blended_default + assert_equal SolidAgent::ModelPricing::DEFAULT_RATE, SolidAgent::ModelPricing.rate_for("totally-unknown-xyz") + assert_equal SolidAgent::ModelPricing::DEFAULT_RATE, SolidAgent::ModelPricing.rate_for(nil) + end + + def test_registry_rate_is_nil_without_ruby_llm + assert_nil SolidAgent::ModelPricing.registry_rate("gpt-4o") + end + + def test_estimate_prices_input_and_output_separately + cost = SolidAgent::ModelPricing.estimate(model: "gpt-4o", input_tokens: 1_000_000, output_tokens: 1_000_000) + assert_in_delta 12.5, cost, 0.001 + end + + def test_estimate_returns_nil_with_nothing_to_price + assert_nil SolidAgent::ModelPricing.estimate(model: "gpt-4o", input_tokens: 0, output_tokens: 0) + end +end From 46aa6ff67a73dd422394627ab5b2bbc56d2cba5c Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 1 Aug 2026 15:40:47 +0000 Subject: [PATCH 4/5] Bump to 0.2.0 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit HasMemory, ToolCache, ModelPricing, and enriched tool persistence are a minor-version feature set — and consumers gate on the version to know whether HasContext persists tool details itself. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01B5PaDq3uaUWuA8UW4K7siP --- lib/solid_agent/version.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/solid_agent/version.rb b/lib/solid_agent/version.rb index 740d49e..5c18928 100644 --- a/lib/solid_agent/version.rb +++ b/lib/solid_agent/version.rb @@ -1,5 +1,5 @@ # frozen_string_literal: true module SolidAgent - VERSION = "0.1.1" + VERSION = "0.2.0" end From 30e2d9d39c51f69c5b46712cbb86a458519d70f9 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 1 Aug 2026 16:06:09 +0000 Subject: [PATCH 5/5] Add AgentRun: durable run records with progress events and cohorts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The install generator now ships an agent_runs table + AgentRun model — the platform-proven shape for run orchestration: lifecycle status (pending/running/complete/failed/cancelled with start!/complete!/fail!/ cancel!), token + duration accounting, trace_id correlation with contexts/generations/telemetry, and an append-only progress-event stream (append_event: eid-paired started/done/error events, byte-capped detail, update_column so the run's own thread appends safely). SolidAgent::RunFingerprint carries the cohort logic: 8-char instruction digests and deterministic adjective-noun codenames ("calm-heron") so configuration cohorts read as names instead of hex. 220 runs, 0 failures. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01B5PaDq3uaUWuA8UW4K7siP --- Gemfile.lock | 2 +- README.md | 19 ++- .../solid_agent/install/install_generator.rb | 4 + .../install/templates/agent_run.rb.erb | 116 ++++++++++++++++++ .../templates/create_agent_runs.rb.erb | 46 +++++++ lib/solid_agent.rb | 1 + lib/solid_agent/run_fingerprint.rb | 51 ++++++++ test/solid_agent/run_fingerprint_test.rb | 44 +++++++ 8 files changed, 281 insertions(+), 2 deletions(-) create mode 100644 lib/generators/solid_agent/install/templates/agent_run.rb.erb create mode 100644 lib/generators/solid_agent/install/templates/create_agent_runs.rb.erb create mode 100644 lib/solid_agent/run_fingerprint.rb create mode 100644 test/solid_agent/run_fingerprint_test.rb diff --git a/Gemfile.lock b/Gemfile.lock index ba9b2d0..204f945 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,7 +1,7 @@ PATH remote: . specs: - solid_agent (0.1.1) + solid_agent (0.2.0) activeagent (>= 1.0.0) activerecord (>= 7.0) activesupport (>= 7.0) diff --git a/README.md b/README.md index 2244ba9..c585c4a 100644 --- a/README.md +++ b/README.md @@ -19,6 +19,7 @@ Agent-side concerns: Model-side and standalone: - **Reasonable** - Persist reasoning content/tokens/metadata on your generation records +- **AgentRun** - Durable run records (installed by the generator): lifecycle status, append-only progress events for live UIs, token/duration accounting, and instruction-fingerprint cohorts for comparing configuration changes - **ToolCache** - Cache tool/MCP/service results by `(tool, normalized args)` with TTL, backed by `Rails.cache`; error results are never cached and replays are tagged `cached: true` - **ModelPricing** - Token-count → estimated USD cost, using RubyLLM's model registry when available with a static pattern-table fallback - **AgentManifest** - Load, validate, export, and build agent classes from portable manifests (`.agent.md`, dotprompt, CrewAI) @@ -41,7 +42,7 @@ $ bundle install ### Quick Start -Install the persistence tables and models (`AgentContext`, `AgentMessage`, `AgentGeneration`, `AgentMemory`, `AgentMemoryEntry`), then generate an agent with context support: +Install the persistence tables and models (`AgentContext`, `AgentMessage`, `AgentGeneration`, `AgentMemory`, `AgentMemoryEntry`, `AgentRun`), then generate an agent with context support: ```bash $ rails generate solid_agent:install @@ -169,6 +170,22 @@ result[:cached] # => true on a replay Error-shaped results (`{ error: ... }`) are never cached, so transient failures don't stick; cache keys are stable across argument ordering and symbol/string keys. +### AgentRun - Durable Run Records + +Executors record each agent execution as an `AgentRun`: lifecycle (`start!`/`complete!`/`fail!`/`cancel!`), correlation with contexts, generations, and telemetry via `trace_id`, and an append-only progress-event stream a UI can poll mid-run: + +```ruby +run = AgentRun.create!(runnable: document, agent_name: "SupportAgent", input_prompt: message) +run.record_instructions(agent.instructions) # cohort fingerprint ("calm-heron") +run.start! +run.append_event(kind: "tool", label: "fetch_url", eid: "e1", status: "started") +# ... execute ... +run.append_event(kind: "tool", label: "fetch_url", eid: "e1", status: "done", duration_ms: 120) +run.complete!(output: response.message.content, input_tokens: usage.input_tokens, output_tokens: usage.output_tokens) +``` + +`AgentRun#instructions_codename` names each instruction cohort deterministically (`SolidAgent::RunFingerprint`), so comparing "what changed between these two batches of runs" reads as `calm-heron` vs `misty-atoll` instead of hex digests. + ### ModelPricing - Estimated Spend ```ruby diff --git a/lib/generators/solid_agent/install/install_generator.rb b/lib/generators/solid_agent/install/install_generator.rb index 31e7381..caebcd5 100644 --- a/lib/generators/solid_agent/install/install_generator.rb +++ b/lib/generators/solid_agent/install/install_generator.rb @@ -32,6 +32,9 @@ def create_migrations migration_template "create_agent_memories.rb.erb", "db/migrate/create_agent_memories.rb" + + migration_template "create_agent_runs.rb.erb", + "db/migrate/create_agent_runs.rb" end def create_models @@ -42,6 +45,7 @@ def create_models template "agent_generation.rb.erb", "app/models/agent_generation.rb" template "agent_memory.rb.erb", "app/models/agent_memory.rb" template "agent_memory_entry.rb.erb", "app/models/agent_memory_entry.rb" + template "agent_run.rb.erb", "app/models/agent_run.rb" end def create_initializer diff --git a/lib/generators/solid_agent/install/templates/agent_run.rb.erb b/lib/generators/solid_agent/install/templates/agent_run.rb.erb new file mode 100644 index 0000000..e7a9907 --- /dev/null +++ b/lib/generators/solid_agent/install/templates/agent_run.rb.erb @@ -0,0 +1,116 @@ +# frozen_string_literal: true + +# A single agent execution: lifecycle status, inputs/outputs, usage, and +# an append-only stream of progress events. Correlates with AgentContext/ +# AgentGeneration rows and telemetry traces via trace_id. +class AgentRun < ApplicationRecord + belongs_to :runnable, polymorphic: true, optional: true + + STATUSES = %w[pending running complete failed cancelled].freeze + + validates :status, inclusion: { in: STATUSES } + + scope :recent, -> { order(created_at: :desc) } + scope :for_agent, ->(agent_name) { where(agent_name: agent_name) } + scope :for_action, ->(action_name) { where(action_name: action_name) } + scope :with_trace, ->(trace_id) { where(trace_id: trace_id) } + scope :for_status, ->(status) { where(status: status) } + + STATUSES.each do |status_name| + define_method("#{status_name}?") { status == status_name } + end + + def in_progress? + pending? || running? + end + + def finished? + complete? || failed? || cancelled? + end + + # === Lifecycle === + + def start! + update!(status: "running", started_at: Time.current) + end + + def complete!(output: nil, metadata: {}, input_tokens: nil, output_tokens: nil) + update!( + status: "complete", + output: output, + output_metadata: (output_metadata || {}).merge(metadata), + input_tokens: input_tokens || self.input_tokens, + output_tokens: output_tokens || self.output_tokens, + completed_at: Time.current, + duration_ms: calculated_duration_ms(fallback_end: Time.current) + ) + end + + def fail!(error) + update!( + status: "failed", + error_message: error.respond_to?(:message) ? error.message : error.to_s, + completed_at: Time.current, + duration_ms: calculated_duration_ms(fallback_end: Time.current) + ) + end + + def cancel! + return false if finished? + + update!(status: "cancelled", completed_at: Time.current) + true + end + + # === Progress events === + + # Appends a progress event mid-run so pollers can stream what the agent + # is doing (pending llm/tool/agent calls). Events pair up by eid: a + # "started" event is pending until a "done"/"error" with the same eid + # lands. update_column: no validations/callbacks, safe from the run's + # own execution thread; reads current DB state so concurrent appends + # interleave safely. + def append_event(kind:, label:, eid: nil, status: "done", detail: nil, duration_ms: nil) + event = { + "at" => Time.current.iso8601(3), + "eid" => eid, + "kind" => kind.to_s, + "label" => label.to_s, + "status" => status.to_s + }.compact + event["detail"] = detail.to_s.byteslice(0, 1200).to_s.scrub if detail + event["duration_ms"] = duration_ms if duration_ms + current = self.class.where(id: id).pick(:events) || [] + update_column(:events, current + [ event ]) + event + end + + # === Cohort fingerprinting === + + # Records the instructions this run executed under as a stable digest — + # the grouping key (with model) for configuration cohorts. + def record_instructions(instructions) + self.instructions_digest = SolidAgent::RunFingerprint.digest(instructions) + end + + # Deterministic memorable name for the digest ("calm-heron") — reads far + # better than hex when comparing cohorts. + def instructions_codename + SolidAgent::RunFingerprint.codename(instructions_digest) + end + + # === Usage === + + def total_tokens + input_tokens.to_i + output_tokens.to_i + end + + def calculated_duration_ms(fallback_end: nil) + return duration_ms if duration_ms.present? + + finish = completed_at || fallback_end + return nil unless started_at && finish + + ((finish - started_at) * 1000).to_i + end +end diff --git a/lib/generators/solid_agent/install/templates/create_agent_runs.rb.erb b/lib/generators/solid_agent/install/templates/create_agent_runs.rb.erb new file mode 100644 index 0000000..2d3fab6 --- /dev/null +++ b/lib/generators/solid_agent/install/templates/create_agent_runs.rb.erb @@ -0,0 +1,46 @@ +# frozen_string_literal: true + +class CreateAgentRuns < ActiveRecord::Migration<%= migration_version %> + def change + create_table :agent_runs do |t| + # What the run executed against (an Agent record, a workflow, any + # AR subject) — optional, so bare executors can still record runs. + t.references :runnable, polymorphic: true, index: true + + # Correlation keys shared with agent_contexts/agent_generations and + # telemetry traces. + t.string :agent_name + t.string :action_name + t.string :trace_id + + t.string :status, null: false, default: "pending" + + t.text :input_prompt + t.jsonb :input_params, default: {} + t.text :output + t.jsonb :output_metadata, default: {} + t.text :error_message + + # Append-only progress events ({at, eid, kind, label, status, + # detail, duration_ms}) streamed to pollers mid-run. + t.jsonb :events, default: [] + + # 8-char fingerprint of the instructions the run executed under — + # the cohort grouping key for instruction/model comparisons. + t.string :instructions_digest + + t.integer :input_tokens, default: 0 + t.integer :output_tokens, default: 0 + t.integer :duration_ms + t.datetime :started_at + t.datetime :completed_at + + t.timestamps + end + + add_index :agent_runs, :status + add_index :agent_runs, :trace_id + add_index :agent_runs, :instructions_digest + add_index :agent_runs, :created_at + end +end diff --git a/lib/solid_agent.rb b/lib/solid_agent.rb index 5f9c657..eca3500 100644 --- a/lib/solid_agent.rb +++ b/lib/solid_agent.rb @@ -3,6 +3,7 @@ require_relative "solid_agent/version" require_relative "solid_agent/tool_cache" require_relative "solid_agent/model_pricing" +require_relative "solid_agent/run_fingerprint" module SolidAgent class Error < StandardError; end diff --git a/lib/solid_agent/run_fingerprint.rb b/lib/solid_agent/run_fingerprint.rb new file mode 100644 index 0000000..e24f350 --- /dev/null +++ b/lib/solid_agent/run_fingerprint.rb @@ -0,0 +1,51 @@ +# frozen_string_literal: true + +require "digest" + +module SolidAgent + # Stable fingerprints for the instructions a run executed under — the + # grouping key (with model) for configuration cohorts when comparing + # instruction/model changes across runs. + # + # @example Digest and codename + # digest = SolidAgent::RunFingerprint.digest("You are a helpful agent.") + # # => "a1b2c3d4" + # SolidAgent::RunFingerprint.codename(digest) + # # => "calm-heron" + module RunFingerprint + # Deterministic memorable names for digests — they read far better + # than hex when comparing cohorts, and are stable across runs and + # deployments because they derive from the digest alone. + ADJECTIVES = %w[ + calm brisk quiet bold amber coral dusky fresh golden keen + lively mellow nimble pale rustic silver tidal vivid wry zesty + arid breezy crisp dapper eager foggy hazy icy jolly lunar + misty polar + ].freeze + NOUNS = %w[ + heron otter falcon cedar willow harbor mesa ridge grove delta + prairie summit canyon reef atoll fjord tundra oasis lagoon dune + glacier meadow bluff cove marsh basin knoll strait quarry vale + hollow crag + ].freeze + + class << self + # @param instructions [String, nil] + # @return [String, nil] 8-hex-char digest, nil for blank input + def digest(instructions) + return nil if instructions.nil? || instructions.to_s.strip.empty? + + Digest::SHA256.hexdigest(instructions.to_s)[0, 8] + end + + # @param digest [String, nil] an 8-hex-char instructions digest + # @return [String, nil] deterministic "adjective-noun" codename + def codename(digest) + return nil if digest.nil? || digest.to_s.empty? + + value = digest.to_s.to_i(16) + "#{ADJECTIVES[value % 32]}-#{NOUNS[(value / 32) % 32]}" + end + end + end +end diff --git a/test/solid_agent/run_fingerprint_test.rb b/test/solid_agent/run_fingerprint_test.rb new file mode 100644 index 0000000..d79fb55 --- /dev/null +++ b/test/solid_agent/run_fingerprint_test.rb @@ -0,0 +1,44 @@ +# frozen_string_literal: true + +require "test_helper" + +# SolidAgent::RunFingerprint — stable instruction digests and their +# deterministic codenames, used to group runs into configuration cohorts. +class RunFingerprintTest < Minitest::Test + def test_digest_is_stable_and_short + a = SolidAgent::RunFingerprint.digest("You are a helpful agent.") + b = SolidAgent::RunFingerprint.digest("You are a helpful agent.") + + assert_equal a, b + assert_equal 8, a.length + assert_match(/\A\h{8}\z/, a) + end + + def test_digest_changes_with_instructions + a = SolidAgent::RunFingerprint.digest("You are a helpful agent.") + b = SolidAgent::RunFingerprint.digest("You are a rigorous agent.") + + refute_equal a, b + end + + def test_digest_is_nil_for_blank_input + assert_nil SolidAgent::RunFingerprint.digest(nil) + assert_nil SolidAgent::RunFingerprint.digest("") + assert_nil SolidAgent::RunFingerprint.digest(" ") + end + + def test_codename_is_deterministic_adjective_noun + digest = SolidAgent::RunFingerprint.digest("You are a helpful agent.") + codename = SolidAgent::RunFingerprint.codename(digest) + + assert_equal codename, SolidAgent::RunFingerprint.codename(digest) + adjective, noun = codename.split("-") + assert_includes SolidAgent::RunFingerprint::ADJECTIVES, adjective + assert_includes SolidAgent::RunFingerprint::NOUNS, noun + end + + def test_codename_is_nil_without_digest + assert_nil SolidAgent::RunFingerprint.codename(nil) + assert_nil SolidAgent::RunFingerprint.codename("") + end +end