From 1000aef82adaea04e436ecf294dc551ad25f8826 Mon Sep 17 00:00:00 2001 From: Denis Carriere Date: Thu, 23 Jul 2026 10:46:27 -0400 Subject: [PATCH 1/2] graphql: Attribute cache status metrics to deployments --- docs/metrics.md | 4 +- graph/src/data/graphql/load_manager.rs | 85 ++++++++++++++++++++------ graphql/src/runner.rs | 1 + graphql/src/store/resolver.rs | 1 + 4 files changed, 71 insertions(+), 20 deletions(-) diff --git a/docs/metrics.md b/docs/metrics.md index 61c223f8256..49921be7e25 100644 --- a/docs/metrics.md +++ b/docs/metrics.md @@ -46,7 +46,7 @@ Counts **Prometheus metrics register errors** - `metrics_unregister_errors` Counts **Prometheus metrics unregister errors** - `query_cache_status_count` -Count **toplevel GraphQL fields executed** and their cache status +Count **toplevel GraphQL fields executed** by `deployment` and `cache_status` - `query_effort_ms` Moving **average of time spent running queries** - `query_execution_time` @@ -69,4 +69,4 @@ The **number of Postgres connections** currently **checked out** - `store_connection_error_count` The **number of Postgres connections errors** - `store_connection_wait_time_ms` -**Average connection wait time** \ No newline at end of file +**Average connection wait time** diff --git a/graph/src/data/graphql/load_manager.rs b/graph/src/data/graphql/load_manager.rs index 79655e25629..6bc9bf2f474 100644 --- a/graph/src/data/graphql/load_manager.rs +++ b/graph/src/data/graphql/load_manager.rs @@ -1,6 +1,5 @@ //! Utilities to keep moving statistics about queries -use prometheus::core::GenericCounter; use rand::{prelude::Rng, rng}; use std::collections::{HashMap, HashSet}; use std::iter::FromIterator; @@ -9,7 +8,7 @@ use std::time::{Duration, Instant}; use crate::parking_lot::RwLock; -use crate::components::metrics::{Counter, GaugeVec, MetricsRegistry}; +use crate::components::metrics::{CounterVec, GaugeVec, MetricsRegistry}; use crate::components::store::{DeploymentId, PoolWaitStats}; use crate::data::graphql::shape_hash::shape_hash; use crate::data::query::{CacheStatus, QueryExecutionError}; @@ -217,7 +216,7 @@ pub struct LoadManager { /// Per shard state of whether we are killing queries or not kill_state: HashMap>, effort_gauge: Box, - query_counters: HashMap, + query_counters: CounterVec, kill_rate_gauge: Box, } @@ -258,19 +257,13 @@ impl LoadManager { shard_label, ) .expect("failed to create `query_kill_rate` counter"); - let query_counters = CacheStatus::iter() - .map(|s| { - let labels = HashMap::from_iter(vec![("cache_status".to_owned(), s.to_string())]); - let counter = registry - .global_counter( - "query_cache_status_count", - "Count toplevel GraphQL fields executed and their cache status", - labels, - ) - .expect("Failed to register query_counter metric"); - (*s, counter) - }) - .collect::>(); + let query_counters = registry + .global_counter_vec( + "query_cache_status_count", + "Count toplevel GraphQL fields executed by deployment and cache status", + &["deployment", "cache_status"], + ) + .expect("Failed to register query_counter metric"); let effort = HashMap::from_iter( shards @@ -302,14 +295,15 @@ impl LoadManager { pub fn record_work( &self, shard: &str, + deployment_hash: &str, deployment: DeploymentId, shape_hash: u64, duration: Duration, cache_status: CacheStatus, ) { self.query_counters - .get(&cache_status) - .map(GenericCounter::inc); + .with_label_values(&[deployment_hash, cache_status.as_str()]) + .inc(); if !ENV_VARS.load_management_is_disabled() { let qref = QueryRef::new(deployment, shape_hash); if let Some(effort) = self.effort.get(shard) { @@ -550,3 +544,58 @@ impl LoadManager { kill_rate } } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn cache_status_metrics_are_scoped_by_deployment() { + let logger = crate::log::logger(false); + let manager = LoadManager::new( + &logger, + vec!["primary".to_owned()], + Vec::new(), + Arc::new(MetricsRegistry::mock()), + ); + + manager.record_work( + "primary", + "QmDeploymentOne", + DeploymentId::new(1), + 1, + Duration::from_millis(10), + CacheStatus::Hit, + ); + manager.record_work( + "primary", + "QmDeploymentTwo", + DeploymentId::new(2), + 2, + Duration::from_millis(20), + CacheStatus::Miss, + ); + + assert_eq!( + manager + .query_counters + .with_label_values(&["QmDeploymentOne", "hit"]) + .get(), + 1.0 + ); + assert_eq!( + manager + .query_counters + .with_label_values(&["QmDeploymentTwo", "miss"]) + .get(), + 1.0 + ); + assert_eq!( + manager + .query_counters + .with_label_values(&["QmDeploymentOne", "miss"]) + .get(), + 0.0 + ); + } +} diff --git a/graphql/src/runner.rs b/graphql/src/runner.rs index dd4b6919500..e4f6e77088b 100644 --- a/graphql/src/runner.rs +++ b/graphql/src/runner.rs @@ -296,6 +296,7 @@ where self.load_manager.record_work( store.shard(), + req.deployment.as_str(), store.deployment_id(), query_hash, query_start.elapsed(), diff --git a/graphql/src/store/resolver.rs b/graphql/src/store/resolver.rs index 1cfe64c4cbc..6ec6539d9de 100644 --- a/graphql/src/store/resolver.rs +++ b/graphql/src/store/resolver.rs @@ -427,6 +427,7 @@ impl Resolver for StoreResolver { fn record_work(&self, query: &Query, elapsed: Duration, cache_status: CacheStatus) { self.load_manager.record_work( self.store.shard(), + self.deployment.as_str(), self.store.deployment_id(), query.shape_hash, elapsed, From 435b023f0ac10328bfb319deb327d15eae2623a2 Mon Sep 17 00:00:00 2001 From: Denis Carriere Date: Thu, 23 Jul 2026 10:48:21 -0400 Subject: [PATCH 2/2] graphql: Track cache outcome latency --- docs/metrics.md | 2 ++ graph/src/data/graphql/load_manager.rs | 26 +++++++++++++++++++++++++- 2 files changed, 27 insertions(+), 1 deletion(-) diff --git a/docs/metrics.md b/docs/metrics.md index 49921be7e25..5682661e045 100644 --- a/docs/metrics.md +++ b/docs/metrics.md @@ -47,6 +47,8 @@ Counts **Prometheus metrics register errors** Counts **Prometheus metrics unregister errors** - `query_cache_status_count` Count **toplevel GraphQL fields executed** by `deployment` and `cache_status` +- `query_cache_status_duration_seconds` +Observe **toplevel GraphQL field execution time** by `deployment` and `cache_status` - `query_effort_ms` Moving **average of time spent running queries** - `query_execution_time` diff --git a/graph/src/data/graphql/load_manager.rs b/graph/src/data/graphql/load_manager.rs index 6bc9bf2f474..ef04f05948a 100644 --- a/graph/src/data/graphql/load_manager.rs +++ b/graph/src/data/graphql/load_manager.rs @@ -8,7 +8,7 @@ use std::time::{Duration, Instant}; use crate::parking_lot::RwLock; -use crate::components::metrics::{CounterVec, GaugeVec, MetricsRegistry}; +use crate::components::metrics::{CounterVec, GaugeVec, HistogramVec, MetricsRegistry}; use crate::components::store::{DeploymentId, PoolWaitStats}; use crate::data::graphql::shape_hash::shape_hash; use crate::data::query::{CacheStatus, QueryExecutionError}; @@ -217,6 +217,7 @@ pub struct LoadManager { kill_state: HashMap>, effort_gauge: Box, query_counters: CounterVec, + query_cache_duration: Box, kill_rate_gauge: Box, } @@ -264,6 +265,14 @@ impl LoadManager { &["deployment", "cache_status"], ) .expect("Failed to register query_counter metric"); + let query_cache_duration = registry + .new_histogram_vec( + "query_cache_status_duration_seconds", + "Duration of toplevel GraphQL field execution by deployment and cache status", + vec!["deployment".to_owned(), "cache_status".to_owned()], + vec![0.01, 0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0], + ) + .expect("Failed to register query_cache_duration metric"); let effort = HashMap::from_iter( shards @@ -285,6 +294,7 @@ impl LoadManager { kill_state, effort_gauge, query_counters, + query_cache_duration, kill_rate_gauge, } } @@ -304,6 +314,9 @@ impl LoadManager { self.query_counters .with_label_values(&[deployment_hash, cache_status.as_str()]) .inc(); + self.query_cache_duration + .with_label_values(&[deployment_hash, cache_status.as_str()]) + .observe(duration.as_secs_f64()); if !ENV_VARS.load_management_is_disabled() { let qref = QueryRef::new(deployment, shape_hash); if let Some(effort) = self.effort.get(shard) { @@ -597,5 +610,16 @@ mod tests { .get(), 0.0 ); + let hit_duration = manager + .query_cache_duration + .with_label_values(&["QmDeploymentOne", "hit"]); + assert_eq!(hit_duration.get_sample_count(), 1); + assert_eq!(hit_duration.get_sample_sum(), 0.01); + + let miss_duration = manager + .query_cache_duration + .with_label_values(&["QmDeploymentTwo", "miss"]); + assert_eq!(miss_duration.get_sample_count(), 1); + assert_eq!(miss_duration.get_sample_sum(), 0.02); } }