edac: Add DIMM/rank layer support

collector/edac_linux.go

@@ -34,6 +34,7 @@ var (
 	edacMemControllerRE = regexp.MustCompile(`.*devices/system/edac/mc/mc([0-9]*)`)
 	edacMemCsrowRE      = regexp.MustCompile(`.*devices/system/edac/mc/mc[0-9]*/csrow([0-9]*)`)
 	edacMemChannelRE    = regexp.MustCompile(`ch([0-9]+)_ce_count`)
+	edacMemDimmRE       = regexp.MustCompile(`^(?:dimm|rank)([0-9]+)$`)
 )
 
 type edacCollector struct {
@@ -75,6 +76,17 @@ var (
 		"Total uncorrectable memory errors for this channel.",
 		[]string{"controller", "csrow", "channel", "dimm_label"}, nil,
 	)
+	edacDimmCECount = prometheus.NewDesc(
+		prometheus.BuildFQName(namespace, edacSubsystem, "dimm_correctable_errors_total"),
+		"Total correctable memory errors for this DIMM.",
+		[]string{"controller", "dimm", "dimm_label"}, nil,
+	)
+	edacDimmUECount = prometheus.NewDesc(
+		prometheus.BuildFQName(namespace, edacSubsystem, "dimm_uncorrectable_errors_total"),
+		"Total uncorrectable memory errors for this DIMM. Best effort: errors the "+
+			"controller cannot localize to a slot are counted in ue_noinfo_count instead.",
+		[]string{"controller", "dimm", "dimm_label"}, nil,
+	)
 )
 
 // NewEdacCollector returns a new Collector exposing edac stats.
@@ -137,11 +149,19 @@ func (c *edacCollector) Update(ch chan<- prometheus.Metric) error {
 		ch <- prometheus.MustNewConstMetric(
 			edacCsRowUECount, prometheus.CounterValue, float64(value), controllerNumber, "unknown")
 
-		// For each controller, walk the csrow directories.
+		// Controllers with no csrow* keep their per-location counters in the DIMM
+		// layer instead. csrow* wins where both exist: amd64_edac_mod repeats the
+		// same counts under rank*, so reading both would double-count.
 		csrows, err := filepath.Glob(controller + "/csrow[0-9]*")
 		if err != nil {
 			return err
 		}
+		if len(csrows) == 0 {
+			if err := c.updateDimms(ch, controller, controllerNumber); err != nil {
+				return err
+			}
+			continue
+		}
 		for _, csrow := range csrows {
 			csrowMatch := edacMemCsrowRE.FindStringSubmatch(csrow)
 			if csrowMatch == nil {
@@ -209,3 +229,66 @@ func (c *edacCollector) Update(ch chan<- prometheus.Metric) error {
 
 	return nil
 }
+
+// updateDimms reads the DIMM/MEM layer, spelled dimm* or rank* depending on the
+// driver. Only populated slots are registered, so the indices are sparse.
+func (c *edacCollector) updateDimms(ch chan<- prometheus.Metric, controller, controllerNumber string) error {
+	var dimms []string
+	for _, pattern := range []string{"/dimm[0-9]*", "/rank[0-9]*"} {
+		matches, err := filepath.Glob(controller + pattern)
+		if err != nil {
+			return err
+		}
+		dimms = append(dimms, matches...)
+	}
+
+	for _, dimm := range dimms {
+		match := edacMemDimmRE.FindStringSubmatch(filepath.Base(dimm))
+		if match == nil {
+			continue
+		}
+		dimmNumber := match[1]
+		label := edacDimmLabelFromDir(dimm)
+
+		value, err := readUintFromFile(filepath.Join(dimm, "dimm_ce_count"))
+		if err != nil {
+			c.logger.Debug("couldn't get dimm_ce_count", "controller", controllerNumber, "dimm", dimmNumber, "err", err)
+			continue
+		}
+		ch <- prometheus.MustNewConstMetric(
+			edacDimmCECount,
+			prometheus.CounterValue,
+			float64(value),
+			controllerNumber,
+			dimmNumber,
+			label,
+		)
+
+		value, err = readUintFromFile(filepath.Join(dimm, "dimm_ue_count"))
+		if err != nil {
+			c.logger.Debug("couldn't get dimm_ue_count", "controller", controllerNumber, "dimm", dimmNumber, "err", err)
+			continue
+		}
+		ch <- prometheus.MustNewConstMetric(
+			edacDimmUECount,
+			prometheus.CounterValue,
+			float64(value),
+			controllerNumber,
+			dimmNumber,
+			label,
+		)
+	}
+
+	return nil
+}
+
+// edacDimmLabelFromDir returns dimm_label verbatim. Unlike the csrow layer's
+// generated ch*_dimm_label, this one is supplied by the platform and names a
+// physical slot, so it is kept byte-identical to what dmidecode reports.
+func edacDimmLabelFromDir(dimm string) string {
+	labelBytes, err := os.ReadFile(filepath.Join(dimm, "dimm_label"))
+	if err != nil {
+		return "unknown"
+	}
+	return strings.TrimSpace(string(labelBytes))
+}

collector/edac_linux_test.go

@@ -0,0 +1,139 @@
+// Copyright The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//go:build !noedac
+
+package collector
+
+import (
+	"fmt"
+	"io"
+	"log/slog"
+	"strings"
+	"testing"
+
+	"github.com/alecthomas/kingpin/v2"
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/testutil"
+	dto "github.com/prometheus/client_model/go"
+)
+
+// newTestEdacRegistry points the collector at the sysfs fixture tree and returns
+// a registry it has been registered against.
+func newTestEdacRegistry(t *testing.T) *prometheus.Registry {
+	t.Helper()
+
+	if _, err := kingpin.CommandLine.Parse([]string{
+		"--path.sysfs", "fixtures/sys",
+		"--collector.edac",
+	}); err != nil {
+		t.Fatal(err)
+	}
+
+	logger := slog.New(slog.NewTextHandler(io.Discard, nil))
+	c, err := NewEdacCollector(logger)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	reg := prometheus.NewRegistry()
+	reg.MustRegister(&testEdacCollector{ec: c})
+	return reg
+}
+
+// Fixture mc1 registers dimm0, dimm1 and dimm3 to cover sparse indices; mc2 uses
+// the rank* spelling and omits dimm_ue_count on rank1.
+func TestEdacDimmLayer(t *testing.T) {
+	reg := newTestEdacRegistry(t)
+
+	expected := `
+# HELP node_edac_dimm_correctable_errors_total Total correctable memory errors for this DIMM.
+# TYPE node_edac_dimm_correctable_errors_total counter
+node_edac_dimm_correctable_errors_total{controller="1",dimm="0",dimm_label="CPU_SrcID#0_MC#1_Chan#0_DIMM#0"} 100
+node_edac_dimm_correctable_errors_total{controller="1",dimm="1",dimm_label="CPU_SrcID#0_MC#1_Chan#1_DIMM#0"} 200
+node_edac_dimm_correctable_errors_total{controller="1",dimm="3",dimm_label="CPU_SrcID#0_MC#1_Chan#3_DIMM#0"} 300
+node_edac_dimm_correctable_errors_total{controller="2",dimm="0",dimm_label="PROC 1 DIMM 8"} 7
+node_edac_dimm_correctable_errors_total{controller="2",dimm="1",dimm_label="PROC 2 DIMM 10"} 9
+# HELP node_edac_dimm_uncorrectable_errors_total Total uncorrectable memory errors for this DIMM. Best effort: errors the controller cannot localize to a slot are counted in ue_noinfo_count instead.
+# TYPE node_edac_dimm_uncorrectable_errors_total counter
+node_edac_dimm_uncorrectable_errors_total{controller="1",dimm="0",dimm_label="CPU_SrcID#0_MC#1_Chan#0_DIMM#0"} 0
+node_edac_dimm_uncorrectable_errors_total{controller="1",dimm="1",dimm_label="CPU_SrcID#0_MC#1_Chan#1_DIMM#0"} 0
+node_edac_dimm_uncorrectable_errors_total{controller="1",dimm="3",dimm_label="CPU_SrcID#0_MC#1_Chan#3_DIMM#0"} 1
+node_edac_dimm_uncorrectable_errors_total{controller="2",dimm="0",dimm_label="PROC 1 DIMM 8"} 0
+`
+
+	if err := testutil.GatherAndCompare(reg, strings.NewReader(expected),
+		"node_edac_dimm_correctable_errors_total",
+		"node_edac_dimm_uncorrectable_errors_total",
+	); err != nil {
+		t.Fatal(err)
+	}
+}
+
+// Fixture mc3 models amd64_edac_mod, where rank* repeats the csrow counts. Its
+// rank* dirs hold 999 so double-counting shows up loudly.
+func TestEdacCsrowTakesPriorityOverDimmLayer(t *testing.T) {
+	reg := newTestEdacRegistry(t)
+
+	families, err := reg.Gather()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	for _, family := range families {
+		if !strings.HasPrefix(family.GetName(), "node_edac_dimm_") {
+			continue
+		}
+		for _, metric := range family.GetMetric() {
+			for _, label := range metric.GetLabel() {
+				if label.GetName() == "controller" && label.GetValue() == "3" {
+					t.Errorf("%s reported for controller 3, which exposes csrow* alongside rank*: "+
+						"the DIMM layer must be skipped there or counts are doubled (value %v, dimm_label %q)",
+						family.GetName(), metric.GetCounter().GetValue(), edacLabelValue(metric, "dimm_label"))
+				}
+			}
+		}
+	}
+}
+
+func edacLabelValue(metric *dto.Metric, name string) string {
+	for _, label := range metric.GetLabel() {
+		if label.GetName() == name {
+			return label.GetValue()
+		}
+	}
+	return ""
+}
+
+// testEdacCollector adapts the Collector interface for use with a registry.
+type testEdacCollector struct {
+	ec Collector
+}
+
+func (tc *testEdacCollector) Collect(ch chan<- prometheus.Metric) {
+	sink := make(chan prometheus.Metric)
+	go func() {
+		if err := tc.ec.Update(sink); err != nil {
+			panic(fmt.Errorf("failed to update collector: %s", err))
+		}
+		close(sink)
+	}()
+
+	for m := range sink {
+		ch <- m
+	}
+}
+
+func (tc *testEdacCollector) Describe(_ chan<- *prometheus.Desc) {
+	// No-op for testing.
+}

collector/fixtures/e2e-output.txt

@@ -1400,33 +1400,62 @@ node_drbd_remote_unacknowledged{device="drbd1"} 12347
 node_edac_channel_correctable_errors_total{channel="0",controller="0",csrow="0",dimm_label="mc0_csrow0_channel0"} 0
 node_edac_channel_correctable_errors_total{channel="0",controller="0",csrow="1",dimm_label="mc0_csrow1_channel0"} 0
 node_edac_channel_correctable_errors_total{channel="0",controller="0",csrow="2",dimm_label="mc0_csrow2_channel0"} 0
+node_edac_channel_correctable_errors_total{channel="0",controller="3",csrow="0",dimm_label="mc3_csrow0_channel0"} 3
 node_edac_channel_correctable_errors_total{channel="1",controller="0",csrow="0",dimm_label="mc0_csrow0_channel1"} 0
 node_edac_channel_correctable_errors_total{channel="1",controller="0",csrow="1",dimm_label="mc0_csrow1_channel1"} 0
 node_edac_channel_correctable_errors_total{channel="1",controller="0",csrow="2",dimm_label="mc0_csrow2_channel1"} 0
 # HELP node_edac_channel_uncorrectable_errors_total Total uncorrectable memory errors for this channel.
 # TYPE node_edac_channel_uncorrectable_errors_total counter
 node_edac_channel_uncorrectable_errors_total{channel="0",controller="0",csrow="0",dimm_label="mc0_csrow0_channel0"} 2
 node_edac_channel_uncorrectable_errors_total{channel="0",controller="0",csrow="1",dimm_label="mc0_csrow1_channel0"} 2
+node_edac_channel_uncorrectable_errors_total{channel="0",controller="3",csrow="0",dimm_label="mc3_csrow0_channel0"} 0
 node_edac_channel_uncorrectable_errors_total{channel="1",controller="0",csrow="0",dimm_label="mc0_csrow0_channel1"} 2
 node_edac_channel_uncorrectable_errors_total{channel="1",controller="0",csrow="1",dimm_label="mc0_csrow1_channel1"} 2
 # HELP node_edac_correctable_errors_total Total correctable memory errors.
 # TYPE node_edac_correctable_errors_total counter
 node_edac_correctable_errors_total{controller="0"} 1
+node_edac_correctable_errors_total{controller="1"} 10
+node_edac_correctable_errors_total{controller="2"} 5
+node_edac_correctable_errors_total{controller="3"} 3
 # HELP node_edac_csrow_correctable_errors_total Total correctable memory errors for this csrow.
 # TYPE node_edac_csrow_correctable_errors_total counter
 node_edac_csrow_correctable_errors_total{controller="0",csrow="0"} 3
 node_edac_csrow_correctable_errors_total{controller="0",csrow="1"} 0
 node_edac_csrow_correctable_errors_total{controller="0",csrow="2"} 0
 node_edac_csrow_correctable_errors_total{controller="0",csrow="unknown"} 2
+node_edac_csrow_correctable_errors_total{controller="1",csrow="unknown"} 1
+node_edac_csrow_correctable_errors_total{controller="2",csrow="unknown"} 0
+node_edac_csrow_correctable_errors_total{controller="3",csrow="0"} 3
+node_edac_csrow_correctable_errors_total{controller="3",csrow="unknown"} 0
 # HELP node_edac_csrow_uncorrectable_errors_total Total uncorrectable memory errors for this csrow.
 # TYPE node_edac_csrow_uncorrectable_errors_total counter
 node_edac_csrow_uncorrectable_errors_total{controller="0",csrow="0"} 4
 node_edac_csrow_uncorrectable_errors_total{controller="0",csrow="1"} 4
 node_edac_csrow_uncorrectable_errors_total{controller="0",csrow="2"} 5
 node_edac_csrow_uncorrectable_errors_total{controller="0",csrow="unknown"} 6
+node_edac_csrow_uncorrectable_errors_total{controller="1",csrow="unknown"} 1
+node_edac_csrow_uncorrectable_errors_total{controller="2",csrow="unknown"} 0
+node_edac_csrow_uncorrectable_errors_total{controller="3",csrow="0"} 0
+node_edac_csrow_uncorrectable_errors_total{controller="3",csrow="unknown"} 0
+# HELP node_edac_dimm_correctable_errors_total Total correctable memory errors for this DIMM.
+# TYPE node_edac_dimm_correctable_errors_total counter
+node_edac_dimm_correctable_errors_total{controller="1",dimm="0",dimm_label="CPU_SrcID#0_MC#1_Chan#0_DIMM#0"} 100
+node_edac_dimm_correctable_errors_total{controller="1",dimm="1",dimm_label="CPU_SrcID#0_MC#1_Chan#1_DIMM#0"} 200
+node_edac_dimm_correctable_errors_total{controller="1",dimm="3",dimm_label="CPU_SrcID#0_MC#1_Chan#3_DIMM#0"} 300
+node_edac_dimm_correctable_errors_total{controller="2",dimm="0",dimm_label="PROC 1 DIMM 8"} 7
+node_edac_dimm_correctable_errors_total{controller="2",dimm="1",dimm_label="PROC 2 DIMM 10"} 9
+# HELP node_edac_dimm_uncorrectable_errors_total Total uncorrectable memory errors for this DIMM. Best effort: errors the controller cannot localize to a slot are counted in ue_noinfo_count instead.
+# TYPE node_edac_dimm_uncorrectable_errors_total counter
+node_edac_dimm_uncorrectable_errors_total{controller="1",dimm="0",dimm_label="CPU_SrcID#0_MC#1_Chan#0_DIMM#0"} 0
+node_edac_dimm_uncorrectable_errors_total{controller="1",dimm="1",dimm_label="CPU_SrcID#0_MC#1_Chan#1_DIMM#0"} 0
+node_edac_dimm_uncorrectable_errors_total{controller="1",dimm="3",dimm_label="CPU_SrcID#0_MC#1_Chan#3_DIMM#0"} 1
+node_edac_dimm_uncorrectable_errors_total{controller="2",dimm="0",dimm_label="PROC 1 DIMM 8"} 0
 # HELP node_edac_uncorrectable_errors_total Total uncorrectable memory errors.
 # TYPE node_edac_uncorrectable_errors_total counter
 node_edac_uncorrectable_errors_total{controller="0"} 5
+node_edac_uncorrectable_errors_total{controller="1"} 2
+node_edac_uncorrectable_errors_total{controller="2"} 0
+node_edac_uncorrectable_errors_total{controller="3"} 0
 # HELP node_entropy_available_bits Bits of available entropy.
 # TYPE node_entropy_available_bits gauge
 node_entropy_available_bits 1337