feat: Add ParseMetadata() function to parity BHCE ingest - BED-7790

README.md

@@ -28,12 +28,12 @@ Want to add the OpenGraph schema to your JSON document?
 
 ```json
 {
-  "$schema": "https://raw.githubusercontent.com/SpecterOps/chow/refs/heads/main/pkg/validator/jsonschema/payload-schema.json"
+  "$schema": "https://raw.githubusercontent.com/SpecterOps/chow/refs/heads/main/pkg/payload/jsonschema/schema.json"
 }
 ```
 
 Most editors will ask you to trust the schema's source. Be sure to add the following URL to your trusted domains
 
 ```text
-https://raw.githubusercontent.com/SpecterOps/chow/refs/heads/main/pkg/validator/jsonschema/
+https://raw.githubusercontent.com/SpecterOps/chow/refs/heads/main/pkg/payload/jsonschema/
 ```

go.mod

@@ -10,7 +10,10 @@ require (
 require (
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/dlclark/regexp2 v1.11.5 // indirect
+	github.com/kr/pretty v0.3.1 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
-	golang.org/x/text v0.34.0 // indirect
+	github.com/rogpeppe/go-internal v1.14.1 // indirect
+	golang.org/x/text v0.35.0 // indirect
+	gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 )

go.sum

@@ -1,16 +1,29 @@
+github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/dlclark/regexp2 v1.11.5 h1:Q/sSnsKerHeCkc/jSTNq1oCm7KiVgUMZRDUoRu0JQZQ=
 github.com/dlclark/regexp2 v1.11.5/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8=
+github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
+github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
+github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
+github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
+github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
+github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
+github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
+github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs=
+github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ=
+github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc=
 github.com/santhosh-tekuri/jsonschema/v6 v6.0.2 h1:KRzFb2m7YtdldCEkzs6KqmJw4nqEVZGK7IN2kJkjTuQ=
 github.com/santhosh-tekuri/jsonschema/v6 v6.0.2/go.mod h1:JXeL+ps8p7/KNMjDQk3TCwPpBy0wYklyWTfbkIzdIFU=
 github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
 github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
-golang.org/x/text v0.34.0 h1:oL/Qq0Kdaqxa1KbNeMKwQq0reLCCaFtqu2eNuSeNHbk=
-golang.org/x/text v0.34.0/go.mod h1:homfLqTYRFyVYemLBFl5GgL/DWEiH5wcsQ5gSh1yziA=
-gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
+golang.org/x/text v0.35.0 h1:JOVx6vVDFokkpaq1AEptVzLTpDe9KGpj5tR4/X+ybL8=
+golang.org/x/text v0.35.0/go.mod h1:khi/HExzZJ2pGnjenulevKNX1W67CUy0AsXcNubPGCA=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
+gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

main.go

@@ -10,7 +10,7 @@ import (
 	"os"
 	"strings"
 
-	"github.com/specterops/chow/pkg/validator"
+	"github.com/specterops/chow/pkg/payload"
 )
 
 var (
@@ -40,13 +40,13 @@ func main() {
 	}
 	defer reader.Close()
 
-	jsonSchema, err := validator.LoadIngestSchema()
+	jsonSchema, err := payload.LoadSchema()
 	if err != nil {
 		slog.Error("Failed to load ingest schema", slog.String("err", err.Error()))
 		os.Exit(1)
 	}
 
-	v := validator.NewValidator(reader, jsonSchema)
+	v := payload.NewValidator(reader, jsonSchema)
 
 	_, report, err := v.ParseAndValidate()
 	validationFailed := err != nil
@@ -76,7 +76,7 @@ func main() {
 	}
 }
 
-func outputReport(w io.WriteCloser, report validator.ValidationReport) error {
+func outputReport(w io.WriteCloser, report payload.ValidationReport) error {
 	for _, e := range report.CriticalErrors {
 		_, err := w.Write([]byte(formatCriticalError(e)))
 		if err != nil {
@@ -108,11 +108,11 @@ func outputReport(w io.WriteCloser, report validator.ValidationReport) error {
 	return nil
 }
 
-func formatCriticalError(e validator.CriticalError) string {
+func formatCriticalError(e payload.CriticalError) string {
 	return fmt.Sprintf("CRITICAL ERROR:\n%s\n%v", e.Message, e.Error)
 }
 
-func formatValidationError(valErr validator.ValidationError) (string, error) {
+func formatValidationError(valErr payload.ValidationError) (string, error) {
 	var (
 		sb       strings.Builder
 		objBytes bytes.Buffer

pkg/validator/jsonschema/edge.json → pkg/payload/jsonschema/edge.json

@@ -57,6 +57,9 @@
         },
         "kind": {
           "type": "string",
+          "not": {
+            "pattern": "^[Tt][Aa][Gg](?:_|$)"
+          },
           "description": "Optional kind filter; the referenced node must have this kind."
         }
       },
@@ -105,7 +108,10 @@
     "kind": {
       "type": "string",
       "description": "Edge kind name must contain only alphanumeric characters and underscores.",
-      "pattern": "^[A-Za-z0-9_]+$"
+      "pattern": "^[A-Za-z0-9_]+$",
+      "not": {
+        "pattern": "^[Tt][Aa][Gg](?:_|$)"
+      }
     },
     "properties": {
       "$ref": "#/$defs/property_map"

pkg/validator/jsonschema/node.json → pkg/payload/jsonschema/node.json

@@ -36,7 +36,12 @@
     },
     "kinds": {
       "type": ["array"],
-      "items": { "type": "string" },
+      "items": {
+        "type": "string",
+        "not": {
+          "pattern": "^[Tt][Aa][Gg](?:_|$)"
+        }
+      },
       "minItems": 0,
       "maxItems": 3,
       "description": "An array of kind labels for the node. The first element is treated as the node's primary kind and is used to determine which icon to display in the graph UI. This primary kind is only used for visual representation and has no semantic significance for data processing."

pkg/validator/schema.go → pkg/payload/schema.go

@@ -13,7 +13,7 @@
 // limitations under the License.
 //
 // SPDX-License-Identifier: Apache-2.0
-package validator
+package payload
 
 import (
 	"bytes"
@@ -26,14 +26,14 @@ import (
 //go:embed jsonschema
 var schemaFiles embed.FS
 
-type IngestSchema struct {
+type Schema struct {
 	NodeSchema *jsonschema.Schema
 	EdgeSchema *jsonschema.Schema
 	MetaSchema *jsonschema.Schema
 }
 
-func LoadIngestSchema() (IngestSchema, error) {
-	var schema IngestSchema
+func LoadSchema() (Schema, error) {
+	var schema Schema
 	if nodeSchema, err := loadSchema("node.json"); err != nil {
 		return schema, err
 	} else if edgeSchema, err := loadSchema("edge.json"); err != nil {

pkg/validator/validator.go → pkg/payload/validator.go

@@ -13,7 +13,7 @@
 // limitations under the License.
 //
 // SPDX-License-Identifier: Apache-2.0
-package validator
+package payload
 
 import (
 	"encoding/json"
@@ -43,7 +43,7 @@ type Validator struct {
 	decoder *json.Decoder
 	depth   int
 
-	schema IngestSchema
+	schema Schema
 
 	originalData  originalData
 	opengraphData opengraphData
@@ -72,7 +72,7 @@ type opengraphData struct {
 	EdgesValidated int
 }
 
-func NewValidator(reader io.Reader, schema IngestSchema) Validator {
+func NewValidator(reader io.Reader, schema Schema) Validator {
 	return Validator{
 		reader:  reader,
 		decoder: json.NewDecoder(reader),
@@ -104,6 +104,31 @@ type ValidationError struct {
 	Errors    []ValidationErrorDetail
 }
 
+func (s ValidationError) Error() string {
+	var (
+		details = make([]string, 0, len(s.Errors))
+		message = "validation error"
+	)
+
+	if s.Location != "" {
+		message = fmt.Sprintf("%s at %s", message, s.Location)
+	}
+
+	for _, validationErrorDetail := range s.Errors {
+		if validationErrorDetail.Location != "" {
+			details = append(details, fmt.Sprintf("%s: %s", validationErrorDetail.Location, validationErrorDetail.Error))
+		} else if validationErrorDetail.Error != "" {
+			details = append(details, validationErrorDetail.Error)
+		}
+	}
+
+	if len(details) > 0 {
+		message = fmt.Sprintf("%s: %s", message, strings.Join(details, "; "))
+	}
+
+	return message
+}
+
 type ValidationErrorDetail struct {
 	Location string
 	Error    string
@@ -122,8 +147,11 @@ type ParsedOpenGraphData struct {
 	EdgesValidated int
 }
 
-// buildParsedData() aggregates data collected during ParseAndValidate() into the ParsedData struct
-func (v *Validator) buildParsedData() ParsedData {
+// buildValidatedData() aggregates data collected during ParseAndValidate() into the ParsedData struct.
+// It is specific to the validation path and relies on signals (GraphFound, NodesValidated, etc.)
+// that are only populated by validationLoop. ParseMetadata() builds its result inline rather than
+// using this helper.
+func (v *Validator) buildValidatedData() ParsedData {
 	p := ParsedData{}
 
 	if (v.opengraphData.GraphFound || v.opengraphData.MetadataFound) && (v.originalData.MetadataFound || v.originalData.DataFound) {
@@ -159,7 +187,7 @@ func (v *Validator) buildValidationReport() ValidationReport {
 
 // result() is a helper for returning the current parsed data, validation report, and provided error.
 func (v *Validator) result(err error) (ParsedData, ValidationReport, error) {
-	return v.buildParsedData(), v.buildValidationReport(), err
+	return v.buildValidatedData(), v.buildValidationReport(), err
 }
 
 // Error Helper functions -------------------------------------------------------------------------
@@ -262,6 +290,32 @@ func (v *Validator) ParseAndValidate() (ParsedData, ValidationReport, error) {
 	return v.result(v.finalizeParse())
 }
 
+// ParseMetadata() walks the top-level JSON object and extracts metadata (either legacy "meta" or
+// opengraph "metadata") without performing schema validation of the payload body. It returns as soon
+// as a metadata tag is successfully decoded; the remainder of the reader is not consumed.
+func (v *Validator) ParseMetadata() (ParsedData, error) {
+	if err := v.enterObject(); err != nil {
+		v.reportCriticalError("failed to enter json object", err)
+		return ParsedData{}, err
+	}
+
+	err := v.parseLoop()
+
+	p := ParsedData{}
+	switch {
+	case v.originalData.MetadataFound:
+		p.PayloadType = v.originalData.Metadata.Type
+		p.LegacyMetadata = v.originalData.Metadata
+	case v.opengraphData.MetadataFound:
+		p.PayloadType = ingest.DataTypeOpenGraph
+		p.OpengraphData.Metadata = v.opengraphData.Metadata
+	case v.opengraphData.GraphFound:
+		p.PayloadType = ingest.DataTypeOpenGraph
+	}
+
+	return p, err
+}
+
 // readToEnd() checks for trailing input if validation succeeded, then consumes all remaining bytes from the decoder
 // buffer and reader while preserving any existing loop error.
 func (v *Validator) readToEnd(loopErr error) error {
@@ -303,7 +357,7 @@ func (v *Validator) finalizeParse() error {
 	return nil
 }
 
-// Validation Loop functions ----------------------------------------------------------------------
+// Loop functions ----------------------------------------------------------------------
 
 // validationLoop() is the primary driver behind the file validation. It walks through the file and directs to
 // child validation functions
@@ -383,6 +437,47 @@ func (v *Validator) validationLoop() error {
 	}
 }
 
+// parseLoop() walks the top-level object looking for tags that identify the payload shape
+// ("meta", "metadata"), decoding any metadata tag into the Validator's internal state and
+// returning as soon as a tag that uniquely identifies the payload type is found or the
+// top-level object is exited.
+func (v *Validator) parseLoop() error {
+	for {
+		if tag, exitedBlock, err := v.nextTagAtDepth(1); err != nil {
+			v.reportCriticalError("failed parsing top level tag", err)
+			return err
+		} else if exitedBlock {
+			return nil
+		} else {
+			switch tag {
+			case "meta":
+				var metadata ingest.OriginalMetadata
+				if err := v.decoder.Decode(&metadata); err != nil {
+					v.reportCriticalError("failed to decode original metadata", err)
+					return err
+				}
+
+				v.originalData.MetadataFound = true
+				v.originalData.Metadata = metadata
+				return nil
+			case "metadata":
+				var metadata ingest.OpengraphMetadata
+				if err := v.decoder.Decode(&metadata); err != nil {
+					v.reportCriticalError("failed to decode opengraph metadata", err)
+					return err
+				}
+
+				v.opengraphData.MetadataFound = true
+				v.opengraphData.Metadata = metadata
+				return nil
+			case "graph":
+				v.opengraphData.GraphFound = true
+			default:
+			}
+		}
+	}
+}
+
 // handleOriginalMetadata() parses and validates original metadata after a "meta" tag is found at the top level
 func (v *Validator) handleOriginalMetadata() (ingest.OriginalMetadata, error) {
 	var originalMetadata ingest.OriginalMetadata