feat: organization-scoped event custom fields with management UI and filter integration

.github/workflows/build.yaml

@@ -217,10 +217,7 @@ jobs:
         run: npm run build
 
       - name: Run Unit Tests
-        run: echo "npm run test:unit"
-
-      - name: Run Integration Tests
-        run: echo "npm run test:integration"
+        run: npm run test:unit
 
   test-e2e:
     runs-on: ubuntu-latest

docs/docs/filtering-and-searching.md

@@ -120,9 +120,11 @@ Specify a `date` or `numeric` range as part of the term.
 
 ## Custom Extended Data
 
-All simple data types (`string`, `boolean`, `date`, `number`) that are stored in extended data will be indexed. _NOTE_: Field names will be lowercased and escaped. Any field name that is not a valid identifier (containing only letter and digits) or is longer than 25 characters will be ignored.
+Extended-data properties are not indexed automatically. An organization administrator must first create a custom event field with the same name and choose its index type. The field name may contain ASCII letters, digits, underscores, dots, and dashes and may be up to 100 characters long.
 
-**Example:** Lets assume that our events extended data contains a property called `Age` with a value of `18`. To search for this value our query would be `data.age:18`.
+Custom-field indexing is forward-only. Only events processed after the definition is created are searchable through it; existing events are not backfilled. Both `data.age:18` and `idx.age:18` resolve to the organization's active custom-field definition.
+
+**Example:** If an administrator creates an integer custom field named `age`, events received afterward with an `age` value of `18` can be found with `data.age:18`.
 
 ***
 

docs/docs/self-hosting/upgrading-self-hosted-instance.md

@@ -8,6 +8,18 @@ title: "Upgrading"
 
 **If you are upgrading from v1 or [v2](https://github.com/exceptionless/Exceptionless/releases/tag/v2.0.0) you will need to upgrade to [v3.0](https://github.com/exceptionless/Exceptionless/releases/tag/v3.0.0) before upgrading to the latest release.**
 
+## Custom event field indexing cutover
+
+The custom event field release replaces automatic indexing of every primitive extended-data property with explicit, organization-scoped definitions. Before upgrading, inventory saved views and integrations that rely on arbitrary `data.*` filters.
+
+After upgrading:
+
+1. Create definitions for the extended-data fields that must remain searchable.
+2. If uninterrupted forward indexing matters, create those definitions before resuming event ingestion.
+3. Plan for forward-only indexing. V1 has no built-in historical backfill, and Elasticsearch reindexing alone does not populate pooled slots. Replaying original payloads is operator-owned and requires a deduplication strategy because it can create duplicate events.
+
+Existing legacy index values remain in Elasticsearch until their events age out, but new custom-field queries use pooled slots and do not search those legacy values. Exceptionless-owned session fields retain dual-read compatibility during this transition.
+
 ## Upgrading from v7.1 to v8
 
 We simplified the self hosting process by integrating the UI into the existing app images. As such `exceptionless/ui` docker images are deprecated and we recommend using `exceptionless/app`.

src/Exceptionless.Core/Bootstrapper.cs

@@ -39,6 +39,7 @@
 using Foundatio.Queues;
 using Foundatio.Repositories.Elasticsearch;
 using Foundatio.Repositories.Elasticsearch.Configuration;
+using Foundatio.Repositories.Elasticsearch.CustomFields;
 using Foundatio.Repositories.Elasticsearch.Jobs;
 using Foundatio.Repositories.Migrations;
 using Foundatio.Resilience;
@@ -78,6 +79,7 @@ public static void RegisterServices(IServiceCollection services, AppOptions appO
         services.AddSingleton<ExceptionlessElasticConfiguration>();
         services.AddSingleton<ElasticsearchClient>(s => s.GetRequiredService<ExceptionlessElasticConfiguration>().Client);
         services.AddSingleton<IElasticConfiguration>(s => s.GetRequiredService<ExceptionlessElasticConfiguration>());
+        services.AddSingleton<ICustomFieldDefinitionRepository>(s => s.GetRequiredService<ExceptionlessElasticConfiguration>().CustomFieldDefinitionRepository!);
         services.AddStartupAction<ExceptionlessElasticConfiguration>();
 
         services.AddSingleton<DataSeedService>();
@@ -178,6 +180,9 @@ public static void RegisterServices(IServiceCollection services, AppOptions appO
         services.AddSingleton<IStripeBillingClient, StripeBillingClient>();
         services.AddSingleton<BillingManager>();
         services.AddSingleton<BillingPlans>();
+        services.AddSingleton<EventCustomFieldQueryPolicy>();
+        services.AddSingleton<EventCustomFieldService>();
+        services.AddStartupAction<EventCustomFieldService>();
         services.AddSingleton<EventPostService>();
         services.AddSingleton<SampleDataService>();
         services.AddSingleton<SemanticVersionParser>();

src/Exceptionless.Core/Configuration/AppOptions.cs

@@ -70,6 +70,7 @@ public class AppOptions
     public int BulkBatchSize { get; internal set; }
 
     public CacheOptions CacheOptions { get; internal set; } = null!;
+    public CustomFieldOptions CustomFieldOptions { get; internal set; } = null!;
     public MessageBusOptions MessageBusOptions { get; internal set; } = null!;
     public QueueOptions QueueOptions { get; internal set; } = null!;
     public StorageOptions StorageOptions { get; internal set; } = null!;
@@ -124,6 +125,7 @@ public static AppOptions ReadFromConfiguration(IConfiguration config)
         catch { }
 
         options.CacheOptions = CacheOptions.ReadFromConfiguration(config, options);
+        options.CustomFieldOptions = CustomFieldOptions.ReadFromConfiguration(config, options);
         options.MessageBusOptions = MessageBusOptions.ReadFromConfiguration(config, options);
         options.QueueOptions = QueueOptions.ReadFromConfiguration(config, options);
         options.StorageOptions = StorageOptions.ReadFromConfiguration(config, options);

src/Exceptionless.Core/Configuration/CustomFieldOptions.cs

@@ -0,0 +1,25 @@
+using Microsoft.Extensions.Configuration;
+
+namespace Exceptionless.Core.Configuration;
+
+public class CustomFieldOptions
+{
+    public int MaxFieldsPerOrganization { get; internal set; }
+    public int MaxLifetimeFieldsPerOrganization { get; internal set; }
+
+    public static CustomFieldOptions ReadFromConfiguration(IConfiguration config, AppOptions appOptions)
+    {
+        int activeLimit = config.GetValue(nameof(MaxFieldsPerOrganization), 20);
+        int lifetimeLimit = config.GetValue(nameof(MaxLifetimeFieldsPerOrganization), activeLimit);
+        if (activeLimit <= 0)
+            throw new ArgumentOutOfRangeException(nameof(MaxFieldsPerOrganization), "Custom field active limit must be greater than zero.");
+        if (lifetimeLimit < activeLimit)
+            throw new ArgumentOutOfRangeException(nameof(MaxLifetimeFieldsPerOrganization), "Custom field lifetime limit must be greater than or equal to the active limit.");
+
+        return new CustomFieldOptions
+        {
+            MaxFieldsPerOrganization = activeLimit,
+            MaxLifetimeFieldsPerOrganization = lifetimeLimit
+        };
+    }
+}

src/Exceptionless.Core/Extensions/PersistentEventExtensions.cs

@@ -10,77 +10,6 @@ public static class PersistentEventExtensions
 {
     private static readonly char[] _commaSeparator = [','];
 
-    public static void CopyDataToIndex(this PersistentEvent ev, string[]? keysToCopy = null)
-    {
-        if (ev.Data is null)
-            return;
-
-        ev.Idx ??= new DataDictionary();
-
-        keysToCopy = keysToCopy?.Length > 0 ? keysToCopy : ev.Data.Keys.ToArray();
-
-        foreach (string key in keysToCopy.Where(k => !String.IsNullOrEmpty(k) && ev.Data.ContainsKey(k)))
-        {
-            string field = key.Trim().ToLowerInvariant();
-
-            if (field.StartsWith("@ref:"))
-            {
-                field = field.Substring(5);
-                if (!field.IsValidFieldName())
-                    continue;
-
-                ev.Idx[field + "-r"] = ev.Data[key]?.ToString();
-                continue;
-            }
-
-            if (field.StartsWith('@') || ev.Data[key] is null)
-                continue;
-
-            if (!field.IsValidFieldName())
-                continue;
-
-            var dataType = ev.Data[key]?.GetType();
-            if (dataType is null)
-                continue;
-
-            if (dataType == typeof(bool))
-            {
-                ev.Idx[field + "-b"] = ev.Data[key];
-            }
-            else if (dataType.IsNumeric())
-            {
-                ev.Idx[field + "-n"] = ev.Data[key];
-            }
-            else if (dataType == typeof(DateTime) || dataType == typeof(DateTimeOffset))
-            {
-                ev.Idx[field + "-d"] = ev.Data[key];
-            }
-            else if (dataType == typeof(string))
-            {
-                string? input = ev.Data[key]?.ToString();
-                if (String.IsNullOrEmpty(input) || input.Length >= 1000)
-                    continue;
-
-                if (input.GetJsonType() != JsonType.None)
-                    continue;
-
-                if (input[0] == '"')
-                    input = input.TrimStart('"').TrimEnd('"');
-
-                if (Boolean.TryParse(input, out bool value))
-                    ev.Idx[field + "-b"] = value;
-                else if (DateTimeOffset.TryParse(input, out var dtoValue))
-                    ev.Idx[field + "-d"] = dtoValue;
-                else if (Decimal.TryParse(input, out decimal decValue))
-                    ev.Idx[field + "-n"] = decValue;
-                else if (Double.TryParse(input, out double dblValue) && !Double.IsNaN(dblValue) && !Double.IsInfinity(dblValue))
-                    ev.Idx[field + "-n"] = dblValue;
-                else
-                    ev.Idx[field + "-s"] = input;
-            }
-        }
-    }
-
     public static string? GetEventReference(this PersistentEvent ev, string name)
     {
         if (String.IsNullOrEmpty(name) || ev.Data is null)
@@ -147,7 +76,7 @@ public static bool HasSessionEndTime(this PersistentEvent ev)
         return null;
     }
 
-    public static bool UpdateSessionStart(this PersistentEvent ev, DateTime lastActivityUtc, bool isSessionEnd = false)
+    public static bool UpdateSessionStart(this PersistentEvent ev, DateTime lastActivityUtc, bool isSessionEnd = false, bool hasError = false)
     {
         if (!ev.IsSessionStart())
             return false;
@@ -168,18 +97,19 @@ public static bool UpdateSessionStart(this PersistentEvent ev, DateTime lastActi
         if (isSessionEnd)
         {
             ev.Data[Event.KnownDataKeys.SessionEnd] = lastActivityUtc;
-            ev.CopyDataToIndex([Event.KnownDataKeys.SessionEnd]);
         }
         else
         {
             ev.Data.Remove(Event.KnownDataKeys.SessionEnd);
-            ev.Idx?.Remove(Event.KnownDataKeys.SessionEnd + "-d");
         }
 
+        if (hasError)
+            ev.Data[Event.KnownDataKeys.SessionHasError] = true;
+
         return true;
     }
 
-    public static PersistentEvent ToSessionStartEvent(this PersistentEvent source, ITextSerializer serializer, ILogger logger, DateTime? lastActivityUtc = null, bool? isSessionEnd = null, bool hasPremiumFeatures = true, bool includePrivateInformation = true)
+    public static PersistentEvent ToSessionStartEvent(this PersistentEvent source, ITextSerializer serializer, ILogger logger, DateTime? lastActivityUtc = null, bool? isSessionEnd = null, bool includePrivateInformation = true, bool hasError = false)
     {
         var startEvent = new PersistentEvent
         {
@@ -238,10 +168,7 @@ public static PersistentEvent ToSessionStartEvent(this PersistentEvent source, I
         }
 
         if (lastActivityUtc.HasValue)
-            startEvent.UpdateSessionStart(lastActivityUtc.Value, isSessionEnd.GetValueOrDefault());
-
-        if (hasPremiumFeatures)
-            startEvent.CopyDataToIndex([]);
+            startEvent.UpdateSessionStart(lastActivityUtc.Value, isSessionEnd.GetValueOrDefault(), hasError);
 
         return startEvent;
     }

src/Exceptionless.Core/Jobs/WorkItemHandlers/OrganizationMaintenanceWorkItemHandler.cs

@@ -2,6 +2,7 @@
 using Exceptionless.Core.Models;
 using Exceptionless.Core.Models.WorkItems;
 using Exceptionless.Core.Repositories;
+using Exceptionless.Core.Services;
 using Foundatio.Jobs;
 using Foundatio.Lock;
 using Foundatio.Repositories;
@@ -13,13 +14,15 @@ public class OrganizationMaintenanceWorkItemHandler : WorkItemHandlerBase
 {
     private readonly IOrganizationRepository _organizationRepository;
     private readonly BillingManager _billingManager;
+    private readonly EventCustomFieldService _eventCustomFieldService;
     private readonly TimeProvider _timeProvider;
     private readonly ILockProvider _lockProvider;
 
-    public OrganizationMaintenanceWorkItemHandler(IOrganizationRepository organizationRepository, ILockProvider lockProvider, BillingManager billingManager, TimeProvider timeProvider, ILoggerFactory loggerFactory) : base(loggerFactory)
+    public OrganizationMaintenanceWorkItemHandler(IOrganizationRepository organizationRepository, ILockProvider lockProvider, BillingManager billingManager, EventCustomFieldService eventCustomFieldService, TimeProvider timeProvider, ILoggerFactory loggerFactory) : base(loggerFactory)
     {
         _organizationRepository = organizationRepository;
         _billingManager = billingManager;
+        _eventCustomFieldService = eventCustomFieldService;
         _timeProvider = timeProvider;
         _lockProvider = lockProvider;
     }
@@ -34,7 +37,8 @@ public override async Task HandleItemAsync(WorkItemContext context)
         const int LIMIT = 100;
         var wi = context.GetData<OrganizationMaintenanceWorkItem>()!;
 
-        Log.LogInformation("Received upgrade organizations work item. Upgrade Plans: {UpgradePlans}", wi.UpgradePlans);
+        Log.LogInformation("Received organization maintenance work item. UpgradePlans: {UpgradePlans} RemoveOldUsageStats: {RemoveOldUsageStats} EnsureSystemCustomFields: {EnsureSystemCustomFields}",
+            wi.UpgradePlans, wi.RemoveOldUsageStats, wi.EnsureSystemCustomFields);
 
         var results = await _organizationRepository.GetAllAsync(o => o.PageLimit(LIMIT));
         while (results.Documents.Count > 0 && !context.CancellationToken.IsCancellationRequested)
@@ -53,6 +57,18 @@ public override async Task HandleItemAsync(WorkItemContext context)
                     foreach (var usage in organization.Usage.Where(u => u.Date < utcNow.Subtract(TimeSpan.FromDays(366))).ToList())
                         organization.Usage.Remove(usage);
                 }
+
+                if (wi.EnsureSystemCustomFields)
+                {
+                    try
+                    {
+                        await _eventCustomFieldService.EnsureSystemFieldsAsync(organization.Id);
+                    }
+                    catch (Exception ex) when (ex is not OperationCanceledException)
+                    {
+                        Log.LogError(ex, "Error ensuring system custom fields for organization {OrganizationId}", organization.Id);
+                    }
+                }
             }
 
             if (wi.UpgradePlans || wi.RemoveOldUsageStats)

src/Exceptionless.Core/Models/PersistentEvent.cs

@@ -2,12 +2,13 @@
 using System.Diagnostics;
 using Exceptionless.Core.Attributes;
 using Exceptionless.Core.Extensions;
+using Foundatio.Repositories.Elasticsearch.CustomFields;
 using Foundatio.Repositories.Models;
 
 namespace Exceptionless.Core.Models;
 
 [DebuggerDisplay("Id: {Id}, Type: {Type}, Date: {Date}, Message: {Message}, Value: {Value}, Count: {Count}")]
-public class PersistentEvent : Event, IOwnedByOrganizationAndProjectAndStackWithIdentity, IHaveCreatedDate, IValidatableObject
+public class PersistentEvent : Event, IOwnedByOrganizationAndProjectAndStackWithIdentity, IHaveCreatedDate, IValidatableObject, IHaveVirtualCustomFields
 {
     /// <summary>
     /// Unique id that identifies an event.
@@ -52,6 +53,28 @@ public class PersistentEvent : Event, IOwnedByOrganizationAndProjectAndStackWith
     [MiniValidation.SkipRecursion]
     public DataDictionary? Idx { get; set; }
 
+    // IHaveVirtualCustomFields explicit implementation
+    IDictionary<string, object> IHaveVirtualCustomFields.Idx => (IDictionary<string, object>)(Idx ??= new DataDictionary());
+
+    public string GetTenantKey() => OrganizationId;
+
+    public IDictionary<string, object?> GetCustomFields()
+    {
+        if (Data is null) return new DataDictionary();
+        var result = new Dictionary<string, object?>(StringComparer.OrdinalIgnoreCase);
+        foreach (var kvp in Data.Where(kvp => !String.IsNullOrEmpty(kvp.Key)
+            && (!kvp.Key.StartsWith('@') || kvp.Key.StartsWith("@ref:", StringComparison.OrdinalIgnoreCase))
+            && kvp.Value is string or bool or int or long or float or double or decimal or DateTime or DateTimeOffset))
+        {
+            result[kvp.Key] = kvp.Value;
+        }
+        return result;
+    }
+
+    public object GetCustomField(string name) => Data is not null && Data.TryGetValue(name, out var v) && v is not null ? v : null!;
+    public void SetCustomField(string name, object value) { Data ??= new DataDictionary(); Data[name] = value; }
+    public void RemoveCustomField(string name) => Data?.Remove(name);
+
     public IEnumerable<ValidationResult> Validate(ValidationContext validationContext)
     {
         if (Date == DateTimeOffset.MinValue)

src/Exceptionless.Core/Models/WorkItems/OrganizationMaintenanceWorkItem.cs

@@ -4,4 +4,5 @@ public class OrganizationMaintenanceWorkItem
 {
     public bool UpgradePlans { get; set; }
     public bool RemoveOldUsageStats { get; set; }
+    public bool EnsureSystemCustomFields { get; set; }
 }