diff --git a/internal/planner/planner.go b/internal/planner/planner.go index 7084dc0..70db6e6 100644 --- a/internal/planner/planner.go +++ b/internal/planner/planner.go @@ -218,14 +218,11 @@ func (tp *TestPlanner) Plan(ctx context.Context) error { } tp.planReport.Split = parallelRunnerSplit - slog.Info("Test execution planning completed", - "parallelRunners", parallelRunners, - "expectedWallTime", parallelRunnerSplit.wallTimeDuration(), - "imbalance", parallelRunnerSplit.imbalanceDuration(), - "expectedTotalRuntime", parallelRunnerSplit.totalRuntimeDuration(), - "testFilesCount", len(tp.testFileWeights)) if settings.GetReportEnabled() { + tp.planReport.DDTestSettings = settings.Get() + tp.planReport.LongSeparateRunnerSuites = tp.longSeparateRunnerSuitesReport(parallelRunners, parallelRunnerSplit) + tp.planReport.SlowestTestSuitesOverall = tp.slowestTestSuitesOverallReport(slowestTestSuitesReportLimit) printPlanReport(tp.reportWriter, tp.planReport) } diff --git a/internal/planner/planner_test.go b/internal/planner/planner_test.go index fab0010..6206be4 100644 --- a/internal/planner/planner_test.go +++ b/internal/planner/planner_test.go @@ -398,6 +398,8 @@ func TestTestPlanner_Setup_WithParallelRunners(t *testing.T) { } runner := NewWithDependencies(mockPlatformDetector, mockOptimizationClient, &MockTestSuiteDurationsClient{}, newDefaultMockCIProviderDetector()) + var reportOutput bytes.Buffer + runner.reportWriter = &reportOutput // Run Setup err := runner.Plan(context.Background()) @@ -416,13 +418,22 @@ func TestTestPlanner_Setup_WithParallelRunners(t *testing.T) { t.Errorf("Expected parallel runners file content '%s', got '%s'", expected, string(content)) } + report := reportOutput.String() + for _, expectedReportLine := range []string{ + "Split\n", + " Runners: 1\n", + " Expected wall time:", + " Imbalance:", + " Total estimated runtime:", + } { + if !strings.Contains(report, expectedReportLine) { + t.Errorf("Expected report to contain %q, got report: %s", expectedReportLine, report) + } + } + logOutput := logs.String() - if !strings.Contains(logOutput, "Test execution planning completed") || - !strings.Contains(logOutput, "parallelRunners=1") || - !strings.Contains(logOutput, "expectedWallTime=") || - !strings.Contains(logOutput, "imbalance=") || - !strings.Contains(logOutput, "expectedTotalRuntime=") { - t.Errorf("Expected planning log with selected split information, got logs: %s", logOutput) + if strings.Contains(logOutput, "Test execution planning completed") { + t.Errorf("Expected selected split information to be reported, not logged, got logs: %s", logOutput) } } diff --git a/internal/planner/report.go b/internal/planner/report.go index 67b9eb5..19f6097 100644 --- a/internal/planner/report.go +++ b/internal/planner/report.go @@ -3,159 +3,23 @@ package planner import ( "fmt" "io" + "slices" "strconv" "strings" "time" ciConstants "github.com/DataDog/ddtest/civisibility/constants" - "github.com/DataDog/ddtest/civisibility/utils/net" "github.com/DataDog/ddtest/internal/runmetadata" + "github.com/DataDog/ddtest/internal/settings" ) -type datadogSettingsReport struct { - Available bool - TestImpactAnalysis bool - TestSkipping bool - TestImpactCollection bool - KnownTests bool - ImpactedTests bool - EarlyFlakeDetection bool - AutoTestRetries bool - FlakyTestManagement bool -} - -func newDatadogSettingsReport(settings *net.SettingsResponseData) datadogSettingsReport { - if settings == nil { - return datadogSettingsReport{} - } - return datadogSettingsReport{ - Available: true, - TestImpactAnalysis: settings.ItrEnabled, - TestSkipping: settings.TestsSkipping, - TestImpactCollection: settings.CodeCoverage, - KnownTests: settings.KnownTestsEnabled, - ImpactedTests: settings.ImpactedTestsEnabled, - EarlyFlakeDetection: settings.EarlyFlakeDetection.Enabled, - AutoTestRetries: settings.FlakyTestRetriesEnabled, - FlakyTestManagement: settings.TestManagement.Enabled, - } -} - -type knownTestsReport struct { - Available bool - Modules int - Suites int - Tests int -} - -func newKnownTestsReport(knownTests *net.KnownTestsResponseData) knownTestsReport { - if knownTests == nil { - return knownTestsReport{} - } - - report := knownTestsReport{ - Available: true, - Modules: len(knownTests.Tests), - } - for _, suites := range knownTests.Tests { - report.Suites += len(suites) - for _, tests := range suites { - report.Tests += len(tests) - } - } - return report -} - -type managedFlakyTestsReport struct { - Available bool - Total int - Quarantined int - Disabled int - AttemptToFix int -} - -func newManagedFlakyTestsReport(testManagementTests *net.TestManagementTestsResponseDataModules) managedFlakyTestsReport { - if testManagementTests == nil { - return managedFlakyTestsReport{} - } - - report := managedFlakyTestsReport{Available: true} - for _, suites := range testManagementTests.Modules { - for _, tests := range suites.Suites { - for _, test := range tests.Tests { - report.Total++ - if test.Properties.Quarantined { - report.Quarantined++ - } - if test.Properties.Disabled { - report.Disabled++ - } - if test.Properties.AttemptToFix { - report.AttemptToFix++ - } - } - } - } - return report -} - -type durationSourceReport struct { - Known int - Default int -} - -type planningReport struct { - TestFilesDiscovered int - FullySkippedFiles int - TestFilesToRun int - DurationSources durationSourceReport - EstimatedTimeSaved float64 -} - -type planReport struct { - RunInfo runmetadata.RunInfo - PlanInfo PlanInfo - DatadogSettings datadogSettingsReport - KnownTests knownTestsReport - SkippableTestsCount int - ManagedFlakyTests managedFlakyTestsReport - Planning planningReport - Split splitScore -} - -func (tp *TestPlanner) newPlanningReport() planningReport { - fullySkippedFiles := len(tp.testFiles) - len(tp.testFileWeights) - if fullySkippedFiles < 0 { - fullySkippedFiles = 0 - } - - return planningReport{ - TestFilesDiscovered: len(tp.testFiles), - FullySkippedFiles: fullySkippedFiles, - TestFilesToRun: len(tp.testFileWeights), - DurationSources: tp.durationSourceReport(), - EstimatedTimeSaved: tp.skippablePercentage, - } -} - -func (tp *TestPlanner) durationSourceReport() durationSourceReport { - var report durationSourceReport - for _, source := range tp.testFileDurationSources { - switch source { - case testFileDurationSourceKnown: - report.Known++ - default: - report.Default++ - } - } - return report -} - func printPlanReport(w io.Writer, report planReport) { reportFprintln(w, "+++ DDTest: plan report") reportFprintln(w) printRunInfoReport(w, report.RunInfo, report.PlanInfo) reportFprintln(w) + printDDTestSettingsReport(w, report.DDTestSettings) + reportFprintln(w) printDatadogSettingsReport(w, report.DatadogSettings) reportFprintln(w) printBackendDataReport(w, report) @@ -163,6 +27,10 @@ func printPlanReport(w io.Writer, report planReport) { printPlanningReport(w, report.Planning) reportFprintln(w) printSplitReport(w, report.Split) + reportFprintln(w) + printLongSeparateRunnerSuitesReport(w, report.LongSeparateRunnerSuites) + reportFprintln(w) + printSlowestTestSuitesOverallReport(w, report.SlowestTestSuitesOverall) } func printRunInfoReport(w io.Writer, runInfo runmetadata.RunInfo, planInfo PlanInfo) { @@ -176,8 +44,29 @@ func printRunInfoReport(w io.Writer, runInfo runmetadata.RunInfo, planInfo PlanI reportFprintf(w, " Runtime tags: %s\n", formatTagList(planInfo.RuntimeTags, ciConstants.RuntimeName, ciConstants.RuntimeVersion)) } +func printDDTestSettingsReport(w io.Writer, config *settings.Config) { + reportFprintln(w, "DDTest settings") + if config == nil { + reportFprintln(w, " Settings: not available") + return + } + + reportFprintf(w, " Platform: %s\n", valueOrNotSet(config.Platform)) + reportFprintf(w, " Framework: %s\n", valueOrNotSet(config.Framework)) + reportFprintf(w, " Min parallelism: %s\n", formatCount(config.MinParallelism)) + reportFprintf(w, " Max parallelism: %s\n", formatCount(config.MaxParallelism)) + reportFprintf(w, " CI job overhead: %s\n", formatDuration(config.ParallelRunnerOverhead)) + reportFprintf(w, " Worker env: %s\n", formatWorkerEnvKeys(config.WorkerEnv)) + reportFprintf(w, " CI node: %s\n", formatCount(config.CiNode)) + reportFprintf(w, " CI node workers: %s\n", formatCount(config.CiNodeWorkers)) + reportFprintf(w, " Command: %s\n", valueOrNotSet(config.Command)) + reportFprintf(w, " Tests location: %s\n", valueOrNotSet(config.TestsLocation)) + reportFprintf(w, " Runtime tags: %s\n", valueOrNotSet(config.RuntimeTags)) + reportFprintf(w, " Report enabled: %s\n", strconv.FormatBool(config.ReportEnabled)) +} + func printDatadogSettingsReport(w io.Writer, report datadogSettingsReport) { - reportFprintln(w, "Datadog") + reportFprintln(w, "Datadog settings") if !report.Available { reportFprintln(w, " Settings: not available") return @@ -219,6 +108,102 @@ func printSplitReport(w io.Writer, report splitScore) { reportFprintf(w, " Total estimated runtime: %s\n", formatDuration(report.totalRuntimeDuration())) } +func printLongSeparateRunnerSuitesReport(w io.Writer, suites []testSuiteTimingReport) { + reportFprintln(w, "Slow suites on dedicated runners") + if len(suites) == 0 { + reportFprintln(w, " None") + return + } + + reportFprintf(w, " ATTENTION: %s\n", formatScheduledTestSuiteCount(len(suites))) + for i, suite := range suites { + printTestSuiteTimingReport(w, i+1, suite, true) + } +} + +func printSlowestTestSuitesOverallReport(w io.Writer, suites []testSuiteTimingReport) { + reportFprintln(w, "10 slowest test suites overall") + if len(suites) == 0 { + reportFprintln(w, " No suite timing data available") + return + } + + for i, suite := range suites { + printTestSuiteTimingReport(w, i+1, suite, false) + } +} + +func printTestSuiteTimingReport(w io.Writer, index int, suite testSuiteTimingReport, includeRunner bool) { + runnerPrefix := "" + if includeRunner { + runnerPrefix = fmt.Sprintf("runner %d, ", suite.Runner) + } + + reportFprintf(w, " %d. %s%s (%s): historical duration %s, estimated runtime %s\n", + index, + runnerPrefix, + formatSuiteLabel(suite), + valueOrNotAvailable(suite.SourceFile), + formatDuration(suite.TotalDuration), + formatDuration(suite.EstimatedDuration)) +} + +func formatSuiteLabel(suite testSuiteTimingReport) string { + switch { + case suite.Module == "" && suite.Suite == "": + return "not available" + case suite.Module == "": + return suite.Suite + case suite.Suite == "": + return suite.Module + default: + return suite.Module + " / " + suite.Suite + } +} + +func formatScheduledTestSuiteCount(count int) string { + if count == 1 { + return "1 dedicated runner" + } + return formatCount(count) + " dedicated runners" +} + +func formatWorkerEnvKeys(workerEnv string) string { + keys := parseWorkerEnvKeys(workerEnv) + if len(keys) == 0 { + return "not set" + } + return strings.Join(keys, ", ") +} + +func parseWorkerEnvKeys(workerEnv string) []string { + if strings.TrimSpace(workerEnv) == "" { + return nil + } + + seen := make(map[string]struct{}) + keys := make([]string, 0) + for pair := range strings.SplitSeq(workerEnv, ";") { + key, _, ok := strings.Cut(pair, "=") + if !ok { + continue + } + key = strings.TrimSpace(key) + if key == "" { + continue + } + if _, ok := seen[key]; ok { + continue + } + + seen[key] = struct{}{} + keys = append(keys, key) + } + + slices.Sort(keys) + return keys +} + func reportFprintln(w io.Writer, args ...any) { _, _ = fmt.Fprintln(w, args...) } @@ -294,6 +279,13 @@ func valueOrNotAvailable(value string) string { return value } +func valueOrNotSet(value string) string { + if value == "" { + return "not set" + } + return value +} + func enabledWord(enabled bool) string { if enabled { return "enabled" diff --git a/internal/planner/report_data.go b/internal/planner/report_data.go new file mode 100644 index 0000000..a2a8bf1 --- /dev/null +++ b/internal/planner/report_data.go @@ -0,0 +1,328 @@ +package planner + +import ( + "slices" + "time" + + "github.com/DataDog/ddtest/civisibility/utils/net" + "github.com/DataDog/ddtest/internal/runmetadata" + "github.com/DataDog/ddtest/internal/settings" +) + +const slowestTestSuitesReportLimit = 10 + +type datadogSettingsReport struct { + Available bool + TestImpactAnalysis bool + TestSkipping bool + TestImpactCollection bool + KnownTests bool + ImpactedTests bool + EarlyFlakeDetection bool + AutoTestRetries bool + FlakyTestManagement bool +} + +func newDatadogSettingsReport(settings *net.SettingsResponseData) datadogSettingsReport { + if settings == nil { + return datadogSettingsReport{} + } + return datadogSettingsReport{ + Available: true, + TestImpactAnalysis: settings.ItrEnabled, + TestSkipping: settings.TestsSkipping, + TestImpactCollection: settings.CodeCoverage, + KnownTests: settings.KnownTestsEnabled, + ImpactedTests: settings.ImpactedTestsEnabled, + EarlyFlakeDetection: settings.EarlyFlakeDetection.Enabled, + AutoTestRetries: settings.FlakyTestRetriesEnabled, + FlakyTestManagement: settings.TestManagement.Enabled, + } +} + +type knownTestsReport struct { + Available bool + Modules int + Suites int + Tests int +} + +func newKnownTestsReport(knownTests *net.KnownTestsResponseData) knownTestsReport { + if knownTests == nil { + return knownTestsReport{} + } + + report := knownTestsReport{ + Available: true, + Modules: len(knownTests.Tests), + } + for _, suites := range knownTests.Tests { + report.Suites += len(suites) + for _, tests := range suites { + report.Tests += len(tests) + } + } + return report +} + +type managedFlakyTestsReport struct { + Available bool + Total int + Quarantined int + Disabled int + AttemptToFix int +} + +func newManagedFlakyTestsReport(testManagementTests *net.TestManagementTestsResponseDataModules) managedFlakyTestsReport { + if testManagementTests == nil { + return managedFlakyTestsReport{} + } + + report := managedFlakyTestsReport{Available: true} + for _, suites := range testManagementTests.Modules { + for _, tests := range suites.Suites { + for _, test := range tests.Tests { + report.Total++ + if test.Properties.Quarantined { + report.Quarantined++ + } + if test.Properties.Disabled { + report.Disabled++ + } + if test.Properties.AttemptToFix { + report.AttemptToFix++ + } + } + } + } + return report +} + +type durationSourceReport struct { + Known int + Default int +} + +type planningReport struct { + TestFilesDiscovered int + FullySkippedFiles int + TestFilesToRun int + DurationSources durationSourceReport + EstimatedTimeSaved float64 +} + +type testSuiteTimingReport struct { + Runner int + Module string + Suite string + SourceFile string + TotalDuration time.Duration + EstimatedDuration time.Duration + DurationSource testFileDurationSource +} + +type planReport struct { + RunInfo runmetadata.RunInfo + PlanInfo PlanInfo + DDTestSettings *settings.Config + DatadogSettings datadogSettingsReport + KnownTests knownTestsReport + SkippableTestsCount int + ManagedFlakyTests managedFlakyTestsReport + Planning planningReport + LongSeparateRunnerSuites []testSuiteTimingReport + SlowestTestSuitesOverall []testSuiteTimingReport + Split splitScore +} + +func (tp *TestPlanner) newPlanningReport() planningReport { + fullySkippedFiles := len(tp.testFiles) - len(tp.testFileWeights) + if fullySkippedFiles < 0 { + fullySkippedFiles = 0 + } + + return planningReport{ + TestFilesDiscovered: len(tp.testFiles), + FullySkippedFiles: fullySkippedFiles, + TestFilesToRun: len(tp.testFileWeights), + DurationSources: tp.durationSourceReport(), + EstimatedTimeSaved: tp.skippablePercentage, + } +} + +func (tp *TestPlanner) durationSourceReport() durationSourceReport { + var report durationSourceReport + for _, source := range tp.testFileDurationSources { + switch source { + case testFileDurationSourceKnown: + report.Known++ + default: + report.Default++ + } + } + return report +} + +func (tp *TestPlanner) longSeparateRunnerSuitesReport(parallelRunners int, split splitScore) []testSuiteTimingReport { + longThreshold := averageRunnerRuntimeDuration(split) + if parallelRunners <= 1 || longThreshold <= 0 { + return nil + } + + distribution := tp.DistributeWeightedTestFiles(tp.testFileWeights, parallelRunners) + suites := make([]testSuiteTimingReport, 0) + for runnerIndex, runnerFiles := range distribution { + if len(runnerFiles) != 1 { + continue + } + + sourceFile := runnerFiles[0] + if time.Duration(tp.testFileWeights[sourceFile])*time.Millisecond <= longThreshold { + continue + } + + for _, key := range tp.suitesBySourceFile[sourceFile] { + aggregate := tp.suiteAggregates[key] + suite := newTestSuiteTimingReport(key, aggregate) + if suite.EstimatedDuration <= longThreshold { + continue + } + suite.Runner = runnerIndex + suites = append(suites, suite) + } + } + + slices.SortFunc(suites, compareSeparateRunnerSuiteTiming) + return suites +} + +func averageRunnerRuntimeDuration(split splitScore) time.Duration { + if split.parallelRunners <= 0 || split.totalRuntime <= 0 { + return 0 + } + return time.Duration(split.totalRuntime/split.parallelRunners) * time.Millisecond +} + +func (tp *TestPlanner) slowestTestSuitesOverallReport(limit int) []testSuiteTimingReport { + if limit <= 0 { + return nil + } + + slowest := make([]testSuiteTimingReport, 0, limit) + for key, aggregate := range tp.suiteAggregates { + suite := newTestSuiteTimingReport(key, aggregate) + if suite.TotalDuration <= 0 && suite.EstimatedDuration <= 0 { + continue + } + slowest = insertSlowestTestSuite(slowest, suite, limit) + } + return slowest +} + +func insertSlowestTestSuite(slowest []testSuiteTimingReport, suite testSuiteTimingReport, limit int) []testSuiteTimingReport { + position := len(slowest) + for position > 0 && compareSuiteTimingByTotalDuration(suite, slowest[position-1]) < 0 { + position-- + } + + if len(slowest) < limit { + slowest = append(slowest, testSuiteTimingReport{}) + copy(slowest[position+1:], slowest[position:]) + slowest[position] = suite + return slowest + } + + if position == len(slowest) { + return slowest + } + + copy(slowest[position+1:], slowest[position:len(slowest)-1]) + slowest[position] = suite + return slowest +} + +func newTestSuiteTimingReport(key testSuiteKey, aggregate testSuiteAggregate) testSuiteTimingReport { + module := aggregate.Module + if module == "" { + module = key.Module + } + suite := aggregate.Suite + if suite == "" { + suite = key.Suite + } + + return testSuiteTimingReport{ + Runner: -1, + Module: module, + Suite: suite, + SourceFile: aggregate.SourceFile, + TotalDuration: durationFromNanoseconds(aggregate.TotalDuration), + EstimatedDuration: durationFromNanoseconds(aggregate.EstimatedDuration), + DurationSource: aggregate.DurationSource, + } +} + +func durationFromNanoseconds(value float64) time.Duration { + if value <= 0 { + return 0 + } + return time.Duration(value) +} + +func compareSeparateRunnerSuiteTiming(a, b testSuiteTimingReport) int { + if a.Runner != b.Runner { + if a.Runner < b.Runner { + return -1 + } + return 1 + } + if result := compareDurationDesc(a.EstimatedDuration, b.EstimatedDuration); result != 0 { + return result + } + if result := compareDurationDesc(a.TotalDuration, b.TotalDuration); result != 0 { + return result + } + return compareSuiteIdentity(a, b) +} + +func compareSuiteTimingByTotalDuration(a, b testSuiteTimingReport) int { + if result := compareDurationDesc(a.TotalDuration, b.TotalDuration); result != 0 { + return result + } + if result := compareDurationDesc(a.EstimatedDuration, b.EstimatedDuration); result != 0 { + return result + } + return compareSuiteIdentity(a, b) +} + +func compareDurationDesc(a, b time.Duration) int { + if a > b { + return -1 + } + if a < b { + return 1 + } + return 0 +} + +func compareSuiteIdentity(a, b testSuiteTimingReport) int { + if a.SourceFile < b.SourceFile { + return -1 + } + if a.SourceFile > b.SourceFile { + return 1 + } + if a.Module < b.Module { + return -1 + } + if a.Module > b.Module { + return 1 + } + if a.Suite < b.Suite { + return -1 + } + if a.Suite > b.Suite { + return 1 + } + return 0 +} diff --git a/internal/planner/report_test.go b/internal/planner/report_test.go index 716868a..9b219ff 100644 --- a/internal/planner/report_test.go +++ b/internal/planner/report_test.go @@ -3,9 +3,11 @@ package planner import ( "strings" "testing" + "time" "github.com/DataDog/ddtest/civisibility/utils/net" "github.com/DataDog/ddtest/internal/runmetadata" + "github.com/DataDog/ddtest/internal/settings" ) func TestPrintPlanReport_AllData(t *testing.T) { @@ -31,6 +33,20 @@ func TestPrintPlanReport_AllData(t *testing.T) { "runtime.version": "3.3.4", }, }, + DDTestSettings: &settings.Config{ + Platform: "ruby", + Framework: "rspec", + MinParallelism: 2, + MaxParallelism: 8, + ParallelRunnerOverhead: 25 * time.Second, + WorkerEnv: "RAILS_ENV=test;DATABASE_PASSWORD=secret", + CiNode: -1, + CiNodeWorkers: 2, + Command: "bundle exec rspec", + TestsLocation: "spec/**/*_spec.rb", + RuntimeTags: `{"runtime.version":"3.3.4"}`, + ReportEnabled: true, + }, DatadogSettings: datadogSettingsReport{ Available: true, TestImpactAnalysis: true, @@ -72,6 +88,35 @@ func TestPrintPlanReport_AllData(t *testing.T) { imbalance: 11000, totalRuntime: 1426000, }, + LongSeparateRunnerSuites: []testSuiteTimingReport{ + { + Runner: 0, + Module: "rspec", + Suite: "Checkout::Slow", + SourceFile: "spec/slow_spec.rb", + TotalDuration: 2 * time.Minute, + EstimatedDuration: 100 * time.Second, + DurationSource: testFileDurationSourceKnown, + }, + }, + SlowestTestSuitesOverall: []testSuiteTimingReport{ + { + Module: "rspec", + Suite: "Checkout::VerySlow", + SourceFile: "spec/very_slow_spec.rb", + TotalDuration: 3 * time.Minute, + EstimatedDuration: 3 * time.Minute, + DurationSource: testFileDurationSourceKnown, + }, + { + Module: "rspec", + Suite: "Checkout::Slow", + SourceFile: "spec/slow_spec.rb", + TotalDuration: 2 * time.Minute, + EstimatedDuration: 100 * time.Second, + DurationSource: testFileDurationSourceKnown, + }, + }, }) expected := `+++ DDTest: plan report @@ -85,7 +130,21 @@ Run OS tags: os.platform=linux, os.architecture=amd64, os.version=6.8.0 Runtime tags: runtime.name=ruby, runtime.version=3.3.4 -Datadog +DDTest settings + Platform: ruby + Framework: rspec + Min parallelism: 2 + Max parallelism: 8 + CI job overhead: 25s + Worker env: DATABASE_PASSWORD, RAILS_ENV + CI node: -1 + CI node workers: 2 + Command: bundle exec rspec + Tests location: spec/**/*_spec.rb + Runtime tags: {"runtime.version":"3.3.4"} + Report enabled: true + +Datadog settings Test Impact Analysis: enabled Test skipping: enabled Test impact collection: disabled @@ -112,6 +171,14 @@ Split Expected wall time: 4m12s Imbalance: 11s Total estimated runtime: 23m46s + +Slow suites on dedicated runners + ATTENTION: 1 dedicated runner + 1. runner 0, rspec / Checkout::Slow (spec/slow_spec.rb): historical duration 2m0s, estimated runtime 1m40s + +10 slowest test suites overall + 1. rspec / Checkout::VerySlow (spec/very_slow_spec.rb): historical duration 3m0s, estimated runtime 3m0s + 2. rspec / Checkout::Slow (spec/slow_spec.rb): historical duration 2m0s, estimated runtime 1m40s ` if output.String() != expected { t.Errorf("unexpected plan report:\n%s", output.String()) @@ -124,6 +191,9 @@ func TestPrintPlanReport_MissingSettingsAndData(t *testing.T) { printPlanReport(&output, planReport{}) report := output.String() + if !strings.Contains(report, "DDTest settings\n Settings: not available") { + t.Errorf("expected missing ddtest settings message, got:\n%s", report) + } if !strings.Contains(report, " Settings: not available") { t.Errorf("expected missing settings message, got:\n%s", report) } @@ -191,3 +261,146 @@ func TestReportSummaries(t *testing.T) { t.Errorf("unexpected managed flaky test summary: %+v", managed) } } + +func TestFormatWorkerEnvKeys(t *testing.T) { + report := formatWorkerEnvKeys("TOKEN=secret; RAILS_ENV = test ;BAD_NO_EQUALS;=NO_KEY;TOKEN=other") + + if report != "RAILS_ENV, TOKEN" { + t.Fatalf("unexpected worker env report: %s", report) + } + if strings.Contains(report, "secret") || strings.Contains(report, "test") || strings.Contains(report, "other") { + t.Fatalf("worker env report leaked values: %s", report) + } +} + +func TestTestPlanner_LongSeparateRunnerSuitesReport(t *testing.T) { + planner := newTestPlannerWithDefaults() + planner.suiteAggregates = map[testSuiteKey]testSuiteAggregate{ + {Module: "rspec", Suite: "SlowSuite"}: { + Module: "rspec", + Suite: "SlowSuite", + SourceFile: "spec/slow_spec.rb", + TotalDuration: float64(2 * time.Minute), + EstimatedDuration: float64(2 * time.Minute), + DurationSource: testFileDurationSourceKnown, + }, + {Module: "rspec", Suite: "ShortSuiteInSlowFile"}: { + Module: "rspec", + Suite: "ShortSuiteInSlowFile", + SourceFile: "spec/slow_spec.rb", + TotalDuration: float64(time.Second), + EstimatedDuration: float64(time.Second), + DurationSource: testFileDurationSourceKnown, + }, + {Module: "rspec", Suite: "MediumSuite"}: { + Module: "rspec", + Suite: "MediumSuite", + SourceFile: "spec/medium_spec.rb", + TotalDuration: float64(30 * time.Second), + EstimatedDuration: float64(30 * time.Second), + DurationSource: testFileDurationSourceKnown, + }, + {Module: "rspec", Suite: "FastSuite"}: { + Module: "rspec", + Suite: "FastSuite", + SourceFile: "spec/fast_spec.rb", + TotalDuration: float64(time.Second), + EstimatedDuration: float64(time.Second), + DurationSource: testFileDurationSourceDefault, + }, + } + planner.suitesBySourceFile = indexSuitesBySourceFile(planner.suiteAggregates) + planner.testFileWeights = map[string]int{ + "spec/slow_spec.rb": int((121 * time.Second) / time.Millisecond), + "spec/medium_spec.rb": int((30 * time.Second) / time.Millisecond), + "spec/fast_spec.rb": int(time.Second / time.Millisecond), + } + + report := planner.longSeparateRunnerSuitesReport(2, splitScore{ + parallelRunners: 2, + totalRuntime: int((152 * time.Second) / time.Millisecond), + }) + + if len(report) != 1 { + t.Fatalf("expected one suite scheduled in a separate runner, got %+v", report) + } + if report[0].Runner != 0 || + report[0].Module != "rspec" || + report[0].Suite != "SlowSuite" || + report[0].SourceFile != "spec/slow_spec.rb" || + report[0].EstimatedDuration != 2*time.Minute { + t.Fatalf("unexpected separate runner suite report: %+v", report[0]) + } +} + +func TestTestPlanner_LongSeparateRunnerSuitesReport_SkipsSingletonRunnersThatAreNotLong(t *testing.T) { + planner := newTestPlannerWithDefaults() + planner.suiteAggregates = map[testSuiteKey]testSuiteAggregate{ + {Module: "rspec", Suite: "SuiteA"}: { + Module: "rspec", + Suite: "SuiteA", + SourceFile: "spec/a_spec.rb", + TotalDuration: float64(10 * time.Second), + EstimatedDuration: float64(10 * time.Second), + DurationSource: testFileDurationSourceKnown, + }, + {Module: "rspec", Suite: "SuiteB"}: { + Module: "rspec", + Suite: "SuiteB", + SourceFile: "spec/b_spec.rb", + TotalDuration: float64(10 * time.Second), + EstimatedDuration: float64(10 * time.Second), + DurationSource: testFileDurationSourceKnown, + }, + {Module: "rspec", Suite: "SuiteC"}: { + Module: "rspec", + Suite: "SuiteC", + SourceFile: "spec/c_spec.rb", + TotalDuration: float64(10 * time.Second), + EstimatedDuration: float64(10 * time.Second), + DurationSource: testFileDurationSourceKnown, + }, + } + planner.suitesBySourceFile = indexSuitesBySourceFile(planner.suiteAggregates) + planner.testFileWeights = map[string]int{ + "spec/a_spec.rb": int((10 * time.Second) / time.Millisecond), + "spec/b_spec.rb": int((10 * time.Second) / time.Millisecond), + "spec/c_spec.rb": int((10 * time.Second) / time.Millisecond), + } + + report := planner.longSeparateRunnerSuitesReport(3, splitScore{ + parallelRunners: 3, + totalRuntime: int((30 * time.Second) / time.Millisecond), + }) + + if len(report) != 0 { + t.Fatalf("expected equal singleton runners not to be reported as long, got %+v", report) + } +} + +func TestTestPlanner_SlowestTestSuitesOverallReport(t *testing.T) { + planner := newTestPlannerWithDefaults() + planner.suiteAggregates = map[testSuiteKey]testSuiteAggregate{} + for i := range 12 { + suite := "Suite" + string(rune('A'+i)) + planner.suiteAggregates[testSuiteKey{Module: "rspec", Suite: suite}] = testSuiteAggregate{ + Module: "rspec", + Suite: suite, + SourceFile: "spec/" + suite + "_spec.rb", + TotalDuration: float64(time.Duration(i+1) * time.Second), + EstimatedDuration: float64(time.Duration(i+1) * time.Second), + DurationSource: testFileDurationSourceKnown, + } + } + + report := planner.slowestTestSuitesOverallReport(10) + if len(report) != 10 { + t.Fatalf("expected 10 slowest suites, got %d", len(report)) + } + if report[0].Suite != "SuiteL" || report[0].TotalDuration != 12*time.Second { + t.Fatalf("expected slowest suite first, got %+v", report[0]) + } + if report[9].Suite != "SuiteC" || report[9].TotalDuration != 3*time.Second { + t.Fatalf("expected 10th slowest suite to be SuiteC, got %+v", report[9]) + } +}