diff --git a/eng/pipelines/common/templates/jobs/ci-run-tests-job.yml b/eng/pipelines/common/templates/jobs/ci-run-tests-job.yml index 02cc6d58de..4d52e45a63 100644 --- a/eng/pipelines/common/templates/jobs/ci-run-tests-job.yml +++ b/eng/pipelines/common/templates/jobs/ci-run-tests-job.yml @@ -89,6 +89,10 @@ parameters: type: boolean default: false + # If true, only flaky tests will be run. If false, all non-flaky tests will be run. + - name: flakyTestsOnly + type: boolean + jobs: - job: ${{ format('{0}', coalesce(parameters.jobDisplayName, parameters.image, 'unknown_image')) }} @@ -284,6 +288,7 @@ jobs: referenceType: ${{ parameters.referenceType }} testSet: ${{ parameters.testSet }} operatingSystem: ${{ parameters.operatingSystem }} + flakyTestsOnly: ${{ parameters.flakyTestsOnly }} - ${{ if and(eq(parameters.enableX86Test, true), eq(parameters.operatingSystem, 'Windows')) }}: # Set up for x86 tests by manually installing dotnet for x86 to an alternative location. This @@ -310,6 +315,7 @@ jobs: msbuildArchitecture: x86 dotnetx86RootPath: $(dotnetx86RootPath) operatingSystem: ${{ parameters.operatingSystem }} + flakyTestsOnly: ${{ parameters.flakyTestsOnly }} - ${{ if eq(parameters.publishTestResults, true) }}: - template: /eng/pipelines/common/templates/steps/publish-test-results-step.yml@self diff --git a/eng/pipelines/common/templates/stages/ci-run-tests-stage.yml b/eng/pipelines/common/templates/stages/ci-run-tests-stage.yml index 54c7b2c61f..0592e53420 100644 --- a/eng/pipelines/common/templates/stages/ci-run-tests-stage.yml +++ b/eng/pipelines/common/templates/stages/ci-run-tests-stage.yml @@ -24,7 +24,7 @@ parameters: - name: mdsPackageVersion type: string - + - name: postTestJobs type: jobList default: [] @@ -46,6 +46,10 @@ parameters: - name: testJobTimeout type: number + # If true, only flaky tests will be run. If false, all non-flaky tests will be run. + - name: flakyTestsOnly + type: boolean + stages: - ${{ each config in parameters.testConfigurations }}: - ${{ each image in config.value.images }}: @@ -79,6 +83,7 @@ stages: configSqlFor: ${{ config.value.configSqlFor }} operatingSystem: ${{ config.value.operatingSystem }} isArm64: ${{ eq(config.value.isArm64, 'true') }} + flakyTestsOnly: ${{ parameters.flakyTestsOnly }} ${{if ne(config.value.configProperties, '{}') }}: ${{ each x86TF in config.value.configProperties.x86TestTargetFrameworks }}: ${{ if eq(x86TF, targetFramework) }}: @@ -103,7 +108,7 @@ stages: useManagedSNI: ${{ useManagedSNI }} mdsArtifactsName: ${{ parameters.mdsArtifactsName }} mdsPackageVersion: ${{ parameters.mdsPackageVersion }} - prebuildSteps: ${{ parameters.prebuildSteps }} + prebuildSteps: ${{ parameters.prebuildSteps }} targetFramework: ${{ targetFramework }} netcoreVersionTestUtils: ${{config.value.netcoreVersionTestUtils }} testSet: ${{ testSet }} @@ -113,6 +118,7 @@ stages: configSqlFor: ${{ config.value.configSqlFor }} operatingSystem: ${{ config.value.operatingSystem }} isArm64: ${{ eq(config.value.isArm64, 'true') }} + flakyTestsOnly: ${{ parameters.flakyTestsOnly }} ${{if and(eq(usemanagedSNI, false), ne(config.value.configProperties, '{}')) }}: ${{ each x86TF in config.value.configProperties.x86TestTargetFrameworks }}: ${{ if eq(x86TF, targetFramework) }}: @@ -122,7 +128,7 @@ stages: - ${{ if ne(length(parameters.postTestJobs), 0) }}: - stage: Post_Test displayName: 'Post Test Jobs' - dependsOn: + dependsOn: - ${{ each config in parameters.testConfigurations }}: - ${{ each image in config.value.images }}: - ${{ image.key }} diff --git a/eng/pipelines/common/templates/steps/run-all-tests-step.yml b/eng/pipelines/common/templates/steps/run-all-tests-step.yml index d88065f44a..77a3d2ac5d 100644 --- a/eng/pipelines/common/templates/steps/run-all-tests-step.yml +++ b/eng/pipelines/common/templates/steps/run-all-tests-step.yml @@ -18,7 +18,7 @@ parameters: - name: platform type: string default: $(Platform) - + - name: buildConfiguration type: string values: @@ -39,11 +39,11 @@ parameters: values: - x64 - x86 - + - name: dotnetx86RootPath # full path to the x86 dotnet root folder with trailing slash type: string default: '' - + - name: operatingSystem type: string default: 'Windows' @@ -52,196 +52,299 @@ parameters: type: number default: 2 + # If true, only flaky tests will be run. If false, all non-flaky tests will be run. + - name: flakyTestsOnly + type: boolean + steps: - ${{ if parameters.debug }}: - powershell: 'dotnet sdk check' displayName: '[Debug] .NET sdk check' condition: succeededOrFailed() +# Run Windows tests. - ${{if eq(parameters.operatingSystem, 'Windows')}}: - - ${{if eq(parameters.referenceType, 'Project')}}: + + # Run non-flaky tests. + - ${{if eq(parameters.flakyTestsOnly, false)}}: + + # Only run Unit Tests for Project reference type. + - ${{if eq(parameters.referenceType, 'Project')}}: + - task: MSBuild@1 + displayName: 'Run Unit Tests ${{parameters.msbuildArchitecture }}' + condition: succeededOrFailed() + inputs: + solution: build.proj + msbuildArchitecture: ${{parameters.msbuildArchitecture }} + platform: '${{parameters.platform }}' + configuration: '${{parameters.buildConfiguration }}' + ${{ if eq(parameters.msbuildArchitecture, 'x64') }}: + msbuildArguments: >- + -t:RunUnitTests + -p:TF=${{ parameters.targetFramework }} + -p:MdsPackageVersion=${{ parameters.mdsPackageVersion }} + ${{ else }}: # x86 + msbuildArguments: >- + -t:RunUnitTests + -p:TF=${{ parameters.targetFramework }} + -p:MdsPackageVersion=${{ parameters.mdsPackageVersion }} + -p:DotnetPath=${{ parameters.dotnetx86RootPath }} + - task: MSBuild@1 - displayName: 'Run Unit Tests ${{parameters.msbuildArchitecture }}' + displayName: 'Run Functional Tests ${{parameters.msbuildArchitecture }}' condition: succeededOrFailed() inputs: solution: build.proj msbuildArchitecture: ${{parameters.msbuildArchitecture }} platform: '${{parameters.platform }}' configuration: '${{parameters.buildConfiguration }}' - ${{ if eq(parameters.msbuildArchitecture, 'x64') }}: + ${{ if eq(parameters.msbuildArchitecture, 'x64') }}: msbuildArguments: >- - -t:RunUnitTests + -t:RunFunctionalTests -p:TF=${{ parameters.targetFramework }} + -p:TestSet=${{ parameters.testSet }} + -p:ReferenceType=${{ parameters.referenceType }} -p:MdsPackageVersion=${{ parameters.mdsPackageVersion }} ${{ else }}: # x86 msbuildArguments: >- - -t:RunUnitTests + -t:RunFunctionalTests -p:TF=${{ parameters.targetFramework }} + -p:TestSet=${{ parameters.testSet }} + -p:ReferenceType=${{ parameters.referenceType }} -p:MdsPackageVersion=${{ parameters.mdsPackageVersion }} -p:DotnetPath=${{ parameters.dotnetx86RootPath }} - task: MSBuild@1 - displayName: 'Run Flaky Unit Tests ${{parameters.msbuildArchitecture }}' condition: succeededOrFailed() + displayName: 'Run Manual Tests ${{parameters.msbuildArchitecture }}' inputs: solution: build.proj msbuildArchitecture: ${{parameters.msbuildArchitecture }} platform: '${{parameters.platform }}' configuration: '${{parameters.buildConfiguration }}' - ${{ if eq(parameters.msbuildArchitecture, 'x64') }}: + ${{ if eq(parameters.msbuildArchitecture, 'x64') }}: msbuildArguments: >- - -t:RunUnitTests + -t:RunManualTests + -p:TF=${{ parameters.targetFramework }} + -p:TestSet=${{ parameters.testSet }} + -p:ReferenceType=${{ parameters.referenceType }} + -p:MdsPackageVersion=${{ parameters.mdsPackageVersion }} + ${{ else }}: # x86 + msbuildArguments: >- + -t:RunManualTests + -p:TF=${{ parameters.targetFramework }} + -p:TestSet=${{ parameters.testSet }} + -p:ReferenceType=${{ parameters.referenceType }} + -p:MdsPackageVersion=${{ parameters.mdsPackageVersion }} + -p:DotnetPath=${{ parameters.dotnetx86RootPath }} + retryCountOnTaskFailure: ${{parameters.retryCountOnManualTests }} + + # Run flaky tests. + - ${{else}}: + + # Only run Unit Tests for Project reference type. + - ${{if eq(parameters.referenceType, 'Project')}}: + - task: MSBuild@1 + displayName: 'Run Flaky Unit Tests ${{parameters.msbuildArchitecture }}' + condition: succeededOrFailed() + inputs: + solution: build.proj + msbuildArchitecture: ${{parameters.msbuildArchitecture }} + platform: '${{parameters.platform }}' + configuration: '${{parameters.buildConfiguration }}' + ${{ if eq(parameters.msbuildArchitecture, 'x64') }}: + msbuildArguments: >- + -t:RunUnitTests + -p:TF=${{ parameters.targetFramework }} + -p:MdsPackageVersion=${{ parameters.mdsPackageVersion }} + -p:Filter="category=flaky" + -p:CollectCodeCoverage=false + ${{ else }}: # x86 + msbuildArguments: >- + -t:RunUnitTests + -p:TF=${{ parameters.targetFramework }} + -p:MdsPackageVersion=${{ parameters.mdsPackageVersion }} + -p:DotnetPath=${{ parameters.dotnetx86RootPath }} + -p:Filter="category=flaky" + -p:CollectCodeCoverage=false + continueOnError: true + + - task: MSBuild@1 + condition: succeededOrFailed() + displayName: 'Run Flaky Functional Tests ${{parameters.msbuildArchitecture }}' + inputs: + solution: build.proj + msbuildArchitecture: ${{parameters.msbuildArchitecture }} + platform: '${{parameters.platform }}' + configuration: '${{parameters.buildConfiguration }}' + ${{ if eq(parameters.msbuildArchitecture, 'x64') }}: + msbuildArguments: >- + -t:RunFunctionalTests -p:TF=${{ parameters.targetFramework }} + -p:TestSet=${{ parameters.testSet }} + -p:ReferenceType=${{ parameters.referenceType }} -p:MdsPackageVersion=${{ parameters.mdsPackageVersion }} -p:Filter="category=flaky" -p:CollectCodeCoverage=false ${{ else }}: # x86 msbuildArguments: >- - -t:RunUnitTests + -t:RunFunctionalTests -p:TF=${{ parameters.targetFramework }} + -p:TestSet=${{ parameters.testSet }} + -p:ReferenceType=${{ parameters.referenceType }} -p:MdsPackageVersion=${{ parameters.mdsPackageVersion }} -p:DotnetPath=${{ parameters.dotnetx86RootPath }} -p:Filter="category=flaky" -p:CollectCodeCoverage=false continueOnError: true - - task: MSBuild@1 - displayName: 'Run Functional Tests ${{parameters.msbuildArchitecture }}' - condition: succeededOrFailed() - inputs: - solution: build.proj - msbuildArchitecture: ${{parameters.msbuildArchitecture }} - platform: '${{parameters.platform }}' - configuration: '${{parameters.buildConfiguration }}' - ${{ if eq(parameters.msbuildArchitecture, 'x64') }}: - msbuildArguments: >- - -t:RunFunctionalTests - -p:TF=${{ parameters.targetFramework }} - -p:TestSet=${{ parameters.testSet }} - -p:ReferenceType=${{ parameters.referenceType }} - -p:MdsPackageVersion=${{ parameters.mdsPackageVersion }} - ${{ else }}: # x86 - msbuildArguments: >- - -t:RunFunctionalTests - -p:TF=${{ parameters.targetFramework }} - -p:TestSet=${{ parameters.testSet }} - -p:ReferenceType=${{ parameters.referenceType }} - -p:MdsPackageVersion=${{ parameters.mdsPackageVersion }} - -p:DotnetPath=${{ parameters.dotnetx86RootPath }} + - task: MSBuild@1 + condition: succeededOrFailed() + displayName: 'Run Flaky Manual Tests ${{parameters.msbuildArchitecture }}' + inputs: + solution: build.proj + msbuildArchitecture: ${{parameters.msbuildArchitecture }} + platform: '${{parameters.platform }}' + configuration: '${{parameters.buildConfiguration }}' + ${{ if eq(parameters.msbuildArchitecture, 'x64') }}: + msbuildArguments: >- + -t:RunManualTests + -p:TF=${{ parameters.targetFramework }} + -p:TestSet=${{ parameters.testSet }} + -p:ReferenceType=${{ parameters.referenceType }} + -p:MdsPackageVersion=${{ parameters.mdsPackageVersion }} + -p:Filter="category=flaky" + -p:CollectCodeCoverage=false + ${{ else }}: # x86 + msbuildArguments: >- + -t:RunManualTests + -p:TF=${{ parameters.targetFramework }} + -p:TestSet=${{ parameters.testSet }} + -p:ReferenceType=${{ parameters.referenceType }} + -p:MdsPackageVersion=${{ parameters.mdsPackageVersion }} + -p:DotnetPath=${{ parameters.dotnetx86RootPath }} + -p:Filter="category=flaky" + -p:CollectCodeCoverage=false + continueOnError: true + retryCountOnTaskFailure: ${{parameters.retryCountOnManualTests }} - - task: MSBuild@1 - condition: succeededOrFailed() - displayName: 'Run Flaky Functional Tests ${{parameters.msbuildArchitecture }}' - inputs: - solution: build.proj - msbuildArchitecture: ${{parameters.msbuildArchitecture }} - platform: '${{parameters.platform }}' - configuration: '${{parameters.buildConfiguration }}' - ${{ if eq(parameters.msbuildArchitecture, 'x64') }}: - msbuildArguments: >- - -t:RunFunctionalTests - -p:TF=${{ parameters.targetFramework }} - -p:TestSet=${{ parameters.testSet }} - -p:ReferenceType=${{ parameters.referenceType }} - -p:MdsPackageVersion=${{ parameters.mdsPackageVersion }} - -p:Filter="category=flaky" - -p:CollectCodeCoverage=false - ${{ else }}: # x86 - msbuildArguments: >- +# Linux or macOS +- ${{ else }}: + + # Run non-flaky tests. + - ${{if eq(parameters.flakyTestsOnly, false)}}: + + # Only run Unit Tests for Project reference type. + - ${{if eq(parameters.referenceType, 'Project')}}: + - task: DotNetCoreCLI@2 + displayName: 'Run Unit Tests' + condition: succeededOrFailed() + inputs: + command: custom + projects: build.proj + custom: msbuild + arguments: >- + -t:RunUnitTests + -p:TF=${{ parameters.targetFramework }} + -p:TestSet=${{ parameters.testSet }} + -p:ReferenceType=${{ parameters.referenceType }} + -p:MdsPackageVersion=${{ parameters.mdsPackageVersion }} + -p:platform=${{ parameters.platform }} + -p:Configuration=${{ parameters.buildConfiguration }} + verbosityRestore: Detailed + verbosityPack: Detailed + + - task: DotNetCoreCLI@2 + displayName: 'Run Functional Tests' + condition: succeededOrFailed() + inputs: + command: custom + projects: build.proj + custom: msbuild + arguments: >- -t:RunFunctionalTests -p:TF=${{ parameters.targetFramework }} -p:TestSet=${{ parameters.testSet }} -p:ReferenceType=${{ parameters.referenceType }} -p:MdsPackageVersion=${{ parameters.mdsPackageVersion }} - -p:DotnetPath=${{ parameters.dotnetx86RootPath }} - -p:Filter="category=flaky" - -p:CollectCodeCoverage=false - continueOnError: true + -p:platform=${{ parameters.platform }} + -p:Configuration=${{ parameters.buildConfiguration }} + verbosityRestore: Detailed + verbosityPack: Detailed - - task: MSBuild@1 - condition: succeededOrFailed() - displayName: 'Run Manual Tests ${{parameters.msbuildArchitecture }}' - inputs: - solution: build.proj - msbuildArchitecture: ${{parameters.msbuildArchitecture }} - platform: '${{parameters.platform }}' - configuration: '${{parameters.buildConfiguration }}' - ${{ if eq(parameters.msbuildArchitecture, 'x64') }}: - msbuildArguments: >- - -t:RunManualTests - -p:TF=${{ parameters.targetFramework }} - -p:TestSet=${{ parameters.testSet }} - -p:ReferenceType=${{ parameters.referenceType }} - -p:MdsPackageVersion=${{ parameters.mdsPackageVersion }} - ${{ else }}: # x86 - msbuildArguments: >- + - task: DotNetCoreCLI@2 + displayName: 'Run Manual Tests' + condition: succeededOrFailed() + inputs: + command: custom + projects: build.proj + custom: msbuild + arguments: >- -t:RunManualTests -p:TF=${{ parameters.targetFramework }} -p:TestSet=${{ parameters.testSet }} -p:ReferenceType=${{ parameters.referenceType }} -p:MdsPackageVersion=${{ parameters.mdsPackageVersion }} - -p:DotnetPath=${{ parameters.dotnetx86RootPath }} - retryCountOnTaskFailure: ${{parameters.retryCountOnManualTests }} + -p:platform=${{ parameters.platform }} + -p:Configuration=${{ parameters.buildConfiguration }} + verbosityRestore: Detailed + verbosityPack: Detailed + retryCountOnTaskFailure: ${{parameters.retryCountOnManualTests }} - - task: MSBuild@1 - condition: succeededOrFailed() - displayName: 'Run Flaky Manual Tests ${{parameters.msbuildArchitecture }}' - inputs: - solution: build.proj - msbuildArchitecture: ${{parameters.msbuildArchitecture }} - platform: '${{parameters.platform }}' - configuration: '${{parameters.buildConfiguration }}' - ${{ if eq(parameters.msbuildArchitecture, 'x64') }}: - msbuildArguments: >- - -t:RunManualTests - -p:TF=${{ parameters.targetFramework }} - -p:TestSet=${{ parameters.testSet }} - -p:ReferenceType=${{ parameters.referenceType }} - -p:MdsPackageVersion=${{ parameters.mdsPackageVersion }} - -p:Filter="category=flaky" - -p:CollectCodeCoverage=false - ${{ else }}: # x86 - msbuildArguments: >- - -t:RunManualTests - -p:TF=${{ parameters.targetFramework }} - -p:TestSet=${{ parameters.testSet }} - -p:ReferenceType=${{ parameters.referenceType }} - -p:MdsPackageVersion=${{ parameters.mdsPackageVersion }} - -p:DotnetPath=${{ parameters.dotnetx86RootPath }} - -p:Filter="category=flaky" - -p:CollectCodeCoverage=false - continueOnError: true - retryCountOnTaskFailure: ${{parameters.retryCountOnManualTests }} + # Run flaky tests. + - ${{else}}: + + # Only run Unit Tests for Project reference type. + - ${{if eq(parameters.referenceType, 'Project')}}: + - task: DotNetCoreCLI@2 + displayName: 'Run Flaky Unit Tests' + condition: succeededOrFailed() + inputs: + command: custom + projects: build.proj + custom: msbuild + arguments: >- + -t:RunUnitTests + -p:TF=${{ parameters.targetFramework }} + -p:TestSet=${{ parameters.testSet }} + -p:ReferenceType=${{ parameters.referenceType }} + -p:MdsPackageVersion=${{ parameters.mdsPackageVersion }} + -p:platform=${{ parameters.platform }} + -p:Configuration=${{ parameters.buildConfiguration }} + -p:Filter="category=flaky" + verbosityRestore: Detailed + verbosityPack: Detailed + continueOnError: true -- ${{ else }}: # Linux or macOS - - ${{if eq(parameters.referenceType, 'Project')}}: - task: DotNetCoreCLI@2 - displayName: 'Run Unit Tests' condition: succeededOrFailed() + displayName: 'Run Flaky Functional Tests' inputs: command: custom projects: build.proj custom: msbuild arguments: >- - -t:RunUnitTests + -t:RunFunctionalTests -p:TF=${{ parameters.targetFramework }} -p:TestSet=${{ parameters.testSet }} -p:ReferenceType=${{ parameters.referenceType }} -p:MdsPackageVersion=${{ parameters.mdsPackageVersion }} -p:platform=${{ parameters.platform }} -p:Configuration=${{ parameters.buildConfiguration }} + -p:Filter="category=flaky" verbosityRestore: Detailed verbosityPack: Detailed - + continueOnError: true + - task: DotNetCoreCLI@2 - displayName: 'Run Flaky Unit Tests' + displayName: 'Run Flaky Manual Tests' condition: succeededOrFailed() inputs: command: custom projects: build.proj custom: msbuild arguments: >- - -t:RunUnitTests + -t:RunManualTests -p:TF=${{ parameters.targetFramework }} -p:TestSet=${{ parameters.testSet }} -p:ReferenceType=${{ parameters.referenceType }} @@ -252,81 +355,4 @@ steps: verbosityRestore: Detailed verbosityPack: Detailed continueOnError: true - - - task: DotNetCoreCLI@2 - displayName: 'Run Functional Tests' - condition: succeededOrFailed() - inputs: - command: custom - projects: build.proj - custom: msbuild - arguments: >- - -t:RunFunctionalTests - -p:TF=${{ parameters.targetFramework }} - -p:TestSet=${{ parameters.testSet }} - -p:ReferenceType=${{ parameters.referenceType }} - -p:MdsPackageVersion=${{ parameters.mdsPackageVersion }} - -p:platform=${{ parameters.platform }} - -p:Configuration=${{ parameters.buildConfiguration }} - verbosityRestore: Detailed - verbosityPack: Detailed - - - task: DotNetCoreCLI@2 - condition: succeededOrFailed() - displayName: 'Run Flaky Functional Tests' - inputs: - command: custom - projects: build.proj - custom: msbuild - arguments: >- - -t:RunFunctionalTests - -p:TF=${{ parameters.targetFramework }} - -p:TestSet=${{ parameters.testSet }} - -p:ReferenceType=${{ parameters.referenceType }} - -p:MdsPackageVersion=${{ parameters.mdsPackageVersion }} - -p:platform=${{ parameters.platform }} - -p:Configuration=${{ parameters.buildConfiguration }} - -p:Filter="category=flaky" - verbosityRestore: Detailed - verbosityPack: Detailed - continueOnError: true - - - task: DotNetCoreCLI@2 - displayName: 'Run Manual Tests' - condition: succeededOrFailed() - inputs: - command: custom - projects: build.proj - custom: msbuild - arguments: >- - -t:RunManualTests - -p:TF=${{ parameters.targetFramework }} - -p:TestSet=${{ parameters.testSet }} - -p:ReferenceType=${{ parameters.referenceType }} - -p:MdsPackageVersion=${{ parameters.mdsPackageVersion }} - -p:platform=${{ parameters.platform }} - -p:Configuration=${{ parameters.buildConfiguration }} - verbosityRestore: Detailed - verbosityPack: Detailed - retryCountOnTaskFailure: ${{parameters.retryCountOnManualTests }} - - - task: DotNetCoreCLI@2 - displayName: 'Run Flaky Manual Tests' - condition: succeededOrFailed() - inputs: - command: custom - projects: build.proj - custom: msbuild - arguments: >- - -t:RunManualTests - -p:TF=${{ parameters.targetFramework }} - -p:TestSet=${{ parameters.testSet }} - -p:ReferenceType=${{ parameters.referenceType }} - -p:MdsPackageVersion=${{ parameters.mdsPackageVersion }} - -p:platform=${{ parameters.platform }} - -p:Configuration=${{ parameters.buildConfiguration }} - -p:Filter="category=flaky" - verbosityRestore: Detailed - verbosityPack: Detailed - continueOnError: true - retryCountOnTaskFailure: ${{parameters.retryCountOnManualTests }} + retryCountOnTaskFailure: ${{parameters.retryCountOnManualTests }} diff --git a/eng/pipelines/dotnet-sqlclient-ci-core.yml b/eng/pipelines/dotnet-sqlclient-ci-core.yml index d287e8b254..957a8da55b 100644 --- a/eng/pipelines/dotnet-sqlclient-ci-core.yml +++ b/eng/pipelines/dotnet-sqlclient-ci-core.yml @@ -98,6 +98,7 @@ parameters: type: boolean default: true +# Dotnet CLI verbosity level. - name: dotnetVerbosity type: string default: normal @@ -108,9 +109,14 @@ parameters: - detailed - diagnostic +# If true, only flaky tests will be run. If false, all non-flaky tests will be run. +- name: flakyTestsOnly + type: boolean + default: false + variables: - template: /eng/pipelines/libraries/ci-build-variables.yml@self - + - name: abstractionsArtifactsName value: Abstractions.Artifacts @@ -190,7 +196,7 @@ stages: mdsArtifactsName: $(mdsArtifactsName) mdsPackageVersion: $(mdsPackageVersion) dotnetVerbosity: ${{ parameters.dotnetVerbosity }} - + # Run the MDS and AKV tests. - template: /eng/pipelines/common/templates/stages/ci-run-tests-stage.yml@self parameters: @@ -200,6 +206,7 @@ stages: mdsArtifactsName: $(mdsArtifactsName) mdsPackageVersion: $(mdsPackageVersion) testJobTimeout: ${{ parameters.testJobTimeout }} + flakyTestsOnly: ${{ parameters.flakyTestsOnly }} # When testing MDS via packages, we must depend on the Abstractions, # MDS, and Azure packages. @@ -341,10 +348,10 @@ stages: # skipSqlConfiguration: # skips the SQL configuration step - windows_sql_22_x64: + windows_sql_22_x64: pool: ${{parameters.defaultPoolName }} images: - Win22_Sql22: ADO-MMS22-SQL22 + Win22_Sql22: ADO-MMS22-SQL22 TargetFrameworks: ${{parameters.targetFrameworks }} netcoreVersionTestUtils: ${{parameters.netcoreVersionTestUtils }} buildPlatforms: ${{parameters.buildPlatforms }} @@ -370,10 +377,10 @@ stages: enableLocalDB: true - windows_sql_22_x86: + windows_sql_22_x86: pool: ${{parameters.defaultPoolName }} images: - Win22_Sql22_x86: ADO-MMS22-SQL22 + Win22_Sql22_x86: ADO-MMS22-SQL22 TargetFrameworks: [net462, net8.0, net9.0] netcoreVersionTestUtils: ${{parameters.netcoreVersionTestUtils }} buildPlatforms: ${{parameters.buildPlatforms }} @@ -424,8 +431,8 @@ stages: # Azure SQL Server - Windows windows_azure_sql: - pool: ${{parameters.defaultPoolName }} - images: + pool: ${{parameters.defaultPoolName }} + images: Win22_Azure_Sql: ADO-MMS22-SQL19 Win11_Azure_Sql: ADO-CI-Win11 TargetFrameworks: ${{parameters.targetFrameworks }} @@ -457,7 +464,7 @@ stages: windows_azure_arm64_sql: pool: ADO-CI-PUBLIC-ARM64-1ES-EUS-POOL - images: + images: Win11_ARM64_Azure_Sql: ADO-WIN11-ARM64 isArm64: true TargetFrameworks: ${{parameters.targetFrameworks }} diff --git a/eng/pipelines/sqlclient/sqlclient-flaky-tests-pipeline.yml b/eng/pipelines/sqlclient/sqlclient-flaky-tests-pipeline.yml new file mode 100644 index 0000000000..0c33cc9498 --- /dev/null +++ b/eng/pipelines/sqlclient/sqlclient-flaky-tests-pipeline.yml @@ -0,0 +1,99 @@ +################################################################################# +# Licensed to the .NET Foundation under one or more agreements. # +# The .NET Foundation licenses this file to you under the MIT license. # +# See the LICENSE file in the project root for more information. # +################################################################################# + +# This pipeline builds and runs flaky tests for the following packages using project references: +# +# - Microsoft.SqlServer.Server +# - Microsoft.Data.SqlClient +# - Microsoft.Data.SqlClient.AlwaysEncrypted.AzureKeyVaultProvider +# +# It runs via CI push triggers and schedules and uses the Debug build configuration: +# +# - Commits to GitHub main +# - Weekdays at 09:00 UTC on GitHub main +# +# Changes are batched together to ensure that the pipeline never runs concurrently. +# +# This pipeline definition is mapped to the following Azure DevOps pipelines: +# +# - CI-SqlClient-Flaky in the Public project: +# +# TODO - Add URL + +# Set the pipeline run name to the day-of-year and the daily run counter. +name: $(DayOfYear)$(Rev:rr) + +# Do not trigger this pipeline for PRs. +pr: none + +# Trigger this pipeline on commits to certain branches. +trigger: + + # Don't trigger a new run until the previous one completes. This may cause + # multiple commits to be batched into a single run. + batch: true + + branches: + include: + # GitHub main branch. + - main + +# Trigger this pipeline on a schedule. +schedules: + + # GitHub main on weekdays + - cron: '0 9 * * Mon-Fri' + displayName: Weekday Run (Debug Config) + branches: + include: + - main + always: true + +# Pipeline parameters, visible in the Azure DevOps UI. +parameters: + + # The build configuration to use; defaults to Debug. + - name: buildConfiguration + displayName: Build Configuration + type: string + default: Debug + values: + - Debug + - Release + + # True to enable debug steps and logging. + - name: debug + displayName: Enable debug output + type: boolean + default: false + + # Dotnet CLI verbosity level. + - name: dotnetVerbosity + displayName: dotnet CLI Verbosity + type: string + default: normal + values: + - quiet + - minimal + - normal + - detailed + - diagnostic + + # The timeout, in minutes, for each test job. + - name: testJobTimeout + displayName: Test job timeout (in minutes) + type: number + default: 60 + +extends: + template: /eng/pipelines/dotnet-sqlclient-ci-core.yml@self + parameters: + buildConfiguration: ${{ parameters.buildConfiguration }} + flakyTestsOnly: true + debug: ${{ parameters.debug }} + dotnetVerbosity: ${{ parameters.dotnetVerbosity }} + referenceType: Project + testJobTimeout: ${{ parameters.testJobTimeout }}