diff --git a/NuGet.config b/NuGet.config index 8d9f70a526f..a86e73bfa95 100644 --- a/NuGet.config +++ b/NuGet.config @@ -4,7 +4,7 @@ - + @@ -19,7 +19,7 @@ - + diff --git a/azure-pipelines-public.yml b/azure-pipelines-public.yml new file mode 100644 index 00000000000..aa704ddcde6 --- /dev/null +++ b/azure-pipelines-public.yml @@ -0,0 +1,316 @@ +schedules: +- cron: 0 9 * * 1 + displayName: "Run CodeQL3000 weekly, Monday at 2:00 AM PDT" + branches: + include: + - release/2.1 + - release/6.0 + - release/7.0 + - main + always: true + +parameters: + # Parameter below is ignored in public builds. + # + # Choose whether to run the CodeQL3000 tasks. + # Manual builds align w/ official builds unless this parameter is true. + - name: runCodeQL3000 + default: false + displayName: Run CodeQL3000 tasks + type: boolean + +variables: + - name: _BuildConfig + value: Release + - name: _TeamName + value: AspNetCore + - name: DOTNET_SKIP_FIRST_TIME_EXPERIENCE + value: true + - name: _PublishUsingPipelines + value: true + - name: _CosmosConnectionUrl + value: https://localhost:8081 + - name: _CosmosToken + value: C2y6yDjf5/R+ob0N8A7Cgv30VRDJIWEHLM+4QDU5DE2nQ9nDuVTqobD4b8mGGyPMbIZnqyMsEcaGQy67XIw/Jw== + - ${{ if or(startswith(variables['Build.SourceBranch'], 'refs/heads/release/'), startswith(variables['Build.SourceBranch'], 'refs/heads/internal/release/'), eq(variables['Build.Reason'], 'Manual')) }}: + - name: PostBuildSign + value: false + - ${{ else }}: + - name: PostBuildSign + value: true + - ${{ if ne(variables['System.TeamProject'], 'public') }}: + - group: DotNet-HelixApi-Access + - group: DotNetBuilds storage account read tokens + - group: AzureDevOps-Artifact-Feeds-Pats + - name: _InternalRuntimeDownloadArgs + value: /p:DotNetRuntimeSourceFeed=https://dotnetbuilds.blob.core.windows.net/internal + /p:DotNetRuntimeSourceFeedKey=$(dotnetbuilds-internal-container-read-token-base64) + - ${{ if eq(variables['System.TeamProject'], 'public') }}: + - name: _InternalRuntimeDownloadArgs + value: '' + - name: LC_ALL + value: 'en_US.UTF-8' + - name: LANG + value: 'en_US.UTF-8' + - name: LANGUAGE + value: 'en_US.UTF-8' + - name: runCodeQL3000 + value: ${{ and(ne(variables['System.TeamProject'], 'public'), or(eq(variables['Build.Reason'], 'Schedule'), and(eq(variables['Build.Reason'], 'Manual'), eq(parameters.runCodeQL3000, 'true')))) }} + - template: /eng/common/templates/variables/pool-providers.yml + +trigger: + batch: true + branches: + include: + - main + - release/* + - feature/* + - internal/release/* + +pr: ['*'] + +stages: +- stage: build + displayName: Build + jobs: + - template: eng/common/templates/jobs/jobs.yml + parameters: + enableMicrobuild: ${{ ne(variables.runCodeQL3000, 'true') }} + enablePublishBuildArtifacts: true + enablePublishBuildAssets: ${{ ne(variables.runCodeQL3000, 'true') }} + enablePublishUsingPipelines: ${{ variables._PublishUsingPipelines }} + publishAssetsImmediately: true + enableSourceIndex: ${{ and(ne(variables['System.TeamProject'], 'public'), eq(variables['Build.SourceBranch'], 'refs/heads/main')) }} + enableTelemetry: true + helixRepo: dotnet/efcore + jobs: + - job: Windows + enablePublishTestResults: ${{ ne(variables.runCodeQL3000, 'true') }} + pool: + ${{ if eq(variables['System.TeamProject'], 'public') }}: + name: $(DncEngPublicBuildPool) + demands: ImageOverride -equals 1es-windows-2019-open + ${{ if ne(variables['System.TeamProject'], 'public') }}: + name: $(DncEngInternalBuildPool) + demands: ImageOverride -equals 1es-windows-2019 + ${{ if eq(variables.runCodeQL3000, 'true') }}: + # Component governance and SBOM creation are not needed here. Disable what Arcade would inject. + disableComponentGovernance: true + enableSbom: false + # CodeQL3000 extends build duration. + timeoutInMinutes: 180 + ${{ else }}: + timeoutInMinutes: 90 + variables: + - _InternalBuildArgs: '' + # Rely on task Arcade injects, not auto-injected build step. + - skipComponentGovernanceDetection: true + - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: + - _SignType: real + - _InternalBuildArgs: /p:DotNetSignType=$(_SignType) /p:TeamName=$(_TeamName) /p:DotNetPublishUsingPipelines=$(_PublishUsingPipelines) /p:OfficialBuildId=$(BUILD.BUILDNUMBER) + - ${{ if eq(variables.runCodeQL3000, 'true') }}: + - _AdditionalBuildArgs: /p:Test=false /p:Sign=false /p:Pack=false /p:Publish=false /p:UseSharedCompilation=false + # Security analysis is included in normal runs. Disable its auto-injection. + - skipNugetSecurityAnalysis: true + # Do not let CodeQL3000 Extension gate scan frequency. + - Codeql.Cadence: 0 + # Enable CodeQL3000 unconditionally so it may be run on any branch. + - Codeql.Enabled: true + # Ignore test and infrastructure code. + - Codeql.SourceRoot: src + # CodeQL3000 needs this plumbed along as a variable to enable TSA. + - Codeql.TSAEnabled: ${{ eq(variables['Build.Reason'], 'Schedule') }} + # Default expects tsaoptions.json under SourceRoot. + - Codeql.TSAOptionsPath: '$(Build.SourcesDirectory)/.config/tsaoptions.json' + - ${{ else }}: + - _AdditionalBuildArgs: '' + steps: + - task: NuGetCommand@2 + displayName: 'Clear NuGet caches' + condition: succeeded() + inputs: + command: custom + arguments: 'locals all -clear' + - script: "echo ##vso[build.addbuildtag]daily-build" + condition: and(notin(variables['Build.Reason'], 'PullRequest'), ne(variables['IsFinalBuild'], 'true')) + displayName: 'Set CI tags' + - script: "echo ##vso[build.addbuildtag]release-candidate" + condition: and(notin(variables['Build.Reason'], 'PullRequest'), eq(variables['IsFinalBuild'], 'true')) + displayName: 'Set CI tags' + - powershell: SqlLocalDB start + displayName: Start LocalDB + - ${{ if ne(variables['System.TeamProject'], 'public') }}: + - task: PowerShell@2 + displayName: Setup Private Feeds Credentials + inputs: + filePath: $(Build.SourcesDirectory)/eng/common/SetupNugetSources.ps1 + arguments: -ConfigFile $(Build.SourcesDirectory)/NuGet.config -Password $Env:Token + env: + Token: $(dn-bot-dnceng-artifact-feeds-rw) + - ${{ if eq(variables.runCodeQL3000, 'true') }}: + - task: CodeQL3000Init@0 + displayName: CodeQL Initialize + - script: "echo ##vso[build.addbuildtag]CodeQL3000" + displayName: 'Set CI CodeQL3000 tag' + condition: ne(variables.CODEQL_DIST,'') + - script: eng\common\cibuild.cmd -configuration $(_BuildConfig) -prepareMachine $(_InternalBuildArgs) + $(_InternalRuntimeDownloadArgs) $(_AdditionalBuildArgs) + env: + Test__Cosmos__DefaultConnection: $(_CosmosConnectionUrl) + name: Build + - ${{ if eq(variables.runCodeQL3000, 'true') }}: + - task: CodeQL3000Finalize@0 + displayName: CodeQL Finalize + - ${{ else }}: + - task: PublishBuildArtifacts@1 + displayName: Upload TestResults + condition: always() + continueOnError: true + inputs: + pathtoPublish: artifacts/TestResults/$(_BuildConfig)/ + artifactName: $(Agent.Os)_$(Agent.JobName) TestResults + artifactType: Container + parallel: true + + - ${{ if ne(variables.runCodeQL3000, 'true') }}: + - job: macOS + enablePublishTestResults: true + pool: + vmImage: macOS-11 + variables: + # Rely on task Arcade injects, not auto-injected build step. + - skipComponentGovernanceDetection: true + steps: + - ${{ if ne(variables['System.TeamProject'], 'public') }}: + - task: Bash@3 + displayName: Setup Private Feeds Credentials + inputs: + filePath: $(Build.SourcesDirectory)/eng/common/SetupNugetSources.sh + arguments: $(Build.SourcesDirectory)/NuGet.config $Token + env: + Token: $(dn-bot-dnceng-artifact-feeds-rw) + - script: eng/common/cibuild.sh --configuration $(_BuildConfig) --prepareMachine $(_InternalRuntimeDownloadArgs) + env: + Test__Cosmos__DefaultConnection: $(_CosmosConnectionUrl) + COMPlus_EnableWriteXorExecute: 0 # Work-around for https://github.com/dotnet/runtime/issues/70758 + name: Build + - task: PublishBuildArtifacts@1 + displayName: Upload TestResults + condition: always() + continueOnError: true + inputs: + pathtoPublish: artifacts/TestResults/$(_BuildConfig)/ + artifactName: $(Agent.Os)_$(Agent.JobName) TestResults + artifactType: Container + parallel: true + + - job: Linux + timeoutInMinutes: 120 + enablePublishTestResults: true + pool: + ${{ if or(ne(variables['System.TeamProject'], 'internal'), in(variables['Build.Reason'], 'Manual', 'PullRequest', 'Schedule')) }}: + vmImage: ubuntu-22.04 + ${{ if and(eq(variables['System.TeamProject'], 'internal'), notin(variables['Build.Reason'], 'Manual', 'PullRequest', 'Schedule')) }}: + name: $(DncEngInternalBuildPool) + demands: ImageOverride -equals Build.Ubuntu.2204.Amd64 + variables: + - _runCounter: $[counter(variables['Build.Reason'], 0)] + # Rely on task Arcade injects, not auto-injected build step. + - skipComponentGovernanceDetection: true + - ${{ if and(eq(variables['System.TeamProject'], 'internal'), notin(variables['Build.Reason'], 'PullRequest', 'Schedule')) }}: + - _CosmosConnectionUrl: 'true' + steps: + - bash: | + echo "##vso[task.setvariable variable=_CosmosConnectionUrl]https://ef-nightly-test.documents.azure.com:443/" + echo "##vso[task.setvariable variable=_CosmosToken]$(ef-nightly-cosmos-key)" + displayName: Prepare to run Cosmos tests on ef-nightly-test + condition: and(eq(variables['_CosmosConnectionUrl'], 'true'), or(endsWith(variables['_runCounter'], '0'), endsWith(variables['_runCounter'], '2'), endsWith(variables['_runCounter'], '4'), endsWith(variables['_runCounter'], '6'), endsWith(variables['_runCounter'], '8'))) + - bash: | + echo "##vso[task.setvariable variable=_CosmosConnectionUrl]https://ef-pr-test.documents.azure.com:443/" + echo "##vso[task.setvariable variable=_CosmosToken]$(ef-pr-cosmos-test)" + displayName: Prepare to run Cosmos tests on ef-pr-test + condition: and(eq(variables['_CosmosConnectionUrl'], 'true'), or(endsWith(variables['_runCounter'], '1'), endsWith(variables['_runCounter'], '3'), endsWith(variables['_runCounter'], '5'), endsWith(variables['_runCounter'], '7'), endsWith(variables['_runCounter'], '9'))) + - ${{ if ne(variables['System.TeamProject'], 'public') }}: + - task: Bash@3 + displayName: Setup Private Feeds Credentials + inputs: + filePath: $(Build.SourcesDirectory)/eng/common/SetupNugetSources.sh + arguments: $(Build.SourcesDirectory)/NuGet.config $Token + env: + Token: $(dn-bot-dnceng-artifact-feeds-rw) + - script: eng/common/cibuild.sh --configuration $(_BuildConfig) --prepareMachine $(_InternalRuntimeDownloadArgs) + env: + Test__Cosmos__DefaultConnection: $(_CosmosConnectionUrl) + Test__Cosmos__AuthToken: $(_CosmosToken) + name: Build + - task: PublishBuildArtifacts@1 + displayName: Upload TestResults + condition: always() + continueOnError: true + inputs: + pathtoPublish: artifacts/TestResults/$(_BuildConfig)/ + artifactName: $(Agent.Os)_$(Agent.JobName) TestResults + artifactType: Container + parallel: true + + - job: Helix + timeoutInMinutes: 180 + pool: + ${{ if eq(variables['System.TeamProject'], 'public') }}: + name: $(DncEngPublicBuildPool) + demands: ImageOverride -equals 1es-windows-2019-open + ${{ if ne(variables['System.TeamProject'], 'public') }}: + name: $(DncEngInternalBuildPool) + demands: ImageOverride -equals 1es-windows-2019 + variables: + # Rely on task Arcade injects, not auto-injected build step. + - skipComponentGovernanceDetection: true + - name: _HelixBuildConfig + value: $(_BuildConfig) + - ${{ if eq(variables['System.TeamProject'], 'public') }}: + - name: HelixTargetQueues + value: OSX.1100.Amd64.Open;(Ubuntu.2004.Amd64.SqlServer)Ubuntu.2004.Amd64.Open@mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-20.04-helix-sqlserver-amd64 + - name: _HelixAccessToken + value: '' # Needed for public queues + - ${{ if ne(variables['System.TeamProject'], 'public') }}: + - name: HelixTargetQueues + value: OSX.1100.Amd64;(Ubuntu.2004.Amd64.SqlServer)Ubuntu.2004.Amd64@mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-20.04-helix-sqlserver-amd64 + - name: _HelixAccessToken + value: $(HelixApiAccessToken) # Needed for internal queues + steps: + - task: NuGetCommand@2 + displayName: 'Clear NuGet caches' + condition: succeeded() + inputs: + command: custom + arguments: 'locals all -clear' + - ${{ if ne(variables['System.TeamProject'], 'public') }}: + - task: PowerShell@2 + displayName: Setup Private Feeds Credentials + inputs: + filePath: $(Build.SourcesDirectory)/eng/common/SetupNugetSources.ps1 + arguments: -ConfigFile $(Build.SourcesDirectory)/NuGet.config -Password $Env:Token + env: + Token: $(dn-bot-dnceng-artifact-feeds-rw) + - script: restore.cmd -ci /p:configuration=$(_BuildConfig) $(_InternalRuntimeDownloadArgs) + displayName: Restore packages + - script: .dotnet\dotnet build eng\helix.proj /restore /t:Test /p:configuration=$(_BuildConfig) /bl:$(Build.SourcesDirectory)/artifacts/log/$(_BuildConfig)/SendToHelix.binlog $(_InternalRuntimeDownloadArgs) + displayName: Send job to helix + env: + HelixAccessToken: $(_HelixAccessToken) + SYSTEM_ACCESSTOKEN: $(System.AccessToken) # We need to set this env var to publish helix results to Azure Dev Ops + MSSQL_SA_PASSWORD: "Password12!" + COMPlus_EnableWriteXorExecute: 0 # Work-around for https://github.com/dotnet/runtime/issues/70758 + DotNetBuildsInternalReadSasToken: $(dotnetbuilds-internal-container-read-token) + +- ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest'), ne(variables.runCodeQL3000, 'true')) }}: + - template: eng\common\templates\post-build\post-build.yml + parameters: + publishingInfraVersion: 3 + # Symbol validation isn't being very reliable lately. This should be enabled back + # once this issue is resolved: https://github.com/dotnet/arcade/issues/2871 + enableSymbolValidation: false + enableSigningValidation: false + enableNugetValidation: false + enableSourceLinkValidation: false + publishAssetsImmediately: true diff --git a/eng/Publishing.props b/eng/Publishing.props index 1da7cb4e448..6f77090ffcd 100644 --- a/eng/Publishing.props +++ b/eng/Publishing.props @@ -5,5 +5,6 @@ false + true diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 3a8e051bb81..6a9637e4386 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -29,9 +29,9 @@ https://dev.azure.com/dnceng/internal/_git/dotnet-runtime 5535e31a712343a63f5d7d796cd874e563e5ac14 - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - 9f4b1f5d664afdfc80e1508ab7ed099dff210fbd + 2d7eea252964e69be94cb9c847b371b23e4dd470 https://dev.azure.com/dnceng/internal/_git/dotnet-runtime @@ -41,32 +41,32 @@ https://dev.azure.com/dnceng/internal/_git/dotnet-runtime 9f4b1f5d664afdfc80e1508ab7ed099dff210fbd - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - 9f4b1f5d664afdfc80e1508ab7ed099dff210fbd + 2d7eea252964e69be94cb9c847b371b23e4dd470 - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - 9f4b1f5d664afdfc80e1508ab7ed099dff210fbd + 2d7eea252964e69be94cb9c847b371b23e4dd470 - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - 9f4b1f5d664afdfc80e1508ab7ed099dff210fbd + 2d7eea252964e69be94cb9c847b371b23e4dd470 - + https://github.com/dotnet/arcade - da98edc4c3ea539f109ea320672136ceb32591a7 + f311667e0587f19c3fa9553a909975662107a351 - + https://github.com/dotnet/arcade - da98edc4c3ea539f109ea320672136ceb32591a7 + f311667e0587f19c3fa9553a909975662107a351 - + https://github.com/dotnet/arcade - da98edc4c3ea539f109ea320672136ceb32591a7 + f311667e0587f19c3fa9553a909975662107a351 diff --git a/eng/Versions.props b/eng/Versions.props index 9507b9b2976..b924af1b8e3 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -1,6 +1,6 @@ - 8.0.3 + 8.0.4 servicing @@ -24,15 +24,15 @@ 8.0.0 8.0.0 8.0.0 - 8.0.3-servicing.24114.23 + 8.0.4-servicing.24169.9 8.0.0 8.0.3 - 8.0.3 - 8.0.3 - 8.0.3-servicing.24114.23 + 8.0.4 + 8.0.4 + 8.0.4-servicing.24169.9 - 8.0.0-beta.24113.2 + 8.0.0-beta.24165.4 diff --git a/eng/common/SetupNugetSources.ps1 b/eng/common/SetupNugetSources.ps1 index 6c65e81925f..efa2fd72bfa 100644 --- a/eng/common/SetupNugetSources.ps1 +++ b/eng/common/SetupNugetSources.ps1 @@ -35,7 +35,7 @@ Set-StrictMode -Version 2.0 . $PSScriptRoot\tools.ps1 # Add source entry to PackageSources -function AddPackageSource($sources, $SourceName, $SourceEndPoint, $creds, $Username, $Password) { +function AddPackageSource($sources, $SourceName, $SourceEndPoint, $creds, $Username, $pwd) { $packageSource = $sources.SelectSingleNode("add[@key='$SourceName']") if ($packageSource -eq $null) @@ -48,12 +48,11 @@ function AddPackageSource($sources, $SourceName, $SourceEndPoint, $creds, $Usern else { Write-Host "Package source $SourceName already present." } - - AddCredential -Creds $creds -Source $SourceName -Username $Username -Password $Password + AddCredential -Creds $creds -Source $SourceName -Username $Username -pwd $pwd } # Add a credential node for the specified source -function AddCredential($creds, $source, $username, $password) { +function AddCredential($creds, $source, $username, $pwd) { # Looks for credential configuration for the given SourceName. Create it if none is found. $sourceElement = $creds.SelectSingleNode($Source) if ($sourceElement -eq $null) @@ -82,17 +81,18 @@ function AddCredential($creds, $source, $username, $password) { $passwordElement.SetAttribute("key", "ClearTextPassword") $sourceElement.AppendChild($passwordElement) | Out-Null } - $passwordElement.SetAttribute("value", $Password) + + $passwordElement.SetAttribute("value", $pwd) } -function InsertMaestroPrivateFeedCredentials($Sources, $Creds, $Username, $Password) { +function InsertMaestroPrivateFeedCredentials($Sources, $Creds, $Username, $pwd) { $maestroPrivateSources = $Sources.SelectNodes("add[contains(@key,'darc-int')]") Write-Host "Inserting credentials for $($maestroPrivateSources.Count) Maestro's private feeds." ForEach ($PackageSource in $maestroPrivateSources) { Write-Host "`tInserting credential for Maestro's feed:" $PackageSource.Key - AddCredential -Creds $creds -Source $PackageSource.Key -Username $Username -Password $Password + AddCredential -Creds $creds -Source $PackageSource.Key -Username $Username -pwd $pwd } } @@ -144,13 +144,13 @@ if ($disabledSources -ne $null) { $userName = "dn-bot" # Insert credential nodes for Maestro's private feeds -InsertMaestroPrivateFeedCredentials -Sources $sources -Creds $creds -Username $userName -Password $Password +InsertMaestroPrivateFeedCredentials -Sources $sources -Creds $creds -Username $userName -pwd $Password # 3.1 uses a different feed url format so it's handled differently here $dotnet31Source = $sources.SelectSingleNode("add[@key='dotnet3.1']") if ($dotnet31Source -ne $null) { - AddPackageSource -Sources $sources -SourceName "dotnet3.1-internal" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal/nuget/v2" -Creds $creds -Username $userName -Password $Password - AddPackageSource -Sources $sources -SourceName "dotnet3.1-internal-transport" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal-transport/nuget/v2" -Creds $creds -Username $userName -Password $Password + AddPackageSource -Sources $sources -SourceName "dotnet3.1-internal" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal/nuget/v2" -Creds $creds -Username $userName -pwd $Password + AddPackageSource -Sources $sources -SourceName "dotnet3.1-internal-transport" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal-transport/nuget/v2" -Creds $creds -Username $userName -pwd $Password } $dotnetVersions = @('5','6','7','8') @@ -159,9 +159,9 @@ foreach ($dotnetVersion in $dotnetVersions) { $feedPrefix = "dotnet" + $dotnetVersion; $dotnetSource = $sources.SelectSingleNode("add[@key='$feedPrefix']") if ($dotnetSource -ne $null) { - AddPackageSource -Sources $sources -SourceName "$feedPrefix-internal" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/internal/_packaging/$feedPrefix-internal/nuget/v2" -Creds $creds -Username $userName -Password $Password - AddPackageSource -Sources $sources -SourceName "$feedPrefix-internal-transport" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/internal/_packaging/$feedPrefix-internal-transport/nuget/v2" -Creds $creds -Username $userName -Password $Password + AddPackageSource -Sources $sources -SourceName "$feedPrefix-internal" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/internal/_packaging/$feedPrefix-internal/nuget/v2" -Creds $creds -Username $userName -pwd $Password + AddPackageSource -Sources $sources -SourceName "$feedPrefix-internal-transport" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/internal/_packaging/$feedPrefix-internal-transport/nuget/v2" -Creds $creds -Username $userName -pwd $Password } } -$doc.Save($filename) +$doc.Save($filename) \ No newline at end of file diff --git a/eng/common/templates-official/job/job.yml b/eng/common/templates-official/job/job.yml new file mode 100644 index 00000000000..a2709d10562 --- /dev/null +++ b/eng/common/templates-official/job/job.yml @@ -0,0 +1,263 @@ +# Internal resources (telemetry, microbuild) can only be accessed from non-public projects, +# and some (Microbuild) should only be applied to non-PR cases for internal builds. + +parameters: +# Job schema parameters - https://docs.microsoft.com/en-us/azure/devops/pipelines/yaml-schema?view=vsts&tabs=schema#job + cancelTimeoutInMinutes: '' + condition: '' + container: '' + continueOnError: false + dependsOn: '' + displayName: '' + pool: '' + steps: [] + strategy: '' + timeoutInMinutes: '' + variables: [] + workspace: '' + templateContext: '' + +# Job base template specific parameters + # See schema documentation - https://github.com/dotnet/arcade/blob/master/Documentation/AzureDevOps/TemplateSchema.md + artifacts: '' + enableMicrobuild: false + enablePublishBuildArtifacts: false + enablePublishBuildAssets: false + enablePublishTestResults: false + enablePublishUsingPipelines: false + enableBuildRetry: false + disableComponentGovernance: '' + componentGovernanceIgnoreDirectories: '' + mergeTestResults: false + testRunTitle: '' + testResultsFormat: '' + name: '' + preSteps: [] + runAsPublic: false +# Sbom related params + enableSbom: true + PackageVersion: 7.0.0 + BuildDropPath: '$(Build.SourcesDirectory)/artifacts' + +jobs: +- job: ${{ parameters.name }} + + ${{ if ne(parameters.cancelTimeoutInMinutes, '') }}: + cancelTimeoutInMinutes: ${{ parameters.cancelTimeoutInMinutes }} + + ${{ if ne(parameters.condition, '') }}: + condition: ${{ parameters.condition }} + + ${{ if ne(parameters.container, '') }}: + container: ${{ parameters.container }} + + ${{ if ne(parameters.continueOnError, '') }}: + continueOnError: ${{ parameters.continueOnError }} + + ${{ if ne(parameters.dependsOn, '') }}: + dependsOn: ${{ parameters.dependsOn }} + + ${{ if ne(parameters.displayName, '') }}: + displayName: ${{ parameters.displayName }} + + ${{ if ne(parameters.pool, '') }}: + pool: ${{ parameters.pool }} + + ${{ if ne(parameters.strategy, '') }}: + strategy: ${{ parameters.strategy }} + + ${{ if ne(parameters.timeoutInMinutes, '') }}: + timeoutInMinutes: ${{ parameters.timeoutInMinutes }} + + ${{ if ne(parameters.templateContext, '') }}: + templateContext: ${{ parameters.templateContext }} + + variables: + - ${{ if ne(parameters.enableTelemetry, 'false') }}: + - name: DOTNET_CLI_TELEMETRY_PROFILE + value: '$(Build.Repository.Uri)' + - ${{ if eq(parameters.enableRichCodeNavigation, 'true') }}: + - name: EnableRichCodeNavigation + value: 'true' + # Retry signature validation up to three times, waiting 2 seconds between attempts. + # See https://learn.microsoft.com/en-us/nuget/reference/errors-and-warnings/nu3028#retry-untrusted-root-failures + - name: NUGET_EXPERIMENTAL_CHAIN_BUILD_RETRY_POLICY + value: 3,2000 + - ${{ each variable in parameters.variables }}: + # handle name-value variable syntax + # example: + # - name: [key] + # value: [value] + - ${{ if ne(variable.name, '') }}: + - name: ${{ variable.name }} + value: ${{ variable.value }} + + # handle variable groups + - ${{ if ne(variable.group, '') }}: + - group: ${{ variable.group }} + + # handle template variable syntax + # example: + # - template: path/to/template.yml + # parameters: + # [key]: [value] + - ${{ if ne(variable.template, '') }}: + - template: ${{ variable.template }} + ${{ if ne(variable.parameters, '') }}: + parameters: ${{ variable.parameters }} + + # handle key-value variable syntax. + # example: + # - [key]: [value] + - ${{ if and(eq(variable.name, ''), eq(variable.group, ''), eq(variable.template, '')) }}: + - ${{ each pair in variable }}: + - name: ${{ pair.key }} + value: ${{ pair.value }} + + # DotNet-HelixApi-Access provides 'HelixApiAccessToken' for internal builds + - ${{ if and(eq(parameters.enableTelemetry, 'true'), eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: + - group: DotNet-HelixApi-Access + + ${{ if ne(parameters.workspace, '') }}: + workspace: ${{ parameters.workspace }} + + steps: + - ${{ if ne(parameters.preSteps, '') }}: + - ${{ each preStep in parameters.preSteps }}: + - ${{ preStep }} + + - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: + - ${{ if eq(parameters.enableMicrobuild, 'true') }}: + - task: MicroBuildSigningPlugin@3 + displayName: Install MicroBuild plugin + inputs: + signType: $(_SignType) + zipSources: false + feedSource: https://dnceng.pkgs.visualstudio.com/_packaging/MicroBuildToolset/nuget/v3/index.json + env: + TeamName: $(_TeamName) + continueOnError: ${{ parameters.continueOnError }} + condition: and(succeeded(), in(variables['_SignType'], 'real', 'test'), eq(variables['Agent.Os'], 'Windows_NT')) + + - ${{ if and(eq(parameters.runAsPublic, 'false'), eq(variables['System.TeamProject'], 'internal')) }}: + - task: NuGetAuthenticate@1 + + - ${{ if and(ne(parameters.artifacts.download, 'false'), ne(parameters.artifacts.download, '')) }}: + - task: DownloadPipelineArtifact@2 + inputs: + buildType: current + artifactName: ${{ coalesce(parameters.artifacts.download.name, 'Artifacts_$(Agent.OS)_$(_BuildConfig)') }} + targetPath: ${{ coalesce(parameters.artifacts.download.path, 'artifacts') }} + itemPattern: ${{ coalesce(parameters.artifacts.download.pattern, '**') }} + + - ${{ each step in parameters.steps }}: + - ${{ step }} + + - ${{ if eq(parameters.enableRichCodeNavigation, true) }}: + - task: RichCodeNavIndexer@0 + displayName: RichCodeNav Upload + inputs: + languages: ${{ coalesce(parameters.richCodeNavigationLanguage, 'csharp') }} + environment: ${{ coalesce(parameters.richCodeNavigationEnvironment, 'production') }} + richNavLogOutputDirectory: $(Build.SourcesDirectory)/artifacts/bin + uploadRichNavArtifacts: ${{ coalesce(parameters.richCodeNavigationUploadArtifacts, false) }} + continueOnError: true + + - template: /eng/common/templates-official/steps/component-governance.yml + parameters: + ${{ if eq(parameters.disableComponentGovernance, '') }}: + ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest'), eq(parameters.runAsPublic, 'false'), or(startsWith(variables['Build.SourceBranch'], 'refs/heads/release/'), startsWith(variables['Build.SourceBranch'], 'refs/heads/dotnet/'), startsWith(variables['Build.SourceBranch'], 'refs/heads/microsoft/'), eq(variables['Build.SourceBranch'], 'refs/heads/main'))) }}: + disableComponentGovernance: false + ${{ else }}: + disableComponentGovernance: true + ${{ else }}: + disableComponentGovernance: ${{ parameters.disableComponentGovernance }} + componentGovernanceIgnoreDirectories: ${{ parameters.componentGovernanceIgnoreDirectories }} + + - ${{ if eq(parameters.enableMicrobuild, 'true') }}: + - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: + - task: MicroBuildCleanup@1 + displayName: Execute Microbuild cleanup tasks + condition: and(always(), in(variables['_SignType'], 'real', 'test'), eq(variables['Agent.Os'], 'Windows_NT')) + continueOnError: ${{ parameters.continueOnError }} + env: + TeamName: $(_TeamName) + + - ${{ if ne(parameters.artifacts.publish, '') }}: + - ${{ if and(ne(parameters.artifacts.publish.artifacts, 'false'), ne(parameters.artifacts.publish.artifacts, '')) }}: + - task: CopyFiles@2 + displayName: Gather binaries for publish to artifacts + inputs: + SourceFolder: 'artifacts/bin' + Contents: '**' + TargetFolder: '$(Build.ArtifactStagingDirectory)/artifacts/bin' + - task: CopyFiles@2 + displayName: Gather packages for publish to artifacts + inputs: + SourceFolder: 'artifacts/packages' + Contents: '**' + TargetFolder: '$(Build.ArtifactStagingDirectory)/artifacts/packages' + - task: 1ES.PublishBuildArtifacts@1 + displayName: Publish pipeline artifacts + inputs: + PathtoPublish: '$(Build.ArtifactStagingDirectory)/artifacts' + PublishLocation: Container + ArtifactName: ${{ coalesce(parameters.artifacts.publish.artifacts.name , 'Artifacts_$(Agent.Os)_$(_BuildConfig)') }} + continueOnError: true + condition: always() + - ${{ if and(ne(parameters.artifacts.publish.logs, 'false'), ne(parameters.artifacts.publish.logs, '')) }}: + - task: 1ES.PublishPipelineArtifact@1 + inputs: + targetPath: 'artifacts/log' + artifactName: ${{ coalesce(parameters.artifacts.publish.logs.name, 'Logs_Build_$(Agent.Os)_$(_BuildConfig)') }} + displayName: 'Publish logs' + continueOnError: true + condition: always() + + - ${{ if ne(parameters.enablePublishBuildArtifacts, 'false') }}: + - task: 1ES.PublishBuildArtifacts@1 + displayName: Publish Logs + inputs: + PathtoPublish: '$(Build.SourcesDirectory)/artifacts/log/$(_BuildConfig)' + PublishLocation: Container + ArtifactName: ${{ coalesce(parameters.enablePublishBuildArtifacts.artifactName, '$(Agent.Os)_$(Agent.JobName)' ) }} + continueOnError: true + condition: always() + + - ${{ if or(and(eq(parameters.enablePublishTestResults, 'true'), eq(parameters.testResultsFormat, '')), eq(parameters.testResultsFormat, 'xunit')) }}: + - task: PublishTestResults@2 + displayName: Publish XUnit Test Results + inputs: + testResultsFormat: 'xUnit' + testResultsFiles: '*.xml' + searchFolder: '$(Build.SourcesDirectory)/artifacts/TestResults/$(_BuildConfig)' + testRunTitle: ${{ coalesce(parameters.testRunTitle, parameters.name, '$(System.JobName)') }}-xunit + mergeTestResults: ${{ parameters.mergeTestResults }} + continueOnError: true + condition: always() + - ${{ if or(and(eq(parameters.enablePublishTestResults, 'true'), eq(parameters.testResultsFormat, '')), eq(parameters.testResultsFormat, 'vstest')) }}: + - task: PublishTestResults@2 + displayName: Publish TRX Test Results + inputs: + testResultsFormat: 'VSTest' + testResultsFiles: '*.trx' + searchFolder: '$(Build.SourcesDirectory)/artifacts/TestResults/$(_BuildConfig)' + testRunTitle: ${{ coalesce(parameters.testRunTitle, parameters.name, '$(System.JobName)') }}-trx + mergeTestResults: ${{ parameters.mergeTestResults }} + continueOnError: true + condition: always() + + - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest'), eq(parameters.enableSbom, 'true')) }}: + - template: /eng/common/templates-official/steps/generate-sbom.yml + parameters: + PackageVersion: ${{ parameters.packageVersion}} + BuildDropPath: ${{ parameters.buildDropPath }} + IgnoreDirectories: ${{ parameters.componentGovernanceIgnoreDirectories }} + + - ${{ if eq(parameters.enableBuildRetry, 'true') }}: + - task: 1ES.PublishPipelineArtifact@1 + inputs: + targetPath: '$(Build.SourcesDirectory)\eng\common\BuildConfiguration' + artifactName: 'BuildConfiguration' + displayName: 'Publish build retry configuration' + continueOnError: true \ No newline at end of file diff --git a/eng/common/templates-official/job/onelocbuild.yml b/eng/common/templates-official/job/onelocbuild.yml new file mode 100644 index 00000000000..ba9ba493032 --- /dev/null +++ b/eng/common/templates-official/job/onelocbuild.yml @@ -0,0 +1,112 @@ +parameters: + # Optional: dependencies of the job + dependsOn: '' + + # Optional: A defined YAML pool - https://docs.microsoft.com/en-us/azure/devops/pipelines/yaml-schema?view=vsts&tabs=schema#pool + pool: '' + + CeapexPat: $(dn-bot-ceapex-package-r) # PAT for the loc AzDO instance https://dev.azure.com/ceapex + GithubPat: $(BotAccount-dotnet-bot-repo-PAT) + + SourcesDirectory: $(Build.SourcesDirectory) + CreatePr: true + AutoCompletePr: false + ReusePr: true + UseLfLineEndings: true + UseCheckedInLocProjectJson: false + SkipLocProjectJsonGeneration: false + LanguageSet: VS_Main_Languages + LclSource: lclFilesInRepo + LclPackageId: '' + RepoType: gitHub + GitHubOrg: dotnet + MirrorRepo: '' + MirrorBranch: main + condition: '' + JobNameSuffix: '' + +jobs: +- job: OneLocBuild${{ parameters.JobNameSuffix }} + + dependsOn: ${{ parameters.dependsOn }} + + displayName: OneLocBuild${{ parameters.JobNameSuffix }} + + variables: + - group: OneLocBuildVariables # Contains the CeapexPat and GithubPat + - name: _GenerateLocProjectArguments + value: -SourcesDirectory ${{ parameters.SourcesDirectory }} + -LanguageSet "${{ parameters.LanguageSet }}" + -CreateNeutralXlfs + - ${{ if eq(parameters.UseCheckedInLocProjectJson, 'true') }}: + - name: _GenerateLocProjectArguments + value: ${{ variables._GenerateLocProjectArguments }} -UseCheckedInLocProjectJson + - template: /eng/common/templates-official/variables/pool-providers.yml + + ${{ if ne(parameters.pool, '') }}: + pool: ${{ parameters.pool }} + ${{ if eq(parameters.pool, '') }}: + pool: + # We don't use the collection uri here because it might vary (.visualstudio.com vs. dev.azure.com) + ${{ if eq(variables['System.TeamProject'], 'DevDiv') }}: + name: AzurePipelines-EO + image: 1ESPT-Windows2022 + demands: Cmd + os: windows + # If it's not devdiv, it's dnceng + ${{ if ne(variables['System.TeamProject'], 'DevDiv') }}: + name: $(DncEngInternalBuildPool) + image: 1es-windows-2022-pt + os: windows + + steps: + - ${{ if ne(parameters.SkipLocProjectJsonGeneration, 'true') }}: + - task: Powershell@2 + inputs: + filePath: $(Build.SourcesDirectory)/eng/common/generate-locproject.ps1 + arguments: $(_GenerateLocProjectArguments) + displayName: Generate LocProject.json + condition: ${{ parameters.condition }} + + - task: OneLocBuild@2 + displayName: OneLocBuild + env: + SYSTEM_ACCESSTOKEN: $(System.AccessToken) + inputs: + locProj: eng/Localize/LocProject.json + outDir: $(Build.ArtifactStagingDirectory) + lclSource: ${{ parameters.LclSource }} + lclPackageId: ${{ parameters.LclPackageId }} + isCreatePrSelected: ${{ parameters.CreatePr }} + isAutoCompletePrSelected: ${{ parameters.AutoCompletePr }} + ${{ if eq(parameters.CreatePr, true) }}: + isUseLfLineEndingsSelected: ${{ parameters.UseLfLineEndings }} + ${{ if eq(parameters.RepoType, 'gitHub') }}: + isShouldReusePrSelected: ${{ parameters.ReusePr }} + packageSourceAuth: patAuth + patVariable: ${{ parameters.CeapexPat }} + ${{ if eq(parameters.RepoType, 'gitHub') }}: + repoType: ${{ parameters.RepoType }} + gitHubPatVariable: "${{ parameters.GithubPat }}" + ${{ if ne(parameters.MirrorRepo, '') }}: + isMirrorRepoSelected: true + gitHubOrganization: ${{ parameters.GitHubOrg }} + mirrorRepo: ${{ parameters.MirrorRepo }} + mirrorBranch: ${{ parameters.MirrorBranch }} + condition: ${{ parameters.condition }} + + - task: 1ES.PublishBuildArtifacts@1 + displayName: Publish Localization Files + inputs: + PathtoPublish: '$(Build.ArtifactStagingDirectory)/loc' + PublishLocation: Container + ArtifactName: Loc + condition: ${{ parameters.condition }} + + - task: 1ES.PublishBuildArtifacts@1 + displayName: Publish LocProject.json + inputs: + PathtoPublish: '$(Build.SourcesDirectory)/eng/Localize/' + PublishLocation: Container + ArtifactName: Loc + condition: ${{ parameters.condition }} \ No newline at end of file diff --git a/eng/common/templates-official/job/publish-build-assets.yml b/eng/common/templates-official/job/publish-build-assets.yml new file mode 100644 index 00000000000..53138622fe7 --- /dev/null +++ b/eng/common/templates-official/job/publish-build-assets.yml @@ -0,0 +1,155 @@ +parameters: + configuration: 'Debug' + + # Optional: condition for the job to run + condition: '' + + # Optional: 'true' if future jobs should run even if this job fails + continueOnError: false + + # Optional: dependencies of the job + dependsOn: '' + + # Optional: Include PublishBuildArtifacts task + enablePublishBuildArtifacts: false + + # Optional: A defined YAML pool - https://docs.microsoft.com/en-us/azure/devops/pipelines/yaml-schema?view=vsts&tabs=schema#pool + pool: {} + + # Optional: should run as a public build even in the internal project + # if 'true', the build won't run any of the internal only steps, even if it is running in non-public projects. + runAsPublic: false + + # Optional: whether the build's artifacts will be published using release pipelines or direct feed publishing + publishUsingPipelines: false + + # Optional: whether the build's artifacts will be published using release pipelines or direct feed publishing + publishAssetsImmediately: false + + artifactsPublishingAdditionalParameters: '' + + signingValidationAdditionalParameters: '' + +jobs: +- job: Asset_Registry_Publish + + dependsOn: ${{ parameters.dependsOn }} + timeoutInMinutes: 150 + + ${{ if eq(parameters.publishAssetsImmediately, 'true') }}: + displayName: Publish Assets + ${{ else }}: + displayName: Publish to Build Asset Registry + + variables: + - template: /eng/common/templates-official/variables/pool-providers.yml + - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: + - group: Publish-Build-Assets + - group: AzureDevOps-Artifact-Feeds-Pats + - name: runCodesignValidationInjection + value: false + - ${{ if eq(parameters.publishAssetsImmediately, 'true') }}: + - template: /eng/common/templates-official/post-build/common-variables.yml + + pool: + # We don't use the collection uri here because it might vary (.visualstudio.com vs. dev.azure.com) + ${{ if eq(variables['System.TeamProject'], 'DevDiv') }}: + name: AzurePipelines-EO + image: 1ESPT-Windows2022 + demands: Cmd + os: windows + # If it's not devdiv, it's dnceng + ${{ if ne(variables['System.TeamProject'], 'DevDiv') }}: + name: $(DncEngInternalBuildPool) + image: 1es-windows-2022-pt + os: windows + steps: + - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: + - task: DownloadBuildArtifacts@0 + displayName: Download artifact + inputs: + artifactName: AssetManifests + downloadPath: '$(Build.StagingDirectory)/Download' + checkDownloadedFiles: true + condition: ${{ parameters.condition }} + continueOnError: ${{ parameters.continueOnError }} + + - task: NuGetAuthenticate@1 + + - task: PowerShell@2 + displayName: Publish Build Assets + inputs: + filePath: eng\common\sdk-task.ps1 + arguments: -task PublishBuildAssets -restore -msbuildEngine dotnet + /p:ManifestsPath='$(Build.StagingDirectory)/Download/AssetManifests' + /p:BuildAssetRegistryToken=$(MaestroAccessToken) + /p:MaestroApiEndpoint=https://maestro-prod.westus2.cloudapp.azure.com + /p:PublishUsingPipelines=${{ parameters.publishUsingPipelines }} + /p:OfficialBuildId=$(Build.BuildNumber) + condition: ${{ parameters.condition }} + continueOnError: ${{ parameters.continueOnError }} + + - task: powershell@2 + displayName: Create ReleaseConfigs Artifact + inputs: + targetType: inline + script: | + New-Item -Path "$(Build.StagingDirectory)/ReleaseConfigs" -ItemType Directory -Force + $filePath = "$(Build.StagingDirectory)/ReleaseConfigs/ReleaseConfigs.txt" + Add-Content -Path $filePath -Value $(BARBuildId) + Add-Content -Path $filePath -Value "$(DefaultChannels)" + Add-Content -Path $filePath -Value $(IsStableBuild) + + - task: 1ES.PublishBuildArtifacts@1 + displayName: Publish ReleaseConfigs Artifact + inputs: + PathtoPublish: '$(Build.StagingDirectory)/ReleaseConfigs' + PublishLocation: Container + ArtifactName: ReleaseConfigs + + - task: powershell@2 + displayName: Check if SymbolPublishingExclusionsFile.txt exists + inputs: + targetType: inline + script: | + $symbolExclusionfile = "$(Build.SourcesDirectory)/eng/SymbolPublishingExclusionsFile.txt" + if(Test-Path -Path $symbolExclusionfile) + { + Write-Host "SymbolExclusionFile exists" + Write-Host "##vso[task.setvariable variable=SymbolExclusionFile]true" + } + else{ + Write-Host "Symbols Exclusion file does not exists" + Write-Host "##vso[task.setvariable variable=SymbolExclusionFile]false" + } + + - task: 1ES.PublishBuildArtifacts@1 + displayName: Publish SymbolPublishingExclusionsFile Artifact + condition: eq(variables['SymbolExclusionFile'], 'true') + inputs: + PathtoPublish: '$(Build.SourcesDirectory)/eng/SymbolPublishingExclusionsFile.txt' + PublishLocation: Container + ArtifactName: ReleaseConfigs + + - ${{ if eq(parameters.publishAssetsImmediately, 'true') }}: + - template: /eng/common/templates-official/post-build/setup-maestro-vars.yml + parameters: + BARBuildId: ${{ parameters.BARBuildId }} + PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} + + - task: PowerShell@2 + displayName: Publish Using Darc + inputs: + filePath: $(Build.SourcesDirectory)/eng/common/post-build/publish-using-darc.ps1 + arguments: -BuildId $(BARBuildId) + -PublishingInfraVersion 3 + -AzdoToken '$(publishing-dnceng-devdiv-code-r-build-re)' + -MaestroToken '$(MaestroApiAccessToken)' + -WaitPublishingFinish true + -ArtifactsPublishingAdditionalParameters '${{ parameters.artifactsPublishingAdditionalParameters }}' + -SymbolPublishingAdditionalParameters '${{ parameters.symbolPublishingAdditionalParameters }}' + + - ${{ if eq(parameters.enablePublishBuildArtifacts, 'true') }}: + - template: /eng/common/templates-official/steps/publish-logs.yml + parameters: + JobLabel: 'Publish_Artifacts_Logs' diff --git a/eng/common/templates-official/job/source-build.yml b/eng/common/templates-official/job/source-build.yml new file mode 100644 index 00000000000..8aba3b44bb2 --- /dev/null +++ b/eng/common/templates-official/job/source-build.yml @@ -0,0 +1,67 @@ +parameters: + # This template adds arcade-powered source-build to CI. The template produces a server job with a + # default ID 'Source_Build_Complete' to put in a dependency list if necessary. + + # Specifies the prefix for source-build jobs added to pipeline. Use this if disambiguation needed. + jobNamePrefix: 'Source_Build' + + # Defines the platform on which to run the job. By default, a linux-x64 machine, suitable for + # managed-only repositories. This is an object with these properties: + # + # name: '' + # The name of the job. This is included in the job ID. + # targetRID: '' + # The name of the target RID to use, instead of the one auto-detected by Arcade. + # nonPortable: false + # Enables non-portable mode. This means a more specific RID (e.g. fedora.32-x64 rather than + # linux-x64), and compiling against distro-provided packages rather than portable ones. + # skipPublishValidation: false + # Disables publishing validation. By default, a check is performed to ensure no packages are + # published by source-build. + # container: '' + # A container to use. Runs in docker. + # pool: {} + # A pool to use. Runs directly on an agent. + # buildScript: '' + # Specifies the build script to invoke to perform the build in the repo. The default + # './build.sh' should work for typical Arcade repositories, but this is customizable for + # difficult situations. + # jobProperties: {} + # A list of job properties to inject at the top level, for potential extensibility beyond + # container and pool. + platform: {} + +jobs: +- job: ${{ parameters.jobNamePrefix }}_${{ parameters.platform.name }} + displayName: Source-Build (${{ parameters.platform.name }}) + + ${{ each property in parameters.platform.jobProperties }}: + ${{ property.key }}: ${{ property.value }} + + ${{ if ne(parameters.platform.container, '') }}: + container: ${{ parameters.platform.container }} + + ${{ if eq(parameters.platform.pool, '') }}: + # The default VM host AzDO pool. This should be capable of running Docker containers: almost all + # source-build builds run in Docker, including the default managed platform. + # /eng/common/templates-official/variables/pool-providers.yml can't be used here (some customers declare variables already), so duplicate its logic + pool: + ${{ if eq(variables['System.TeamProject'], 'public') }}: + name: $[replace(replace(eq(contains(coalesce(variables['System.PullRequest.TargetBranch'], variables['Build.SourceBranch'], 'refs/heads/main'), 'release'), 'true'), True, 'NetCore-Svc-Public' ), False, 'NetCore-Public')] + demands: ImageOverride -equals Build.Ubuntu.1804.Amd64.Open + + ${{ if eq(variables['System.TeamProject'], 'internal') }}: + name: $[replace(replace(eq(contains(coalesce(variables['System.PullRequest.TargetBranch'], variables['Build.SourceBranch'], 'refs/heads/main'), 'release'), 'true'), True, 'NetCore1ESPool-Svc-Internal'), False, 'NetCore1ESPool-Internal')] + image: 1es-mariner-2-pt + os: linux + + ${{ if ne(parameters.platform.pool, '') }}: + pool: ${{ parameters.platform.pool }} + + workspace: + clean: all + + steps: + - template: /eng/common/templates-official/steps/source-build.yml + parameters: + platform: ${{ parameters.platform }} diff --git a/eng/common/templates-official/job/source-index-stage1.yml b/eng/common/templates-official/job/source-index-stage1.yml new file mode 100644 index 00000000000..4b633739170 --- /dev/null +++ b/eng/common/templates-official/job/source-index-stage1.yml @@ -0,0 +1,68 @@ +parameters: + runAsPublic: false + sourceIndexPackageVersion: 1.0.1-20230228.2 + sourceIndexPackageSource: https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json + sourceIndexBuildCommand: powershell -NoLogo -NoProfile -ExecutionPolicy Bypass -Command "eng/common/build.ps1 -restore -build -binarylog -ci" + preSteps: [] + binlogPath: artifacts/log/Debug/Build.binlog + condition: '' + dependsOn: '' + pool: '' + +jobs: +- job: SourceIndexStage1 + dependsOn: ${{ parameters.dependsOn }} + condition: ${{ parameters.condition }} + variables: + - name: SourceIndexPackageVersion + value: ${{ parameters.sourceIndexPackageVersion }} + - name: SourceIndexPackageSource + value: ${{ parameters.sourceIndexPackageSource }} + - name: BinlogPath + value: ${{ parameters.binlogPath }} + - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: + - group: source-dot-net stage1 variables + - template: /eng/common/templates-official/variables/pool-providers.yml + + ${{ if ne(parameters.pool, '') }}: + pool: ${{ parameters.pool }} + ${{ if eq(parameters.pool, '') }}: + pool: + ${{ if eq(variables['System.TeamProject'], 'public') }}: + name: $(DncEngPublicBuildPool) + demands: ImageOverride -equals windows.vs2019.amd64.open + ${{ if eq(variables['System.TeamProject'], 'internal') }}: + name: $(DncEngInternalBuildPool) + image: 1es-windows-2022-pt + os: windows + + steps: + - ${{ each preStep in parameters.preSteps }}: + - ${{ preStep }} + + - task: UseDotNet@2 + displayName: Use .NET Core SDK 6 + inputs: + packageType: sdk + version: 6.0.x + installationPath: $(Agent.TempDirectory)/dotnet + workingDirectory: $(Agent.TempDirectory) + + - script: | + $(Agent.TempDirectory)/dotnet/dotnet tool install BinLogToSln --version $(SourceIndexPackageVersion) --add-source $(SourceIndexPackageSource) --tool-path $(Agent.TempDirectory)/.source-index/tools + $(Agent.TempDirectory)/dotnet/dotnet tool install UploadIndexStage1 --version $(SourceIndexPackageVersion) --add-source $(SourceIndexPackageSource) --tool-path $(Agent.TempDirectory)/.source-index/tools + displayName: Download Tools + # Set working directory to temp directory so 'dotnet' doesn't try to use global.json and use the repo's sdk. + workingDirectory: $(Agent.TempDirectory) + + - script: ${{ parameters.sourceIndexBuildCommand }} + displayName: Build Repository + + - script: $(Agent.TempDirectory)/.source-index/tools/BinLogToSln -i $(BinlogPath) -r $(Build.SourcesDirectory) -n $(Build.Repository.Name) -o .source-index/stage1output + displayName: Process Binlog into indexable sln + + - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: + - script: $(Agent.TempDirectory)/.source-index/tools/UploadIndexStage1 -i .source-index/stage1output -n $(Build.Repository.Name) + displayName: Upload stage1 artifacts to source index + env: + BLOB_CONTAINER_URL: $(source-dot-net-stage1-blob-container-url) diff --git a/eng/common/templates-official/jobs/codeql-build.yml b/eng/common/templates-official/jobs/codeql-build.yml new file mode 100644 index 00000000000..b68d3c2f319 --- /dev/null +++ b/eng/common/templates-official/jobs/codeql-build.yml @@ -0,0 +1,31 @@ +parameters: + # See schema documentation in /Documentation/AzureDevOps/TemplateSchema.md + continueOnError: false + # Required: A collection of jobs to run - https://docs.microsoft.com/en-us/azure/devops/pipelines/yaml-schema?view=vsts&tabs=schema#job + jobs: [] + # Optional: if specified, restore and use this version of Guardian instead of the default. + overrideGuardianVersion: '' + +jobs: +- template: /eng/common/templates-official/jobs/jobs.yml + parameters: + enableMicrobuild: false + enablePublishBuildArtifacts: false + enablePublishTestResults: false + enablePublishBuildAssets: false + enablePublishUsingPipelines: false + enableTelemetry: true + + variables: + - group: Publish-Build-Assets + # The Guardian version specified in 'eng/common/sdl/packages.config'. This value must be kept in + # sync with the packages.config file. + - name: DefaultGuardianVersion + value: 0.109.0 + - name: GuardianPackagesConfigFile + value: $(Build.SourcesDirectory)\eng\common\sdl\packages.config + - name: GuardianVersion + value: ${{ coalesce(parameters.overrideGuardianVersion, '$(DefaultGuardianVersion)') }} + + jobs: ${{ parameters.jobs }} + diff --git a/eng/common/templates-official/jobs/jobs.yml b/eng/common/templates-official/jobs/jobs.yml new file mode 100644 index 00000000000..857a0f8ba43 --- /dev/null +++ b/eng/common/templates-official/jobs/jobs.yml @@ -0,0 +1,97 @@ +parameters: + # See schema documentation in /Documentation/AzureDevOps/TemplateSchema.md + continueOnError: false + + # Optional: Include PublishBuildArtifacts task + enablePublishBuildArtifacts: false + + # Optional: Enable publishing using release pipelines + enablePublishUsingPipelines: false + + # Optional: Enable running the source-build jobs to build repo from source + enableSourceBuild: false + + # Optional: Parameters for source-build template. + # See /eng/common/templates-official/jobs/source-build.yml for options + sourceBuildParameters: [] + + graphFileGeneration: + # Optional: Enable generating the graph files at the end of the build + enabled: false + # Optional: Include toolset dependencies in the generated graph files + includeToolset: false + + # Required: A collection of jobs to run - https://docs.microsoft.com/en-us/azure/devops/pipelines/yaml-schema?view=vsts&tabs=schema#job + jobs: [] + + # Optional: Override automatically derived dependsOn value for "publish build assets" job + publishBuildAssetsDependsOn: '' + + # Optional: Publish the assets as soon as the publish to BAR stage is complete, rather doing so in a separate stage. + publishAssetsImmediately: false + + # Optional: If using publishAssetsImmediately and additional parameters are needed, can be used to send along additional parameters (normally sent to post-build.yml) + artifactsPublishingAdditionalParameters: '' + signingValidationAdditionalParameters: '' + + # Optional: should run as a public build even in the internal project + # if 'true', the build won't run any of the internal only steps, even if it is running in non-public projects. + runAsPublic: false + + enableSourceIndex: false + sourceIndexParams: {} + +# Internal resources (telemetry, microbuild) can only be accessed from non-public projects, +# and some (Microbuild) should only be applied to non-PR cases for internal builds. + +jobs: +- ${{ each job in parameters.jobs }}: + - template: ../job/job.yml + parameters: + # pass along parameters + ${{ each parameter in parameters }}: + ${{ if ne(parameter.key, 'jobs') }}: + ${{ parameter.key }}: ${{ parameter.value }} + + # pass along job properties + ${{ each property in job }}: + ${{ if ne(property.key, 'job') }}: + ${{ property.key }}: ${{ property.value }} + + name: ${{ job.job }} + +- ${{ if eq(parameters.enableSourceBuild, true) }}: + - template: /eng/common/templates-official/jobs/source-build.yml + parameters: + allCompletedJobId: Source_Build_Complete + ${{ each parameter in parameters.sourceBuildParameters }}: + ${{ parameter.key }}: ${{ parameter.value }} + +- ${{ if eq(parameters.enableSourceIndex, 'true') }}: + - template: ../job/source-index-stage1.yml + parameters: + runAsPublic: ${{ parameters.runAsPublic }} + ${{ each parameter in parameters.sourceIndexParams }}: + ${{ parameter.key }}: ${{ parameter.value }} + +- ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: + - ${{ if or(eq(parameters.enablePublishBuildAssets, true), eq(parameters.artifacts.publish.manifests, 'true'), ne(parameters.artifacts.publish.manifests, '')) }}: + - template: ../job/publish-build-assets.yml + parameters: + continueOnError: ${{ parameters.continueOnError }} + dependsOn: + - ${{ if ne(parameters.publishBuildAssetsDependsOn, '') }}: + - ${{ each job in parameters.publishBuildAssetsDependsOn }}: + - ${{ job.job }} + - ${{ if eq(parameters.publishBuildAssetsDependsOn, '') }}: + - ${{ each job in parameters.jobs }}: + - ${{ job.job }} + - ${{ if eq(parameters.enableSourceBuild, true) }}: + - Source_Build_Complete + + runAsPublic: ${{ parameters.runAsPublic }} + publishUsingPipelines: ${{ parameters.enablePublishUsingPipelines }} + publishAssetsImmediately: ${{ parameters.publishAssetsImmediately }} + enablePublishBuildArtifacts: ${{ parameters.enablePublishBuildArtifacts }} + artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} + signingValidationAdditionalParameters: ${{ parameters.signingValidationAdditionalParameters }} diff --git a/eng/common/templates-official/jobs/source-build.yml b/eng/common/templates-official/jobs/source-build.yml new file mode 100644 index 00000000000..08e5db9bb11 --- /dev/null +++ b/eng/common/templates-official/jobs/source-build.yml @@ -0,0 +1,46 @@ +parameters: + # This template adds arcade-powered source-build to CI. A job is created for each platform, as + # well as an optional server job that completes when all platform jobs complete. + + # The name of the "join" job for all source-build platforms. If set to empty string, the job is + # not included. Existing repo pipelines can use this job depend on all source-build jobs + # completing without maintaining a separate list of every single job ID: just depend on this one + # server job. By default, not included. Recommended name if used: 'Source_Build_Complete'. + allCompletedJobId: '' + + # See /eng/common/templates-official/job/source-build.yml + jobNamePrefix: 'Source_Build' + + # This is the default platform provided by Arcade, intended for use by a managed-only repo. + defaultManagedPlatform: + name: 'Managed' + container: 'mcr.microsoft.com/dotnet-buildtools/prereqs:centos-stream8' + + # Defines the platforms on which to run build jobs. One job is created for each platform, and the + # object in this array is sent to the job template as 'platform'. If no platforms are specified, + # one job runs on 'defaultManagedPlatform'. + platforms: [] + +jobs: + +- ${{ if ne(parameters.allCompletedJobId, '') }}: + - job: ${{ parameters.allCompletedJobId }} + displayName: Source-Build Complete + pool: server + dependsOn: + - ${{ each platform in parameters.platforms }}: + - ${{ parameters.jobNamePrefix }}_${{ platform.name }} + - ${{ if eq(length(parameters.platforms), 0) }}: + - ${{ parameters.jobNamePrefix }}_${{ parameters.defaultManagedPlatform.name }} + +- ${{ each platform in parameters.platforms }}: + - template: /eng/common/templates-official/job/source-build.yml + parameters: + jobNamePrefix: ${{ parameters.jobNamePrefix }} + platform: ${{ platform }} + +- ${{ if eq(length(parameters.platforms), 0) }}: + - template: /eng/common/templates-official/job/source-build.yml + parameters: + jobNamePrefix: ${{ parameters.jobNamePrefix }} + platform: ${{ parameters.defaultManagedPlatform }} diff --git a/eng/common/templates-official/post-build/common-variables.yml b/eng/common/templates-official/post-build/common-variables.yml new file mode 100644 index 00000000000..c24193acfc9 --- /dev/null +++ b/eng/common/templates-official/post-build/common-variables.yml @@ -0,0 +1,22 @@ +variables: + - group: Publish-Build-Assets + + # Whether the build is internal or not + - name: IsInternalBuild + value: ${{ and(ne(variables['System.TeamProject'], 'public'), contains(variables['Build.SourceBranch'], 'internal')) }} + + # Default Maestro++ API Endpoint and API Version + - name: MaestroApiEndPoint + value: "https://maestro-prod.westus2.cloudapp.azure.com" + - name: MaestroApiAccessToken + value: $(MaestroAccessToken) + - name: MaestroApiVersion + value: "2020-02-20" + + - name: SourceLinkCLIVersion + value: 3.0.0 + - name: SymbolToolVersion + value: 1.0.1 + + - name: runCodesignValidationInjection + value: false diff --git a/eng/common/templates-official/post-build/post-build.yml b/eng/common/templates-official/post-build/post-build.yml new file mode 100644 index 00000000000..5c98fe1c0f3 --- /dev/null +++ b/eng/common/templates-official/post-build/post-build.yml @@ -0,0 +1,285 @@ +parameters: + # Which publishing infra should be used. THIS SHOULD MATCH THE VERSION ON THE BUILD MANIFEST. + # Publishing V1 is no longer supported + # Publishing V2 is no longer supported + # Publishing V3 is the default + - name: publishingInfraVersion + displayName: Which version of publishing should be used to promote the build definition? + type: number + default: 3 + values: + - 3 + + - name: BARBuildId + displayName: BAR Build Id + type: number + default: 0 + + - name: PromoteToChannelIds + displayName: Channel to promote BARBuildId to + type: string + default: '' + + - name: enableSourceLinkValidation + displayName: Enable SourceLink validation + type: boolean + default: false + + - name: enableSigningValidation + displayName: Enable signing validation + type: boolean + default: true + + - name: enableSymbolValidation + displayName: Enable symbol validation + type: boolean + default: false + + - name: enableNugetValidation + displayName: Enable NuGet validation + type: boolean + default: true + + - name: publishInstallersAndChecksums + displayName: Publish installers and checksums + type: boolean + default: true + + - name: SDLValidationParameters + type: object + default: + enable: false + publishGdn: false + continueOnError: false + params: '' + artifactNames: '' + downloadArtifacts: true + + # These parameters let the user customize the call to sdk-task.ps1 for publishing + # symbols & general artifacts as well as for signing validation + - name: symbolPublishingAdditionalParameters + displayName: Symbol publishing additional parameters + type: string + default: '' + + - name: artifactsPublishingAdditionalParameters + displayName: Artifact publishing additional parameters + type: string + default: '' + + - name: signingValidationAdditionalParameters + displayName: Signing validation additional parameters + type: string + default: '' + + # Which stages should finish execution before post-build stages start + - name: validateDependsOn + type: object + default: + - build + + - name: publishDependsOn + type: object + default: + - Validate + + # Optional: Call asset publishing rather than running in a separate stage + - name: publishAssetsImmediately + type: boolean + default: false + +stages: +- ${{ if or(eq( parameters.enableNugetValidation, 'true'), eq(parameters.enableSigningValidation, 'true'), eq(parameters.enableSourceLinkValidation, 'true'), eq(parameters.SDLValidationParameters.enable, 'true')) }}: + - stage: Validate + dependsOn: ${{ parameters.validateDependsOn }} + displayName: Validate Build Assets + variables: + - template: common-variables.yml + - template: /eng/common/templates-official/variables/pool-providers.yml + jobs: + - job: + displayName: NuGet Validation + condition: and(succeededOrFailed(), eq( ${{ parameters.enableNugetValidation }}, 'true')) + pool: + # We don't use the collection uri here because it might vary (.visualstudio.com vs. dev.azure.com) + ${{ if eq(variables['System.TeamProject'], 'DevDiv') }}: + name: AzurePipelines-EO + image: 1ESPT-Windows2022 + demands: Cmd + os: windows + # If it's not devdiv, it's dnceng + ${{ else }}: + name: $(DncEngInternalBuildPool) + image: 1es-windows-2022-pt + os: windows + + steps: + - template: setup-maestro-vars.yml + parameters: + BARBuildId: ${{ parameters.BARBuildId }} + PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} + + - task: DownloadBuildArtifacts@0 + displayName: Download Package Artifacts + inputs: + buildType: specific + buildVersionToDownload: specific + project: $(AzDOProjectName) + pipeline: $(AzDOPipelineId) + buildId: $(AzDOBuildId) + artifactName: PackageArtifacts + checkDownloadedFiles: true + + - task: PowerShell@2 + displayName: Validate + inputs: + filePath: $(Build.SourcesDirectory)/eng/common/post-build/nuget-validation.ps1 + arguments: -PackagesPath $(Build.ArtifactStagingDirectory)/PackageArtifacts/ + -ToolDestinationPath $(Agent.BuildDirectory)/Extract/ + + - job: + displayName: Signing Validation + condition: and( eq( ${{ parameters.enableSigningValidation }}, 'true'), ne( variables['PostBuildSign'], 'true')) + pool: + # We don't use the collection uri here because it might vary (.visualstudio.com vs. dev.azure.com) + ${{ if eq(variables['System.TeamProject'], 'DevDiv') }}: + name: AzurePipelines-EO + image: 1ESPT-Windows2022 + demands: Cmd + os: windows + # If it's not devdiv, it's dnceng + ${{ else }}: + name: $(DncEngInternalBuildPool) + image: 1es-windows-2022-pt + os: windows + steps: + - template: setup-maestro-vars.yml + parameters: + BARBuildId: ${{ parameters.BARBuildId }} + PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} + + - task: DownloadBuildArtifacts@0 + displayName: Download Package Artifacts + inputs: + buildType: specific + buildVersionToDownload: specific + project: $(AzDOProjectName) + pipeline: $(AzDOPipelineId) + buildId: $(AzDOBuildId) + artifactName: PackageArtifacts + checkDownloadedFiles: true + itemPattern: | + ** + !**/Microsoft.SourceBuild.Intermediate.*.nupkg + + # This is necessary whenever we want to publish/restore to an AzDO private feed + # Since sdk-task.ps1 tries to restore packages we need to do this authentication here + # otherwise it'll complain about accessing a private feed. + - task: NuGetAuthenticate@1 + displayName: 'Authenticate to AzDO Feeds' + + # Signing validation will optionally work with the buildmanifest file which is downloaded from + # Azure DevOps above. + - task: PowerShell@2 + displayName: Validate + inputs: + filePath: eng\common\sdk-task.ps1 + arguments: -task SigningValidation -restore -msbuildEngine vs + /p:PackageBasePath='$(Build.ArtifactStagingDirectory)/PackageArtifacts' + /p:SignCheckExclusionsFile='$(Build.SourcesDirectory)/eng/SignCheckExclusionsFile.txt' + ${{ parameters.signingValidationAdditionalParameters }} + + - template: ../steps/publish-logs.yml + parameters: + StageLabel: 'Validation' + JobLabel: 'Signing' + BinlogToolVersion: $(BinlogToolVersion) + + - job: + displayName: SourceLink Validation + condition: eq( ${{ parameters.enableSourceLinkValidation }}, 'true') + pool: + # We don't use the collection uri here because it might vary (.visualstudio.com vs. dev.azure.com) + ${{ if eq(variables['System.TeamProject'], 'DevDiv') }}: + name: AzurePipelines-EO + image: 1ESPT-Windows2022 + demands: Cmd + os: windows + # If it's not devdiv, it's dnceng + ${{ else }}: + name: $(DncEngInternalBuildPool) + image: 1es-windows-2022-pt + os: windows + steps: + - template: setup-maestro-vars.yml + parameters: + BARBuildId: ${{ parameters.BARBuildId }} + PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} + + - task: DownloadBuildArtifacts@0 + displayName: Download Blob Artifacts + inputs: + buildType: specific + buildVersionToDownload: specific + project: $(AzDOProjectName) + pipeline: $(AzDOPipelineId) + buildId: $(AzDOBuildId) + artifactName: BlobArtifacts + checkDownloadedFiles: true + + - task: PowerShell@2 + displayName: Validate + inputs: + filePath: $(Build.SourcesDirectory)/eng/common/post-build/sourcelink-validation.ps1 + arguments: -InputPath $(Build.ArtifactStagingDirectory)/BlobArtifacts/ + -ExtractPath $(Agent.BuildDirectory)/Extract/ + -GHRepoName $(Build.Repository.Name) + -GHCommit $(Build.SourceVersion) + -SourcelinkCliVersion $(SourceLinkCLIVersion) + continueOnError: true + +- ${{ if ne(parameters.publishAssetsImmediately, 'true') }}: + - stage: publish_using_darc + ${{ if or(eq(parameters.enableNugetValidation, 'true'), eq(parameters.enableSigningValidation, 'true'), eq(parameters.enableSourceLinkValidation, 'true'), eq(parameters.SDLValidationParameters.enable, 'true')) }}: + dependsOn: ${{ parameters.publishDependsOn }} + ${{ else }}: + dependsOn: ${{ parameters.validateDependsOn }} + displayName: Publish using Darc + variables: + - template: common-variables.yml + - template: /eng/common/templates-official/variables/pool-providers.yml + jobs: + - job: + displayName: Publish Using Darc + timeoutInMinutes: 120 + pool: + # We don't use the collection uri here because it might vary (.visualstudio.com vs. dev.azure.com) + ${{ if eq(variables['System.TeamProject'], 'DevDiv') }}: + name: AzurePipelines-EO + image: 1ESPT-Windows2022 + demands: Cmd + os: windows + # If it's not devdiv, it's dnceng + ${{ else }}: + name: $(DncEngInternalBuildPool) + image: 1es-windows-2022-pt + os: windows + steps: + - template: setup-maestro-vars.yml + parameters: + BARBuildId: ${{ parameters.BARBuildId }} + PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} + + - task: NuGetAuthenticate@1 + + - task: PowerShell@2 + displayName: Publish Using Darc + inputs: + filePath: $(Build.SourcesDirectory)/eng/common/post-build/publish-using-darc.ps1 + arguments: -BuildId $(BARBuildId) + -PublishingInfraVersion ${{ parameters.publishingInfraVersion }} + -AzdoToken '$(publishing-dnceng-devdiv-code-r-build-re)' + -MaestroToken '$(MaestroApiAccessToken)' + -WaitPublishingFinish true + -ArtifactsPublishingAdditionalParameters '${{ parameters.artifactsPublishingAdditionalParameters }}' + -SymbolPublishingAdditionalParameters '${{ parameters.symbolPublishingAdditionalParameters }}' diff --git a/eng/common/templates-official/post-build/setup-maestro-vars.yml b/eng/common/templates-official/post-build/setup-maestro-vars.yml new file mode 100644 index 00000000000..0c87f149a4a --- /dev/null +++ b/eng/common/templates-official/post-build/setup-maestro-vars.yml @@ -0,0 +1,70 @@ +parameters: + BARBuildId: '' + PromoteToChannelIds: '' + +steps: + - ${{ if eq(coalesce(parameters.PromoteToChannelIds, 0), 0) }}: + - task: DownloadBuildArtifacts@0 + displayName: Download Release Configs + inputs: + buildType: current + artifactName: ReleaseConfigs + checkDownloadedFiles: true + + - task: PowerShell@2 + name: setReleaseVars + displayName: Set Release Configs Vars + inputs: + targetType: inline + pwsh: true + script: | + try { + if (!$Env:PromoteToMaestroChannels -or $Env:PromoteToMaestroChannels.Trim() -eq '') { + $Content = Get-Content $(Build.StagingDirectory)/ReleaseConfigs/ReleaseConfigs.txt + + $BarId = $Content | Select -Index 0 + $Channels = $Content | Select -Index 1 + $IsStableBuild = $Content | Select -Index 2 + + $AzureDevOpsProject = $Env:System_TeamProject + $AzureDevOpsBuildDefinitionId = $Env:System_DefinitionId + $AzureDevOpsBuildId = $Env:Build_BuildId + } + else { + $buildApiEndpoint = "${Env:MaestroApiEndPoint}/api/builds/${Env:BARBuildId}?api-version=${Env:MaestroApiVersion}" + + $apiHeaders = New-Object 'System.Collections.Generic.Dictionary[[String],[String]]' + $apiHeaders.Add('Accept', 'application/json') + $apiHeaders.Add('Authorization',"Bearer ${Env:MAESTRO_API_TOKEN}") + + $buildInfo = try { Invoke-WebRequest -Method Get -Uri $buildApiEndpoint -Headers $apiHeaders | ConvertFrom-Json } catch { Write-Host "Error: $_" } + + $BarId = $Env:BARBuildId + $Channels = $Env:PromoteToMaestroChannels -split "," + $Channels = $Channels -join "][" + $Channels = "[$Channels]" + + $IsStableBuild = $buildInfo.stable + $AzureDevOpsProject = $buildInfo.azureDevOpsProject + $AzureDevOpsBuildDefinitionId = $buildInfo.azureDevOpsBuildDefinitionId + $AzureDevOpsBuildId = $buildInfo.azureDevOpsBuildId + } + + Write-Host "##vso[task.setvariable variable=BARBuildId]$BarId" + Write-Host "##vso[task.setvariable variable=TargetChannels]$Channels" + Write-Host "##vso[task.setvariable variable=IsStableBuild]$IsStableBuild" + + Write-Host "##vso[task.setvariable variable=AzDOProjectName]$AzureDevOpsProject" + Write-Host "##vso[task.setvariable variable=AzDOPipelineId]$AzureDevOpsBuildDefinitionId" + Write-Host "##vso[task.setvariable variable=AzDOBuildId]$AzureDevOpsBuildId" + } + catch { + Write-Host $_ + Write-Host $_.Exception + Write-Host $_.ScriptStackTrace + exit 1 + } + env: + MAESTRO_API_TOKEN: $(MaestroApiAccessToken) + BARBuildId: ${{ parameters.BARBuildId }} + PromoteToMaestroChannels: ${{ parameters.PromoteToChannelIds }} diff --git a/eng/common/templates-official/post-build/trigger-subscription.yml b/eng/common/templates-official/post-build/trigger-subscription.yml new file mode 100644 index 00000000000..da669030daf --- /dev/null +++ b/eng/common/templates-official/post-build/trigger-subscription.yml @@ -0,0 +1,13 @@ +parameters: + ChannelId: 0 + +steps: +- task: PowerShell@2 + displayName: Triggering subscriptions + inputs: + filePath: $(Build.SourcesDirectory)/eng/common/post-build/trigger-subscriptions.ps1 + arguments: -SourceRepo $(Build.Repository.Uri) + -ChannelId ${{ parameters.ChannelId }} + -MaestroApiAccessToken $(MaestroAccessToken) + -MaestroApiEndPoint $(MaestroApiEndPoint) + -MaestroApiVersion $(MaestroApiVersion) diff --git a/eng/common/templates-official/steps/add-build-to-channel.yml b/eng/common/templates-official/steps/add-build-to-channel.yml new file mode 100644 index 00000000000..f67a210d62f --- /dev/null +++ b/eng/common/templates-official/steps/add-build-to-channel.yml @@ -0,0 +1,13 @@ +parameters: + ChannelId: 0 + +steps: +- task: PowerShell@2 + displayName: Add Build to Channel + inputs: + filePath: $(Build.SourcesDirectory)/eng/common/post-build/add-build-to-channel.ps1 + arguments: -BuildId $(BARBuildId) + -ChannelId ${{ parameters.ChannelId }} + -MaestroApiAccessToken $(MaestroApiAccessToken) + -MaestroApiEndPoint $(MaestroApiEndPoint) + -MaestroApiVersion $(MaestroApiVersion) diff --git a/eng/common/templates-official/steps/build-reason.yml b/eng/common/templates-official/steps/build-reason.yml new file mode 100644 index 00000000000..eba58109b52 --- /dev/null +++ b/eng/common/templates-official/steps/build-reason.yml @@ -0,0 +1,12 @@ +# build-reason.yml +# Description: runs steps if build.reason condition is valid. conditions is a string of valid build reasons +# to include steps (',' separated). +parameters: + conditions: '' + steps: [] + +steps: + - ${{ if and( not(startsWith(parameters.conditions, 'not')), contains(parameters.conditions, variables['build.reason'])) }}: + - ${{ parameters.steps }} + - ${{ if and( startsWith(parameters.conditions, 'not'), not(contains(parameters.conditions, variables['build.reason']))) }}: + - ${{ parameters.steps }} diff --git a/eng/common/templates-official/steps/component-governance.yml b/eng/common/templates-official/steps/component-governance.yml new file mode 100644 index 00000000000..0ecec47b0c9 --- /dev/null +++ b/eng/common/templates-official/steps/component-governance.yml @@ -0,0 +1,13 @@ +parameters: + disableComponentGovernance: false + componentGovernanceIgnoreDirectories: '' + +steps: +- ${{ if eq(parameters.disableComponentGovernance, 'true') }}: + - script: "echo ##vso[task.setvariable variable=skipComponentGovernanceDetection]true" + displayName: Set skipComponentGovernanceDetection variable +- ${{ if ne(parameters.disableComponentGovernance, 'true') }}: + - task: ComponentGovernanceComponentDetection@0 + continueOnError: true + inputs: + ignoreDirectories: ${{ parameters.componentGovernanceIgnoreDirectories }} \ No newline at end of file diff --git a/eng/common/templates-official/steps/execute-codeql.yml b/eng/common/templates-official/steps/execute-codeql.yml new file mode 100644 index 00000000000..9b4a5ffa30a --- /dev/null +++ b/eng/common/templates-official/steps/execute-codeql.yml @@ -0,0 +1,32 @@ +parameters: + # Language that should be analyzed. Defaults to csharp + language: csharp + # Build Commands + buildCommands: '' + overrideParameters: '' # Optional: to override values for parameters. + additionalParameters: '' # Optional: parameters that need user specific values eg: '-SourceToolsList @("abc","def") -ArtifactToolsList @("ghi","jkl")' + # Optional: if specified, restore and use this version of Guardian instead of the default. + overrideGuardianVersion: '' + # Optional: if true, publish the '.gdn' folder as a pipeline artifact. This can help with in-depth + # diagnosis of problems with specific tool configurations. + publishGuardianDirectoryToPipeline: false + # The script to run to execute all SDL tools. Use this if you want to use a script to define SDL + # parameters rather than relying on YAML. It may be better to use a local script, because you can + # reproduce results locally without piecing together a command based on the YAML. + executeAllSdlToolsScript: 'eng/common/sdl/execute-all-sdl-tools.ps1' + # There is some sort of bug (has been reported) in Azure DevOps where if this parameter is named + # 'continueOnError', the parameter value is not correctly picked up. + # This can also be remedied by the caller (post-build.yml) if it does not use a nested parameter + # optional: determines whether to continue the build if the step errors; + sdlContinueOnError: false + +steps: +- template: /eng/common/templates-official/steps/execute-sdl.yml + parameters: + overrideGuardianVersion: ${{ parameters.overrideGuardianVersion }} + executeAllSdlToolsScript: ${{ parameters.executeAllSdlToolsScript }} + overrideParameters: ${{ parameters.overrideParameters }} + additionalParameters: '${{ parameters.additionalParameters }} + -CodeQLAdditionalRunConfigParams @("BuildCommands < ${{ parameters.buildCommands }}", "Language < ${{ parameters.language }}")' + publishGuardianDirectoryToPipeline: ${{ parameters.publishGuardianDirectoryToPipeline }} + sdlContinueOnError: ${{ parameters.sdlContinueOnError }} \ No newline at end of file diff --git a/eng/common/templates-official/steps/execute-sdl.yml b/eng/common/templates-official/steps/execute-sdl.yml new file mode 100644 index 00000000000..07426fde05d --- /dev/null +++ b/eng/common/templates-official/steps/execute-sdl.yml @@ -0,0 +1,88 @@ +parameters: + overrideGuardianVersion: '' + executeAllSdlToolsScript: '' + overrideParameters: '' + additionalParameters: '' + publishGuardianDirectoryToPipeline: false + sdlContinueOnError: false + condition: '' + +steps: +- task: NuGetAuthenticate@1 + inputs: + nuGetServiceConnections: GuardianConnect + +- task: NuGetToolInstaller@1 + displayName: 'Install NuGet.exe' + +- ${{ if ne(parameters.overrideGuardianVersion, '') }}: + - pwsh: | + Set-Location -Path $(Build.SourcesDirectory)\eng\common\sdl + . .\sdl.ps1 + $guardianCliLocation = Install-Gdn -Path $(Build.SourcesDirectory)\.artifacts -Version ${{ parameters.overrideGuardianVersion }} + Write-Host "##vso[task.setvariable variable=GuardianCliLocation]$guardianCliLocation" + displayName: Install Guardian (Overridden) + +- ${{ if eq(parameters.overrideGuardianVersion, '') }}: + - pwsh: | + Set-Location -Path $(Build.SourcesDirectory)\eng\common\sdl + . .\sdl.ps1 + $guardianCliLocation = Install-Gdn -Path $(Build.SourcesDirectory)\.artifacts + Write-Host "##vso[task.setvariable variable=GuardianCliLocation]$guardianCliLocation" + displayName: Install Guardian + +- ${{ if ne(parameters.overrideParameters, '') }}: + - powershell: ${{ parameters.executeAllSdlToolsScript }} ${{ parameters.overrideParameters }} + displayName: Execute SDL (Overridden) + continueOnError: ${{ parameters.sdlContinueOnError }} + condition: ${{ parameters.condition }} + +- ${{ if eq(parameters.overrideParameters, '') }}: + - powershell: ${{ parameters.executeAllSdlToolsScript }} + -GuardianCliLocation $(GuardianCliLocation) + -NugetPackageDirectory $(Build.SourcesDirectory)\.packages + -AzureDevOpsAccessToken $(dn-bot-dotnet-build-rw-code-rw) + ${{ parameters.additionalParameters }} + displayName: Execute SDL + continueOnError: ${{ parameters.sdlContinueOnError }} + condition: ${{ parameters.condition }} + +- ${{ if ne(parameters.publishGuardianDirectoryToPipeline, 'false') }}: + # We want to publish the Guardian results and configuration for easy diagnosis. However, the + # '.gdn' dir is a mix of configuration, results, extracted dependencies, and Guardian default + # tooling files. Some of these files are large and aren't useful during an investigation, so + # exclude them by simply deleting them before publishing. (As of writing, there is no documented + # way to selectively exclude a dir from the pipeline artifact publish task.) + - task: DeleteFiles@1 + displayName: Delete Guardian dependencies to avoid uploading + inputs: + SourceFolder: $(Agent.BuildDirectory)/.gdn + Contents: | + c + i + condition: succeededOrFailed() + + - publish: $(Agent.BuildDirectory)/.gdn + artifact: GuardianConfiguration + displayName: Publish GuardianConfiguration + condition: succeededOrFailed() + + # Publish the SARIF files in a container named CodeAnalysisLogs to enable integration + # with the "SARIF SAST Scans Tab" Azure DevOps extension + - task: CopyFiles@2 + displayName: Copy SARIF files + inputs: + flattenFolders: true + sourceFolder: $(Agent.BuildDirectory)/.gdn/rc/ + contents: '**/*.sarif' + targetFolder: $(Build.SourcesDirectory)/CodeAnalysisLogs + condition: succeededOrFailed() + + # Use PublishBuildArtifacts because the SARIF extension only checks this case + # see microsoft/sarif-azuredevops-extension#4 + - task: PublishBuildArtifacts@1 + displayName: Publish SARIF files to CodeAnalysisLogs container + inputs: + pathToPublish: $(Build.SourcesDirectory)/CodeAnalysisLogs + artifactName: CodeAnalysisLogs + condition: succeededOrFailed() \ No newline at end of file diff --git a/eng/common/templates-official/steps/generate-sbom.yml b/eng/common/templates-official/steps/generate-sbom.yml new file mode 100644 index 00000000000..1bf43bf807a --- /dev/null +++ b/eng/common/templates-official/steps/generate-sbom.yml @@ -0,0 +1,48 @@ +# BuildDropPath - The root folder of the drop directory for which the manifest file will be generated. +# PackageName - The name of the package this SBOM represents. +# PackageVersion - The version of the package this SBOM represents. +# ManifestDirPath - The path of the directory where the generated manifest files will be placed +# IgnoreDirectories - Directories to ignore for SBOM generation. This will be passed through to the CG component detector. + +parameters: + PackageVersion: 8.0.0 + BuildDropPath: '$(Build.SourcesDirectory)/artifacts' + PackageName: '.NET' + ManifestDirPath: $(Build.ArtifactStagingDirectory)/sbom + IgnoreDirectories: '' + sbomContinueOnError: true + +steps: +- task: PowerShell@2 + displayName: Prep for SBOM generation in (Non-linux) + condition: or(eq(variables['Agent.Os'], 'Windows_NT'), eq(variables['Agent.Os'], 'Darwin')) + inputs: + filePath: ./eng/common/generate-sbom-prep.ps1 + arguments: ${{parameters.manifestDirPath}} + +# Chmodding is a workaround for https://github.com/dotnet/arcade/issues/8461 +- script: | + chmod +x ./eng/common/generate-sbom-prep.sh + ./eng/common/generate-sbom-prep.sh ${{parameters.manifestDirPath}} + displayName: Prep for SBOM generation in (Linux) + condition: eq(variables['Agent.Os'], 'Linux') + continueOnError: ${{ parameters.sbomContinueOnError }} + +- task: AzureArtifacts.manifest-generator-task.manifest-generator-task.ManifestGeneratorTask@0 + displayName: 'Generate SBOM manifest' + continueOnError: ${{ parameters.sbomContinueOnError }} + inputs: + PackageName: ${{ parameters.packageName }} + BuildDropPath: ${{ parameters.buildDropPath }} + PackageVersion: ${{ parameters.packageVersion }} + ManifestDirPath: ${{ parameters.manifestDirPath }} + ${{ if ne(parameters.IgnoreDirectories, '') }}: + AdditionalComponentDetectorArgs: '--IgnoreDirectories ${{ parameters.IgnoreDirectories }}' + +- task: 1ES.PublishPipelineArtifact@1 + displayName: Publish SBOM manifest + continueOnError: ${{parameters.sbomContinueOnError}} + inputs: + targetPath: '${{parameters.manifestDirPath}}' + artifactName: $(ARTIFACT_NAME) + diff --git a/eng/common/templates-official/steps/publish-logs.yml b/eng/common/templates-official/steps/publish-logs.yml new file mode 100644 index 00000000000..04012fed182 --- /dev/null +++ b/eng/common/templates-official/steps/publish-logs.yml @@ -0,0 +1,23 @@ +parameters: + StageLabel: '' + JobLabel: '' + +steps: +- task: Powershell@2 + displayName: Prepare Binlogs to Upload + inputs: + targetType: inline + script: | + New-Item -ItemType Directory $(Build.SourcesDirectory)/PostBuildLogs/${{parameters.StageLabel}}/${{parameters.JobLabel}}/ + Move-Item -Path $(Build.SourcesDirectory)/artifacts/log/Debug/* $(Build.SourcesDirectory)/PostBuildLogs/${{parameters.StageLabel}}/${{parameters.JobLabel}}/ + continueOnError: true + condition: always() + +- task: 1ES.PublishBuildArtifacts@1 + displayName: Publish Logs + inputs: + PathtoPublish: '$(Build.SourcesDirectory)/PostBuildLogs' + PublishLocation: Container + ArtifactName: PostBuildLogs + continueOnError: true + condition: always() diff --git a/eng/common/templates-official/steps/retain-build.yml b/eng/common/templates-official/steps/retain-build.yml new file mode 100644 index 00000000000..83d97a26a01 --- /dev/null +++ b/eng/common/templates-official/steps/retain-build.yml @@ -0,0 +1,28 @@ +parameters: + # Optional azure devops PAT with build execute permissions for the build's organization, + # only needed if the build that should be retained ran on a different organization than + # the pipeline where this template is executing from + Token: '' + # Optional BuildId to retain, defaults to the current running build + BuildId: '' + # Azure devops Organization URI for the build in the https://dev.azure.com/ format. + # Defaults to the organization the current pipeline is running on + AzdoOrgUri: '$(System.CollectionUri)' + # Azure devops project for the build. Defaults to the project the current pipeline is running on + AzdoProject: '$(System.TeamProject)' + +steps: + - task: powershell@2 + inputs: + targetType: 'filePath' + filePath: eng/common/retain-build.ps1 + pwsh: true + arguments: > + -AzdoOrgUri: ${{parameters.AzdoOrgUri}} + -AzdoProject ${{parameters.AzdoProject}} + -Token ${{coalesce(parameters.Token, '$env:SYSTEM_ACCESSTOKEN') }} + -BuildId ${{coalesce(parameters.BuildId, '$env:BUILD_ID')}} + displayName: Enable permanent build retention + env: + SYSTEM_ACCESSTOKEN: $(System.AccessToken) + BUILD_ID: $(Build.BuildId) \ No newline at end of file diff --git a/eng/common/templates-official/steps/send-to-helix.yml b/eng/common/templates-official/steps/send-to-helix.yml new file mode 100644 index 00000000000..3eb7e2d5f84 --- /dev/null +++ b/eng/common/templates-official/steps/send-to-helix.yml @@ -0,0 +1,91 @@ +# Please remember to update the documentation if you make changes to these parameters! +parameters: + HelixSource: 'pr/default' # required -- sources must start with pr/, official/, prodcon/, or agent/ + HelixType: 'tests/default/' # required -- Helix telemetry which identifies what type of data this is; should include "test" for clarity and must end in '/' + HelixBuild: $(Build.BuildNumber) # required -- the build number Helix will use to identify this -- automatically set to the AzDO build number + HelixTargetQueues: '' # required -- semicolon-delimited list of Helix queues to test on; see https://helix.dot.net/ for a list of queues + HelixAccessToken: '' # required -- access token to make Helix API requests; should be provided by the appropriate variable group + HelixConfiguration: '' # optional -- additional property attached to a job + HelixPreCommands: '' # optional -- commands to run before Helix work item execution + HelixPostCommands: '' # optional -- commands to run after Helix work item execution + WorkItemDirectory: '' # optional -- a payload directory to zip up and send to Helix; requires WorkItemCommand; incompatible with XUnitProjects + WorkItemCommand: '' # optional -- a command to execute on the payload; requires WorkItemDirectory; incompatible with XUnitProjects + WorkItemTimeout: '' # optional -- a timeout in TimeSpan.Parse-ready value (e.g. 00:02:00) for the work item command; requires WorkItemDirectory; incompatible with XUnitProjects + CorrelationPayloadDirectory: '' # optional -- a directory to zip up and send to Helix as a correlation payload + XUnitProjects: '' # optional -- semicolon-delimited list of XUnitProjects to parse and send to Helix; requires XUnitRuntimeTargetFramework, XUnitPublishTargetFramework, XUnitRunnerVersion, and IncludeDotNetCli=true + XUnitWorkItemTimeout: '' # optional -- the workitem timeout in seconds for all workitems created from the xUnit projects specified by XUnitProjects + XUnitPublishTargetFramework: '' # optional -- framework to use to publish your xUnit projects + XUnitRuntimeTargetFramework: '' # optional -- framework to use for the xUnit console runner + XUnitRunnerVersion: '' # optional -- version of the xUnit nuget package you wish to use on Helix; required for XUnitProjects + IncludeDotNetCli: false # optional -- true will download a version of the .NET CLI onto the Helix machine as a correlation payload; requires DotNetCliPackageType and DotNetCliVersion + DotNetCliPackageType: '' # optional -- either 'sdk', 'runtime' or 'aspnetcore-runtime'; determines whether the sdk or runtime will be sent to Helix; see https://raw.githubusercontent.com/dotnet/core/main/release-notes/releases-index.json + DotNetCliVersion: '' # optional -- version of the CLI to send to Helix; based on this: https://raw.githubusercontent.com/dotnet/core/main/release-notes/releases-index.json + WaitForWorkItemCompletion: true # optional -- true will make the task wait until work items have been completed and fail the build if work items fail. False is "fire and forget." + IsExternal: false # [DEPRECATED] -- doesn't do anything, jobs are external if HelixAccessToken is empty and Creator is set + HelixBaseUri: 'https://helix.dot.net/' # optional -- sets the Helix API base URI (allows targeting https://helix.int-dot.net ) + Creator: '' # optional -- if the build is external, use this to specify who is sending the job + DisplayNamePrefix: 'Run Tests' # optional -- rename the beginning of the displayName of the steps in AzDO + condition: succeeded() # optional -- condition for step to execute; defaults to succeeded() + continueOnError: false # optional -- determines whether to continue the build if the step errors; defaults to false + +steps: + - powershell: 'powershell "$env:BUILD_SOURCESDIRECTORY\eng\common\msbuild.ps1 $env:BUILD_SOURCESDIRECTORY\eng\common\helixpublish.proj /restore /p:TreatWarningsAsErrors=false /t:Test /bl:$env:BUILD_SOURCESDIRECTORY\artifacts\log\$env:BuildConfig\SendToHelix.binlog"' + displayName: ${{ parameters.DisplayNamePrefix }} (Windows) + env: + BuildConfig: $(_BuildConfig) + HelixSource: ${{ parameters.HelixSource }} + HelixType: ${{ parameters.HelixType }} + HelixBuild: ${{ parameters.HelixBuild }} + HelixConfiguration: ${{ parameters.HelixConfiguration }} + HelixTargetQueues: ${{ parameters.HelixTargetQueues }} + HelixAccessToken: ${{ parameters.HelixAccessToken }} + HelixPreCommands: ${{ parameters.HelixPreCommands }} + HelixPostCommands: ${{ parameters.HelixPostCommands }} + WorkItemDirectory: ${{ parameters.WorkItemDirectory }} + WorkItemCommand: ${{ parameters.WorkItemCommand }} + WorkItemTimeout: ${{ parameters.WorkItemTimeout }} + CorrelationPayloadDirectory: ${{ parameters.CorrelationPayloadDirectory }} + XUnitProjects: ${{ parameters.XUnitProjects }} + XUnitWorkItemTimeout: ${{ parameters.XUnitWorkItemTimeout }} + XUnitPublishTargetFramework: ${{ parameters.XUnitPublishTargetFramework }} + XUnitRuntimeTargetFramework: ${{ parameters.XUnitRuntimeTargetFramework }} + XUnitRunnerVersion: ${{ parameters.XUnitRunnerVersion }} + IncludeDotNetCli: ${{ parameters.IncludeDotNetCli }} + DotNetCliPackageType: ${{ parameters.DotNetCliPackageType }} + DotNetCliVersion: ${{ parameters.DotNetCliVersion }} + WaitForWorkItemCompletion: ${{ parameters.WaitForWorkItemCompletion }} + HelixBaseUri: ${{ parameters.HelixBaseUri }} + Creator: ${{ parameters.Creator }} + SYSTEM_ACCESSTOKEN: $(System.AccessToken) + condition: and(${{ parameters.condition }}, eq(variables['Agent.Os'], 'Windows_NT')) + continueOnError: ${{ parameters.continueOnError }} + - script: $BUILD_SOURCESDIRECTORY/eng/common/msbuild.sh $BUILD_SOURCESDIRECTORY/eng/common/helixpublish.proj /restore /p:TreatWarningsAsErrors=false /t:Test /bl:$BUILD_SOURCESDIRECTORY/artifacts/log/$BuildConfig/SendToHelix.binlog + displayName: ${{ parameters.DisplayNamePrefix }} (Unix) + env: + BuildConfig: $(_BuildConfig) + HelixSource: ${{ parameters.HelixSource }} + HelixType: ${{ parameters.HelixType }} + HelixBuild: ${{ parameters.HelixBuild }} + HelixConfiguration: ${{ parameters.HelixConfiguration }} + HelixTargetQueues: ${{ parameters.HelixTargetQueues }} + HelixAccessToken: ${{ parameters.HelixAccessToken }} + HelixPreCommands: ${{ parameters.HelixPreCommands }} + HelixPostCommands: ${{ parameters.HelixPostCommands }} + WorkItemDirectory: ${{ parameters.WorkItemDirectory }} + WorkItemCommand: ${{ parameters.WorkItemCommand }} + WorkItemTimeout: ${{ parameters.WorkItemTimeout }} + CorrelationPayloadDirectory: ${{ parameters.CorrelationPayloadDirectory }} + XUnitProjects: ${{ parameters.XUnitProjects }} + XUnitWorkItemTimeout: ${{ parameters.XUnitWorkItemTimeout }} + XUnitPublishTargetFramework: ${{ parameters.XUnitPublishTargetFramework }} + XUnitRuntimeTargetFramework: ${{ parameters.XUnitRuntimeTargetFramework }} + XUnitRunnerVersion: ${{ parameters.XUnitRunnerVersion }} + IncludeDotNetCli: ${{ parameters.IncludeDotNetCli }} + DotNetCliPackageType: ${{ parameters.DotNetCliPackageType }} + DotNetCliVersion: ${{ parameters.DotNetCliVersion }} + WaitForWorkItemCompletion: ${{ parameters.WaitForWorkItemCompletion }} + HelixBaseUri: ${{ parameters.HelixBaseUri }} + Creator: ${{ parameters.Creator }} + SYSTEM_ACCESSTOKEN: $(System.AccessToken) + condition: and(${{ parameters.condition }}, ne(variables['Agent.Os'], 'Windows_NT')) + continueOnError: ${{ parameters.continueOnError }} diff --git a/eng/common/templates-official/steps/source-build.yml b/eng/common/templates-official/steps/source-build.yml new file mode 100644 index 00000000000..829f17c34d1 --- /dev/null +++ b/eng/common/templates-official/steps/source-build.yml @@ -0,0 +1,129 @@ +parameters: + # This template adds arcade-powered source-build to CI. + + # This is a 'steps' template, and is intended for advanced scenarios where the existing build + # infra has a careful build methodology that must be followed. For example, a repo + # (dotnet/runtime) might choose to clone the GitHub repo only once and store it as a pipeline + # artifact for all subsequent jobs to use, to reduce dependence on a strong network connection to + # GitHub. Using this steps template leaves room for that infra to be included. + + # Defines the platform on which to run the steps. See 'eng/common/templates-official/job/source-build.yml' + # for details. The entire object is described in the 'job' template for simplicity, even though + # the usage of the properties on this object is split between the 'job' and 'steps' templates. + platform: {} + +steps: +# Build. Keep it self-contained for simple reusability. (No source-build-specific job variables.) +- script: | + set -x + df -h + + # If building on the internal project, the artifact feeds variable may be available (usually only if needed) + # In that case, call the feed setup script to add internal feeds corresponding to public ones. + # In addition, add an msbuild argument to copy the WIP from the repo to the target build location. + # This is because SetupNuGetSources.sh will alter the current NuGet.config file, and we need to preserve those + # changes. + internalRestoreArgs= + if [ '$(dn-bot-dnceng-artifact-feeds-rw)' != '$''(dn-bot-dnceng-artifact-feeds-rw)' ]; then + # Temporarily work around https://github.com/dotnet/arcade/issues/7709 + chmod +x $(Build.SourcesDirectory)/eng/common/SetupNugetSources.sh + $(Build.SourcesDirectory)/eng/common/SetupNugetSources.sh $(Build.SourcesDirectory)/NuGet.config $(dn-bot-dnceng-artifact-feeds-rw) + internalRestoreArgs='/p:CopyWipIntoInnerSourceBuildRepo=true' + + # The 'Copy WIP' feature of source build uses git stash to apply changes from the original repo. + # This only works if there is a username/email configured, which won't be the case in most CI runs. + git config --get user.email + if [ $? -ne 0 ]; then + git config user.email dn-bot@microsoft.com + git config user.name dn-bot + fi + fi + + # If building on the internal project, the internal storage variable may be available (usually only if needed) + # In that case, add variables to allow the download of internal runtimes if the specified versions are not found + # in the default public locations. + internalRuntimeDownloadArgs= + if [ '$(dotnetbuilds-internal-container-read-token-base64)' != '$''(dotnetbuilds-internal-container-read-token-base64)' ]; then + internalRuntimeDownloadArgs='/p:DotNetRuntimeSourceFeed=https://dotnetbuilds.blob.core.windows.net/internal /p:DotNetRuntimeSourceFeedKey=$(dotnetbuilds-internal-container-read-token-base64) --runtimesourcefeed https://dotnetbuilds.blob.core.windows.net/internal --runtimesourcefeedkey $(dotnetbuilds-internal-container-read-token-base64)' + fi + + buildConfig=Release + # Check if AzDO substitutes in a build config from a variable, and use it if so. + if [ '$(_BuildConfig)' != '$''(_BuildConfig)' ]; then + buildConfig='$(_BuildConfig)' + fi + + officialBuildArgs= + if [ '${{ and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}' = 'True' ]; then + officialBuildArgs='/p:DotNetPublishUsingPipelines=true /p:OfficialBuildId=$(BUILD.BUILDNUMBER)' + fi + + targetRidArgs= + if [ '${{ parameters.platform.targetRID }}' != '' ]; then + targetRidArgs='/p:TargetRid=${{ parameters.platform.targetRID }}' + fi + + runtimeOsArgs= + if [ '${{ parameters.platform.runtimeOS }}' != '' ]; then + runtimeOsArgs='/p:RuntimeOS=${{ parameters.platform.runtimeOS }}' + fi + + baseOsArgs= + if [ '${{ parameters.platform.baseOS }}' != '' ]; then + baseOsArgs='/p:BaseOS=${{ parameters.platform.baseOS }}' + fi + + publishArgs= + if [ '${{ parameters.platform.skipPublishValidation }}' != 'true' ]; then + publishArgs='--publish' + fi + + assetManifestFileName=SourceBuild_RidSpecific.xml + if [ '${{ parameters.platform.name }}' != '' ]; then + assetManifestFileName=SourceBuild_${{ parameters.platform.name }}.xml + fi + + ${{ coalesce(parameters.platform.buildScript, './build.sh') }} --ci \ + --configuration $buildConfig \ + --restore --build --pack $publishArgs -bl \ + $officialBuildArgs \ + $internalRuntimeDownloadArgs \ + $internalRestoreArgs \ + $targetRidArgs \ + $runtimeOsArgs \ + $baseOsArgs \ + /p:SourceBuildNonPortable=${{ parameters.platform.nonPortable }} \ + /p:ArcadeBuildFromSource=true \ + /p:AssetManifestFileName=$assetManifestFileName + displayName: Build + +# Upload build logs for diagnosis. +- task: CopyFiles@2 + displayName: Prepare BuildLogs staging directory + inputs: + SourceFolder: '$(Build.SourcesDirectory)' + Contents: | + **/*.log + **/*.binlog + artifacts/source-build/self/prebuilt-report/** + TargetFolder: '$(Build.StagingDirectory)/BuildLogs' + CleanTargetFolder: true + continueOnError: true + condition: succeededOrFailed() + +- task: 1ES.PublishPipelineArtifact@1 + displayName: Publish BuildLogs + inputs: + targetPath: '$(Build.StagingDirectory)/BuildLogs' + artifactName: BuildLogs_SourceBuild_${{ parameters.platform.name }}_Attempt$(System.JobAttempt) + continueOnError: true + condition: succeededOrFailed() + +# Manually inject component detection so that we can ignore the source build upstream cache, which contains +# a nupkg cache of input packages (a local feed). +# This path must match the upstream cache path in property 'CurrentRepoSourceBuiltNupkgCacheDir' +# in src\Microsoft.DotNet.Arcade.Sdk\tools\SourceBuild\SourceBuildArcade.targets +- task: ComponentGovernanceComponentDetection@0 + displayName: Component Detection (Exclude upstream cache) + inputs: + ignoreDirectories: '$(Build.SourcesDirectory)/artifacts/source-build/self/src/artifacts/obj/source-built-upstream-cache' diff --git a/eng/common/templates-official/variables/pool-providers.yml b/eng/common/templates-official/variables/pool-providers.yml new file mode 100644 index 00000000000..beab7d1bfba --- /dev/null +++ b/eng/common/templates-official/variables/pool-providers.yml @@ -0,0 +1,45 @@ +# Select a pool provider based off branch name. Anything with branch name containing 'release' must go into an -Svc pool, +# otherwise it should go into the "normal" pools. This separates out the queueing and billing of released branches. + +# Motivation: +# Once a given branch of a repository's output has been officially "shipped" once, it is then considered to be COGS +# (Cost of goods sold) and should be moved to a servicing pool provider. This allows both separation of queueing +# (allowing release builds and main PR builds to not intefere with each other) and billing (required for COGS. +# Additionally, the pool provider name itself may be subject to change when the .NET Core Engineering Services +# team needs to move resources around and create new and potentially differently-named pools. Using this template +# file from an Arcade-ified repo helps guard against both having to update one's release/* branches and renaming. + +# How to use: +# This yaml assumes your shipped product branches use the naming convention "release/..." (which many do). +# If we find alternate naming conventions in broad usage it can be added to the condition below. +# +# First, import the template in an arcade-ified repo to pick up the variables, e.g.: +# +# variables: +# - template: /eng/common/templates-official/variables/pool-providers.yml +# +# ... then anywhere specifying the pool provider use the runtime variables, +# $(DncEngInternalBuildPool) +# +# pool: +# name: $(DncEngInternalBuildPool) +# image: 1es-windows-2022-pt + +variables: + # Coalesce the target and source branches so we know when a PR targets a release branch + # If these variables are somehow missing, fall back to main (tends to have more capacity) + + # Any new -Svc alternative pools should have variables added here to allow for splitting work + + - name: DncEngInternalBuildPool + value: $[ + replace( + replace( + eq(contains(coalesce(variables['System.PullRequest.TargetBranch'], variables['Build.SourceBranch'], 'refs/heads/main'), 'release'), 'true'), + True, + 'NetCore1ESPool-Svc-Internal' + ), + False, + 'NetCore1ESPool-Internal' + ) + ] \ No newline at end of file diff --git a/eng/common/templates-official/variables/sdl-variables.yml b/eng/common/templates-official/variables/sdl-variables.yml new file mode 100644 index 00000000000..dbdd66d4a4b --- /dev/null +++ b/eng/common/templates-official/variables/sdl-variables.yml @@ -0,0 +1,7 @@ +variables: +# The Guardian version specified in 'eng/common/sdl/packages.config'. This value must be kept in +# sync with the packages.config file. +- name: DefaultGuardianVersion + value: 0.109.0 +- name: GuardianPackagesConfigFile + value: $(Build.SourcesDirectory)\eng\common\sdl\packages.config \ No newline at end of file diff --git a/eng/common/templates/job/job.yml b/eng/common/templates/job/job.yml index e24ca2f46f9..8ec5c4f2d9f 100644 --- a/eng/common/templates/job/job.yml +++ b/eng/common/templates/job/job.yml @@ -15,6 +15,7 @@ parameters: timeoutInMinutes: '' variables: [] workspace: '' + templateContext: '' # Job base template specific parameters # See schema documentation - https://github.com/dotnet/arcade/blob/master/Documentation/AzureDevOps/TemplateSchema.md @@ -68,6 +69,9 @@ jobs: ${{ if ne(parameters.timeoutInMinutes, '') }}: timeoutInMinutes: ${{ parameters.timeoutInMinutes }} + ${{ if ne(parameters.templateContext, '') }}: + templateContext: ${{ parameters.templateContext }} + variables: - ${{ if ne(parameters.enableTelemetry, 'false') }}: - name: DOTNET_CLI_TELEMETRY_PROFILE diff --git a/eng/common/templates/jobs/jobs.yml b/eng/common/templates/jobs/jobs.yml index 67a2e2c747c..289bb2396ce 100644 --- a/eng/common/templates/jobs/jobs.yml +++ b/eng/common/templates/jobs/jobs.yml @@ -20,7 +20,7 @@ parameters: enabled: false # Optional: Include toolset dependencies in the generated graph files includeToolset: false - + # Required: A collection of jobs to run - https://docs.microsoft.com/en-us/azure/devops/pipelines/yaml-schema?view=vsts&tabs=schema#job jobs: [] @@ -47,7 +47,7 @@ parameters: jobs: - ${{ each job in parameters.jobs }}: - template: ../job/job.yml - parameters: + parameters: # pass along parameters ${{ each parameter in parameters }}: ${{ if ne(parameter.key, 'jobs') }}: diff --git a/eng/common/templates/steps/generate-sbom.yml b/eng/common/templates/steps/generate-sbom.yml index a06373f38fa..2b21eae4273 100644 --- a/eng/common/templates/steps/generate-sbom.yml +++ b/eng/common/templates/steps/generate-sbom.yml @@ -5,7 +5,7 @@ # IgnoreDirectories - Directories to ignore for SBOM generation. This will be passed through to the CG component detector. parameters: - PackageVersion: 7.0.0 + PackageVersion: 8.0.0 BuildDropPath: '$(Build.SourcesDirectory)/artifacts' PackageName: '.NET' ManifestDirPath: $(Build.ArtifactStagingDirectory)/sbom diff --git a/global.json b/global.json index 29f1d538878..bd1f68c7529 100644 --- a/global.json +++ b/global.json @@ -13,7 +13,7 @@ } }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "8.0.0-beta.24113.2", - "Microsoft.DotNet.Helix.Sdk": "8.0.0-beta.24113.2" + "Microsoft.DotNet.Arcade.Sdk": "8.0.0-beta.24165.4", + "Microsoft.DotNet.Helix.Sdk": "8.0.0-beta.24165.4" } } diff --git a/src/EFCore.Relational/Extensions/RelationalPropertyExtensions.cs b/src/EFCore.Relational/Extensions/RelationalPropertyExtensions.cs index 0b0d0b7f410..929fb4f470e 100644 --- a/src/EFCore.Relational/Extensions/RelationalPropertyExtensions.cs +++ b/src/EFCore.Relational/Extensions/RelationalPropertyExtensions.cs @@ -20,6 +20,9 @@ public static class RelationalPropertyExtensions private static readonly bool UseOldBehavior32763 = AppContext.TryGetSwitch("Microsoft.EntityFrameworkCore.Issue32763", out var enabled32763) && enabled32763; + private static readonly bool UseOldBehavior33004 = + AppContext.TryGetSwitch("Microsoft.EntityFrameworkCore.Issue33004", out var enabled33004) && enabled33004; + private static readonly MethodInfo GetFieldValueMethod = typeof(DbDataReader).GetRuntimeMethod(nameof(DbDataReader.GetFieldValue), new[] { typeof(int) })!; @@ -1191,8 +1194,12 @@ public static bool IsColumnNullable(this IReadOnlyProperty property, in StoreObj return sharedTableRootProperty.IsColumnNullable(storeObject); } + var declaringEntityType = UseOldBehavior33004 + ? property.DeclaringType + : property.DeclaringType.ContainingEntityType; + return property.IsNullable - || (property.DeclaringType is IReadOnlyEntityType entityType + || (declaringEntityType is IReadOnlyEntityType entityType && ((entityType.BaseType != null && entityType.GetMappingStrategy() == RelationalAnnotationNames.TphMappingStrategy) || IsOptionalSharingDependent(entityType, storeObject, 0))); diff --git a/src/EFCore.Relational/Migrations/Internal/MigrationsModelDiffer.cs b/src/EFCore.Relational/Migrations/Internal/MigrationsModelDiffer.cs index a14998c9fd4..04cc44f1621 100644 --- a/src/EFCore.Relational/Migrations/Internal/MigrationsModelDiffer.cs +++ b/src/EFCore.Relational/Migrations/Internal/MigrationsModelDiffer.cs @@ -16,6 +16,9 @@ namespace Microsoft.EntityFrameworkCore.Migrations.Internal; /// public class MigrationsModelDiffer : IMigrationsModelDiffer { + private static readonly bool UseOldBehavior32972 = + AppContext.TryGetSwitch("Microsoft.EntityFrameworkCore.Issue32972", out var enabled32972) && enabled32972; + private static readonly Type[] DropOperationTypes = { typeof(DropIndexOperation), @@ -1190,7 +1193,14 @@ private void Initialize( if (!column.TryGetDefaultValue(out var defaultValue)) { - defaultValue = null; + // for non-nullable collections of primitives that are mapped to JSON we set a default value corresponding to empty JSON collection + defaultValue = !UseOldBehavior32972 + && !inline + && column is { IsNullable: false, StoreTypeMapping: { ElementTypeMapping: not null, Converter: ValueConverter columnValueConverter } } + && columnValueConverter.GetType() is Type { IsGenericType: true } columnValueConverterType + && columnValueConverterType.GetGenericTypeDefinition() == typeof(CollectionToJsonStringConverter<>) + ? "[]" + : null; } columnOperation.ColumnType = column.StoreType; diff --git a/src/EFCore.Relational/Query/SqlExpressions/SelectExpression.cs b/src/EFCore.Relational/Query/SqlExpressions/SelectExpression.cs index 4ddcc348373..7ac113066b1 100644 --- a/src/EFCore.Relational/Query/SqlExpressions/SelectExpression.cs +++ b/src/EFCore.Relational/Query/SqlExpressions/SelectExpression.cs @@ -30,6 +30,8 @@ public sealed partial class SelectExpression : TableExpressionBase AppContext.TryGetSwitch("Microsoft.EntityFrameworkCore.Issue31107", out var enabled31107) && enabled31107; private static readonly bool UseOldBehavior32234 = AppContext.TryGetSwitch("Microsoft.EntityFrameworkCore.Issue32234", out var enabled32234) && enabled32234; + private static readonly bool UseOldBehavior32911 = + AppContext.TryGetSwitch("Microsoft.EntityFrameworkCore.Issue32911", out var enabled32911) && enabled32911; private static readonly IdentifierComparer IdentifierComparerInstance = new(); @@ -2706,116 +2708,243 @@ void HandleStructuralTypeProjection( projection1.StructuralType.DisplayName(), projection2.StructuralType.DisplayName())); } - var propertyExpressions = new Dictionary(); - - ProcessStructuralType(projection1, projection2); - - void ProcessStructuralType( - StructuralTypeProjectionExpression nestedProjection1, - StructuralTypeProjectionExpression nestedProjection2) + if (UseOldBehavior32911) { - var type = nestedProjection1.StructuralType; + var propertyExpressions = new Dictionary(); + + ProcessStructuralType(projection1, projection2); - foreach (var property in GetAllPropertiesInHierarchy(type)) + void ProcessStructuralType( + StructuralTypeProjectionExpression nestedProjection1, + StructuralTypeProjectionExpression nestedProjection2) { - var column1 = nestedProjection1.BindProperty(property); - var column2 = nestedProjection2.BindProperty(property); - var alias = GenerateUniqueColumnAlias(column1.Name); - var innerProjection = new ProjectionExpression(column1, alias); - select1._projection.Add(innerProjection); - select2._projection.Add(new ProjectionExpression(column2, alias)); - var outerColumn = new ConcreteColumnExpression(innerProjection, tableReferenceExpression); - if (column1.IsNullable - || column2.IsNullable) + var type = nestedProjection1.StructuralType; + + foreach (var property in GetAllPropertiesInHierarchy(type)) { - outerColumn = outerColumn.MakeNullable(); - } + var column1 = nestedProjection1.BindProperty(property); + var column2 = nestedProjection2.BindProperty(property); + var alias = GenerateUniqueColumnAlias(column1.Name); + var innerProjection = new ProjectionExpression(column1, alias); + select1._projection.Add(innerProjection); + select2._projection.Add(new ProjectionExpression(column2, alias)); + var outerColumn = new ConcreteColumnExpression(innerProjection, tableReferenceExpression); + if (column1.IsNullable + || column2.IsNullable) + { + outerColumn = outerColumn.MakeNullable(); + } - propertyExpressions[property] = outerColumn; + propertyExpressions[property] = outerColumn; - // Lift up any identifier columns to the set operation result (the outer). - // This is typically the entity primary key columns, but can also be all of a complex type's properties if Distinct - // was previously called. - if (outerIdentifiers.Length > 0) - { - var index = select1._identifier.FindIndex(e => e.Column.Equals(column1)); - if (index != -1) + // Lift up any identifier columns to the set operation result (the outer). + // This is typically the entity primary key columns, but can also be all of a complex type's properties if Distinct + // was previously called. + if (outerIdentifiers.Length > 0) { - if (select2._identifier[index].Column.Equals(column2)) + var index = select1._identifier.FindIndex(e => e.Column.Equals(column1)); + if (index != -1) { - outerIdentifiers[index] = outerColumn; + if (select2._identifier[index].Column.Equals(column2)) + { + outerIdentifiers[index] = outerColumn; + } + else + { + // If select1 matched but select2 did not then we erase all identifiers + // TODO: We could make this little more robust by allow the indexes to be different. See issue#24475 + // i.e. Identifier ordering being different. + outerIdentifiers = Array.Empty(); + } } - else + // If the top-level projection - not the current nested one - is a complex type and not an entity type, then add + // all its columns to the "otherExpressions" list (i.e. columns not part of a an entity primary key). This is + // the same as with a non-structural type projection. + else if (projection1.StructuralType is IComplexType) { - // If select1 matched but select2 did not then we erase all identifiers - // TODO: We could make this little more robust by allow the indexes to be different. See issue#24475 - // i.e. Identifier ordering being different. - outerIdentifiers = Array.Empty(); + var outerTypeMapping = column1.TypeMapping ?? column1.TypeMapping; + if (outerTypeMapping == null) + { + throw new InvalidOperationException( + RelationalStrings.SetOperationsRequireAtLeastOneSideWithValidTypeMapping(setOperationType)); + } + + otherExpressions.Add((outerColumn, outerTypeMapping.KeyComparer)); } } - // If the top-level projection - not the current nested one - is a complex type and not an entity type, then add - // all its columns to the "otherExpressions" list (i.e. columns not part of a an entity primary key). This is - // the same as with a non-structural type projection. - else if (projection1.StructuralType is IComplexType) - { - var outerTypeMapping = column1.TypeMapping ?? column1.TypeMapping; - if (outerTypeMapping == null) - { - throw new InvalidOperationException( - RelationalStrings.SetOperationsRequireAtLeastOneSideWithValidTypeMapping(setOperationType)); - } + } - otherExpressions.Add((outerColumn, outerTypeMapping.KeyComparer)); - } + foreach (var complexProperty in GetAllComplexPropertiesInHierarchy(nestedProjection1.StructuralType)) + { + ProcessStructuralType( + (StructuralTypeProjectionExpression)nestedProjection1.BindComplexProperty(complexProperty).ValueBufferExpression, + (StructuralTypeProjectionExpression)nestedProjection2.BindComplexProperty(complexProperty).ValueBufferExpression); } } - foreach (var complexProperty in GetAllComplexPropertiesInHierarchy(nestedProjection1.StructuralType)) + Check.DebugAssert( + projection1.TableMap.Count == projection2.TableMap.Count, + "Set operation over entity projections with different table map counts"); + Check.DebugAssert( + projection1.TableMap.Keys.All(t => projection2.TableMap.ContainsKey(t)), + "Set operation over entity projections with table map discrepancy"); + + var tableMap = projection1.TableMap.ToDictionary(kvp => kvp.Key, kvp => tableReferenceExpression); + + var discriminatorExpression = projection1.DiscriminatorExpression; + if (projection1.DiscriminatorExpression != null + && projection2.DiscriminatorExpression != null) { - ProcessStructuralType( - (StructuralTypeProjectionExpression)nestedProjection1.BindComplexProperty(complexProperty).ValueBufferExpression, - (StructuralTypeProjectionExpression)nestedProjection2.BindComplexProperty(complexProperty).ValueBufferExpression); + var alias = GenerateUniqueColumnAlias(DiscriminatorColumnAlias); + var innerProjection = new ProjectionExpression(projection1.DiscriminatorExpression, alias); + select1._projection.Add(innerProjection); + select2._projection.Add(new ProjectionExpression(projection2.DiscriminatorExpression, alias)); + discriminatorExpression = new ConcreteColumnExpression(innerProjection, tableReferenceExpression); } - } - - Check.DebugAssert( - projection1.TableMap.Count == projection2.TableMap.Count, - "Set operation over entity projections with different table map counts"); - Check.DebugAssert( - projection1.TableMap.Keys.All(t => projection2.TableMap.ContainsKey(t)), - "Set operation over entity projections with table map discrepancy"); - var tableMap = projection1.TableMap.ToDictionary(kvp => kvp.Key, kvp => tableReferenceExpression); + var outerProjection = new StructuralTypeProjectionExpression( + projection1.StructuralType, propertyExpressions, tableMap, nullable: false, discriminatorExpression); - var discriminatorExpression = projection1.DiscriminatorExpression; - if (projection1.DiscriminatorExpression != null - && projection2.DiscriminatorExpression != null) - { - var alias = GenerateUniqueColumnAlias(DiscriminatorColumnAlias); - var innerProjection = new ProjectionExpression(projection1.DiscriminatorExpression, alias); - select1._projection.Add(innerProjection); - select2._projection.Add(new ProjectionExpression(projection2.DiscriminatorExpression, alias)); - discriminatorExpression = new ConcreteColumnExpression(innerProjection, tableReferenceExpression); - } + if (outerIdentifiers.Length > 0 && outerProjection is { StructuralType: IEntityType entityType }) + { + var primaryKey = entityType.FindPrimaryKey(); - var outerProjection = new StructuralTypeProjectionExpression( - projection1.StructuralType, propertyExpressions, tableMap, nullable: false, discriminatorExpression); + // We know that there are existing identifiers (see condition above); we know we must have a key since a keyless + // entity type would have wiped the identifiers when generating the join. + Check.DebugAssert(primaryKey != null, "primary key is null."); + foreach (var property in primaryKey.Properties) + { + entityProjectionIdentifiers.Add(outerProjection.BindProperty(property)); + entityProjectionValueComparers.Add(property.GetKeyValueComparer()); + } + } - if (outerIdentifiers.Length > 0 && outerProjection is { StructuralType: IEntityType entityType }) + _projectionMapping[projectionMember] = outerProjection; + } + else { - var primaryKey = entityType.FindPrimaryKey(); + var resultProjection = ProcessStructuralType(projection1, projection2); + _projectionMapping[projectionMember] = resultProjection; - // We know that there are existing identifiers (see condition above); we know we must have a key since a keyless - // entity type would have wiped the identifiers when generating the join. - Check.DebugAssert(primaryKey != null, "primary key is null."); - foreach (var property in primaryKey.Properties) + StructuralTypeProjectionExpression ProcessStructuralType( + StructuralTypeProjectionExpression structuralProjection1, + StructuralTypeProjectionExpression structuralProjection2) { - entityProjectionIdentifiers.Add(outerProjection.BindProperty(property)); - entityProjectionValueComparers.Add(property.GetKeyValueComparer()); + var propertyExpressions = new Dictionary(); + var complexPropertyCache = new Dictionary(); + var type = structuralProjection1.StructuralType; + + foreach (var property in GetAllPropertiesInHierarchy(type)) + { + var column1 = structuralProjection1.BindProperty(property); + var column2 = structuralProjection2.BindProperty(property); + var alias = GenerateUniqueColumnAlias(column1.Name); + var innerProjection = new ProjectionExpression(column1, alias); + select1._projection.Add(innerProjection); + select2._projection.Add(new ProjectionExpression(column2, alias)); + var outerColumn = new ConcreteColumnExpression(innerProjection, tableReferenceExpression); + if (column1.IsNullable + || column2.IsNullable) + { + outerColumn = outerColumn.MakeNullable(); + } + + propertyExpressions[property] = outerColumn; + + // Lift up any identifier columns to the set operation result (the outer). + // This is typically the entity primary key columns, but can also be all of a complex type's properties if Distinct + // was previously called. + if (outerIdentifiers.Length > 0) + { + var index = select1._identifier.FindIndex(e => e.Column.Equals(column1)); + if (index != -1) + { + if (select2._identifier[index].Column.Equals(column2)) + { + outerIdentifiers[index] = outerColumn; + } + else + { + // If select1 matched but select2 did not then we erase all identifiers + // TODO: We could make this little more robust by allow the indexes to be different. See issue#24475 + // i.e. Identifier ordering being different. + outerIdentifiers = Array.Empty(); + } + } + // If the top-level projection - not the current nested one - is a complex type and not an entity type, then add + // all its columns to the "otherExpressions" list (i.e. columns not part of a an entity primary key). This is + // the same as with a non-structural type projection. + else if (projection1.StructuralType is IComplexType) + { + var outerTypeMapping = column1.TypeMapping ?? column1.TypeMapping; + if (outerTypeMapping == null) + { + throw new InvalidOperationException( + RelationalStrings.SetOperationsRequireAtLeastOneSideWithValidTypeMapping(setOperationType)); + } + + otherExpressions.Add((outerColumn, outerTypeMapping.KeyComparer)); + } + } + } + + foreach (var complexProperty in GetAllComplexPropertiesInHierarchy(type)) + { + var complexPropertyShaper1 = structuralProjection1.BindComplexProperty(complexProperty); + var complexPropertyShaper2 = structuralProjection2.BindComplexProperty(complexProperty); + + var resultComplexProjection = ProcessStructuralType( + (StructuralTypeProjectionExpression)complexPropertyShaper1.ValueBufferExpression, + (StructuralTypeProjectionExpression)complexPropertyShaper2.ValueBufferExpression); + + var resultComplexShaper = new RelationalStructuralTypeShaperExpression( + complexProperty.ComplexType, + resultComplexProjection, + resultComplexProjection.IsNullable); + + complexPropertyCache[complexProperty] = resultComplexShaper; + } + + Check.DebugAssert( + structuralProjection1.TableMap.Count == structuralProjection2.TableMap.Count, + "Set operation over entity projections with different table map counts"); + Check.DebugAssert( + structuralProjection1.TableMap.Keys.All(t => structuralProjection2.TableMap.ContainsKey(t)), + "Set operation over entity projections with table map discrepancy"); + + var tableMap = structuralProjection1.TableMap.ToDictionary(kvp => kvp.Key, _ => tableReferenceExpression); + + var discriminatorExpression = structuralProjection1.DiscriminatorExpression; + if (structuralProjection1.DiscriminatorExpression != null + && structuralProjection2.DiscriminatorExpression != null) + { + var alias = GenerateUniqueColumnAlias(DiscriminatorColumnAlias); + var innerProjection = new ProjectionExpression(structuralProjection1.DiscriminatorExpression, alias); + select1._projection.Add(innerProjection); + select2._projection.Add(new ProjectionExpression(structuralProjection2.DiscriminatorExpression, alias)); + discriminatorExpression = new ConcreteColumnExpression(innerProjection, tableReferenceExpression); + } + + var outerProjection = new StructuralTypeProjectionExpression( + type, propertyExpressions, complexPropertyCache, tableMap, nullable: false, discriminatorExpression); + + if (outerIdentifiers.Length > 0 && outerProjection is { StructuralType: IEntityType entityType }) + { + var primaryKey = entityType.FindPrimaryKey(); + + // We know that there are existing identifiers (see condition above); we know we must have a key since a keyless + // entity type would have wiped the identifiers when generating the join. + Check.DebugAssert(primaryKey != null, "primary key is null."); + foreach (var property in primaryKey.Properties) + { + entityProjectionIdentifiers.Add(outerProjection.BindProperty(property)); + entityProjectionValueComparers.Add(property.GetKeyValueComparer()); + } + } + + return outerProjection; } } - - _projectionMapping[projectionMember] = outerProjection; } string GenerateUniqueColumnAlias(string baseAlias) @@ -4184,32 +4313,67 @@ StructuralTypeProjectionExpression LiftEntityProjectionFromSubquery( TableReferenceExpression subqueryTableReference) { var propertyExpressions = new Dictionary(); + var complexPropertyCache = new Dictionary(); + + if (UseOldBehavior32911) + { + HandleTypeProjection(projection); - HandleTypeProjection(projection); + void HandleTypeProjection(StructuralTypeProjectionExpression typeProjection) + { + foreach (var property in GetAllPropertiesInHierarchy(typeProjection.StructuralType)) + { + // json entity projection (i.e. JSON entity that was transformed into query root) may have synthesized keys + // but they don't correspond to any columns - we need to skip those + if (typeProjection is { StructuralType: IEntityType entityType } + && entityType.IsMappedToJson() + && property.IsOrdinalKeyProperty()) + { + continue; + } + + var innerColumn = typeProjection.BindProperty(property); + var outerColumn = subquery.GenerateOuterColumn(subqueryTableReferenceExpression, innerColumn); + projectionMap[innerColumn] = outerColumn; + propertyExpressions[property] = outerColumn; + } - void HandleTypeProjection(StructuralTypeProjectionExpression typeProjection) + foreach (var complexProperty in GetAllComplexPropertiesInHierarchy(typeProjection.StructuralType)) + { + HandleTypeProjection( + (StructuralTypeProjectionExpression)typeProjection.BindComplexProperty(complexProperty).ValueBufferExpression); + } + } + } + else { - foreach (var property in GetAllPropertiesInHierarchy(typeProjection.StructuralType)) + foreach (var property in GetAllPropertiesInHierarchy(projection.StructuralType)) { // json entity projection (i.e. JSON entity that was transformed into query root) may have synthesized keys // but they don't correspond to any columns - we need to skip those - if (typeProjection is { StructuralType: IEntityType entityType } + if (projection is { StructuralType: IEntityType entityType } && entityType.IsMappedToJson() && property.IsOrdinalKeyProperty()) { continue; } - var innerColumn = typeProjection.BindProperty(property); - var outerColumn = subquery.GenerateOuterColumn(subqueryTableReferenceExpression, innerColumn); + var innerColumn = projection.BindProperty(property); + var outerColumn = subquery.GenerateOuterColumn(subqueryTableReference, innerColumn); + projectionMap[innerColumn] = outerColumn; propertyExpressions[property] = outerColumn; } - foreach (var complexProperty in GetAllComplexPropertiesInHierarchy(typeProjection.StructuralType)) + foreach (var complexProperty in GetAllComplexPropertiesInHierarchy(projection.StructuralType)) { - HandleTypeProjection( - (StructuralTypeProjectionExpression)typeProjection.BindComplexProperty(complexProperty).ValueBufferExpression); + var complexPropertyShaper = projection.BindComplexProperty(complexProperty); + + var complexTypeProjectionExpression = LiftEntityProjectionFromSubquery( + (StructuralTypeProjectionExpression)complexPropertyShaper.ValueBufferExpression, + subqueryTableReference); + + complexPropertyCache[complexProperty] = complexPropertyShaper.Update(complexTypeProjectionExpression); } } @@ -4223,8 +4387,11 @@ void HandleTypeProjection(StructuralTypeProjectionExpression typeProjection) var tableMap = projection.TableMap.ToDictionary(kvp => kvp.Key, _ => subqueryTableReference); - var newEntityProjection = new StructuralTypeProjectionExpression( - projection.StructuralType, propertyExpressions, tableMap, nullable: false, discriminatorExpression); + var newEntityProjection = UseOldBehavior32911 + ? new StructuralTypeProjectionExpression( + projection.StructuralType, propertyExpressions, tableMap, nullable: false, discriminatorExpression) + : new StructuralTypeProjectionExpression( + projection.StructuralType, propertyExpressions, complexPropertyCache, tableMap, nullable: false, discriminatorExpression); if (projection.StructuralType is IEntityType entityType2) { diff --git a/src/EFCore.Relational/Query/StructuralTypeProjectionExpression.cs b/src/EFCore.Relational/Query/StructuralTypeProjectionExpression.cs index 828a7c95b2b..5b9c7dd152d 100644 --- a/src/EFCore.Relational/Query/StructuralTypeProjectionExpression.cs +++ b/src/EFCore.Relational/Query/StructuralTypeProjectionExpression.cs @@ -17,6 +17,9 @@ namespace Microsoft.EntityFrameworkCore.Query; /// public class StructuralTypeProjectionExpression : Expression { + private static readonly bool UseOldBehavior32911 = + AppContext.TryGetSwitch("Microsoft.EntityFrameworkCore.Issue32911", out var enabled32911) && enabled32911; + private readonly IReadOnlyDictionary _propertyExpressionMap; private readonly Dictionary _ownedNavigationMap; private Dictionary? _complexPropertyCache; @@ -38,6 +41,32 @@ public StructuralTypeProjectionExpression( type, propertyExpressionMap, new Dictionary(), + null, + tableMap, + nullable, + discriminatorExpression) + { + } + + /// + /// This is an internal API that supports the Entity Framework Core infrastructure and not subject to + /// the same compatibility standards as public APIs. It may be changed or removed without notice in + /// any release. You should only use it directly in your code with extreme caution and knowing that + /// doing so can result in application failures when updating to a new Entity Framework Core release. + /// + [EntityFrameworkInternal] + public StructuralTypeProjectionExpression( + ITypeBase type, + IReadOnlyDictionary propertyExpressionMap, + Dictionary complexPropertyCache, + IReadOnlyDictionary tableMap, + bool nullable = false, + SqlExpression? discriminatorExpression = null) + : this( + type, + propertyExpressionMap, + new Dictionary(), + complexPropertyCache, tableMap, nullable, discriminatorExpression) @@ -48,6 +77,7 @@ private StructuralTypeProjectionExpression( ITypeBase type, IReadOnlyDictionary propertyExpressionMap, Dictionary ownedNavigationMap, + Dictionary? complexPropertyCache, IReadOnlyDictionary tableMap, bool nullable, SqlExpression? discriminatorExpression = null) @@ -55,6 +85,7 @@ private StructuralTypeProjectionExpression( StructuralType = type; _propertyExpressionMap = propertyExpressionMap; _ownedNavigationMap = ownedNavigationMap; + _complexPropertyCache = complexPropertyCache; TableMap = tableMap; IsNullable = nullable; DiscriminatorExpression = discriminatorExpression; @@ -115,6 +146,21 @@ protected override Expression VisitChildren(ExpressionVisitor visitor) propertyExpressionMap[property] = newExpression; } + var complexPropertyCache = default(Dictionary); + if (!UseOldBehavior32911) + { + if (_complexPropertyCache != null) + { + complexPropertyCache = new(); + foreach (var (complexProperty, complexShaper) in _complexPropertyCache) + { + var newComplexShaper = (StructuralTypeShaperExpression)visitor.Visit(complexShaper); + changed |= complexShaper != newComplexShaper; + complexPropertyCache[complexProperty] = newComplexShaper; + } + } + } + // We only need to visit the table map since TableReferenceUpdatingExpressionVisitor may need to modify it; it mutates // TableReferenceExpression (a new TableReferenceExpression is never returned), so we never need a new table map. foreach (var (_, tableExpression) in TableMap) @@ -136,7 +182,7 @@ protected override Expression VisitChildren(ExpressionVisitor visitor) return changed ? new StructuralTypeProjectionExpression( - StructuralType, propertyExpressionMap, ownedNavigationMap, TableMap, IsNullable, discriminatorExpression) + StructuralType, propertyExpressionMap, ownedNavigationMap, complexPropertyCache, TableMap, IsNullable, discriminatorExpression) : this; } @@ -159,6 +205,19 @@ public virtual StructuralTypeProjectionExpression MakeNullable() discriminatorExpression = ce.MakeNullable(); } + var complexPropertyCache = default(Dictionary); + if (!UseOldBehavior32911) + { + if (_complexPropertyCache != null) + { + complexPropertyCache = new(); + foreach (var (complexProperty, complexShaper) in _complexPropertyCache) + { + complexPropertyCache[complexProperty] = complexShaper.MakeNullable(); + } + } + } + var ownedNavigationMap = new Dictionary(); foreach (var (navigation, shaper) in _ownedNavigationMap) { @@ -178,6 +237,7 @@ public virtual StructuralTypeProjectionExpression MakeNullable() StructuralType, propertyExpressionMap, ownedNavigationMap, + complexPropertyCache, TableMap, nullable: true, discriminatorExpression); @@ -212,6 +272,23 @@ public virtual StructuralTypeProjectionExpression UpdateEntityType(IEntityType d } } + var complexPropertyCache = default(Dictionary); + if (!UseOldBehavior32911) + { + if (_complexPropertyCache != null) + { + complexPropertyCache = new(); + foreach (var (complexProperty, complexShaper) in _complexPropertyCache) + { + if (derivedType.IsAssignableFrom(complexProperty.DeclaringType) + || complexProperty.DeclaringType.IsAssignableFrom(derivedType)) + { + complexPropertyCache[complexProperty] = complexShaper; + } + } + } + } + var ownedNavigationMap = new Dictionary(); foreach (var (navigation, entityShaperExpression) in _ownedNavigationMap) { @@ -263,7 +340,7 @@ public virtual StructuralTypeProjectionExpression UpdateEntityType(IEntityType d } return new StructuralTypeProjectionExpression( - derivedType, propertyExpressionMap, ownedNavigationMap, newTableMap ?? TableMap, IsNullable, discriminatorExpression); + derivedType, propertyExpressionMap, ownedNavigationMap, complexPropertyCache, newTableMap ?? TableMap, IsNullable, discriminatorExpression); } /// diff --git a/src/EFCore.SqlServer/Query/Internal/SqlServerJsonPostprocessor.cs b/src/EFCore.SqlServer/Query/Internal/SqlServerJsonPostprocessor.cs index 4baa71272c9..28c74f76cbc 100644 --- a/src/EFCore.SqlServer/Query/Internal/SqlServerJsonPostprocessor.cs +++ b/src/EFCore.SqlServer/Query/Internal/SqlServerJsonPostprocessor.cs @@ -32,6 +32,9 @@ private readonly private RelationalTypeMapping? _nvarcharMaxTypeMapping; + private static readonly bool UseOldBehavior32976 = + AppContext.TryGetSwitch("Microsoft.EntityFrameworkCore.Issue32976", out var enabled32976) && enabled32976; + /// /// This is an internal API that supports the Entity Framework Core infrastructure and not subject to /// the same compatibility standards as public APIs. It may be changed or removed without notice in @@ -70,7 +73,14 @@ public virtual Expression Process(Expression expression) switch (expression) { case ShapedQueryExpression shapedQueryExpression: - return shapedQueryExpression.UpdateQueryExpression(Visit(shapedQueryExpression.QueryExpression)); + shapedQueryExpression = shapedQueryExpression.UpdateQueryExpression(Visit(shapedQueryExpression.QueryExpression)); + + if (!UseOldBehavior32976) + { + shapedQueryExpression = shapedQueryExpression.UpdateShaperExpression(Visit(shapedQueryExpression.ShaperExpression)); + } + + return shapedQueryExpression; case SelectExpression selectExpression: { diff --git a/src/EFCore.SqlServer/Storage/Internal/SqlServerDatabaseCreator.cs b/src/EFCore.SqlServer/Storage/Internal/SqlServerDatabaseCreator.cs index c85be77e61c..589440790a1 100644 --- a/src/EFCore.SqlServer/Storage/Internal/SqlServerDatabaseCreator.cs +++ b/src/EFCore.SqlServer/Storage/Internal/SqlServerDatabaseCreator.cs @@ -1,6 +1,7 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. +using System.ComponentModel; using System.Transactions; using Microsoft.Data.SqlClient; using Microsoft.EntityFrameworkCore.SqlServer.Internal; @@ -322,7 +323,8 @@ private bool RetryOnExistsFailure(SqlException exception) // Microsoft.Data.SqlClient.SqlException: Unable to open the physical file xxxxxxx. // And (Number 18456) // Microsoft.Data.SqlClient.SqlException: Login failed for user 'xxxxxxx'. - if (exception.Number is 233 or -2 or 4060 or 1832 or 5120 or 18456) + if ((exception.Number is 203 && exception.InnerException is Win32Exception) + || (exception.Number is 233 or -2 or 4060 or 1832 or 5120 or 18456)) { ClearPool(); return true; diff --git a/src/EFCore.SqlServer/Storage/Internal/SqlServerTransientExceptionDetector.cs b/src/EFCore.SqlServer/Storage/Internal/SqlServerTransientExceptionDetector.cs index 766d0c0504d..6b11941e6b3 100644 --- a/src/EFCore.SqlServer/Storage/Internal/SqlServerTransientExceptionDetector.cs +++ b/src/EFCore.SqlServer/Storage/Internal/SqlServerTransientExceptionDetector.cs @@ -1,6 +1,7 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. +using System.ComponentModel; using Microsoft.Data.SqlClient; namespace Microsoft.EntityFrameworkCore.SqlServer.Storage.Internal; @@ -670,6 +671,16 @@ public static bool ShouldRetryOn(Exception? ex) // allowed connections) on the server. (provider: TCP Provider, error: 0 - An existing connection was forcibly closed by // the remote host.) case 233: + return true; + // SQL Error Code: 203 + // A connection was successfully established with the server, but then an error occurred during the pre-login handshake. + // (provider: TCP Provider, error: 0 - 20) ---> System.ComponentModel.Win32Exception (203): Unknown error: 203 + case 203: + if (ex.InnerException is Win32Exception) + { + return true; + } + continue; // SQL Error Code: 121 // The semaphore timeout period has expired case 121: diff --git a/src/EFCore/Metadata/Internal/Property.cs b/src/EFCore/Metadata/Internal/Property.cs index a8fcb4eee96..e88f9f4bef5 100644 --- a/src/EFCore/Metadata/Internal/Property.cs +++ b/src/EFCore/Metadata/Internal/Property.cs @@ -40,6 +40,9 @@ public class Property : PropertyBase, IMutableProperty, IConventionProperty, IPr public static readonly bool UseOldBehavior32422 = AppContext.TryGetSwitch("Microsoft.EntityFrameworkCore.Issue32422", out var enabled32422) && enabled32422; + private static readonly bool UseOldBehavior33176 = + AppContext.TryGetSwitch("Microsoft.EntityFrameworkCore.Issue33176", out var enabled33176) && enabled33176; + /// /// This is an internal API that supports the Entity Framework Core infrastructure and not subject to /// the same compatibility standards as public APIs. It may be changed or removed without notice in @@ -911,9 +914,10 @@ public virtual (ValueConverter? ValueConverter, Type? ValueConverterType, Type? bool throwOnValueConverterConflict = true, bool throwOnProviderClrTypeConflict = true) { - Queue<(Property CurrentProperty, Property CycleBreakingPropert, int CyclePosition, int MaxCycleLength)>? queue = null; - (Property CurrentProperty, Property CycleBreakingPropert, int CyclePosition, int MaxCycleLength)? currentNode = + Queue<(Property CurrentProperty, Property CycleBreakingProperty, int CyclePosition, int MaxCycleLength)>? queue = null; + (Property CurrentProperty, Property CycleBreakingProperty, int CyclePosition, int MaxCycleLength)? currentNode = (this, this, 0, 2); + HashSet? visitedProperties = null; ValueConverter? valueConverter = null; Type? valueConverterType = null; @@ -922,12 +926,17 @@ public virtual (ValueConverter? ValueConverter, Type? ValueConverterType, Type? { var (property, cycleBreakingProperty, cyclePosition, maxCycleLength) = currentNode ?? queue!.Dequeue(); currentNode = null; - if (cyclePosition >= ForeignKey.LongestFkChainAllowedLength) + if (cyclePosition >= ForeignKey.LongestFkChainAllowedLength + || (!UseOldBehavior33176 + && queue is not null + && queue.Count >= ForeignKey.LongestFkChainAllowedLength)) { throw new InvalidOperationException( CoreStrings.RelationshipCycle(DeclaringType.DisplayName(), Name, "ValueConverter")); } + visitedProperties?.Add(property); + foreach (var foreignKey in property.GetContainingForeignKeys()) { for (var propertyIndex = 0; propertyIndex < foreignKey.Properties.Count; propertyIndex++) @@ -956,6 +965,13 @@ public virtual (ValueConverter? ValueConverter, Type? ValueConverterType, Type? { queue = new(); queue.Enqueue(currentNode.Value); + visitedProperties = new() { property }; + } + + if (!UseOldBehavior33176 + && visitedProperties?.Contains(principalProperty) == true) + { + break; } if (cyclePosition == maxCycleLength - 1) diff --git a/test/EFCore.InMemory.FunctionalTests/Query/AdHocAdvancedMappingsQueryInMemoryTest.cs b/test/EFCore.InMemory.FunctionalTests/Query/AdHocAdvancedMappingsQueryInMemoryTest.cs new file mode 100644 index 00000000000..bcd988e7562 --- /dev/null +++ b/test/EFCore.InMemory.FunctionalTests/Query/AdHocAdvancedMappingsQueryInMemoryTest.cs @@ -0,0 +1,10 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +namespace Microsoft.EntityFrameworkCore.Query; + +public class AdHocAdvancedMappingsQueryInMemoryTest : AdHocAdvancedMappingsQueryTestBase +{ + protected override ITestStoreFactory TestStoreFactory + => InMemoryTestStoreFactory.Instance; +} diff --git a/test/EFCore.Relational.Specification.Tests/Migrations/MigrationsTestBase.cs b/test/EFCore.Relational.Specification.Tests/Migrations/MigrationsTestBase.cs index 8af3b5cbc97..eef96d80c95 100644 --- a/test/EFCore.Relational.Specification.Tests/Migrations/MigrationsTestBase.cs +++ b/test/EFCore.Relational.Specification.Tests/Migrations/MigrationsTestBase.cs @@ -1,6 +1,7 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. +using System.ComponentModel.DataAnnotations; using Microsoft.EntityFrameworkCore.Metadata.Internal; using Microsoft.EntityFrameworkCore.Migrations.Internal; using Microsoft.EntityFrameworkCore.Scaffolding.Metadata; @@ -1874,6 +1875,339 @@ await Test( """); } + [ConditionalFact] + public virtual Task Add_required_primitve_collection_to_existing_table() + => Test( + builder => builder.Entity( + "Customer", e => + { + e.Property("Id").ValueGeneratedOnAdd(); + e.HasKey("Id"); + e.Property("Name"); + e.ToTable("Customers"); + }), + builder => builder.Entity( + "Customer", e => + { + e.Property("Id").ValueGeneratedOnAdd(); + e.HasKey("Id"); + e.Property("Name"); + e.Property>("Numbers").IsRequired(); + e.ToTable("Customers"); + }), + model => + { + var customersTable = Assert.Single(model.Tables.Where(t => t.Name == "Customers")); + + Assert.Collection( + customersTable.Columns, + c => Assert.Equal("Id", c.Name), + c => Assert.Equal("Name", c.Name), + c => Assert.Equal("Numbers", c.Name)); + Assert.Same( + customersTable.Columns.Single(c => c.Name == "Id"), + Assert.Single(customersTable.PrimaryKey!.Columns)); + }); + + [ConditionalFact] + public virtual Task Add_required_primitve_collection_with_custom_default_value_to_existing_table() + => Test( + builder => builder.Entity( + "Customer", e => + { + e.Property("Id").ValueGeneratedOnAdd(); + e.HasKey("Id"); + e.Property("Name"); + e.ToTable("Customers"); + }), + builder => builder.Entity( + "Customer", e => + { + e.Property("Id").ValueGeneratedOnAdd(); + e.HasKey("Id"); + e.Property("Name"); + e.Property>("Numbers").IsRequired().HasDefaultValue(new List { 1, 2, 3 }); + e.ToTable("Customers"); + }), + model => + { + var customersTable = Assert.Single(model.Tables.Where(t => t.Name == "Customers")); + + Assert.Collection( + customersTable.Columns, + c => Assert.Equal("Id", c.Name), + c => Assert.Equal("Name", c.Name), + c => Assert.Equal("Numbers", c.Name)); + Assert.Same( + customersTable.Columns.Single(c => c.Name == "Id"), + Assert.Single(customersTable.PrimaryKey!.Columns)); + }); + + [ConditionalFact] + public abstract Task Add_required_primitve_collection_with_custom_default_value_sql_to_existing_table(); + + protected virtual Task Add_required_primitve_collection_with_custom_default_value_sql_to_existing_table_core(string defaultValueSql) + => Test( + builder => builder.Entity( + "Customer", e => + { + e.Property("Id").ValueGeneratedOnAdd(); + e.HasKey("Id"); + e.Property("Name"); + e.ToTable("Customers"); + }), + builder => builder.Entity( + "Customer", e => + { + e.Property("Id").ValueGeneratedOnAdd(); + e.HasKey("Id"); + e.Property("Name"); + e.Property>("Numbers").IsRequired().HasDefaultValueSql(defaultValueSql); + e.ToTable("Customers"); + }), + model => + { + var customersTable = Assert.Single(model.Tables.Where(t => t.Name == "Customers")); + + Assert.Collection( + customersTable.Columns, + c => Assert.Equal("Id", c.Name), + c => Assert.Equal("Name", c.Name), + c => Assert.Equal("Numbers", c.Name)); + Assert.Same( + customersTable.Columns.Single(c => c.Name == "Id"), + Assert.Single(customersTable.PrimaryKey!.Columns)); + }); + + [ConditionalFact(Skip = "issue #33038")] + public virtual Task Add_required_primitve_collection_with_custom_converter_to_existing_table() + => Test( + builder => builder.Entity( + "Customer", e => + { + e.Property("Id").ValueGeneratedOnAdd(); + e.HasKey("Id"); + e.Property("Name"); + e.ToTable("Customers"); + }), + builder => builder.Entity( + "Customer", e => + { + e.Property("Id").ValueGeneratedOnAdd(); + e.HasKey("Id"); + e.Property("Name"); + e.Property>("Numbers").HasConversion(new ValueConverter, string>( + convertToProviderExpression: x => x != null && x.Count > 0 ? "some numbers" : "nothing", + convertFromProviderExpression: x => x == "nothing" ? new List { } : new List { 7, 8, 9 })) + .IsRequired(); + e.ToTable("Customers"); + }), + model => + { + var customersTable = Assert.Single(model.Tables.Where(t => t.Name == "Customers")); + + Assert.Collection( + customersTable.Columns, + c => Assert.Equal("Id", c.Name), + c => Assert.Equal("Name", c.Name), + c => Assert.Equal("Numbers", c.Name)); + Assert.Same( + customersTable.Columns.Single(c => c.Name == "Id"), + Assert.Single(customersTable.PrimaryKey!.Columns)); + }); + + [ConditionalFact] + public virtual Task Add_required_primitve_collection_with_custom_converter_and_custom_default_value_to_existing_table() + => Test( + builder => builder.Entity( + "Customer", e => + { + e.Property("Id").ValueGeneratedOnAdd(); + e.HasKey("Id"); + e.Property("Name"); + e.ToTable("Customers"); + }), + builder => builder.Entity( + "Customer", e => + { + e.Property("Id").ValueGeneratedOnAdd(); + e.HasKey("Id"); + e.Property("Name"); + e.Property>("Numbers").HasConversion(new ValueConverter, string>( + convertToProviderExpression: x => x != null && x.Count > 0 ? "some numbers" : "nothing", + convertFromProviderExpression: x => x == "nothing" ? new List { } : new List { 7, 8, 9 })) + .HasDefaultValue(new List { 42 }) + .IsRequired(); + e.ToTable("Customers"); + }), + model => + { + var customersTable = Assert.Single(model.Tables.Where(t => t.Name == "Customers")); + + Assert.Collection( + customersTable.Columns, + c => Assert.Equal("Id", c.Name), + c => Assert.Equal("Name", c.Name), + c => Assert.Equal("Numbers", c.Name)); + Assert.Same( + customersTable.Columns.Single(c => c.Name == "Id"), + Assert.Single(customersTable.PrimaryKey!.Columns)); + }); + + [ConditionalFact] + public virtual Task Add_optional_primitive_collection_to_existing_table() + => Test( + builder => builder.Entity( + "Customer", e => + { + e.Property("Id").ValueGeneratedOnAdd(); + e.HasKey("Id"); + e.Property("Name"); + e.ToTable("Customers"); + }), + builder => builder.Entity( + "Customer", e => + { + e.Property("Id").ValueGeneratedOnAdd(); + e.HasKey("Id"); + e.Property("Name"); + e.Property>("Numbers"); + e.ToTable("Customers"); + }), + model => + { + var customersTable = Assert.Single(model.Tables.Where(t => t.Name == "Customers")); + + Assert.Collection( + customersTable.Columns, + c => Assert.Equal("Id", c.Name), + c => Assert.Equal("Name", c.Name), + c => Assert.Equal("Numbers", c.Name)); + Assert.Same( + customersTable.Columns.Single(c => c.Name == "Id"), + Assert.Single(customersTable.PrimaryKey!.Columns)); + }); + + [ConditionalFact] + public virtual Task Create_table_with_required_primitive_collection() + => Test( + builder => { }, + builder => builder.Entity( + "Customer", e => + { + e.Property("Id").ValueGeneratedOnAdd(); + e.HasKey("Id"); + e.Property("Name"); + e.Property>("Numbers").IsRequired(); + e.ToTable("Customers"); + }), + model => + { + var customersTable = Assert.Single(model.Tables.Where(t => t.Name == "Customers")); + + Assert.Collection( + customersTable.Columns, + c => Assert.Equal("Id", c.Name), + c => Assert.Equal("Name", c.Name), + c => Assert.Equal("Numbers", c.Name)); + Assert.Same( + customersTable.Columns.Single(c => c.Name == "Id"), + Assert.Single(customersTable.PrimaryKey!.Columns)); + }); + + [ConditionalFact] + public virtual Task Create_table_with_optional_primitive_collection() + => Test( + builder => { }, + builder => builder.Entity( + "Customer", e => + { + e.Property("Id").ValueGeneratedOnAdd(); + e.HasKey("Id"); + e.Property("Name"); + e.Property>("Numbers"); + e.ToTable("Customers"); + }), + model => + { + var customersTable = Assert.Single(model.Tables.Where(t => t.Name == "Customers")); + + Assert.Collection( + customersTable.Columns, + c => Assert.Equal("Id", c.Name), + c => Assert.Equal("Name", c.Name), + c => Assert.Equal("Numbers", c.Name)); + Assert.Same( + customersTable.Columns.Single(c => c.Name == "Id"), + Assert.Single(customersTable.PrimaryKey!.Columns)); + }); + + [ConditionalFact] + public virtual Task Create_table_with_complex_type_with_required_properties_on_derived_entity_in_TPH() + => Test( + builder => { }, + builder => + { + builder.Entity( + "Contact", e => + { + e.Property("Id").ValueGeneratedOnAdd(); + e.HasKey("Id"); + e.Property("Name"); + e.ToTable("Contacts"); + }); + builder.Entity( + "Supplier", e => + { + e.HasBaseType("Contact"); + e.Property("Number"); + e.ComplexProperty("MyComplex", ct => + { + ct.ComplexProperty("MyNestedComplex").IsRequired(); + }); + }); + }, + model => + { + var contactsTable = Assert.Single(model.Tables.Where(t => t.Name == "Contacts")); + Assert.Collection( + contactsTable.Columns, + c => Assert.Equal("Id", c.Name), + c => Assert.Equal("Discriminator", c.Name), + c => Assert.Equal("Name", c.Name), + c => Assert.Equal("Number", c.Name), + c => + { + Assert.Equal("MyComplex_Prop", c.Name); + Assert.Equal(true, c.IsNullable); + }, + c => + { + Assert.Equal("MyComplex_MyNestedComplex_Bar", c.Name); + Assert.Equal(true, c.IsNullable); + }, + c => + { + Assert.Equal("MyComplex_MyNestedComplex_Foo", c.Name); + Assert.Equal(true, c.IsNullable); + }); + }); + + protected class MyComplex + { + [Required] + public string Prop { get; set; } + + [Required] + public MyNestedComplex Nested { get; set; } + } + + public class MyNestedComplex + { + public int Foo { get; set; } + public DateTime Bar { get; set; } + } + protected class Person { public int Id { get; set; } diff --git a/test/EFCore.Relational.Specification.Tests/Query/AdHocAdvancedMappingsQueryRelationalTestBase.cs b/test/EFCore.Relational.Specification.Tests/Query/AdHocAdvancedMappingsQueryRelationalTestBase.cs new file mode 100644 index 00000000000..d001b5488f2 --- /dev/null +++ b/test/EFCore.Relational.Specification.Tests/Query/AdHocAdvancedMappingsQueryRelationalTestBase.cs @@ -0,0 +1,230 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +namespace Microsoft.EntityFrameworkCore.Query; + +public abstract class AdHocAdvancedMappingsQueryRelationalTestBase : AdHocAdvancedMappingsQueryTestBase +{ + protected TestSqlLoggerFactory TestSqlLoggerFactory + => (TestSqlLoggerFactory)ListLoggerFactory; + + protected void ClearLog() + => TestSqlLoggerFactory.Clear(); + + protected void AssertSql(params string[] expected) + => TestSqlLoggerFactory.AssertBaseline(expected); + + #region 32911 + + [ConditionalFact] + public virtual async Task Two_similar_complex_properties_projected_with_split_query1() + { + var contextFactory = await InitializeAsync(seed: c => c.Seed()); + + using var context = contextFactory.CreateContext(); + var query = context.Offers + .Include(e => e.Variations) + .ThenInclude(v => v.Nested) + .AsSplitQuery() + .ToList(); + + var resultElement = query.Single(); + foreach (var variation in resultElement.Variations) + { + Assert.NotEqual(variation.Payment.Brutto, variation.Nested.Payment.Brutto); + Assert.NotEqual(variation.Payment.Netto, variation.Nested.Payment.Netto); + } + } + + [ConditionalFact] + public virtual async Task Two_similar_complex_properties_projected_with_split_query2() + { + var contextFactory = await InitializeAsync(seed: c => c.Seed()); + + using var context = contextFactory.CreateContext(); + var query = context.Offers + .Include(e => e.Variations) + .ThenInclude(v => v.Nested) + .AsSplitQuery() + .Single(x => x.Id == 1); + + foreach (var variation in query.Variations) + { + Assert.NotEqual(variation.Payment.Brutto, variation.Nested.Payment.Brutto); + Assert.NotEqual(variation.Payment.Netto, variation.Nested.Payment.Netto); + } + } + + [ConditionalFact] + public virtual async Task Projecting_one_of_two_similar_complex_types_picks_the_correct_one() + { + var contextFactory = await InitializeAsync(seed: c => c.Seed()); + + using var context = contextFactory.CreateContext(); + + var query = context.Cs + .Where(x => x.B.AId.Value == 1) + .OrderBy(x => x.Id) + .Take(10) + .Select(x => new + { + x.B.A.Id, + x.B.Info.Created, + }).ToList(); + + Assert.Equal(new DateTime(2000, 1, 1), query[0].Created); + } + + protected class Context32911(DbContextOptions options) : DbContext(options) + { + public DbSet Offers { get; set; } + + protected override void OnModelCreating(ModelBuilder modelBuilder) + { + modelBuilder.Entity().Property(x => x.Id).ValueGeneratedNever(); + modelBuilder.Entity().Property(x => x.Id).ValueGeneratedNever(); + modelBuilder.Entity().ComplexProperty(x => x.Payment, cpb => + { + cpb.IsRequired(); + cpb.Property(p => p.Netto).HasColumnName("payment_netto"); + cpb.Property(p => p.Brutto).HasColumnName("payment_brutto"); + }); + modelBuilder.Entity().Property(x => x.Id).ValueGeneratedNever(); + modelBuilder.Entity().ComplexProperty(x => x.Payment, cpb => + { + cpb.IsRequired(); + cpb.Property(p => p.Netto).HasColumnName("payment_netto"); + cpb.Property(p => p.Brutto).HasColumnName("payment_brutto"); + }); + } + + public void Seed() + { + var v1 = new Variation + { + Id = 1, + Payment = new Payment(1, 10), + Nested = new NestedEntity + { + Id = 1, + Payment = new Payment(10, 100) + } + }; + + var v2 = new Variation + { + Id = 2, + Payment = new Payment(2, 20), + Nested = new NestedEntity + { + Id = 2, + Payment = new Payment(20, 200) + } + }; + + var v3 = new Variation + { + Id = 3, + Payment = new Payment(3, 30), + Nested = new NestedEntity + { + Id = 3, + Payment = new Payment(30, 300) + } + }; + + Offers.Add(new Offer { Id = 1, Variations = new List { v1, v2, v3 } }); + + SaveChanges(); + } + + public abstract class EntityBase + { + public int Id { get; set; } + } + + public class Offer : EntityBase + { + public ICollection Variations { get; set; } + } + + public class Variation : EntityBase + { + public Payment Payment { get; set; } = new Payment(0, 0); + + public NestedEntity Nested { get; set; } + } + + public class NestedEntity : EntityBase + { + public Payment Payment { get; set; } = new Payment(0, 0); + } + + public record Payment(decimal Netto, decimal Brutto); + } + + protected class Context32911_2(DbContextOptions options) : DbContext(options) + { + public DbSet As { get; set; } + public DbSet Bs { get; set; } + public DbSet Cs { get; set; } + + protected override void OnModelCreating(ModelBuilder modelBuilder) + { + modelBuilder.Entity().Property(x => x.Id).ValueGeneratedNever(); + modelBuilder.Entity().Property(x => x.Id).ValueGeneratedNever(); + modelBuilder.Entity().Property(x => x.Id).ValueGeneratedNever(); + + modelBuilder.Entity(x => x.ComplexProperty(b => b.Info).IsRequired()); + modelBuilder.Entity(x => x.ComplexProperty(c => c.Info).IsRequired()); + } + + public void Seed() + { + var c = new C + { + Id = 100, + Info = new Metadata { Created = new DateTime(2020, 10, 10) }, + B = new B + { + Id = 10, + Info = new Metadata { Created = new DateTime(2000, 1, 1) }, + A = new A { Id = 1 } + } + }; + + Cs.Add(c); + SaveChanges(); + } + + public class Metadata + { + public DateTime Created { get; set; } + } + + public class A + { + public int Id { get; set; } + } + + public class B + { + public int Id { get; set; } + public Metadata Info { get; set; } + public int? AId { get; set; } + + public A A { get; set; } + } + + public class C + { + public int Id { get; set; } + public Metadata Info { get; set; } + public int BId { get; set; } + + public B B { get; set; } + } + } + + #endregion +} diff --git a/test/EFCore.Specification.Tests/Query/AdHocAdvancedMappingsQueryTestBase.cs b/test/EFCore.Specification.Tests/Query/AdHocAdvancedMappingsQueryTestBase.cs new file mode 100644 index 00000000000..7782e297b48 --- /dev/null +++ b/test/EFCore.Specification.Tests/Query/AdHocAdvancedMappingsQueryTestBase.cs @@ -0,0 +1,10 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +namespace Microsoft.EntityFrameworkCore.Query; + +public abstract class AdHocAdvancedMappingsQueryTestBase : NonSharedModelTestBase +{ + protected override string StoreName + => "AdHocAdvancedMappingsQueryTests"; +} diff --git a/test/EFCore.Specification.Tests/Query/ComplexTypeQueryTestBase.cs b/test/EFCore.Specification.Tests/Query/ComplexTypeQueryTestBase.cs index ceb4a5d65af..1f45144803e 100644 --- a/test/EFCore.Specification.Tests/Query/ComplexTypeQueryTestBase.cs +++ b/test/EFCore.Specification.Tests/Query/ComplexTypeQueryTestBase.cs @@ -506,6 +506,292 @@ public virtual Task Union_two_different_struct_complex_type(bool async) async, ss => ss.Set().Select(c => c.ShippingAddress).Union(ss.Set().Select(c => c.BillingAddress))); + [ConditionalTheory] + [MemberData(nameof(IsAsyncData))] + public virtual Task Project_same_nested_complex_type_twice_with_pushdown(bool async) + => AssertQuery( + async, + ss => (from c1 in ss.Set() + from c2 in ss.Set() + select new { BA1 = c1.BillingAddress, BA2 = c2.BillingAddress }) + .Distinct() + .Select(x => new { x.BA1, x.BA2 }), + elementSorter: e => (e.BA1.ZipCode, e.BA2.ZipCode), + elementAsserter: (e, a) => + { + AssertEqual(e.BA1, a.BA1); + AssertEqual(e.BA2, a.BA2); + }); + + [ConditionalTheory] + [MemberData(nameof(IsAsyncData))] + public virtual Task Project_same_entity_with_nested_complex_type_twice_with_pushdown(bool async) + => AssertQuery( + async, + ss => (from c1 in ss.Set() + from c2 in ss.Set() + select new { c1, c2 }) + .Distinct() + .Select(x => new { x.c1, x.c2 }), + elementSorter: e => (e.c1.Id, e.c2.Id), + elementAsserter: (e, a) => + { + AssertEqual(e.c1, a.c1); + AssertEqual(e.c2, a.c2); + }); + + [ConditionalTheory] + [MemberData(nameof(IsAsyncData))] + public virtual Task Project_same_nested_complex_type_twice_with_double_pushdown(bool async) + => AssertQuery( + async, + ss => (from c1 in ss.Set() + from c2 in ss.Set() + orderby c1.Id, c2.Id + select new { BA1 = c1.BillingAddress, BA2 = c2.BillingAddress }) + .Take(50) + .Distinct() + .Select(x => new { x.BA1, x.BA2 }), + elementSorter: e => (e.BA1.ZipCode, e.BA2.ZipCode), + elementAsserter: (e, a) => + { + AssertEqual(e.BA1, a.BA1); + AssertEqual(e.BA2, a.BA2); + }); + + [ConditionalTheory] + [MemberData(nameof(IsAsyncData))] + public virtual Task Project_same_entity_with_nested_complex_type_twice_with_double_pushdown(bool async) + => AssertQuery( + async, + ss => (from c1 in ss.Set() + from c2 in ss.Set() + orderby c1.Id, c2.Id + select new { c1, c2 }) + .Take(50) + .Distinct() + .Select(x => new { x.c1, x.c2 }), + elementSorter: e => (e.c1.Id, e.c2.Id), + elementAsserter: (e, a) => + { + AssertEqual(e.c1, a.c1); + AssertEqual(e.c2, a.c2); + }); + + [ConditionalTheory] + [MemberData(nameof(IsAsyncData))] + public virtual Task Project_same_struct_nested_complex_type_twice_with_pushdown(bool async) + => AssertQuery( + async, + ss => (from c1 in ss.Set() + from c2 in ss.Set() + select new { BA1 = c1.BillingAddress, BA2 = c2.BillingAddress }) + .Distinct() + .Select(x => new { x.BA1, x.BA2 }), + elementSorter: e => (e.BA1.ZipCode, e.BA2.ZipCode), + elementAsserter: (e, a) => + { + AssertEqual(e.BA1, a.BA1); + AssertEqual(e.BA2, a.BA2); + }); + + [ConditionalTheory] + [MemberData(nameof(IsAsyncData))] + public virtual Task Project_same_entity_with_struct_nested_complex_type_twice_with_pushdown(bool async) + => AssertQuery( + async, + ss => (from c1 in ss.Set() + from c2 in ss.Set() + select new { c1, c2 }) + .Distinct() + .Select(x => new { x.c1, x.c2 }), + elementSorter: e => (e.c1.Id, e.c2.Id), + elementAsserter: (e, a) => + { + AssertEqual(e.c1, a.c1); + AssertEqual(e.c2, a.c2); + }); + + [ConditionalTheory] + [MemberData(nameof(IsAsyncData))] + public virtual Task Project_same_struct_nested_complex_type_twice_with_double_pushdown(bool async) + => AssertQuery( + async, + ss => (from c1 in ss.Set() + from c2 in ss.Set() + orderby c1.Id, c2.Id + select new { BA1 = c1.BillingAddress, BA2 = c2.BillingAddress }) + .Take(50) + .Distinct() + .Select(x => new { x.BA1, x.BA2 }), + elementSorter: e => (e.BA1.ZipCode, e.BA2.ZipCode), + elementAsserter: (e, a) => + { + AssertEqual(e.BA1, a.BA1); + AssertEqual(e.BA2, a.BA2); + }); + + [ConditionalTheory] + [MemberData(nameof(IsAsyncData))] + public virtual Task Project_same_entity_with_struct_nested_complex_type_twice_with_double_pushdown(bool async) + => AssertQuery( + async, + ss => (from c1 in ss.Set() + from c2 in ss.Set() + orderby c1.Id, c2.Id + select new { c1, c2 }) + .Take(50) + .Distinct() + .Select(x => new { x.c1, x.c2 }), + elementSorter: e => (e.c1.Id, e.c2.Id), + elementAsserter: (e, a) => + { + AssertEqual(e.c1, a.c1); + AssertEqual(e.c2, a.c2); + }); + + [ConditionalTheory] + [MemberData(nameof(IsAsyncData))] + public virtual Task Union_of_same_entity_with_nested_complex_type_projected_twice_with_pushdown(bool async) + => AssertQuery( + async, + ss => (from c1 in ss.Set() + from c2 in ss.Set() + orderby c1.Id, c2.Id + select new { c1, c2 }) + .Union(from c1 in ss.Set() + from c2 in ss.Set() + orderby c1.Id, c2.Id + select new { c1, c2 }) + .OrderBy(x => x.c1.Id).ThenBy(x => x.c2.Id) + .Take(50) + .Select(x => new { x.c1, x.c2 }), + assertOrder: true, + elementAsserter: (e, a) => + { + AssertEqual(e.c1, a.c1); + AssertEqual(e.c2, a.c2); + }); + + [ConditionalTheory] + [MemberData(nameof(IsAsyncData))] + public virtual Task Union_of_same_entity_with_nested_complex_type_projected_twice_with_double_pushdown(bool async) + => AssertQuery( + async, + ss => (from c1 in ss.Set() + from c2 in ss.Set() + orderby c1.Id, c2.Id + select new { c1, c2 }) + .Union(from c1 in ss.Set() + from c2 in ss.Set() + orderby c1.Id, c2.Id + select new { c1, c2 }) + .OrderBy(x => x.c1.Id).ThenBy(x => x.c2.Id) + .Take(50) + .Select(x => new { x.c1, x.c2 }) + .Distinct() + .OrderBy(x => x.c1.Id).ThenBy(x => x.c2.Id) + .Take(50) + .Select(x => new { x.c1, x.c2 }), + assertOrder: true, + elementAsserter: (e, a) => + { + AssertEqual(e.c1, a.c1); + AssertEqual(e.c2, a.c2); + }); + + [ConditionalTheory] + [MemberData(nameof(IsAsyncData))] + public virtual Task Union_of_same_nested_complex_type_projected_twice_with_pushdown(bool async) + => AssertQuery( + async, + ss => (from c1 in ss.Set() + from c2 in ss.Set() + orderby c1.Id, c2.Id + select new { BA1 = c1.BillingAddress, BA2 = c2.BillingAddress }) + .Union(from c1 in ss.Set() + from c2 in ss.Set() + orderby c1.Id, c2.Id + select new { BA1 = c1.BillingAddress, BA2 = c2.BillingAddress }) + .OrderBy(x => x.BA1.ZipCode).ThenBy(x => x.BA2.ZipCode) + .Take(50) + .Select(x => new { x.BA1, x.BA2 }), + assertOrder: true, + elementAsserter: (e, a) => + { + AssertEqual(e.BA1, a.BA1); + AssertEqual(e.BA2, a.BA2); + }); + + [ConditionalTheory] + [MemberData(nameof(IsAsyncData))] + public virtual Task Union_of_same_nested_complex_type_projected_twice_with_double_pushdown(bool async) + => AssertQuery( + async, + ss => (from c1 in ss.Set() + from c2 in ss.Set() + orderby c1.Id, c2.Id + select new { BA1 = c1.BillingAddress, BA2 = c2.BillingAddress }) + .Union(from c1 in ss.Set() + from c2 in ss.Set() + orderby c1.Id, c2.Id + select new { BA1 = c1.BillingAddress, BA2 = c2.BillingAddress }) + .OrderBy(x => x.BA1.ZipCode).ThenBy(x => x.BA2.ZipCode) + .Take(50) + .Select(x => new { x.BA1, x.BA2 }) + .Distinct() + .OrderBy(x => x.BA1.ZipCode).ThenBy(x => x.BA2.ZipCode) + .Take(50) + .Select(x => new { x.BA1, x.BA2 }), + assertOrder: true, + elementAsserter: (e, a) => + { + AssertEqual(e.BA1, a.BA1); + AssertEqual(e.BA2, a.BA2); + }); + + [ConditionalTheory] + [MemberData(nameof(IsAsyncData))] + public virtual Task Same_entity_with_complex_type_projected_twice_with_pushdown_as_part_of_another_projection(bool async) + => AssertQuery( + async, + ss => ss.Set().Select(x => new + { + x.Id, + Complex = (from c1 in ss.Set() + from c2 in ss.Set() + orderby c1.Id, c2.Id descending + select new { One = c1, Two = c2 }).FirstOrDefault() + }), + elementSorter: e => e.Id, + elementAsserter: (e, a) => + { + AssertEqual(e.Id, a.Id); + AssertEqual(e.Complex?.One, a.Complex?.One); + AssertEqual(e.Complex?.Two, a.Complex?.Two); + }); + + [ConditionalTheory(Skip = "issue #31376")] + [MemberData(nameof(IsAsyncData))] + public virtual Task Same_complex_type_projected_twice_with_pushdown_as_part_of_another_projection(bool async) + => AssertQuery( + async, + ss => ss.Set().Select(x => new + { + x.Id, + Complex = (from c1 in ss.Set() + from c2 in ss.Set() + orderby c1.Id, c2.Id descending + select new { One = c1.BillingAddress, Two = c2.BillingAddress }).FirstOrDefault() + }), + elementSorter: e => e.Id, + elementAsserter: (e, a) => + { + AssertEqual(e.Id, a.Id); + AssertEqual(e.Complex?.One, a.Complex?.One); + AssertEqual(e.Complex?.Two, a.Complex?.Two); + }); + protected DbContext CreateContext() => Fixture.CreateContext(); } diff --git a/test/EFCore.SqlServer.FunctionalTests/Migrations/MigrationsSqlServerTest.cs b/test/EFCore.SqlServer.FunctionalTests/Migrations/MigrationsSqlServerTest.cs index a2c64defd70..f8c618f88ca 100644 --- a/test/EFCore.SqlServer.FunctionalTests/Migrations/MigrationsSqlServerTest.cs +++ b/test/EFCore.SqlServer.FunctionalTests/Migrations/MigrationsSqlServerTest.cs @@ -9230,6 +9230,123 @@ await Test( AssertSql(); } + [ConditionalFact] + public override async Task Add_required_primitve_collection_to_existing_table() + { + await base.Add_required_primitve_collection_to_existing_table(); + + AssertSql( +""" +ALTER TABLE [Customers] ADD [Numbers] nvarchar(max) NOT NULL DEFAULT N'[]'; +"""); + } + + [ConditionalFact] + public override async Task Add_required_primitve_collection_with_custom_default_value_to_existing_table() + { + await base.Add_required_primitve_collection_with_custom_default_value_to_existing_table(); + + AssertSql( +""" +ALTER TABLE [Customers] ADD [Numbers] nvarchar(max) NOT NULL DEFAULT N'[1,2,3]'; +"""); + } + + [ConditionalFact] + public override async Task Add_required_primitve_collection_with_custom_default_value_sql_to_existing_table() + { + await base.Add_required_primitve_collection_with_custom_default_value_sql_to_existing_table_core("N'[3, 2, 1]'"); + + AssertSql( +""" +ALTER TABLE [Customers] ADD [Numbers] nvarchar(max) NOT NULL DEFAULT (N'[3, 2, 1]'); +"""); + } + + [ConditionalFact(Skip = "issue #33038")] + public override async Task Add_required_primitve_collection_with_custom_converter_to_existing_table() + { + await base.Add_required_primitve_collection_with_custom_converter_to_existing_table(); + + AssertSql( +""" +ALTER TABLE [Customers] ADD [Numbers] nvarchar(max) NOT NULL DEFAULT N'nothing'; +"""); + } + + [ConditionalFact] + public override async Task Add_required_primitve_collection_with_custom_converter_and_custom_default_value_to_existing_table() + { + await base.Add_required_primitve_collection_with_custom_converter_and_custom_default_value_to_existing_table(); + + AssertSql( +""" +ALTER TABLE [Customers] ADD [Numbers] nvarchar(max) NOT NULL DEFAULT N'some numbers'; +"""); + } + + [ConditionalFact] + public override async Task Add_optional_primitive_collection_to_existing_table() + { + await base.Add_optional_primitive_collection_to_existing_table(); + + AssertSql( +""" +ALTER TABLE [Customers] ADD [Numbers] nvarchar(max) NULL; +"""); + } + + [ConditionalFact] + public override async Task Create_table_with_required_primitive_collection() + { + await base.Create_table_with_required_primitive_collection(); + + AssertSql( +""" +CREATE TABLE [Customers] ( + [Id] int NOT NULL IDENTITY, + [Name] nvarchar(max) NULL, + [Numbers] nvarchar(max) NOT NULL, + CONSTRAINT [PK_Customers] PRIMARY KEY ([Id]) +); +"""); + } + + [ConditionalFact] + public override async Task Create_table_with_optional_primitive_collection() + { + await base.Create_table_with_optional_primitive_collection(); + + AssertSql( +""" +CREATE TABLE [Customers] ( + [Id] int NOT NULL IDENTITY, + [Name] nvarchar(max) NULL, + [Numbers] nvarchar(max) NULL, + CONSTRAINT [PK_Customers] PRIMARY KEY ([Id]) +); +"""); + } + + public override async Task Create_table_with_complex_type_with_required_properties_on_derived_entity_in_TPH() + { + await base.Create_table_with_complex_type_with_required_properties_on_derived_entity_in_TPH(); + + AssertSql( +""" +CREATE TABLE [Contacts] ( + [Id] int NOT NULL IDENTITY, + [Discriminator] nvarchar(8) NOT NULL, + [Name] nvarchar(max) NULL, + [Number] int NULL, + [MyComplex_Prop] nvarchar(max) NULL, + [MyComplex_MyNestedComplex_Bar] datetime2 NULL, + [MyComplex_MyNestedComplex_Foo] int NULL, + CONSTRAINT [PK_Contacts] PRIMARY KEY ([Id]) +); +"""); + } + protected override string NonDefaultCollation => _nonDefaultCollation ??= GetDatabaseCollation() == "German_PhoneBook_CI_AS" ? "French_CI_AS" diff --git a/test/EFCore.SqlServer.FunctionalTests/Query/AdHocAdvancedMappingsQuerySqlServerTest.cs b/test/EFCore.SqlServer.FunctionalTests/Query/AdHocAdvancedMappingsQuerySqlServerTest.cs new file mode 100644 index 00000000000..122d76763bf --- /dev/null +++ b/test/EFCore.SqlServer.FunctionalTests/Query/AdHocAdvancedMappingsQuerySqlServerTest.cs @@ -0,0 +1,82 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +namespace Microsoft.EntityFrameworkCore.Query; + +public class AdHocAdvancedMappingsQuerySqlServerTest : AdHocAdvancedMappingsQueryRelationalTestBase +{ + protected override ITestStoreFactory TestStoreFactory + => SqlServerTestStoreFactory.Instance; + + public override async Task Two_similar_complex_properties_projected_with_split_query1() + { + await base.Two_similar_complex_properties_projected_with_split_query1(); + + AssertSql( +""" +SELECT [o].[Id] +FROM [Offers] AS [o] +ORDER BY [o].[Id] +""", + // + """ +SELECT [t].[Id], [t].[NestedId], [t].[OfferId], [t].[payment_brutto], [t].[payment_netto], [t].[Id0], [t].[payment_brutto0], [t].[payment_netto0], [o].[Id] +FROM [Offers] AS [o] +INNER JOIN ( + SELECT [v].[Id], [v].[NestedId], [v].[OfferId], [v].[payment_brutto], [v].[payment_netto], [n].[Id] AS [Id0], [n].[payment_brutto] AS [payment_brutto0], [n].[payment_netto] AS [payment_netto0] + FROM [Variation] AS [v] + LEFT JOIN [NestedEntity] AS [n] ON [v].[NestedId] = [n].[Id] +) AS [t] ON [o].[Id] = [t].[OfferId] +ORDER BY [o].[Id] +"""); + } + + public override async Task Two_similar_complex_properties_projected_with_split_query2() + { + await base.Two_similar_complex_properties_projected_with_split_query2(); + + AssertSql( +""" +SELECT TOP(2) [o].[Id] +FROM [Offers] AS [o] +WHERE [o].[Id] = 1 +ORDER BY [o].[Id] +""", + // + """ +SELECT [t0].[Id], [t0].[NestedId], [t0].[OfferId], [t0].[payment_brutto], [t0].[payment_netto], [t0].[Id0], [t0].[payment_brutto0], [t0].[payment_netto0], [t].[Id] +FROM ( + SELECT TOP(1) [o].[Id] + FROM [Offers] AS [o] + WHERE [o].[Id] = 1 +) AS [t] +INNER JOIN ( + SELECT [v].[Id], [v].[NestedId], [v].[OfferId], [v].[payment_brutto], [v].[payment_netto], [n].[Id] AS [Id0], [n].[payment_brutto] AS [payment_brutto0], [n].[payment_netto] AS [payment_netto0] + FROM [Variation] AS [v] + LEFT JOIN [NestedEntity] AS [n] ON [v].[NestedId] = [n].[Id] +) AS [t0] ON [t].[Id] = [t0].[OfferId] +ORDER BY [t].[Id] +"""); + } + + public override async Task Projecting_one_of_two_similar_complex_types_picks_the_correct_one() + { + await base.Projecting_one_of_two_similar_complex_types_picks_the_correct_one(); + + AssertSql( +""" +@__p_0='10' + +SELECT [a].[Id], [t].[Info_Created0] AS [Created] +FROM ( + SELECT TOP(@__p_0) [c].[Id], [b].[AId], [b].[Info_Created] AS [Info_Created0] + FROM [Cs] AS [c] + INNER JOIN [Bs] AS [b] ON [c].[BId] = [b].[Id] + WHERE [b].[AId] = 1 + ORDER BY [c].[Id] +) AS [t] +LEFT JOIN [As] AS [a] ON [t].[AId] = [a].[Id] +ORDER BY [t].[Id] +"""); + } +} diff --git a/test/EFCore.SqlServer.FunctionalTests/Query/ComplexTypeQuerySqlServerTest.cs b/test/EFCore.SqlServer.FunctionalTests/Query/ComplexTypeQuerySqlServerTest.cs index 9c72bf3aebe..cbaf021a01c 100644 --- a/test/EFCore.SqlServer.FunctionalTests/Query/ComplexTypeQuerySqlServerTest.cs +++ b/test/EFCore.SqlServer.FunctionalTests/Query/ComplexTypeQuerySqlServerTest.cs @@ -858,6 +858,276 @@ FROM [Customer] AS [c] """).Select(c => c.ShippingAddress).Distinct(), ss => ss.Set().Select(c => c.ShippingAddress).Distinct()); + public override async Task Project_same_entity_with_nested_complex_type_twice_with_pushdown(bool async) + { + await base.Project_same_entity_with_nested_complex_type_twice_with_pushdown(async); + + AssertSql( +""" +SELECT [t].[Id], [t].[Name], [t].[BillingAddress_AddressLine1], [t].[BillingAddress_AddressLine2], [t].[BillingAddress_ZipCode], [t].[BillingAddress_Country_Code], [t].[BillingAddress_Country_FullName], [t].[ShippingAddress_AddressLine1], [t].[ShippingAddress_AddressLine2], [t].[ShippingAddress_ZipCode], [t].[ShippingAddress_Country_Code], [t].[ShippingAddress_Country_FullName], [t].[Id0], [t].[Name0], [t].[BillingAddress_AddressLine10], [t].[BillingAddress_AddressLine20], [t].[BillingAddress_ZipCode0], [t].[BillingAddress_Country_Code0], [t].[BillingAddress_Country_FullName0], [t].[ShippingAddress_AddressLine10], [t].[ShippingAddress_AddressLine20], [t].[ShippingAddress_ZipCode0], [t].[ShippingAddress_Country_Code0], [t].[ShippingAddress_Country_FullName0] +FROM ( + SELECT DISTINCT [c].[Id], [c].[Name], [c].[BillingAddress_AddressLine1], [c].[BillingAddress_AddressLine2], [c].[BillingAddress_ZipCode], [c].[BillingAddress_Country_Code], [c].[BillingAddress_Country_FullName], [c].[ShippingAddress_AddressLine1], [c].[ShippingAddress_AddressLine2], [c].[ShippingAddress_ZipCode], [c].[ShippingAddress_Country_Code], [c].[ShippingAddress_Country_FullName], [c0].[Id] AS [Id0], [c0].[Name] AS [Name0], [c0].[BillingAddress_AddressLine1] AS [BillingAddress_AddressLine10], [c0].[BillingAddress_AddressLine2] AS [BillingAddress_AddressLine20], [c0].[BillingAddress_ZipCode] AS [BillingAddress_ZipCode0], [c0].[BillingAddress_Country_Code] AS [BillingAddress_Country_Code0], [c0].[BillingAddress_Country_FullName] AS [BillingAddress_Country_FullName0], [c0].[ShippingAddress_AddressLine1] AS [ShippingAddress_AddressLine10], [c0].[ShippingAddress_AddressLine2] AS [ShippingAddress_AddressLine20], [c0].[ShippingAddress_ZipCode] AS [ShippingAddress_ZipCode0], [c0].[ShippingAddress_Country_Code] AS [ShippingAddress_Country_Code0], [c0].[ShippingAddress_Country_FullName] AS [ShippingAddress_Country_FullName0] + FROM [Customer] AS [c] + CROSS JOIN [Customer] AS [c0] +) AS [t] +"""); + } + + public override async Task Project_same_nested_complex_type_twice_with_pushdown(bool async) + { + await base.Project_same_nested_complex_type_twice_with_pushdown(async); + + AssertSql( +""" +SELECT [t].[BillingAddress_AddressLine1], [t].[BillingAddress_AddressLine2], [t].[BillingAddress_ZipCode], [t].[BillingAddress_Country_Code], [t].[BillingAddress_Country_FullName], [t].[BillingAddress_AddressLine10], [t].[BillingAddress_AddressLine20], [t].[BillingAddress_ZipCode0], [t].[BillingAddress_Country_Code0], [t].[BillingAddress_Country_FullName0] +FROM ( + SELECT DISTINCT [c].[BillingAddress_AddressLine1], [c].[BillingAddress_AddressLine2], [c].[BillingAddress_ZipCode], [c].[BillingAddress_Country_Code], [c].[BillingAddress_Country_FullName], [c0].[BillingAddress_AddressLine1] AS [BillingAddress_AddressLine10], [c0].[BillingAddress_AddressLine2] AS [BillingAddress_AddressLine20], [c0].[BillingAddress_ZipCode] AS [BillingAddress_ZipCode0], [c0].[BillingAddress_Country_Code] AS [BillingAddress_Country_Code0], [c0].[BillingAddress_Country_FullName] AS [BillingAddress_Country_FullName0] + FROM [Customer] AS [c] + CROSS JOIN [Customer] AS [c0] +) AS [t] +"""); + } + + public override async Task Project_same_entity_with_nested_complex_type_twice_with_double_pushdown(bool async) + { + await base.Project_same_entity_with_nested_complex_type_twice_with_double_pushdown(async); + + AssertSql( +""" +@__p_0='50' + +SELECT [t0].[Id], [t0].[Name], [t0].[BillingAddress_AddressLine1], [t0].[BillingAddress_AddressLine2], [t0].[BillingAddress_ZipCode], [t0].[BillingAddress_Country_Code], [t0].[BillingAddress_Country_FullName], [t0].[ShippingAddress_AddressLine1], [t0].[ShippingAddress_AddressLine2], [t0].[ShippingAddress_ZipCode], [t0].[ShippingAddress_Country_Code], [t0].[ShippingAddress_Country_FullName], [t0].[Id0], [t0].[Name0], [t0].[BillingAddress_AddressLine10], [t0].[BillingAddress_AddressLine20], [t0].[BillingAddress_ZipCode0], [t0].[BillingAddress_Country_Code0], [t0].[BillingAddress_Country_FullName0], [t0].[ShippingAddress_AddressLine10], [t0].[ShippingAddress_AddressLine20], [t0].[ShippingAddress_ZipCode0], [t0].[ShippingAddress_Country_Code0], [t0].[ShippingAddress_Country_FullName0] +FROM ( + SELECT DISTINCT [t].[Id], [t].[Name], [t].[BillingAddress_AddressLine1], [t].[BillingAddress_AddressLine2], [t].[BillingAddress_ZipCode], [t].[BillingAddress_Country_Code], [t].[BillingAddress_Country_FullName], [t].[ShippingAddress_AddressLine1], [t].[ShippingAddress_AddressLine2], [t].[ShippingAddress_ZipCode], [t].[ShippingAddress_Country_Code], [t].[ShippingAddress_Country_FullName], [t].[Id0], [t].[Name0], [t].[BillingAddress_AddressLine10], [t].[BillingAddress_AddressLine20], [t].[BillingAddress_ZipCode0], [t].[BillingAddress_Country_Code0], [t].[BillingAddress_Country_FullName0], [t].[ShippingAddress_AddressLine10], [t].[ShippingAddress_AddressLine20], [t].[ShippingAddress_ZipCode0], [t].[ShippingAddress_Country_Code0], [t].[ShippingAddress_Country_FullName0] + FROM ( + SELECT TOP(@__p_0) [c].[Id], [c].[Name], [c].[BillingAddress_AddressLine1], [c].[BillingAddress_AddressLine2], [c].[BillingAddress_ZipCode], [c].[BillingAddress_Country_Code], [c].[BillingAddress_Country_FullName], [c].[ShippingAddress_AddressLine1], [c].[ShippingAddress_AddressLine2], [c].[ShippingAddress_ZipCode], [c].[ShippingAddress_Country_Code], [c].[ShippingAddress_Country_FullName], [c0].[Id] AS [Id0], [c0].[Name] AS [Name0], [c0].[BillingAddress_AddressLine1] AS [BillingAddress_AddressLine10], [c0].[BillingAddress_AddressLine2] AS [BillingAddress_AddressLine20], [c0].[BillingAddress_ZipCode] AS [BillingAddress_ZipCode0], [c0].[BillingAddress_Country_Code] AS [BillingAddress_Country_Code0], [c0].[BillingAddress_Country_FullName] AS [BillingAddress_Country_FullName0], [c0].[ShippingAddress_AddressLine1] AS [ShippingAddress_AddressLine10], [c0].[ShippingAddress_AddressLine2] AS [ShippingAddress_AddressLine20], [c0].[ShippingAddress_ZipCode] AS [ShippingAddress_ZipCode0], [c0].[ShippingAddress_Country_Code] AS [ShippingAddress_Country_Code0], [c0].[ShippingAddress_Country_FullName] AS [ShippingAddress_Country_FullName0] + FROM [Customer] AS [c] + CROSS JOIN [Customer] AS [c0] + ORDER BY [c].[Id], [c0].[Id] + ) AS [t] +) AS [t0] +"""); + } + + public override async Task Project_same_nested_complex_type_twice_with_double_pushdown(bool async) + { + await base.Project_same_nested_complex_type_twice_with_double_pushdown(async); + + AssertSql( +""" +@__p_0='50' + +SELECT [t0].[BillingAddress_AddressLine1], [t0].[BillingAddress_AddressLine2], [t0].[BillingAddress_ZipCode], [t0].[BillingAddress_Country_Code], [t0].[BillingAddress_Country_FullName], [t0].[BillingAddress_AddressLine10], [t0].[BillingAddress_AddressLine20], [t0].[BillingAddress_ZipCode0], [t0].[BillingAddress_Country_Code0], [t0].[BillingAddress_Country_FullName0] +FROM ( + SELECT DISTINCT [t].[BillingAddress_AddressLine1], [t].[BillingAddress_AddressLine2], [t].[BillingAddress_ZipCode], [t].[BillingAddress_Country_Code], [t].[BillingAddress_Country_FullName], [t].[BillingAddress_AddressLine10], [t].[BillingAddress_AddressLine20], [t].[BillingAddress_ZipCode0], [t].[BillingAddress_Country_Code0], [t].[BillingAddress_Country_FullName0] + FROM ( + SELECT TOP(@__p_0) [c].[BillingAddress_AddressLine1], [c].[BillingAddress_AddressLine2], [c].[BillingAddress_ZipCode], [c].[BillingAddress_Country_Code], [c].[BillingAddress_Country_FullName], [c0].[BillingAddress_AddressLine1] AS [BillingAddress_AddressLine10], [c0].[BillingAddress_AddressLine2] AS [BillingAddress_AddressLine20], [c0].[BillingAddress_ZipCode] AS [BillingAddress_ZipCode0], [c0].[BillingAddress_Country_Code] AS [BillingAddress_Country_Code0], [c0].[BillingAddress_Country_FullName] AS [BillingAddress_Country_FullName0] + FROM [Customer] AS [c] + CROSS JOIN [Customer] AS [c0] + ORDER BY [c].[Id], [c0].[Id] + ) AS [t] +) AS [t0] +"""); + } + + public override async Task Project_same_entity_with_struct_nested_complex_type_twice_with_pushdown(bool async) + { + await base.Project_same_entity_with_struct_nested_complex_type_twice_with_pushdown(async); + + AssertSql( +""" +SELECT [t].[Id], [t].[Name], [t].[BillingAddress_AddressLine1], [t].[BillingAddress_AddressLine2], [t].[BillingAddress_ZipCode], [t].[BillingAddress_Country_Code], [t].[BillingAddress_Country_FullName], [t].[ShippingAddress_AddressLine1], [t].[ShippingAddress_AddressLine2], [t].[ShippingAddress_ZipCode], [t].[ShippingAddress_Country_Code], [t].[ShippingAddress_Country_FullName], [t].[Id0], [t].[Name0], [t].[BillingAddress_AddressLine10], [t].[BillingAddress_AddressLine20], [t].[BillingAddress_ZipCode0], [t].[BillingAddress_Country_Code0], [t].[BillingAddress_Country_FullName0], [t].[ShippingAddress_AddressLine10], [t].[ShippingAddress_AddressLine20], [t].[ShippingAddress_ZipCode0], [t].[ShippingAddress_Country_Code0], [t].[ShippingAddress_Country_FullName0] +FROM ( + SELECT DISTINCT [v].[Id], [v].[Name], [v].[BillingAddress_AddressLine1], [v].[BillingAddress_AddressLine2], [v].[BillingAddress_ZipCode], [v].[BillingAddress_Country_Code], [v].[BillingAddress_Country_FullName], [v].[ShippingAddress_AddressLine1], [v].[ShippingAddress_AddressLine2], [v].[ShippingAddress_ZipCode], [v].[ShippingAddress_Country_Code], [v].[ShippingAddress_Country_FullName], [v0].[Id] AS [Id0], [v0].[Name] AS [Name0], [v0].[BillingAddress_AddressLine1] AS [BillingAddress_AddressLine10], [v0].[BillingAddress_AddressLine2] AS [BillingAddress_AddressLine20], [v0].[BillingAddress_ZipCode] AS [BillingAddress_ZipCode0], [v0].[BillingAddress_Country_Code] AS [BillingAddress_Country_Code0], [v0].[BillingAddress_Country_FullName] AS [BillingAddress_Country_FullName0], [v0].[ShippingAddress_AddressLine1] AS [ShippingAddress_AddressLine10], [v0].[ShippingAddress_AddressLine2] AS [ShippingAddress_AddressLine20], [v0].[ShippingAddress_ZipCode] AS [ShippingAddress_ZipCode0], [v0].[ShippingAddress_Country_Code] AS [ShippingAddress_Country_Code0], [v0].[ShippingAddress_Country_FullName] AS [ShippingAddress_Country_FullName0] + FROM [ValuedCustomer] AS [v] + CROSS JOIN [ValuedCustomer] AS [v0] +) AS [t] +"""); + } + + public override async Task Project_same_struct_nested_complex_type_twice_with_pushdown(bool async) + { + await base.Project_same_struct_nested_complex_type_twice_with_pushdown(async); + + AssertSql( +""" +SELECT [t].[BillingAddress_AddressLine1], [t].[BillingAddress_AddressLine2], [t].[BillingAddress_ZipCode], [t].[BillingAddress_Country_Code], [t].[BillingAddress_Country_FullName], [t].[BillingAddress_AddressLine10], [t].[BillingAddress_AddressLine20], [t].[BillingAddress_ZipCode0], [t].[BillingAddress_Country_Code0], [t].[BillingAddress_Country_FullName0] +FROM ( + SELECT DISTINCT [v].[BillingAddress_AddressLine1], [v].[BillingAddress_AddressLine2], [v].[BillingAddress_ZipCode], [v].[BillingAddress_Country_Code], [v].[BillingAddress_Country_FullName], [v0].[BillingAddress_AddressLine1] AS [BillingAddress_AddressLine10], [v0].[BillingAddress_AddressLine2] AS [BillingAddress_AddressLine20], [v0].[BillingAddress_ZipCode] AS [BillingAddress_ZipCode0], [v0].[BillingAddress_Country_Code] AS [BillingAddress_Country_Code0], [v0].[BillingAddress_Country_FullName] AS [BillingAddress_Country_FullName0] + FROM [ValuedCustomer] AS [v] + CROSS JOIN [ValuedCustomer] AS [v0] +) AS [t] +"""); + } + + public override async Task Project_same_entity_with_struct_nested_complex_type_twice_with_double_pushdown(bool async) + { + await base.Project_same_entity_with_struct_nested_complex_type_twice_with_double_pushdown(async); + + AssertSql( +""" +@__p_0='50' + +SELECT [t0].[Id], [t0].[Name], [t0].[BillingAddress_AddressLine1], [t0].[BillingAddress_AddressLine2], [t0].[BillingAddress_ZipCode], [t0].[BillingAddress_Country_Code], [t0].[BillingAddress_Country_FullName], [t0].[ShippingAddress_AddressLine1], [t0].[ShippingAddress_AddressLine2], [t0].[ShippingAddress_ZipCode], [t0].[ShippingAddress_Country_Code], [t0].[ShippingAddress_Country_FullName], [t0].[Id0], [t0].[Name0], [t0].[BillingAddress_AddressLine10], [t0].[BillingAddress_AddressLine20], [t0].[BillingAddress_ZipCode0], [t0].[BillingAddress_Country_Code0], [t0].[BillingAddress_Country_FullName0], [t0].[ShippingAddress_AddressLine10], [t0].[ShippingAddress_AddressLine20], [t0].[ShippingAddress_ZipCode0], [t0].[ShippingAddress_Country_Code0], [t0].[ShippingAddress_Country_FullName0] +FROM ( + SELECT DISTINCT [t].[Id], [t].[Name], [t].[BillingAddress_AddressLine1], [t].[BillingAddress_AddressLine2], [t].[BillingAddress_ZipCode], [t].[BillingAddress_Country_Code], [t].[BillingAddress_Country_FullName], [t].[ShippingAddress_AddressLine1], [t].[ShippingAddress_AddressLine2], [t].[ShippingAddress_ZipCode], [t].[ShippingAddress_Country_Code], [t].[ShippingAddress_Country_FullName], [t].[Id0], [t].[Name0], [t].[BillingAddress_AddressLine10], [t].[BillingAddress_AddressLine20], [t].[BillingAddress_ZipCode0], [t].[BillingAddress_Country_Code0], [t].[BillingAddress_Country_FullName0], [t].[ShippingAddress_AddressLine10], [t].[ShippingAddress_AddressLine20], [t].[ShippingAddress_ZipCode0], [t].[ShippingAddress_Country_Code0], [t].[ShippingAddress_Country_FullName0] + FROM ( + SELECT TOP(@__p_0) [v].[Id], [v].[Name], [v].[BillingAddress_AddressLine1], [v].[BillingAddress_AddressLine2], [v].[BillingAddress_ZipCode], [v].[BillingAddress_Country_Code], [v].[BillingAddress_Country_FullName], [v].[ShippingAddress_AddressLine1], [v].[ShippingAddress_AddressLine2], [v].[ShippingAddress_ZipCode], [v].[ShippingAddress_Country_Code], [v].[ShippingAddress_Country_FullName], [v0].[Id] AS [Id0], [v0].[Name] AS [Name0], [v0].[BillingAddress_AddressLine1] AS [BillingAddress_AddressLine10], [v0].[BillingAddress_AddressLine2] AS [BillingAddress_AddressLine20], [v0].[BillingAddress_ZipCode] AS [BillingAddress_ZipCode0], [v0].[BillingAddress_Country_Code] AS [BillingAddress_Country_Code0], [v0].[BillingAddress_Country_FullName] AS [BillingAddress_Country_FullName0], [v0].[ShippingAddress_AddressLine1] AS [ShippingAddress_AddressLine10], [v0].[ShippingAddress_AddressLine2] AS [ShippingAddress_AddressLine20], [v0].[ShippingAddress_ZipCode] AS [ShippingAddress_ZipCode0], [v0].[ShippingAddress_Country_Code] AS [ShippingAddress_Country_Code0], [v0].[ShippingAddress_Country_FullName] AS [ShippingAddress_Country_FullName0] + FROM [ValuedCustomer] AS [v] + CROSS JOIN [ValuedCustomer] AS [v0] + ORDER BY [v].[Id], [v0].[Id] + ) AS [t] +) AS [t0] +"""); + } + + public override async Task Project_same_struct_nested_complex_type_twice_with_double_pushdown(bool async) + { + await base.Project_same_struct_nested_complex_type_twice_with_double_pushdown(async); + + AssertSql( +""" +@__p_0='50' + +SELECT [t0].[BillingAddress_AddressLine1], [t0].[BillingAddress_AddressLine2], [t0].[BillingAddress_ZipCode], [t0].[BillingAddress_Country_Code], [t0].[BillingAddress_Country_FullName], [t0].[BillingAddress_AddressLine10], [t0].[BillingAddress_AddressLine20], [t0].[BillingAddress_ZipCode0], [t0].[BillingAddress_Country_Code0], [t0].[BillingAddress_Country_FullName0] +FROM ( + SELECT DISTINCT [t].[BillingAddress_AddressLine1], [t].[BillingAddress_AddressLine2], [t].[BillingAddress_ZipCode], [t].[BillingAddress_Country_Code], [t].[BillingAddress_Country_FullName], [t].[BillingAddress_AddressLine10], [t].[BillingAddress_AddressLine20], [t].[BillingAddress_ZipCode0], [t].[BillingAddress_Country_Code0], [t].[BillingAddress_Country_FullName0] + FROM ( + SELECT TOP(@__p_0) [v].[BillingAddress_AddressLine1], [v].[BillingAddress_AddressLine2], [v].[BillingAddress_ZipCode], [v].[BillingAddress_Country_Code], [v].[BillingAddress_Country_FullName], [v0].[BillingAddress_AddressLine1] AS [BillingAddress_AddressLine10], [v0].[BillingAddress_AddressLine2] AS [BillingAddress_AddressLine20], [v0].[BillingAddress_ZipCode] AS [BillingAddress_ZipCode0], [v0].[BillingAddress_Country_Code] AS [BillingAddress_Country_Code0], [v0].[BillingAddress_Country_FullName] AS [BillingAddress_Country_FullName0] + FROM [ValuedCustomer] AS [v] + CROSS JOIN [ValuedCustomer] AS [v0] + ORDER BY [v].[Id], [v0].[Id] + ) AS [t] +) AS [t0] +"""); + } + + public override async Task Union_of_same_entity_with_nested_complex_type_projected_twice_with_pushdown(bool async) + { + await base.Union_of_same_entity_with_nested_complex_type_projected_twice_with_pushdown(async); + + AssertSql( +""" +@__p_0='50' + +SELECT TOP(@__p_0) [t].[Id], [t].[Name], [t].[BillingAddress_AddressLine1], [t].[BillingAddress_AddressLine2], [t].[BillingAddress_ZipCode], [t].[BillingAddress_Country_Code], [t].[BillingAddress_Country_FullName], [t].[ShippingAddress_AddressLine1], [t].[ShippingAddress_AddressLine2], [t].[ShippingAddress_ZipCode], [t].[ShippingAddress_Country_Code], [t].[ShippingAddress_Country_FullName], [t].[Id0], [t].[Name0], [t].[BillingAddress_AddressLine10], [t].[BillingAddress_AddressLine20], [t].[BillingAddress_ZipCode0], [t].[BillingAddress_Country_Code0], [t].[BillingAddress_Country_FullName0], [t].[ShippingAddress_AddressLine10], [t].[ShippingAddress_AddressLine20], [t].[ShippingAddress_ZipCode0], [t].[ShippingAddress_Country_Code0], [t].[ShippingAddress_Country_FullName0] +FROM ( + SELECT [c].[Id], [c].[Name], [c].[BillingAddress_AddressLine1], [c].[BillingAddress_AddressLine2], [c].[BillingAddress_ZipCode], [c].[BillingAddress_Country_Code], [c].[BillingAddress_Country_FullName], [c].[ShippingAddress_AddressLine1], [c].[ShippingAddress_AddressLine2], [c].[ShippingAddress_ZipCode], [c].[ShippingAddress_Country_Code], [c].[ShippingAddress_Country_FullName], [c0].[Id] AS [Id0], [c0].[Name] AS [Name0], [c0].[BillingAddress_AddressLine1] AS [BillingAddress_AddressLine10], [c0].[BillingAddress_AddressLine2] AS [BillingAddress_AddressLine20], [c0].[BillingAddress_ZipCode] AS [BillingAddress_ZipCode0], [c0].[BillingAddress_Country_Code] AS [BillingAddress_Country_Code0], [c0].[BillingAddress_Country_FullName] AS [BillingAddress_Country_FullName0], [c0].[ShippingAddress_AddressLine1] AS [ShippingAddress_AddressLine10], [c0].[ShippingAddress_AddressLine2] AS [ShippingAddress_AddressLine20], [c0].[ShippingAddress_ZipCode] AS [ShippingAddress_ZipCode0], [c0].[ShippingAddress_Country_Code] AS [ShippingAddress_Country_Code0], [c0].[ShippingAddress_Country_FullName] AS [ShippingAddress_Country_FullName0] + FROM [Customer] AS [c] + CROSS JOIN [Customer] AS [c0] + UNION + SELECT [c1].[Id], [c1].[Name], [c1].[BillingAddress_AddressLine1], [c1].[BillingAddress_AddressLine2], [c1].[BillingAddress_ZipCode], [c1].[BillingAddress_Country_Code], [c1].[BillingAddress_Country_FullName], [c1].[ShippingAddress_AddressLine1], [c1].[ShippingAddress_AddressLine2], [c1].[ShippingAddress_ZipCode], [c1].[ShippingAddress_Country_Code], [c1].[ShippingAddress_Country_FullName], [c2].[Id] AS [Id0], [c2].[Name] AS [Name0], [c2].[BillingAddress_AddressLine1] AS [BillingAddress_AddressLine10], [c2].[BillingAddress_AddressLine2] AS [BillingAddress_AddressLine20], [c2].[BillingAddress_ZipCode] AS [BillingAddress_ZipCode0], [c2].[BillingAddress_Country_Code] AS [BillingAddress_Country_Code0], [c2].[BillingAddress_Country_FullName] AS [BillingAddress_Country_FullName0], [c2].[ShippingAddress_AddressLine1] AS [ShippingAddress_AddressLine10], [c2].[ShippingAddress_AddressLine2] AS [ShippingAddress_AddressLine20], [c2].[ShippingAddress_ZipCode] AS [ShippingAddress_ZipCode0], [c2].[ShippingAddress_Country_Code] AS [ShippingAddress_Country_Code0], [c2].[ShippingAddress_Country_FullName] AS [ShippingAddress_Country_FullName0] + FROM [Customer] AS [c1] + CROSS JOIN [Customer] AS [c2] +) AS [t] +ORDER BY [t].[Id], [t].[Id0] +"""); + } + + public override async Task Union_of_same_entity_with_nested_complex_type_projected_twice_with_double_pushdown(bool async) + { + await base.Union_of_same_entity_with_nested_complex_type_projected_twice_with_double_pushdown(async); + + AssertSql( +""" +@__p_0='50' + +SELECT TOP(@__p_0) [t1].[Id], [t1].[Name], [t1].[BillingAddress_AddressLine1], [t1].[BillingAddress_AddressLine2], [t1].[BillingAddress_ZipCode], [t1].[BillingAddress_Country_Code], [t1].[BillingAddress_Country_FullName], [t1].[ShippingAddress_AddressLine1], [t1].[ShippingAddress_AddressLine2], [t1].[ShippingAddress_ZipCode], [t1].[ShippingAddress_Country_Code], [t1].[ShippingAddress_Country_FullName], [t1].[Id0], [t1].[Name0], [t1].[BillingAddress_AddressLine10], [t1].[BillingAddress_AddressLine20], [t1].[BillingAddress_ZipCode0], [t1].[BillingAddress_Country_Code0], [t1].[BillingAddress_Country_FullName0], [t1].[ShippingAddress_AddressLine10], [t1].[ShippingAddress_AddressLine20], [t1].[ShippingAddress_ZipCode0], [t1].[ShippingAddress_Country_Code0], [t1].[ShippingAddress_Country_FullName0] +FROM ( + SELECT DISTINCT [t0].[Id], [t0].[Name], [t0].[BillingAddress_AddressLine1], [t0].[BillingAddress_AddressLine2], [t0].[BillingAddress_ZipCode], [t0].[BillingAddress_Country_Code], [t0].[BillingAddress_Country_FullName], [t0].[ShippingAddress_AddressLine1], [t0].[ShippingAddress_AddressLine2], [t0].[ShippingAddress_ZipCode], [t0].[ShippingAddress_Country_Code], [t0].[ShippingAddress_Country_FullName], [t0].[Id0], [t0].[Name0], [t0].[BillingAddress_AddressLine10], [t0].[BillingAddress_AddressLine20], [t0].[BillingAddress_ZipCode0], [t0].[BillingAddress_Country_Code0], [t0].[BillingAddress_Country_FullName0], [t0].[ShippingAddress_AddressLine10], [t0].[ShippingAddress_AddressLine20], [t0].[ShippingAddress_ZipCode0], [t0].[ShippingAddress_Country_Code0], [t0].[ShippingAddress_Country_FullName0] + FROM ( + SELECT TOP(@__p_0) [t].[Id], [t].[Name], [t].[BillingAddress_AddressLine1], [t].[BillingAddress_AddressLine2], [t].[BillingAddress_ZipCode], [t].[BillingAddress_Country_Code], [t].[BillingAddress_Country_FullName], [t].[ShippingAddress_AddressLine1], [t].[ShippingAddress_AddressLine2], [t].[ShippingAddress_ZipCode], [t].[ShippingAddress_Country_Code], [t].[ShippingAddress_Country_FullName], [t].[Id0], [t].[Name0], [t].[BillingAddress_AddressLine10], [t].[BillingAddress_AddressLine20], [t].[BillingAddress_ZipCode0], [t].[BillingAddress_Country_Code0], [t].[BillingAddress_Country_FullName0], [t].[ShippingAddress_AddressLine10], [t].[ShippingAddress_AddressLine20], [t].[ShippingAddress_ZipCode0], [t].[ShippingAddress_Country_Code0], [t].[ShippingAddress_Country_FullName0] + FROM ( + SELECT [c].[Id], [c].[Name], [c].[BillingAddress_AddressLine1], [c].[BillingAddress_AddressLine2], [c].[BillingAddress_ZipCode], [c].[BillingAddress_Country_Code], [c].[BillingAddress_Country_FullName], [c].[ShippingAddress_AddressLine1], [c].[ShippingAddress_AddressLine2], [c].[ShippingAddress_ZipCode], [c].[ShippingAddress_Country_Code], [c].[ShippingAddress_Country_FullName], [c0].[Id] AS [Id0], [c0].[Name] AS [Name0], [c0].[BillingAddress_AddressLine1] AS [BillingAddress_AddressLine10], [c0].[BillingAddress_AddressLine2] AS [BillingAddress_AddressLine20], [c0].[BillingAddress_ZipCode] AS [BillingAddress_ZipCode0], [c0].[BillingAddress_Country_Code] AS [BillingAddress_Country_Code0], [c0].[BillingAddress_Country_FullName] AS [BillingAddress_Country_FullName0], [c0].[ShippingAddress_AddressLine1] AS [ShippingAddress_AddressLine10], [c0].[ShippingAddress_AddressLine2] AS [ShippingAddress_AddressLine20], [c0].[ShippingAddress_ZipCode] AS [ShippingAddress_ZipCode0], [c0].[ShippingAddress_Country_Code] AS [ShippingAddress_Country_Code0], [c0].[ShippingAddress_Country_FullName] AS [ShippingAddress_Country_FullName0] + FROM [Customer] AS [c] + CROSS JOIN [Customer] AS [c0] + UNION + SELECT [c1].[Id], [c1].[Name], [c1].[BillingAddress_AddressLine1], [c1].[BillingAddress_AddressLine2], [c1].[BillingAddress_ZipCode], [c1].[BillingAddress_Country_Code], [c1].[BillingAddress_Country_FullName], [c1].[ShippingAddress_AddressLine1], [c1].[ShippingAddress_AddressLine2], [c1].[ShippingAddress_ZipCode], [c1].[ShippingAddress_Country_Code], [c1].[ShippingAddress_Country_FullName], [c2].[Id] AS [Id0], [c2].[Name] AS [Name0], [c2].[BillingAddress_AddressLine1] AS [BillingAddress_AddressLine10], [c2].[BillingAddress_AddressLine2] AS [BillingAddress_AddressLine20], [c2].[BillingAddress_ZipCode] AS [BillingAddress_ZipCode0], [c2].[BillingAddress_Country_Code] AS [BillingAddress_Country_Code0], [c2].[BillingAddress_Country_FullName] AS [BillingAddress_Country_FullName0], [c2].[ShippingAddress_AddressLine1] AS [ShippingAddress_AddressLine10], [c2].[ShippingAddress_AddressLine2] AS [ShippingAddress_AddressLine20], [c2].[ShippingAddress_ZipCode] AS [ShippingAddress_ZipCode0], [c2].[ShippingAddress_Country_Code] AS [ShippingAddress_Country_Code0], [c2].[ShippingAddress_Country_FullName] AS [ShippingAddress_Country_FullName0] + FROM [Customer] AS [c1] + CROSS JOIN [Customer] AS [c2] + ) AS [t] + ORDER BY [t].[Id], [t].[Id0] + ) AS [t0] +) AS [t1] +ORDER BY [t1].[Id], [t1].[Id0] +"""); + } + + public override async Task Union_of_same_nested_complex_type_projected_twice_with_pushdown(bool async) + { + await base.Union_of_same_nested_complex_type_projected_twice_with_pushdown(async); + + AssertSql( +""" +@__p_0='50' + +SELECT TOP(@__p_0) [t].[BillingAddress_AddressLine1], [t].[BillingAddress_AddressLine2], [t].[BillingAddress_ZipCode], [t].[BillingAddress_Country_Code], [t].[BillingAddress_Country_FullName], [t].[BillingAddress_AddressLine10], [t].[BillingAddress_AddressLine20], [t].[BillingAddress_ZipCode0], [t].[BillingAddress_Country_Code0], [t].[BillingAddress_Country_FullName0] +FROM ( + SELECT [c].[BillingAddress_AddressLine1], [c].[BillingAddress_AddressLine2], [c].[BillingAddress_ZipCode], [c].[BillingAddress_Country_Code], [c].[BillingAddress_Country_FullName], [c0].[BillingAddress_AddressLine1] AS [BillingAddress_AddressLine10], [c0].[BillingAddress_AddressLine2] AS [BillingAddress_AddressLine20], [c0].[BillingAddress_ZipCode] AS [BillingAddress_ZipCode0], [c0].[BillingAddress_Country_Code] AS [BillingAddress_Country_Code0], [c0].[BillingAddress_Country_FullName] AS [BillingAddress_Country_FullName0] + FROM [Customer] AS [c] + CROSS JOIN [Customer] AS [c0] + UNION + SELECT [c1].[BillingAddress_AddressLine1], [c1].[BillingAddress_AddressLine2], [c1].[BillingAddress_ZipCode], [c1].[BillingAddress_Country_Code], [c1].[BillingAddress_Country_FullName], [c2].[BillingAddress_AddressLine1] AS [BillingAddress_AddressLine10], [c2].[BillingAddress_AddressLine2] AS [BillingAddress_AddressLine20], [c2].[BillingAddress_ZipCode] AS [BillingAddress_ZipCode0], [c2].[BillingAddress_Country_Code] AS [BillingAddress_Country_Code0], [c2].[BillingAddress_Country_FullName] AS [BillingAddress_Country_FullName0] + FROM [Customer] AS [c1] + CROSS JOIN [Customer] AS [c2] +) AS [t] +ORDER BY [t].[BillingAddress_ZipCode], [t].[BillingAddress_ZipCode0] +"""); + } + + public override async Task Union_of_same_nested_complex_type_projected_twice_with_double_pushdown(bool async) + { + await base.Union_of_same_nested_complex_type_projected_twice_with_double_pushdown(async); + + AssertSql( +""" +@__p_0='50' + +SELECT TOP(@__p_0) [t1].[BillingAddress_AddressLine1], [t1].[BillingAddress_AddressLine2], [t1].[BillingAddress_ZipCode], [t1].[BillingAddress_Country_Code], [t1].[BillingAddress_Country_FullName], [t1].[BillingAddress_AddressLine10], [t1].[BillingAddress_AddressLine20], [t1].[BillingAddress_ZipCode0], [t1].[BillingAddress_Country_Code0], [t1].[BillingAddress_Country_FullName0] +FROM ( + SELECT DISTINCT [t0].[BillingAddress_AddressLine1], [t0].[BillingAddress_AddressLine2], [t0].[BillingAddress_ZipCode], [t0].[BillingAddress_Country_Code], [t0].[BillingAddress_Country_FullName], [t0].[BillingAddress_AddressLine10], [t0].[BillingAddress_AddressLine20], [t0].[BillingAddress_ZipCode0], [t0].[BillingAddress_Country_Code0], [t0].[BillingAddress_Country_FullName0] + FROM ( + SELECT TOP(@__p_0) [t].[BillingAddress_AddressLine1], [t].[BillingAddress_AddressLine2], [t].[BillingAddress_ZipCode], [t].[BillingAddress_Country_Code], [t].[BillingAddress_Country_FullName], [t].[BillingAddress_AddressLine10], [t].[BillingAddress_AddressLine20], [t].[BillingAddress_ZipCode0], [t].[BillingAddress_Country_Code0], [t].[BillingAddress_Country_FullName0] + FROM ( + SELECT [c].[BillingAddress_AddressLine1], [c].[BillingAddress_AddressLine2], [c].[BillingAddress_ZipCode], [c].[BillingAddress_Country_Code], [c].[BillingAddress_Country_FullName], [c0].[BillingAddress_AddressLine1] AS [BillingAddress_AddressLine10], [c0].[BillingAddress_AddressLine2] AS [BillingAddress_AddressLine20], [c0].[BillingAddress_ZipCode] AS [BillingAddress_ZipCode0], [c0].[BillingAddress_Country_Code] AS [BillingAddress_Country_Code0], [c0].[BillingAddress_Country_FullName] AS [BillingAddress_Country_FullName0] + FROM [Customer] AS [c] + CROSS JOIN [Customer] AS [c0] + UNION + SELECT [c1].[BillingAddress_AddressLine1], [c1].[BillingAddress_AddressLine2], [c1].[BillingAddress_ZipCode], [c1].[BillingAddress_Country_Code], [c1].[BillingAddress_Country_FullName], [c2].[BillingAddress_AddressLine1] AS [BillingAddress_AddressLine10], [c2].[BillingAddress_AddressLine2] AS [BillingAddress_AddressLine20], [c2].[BillingAddress_ZipCode] AS [BillingAddress_ZipCode0], [c2].[BillingAddress_Country_Code] AS [BillingAddress_Country_Code0], [c2].[BillingAddress_Country_FullName] AS [BillingAddress_Country_FullName0] + FROM [Customer] AS [c1] + CROSS JOIN [Customer] AS [c2] + ) AS [t] + ORDER BY [t].[BillingAddress_ZipCode], [t].[BillingAddress_ZipCode0] + ) AS [t0] +) AS [t1] +ORDER BY [t1].[BillingAddress_ZipCode], [t1].[BillingAddress_ZipCode0] +"""); + } + + public override async Task Same_entity_with_complex_type_projected_twice_with_pushdown_as_part_of_another_projection(bool async) + { + await base.Same_entity_with_complex_type_projected_twice_with_pushdown_as_part_of_another_projection(async); + + AssertSql( +""" +SELECT [c].[Id], [t].[Id], [t].[Name], [t].[BillingAddress_AddressLine1], [t].[BillingAddress_AddressLine2], [t].[BillingAddress_ZipCode], [t].[BillingAddress_Country_Code], [t].[BillingAddress_Country_FullName], [t].[ShippingAddress_AddressLine1], [t].[ShippingAddress_AddressLine2], [t].[ShippingAddress_ZipCode], [t].[ShippingAddress_Country_Code], [t].[ShippingAddress_Country_FullName], [t].[Id0], [t].[Name0], [t].[BillingAddress_AddressLine10], [t].[BillingAddress_AddressLine20], [t].[BillingAddress_ZipCode0], [t].[BillingAddress_Country_Code0], [t].[BillingAddress_Country_FullName0], [t].[ShippingAddress_AddressLine10], [t].[ShippingAddress_AddressLine20], [t].[ShippingAddress_ZipCode0], [t].[ShippingAddress_Country_Code0], [t].[ShippingAddress_Country_FullName0], [t].[c] +FROM [Customer] AS [c] +OUTER APPLY ( + SELECT TOP(1) [c0].[Id], [c0].[Name], [c0].[BillingAddress_AddressLine1], [c0].[BillingAddress_AddressLine2], [c0].[BillingAddress_ZipCode], [c0].[BillingAddress_Country_Code], [c0].[BillingAddress_Country_FullName], [c0].[ShippingAddress_AddressLine1], [c0].[ShippingAddress_AddressLine2], [c0].[ShippingAddress_ZipCode], [c0].[ShippingAddress_Country_Code], [c0].[ShippingAddress_Country_FullName], [c1].[Id] AS [Id0], [c1].[Name] AS [Name0], [c1].[BillingAddress_AddressLine1] AS [BillingAddress_AddressLine10], [c1].[BillingAddress_AddressLine2] AS [BillingAddress_AddressLine20], [c1].[BillingAddress_ZipCode] AS [BillingAddress_ZipCode0], [c1].[BillingAddress_Country_Code] AS [BillingAddress_Country_Code0], [c1].[BillingAddress_Country_FullName] AS [BillingAddress_Country_FullName0], [c1].[ShippingAddress_AddressLine1] AS [ShippingAddress_AddressLine10], [c1].[ShippingAddress_AddressLine2] AS [ShippingAddress_AddressLine20], [c1].[ShippingAddress_ZipCode] AS [ShippingAddress_ZipCode0], [c1].[ShippingAddress_Country_Code] AS [ShippingAddress_Country_Code0], [c1].[ShippingAddress_Country_FullName] AS [ShippingAddress_Country_FullName0], 1 AS [c] + FROM [Customer] AS [c0] + CROSS JOIN [Customer] AS [c1] + ORDER BY [c0].[Id], [c1].[Id] DESC +) AS [t] +"""); + } + + public override async Task Same_complex_type_projected_twice_with_pushdown_as_part_of_another_projection(bool async) + { + await base.Same_complex_type_projected_twice_with_pushdown_as_part_of_another_projection(async); + + AssertSql(""); + } + [ConditionalFact] public virtual void Check_all_tests_overridden() => TestHelpers.AssertAllMethodsOverridden(GetType()); diff --git a/test/EFCore.SqlServer.FunctionalTests/Query/NonSharedPrimitiveCollectionsQuerySqlServerTest.cs b/test/EFCore.SqlServer.FunctionalTests/Query/NonSharedPrimitiveCollectionsQuerySqlServerTest.cs index 28703f1f91d..a28150f3ad6 100644 --- a/test/EFCore.SqlServer.FunctionalTests/Query/NonSharedPrimitiveCollectionsQuerySqlServerTest.cs +++ b/test/EFCore.SqlServer.FunctionalTests/Query/NonSharedPrimitiveCollectionsQuerySqlServerTest.cs @@ -871,6 +871,42 @@ public virtual async Task Same_collection_with_conflicting_type_mappings_not_sup #endregion Type mapping inference + [ConditionalFact] + public virtual async Task Ordered_collection_with_split_query() + { + var contextFactory = await InitializeAsync( + onModelCreating: mb => mb.Entity(), + seed: context => + { + context.Add(new Context32976.Principal { Ints = [2, 3, 4]}); + context.SaveChanges(); + }); + + await using var context = contextFactory.CreateContext(); + + _ = await context.Set() + .Where(p => p.Ints.Skip(1).Contains(3)) + .Include(p => p.Dependents) + .AsSplitQuery() + .SingleAsync(); + } + + public class Context32976(DbContextOptions options) : DbContext(options) + { + public class Principal + { + public int Id { get; set; } + public List Ints { get; set; } + public List Dependents { get; set; } + } + + public class Dependent + { + public int Id { get; set; } + public Principal Principal { get; set; } + } + } + [ConditionalFact] public virtual void Check_all_tests_overridden() => TestHelpers.AssertAllMethodsOverridden(GetType()); diff --git a/test/EFCore.SqlServer.Tests/Infrastructure/SqlServerModelValidatorTest.cs b/test/EFCore.SqlServer.Tests/Infrastructure/SqlServerModelValidatorTest.cs index 7019f8ada98..a311cb5d41b 100644 --- a/test/EFCore.SqlServer.Tests/Infrastructure/SqlServerModelValidatorTest.cs +++ b/test/EFCore.SqlServer.Tests/Infrastructure/SqlServerModelValidatorTest.cs @@ -11,6 +11,20 @@ namespace Microsoft.EntityFrameworkCore.Infrastructure; public class SqlServerModelValidatorTest : RelationalModelValidatorTest { + [ConditionalFact] + public virtual void Passes_on_TPT_with_nested_owned_types() + { + var modelBuilder = base.CreateConventionModelBuilder(); + + modelBuilder.Entity().UseTptMappingStrategy(); + modelBuilder.Entity(); + modelBuilder.Entity(); + modelBuilder.Entity(); + modelBuilder.Entity(); + + Validate(modelBuilder); + } + public override void Detects_duplicate_columns_in_derived_types_with_different_types() { var modelBuilder = CreateConventionModelBuilder(); diff --git a/test/EFCore.Sqlite.FunctionalTests/Migrations/MigrationsSqliteTest.cs b/test/EFCore.Sqlite.FunctionalTests/Migrations/MigrationsSqliteTest.cs index c5de28da7a2..1dfe9c2d5a1 100644 --- a/test/EFCore.Sqlite.FunctionalTests/Migrations/MigrationsSqliteTest.cs +++ b/test/EFCore.Sqlite.FunctionalTests/Migrations/MigrationsSqliteTest.cs @@ -1702,6 +1702,24 @@ await Test( """); } + public override async Task Create_table_with_complex_type_with_required_properties_on_derived_entity_in_TPH() + { + await base.Create_table_with_complex_type_with_required_properties_on_derived_entity_in_TPH(); + + AssertSql( +""" +CREATE TABLE "Contacts" ( + "Id" INTEGER NOT NULL CONSTRAINT "PK_Contacts" PRIMARY KEY AUTOINCREMENT, + "Discriminator" TEXT NOT NULL, + "Name" TEXT NULL, + "Number" INTEGER NULL, + "MyComplex_Prop" TEXT NULL, + "MyComplex_MyNestedComplex_Bar" TEXT NULL, + "MyComplex_MyNestedComplex_Foo" INTEGER NULL +); +"""); + } + public override Task Create_sequence() => AssertNotSupportedAsync(base.Create_sequence, SqliteStrings.SequencesNotSupported); @@ -1732,6 +1750,103 @@ public override Task Rename_sequence() public override Task Move_sequence() => AssertNotSupportedAsync(base.Move_sequence, SqliteStrings.SequencesNotSupported); + + [ConditionalFact] + public override async Task Add_required_primitve_collection_to_existing_table() + { + await base.Add_required_primitve_collection_to_existing_table(); + + AssertSql( +""" +ALTER TABLE "Customers" ADD "Numbers" TEXT NOT NULL DEFAULT '[]'; +"""); + } + + [ConditionalFact] + public override async Task Add_required_primitve_collection_with_custom_default_value_to_existing_table() + { + await base.Add_required_primitve_collection_with_custom_default_value_to_existing_table(); + + AssertSql( +""" +ALTER TABLE "Customers" ADD "Numbers" TEXT NOT NULL DEFAULT '[1,2,3]'; +"""); + } + + [ConditionalFact] + public override async Task Add_required_primitve_collection_with_custom_default_value_sql_to_existing_table() + { + await base.Add_required_primitve_collection_with_custom_default_value_sql_to_existing_table_core("'[3, 2, 1]'"); + + AssertSql( +""" +ALTER TABLE "Customers" ADD "Numbers" TEXT NOT NULL DEFAULT ('[3, 2, 1]'); +"""); + } + + [ConditionalFact(Skip = "issue #33038")] + public override async Task Add_required_primitve_collection_with_custom_converter_to_existing_table() + { + await base.Add_required_primitve_collection_with_custom_converter_to_existing_table(); + + AssertSql( +""" +ALTER TABLE [Customers] ADD [Numbers] nvarchar(max) NOT NULL DEFAULT N'nothing'; +"""); + } + + [ConditionalFact] + public override async Task Add_required_primitve_collection_with_custom_converter_and_custom_default_value_to_existing_table() + { + await base.Add_required_primitve_collection_with_custom_converter_and_custom_default_value_to_existing_table(); + + AssertSql( +""" +ALTER TABLE "Customers" ADD "Numbers" TEXT NOT NULL DEFAULT 'some numbers'; +"""); + } + + [ConditionalFact] + public override async Task Add_optional_primitive_collection_to_existing_table() + { + await base.Add_optional_primitive_collection_to_existing_table(); + + AssertSql( +""" +ALTER TABLE "Customers" ADD "Numbers" TEXT NULL; +"""); + } + + [ConditionalFact] + public override async Task Create_table_with_required_primitive_collection() + { + await base.Create_table_with_required_primitive_collection(); + + AssertSql( +""" +CREATE TABLE "Customers" ( + "Id" INTEGER NOT NULL CONSTRAINT "PK_Customers" PRIMARY KEY AUTOINCREMENT, + "Name" TEXT NULL, + "Numbers" TEXT NOT NULL +); +"""); + } + + [ConditionalFact] + public override async Task Create_table_with_optional_primitive_collection() + { + await base.Create_table_with_optional_primitive_collection(); + + AssertSql( +""" +CREATE TABLE "Customers" ( + "Id" INTEGER NOT NULL CONSTRAINT "PK_Customers" PRIMARY KEY AUTOINCREMENT, + "Name" TEXT NULL, + "Numbers" TEXT NULL +); +"""); + } + // SQLite does not support schemas protected override bool AssertSchemaNames => false; diff --git a/test/EFCore.Sqlite.FunctionalTests/Query/AdHocAdvancedMappingsQuerySqliteTest.cs b/test/EFCore.Sqlite.FunctionalTests/Query/AdHocAdvancedMappingsQuerySqliteTest.cs new file mode 100644 index 00000000000..1807f23b4a2 --- /dev/null +++ b/test/EFCore.Sqlite.FunctionalTests/Query/AdHocAdvancedMappingsQuerySqliteTest.cs @@ -0,0 +1,10 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +namespace Microsoft.EntityFrameworkCore.Query; + +public class AdHocAdvancedMappingsQuerySqliteTest : AdHocAdvancedMappingsQueryRelationalTestBase +{ + protected override ITestStoreFactory TestStoreFactory + => SqliteTestStoreFactory.Instance; +} diff --git a/test/EFCore.Sqlite.FunctionalTests/Query/ComplexTypeQuerySqliteTest.cs b/test/EFCore.Sqlite.FunctionalTests/Query/ComplexTypeQuerySqliteTest.cs index 51c426cd289..ace097abb2a 100644 --- a/test/EFCore.Sqlite.FunctionalTests/Query/ComplexTypeQuerySqliteTest.cs +++ b/test/EFCore.Sqlite.FunctionalTests/Query/ComplexTypeQuerySqliteTest.cs @@ -1,6 +1,8 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. +using Microsoft.EntityFrameworkCore.Sqlite.Internal; + namespace Microsoft.EntityFrameworkCore.Query; public class ComplexTypeQuerySqliteTest : ComplexTypeQueryRelationalTestBase< @@ -740,6 +742,274 @@ public override async Task Union_two_different_struct_complex_type(bool async) AssertSql(); } + public override async Task Project_same_entity_with_nested_complex_type_twice_with_pushdown(bool async) + { + await base.Project_same_entity_with_nested_complex_type_twice_with_pushdown(async); + + AssertSql( +""" +SELECT "t"."Id", "t"."Name", "t"."BillingAddress_AddressLine1", "t"."BillingAddress_AddressLine2", "t"."BillingAddress_ZipCode", "t"."BillingAddress_Country_Code", "t"."BillingAddress_Country_FullName", "t"."ShippingAddress_AddressLine1", "t"."ShippingAddress_AddressLine2", "t"."ShippingAddress_ZipCode", "t"."ShippingAddress_Country_Code", "t"."ShippingAddress_Country_FullName", "t"."Id0", "t"."Name0", "t"."BillingAddress_AddressLine10", "t"."BillingAddress_AddressLine20", "t"."BillingAddress_ZipCode0", "t"."BillingAddress_Country_Code0", "t"."BillingAddress_Country_FullName0", "t"."ShippingAddress_AddressLine10", "t"."ShippingAddress_AddressLine20", "t"."ShippingAddress_ZipCode0", "t"."ShippingAddress_Country_Code0", "t"."ShippingAddress_Country_FullName0" +FROM ( + SELECT DISTINCT "c"."Id", "c"."Name", "c"."BillingAddress_AddressLine1", "c"."BillingAddress_AddressLine2", "c"."BillingAddress_ZipCode", "c"."BillingAddress_Country_Code", "c"."BillingAddress_Country_FullName", "c"."ShippingAddress_AddressLine1", "c"."ShippingAddress_AddressLine2", "c"."ShippingAddress_ZipCode", "c"."ShippingAddress_Country_Code", "c"."ShippingAddress_Country_FullName", "c0"."Id" AS "Id0", "c0"."Name" AS "Name0", "c0"."BillingAddress_AddressLine1" AS "BillingAddress_AddressLine10", "c0"."BillingAddress_AddressLine2" AS "BillingAddress_AddressLine20", "c0"."BillingAddress_ZipCode" AS "BillingAddress_ZipCode0", "c0"."BillingAddress_Country_Code" AS "BillingAddress_Country_Code0", "c0"."BillingAddress_Country_FullName" AS "BillingAddress_Country_FullName0", "c0"."ShippingAddress_AddressLine1" AS "ShippingAddress_AddressLine10", "c0"."ShippingAddress_AddressLine2" AS "ShippingAddress_AddressLine20", "c0"."ShippingAddress_ZipCode" AS "ShippingAddress_ZipCode0", "c0"."ShippingAddress_Country_Code" AS "ShippingAddress_Country_Code0", "c0"."ShippingAddress_Country_FullName" AS "ShippingAddress_Country_FullName0" + FROM "Customer" AS "c" + CROSS JOIN "Customer" AS "c0" +) AS "t" +"""); + } + + public override async Task Project_same_nested_complex_type_twice_with_pushdown(bool async) + { + await base.Project_same_nested_complex_type_twice_with_pushdown(async); + + AssertSql( +""" +SELECT "t"."BillingAddress_AddressLine1", "t"."BillingAddress_AddressLine2", "t"."BillingAddress_ZipCode", "t"."BillingAddress_Country_Code", "t"."BillingAddress_Country_FullName", "t"."BillingAddress_AddressLine10", "t"."BillingAddress_AddressLine20", "t"."BillingAddress_ZipCode0", "t"."BillingAddress_Country_Code0", "t"."BillingAddress_Country_FullName0" +FROM ( + SELECT DISTINCT "c"."BillingAddress_AddressLine1", "c"."BillingAddress_AddressLine2", "c"."BillingAddress_ZipCode", "c"."BillingAddress_Country_Code", "c"."BillingAddress_Country_FullName", "c0"."BillingAddress_AddressLine1" AS "BillingAddress_AddressLine10", "c0"."BillingAddress_AddressLine2" AS "BillingAddress_AddressLine20", "c0"."BillingAddress_ZipCode" AS "BillingAddress_ZipCode0", "c0"."BillingAddress_Country_Code" AS "BillingAddress_Country_Code0", "c0"."BillingAddress_Country_FullName" AS "BillingAddress_Country_FullName0" + FROM "Customer" AS "c" + CROSS JOIN "Customer" AS "c0" +) AS "t" +"""); + } + + public override async Task Project_same_entity_with_nested_complex_type_twice_with_double_pushdown(bool async) + { + await base.Project_same_entity_with_nested_complex_type_twice_with_double_pushdown(async); + + AssertSql( +""" +@__p_0='50' + +SELECT "t0"."Id", "t0"."Name", "t0"."BillingAddress_AddressLine1", "t0"."BillingAddress_AddressLine2", "t0"."BillingAddress_ZipCode", "t0"."BillingAddress_Country_Code", "t0"."BillingAddress_Country_FullName", "t0"."ShippingAddress_AddressLine1", "t0"."ShippingAddress_AddressLine2", "t0"."ShippingAddress_ZipCode", "t0"."ShippingAddress_Country_Code", "t0"."ShippingAddress_Country_FullName", "t0"."Id0", "t0"."Name0", "t0"."BillingAddress_AddressLine10", "t0"."BillingAddress_AddressLine20", "t0"."BillingAddress_ZipCode0", "t0"."BillingAddress_Country_Code0", "t0"."BillingAddress_Country_FullName0", "t0"."ShippingAddress_AddressLine10", "t0"."ShippingAddress_AddressLine20", "t0"."ShippingAddress_ZipCode0", "t0"."ShippingAddress_Country_Code0", "t0"."ShippingAddress_Country_FullName0" +FROM ( + SELECT DISTINCT "t"."Id", "t"."Name", "t"."BillingAddress_AddressLine1", "t"."BillingAddress_AddressLine2", "t"."BillingAddress_ZipCode", "t"."BillingAddress_Country_Code", "t"."BillingAddress_Country_FullName", "t"."ShippingAddress_AddressLine1", "t"."ShippingAddress_AddressLine2", "t"."ShippingAddress_ZipCode", "t"."ShippingAddress_Country_Code", "t"."ShippingAddress_Country_FullName", "t"."Id0", "t"."Name0", "t"."BillingAddress_AddressLine10", "t"."BillingAddress_AddressLine20", "t"."BillingAddress_ZipCode0", "t"."BillingAddress_Country_Code0", "t"."BillingAddress_Country_FullName0", "t"."ShippingAddress_AddressLine10", "t"."ShippingAddress_AddressLine20", "t"."ShippingAddress_ZipCode0", "t"."ShippingAddress_Country_Code0", "t"."ShippingAddress_Country_FullName0" + FROM ( + SELECT "c"."Id", "c"."Name", "c"."BillingAddress_AddressLine1", "c"."BillingAddress_AddressLine2", "c"."BillingAddress_ZipCode", "c"."BillingAddress_Country_Code", "c"."BillingAddress_Country_FullName", "c"."ShippingAddress_AddressLine1", "c"."ShippingAddress_AddressLine2", "c"."ShippingAddress_ZipCode", "c"."ShippingAddress_Country_Code", "c"."ShippingAddress_Country_FullName", "c0"."Id" AS "Id0", "c0"."Name" AS "Name0", "c0"."BillingAddress_AddressLine1" AS "BillingAddress_AddressLine10", "c0"."BillingAddress_AddressLine2" AS "BillingAddress_AddressLine20", "c0"."BillingAddress_ZipCode" AS "BillingAddress_ZipCode0", "c0"."BillingAddress_Country_Code" AS "BillingAddress_Country_Code0", "c0"."BillingAddress_Country_FullName" AS "BillingAddress_Country_FullName0", "c0"."ShippingAddress_AddressLine1" AS "ShippingAddress_AddressLine10", "c0"."ShippingAddress_AddressLine2" AS "ShippingAddress_AddressLine20", "c0"."ShippingAddress_ZipCode" AS "ShippingAddress_ZipCode0", "c0"."ShippingAddress_Country_Code" AS "ShippingAddress_Country_Code0", "c0"."ShippingAddress_Country_FullName" AS "ShippingAddress_Country_FullName0" + FROM "Customer" AS "c" + CROSS JOIN "Customer" AS "c0" + ORDER BY "c"."Id", "c0"."Id" + LIMIT @__p_0 + ) AS "t" +) AS "t0" +"""); + } + + public override async Task Project_same_nested_complex_type_twice_with_double_pushdown(bool async) + { + await base.Project_same_nested_complex_type_twice_with_double_pushdown(async); + + AssertSql( +""" +@__p_0='50' + +SELECT "t0"."BillingAddress_AddressLine1", "t0"."BillingAddress_AddressLine2", "t0"."BillingAddress_ZipCode", "t0"."BillingAddress_Country_Code", "t0"."BillingAddress_Country_FullName", "t0"."BillingAddress_AddressLine10", "t0"."BillingAddress_AddressLine20", "t0"."BillingAddress_ZipCode0", "t0"."BillingAddress_Country_Code0", "t0"."BillingAddress_Country_FullName0" +FROM ( + SELECT DISTINCT "t"."BillingAddress_AddressLine1", "t"."BillingAddress_AddressLine2", "t"."BillingAddress_ZipCode", "t"."BillingAddress_Country_Code", "t"."BillingAddress_Country_FullName", "t"."BillingAddress_AddressLine10", "t"."BillingAddress_AddressLine20", "t"."BillingAddress_ZipCode0", "t"."BillingAddress_Country_Code0", "t"."BillingAddress_Country_FullName0" + FROM ( + SELECT "c"."BillingAddress_AddressLine1", "c"."BillingAddress_AddressLine2", "c"."BillingAddress_ZipCode", "c"."BillingAddress_Country_Code", "c"."BillingAddress_Country_FullName", "c0"."BillingAddress_AddressLine1" AS "BillingAddress_AddressLine10", "c0"."BillingAddress_AddressLine2" AS "BillingAddress_AddressLine20", "c0"."BillingAddress_ZipCode" AS "BillingAddress_ZipCode0", "c0"."BillingAddress_Country_Code" AS "BillingAddress_Country_Code0", "c0"."BillingAddress_Country_FullName" AS "BillingAddress_Country_FullName0" + FROM "Customer" AS "c" + CROSS JOIN "Customer" AS "c0" + ORDER BY "c"."Id", "c0"."Id" + LIMIT @__p_0 + ) AS "t" +) AS "t0" +"""); + } + + public override async Task Project_same_entity_with_struct_nested_complex_type_twice_with_pushdown(bool async) + { + await base.Project_same_entity_with_struct_nested_complex_type_twice_with_pushdown(async); + + AssertSql( +""" +SELECT "t"."Id", "t"."Name", "t"."BillingAddress_AddressLine1", "t"."BillingAddress_AddressLine2", "t"."BillingAddress_ZipCode", "t"."BillingAddress_Country_Code", "t"."BillingAddress_Country_FullName", "t"."ShippingAddress_AddressLine1", "t"."ShippingAddress_AddressLine2", "t"."ShippingAddress_ZipCode", "t"."ShippingAddress_Country_Code", "t"."ShippingAddress_Country_FullName", "t"."Id0", "t"."Name0", "t"."BillingAddress_AddressLine10", "t"."BillingAddress_AddressLine20", "t"."BillingAddress_ZipCode0", "t"."BillingAddress_Country_Code0", "t"."BillingAddress_Country_FullName0", "t"."ShippingAddress_AddressLine10", "t"."ShippingAddress_AddressLine20", "t"."ShippingAddress_ZipCode0", "t"."ShippingAddress_Country_Code0", "t"."ShippingAddress_Country_FullName0" +FROM ( + SELECT DISTINCT "v"."Id", "v"."Name", "v"."BillingAddress_AddressLine1", "v"."BillingAddress_AddressLine2", "v"."BillingAddress_ZipCode", "v"."BillingAddress_Country_Code", "v"."BillingAddress_Country_FullName", "v"."ShippingAddress_AddressLine1", "v"."ShippingAddress_AddressLine2", "v"."ShippingAddress_ZipCode", "v"."ShippingAddress_Country_Code", "v"."ShippingAddress_Country_FullName", "v0"."Id" AS "Id0", "v0"."Name" AS "Name0", "v0"."BillingAddress_AddressLine1" AS "BillingAddress_AddressLine10", "v0"."BillingAddress_AddressLine2" AS "BillingAddress_AddressLine20", "v0"."BillingAddress_ZipCode" AS "BillingAddress_ZipCode0", "v0"."BillingAddress_Country_Code" AS "BillingAddress_Country_Code0", "v0"."BillingAddress_Country_FullName" AS "BillingAddress_Country_FullName0", "v0"."ShippingAddress_AddressLine1" AS "ShippingAddress_AddressLine10", "v0"."ShippingAddress_AddressLine2" AS "ShippingAddress_AddressLine20", "v0"."ShippingAddress_ZipCode" AS "ShippingAddress_ZipCode0", "v0"."ShippingAddress_Country_Code" AS "ShippingAddress_Country_Code0", "v0"."ShippingAddress_Country_FullName" AS "ShippingAddress_Country_FullName0" + FROM "ValuedCustomer" AS "v" + CROSS JOIN "ValuedCustomer" AS "v0" +) AS "t" +"""); + } + + public override async Task Project_same_struct_nested_complex_type_twice_with_pushdown(bool async) + { + await base.Project_same_struct_nested_complex_type_twice_with_pushdown(async); + + AssertSql( +""" +SELECT "t"."BillingAddress_AddressLine1", "t"."BillingAddress_AddressLine2", "t"."BillingAddress_ZipCode", "t"."BillingAddress_Country_Code", "t"."BillingAddress_Country_FullName", "t"."BillingAddress_AddressLine10", "t"."BillingAddress_AddressLine20", "t"."BillingAddress_ZipCode0", "t"."BillingAddress_Country_Code0", "t"."BillingAddress_Country_FullName0" +FROM ( + SELECT DISTINCT "v"."BillingAddress_AddressLine1", "v"."BillingAddress_AddressLine2", "v"."BillingAddress_ZipCode", "v"."BillingAddress_Country_Code", "v"."BillingAddress_Country_FullName", "v0"."BillingAddress_AddressLine1" AS "BillingAddress_AddressLine10", "v0"."BillingAddress_AddressLine2" AS "BillingAddress_AddressLine20", "v0"."BillingAddress_ZipCode" AS "BillingAddress_ZipCode0", "v0"."BillingAddress_Country_Code" AS "BillingAddress_Country_Code0", "v0"."BillingAddress_Country_FullName" AS "BillingAddress_Country_FullName0" + FROM "ValuedCustomer" AS "v" + CROSS JOIN "ValuedCustomer" AS "v0" +) AS "t" +"""); + } + + public override async Task Project_same_entity_with_struct_nested_complex_type_twice_with_double_pushdown(bool async) + { + await base.Project_same_entity_with_struct_nested_complex_type_twice_with_double_pushdown(async); + + AssertSql( +""" +@__p_0='50' + +SELECT "t0"."Id", "t0"."Name", "t0"."BillingAddress_AddressLine1", "t0"."BillingAddress_AddressLine2", "t0"."BillingAddress_ZipCode", "t0"."BillingAddress_Country_Code", "t0"."BillingAddress_Country_FullName", "t0"."ShippingAddress_AddressLine1", "t0"."ShippingAddress_AddressLine2", "t0"."ShippingAddress_ZipCode", "t0"."ShippingAddress_Country_Code", "t0"."ShippingAddress_Country_FullName", "t0"."Id0", "t0"."Name0", "t0"."BillingAddress_AddressLine10", "t0"."BillingAddress_AddressLine20", "t0"."BillingAddress_ZipCode0", "t0"."BillingAddress_Country_Code0", "t0"."BillingAddress_Country_FullName0", "t0"."ShippingAddress_AddressLine10", "t0"."ShippingAddress_AddressLine20", "t0"."ShippingAddress_ZipCode0", "t0"."ShippingAddress_Country_Code0", "t0"."ShippingAddress_Country_FullName0" +FROM ( + SELECT DISTINCT "t"."Id", "t"."Name", "t"."BillingAddress_AddressLine1", "t"."BillingAddress_AddressLine2", "t"."BillingAddress_ZipCode", "t"."BillingAddress_Country_Code", "t"."BillingAddress_Country_FullName", "t"."ShippingAddress_AddressLine1", "t"."ShippingAddress_AddressLine2", "t"."ShippingAddress_ZipCode", "t"."ShippingAddress_Country_Code", "t"."ShippingAddress_Country_FullName", "t"."Id0", "t"."Name0", "t"."BillingAddress_AddressLine10", "t"."BillingAddress_AddressLine20", "t"."BillingAddress_ZipCode0", "t"."BillingAddress_Country_Code0", "t"."BillingAddress_Country_FullName0", "t"."ShippingAddress_AddressLine10", "t"."ShippingAddress_AddressLine20", "t"."ShippingAddress_ZipCode0", "t"."ShippingAddress_Country_Code0", "t"."ShippingAddress_Country_FullName0" + FROM ( + SELECT "v"."Id", "v"."Name", "v"."BillingAddress_AddressLine1", "v"."BillingAddress_AddressLine2", "v"."BillingAddress_ZipCode", "v"."BillingAddress_Country_Code", "v"."BillingAddress_Country_FullName", "v"."ShippingAddress_AddressLine1", "v"."ShippingAddress_AddressLine2", "v"."ShippingAddress_ZipCode", "v"."ShippingAddress_Country_Code", "v"."ShippingAddress_Country_FullName", "v0"."Id" AS "Id0", "v0"."Name" AS "Name0", "v0"."BillingAddress_AddressLine1" AS "BillingAddress_AddressLine10", "v0"."BillingAddress_AddressLine2" AS "BillingAddress_AddressLine20", "v0"."BillingAddress_ZipCode" AS "BillingAddress_ZipCode0", "v0"."BillingAddress_Country_Code" AS "BillingAddress_Country_Code0", "v0"."BillingAddress_Country_FullName" AS "BillingAddress_Country_FullName0", "v0"."ShippingAddress_AddressLine1" AS "ShippingAddress_AddressLine10", "v0"."ShippingAddress_AddressLine2" AS "ShippingAddress_AddressLine20", "v0"."ShippingAddress_ZipCode" AS "ShippingAddress_ZipCode0", "v0"."ShippingAddress_Country_Code" AS "ShippingAddress_Country_Code0", "v0"."ShippingAddress_Country_FullName" AS "ShippingAddress_Country_FullName0" + FROM "ValuedCustomer" AS "v" + CROSS JOIN "ValuedCustomer" AS "v0" + ORDER BY "v"."Id", "v0"."Id" + LIMIT @__p_0 + ) AS "t" +) AS "t0" +"""); + } + + public override async Task Project_same_struct_nested_complex_type_twice_with_double_pushdown(bool async) + { + await base.Project_same_struct_nested_complex_type_twice_with_double_pushdown(async); + + AssertSql( +""" +@__p_0='50' + +SELECT "t0"."BillingAddress_AddressLine1", "t0"."BillingAddress_AddressLine2", "t0"."BillingAddress_ZipCode", "t0"."BillingAddress_Country_Code", "t0"."BillingAddress_Country_FullName", "t0"."BillingAddress_AddressLine10", "t0"."BillingAddress_AddressLine20", "t0"."BillingAddress_ZipCode0", "t0"."BillingAddress_Country_Code0", "t0"."BillingAddress_Country_FullName0" +FROM ( + SELECT DISTINCT "t"."BillingAddress_AddressLine1", "t"."BillingAddress_AddressLine2", "t"."BillingAddress_ZipCode", "t"."BillingAddress_Country_Code", "t"."BillingAddress_Country_FullName", "t"."BillingAddress_AddressLine10", "t"."BillingAddress_AddressLine20", "t"."BillingAddress_ZipCode0", "t"."BillingAddress_Country_Code0", "t"."BillingAddress_Country_FullName0" + FROM ( + SELECT "v"."BillingAddress_AddressLine1", "v"."BillingAddress_AddressLine2", "v"."BillingAddress_ZipCode", "v"."BillingAddress_Country_Code", "v"."BillingAddress_Country_FullName", "v0"."BillingAddress_AddressLine1" AS "BillingAddress_AddressLine10", "v0"."BillingAddress_AddressLine2" AS "BillingAddress_AddressLine20", "v0"."BillingAddress_ZipCode" AS "BillingAddress_ZipCode0", "v0"."BillingAddress_Country_Code" AS "BillingAddress_Country_Code0", "v0"."BillingAddress_Country_FullName" AS "BillingAddress_Country_FullName0" + FROM "ValuedCustomer" AS "v" + CROSS JOIN "ValuedCustomer" AS "v0" + ORDER BY "v"."Id", "v0"."Id" + LIMIT @__p_0 + ) AS "t" +) AS "t0" +"""); + } + + public override async Task Union_of_same_entity_with_nested_complex_type_projected_twice_with_pushdown(bool async) + { + await base.Union_of_same_entity_with_nested_complex_type_projected_twice_with_pushdown(async); + + AssertSql( +""" +@__p_0='50' + +SELECT "t"."Id", "t"."Name", "t"."BillingAddress_AddressLine1", "t"."BillingAddress_AddressLine2", "t"."BillingAddress_ZipCode", "t"."BillingAddress_Country_Code", "t"."BillingAddress_Country_FullName", "t"."ShippingAddress_AddressLine1", "t"."ShippingAddress_AddressLine2", "t"."ShippingAddress_ZipCode", "t"."ShippingAddress_Country_Code", "t"."ShippingAddress_Country_FullName", "t"."Id0", "t"."Name0", "t"."BillingAddress_AddressLine10", "t"."BillingAddress_AddressLine20", "t"."BillingAddress_ZipCode0", "t"."BillingAddress_Country_Code0", "t"."BillingAddress_Country_FullName0", "t"."ShippingAddress_AddressLine10", "t"."ShippingAddress_AddressLine20", "t"."ShippingAddress_ZipCode0", "t"."ShippingAddress_Country_Code0", "t"."ShippingAddress_Country_FullName0" +FROM ( + SELECT "c"."Id", "c"."Name", "c"."BillingAddress_AddressLine1", "c"."BillingAddress_AddressLine2", "c"."BillingAddress_ZipCode", "c"."BillingAddress_Country_Code", "c"."BillingAddress_Country_FullName", "c"."ShippingAddress_AddressLine1", "c"."ShippingAddress_AddressLine2", "c"."ShippingAddress_ZipCode", "c"."ShippingAddress_Country_Code", "c"."ShippingAddress_Country_FullName", "c0"."Id" AS "Id0", "c0"."Name" AS "Name0", "c0"."BillingAddress_AddressLine1" AS "BillingAddress_AddressLine10", "c0"."BillingAddress_AddressLine2" AS "BillingAddress_AddressLine20", "c0"."BillingAddress_ZipCode" AS "BillingAddress_ZipCode0", "c0"."BillingAddress_Country_Code" AS "BillingAddress_Country_Code0", "c0"."BillingAddress_Country_FullName" AS "BillingAddress_Country_FullName0", "c0"."ShippingAddress_AddressLine1" AS "ShippingAddress_AddressLine10", "c0"."ShippingAddress_AddressLine2" AS "ShippingAddress_AddressLine20", "c0"."ShippingAddress_ZipCode" AS "ShippingAddress_ZipCode0", "c0"."ShippingAddress_Country_Code" AS "ShippingAddress_Country_Code0", "c0"."ShippingAddress_Country_FullName" AS "ShippingAddress_Country_FullName0" + FROM "Customer" AS "c" + CROSS JOIN "Customer" AS "c0" + UNION + SELECT "c1"."Id", "c1"."Name", "c1"."BillingAddress_AddressLine1", "c1"."BillingAddress_AddressLine2", "c1"."BillingAddress_ZipCode", "c1"."BillingAddress_Country_Code", "c1"."BillingAddress_Country_FullName", "c1"."ShippingAddress_AddressLine1", "c1"."ShippingAddress_AddressLine2", "c1"."ShippingAddress_ZipCode", "c1"."ShippingAddress_Country_Code", "c1"."ShippingAddress_Country_FullName", "c2"."Id" AS "Id0", "c2"."Name" AS "Name0", "c2"."BillingAddress_AddressLine1" AS "BillingAddress_AddressLine10", "c2"."BillingAddress_AddressLine2" AS "BillingAddress_AddressLine20", "c2"."BillingAddress_ZipCode" AS "BillingAddress_ZipCode0", "c2"."BillingAddress_Country_Code" AS "BillingAddress_Country_Code0", "c2"."BillingAddress_Country_FullName" AS "BillingAddress_Country_FullName0", "c2"."ShippingAddress_AddressLine1" AS "ShippingAddress_AddressLine10", "c2"."ShippingAddress_AddressLine2" AS "ShippingAddress_AddressLine20", "c2"."ShippingAddress_ZipCode" AS "ShippingAddress_ZipCode0", "c2"."ShippingAddress_Country_Code" AS "ShippingAddress_Country_Code0", "c2"."ShippingAddress_Country_FullName" AS "ShippingAddress_Country_FullName0" + FROM "Customer" AS "c1" + CROSS JOIN "Customer" AS "c2" +) AS "t" +ORDER BY "t"."Id", "t"."Id0" +LIMIT @__p_0 +"""); + } + + public override async Task Union_of_same_entity_with_nested_complex_type_projected_twice_with_double_pushdown(bool async) + { + await base.Union_of_same_entity_with_nested_complex_type_projected_twice_with_double_pushdown(async); + + AssertSql( +""" +@__p_0='50' + +SELECT "t1"."Id", "t1"."Name", "t1"."BillingAddress_AddressLine1", "t1"."BillingAddress_AddressLine2", "t1"."BillingAddress_ZipCode", "t1"."BillingAddress_Country_Code", "t1"."BillingAddress_Country_FullName", "t1"."ShippingAddress_AddressLine1", "t1"."ShippingAddress_AddressLine2", "t1"."ShippingAddress_ZipCode", "t1"."ShippingAddress_Country_Code", "t1"."ShippingAddress_Country_FullName", "t1"."Id0", "t1"."Name0", "t1"."BillingAddress_AddressLine10", "t1"."BillingAddress_AddressLine20", "t1"."BillingAddress_ZipCode0", "t1"."BillingAddress_Country_Code0", "t1"."BillingAddress_Country_FullName0", "t1"."ShippingAddress_AddressLine10", "t1"."ShippingAddress_AddressLine20", "t1"."ShippingAddress_ZipCode0", "t1"."ShippingAddress_Country_Code0", "t1"."ShippingAddress_Country_FullName0" +FROM ( + SELECT DISTINCT "t0"."Id", "t0"."Name", "t0"."BillingAddress_AddressLine1", "t0"."BillingAddress_AddressLine2", "t0"."BillingAddress_ZipCode", "t0"."BillingAddress_Country_Code", "t0"."BillingAddress_Country_FullName", "t0"."ShippingAddress_AddressLine1", "t0"."ShippingAddress_AddressLine2", "t0"."ShippingAddress_ZipCode", "t0"."ShippingAddress_Country_Code", "t0"."ShippingAddress_Country_FullName", "t0"."Id0", "t0"."Name0", "t0"."BillingAddress_AddressLine10", "t0"."BillingAddress_AddressLine20", "t0"."BillingAddress_ZipCode0", "t0"."BillingAddress_Country_Code0", "t0"."BillingAddress_Country_FullName0", "t0"."ShippingAddress_AddressLine10", "t0"."ShippingAddress_AddressLine20", "t0"."ShippingAddress_ZipCode0", "t0"."ShippingAddress_Country_Code0", "t0"."ShippingAddress_Country_FullName0" + FROM ( + SELECT "t"."Id", "t"."Name", "t"."BillingAddress_AddressLine1", "t"."BillingAddress_AddressLine2", "t"."BillingAddress_ZipCode", "t"."BillingAddress_Country_Code", "t"."BillingAddress_Country_FullName", "t"."ShippingAddress_AddressLine1", "t"."ShippingAddress_AddressLine2", "t"."ShippingAddress_ZipCode", "t"."ShippingAddress_Country_Code", "t"."ShippingAddress_Country_FullName", "t"."Id0", "t"."Name0", "t"."BillingAddress_AddressLine10", "t"."BillingAddress_AddressLine20", "t"."BillingAddress_ZipCode0", "t"."BillingAddress_Country_Code0", "t"."BillingAddress_Country_FullName0", "t"."ShippingAddress_AddressLine10", "t"."ShippingAddress_AddressLine20", "t"."ShippingAddress_ZipCode0", "t"."ShippingAddress_Country_Code0", "t"."ShippingAddress_Country_FullName0" + FROM ( + SELECT "c"."Id", "c"."Name", "c"."BillingAddress_AddressLine1", "c"."BillingAddress_AddressLine2", "c"."BillingAddress_ZipCode", "c"."BillingAddress_Country_Code", "c"."BillingAddress_Country_FullName", "c"."ShippingAddress_AddressLine1", "c"."ShippingAddress_AddressLine2", "c"."ShippingAddress_ZipCode", "c"."ShippingAddress_Country_Code", "c"."ShippingAddress_Country_FullName", "c0"."Id" AS "Id0", "c0"."Name" AS "Name0", "c0"."BillingAddress_AddressLine1" AS "BillingAddress_AddressLine10", "c0"."BillingAddress_AddressLine2" AS "BillingAddress_AddressLine20", "c0"."BillingAddress_ZipCode" AS "BillingAddress_ZipCode0", "c0"."BillingAddress_Country_Code" AS "BillingAddress_Country_Code0", "c0"."BillingAddress_Country_FullName" AS "BillingAddress_Country_FullName0", "c0"."ShippingAddress_AddressLine1" AS "ShippingAddress_AddressLine10", "c0"."ShippingAddress_AddressLine2" AS "ShippingAddress_AddressLine20", "c0"."ShippingAddress_ZipCode" AS "ShippingAddress_ZipCode0", "c0"."ShippingAddress_Country_Code" AS "ShippingAddress_Country_Code0", "c0"."ShippingAddress_Country_FullName" AS "ShippingAddress_Country_FullName0" + FROM "Customer" AS "c" + CROSS JOIN "Customer" AS "c0" + UNION + SELECT "c1"."Id", "c1"."Name", "c1"."BillingAddress_AddressLine1", "c1"."BillingAddress_AddressLine2", "c1"."BillingAddress_ZipCode", "c1"."BillingAddress_Country_Code", "c1"."BillingAddress_Country_FullName", "c1"."ShippingAddress_AddressLine1", "c1"."ShippingAddress_AddressLine2", "c1"."ShippingAddress_ZipCode", "c1"."ShippingAddress_Country_Code", "c1"."ShippingAddress_Country_FullName", "c2"."Id" AS "Id0", "c2"."Name" AS "Name0", "c2"."BillingAddress_AddressLine1" AS "BillingAddress_AddressLine10", "c2"."BillingAddress_AddressLine2" AS "BillingAddress_AddressLine20", "c2"."BillingAddress_ZipCode" AS "BillingAddress_ZipCode0", "c2"."BillingAddress_Country_Code" AS "BillingAddress_Country_Code0", "c2"."BillingAddress_Country_FullName" AS "BillingAddress_Country_FullName0", "c2"."ShippingAddress_AddressLine1" AS "ShippingAddress_AddressLine10", "c2"."ShippingAddress_AddressLine2" AS "ShippingAddress_AddressLine20", "c2"."ShippingAddress_ZipCode" AS "ShippingAddress_ZipCode0", "c2"."ShippingAddress_Country_Code" AS "ShippingAddress_Country_Code0", "c2"."ShippingAddress_Country_FullName" AS "ShippingAddress_Country_FullName0" + FROM "Customer" AS "c1" + CROSS JOIN "Customer" AS "c2" + ) AS "t" + ORDER BY "t"."Id", "t"."Id0" + LIMIT @__p_0 + ) AS "t0" +) AS "t1" +ORDER BY "t1"."Id", "t1"."Id0" +LIMIT @__p_0 +"""); + } + + public override async Task Union_of_same_nested_complex_type_projected_twice_with_pushdown(bool async) + { + await base.Union_of_same_nested_complex_type_projected_twice_with_pushdown(async); + + AssertSql( +""" +@__p_0='50' + +SELECT "t"."BillingAddress_AddressLine1", "t"."BillingAddress_AddressLine2", "t"."BillingAddress_ZipCode", "t"."BillingAddress_Country_Code", "t"."BillingAddress_Country_FullName", "t"."BillingAddress_AddressLine10", "t"."BillingAddress_AddressLine20", "t"."BillingAddress_ZipCode0", "t"."BillingAddress_Country_Code0", "t"."BillingAddress_Country_FullName0" +FROM ( + SELECT "c"."BillingAddress_AddressLine1", "c"."BillingAddress_AddressLine2", "c"."BillingAddress_ZipCode", "c"."BillingAddress_Country_Code", "c"."BillingAddress_Country_FullName", "c0"."BillingAddress_AddressLine1" AS "BillingAddress_AddressLine10", "c0"."BillingAddress_AddressLine2" AS "BillingAddress_AddressLine20", "c0"."BillingAddress_ZipCode" AS "BillingAddress_ZipCode0", "c0"."BillingAddress_Country_Code" AS "BillingAddress_Country_Code0", "c0"."BillingAddress_Country_FullName" AS "BillingAddress_Country_FullName0" + FROM "Customer" AS "c" + CROSS JOIN "Customer" AS "c0" + UNION + SELECT "c1"."BillingAddress_AddressLine1", "c1"."BillingAddress_AddressLine2", "c1"."BillingAddress_ZipCode", "c1"."BillingAddress_Country_Code", "c1"."BillingAddress_Country_FullName", "c2"."BillingAddress_AddressLine1" AS "BillingAddress_AddressLine10", "c2"."BillingAddress_AddressLine2" AS "BillingAddress_AddressLine20", "c2"."BillingAddress_ZipCode" AS "BillingAddress_ZipCode0", "c2"."BillingAddress_Country_Code" AS "BillingAddress_Country_Code0", "c2"."BillingAddress_Country_FullName" AS "BillingAddress_Country_FullName0" + FROM "Customer" AS "c1" + CROSS JOIN "Customer" AS "c2" +) AS "t" +ORDER BY "t"."BillingAddress_ZipCode", "t"."BillingAddress_ZipCode0" +LIMIT @__p_0 +"""); + } + + public override async Task Union_of_same_nested_complex_type_projected_twice_with_double_pushdown(bool async) + { + await base.Union_of_same_nested_complex_type_projected_twice_with_double_pushdown(async); + + AssertSql( +""" +@__p_0='50' + +SELECT "t1"."BillingAddress_AddressLine1", "t1"."BillingAddress_AddressLine2", "t1"."BillingAddress_ZipCode", "t1"."BillingAddress_Country_Code", "t1"."BillingAddress_Country_FullName", "t1"."BillingAddress_AddressLine10", "t1"."BillingAddress_AddressLine20", "t1"."BillingAddress_ZipCode0", "t1"."BillingAddress_Country_Code0", "t1"."BillingAddress_Country_FullName0" +FROM ( + SELECT DISTINCT "t0"."BillingAddress_AddressLine1", "t0"."BillingAddress_AddressLine2", "t0"."BillingAddress_ZipCode", "t0"."BillingAddress_Country_Code", "t0"."BillingAddress_Country_FullName", "t0"."BillingAddress_AddressLine10", "t0"."BillingAddress_AddressLine20", "t0"."BillingAddress_ZipCode0", "t0"."BillingAddress_Country_Code0", "t0"."BillingAddress_Country_FullName0" + FROM ( + SELECT "t"."BillingAddress_AddressLine1", "t"."BillingAddress_AddressLine2", "t"."BillingAddress_ZipCode", "t"."BillingAddress_Country_Code", "t"."BillingAddress_Country_FullName", "t"."BillingAddress_AddressLine10", "t"."BillingAddress_AddressLine20", "t"."BillingAddress_ZipCode0", "t"."BillingAddress_Country_Code0", "t"."BillingAddress_Country_FullName0" + FROM ( + SELECT "c"."BillingAddress_AddressLine1", "c"."BillingAddress_AddressLine2", "c"."BillingAddress_ZipCode", "c"."BillingAddress_Country_Code", "c"."BillingAddress_Country_FullName", "c0"."BillingAddress_AddressLine1" AS "BillingAddress_AddressLine10", "c0"."BillingAddress_AddressLine2" AS "BillingAddress_AddressLine20", "c0"."BillingAddress_ZipCode" AS "BillingAddress_ZipCode0", "c0"."BillingAddress_Country_Code" AS "BillingAddress_Country_Code0", "c0"."BillingAddress_Country_FullName" AS "BillingAddress_Country_FullName0" + FROM "Customer" AS "c" + CROSS JOIN "Customer" AS "c0" + UNION + SELECT "c1"."BillingAddress_AddressLine1", "c1"."BillingAddress_AddressLine2", "c1"."BillingAddress_ZipCode", "c1"."BillingAddress_Country_Code", "c1"."BillingAddress_Country_FullName", "c2"."BillingAddress_AddressLine1" AS "BillingAddress_AddressLine10", "c2"."BillingAddress_AddressLine2" AS "BillingAddress_AddressLine20", "c2"."BillingAddress_ZipCode" AS "BillingAddress_ZipCode0", "c2"."BillingAddress_Country_Code" AS "BillingAddress_Country_Code0", "c2"."BillingAddress_Country_FullName" AS "BillingAddress_Country_FullName0" + FROM "Customer" AS "c1" + CROSS JOIN "Customer" AS "c2" + ) AS "t" + ORDER BY "t"."BillingAddress_ZipCode", "t"."BillingAddress_ZipCode0" + LIMIT @__p_0 + ) AS "t0" +) AS "t1" +ORDER BY "t1"."BillingAddress_ZipCode", "t1"."BillingAddress_ZipCode0" +LIMIT @__p_0 +"""); + } + + public override async Task Same_entity_with_complex_type_projected_twice_with_pushdown_as_part_of_another_projection(bool async) + => Assert.Equal( + SqliteStrings.ApplyNotSupported, + (await Assert.ThrowsAsync( + () => base.Same_entity_with_complex_type_projected_twice_with_pushdown_as_part_of_another_projection(async))).Message); + + public override async Task Same_complex_type_projected_twice_with_pushdown_as_part_of_another_projection(bool async) + => Assert.Equal( + SqliteStrings.ApplyNotSupported, + (await Assert.ThrowsAsync( + () => base.Same_complex_type_projected_twice_with_pushdown_as_part_of_another_projection(async))).Message); + [ConditionalFact] public virtual void Check_all_tests_overridden() => TestHelpers.AssertAllMethodsOverridden(GetType()); diff --git a/test/EFCore.Tests/Infrastructure/ModelValidatorTestBase.cs b/test/EFCore.Tests/Infrastructure/ModelValidatorTestBase.cs index e6d2953e258..ddee4e89e0b 100644 --- a/test/EFCore.Tests/Infrastructure/ModelValidatorTestBase.cs +++ b/test/EFCore.Tests/Infrastructure/ModelValidatorTestBase.cs @@ -118,7 +118,43 @@ protected class Generic : Abstract { } - public class SampleEntity +#nullable enable + protected class BaseEntity + { + public int Id { get; set; } + } + + protected class ChildA : BaseEntity + { + public OwnedType OwnedType { get; set; } = null!; + } + + protected class ChildB : BaseEntity + { + } + + protected class ChildC : BaseEntity + { + } + + protected class ChildD : BaseEntity + { + } + + [Owned] + protected class OwnedType + { + public NestedOwnedType NestedOwnedType { get; set; } = null!; + } + + [Owned] + protected class NestedOwnedType + { + } + +#nullable restore + + protected class SampleEntity { public int Id { get; set; } public int Number { get; set; } @@ -131,29 +167,29 @@ public class SampleEntity public ICollection OtherSamples { get; set; } } - public class AnotherSampleEntity + protected class AnotherSampleEntity { public int Id { get; set; } public ReferencedEntity ReferencedEntity { get; set; } } - public class ReferencedEntity + protected class ReferencedEntity { public int Id { get; set; } public int SampleEntityId { get; set; } } - public class SampleEntityMinimal + protected class SampleEntityMinimal { public int Id { get; set; } public ReferencedEntityMinimal ReferencedEntity { get; set; } } - public class ReferencedEntityMinimal + protected class ReferencedEntityMinimal { } - public class AnotherSampleEntityMinimal + protected class AnotherSampleEntityMinimal { public int Id { get; set; } public ReferencedEntityMinimal ReferencedEntity { get; set; } @@ -373,7 +409,7 @@ protected class DependentFour public PrincipalFour PrincipalFour { get; set; } } - public class Blog + protected class Blog { public int BlogId { get; set; } public bool IsDeleted { get; set; } @@ -381,14 +417,14 @@ public class Blog public List BlogOwnedEntities { get; set; } } - public class BlogOwnedEntity + protected class BlogOwnedEntity { public int BlogOwnedEntityId { get; set; } public int BlogId { get; set; } public Blog Blog { get; set; } } - public class Post + protected class Post { public int PostId { get; set; } public int BlogId { get; set; } @@ -397,13 +433,13 @@ public class Post public Blog Blog { get; set; } } - public class PicturePost : Post + protected class PicturePost : Post { public string PictureUrl { get; set; } public List Pictures { get; set; } } - public class Picture + protected class Picture { public int PictureId { get; set; } public bool IsDeleted { get; set; }