diff --git a/.vsts.pipelines/builds/ci.yml b/.vsts.pipelines/builds/ci.yml index 977f0bc46d..3dc0926931 100644 --- a/.vsts.pipelines/builds/ci.yml +++ b/.vsts.pipelines/builds/ci.yml @@ -38,7 +38,12 @@ stages: - job: SourceBuild_Managed displayName: Source-Build (Managed) pool: - vmImage: ubuntu-20.04 + ${{ if eq(variables['System.TeamProject'], 'public') }}: + name: NetCore-Svc-Public + demands: ImageOverride -equals Build.Ubuntu.1804.Amd64.Open + ${{ if eq(variables['System.TeamProject'], 'internal') }}: + name: NetCore1ESPool-Svc-Internal + demands: ImageOverride -equals Build.Ubuntu.1804.Amd64 container: 'mcr.microsoft.com/dotnet-buildtools/prereqs:centos-7-3e800f1-20190501005343' workspace: clean: all @@ -50,24 +55,6 @@ stages: submodules: true - template: /eng/common/templates/steps/source-build.yml - # Jobs that are useful to validate the build runs on the platform, but shouldn't run in the - # official build. In dotnet/source-build, we should *only* produce the intermediate nupkg in - # the official build. Our source-built packages are not necessarily the same as the versions - # on nuget.org and must not make their way into the channel. (E.g. ours are not signed.) - - ${{ if or(eq(variables['System.TeamProject'], 'public'), in(variables['Build.Reason'], 'PullRequest')) }}: - - - job: macOS - pool: - name: Hosted macOS - variables: - - name: _BuildConfig - value: Release - steps: - - checkout: self - submodules: true - - script: ./build.sh --ci --restore --build --pack --publish - displayName: Build - - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - template: /eng/common/templates/post-build/post-build.yml parameters: diff --git a/Documentation/sourcebuild-in-repos/README.md b/Documentation/sourcebuild-in-repos/README.md new file mode 100644 index 0000000000..cd29555615 --- /dev/null +++ b/Documentation/sourcebuild-in-repos/README.md @@ -0,0 +1,12 @@ +# Repo owner's handbook for source-build + +These docs aim to be a collection of useful information for repo owners who work +with and participate in source-build. + +## Index + +* [Adding new dependencies](new-dependencies.md) +* [Updating dependencies](update-dependencies.md) +* [Considerations when adding features](adding-features.md) +* [General notes on the source-build build](build-info.md) +* [Adding new repositories](new-repo.md) diff --git a/Documentation/sourcebuild-in-repos/adding-features.md b/Documentation/sourcebuild-in-repos/adding-features.md new file mode 100644 index 0000000000..ae98d3d7b0 --- /dev/null +++ b/Documentation/sourcebuild-in-repos/adding-features.md @@ -0,0 +1,43 @@ +# New features + +New features are great! They also have source-build considerations though. +.NET is no longer just multi-platform but also multi-distribution: there's +a Linux SDK produced by Microsoft, but there's also a Linux SDK produced +by Red Hat, one produced by Fedora, and one that anyone in the community +can build himself from source. + +## Things to consider + +New features, or expansions of current features, should act sensibly +across Windows, Linux, and OSX. This also involves taking into account +the limitations and conventions of the different platforms - for instance, +on Linux, it's typical for the .NET SDK to be installed by root and +therefore be unchangeable by normal users, so installing global tools +needs to take into account the fact that the user might not be able to +add anything to the SDK directories (see +[this docker global tools issue](https://github.com/dotnet/dotnet-docker/issues/520) +and [a similar installer issue](https://github.com/dotnet/installer/issues/7069)). + +New features also need to be compatible across all distributions of +.NET - for instance, Fedora and Debian cannot distribute *any* +non-open-source code, even samples, tests, or minor additions to +licenses like restricting the field that the code can be used in. +This includes any code or packages that are used to build product +code as well. Microsoft generally prefers the MIT license. + +One example of a licensing issue that source-build found was with +dotnet-check and dotnet-format: these tools were brought into the +product build but had a dependency that was not licensed compatibly - +even without the source-build aspect. We had to scramble to replace +the dependency before the release. Dependencies need to be carefully +checked for license compatibility as well. + +## Resources + +Fedora's approved open-source licenses can be found +[on their wiki](https://fedoraproject.org/wiki/Licensing:Main#Good_Licenses), +or you can check on the [OSI-approved list of licenses](https://opensource.org/licenses/alphabetical). + +If you would like some of our distribution maintainer partners to +review your new feature you can ping @dotnet/distro-maintainers +on a PR or issue. \ No newline at end of file diff --git a/Documentation/sourcebuild-in-repos/build-info.md b/Documentation/sourcebuild-in-repos/build-info.md new file mode 100644 index 0000000000..32d474842c --- /dev/null +++ b/Documentation/sourcebuild-in-repos/build-info.md @@ -0,0 +1,50 @@ +# Source-build build info + +This is a collection of notes about how source-build can differ in general +from your repo's build and what kind of issues that can create. + +## Single-version and single-RID build + +Source-build is required to build on a single machine with no internet +access. This means that we build targeting a single RID, usually the +non-portable RID for the build machines (like rhel.7-x64). We do +support building portable (linux-x64) as well - this is useful for +bootstrapping new distributions. + +Source-build cannot build with any *prebuilts*. This is our term for +any package that comes from *outside the current source-build*. This means +that everything that ships out of source-build and everything that is used to +build those shipping products must come from the source-build in progress. +Packages from nuget.org, Microsoft builds, or other unrelated source-builds +cannot be used in source-build except in a limited bootstrapping process. + +Source-build supplies a *previously-source-built* set of packages for this +bootstrapping process. This is one way we have of breaking cycles in the +build. However, none of these packages can make it to the final build output. +This also means that your repo should be buildable with the immediately +previous version of the SDK than you are building for; i.e., if you are +building for 6.0.103, everything should be buildable with the 6.0.102 SDK. + +We also only build one version of each repo. This means that if your repo +turns production of some packages on and off, for instance, if you only +produce packages if they are changed, source-build will need a workaround +to force all packages to be produced. Additionally, we can only supply +one version of each package to a repo. This is injected into the +`$({PackageName}PackageVersion)` variables, e.g. SystemReflectionMetadataPackageVersion. +One exception is reference-only packages - +[dotnet/source-build-reference-packages](https://github.com/dotnet/source-build-reference-packages) +produces multiple versions and these can be hard-coded or use a +`$(ReferenceVersion)` property or similar if you don't +need source-build to change them. + +## Platform-specific packages + +Packages that require components or packages only available on some other +operating system than the building OS cannot be built in source-build. +These should use `true` or +other options to be excluded from the source-build. For instance, if a +project depends on a package that can only be built on Windows, it will need +to be disabled or worked around in source-build. As an example, +[Roslyn removes](https://github.com/dotnet/roslyn/blob/b999a65c8b0feeccb2b58da3d7a6e80e5f08feab/src/Workspaces/Core/Portable/Storage/PersistentStorageExtensions.cs#L23) +a small performance improvement when building for source-build because it +requires a component that isn't available. \ No newline at end of file diff --git a/Documentation/sourcebuild-in-repos/img/ci-job.png b/Documentation/sourcebuild-in-repos/img/ci-job.png new file mode 100644 index 0000000000..398208ca28 Binary files /dev/null and b/Documentation/sourcebuild-in-repos/img/ci-job.png differ diff --git a/Documentation/sourcebuild-in-repos/new-dependencies.md b/Documentation/sourcebuild-in-repos/new-dependencies.md new file mode 100644 index 0000000000..dde92c9a63 --- /dev/null +++ b/Documentation/sourcebuild-in-repos/new-dependencies.md @@ -0,0 +1,118 @@ +# Adding a new dependency + +## Basics + +When adding a new dependency, there are a few steps common between any type +of dependency. If you use darc, `darc add-dependency` will take care of +this process for you. Otherwise: + +1. Add the package and a default version to eng/Versions.props. The default + version property should be named `PackageVersion`, + e.g. `SystemCollectionsImmutablePackageVersion`. +1. Add the package and a repo reference to eng/Version.Details.xml. This + should include a SourceBuild metadata entry, looking something like: + +```xml + + https://github.com/dotnet/roslyn + 0167599e0e1634ea3ed8d0e41390a3c0d9b3e4e9 + + +``` + +1. Set up dependency flow from the foreign repo to your repo. + +In general, you should aim to use one version of each package. If you are +using a package as reference-only, it is possible to use multiple versions, +but only one implementation version of each package will be used - +source-build will override it to the version that is being built in this +version of the SDK. + +The source-build metadata is important - this tells source-build which repo +package contains the specific nupkg you want. + +Another uncommon case that can cause problems is when the repo `` +version and the source-built intermediate package version don't match. +In this case, a direct dependency should be added to the source-build +intermediate package and the CoherentParentDependency attribute should be +set to the repo that consumes the dependency. For an example, +see [installer's F# dependency](https://github.com/dotnet/installer/blob/ba1739a2363b1062f03ea386ec67174c6468d3b2/eng/Version.Details.xml#L128). +You can find the version needed by running `darc get-build` +or using [BAR](https://aka.ms/bar). + +## Adding dependencies + +Source build classifies dependencies in the following ways + +1. .NET - a dependency on a component from the .NET org - e.g. dotnet/runtime +1. Microsoft - a dependency on a component from the Microsoft org - e.g. microsoft/vstest +1. External - a dependency on a component outside of Microsoft/.NET - e.g. JamesNK/Newtonsoft.Json + +The following checklist can be used to determine how to handle each type of +dependency and the nuances it may have. + +1. Are you already using a package from the same repo and is the new + dependency already source-built? You can determine this by checking if + the repo has a SourceBuild leg building, e.g. runtime's is + `runtime-dev-innerloop (Build Linux x64 release SourceBuild)`. + + 1. This is the simplest case. You can add new dependencies from repos + you are already using freely. Note that you need to use the same version + for all packages from the repo. +1. Is the repo already built in source-build including the specific + package you want? + 1. Add the dependency using `darc add-dependency` as normal, then + add the [source-build metadata](#Basics) as above. +1. Is the repo already built in source-build but the specific package is not? + 1. There's probably an issue with source-building this package. Please + talk to a [source-build team member](https://github.com/orgs/dotnet/teams/source-build-internal) + about why that is and whether we can fix it. +1. Is this a repo that uses Arcade to build? + 1. Does the foreign repo depend on your repo, directly or indirectly? + i.e. would adding the dependency create a cycle? + 1. This isn't necessarily a deal-breaker - it can sometimes be worked + around with reference-only packages. Please contact a + [source-build team member](https://github.com/orgs/dotnet/teams/source-build-internal) + to discuss. + 1. Does the foreign repo publish to BAR? + 1. If not, please contact them to get them publishing to BAR + in an appropriate channel. + 1. If neither of these caveats apply you should be in good shape. + Follow the instructions under "Basics" above. +1. Dependencies that have no code (e.g. SDKs with just props and targets) can + usually be added using the [source-build text-only-package process](https://github.com/dotnet/source-build-reference-packages/tree/main/src/textOnlyPackages/src). + If the package is not already included there, you can [open a PR](https://github.com/dotnet/source-build-reference-packages/pulls) + or [file an issue](https://github.com/dotnet/source-build/issues/new/choose) + to include it. +1. Source-build has in the past used Arcade shims to allow non-Arcade repos + to build appropriate packages for source-build. Please + [log an issue](https://github.com/dotnet/source-build/issues/new/choose) + to determine if this is a workable approach for your foreign repo. +1. We build some external dependencies in the [dotnet/source-build-externals](https://github.com/dotnet/source-build-externals) + repo. Good targets for this generally have very few if any dependencies and + very simple build processes. + [Please log an issue](https://github.com/dotnet/source-build/issues/new/choose) + to get the process started. + +## Deciding what version to use + +- If you are using the package as reference-only and want the version to be + pinned, use a literal version number in the csprojs that reference the + project. You can also set up a reference-only package version variable + in eng/Versions.props, for instance `1.2.3` + in addition to `4.5.6`. + Also verify that the package is available in + [source-build-reference-packages](https://github.com/dotnet/source-build-reference-packages), + and if not, [file a source-build issue](https://github.com/dotnet/source-build/issues). +- If you are using the package in the actual build or want the version + to be updated whenever the foreign repo publishes to your channel, use + the version number property set up in eng/Versions.props. When performing + a source-build, the version number will get updated to the current + source-built version. +- If you are using an external or non-Arcade package, please coordinate as + much as possible with other teams using that package. Each + package-version is essentially maintained as a separate concern, so + something like repo A requiring Newtonsoft.Json 9.0.1 and repo B requiring + 12.0.2 essentially doubles the source-build team's work. \ No newline at end of file diff --git a/Documentation/sourcebuild-in-repos/new-repo.md b/Documentation/sourcebuild-in-repos/new-repo.md new file mode 100644 index 0000000000..2189f59690 --- /dev/null +++ b/Documentation/sourcebuild-in-repos/new-repo.md @@ -0,0 +1,335 @@ +# Adding a new repository to source build + +This document describes the overall process of onboarding new repos onto +arcade powered source build (ArPow). + +1. [Source Build Configuration](#source-build-configuration) +1. [Setup CI](#setup-ci) +1. [Add Downstream Dependency](#add-downstream-dependency) +1. [Validate](##validate) + +## Source Build Configuration + +Configuring source build involves setting up files in `eng/` that +determine the behavior of source build in the repo. + +These changes are all needed before source build will work: + +* [`eng/SourceBuild.props`](#engsourcebuildprops) - + Basic properties, such as repo name. +* [`eng/SourceBuildPrebuiltBaseline.xml`](#engsourcebuildprebuiltbaselinexml) - + Allow prebuilts. Until prebuilt detection is enabled/enforced, we allow all + prebuilts. +* [`eng/Version.Details.xml`](#engversiondetailsxml) - + Already exists, but modifications are needed to pull dependencies from + upstream [intermediate nupkgs](planning/arcade-powered-source-build/README.md#intermediate-nupkg-outputsinputs). + +See the following sections for details: + +### `eng/SourceBuild.props` + +```xml + + + + this-repo + true + + + +``` + +This file contains basic configuration used to restore the correct +dependencies (managed-only or not) and produce the right name for +the repo's [intermediate nupkg]. + +* `this-repo` should be the same as the repo's name on GitHub, + without the GitHub organization name. +* `SourceBuildManagedOnly` defaults to false if omitted. `true` means + that the repo doesn't produce any RID-specific artifacts like + `Microsoft.NETCore.App.Runtime.linux-x64`, only managed code. + +These two properties determine the name of the [intermediate nupkg]: +`Microsoft.SourceBuild.Intermediate.$(GitHubRepositoryName)[.$(RidSuffix)]`. + +It's possible more configuration will be required for specific repos. +`eng/SourceBuild.props`, similar to `eng/Build.props`, is a place to add extra +MSBuild code that can change the way source build behaves. + +### `eng/SourceBuildPrebuiltBaseline.xml` + +```xml + + + + + +``` + +This defines which prebuilt NuGet packages are allowed to be used during +the build. Initially, all package IDs and versions are permitted (`*/*`). + +The `*/*` glob means "any nupkg id, any version". It will be replaced with +more specific rules at a later phase once the repo level prebuilt detection +infrastructure is completed. The goal is to make it specific, so when a PR +introduces an unexpected prebuilt, PR validation will fail and let us resolve +the source-buildability issue before the PR gets merged. + +### Trying it out locally + +After setting up ArPow in a repo, running it locally is done by passing +`/p:ArcadeBuildFromSource=true` at the end of the usual arcade-based build +command for the repo. For example: + +```bash +./build.sh -c Release --restore --build --pack /p:ArcadeBuildFromSource=true -bl +``` + +> Note: [source build is not supported on Windows](https://github.com/dotnet/source-build/issues/1190), +only Linux and macOS. + +After running the build, source build artifacts will be in `artifacts/source-build`, +and the [intermediate nupkg] will be something like +`artifacts/packages/*/Microsoft.SourceBuild.Intermediate.*.nupkg`. + +The MSBuild binlog will be placed somewhere like: `artifacts/log/Debug/Build.binlog`. +However, this "outer" binlog doesn't contain the meat of the build: the +"inner" build runs inside an `Exec` task. The inner binlog will be written to: +`artifacts/source-build/self/src/artifacts/sourcebuild.binlog`. + +### Excluding components + +It is not always necessary or correct to include all repo components in +source build. Components should be excluded if they are not required for +the platform being source-built. Including them expands the source-build +graph and may not be possible because of licensing. +Examples include tests, Windows components (rmember source build currently +only supports Linux and macOS), etc. To exlcude these components +use the `DotNetBuildFromSource` msbuild property to conditionally exclude. + +```code +Condition="'$(DotNetBuildFromSource)' != 'true'" +``` + +### `eng/Version.Details.xml` + +```xml +... + + https://github.com/dotnet/runtime + a820ca1c4f9cb5892331e2624d3999c39161fe2a + + +... + + https://github.com/dotnet/source-build-reference-packages + def2e2c6dc5064319250e2868a041a3dc07f9579 + + +... +``` + +Dependency changes may include adding `SourceBuild` to existing dependency +elements, and adding a new `source-build-reference-packages` element. + +`SourceBuild` causes the source build targets in the Arcade SDK to download +[intermediate nupkg]s from the upstream repo's official build, rather than +using prebuilt binaries to fulfill the dependencies. Note that `RepoName` +is used to calculate the ID of the +[intermediate nupkg](planning/arcade-powered-source-build/README.md#intermediate-nupkg-outputsinputs): +the `Dependency` `Name` is ignored by source build. + +Building with the source-built versions of your dependencies also means that +any upstream repos will have been built in a source-build context, including +things projects that are excluded from the source-build. This can help you +find issues where your source-build depends on an upstream component that +isn't actually built in source-build. + +`ManagedOnly` determines whether a RID suffix is necessary on the +[intermediate nupkg](planning/arcade-powered-source-build/README.md#intermediate-nupkg-outputsinputs) +ID. For example, running source build on `dotnet/installer` with +`linux-x64` with the above example configuration will restore: + +* `Microsoft.SourceBuild.Intermediate.runtime.linux-x64` + * `.linux-x64` because `ManagedOnly` is not `true`. +* `Microsoft.SourceBuild.Intermediate.source-build-reference-packages` + * Ends with the `RepoName` without a suffix because `ManagedOnly="true"`. + +## Setup CI + +ArPow needs to run during official builds to create source build +[intermediate nupkg]s for the downstream repos that will consume them. +ArPow should also run in PR validation builds, to prevent regression +in the ArPow flow. + +The ArPow implementation can be activated with a single flag in the +ordinary Arcade SDK jobs template for easy consumption. If a repo can't +simply use the Arcade SDK jobs template, more granular templates are +also available. + +See +for the current template source code. The inline comments in the +`parameters:` section in those files are the most up to date docs, maintained +with higher priority than this general onboarding doc. + +### `eng/common/templates/jobs/jobs.yml` opt-in switch + +The simplest way to onboard. This approach applies if the repo already +uses the `eng/common/templates/jobs/jobs.yml` template. + +To opt in: + +1. Set `enableSourceBuild: true` + + Set `enableSourceBuild: true` in the template parameters. + + This should look something like [this sourcelink implementation:](https://github.com/dotnet/sourcelink/blob/dfe619dc722be42d475595c755c958afe6177554/azure-pipelines.yml#L40) + + ```yaml + stages: + - stage: build + displayName: Build + jobs: + - template: /eng/common/templates/jobs/jobs.yml + parameters: + enablePublishUsingPipelines: true + enablePublishBuildArtifacts: true + enablePublishBuildAssets: true + enableSourceBuild: true + artifacts: + publish: + artifacts: true + manifests: true + ``` + +1. Specify platforms (if repo is not managed-only) + + A repo is managed-only if `eng/SourceBuild.props` contains +h `true`. If this is true, + this step is not necessary. Otherwise, specify `sourceBuildParameters` + in the `jobs.yml` template's parameters like this: + + ```yaml + sourceBuildParameters: + platforms: + - name: 'Centos71_Portable' + container: 'mcr.microsoft.com/dotnet-buildtools/prereqs:centos-7-3e800f1-20190501005343' + - name: 'MacOS_Portable' + pool: { vmImage: 'macOS-10.14' } + - name: 'Centos71' + nonPortable: true + container: 'mcr.microsoft.com/dotnet-buildtools/prereqs:centos-7-3e800f1-20190501005343' + - name: 'Centos8' + nonPortable: true + container: 'mcr.microsoft.com/dotnet-buildtools/prereqs:centos-8-daa5116-20200325130212' + - name: 'Debian9' + nonPortable: true + container: 'mcr.microsoft.com/dotnet-buildtools/prereqs:debian-stretch-20200918130533-047508b' + - name: 'Fedora30' + nonPortable: true + container: 'mcr.microsoft.com/dotnet-buildtools/prereqs:fedora-30-38e0f29-20191126135223' + - name: 'Fedora32' + nonPortable: true + container: 'mcr.microsoft.com/dotnet-buildtools/prereqs:fedora-32-20200512010618-163ed2a' + - name: 'MacOS' + nonPortable: true + pool: { vmImage: 'macOS-10.14' } + - name: 'Ubuntu1804' + nonPortable: true + container: 'mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-18.04-20200918145614-047508b' + ``` + +#### End result + +Submit the changes above in a PR and include +[@source-build-internal](https://github.com/orgs/dotnet/teams/source-build-internal) +as a reviewer. The jobs (or job, if managed-only) are automatically be added +to CI in the existing pipeline alongside existing jobs, +with a name like `Build Source-Build ()`: + +![Build jobs image with source build leg](img/ci-job.png) + +Once this PR works, run a mock official build (AKA "validation build") in your +official build definition. The usual workflow is to push a +`dev//arpow` branch to the AzDO repo and then queue a build on that +branch. This makes sure that merging the PR won't immediately break the +official build: `enableSourceBuild: true` does add job(s) to the official +build, not just PR validation. + +If the PR is green, but merging it produces red PR or Official builds, +immediately let the source build team know about the failure and revert the +source build PR to unblock dev work. + +### Advanced: granular templates + +If the repo doesn't use the basic Arcade jobs template, or has advanced job +templating infra built on top of the Arcade jobs template, the simple +`enableSourceBuild` flag might not work out. There are a few more granular +templates to use in this case. + +Look at the documentation in each YAML file itself to figure out how to use +it properly, and if it fits the scenario. This list is only an overview. + +#### `eng/common/templates/jobs/source-build.yml` + +This is one level deeper than `eng/common/templates/jobs/jobs.yml`. It is +a `jobs` template that produces just the set of source-build jobs based +on the specified `platforms`. Or, just one job with the default platform, +if managed-only. + +#### `eng/common/templates/job/source-build.yml` + +This template defines a single `job` that runs source build on a +specifiedplatform. + +#### `eng/common/templates/steps/source-build.yml` + +This template defines the build `steps` for a single source build job. +This is the most granular template, and may be useful if some +repo-specific preamble or cleanup steps are required, or if the repo +already has job matrix templates and this just happens to fit in nicely. + +### Official build publishing + +Publishing [intermediate nupkg]s in the official build is handled by the +standard Arcade publish infrastructure, which should already be set up +in the repo. The source build steps handle uploading the +[intermediate nupkg] to the pipeline in the standard way that Arcade +will detect and publish. + +[intermediate nupkg]: https://github.com/dotnet/source-build/blob/master/Documentation/planning/arcade-powered-source-build/intermediate-nupkg.md + +## Add Downstream Dependency + +The graph defined by `eng/Version.Details.xml` defines what is included +in source build. This dependency graph starts at +[dotnet/installer](https://github.com/dotnet/installer/blob/main/eng/Version.Details.xml). +The .NET source build insfrastructure walks this graph and determines +the full set of repos to build. Source build will only build one copy/commit +of each repo (the first commit found in the dependency graph). Therefore, +when adding a new repositories, a dependency must be created within the +`eng/Version.Details.xml` graph. To do this, go to the repo which depends on +the new repo and add a +[new source build dependency](#engversiondetailsxml) to the new source build +repo. + +## Validate + +Once the downstream dependency(s) are added to the new repo and those changes +flow into `dotnet/installer`, a complete .NET source tarball can be built. +This will validate that no prebuilts were added to the system and everything +is functioning correctly. Please notify +[@source-build-internal](https://github.com/orgs/dotnet/teams/source-build-internal) +to be on the lookout for the new repo and they will validate as necessary. + +## Additional resources + +* For more details about how the build executes, see + [Arcade's build tools](https://github.com/dotnet/arcade/tree/master/src/Microsoft.DotNet.Arcade.Sdk/tools/SourceBuild). +* The source code for the build tasks that run for prebuilt validation and + intermediate nupkg dependency reading are maintained in + [Arcade](https://github.com/dotnet/arcade/tree/master/src/Microsoft.DotNet.SourceBuild) + as well. diff --git a/Documentation/sourcebuild-in-repos/update-dependencies.md b/Documentation/sourcebuild-in-repos/update-dependencies.md new file mode 100644 index 0000000000..b349afb39a --- /dev/null +++ b/Documentation/sourcebuild-in-repos/update-dependencies.md @@ -0,0 +1,53 @@ +# Updating a dependency + +In most cases, dependency flow should automatically update the packages you +depend on. There are some special considerations and things to keep in mind +when doing manual updates. If you are manually updating a version, also see +[the doc on new dependencies](new-dependencies.md) for special considerations. + +## Internal packages + +If you are manually updating a package, please make sure it's from a +compatible branch (e.g. runtime release/6.0 to sdk release/6.0.1xx, +release/6.0.2xx, etc). Package versions that you are updating to should +be source-built in their respective repos. If the version you need is +produced in a branch that is not yet source-build-compatible please let the +[source-build team](https://github.com/orgs/dotnet/teams/source-build-internal) +know. + +Another common error we see is updating eng/Versions.props but not +eng/Version.Details.xml. This causes problems for source-build because we +depend on these files being updated in lockstep. Please prefer updating with +Darc - it takes care of these issues - or, if a manual update really is +necessary, make sure you update both files. + +Example Darc command: +`darc update-dependencies --name MyCoolPackage -v 1.2.3` + +## External packages + +Updating a non-Microsoft (e.g. Newtonsoft.Json 9.0.1 to 13.0.1) or +non-Arcade (e.g. ApplicationInsights) package is typically equivalent to +adding the new version all over again. Please +[log an issue](https://github.com/dotnet/source-build/issues/new/choose) +to discuss this. You can check if the external package you want is already +included in source-build in the +[source-build-externals](https://github.com/dotnet/source-build-externals) repo. + +## Splitting, combining, and moving packages + +These operations often end up causing source-build issues. A typical case of +what can happen is + +1. Package A is produced in version 1. +1. In version 2, Package A is split into Package A1 and Package A2. +1. A downstream repo is never updated to take the split packages. +1. The version 2 source-build succeeds because Package A exists in the + [previously-source-built archive](build-info.md#Single-version_and_single-RID_build), + but no new version of Package A is produced. +1. Source-build version 3 fails because Package A no longer exists in + the previously-source-built archive either. + +So please keep in mind consumers of your packages and try to keep them +informed of these kinds of changes. If you are the consuming repo, consider +tracking PRs in the repos that produce your dependencies. \ No newline at end of file diff --git a/NuGet.config b/NuGet.config index 14a16f19a9..69a5757cd3 100644 --- a/NuGet.config +++ b/NuGet.config @@ -2,9 +2,16 @@ + + - + + + + + + diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index f52cd28f95..af9e46d956 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -1,14 +1,19 @@ - + https://github.com/dotnet/arcade - 200adbc809c4451973d1929a53a75502b7cada01 + bd9b4590681e602535ed414e98a17e6ba121dca4 - + + https://github.com/dotnet/sourcelink + 8031e5220baf2acad991e661d8308b783d2acf3e + + + https://github.com/dotnet/source-build-reference-packages - def2e2c6dc5064319250e2868a041a3dc07f9579 + b6e3937b3818bb214a967e990da7002dd8f20fad diff --git a/eng/Versions.props b/eng/Versions.props index df7b892f6a..b299189371 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -2,8 +2,10 @@ - 0.1.0 - alpha.1 + 6 + 0 + 0 + servicing diff --git a/eng/common/templates/job/source-build.yml b/eng/common/templates/job/source-build.yml index 5cd5325d7b..88f6f75a62 100644 --- a/eng/common/templates/job/source-build.yml +++ b/eng/common/templates/job/source-build.yml @@ -46,7 +46,7 @@ jobs: # source-build builds run in Docker, including the default managed platform. pool: ${{ if eq(variables['System.TeamProject'], 'public') }}: - name: NetCore1ESPool-Public + name: NetCore-Public demands: ImageOverride -equals Build.Ubuntu.1804.Amd64.Open ${{ if eq(variables['System.TeamProject'], 'internal') }}: name: NetCore1ESPool-Internal diff --git a/eng/common/templates/job/source-index-stage1.yml b/eng/common/templates/job/source-index-stage1.yml index 4af724eb1a..f6b80407ee 100644 --- a/eng/common/templates/job/source-index-stage1.yml +++ b/eng/common/templates/job/source-index-stage1.yml @@ -24,7 +24,7 @@ jobs: pool: ${{ if eq(variables['System.TeamProject'], 'public') }}: - name: NetCore1ESPool-Public + name: NetCore-Public demands: ImageOverride -equals Build.Server.Amd64.VS2019.Open ${{ if eq(variables['System.TeamProject'], 'internal') }}: name: NetCore1ESPool-Internal diff --git a/global.json b/global.json index 733270e033..22426441a8 100644 --- a/global.json +++ b/global.json @@ -1,8 +1,8 @@ { "tools": { - "dotnet": "6.0.100" + "dotnet": "6.0.128" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "7.0.0-beta.21608.1" + "Microsoft.DotNet.Arcade.Sdk": "6.0.0-beta.24204.4" } } diff --git a/patches/application-insights/0001-Eliminate-prebuilts.patch b/patches/application-insights/0001-Eliminate-prebuilts.patch new file mode 100644 index 0000000000..094460f816 --- /dev/null +++ b/patches/application-insights/0001-Eliminate-prebuilts.patch @@ -0,0 +1,29 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: MichaelSimons +Date: Wed, 4 May 2022 17:42:31 +0000 +Subject: [PATCH] Eliminate prebuilts + +--- + .props/Product.props | 9 +-------- + 1 file changed, 1 insertion(+), 8 deletions(-) + +diff --git a/.props/Product.props b/.props/Product.props +index da6f6ec3..cec2ee58 100644 +--- a/.props/Product.props ++++ b/.props/Product.props +@@ -32,14 +32,7 @@ + + + +- +- +- All +- +- +- +- +- ++ + + + diff --git a/repos/Directory.Build.targets b/repos/Directory.Build.targets index d00b4119bf..e0e88eac3b 100644 --- a/repos/Directory.Build.targets +++ b/repos/Directory.Build.targets @@ -29,9 +29,15 @@ git --work-tree=$(ProjectDirectory) apply --ignore-whitespace --whitespace=nowarn + + Condition="'@(PatchesToApply)' != ''" + EnvironmentVariables="GIT_DIR=/dev/null" /> diff --git a/repos/application-insights.proj b/repos/application-insights.proj index a2b68986e5..358839eb6c 100644 --- a/repos/application-insights.proj +++ b/repos/application-insights.proj @@ -7,17 +7,27 @@ true + + + + + - $(ProjectDirectory)/Microsoft.ApplicationInsights.csproj + $(ProjectDirectory)/BASE/src/Microsoft.ApplicationInsights/Microsoft.ApplicationInsights.csproj $(BuildCommandArgs) /p:Configuration=$(Configuration) - - $(BuildCommandArgs) /p:EnlistmentRoot=$(ProjectDirectory)/src + $(BuildCommandArgs) /p:MicrosoftSourceLinkVersion=$(MicrosoftSourceLinkVersion) + $(BuildCommandArgs) /p:StableRelease=True $(BuildCommandArgs) /p:RelativeOutputPathBase= $(BuildCommandArgs) /v:$(LogVerbosity) $(BuildCommandArgs) $(RedirectRepoOutputToLog) + + + $(BuildCommandArgs) /p:BinRoot=$(ProjectDirectory)/bin + $(BuildCommandArgs) /p:ObjRoot=$(ProjectDirectory)/obj + $(BuildCommandArgs) /p:PackageOutputDir=$(ProjectDirectory)/bin/$(Configuration)