Skip to content

Commit ce346d9

Browse files
authored
Allow user to skip downloading the previously-source-built packages and supply their own. (#1434)
* Allow user to skip downloading the previously-source-built packages and supply their own. * Need list of packages while creating tarball * Also add custom ref package path * Don't try to copy externals tarballs if they're not there * Correct help for some options * Some fixups from trying local builds. * Separate out arguments to skip ref packages and previously-source-built.
1 parent 3fb0944 commit ce346d9

File tree

6 files changed

+147
-33
lines changed

6 files changed

+147
-33
lines changed

.vsts.pipelines/jobs/ci-linux.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -201,7 +201,7 @@ jobs:
201201
if [ "$(reportPrebuiltLeaks)" = "true" ]; then
202202
poisonArg="/p:EnablePoison=true"
203203
fi
204-
$(docker.run) $(docker.tb.map) $(docker.tb.work) $networkArg $(imageName) "$(tarballName)/build.sh" \
204+
$(docker.run) $(docker.tb.map) $(docker.tb.work) $networkArg $(imageName) "$(tarballName)/build.sh" -- \
205205
/p:Configuration=$(sb.configuration) \
206206
/p:PortableBuild=$(sb.portable) \
207207
/p:FailOnPrebuiltBaselineError=true \

build-source-tarball.sh

Lines changed: 44 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -3,8 +3,14 @@ set -euo pipefail
33
IFS=$'\n\t'
44

55
usage() {
6-
echo "usage: $0 <path-to-tarball-root> [--skip-build] [--enable-leak-detection] [--skip-prebuilt-check] [-- [extra build.sh args]]"
7-
echo ""
6+
echo "usage: $0 <path-to-tarball-root> [options]"
7+
echo "options:"
8+
echo " --skip-build assume we have already built (requires you have built with the /p:ArchiveDownloadedPackages=true flag)"
9+
echo " --enable-leak-detection build leaked-binary detection tasts for later use while building from this tarball"
10+
echo " --skip-prebuilt-check do not confirm that all prebuilt packages used are either reference packages, previously-built, or known extras"
11+
echo " --with-ref-packages <dir> use the specified directory of available reference packages to determine what prebuilts to delete, instead of downloading the most recent version"
12+
echo " --with-packages <dir> use the specified directory of available previously-built packages to determine what prebuilts to delete, instead of downloading the most recent version"
13+
echo "use -- to send the remaining arguments to build.sh"
814
}
915

1016
if [ -z "${1:-}" ]; then
@@ -45,6 +51,9 @@ SKIP_BUILD=0
4551
INCLUDE_LEAK_DETECTION=0
4652
MINIMIZE_DISK_USAGE=0
4753
SKIP_PREBUILT_ENFORCEMENT=0
54+
CUSTOM_REF_PACKAGES_DIR=''
55+
CUSTOM_PREVIOUSLY_BUILT_PACKAGES_DIR=''
56+
MAIN_BUILD_ARGS=("/p:ArchiveDownloadedPackages=true")
4857
export DOTNET_SKIP_FIRST_TIME_EXPERIENCE=1
4958

5059
while :; do
@@ -66,6 +75,24 @@ while :; do
6675
--skip-prebuilt-check)
6776
SKIP_PREBUILT_ENFORCEMENT=1
6877
;;
78+
--with-ref-packages)
79+
CUSTOM_REF_PACKAGES_DIR="$2"
80+
if [ ! -d "$CUSTOM_REF_PACKAGES_DIR" ]; then
81+
echo "Custom reference packages directory '$CUSTOM_REF_PACKAGES_DIR' does not exist"
82+
exit 1
83+
fi
84+
MAIN_BUILD_ARGS+=( "/p:SkipDownloadingReferencePackages=true" )
85+
shift
86+
;;
87+
--with-packages)
88+
CUSTOM_PREVIOUSLY_BUILT_PACKAGES_DIR="$2"
89+
if [ ! -d "$CUSTOM_PREVIOUSLY_BUILT_PACKAGES_DIR" ]; then
90+
echo "Custom reference packages directory '$CUSTOM_PREVIOUSLY_BUILT_PACKAGES_DIR' does not exist"
91+
exit 1
92+
fi
93+
MAIN_BUILD_ARGS+=( "/p:SkipDownloadingPreviouslySourceBuiltPackages=true" )
94+
shift
95+
;;
6996
--)
7097
shift
7198
echo "Detected '--': passing remaining parameters '$@' as build.sh arguments."
@@ -119,7 +146,7 @@ if [ $SKIP_BUILD -ne 1 ]; then
119146
fi
120147

121148
$SCRIPT_ROOT/clean.sh
122-
$SCRIPT_ROOT/build.sh /p:ArchiveDownloadedPackages=true "$@"
149+
$SCRIPT_ROOT/build.sh ${MAIN_BUILD_ARGS[@]} "$@"
123150
fi
124151

125152
mkdir -p "$TARBALL_ROOT"
@@ -255,8 +282,10 @@ cp -r $SCRIPT_ROOT/bin/obj/$targetArchitecture/Release/reference-packages/source
255282
cp -r $SCRIPT_ROOT/bin/obj/$targetArchitecture/Release/reference-packages/staging $TARBALL_ROOT/packages/reference/staging
256283

257284
# Copy tarballs to ./packages/archive directory
258-
mkdir -p $TARBALL_ROOT/packages/archive
259-
cp -r $SCRIPT_ROOT/bin/obj/$targetArchitecture/Release/external-tarballs/*.tar.gz $TARBALL_ROOT/packages/archive/
285+
if [[ -d "$SCRIPT_ROOT/bin/obj/$targetArchitecture/Release/external-tarballs" && ! -z "$(find $SCRIPT_ROOT/bin/obj/$targetArchitecture/Release/external-tarballs -iname '*.tar.gz')" ]]; then
286+
mkdir -p $TARBALL_ROOT/packages/archive
287+
cp -r $SCRIPT_ROOT/bin/obj/$targetArchitecture/Release/external-tarballs/*.tar.gz $TARBALL_ROOT/packages/archive/
288+
fi
260289

261290
# Copy generated source from bin to src/generatedSrc
262291
cp -r $SCRIPT_ROOT/bin/obj/$targetArchitecture/Release/generatedSrc $TARBALL_ROOT/src/generatedSrc
@@ -300,8 +329,16 @@ do
300329
fi
301330
done
302331

303-
allRefPkgs=(`tar -tf $TARBALL_ROOT/packages/archive/Private.SourceBuild.ReferencePackages.*.tar.gz | tr '[:upper:]' '[:lower:]'`)
304-
allSourceBuiltPkgs=(`tar -tf $TARBALL_ROOT/packages/archive/Private.SourceBuilt.Artifacts.*.tar.gz | tr '[:upper:]' '[:lower:]'`)
332+
if [ -d "$CUSTOM_REF_PACKAGES_DIR" ]; then
333+
allRefPkgs=(`ls "$CUSTOM_REF_PACKAGES_DIR" | tr '[:upper:]' '[:lower:]'`)
334+
else
335+
allRefPkgs=(`tar -tf $TARBALL_ROOT/packages/archive/Private.SourceBuild.ReferencePackages.*.tar.gz | tr '[:upper:]' '[:lower:]'`)
336+
fi
337+
if [ -d "$CUSTOM_PREVIOUSLY_BUILT_PACKAGES_DIR" ]; then
338+
allSourceBuiltPkgs=(`ls "$CUSTOM_PREVIOUSLY_BUILT_PACKAGES_DIR" | tr '[:upper:]' '[:lower:]'`)
339+
else
340+
allSourceBuiltPkgs=(`tar -tf $TARBALL_ROOT/packages/archive/Private.SourceBuilt.Artifacts.*.tar.gz | tr '[:upper:]' '[:lower:]'`)
341+
fi
305342

306343
echo 'Removing reference-packages from tarball prebuilts...'
307344

build.proj

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -147,15 +147,15 @@
147147

148148
<Target Name="DownloadSourceBuildReferencePackages"
149149
AfterTargets="Build"
150-
Condition="'$(OfflineBuild)' != 'true' and '$(OS)' != 'Windows_NT'">
150+
Condition="'$(OfflineBuild)' != 'true' and '$(OS)' != 'Windows_NT' and '$(SkipDownloadingReferencePackages)' != 'true'">
151151
<DownloadFile
152152
SourceUrl="$(ReferencePackagesTarballUrl)$(ReferencePackagesTarballName).$(PrivateSourceBuildReferencePackagesPackageVersion).tar.gz"
153153
DestinationFolder="$(ExternalTarballsDir)" />
154154
</Target>
155155

156156
<Target Name="DownloadSourceBuiltArtifacts"
157157
AfterTargets="Build"
158-
Condition="'$(OfflineBuild)' != 'true' and '$(OS)' != 'Windows_NT'">
158+
Condition="'$(OfflineBuild)' != 'true' and '$(OS)' != 'Windows_NT' and '$(SkipDownloadingPreviouslySourceBuiltPackages)' != 'true'">
159159
<DownloadFile
160160
SourceUrl="$(SourceBuiltArtifactsTarballUrl)$(SourceBuiltArtifactsTarballName).$(PrivateSourceBuiltArtifactsPackageVersion).tar.gz"
161161
DestinationFolder="$(ExternalTarballsDir)" />

dir.props

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -84,6 +84,7 @@
8484
<PrebuiltPackagesPath>$(ProjectDir)packages/prebuilt/</PrebuiltPackagesPath>
8585
<PreviouslyRestoredPackagesPath>$(ProjectDir)packages/previouslyRestored/</PreviouslyRestoredPackagesPath>
8686
<PrebuiltSourceBuiltPackagesPath>$(ProjectDir)packages/source-built/</PrebuiltSourceBuiltPackagesPath>
87+
<PrebuiltSourceBuiltPackagesPath Condition="'$(CustomPrebuiltSourceBuiltPackagesPath)' != ''">$(CustomPrebuiltSourceBuiltPackagesPath)/</PrebuiltSourceBuiltPackagesPath>
8788
<SourceBuiltTarBallPath>$(OutputPath)</SourceBuiltTarBallPath>
8889
<SourceBuiltToolsetDir>$(LocalBlobStorageRoot)Toolset/</SourceBuiltToolsetDir>
8990
<SourceBuiltRuntimeDir>$(LocalBlobStorageRoot)Runtime/</SourceBuiltRuntimeDir>
@@ -135,6 +136,7 @@
135136
<ReferencePackagesStagingDir>$(ReferencePackagesBaseDir)staging/</ReferencePackagesStagingDir>
136137
<ReferencePackagesSourceDir>$(ReferencePackagesBaseDir)source/</ReferencePackagesSourceDir>
137138
<ReferencePackagesDir>$(ReferencePackagesBaseDir)packages/</ReferencePackagesDir>
139+
<ReferencePackagesDir Condition="'$(CustomReferencePackagesPath)' != ''">$(CustomReferencePackagesPath)/</ReferencePackagesDir>
138140
<SourceBuiltArtifactsTarballName>Private.SourceBuilt.Artifacts</SourceBuiltArtifactsTarballName>
139141
<SourceBuiltArtifactsTarballUrl>https://dotnetcli.blob.core.windows.net/source-built-artifacts/assets/</SourceBuiltArtifactsTarballUrl>
140142
<ReferencePackagesTarballName>Private.SourceBuild.ReferencePackages</ReferencePackagesTarballName>

support/tarball/build.sh

Lines changed: 92 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -2,43 +2,116 @@
22
set -euo pipefail
33
IFS=$'\n\t'
44

5+
usage() {
6+
echo "usage: $0 [options]"
7+
echo "options:"
8+
echo " --with-ref-packages <dir> use the specified directory of reference packages"
9+
echo " --with-packages <dir> use the specified directory of previously-built packages"
10+
echo "use -- to send the remaining arguments to MSBuild"
11+
echo ""
12+
}
13+
514
SCRIPT_ROOT="$(cd -P "$( dirname "$0" )" && pwd)"
15+
16+
MSBUILD_ARGUMENTS=("/p:OfflineBuild=true" "/flp:v=detailed")
17+
CUSTOM_REF_PACKAGES_DIR=''
18+
CUSTOM_PREVIOUSLY_BUILT_PACKAGES_DIR=''
19+
20+
while :; do
21+
if [ $# -le 0 ]; then
22+
break
23+
fi
24+
25+
lowerI="$(echo $1 | awk '{print tolower($0)}')"
26+
case $lowerI in
27+
--with-ref-packages)
28+
CUSTOM_REF_PACKAGES_DIR="$(cd -P "$2" && pwd)"
29+
if [ ! -d "$CUSTOM_REF_PACKAGES_DIR" ]; then
30+
echo "Custom reference packages directory '$CUSTOM_REF_PACKAGES_DIR' does not exist"
31+
exit 1
32+
fi
33+
MSBUILD_ARGUMENTS+=( "/p:CustomReferencePackagesPath=$CUSTOM_REF_PACKAGES_DIR" )
34+
shift
35+
;;
36+
--with-packages)
37+
CUSTOM_PREVIOUSLY_BUILT_PACKAGES_DIR="$(cd -P "$2" && pwd)"
38+
if [ ! -d "$CUSTOM_PREVIOUSLY_BUILT_PACKAGES_DIR" ]; then
39+
echo "Custom prviously built packages directory '$CUSTOM_PREVIOUSLY_BUILT_PACKAGES_DIR' does not exist"
40+
exit 1
41+
fi
42+
MSBUILD_ARGUMENTS+=( "/p:CustomPrebuiltSourceBuiltPackagesPath=$CUSTOM_PREVIOUSLY_BUILT_PACKAGES_DIR" )
43+
shift
44+
;;
45+
--)
46+
shift
47+
echo "Detected '--': passing remaining parameters '$@' as build.sh arguments."
48+
break
49+
;;
50+
-?|-h|--help)
51+
usage
52+
exit 0
53+
;;
54+
*)
55+
echo "Unrecognized argument '$1'"
56+
usage
57+
exit 1
58+
;;
59+
esac
60+
shift
61+
done
62+
663
sdkLine=`grep -m 1 'dotnet' "$SCRIPT_ROOT/global.json"`
764
sdkPattern="\"dotnet\" *: *\"(.*)\""
865
if [[ $sdkLine =~ $sdkPattern ]]; then
966
export SDK_VERSION=${BASH_REMATCH[1]}
1067
fi
1168

12-
sourceBuiltArchive=`find $SCRIPT_ROOT/packages/archive -maxdepth 1 -name 'Private.SourceBuilt.Artifacts*.tar.gz'`
13-
if [ -f "$sourceBuiltArchive" ]; then
14-
tar -xzf "$sourceBuiltArchive" -C /tmp PackageVersions.props
15-
arcadeSdkLine=`grep -m 1 'MicrosoftDotNetArcadeSdkVersion' /tmp/PackageVersions.props`
16-
versionPattern="<MicrosoftDotNetArcadeSdkVersion>(.*)</MicrosoftDotNetArcadeSdkVersion>"
17-
if [[ $arcadeSdkLine =~ $versionPattern ]]; then
18-
export ARCADE_BOOTSTRAP_VERSION=${BASH_REMATCH[1]}
19-
fi
69+
packageVersionsPath=''
2070

21-
sourceLinkLine=`grep -m 1 'MicrosoftSourceLinkCommonVersion' /tmp/PackageVersions.props`
22-
versionPattern="<MicrosoftSourceLinkCommonVersion>(.*)</MicrosoftSourceLinkCommonVersion>"
23-
if [[ $sourceLinkLine =~ $versionPattern ]]; then
24-
export SOURCE_LINK_BOOTSTRAP_VERSION=${BASH_REMATCH[1]}
71+
if [ -d "$SCRIPT_ROOT/packages/archive" ]; then
72+
sourceBuiltArchive=`find $SCRIPT_ROOT/packages/archive -maxdepth 1 -name 'Private.SourceBuilt.Artifacts*.tar.gz'`
73+
if [ -f "$sourceBuiltArchive" ]; then
74+
tar -xzf "$sourceBuiltArchive" -C /tmp PackageVersions.props
75+
packageVersionsPath=/tmp/PackageVersions.props
2576
fi
26-
27-
dotNetHostLine=`grep -m 1 'MicrosoftNETCoreDotNetHostVersion' /tmp/PackageVersions.props`
28-
versionPattern="<MicrosoftNETCoreDotNetHostVersion>(.*)</MicrosoftNETCoreDotNetHostVersion>"
29-
if [[ $dotNetHostLine =~ $versionPattern ]]; then
30-
export DOTNET_HOST_BOOTSTRAP_VERSION=${BASH_REMATCH[1]}
77+
else
78+
if [ -f "$CUSTOM_PREVIOUSLY_BUILT_PACKAGES_DIR/PackageVersions.props" ]; then
79+
packageVersionsPath="$CUSTOM_PREVIOUSLY_BUILT_PACKAGES_DIR/PackageVersions.props"
3180
fi
3281
fi
82+
83+
if [ ! -f "$packageVersionsPath" ]; then
84+
echo "Cannot find PackagesVersions.props. Debugging info:"
85+
echo " Attempted archive path: $SCRIPT_ROOT/packages/archive"
86+
echo " Attempted custom PVP path: $CUSTOM_PREVIOUSLY_BUILT_PACKAGES_DIR/PackageVersions.props"
87+
exit 1
88+
fi
89+
90+
arcadeSdkLine=`grep -m 1 'MicrosoftDotNetArcadeSdkVersion' "$packageVersionsPath"`
91+
versionPattern="<MicrosoftDotNetArcadeSdkVersion>(.*)</MicrosoftDotNetArcadeSdkVersion>"
92+
if [[ $arcadeSdkLine =~ $versionPattern ]]; then
93+
export ARCADE_BOOTSTRAP_VERSION=${BASH_REMATCH[1]}
94+
fi
95+
96+
sourceLinkLine=`grep -m 1 'MicrosoftSourceLinkCommonVersion' "$packageVersionsPath"`
97+
versionPattern="<MicrosoftSourceLinkCommonVersion>(.*)</MicrosoftSourceLinkCommonVersion>"
98+
if [[ $sourceLinkLine =~ $versionPattern ]]; then
99+
export SOURCE_LINK_BOOTSTRAP_VERSION=${BASH_REMATCH[1]}
100+
fi
101+
102+
dotNetHostLine=`grep -m 1 'MicrosoftNETCoreDotNetHostVersion' "$packageVersionsPath"`
103+
versionPattern="<MicrosoftNETCoreDotNetHostVersion>(.*)</MicrosoftNETCoreDotNetHostVersion>"
104+
if [[ $dotNetHostLine =~ $versionPattern ]]; then
105+
export DOTNET_HOST_BOOTSTRAP_VERSION=${BASH_REMATCH[1]}
106+
fi
107+
33108
echo "Found bootstrap SDK $SDK_VERSION, bootstrap Arcade $ARCADE_BOOTSTRAP_VERSION, bootstrap SourceLink $SOURCE_LINK_BOOTSTRAP_VERSION, bootstrap DotNetHost $DOTNET_HOST_BOOTSTRAP_VERSION"
34109
CLI_ROOT="$SCRIPT_ROOT/.dotnet"
35110

36111
export DOTNET_CLI_TELEMETRY_OPTOUT=1
37112
export DOTNET_SKIP_FIRST_TIME_EXPERIENCE=1
38113
export NUGET_PACKAGES="$SCRIPT_ROOT/packages/restored/"
39114

40-
MSBUILD_ARGUMENTS=("/p:OfflineBuild=true" "/flp:v=detailed")
41-
42115
$CLI_ROOT/dotnet $CLI_ROOT/sdk/$SDK_VERSION/MSBuild.dll /bl:BuildXPlatTasks.binlog $SCRIPT_ROOT/tools-local/init-build.proj /t:BuildXPlatTasks ${MSBUILD_ARGUMENTS[@]} "$@"
43116

44117
echo "Rebuild reference assemblies"

tools-local/init-build.proj

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -38,13 +38,15 @@
3838

3939
<Target Name="UnpackTarballs"
4040
Condition="'$(OfflineBuild)' == 'true'">
41-
<MakeDir Directories="$(ReferencePackagesDir)" />
41+
<MakeDir Directories="$(ReferencePackagesDir)" Condition="'$(CustomReferencePackagesPath)' == ''" />
4242
<Exec Command="tar -xzf $(ExternalTarballsDir)$(ReferencePackagesTarballName).*.tar.gz"
43-
WorkingDirectory="$(ReferencePackagesDir)" />
43+
WorkingDirectory="$(ReferencePackagesDir)"
44+
Condition="'$(CustomReferencePackagesPath)' == ''" />
4445

45-
<MakeDir Directories="$(PrebuiltSourceBuiltPackagesPath)" />
46+
<MakeDir Directories="$(PrebuiltSourceBuiltPackagesPath)" Condition="'$(CustomPrebuiltSourceBuiltPackagesPath)' == ''" />
4647
<Exec Command="tar -xzf $(ExternalTarballsDir)$(SourceBuiltArtifactsTarballName).*.tar.gz"
47-
WorkingDirectory="$(PrebuiltSourceBuiltPackagesPath)" />
48+
WorkingDirectory="$(PrebuiltSourceBuiltPackagesPath)"
49+
Condition="'$(CustomPrebuiltSourceBuiltPackagesPath)' == ''" />
4850

4951
<Copy SourceFiles="$(PrebuiltSourceBuiltPackagesPath)PackageVersions.props" DestinationFiles="$(IntermediatePath)SourceBuiltPackageVersions.props" />
5052
</Target>

0 commit comments

Comments
 (0)