Skip to content

Commit aba4099

Browse files
dagooddagood
authored
More complete release scripts, fix checklist ordering and reference to scripts (#1774)
1 parent a5bf06c commit aba4099

File tree

2 files changed

+151
-13
lines changed

2 files changed

+151
-13
lines changed

‎.github/ISSUE_TEMPLATE/releases/release_checklist.md

+7-6
Original file line numberDiff line numberDiff line change
@@ -83,12 +83,6 @@
8383
- Note: The PR is internal for internal builds and on GitHub for non-internal builds.
8484
- [Non-Internal] Merge the PR when green and accepted.
8585
- [Internal] Don't merge the PR.
86-
- [ ] Download the artifacts from the green PR validation build.
87-
- Download the tarball and smoke-test prereqs from each job.
88-
- We may require only a subset for certain releases: refer to the previous release email thread corresponding to 2.1 vs. 3.1 build.
89-
- Rename tarball and smoke-test prereqs to human-friendly names. Refer to previous release email for naming pattern.
90-
- Can use <https://github.com/dagood/terminal-setup/blob/master/bash-util/rename-inner-targz> to do this.
91-
1. - [ ] Upload tarballs and smoke-test prereqs to blob storage.
9286
1. - [ ] Coordinate with team and complete manual distributed smoke-testing. Suggested assignment below, but it's flexible. Send email with direct links to artifacts from green PR.
9387
- [2.1]
9488
- [ ] RHEL7 and RHEL8 VMs - crummel
@@ -103,6 +97,13 @@
10397
- [ ] CentOS 7 - dagood
10498
- [ ] CentOS 8 - dseefeld
10599
- [ ] Debian - dagood
100+
1. - [ ] [Internal] Download the artifacts from the green PR validation build onto a machine with Bash.
101+
- We may require only a subset of the artifacts for certain releases: refer to the previous release email thread corresponding to 2.1 vs. 3.1 build.
102+
- Can use <https://github.com/dotnet/source-build/blob/release/3.1/scripts/fetch-azdo-artifact-tarball.sh> as a library.
103+
- E.g. download the tarballs with `download_tarball`, and fix them up with `fix_azdo_tarball` if necessary.
104+
1. - [ ] [Internal] Rename tarball and smoke-test prereqs to human-friendly names. (Can use `rename_tarball_inner_dir`.)
105+
- Refer to previous release's email threads for naming pattern.
106+
1. - [ ] [Internal] Upload tarballs and smoke-test prereqs to blob storage.
106107
1. - [ ] [Internal] Send the tarball to partners. Include info about how certain we are that this will be the final Microsoft build.
107108
- Never overwrite a tarball. At least change the blob storage virtual dir to represent a new build. This can help avoid timing issues and make it more obvious if stale links were accidentally re-sent rather than new ones.
108109
1. - [ ] SYNC POINT: Wait for Microsoft build release.

‎scripts/fetch-azdo-artifact-tarball.sh

+144-7
Original file line numberDiff line numberDiff line change
@@ -7,31 +7,168 @@
77
# that doesn't have a clone of this repository, which is common when downloading a tarball onto a
88
# fresh machine to validate it.
99
#
10+
# If env var 'azdo_build_pat' isn't detected, prompts to set it. The prompt is intended to help a
11+
# little bit by keeping the PAT off 'history' while also making multiple calls to the function
12+
# convenient. Be aware that the PAT is put into the current shell's env.
13+
#
1014
# Usage: download_tarball <url>
1115
# url: the URL to download. Wrap it in single quotes to avoid issues with special chars.
16+
#
17+
# Usage: name=<targz-path> download_tarball <url>
18+
# Downloads the tarball to a specific name/location.
1219
download_tarball() {
1320
[ "${azdo_build_pat:-}" ] || read -p 'AzDO dnceng PAT with build read permission: ' -s azdo_build_pat
1421
(
1522
set -euo pipefail
1623
url=$1
17-
temp_name=${temp_name:-tarball.tar.gz}
24+
name=${name:-tarball.tar.gz}
1825

1926
echo; echo 'Starting download...'
2027

21-
curl -o "$temp_name" -u "intentionally_blank:$azdo_build_pat" "$url"
22-
if tar -xf "$temp_name"; then
28+
# Sometimes tooling converts %-encoding back to spaces. AzDO doesn't like that (400 error), so
29+
# put the % back. (Full encoding might be better, but we just have spaces to deal with so far.)
30+
url=${url// /%20}
31+
32+
curl -fSL -o "$name" -u "intentionally_blank:$azdo_build_pat" "$url"
33+
34+
echo "Completed download: '$url' -> '$name'"
35+
)
36+
}
37+
38+
# Extract a tarball into the working directory. This includes a workaround for tarballs downloaded
39+
# from AzDO, which may have been double-compressed by the server.
40+
#
41+
# Usage: name=<tarball-path> extract_tarball
42+
extract_tarball() {
43+
(
44+
set -euo pipefail
45+
46+
if tar -xf "$name"; then
2347
echo "Extracted using tar -xf"
2448
# Fall back to 'gunzip' as an intermediate to avoid problems found
2549
# decompressing archives with 'tar' when downloaded directly from AzDO.
2650
# AzDO seems to double-compress the tar.gz: https://superuser.com/a/841876
27-
elif gunzip -d < "$temp_name" | tar -zx; then
51+
elif gunzip -d < "$name" | tar -zx; then
2852
echo "Extracted using gunzip"
2953
else
30-
echo "Failed to extract $temp_name"
54+
echo "Failed to extract $name"
3155
exit 1
3256
fi
3357

34-
rm "$temp_name"
35-
echo 'Complete!'
58+
echo "Completed extracting: $name"
59+
)
60+
}
61+
62+
# Use download_tarball to download a tarball to a temporary file in the working dir, extract the
63+
# tar.gz in the working dir, and delete the temporary file.
64+
#
65+
# Usage: download_extract_tarball <url>
66+
# url: the URL to download. Wrap it in single quotes to avoid issues with special chars.
67+
download_extract_tarball() {
68+
(
69+
set -euo pipefail
70+
71+
url=$1
72+
73+
date_now_for_path=$(date +%Y-%m-%d_%H%M%S)
74+
export name=${name:-tarball-${date_now_for_path}.tar.gz}
75+
76+
download_tarball "$url"
77+
extract_tarball
78+
79+
rm "$name"
80+
echo "Completed download_extract_tarball."
81+
)
82+
}
83+
84+
# AzDO sometimes double-compresses tarballs unexpectedly, which can cause problems with
85+
# rename_inner_targz during the release process. This function extracts the tarball file with a
86+
# workaround and re-creates it. Uses a temporary dir in the working directory to store extracted
87+
# contents. The targz file must be in the working dir.
88+
#
89+
# Usage: fix_azdo_tarball <targz-file>
90+
fix_azdo_tarball() {
91+
(
92+
set -euo pipefail
93+
94+
tar=$1
95+
tar_temp=${tar}-temp-extract
96+
97+
set -x
98+
99+
mkdir "$tar_temp"
100+
(
101+
cd "$tar_temp"
102+
name="../$tar" extract_tarball
103+
rm "../$tar"
104+
105+
tar --numeric-owner -zcf "../$tar" *
106+
)
107+
rm -rf "$tar_temp"
36108
)
37109
}
110+
111+
# Renames the inner top-level dir inside a tarball to something else. Creates a new tar.gz in the
112+
# working dir that matches the new inner top-level dir. The input tar.gz file must contain only a
113+
# single directory at its root. The working directory must not have any exiting files that match the
114+
# old inner dir name, target inner dir name, or output tarball name.
115+
#
116+
# This is only intended to be useful to manually create a nice-looking final source-build tarball to
117+
# upload to blob storage. We should instead have the build produce a nice name to begin with:
118+
# https://github.com/dotnet/source-build/issues/747
119+
#
120+
# Usage: rename_tarball_inner_dir <targz-name> <target-name>
121+
# targz-name: The name of the tarball file. Includes extension.
122+
# target-name:
123+
# The desired name of the inner dir in the output tar.gz. The output tar.gz is named
124+
# {target-name}.tar.gz in the working directory.
125+
rename_tarball_inner_dir() {
126+
(
127+
set -u
128+
129+
tar=$1
130+
target=$2
131+
132+
first=$(tar -ztf "$tar" | head -1)
133+
inner=${first%%/*}
134+
135+
abort_if_exists "$inner"
136+
abort_if_exists "$target"
137+
abort_if_exists "$target.tar.gz"
138+
139+
set -x
140+
141+
tar -xf "$tar"
142+
mv "$inner" "$target"
143+
tar --numeric-owner -zcf "$target.tar.gz" "$target"
144+
rm -rf "$target"
145+
)
146+
}
147+
148+
# Renames the given targz so that its filename matches its top-level directory. The output tarball
149+
# ends up in the working dir.
150+
#
151+
# Usage: rename_targz_to_match_first_dir <targz-name>
152+
rename_targz_to_match_first_dir() {
153+
(
154+
set -u
155+
156+
tar=$1
157+
first_file=$(tar -ztf "$tar" | head -1)
158+
first_dir=${first_file%%/*}
159+
tar_dest=${first_dir}.tar.gz
160+
161+
abort_if_exists "$tar_dest"
162+
163+
set -x
164+
mv "$tar" "$tar_dest"
165+
)
166+
}
167+
168+
abort_if_exists() {
169+
if [ -a "$1" ]
170+
then
171+
echo "Aborting: file exists: $1"
172+
exit 1
173+
fi
174+
}

0 commit comments

Comments
 (0)