Skip to content

Commit 8e1764f

Browse files
ytmimiytmimi
committed
automate git subtree-push pull requests
1 parent 182a203 commit 8e1764f

File tree

2 files changed

+383
-0
lines changed

2 files changed

+383
-0
lines changed

‎.github/workflows/subtree_push.yml

+40
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
name: Git Subtree Push
2+
on:
3+
workflow_dispatch:
4+
inputs:
5+
rustlang_rust_url:
6+
description: 'rust-lang/rust repository URL'
7+
default: 'https://github.com/rust-lang/rust'
8+
required: true
9+
10+
jobs:
11+
subtree-push:
12+
runs-on: ubuntu-latest
13+
14+
steps:
15+
- name: checkout
16+
uses: actions/checkout@v4
17+
with:
18+
# 0 indicates all history for all branches and tags.
19+
# https://github.com/actions/checkout?tab=readme-ov-file#fetch-all-history-for-all-tags-and-branches
20+
# Grabbing everything should help us avoid issues where `git commit --no-ff` complains that
21+
# it can't merge unrelated histories.
22+
fetch-depth: 0
23+
24+
# Based on https://github.com/rust-lang/rustup/issues/3409
25+
# rustup should already be installed in GitHub Actions.
26+
- name: install current toolchain with rustup
27+
run: |
28+
CURRENT_TOOLCHAIN=$(cut -d ' ' -f3 <<< $(cat rust-toolchain | grep "channel =") | tr -d '"')
29+
rustup install $CURRENT_TOOLCHAIN --no-self-update
30+
31+
- name: Setup Rustfmt Bot Git Details
32+
run: |
33+
git config user.name "rustfmt bot"
34+
git config user.email "[email protected]"
35+
36+
- name: subtree-push
37+
env:
38+
# Need to set the `GH_TOKEN` env variable so we can use the GitHub CLI in `/ci/subtree_sync.sh`
39+
GH_TOKEN: ${{ github.token }}
40+
run: ${GITHUB_WORKSPACE}/ci/subtree_sync.sh subtree-push ${{ inputs.rustlang_rust_url }}

‎ci/subtree_sync.sh

+343
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,343 @@
1+
#!/bin/bash
2+
3+
# Install the latest nightly rust toolchain
4+
# We want to perform a subtree-push from the latest nightly rust-lang/rust -> rustfmt
5+
# In order to do so, make sure we've go the latest nightly toolchain installed
6+
function install_latest_nightly() {
7+
rustup update nightly --no-self-update
8+
}
9+
10+
# Follows the steps outlined in the Clippy docs to get a patched version of git-subtree that works
11+
# with larger repos.
12+
# This is necessary to push commits from rust-lang/rust -> rustfmt
13+
# https://doc.rust-lang.org/nightly/clippy/development/infrastructure/sync.html#patching-git-subtree-to-work-with-big-repos
14+
function get_patched_subtree() {
15+
local CLONE_DIR=$1
16+
local PATCHED_GIT_SUBTREE_FORK="https://github.com/tqc/git.git"
17+
local PATCHED_BRANCH="tqc/subtree"
18+
19+
GIT_TERMINAL_PROMPT=0 git clone --branch $PATCHED_BRANCH --single-branch -q --depth 1 $PATCHED_GIT_SUBTREE_FORK $CLONE_DIR
20+
21+
local SUBTREE_SCRIPT_PATH="contrib/subtree/git-subtree.sh"
22+
local FULL_SUBTREE_SCRIPT_PATH="$CLONE_DIR/$SUBTREE_SCRIPT_PATH"
23+
24+
echo "Patching git-subtree using fork:$PATCHED_GIT_SUBTREE_FORK branch:$PATCHED_BRANCH"
25+
26+
sudo cp --backup $FULL_SUBTREE_SCRIPT_PATH /usr/lib/git-core/git-subtree
27+
sudo chmod --reference=/usr/lib/git-core/git-subtree~ /usr/lib/git-core/git-subtree
28+
sudo chown --reference=/usr/lib/git-core/git-subtree~ /usr/lib/git-core/git-subtree
29+
}
30+
31+
# Extract various details from rustc's verbose version output e.g `rustc -Vv`
32+
function parse_rustc_verbose_version_info() {
33+
local RUSTC_VERBOSE_VERSION_INFO=$1
34+
# valid values are: `binary`, `commit-hash`, `commit-date`, `host`, `release`, `LLVM version`
35+
local INFO_KEY=$2
36+
echo $(cut -d ' ' -f2 <<< $(echo "$RUSTC_VERBOSE_VERSION_INFO=" | grep "$INFO_KEY:"))
37+
}
38+
39+
# Parses the `commit-hash` from rustc verbose version output e.g `rustc -Vv`
40+
function get_commit_hash() {
41+
local RUSTC_VERBOSE_VERSION_INFO=$1
42+
echo $(parse_rustc_verbose_version_info "$RUSTC_VERBOSE_VERSION_INFO" "commit-hash")
43+
}
44+
45+
# Parses the `commit-date` from rustc verbose version output e.g `rustc -Vv`
46+
function get_commit_date() {
47+
local RUSTC_VERBOSE_VERSION_INFO==$1
48+
echo $(parse_rustc_verbose_version_info "$RUSTC_VERBOSE_VERSION_INFO" "commit-date")
49+
}
50+
51+
# Parses the `release` from rustc verbose version output e.g `rustc -Vv`
52+
function get_release_number() {
53+
local RUSTC_VERBOSE_VERSION_INFO==$1
54+
echo $(parse_rustc_verbose_version_info "$RUSTC_VERBOSE_VERSION_INFO" "release")
55+
}
56+
57+
# The nightly toolchain always has a commit date that is 1 day behind.
58+
# This will help us get the correct release date for the toolchain
59+
function toolchain_date() {
60+
# Should be a date string in the form YYYY-MM-DD.
61+
# We should get this date using `get_commit_date`
62+
local DATE=$1
63+
echo $(date --rfc-3339=date --date="$DATE+1day")
64+
}
65+
66+
# Sets the new toolchain version in rustfmt's `rust-toolchain` file
67+
function bump_rust_toolchain_version() {
68+
local CURRENT_TOOLCHAIN=$1
69+
local LATEST_TOOLCHAIN=$2
70+
local TOOLCHAIN_FILE="rust-toolchain"
71+
echo "Bumping the toolchain listed in $TOOLCHAIN_FILE from $CURRENT_TOOLCHAIN -> $LATEST_TOOLCHAIN"
72+
73+
NEW_TOOLCHAIN_FILE=$(cat $TOOLCHAIN_FILE | sed "s/$CURRENT_TOOLCHAIN/$LATEST_TOOLCHAIN/g")
74+
echo "$NEW_TOOLCHAIN_FILE" > $TOOLCHAIN_FILE
75+
}
76+
77+
# Clone the master branch of the rust-lang/rust repo
78+
function clone_rustlang_rust() {
79+
local CLONE_DIR=$1
80+
local RUSTLANG_RUST_GIT_URL=$2
81+
echo "Cloning $RUSTLANG_RUST_GIT_URL into $CLONE_DIR"
82+
# Do we need the entire git history? Would it suffice to just get the history from the last full subtree sync?
83+
git clone -q --branch master --single-branch $RUSTLANG_RUST_GIT_URL $CLONE_DIR
84+
cd $CLONE_DIR
85+
}
86+
87+
# Dynamically get the name of the HEAD branch.
88+
function get_main_branch_name() {
89+
REMOTE_NAME=$1
90+
echo $(git remote show $REMOTE_NAME | grep 'HEAD branch' | cut -d' ' -f5)
91+
}
92+
93+
# Get rustfmt's repository URL
94+
function rustfmt_repository_url() {
95+
# This is one of the default environment variables set by GitHub Actions
96+
# https://docs.github.com/en/actions/learn-github-actions/variables#default-environment-variables
97+
local GITHUB_REPOSITORY=$GITHUB_REPOSITORY
98+
echo "https://github.com/$GITHUB_REPOSITORY"
99+
}
100+
101+
# Get rustfmt's git URL
102+
function rustfmt_git_url() {
103+
echo "$(rustfmt_repository_url).git"
104+
}
105+
106+
107+
# Follows instructions outlined by the Clippy docs to perform a subtree push
108+
# https://doc.rust-lang.org/nightly/clippy/development/infrastructure/sync.html#syncing-changes-between-clippy-and-rust-langrust
109+
function rustc_to_rustfmt_subtree_push() {
110+
local CLONE_DIR=$1
111+
local LATEST_NIGHTLY_COMMIT=$2
112+
local LAST_SUBTREE_PUSH_COMMIT=$3
113+
local RUSTFMT_LOCAL_PATH=$4
114+
local NEW_BRANCH_NAME=$5
115+
local RUSTLANG_RUST_GIT_URL=$6
116+
local LOCAL_RUSTFMT_REPO_ALIAS="rustfmt-local"
117+
# Path to the rustfmt subtree within the rust-lang/rust repo
118+
local RUSTFMT_TOOLS_PATH="src/tools/rustfmt"
119+
120+
get_patched_subtree "$CLONE_DIR/git"
121+
122+
# cloning will also CD into the rust-lang/rust repo
123+
clone_rustlang_rust "$CLONE_DIR/rust" $RUSTLANG_RUST_GIT_URL
124+
git remote add $LOCAL_RUSTFMT_REPO_ALIAS $RUSTFMT_LOCAL_PATH
125+
126+
# The subtree-push doesn't necessarily happen with the HEAD of the rust-lang/rust repo.
127+
# We want to `push` changes up to whatever commit was last released.
128+
git switch --detach $LATEST_NIGHTLY_COMMIT
129+
# Need to bump up the stack size as we're about to go through the entire rustfmt history
130+
ulimit -s 60000
131+
echo "Running the subtree push"
132+
# The logs get really noisy so redirect everything to /dev/null
133+
git subtree push -P $RUSTFMT_TOOLS_PATH $LOCAL_RUSTFMT_REPO_ALIAS $NEW_BRANCH_NAME > /dev/null 2>&1
134+
}
135+
136+
# Try to create a merge commit for the latest subtree-push
137+
# A merge commit is only created if the changes from rust-lang/rust apply cleanly to rustfmt.
138+
function try_create_subtree_push_merge_commit() {
139+
local RUSTFMT_REPO_PATH=$1
140+
local SUBTREE_PUSH_BRANCH_NAME=$2
141+
local REMOTE_HEAD_BRANCH=$3
142+
local REMOTE_REF="origin/$REMOTE_HEAD_BRANCH"
143+
144+
cd $RUSTFMT_REPO_PATH
145+
git fetch origin $REMOTE_HEAD_BRANCH
146+
git switch $SUBTREE_PUSH_BRANCH_NAME
147+
148+
echo "Trying create merge commit between $SUBTREE_PUSH_BRANCH_NAME and $REMOTE_REF"
149+
150+
git merge $REMOTE_REF --no-ff --no-commit
151+
if [ $? -eq 0 ]; then
152+
echo "The subtree push was clean 😁. Creating a merge commit."
153+
git commit --no-edit
154+
return 0
155+
else
156+
echo "Unfortunately there are merge conflicts that need to be addressed 😢"
157+
git merge --abort
158+
return 1
159+
fi
160+
}
161+
162+
# Create a Pull Request for the subtree-push
163+
function create_subtree_push_pull_request() {
164+
local CLEAN_MERGE_COMMIT=$1
165+
local CURRENT_TOOLCHAIN=$2
166+
local LATEST_TOOLCHAIN=$3
167+
local COMMIT_MESSAGE=$4
168+
local NEW_BRANCH_NAME=$5
169+
local REMOTE_HEAD_BRANCH=$6
170+
171+
# This is one of the default environment variables set by GitHub Actions
172+
# https://docs.github.com/en/actions/learn-github-actions/variables#default-environment-variables
173+
local GITHUB_REPOSITORY=$GITHUB_REPOSITORY
174+
local REPOSITORY_URL=$(rustfmt_repository_url)
175+
local GIT_URL=$(rustfmt_git_url)
176+
177+
if [ $CLEAN_MERGE_COMMIT -eq 0 ]; then
178+
# No merge conflicts!!
179+
bump_rust_toolchain_version "$CURRENT_TOOLCHAIN" "$LATEST_TOOLCHAIN"
180+
git add rust-toolchain
181+
echo "Creating commit for the new $LATEST_TOOLCHAIN toolchain"
182+
git commit -m "$COMMIT_MESSAGE"
183+
fi
184+
185+
# whether the merge commit applied cleanly or not create a PR
186+
# I believe the remote repo will always be `origin` in GitHub Actions
187+
echo "Pushing $NEW_BRANCH_NAME to $GIT_URL"
188+
git push origin $NEW_BRANCH_NAME
189+
# Skip the title of the commit
190+
local PR_MESSAGE=$(echo "$COMMIT_MESSAGE" | tail -n +2)
191+
local PR_URL=$(gh pr create --title "subtree-push $LATEST_TOOLCHAIN" --body "$PR_MESSAGE")
192+
193+
echo "Created Pull Request $PR_URL"
194+
195+
# TODO(ytmimi)
196+
# For convenience, if the commit applied cleanly, we could kick off the
197+
# Diff-Check Job using the GitHub CLI: https://cli.github.com/manual/gh_workflow_run
198+
DIFF_CHECK_URL="$REPOSITORY_URL/actions/workflows/check_diff.yml"
199+
200+
if [ $CLEAN_MERGE_COMMIT -eq 0 ]; then
201+
gh pr comment $PR_URL --body "The subtree-push applied cleanly ✅.
202+
203+
Take a moment to review the changes. You'll also want to Run the [Diff-Check] job.
204+
205+
**Diff-Check Job Parameters**:
206+
- Git URL: $GIT_URL
207+
- Feature Branch: $NEW_BRANCH_NAME
208+
209+
After CI and the [Diff-Check] job pass this PR should be good to merge!
210+
211+
[Diff-Check]: $DIFF_CHECK_URL
212+
"
213+
else
214+
gh pr comment $PR_URL --body "There was an issue and this subtree-push can't automatically be merged ⚠️
215+
216+
1. Please checkout branch \`$NEW_BRANCH_NAME\`, fix any merge conflicts, and then run \`git merge upstream/$REMOTE_HEAD_BRANCH --no-ff\`
217+
2. Bump the toolchain listed in the \`rust-toolchain\` file to \`$LATEST_TOOLCHAIN\`, and commit those changes.
218+
3. Run the [Diff-Check] Job.
219+
- **Diff-Check Job Parameters**:
220+
- Git URL: $GIT_URL
221+
- Feature Branch: $NEW_BRANCH_NAME
222+
4. Wait for CI checks to pass.
223+
224+
[Diff-Check]: $DIFF_CHECK_URL
225+
"
226+
fi
227+
228+
# TODO(ytmimi): notify the team that a new subtree-push PR was created.
229+
# Additionally, include whether the subtree-push applied cleanly or not.
230+
# miri publishes messages to Zulip. I feel like we could do the same.
231+
# https://github.com/rust-lang/miri/blob/f006d42618a038f7e38d2b59d1b0664727e51382/.github/workflows/ci.yml#L205-L210
232+
}
233+
234+
# Create a new Pull Request in the rustfmt repository for the git subtree-push
235+
#
236+
# **Note:** The Pull Request is created regardless if the changes from rust-lang/rust
237+
# apply cleanly to rustfmt or not. If they apply cleanly, then great! All that's left to
238+
# do is review and merge the changes. If there are conflicts one of the rustfmt maintainers
239+
# will need to address those, create a merge commit, bump the nightly toolchain, and wait for CI to pass.
240+
function run_rustfmt_subtree_push() {
241+
local RUSTLANG_RUST_URL=$1
242+
# Assumes that the current working directory is the root of the rustfmt repo
243+
local CWD=$(pwd)
244+
# TMP DIR used to clone the rust-lang/rust repo and a patched `git subtree` command
245+
local TMP_DIR=$(mktemp -d)
246+
local NIGHTLY="nightly"
247+
248+
# Running `rustc -Vv` in the rustfmt repo should give us details for the nightly toolchain
249+
# specified in the `rust-toolchain` file
250+
CURRENT_RUSTFMT_RUSTC_VERSION=$(rustc -Vv)
251+
CURRENT_RUSTFMT_RUSTC_COMMIT_HASH=$(get_commit_hash "$CURRENT_RUSTFMT_RUSTC_VERSION")
252+
CURRENT_RUSTFMT_RUSTC_COMMIT_DATE=$(get_commit_date "$CURRENT_RUSTFMT_RUSTC_VERSION")
253+
CURRENT_RUSTFMT_RUSTC_RELEASE=$(get_release_number "$CURRENT_RUSTFMT_RUSTC_VERSION")
254+
CURRENT_TOOLCHAIN_DATE=$(toolchain_date "$CURRENT_RUSTFMT_RUSTC_COMMIT_DATE")
255+
CURRENT_TOOLCHAIN="$NIGHTLY-$CURRENT_TOOLCHAIN_DATE"
256+
257+
echo "CURRENT_TOOLCHAIN: $CURRENT_TOOLCHAIN"
258+
259+
# Running `rustc +nightly -Vv` should give us details about the latest nightly toolchain
260+
LATEST_NIGHTLY_RUSTC_VERSION=$(rustc +$NIGHTLY -Vv)
261+
LATEST_NIGHTLY_RUSTC_COMMIT_HASH=$(get_commit_hash "$LATEST_NIGHTLY_RUSTC_VERSION")
262+
LATEST_NIGHTLY_COMMIT_DATE=$(get_commit_date "$LATEST_NIGHTLY_RUSTC_VERSION")
263+
LATEST_NIGHTLY_RELEASE=$(get_release_number "$LATEST_NIGHTLY_RUSTC_VERSION")
264+
LATEST_TOOLCHAIN_DATE=$(toolchain_date "$LATEST_NIGHTLY_COMMIT_DATE")
265+
LATEST_TOOLCHAIN="$NIGHTLY-$LATEST_TOOLCHAIN_DATE"
266+
267+
echo "LATEST_TOOLCHAIN $LATEST_TOOLCHAIN"
268+
269+
COMMIT_MESSAGE="chore: bump rustfmt toolchain to $LATEST_TOOLCHAIN
270+
271+
Bumping the toolchain version as part of a git subtree push
272+
273+
current toolchain ($CURRENT_TOOLCHAIN):
274+
- $CURRENT_RUSTFMT_RUSTC_RELEASE (${CURRENT_RUSTFMT_RUSTC_COMMIT_HASH:0:9} $CURRENT_RUSTFMT_RUSTC_COMMIT_DATE)
275+
276+
latest toolchain ($LATEST_TOOLCHAIN):
277+
- $LATEST_NIGHTLY_RELEASE (${LATEST_NIGHTLY_RUSTC_COMMIT_HASH:0:9} $LATEST_NIGHTLY_COMMIT_DATE)
278+
"
279+
280+
NEW_BRANCH_NAME="subtree-push-$LATEST_TOOLCHAIN"
281+
282+
rustc_to_rustfmt_subtree_push \
283+
$TMP_DIR \
284+
$LATEST_NIGHTLY_RUSTC_COMMIT_HASH \
285+
$CURRENT_RUSTFMT_RUSTC_COMMIT_HASH \
286+
$CWD \
287+
$NEW_BRANCH_NAME \
288+
"${RUSTLANG_RUST_URL}.git"
289+
290+
if [ $? -eq 0 ]; then
291+
echo "subtree push was successfull. The $NEW_BRANCH_NAME branch should be available in the local rustfmt repo"
292+
else
293+
echo "Failed to create a subtree push from the local rust-lang/rust clone"
294+
return 1
295+
fi
296+
297+
echo "Switching back to rustfmt to finish the subtree push"
298+
cd $CWD
299+
git switch $NEW_RUSTFMT_BRANCH
300+
301+
local RUSTFMT_REMOTE_HEAD=$(get_main_branch_name "origin")
302+
try_create_subtree_push_merge_commit $CWD $NEW_BRANCH_NAME $RUSTFMT_REMOTE_HEAD
303+
CLEAN_MERGE_COMMIT=$?
304+
305+
create_subtree_push_pull_request \
306+
$CLEAN_MERGE_COMMIT \
307+
$CURRENT_TOOLCHAIN \
308+
$LATEST_TOOLCHAIN \
309+
"$COMMIT_MESSAGE" \
310+
$NEW_BRANCH_NAME \
311+
$RUSTFMT_REMOTE_HEAD
312+
313+
rm -rf $TMP_DIR
314+
}
315+
316+
function print_help() {
317+
echo "Tools to help automate subtree syncs
318+
319+
usage: subtree_sync.sh <command> [<args>]
320+
321+
commands:
322+
subtree-push Push changes from rust-lang/rust back to rustfmt.
323+
"
324+
}
325+
326+
function main() {
327+
local COMMAND=$1
328+
local RUSTLANG_RUST_URL=$2
329+
330+
echo "Running Command $COMMAND"
331+
332+
case $COMMAND in
333+
subtree-push)
334+
install_latest_nightly
335+
run_rustfmt_subtree_push $RUSTLANG_RUST_URL
336+
;;
337+
*)
338+
print_help
339+
;;
340+
esac
341+
}
342+
343+
main $@

0 commit comments

Comments
 (0)