Extend the release script to tag and create the releases. (#10496)

Author: erikjohnston

changelog.d/10496.misc

@@ -0,0 +1 @@
+Extend release script to also tag and create GitHub releases.

scripts-dev/release.py

@@ -14,29 +14,57 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-"""An interactive script for doing a release. See `run()` below.
+"""An interactive script for doing a release. See `cli()` below.
 """
 
+import re
 import subprocess
 import sys
-from typing import Optional
+import urllib.request
+from os import path
+from tempfile import TemporaryDirectory
+from typing import List, Optional, Tuple
 
+import attr
 import click
+import commonmark
 import git
+import redbaron
+from click.exceptions import ClickException
+from github import Github
 from packaging import version
-from redbaron import RedBaron
 
 
-@click.command()
-def run():
-    """An interactive script to walk through the initial stages of creating a
-    release, including creating release branch, updating changelog and pushing to
-    GitHub.
+@click.group()
+def cli():
+    """An interactive script to walk through the parts of creating a release.
 
     Requires the dev dependencies be installed, which can be done via:
 
         pip install -e .[dev]
 
+    Then to use:
+
+        ./scripts-dev/release.py prepare
+
+        # ... ask others to look at the changelog ...
+
+        ./scripts-dev/release.py tag
+
+        # ... wait for asssets to build ...
+
+        ./scripts-dev/release.py publish
+        ./scripts-dev/release.py upload
+
+    If the env var GH_TOKEN (or GITHUB_TOKEN) is set, or passed into the
+    `tag`/`publish` command, then a new draft release will be created/published.
+    """
+
+
+@cli.command()
+def prepare():
+    """Do the initial stages of creating a release, including creating release
+    branch, updating changelog and pushing to GitHub.
     """
 
     # Make sure we're in a git repo.
@@ -51,32 +79,8 @@ def run():
     click.secho("Updating git repo...")
     repo.remote().fetch()
 
-    # Parse the AST and load the `__version__` node so that we can edit it
-    # later.
-    with open("synapse/__init__.py") as f:
-        red = RedBaron(f.read())
-
-    version_node = None
-    for node in red:
-        if node.type != "assignment":
-            continue
-
-        if node.target.type != "name":
-            continue
-
-        if node.target.value != "__version__":
-            continue
-
-        version_node = node
-        break
-
-    if not version_node:
-        print("Failed to find '__version__' definition in synapse/__init__.py")
-        sys.exit(1)
-
-    # Parse the current version.
-    current_version = version.parse(version_node.value.value.strip('"'))
-    assert isinstance(current_version, version.Version)
+    # Get the current version and AST from root Synapse module.
+    current_version, parsed_synapse_ast, version_node = parse_version_from_module()
 
     # Figure out what sort of release we're doing and calcuate the new version.
     rc = click.confirm("RC", default=True)
@@ -190,7 +194,7 @@ def run():
     # Update the `__version__` variable and write it back to the file.
     version_node.value = '"' + new_version + '"'
     with open("synapse/__init__.py", "w") as f:
-        f.write(red.dumps())
+        f.write(parsed_synapse_ast.dumps())
 
     # Generate changelogs
     subprocess.run("python3 -m towncrier", shell=True)
@@ -240,6 +244,180 @@ def run():
     )
 
 
+@cli.command()
+@click.option("--gh-token", envvar=["GH_TOKEN", "GITHUB_TOKEN"])
+def tag(gh_token: Optional[str]):
+    """Tags the release and generates a draft GitHub release"""
+
+    # Make sure we're in a git repo.
+    try:
+        repo = git.Repo()
+    except git.InvalidGitRepositoryError:
+        raise click.ClickException("Not in Synapse repo.")
+
+    if repo.is_dirty():
+        raise click.ClickException("Uncommitted changes exist.")
+
+    click.secho("Updating git repo...")
+    repo.remote().fetch()
+
+    # Find out the version and tag name.
+    current_version, _, _ = parse_version_from_module()
+    tag_name = f"v{current_version}"
+
+    # Check we haven't released this version.
+    if tag_name in repo.tags:
+        raise click.ClickException(f"Tag {tag_name} already exists!\n")
+
+    # Get the appropriate changelogs and tag.
+    changes = get_changes_for_version(current_version)
+
+    click.echo_via_pager(changes)
+    if click.confirm("Edit text?", default=False):
+        changes = click.edit(changes, require_save=False)
+
+    repo.create_tag(tag_name, message=changes)
+
+    if not click.confirm("Push tag to GitHub?", default=True):
+        print("")
+        print("Run when ready to push:")
+        print("")
+        print(f"\tgit push {repo.remote().name} tag {current_version}")
+        print("")
+        return
+
+    repo.git.push(repo.remote().name, "tag", tag_name)
+
+    # If no token was given, we bail here
+    if not gh_token:
+        click.launch(f"https://github.com/matrix-org/synapse/releases/edit/{tag_name}")
+        return
+
+    # Create a new draft release
+    gh = Github(gh_token)
+    gh_repo = gh.get_repo("matrix-org/synapse")
+    release = gh_repo.create_git_release(
+        tag=tag_name,
+        name=tag_name,
+        message=changes,
+        draft=True,
+        prerelease=current_version.is_prerelease,
+    )
+
+    # Open the release and the actions where we are building the assets.
+    click.launch(release.url)
+    click.launch(
+        f"https://github.com/matrix-org/synapse/actions?query=branch%3A{tag_name}"
+    )
+
+    click.echo("Wait for release assets to be built")
+
+
+@cli.command()
+@click.option("--gh-token", envvar=["GH_TOKEN", "GITHUB_TOKEN"], required=True)
+def publish(gh_token: str):
+    """Publish release."""
+
+    # Make sure we're in a git repo.
+    try:
+        repo = git.Repo()
+    except git.InvalidGitRepositoryError:
+        raise click.ClickException("Not in Synapse repo.")
+
+    if repo.is_dirty():
+        raise click.ClickException("Uncommitted changes exist.")
+
+    current_version, _, _ = parse_version_from_module()
+    tag_name = f"v{current_version}"
+
+    if not click.confirm(f"Publish {tag_name}?", default=True):
+        return
+
+    # Publish the draft release
+    gh = Github(gh_token)
+    gh_repo = gh.get_repo("matrix-org/synapse")
+    for release in gh_repo.get_releases():
+        if release.title == tag_name:
+            break
+    else:
+        raise ClickException(f"Failed to find GitHub release for {tag_name}")
+
+    assert release.title == tag_name
+
+    if not release.draft:
+        click.echo("Release already published.")
+        return
+
+    release = release.update_release(
+        name=release.title,
+        message=release.body,
+        tag_name=release.tag_name,
+        prerelease=release.prerelease,
+        draft=False,
+    )
+
+
+@cli.command()
+def upload():
+    """Upload release to pypi."""
+
+    current_version, _, _ = parse_version_from_module()
+    tag_name = f"v{current_version}"
+
+    pypi_asset_names = [
+        f"matrix_synapse-{current_version}-py3-none-any.whl",
+        f"matrix-synapse-{current_version}.tar.gz",
+    ]
+
+    with TemporaryDirectory(prefix=f"synapse_upload_{tag_name}_") as tmpdir:
+        for name in pypi_asset_names:
+            filename = path.join(tmpdir, name)
+            url = f"https://github.com/matrix-org/synapse/releases/download/{tag_name}/{name}"
+
+            click.echo(f"Downloading {name} into {filename}")
+            urllib.request.urlretrieve(url, filename=filename)
+
+        if click.confirm("Upload to PyPI?", default=True):
+            subprocess.run("twine upload *", shell=True, cwd=tmpdir)
+
+    click.echo(
+        f"Done! Remember to merge the tag {tag_name} into the appropriate branches"
+    )
+
+
+def parse_version_from_module() -> Tuple[
+    version.Version, redbaron.RedBaron, redbaron.Node
+]:
+    # Parse the AST and load the `__version__` node so that we can edit it
+    # later.
+    with open("synapse/__init__.py") as f:
+        red = redbaron.RedBaron(f.read())
+
+    version_node = None
+    for node in red:
+        if node.type != "assignment":
+            continue
+
+        if node.target.type != "name":
+            continue
+
+        if node.target.value != "__version__":
+            continue
+
+        version_node = node
+        break
+
+    if not version_node:
+        print("Failed to find '__version__' definition in synapse/__init__.py")
+        sys.exit(1)
+
+    # Parse the current version.
+    current_version = version.parse(version_node.value.value.strip('"'))
+    assert isinstance(current_version, version.Version)
+
+    return current_version, red, version_node
+
+
 def find_ref(repo: git.Repo, ref_name: str) -> Optional[git.HEAD]:
     """Find the branch/ref, looking first locally then in the remote."""
     if ref_name in repo.refs:
@@ -256,5 +434,66 @@ def update_branch(repo: git.Repo):
         repo.git.merge(repo.active_branch.tracking_branch().name)
 
 
+def get_changes_for_version(wanted_version: version.Version) -> str:
+    """Get the changelogs for the given version.
+
+    If an RC then will only get the changelog for that RC version, otherwise if
+    its a full release will get the changelog for the release and all its RCs.
+    """
+
+    with open("CHANGES.md") as f:
+        changes = f.read()
+
+    # First we parse the changelog so that we can split it into sections based
+    # on the release headings.
+    ast = commonmark.Parser().parse(changes)
+
+    @attr.s(auto_attribs=True)
+    class VersionSection:
+        title: str
+
+        # These are 0-based.
+        start_line: int
+        end_line: Optional[int] = None  # Is none if its the last entry
+
+    headings: List[VersionSection] = []
+    for node, _ in ast.walker():
+        # We look for all text nodes that are in a level 1 heading.
+        if node.t != "text":
+            continue
+
+        if node.parent.t != "heading" or node.parent.level != 1:
+            continue
+
+        # If we have a previous heading then we update its `end_line`.
+        if headings:
+            headings[-1].end_line = node.parent.sourcepos[0][0] - 1
+
+        headings.append(VersionSection(node.literal, node.parent.sourcepos[0][0] - 1))
+
+    changes_by_line = changes.split("\n")
+
+    version_changelog = []  # The lines we want to include in the changelog
+
+    # Go through each section and find any that match the requested version.
+    regex = re.compile(r"^Synapse v?(\S+)")
+    for section in headings:
+        groups = regex.match(section.title)
+        if not groups:
+            continue
+
+        heading_version = version.parse(groups.group(1))
+        heading_base_version = version.parse(heading_version.base_version)
+
+        # Check if heading version matches the requested version, or if its an
+        # RC of the requested version.
+        if wanted_version not in (heading_version, heading_base_version):
+            continue
+
+        version_changelog.extend(changes_by_line[section.start_line : section.end_line])
+
+    return "\n".join(version_changelog)
+
+
 if __name__ == "__main__":
-    run()
+    cli()

setup.py

@@ -108,6 +108,8 @@ def exec_file(path_segments):
     "click==7.1.2",
     "redbaron==0.9.2",
     "GitPython==3.1.14",
+    "commonmark==0.9.1",
+    "pygithub==1.55",
 ]
 
 CONDITIONAL_REQUIREMENTS["mypy"] = ["mypy==0.812", "mypy-zope==0.2.13"]