diff --git a/.github/actions/check-links/action.yml b/.github/actions/check-links/action.yml index 1aab64f3..a66b5d08 100644 --- a/.github/actions/check-links/action.yml +++ b/.github/actions/check-links/action.yml @@ -4,7 +4,14 @@ runs: using: "composite" steps: - name: install-releaser - uses: jupyter-server/jupyter_releaser/.github/actions/install-releaser@v1 + shell: bash -eux {0} + id: install-releaser + run: | + # Install Jupyter Releaser from git unless we are testing Releaser itself + if ! command -v jupyter-releaser &> /dev/null + then + pip install -q git+https://github.com/jupyter-server/jupyter_releaser.git@v1 + fi - name: Cache checked links uses: actions/cache@v2 diff --git a/.github/actions/check-release/action.yml b/.github/actions/check-release/action.yml index 6a86ddd9..e35a5610 100644 --- a/.github/actions/check-release/action.yml +++ b/.github/actions/check-release/action.yml @@ -4,45 +4,48 @@ inputs: token: description: "GitHub access token" required: true - changelog: - description: "Changelog file" - default: "CHANGELOG.md" - required: false version_spec: description: "New Version Specifier" required: false default: "" + steps_to_skip: + description: "Comma separated list of steps to skip" + required: false runs: using: "composite" steps: - - name: install-releaser - uses: jupyter-server/jupyter_releaser/.github/actions/install-releaser@v1 + - shell: bash -eux {0} + id: install-releaser + run: | + # Install Jupyter Releaser from git unless we are testing Releaser itself + if ! command -v jupyter-releaser &> /dev/null + then + pip install -q git+https://github.com/jupyter-server/jupyter_releaser.git@v1 + fi - - name: draft-changelog - uses: jupyter-server/jupyter_releaser/.github/actions/draft-changelog@v1 - env: - RH_IS_CHECK_RELEASE: "true" - with: - dry_run: true - token: ${{ inputs.token }} - changelog: ${{ inputs.changelog }} - version_spec: ${{ inputs.version_spec }} + - id: draft-changelog + shell: bash -eux {0} + run: | + export RH_IS_CHECK_RELEASE="true" + export GITHUB_ACCESS_TOKEN=${{ inputs.token }} + export RH_VERSION_SPEC=${{ inputs.version_spec }} + export RH_STEPS_TO_SKIP=${{ inputs.steps_to_skip }} + python -m jupyter_releaser.actions.draft_changelog - - name: draft-release - uses: jupyter-server/jupyter_releaser/.github/actions/draft-release@v1 - env: - RH_IS_CHECK_RELEASE: "true" - with: - dry_run: true - token: ${{ inputs.token }} - changelog: ${{ inputs.changelog }} - version_spec: ${{ inputs.version_spec }} + - id: draft-release + shell: bash -eux {0} + run: | + export RH_IS_CHECK_RELEASE="true" + export GITHUB_ACCESS_TOKEN=${{ inputs.token }} + export RH_RELEASE_URL=${{ steps.draft-changelog.outputs.release_url }} + export RH_STEPS_TO_SKIP=${{ inputs.steps_to_skip }} + python -m jupyter_releaser.actions.draft_release - - name: publish-release - uses: jupyter-server/jupyter_releaser/.github/actions/publish-release@v1 - env: - RH_IS_CHECK_RELEASE: "true" - with: - dry_run: true - token: ${{ inputs.token }} - release_url: "" + - id: publish-release + shell: bash -eux {0} + run: | + export RH_IS_CHECK_RELEASE="true" + export GITHUB_ACCESS_TOKEN=${{ inputs.token }} + export RH_RELEASE_URL=${{ steps.draft-changelog.outputs.release_url }} + export RH_STEPS_TO_SKIP=${{ inputs.steps_to_skip }} + python -m jupyter_releaser.actions.publish_release diff --git a/.github/actions/draft-changelog/action.yml b/.github/actions/draft-changelog/action.yml index 01bbeb44..0c2d0e35 100644 --- a/.github/actions/draft-changelog/action.yml +++ b/.github/actions/draft-changelog/action.yml @@ -6,53 +6,58 @@ inputs: required: true version_spec: description: "New Version Specifier" - required: true + default: "next" + required: false + post_version_spec: + description: "Post Version Specifier" + required: false target: description: "The owner/repo GitHub target" required: true branch: - description: The branch to target" - required: false - changelog: - description: "Changelog file" - default: "CHANGELOG.md" + description: "The branch to target" required: false dry_run: description: "If set, do not make a PR" default: "false" required: false since: - description: Use PRs with activity since this date or git reference + description: "Use PRs with activity since this date or git reference" required: false since_last_stable: - description: Use PRs with activity since the last stable git tag + description: "Use PRs with activity since the last stable git tag" required: false outputs: pr_url: description: "The URL of the Changelog Pull Request" value: ${{ steps.draft-changelog.outputs.pr_url }} + release_url: + description: "The html URL of the draft GitHub release" + value: ${{ steps.draft-release.outputs.release_url }} runs: using: "composite" steps: - name: install-releaser - uses: jupyter-server/jupyter_releaser/.github/actions/install-releaser@v1 - - - shell: bash - id: draft-changelog + shell: bash -eux {0} run: | - set -eux + # Install Jupyter Releaser from git unless we are testing Releaser itself + if ! command -v jupyter-releaser &> /dev/null + then + pip install -q git+https://github.com/jupyter-server/jupyter_releaser.git@v1 + fi - # Set up env variables + - id: draft-changelog + shell: bash -eux {0} + run: | export GITHUB_ACCESS_TOKEN=${{ inputs.token }} export RH_REPOSITORY=${{ inputs.target }} if [ ! -z ${{ inputs.branch }} ]; then export RH_BRANCH=${{ inputs.branch }} fi export RH_VERSION_SPEC=${{ inputs.version_spec }} - export RH_CHANGELOG=${{ inputs.changelog }} + export RH_POST_VERSION_SPEC=${{ inputs.post_version_spec }} export RH_DRY_RUN=${{ inputs.dry_run }} export RH_SINCE=${{ inputs.since }} export RH_SINCE_LAST_STABLE=${{ inputs.since_last_stable }} - # Draft Changelog python -m jupyter_releaser.actions.draft_changelog diff --git a/.github/actions/draft-release/action.yml b/.github/actions/draft-release/action.yml index 2b1a9af7..5d795142 100644 --- a/.github/actions/draft-release/action.yml +++ b/.github/actions/draft-release/action.yml @@ -6,28 +6,16 @@ inputs: required: true target: description: "The owner/repo GitHub target" - required: true - branch: - description: The branch to target - required: true - version_spec: - description: "New Version Specifier" - required: true - post_version_spec: - description: "Post Version Specifier" + required: false + release_url: + description: "The full url to the GitHub release page" required: false dry_run: description: "If set, do not push permanent changes" default: "false" required: false - since: - description: Use PRs with activity since this date or git reference - required: false - since_last_stable: - description: Use PRs with activity since the last stable git tag - required: false steps_to_skip: - description: Comma separated list of steps to skip + description: "Comma separated list of steps to skip" required: false outputs: @@ -39,25 +27,20 @@ runs: using: "composite" steps: - name: install-releaser - uses: jupyter-server/jupyter_releaser/.github/actions/install-releaser@v1 - - - shell: bash - id: draft-release + shell: bash -eux {0} run: | - set -eux + # Install Jupyter Releaser from git unless we are testing Releaser itself + if ! command -v jupyter-releaser &> /dev/null + then + pip install -q git+https://github.com/jupyter-server/jupyter_releaser.git@v1 + fi - # Set up env variables + - id: draft-release + shell: bash -eux {0} + run: | export GITHUB_ACCESS_TOKEN=${{ inputs.token }} export RH_REPOSITORY=${{ inputs.target }} - if [ ! -z ${{ inputs.branch }} ]; then - export RH_BRANCH=${{ inputs.branch }} - fi - export RH_VERSION_SPEC=${{ inputs.version_spec }} - export RH_POST_VERSION_SPEC=${{ inputs.post_version_spec }} export RH_DRY_RUN=${{ inputs.dry_run }} - export RH_SINCE=${{ inputs.since }} - export RH_SINCE_LAST_STABLE=${{ inputs.since_last_stable }} export RH_STEPS_TO_SKIP=${{ inputs.steps_to_skip }} - - # Draft Release + export RH_RELEASE_URL=${{ inputs.release_url }} python -m jupyter_releaser.actions.draft_release diff --git a/.github/actions/publish-release/action.yml b/.github/actions/publish-release/action.yml index bd8d30a8..bad6b4c6 100644 --- a/.github/actions/publish-release/action.yml +++ b/.github/actions/publish-release/action.yml @@ -4,15 +4,18 @@ inputs: token: description: "GitHub access token" required: true + target: + description: "The owner/repo GitHub target" + required: false release_url: description: "The full url to the GitHub release page" - required: true + required: false dry_run: description: "If set, do not push permanent changes" default: "false" required: false steps_to_skip: - description: Comma separated list of steps to skip + description: "Comma separated list of steps to skip" required: false outputs: @@ -27,18 +30,19 @@ runs: using: "composite" steps: - name: install-releaser - uses: jupyter-server/jupyter_releaser/.github/actions/install-releaser@v1 - - - shell: bash - id: publish-release + shell: bash -eux {0} + run: | + # Install Jupyter Releaser from git unless we are testing Releaser itself + if ! command -v jupyter-releaser &> /dev/null + then + pip install -q git+https://github.com/jupyter-server/jupyter_releaser.git@v1 + fi + - id: publish-release + shell: bash -eux {0} run: | - set -eux - - # Set up env variables export GITHUB_ACCESS_TOKEN=${{ inputs.token }} + export RH_REPOSITORY=${{ inputs.target }} export RH_DRY_RUN=${{ inputs.dry_run }} - export release_url=${{ inputs.release_url }} + export RH_RELEASE_URL=${{ inputs.release_url }} export RH_STEPS_TO_SKIP=${{ inputs.steps_to_skip }} - - # Publish release python -m jupyter_releaser.actions.publish_release diff --git a/.github/workflows/draft-changelog.yml b/.github/workflows/draft-changelog.yml index dedb5b8e..d54d6c4f 100644 --- a/.github/workflows/draft-changelog.yml +++ b/.github/workflows/draft-changelog.yml @@ -2,22 +2,23 @@ name: "Step 1: Draft Changelog" on: workflow_dispatch: inputs: + version_spec: + description: "New Version Specifier" + required: true + post_version_spec: + description: "Post Version Specifier" + required: false target: description: "The owner/repo GitHub target" required: true branch: - description: "The branch to target (defaults to default branch)" - required: false - version_spec: - description: "New Version Spec" - default: "next" + description: "The branch to target" required: false since: description: "Use PRs with activity since this date or git reference" required: false since_last_stable: description: "Use PRs with activity since the last stable git tag" - type: boolean required: false jobs: draft_changelog: @@ -39,6 +40,7 @@ jobs: with: token: ${{ secrets.ADMIN_GITHUB_TOKEN }} version_spec: ${{ github.event.inputs.version_spec }} + post_version_spec: ${{ github.event.inputs.post_version_spec }} target: ${{ github.event.inputs.target }} branch: ${{ github.event.inputs.branch }} since: ${{ github.event.inputs.since }} @@ -47,5 +49,7 @@ jobs: - name: "** Next Step **" run: | echo "Review PR: ${{ steps.draft-changelog.outputs.pr_url }}" + echo "Optional): Review Draft Release: ${{ steps.draft-changelog.outputs.release_url }}" echo "## Next Step" >> $GITHUB_STEP_SUMMARY echo "Review PR: ${{ steps.draft-changelog.outputs.pr_url }}" >> $GITHUB_STEP_SUMMARY + echo "(Optional): Review Draft Release: ${{ steps.draft-changelog.outputs.release_url }}" >> $GITHUB_STEP_SUMMARY diff --git a/.github/workflows/draft-release.yml b/.github/workflows/draft-release.yml index 0424456b..58e96ef7 100644 --- a/.github/workflows/draft-release.yml +++ b/.github/workflows/draft-release.yml @@ -4,27 +4,14 @@ on: inputs: target: description: "The owner/repo GitHub target" - required: true - branch: - description: "The branch to target (defaults to default branch)" required: false - version_spec: - description: "New Version Spec" - default: "next" + release_url: + description: "The URL of the draft GitHub release" required: false - post_version_spec: - description: "Post Version Specifier" - required: false - since: - description: "Use PRs with activity since this date or git reference" - required: false - since_last_stable: - description: "Use PRs with activity since the last stable git tag" - required: false - type: boolean steps_to_skip: - description: "Comma separated list of steps to skip" + description: "comma-separated list of steps to skip" required: false + jobs: draft_release: runs-on: ubuntu-latest @@ -32,9 +19,6 @@ jobs: fail-fast: true matrix: python-version: ["3.10"] - env: - VERSION_SPEC: ${{ github.event.inputs.version_spec }} - POST_VERSION_SPEC: ${{ github.event.inputs.post_version_spec }} steps: - name: Checkout uses: actions/checkout@v2 @@ -48,11 +32,7 @@ jobs: with: token: ${{ secrets.ADMIN_GITHUB_TOKEN }} target: ${{ github.event.inputs.target }} - branch: ${{ github.event.inputs.branch }} - version_spec: ${{ github.event.inputs.version_spec }} - post_version_spec: ${{ github.event.inputs.post_version_spec }} - since: ${{ github.event.inputs.since }} - since_last_stable: ${{ github.event.inputs.since_last_stable }} + release_url: ${{ github.event.inputs.release_url }} steps_to_skip: ${{ github.event.inputs.steps_to_skip }} - name: "** Next Step **" diff --git a/.github/workflows/full-release.yml b/.github/workflows/full-release.yml index 1e55bdd6..9a08df3a 100644 --- a/.github/workflows/full-release.yml +++ b/.github/workflows/full-release.yml @@ -4,24 +4,10 @@ on: inputs: target: description: "The owner/repo GitHub target" - required: true - branch: - description: "The branch to target (defaults to default branch)" required: false - version_spec: - description: "New Version Spec" + release_url: + description: "The URL of the draft GitHub release" required: false - default: "next" - post_version_spec: - description: "Post Version Specifier" - required: false - since: - description: "Use PRs with activity since this date or git reference" - required: false - since_last_stable: - description: "Use PRs with activity since the last stable git tag" - required: false - type: boolean steps_to_skip: description: "Comma separated list of steps to skip" required: false @@ -33,9 +19,6 @@ jobs: fail-fast: true matrix: python-version: ["3.10"] - env: - VERSION_SPEC: ${{ github.event.inputs.version_spec }} - POST_VERSION_SPEC: ${{ github.event.inputs.post_version_spec }} steps: - name: Checkout uses: actions/checkout@v2 @@ -49,11 +32,7 @@ jobs: with: token: ${{ secrets.ADMIN_GITHUB_TOKEN }} target: ${{ github.event.inputs.target }} - branch: ${{ github.event.inputs.branch }} - version_spec: ${{ github.event.inputs.version_spec }} - post_version_spec: ${{ github.event.inputs.post_version_spec }} - since: ${{ github.event.inputs.since }} - since_last_stable: ${{ github.event.inputs.since_last_stable }} + release_url: ${{ github.event.inputs.release_url }} steps_to_skip: ${{ github.event.inputs.steps_to_skip }} - name: Publish Release diff --git a/.github/workflows/publish-release.yml b/.github/workflows/publish-release.yml index 3c3a0ab3..3c4533dc 100644 --- a/.github/workflows/publish-release.yml +++ b/.github/workflows/publish-release.yml @@ -2,9 +2,12 @@ name: Publish Release on: workflow_dispatch: inputs: + target: + description: "The owner/repo GitHub target" + required: false release_url: description: "The URL of the draft GitHub release" - required: true + required: false steps_to_skip: description: "comma-separated list of steps to skip" required: false @@ -28,6 +31,7 @@ jobs: uses: ./.github/actions/publish-release with: token: ${{ secrets.ADMIN_GITHUB_TOKEN }} + target: ${{ github.event.inputs.target }} release_url: ${{ github.event.inputs.release_url }} steps_to_skip: ${{ github.event.inputs.steps_to_skip }} diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index c95f72a2..a0d6e301 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -131,6 +131,42 @@ jobs: make html popd + check_local_actions: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v2 + + - name: Setup + uses: ./.github/actions/common + + - name: draft-changelog + uses: ./.github/actions/draft-changelog + env: + RH_IS_CHECK_RELEASE: "true" + with: + token: ${{ secrets.GITHUB_TOKEN }} + version_spec: 10.10.10 + dry_run: true + + - name: draft-release + uses: ./.github/actions/draft-release + env: + RH_IS_CHECK_RELEASE: "true" + with: + token: ${{ secrets.GITHUB_TOKEN }} + release_url: ${{ steps.draft-changelog.outputs.release_url }} + dry_run: true + + - name: publish-release + uses: ./.github/actions/publish-release + env: + RH_IS_CHECK_RELEASE: "true" + with: + token: ${{ secrets.GITHUB_TOKEN }} + release_url: ${{ steps.draft-changelog.outputs.release_url }} + dry_run: true + check: # This job does nothing and is only used for the branch protection if: always() needs: diff --git a/README.md b/README.md index a45e52a8..e21279fc 100644 --- a/README.md +++ b/README.md @@ -27,77 +27,4 @@ See the [adoption docs](https://jupyter-releaser.readthedocs.io/en/latest/how_to Detailed workflows are available to draft a changelog, draft a release, publish a release, and check a release. -### Draft ChangeLog Workflow - -- Manual Github workflow - - Inputs are the target repo, branch, and the version spec -- Bumps the version - - By default, uses [tbump](https://github.com/tankerhq/tbump) or [bump2version](https://github.com/c4urself/bump2version) to bump the version based on presence of config files - - We recommend `tbump` instead of `bump2version` for most cases because it does not handle patch releases well when using [prereleases](https://github.com/c4urself/bump2version/issues/190). -- Prepares the environment - - Sets up git config and branch -- Generates a changelog (using [github-activity](https://github.com/executablebooks/github-activity)) using the PRs since the last tag on this branch. - - Gets the current version and then does a git checkout to clear state - - Adds a new version entry using a HTML comment markers in the changelog file - - Optionally resolves [meeseeks](https://github.com/MeeseeksBox/MeeseeksDev) backport PRs to their original PR -- Creates a PR with the changelog changes -- Can be re-run using the same version spec. It will add new entries but preserve existing ones (in case they have been hand modified). -- Note: Pre-release changelog sections are not automatically combined, - but you may wish to do so manually. - -### Draft Release Workflow - -- Manual Github workflow - - Inputs are target repository, branch, version spec and optional post version spec -- Bumps version using the same method as the changelog action -- Prepares the environment using the same method as the changelog action -- Checks the changelog entry - - Looks for the current entry using the HTML comment markers - - Gets the expected changelog values using `github-activity` - - Ensures that all PRs are the same between the two -- For Python packages: - - Builds the wheel and source distributions if applicable - - Makes sure Python dists can be installed and imported in a virtual environment -- For npm package(s) (including workspace support): - - Builds tarball(s) using `npm pack` - - Make sure tarball(s) can be installed and imported in a new npm package -- Checks the package manifest using [`check-manifest`](https://github.com/mgedmin/check-manifest) -- Checks the links in Markdown and reStructuredText files -- Adds a commit that includes the hashes of the dist files -- Creates an annotated version tag in standard format -- If given, bumps the version using the post version spec. The post version - spec can also be given as a setting, see the [Write Releaser Config Guide](https://jupyter-releaser.readthedocs.io/en/latest/how_to_guides/write_config.html). -- Pushes the commits and tag to the target `branch` -- Publishes a draft GitHub release for the tag with the changelog entry as the text - -### Publish Release Workflow - -- Manual Github workflow - - Input is the url of the draft release -- Downloads the dist assets from the release -- Verifies shas and integrity of release assets -- Publishes assets to appropriate registries -- If the tag is on a backport branch, makes a forwardport PR for the changelog entry - -### Check Release Workflow - -- Runs on CI in the target repository to verify compatibility and release-ability. -- Runs the `Draft Changelog` and `Draft Release` actions in dry run mode -- Publishes to the local PyPI server and/or dry-run `npm publish`. -- Does not make PRs or push git changes - -## FAQs - -### My changelog is out of sync - -Create a new manual PR to fix the PR and re-orient the changelog entry markers. - -### PR is merged to the target branch in the middle of a "Draft Release" - -The release will fail to push commits because it will not be up to date. Delete the pushed tags and re-start with "Draft Changelog" to -pick up the new PR. - -## How to keep fork of Jupyter Releaser up to date - -The manual workflow files target the `@v1` actions in the source repository, which means that as long as -the workflow files themselves are up to date, you will always be running the most up to date actions. +See the [workflow details documentation](https://jupyter-releaser.readthedocs.io/en/latest/background/theory.html#workflow-details) for more information. diff --git a/docs/source/background/theory.md b/docs/source/background/theory.md index 6059330b..e9c21ad5 100644 --- a/docs/source/background/theory.md +++ b/docs/source/background/theory.md @@ -40,7 +40,8 @@ Detailed workflows are available to draft a changelog, draft a release, publish ### Draft Release Workflow - Manual Github workflow - - Inputs are target repository, branch, version spec and optional post version spec + - Input is the URL of the draft GitHub Release created in the Draft Changelog + workflow. - Bumps version using the same method as the changelog action - Prepares the environment using the same method as the changelog action - Checks the changelog entry @@ -60,17 +61,23 @@ Detailed workflows are available to draft a changelog, draft a release, publish - If given, bumps the version using the post version spec. he post version spec can also be given as a setting, [Write Releaser Config Guide](../how_to_guides/write_config.html#automatic-dev-versions). - Pushes the commits and tag to the target `branch` -- Publishes a draft GitHub release for the tag with the changelog entry as the text +- Updates the draft GitHub release for the tag with the changelog entry as the text ### Publish Release Workflow - Manual Github workflow - - Input is the url of the draft release + - Input is the url of the draft GitHub release - Downloads the dist assets from the release - Verifies shas and integrity of release assets - Publishes assets to appropriate registries - If the tag is on a backport branch, makes a forwardport PR for the changelog entry +### Full Release Workflow + +- Combines the Draft and Publish workflows into a single workflow. +- If this workflow fails during the publish step, you can address any + credential errors and run the Publish Release Workflow to publish assets. + ### Check Release Workflow - Runs on CI in the target repository to verify compatibility and release-ability. diff --git a/docs/source/get_started/making_first_release.md b/docs/source/get_started/making_first_release.md index 2dc7fb5c..2296e026 100644 --- a/docs/source/get_started/making_first_release.md +++ b/docs/source/get_started/making_first_release.md @@ -35,7 +35,7 @@ already uses Jupyter Releaser. ## Draft Changelog - Go to the "Actions" tab in your fork of `jupyter_releaser` -- Select the "Draft Changelog" workflow on the left +- Select the "Step 1: Draft Changelog" workflow on the left - Click on the "Run workflow" dropdown button on the right - Fill in the appropriate parameters @@ -48,6 +48,7 @@ already uses Jupyter Releaser. instead. - Use the "since" field to select PRs prior to the latest tag to include in the release - Type "true" in the "since the last stable git tag" if you would like to include PRs since the last non-prerelease version tagged on the target repository and branch. + - The additional "Post Version Spec" field should be used if your repo uses a dev version (e.g. 0.7.0.dev0) - The workflow will use the GitHub API to find the relevant pull requests and make an appropriate changelog entry. - The workflow will create a pull request to the target repository and branch. It will print the link in the "\*\* Next Step \*\*" job step. @@ -72,13 +73,11 @@ already uses Jupyter Releaser. - Click on the "Actions" tab - Select the "Full Release" workflow on the left - Click on the "Run workflow" button on the right -- Fill in the entries as prompted by the automated changelog text +- Fill in draft GitHub Release URL given by the Changelog PR. ![Full Release Workflow Dialog](../images/draft_release.png) - - The additional "Post Version Spec" field should be used if your repo uses a dev version (e.g. 0.7.0.dev0) - -- The workflow will draft a GitHub release, publish assets to the appropriate registries. +- The workflow will finish the GitHub release and publish assets to the appropriate registries. - If the workflow is not targeting the default branch, it will also generate a forward-port pull request for the changelog entry to the default branch. - When the workflow finishes it will print a link to the GitHub release and the forward-port PR (if appropriate) in the "\*\* Next Step \*\*" output. diff --git a/jupyter_releaser/actions/common.py b/jupyter_releaser/actions/common.py index 2a155685..e3f95f9c 100644 --- a/jupyter_releaser/actions/common.py +++ b/jupyter_releaser/actions/common.py @@ -1,8 +1,6 @@ -import os -import tempfile from contextlib import contextmanager -from jupyter_releaser.util import ensure_mock_github +from jupyter_releaser.util import prepare_environment from jupyter_releaser.util import run as _run @@ -14,29 +12,8 @@ def make_group(name): def setup(): - with make_group("Common Setup"): - # Set up env variables - os.environ.setdefault("RH_REPOSITORY", os.environ["GITHUB_REPOSITORY"]) - os.environ.setdefault("RH_REF", os.environ["GITHUB_REF"]) - - check_release = os.environ.get("RH_IS_CHECK_RELEASE", "").lower() == "true" - if not os.environ.get("RH_BRANCH") and check_release: - if os.environ.get("GITHUB_BASE_REF"): - base_ref = os.environ.get("GITHUB_BASE_REF", "") - print(f"Using GITHUB_BASE_REF: ${base_ref}") - os.environ["RH_BRANCH"] = base_ref - - else: - # e.g refs/head/foo or refs/tag/bar - ref = os.environ["GITHUB_REF"] - print(f"Using GITHUB_REF: {ref}") - os.environ["RH_BRANCH"] = "/".join(ref.split("/")[2:]) - - if os.environ.get("RH_DRY_RUN", "").lower() == "true": - static_dir = os.path.join(tempfile.gettempdir(), "gh_static") - os.makedirs(static_dir, exist_ok=True) - os.environ["RH_GITHUB_STATIC_DIR"] = static_dir - ensure_mock_github() + with make_group("Prepare Environment"): + prepare_environment() def run_action(target, *args, **kwargs): diff --git a/jupyter_releaser/actions/draft_changelog.py b/jupyter_releaser/actions/draft_changelog.py index 9729d478..9235d823 100644 --- a/jupyter_releaser/actions/draft_changelog.py +++ b/jupyter_releaser/actions/draft_changelog.py @@ -1,28 +1,16 @@ # Copyright (c) Jupyter Development Team. # Distributed under the terms of the Modified BSD License. -import os - from jupyter_releaser.actions.common import make_group, run_action, setup -from jupyter_releaser.util import CHECKOUT_NAME, get_latest_tag, log +from jupyter_releaser.util import handle_since setup() run_action("jupyter-releaser prep-git") - -# Capture the "since" argument in case we add tags befor checking changelog -# Do this before bumping the version +# Capture the "since" variable in case we add tags before checking changelog +# Do this before bumping the version. with make_group("Handle RH_SINCE"): - if not os.environ.get("RH_SINCE"): - curr_dir = os.getcwd() - os.chdir(CHECKOUT_NAME) - since_last_stable_env = os.environ.get("RH_SINCE_LAST_STABLE") - since_last_stable = since_last_stable_env == "true" - since = get_latest_tag(os.environ.get("RH_BRANCH"), since_last_stable) - if since: - log(f"Capturing {since} in RH_SINCE variable") - os.environ["RH_SINCE"] = since - os.chdir(curr_dir) + handle_since() run_action("jupyter-releaser bump-version") run_action("jupyter-releaser build-changelog") diff --git a/jupyter_releaser/actions/draft_release.py b/jupyter_releaser/actions/draft_release.py index 5cfd58e6..1cf3df85 100644 --- a/jupyter_releaser/actions/draft_release.py +++ b/jupyter_releaser/actions/draft_release.py @@ -1,5 +1,6 @@ # Copyright (c) Jupyter Development Team. # Distributed under the terms of the Modified BSD License. + import os import shutil from pathlib import Path @@ -7,7 +8,7 @@ from jupyter_releaser.actions.common import make_group, run_action, setup from jupyter_releaser.changelog import extract_current -from jupyter_releaser.util import CHECKOUT_NAME, get_latest_tag, log, run +from jupyter_releaser.util import CHECKOUT_NAME, log, run setup() @@ -36,23 +37,6 @@ run_action("jupyter-releaser prep-git") - - -with make_group("Handle RH_SINCE"): - # Capture the "since" argument in case we add tags befor checking changelog - # Do this before bumping the version - if not os.environ.get("RH_SINCE"): - curr_dir = os.getcwd() - os.chdir(CHECKOUT_NAME) - since_last_stable_env = os.environ.get("RH_SINCE_LAST_STABLE") - since_last_stable = since_last_stable_env == "true" - since = get_latest_tag(os.environ.get("RH_BRANCH"), since_last_stable) - if since: - log(f"Capturing {since} in RH_SINCE variable") - os.environ["RH_SINCE"] = since - os.chdir(curr_dir) - - run_action("jupyter-releaser bump-version") with make_group("Handle Check Release"): @@ -63,6 +47,7 @@ Path(changelog_location).write_text(changelog_text) log(extract_current(changelog_location)) +# TODO: make this compare files changed since the sha in the metadata. run_action("jupyter-releaser check-changelog") # Make sure npm comes before python in case it produces diff --git a/jupyter_releaser/actions/publish_release.py b/jupyter_releaser/actions/publish_release.py index 679ebd50..1a0e06c0 100644 --- a/jupyter_releaser/actions/publish_release.py +++ b/jupyter_releaser/actions/publish_release.py @@ -2,15 +2,17 @@ # Distributed under the terms of the Modified BSD License. import os -from jupyter_releaser.actions.common import run_action +from jupyter_releaser.actions.common import run_action, setup -release_url = os.environ["release_url"] +setup() + +release_url = os.environ["RH_RELEASE_URL"] if release_url: - run_action(f"jupyter-releaser extract-release {release_url}") + run_action("jupyter-releaser extract-release") -run_action(f"jupyter-releaser publish-assets {release_url}") +run_action("jupyter-releaser publish-assets") if release_url: - run_action(f"jupyter-releaser forwardport-changelog {release_url}") - run_action(f"jupyter-releaser publish-release {release_url}") + run_action("jupyter-releaser forwardport-changelog") + run_action("jupyter-releaser publish-release") diff --git a/jupyter_releaser/cli.py b/jupyter_releaser/cli.py index 67b831b5..912529c7 100644 --- a/jupyter_releaser/cli.py +++ b/jupyter_releaser/cli.py @@ -140,6 +140,22 @@ def main(force): ) ] + +post_version_spec_options = [ + click.option( + "--post-version-spec", + envvar="RH_POST_VERSION_SPEC", + default="", + help="The post release version (usually dev)", + ), + click.option( + "--post-version-message", + default="Bumped version to {post_version}", + envvar="RH_POST_VERSION_MESSAGE", + help="The post release message", + ), +] + version_cmd_options = [ click.option("--version-cmd", envvar="RH_VERSION_COMMAND", help="The version command") ] @@ -194,6 +210,11 @@ def main(force): git_url_options = [click.option("--git-url", help="A custom url for the git repository")] +release_url_options = [ + click.option("--release-url", envvar="RH_RELEASE_URL", help="A draft GitHub release url") +] + + changelog_path_options = [ click.option( "--changelog-path", @@ -337,6 +358,7 @@ def build_changelog( @add_options(auth_options) @add_options(changelog_path_options) @add_options(dry_run_options) +@add_options(post_version_spec_options) @use_checkout_dir() def draft_changelog( version_spec, @@ -348,6 +370,8 @@ def draft_changelog( auth, changelog_path, dry_run, + post_version_spec, + post_version_message, ): """Create a changelog entry PR""" lib.draft_changelog( @@ -359,6 +383,8 @@ def draft_changelog( auth, changelog_path, dry_run, + post_version_spec, + post_version_message, ) @@ -529,33 +555,25 @@ def tag_release(dist_dir, release_message, tag_format, tag_message, no_git_tag_w @main.command() @add_options(branch_options) +@add_options(version_cmd_options) @add_options(auth_options) @add_options(changelog_path_options) -@add_options(version_cmd_options) @add_options(dist_dir_options) @add_options(dry_run_options) -@click.option( - "--post-version-spec", - envvar="RH_POST_VERSION_SPEC", - help="The post release version (usually dev)", -) -@click.option( - "--post-version-message", - default="Bumped version to {post_version}", - envvar="RH_POST_VERSION_MESSAGE", - help="The post release message", -) +@add_options(release_url_options) +@add_options(post_version_spec_options) @click.argument("assets", nargs=-1) @use_checkout_dir() def draft_release( ref, branch, repo, + version_cmd, auth, changelog_path, - version_cmd, dist_dir, dry_run, + release_url, post_version_spec, post_version_message, assets, @@ -565,11 +583,12 @@ def draft_release( ref, branch, repo, + version_cmd, auth, changelog_path, - version_cmd, dist_dir, dry_run, + release_url, post_version_spec, post_version_message, assets, @@ -579,7 +598,7 @@ def draft_release( @main.command() @add_options(auth_options) @add_options(dry_run_options) -@click.argument("release-url", nargs=1) +@add_options(release_url_options) @use_checkout_dir() def delete_release(auth, dry_run, release_url): """Delete a draft GitHub release by url to the release page""" @@ -593,7 +612,7 @@ def delete_release(auth, dry_run, release_url): @add_options(npm_install_options) @add_options(pydist_check_options) @add_options(check_imports_options) -@click.argument("release-url", nargs=1) +@add_options(release_url_options) def extract_release( auth, dist_dir, @@ -646,7 +665,7 @@ def extract_release( ) @add_options(dry_run_options) @add_options(python_packages_options) -@click.argument("release-url", nargs=1, required=False) +@add_options(release_url_options) @use_checkout_dir() def publish_assets( dist_dir, @@ -677,7 +696,7 @@ def publish_assets( @main.command() @add_options(auth_options) @add_options(dry_run_options) -@click.argument("release-url", nargs=1) +@add_options(release_url_options) @use_checkout_dir() def publish_release(auth, dry_run, release_url): """Publish GitHub release""" @@ -690,7 +709,7 @@ def publish_release(auth, dry_run, release_url): @add_options(username_options) @add_options(changelog_path_options) @add_options(dry_run_options) -@click.argument("release-url") +@add_options(release_url_options) @use_checkout_dir() def forwardport_changelog(auth, ref, branch, repo, username, changelog_path, dry_run, release_url): """Forwardport Changelog Entries to the Default Branch""" diff --git a/jupyter_releaser/lib.py b/jupyter_releaser/lib.py index 8e97c57e..187a3997 100644 --- a/jupyter_releaser/lib.py +++ b/jupyter_releaser/lib.py @@ -1,10 +1,12 @@ # Copyright (c) Jupyter Development Team. # Distributed under the terms of the Modified BSD License. +import json import os import os.path as osp import re import shutil import sys +import tempfile import typing as t import uuid from datetime import datetime @@ -106,12 +108,24 @@ def check_links(ignore_glob, ignore_links, cache_file, links_expire): def draft_changelog( - version_spec, branch, repo, since, since_last_stable, auth, changelog_path, dry_run + version_spec, + branch, + repo, + since, + since_last_stable, + auth, + changelog_path, + dry_run, + post_version_spec, + post_version_message, ): """Create a changelog entry PR""" repo = repo or util.get_repo() branch = branch or util.get_branch() version = util.get_version() + prerelease = util.is_prerelease(version) + + current_sha = util.run("git rev-parse HEAD") # Check for multiple versions npm_versions = None @@ -137,6 +151,30 @@ def draft_changelog( commit_message = f'git commit -a -m "{title}"' body = title + util.log(f"Creating draft GitHub release for {version}") + owner, repo_name = repo.split("/") + gh = util.get_gh_object(dry_run=dry_run, owner=owner, repo=repo_name, token=auth) + + data = dict( + version_spec=version_spec, + branch=branch, + repo=repo, + since=since, + since_last_stable=since_last_stable, + version=version, + post_version_spec=post_version_spec, + post_version_message=post_version_message, + current_sha=current_sha, + ) + with tempfile.TemporaryDirectory() as d: + metadata_path = Path(d) / "metadata.json" + with open(metadata_path, "w") as fid: + json.dump(data, fid) + + release = gh.create_release( + f"v{version}", branch, f"v{version}", body, True, prerelease, files=[metadata_path] + ) + if npm_versions: body += f"\n```{npm_versions}\n```" @@ -144,9 +182,7 @@ def draft_changelog( body += f""" | Input | Value | | ------------- | ------------- | -| Target | {repo} | -| Branch | {branch} | -| Version Spec | {version_spec} | +| Draft Release | {release.html_url} | """ if since_last_stable: body += "| Since Last Stable | true |" @@ -154,8 +190,22 @@ def draft_changelog( body += f"| Since | {since} |" util.log(body) + # Remove draft releases over a day old + if bool(os.environ.get("GITHUB_ACTIONS")): + for rel in gh.repos.list_releases(): + if str(rel.draft).lower() == "false": + continue + created = rel.created_at + d_created = datetime.strptime(created, r"%Y-%m-%dT%H:%M:%SZ") + delta = datetime.utcnow() - d_created + if delta.days > 0: + gh.repos.delete_release(rel.id) + make_changelog_pr(auth, branch, repo, title, commit_message, body, dry_run=dry_run) + # Set the GitHub action output for the release url. + util.actions_output("release_url", release.html_url) + def make_changelog_pr(auth, branch, repo, title, commit_message, body, dry_run=False): repo = repo or util.get_repo() @@ -226,24 +276,25 @@ def draft_release( ref, branch, repo, + version_cmd, auth, changelog_path, - version_cmd, dist_dir, dry_run, + release_url, post_version_spec, post_version_message, assets, ): """Publish Draft GitHub release and handle post version bump""" branch = branch or util.get_branch() - repo = repo or util.get_repo() assets = assets or glob(f"{dist_dir}/*") - version = util.get_version() body = changelog.extract_current(changelog_path) - prerelease = util.is_prerelease(version) - # Bump to post version if given + match = util.parse_release_url(release_url) + owner, repo_name = match["owner"], match["repo"] + + # Bump to post version if given. if post_version_spec: post_version = bump_version( post_version_spec, version_cmd=version_cmd, changelog_path=changelog_path @@ -251,35 +302,28 @@ def draft_release( util.log(post_version_message.format(post_version=post_version)) util.run(f'git commit -a -m "Bump to {post_version}"') - owner, repo_name = repo.split("/") gh = util.get_gh_object(dry_run=dry_run, owner=owner, repo=repo_name, token=auth) - - # Remove draft releases over a day old - if bool(os.environ.get("GITHUB_ACTIONS")): - for release in gh.repos.list_releases(): - if str(release.draft).lower() == "false": - continue - created = release.created_at - d_created = datetime.strptime(created, r"%Y-%m-%dT%H:%M:%SZ") - delta = datetime.utcnow() - d_created - if delta.days > 0: - gh.repos.delete_release(release.id) + release = util.release_for_url(gh, release_url) remote_name = util.get_remote_name(dry_run) remote_url = util.run(f"git config --get remote.{remote_name}.url") if not os.path.exists(remote_url): util.run(f"git push {remote_name} HEAD:{branch} --follow-tags --tags") - util.log(f"Creating release for {version}") - util.log(f"With assets: {assets}") - release = gh.create_release( - f"v{version}", - branch, - f"v{version}", + # Upload the assets to the draft release. + util.log(f"Uploading assets: {assets}") + for fpath in assets: + gh.upload_file(release, fpath) + + # Set the body of the release with the changelog contents. + gh.repos.update_release( + release.id, + release.tag_name, + release.target_commitish, + release.name, body, True, - prerelease, - files=assets, + release.prerelease, ) # Set the GitHub action output @@ -312,7 +356,7 @@ def extract_release( python_imports, ): """Download and verify assets from a draft GitHub release""" - match = parse_release_url(release_url) + match = util.parse_release_url(release_url) owner, repo = match["owner"], match["repo"] gh = util.get_gh_object(dry_run=dry_run, owner=owner, repo=repo, token=auth) @@ -369,6 +413,7 @@ def extract_release( # Skip sha validation for dry runs since the remote tag will not exist if dry_run: + os.chdir(orig_dir) return tag_name = release.tag_name @@ -404,15 +449,6 @@ def extract_release( os.chdir(orig_dir) -def parse_release_url(release_url): - """Parse a release url into a regex match""" - match = re.match(util.RELEASE_HTML_PATTERN, release_url) - match = match or re.match(util.RELEASE_API_PATTERN, release_url) - if not match: - raise ValueError(f"Release url is not valid: {release_url}") - return match - - def publish_assets( dist_dir, npm_token, @@ -441,7 +477,7 @@ def publish_assets( else: python_package_name = "" - if len(glob(f"{dist_dir}/*.whl")): + if release_url and len(glob(f"{dist_dir}/*.whl")): twine_token = python.get_pypi_token(release_url, python_package_path) if dry_run: @@ -459,6 +495,7 @@ def publish_assets( found = False for path in sorted(glob(f"{dist_dir}/*.*")): name = Path(path).name + util.log(f"Handling dist file {path}") suffix = Path(path).suffix if suffix in [".gz", ".whl"]: if suffix == ".gz": @@ -471,12 +508,12 @@ def publish_assets( env["TWINE_PASSWORD"] = twine_token # NOTE: Do not print the env since a twine token extracted from # a PYPI_TOKEN_MAP will not be sanitized in output - util.retry(f"{twine_cmd} {name}", cwd=dist_dir, env=env) + util.retry(f"{twine_cmd} {name}", cwd=dist_dir, env=env, echo=True) found = True elif suffix == ".tgz": # Ignore already published versions try: - util.run(f"{npm_cmd} {name}", cwd=dist_dir, quiet=True, quiet_error=True) + util.run(f"{npm_cmd} {name}", cwd=dist_dir, quiet=True, quiet_error=True, echo=True) except CalledProcessError as e: stderr = e.stderr if "EPUBLISHCONFLICT" in stderr or "previously published versions" in stderr: @@ -493,7 +530,7 @@ def publish_release(auth, dry_run, release_url): """Publish GitHub release""" util.log(f"Publishing {release_url}") - match = parse_release_url(release_url) + match = util.parse_release_url(release_url) # Take the release out of draft gh = util.get_gh_object(dry_run=dry_run, owner=match["owner"], repo=match["repo"], token=auth) @@ -619,7 +656,7 @@ def prep_git(ref, branch, repo, auth, username, url): def forwardport_changelog(auth, ref, branch, repo, username, changelog_path, dry_run, release_url): """Forwardport Changelog Entries to the Default Branch""" # Set up the git repo with the branch - match = parse_release_url(release_url) + match = util.parse_release_url(release_url) gh = util.get_gh_object(dry_run=dry_run, owner=match["owner"], repo=match["repo"], token=auth) release = util.release_for_url(gh, release_url) diff --git a/jupyter_releaser/mock_github.py b/jupyter_releaser/mock_github.py index 44c85892..ce32aee3 100644 --- a/jupyter_releaser/mock_github.py +++ b/jupyter_releaser/mock_github.py @@ -1,7 +1,7 @@ import atexit import datetime +import json import os -import pickle import tempfile import uuid from typing import Dict, List @@ -24,24 +24,31 @@ app.mount("/static", StaticFiles(directory=static_dir), name="static") -def load_from_pickle(name): - source_file = os.path.join(static_dir, name + ".pkl") +def load_from_file(name, klass): + source_file = os.path.join(static_dir, name + ".json") if not os.path.exists(source_file): return {} - with open(source_file, "rb") as fid: - return pickle.load(fid) - - -def write_to_pickle(name, data): - source_file = os.path.join(static_dir, name + ".pkl") - with open(source_file, "wb") as fid: - pickle.dump(data, fid) - - -releases: Dict[int, "Release"] = load_from_pickle("releases") -pulls: Dict[int, "PullRequest"] = load_from_pickle("pulls") -release_ids_for_asset: Dict[int, int] = load_from_pickle("release_ids_for_asset") -tag_refs: Dict[str, "Tag"] = load_from_pickle("tag_refs") + with open(source_file) as fid: + data = json.load(fid) + results = {} + for key in data: + if issubclass(klass, BaseModel): + results[key] = klass(**data[key]) + else: + results[key] = data[key] + return results + + +def write_to_file(name, data): + source_file = os.path.join(static_dir, name + ".json") + result = {} + for key in data: + value = data[key] + if isinstance(value, BaseModel): + value = json.loads(value.json()) + result[key] = value + with open(source_file, "w") as fid: + json.dump(result, fid) class Asset(BaseModel): @@ -102,6 +109,12 @@ class Tag(BaseModel): object: TagObject +releases: Dict[str, "Release"] = load_from_file("releases", Release) +pulls: Dict[str, "PullRequest"] = load_from_file("pulls", PullRequest) +release_ids_for_asset: Dict[str, str] = load_from_file("release_ids_for_asset", int) +tag_refs: Dict[str, "Tag"] = load_from_file("tag_refs", Tag) + + @app.get("/") def read_root(): return {"Hello": "World"} @@ -118,8 +131,8 @@ async def create_a_release(owner: str, repo: str, request: Request) -> Release: """https://docs.github.com/en/rest/releases/releases#create-a-release""" release_id = uuid.uuid4().int data = await request.json() - url = f"https://github.com/repos/{owner}/{repo}/releases/{release_id}" - html_url = f"https://github.com/{owner}/{repo}/releases/tag/{data['tag_name']}" + url = f"{MOCK_GITHUB_URL}/repos/{owner}/{repo}/releases/{release_id}" + html_url = f"{MOCK_GITHUB_URL}/{owner}/{repo}/releases/tag/{data['tag_name']}" upload_url = f"{MOCK_GITHUB_URL}/repos/{owner}/{repo}/releases/{release_id}/assets" fmt_str = r"%Y-%m-%dT%H:%M:%SZ" created_at = datetime.datetime.utcnow().strftime(fmt_str) @@ -132,8 +145,8 @@ async def create_a_release(owner: str, repo: str, request: Request) -> Release: created_at=created_at, **data, ) - releases[model.id] = model - write_to_pickle("releases", releases) + releases[str(model.id)] = model + write_to_file("releases", releases) return model @@ -141,17 +154,17 @@ async def create_a_release(owner: str, repo: str, request: Request) -> Release: async def update_a_release(owner: str, repo: str, release_id: int, request: Request) -> Release: """https://docs.github.com/en/rest/releases/releases#update-a-release""" data = await request.json() - model = releases[release_id] + model = releases[str(release_id)] for name, value in data.items(): setattr(model, name, value) - write_to_pickle("releases", releases) + write_to_file("releases", releases) return model @app.post("/repos/{owner}/{repo}/releases/{release_id}/assets") async def upload_a_release_asset(owner: str, repo: str, release_id: int, request: Request) -> None: """https://docs.github.com/en/rest/releases/assets#upload-a-release-asset""" - model = releases[release_id] + model = releases[str(release_id)] asset_id = uuid.uuid4().int name = request.query_params["name"] with open(f"{static_dir}/{asset_id}", "wb") as fid: @@ -166,45 +179,45 @@ async def upload_a_release_asset(owner: str, repo: str, release_id: int, request url=url, content_type=headers["content-type"], ) - release_ids_for_asset[asset_id] = release_id + release_ids_for_asset[str(asset_id)] = str(release_id) model.assets.append(asset) - write_to_pickle("releases", releases) - write_to_pickle("release_ids_for_asset", release_ids_for_asset) + write_to_file("releases", releases) + write_to_file("release_ids_for_asset", release_ids_for_asset) @app.delete("/repos/{owner}/{repo}/releases/assets/{asset_id}") async def delete_a_release_asset(owner: str, repo: str, asset_id: int) -> None: """https://docs.github.com/en/rest/releases/assets#delete-a-release-asset""" - release = releases[release_ids_for_asset[asset_id]] + release = releases[release_ids_for_asset[str(asset_id)]] os.remove(f"{static_dir}/{asset_id}") release.assets = [a for a in release.assets if a.id != asset_id] - del release_ids_for_asset[asset_id] - write_to_pickle("releases", releases) - write_to_pickle("release_ids_for_asset", release_ids_for_asset) + del release_ids_for_asset[str(asset_id)] + write_to_file("releases", releases) + write_to_file("release_ids_for_asset", release_ids_for_asset) @app.delete("/repos/{owner}/{repo}/releases/{release_id}") def delete_a_release(owner: str, repo: str, release_id: int) -> None: """https://docs.github.com/en/rest/releases/releases#delete-a-release""" - del releases[release_id] - write_to_pickle("releases", releases) + del releases[str(release_id)] + write_to_file("releases", releases) @app.get("/repos/{owner}/{repo}/pulls/{pull_number}") def get_a_pull_request(owner: str, repo: str, pull_number: int) -> PullRequest: """https://docs.github.com/en/rest/pulls/pulls#get-a-pull-request""" - if pull_number not in pulls: - pulls[pull_number] = PullRequest() - write_to_pickle("pulls", pulls) - return pulls[pull_number] + if str(pull_number) not in pulls: + pulls[str(pull_number)] = PullRequest() + write_to_file("pulls", pulls) + return pulls[str(pull_number)] @app.post("/repos/{owner}/{repo}/pulls") def create_a_pull_request(owner: str, repo: str) -> PullRequest: """https://docs.github.com/en/rest/pulls/pulls#create-a-pull-request""" pull = PullRequest() - pulls[pull.number] = pull - write_to_pickle("releases", releases) + pulls[str(pull.number)] = pull + write_to_file("pulls", pulls) return pull @@ -214,12 +227,15 @@ def add_labels_to_an_issue(owner: str, repo: str, issue_number: int) -> BaseMode return BaseModel() -@app.post("/create_tag_ref/{tag_ref}/{sha}") -def create_tag_ref(tag_ref: str, sha: str) -> None: - """Create a remote tag ref object for testing""" +@app.post("/repos/{owner}/{repo}/git/refs") +async def create_tag_ref(owner: str, repo: str, request: Request) -> None: + """https://docs.github.com/en/rest/git/refs#create-a-reference""" + data = await request.json() + tag_ref = data["ref"] + sha = data["sha"] tag = Tag(ref=f"refs/tags/{tag_ref}", object=TagObject(sha=sha)) tag_refs[tag_ref] = tag - write_to_pickle("tag_refs", tag_refs) + write_to_file("tag_refs", tag_refs) @app.get("/repos/{owner}/{repo}/git/matching-refs/tags/{tag_ref}") diff --git a/jupyter_releaser/python.py b/jupyter_releaser/python.py index 8502ff65..ac5e7c55 100644 --- a/jupyter_releaser/python.py +++ b/jupyter_releaser/python.py @@ -102,7 +102,11 @@ def get_pypi_token(release_url, python_package): twine_pwd = os.environ.get("PYPI_TOKEN", "") pypi_token_map = os.environ.get("PYPI_TOKEN_MAP", "").replace(r"\n", "\n") if pypi_token_map and release_url: - parts = release_url.replace("https://github.com/", "").split("/") + parts = ( + release_url.replace(util.MOCK_GITHUB_URL + "/", "") + .replace("https://github.com/", "") + .split("/") + ) repo_name = f"{parts[0]}/{parts[1]}" if python_package != ".": repo_name += f"/{python_package}" diff --git a/jupyter_releaser/tests/conftest.py b/jupyter_releaser/tests/conftest.py index f2e3f5b1..b1f33e7e 100644 --- a/jupyter_releaser/tests/conftest.py +++ b/jupyter_releaser/tests/conftest.py @@ -3,9 +3,13 @@ import json import os import os.path as osp +import tempfile +import time +import uuid from pathlib import Path from click.testing import CliRunner +from ghapi.core import GhApi from pytest import fixture from jupyter_releaser import cli, util @@ -195,3 +199,33 @@ def mock_github(): if proc: proc.kill() proc.wait() + + +@fixture +def draft_release(mock_github): + gh = GhApi(owner="foo", repo="bar") + tag = uuid.uuid4().hex + data = dict( + version_spec="foo", + branch="bar", + repo="fizz", + since="buzz", + since_last_stable=False, + version=tag, + post_version_spec="dev", + post_version_message="hi", + ) + + with tempfile.TemporaryDirectory() as d: + metadata_path = Path(d) / "metadata.json" + with open(metadata_path, "w") as fid: + json.dump(data, fid) + + # Ensure this is the latest release. + time.sleep(1) + release = gh.create_release(tag, "bar", tag, "hi", True, True, files=[metadata_path]) + yield release.html_url + try: + gh.repos.delete_release(release.id) + except Exception as e: + print(e) diff --git a/jupyter_releaser/tests/test_cli.py b/jupyter_releaser/tests/test_cli.py index f0f41305..3b094249 100644 --- a/jupyter_releaser/tests/test_cli.py +++ b/jupyter_releaser/tests/test_cli.py @@ -163,6 +163,7 @@ def test_list_envvars(runner): python-packages: RH_PYTHON_PACKAGES ref: RH_REF release-message: RH_RELEASE_MESSAGE +release-url: RH_RELEASE_URL repo: RH_REPOSITORY resolve-backports: RH_RESOLVE_BACKPORTS since: RH_SINCE @@ -503,7 +504,7 @@ def test_tag_release(py_package, runner, build_mock, git_prep): assert "after-tag-release" in log -def test_draft_release_dry_run(py_dist, mocker, runner, git_prep): +def test_draft_release_dry_run(py_dist, mocker, runner, git_prep, draft_release): # Publish the release - dry run runner( [ @@ -513,6 +514,8 @@ def test_draft_release_dry_run(py_dist, mocker, runner, git_prep): "1.1.0.dev0", "--post-version-message", "haha", + "--release-url", + draft_release, ] ) @@ -521,26 +524,22 @@ def test_draft_release_dry_run(py_dist, mocker, runner, git_prep): assert "after-draft-release" in log -def test_draft_release_final(npm_dist, runner, mock_github, git_prep): +def test_draft_release_final(npm_dist, runner, mock_github, git_prep, draft_release): # Publish the release os.environ["GITHUB_ACTIONS"] = "true" + os.environ["RH_RELEASE_URL"] = draft_release runner(["draft-release"]) -def test_delete_release(npm_dist, runner, mock_github, git_prep): +def test_delete_release(npm_dist, runner, mock_github, git_prep, draft_release): # Publish the release # Mimic being on GitHub actions so we get the magic output os.environ["GITHUB_ACTIONS"] = "true" + os.environ["RH_RELEASE_URL"] = draft_release result = runner(["draft-release"]) - url = "" - for line in result.output.splitlines(): - match = re.match(r"::set-output name=release_url::(.*)", line) - if match: - url = match.groups()[0] - # Delete the release - runner(["delete-release", url]) + runner(["delete-release"]) log = get_log() assert "before-delete-release" in log @@ -568,7 +567,8 @@ def test_extract_dist_py(py_package, runner, mocker, mock_github, tmp_path, git_ release = create_draft_release(ref, glob(f"{dist_dir}/*.*")) shutil.rmtree(f"{util.CHECKOUT_NAME}/dist") - runner(["extract-release", release.html_url]) + os.environ["RH_RELEASE_URL"] = release.html_url + runner(["extract-release"]) log = get_log() assert "before-extract-release" not in log @@ -601,10 +601,12 @@ def test_extract_dist_multipy(py_multipackage, runner, mocker, mock_github, tmp_ # Create the release. dist_dir = os.path.join(util.CHECKOUT_NAME, "dist") - release = create_draft_release(ref, files) + release = create_draft_release(ref, glob(f"{dist_dir}/*.*")) + shutil.rmtree(f"{util.CHECKOUT_NAME}/dist") - runner(["extract-release", release.html_url]) + os.environ["RH_RELEASE_URL"] = release.html_url + runner(["extract-release"]) log = get_log() assert "before-extract-release" not in log @@ -616,7 +618,6 @@ def test_extract_dist_multipy(py_multipackage, runner, mocker, mock_github, tmp_ reason="See https://bugs.python.org/issue26660", ) def test_extract_dist_npm(npm_dist, runner, mocker, mock_github, tmp_path): - # Create a tag ref ref = create_tag_ref() @@ -625,7 +626,8 @@ def test_extract_dist_npm(npm_dist, runner, mocker, mock_github, tmp_path): release = create_draft_release(ref, glob(f"{dist_dir}/*.*")) shutil.rmtree(f"{util.CHECKOUT_NAME}/dist") - runner(["extract-release", release.html_url]) + os.environ["RH_RELEASE_URL"] = release.html_url + runner(["extract-release"]) log = get_log() assert "before-extract-release" not in log @@ -641,7 +643,7 @@ def test_publish_assets_py(py_package, runner, mocker, git_prep, mock_github): orig_run = util.run called = 0 - os.environ["PYPI_TOKEN_MAP"] = "snuffy/test,foo-token\nfizz/buzz,bar" + os.environ["PYPI_TOKEN_MAP"] = "foo/bar,foo-token\nfizz/buzz,bar" def wrapped(cmd, **kwargs): nonlocal called @@ -654,7 +656,8 @@ def wrapped(cmd, **kwargs): dist_dir = py_package / util.CHECKOUT_NAME / "dist" release = create_draft_release() - runner(["publish-assets", "--dist-dir", dist_dir, "--dry-run", release.html_url]) + os.environ["RH_RELEASE_URL"] = release.html_url + runner(["publish-assets", "--dist-dir", dist_dir, "--dry-run"]) assert called == 2, called log = get_log() @@ -680,7 +683,8 @@ def wrapped(cmd, **kwargs): assert called == 3, called -def test_publish_assets_npm_exists(npm_dist, runner, mocker, mock_github): +def test_publish_assets_npm_exists(npm_dist, runner, mocker, mock_github, draft_release): + os.environ["RH_RELEASE_URL"] = draft_release dist_dir = npm_dist / util.CHECKOUT_NAME / "dist" called = 0 @@ -694,7 +698,6 @@ def wrapped(cmd, **kwargs): raise err mock_run = mocker.patch("jupyter_releaser.util.run", wraps=wrapped) - release = create_draft_release() runner( [ "publish-assets", @@ -704,14 +707,14 @@ def wrapped(cmd, **kwargs): "npm publish --dry-run", "--dist-dir", dist_dir, - release.html_url, ] ) assert called == 3, called -def test_publish_assets_npm_all_exists(npm_dist, runner, mocker, mock_github): +def test_publish_assets_npm_all_exists(npm_dist, runner, mocker, mock_github, draft_release): + os.environ["RH_RELEASE_URL"] = draft_release dist_dir = npm_dist / util.CHECKOUT_NAME / "dist" called = 0 @@ -724,7 +727,6 @@ def wrapped(cmd, **kwargs): raise err mocker.patch("jupyter_releaser.util.run", wraps=wrapped) - release = create_draft_release() runner( [ "publish-assets", @@ -734,16 +736,15 @@ def wrapped(cmd, **kwargs): "npm publish --dry-run", "--dist-dir", dist_dir, - release.html_url, ] ) assert called == 3, called -def test_publish_release(npm_dist, runner, mocker, mock_github): - release = create_draft_release("bar") - runner(["publish-release", release.html_url]) +def test_publish_release(npm_dist, runner, mocker, mock_github, draft_release): + os.environ["RH_RELEASE_URL"] = draft_release + runner(["publish-release"]) log = get_log() assert "before-publish-release" in log @@ -823,6 +824,7 @@ def wrapped(cmd, **kwargs): def test_forwardport_changelog_no_new(npm_package, runner, mocker, mock_github, git_prep): release = create_draft_release("bar") + os.environ["RH_RELEASE_URL"] = release.html_url # Create a branch with a changelog entry util.run("git checkout -b backport_branch", cwd=util.CHECKOUT_NAME) @@ -832,7 +834,7 @@ def test_forwardport_changelog_no_new(npm_package, runner, mocker, mock_github, util.run(f"git tag v{VERSION_SPEC}", cwd=util.CHECKOUT_NAME) # Run the forwardport workflow against default branch - runner(["forwardport-changelog", release.html_url]) + runner(["forwardport-changelog"]) log = get_log() assert "before-forwardport-changelog" in log @@ -841,6 +843,7 @@ def test_forwardport_changelog_no_new(npm_package, runner, mocker, mock_github, def test_forwardport_changelog_has_new(npm_package, runner, mocker, mock_github, git_prep): release = create_draft_release("bar") + os.environ["RH_RELEASE_URL"] = release.html_url current = util.run("git branch --show-current", cwd=util.CHECKOUT_NAME) @@ -866,7 +869,7 @@ def test_forwardport_changelog_has_new(npm_package, runner, mocker, mock_github, # Run the forwardport workflow against default branch url = osp.abspath(npm_package) os.chdir(npm_package) - runner(["forwardport-changelog", release.html_url, "--branch", current]) + runner(["forwardport-changelog", "--branch", current]) util.run(f"git checkout {current}", cwd=npm_package) diff --git a/jupyter_releaser/tests/test_functions.py b/jupyter_releaser/tests/test_functions.py index a15dfc6d..b0aa4b13 100644 --- a/jupyter_releaser/tests/test_functions.py +++ b/jupyter_releaser/tests/test_functions.py @@ -3,6 +3,7 @@ import json import os import shutil +import time from pathlib import Path import pytest @@ -340,5 +341,62 @@ def test_get_latest_draft_release(mock_github): True, files=[], ) - latest = util.lastest_draft_release(gh) + latest = util.latest_draft_release(gh) assert latest.name == "v1.0.0" + + # Ensure a different timestamp. + time.sleep(1) + gh.create_release( + "v1.1.0", + "bob", + "v1.1.0", + "body", + True, + True, + files=[], + ) + latest = util.latest_draft_release(gh) + assert latest.name == "v1.1.0" + latest = util.latest_draft_release(gh, "main") + assert latest.name == "v1.0.0" + + +def test_parse_release_url(): + match = util.parse_release_url("https://github.com/foo/bar/releases/tag/fizz") + assert match.groupdict() == {"owner": "foo", "repo": "bar", "tag": "fizz"} + match = util.parse_release_url("https://api.github.com/repos/fizz/buzz/releases/tags/foo") + assert match.groupdict() == {"owner": "fizz", "repo": "buzz", "tag": "foo"} + match = util.parse_release_url( + "https://github.com/foo/bar/releases/tag/untagged-8a3c19f85a0a51d3ea66" + ) + assert match.groupdict() == { + "owner": "foo", + "repo": "bar", + "tag": "untagged-8a3c19f85a0a51d3ea66", + } + + +def test_extract_metadata_from_release_url(mock_github, draft_release): + gh = GhApi(owner="foo", repo="bar") + data = util.extract_metadata_from_release_url(gh, draft_release, "") + assert os.environ["RH_BRANCH"] == data["branch"] + + +def test_prepare_environment(mock_github, draft_release): + os.environ["GITHUB_REPOSITORY"] = "foo/bar" + tag = draft_release.split("/")[-1] + os.environ["GITHUB_REF"] = f"refs/tag/{tag}" + os.environ["RH_DRY_RUN"] = "true" + data = util.prepare_environment() + assert os.environ["RH_RELEASE_URL"] == draft_release + assert data["version_spec"] == os.environ["RH_VERSION_SPEC"] + + +def test_handle_since(npm_package, runner): + runner(["prep-git", "--git-url", npm_package]) + since = util.handle_since() + assert not since + + run("git tag v1.0.1", cwd=util.CHECKOUT_NAME) + since = util.handle_since() + assert since == "v1.0.1" diff --git a/jupyter_releaser/tests/test_mock_github.py b/jupyter_releaser/tests/test_mock_github.py index 393dc286..f9ec3790 100644 --- a/jupyter_releaser/tests/test_mock_github.py +++ b/jupyter_releaser/tests/test_mock_github.py @@ -3,6 +3,8 @@ import requests from ghapi.core import GhApi +from jupyter_releaser.mock_github import Asset, Release, load_from_file, write_to_file + def test_mock_github(mock_github): owner = "foo" @@ -47,7 +49,38 @@ def test_mock_github(mock_github): for _ in r.iter_content(chunk_size=8192): pass + gh.git.create_ref("v1.1.0", "aaaa") + tags = gh.list_tags("v1.1.0") + assert tags[0]["object"]["sha"] == "aaaa" + gh.repos.delete_release(release.id) pull = gh.pulls.create("title", "head", "base", "body", True, False, None) gh.issues.add_labels(pull.number, ["documentation"]) + + +def test_cache_storage(): + asset = Asset( + id=1, + name="hi", + size=122, + url="hi", + content_type="hi", + ) + model = Release( + id=1, + url="hi", + html_url="ho", + assets=[asset], + upload_url="hi", + created_at="1", + draft=False, + prerelease=False, + target_commitish="1", + tag_name="1", + ) + write_to_file("releases", dict(test=model)) + data = load_from_file("releases", Release) + assert isinstance(data["test"], Release) + assert isinstance(data["test"].assets[0], Asset) + assert data["test"].assets[0].url == asset.url diff --git a/jupyter_releaser/tests/util.py b/jupyter_releaser/tests/util.py index aa2db2b3..9b7731bc 100644 --- a/jupyter_releaser/tests/util.py +++ b/jupyter_releaser/tests/util.py @@ -4,11 +4,10 @@ import shutil from pathlib import Path -import requests from ghapi.core import GhApi from jupyter_releaser import changelog, cli, util -from jupyter_releaser.util import MOCK_GITHUB_URL, get_latest_tag, run +from jupyter_releaser.util import get_latest_tag, run VERSION_SPEC = "1.0.1" @@ -284,7 +283,7 @@ def write_files(git_repo, sub_packages=None, package_name="foo", module_name=Non def create_draft_release(ref="bar", files=None): - gh = GhApi("snuffy", "test") + gh = GhApi("foo", "bar") return gh.create_release( ref, "bar", @@ -301,7 +300,7 @@ def create_tag_ref(): os.chdir(util.CHECKOUT_NAME) ref = get_latest_tag(None) sha = run("git rev-parse HEAD") - url = f"{MOCK_GITHUB_URL}/create_tag_ref/{ref}/{sha}" - requests.post(url) + gh = GhApi("foo", "bar") + gh.git.create_ref(ref, sha) os.chdir(curr_dir) return ref diff --git a/jupyter_releaser/util.py b/jupyter_releaser/util.py index 727d8982..2d787ea0 100644 --- a/jupyter_releaser/util.py +++ b/jupyter_releaser/util.py @@ -17,6 +17,7 @@ import warnings from datetime import datetime from glob import glob +from io import BytesIO from pathlib import Path from subprocess import PIPE, CalledProcessError, check_output @@ -44,9 +45,8 @@ CHECKOUT_NAME = ".jupyter_releaser_checkout" -RELEASE_HTML_PATTERN = ( - "https://github.com/(?P[^/]+)/(?P[^/]+)/releases/tag/(?P.*)" -) +MOCK_GITHUB_URL = "http://127.0.0.1:8000" +RELEASE_HTML_PATTERN = f"(?:https://github.com|{MOCK_GITHUB_URL})/(?P[^/]+)/(?P[^/]+)/releases/tag/(?P.*)" RELEASE_API_PATTERN = ( "https://api.github.com/repos/(?P[^/]+)/(?P[^/]+)/releases/tags/(?P.*)" ) @@ -57,8 +57,6 @@ GIT_FETCH_CMD = "git fetch origin --filter=blob:none --quiet" -MOCK_GITHUB_URL = "http://127.0.0.1:8000" - def run(cmd, **kwargs): """Run a command as a subprocess and get the output as a string""" @@ -72,7 +70,7 @@ def run(cmd, **kwargs): if show_cwd: prefix += f" (in '{os.getcwd()}')" prefix += ":" - print(f"{prefix} {cmd}", file=sys.stderr) + log(f"{prefix} {cmd}") if sys.platform.startswith("win"): # Async subprocesses do not work well on Windows, use standard @@ -177,7 +175,7 @@ def get_version(): try: return run("python setup.py --version").split("\n")[-1] except CalledProcessError as e: - print(e) + log(e) # Build the wheel and extract the version. if PYPROJECT.exists(): @@ -336,20 +334,20 @@ def release_for_url(gh, url): return release -def lastest_draft_release(gh): +def latest_draft_release(gh, branch=None): """Get the latest draft release for a given repo""" newest_time = None newest_release = None for release in gh.repos.list_releases(): if str(release.draft).lower() == "false": continue + if branch and release.target_commitish != branch: + continue created = release.created_at d_created = datetime.strptime(created, r"%Y-%m-%dT%H:%M:%SZ") if newest_time is None or d_created > newest_time: newest_time = d_created newest_release = release - if not newest_release: - raise ValueError("No draft releases found") return newest_release @@ -357,7 +355,7 @@ def actions_output(name, value): "Print the special GitHub Actions `::set-output` line for `name::value`" log(f"\n\nSetting output {name}={value}") if "GITHUB_ACTIONS" in os.environ: - print(f"::set-output name={name}::{value}") + log(f"::set-output name={name}::{value}") def get_latest_tag(source, since_last_stable=False): @@ -424,6 +422,125 @@ def read_config(): return config +def parse_release_url(release_url): + """Parse a release url into a regex match""" + match = re.match(RELEASE_HTML_PATTERN, release_url) + match = match or re.match(RELEASE_API_PATTERN, release_url) + if not match: + raise ValueError(f"Release url is not valid: {release_url}") + return match + + +def extract_metadata_from_release_url(gh, release_url, auth): + log(f"Extracting metadata for release: {release_url}") + release = release_for_url(gh, release_url) + + data = None + for asset in release.assets: + if asset.name != "metadata.json": + continue + + log(f"Fetching {asset.name}...") + url = asset.url + headers = dict(Authorization=f"token {auth}", Accept="application/octet-stream") + + sink = BytesIO() + with requests.get(url, headers=headers, stream=True) as r: + r.raise_for_status() + for chunk in r.iter_content(chunk_size=8192): + sink.write(chunk) + sink.seek(0) + data = json.loads(sink.read().decode("utf-8")) + + if data is None: + raise ValueError(f'Could not find "metadata.json" file in draft release {release_url}') + + # Update environment variables. + if "post_version_spec" in data: + os.environ["RH_POST_VERSION_SPEC"] = data["post_version_spec"] + if "post_version_message" in data: + os.environ["RH_POST_VERSION_MESSAGE"] = data["post_version_message"] + if "version_spec" in data: + os.environ["RH_VERSION_SPEC"] = data["version_spec"] + if "branch" in data: + os.environ["RH_BRANCH"] = data["branch"] + if "since" in data: + os.environ["RH_SINCE"] = data["since"] + if "since_last_stable" in data: + os.environ["RH_SINCE_LAST_STABLE"] = str(data["since_last_stable"]) + + return data + + +def prepare_environment(): + """Prepare the environment variables, for use when running one of the + action scripts.""" + # Set up env variables + os.environ.setdefault("RH_REPOSITORY", os.environ["GITHUB_REPOSITORY"]) + os.environ.setdefault("RH_REF", os.environ["GITHUB_REF"]) + + check_release = os.environ.get("RH_IS_CHECK_RELEASE", "").lower() == "true" + if not os.environ.get("RH_DRY_RUN") and check_release: + os.environ["RH_DRY_RUN"] = "true" + dry_run = os.environ.get("RH_DRY_RUN", "").lower() == "true" + + # Set the branch when using check release. + if not os.environ.get("RH_BRANCH") and check_release: + if os.environ.get("GITHUB_BASE_REF"): + base_ref = os.environ.get("GITHUB_BASE_REF", "") + log(f"Using GITHUB_BASE_REF: ${base_ref}") + os.environ["RH_BRANCH"] = base_ref + + else: + # e.g refs/head/foo or refs/tag/bar + ref = os.environ["GITHUB_REF"] + log(f"Using GITHUB_REF: {ref}") + os.environ["RH_BRANCH"] = "/".join(ref.split("/")[2:]) + + # Start the mock GitHub server if in a dry run. + if dry_run: + static_dir = os.path.join(tempfile.gettempdir(), "gh_static") + os.makedirs(static_dir, exist_ok=True) + os.environ["RH_GITHUB_STATIC_DIR"] = static_dir + ensure_mock_github() + + # Set up GitHub object. + branch = os.environ.get("RH_BRANCH") + owner, repo_name = os.environ["GITHUB_REPOSITORY"].split("/") + auth = os.environ.get("GITHUB_ACCESS_TOKEN", "") + gh = get_gh_object(dry_run=dry_run, owner=owner, repo=repo_name, token=auth) + + # Get the latest draft release if none is given. + release_url = os.environ.get("RH_RELEASE_URL") + log(f"Environment release url was {release_url}") + if not release_url: + release = latest_draft_release(gh, branch) + if release: + release_url = release.html_url + + if release_url: + os.environ["RH_RELEASE_URL"] = release_url + + # Extract the metadata from the release url. + return extract_metadata_from_release_url(gh, release_url, auth) + + +def handle_since(): + """Capture the "since" argument in case we add tags before checking changelog.""" + if os.environ.get("RH_SINCE"): + return + curr_dir = os.getcwd() + os.chdir(CHECKOUT_NAME) + since_last_stable_env = os.environ.get("RH_SINCE_LAST_STABLE") + since_last_stable = since_last_stable_env == "true" + since = get_latest_tag(os.environ.get("RH_BRANCH"), since_last_stable) + if since: + log(f"Capturing {since} in RH_SINCE variable") + os.environ["RH_SINCE"] = since + os.chdir(curr_dir) + return since + + def get_gh_object(dry_run=False, **kwargs): """Get a properly configured GhAPi object""" if dry_run: @@ -459,7 +576,7 @@ def get_remote_name(dry_run): def ensure_mock_github(): """Check for or start a mock github server.""" core.GH_HOST = MOCK_GITHUB_URL - + log("Ensuring mock GitHub") # First see if it is already running. try: requests.get(MOCK_GITHUB_URL) @@ -483,6 +600,7 @@ def ensure_mock_github(): raise ValueError(f"mock_github failed with {proc.returncode}") except subprocess.TimeoutExpired: pass + log("Mock GitHub started") atexit.register(proc.kill) while 1: