diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 860be11..9f1c464 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 1.0.0 +current_version = 1.2.0 commit = True tag = False diff --git a/.github/dependabot.yml b/.github/dependabot.yml index afb98ae..0e0584e 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -1,7 +1,16 @@ version: 2 updates: -- package-ecosystem: github-actions - directory: "/" - schedule: - interval: daily - open-pull-requests-limit: 10 + - package-ecosystem: github-actions + directory: "/" + schedule: + interval: daily + open-pull-requests-limit: 10 + labels: + - "merge when passing" + - package-ecosystem: pip + directory: "/" + schedule: + interval: daily + open-pull-requests-limit: 10 + labels: + - "merge when passing" diff --git a/.github/workflows/auto-approve.yml b/.github/workflows/auto-approve.yml index d537315..6c8dc38 100644 --- a/.github/workflows/auto-approve.yml +++ b/.github/workflows/auto-approve.yml @@ -7,7 +7,7 @@ jobs: auto-approve: runs-on: ubuntu-latest steps: - - uses: hmarr/auto-approve-action@v2 + - uses: hmarr/auto-approve-action@v4 if: | ( github.event.pull_request.user.login == 'dependabot[bot]' || diff --git a/.github/workflows/auto-merge.yml b/.github/workflows/auto-merge.yml deleted file mode 100644 index 59f687f..0000000 --- a/.github/workflows/auto-merge.yml +++ /dev/null @@ -1,26 +0,0 @@ -name: automerge -on: - check_suite: - types: - - completed - -jobs: - automerge: - runs-on: ubuntu-latest - if: | - github.actor == 'dependabot[bot]' || - github.actor == 'dependabot' || - github.actor == 'dependabot-preview[bot]' || - github.actor == 'dependabot-preview' || - github.actor == 'renovate[bot]' || - github.actor == 'renovate' - steps: - - name: automerge - uses: pascalgn/automerge-action@v0.15.3 - env: - GITHUB_TOKEN: ${{ secrets.PAT_TOKEN }} - MERGE_METHOD: "rebase" - UPDATE_METHOD: "rebase" - MERGE_RETRIES: "6" - MERGE_RETRY_SLEEP: "100000" - MERGE_LABELS: "" diff --git a/.github/workflows/codacy.yml b/.github/workflows/codacy.yml new file mode 100644 index 0000000..127070e --- /dev/null +++ b/.github/workflows/codacy.yml @@ -0,0 +1,60 @@ +# This workflow uses actions that are not certified by GitHub. +# They are provided by a third-party and are governed by +# separate terms of service, privacy policy, and support +# documentation. + +# This workflow checks out code, performs a Codacy security scan +# and integrates the results with the +# GitHub Advanced Security code scanning feature. For more information on +# the Codacy security scan action usage and parameters, see +# https://github.com/codacy/codacy-analysis-cli-action. +# For more information on Codacy Analysis CLI in general, see +# https://github.com/codacy/codacy-analysis-cli. + +name: Codacy Security Scan + +on: + push: + branches: [ "main" ] + pull_request: + # The branches below must be a subset of the branches above + branches: [ "main" ] + schedule: + - cron: '39 21 * * 4' + +permissions: + contents: read + +jobs: + codacy-security-scan: + permissions: + contents: read # for actions/checkout to fetch code + security-events: write # for github/codeql-action/upload-sarif to upload SARIF results + name: Codacy Security Scan + runs-on: ubuntu-latest + steps: + # Checkout the repository to the GitHub Actions runner + - name: Checkout code + uses: actions/checkout@v4 + + # Execute Codacy Analysis CLI and generate a SARIF output with the security issues identified during the analysis + - name: Run Codacy Analysis CLI + uses: codacy/codacy-analysis-cli-action@3ff8e64eb4b714c4bee91b7b4eea31c6fc2c4f93 + with: + # Check https://github.com/codacy/codacy-analysis-cli#project-token to get your project token from your Codacy repository + # You can also omit the token and run the tools that support default configurations + project-token: ${{ secrets.CODACY_PROJECT_TOKEN }} + verbose: true + output: results.sarif + format: sarif + # Adjust severity of non-security issues + gh-code-scanning-compat: true + # Force 0 exit code to allow SARIF file generation + # This will handover control about PR rejection to the GitHub side + max-allowed-issues: 2147483647 + + # Upload the SARIF file generated in the previous step + - name: Upload SARIF results file + uses: github/codeql-action/upload-sarif@v3 + with: + sarif_file: results.sarif diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml new file mode 100644 index 0000000..74677b6 --- /dev/null +++ b/.github/workflows/codeql.yml @@ -0,0 +1,41 @@ +name: "CodeQL" + +on: + push: + branches: [ "main" ] + pull_request: + branches: [ "main" ] + schedule: + - cron: "32 0 * * 4" + +jobs: + analyze: + name: Analyze + runs-on: ubuntu-latest + permissions: + actions: read + contents: read + security-events: write + + strategy: + fail-fast: false + matrix: + language: [ python ] + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Initialize CodeQL + uses: github/codeql-action/init@v3 + with: + languages: ${{ matrix.language }} + queries: +security-and-quality + + - name: Autobuild + uses: github/codeql-action/autobuild@v3 + + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@v3 + with: + category: "/language:${{ matrix.language }}" diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 07d47a7..2a6203f 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -8,18 +8,18 @@ jobs: deploy: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: fetch-depth: 0 - name: Run semver-diff id: semver-diff - uses: tj-actions/semver-diff@v2.0.0 + uses: tj-actions/semver-diff@v3.0.1 - name: Set up Python - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: - python-version: '3.6.x' + python-version: '3.7.x' - name: Upgrade pip run: pip install -U pip @@ -43,13 +43,14 @@ jobs: TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} - name: Generate CHANGELOG - uses: tj-actions/github-changelog-generator@v1.13 + uses: tj-actions/github-changelog-generator@v1.20 - name: Create Pull Request - uses: peter-evans/create-pull-request@v4 + uses: peter-evans/create-pull-request@v6 with: base: "main" title: "Upgraded ${{ steps.semver-diff.outputs.old_version }} → ${{ steps.semver-diff.outputs.new_version }}" + labels: "merge when passing" branch: "chore/upgrade-${{ steps.semver-diff.outputs.old_version }}-to-${{ steps.semver-diff.outputs.new_version }}" commit-message: "Upgraded from ${{ steps.semver-diff.outputs.old_version }} → ${{ steps.semver-diff.outputs.new_version }}" body: "View [CHANGES](https://github.com/${{ github.repository }}/compare/${{ steps.semver-diff.outputs.old_version }}...${{ steps.semver-diff.outputs.new_version }})" diff --git a/CHANGELOG.md b/CHANGELOG.md index 4b51c49..18ef1cf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,83 @@ # Changelog +## [v1.2.0](https://github.com/tj-python/github-deploy/tree/v1.2.0) (2023-07-05) + +[Full Changelog](https://github.com/tj-python/github-deploy/compare/v1.1.2...v1.2.0) + +**Merged pull requests:** + +- Upgraded v1.1.1 → v1.1.2 [\#46](https://github.com/tj-python/github-deploy/pull/46) ([jackton1](https://github.com/jackton1)) + +## [v1.1.2](https://github.com/tj-python/github-deploy/tree/v1.1.2) (2023-05-29) + +[Full Changelog](https://github.com/tj-python/github-deploy/compare/v1.1.1...v1.1.2) + +**Merged pull requests:** + +- Bump tj-actions/github-changelog-generator from 1.18 to 1.19 [\#45](https://github.com/tj-python/github-deploy/pull/45) ([dependabot[bot]](https://github.com/apps/dependabot)) +- Upgraded v1.1.0 → v1.1.1 [\#44](https://github.com/tj-python/github-deploy/pull/44) ([jackton1](https://github.com/jackton1)) + +## [v1.1.1](https://github.com/tj-python/github-deploy/tree/v1.1.1) (2023-04-05) + +[Full Changelog](https://github.com/tj-python/github-deploy/compare/v1.1.0...v1.1.1) + +**Merged pull requests:** + +- Bump peter-evans/create-pull-request from 4 to 5 [\#43](https://github.com/tj-python/github-deploy/pull/43) ([dependabot[bot]](https://github.com/apps/dependabot)) +- Bump tj-actions/github-changelog-generator from 1.17 to 1.18 [\#42](https://github.com/tj-python/github-deploy/pull/42) ([dependabot[bot]](https://github.com/apps/dependabot)) +- Bump codacy/codacy-analysis-cli-action from 4.2.0 to 4.3.0 [\#41](https://github.com/tj-python/github-deploy/pull/41) ([dependabot[bot]](https://github.com/apps/dependabot)) +- Bump pascalgn/automerge-action from 0.15.5 to 0.15.6 [\#40](https://github.com/tj-python/github-deploy/pull/40) ([dependabot[bot]](https://github.com/apps/dependabot)) +- Bump tj-actions/semver-diff from 2.4.0 to 2.4.1 [\#39](https://github.com/tj-python/github-deploy/pull/39) ([dependabot[bot]](https://github.com/apps/dependabot)) +- Bump tj-actions/semver-diff from 2.1.0 to 2.4.0 [\#38](https://github.com/tj-python/github-deploy/pull/38) ([dependabot[bot]](https://github.com/apps/dependabot)) +- Upgraded v1.0.2 → v1.1.0 [\#37](https://github.com/tj-python/github-deploy/pull/37) ([jackton1](https://github.com/jackton1)) + +## [v1.1.0](https://github.com/tj-python/github-deploy/tree/v1.1.0) (2023-01-06) + +[Full Changelog](https://github.com/tj-python/github-deploy/compare/v1.0.2...v1.1.0) + +**Merged pull requests:** + +- feat: update api url and add new features [\#36](https://github.com/tj-python/github-deploy/pull/36) ([jackton1](https://github.com/jackton1)) +- feat: add support for updating existing files [\#32](https://github.com/tj-python/github-deploy/pull/32) ([jackton1](https://github.com/jackton1)) +- Bump tj-actions/github-changelog-generator from 1.15 to 1.17 [\#31](https://github.com/tj-python/github-deploy/pull/31) ([dependabot[bot]](https://github.com/apps/dependabot)) +- Add CodeQL workflow for GitHub code scanning [\#30](https://github.com/tj-python/github-deploy/pull/30) ([lgtm-com[bot]](https://github.com/apps/lgtm-com)) +- Bump hmarr/auto-approve-action from 2 to 3 [\#29](https://github.com/tj-python/github-deploy/pull/29) ([dependabot[bot]](https://github.com/apps/dependabot)) +- Upgraded v1.0.1 → v1.0.2 [\#28](https://github.com/tj-python/github-deploy/pull/28) ([jackton1](https://github.com/jackton1)) + +## [v1.0.2](https://github.com/tj-python/github-deploy/tree/v1.0.2) (2022-10-28) + +[Full Changelog](https://github.com/tj-python/github-deploy/compare/v1.0.1...v1.0.2) + +**Merged pull requests:** + +- chore: upgrade required python version to 3.7 [\#27](https://github.com/tj-python/github-deploy/pull/27) ([jackton1](https://github.com/jackton1)) +- chore: reformatted and restructured modules [\#26](https://github.com/tj-python/github-deploy/pull/26) ([jackton1](https://github.com/jackton1)) +- fix: bug with listing repositories [\#25](https://github.com/tj-python/github-deploy/pull/25) ([jackton1](https://github.com/jackton1)) +- Bump pascalgn/automerge-action from 0.15.3 to 0.15.5 [\#24](https://github.com/tj-python/github-deploy/pull/24) ([dependabot[bot]](https://github.com/apps/dependabot)) +- Bump codacy/codacy-analysis-cli-action from 4.1.0 to 4.2.0 [\#23](https://github.com/tj-python/github-deploy/pull/23) ([dependabot[bot]](https://github.com/apps/dependabot)) +- Bump tj-actions/semver-diff from 2.0.0 to 2.1.0 [\#22](https://github.com/tj-python/github-deploy/pull/22) ([dependabot[bot]](https://github.com/apps/dependabot)) +- Bump tj-actions/github-changelog-generator from 1.14 to 1.15 [\#21](https://github.com/tj-python/github-deploy/pull/21) ([dependabot[bot]](https://github.com/apps/dependabot)) +- Bump codacy/codacy-analysis-cli-action from 4.0.2 to 4.1 [\#20](https://github.com/tj-python/github-deploy/pull/20) ([dependabot[bot]](https://github.com/apps/dependabot)) +- Bump tj-actions/github-changelog-generator from 1.13 to 1.14 [\#19](https://github.com/tj-python/github-deploy/pull/19) ([dependabot[bot]](https://github.com/apps/dependabot)) +- Bump codacy/codacy-analysis-cli-action from 1.1.0 to 4.0.2 [\#18](https://github.com/tj-python/github-deploy/pull/18) ([dependabot[bot]](https://github.com/apps/dependabot)) +- chore: Fixed lint errors. [\#17](https://github.com/tj-python/github-deploy/pull/17) ([jackton1](https://github.com/jackton1)) +- Update README.md [\#16](https://github.com/tj-python/github-deploy/pull/16) ([jackton1](https://github.com/jackton1)) +- Upgraded v1.0.0 → v1.0.1 [\#15](https://github.com/tj-python/github-deploy/pull/15) ([jackton1](https://github.com/jackton1)) + +## [v1.0.1](https://github.com/tj-python/github-deploy/tree/v1.0.1) (2022-06-12) + +[Full Changelog](https://github.com/tj-python/github-deploy/compare/v1.0.0...v1.0.1) + +**Merged pull requests:** + +- Bump actions/setup-python from 2 to 4 [\#14](https://github.com/tj-python/github-deploy/pull/14) ([dependabot[bot]](https://github.com/apps/dependabot)) +- Bump peter-evans/create-pull-request from 3 to 4 [\#13](https://github.com/tj-python/github-deploy/pull/13) ([dependabot[bot]](https://github.com/apps/dependabot)) +- Bump tj-actions/semver-diff from 1.2.0 to 2.0.0 [\#12](https://github.com/tj-python/github-deploy/pull/12) ([dependabot[bot]](https://github.com/apps/dependabot)) +- Bump tj-actions/github-changelog-generator from 1.8 to 1.13 [\#11](https://github.com/tj-python/github-deploy/pull/11) ([dependabot[bot]](https://github.com/apps/dependabot)) +- Bump actions/checkout from 2 to 3 [\#10](https://github.com/tj-python/github-deploy/pull/10) ([dependabot[bot]](https://github.com/apps/dependabot)) +- feat: Improve error handling [\#9](https://github.com/tj-python/github-deploy/pull/9) ([jackton1](https://github.com/jackton1)) +- Upgraded 0.0.9 → v1.0.0 [\#8](https://github.com/tj-python/github-deploy/pull/8) ([jackton1](https://github.com/jackton1)) + ## [v1.0.0](https://github.com/tj-python/github-deploy/tree/v1.0.0) (2022-02-12) [Full Changelog](https://github.com/tj-python/github-deploy/compare/0.0.9...v1.0.0) @@ -22,15 +100,15 @@ ## [0.0.7](https://github.com/tj-python/github-deploy/tree/0.0.7) (2021-11-15) -[Full Changelog](https://github.com/tj-python/github-deploy/compare/0.0.5...0.0.7) +[Full Changelog](https://github.com/tj-python/github-deploy/compare/0.0.6...0.0.7) -## [0.0.5](https://github.com/tj-python/github-deploy/tree/0.0.5) (2021-11-15) +## [0.0.6](https://github.com/tj-python/github-deploy/tree/0.0.6) (2021-11-15) -[Full Changelog](https://github.com/tj-python/github-deploy/compare/0.0.6...0.0.5) +[Full Changelog](https://github.com/tj-python/github-deploy/compare/0.0.5...0.0.6) -## [0.0.6](https://github.com/tj-python/github-deploy/tree/0.0.6) (2021-11-15) +## [0.0.5](https://github.com/tj-python/github-deploy/tree/0.0.5) (2021-11-15) -[Full Changelog](https://github.com/tj-python/github-deploy/compare/0.0.4...0.0.6) +[Full Changelog](https://github.com/tj-python/github-deploy/compare/0.0.4...0.0.5) **Merged pull requests:** diff --git a/Makefile b/Makefile index 2904151..b7e4461 100644 --- a/Makefile +++ b/Makefile @@ -36,8 +36,8 @@ release: dist ## package and upload a release @twine upload dist/* dist: clean install-deploy ## builds source and wheel package - @pip install twine==3.4.1 - @python setup.py sdist bdist_wheel + @pip install build twine + @python -m build increase-version: guard-PART ## Increase project version @bump2version $(PART) diff --git a/README.md b/README.md index b099986..b014e1a 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,9 @@ +[![Codacy Badge](https://app.codacy.com/project/badge/Grade/867aeabe457f4367b9e0013b713add6b)](https://www.codacy.com/gh/tj-python/github-deploy/dashboard?utm_source=github.com&utm_medium=referral&utm_content=tj-python/github-deploy&utm_campaign=Badge_Grade) [![PyPI version](https://badge.fury.io/py/github-deploy.svg)](https://badge.fury.io/py/github-deploy) -[![Upload Python Package](https://github.com/tj-python/github-deploy/actions/workflows/deploy.yml/badge.svg)](https://github.com/tj-python/github-deploy/actions/workflows/deploy.yml) [![Downloads](https://pepy.tech/badge/github-deploy)](https://pepy.tech/project/github-deploy) -[![Total alerts](https://img.shields.io/lgtm/alerts/g/tj-python/github-deploy.svg?logo=lgtm&logoWidth=18)](https://lgtm.com/projects/g/tj-python/github-deploy/alerts/) -[![Language grade: Python](https://img.shields.io/lgtm/grade/python/g/tj-python/github-deploy.svg?logo=lgtm&logoWidth=18)](https://lgtm.com/projects/g/tj-python/github-deploy/context:python) +[![Upload Python Package](https://github.com/tj-python/github-deploy/actions/workflows/deploy.yml/badge.svg)](https://github.com/tj-python/github-deploy/actions/workflows/deploy.yml) +[![Downloads](https://static.pepy.tech/badge/github-deploy)](https://pepy.tech/project/github-deploy) + + # github-deploy @@ -12,7 +14,6 @@ This can introduce a number challenges one of which is maintaining consistency a > For example adding a github action or maintaing a consistent pull request template accross your organization. - ## Solution `github-deploy` makes maintaining such configurations as easy as a single command. diff --git a/github_deploy/__init__.py b/github_deploy/__init__.py index 0260537..8db66d3 100644 --- a/github_deploy/__init__.py +++ b/github_deploy/__init__.py @@ -1 +1 @@ -__path__ = __import__('pkgutil').extend_path(__path__, __name__) \ No newline at end of file +__path__ = __import__("pkgutil").extend_path(__path__, __name__) diff --git a/github_deploy/commands/__init__.py b/github_deploy/commands/__init__.py index 0260537..8db66d3 100644 --- a/github_deploy/commands/__init__.py +++ b/github_deploy/commands/__init__.py @@ -1 +1 @@ -__path__ = __import__('pkgutil').extend_path(__path__, __name__) \ No newline at end of file +__path__ = __import__("pkgutil").extend_path(__path__, __name__) diff --git a/github_deploy/commands/_constants.py b/github_deploy/commands/_constants.py index 1249c71..174b0a3 100644 --- a/github_deploy/commands/_constants.py +++ b/github_deploy/commands/_constants.py @@ -1,2 +1,2 @@ -REPOS_URL = "https://api.github.com/search/repositories?q=org:{org}" -BASE_URL = "https://api.github.com/repos/{repo}/contents/{path}" +REPOS_URL = "https://api.github.com/users/{org}/repos?per_page=100" +FILE_CONTENTS_URL = "https://api.github.com/repos/{repo}/contents/{path}" diff --git a/github_deploy/commands/_http_utils.py b/github_deploy/commands/_http_utils.py new file mode 100644 index 0000000..539b9ec --- /dev/null +++ b/github_deploy/commands/_http_utils.py @@ -0,0 +1,50 @@ +import ssl + +import certifi + + +async def get(*, session, url, headers=None, skip_missing=False): + ssl_context = ssl.create_default_context(cafile=certifi.where()) + + async with session.get( + url, + headers=headers, + timeout=70, + ssl_context=ssl_context, + raise_for_status=not skip_missing, + ) as response: + if skip_missing and response.status == 404: + return {} + + value = await response.json() + return value + + +async def put(*, session, url, data, headers=None): + ssl_context = ssl.create_default_context(cafile=certifi.where()) + + async with session.put( + url, + json=data, + headers=headers, + timeout=70, + ssl_context=ssl_context, + raise_for_status=True, + ) as response: + value = await response.json() + return value + + +async def delete(*, session, url, data, headers=None): + ssl_context = ssl.create_default_context(cafile=certifi.where()) + + async with session.delete( + url, + json=data, + headers=headers, + timeout=70, + ssl_context=ssl_context, + raise_for_status=True, + ) as response: + value = await response.json() + return value diff --git a/github_deploy/commands/_repo_utils.py b/github_deploy/commands/_repo_utils.py new file mode 100644 index 0000000..a7aa389 --- /dev/null +++ b/github_deploy/commands/_repo_utils.py @@ -0,0 +1,135 @@ +import base64 +import os + +import asyncclick as click +from aiofiles import os as aiofiles_os, open as aiofiles_open + +from github_deploy.commands._constants import REPOS_URL, FILE_CONTENTS_URL +from github_deploy.commands._http_utils import get, delete, put +from github_deploy.commands._utils import get_headers + + +async def list_repos(*, session, org, token): + url = REPOS_URL.format(org=org) + click.echo(f"Retrieving repos at {url}") + response = await get(session=session, url=url, headers=get_headers(token=token)) + return response + + +async def delete_content( + *, + session, + repo, + dest, + token, + semaphore, + exists, + current_sha, +): + data = {"message": f"Deleted {dest}"} + if exists: + data["sha"] = current_sha + + url = FILE_CONTENTS_URL.format(repo=repo, path=dest) + + async with semaphore: + response = await delete( + session=session, url=url, data=data, headers=get_headers(token=token) + ) + + return response + + +async def check_exists(*, session, repo, dest, token, semaphore, skip_missing): + url = FILE_CONTENTS_URL.format(repo=repo, path=dest) + + async with semaphore: + response = await get( + session=session, + url=url, + headers=get_headers(token=token), + skip_missing=skip_missing, + ) + + return response + + +async def upload_content( + *, + session, + repo, + source, + dest, + token, + semaphore, + exists, + only_update, + current_sha, + current_content +): + async with semaphore: + async with aiofiles_open(source, mode="rb") as f: + output = await f.read() + base64_content = base64.b64encode(output).decode("ascii") + + if current_content == base64_content: + click.echo( + click.style( + f"Skipped uploading {source} to {repo}/{dest}: No changes detected.", + fg="yellow", + bold=True, + ) + ) + return + else: + if exists: + click.echo( + click.style( + "Storing backup of existing file at {repo}/{path}...".format( + repo=repo, path=dest + ), + fg="cyan", + ), + ) + + dirname, filename = os.path.split(f"{repo}/{dest}") + + await aiofiles_os.makedirs(dirname, exist_ok=True) + + async with aiofiles_open(f"{dirname}/{filename}", mode="wb") as f: + await f.write(base64.b64decode(current_content)) + elif only_update: + click.echo( + click.style( + f"Updates only: Skipped uploading {source} to {repo}/{dest}. File does not exist.", + fg="yellow", + bold=True, + ) + ) + return + + data = { + "message": f"Updated {dest}" + if exists + else f"Added {dest}", + "content": base64_content, + } + if exists: + data["sha"] = current_sha + + url = FILE_CONTENTS_URL.format(repo=repo, path=dest) + + click.echo( + click.style( + f"Uploading {source} to {repo}/{dest}...", + fg="green", + bold=True, + ) + ) + + async with semaphore: + response = await put( + session=session, url=url, data=data, headers=get_headers(token=token) + ) + + return response diff --git a/github_deploy/commands/_utils.py b/github_deploy/commands/_utils.py index 67cdb6b..afc4b79 100644 --- a/github_deploy/commands/_utils.py +++ b/github_deploy/commands/_utils.py @@ -1,6 +1,17 @@ def get_repo(*, org, project): - return "{org}/{project}".format(project=project, org=org) + return f"{org}/{project}" def can_upload(*, repo, include_private): - return True if include_private and repo['private'] == True else not repo['private'] + return ( + True + if include_private and repo["private"] is True + else not repo["private"] + ) + + +def get_headers(*, token): + return { + "Authorization": f"Bearer {token}", + "Accept": "application/vnd.github+json", + } diff --git a/github_deploy/commands/delete.py b/github_deploy/commands/delete.py index f85c9c0..f34a8ef 100644 --- a/github_deploy/commands/delete.py +++ b/github_deploy/commands/delete.py @@ -1,88 +1,13 @@ import asyncio -import ssl import aiohttp import asyncclick as click -import certifi -from github_deploy.commands._constants import BASE_URL, REPOS_URL +from github_deploy.commands._repo_utils import list_repos, delete_content, check_exists from github_deploy.commands._utils import get_repo -async def get(*, session, url, headers=None, skip_missing=False): - ssl_context = ssl.create_default_context(cafile=certifi.where()) - - async with session.get( - url, - headers=headers, - timeout=70, - ssl_context=ssl_context, - raise_for_status=not skip_missing, - ) as response: - if skip_missing and response.status == 404: - return {} - - value = await response.json() - return value - - -async def delete(*, session, url, data, headers=None): - ssl_context = ssl.create_default_context(cafile=certifi.where()) - - async with session.delete( - url, - json=data, - headers=headers, - timeout=70, - ssl_context=ssl_context, - raise_for_status=True, - ) as response: - value = await response.json() - return value - - -async def delete_content( - *, - session, - repo, - dest, - token, - semaphore, - exists, - current_sha, -): - headers = { - "Authorization": "token {token}".format(token=token), - "Accept": "application/vnd.github.v3+json", - } - - data = {"message": "Deleted {}".format(dest)} - if exists: - data["sha"] = current_sha - - url = BASE_URL.format(repo=repo, path=dest) - - async with semaphore: - response = await delete(session=session, url=url, data=data, headers=headers) - - return response - - -async def check_exists(*, session, repo, dest, token, semaphore, skip_missing): - headers = {"Authorization": "token {token}".format(token=token)} - url = BASE_URL.format(repo=repo, path=dest) - - async with semaphore: - response = await get( - session=session, url=url, headers=headers, skip_missing=skip_missing - ) - - return response - - -async def handle_file_delete( - *, repo, dest, token, semaphore, session -): +async def handle_file_delete(*, repo, dest, token, semaphore, session): check_exists_response = await check_exists( session=session, repo=repo, @@ -114,7 +39,7 @@ async def handle_file_delete( exists=exists, current_sha=current_sha, ) - + if delete_response: return click.style( "Successfully deleted contents at {repo}/{dest}".format( @@ -124,25 +49,14 @@ async def handle_file_delete( fg="green", bold=True, ) - + return click.style( - "No content found at {repo}/{dest}".format(repo=repo, dest=dest), + f"No content found at {repo}/{dest}", fg="blue", bold=True, ) -async def list_repos(*, session, org, token): - headers = { - "Authorization": "token {token}".format(token=token), - "Accept": "application/vnd.github.v3+json", - } - url = REPOS_URL.format(org=org) - click.echo("Retrieving repos at {}".format(url)) - response = await get(session=session, url=url, headers=headers) - return response - - @click.command() @click.option( "--org", @@ -154,7 +68,7 @@ async def list_repos(*, session, org, token): prompt=click.style("Enter your personal access token", bold=True), help="Personal Access token with read and write access to org.", hide_input=True, - envvar='TOKEN', + envvar="TOKEN", ) @click.option( "--dest", @@ -173,16 +87,23 @@ async def main(org, token, dest): response = await list_repos(org=org, token=token, session=session) repos = [ get_repo(org=org, project=v["name"]) - for v in response["items"] + for v in response if not v["archived"] ] click.echo( click.style( - "Found '{}' repositories non archived repositories".format(len(repos)), + "Found '{}' repositories non archived repositories".format( + len(repos) + ), fg="green", ) ) - click.echo(click.style('Deleting "{path}" for all repositories:'.format(path=dest), fg="blue")) + click.echo( + click.style( + f'Deleting "{dest}" for all repositories:', + fg="blue", + ) + ) click.echo("\n".join(repos)) c = click.prompt(click.style("Continue? [YN] ", fg="blue")) diff --git a/github_deploy/commands/upload.py b/github_deploy/commands/upload.py index 47d453c..52813b0 100644 --- a/github_deploy/commands/upload.py +++ b/github_deploy/commands/upload.py @@ -1,103 +1,14 @@ import asyncio -import base64 -import ssl -import aiofiles import aiohttp import asyncclick as click -import certifi -from github_deploy.commands._constants import BASE_URL, REPOS_URL +from github_deploy.commands._repo_utils import list_repos, check_exists, upload_content from github_deploy.commands._utils import get_repo, can_upload -async def get(*, session, url, headers=None, skip_missing=False): - ssl_context = ssl.create_default_context(cafile=certifi.where()) - - async with session.get( - url, - headers=headers, - timeout=70, - ssl_context=ssl_context, - raise_for_status=not skip_missing, - ) as response: - if skip_missing and response.status == 404: - return {} - - value = await response.json() - return value - - -async def put(*, session, url, data, headers=None): - ssl_context = ssl.create_default_context(cafile=certifi.where()) - - async with session.put( - url, - json=data, - headers=headers, - timeout=70, - ssl_context=ssl_context, - raise_for_status=True, - ) as response: - value = await response.json() - return value - - -async def upload_content( - *, - session, - repo, - source, - dest, - token, - semaphore, - exists, - current_sha, - current_content -): - headers = { - "Authorization": "token {token}".format(token=token), - "Accept": "application/vnd.github.v3+json", - } - - async with semaphore: - async with aiofiles.open(source, mode="rb") as f: - output = await f.read() - base64_content = base64.b64encode(output).decode("ascii") - - if current_content == base64_content: - click.echo("Skipping: Contents are the same.") - return - - data = { - "message": "Updated {}".format(dest) if exists else "Added {}".format(dest), - "content": base64_content, - } - if exists: - data["sha"] = current_sha - - url = BASE_URL.format(repo=repo, path=dest) - - async with semaphore: - response = await put(session=session, url=url, data=data, headers=headers) - - return response - - -async def check_exists(*, session, repo, dest, token, semaphore, skip_missing): - headers = {"Authorization": "token {token}".format(token=token)} - url = BASE_URL.format(repo=repo, path=dest) - - async with semaphore: - response = await get( - session=session, url=url, headers=headers, skip_missing=skip_missing - ) - - return response - - async def handle_file_upload( - *, repo, source, dest, overwrite, token, semaphore, session + *, repo, source, dest, overwrite, only_update, token, semaphore, session ): check_exists_response = await check_exists( session=session, @@ -112,19 +23,18 @@ async def handle_file_upload( current_content = check_exists_response.get("content") exists = current_sha is not None - if exists and not overwrite: - return click.style( - "Skipped uploading {source} to {repo}/{path}: Found an existing copy.".format( - source=source, - repo=repo, - path=dest, - ), - fg="blue", - bold=True - ) - - else: - if exists: + if exists: + if not overwrite: + click.style( + "Skipped uploading {source} to {repo}/{path}: Found an existing copy.".format( + source=source, + repo=repo, + path=dest, + ), + fg="blue", + bold=True, + ) + else: click.echo( click.style( "Found an existing copy at {repo}/{path} overwriting it's contents...".format( @@ -134,53 +44,43 @@ async def handle_file_upload( ), ) - upload_response = await upload_content( - session=session, - repo=repo, - source=source, - dest=dest, - token=token, - semaphore=semaphore, - exists=exists, - current_sha=current_sha, - current_content=current_content, - ) - - if upload_response: - return click.style( - "Successfully uploaded '{source}' to {repo}/{dest}".format( - source=upload_response["content"]["name"], - repo=repo, - dest=upload_response["content"]["path"], - ), - fg="green", - bold=True - ) - + upload_response = await upload_content( + session=session, + repo=repo, + source=source, + dest=dest, + token=token, + semaphore=semaphore, + exists=exists, + only_update=only_update, + current_sha=current_sha, + current_content=current_content, + ) -async def list_repos(*, session, org, token): - headers = { - "Authorization": "token {token}".format(token=token), - "Accept": "application/vnd.github.v3+json", - } - url = REPOS_URL.format(org=org) - click.echo("Retrieving repos at {}".format(url)) - response = await get(session=session, url=url, headers=headers) - return response + if upload_response: + return click.style( + "Successfully uploaded '{source}' to {repo}/{dest}".format( + source=upload_response["content"]["name"], + repo=repo, + dest=upload_response["content"]["path"], + ), + fg="green", + bold=True, + ) @click.command() -@click.option( - "--org", - prompt=click.style("Enter your github user/organization", bold=True), - help="The github organization.", -) @click.option( "--token", prompt=click.style("Enter your personal access token", bold=True), help="Personal Access token with read and write access to org.", hide_input=True, - envvar='TOKEN', + envvar="TOKEN", +) +@click.option( + "--org", + prompt=click.style("Enter your github user/organization", bold=True), + help="The github organization.", ) @click.option( "--source", @@ -195,17 +95,33 @@ async def list_repos(*, session, org, token): ) @click.option( "--overwrite/--no-overwrite", - prompt=click.style("Should we overwrite existing contents at this path", fg="blue"), + prompt=click.style( + "Should we overwrite existing contents at this path", fg="blue" + ), + is_flag=True, + show_default=True, help="Overwrite existing files.", default=False, ) +@click.option( + "--only-update/--no-only-update", + prompt=click.style( + "Should we only update existing files at this path", fg="blue" + ), + is_flag=True, + show_default=True, + help="Only update existing files.", + default=False, +) @click.option( "--private/--no-private", prompt=click.style("Should we Include private repositories", bold=True), + is_flag=True, + show_default=True, help="Upload files to private repositories.", default=True, ) -async def main(org, token, source, dest, overwrite, private): +async def main(org, token, source, dest, overwrite, only_update, private): """Upload a file to all repositories owned by an organization/user.""" # create instance of Semaphore: max concurrent requests. semaphore = asyncio.Semaphore(1000) @@ -216,13 +132,16 @@ async def main(org, token, source, dest, overwrite, private): response = await list_repos(org=org, token=token, session=session) repos = [ get_repo(org=org, project=r["name"]) - for r in response["items"] - if not r["archived"] and can_upload(repo=r, include_private=private) + for r in response + if not r["archived"] + and can_upload(repo=r, include_private=private) ] - repo_type = 'public and private' if private else 'public' + repo_type = "public and private" if private else "public" click.echo( click.style( - "Found '{}' repositories non archived {} repositories:".format(len(repos), repo_type), + "Found '{}' repositories non archived {} repositories:".format( + len(repos), repo_type + ), fg="green", ) ) @@ -237,14 +156,21 @@ async def main(org, token, source, dest, overwrite, private): fg="bright_red", ) ) - deploy_msg = ( - 'Deploying "{source}" to "{path}" for all repositories'.format(source=source, path=dest) - if overwrite - else 'Deploying "{source}" to repositories that don\'t already have contents at "{path}"'.format( - source=source, - path=dest + + if overwrite: + if only_update: + deploy_msg = "Updating '{source}' for existing files located at '{dest}'".format( + source=source, dest=dest + ) + else: + deploy_msg = "Overwriting '{dest}' with '{source}'".format( + source=source, dest=dest + ) + else: + deploy_msg = "Deploying '{source}' to repositories that don\'t already have contents at '{path}'".format( + source=source, path=dest ) - ) + click.echo(click.style(deploy_msg, fg="blue")) c = click.prompt(click.style("Continue? [YN] ", fg="blue")) @@ -265,6 +191,7 @@ async def main(org, token, source, dest, overwrite, private): dest=dest, token=token, overwrite=overwrite, + only_update=only_update, session=session, semaphore=semaphore, ) diff --git a/github_deploy/main.py b/github_deploy/main.py index 2acef84..c93ad51 100644 --- a/github_deploy/main.py +++ b/github_deploy/main.py @@ -1,35 +1,42 @@ -import asyncclick as click import os -plugin_folder = os.path.join(os.path.dirname(__file__), 'commands') +import asyncclick as click + +plugin_folder = os.path.join(os.path.dirname(__file__), "commands") class GithubDeploy(click.MultiCommand): - def list_commands(self, ctx): rv = [] for filename in os.listdir(plugin_folder): - if filename.endswith('.py') and not filename.startswith('__init__') and not filename.startswith('_'): + if ( + filename.endswith(".py") + and not filename.startswith("__init__") + and not filename.startswith("_") + ): rv.append(filename[:-3]) rv.sort() return rv def get_command(self, ctx, name): ns = {} - fn = os.path.join(plugin_folder, name + '.py') + fn = os.path.join(plugin_folder, name + ".py") if os.path.exists(fn): with open(fn) as f: - code = compile(f.read(), fn, 'exec') + code = compile(f.read(), fn, "exec") eval(code, ns, ns) - return ns['main'] + return ns["main"] - ctx.fail("Invalid Command: {name}".format(name=name)) + ctx.fail(f"Invalid Command \"{name}\"") main = GithubDeploy( - help='Deploy changes to multiple github repositories using a single command.', + help=( + "Deploy changes to multiple github repositories using " + "a single command." + ), ) -if __name__ == '__main__': +if __name__ == "__main__": main() diff --git a/setup.py b/setup.py index d5a2463..29124f3 100644 --- a/setup.py +++ b/setup.py @@ -23,7 +23,7 @@ setup( name="github-deploy", - version="1.0.0", + version="1.2.0", description="Deploy yaml files to a large number of repositories in seconds.", long_description=LONG_DESCRIPTION, long_description_content_type=LONG_DESCRIPTION_TYPE, @@ -34,12 +34,18 @@ "gh-deploy=github_deploy.main:main", ], }, - keywords=["yaml", "deploy", "poly repository", "github", "single configuration"], + keywords=[ + "yaml", + "deploy", + "poly repository", + "github", + "single configuration", + ], author="Tonye Jack", author_email="jtonye@ymail.com", license="MIT", packages=find_packages(), - python_requires='>=3.6', + python_requires=">=3.7", extras_require=extras_require, install_requires=[ "asyncclick",