diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 860be11..741cf0e 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 1.0.0 +current_version = 1.0.1 commit = True tag = False diff --git a/.github/dependabot.yml b/.github/dependabot.yml index afb98ae..ae760f7 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -1,7 +1,12 @@ version: 2 updates: -- package-ecosystem: github-actions - directory: "/" - schedule: - interval: daily - open-pull-requests-limit: 10 + - package-ecosystem: github-actions + directory: "/" + schedule: + interval: daily + open-pull-requests-limit: 10 + - package-ecosystem: pip + directory: "/" + schedule: + interval: daily + open-pull-requests-limit: 10 diff --git a/.github/workflows/auto-merge.yml b/.github/workflows/auto-merge.yml index 59f687f..cb1f956 100644 --- a/.github/workflows/auto-merge.yml +++ b/.github/workflows/auto-merge.yml @@ -16,7 +16,7 @@ jobs: github.actor == 'renovate' steps: - name: automerge - uses: pascalgn/automerge-action@v0.15.3 + uses: pascalgn/automerge-action@v0.15.5 env: GITHUB_TOKEN: ${{ secrets.PAT_TOKEN }} MERGE_METHOD: "rebase" diff --git a/.github/workflows/codacy.yml b/.github/workflows/codacy.yml new file mode 100644 index 0000000..ae02244 --- /dev/null +++ b/.github/workflows/codacy.yml @@ -0,0 +1,60 @@ +# This workflow uses actions that are not certified by GitHub. +# They are provided by a third-party and are governed by +# separate terms of service, privacy policy, and support +# documentation. + +# This workflow checks out code, performs a Codacy security scan +# and integrates the results with the +# GitHub Advanced Security code scanning feature. For more information on +# the Codacy security scan action usage and parameters, see +# https://github.com/codacy/codacy-analysis-cli-action. +# For more information on Codacy Analysis CLI in general, see +# https://github.com/codacy/codacy-analysis-cli. + +name: Codacy Security Scan + +on: + push: + branches: [ "main" ] + pull_request: + # The branches below must be a subset of the branches above + branches: [ "main" ] + schedule: + - cron: '39 21 * * 4' + +permissions: + contents: read + +jobs: + codacy-security-scan: + permissions: + contents: read # for actions/checkout to fetch code + security-events: write # for github/codeql-action/upload-sarif to upload SARIF results + name: Codacy Security Scan + runs-on: ubuntu-latest + steps: + # Checkout the repository to the GitHub Actions runner + - name: Checkout code + uses: actions/checkout@v3 + + # Execute Codacy Analysis CLI and generate a SARIF output with the security issues identified during the analysis + - name: Run Codacy Analysis CLI + uses: codacy/codacy-analysis-cli-action@d43127fe38d20c527dc1951ae5aea23148bab738 + with: + # Check https://github.com/codacy/codacy-analysis-cli#project-token to get your project token from your Codacy repository + # You can also omit the token and run the tools that support default configurations + project-token: ${{ secrets.CODACY_PROJECT_TOKEN }} + verbose: true + output: results.sarif + format: sarif + # Adjust severity of non-security issues + gh-code-scanning-compat: true + # Force 0 exit code to allow SARIF file generation + # This will handover control about PR rejection to the GitHub side + max-allowed-issues: 2147483647 + + # Upload the SARIF file generated in the previous step + - name: Upload SARIF results file + uses: github/codeql-action/upload-sarif@v2 + with: + sarif_file: results.sarif diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 07d47a7..b02f205 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -14,12 +14,12 @@ jobs: - name: Run semver-diff id: semver-diff - uses: tj-actions/semver-diff@v2.0.0 + uses: tj-actions/semver-diff@v2.1.0 - name: Set up Python uses: actions/setup-python@v4 with: - python-version: '3.6.x' + python-version: '3.7.x' - name: Upgrade pip run: pip install -U pip @@ -43,7 +43,7 @@ jobs: TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} - name: Generate CHANGELOG - uses: tj-actions/github-changelog-generator@v1.13 + uses: tj-actions/github-changelog-generator@v1.15 - name: Create Pull Request uses: peter-evans/create-pull-request@v4 diff --git a/CHANGELOG.md b/CHANGELOG.md index 4b51c49..27a9493 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,19 @@ # Changelog +## [v1.0.1](https://github.com/tj-python/github-deploy/tree/v1.0.1) (2022-06-12) + +[Full Changelog](https://github.com/tj-python/github-deploy/compare/v1.0.0...v1.0.1) + +**Merged pull requests:** + +- Bump actions/setup-python from 2 to 4 [\#14](https://github.com/tj-python/github-deploy/pull/14) ([dependabot[bot]](https://github.com/apps/dependabot)) +- Bump peter-evans/create-pull-request from 3 to 4 [\#13](https://github.com/tj-python/github-deploy/pull/13) ([dependabot[bot]](https://github.com/apps/dependabot)) +- Bump tj-actions/semver-diff from 1.2.0 to 2.0.0 [\#12](https://github.com/tj-python/github-deploy/pull/12) ([dependabot[bot]](https://github.com/apps/dependabot)) +- Bump tj-actions/github-changelog-generator from 1.8 to 1.13 [\#11](https://github.com/tj-python/github-deploy/pull/11) ([dependabot[bot]](https://github.com/apps/dependabot)) +- Bump actions/checkout from 2 to 3 [\#10](https://github.com/tj-python/github-deploy/pull/10) ([dependabot[bot]](https://github.com/apps/dependabot)) +- feat: Improve error handling [\#9](https://github.com/tj-python/github-deploy/pull/9) ([jackton1](https://github.com/jackton1)) +- Upgraded 0.0.9 → v1.0.0 [\#8](https://github.com/tj-python/github-deploy/pull/8) ([jackton1](https://github.com/jackton1)) + ## [v1.0.0](https://github.com/tj-python/github-deploy/tree/v1.0.0) (2022-02-12) [Full Changelog](https://github.com/tj-python/github-deploy/compare/0.0.9...v1.0.0) @@ -22,15 +36,15 @@ ## [0.0.7](https://github.com/tj-python/github-deploy/tree/0.0.7) (2021-11-15) -[Full Changelog](https://github.com/tj-python/github-deploy/compare/0.0.5...0.0.7) +[Full Changelog](https://github.com/tj-python/github-deploy/compare/0.0.6...0.0.7) -## [0.0.5](https://github.com/tj-python/github-deploy/tree/0.0.5) (2021-11-15) +## [0.0.6](https://github.com/tj-python/github-deploy/tree/0.0.6) (2021-11-15) -[Full Changelog](https://github.com/tj-python/github-deploy/compare/0.0.6...0.0.5) +[Full Changelog](https://github.com/tj-python/github-deploy/compare/0.0.5...0.0.6) -## [0.0.6](https://github.com/tj-python/github-deploy/tree/0.0.6) (2021-11-15) +## [0.0.5](https://github.com/tj-python/github-deploy/tree/0.0.5) (2021-11-15) -[Full Changelog](https://github.com/tj-python/github-deploy/compare/0.0.4...0.0.6) +[Full Changelog](https://github.com/tj-python/github-deploy/compare/0.0.4...0.0.5) **Merged pull requests:** diff --git a/Makefile b/Makefile index 2904151..b7e4461 100644 --- a/Makefile +++ b/Makefile @@ -36,8 +36,8 @@ release: dist ## package and upload a release @twine upload dist/* dist: clean install-deploy ## builds source and wheel package - @pip install twine==3.4.1 - @python setup.py sdist bdist_wheel + @pip install build twine + @python -m build increase-version: guard-PART ## Increase project version @bump2version $(PART) diff --git a/README.md b/README.md index b099986..2f9fd94 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,6 @@ This can introduce a number challenges one of which is maintaining consistency a > For example adding a github action or maintaing a consistent pull request template accross your organization. - ## Solution `github-deploy` makes maintaining such configurations as easy as a single command. diff --git a/github_deploy/__init__.py b/github_deploy/__init__.py index 0260537..8db66d3 100644 --- a/github_deploy/__init__.py +++ b/github_deploy/__init__.py @@ -1 +1 @@ -__path__ = __import__('pkgutil').extend_path(__path__, __name__) \ No newline at end of file +__path__ = __import__("pkgutil").extend_path(__path__, __name__) diff --git a/github_deploy/commands/__init__.py b/github_deploy/commands/__init__.py index 0260537..8db66d3 100644 --- a/github_deploy/commands/__init__.py +++ b/github_deploy/commands/__init__.py @@ -1 +1 @@ -__path__ = __import__('pkgutil').extend_path(__path__, __name__) \ No newline at end of file +__path__ = __import__("pkgutil").extend_path(__path__, __name__) diff --git a/github_deploy/commands/_http_utils.py b/github_deploy/commands/_http_utils.py new file mode 100644 index 0000000..539b9ec --- /dev/null +++ b/github_deploy/commands/_http_utils.py @@ -0,0 +1,50 @@ +import ssl + +import certifi + + +async def get(*, session, url, headers=None, skip_missing=False): + ssl_context = ssl.create_default_context(cafile=certifi.where()) + + async with session.get( + url, + headers=headers, + timeout=70, + ssl_context=ssl_context, + raise_for_status=not skip_missing, + ) as response: + if skip_missing and response.status == 404: + return {} + + value = await response.json() + return value + + +async def put(*, session, url, data, headers=None): + ssl_context = ssl.create_default_context(cafile=certifi.where()) + + async with session.put( + url, + json=data, + headers=headers, + timeout=70, + ssl_context=ssl_context, + raise_for_status=True, + ) as response: + value = await response.json() + return value + + +async def delete(*, session, url, data, headers=None): + ssl_context = ssl.create_default_context(cafile=certifi.where()) + + async with session.delete( + url, + json=data, + headers=headers, + timeout=70, + ssl_context=ssl_context, + raise_for_status=True, + ) as response: + value = await response.json() + return value diff --git a/github_deploy/commands/_repo_utils.py b/github_deploy/commands/_repo_utils.py new file mode 100644 index 0000000..438d970 --- /dev/null +++ b/github_deploy/commands/_repo_utils.py @@ -0,0 +1,93 @@ +import base64 + +import aiofiles +import asyncclick as click + +from github_deploy.commands._constants import REPOS_URL, BASE_URL +from github_deploy.commands._http_utils import get, delete, put +from github_deploy.commands._utils import get_headers + + +async def list_repos(*, session, org, token): + url = REPOS_URL.format(org=org) + click.echo(f"Retrieving repos at {url}") + response = await get(session=session, url=url, headers=get_headers(token=token)) + return response + + +async def delete_content( + *, + session, + repo, + dest, + token, + semaphore, + exists, + current_sha, +): + data = {"message": f"Deleted {dest}"} + if exists: + data["sha"] = current_sha + + url = BASE_URL.format(repo=repo, path=dest) + + async with semaphore: + response = await delete( + session=session, url=url, data=data, headers=get_headers(token=token) + ) + + return response + + +async def check_exists(*, session, repo, dest, token, semaphore, skip_missing): + url = BASE_URL.format(repo=repo, path=dest) + + async with semaphore: + response = await get( + session=session, + url=url, + headers=get_headers(token=token), + skip_missing=skip_missing, + ) + + return response + + +async def upload_content( + *, + session, + repo, + source, + dest, + token, + semaphore, + exists, + current_sha, + current_content +): + async with semaphore: + async with aiofiles.open(source, mode="rb") as f: + output = await f.read() + base64_content = base64.b64encode(output).decode("ascii") + + if current_content == base64_content: + click.echo("Skipping: Contents are the same.") + return + + data = { + "message": f"Updated {dest}" + if exists + else f"Added {dest}", + "content": base64_content, + } + if exists: + data["sha"] = current_sha + + url = BASE_URL.format(repo=repo, path=dest) + + async with semaphore: + response = await put( + session=session, url=url, data=data, headers=get_headers(token=token) + ) + + return response diff --git a/github_deploy/commands/_utils.py b/github_deploy/commands/_utils.py index 67cdb6b..afc4b79 100644 --- a/github_deploy/commands/_utils.py +++ b/github_deploy/commands/_utils.py @@ -1,6 +1,17 @@ def get_repo(*, org, project): - return "{org}/{project}".format(project=project, org=org) + return f"{org}/{project}" def can_upload(*, repo, include_private): - return True if include_private and repo['private'] == True else not repo['private'] + return ( + True + if include_private and repo["private"] is True + else not repo["private"] + ) + + +def get_headers(*, token): + return { + "Authorization": f"Bearer {token}", + "Accept": "application/vnd.github+json", + } diff --git a/github_deploy/commands/delete.py b/github_deploy/commands/delete.py index f85c9c0..9b7209e 100644 --- a/github_deploy/commands/delete.py +++ b/github_deploy/commands/delete.py @@ -1,88 +1,13 @@ import asyncio -import ssl import aiohttp import asyncclick as click -import certifi -from github_deploy.commands._constants import BASE_URL, REPOS_URL +from github_deploy.commands._repo_utils import list_repos, delete_content, check_exists from github_deploy.commands._utils import get_repo -async def get(*, session, url, headers=None, skip_missing=False): - ssl_context = ssl.create_default_context(cafile=certifi.where()) - - async with session.get( - url, - headers=headers, - timeout=70, - ssl_context=ssl_context, - raise_for_status=not skip_missing, - ) as response: - if skip_missing and response.status == 404: - return {} - - value = await response.json() - return value - - -async def delete(*, session, url, data, headers=None): - ssl_context = ssl.create_default_context(cafile=certifi.where()) - - async with session.delete( - url, - json=data, - headers=headers, - timeout=70, - ssl_context=ssl_context, - raise_for_status=True, - ) as response: - value = await response.json() - return value - - -async def delete_content( - *, - session, - repo, - dest, - token, - semaphore, - exists, - current_sha, -): - headers = { - "Authorization": "token {token}".format(token=token), - "Accept": "application/vnd.github.v3+json", - } - - data = {"message": "Deleted {}".format(dest)} - if exists: - data["sha"] = current_sha - - url = BASE_URL.format(repo=repo, path=dest) - - async with semaphore: - response = await delete(session=session, url=url, data=data, headers=headers) - - return response - - -async def check_exists(*, session, repo, dest, token, semaphore, skip_missing): - headers = {"Authorization": "token {token}".format(token=token)} - url = BASE_URL.format(repo=repo, path=dest) - - async with semaphore: - response = await get( - session=session, url=url, headers=headers, skip_missing=skip_missing - ) - - return response - - -async def handle_file_delete( - *, repo, dest, token, semaphore, session -): +async def handle_file_delete(*, repo, dest, token, semaphore, session): check_exists_response = await check_exists( session=session, repo=repo, @@ -114,7 +39,7 @@ async def handle_file_delete( exists=exists, current_sha=current_sha, ) - + if delete_response: return click.style( "Successfully deleted contents at {repo}/{dest}".format( @@ -124,25 +49,14 @@ async def handle_file_delete( fg="green", bold=True, ) - + return click.style( - "No content found at {repo}/{dest}".format(repo=repo, dest=dest), + f"No content found at {repo}/{dest}", fg="blue", bold=True, ) -async def list_repos(*, session, org, token): - headers = { - "Authorization": "token {token}".format(token=token), - "Accept": "application/vnd.github.v3+json", - } - url = REPOS_URL.format(org=org) - click.echo("Retrieving repos at {}".format(url)) - response = await get(session=session, url=url, headers=headers) - return response - - @click.command() @click.option( "--org", @@ -154,7 +68,7 @@ async def list_repos(*, session, org, token): prompt=click.style("Enter your personal access token", bold=True), help="Personal Access token with read and write access to org.", hide_input=True, - envvar='TOKEN', + envvar="TOKEN", ) @click.option( "--dest", @@ -178,11 +92,18 @@ async def main(org, token, dest): ] click.echo( click.style( - "Found '{}' repositories non archived repositories".format(len(repos)), + "Found '{}' repositories non archived repositories".format( + len(repos) + ), fg="green", ) ) - click.echo(click.style('Deleting "{path}" for all repositories:'.format(path=dest), fg="blue")) + click.echo( + click.style( + f'Deleting "{dest}" for all repositories:', + fg="blue", + ) + ) click.echo("\n".join(repos)) c = click.prompt(click.style("Continue? [YN] ", fg="blue")) diff --git a/github_deploy/commands/upload.py b/github_deploy/commands/upload.py index 47d453c..3baf4a3 100644 --- a/github_deploy/commands/upload.py +++ b/github_deploy/commands/upload.py @@ -1,101 +1,12 @@ import asyncio -import base64 -import ssl -import aiofiles import aiohttp import asyncclick as click -import certifi -from github_deploy.commands._constants import BASE_URL, REPOS_URL +from github_deploy.commands._repo_utils import list_repos, check_exists, upload_content from github_deploy.commands._utils import get_repo, can_upload -async def get(*, session, url, headers=None, skip_missing=False): - ssl_context = ssl.create_default_context(cafile=certifi.where()) - - async with session.get( - url, - headers=headers, - timeout=70, - ssl_context=ssl_context, - raise_for_status=not skip_missing, - ) as response: - if skip_missing and response.status == 404: - return {} - - value = await response.json() - return value - - -async def put(*, session, url, data, headers=None): - ssl_context = ssl.create_default_context(cafile=certifi.where()) - - async with session.put( - url, - json=data, - headers=headers, - timeout=70, - ssl_context=ssl_context, - raise_for_status=True, - ) as response: - value = await response.json() - return value - - -async def upload_content( - *, - session, - repo, - source, - dest, - token, - semaphore, - exists, - current_sha, - current_content -): - headers = { - "Authorization": "token {token}".format(token=token), - "Accept": "application/vnd.github.v3+json", - } - - async with semaphore: - async with aiofiles.open(source, mode="rb") as f: - output = await f.read() - base64_content = base64.b64encode(output).decode("ascii") - - if current_content == base64_content: - click.echo("Skipping: Contents are the same.") - return - - data = { - "message": "Updated {}".format(dest) if exists else "Added {}".format(dest), - "content": base64_content, - } - if exists: - data["sha"] = current_sha - - url = BASE_URL.format(repo=repo, path=dest) - - async with semaphore: - response = await put(session=session, url=url, data=data, headers=headers) - - return response - - -async def check_exists(*, session, repo, dest, token, semaphore, skip_missing): - headers = {"Authorization": "token {token}".format(token=token)} - url = BASE_URL.format(repo=repo, path=dest) - - async with semaphore: - response = await get( - session=session, url=url, headers=headers, skip_missing=skip_missing - ) - - return response - - async def handle_file_upload( *, repo, source, dest, overwrite, token, semaphore, session ): @@ -120,7 +31,7 @@ async def handle_file_upload( path=dest, ), fg="blue", - bold=True + bold=True, ) else: @@ -154,21 +65,10 @@ async def handle_file_upload( dest=upload_response["content"]["path"], ), fg="green", - bold=True + bold=True, ) -async def list_repos(*, session, org, token): - headers = { - "Authorization": "token {token}".format(token=token), - "Accept": "application/vnd.github.v3+json", - } - url = REPOS_URL.format(org=org) - click.echo("Retrieving repos at {}".format(url)) - response = await get(session=session, url=url, headers=headers) - return response - - @click.command() @click.option( "--org", @@ -180,7 +80,7 @@ async def list_repos(*, session, org, token): prompt=click.style("Enter your personal access token", bold=True), help="Personal Access token with read and write access to org.", hide_input=True, - envvar='TOKEN', + envvar="TOKEN", ) @click.option( "--source", @@ -195,7 +95,9 @@ async def list_repos(*, session, org, token): ) @click.option( "--overwrite/--no-overwrite", - prompt=click.style("Should we overwrite existing contents at this path", fg="blue"), + prompt=click.style( + "Should we overwrite existing contents at this path", fg="blue" + ), help="Overwrite existing files.", default=False, ) @@ -217,12 +119,15 @@ async def main(org, token, source, dest, overwrite, private): repos = [ get_repo(org=org, project=r["name"]) for r in response["items"] - if not r["archived"] and can_upload(repo=r, include_private=private) + if not r["archived"] + and can_upload(repo=r, include_private=private) ] - repo_type = 'public and private' if private else 'public' + repo_type = "public and private" if private else "public" click.echo( click.style( - "Found '{}' repositories non archived {} repositories:".format(len(repos), repo_type), + "Found '{}' repositories non archived {} repositories:".format( + len(repos), repo_type + ), fg="green", ) ) @@ -238,11 +143,12 @@ async def main(org, token, source, dest, overwrite, private): ) ) deploy_msg = ( - 'Deploying "{source}" to "{path}" for all repositories'.format(source=source, path=dest) + 'Deploying "{source}" to "{path}" for all repositories'.format( + source=source, path=dest + ) if overwrite else 'Deploying "{source}" to repositories that don\'t already have contents at "{path}"'.format( - source=source, - path=dest + source=source, path=dest ) ) click.echo(click.style(deploy_msg, fg="blue")) diff --git a/github_deploy/main.py b/github_deploy/main.py index 2acef84..c93ad51 100644 --- a/github_deploy/main.py +++ b/github_deploy/main.py @@ -1,35 +1,42 @@ -import asyncclick as click import os -plugin_folder = os.path.join(os.path.dirname(__file__), 'commands') +import asyncclick as click + +plugin_folder = os.path.join(os.path.dirname(__file__), "commands") class GithubDeploy(click.MultiCommand): - def list_commands(self, ctx): rv = [] for filename in os.listdir(plugin_folder): - if filename.endswith('.py') and not filename.startswith('__init__') and not filename.startswith('_'): + if ( + filename.endswith(".py") + and not filename.startswith("__init__") + and not filename.startswith("_") + ): rv.append(filename[:-3]) rv.sort() return rv def get_command(self, ctx, name): ns = {} - fn = os.path.join(plugin_folder, name + '.py') + fn = os.path.join(plugin_folder, name + ".py") if os.path.exists(fn): with open(fn) as f: - code = compile(f.read(), fn, 'exec') + code = compile(f.read(), fn, "exec") eval(code, ns, ns) - return ns['main'] + return ns["main"] - ctx.fail("Invalid Command: {name}".format(name=name)) + ctx.fail(f"Invalid Command \"{name}\"") main = GithubDeploy( - help='Deploy changes to multiple github repositories using a single command.', + help=( + "Deploy changes to multiple github repositories using " + "a single command." + ), ) -if __name__ == '__main__': +if __name__ == "__main__": main() diff --git a/setup.py b/setup.py index d5a2463..b8719f4 100644 --- a/setup.py +++ b/setup.py @@ -23,7 +23,7 @@ setup( name="github-deploy", - version="1.0.0", + version="1.0.1", description="Deploy yaml files to a large number of repositories in seconds.", long_description=LONG_DESCRIPTION, long_description_content_type=LONG_DESCRIPTION_TYPE, @@ -34,12 +34,18 @@ "gh-deploy=github_deploy.main:main", ], }, - keywords=["yaml", "deploy", "poly repository", "github", "single configuration"], + keywords=[ + "yaml", + "deploy", + "poly repository", + "github", + "single configuration", + ], author="Tonye Jack", author_email="jtonye@ymail.com", license="MIT", packages=find_packages(), - python_requires='>=3.6', + python_requires=">=3.7", extras_require=extras_require, install_requires=[ "asyncclick",