diff --git a/.cookiecutter.json b/.cookiecutter.json index ea65bfeb..d8d0ee46 100644 --- a/.cookiecutter.json +++ b/.cookiecutter.json @@ -10,12 +10,10 @@ "project_slug": "nautobot-plugin-golden-config", "repo_url": "https://github.com/nautobot/nautobot-plugin-golden-config", "base_url": "golden-config", - "min_nautobot_version": "1.4.0", - "max_nautobot_version": "1.9999", - "nautobot_version": "latest", + "min_nautobot_version": "2.0.0", + "max_nautobot_version": "2.9999", "camel_name": "NautobotGoldenConfig", "project_short_description": "A plugin for configuration on nautobot", - "version": "1.0.0", "model_class_name": "None", "open_source_license": "Apache-2.0", "docs_base_url": "https://docs.nautobot.com", @@ -27,7 +25,11 @@ "cookie_dir": "", "branch_prefix": "drift-manager", "pull_request_strategy": "create", - "post_actions": [] + "post_actions": [ + "black" + ], + "draft": true, + "baked_commit_ref": "c6963e367d835faf66a9f4ac63267b6ce5811577" } } } diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md index cbb194ae..8d0594cb 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.md +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -4,8 +4,8 @@ about: Report a reproducible bug in the current release of nautobot-golden-confi --- ### Environment -* Python version: -* Nautobot version: +* Python version: +* Nautobot version: * nautobot-golden-config version: diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md index a8d54617..3506acde 100644 --- a/.github/ISSUE_TEMPLATE/feature_request.md +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -5,7 +5,7 @@ about: Propose a new feature or enhancement --- ### Environment -* Nautobot version: +* Nautobot version: * nautobot-golden-config version: -## Change Notes +# Closes: # -## Justification +## What's Changed + + + +## To Do + + +- [ ] Explanation of Change(s) +- [ ] Attached Screenshots, Payload Example +- [ ] Unit, Integration Tests +- [ ] Documentation Updates (when adding/changing features) +- [ ] Example Plugin Updates (when adding/changing features) +- [ ] Outline Remaining Work, Constraints from Design diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 632b3dc2..573d26b9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -17,56 +17,67 @@ env: jobs: black: - runs-on: "ubuntu-20.04" + runs-on: "ubuntu-22.04" env: INVOKE_NAUTOBOT_GOLDEN_CONFIG_LOCAL: "True" steps: - name: "Check out repository code" - uses: "actions/checkout@v3" + uses: "actions/checkout@v4" - name: "Setup environment" uses: "networktocode/gh-action-setup-poetry-environment@v4" - name: "Linting: black" run: "poetry run invoke black" bandit: - runs-on: "ubuntu-20.04" + runs-on: "ubuntu-22.04" env: INVOKE_NAUTOBOT_GOLDEN_CONFIG_LOCAL: "True" steps: - name: "Check out repository code" - uses: "actions/checkout@v3" + uses: "actions/checkout@v4" - name: "Setup environment" uses: "networktocode/gh-action-setup-poetry-environment@v4" - name: "Linting: bandit" run: "poetry run invoke bandit" pydocstyle: - runs-on: "ubuntu-20.04" + runs-on: "ubuntu-22.04" env: INVOKE_NAUTOBOT_GOLDEN_CONFIG_LOCAL: "True" steps: - name: "Check out repository code" - uses: "actions/checkout@v3" + uses: "actions/checkout@v4" - name: "Setup environment" uses: "networktocode/gh-action-setup-poetry-environment@v4" - name: "Linting: pydocstyle" run: "poetry run invoke pydocstyle" flake8: - runs-on: "ubuntu-20.04" + runs-on: "ubuntu-22.04" env: INVOKE_NAUTOBOT_GOLDEN_CONFIG_LOCAL: "True" steps: - name: "Check out repository code" - uses: "actions/checkout@v3" + uses: "actions/checkout@v4" - name: "Setup environment" uses: "networktocode/gh-action-setup-poetry-environment@v4" - name: "Linting: flake8" run: "poetry run invoke flake8" + poetry: + runs-on: "ubuntu-22.04" + env: + INVOKE_NAUTOBOT_GOLDEN_CONFIG_LOCAL: "True" + steps: + - name: "Check out repository code" + uses: "actions/checkout@v4" + - name: "Setup environment" + uses: "networktocode/gh-action-setup-poetry-environment@v4" + - name: "Checking: poetry lock file" + run: "poetry run invoke lock --check" yamllint: - runs-on: "ubuntu-20.04" + runs-on: "ubuntu-22.04" env: INVOKE_NAUTOBOT_GOLDEN_CONFIG_LOCAL: "True" steps: - name: "Check out repository code" - uses: "actions/checkout@v3" + uses: "actions/checkout@v4" - name: "Setup environment" uses: "networktocode/gh-action-setup-poetry-environment@v4" - name: "Linting: yamllint" @@ -76,9 +87,10 @@ jobs: - "bandit" - "pydocstyle" - "flake8" + - "poetry" - "yamllint" - "black" - runs-on: "ubuntu-20.04" + runs-on: "ubuntu-22.04" strategy: fail-fast: true matrix: @@ -89,14 +101,14 @@ jobs: INVOKE_NAUTOBOT_GOLDEN_CONFIG_NAUTOBOT_VER: "${{ matrix.nautobot-version }}" steps: - name: "Check out repository code" - uses: "actions/checkout@v3" + uses: "actions/checkout@v4" - name: "Setup environment" uses: "networktocode/gh-action-setup-poetry-environment@v4" - name: "Set up Docker Buildx" id: "buildx" - uses: "docker/setup-buildx-action@v1" + uses: "docker/setup-buildx-action@v3" - name: "Build" - uses: "docker/build-push-action@v2" + uses: "docker/build-push-action@v5" with: builder: "${{ steps.buildx.outputs.name }}" context: "./" @@ -113,42 +125,80 @@ jobs: run: "cp development/creds.example.env development/creds.env" - name: "Linting: pylint" run: "poetry run invoke pylint" + check-migrations: + needs: + - "bandit" + - "pydocstyle" + - "flake8" + - "poetry" + - "yamllint" + - "black" + runs-on: "ubuntu-22.04" + strategy: + fail-fast: true + matrix: + python-version: ["3.11"] + nautobot-version: ["2.0.0"] + env: + INVOKE_NAUTOBOT_GOLDEN_CONFIG_PYTHON_VER: "${{ matrix.python-version }}" + INVOKE_NAUTOBOT_GOLDEN_CONFIG_NAUTOBOT_VER: "${{ matrix.nautobot-version }}" + steps: + - name: "Check out repository code" + uses: "actions/checkout@v4" + - name: "Setup environment" + uses: "networktocode/gh-action-setup-poetry-environment@v4" + - name: "Set up Docker Buildx" + id: "buildx" + uses: "docker/setup-buildx-action@v3" + - name: "Build" + uses: "docker/build-push-action@v5" + with: + builder: "${{ steps.buildx.outputs.name }}" + context: "./" + push: false + load: true + tags: "${{ env.PLUGIN_NAME }}/nautobot:${{ matrix.nautobot-version }}-py${{ matrix.python-version }}" + file: "./development/Dockerfile" + cache-from: "type=gha,scope=${{ matrix.nautobot-version }}-py${{ matrix.python-version }}" + cache-to: "type=gha,scope=${{ matrix.nautobot-version }}-py${{ matrix.python-version }}" + build-args: | + NAUTOBOT_VER=${{ matrix.nautobot-version }} + PYTHON_VER=${{ matrix.python-version }} + - name: "Copy credentials" + run: "cp development/creds.example.env development/creds.env" + - name: "Checking: migrations" + run: "poetry run invoke check-migrations" unittest: needs: - "pylint" + - "check-migrations" strategy: fail-fast: true matrix: - python-version: ["3.8", "3.9", "3.10", "3.11"] + python-version: ["3.8", "3.11"] db-backend: ["postgresql"] nautobot-version: ["stable"] - # The include is a method to limit the amount of jobs ran. This essentially - # means that in addition to standard postgres and stable, also the lowest - # supported version and with mysql include: - python-version: "3.11" db-backend: "postgresql" nautobot-version: "2.0.0" - python-version: "3.11" - db-backend: "mysql" - nautobot-version: "2.0.0" - - python-version: "3.8" db-backend: "mysql" nautobot-version: "stable" - runs-on: "ubuntu-20.04" + runs-on: "ubuntu-22.04" env: INVOKE_NAUTOBOT_GOLDEN_CONFIG_PYTHON_VER: "${{ matrix.python-version }}" INVOKE_NAUTOBOT_GOLDEN_CONFIG_NAUTOBOT_VER: "${{ matrix.nautobot-version }}" steps: - name: "Check out repository code" - uses: "actions/checkout@v3" + uses: "actions/checkout@v4" - name: "Setup environment" uses: "networktocode/gh-action-setup-poetry-environment@v4" - name: "Set up Docker Buildx" id: "buildx" - uses: "docker/setup-buildx-action@v1" + uses: "docker/setup-buildx-action@v3" - name: "Build" - uses: "docker/build-push-action@v2" + uses: "docker/build-push-action@v5" with: builder: "${{ steps.buildx.outputs.name }}" context: "./" @@ -172,13 +222,13 @@ jobs: needs: - "unittest" name: "Publish to GitHub" - runs-on: "ubuntu-20.04" + runs-on: "ubuntu-22.04" if: "startsWith(github.ref, 'refs/tags/v')" steps: - name: "Check out repository code" - uses: "actions/checkout@v3" + uses: "actions/checkout@v4" - name: "Set up Python" - uses: "actions/setup-python@v2" + uses: "actions/setup-python@v4" with: python-version: "3.11" - name: "Install Python Packages" @@ -201,13 +251,13 @@ jobs: needs: - "unittest" name: "Push Package to PyPI" - runs-on: "ubuntu-20.04" + runs-on: "ubuntu-22.04" if: "startsWith(github.ref, 'refs/tags/v')" steps: - name: "Check out repository code" - uses: "actions/checkout@v3" + uses: "actions/checkout@v4" - name: "Set up Python" - uses: "actions/setup-python@v2" + uses: "actions/setup-python@v4" with: python-version: "3.11" - name: "Install Python Packages" @@ -227,7 +277,7 @@ jobs: needs: - "publish_gh" - "publish_pypi" - runs-on: "ubuntu-20.04" + runs-on: "ubuntu-22.04" env: SLACK_WEBHOOK_URL: "${{ secrets.SLACK_WEBHOOK_URL }}" SLACK_MESSAGE: >- @@ -240,7 +290,7 @@ jobs: # ENVs cannot be used directly in job.if. This is a workaround to check # if SLACK_WEBHOOK_URL is present. if: "env.SLACK_WEBHOOK_URL != ''" - uses: "slackapi/slack-github-action@v1.17.0" + uses: "slackapi/slack-github-action@v1" with: payload: | { diff --git a/.github/workflows/rebake.yaml b/.github/workflows/rebake.yaml deleted file mode 100644 index 6b91c3fd..00000000 --- a/.github/workflows/rebake.yaml +++ /dev/null @@ -1,34 +0,0 @@ ---- -name: "Rebake Cookie" -on: # yamllint disable-line rule:truthy - schedule: - # Every Saturday at 5:00 am UTC - - cron: "0 5 * * 6" - workflow_dispatch: - inputs: - template-ref: - description: "The branch or tag to use for the template, using the value from .cookiecutter.json when empty" - default: "" -jobs: - rebake: - runs-on: "ubuntu-22.04" - permissions: - actions: "write" - contents: "write" - packages: "read" - pull-requests: "write" - container: "ghcr.io/nautobot/cookiecutter-nautobot-app-drift-manager/prod:latest" - env: - GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" - steps: - - name: "Configure" - id: "config" - run: | - export REBAKE_ARGS='--push' - if [[ ${{ github.event_name }} == 'workflow_dispatch' ]]; then - export REBAKE_ARGS="$REBAKE_ARGS --template-ref='${{ github.event.inputs.template-ref }}'" - fi - echo "rebake-args='$REBAKE_ARGS'" >> $GITHUB_OUTPUT - - name: "Rebake" - run: | - python -m ntc_cookie_drift_manager rebake ${{ steps.config.outputs.rebake-args }} '${{ github.repositoryUrl }}' diff --git a/.github/workflows/rebake.yml b/.github/workflows/rebake.yml new file mode 100644 index 00000000..b4688842 --- /dev/null +++ b/.github/workflows/rebake.yml @@ -0,0 +1,118 @@ +--- +name: "Rebake Cookie" +on: # yamllint disable-line rule:truthy + workflow_call: + inputs: + cookie: + description: "The cookie to rebake" + type: "string" + default: "" + draft: + description: "Whether to create the pull request as a draft" + type: "string" + default: "" + pull-request: + description: "The pull request strategy" + type: "string" + default: "" + template: + description: "The template repository URL" + type: "string" + default: "" + template-dir: + description: "The directory within the template repository to use as the template" + type: "string" + default: "" + template-ref: + description: "The branch or tag to use for the template" + type: "string" + default: "" + drift-manager-tag: + description: "The drift manager Docker image tag to use" + type: "string" + default: "prod" + workflow_dispatch: + inputs: + cookie: + description: "The cookie to rebake" + type: "string" + default: "" + draft: + description: "Whether to create the pull request as a draft" + type: "string" + default: "" + pull-request: + description: "The pull request strategy" + type: "string" + default: "" + template: + description: "The template repository URL" + type: "string" + default: "" + template-dir: + description: "The directory within the template repository to use as the template" + type: "string" + default: "" + template-ref: + description: "The branch or tag to use for the template" + type: "string" + default: "" + drift-manager-tag: + description: "The drift manager Docker image tag to use" + type: "string" + default: "prod" +jobs: + rebake: + runs-on: "ubuntu-22.04" + permissions: + actions: "write" + contents: "write" + packages: "read" + pull-requests: "write" + container: "ghcr.io/nautobot/cookiecutter-nautobot-app-drift-manager/prod:${{ github.event.inputs.drift-manager-tag }}" + env: + GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" + steps: + - name: "Configure Rebake Arguments" + id: "config" + shell: "bash" + run: | + ARGS='--push' + + if [[ '${{ github.event.inputs.draft }}' == 'true' ]]; then + ARGS="$ARGS --draft" + elif [[ '${{ github.event.inputs.draft }}' == 'false' ]]; then + ARGS="$ARGS --no-draft" + elif [[ '${{ github.event.inputs.draft }}' == '' ]]; then + echo "Using repo default value for --draft" + else + echo "ERROR: Invalid value for draft: '${{ github.event.inputs.draft }}'" + exit 1 + fi + + if [[ '${{ github.event.inputs.pull-request }}' != '' ]]; then + ARGS="$ARGS --pull-request='${{ github.event.inputs.pull-request }}'" + fi + + if [[ '${{ github.event.inputs.template }}' != '' ]]; then + ARGS="$ARGS --template='${{ github.event.inputs.template }}'" + fi + + if [[ '${{ github.event.inputs.template-dir }}' != '' ]]; then + ARGS="$ARGS --template-dir='${{ github.event.inputs.template-dir }}'" + fi + + if [[ '${{ github.event.inputs.template-ref }}' != '' ]]; then + ARGS="$ARGS --template-ref='${{ github.event.inputs.template-ref }}'" + fi + + if [[ '${{ github.event.inputs.cookie }}' == '' ]]; then + ARGS="$ARGS '${{ github.repositoryUrl }}'" + else + ARGS="$ARGS '${{ github.event.inputs.cookie }}'" + fi + + echo "args=$ARGS" >> $GITHUB_OUTPUT + - name: "Rebake" + run: | + python -m ntc_cookie_drift_manager rebake ${{ steps.config.outputs.args }} diff --git a/.gitignore b/.gitignore index 9e81c36c..c812ea13 100644 --- a/.gitignore +++ b/.gitignore @@ -310,3 +310,4 @@ invoke.yml public /compose.yaml /dump.sql +/nautobot_golden_config/static/nautobot_golden_config/docs diff --git a/development/Dockerfile b/development/Dockerfile index e109a262..a7a0870d 100644 --- a/development/Dockerfile +++ b/development/Dockerfile @@ -19,20 +19,21 @@ FROM ghcr.io/nautobot/nautobot-dev:${NAUTOBOT_VER}-py${PYTHON_VER} ARG NAUTOBOT_ROOT=/opt/nautobot ENV prometheus_multiproc_dir=/prom_cache -ENV NAUTOBOT_ROOT ${NAUTOBOT_ROOT} +ENV NAUTOBOT_ROOT=${NAUTOBOT_ROOT} +ENV INVOKE_NAUTOBOT_GOLDEN_CONFIG_LOCAL=true # Install Poetry manually via its installer script; # We might be using an older version of Nautobot that includes an older version of Poetry # and CI and local development may have a newer version of Poetry # Since this is only used for development and we don't ship this container, pinning Poetry back is not expressly necessary # We also don't need virtual environments in container -RUN curl -sSL https://install.python-poetry.org | python3 - && \ +RUN which poetry || curl -sSL https://install.python-poetry.org | python3 - && \ poetry config virtualenvs.create false # !!! USE CAUTION WHEN MODIFYING LINES ABOVE # ------------------------------------------------------------------------------------- # App-specifc system build/test dependencies. -# +# # Example: LDAP requires `libldap2-dev` to be apt-installed before the Python package. # ------------------------------------------------------------------------------------- # --> Start safe to modify section @@ -68,11 +69,13 @@ RUN sort poetry_freeze_base.txt poetry_freeze_all.txt | uniq -u > poetry_freeze_ # Install all local project as editable, constrained on Nautobot version, to get any additional # direct dependencies of the app -RUN pip install -c constraints.txt -e . +RUN --mount=type=cache,target="/root/.cache/pip",sharing=locked \ + pip install -c constraints.txt -e .[all] # Install any dev dependencies frozen from Poetry # Can be improved in Poetry 1.2 which allows `poetry install --only dev` -RUN pip install -c constraints.txt -r poetry_freeze_dev.txt +RUN --mount=type=cache,target="/root/.cache/pip",sharing=locked \ + pip install -c constraints.txt -r poetry_freeze_dev.txt COPY development/nautobot_config.py ${NAUTOBOT_ROOT}/nautobot_config.py # !!! USE CAUTION WHEN MODIFYING LINES ABOVE diff --git a/development/docker-compose.base.yml b/development/docker-compose.base.yml index 6a0a4c72..3dda2b07 100644 --- a/development/docker-compose.base.yml +++ b/development/docker-compose.base.yml @@ -38,3 +38,13 @@ services: retries: 3 test: ["CMD", "bash", "-c", "nautobot-server celery inspect ping --destination celery@$$HOSTNAME"] ## $$ because of docker-compose <<: *nautobot-base + beat: + entrypoint: + - "sh" + - "-c" # this is to evaluate the $NAUTOBOT_LOG_LEVEL from the env + - "nautobot-server celery beat -l $$NAUTOBOT_LOG_LEVEL" ## $$ because of docker-compose + depends_on: + - "nautobot" + healthcheck: + disable: true + <<: *nautobot-base diff --git a/docs/dev/dev_environment.md b/docs/dev/dev_environment.md index f5ce31a8..a0045039 100644 --- a/docs/dev/dev_environment.md +++ b/docs/dev/dev_environment.md @@ -13,8 +13,8 @@ This is a quick reference guide if you're already familiar with the development The [Invoke](http://www.pyinvoke.org/) library is used to provide some helper commands based on the environment. There are a few configuration parameters which can be passed to Invoke to override the default configuration: -- `nautobot_ver`: the version of Nautobot to use as a base for any built docker containers (default: latest) -- `project_name`: the default docker compose project name (default: `nautobot_golden_config`) +- `nautobot_ver`: the version of Nautobot to use as a base for any built docker containers (default: 2.0.0) +- `project_name`: the default docker compose project name (default: `nautobot-golden-config`) - `python_ver`: the version of Python to use as a base for any built docker containers (default: 3.11) - `local`: a boolean flag indicating if invoke tasks should be run on the host or inside the docker containers (default: False, commands will be run in docker containers) - `compose_dir`: the full path to a directory containing the project compose files @@ -57,8 +57,6 @@ To either stop or destroy the development environment use the following options. --- nautobot_golden_config: local: true - compose_files: - - "docker-compose.requirements.yml" ``` Run the following commands: @@ -136,7 +134,6 @@ Each command can be executed with `invoke `. All commands support the a unittest Run Django unit tests for the plugin. ``` - ## Project Overview This project provides the ability to develop and manage the Nautobot server locally (with supporting services being *Dockerized*) or by using only Docker containers to manage Nautobot. The main difference between the two environments is the ability to debug and use **pdb** when developing locally. Debugging with **pdb** within the Docker container is more complicated, but can still be accomplished by either entering into the container (via `docker exec`) or attaching your IDE to the container and running the Nautobot service manually within the container. @@ -187,7 +184,7 @@ The first thing you need to do is build the necessary Docker image for Nautobot #14 exporting layers #14 exporting layers 1.2s done #14 writing image sha256:2d524bc1665327faa0d34001b0a9d2ccf450612bf8feeb969312e96a2d3e3503 done -#14 naming to docker.io/nautobot-golden-config/nautobot:latest-py3.11 done +#14 naming to docker.io/nautobot-golden-config/nautobot:2.0.0-py3.11 done ``` ### Invoke - Starting the Development Environment @@ -218,9 +215,9 @@ This will start all of the Docker containers used for hosting Nautobot. You shou ```bash ➜ docker ps ****CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES -ee90fbfabd77 nautobot-golden-config/nautobot:latest-py3.11 "nautobot-server rqw…" 16 seconds ago Up 13 seconds nautobot_golden_config_worker_1 -b8adb781d013 nautobot-golden-config/nautobot:latest-py3.11 "/docker-entrypoint.…" 20 seconds ago Up 15 seconds 0.0.0.0:8080->8080/tcp, :::8080->8080/tcp nautobot_golden_config_nautobot_1 -d64ebd60675d nautobot-golden-config/nautobot:latest-py3.11 "mkdocs serve -v -a …" 25 seconds ago Up 18 seconds 0.0.0.0:8001->8080/tcp, :::8001->8080/tcp nautobot_golden_config_docs_1 +ee90fbfabd77 nautobot-golden-config/nautobot:2.0.0-py3.11 "nautobot-server rqw…" 16 seconds ago Up 13 seconds nautobot_golden_config_worker_1 +b8adb781d013 nautobot-golden-config/nautobot:2.0.0-py3.11 "/docker-entrypoint.…" 20 seconds ago Up 15 seconds 0.0.0.0:8080->8080/tcp, :::8080->8080/tcp nautobot_golden_config_nautobot_1 +d64ebd60675d nautobot-golden-config/nautobot:2.0.0-py3.11 "mkdocs serve -v -a …" 25 seconds ago Up 18 seconds 0.0.0.0:8001->8080/tcp, :::8001->8080/tcp nautobot_golden_config_docs_1 e72d63129b36 postgres:13-alpine "docker-entrypoint.s…" 25 seconds ago Up 19 seconds 0.0.0.0:5432->5432/tcp, :::5432->5432/tcp nautobot_golden_config_postgres_1 96c6ff66997c redis:6-alpine "docker-entrypoint.s…" 25 seconds ago Up 21 seconds 0.0.0.0:6379->6379/tcp, :::6379->6379/tcp nautobot_golden_config_redis_1 ``` diff --git a/docs/requirements.txt b/docs/requirements.txt index e8e7f513..3aed102e 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -2,4 +2,4 @@ mkdocs==1.5.2 mkdocs-material==9.2.4 mkdocs-version-annotations==1.0.0 mkdocstrings==0.22.0 -mkdocstrings-python==1.5.2 \ No newline at end of file +mkdocstrings-python==1.5.2 diff --git a/poetry.lock b/poetry.lock index 37729892..95c72f1d 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 1.5.1 and should not be changed by hand. +# This file is automatically @generated by Poetry 1.6.1 and should not be changed by hand. [[package]] name = "amqp" @@ -2267,16 +2267,6 @@ files = [ {file = "MarkupSafe-2.1.3-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:5bbe06f8eeafd38e5d0a4894ffec89378b6c6a625ff57e3028921f8ff59318ac"}, {file = "MarkupSafe-2.1.3-cp311-cp311-win32.whl", hash = "sha256:dd15ff04ffd7e05ffcb7fe79f1b98041b8ea30ae9234aed2a9168b5797c3effb"}, {file = "MarkupSafe-2.1.3-cp311-cp311-win_amd64.whl", hash = "sha256:134da1eca9ec0ae528110ccc9e48041e0828d79f24121a1a146161103c76e686"}, - {file = "MarkupSafe-2.1.3-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:f698de3fd0c4e6972b92290a45bd9b1536bffe8c6759c62471efaa8acb4c37bc"}, - {file = "MarkupSafe-2.1.3-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:aa57bd9cf8ae831a362185ee444e15a93ecb2e344c8e52e4d721ea3ab6ef1823"}, - {file = "MarkupSafe-2.1.3-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ffcc3f7c66b5f5b7931a5aa68fc9cecc51e685ef90282f4a82f0f5e9b704ad11"}, - {file = "MarkupSafe-2.1.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:47d4f1c5f80fc62fdd7777d0d40a2e9dda0a05883ab11374334f6c4de38adffd"}, - {file = "MarkupSafe-2.1.3-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1f67c7038d560d92149c060157d623c542173016c4babc0c1913cca0564b9939"}, - {file = "MarkupSafe-2.1.3-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:9aad3c1755095ce347e26488214ef77e0485a3c34a50c5a5e2471dff60b9dd9c"}, - {file = "MarkupSafe-2.1.3-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:14ff806850827afd6b07a5f32bd917fb7f45b046ba40c57abdb636674a8b559c"}, - {file = "MarkupSafe-2.1.3-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8f9293864fe09b8149f0cc42ce56e3f0e54de883a9de90cd427f191c346eb2e1"}, - {file = "MarkupSafe-2.1.3-cp312-cp312-win32.whl", hash = "sha256:715d3562f79d540f251b99ebd6d8baa547118974341db04f5ad06d5ea3eb8007"}, - {file = "MarkupSafe-2.1.3-cp312-cp312-win_amd64.whl", hash = "sha256:1b8dd8c3fd14349433c79fa8abeb573a55fc0fdd769133baac1f5e07abf54aeb"}, {file = "MarkupSafe-2.1.3-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:8e254ae696c88d98da6555f5ace2279cf7cd5b3f52be2b5cf97feafe883b58d2"}, {file = "MarkupSafe-2.1.3-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cb0932dc158471523c9637e807d9bfb93e06a95cbf010f1a38b98623b929ef2b"}, {file = "MarkupSafe-2.1.3-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9402b03f1a1b4dc4c19845e5c749e3ab82d5078d16a2a4c2cd2df62d57bb0707"}, @@ -3590,20 +3580,6 @@ files = [ [package.dependencies] six = ">=1.5" -[[package]] -name = "python-dotenv" -version = "1.0.0" -description = "Read key-value pairs from a .env file and set them as environment variables" -optional = false -python-versions = ">=3.8" -files = [ - {file = "python-dotenv-1.0.0.tar.gz", hash = "sha256:a8df96034aae6d2d50a4ebe8216326c61c3eb64836776504fcca410e5937a3ba"}, - {file = "python_dotenv-1.0.0-py3-none-any.whl", hash = "sha256:f5971a9226b701070a4bf2c38c89e5a3f0d64de8debda981d1db98583009122a"}, -] - -[package.extras] -cli = ["click (>=5.0)"] - [[package]] name = "python-slugify" version = "8.0.1" @@ -3714,7 +3690,6 @@ files = [ {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:69b023b2b4daa7548bcfbd4aa3da05b3a74b772db9e23b982788168117739938"}, {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:81e0b275a9ecc9c0c0c07b4b90ba548307583c125f54d5b6946cfee6360c733d"}, {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ba336e390cd8e4d1739f42dfe9bb83a3cc2e80f567d8805e11b46f4a943f5515"}, - {file = "PyYAML-6.0.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:326c013efe8048858a6d312ddd31d56e468118ad4cdeda36c719bf5bb6192290"}, {file = "PyYAML-6.0.1-cp310-cp310-win32.whl", hash = "sha256:bd4af7373a854424dabd882decdc5579653d7868b8fb26dc7d0e99f823aa5924"}, {file = "PyYAML-6.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:fd1592b3fdf65fff2ad0004b5e363300ef59ced41c2e6b3a99d4089fa8c5435d"}, {file = "PyYAML-6.0.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6965a7bc3cf88e5a1c3bd2e0b5c22f8d677dc88a455344035f03399034eb3007"}, @@ -3722,15 +3697,8 @@ files = [ {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:42f8152b8dbc4fe7d96729ec2b99c7097d656dc1213a3229ca5383f973a5ed6d"}, {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:062582fca9fabdd2c8b54a3ef1c978d786e0f6b3a1510e0ac93ef59e0ddae2bc"}, {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d2b04aac4d386b172d5b9692e2d2da8de7bfb6c387fa4f801fbf6fb2e6ba4673"}, - {file = "PyYAML-6.0.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:e7d73685e87afe9f3b36c799222440d6cf362062f78be1013661b00c5c6f678b"}, {file = "PyYAML-6.0.1-cp311-cp311-win32.whl", hash = "sha256:1635fd110e8d85d55237ab316b5b011de701ea0f29d07611174a1b42f1444741"}, {file = "PyYAML-6.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34"}, - {file = "PyYAML-6.0.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28"}, - {file = "PyYAML-6.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9"}, - {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0"}, - {file = "PyYAML-6.0.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4"}, - {file = "PyYAML-6.0.1-cp312-cp312-win32.whl", hash = "sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54"}, - {file = "PyYAML-6.0.1-cp312-cp312-win_amd64.whl", hash = "sha256:0d3304d8c0adc42be59c5f8a4d9e3d7379e6955ad754aa9d6ab7a398b59dd1df"}, {file = "PyYAML-6.0.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:50550eb667afee136e9a77d6dc71ae76a44df8b3e51e41b77f6de2932bfe0f47"}, {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1fe35611261b29bd1de0070f0b2f47cb6ff71fa6595c077e42bd0c419fa27b98"}, {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:704219a11b772aea0d8ecd7058d0082713c3562b4e271b849ad7dc4a5c90c13c"}, @@ -3747,7 +3715,6 @@ files = [ {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a0cd17c15d3bb3fa06978b4e8958dcdc6e0174ccea823003a106c7d4d7899ac5"}, {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:28c119d996beec18c05208a8bd78cbe4007878c6dd15091efb73a30e90539696"}, {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7e07cbde391ba96ab58e532ff4803f79c4129397514e1413a7dc761ccd755735"}, - {file = "PyYAML-6.0.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:49a183be227561de579b4a36efbb21b3eab9651dd81b1858589f796549873dd6"}, {file = "PyYAML-6.0.1-cp38-cp38-win32.whl", hash = "sha256:184c5108a2aca3c5b3d3bf9395d50893a7ab82a38004c8f61c258d4428e80206"}, {file = "PyYAML-6.0.1-cp38-cp38-win_amd64.whl", hash = "sha256:1e2722cc9fbb45d9b87631ac70924c11d3a401b2d7f410cc0e3bbf249f2dca62"}, {file = "PyYAML-6.0.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9eb6caa9a297fc2c2fb8862bc5370d0303ddba53ba97e71f08023b6cd73d16a8"}, @@ -3755,7 +3722,6 @@ files = [ {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5773183b6446b2c99bb77e77595dd486303b4faab2b086e7b17bc6bef28865f6"}, {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b786eecbdf8499b9ca1d697215862083bd6d2a99965554781d0d8d1ad31e13a0"}, {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bc1bf2925a1ecd43da378f4db9e4f799775d6367bdb94671027b73b393a7c42c"}, - {file = "PyYAML-6.0.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:04ac92ad1925b2cff1db0cfebffb6ffc43457495c9b3c39d3fcae417d7125dc5"}, {file = "PyYAML-6.0.1-cp39-cp39-win32.whl", hash = "sha256:faca3bdcf85b2fc05d06ff3fbc1f83e1391b3e724afa3feba7d13eeab355484c"}, {file = "PyYAML-6.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:510c9deebc5c0225e8c96813043e62b680ba2f9c50a08d3724c7f28a747d1486"}, {file = "PyYAML-6.0.1.tar.gz", hash = "sha256:bfdf460b1736c775f2ba9f6a92bca30bc2095067b8a9d77876d1fad6cc3b4a43"}, @@ -4764,4 +4730,4 @@ testing = ["big-O", "jaraco.functools", "jaraco.itertools", "more-itertools", "p [metadata] lock-version = "2.0" python-versions = ">=3.8,<3.12" -content-hash = "1bc38867f792a6bd269ca1bebb4a406e5e8b9e1e361e2ddc96a90795c44fb342" +content-hash = "f4c4dfa5323488895b4d7b059cef8c825cdb3041c7728e68d5599d0ba49f1945" diff --git a/pyproject.toml b/pyproject.toml index 07db5609..7dfece0a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -54,9 +54,7 @@ pydocstyle = "*" pylint = "*" pylint-django = "*" pylint-nautobot = "*" -python-dotenv = "^1.0.0" yamllint = "*" -toml = "*" Markdown = "*" # Rendering docs to HTML mkdocs = "1.5.2" @@ -135,11 +133,5 @@ add_ignore = "D212,D417" requires = ["poetry_core>=1.0.0"] build-backend = "poetry.core.masonry.api" -[tool.pytest.ini_options] -testpaths = [ - "tests" -] -addopts = "-vv --doctest-modules" - [tool.pylint-nautobot] supported_nautobot_versions = ["2"] diff --git a/tasks.py b/tasks.py index 77fc61ee..d7844e5c 100644 --- a/tasks.py +++ b/tasks.py @@ -13,17 +13,9 @@ """ import os -from distutils.util import strtobool -from invoke import Collection -from invoke import task as invoke_task - -# from dotenv import load_dotenv - - -# def _load_dotenv(): -# load_dotenv("./development/development.env") -# load_dotenv("./development/creds.env") +from invoke.collection import Collection +from invoke.tasks import task as invoke_task def is_truthy(arg): @@ -38,7 +30,14 @@ def is_truthy(arg): """ if isinstance(arg, bool): return arg - return bool(strtobool(arg)) + + val = str(arg).lower() + if val in ("y", "yes", "t", "true", "on", "1"): + return True + elif val in ("n", "no", "f", "false", "off", "0"): + return False + else: + raise ValueError(f"Invalid truthy value: `{arg}`") # Use pyinvoke configuration for default values, see http://docs.pyinvoke.org/en/stable/concepts/configuration.html @@ -64,6 +63,10 @@ def is_truthy(arg): ) +def _is_compose_included(context, name): + return f"docker-compose.{name}.yml" in context.nautobot_golden_config.compose_files + + def task(function=None, *args, **kwargs): """Task decorator to override the default Invoke task decorator and add each task to the invoke namespace.""" @@ -100,7 +103,7 @@ def docker_compose(context, command, **kwargs): **kwargs.pop("env", {}), } compose_command_tokens = [ - "docker-compose", + "docker compose", f"--project-name {context.nautobot_golden_config.project_name}", f'--project-directory "{context.nautobot_golden_config.compose_dir}"', ] @@ -169,10 +172,17 @@ def generate_packages(context): run_command(context, command) -@task -def lock(context): +@task( + help={ + "check": ( + "If enabled, check for outdated dependencies in the poetry.lock file, " + "instead of generating a new one. (default: disabled)" + ) + } +) +def lock(context, check=False): """Generate poetry.lock inside the Nautobot container.""" - run_command(context, "poetry lock --no-update") + run_command(context, f"poetry {'check' if check else 'lock --no-update'}") # ------------------------------------------------------------------------------ @@ -360,157 +370,141 @@ def exec(context, service="nautobot", command="bash", file=""): @task( help={ + "db-name": "Database name (default: Nautobot database)", + "input-file": "SQL file to execute and quit (default: empty, start interactive CLI)", + "output-file": "Ouput file, overwrite if exists (default: empty, output to stdout)", "query": "SQL command to execute and quit (default: empty)", - "input": "SQL file to execute and quit (default: empty)", - "output": "Ouput file, overwrite if exists (default: empty)", } ) -def dbshell(context, query="", input="", output=""): +def dbshell(context, db_name="", input_file="", output_file="", query=""): """Start database CLI inside the running `db` container. Doesn't use `nautobot-server dbshell`, using started `db` service container only. """ - if input and query: - raise ValueError("Cannot specify both, `input` and `query` arguments") - if output and not (input or query): - raise ValueError("`output` argument requires `input` or `query` argument") + if input_file and query: + raise ValueError("Cannot specify both, `input_file` and `query` arguments") + if output_file and not (input_file or query): + raise ValueError("`output_file` argument requires `input_file` or `query` argument") - # _load_dotenv() + env = {} + if query: + env["_SQL_QUERY"] = query - service = "db" - env_vars = {} - command = ["exec"] + command = [ + "exec", + "--env=_SQL_QUERY" if query else "", + "-- db sh -c '", + ] - if "docker-compose.mysql.yml" in context.nautobot_golden_config.compose_files: - env_vars["MYSQL_PWD"] = os.getenv("MYSQL_PASSWORD") + if _is_compose_included(context, "mysql"): command += [ - "--env=MYSQL_PWD", - "--", - service, "mysql", - f"--user='{os.getenv('MYSQL_USER')}'", - f"--database='{os.getenv('MYSQL_DATABASE')}'", + "--user=$MYSQL_USER", + "--password=$MYSQL_PASSWORD", + f"--database={db_name or '$MYSQL_DATABASE'}", ] - if query: - command += [f"--execute='{query}'"] - elif "docker-compose.postgres.yml" in context.nautobot_golden_config.compose_files: + elif _is_compose_included(context, "postgres"): command += [ - "--", - service, "psql", - f"--username='{os.getenv('POSTGRES_USER')}'", - f"--dbname='{os.getenv('POSTGRES_DB')}'", + "--username=$POSTGRES_USER", + f"--dbname={db_name or '$POSTGRES_DB'}", ] - if query: - command += [f"--command='{query}'"] else: raise ValueError("Unsupported database backend.") - if input: - command += [f"< '{input}'"] - if output: - command += [f"> '{output}'"] + command += [ + "'", + '<<<"$_SQL_QUERY"' if query else "", + f"< '{input_file}'" if input_file else "", + f"> '{output_file}'" if output_file else "", + ] - docker_compose(context, " ".join(command), env=env_vars, pty=not (input or output or query)) + docker_compose(context, " ".join(command), env=env, pty=not (input_file or output_file or query)) @task( help={ - "input": "SQL dump file to replace the existing database with. This can be generated using `invoke backup-db` (default: `dump.sql`).", + "input-file": "SQL dump file to replace the existing database with. This can be generated using `invoke backup-db` (default: `dump.sql`).", } ) -def import_db(context, input="dump.sql"): +def import_db(context, input_file="dump.sql"): """Stop Nautobot containers and replace the current database with the dump into the running `db` container.""" docker_compose(context, "stop -- nautobot worker") - # _load_dotenv() + command = ["exec -- db sh -c '"] - service = "db" - env_vars = {} - command = ["exec"] - - if "docker-compose.mysql.yml" in context.nautobot_golden_config.compose_files: - env_vars["MYSQL_PWD"] = os.getenv("MYSQL_PASSWORD") + if _is_compose_included(context, "mysql"): command += [ - "--env=MYSQL_PWD", - "--", - service, "mysql", - f"--user='{os.getenv('MYSQL_USER')}'", - f"--database='{os.getenv('MYSQL_DATABASE')}'", + "--database=$MYSQL_DATABASE", + "--user=$MYSQL_USER", + "--password=$MYSQL_PASSWORD", ] - elif "docker-compose.postgres.yml" in context.nautobot_golden_config.compose_files: + elif _is_compose_included(context, "postgres"): command += [ - "--", - service, "psql", - f"--username='{os.getenv('POSTGRES_USER')}'", + "--username=$POSTGRES_USER", "postgres", ] else: raise ValueError("Unsupported database backend.") - command += [f"< '{input}'"] + command += [ + "'", + f"< '{input_file}'", + ] - docker_compose(context, " ".join(command), env=env_vars, pty=False) + docker_compose(context, " ".join(command), pty=False) print("Database import complete, you can start Nautobot now: `invoke start`") @task( help={ - "output": "Ouput file, overwrite if exists (default: `dump.sql`)", + "db-name": "Database name to backup (default: Nautobot database)", + "output-file": "Ouput file, overwrite if exists (default: `dump.sql`)", "readable": "Flag to dump database data in more readable format (default: `True`)", } ) -def backup_db(context, output="dump.sql", readable=True): - """Dump database into `output` file from running `db` container.""" - # _load_dotenv() - - service = "db" - env_vars = {} - command = ["exec"] +def backup_db(context, db_name="", output_file="dump.sql", readable=True): + """Dump database into `output_file` file from running `db` container.""" + command = ["exec -- db sh -c '"] - if "docker-compose.mysql.yml" in context.nautobot_golden_config.compose_files: - env_vars["MYSQL_PWD"] = os.getenv("MYSQL_ROOT_PASSWORD") + if _is_compose_included(context, "mysql"): command += [ - "--env=MYSQL_PWD", - "--", - service, "mysqldump", "--user=root", + "--password=$MYSQL_ROOT_PASSWORD", "--add-drop-database", "--skip-extended-insert" if readable else "", "--databases", - os.getenv("MYSQL_DATABASE", ""), + db_name if db_name else "$MYSQL_DATABASE", ] - elif "docker-compose.postgres.yml" in context.nautobot_golden_config.compose_files: + elif _is_compose_included(context, "postgres"): command += [ - "--", - service, "pg_dump", "--clean", "--create", "--if-exists", - f"--username='{os.getenv('POSTGRES_USER')}'", - f"--dbname='{os.getenv('POSTGRES_DB')}'", + "--username=$POSTGRES_USER", + f"--dbname={db_name or '$POSTGRES_DB'}", + "--inserts" if readable else "", ] - - if readable: - command += ["--inserts"] else: raise ValueError("Unsupported database backend.") - if output: - command += [f"> '{output}'"] + command += [ + "'", + f"> '{output_file}'", + ] - docker_compose(context, " ".join(command), env=env_vars, pty=False) + docker_compose(context, " ".join(command), pty=False) print(50 * "=") - print("The database backup has been successfully completed and saved to the file:") - print(output) - print("If you want to import this database backup, please execute the following command:") - print(f"invoke import-db --input '{output}'") + print("The database backup has been successfully completed and saved to the following file:") + print(output_file) + print("You can import this database backup with the following command:") + print(f"invoke import-db --input-file '{output_file}'") print(50 * "=") @@ -691,6 +685,10 @@ def tests(context, failfast=False, keepdb=False, lint_only=False): pydocstyle(context) print("Running yamllint...") yamllint(context) + print("Running poetry check...") + lock(context, check=True) + print("Running migrations check...") + check_migrations(context) print("Running pylint...") pylint(context) print("Running mkdocs...") @@ -698,5 +696,5 @@ def tests(context, failfast=False, keepdb=False, lint_only=False): if not lint_only: print("Running unit tests...") unittest(context, failfast=failfast, keepdb=keepdb) + unittest_coverage(context) print("All tests have passed!") - unittest_coverage(context)