diff --git a/.cookiecutter.json b/.cookiecutter.json index ea65bfeb..d8d0ee46 100644 --- a/.cookiecutter.json +++ b/.cookiecutter.json @@ -10,12 +10,10 @@ "project_slug": "nautobot-plugin-golden-config", "repo_url": "https://github.com/nautobot/nautobot-plugin-golden-config", "base_url": "golden-config", - "min_nautobot_version": "1.4.0", - "max_nautobot_version": "1.9999", - "nautobot_version": "latest", + "min_nautobot_version": "2.0.0", + "max_nautobot_version": "2.9999", "camel_name": "NautobotGoldenConfig", "project_short_description": "A plugin for configuration on nautobot", - "version": "1.0.0", "model_class_name": "None", "open_source_license": "Apache-2.0", "docs_base_url": "https://docs.nautobot.com", @@ -27,7 +25,11 @@ "cookie_dir": "", "branch_prefix": "drift-manager", "pull_request_strategy": "create", - "post_actions": [] + "post_actions": [ + "black" + ], + "draft": true, + "baked_commit_ref": "c6963e367d835faf66a9f4ac63267b6ce5811577" } } } diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md index cbb194ae..8d0594cb 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.md +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -4,8 +4,8 @@ about: Report a reproducible bug in the current release of nautobot-golden-confi --- ### Environment -* Python version: -* Nautobot version: +* Python version: +* Nautobot version: * nautobot-golden-config version: diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md index a8d54617..3506acde 100644 --- a/.github/ISSUE_TEMPLATE/feature_request.md +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -5,7 +5,7 @@ about: Propose a new feature or enhancement --- ### Environment -* Nautobot version: +* Nautobot version: * nautobot-golden-config version: -## Change Notes +# Closes: # -## Justification +## What's Changed + + + +## To Do + + +- [ ] Explanation of Change(s) +- [ ] Attached Screenshots, Payload Example +- [ ] Unit, Integration Tests +- [ ] Documentation Updates (when adding/changing features) +- [ ] Example Plugin Updates (when adding/changing features) +- [ ] Outline Remaining Work, Constraints from Design diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 632b3dc2..68e3d45e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -8,6 +8,7 @@ on: # yamllint disable-line rule:truthy rule:comments branches: - "main" - "develop" + - "ltm-1.6" tags: - "v*" pull_request: ~ @@ -17,56 +18,67 @@ env: jobs: black: - runs-on: "ubuntu-20.04" + runs-on: "ubuntu-22.04" env: INVOKE_NAUTOBOT_GOLDEN_CONFIG_LOCAL: "True" steps: - name: "Check out repository code" - uses: "actions/checkout@v3" + uses: "actions/checkout@v4" - name: "Setup environment" uses: "networktocode/gh-action-setup-poetry-environment@v4" - name: "Linting: black" run: "poetry run invoke black" bandit: - runs-on: "ubuntu-20.04" + runs-on: "ubuntu-22.04" env: INVOKE_NAUTOBOT_GOLDEN_CONFIG_LOCAL: "True" steps: - name: "Check out repository code" - uses: "actions/checkout@v3" + uses: "actions/checkout@v4" - name: "Setup environment" uses: "networktocode/gh-action-setup-poetry-environment@v4" - name: "Linting: bandit" run: "poetry run invoke bandit" pydocstyle: - runs-on: "ubuntu-20.04" + runs-on: "ubuntu-22.04" env: INVOKE_NAUTOBOT_GOLDEN_CONFIG_LOCAL: "True" steps: - name: "Check out repository code" - uses: "actions/checkout@v3" + uses: "actions/checkout@v4" - name: "Setup environment" uses: "networktocode/gh-action-setup-poetry-environment@v4" - name: "Linting: pydocstyle" run: "poetry run invoke pydocstyle" flake8: - runs-on: "ubuntu-20.04" + runs-on: "ubuntu-22.04" env: INVOKE_NAUTOBOT_GOLDEN_CONFIG_LOCAL: "True" steps: - name: "Check out repository code" - uses: "actions/checkout@v3" + uses: "actions/checkout@v4" - name: "Setup environment" uses: "networktocode/gh-action-setup-poetry-environment@v4" - name: "Linting: flake8" run: "poetry run invoke flake8" + poetry: + runs-on: "ubuntu-22.04" + env: + INVOKE_NAUTOBOT_GOLDEN_CONFIG_LOCAL: "True" + steps: + - name: "Check out repository code" + uses: "actions/checkout@v4" + - name: "Setup environment" + uses: "networktocode/gh-action-setup-poetry-environment@v4" + - name: "Checking: poetry lock file" + run: "poetry run invoke lock --check" yamllint: - runs-on: "ubuntu-20.04" + runs-on: "ubuntu-22.04" env: INVOKE_NAUTOBOT_GOLDEN_CONFIG_LOCAL: "True" steps: - name: "Check out repository code" - uses: "actions/checkout@v3" + uses: "actions/checkout@v4" - name: "Setup environment" uses: "networktocode/gh-action-setup-poetry-environment@v4" - name: "Linting: yamllint" @@ -76,9 +88,10 @@ jobs: - "bandit" - "pydocstyle" - "flake8" + - "poetry" - "yamllint" - "black" - runs-on: "ubuntu-20.04" + runs-on: "ubuntu-22.04" strategy: fail-fast: true matrix: @@ -89,14 +102,14 @@ jobs: INVOKE_NAUTOBOT_GOLDEN_CONFIG_NAUTOBOT_VER: "${{ matrix.nautobot-version }}" steps: - name: "Check out repository code" - uses: "actions/checkout@v3" + uses: "actions/checkout@v4" - name: "Setup environment" uses: "networktocode/gh-action-setup-poetry-environment@v4" - name: "Set up Docker Buildx" id: "buildx" - uses: "docker/setup-buildx-action@v1" + uses: "docker/setup-buildx-action@v3" - name: "Build" - uses: "docker/build-push-action@v2" + uses: "docker/build-push-action@v5" with: builder: "${{ steps.buildx.outputs.name }}" context: "./" @@ -113,42 +126,80 @@ jobs: run: "cp development/creds.example.env development/creds.env" - name: "Linting: pylint" run: "poetry run invoke pylint" + check-migrations: + needs: + - "bandit" + - "pydocstyle" + - "flake8" + - "poetry" + - "yamllint" + - "black" + runs-on: "ubuntu-22.04" + strategy: + fail-fast: true + matrix: + python-version: ["3.11"] + nautobot-version: ["2.0.0"] + env: + INVOKE_NAUTOBOT_GOLDEN_CONFIG_PYTHON_VER: "${{ matrix.python-version }}" + INVOKE_NAUTOBOT_GOLDEN_CONFIG_NAUTOBOT_VER: "${{ matrix.nautobot-version }}" + steps: + - name: "Check out repository code" + uses: "actions/checkout@v4" + - name: "Setup environment" + uses: "networktocode/gh-action-setup-poetry-environment@v4" + - name: "Set up Docker Buildx" + id: "buildx" + uses: "docker/setup-buildx-action@v3" + - name: "Build" + uses: "docker/build-push-action@v5" + with: + builder: "${{ steps.buildx.outputs.name }}" + context: "./" + push: false + load: true + tags: "${{ env.PLUGIN_NAME }}/nautobot:${{ matrix.nautobot-version }}-py${{ matrix.python-version }}" + file: "./development/Dockerfile" + cache-from: "type=gha,scope=${{ matrix.nautobot-version }}-py${{ matrix.python-version }}" + cache-to: "type=gha,scope=${{ matrix.nautobot-version }}-py${{ matrix.python-version }}" + build-args: | + NAUTOBOT_VER=${{ matrix.nautobot-version }} + PYTHON_VER=${{ matrix.python-version }} + - name: "Copy credentials" + run: "cp development/creds.example.env development/creds.env" + - name: "Checking: migrations" + run: "poetry run invoke check-migrations" unittest: needs: - "pylint" + - "check-migrations" strategy: fail-fast: true matrix: - python-version: ["3.8", "3.9", "3.10", "3.11"] + python-version: ["3.8", "3.11"] db-backend: ["postgresql"] nautobot-version: ["stable"] - # The include is a method to limit the amount of jobs ran. This essentially - # means that in addition to standard postgres and stable, also the lowest - # supported version and with mysql include: - python-version: "3.11" db-backend: "postgresql" nautobot-version: "2.0.0" - python-version: "3.11" - db-backend: "mysql" - nautobot-version: "2.0.0" - - python-version: "3.8" db-backend: "mysql" nautobot-version: "stable" - runs-on: "ubuntu-20.04" + runs-on: "ubuntu-22.04" env: INVOKE_NAUTOBOT_GOLDEN_CONFIG_PYTHON_VER: "${{ matrix.python-version }}" INVOKE_NAUTOBOT_GOLDEN_CONFIG_NAUTOBOT_VER: "${{ matrix.nautobot-version }}" steps: - name: "Check out repository code" - uses: "actions/checkout@v3" + uses: "actions/checkout@v4" - name: "Setup environment" uses: "networktocode/gh-action-setup-poetry-environment@v4" - name: "Set up Docker Buildx" id: "buildx" - uses: "docker/setup-buildx-action@v1" + uses: "docker/setup-buildx-action@v3" - name: "Build" - uses: "docker/build-push-action@v2" + uses: "docker/build-push-action@v5" with: builder: "${{ steps.buildx.outputs.name }}" context: "./" @@ -172,13 +223,13 @@ jobs: needs: - "unittest" name: "Publish to GitHub" - runs-on: "ubuntu-20.04" + runs-on: "ubuntu-22.04" if: "startsWith(github.ref, 'refs/tags/v')" steps: - name: "Check out repository code" - uses: "actions/checkout@v3" + uses: "actions/checkout@v4" - name: "Set up Python" - uses: "actions/setup-python@v2" + uses: "actions/setup-python@v4" with: python-version: "3.11" - name: "Install Python Packages" @@ -201,13 +252,13 @@ jobs: needs: - "unittest" name: "Push Package to PyPI" - runs-on: "ubuntu-20.04" + runs-on: "ubuntu-22.04" if: "startsWith(github.ref, 'refs/tags/v')" steps: - name: "Check out repository code" - uses: "actions/checkout@v3" + uses: "actions/checkout@v4" - name: "Set up Python" - uses: "actions/setup-python@v2" + uses: "actions/setup-python@v4" with: python-version: "3.11" - name: "Install Python Packages" @@ -227,7 +278,7 @@ jobs: needs: - "publish_gh" - "publish_pypi" - runs-on: "ubuntu-20.04" + runs-on: "ubuntu-22.04" env: SLACK_WEBHOOK_URL: "${{ secrets.SLACK_WEBHOOK_URL }}" SLACK_MESSAGE: >- @@ -240,7 +291,7 @@ jobs: # ENVs cannot be used directly in job.if. This is a workaround to check # if SLACK_WEBHOOK_URL is present. if: "env.SLACK_WEBHOOK_URL != ''" - uses: "slackapi/slack-github-action@v1.17.0" + uses: "slackapi/slack-github-action@v1" with: payload: | { diff --git a/.github/workflows/rebake.yaml b/.github/workflows/rebake.yaml deleted file mode 100644 index 6b91c3fd..00000000 --- a/.github/workflows/rebake.yaml +++ /dev/null @@ -1,34 +0,0 @@ ---- -name: "Rebake Cookie" -on: # yamllint disable-line rule:truthy - schedule: - # Every Saturday at 5:00 am UTC - - cron: "0 5 * * 6" - workflow_dispatch: - inputs: - template-ref: - description: "The branch or tag to use for the template, using the value from .cookiecutter.json when empty" - default: "" -jobs: - rebake: - runs-on: "ubuntu-22.04" - permissions: - actions: "write" - contents: "write" - packages: "read" - pull-requests: "write" - container: "ghcr.io/nautobot/cookiecutter-nautobot-app-drift-manager/prod:latest" - env: - GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" - steps: - - name: "Configure" - id: "config" - run: | - export REBAKE_ARGS='--push' - if [[ ${{ github.event_name }} == 'workflow_dispatch' ]]; then - export REBAKE_ARGS="$REBAKE_ARGS --template-ref='${{ github.event.inputs.template-ref }}'" - fi - echo "rebake-args='$REBAKE_ARGS'" >> $GITHUB_OUTPUT - - name: "Rebake" - run: | - python -m ntc_cookie_drift_manager rebake ${{ steps.config.outputs.rebake-args }} '${{ github.repositoryUrl }}' diff --git a/.github/workflows/rebake.yml b/.github/workflows/rebake.yml new file mode 100644 index 00000000..b4688842 --- /dev/null +++ b/.github/workflows/rebake.yml @@ -0,0 +1,118 @@ +--- +name: "Rebake Cookie" +on: # yamllint disable-line rule:truthy + workflow_call: + inputs: + cookie: + description: "The cookie to rebake" + type: "string" + default: "" + draft: + description: "Whether to create the pull request as a draft" + type: "string" + default: "" + pull-request: + description: "The pull request strategy" + type: "string" + default: "" + template: + description: "The template repository URL" + type: "string" + default: "" + template-dir: + description: "The directory within the template repository to use as the template" + type: "string" + default: "" + template-ref: + description: "The branch or tag to use for the template" + type: "string" + default: "" + drift-manager-tag: + description: "The drift manager Docker image tag to use" + type: "string" + default: "prod" + workflow_dispatch: + inputs: + cookie: + description: "The cookie to rebake" + type: "string" + default: "" + draft: + description: "Whether to create the pull request as a draft" + type: "string" + default: "" + pull-request: + description: "The pull request strategy" + type: "string" + default: "" + template: + description: "The template repository URL" + type: "string" + default: "" + template-dir: + description: "The directory within the template repository to use as the template" + type: "string" + default: "" + template-ref: + description: "The branch or tag to use for the template" + type: "string" + default: "" + drift-manager-tag: + description: "The drift manager Docker image tag to use" + type: "string" + default: "prod" +jobs: + rebake: + runs-on: "ubuntu-22.04" + permissions: + actions: "write" + contents: "write" + packages: "read" + pull-requests: "write" + container: "ghcr.io/nautobot/cookiecutter-nautobot-app-drift-manager/prod:${{ github.event.inputs.drift-manager-tag }}" + env: + GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" + steps: + - name: "Configure Rebake Arguments" + id: "config" + shell: "bash" + run: | + ARGS='--push' + + if [[ '${{ github.event.inputs.draft }}' == 'true' ]]; then + ARGS="$ARGS --draft" + elif [[ '${{ github.event.inputs.draft }}' == 'false' ]]; then + ARGS="$ARGS --no-draft" + elif [[ '${{ github.event.inputs.draft }}' == '' ]]; then + echo "Using repo default value for --draft" + else + echo "ERROR: Invalid value for draft: '${{ github.event.inputs.draft }}'" + exit 1 + fi + + if [[ '${{ github.event.inputs.pull-request }}' != '' ]]; then + ARGS="$ARGS --pull-request='${{ github.event.inputs.pull-request }}'" + fi + + if [[ '${{ github.event.inputs.template }}' != '' ]]; then + ARGS="$ARGS --template='${{ github.event.inputs.template }}'" + fi + + if [[ '${{ github.event.inputs.template-dir }}' != '' ]]; then + ARGS="$ARGS --template-dir='${{ github.event.inputs.template-dir }}'" + fi + + if [[ '${{ github.event.inputs.template-ref }}' != '' ]]; then + ARGS="$ARGS --template-ref='${{ github.event.inputs.template-ref }}'" + fi + + if [[ '${{ github.event.inputs.cookie }}' == '' ]]; then + ARGS="$ARGS '${{ github.repositoryUrl }}'" + else + ARGS="$ARGS '${{ github.event.inputs.cookie }}'" + fi + + echo "args=$ARGS" >> $GITHUB_OUTPUT + - name: "Rebake" + run: | + python -m ntc_cookie_drift_manager rebake ${{ steps.config.outputs.args }} diff --git a/.gitignore b/.gitignore index 9e81c36c..c812ea13 100644 --- a/.gitignore +++ b/.gitignore @@ -310,3 +310,4 @@ invoke.yml public /compose.yaml /dump.sql +/nautobot_golden_config/static/nautobot_golden_config/docs diff --git a/.readthedocs.yaml b/.readthedocs.yaml index a6334706..1aa8ad22 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -14,7 +14,7 @@ build: mkdocs: configuration: "mkdocs.yml" - # fail_on_warning: true + fail_on_warning: true # Use our docs/requirements.txt during installation. python: diff --git a/development/Dockerfile b/development/Dockerfile index e109a262..a7a0870d 100644 --- a/development/Dockerfile +++ b/development/Dockerfile @@ -19,20 +19,21 @@ FROM ghcr.io/nautobot/nautobot-dev:${NAUTOBOT_VER}-py${PYTHON_VER} ARG NAUTOBOT_ROOT=/opt/nautobot ENV prometheus_multiproc_dir=/prom_cache -ENV NAUTOBOT_ROOT ${NAUTOBOT_ROOT} +ENV NAUTOBOT_ROOT=${NAUTOBOT_ROOT} +ENV INVOKE_NAUTOBOT_GOLDEN_CONFIG_LOCAL=true # Install Poetry manually via its installer script; # We might be using an older version of Nautobot that includes an older version of Poetry # and CI and local development may have a newer version of Poetry # Since this is only used for development and we don't ship this container, pinning Poetry back is not expressly necessary # We also don't need virtual environments in container -RUN curl -sSL https://install.python-poetry.org | python3 - && \ +RUN which poetry || curl -sSL https://install.python-poetry.org | python3 - && \ poetry config virtualenvs.create false # !!! USE CAUTION WHEN MODIFYING LINES ABOVE # ------------------------------------------------------------------------------------- # App-specifc system build/test dependencies. -# +# # Example: LDAP requires `libldap2-dev` to be apt-installed before the Python package. # ------------------------------------------------------------------------------------- # --> Start safe to modify section @@ -68,11 +69,13 @@ RUN sort poetry_freeze_base.txt poetry_freeze_all.txt | uniq -u > poetry_freeze_ # Install all local project as editable, constrained on Nautobot version, to get any additional # direct dependencies of the app -RUN pip install -c constraints.txt -e . +RUN --mount=type=cache,target="/root/.cache/pip",sharing=locked \ + pip install -c constraints.txt -e .[all] # Install any dev dependencies frozen from Poetry # Can be improved in Poetry 1.2 which allows `poetry install --only dev` -RUN pip install -c constraints.txt -r poetry_freeze_dev.txt +RUN --mount=type=cache,target="/root/.cache/pip",sharing=locked \ + pip install -c constraints.txt -r poetry_freeze_dev.txt COPY development/nautobot_config.py ${NAUTOBOT_ROOT}/nautobot_config.py # !!! USE CAUTION WHEN MODIFYING LINES ABOVE diff --git a/development/docker-compose.base.yml b/development/docker-compose.base.yml index 6a0a4c72..3dda2b07 100644 --- a/development/docker-compose.base.yml +++ b/development/docker-compose.base.yml @@ -38,3 +38,13 @@ services: retries: 3 test: ["CMD", "bash", "-c", "nautobot-server celery inspect ping --destination celery@$$HOSTNAME"] ## $$ because of docker-compose <<: *nautobot-base + beat: + entrypoint: + - "sh" + - "-c" # this is to evaluate the $NAUTOBOT_LOG_LEVEL from the env + - "nautobot-server celery beat -l $$NAUTOBOT_LOG_LEVEL" ## $$ because of docker-compose + depends_on: + - "nautobot" + healthcheck: + disable: true + <<: *nautobot-base diff --git a/docs/admin/admin_uninstall.md b/docs/admin/admin_uninstall.md deleted file mode 100644 index 2bc048c7..00000000 --- a/docs/admin/admin_uninstall.md +++ /dev/null @@ -1,9 +0,0 @@ -# Uninstall the App from Nautobot - -## Uninstall Guide - -Remove the configuration you added in `nautobot_config.py` from `PLUGINS` & `PLUGINS_CONFIG`. - -## Database Cleanup - -Drop all tables from the plugin: `nautobot_plugin_goldenconfig*`. diff --git a/docs/admin/compatibility_matrix.md b/docs/admin/compatibility_matrix.md index 8a3e2e55..e24d35fd 100644 --- a/docs/admin/compatibility_matrix.md +++ b/docs/admin/compatibility_matrix.md @@ -17,4 +17,4 @@ While that last supported version will not be strictly enforced via the `max_ver | 1.4.X | 1.5.3 | 1.5.99 [Official] | | 1.5.X | 1.6.1 | 1.6.99 [Official] | | 1.6.X | 1.6.1 | 1.6.99 [Official] | -| 2.0.x | 2.0.0 | TBD | +| 2.0.x | 2.0.0 | 2.0.99 | diff --git a/docs/admin/admin_install.md b/docs/admin/install.md similarity index 94% rename from docs/admin/admin_install.md rename to docs/admin/install.md index df2653d7..0e9f4bf0 100644 --- a/docs/admin/admin_install.md +++ b/docs/admin/install.md @@ -1,13 +1,16 @@ # Installing the App in Nautobot +Here you will find detailed instructions on how to **install** and **configure** the App within your Nautobot environment. + ## Prerequisites - The plugin relies on [`nautobot_plugin_nornir`](https://pypi.org/project/nautobot-plugin-nornir/) to be installed and both plugins to be enabled in your configuration settings. -- The plugin is compatible with Nautobot 1.4.0 and higher. +- The latest version of this plugin is compatible with Nautobot 2.0.0 and higher, see [this dedicated page](compatibility_matrix.md) for a full compatibility matrix and the deprecation policy. - Databases supported: PostgreSQL, MySQL -!!! note - Please check the [dedicated page](compatibility_matrix.md) for a full compatibility matrix and the deprecation policy. +### Access Requirements + +There are no access requirements from external systems to this plugin. ## Install Guide @@ -71,13 +74,13 @@ PLUGINS_CONFIG = { } ``` -Once the Nautobot configuration is updated, run the Post Upgrade command (`nautobot-server post_upgrade`) to run migrations and clear any cache. +Once the Nautobot configuration is updated, run the Post Upgrade command (`nautobot-server post_upgrade`) to run migrations and clear any cache: ```shell nautobot-server post_upgrade ``` -Then restart the Nautobot services which may include: +Then restart (if necessary) the Nautobot services which may include: - Nautobot - Nautobot Workers @@ -89,7 +92,7 @@ sudo systemctl restart nautobot nautobot-worker nautobot-scheduler ## App Configuration -The plugin behavior can be controlled with the following list of settings. +The plugin behavior can be controlled with the following list of settings: !!! note The `enable_backup`, `enable_compliance`, `enable_intended`, `enable_sotagg`, `enable_plan`, `enable_deploy`, and `enable_postprocessing` will toggle inclusion of the entire component. @@ -153,4 +156,4 @@ PLUGINS_CONFIG = { } ``` -The format for defining these methods is via the dotted string format that will be imported by Django. For example, the Netmiko Cisco IOS dispatcher is defined as `nornir_nautobot.plugins.tasks.dispatcher.cisco_ios.NetmikoCiscoIos`. You also must hand any installation of the packaging and assurance that the value you provide is importable in the environment you run it on. \ No newline at end of file +The format for defining these methods is via the dotted string format that will be imported by Django. For example, the Netmiko Cisco IOS dispatcher is defined as `nornir_nautobot.plugins.tasks.dispatcher.cisco_ios.NetmikoCiscoIos`. You also must hand any installation of the packaging and assurance that the value you provide is importable in the environment you run it on. diff --git a/docs/admin/migrating_to_v2.md b/docs/admin/migrating_to_v2.md index 5aa4baaa..e1452cfb 100644 --- a/docs/admin/migrating_to_v2.md +++ b/docs/admin/migrating_to_v2.md @@ -41,14 +41,14 @@ If previously you have leveraged the `dispatcher_mapping` to use your preferred ## Dynamic Group !!! tip - You can safely skip this section if your Dynamic Groups did not use slugs or one of the removed models or you Dynamic Groups are currently in the required state. + You can safely skip this section if your Dynamic Groups was not using slugs/Site/Region/DeviceRole or your Dynamic Groups are currently in the required state. In an effort to guide you along, you are highly encouraged to leverage the `nautobot-server audit_dynamic_groups` as [documented](https://docs.nautobot.com/projects/core/en/v2.0.0/user-guide/administration/tools/nautobot-server/#audit_dynamic_groups). You will know you have completed this step, when the scope of devices in your Dynamic Group match your expectations. ## GraphQL !!! tip - You can safely skip this section if your GraphQL did not use slugs or one of the removed models or your saved GraphQL currently renders to the appropriate state. + You can safely skip this section if your GraphQL Query was not using slugs/Site/Region/DeviceRole or your saved GraphQL Query currently renders to the appropriate state. As mentioned, any reference to slug or to one of the removed models will need to be updated to reflect Nautobot 2.0 standards, in this example we will review what would need to change. @@ -145,9 +145,8 @@ _Path for backup and intended_ _Path for templates_ ```jinja -{{obj.platform.slug}}.j2 -{{obj.platform.network_driver}}.j2 <---- old way of doing it -{{obj.location.name|slugify}}/{{obj.name}} <---- new way of doing it +{{obj.platform.slug}}.j2 <---- old way of doing it +{{obj.platform.network_driver}}.j2 <---- new way of doing it ``` ## Custom Dispatcher @@ -190,13 +189,13 @@ PLUGINS_CONFIG = { } ``` -You can also see information within the [custom dispatcher docs](../admin/admin_install.md#custom-dispatcher). +The [custom dispatcher docs](../admin/install.md#custom-dispatcher) will provide further clarification if needed. ## Secrets !!! tip - You can safely skip this section if you have already been using Nautobot Secrets vs Git Repository Token. + You can safely skip this section if you have already been using Nautobot Secrets and not Git Repository Token. Nautobot initially had the ability to store some secrets, this was deprecated when [Secrets framework](https://docs.nautobot.com/projects/core/en/stable/user-guide/platform-functionality/secret/) was added in Nautobot 1.2. The feature to directly store Secrets in the database has been removed in 2.0. -The documentation has been updated in docs covering [secret groups](../user/app_use_cases.md#create-secret-groups). \ No newline at end of file +The documentation has been updated in docs covering [secret groups](../user/app_use_cases.md#create-secret-groups). diff --git a/docs/admin/release_notes/version_1.6.md b/docs/admin/release_notes/version_1.6.md index 1e047824..259d890a 100755 --- a/docs/admin/release_notes/version_1.6.md +++ b/docs/admin/release_notes/version_1.6.md @@ -5,6 +5,12 @@ - Add functionality to compliance result to provide a Remediation plan. - Supports Nautobot >=1.6.1,<2.0.0. +## v1.6.3 - 2023-10 + +### Fixed + +- [#668](https://github.com/nautobot/nautobot-plugin-golden-config/issue/668) - Removed unneeded lookup for GoldenConfigSetting + ## v1.6.2 - 2023-09 ### Fixed diff --git a/docs/admin/release_notes/version_2.0.md b/docs/admin/release_notes/version_2.0.md index b3f2da3d..85ab0852 100755 --- a/docs/admin/release_notes/version_2.0.md +++ b/docs/admin/release_notes/version_2.0.md @@ -11,6 +11,23 @@ !!! note Please see [migrating guide](../migrating_to_v2.md) for details on migration. + +## v2.0.1 - 2023-12 + +### Fixed + +- [#676](https://github.com/nautobot/nautobot-plugin-golden-config/pull/676) - Fixes docs for running config plan job in 2.0. +- [#680](https://github.com/nautobot/nautobot-plugin-golden-config/pull/680) - Resolve RTD build issue. +- [#684](https://github.com/nautobot/nautobot-plugin-golden-config/pull/684) - Fix repo sync not executing on any task failure. +- [#685](https://github.com/nautobot/nautobot-plugin-golden-config/pull/685) - Cherry-pick #669 - Removed unneeded lookup for GoldenConfigSetting. +- [#686](https://github.com/nautobot/nautobot-plugin-golden-config/pull/686) - Fix incorrect permissions. + + +### Changed + +- [#658](https://github.com/nautobot/nautobot-plugin-golden-config/pull/658) - Cookie updated by NetworkToCode Cookie Drift Manager Tool +- [#671](https://github.com/nautobot/nautobot-plugin-golden-config/pull/671) - Finish Documentation Updates from Drift Manager + ## v2.0.0 - 2023-09 ### Changed diff --git a/docs/admin/uninstall.md b/docs/admin/uninstall.md new file mode 100644 index 00000000..44cfc88a --- /dev/null +++ b/docs/admin/uninstall.md @@ -0,0 +1,15 @@ +# Uninstall the App from Nautobot + +Here you will find any steps necessary to cleanly remove the App from your Nautobot environment. + +## Database Cleanup + +Prior to removing the plugin from the `nautobot_config.py`, run the following command to roll back any migration specific to this plugin. + +```shell +nautobot-server migrate nautobot_golden_config zero +``` + +## Remove App configuration + +Remove the configuration you added in `nautobot_config.py` from `PLUGINS` & `PLUGINS_CONFIG`. diff --git a/docs/admin/admin_upgrade.md b/docs/admin/upgrade.md similarity index 72% rename from docs/admin/admin_upgrade.md rename to docs/admin/upgrade.md index 53109384..f4192592 100644 --- a/docs/admin/admin_upgrade.md +++ b/docs/admin/upgrade.md @@ -1,8 +1,10 @@ # Upgrading the App +Here you will find any steps necessary to upgrade the App in your Nautobot environment. + ## Upgrade Guide -When a new release comes out it may be necessary to run a migration of the database to account for any changes in the data models used by this plugin. Execute the command `nautobot-server migrate` from the Nautobot install nautobot/ directory after updating the package. +When a new release comes out it may be necessary to run a migration of the database to account for any changes in the data models used by this plugin. Execute the command `nautobot-server post-upgrade` within the runtime environment of your Nautobot installation after updating the `nautobot-golden-config` package via `pip`. ## v1.0.0 diff --git a/docs/assets/extra.css b/docs/assets/extra.css index 131cef77..dfe2e4b1 100644 --- a/docs/assets/extra.css +++ b/docs/assets/extra.css @@ -18,6 +18,15 @@ font-size: 0.7rem; } +/* +* The default max-width is 61rem which does not provide nearly enough space to present code examples or larger tables +*/ +.md-grid { + margin-left: auto; + margin-right: auto; + max-width: 95%; +} + .md-tabs__link { font-size: 0.8rem; } @@ -39,7 +48,7 @@ } img.logo { - height: 100px; + height: 200px; } img.copyright-logo { @@ -85,3 +94,68 @@ a.autorefs-external::after { a.autorefs-external:hover::after { background-color: var(--md-accent-fg-color); } + + +/* Customization for mkdocs-version-annotations */ +:root { + /* Icon for "version-added" admonition: Material Design Icons "plus-box-outline" */ + --md-admonition-icon--version-added: url('data:image/svg+xml;charset=utf-8,'); + /* Icon for "version-changed" admonition: Material Design Icons "delta" */ + --md-admonition-icon--version-changed: url('data:image/svg+xml;charset=utf-8,'); + /* Icon for "version-removed" admonition: Material Design Icons "minus-circle-outline" */ + --md-admonition-icon--version-removed: url('data:image/svg+xml;charset=utf-8,'); +} + +/* "version-added" admonition in green */ +.md-typeset .admonition.version-added, +.md-typeset details.version-added { + border-color: rgb(0, 200, 83); +} + +.md-typeset .version-added>.admonition-title, +.md-typeset .version-added>summary { + background-color: rgba(0, 200, 83, .1); +} + +.md-typeset .version-added>.admonition-title::before, +.md-typeset .version-added>summary::before { + background-color: rgb(0, 200, 83); + -webkit-mask-image: var(--md-admonition-icon--version-added); + mask-image: var(--md-admonition-icon--version-added); +} + +/* "version-changed" admonition in orange */ +.md-typeset .admonition.version-changed, +.md-typeset details.version-changed { + border-color: rgb(255, 145, 0); +} + +.md-typeset .version-changed>.admonition-title, +.md-typeset .version-changed>summary { + background-color: rgba(255, 145, 0, .1); +} + +.md-typeset .version-changed>.admonition-title::before, +.md-typeset .version-changed>summary::before { + background-color: rgb(255, 145, 0); + -webkit-mask-image: var(--md-admonition-icon--version-changed); + mask-image: var(--md-admonition-icon--version-changed); +} + +/* "version-removed" admonition in red */ +.md-typeset .admonition.version-removed, +.md-typeset details.version-removed { + border-color: rgb(255, 82, 82); +} + +.md-typeset .version-removed>.admonition-title, +.md-typeset .version-removed>summary { + background-color: rgba(255, 82, 82, .1); +} + +.md-typeset .version-removed>.admonition-title::before, +.md-typeset .version-removed>summary::before { + background-color: rgb(255, 82, 82); + -webkit-mask-image: var(--md-admonition-icon--version-removed); + mask-image: var(--md-admonition-icon--version-removed); +} diff --git a/docs/assets/overrides/partials/copyright.html b/docs/assets/overrides/partials/copyright.html index e0b77e5f..b92cf5e3 100644 --- a/docs/assets/overrides/partials/copyright.html +++ b/docs/assets/overrides/partials/copyright.html @@ -1,3 +1,4 @@ +
{% if config.copyright %}
@@ -18,3 +19,4 @@
+ diff --git a/docs/dev/dev_adr.md b/docs/dev/arch_decision.md similarity index 100% rename from docs/dev/dev_adr.md rename to docs/dev/arch_decision.md diff --git a/docs/dev/code_reference/api.md b/docs/dev/code_reference/api.md index ae2a8545..2ce5c164 100644 --- a/docs/dev/code_reference/api.md +++ b/docs/dev/code_reference/api.md @@ -3,4 +3,3 @@ ::: nautobot_golden_config.api options: show_submodules: True - #docstring_section_style: spacy diff --git a/docs/dev/code_reference/package.md b/docs/dev/code_reference/package.md new file mode 100644 index 00000000..8a3cfda5 --- /dev/null +++ b/docs/dev/code_reference/package.md @@ -0,0 +1 @@ +::: nautobot_golden_config diff --git a/docs/dev/dev_contributing.md b/docs/dev/contributing.md similarity index 96% rename from docs/dev/dev_contributing.md rename to docs/dev/contributing.md index 0043135c..5ca961d4 100644 --- a/docs/dev/dev_contributing.md +++ b/docs/dev/contributing.md @@ -4,7 +4,8 @@ The project is packaged with a light [development environment](dev_environment.m The project is following Network to Code software development guidelines and is leveraging the following: -- Black, Pylint, Bandit, flake8, yamllint, and pydocstyle for Python linting and formatting. +- Python linting and formatting: `black`, `pylint`, `bandit`, `flake8`, and `pydocstyle`. +- YAML linting is done with `yamllint`. - Django unit test to ensure the plugin is working properly. Documentation is built using [mkdocs](https://www.mkdocs.org/). The [Docker based development environment](dev_environment.md#docker-development-environment) automatically starts a container hosting a live version of the documentation website on [http://localhost:8001](http://localhost:8001) that auto-refreshes when you make any changes to your local files. diff --git a/docs/dev/dev_environment.md b/docs/dev/dev_environment.md index f5ce31a8..49495441 100644 --- a/docs/dev/dev_environment.md +++ b/docs/dev/dev_environment.md @@ -1,11 +1,11 @@ # Building Your Development Environment -## Quickstart +## Quickstart Guide The development environment can be used in two ways: -1. `Recommended` All services are spun up using Docker and a local volume mount attached so you can develop locally, but Nautobot is spun up within the Docker container. -2. With a local poetry environment if you wish to develop outside of Docker with the caveat of using external services provided by Docker for PostgresQL and Redis. +1. **(Recommended)** All services, including Nautobot, are spun up using Docker containers and a volume mount so you can develop locally. +2. With a local Poetry environment if you wish to develop outside of Docker, with the caveat of using external services provided by Docker for the database (PostgreSQL by default, MySQL optionally) and Redis services. This is a quick reference guide if you're already familiar with the development environment provided, which you can read more about later in this document. @@ -13,8 +13,8 @@ This is a quick reference guide if you're already familiar with the development The [Invoke](http://www.pyinvoke.org/) library is used to provide some helper commands based on the environment. There are a few configuration parameters which can be passed to Invoke to override the default configuration: -- `nautobot_ver`: the version of Nautobot to use as a base for any built docker containers (default: latest) -- `project_name`: the default docker compose project name (default: `nautobot_golden_config`) +- `nautobot_ver`: the version of Nautobot to use as a base for any built docker containers (default: 2.0.0) +- `project_name`: the default docker compose project name (default: `nautobot-golden-config`) - `python_ver`: the version of Python to use as a base for any built docker containers (default: 3.11) - `local`: a boolean flag indicating if invoke tasks should be run on the host or inside the docker containers (default: False, commands will be run in docker containers) - `compose_dir`: the full path to a directory containing the project compose files @@ -57,8 +57,6 @@ To either stop or destroy the development environment use the following options. --- nautobot_golden_config: local: true - compose_files: - - "docker-compose.requirements.yml" ``` Run the following commands: @@ -101,12 +99,9 @@ The project features a CLI helper based on [Invoke](https://www.pyinvoke.org/) t Each command can be executed with `invoke `. All commands support the arguments `--nautobot-ver` and `--python-ver` if you want to manually define the version of Python and Nautobot to use. Each command also has its own help `invoke --help` -!!! note - To run the mysql (mariadb) development environment, set the environment variable as such `export NAUTOBOT_USE_MYSQL=1`. - #### Local Development Environment -```shell +``` build Build all docker images. debug Start Nautobot and its dependencies in debug mode. destroy Destroy all containers and volumes. @@ -117,7 +112,7 @@ Each command can be executed with `invoke `. All commands support the a #### Utility -```shell +``` cli Launch a bash shell inside the running Nautobot container. create-user Create a new user in django (default: admin), will prompt for password. makemigrations Run Make Migration in Django. @@ -126,7 +121,7 @@ Each command can be executed with `invoke `. All commands support the a #### Testing -```shell +``` bandit Run bandit to validate basic static code security analysis. black Run black to check that Python files adhere to its style standards. flake8 Run flake8 to check that Python files adhere to its style standards. @@ -136,7 +131,6 @@ Each command can be executed with `invoke `. All commands support the a unittest Run Django unit tests for the plugin. ``` - ## Project Overview This project provides the ability to develop and manage the Nautobot server locally (with supporting services being *Dockerized*) or by using only Docker containers to manage Nautobot. The main difference between the two environments is the ability to debug and use **pdb** when developing locally. Debugging with **pdb** within the Docker container is more complicated, but can still be accomplished by either entering into the container (via `docker exec`) or attaching your IDE to the container and running the Nautobot service manually within the container. @@ -161,8 +155,6 @@ The `poetry shell` command is used to create and enable a virtual environment ma For more details about Poetry and its commands please check out its [online documentation](https://python-poetry.org/docs/). - - ## Full Docker Development Environment This project is set up with a number of **Invoke** tasks consumed as simple CLI commands to get developing fast. You'll use a few `invoke` commands to get your environment up and running. @@ -187,7 +179,7 @@ The first thing you need to do is build the necessary Docker image for Nautobot #14 exporting layers #14 exporting layers 1.2s done #14 writing image sha256:2d524bc1665327faa0d34001b0a9d2ccf450612bf8feeb969312e96a2d3e3503 done -#14 naming to docker.io/nautobot-golden-config/nautobot:latest-py3.11 done +#14 naming to docker.io/nautobot-golden-config/nautobot:2.0.0-py3.11 done ``` ### Invoke - Starting the Development Environment @@ -218,9 +210,9 @@ This will start all of the Docker containers used for hosting Nautobot. You shou ```bash ➜ docker ps ****CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES -ee90fbfabd77 nautobot-golden-config/nautobot:latest-py3.11 "nautobot-server rqw…" 16 seconds ago Up 13 seconds nautobot_golden_config_worker_1 -b8adb781d013 nautobot-golden-config/nautobot:latest-py3.11 "/docker-entrypoint.…" 20 seconds ago Up 15 seconds 0.0.0.0:8080->8080/tcp, :::8080->8080/tcp nautobot_golden_config_nautobot_1 -d64ebd60675d nautobot-golden-config/nautobot:latest-py3.11 "mkdocs serve -v -a …" 25 seconds ago Up 18 seconds 0.0.0.0:8001->8080/tcp, :::8001->8080/tcp nautobot_golden_config_docs_1 +ee90fbfabd77 nautobot-golden-config/nautobot:2.0.0-py3.11 "nautobot-server rqw…" 16 seconds ago Up 13 seconds nautobot_golden_config_worker_1 +b8adb781d013 nautobot-golden-config/nautobot:2.0.0-py3.11 "/docker-entrypoint.…" 20 seconds ago Up 15 seconds 0.0.0.0:8080->8080/tcp, :::8080->8080/tcp nautobot_golden_config_nautobot_1 +d64ebd60675d nautobot-golden-config/nautobot:2.0.0-py3.11 "mkdocs serve -v -a …" 25 seconds ago Up 18 seconds 0.0.0.0:8001->8080/tcp, :::8001->8080/tcp nautobot_golden_config_docs_1 e72d63129b36 postgres:13-alpine "docker-entrypoint.s…" 25 seconds ago Up 19 seconds 0.0.0.0:5432->5432/tcp, :::5432->5432/tcp nautobot_golden_config_postgres_1 96c6ff66997c redis:6-alpine "docker-entrypoint.s…" 25 seconds ago Up 21 seconds 0.0.0.0:6379->6379/tcp, :::6379->6379/tcp nautobot_golden_config_redis_1 ``` diff --git a/docs/dev/dev_extending.md b/docs/dev/extending.md similarity index 100% rename from docs/dev/dev_extending.md rename to docs/dev/extending.md diff --git a/docs/requirements.txt b/docs/requirements.txt index e8e7f513..4a079cd3 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,5 +1,5 @@ mkdocs==1.5.2 mkdocs-material==9.2.4 mkdocs-version-annotations==1.0.0 +mkdocstrings-python==1.5.2 mkdocstrings==0.22.0 -mkdocstrings-python==1.5.2 \ No newline at end of file diff --git a/docs/user/app_feature_compliance.md b/docs/user/app_feature_compliance.md index 416820c1..1647370d 100644 --- a/docs/user/app_feature_compliance.md +++ b/docs/user/app_feature_compliance.md @@ -91,7 +91,7 @@ Please note the following about the compliance details page. Platforms support technically come from the options provided by [nornir-nautobot](https://github.com/nautobot/nornir-nautobot) for Nornir dispatcher tasks and [netutils](https://github.com/networktocode/netutils) for configuration compliance and parsing. However, for reference, the valid network_driver's of the platforms are -provided in the [FAQ](./app_faq.md). +provided in the [FAQ](./faq.md). ## Overview Report diff --git a/docs/user/app_feature_compliancecustom.md b/docs/user/app_feature_compliancecustom.md index cc330feb..8f43b914 100644 --- a/docs/user/app_feature_compliancecustom.md +++ b/docs/user/app_feature_compliancecustom.md @@ -1,11 +1,9 @@ # Navigating Compliance With Custom Logic ## Introduction -Custom compliance is a powerful method to alter the default behaviour and results of the natively provided `CLI` and `JSON` config type based configuration compliance. +Custom compliance is a powerful method to alter the default behavior and results of the natively provided `CLI` and `JSON` config type based configuration compliance. The high level idea is to provide a Python code containing custom logic defined by the User. This code is called by the Golden Configuration Plugin in the compliance process, -allowing the user to change the default behaviour of the Golden Configuration Plugin. - - +allowing the user to change the default behavior of the Golden Configuration Plugin. ## Caveats diff --git a/docs/user/app_feature_config_plans.md b/docs/user/app_feature_config_plans.md index fbb516a2..ff49831a 100644 --- a/docs/user/app_feature_config_plans.md +++ b/docs/user/app_feature_config_plans.md @@ -48,7 +48,7 @@ Once you have selected the appropriate options, you can click the **Generate** b ### Generating Config Plans via API -The HTTP(S) POST method is not currently enabled for the Config Plan serializer to create plans directly via API. Instead you may run the **GenerateConfigPlans** Job directly via the `plugins/nautobot_golden_config.jobs/GenerateConfigPlans` API endpoint. +The HTTP(S) POST method is not currently enabled for the Config Plan serializer to create plans directly via API. Instead you may run the **Generate Config Plans** Job directly via the `/api/extras/jobs/Generate Config Plans/run/` API endpoint. ## Editing a Config Plan diff --git a/docs/user/app_getting_started.md b/docs/user/app_getting_started.md index 224d3bb9..22b3a82d 100644 --- a/docs/user/app_getting_started.md +++ b/docs/user/app_getting_started.md @@ -1,15 +1,14 @@ -# Quick Start Guides +# Getting Started with the App -- [Quick Start Guides](#quick-start-guides) -- [Backup Configuration](#backup-configuration) -- [Intended Configuration](#intended-configuration) -- [Compliance](#compliance) -- [Config Remediation](#config-remediation) -- [Config Plans](#config-plans) -- [Config Deploy](#config-deploy) -- [Load Properties from Git](#load-properties-from-git) +This document provides a step-by-step tutorial on how to get the App going and how to use it. -## Backup Configuration +## Install the App + +To install the App, please follow the instructions detailed in the [Installation Guide](../admin/install.md). + +## First steps with the App + +### Backup Configuration Follow the steps below to get up and running for the configuration backup element of the plugin. @@ -44,7 +43,7 @@ Follow the steps below to get up and running for the configuration backup elemen > For in-depth details see [Navigating Backup](./app_feature_backup.md) -## Intended Configuration +### Intended Configuration Follow the steps below to get up and running for the intended configuration element of the plugin. @@ -99,7 +98,7 @@ Follow the steps below to get up and running for the intended configuration elem > For in-depth details see [Navigating Intended](./app_feature_intended.md) -## Compliance +### Compliance Compliance requires Backups and Intended Configurations in order to be executed. @@ -125,7 +124,7 @@ Compliance requires Backups and Intended Configurations in order to be executed. > For in-depth details see [Navigating Compliance](./app_feature_compliance.md) -## Config Remediation +### Config Remediation Follow the steps below to get up and running for the configuration remediation element of the plugin. @@ -137,7 +136,7 @@ Follow the steps below to get up and running for the configuration remediation e > For in-depth details see [Navigating Config Plans](./app_feature_remediation.md) -## Config Plans +### Config Plans Follow the steps below to get up and running for the configuration plans element of the plugin. @@ -159,7 +158,7 @@ Follow the steps below to get up and running for the configuration plans element > For in-depth details see [Navigating Config Plans](./app_feature_config_plans.md) -## Config Deploy +### Config Deploy Follow the steps below to get up and running for the configuration deployment element of the plugin. @@ -172,7 +171,7 @@ Follow the steps below to get up and running for the configuration deployment el > Config Deployments utilize the dispatchers from nornir-nautobot just like the other functionality of Golden Config. See [Troubleshooting Dispatchers](./troubleshooting/troubleshoot_dispatchers.md) for more details. -## Load Properties from Git +### Load Properties from Git Golden Config properties include: Compliance Features, Compliance Rules, Config Removals, and Config Replacements. They can be created via the UI, API, or alternatively you can load these properties from a Git repository, defined in YAML files following the this directory structure (you can skip any of them if not apply): @@ -274,7 +273,7 @@ CustomField data can be added using the `_custom_field_data` attribute, that tak 2. Run `sync` and all the properties will be created/updated in a declarative way and following the right order to respect the dependencies between objects. The import task will raise a `warning` if the dependencies are not available yet (for instance, a referenced `Platform` is not created), so the `sync` process will continue, and you could then fix these warnings by reviewing the mismatch (maybe creating the required object) and run the `sync` process again. -## Constance Settings +### Constance Settings Golden config uses the `dispatch_params()` function in conjunction with the constance settings DEFAULT_FRAMEWORK, GET_CONFIG_FRAMEWORK, MERGE_CONFIG_FRAMEWORK, and REPLACE_CONFIG_FRAMEWORK. This allows you to define in this order of precedence: @@ -328,3 +327,6 @@ As you can see, you now have the flexibility to control which network_driver wil Golden Config leverages the [config framework](https://docs.nautobot.com/projects/core/en/stable/development/apps/api/database-backend-config/) from [constance](https://django-constance.readthedocs.io/en/latest/), please refer to that documentation for how to use. You can access your configurations from your name in the top right of the UI, followed by `Admin -> Configuration -> Config` and locate your setting. +## What are the next steps? + +You can check out the [Use Cases](app_use_cases.md) section for more examples. diff --git a/docs/user/app_overview.md b/docs/user/app_overview.md index 74f02514..82d05dbe 100644 --- a/docs/user/app_overview.md +++ b/docs/user/app_overview.md @@ -1,11 +1,11 @@ # App Overview -The golden configuration plugin is a Nautobot plugin that aims to solve common configuration management challenges. +This document provides an overview of the App including critical information and important considerations when applying it to your Nautobot environment. !!! note Throughout this documentation, the terms "app" and "plugin" will be used interchangeably. -## Description/Overview +## Description When engineers are starting their network automation journey, everybody asks where and how they should start. Their immediate thought is coming up with methods of automating changes within their environments. However, doing so can be scary for those who are risk averse about automation making changes. The question then comes about how automation can be used to help solve some of the big problems facing network teams today. One of those problems that we’ve repeatedly heard from our customers and fellow network engineers is around configuration drift. This issue typically occurs for multiple reasons: @@ -36,3 +36,17 @@ Utilizing these components, you can define the Golden Configuration standard for - Ken Celenza (@itdependsnetworks) - Jeff Kala (@jeffkala) + +## Nautobot Features Used + +- Dynamic Groups +- Jobs +- Job Buttons +- Secret Groups +- Git Repositories +- Git as a Data Source +- GraphQL Saved Queries + +### Extras + +The extras and features that Golden Config utilizes are covered in depth in [App Use Cases](./app_use_cases.md). diff --git a/docs/user/app_use_cases.md b/docs/user/app_use_cases.md index 9dc4bd35..a9d2c875 100644 --- a/docs/user/app_use_cases.md +++ b/docs/user/app_use_cases.md @@ -1,5 +1,7 @@ # Using the App +This document describes common use-cases and scenarios for this App. + ## General Usage ### Home @@ -228,7 +230,7 @@ garbage collection and it is up to the operator to remove such data. ### Network Operating System Support -The version of OS's supported is documented in the [FAQ](./app_faq.md) and is controlled the platform network_driver. The platform network_driver must be exactly as expected or leverage +The version of OS's supported is documented in the [FAQ](./faq.md) and is controlled the platform network_driver. The platform network_driver must be exactly as expected or leverage a configuration option--which is described the the FAQ--for the plugin to work. ### Use-cases and common workflows diff --git a/docs/user/app_external_interactions.md b/docs/user/external_interactions.md similarity index 70% rename from docs/user/app_external_interactions.md rename to docs/user/external_interactions.md index e80b8424..b6788d92 100644 --- a/docs/user/app_external_interactions.md +++ b/docs/user/external_interactions.md @@ -1,5 +1,7 @@ # External Interactions +This document describes external dependencies and prerequisites for this App to operate, including system requirements, API endpoints, interconnection or integrations to other applications or services, and similar topics. + ### From the App to Other Systems - Git integrations are required, this is generally `tcp/22` or `tcp/443` to the git repository. diff --git a/docs/user/app_faq.md b/docs/user/faq.md similarity index 100% rename from docs/user/app_faq.md rename to docs/user/faq.md diff --git a/mkdocs.yml b/mkdocs.yml index a5b7ef36..a392bdfc 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -1,11 +1,11 @@ --- dev_addr: "127.0.0.1:8001" -edit_uri: "edit/main/docs" -site_dir: "golden-config/static/golden-config/docs" -site_name: "Golden Configuration Documentation" +edit_uri: "edit/main/nautobot-plugin-golden-config/docs" +site_dir: "nautobot_golden_config/static/nautobot_golden_config/docs" +site_name: "Nautobot Golden Configuration Documentation" site_url: "https://docs.nautobot.com/projects/golden-config/en/latest/" repo_url: "https://github.com/nautobot/nautobot-plugin-golden-config" -copyright: "Copyright © 2023 The Authors" +copyright: "Copyright © The Authors" theme: name: "material" navigation_depth: 4 @@ -14,14 +14,17 @@ theme: - "django" - "yaml" features: + - "content.action.edit" + - "content.action.view" - "content.code.copy" - - "navigation.tracking" + - "navigation.footer" + - "navigation.indexes" - "navigation.tabs" - "navigation.tabs.sticky" - - "search.suggest" + - "navigation.tracking" - "search.highlight" - "search.share" - - "navigation.indexes" + - "search.suggest" favicon: "assets/favicon.ico" logo: "assets/nautobot_logo.svg" palette: @@ -79,8 +82,10 @@ markdown_extensions: - "pymdownx.inlinehilite" - "pymdownx.snippets" - "pymdownx.superfences" + - "footnotes" plugins: - "search" + - "mkdocs-version-annotations" - "mkdocstrings": default_handler: "python" handlers: @@ -95,6 +100,7 @@ nav: - Overview: "index.md" - User Guide: - App Overview: "user/app_overview.md" + - Getting Started: "user/app_getting_started.md" - Using the App: "user/app_use_cases.md" - Navigate Backup: "user/app_feature_backup.md" - Navigate Compliance: "user/app_feature_compliance.md" @@ -106,17 +112,16 @@ nav: - Navigate Configuration Post-Processing: "user/app_feature_config_postprocessing.md" - Navigate Config Plans: "user/app_feature_config_plans.md" - Navigate Remediation: "user/app_feature_remediation.md" - - Getting Started: "user/app_getting_started.md" - - Frequently Asked Questions: "user/app_faq.md" - - External Interactions: "user/app_external_interactions.md" + - Frequently Asked Questions: "user/faq.md" + - External Interactions: "user/external_interactions.md" - Troubleshooting: - General: "user/troubleshooting/troubleshoot_general.md" - Credentials: "user/troubleshooting/troubleshoot_credentials.md" - Dispatchers: "user/troubleshooting/troubleshoot_dispatchers.md" - Administrator Guide: - - Install and Configure: "admin/admin_install.md" - - Upgrade: "admin/admin_upgrade.md" - - Uninstall: "admin/admin_uninstall.md" + - Install and Configure: "admin/install.md" + - Upgrade: "admin/upgrade.md" + - Uninstall: "admin/uninstall.md" - Compatibility Matrix: "admin/compatibility_matrix.md" - Troubleshooting: - "admin/troubleshooting/index.md" @@ -160,12 +165,13 @@ nav: - v0.10: "admin/release_notes/version_0.10.md" - v0.9: "admin/release_notes/version_0.9.md" - Developer Guide: - - Extending the App: "dev/dev_extending.md" - - Contributing to the App: "dev/dev_contributing.md" + - Extending the App: "dev/extending.md" + - Contributing to the App: "dev/contributing.md" - Development Environment: "dev/dev_environment.md" - - Architecture Decision Record: "dev/dev_adr.md" + - Architecture Decision Records: "dev/arch_decision.md" - Code Reference: - "dev/code_reference/index.md" - Models: "dev/code_reference/models.md" + - Package: "dev/code_reference/package.md" - API: "dev/code_reference/api.md" - - Nautobot Docs Home ↗︎: "https://docs.nautobot.com/" + - Nautobot Docs Home ↗︎: "https://docs.nautobot.com" diff --git a/nautobot_golden_config/jobs.py b/nautobot_golden_config/jobs.py index 0f8550f3..6f3c5be7 100644 --- a/nautobot_golden_config/jobs.py +++ b/nautobot_golden_config/jobs.py @@ -4,11 +4,11 @@ # pylint: disable=arguments-differ from datetime import datetime -from django.utils.timezone import make_aware +from django.utils.timezone import make_aware from nautobot.core.celery import register_jobs -from nautobot.dcim.models import Device, DeviceType, Manufacturer, Platform, Rack, RackGroup, Location -from nautobot.extras.datasources.git import ensure_git_repository +from nautobot.dcim.models import Device, DeviceType, Location, Manufacturer, Platform, Rack, RackGroup +from nautobot.extras.datasources.git import ensure_git_repository, get_repo_from_url_to_path_and_from_branch from nautobot.extras.jobs import ( BooleanVar, ChoiceVar, @@ -19,13 +19,9 @@ StringVar, TextVar, ) -from nautobot.extras.models import DynamicGroup, GitRepository, Status, Tag, Role +from nautobot.extras.models import DynamicGroup, GitRepository, Role, Status, Tag from nautobot.tenancy.models import Tenant, TenantGroup -from nautobot.extras.datasources.git import get_repo_from_url_to_path_and_from_branch - - from nornir_nautobot.exceptions import NornirNautobotException - from nautobot_golden_config.choices import ConfigPlanTypeChoice from nautobot_golden_config.models import ComplianceFeature, ConfigPlan, GoldenConfig from nautobot_golden_config.nornir_plays.config_backup import config_backup @@ -94,7 +90,43 @@ class FormEntry: # pylint disable=too-few-public-method debug = BooleanVar(description="Enable for more verbose debug logging") -class ComplianceJob(Job, FormEntry): +class GoldenConfigJobMixin(Job): # pylint: disable=abstract-method + """Reused mixin to be able to reuse common celery primitives in all GC jobs.""" + + def before_start(self, task_id, args, kwargs): + """Ensure repos before tasks runs.""" + super().before_start(task_id, args, kwargs) + self.repos = [] + self.logger.debug( + f"Repository types to sync: {', '.join(self.Meta.repo_types)}", # pylint: disable=no-member + extra={"grouping": "GC Repo Syncs"}, + ) + for repo_type in self.Meta.repo_types: # pylint: disable=no-member + self.logger.debug(f"Refreshing repositories of type {repo_type}.", extra={"grouping": "GC Repo Syncs"}) + current_repos = get_refreshed_repos(job_obj=self, repo_type=repo_type, data=self.deserialize_data(kwargs)) + if not repo_type == "jinja_repository": + for current_repo in current_repos: + self.repos.append(current_repo) + + def after_return(self, status, retval, task_id, args, kwargs, einfo): # pylint: disable=too-many-arguments + """Commit and Push each repo after job is completed.""" + now = make_aware(datetime.now()) + self.logger.debug( + f"Finished the {self.Meta.name} job execution.", # pylint: disable=no-member + extra={"grouping": "GC After Run"}, + ) + if self.repos: + for repo in self.repos: + self.logger.debug( + f"Pushing {self.Meta.name} results to repo {repo.base_url}.", # pylint: disable=no-member + extra={"grouping": "GC Repo Commit and Push"}, + ) + repo.commit_with_added(f"{self.Meta.name.upper()} JOB {now}") # pylint: disable=no-member + repo.push() + super().after_return(status, retval, task_id, args, kwargs, einfo=einfo) + + +class ComplianceJob(GoldenConfigJobMixin, FormEntry): """Job to to run the compliance engine.""" class Meta: @@ -103,21 +135,19 @@ class Meta: name = "Perform Configuration Compliance" description = "Run configuration compliance on your network infrastructure." has_sensitive_variables = False + repo_types = ["intended_repository", "backup_repository"] def run(self, *args, **data): """Run config compliance report script.""" - self.logger.debug("Starting compliance job.") - self.logger.debug("Refreshing intended configuration git repository.") - get_refreshed_repos(job_obj=self, repo_type="intended_repository", data=data) - self.logger.debug("Refreshing backup configuration git repository.") - get_refreshed_repos(job_obj=self, repo_type="backup_repository", data=data) - self.logger.debug("Starting config compliance nornir play.") - # config_compliance(self.logger, data, self.job_result) config_compliance(self.job_result, self.logger.getEffectiveLevel(), data) + def after_return(self, *args): + """Commit and Push each repo after job is completed.""" + self.logger.debug("Compliance job completed, no repositories need to be synced in this task.") -class IntendedJob(Job, FormEntry): + +class IntendedJob(GoldenConfigJobMixin, FormEntry): """Job to to run generation of intended configurations.""" class Meta: @@ -126,29 +156,15 @@ class Meta: name = "Generate Intended Configurations" description = "Generate the configuration for your intended state." has_sensitive_variables = False + repo_types = ["jinja_repository", "intended_repository"] def run(self, *args, **data): """Run config generation script.""" - self.logger.debug("Starting intended job.") - now = make_aware(datetime.now()) - self.logger.debug("Pull Jinja template repos.") - get_refreshed_repos(job_obj=self, repo_type="jinja_repository", data=data) - - self.logger.debug("Pull Intended config repos.") - # Instantiate a GitRepo object for each GitRepository in GoldenConfigSettings. - intended_repos = get_refreshed_repos(job_obj=self, repo_type="intended_repository", data=data) - self.logger.debug("Building device settings mapping and running intended config nornir play.") config_intended(self.job_result, self.logger.getEffectiveLevel(), data, self) - # Commit / Push each repo after job is completed. - for intended_repo in intended_repos: - self.logger.debug("Push new intended configs to repo %s.", intended_repo.base_url) - intended_repo.commit_with_added(f"INTENDED CONFIG CREATION JOB - {now}") - intended_repo.push() - -class BackupJob(Job, FormEntry): +class BackupJob(GoldenConfigJobMixin, FormEntry): """Job to to run the backup job.""" class Meta: @@ -157,28 +173,15 @@ class Meta: name = "Backup Configurations" description = "Backup the configurations of your network devices." has_sensitive_variables = False + repo_types = ["backup_repository"] def run(self, *args, **data): """Run config backup process.""" - self.logger.debug("Starting backup job.") - now = make_aware(datetime.now()) - self.logger.debug("Pull Backup config repo.") - - # Instantiate a GitRepo object for each GitRepository in GoldenConfigSettings. - backup_repos = get_refreshed_repos(job_obj=self, repo_type="backup_repository", data=data) - - self.logger.debug("Starting backup jobs to the following repos: %s", backup_repos) self.logger.debug("Starting config backup nornir play.") config_backup(self.job_result, self.logger.getEffectiveLevel(), data) - # Commit / Push each repo after job is completed. - for backup_repo in backup_repos: - self.logger.debug("Pushing Backup config repo %s.", backup_repo.base_url) - backup_repo.commit_with_added(f"BACKUP JOB {now}") - backup_repo.push() - -class AllGoldenConfig(Job): +class AllGoldenConfig(GoldenConfigJobMixin): """Job to to run all three jobs against a single device.""" device = ObjectVar(model=Device, required=True) @@ -190,9 +193,20 @@ class Meta: name = "Execute All Golden Configuration Jobs - Single Device" description = "Process to run all Golden Configuration jobs configured." has_sensitive_variables = False + repo_types = [] def run(self, *args, **data): """Run all jobs.""" + repo_types = [] + if constant.ENABLE_INTENDED: + repo_types.extend(["jinja_repository", "intended_repository"]) + if constant.ENABLE_BACKUP: + repo_types.extend(["backup_repository"]) + repo_types = list(set(repo_types) - set()) + if constant.ENABLE_COMPLIANCE: + repo_types.extend(["intended_repository", "backup_repository"]) + + self.Meta.repo_types = repo_types if constant.ENABLE_INTENDED: IntendedJob().run.__func__(self, **data) # pylint: disable=too-many-function-args if constant.ENABLE_BACKUP: @@ -201,7 +215,7 @@ def run(self, *args, **data): ComplianceJob().run.__func__(self, **data) # pylint: disable=too-many-function-args -class AllDevicesGoldenConfig(Job, FormEntry): +class AllDevicesGoldenConfig(GoldenConfigJobMixin, FormEntry): """Job to to run all three jobs against multiple devices.""" class Meta: @@ -210,9 +224,20 @@ class Meta: name = "Execute All Golden Configuration Jobs - Multiple Device" description = "Process to run all Golden Configuration jobs configured against multiple devices." has_sensitive_variables = False + repo_types = [] def run(self, *args, **data): """Run all jobs.""" + repo_types = [] + if constant.ENABLE_INTENDED: + repo_types.extend(["jinja_repository", "intended_repository"]) + if constant.ENABLE_BACKUP: + repo_types.extend(["backup_repository"]) + repo_types = list(set(repo_types) - set()) + if constant.ENABLE_COMPLIANCE: + repo_types.extend(["intended_repository", "backup_repository"]) + + self.Meta.repo_types = repo_types if constant.ENABLE_INTENDED: IntendedJob().run.__func__(self, **data) # pylint: disable=too-many-function-args if constant.ENABLE_BACKUP: diff --git a/nautobot_golden_config/template_content.py b/nautobot_golden_config/template_content.py index 34af6439..b43029c3 100644 --- a/nautobot_golden_config/template_content.py +++ b/nautobot_golden_config/template_content.py @@ -2,11 +2,9 @@ from django.core.exceptions import ObjectDoesNotExist from django.db.models import Count, Q from django.urls import reverse -from nautobot.dcim.models import Device from nautobot.extras.plugins import PluginTemplateExtension from nautobot_golden_config.models import ConfigCompliance, GoldenConfig from nautobot_golden_config.utilities.constant import CONFIG_FEATURES, ENABLE_COMPLIANCE -from nautobot_golden_config.utilities.helper import get_device_to_settings_map class ConfigComplianceDeviceCheck(PluginTemplateExtension): # pylint: disable=abstract-method @@ -21,6 +19,8 @@ def get_device(self): def right_page(self): """Content to add to the configuration compliance.""" comp_obj = ConfigCompliance.objects.filter(device=self.get_device()).values("rule__feature__name", "compliance") + if not comp_obj: + return "" extra_context = { "compliance": comp_obj, "device": self.get_device(), @@ -72,6 +72,8 @@ def right_page(self): .order_by("rule__feature__name") .values("rule__feature__name", "compliant", "non_compliant") ) + if not comp_obj: + return "" extra_context = {"compliance": comp_obj, "template_type": "location"} return self.render( "nautobot_golden_config/content_template.html", @@ -92,13 +94,13 @@ def right_page(self): """Content to add to the configuration compliance.""" device = self.get_device() golden_config = GoldenConfig.objects.filter(device=device).first() - settings = get_device_to_settings_map(queryset=Device.objects.filter(id=device.id)) + if not golden_config: + return "" extra_context = { "device": self.get_device(), # device, "golden_config": golden_config, "template_type": "device-configs", "config_features": CONFIG_FEATURES, - "matched_config_setting": settings.get(device.id, False), } return self.render( "nautobot_golden_config/content_template.html", @@ -128,6 +130,8 @@ def right_page(self): .order_by("rule__feature__name") .values("rule__feature__name", "compliant", "non_compliant") ) + if not comp_obj: + return "" extra_context = {"compliance": comp_obj, "template_type": "location"} return self.render( "nautobot_golden_config/content_template.html", diff --git a/nautobot_golden_config/templates/nautobot_golden_config/content_template.html b/nautobot_golden_config/templates/nautobot_golden_config/content_template.html index eccc69f8..879ca4d6 100644 --- a/nautobot_golden_config/templates/nautobot_golden_config/content_template.html +++ b/nautobot_golden_config/templates/nautobot_golden_config/content_template.html @@ -64,7 +64,6 @@ Config - {% if matched_config_setting %} {% if config_features.compliance and golden_config.compliance_config %} @@ -118,7 +117,6 @@ {% endif %} - {% endif %}
{% endif %} diff --git a/nautobot_golden_config/views.py b/nautobot_golden_config/views.py index 0a227f55..633ba910 100644 --- a/nautobot_golden_config/views.py +++ b/nautobot_golden_config/views.py @@ -33,11 +33,11 @@ # TODO: Future #4512 PERMISSIONS_ACTION_MAP.update( { - "backup": "change", - "compliance": "change", - "intended": "change", - "sotagg": "change", - "postprocessing": "change", + "backup": "view", + "compliance": "view", + "intended": "view", + "sotagg": "view", + "postprocessing": "view", "devicetab": "view", } ) diff --git a/poetry.lock b/poetry.lock index 37729892..95c72f1d 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 1.5.1 and should not be changed by hand. +# This file is automatically @generated by Poetry 1.6.1 and should not be changed by hand. [[package]] name = "amqp" @@ -2267,16 +2267,6 @@ files = [ {file = "MarkupSafe-2.1.3-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:5bbe06f8eeafd38e5d0a4894ffec89378b6c6a625ff57e3028921f8ff59318ac"}, {file = "MarkupSafe-2.1.3-cp311-cp311-win32.whl", hash = "sha256:dd15ff04ffd7e05ffcb7fe79f1b98041b8ea30ae9234aed2a9168b5797c3effb"}, {file = "MarkupSafe-2.1.3-cp311-cp311-win_amd64.whl", hash = "sha256:134da1eca9ec0ae528110ccc9e48041e0828d79f24121a1a146161103c76e686"}, - {file = "MarkupSafe-2.1.3-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:f698de3fd0c4e6972b92290a45bd9b1536bffe8c6759c62471efaa8acb4c37bc"}, - {file = "MarkupSafe-2.1.3-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:aa57bd9cf8ae831a362185ee444e15a93ecb2e344c8e52e4d721ea3ab6ef1823"}, - {file = "MarkupSafe-2.1.3-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ffcc3f7c66b5f5b7931a5aa68fc9cecc51e685ef90282f4a82f0f5e9b704ad11"}, - {file = "MarkupSafe-2.1.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:47d4f1c5f80fc62fdd7777d0d40a2e9dda0a05883ab11374334f6c4de38adffd"}, - {file = "MarkupSafe-2.1.3-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1f67c7038d560d92149c060157d623c542173016c4babc0c1913cca0564b9939"}, - {file = "MarkupSafe-2.1.3-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:9aad3c1755095ce347e26488214ef77e0485a3c34a50c5a5e2471dff60b9dd9c"}, - {file = "MarkupSafe-2.1.3-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:14ff806850827afd6b07a5f32bd917fb7f45b046ba40c57abdb636674a8b559c"}, - {file = "MarkupSafe-2.1.3-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8f9293864fe09b8149f0cc42ce56e3f0e54de883a9de90cd427f191c346eb2e1"}, - {file = "MarkupSafe-2.1.3-cp312-cp312-win32.whl", hash = "sha256:715d3562f79d540f251b99ebd6d8baa547118974341db04f5ad06d5ea3eb8007"}, - {file = "MarkupSafe-2.1.3-cp312-cp312-win_amd64.whl", hash = "sha256:1b8dd8c3fd14349433c79fa8abeb573a55fc0fdd769133baac1f5e07abf54aeb"}, {file = "MarkupSafe-2.1.3-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:8e254ae696c88d98da6555f5ace2279cf7cd5b3f52be2b5cf97feafe883b58d2"}, {file = "MarkupSafe-2.1.3-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cb0932dc158471523c9637e807d9bfb93e06a95cbf010f1a38b98623b929ef2b"}, {file = "MarkupSafe-2.1.3-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9402b03f1a1b4dc4c19845e5c749e3ab82d5078d16a2a4c2cd2df62d57bb0707"}, @@ -3590,20 +3580,6 @@ files = [ [package.dependencies] six = ">=1.5" -[[package]] -name = "python-dotenv" -version = "1.0.0" -description = "Read key-value pairs from a .env file and set them as environment variables" -optional = false -python-versions = ">=3.8" -files = [ - {file = "python-dotenv-1.0.0.tar.gz", hash = "sha256:a8df96034aae6d2d50a4ebe8216326c61c3eb64836776504fcca410e5937a3ba"}, - {file = "python_dotenv-1.0.0-py3-none-any.whl", hash = "sha256:f5971a9226b701070a4bf2c38c89e5a3f0d64de8debda981d1db98583009122a"}, -] - -[package.extras] -cli = ["click (>=5.0)"] - [[package]] name = "python-slugify" version = "8.0.1" @@ -3714,7 +3690,6 @@ files = [ {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:69b023b2b4daa7548bcfbd4aa3da05b3a74b772db9e23b982788168117739938"}, {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:81e0b275a9ecc9c0c0c07b4b90ba548307583c125f54d5b6946cfee6360c733d"}, {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ba336e390cd8e4d1739f42dfe9bb83a3cc2e80f567d8805e11b46f4a943f5515"}, - {file = "PyYAML-6.0.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:326c013efe8048858a6d312ddd31d56e468118ad4cdeda36c719bf5bb6192290"}, {file = "PyYAML-6.0.1-cp310-cp310-win32.whl", hash = "sha256:bd4af7373a854424dabd882decdc5579653d7868b8fb26dc7d0e99f823aa5924"}, {file = "PyYAML-6.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:fd1592b3fdf65fff2ad0004b5e363300ef59ced41c2e6b3a99d4089fa8c5435d"}, {file = "PyYAML-6.0.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6965a7bc3cf88e5a1c3bd2e0b5c22f8d677dc88a455344035f03399034eb3007"}, @@ -3722,15 +3697,8 @@ files = [ {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:42f8152b8dbc4fe7d96729ec2b99c7097d656dc1213a3229ca5383f973a5ed6d"}, {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:062582fca9fabdd2c8b54a3ef1c978d786e0f6b3a1510e0ac93ef59e0ddae2bc"}, {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d2b04aac4d386b172d5b9692e2d2da8de7bfb6c387fa4f801fbf6fb2e6ba4673"}, - {file = "PyYAML-6.0.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:e7d73685e87afe9f3b36c799222440d6cf362062f78be1013661b00c5c6f678b"}, {file = "PyYAML-6.0.1-cp311-cp311-win32.whl", hash = "sha256:1635fd110e8d85d55237ab316b5b011de701ea0f29d07611174a1b42f1444741"}, {file = "PyYAML-6.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34"}, - {file = "PyYAML-6.0.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28"}, - {file = "PyYAML-6.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9"}, - {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0"}, - {file = "PyYAML-6.0.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4"}, - {file = "PyYAML-6.0.1-cp312-cp312-win32.whl", hash = "sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54"}, - {file = "PyYAML-6.0.1-cp312-cp312-win_amd64.whl", hash = "sha256:0d3304d8c0adc42be59c5f8a4d9e3d7379e6955ad754aa9d6ab7a398b59dd1df"}, {file = "PyYAML-6.0.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:50550eb667afee136e9a77d6dc71ae76a44df8b3e51e41b77f6de2932bfe0f47"}, {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1fe35611261b29bd1de0070f0b2f47cb6ff71fa6595c077e42bd0c419fa27b98"}, {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:704219a11b772aea0d8ecd7058d0082713c3562b4e271b849ad7dc4a5c90c13c"}, @@ -3747,7 +3715,6 @@ files = [ {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a0cd17c15d3bb3fa06978b4e8958dcdc6e0174ccea823003a106c7d4d7899ac5"}, {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:28c119d996beec18c05208a8bd78cbe4007878c6dd15091efb73a30e90539696"}, {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7e07cbde391ba96ab58e532ff4803f79c4129397514e1413a7dc761ccd755735"}, - {file = "PyYAML-6.0.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:49a183be227561de579b4a36efbb21b3eab9651dd81b1858589f796549873dd6"}, {file = "PyYAML-6.0.1-cp38-cp38-win32.whl", hash = "sha256:184c5108a2aca3c5b3d3bf9395d50893a7ab82a38004c8f61c258d4428e80206"}, {file = "PyYAML-6.0.1-cp38-cp38-win_amd64.whl", hash = "sha256:1e2722cc9fbb45d9b87631ac70924c11d3a401b2d7f410cc0e3bbf249f2dca62"}, {file = "PyYAML-6.0.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9eb6caa9a297fc2c2fb8862bc5370d0303ddba53ba97e71f08023b6cd73d16a8"}, @@ -3755,7 +3722,6 @@ files = [ {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5773183b6446b2c99bb77e77595dd486303b4faab2b086e7b17bc6bef28865f6"}, {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b786eecbdf8499b9ca1d697215862083bd6d2a99965554781d0d8d1ad31e13a0"}, {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bc1bf2925a1ecd43da378f4db9e4f799775d6367bdb94671027b73b393a7c42c"}, - {file = "PyYAML-6.0.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:04ac92ad1925b2cff1db0cfebffb6ffc43457495c9b3c39d3fcae417d7125dc5"}, {file = "PyYAML-6.0.1-cp39-cp39-win32.whl", hash = "sha256:faca3bdcf85b2fc05d06ff3fbc1f83e1391b3e724afa3feba7d13eeab355484c"}, {file = "PyYAML-6.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:510c9deebc5c0225e8c96813043e62b680ba2f9c50a08d3724c7f28a747d1486"}, {file = "PyYAML-6.0.1.tar.gz", hash = "sha256:bfdf460b1736c775f2ba9f6a92bca30bc2095067b8a9d77876d1fad6cc3b4a43"}, @@ -4764,4 +4730,4 @@ testing = ["big-O", "jaraco.functools", "jaraco.itertools", "more-itertools", "p [metadata] lock-version = "2.0" python-versions = ">=3.8,<3.12" -content-hash = "1bc38867f792a6bd269ca1bebb4a406e5e8b9e1e361e2ddc96a90795c44fb342" +content-hash = "f4c4dfa5323488895b4d7b059cef8c825cdb3041c7728e68d5599d0ba49f1945" diff --git a/pyproject.toml b/pyproject.toml index 07db5609..7dfece0a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -54,9 +54,7 @@ pydocstyle = "*" pylint = "*" pylint-django = "*" pylint-nautobot = "*" -python-dotenv = "^1.0.0" yamllint = "*" -toml = "*" Markdown = "*" # Rendering docs to HTML mkdocs = "1.5.2" @@ -135,11 +133,5 @@ add_ignore = "D212,D417" requires = ["poetry_core>=1.0.0"] build-backend = "poetry.core.masonry.api" -[tool.pytest.ini_options] -testpaths = [ - "tests" -] -addopts = "-vv --doctest-modules" - [tool.pylint-nautobot] supported_nautobot_versions = ["2"] diff --git a/tasks.py b/tasks.py index 77fc61ee..acad3324 100644 --- a/tasks.py +++ b/tasks.py @@ -13,17 +13,9 @@ """ import os -from distutils.util import strtobool -from invoke import Collection -from invoke import task as invoke_task - -# from dotenv import load_dotenv - - -# def _load_dotenv(): -# load_dotenv("./development/development.env") -# load_dotenv("./development/creds.env") +from invoke.collection import Collection +from invoke.tasks import task as invoke_task def is_truthy(arg): @@ -38,7 +30,14 @@ def is_truthy(arg): """ if isinstance(arg, bool): return arg - return bool(strtobool(arg)) + + val = str(arg).lower() + if val in ("y", "yes", "t", "true", "on", "1"): + return True + elif val in ("n", "no", "f", "false", "off", "0"): + return False + else: + raise ValueError(f"Invalid truthy value: `{arg}`") # Use pyinvoke configuration for default values, see http://docs.pyinvoke.org/en/stable/concepts/configuration.html @@ -47,7 +46,7 @@ def is_truthy(arg): namespace.configure( { "nautobot_golden_config": { - "nautobot_ver": "2.0.0", + "nautobot_ver": "2.0.4", "project_name": "nautobot-golden-config", "python_ver": "3.11", "local": False, @@ -64,6 +63,10 @@ def is_truthy(arg): ) +def _is_compose_included(context, name): + return f"docker-compose.{name}.yml" in context.nautobot_golden_config.compose_files + + def task(function=None, *args, **kwargs): """Task decorator to override the default Invoke task decorator and add each task to the invoke namespace.""" @@ -100,7 +103,7 @@ def docker_compose(context, command, **kwargs): **kwargs.pop("env", {}), } compose_command_tokens = [ - "docker-compose", + "docker compose", f"--project-name {context.nautobot_golden_config.project_name}", f'--project-directory "{context.nautobot_golden_config.compose_dir}"', ] @@ -169,10 +172,17 @@ def generate_packages(context): run_command(context, command) -@task -def lock(context): +@task( + help={ + "check": ( + "If enabled, check for outdated dependencies in the poetry.lock file, " + "instead of generating a new one. (default: disabled)" + ) + } +) +def lock(context, check=False): """Generate poetry.lock inside the Nautobot container.""" - run_command(context, "poetry lock --no-update") + run_command(context, f"poetry {'check' if check else 'lock --no-update'}") # ------------------------------------------------------------------------------ @@ -360,157 +370,141 @@ def exec(context, service="nautobot", command="bash", file=""): @task( help={ + "db-name": "Database name (default: Nautobot database)", + "input-file": "SQL file to execute and quit (default: empty, start interactive CLI)", + "output-file": "Ouput file, overwrite if exists (default: empty, output to stdout)", "query": "SQL command to execute and quit (default: empty)", - "input": "SQL file to execute and quit (default: empty)", - "output": "Ouput file, overwrite if exists (default: empty)", } ) -def dbshell(context, query="", input="", output=""): +def dbshell(context, db_name="", input_file="", output_file="", query=""): """Start database CLI inside the running `db` container. Doesn't use `nautobot-server dbshell`, using started `db` service container only. """ - if input and query: - raise ValueError("Cannot specify both, `input` and `query` arguments") - if output and not (input or query): - raise ValueError("`output` argument requires `input` or `query` argument") + if input_file and query: + raise ValueError("Cannot specify both, `input_file` and `query` arguments") + if output_file and not (input_file or query): + raise ValueError("`output_file` argument requires `input_file` or `query` argument") - # _load_dotenv() + env = {} + if query: + env["_SQL_QUERY"] = query - service = "db" - env_vars = {} - command = ["exec"] + command = [ + "exec", + "--env=_SQL_QUERY" if query else "", + "-- db sh -c '", + ] - if "docker-compose.mysql.yml" in context.nautobot_golden_config.compose_files: - env_vars["MYSQL_PWD"] = os.getenv("MYSQL_PASSWORD") + if _is_compose_included(context, "mysql"): command += [ - "--env=MYSQL_PWD", - "--", - service, "mysql", - f"--user='{os.getenv('MYSQL_USER')}'", - f"--database='{os.getenv('MYSQL_DATABASE')}'", + "--user=$MYSQL_USER", + "--password=$MYSQL_PASSWORD", + f"--database={db_name or '$MYSQL_DATABASE'}", ] - if query: - command += [f"--execute='{query}'"] - elif "docker-compose.postgres.yml" in context.nautobot_golden_config.compose_files: + elif _is_compose_included(context, "postgres"): command += [ - "--", - service, "psql", - f"--username='{os.getenv('POSTGRES_USER')}'", - f"--dbname='{os.getenv('POSTGRES_DB')}'", + "--username=$POSTGRES_USER", + f"--dbname={db_name or '$POSTGRES_DB'}", ] - if query: - command += [f"--command='{query}'"] else: raise ValueError("Unsupported database backend.") - if input: - command += [f"< '{input}'"] - if output: - command += [f"> '{output}'"] + command += [ + "'", + '<<<"$_SQL_QUERY"' if query else "", + f"< '{input_file}'" if input_file else "", + f"> '{output_file}'" if output_file else "", + ] - docker_compose(context, " ".join(command), env=env_vars, pty=not (input or output or query)) + docker_compose(context, " ".join(command), env=env, pty=not (input_file or output_file or query)) @task( help={ - "input": "SQL dump file to replace the existing database with. This can be generated using `invoke backup-db` (default: `dump.sql`).", + "input-file": "SQL dump file to replace the existing database with. This can be generated using `invoke backup-db` (default: `dump.sql`).", } ) -def import_db(context, input="dump.sql"): +def import_db(context, input_file="dump.sql"): """Stop Nautobot containers and replace the current database with the dump into the running `db` container.""" docker_compose(context, "stop -- nautobot worker") - # _load_dotenv() + command = ["exec -- db sh -c '"] - service = "db" - env_vars = {} - command = ["exec"] - - if "docker-compose.mysql.yml" in context.nautobot_golden_config.compose_files: - env_vars["MYSQL_PWD"] = os.getenv("MYSQL_PASSWORD") + if _is_compose_included(context, "mysql"): command += [ - "--env=MYSQL_PWD", - "--", - service, "mysql", - f"--user='{os.getenv('MYSQL_USER')}'", - f"--database='{os.getenv('MYSQL_DATABASE')}'", + "--database=$MYSQL_DATABASE", + "--user=$MYSQL_USER", + "--password=$MYSQL_PASSWORD", ] - elif "docker-compose.postgres.yml" in context.nautobot_golden_config.compose_files: + elif _is_compose_included(context, "postgres"): command += [ - "--", - service, "psql", - f"--username='{os.getenv('POSTGRES_USER')}'", + "--username=$POSTGRES_USER", "postgres", ] else: raise ValueError("Unsupported database backend.") - command += [f"< '{input}'"] + command += [ + "'", + f"< '{input_file}'", + ] - docker_compose(context, " ".join(command), env=env_vars, pty=False) + docker_compose(context, " ".join(command), pty=False) print("Database import complete, you can start Nautobot now: `invoke start`") @task( help={ - "output": "Ouput file, overwrite if exists (default: `dump.sql`)", + "db-name": "Database name to backup (default: Nautobot database)", + "output-file": "Ouput file, overwrite if exists (default: `dump.sql`)", "readable": "Flag to dump database data in more readable format (default: `True`)", } ) -def backup_db(context, output="dump.sql", readable=True): - """Dump database into `output` file from running `db` container.""" - # _load_dotenv() - - service = "db" - env_vars = {} - command = ["exec"] +def backup_db(context, db_name="", output_file="dump.sql", readable=True): + """Dump database into `output_file` file from running `db` container.""" + command = ["exec -- db sh -c '"] - if "docker-compose.mysql.yml" in context.nautobot_golden_config.compose_files: - env_vars["MYSQL_PWD"] = os.getenv("MYSQL_ROOT_PASSWORD") + if _is_compose_included(context, "mysql"): command += [ - "--env=MYSQL_PWD", - "--", - service, "mysqldump", "--user=root", + "--password=$MYSQL_ROOT_PASSWORD", "--add-drop-database", "--skip-extended-insert" if readable else "", "--databases", - os.getenv("MYSQL_DATABASE", ""), + db_name if db_name else "$MYSQL_DATABASE", ] - elif "docker-compose.postgres.yml" in context.nautobot_golden_config.compose_files: + elif _is_compose_included(context, "postgres"): command += [ - "--", - service, "pg_dump", "--clean", "--create", "--if-exists", - f"--username='{os.getenv('POSTGRES_USER')}'", - f"--dbname='{os.getenv('POSTGRES_DB')}'", + "--username=$POSTGRES_USER", + f"--dbname={db_name or '$POSTGRES_DB'}", + "--inserts" if readable else "", ] - - if readable: - command += ["--inserts"] else: raise ValueError("Unsupported database backend.") - if output: - command += [f"> '{output}'"] + command += [ + "'", + f"> '{output_file}'", + ] - docker_compose(context, " ".join(command), env=env_vars, pty=False) + docker_compose(context, " ".join(command), pty=False) print(50 * "=") - print("The database backup has been successfully completed and saved to the file:") - print(output) - print("If you want to import this database backup, please execute the following command:") - print(f"invoke import-db --input '{output}'") + print("The database backup has been successfully completed and saved to the following file:") + print(output_file) + print("You can import this database backup with the following command:") + print(f"invoke import-db --input-file '{output_file}'") print(50 * "=") @@ -691,6 +685,10 @@ def tests(context, failfast=False, keepdb=False, lint_only=False): pydocstyle(context) print("Running yamllint...") yamllint(context) + print("Running poetry check...") + lock(context, check=True) + print("Running migrations check...") + check_migrations(context) print("Running pylint...") pylint(context) print("Running mkdocs...") @@ -698,5 +696,5 @@ def tests(context, failfast=False, keepdb=False, lint_only=False): if not lint_only: print("Running unit tests...") unittest(context, failfast=failfast, keepdb=keepdb) + unittest_coverage(context) print("All tests have passed!") - unittest_coverage(context)