ci: periodically run the unit tests of all GitHub-hosted published ch…

…arms (canonical#1365)

Once a month, before the typical release time, run a workflow that tests
all charms published on CharmHub where the source location is in the
charm metadata and that location is in a `canonical` GitHub repository.

This is much the same as the existing data, hello, and observability
tests, but at a larger scale and without the expectation that all of the
tests would pass in every PR. The intention is that this gives us an
insight into when ops changes break the most important charms - not that
we would necessarily ensure that the tests were always passing (but we
would at least look into it).

This also tests the `charmcraft init` profiles (other than the framework
profiles, which don't start out with usable tests) to detect breakage
there.

To avoid manually maintaining the list of charms, a script is added that
will update the workflow file with the latest list, pulled from
CharmHub. This is not currently automatically run.

---------

Co-authored-by: Ben Hoyt <[email protected]>

.github/update-published-charms-tests-workflow.py

@@ -0,0 +1,132 @@
+#! /usr/bin/env python
+
+# Copyright 2024 Canonical Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Update a GitHub workload that runs `tox -e unit` on all published charms.
+
+Charms that are not hosted on GitHub are skipped, as well as any charms where
+the source URL could not be found.
+"""
+
+import json
+import logging
+import pathlib
+import re
+import typing
+import urllib.error
+import urllib.parse
+import urllib.request
+
+logger = logging.getLogger(__name__)
+
+
+URL_BASE = 'https://api.charmhub.io/v2/charms/info'
+WORKFLOW = pathlib.Path(__file__).parent / 'workflows' / 'published-charms-tests.yaml'
+
+SKIP = {
+    # Handled by db-charm-tests.yaml
+    'postgresql-operator',
+    'postgresql-k8s-operator',
+    'mysql-operator',
+    'mysql-k8s-operator',
+    # Handled by hello-charm-tests.yaml
+    'hello-kubecon',  # Not in the canonical org anyway (jnsgruk).
+    'hello-juju-charm',  # Not in the canonical org anyway (juju).
+    # Handled by observability-charms-tests.yaml
+    'alertmanager-k8s-operator',
+    'prometheus-k8s-operator',
+    'grafana-k8s-operator',
+    # This has a redirect, which is too complicated to handle for now.
+    'bundle-jupyter',
+    # The charms are in a subfolder, which this can't handle yet.
+    'jimm',
+    'notebook-operators',
+    # Not ops.
+    'charm-prometheus-libvirt-exporter',
+    'juju-dashboard',
+    'charm-openstack-service-checks',
+}
+
+
+def packages():
+    """Get the list of published charms from Charmhub."""
+    logger.info('Fetching the list of published charms')
+    url = 'https://charmhub.io/packages.json'
+    with urllib.request.urlopen(url, timeout=120) as response:  # noqa: S310 (unsafe URL)
+        data = response.read().decode()
+        packages = json.loads(data)['packages']
+    return packages
+
+
+def get_source_url(charm: str):
+    """Get the source URL for a charm."""
+    logger.info("Looking for a 'source' URL for %s", charm)
+    try:
+        with urllib.request.urlopen(  # noqa: S310 (unsafe URL)
+            f'{URL_BASE}/{charm}?fields=result.links', timeout=30
+        ) as response:
+            data = json.loads(response.read().decode())
+            return data['result']['links']['source'][0]
+    except (urllib.error.HTTPError, KeyError):
+        pass
+    logger.info("Looking for a 'bugs-url' URL for %s", charm)
+    try:
+        with urllib.request.urlopen(  # noqa: S310 (unsafe URL)
+            f'{URL_BASE}/{charm}?fields=result.bugs-url', timeout=30
+        ) as response:
+            data = json.loads(response.read().decode())
+            return data['result']['bugs-url']
+    except (urllib.error.HTTPError, KeyError):
+        pass
+    logger.warning('Could not find a source URL for %s', charm)
+    return None
+
+
+def url_to_charm_name(url: typing.Optional[str]):
+    """Get the charm name from a URL."""
+    if not url:
+        return None
+    parsed = urllib.parse.urlparse(url)
+    if parsed.netloc != 'github.com':
+        logger.info('URL %s is not a GitHub URL', url)
+        return None
+    if not parsed.path.startswith('/canonical'):
+        # TODO: Maybe we can include some of these anyway?
+        # 'juju-solutions' and 'charmed-kubernetes' seem viable, for example.
+        logger.info('URL %s is not a Canonical charm', url)
+        return None
+    try:
+        return urllib.parse.urlparse(url).path.split('/')[2]
+    except IndexError:
+        logger.warning('Could not get charm name from URL %s', url)
+        return None
+
+
+def main():
+    """Update the workflow file."""
+    logging.basicConfig(level=logging.INFO)
+    charms = [url_to_charm_name(get_source_url(package['name'])) for package in packages()]
+    charms = [charm for charm in charms if charm and charm not in SKIP]
+    charms.sort()
+    with WORKFLOW.open('r') as f:
+        workflow = f.read()
+    repos = '\n'.join(f'          - charm-repo: canonical/{charm}' for charm in charms)
+    workflow = re.sub(r'(\s{10}- charm-repo: \S+\n)+', repos + '\n', workflow, count=1)
+    with WORKFLOW.open('w') as f:
+        f.write(workflow)
+
+
+if __name__ == '__main__':
+    main()

.github/workflows/published-charms-tests.yaml

@@ -0,0 +1,131 @@
+# To update the list of charms included here, run:
+# python .github/update-published-charms-tests-workflow.py
+
+name: Broad Charm Compatibility Tests
+
+on:
+  schedule:
+    - cron: '0 1 25 * *'
+  workflow_dispatch:
+
+jobs:
+  charm-tests:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - charm-repo: canonical/content-cache-k8s-operator
+          - charm-repo: canonical/data-platform-libs
+          - charm-repo: canonical/dex-auth-operator
+          - charm-repo: canonical/discourse-k8s-operator
+          - charm-repo: canonical/grafana-agent-k8s-operator
+          - charm-repo: canonical/hardware-observer-operator
+          - charm-repo: canonical/identity-platform-login-ui-operator
+          - charm-repo: canonical/indico-operator
+          - charm-repo: canonical/jenkins-agent-k8s-operator
+          - charm-repo: canonical/jenkins-agent-operator
+          - charm-repo: canonical/kafka-operator
+          - charm-repo: canonical/livepatch-k8s-operator
+          - charm-repo: canonical/loki-k8s-operator
+          - charm-repo: canonical/manual-tls-certificates-operator
+          - charm-repo: canonical/mongodb-operator
+          - charm-repo: canonical/mysql-router-k8s-operator
+          - charm-repo: canonical/namecheap-lego-k8s-operator
+          - charm-repo: canonical/nginx-ingress-integrator-operator
+          - charm-repo: canonical/oathkeeper-operator
+          - charm-repo: canonical/oauth2-proxy-k8s-operator
+          - charm-repo: canonical/openfga-operator
+          - charm-repo: canonical/pgbouncer-k8s-operator
+          - charm-repo: canonical/ranger-k8s-operator
+          - charm-repo: canonical/route53-lego-k8s-operator
+          - charm-repo: canonical/s3-integrator
+          - charm-repo: canonical/saml-integrator-operator
+          - charm-repo: canonical/seldon-core-operator
+          - charm-repo: canonical/self-signed-certificates-operator
+          - charm-repo: canonical/smtp-integrator-operator
+          - charm-repo: canonical/superset-k8s-operator
+          - charm-repo: canonical/temporal-admin-k8s-operator
+          - charm-repo: canonical/temporal-k8s-operator
+          - charm-repo: canonical/temporal-ui-k8s-operator
+          - charm-repo: canonical/temporal-worker-k8s-operator
+          - charm-repo: canonical/traefik-k8s-operator
+          - charm-repo: canonical/trino-k8s-operator
+          - charm-repo: canonical/wordpress-k8s-operator
+          - charm-repo: canonical/zookeeper-operator
+    steps:
+      - name: Checkout the ${{ matrix.charm-repo }} repository
+        uses: actions/checkout@v4
+        with:
+          repository: ${{ matrix.charm-repo }}
+
+      - name: Checkout the operator repository
+        uses: actions/checkout@v4
+        with:
+          path: myops
+
+      - name: Install patch dependencies
+        run: pip install poetry~=1.6
+
+      - name: Update 'ops' dependency in test charm to latest
+        run: |
+          rm -rf myops/test
+          if [ -e "test-requirements.txt" ]; then
+            sed -i -e "/^ops[ ><=]/d" -e "/canonical\/operator/d" -e "/#egg=ops/d" test-requirements.txt
+            echo -e "\ngit+$GITHUB_SERVER_URL/$GITHUB_REPOSITORY@$GITHUB_SHA#egg=ops" >> test-requirements.txt
+          fi
+          if [ -e "requirements-charmcraft.txt" ]; then
+            sed -i -e "/^ops[ ><=]/d" -e "/canonical\/operator/d" -e "/#egg=ops/d" requirements-charmcraft.txt
+            echo -e "\ngit+$GITHUB_SERVER_URL/$GITHUB_REPOSITORY@$GITHUB_SHA#egg=ops" >> requirements-charmcraft.txt
+          fi
+          if [ -e "requirements.txt" ]; then
+            sed -i -e "/^ops[ ><=]/d" -e "/canonical\/operator/d" -e "/#egg=ops/d" requirements.txt
+            echo -e "\ngit+$GITHUB_SERVER_URL/$GITHUB_REPOSITORY@$GITHUB_SHA#egg=ops" >> requirements.txt
+          elif [ -e "poetry.lock" ]; then
+            sed -i -e "s/^ops[ ><=].*/ops = {path = \"myops\"}/" pyproject.toml
+            poetry lock --no-update
+          else
+            echo "Error: No requirements.txt or poetry.lock file found"
+            exit 1
+          fi
+
+      - name: Install dependencies
+        run: pip install tox~=4.2
+
+      - name: Run the charm's unit tests
+        run: tox -vve unit
+
+  charmcraft-profile-tests:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - profile: machine
+          - profile: kubernetes
+          - profile: simple
+    steps:
+      - name: Install charmcraft
+        run: sudo snap install charmcraft --classic
+
+      - name: Charmcraft init
+        run: charmcraft init --profile=${{ matrix.profile }} --author=charm-tech
+
+      - name: Checkout the operator repository
+        uses: actions/checkout@v4
+        with:
+          path: myops
+
+      - name: Update 'ops' dependency in test charm to latest
+        run: |
+          rm -rf myops/test
+          if [ -e "requirements.txt" ]; then
+            sed -i -e "/^ops[ ><=]/d" -e "/canonical\/operator/d" -e "/#egg=ops/d" requirements.txt
+            echo -e "\ngit+$GITHUB_SERVER_URL/$GITHUB_REPOSITORY@$GITHUB_SHA#egg=ops" >> requirements.txt
+          fi
+
+      - name: Install dependencies
+        run: pip install tox~=4.2
+
+      - name: Run the charm's unit tests
+        run: tox -vve unit