Merge branch 'main' into archive-ref

Author: pombredanne

.github/workflows/pypi-release.yml

@@ -85,3 +85,5 @@ jobs:
       - name: Publish to PyPI
         if: startsWith(github.ref, 'refs/tags/')
         uses: pypa/gh-action-pypi-publish@ed0c53931b1dc9bd32cbe73a98c7f6766f8a527e  # v1.13.0
+        with:
+          verbose: true

CHANGELOG.rst

@@ -1,6 +1,37 @@
 Release notes
 =============
 
+next release
+---------------------
+
+- WARNING: Vulnerablecode V1 API and UI has stopped supporting Ubuntu OVAL advisories, please shift to V3 API for new Ubuntu advisories. 
+
+Version v38.6.0
+---------------------
+
+- fix: add missing clean_downloads step for alpine_linux v2 importer (https://github.com/aboutcode-org/vulnerablecode/pull/2282)
+- fix: fix AdvisoryDataV2 deserialization and validate version ranges correctly (https://github.com/aboutcode-org/vulnerablecode/pull/2285)
+- fix: ignore conflicts while bulk creating v2 packages (https://github.com/aboutcode-org/vulnerablecode/pull/2289)
+- fix: use shared cache backend across WSGI workers (https://github.com/aboutcode-org/vulnerablecode/pull/2290)
+- fix: delete AdvisorySet associated with malformed aliases (https://github.com/aboutcode-org/vulnerablecode/pull/2291)
+
+
+Version v38.5.0
+---------------------
+
+- fix: Make package_url field unique for PackageV2
+
+Version v38.4.0
+---------------------
+
+- fix: run pipeline scheduling jobs in respective queues (https://github.com/aboutcode-org/vulnerablecode/pull/2263)
+- feat: show queue load factors on the pipeline dashboard (https://github.com/aboutcode-org/vulnerablecode/pull/2264)
+
+Version v38.3.0
+---------------------
+
+- feat: add high priority queue and run version range unfurling pipeline more frequently (https://github.com/aboutcode-org/vulnerablecode/pull/2256)
+
 Version v38.1.0
 ---------------------
 

docker-compose.yml

@@ -57,6 +57,17 @@ services:
       - db
       - vulnerablecode
 
+  vulnerablecode_rqworker_high:
+    build: .
+    command: wait-for-it web:8000 -- python ./manage.py rqworker high
+    env_file:
+      - docker.env
+    volumes:
+      - /etc/vulnerablecode/:/etc/vulnerablecode/
+    depends_on:
+      - vulnerablecode_redis
+      - db
+      - vulnerablecode
 
   nginx:
     image: nginx

docs/source/PIPELINES-AVID.rst

@@ -0,0 +1,79 @@
+.. _pipelines_avid:
+
+Pipelines AVID Mapping
+=========================
+
+.. list-table:: Pipeline AVID Mapping
+   :header-rows: 1
+   :widths: 35 65
+
+   * - pipeline name
+     - AVID
+   * - alpine_linux_importer_v2
+     - {package_name}/{distroversion}/{version}/{vulnerability_id}
+   * - aosp_dataset_fix_commits
+     - CVE ID of the record
+   * - apache_httpd_importer_v2
+     - CVE ID of the record
+   * - apache_kafka_importer_v2
+     - CVE ID of the record
+   * - apache_tomcat_importer_v2
+     - {page_id}/{cve_id}
+   * - archlinux_importer_v2
+     - AVG ID of the record
+   * - curl_importer_v2
+     - CURL-CVE ID of the record
+   * - debian_importer_v2
+     - {package_name}/{debian_record_id}
+   * - elixir_security_importer_v2
+     - {package_name}/{file_id}
+   * - epss_importer_v2
+     - CVE ID of the record
+   * - fireeye_importer_v2
+     - {file_id}
+   * - gentoo_importer_v2
+     - GLSA ID of the record
+   * - github_osv_importer_v2
+     - ID of the OSV record
+   * - gitlab_importer_v2
+     - Identifier of the GitLab community advisory record
+   * - istio_importer_v2
+     - ISTIO-SECURITY-<ID>
+   * - mattermost_importer_v2
+     - MMSA-<ID>
+   * - mozilla_importer_v2
+     - MFSA-<ID>
+   * - nginx_importer_v2
+     - First alias of the record
+   * - nodejs_security_wg
+     - NPM-<ID>
+   * - nvd_importer_v2
+     - CVE ID of the record
+   * - openssl_importer_v2
+     - CVE ID of the record
+   * - oss_fuzz_importer_v2
+     - ID of the OSV record
+   * - postgresql_importer_v2
+     - CVE ID of the record
+   * - project-kb-msr-2019_v2
+     - Vulnerability ID of the record
+   * - project-kb-statements_v2
+     - Vulnerability ID of the record
+   * - pypa_importer_v2
+     - {package_name}/{ID of the OSV record}
+   * - pysec_importer_v2
+     - ID of the OSV record
+   * - redhat_importer_v2
+     - RHSA ID of the record
+   * - retiredotnet_importer_v2
+     - retiredotnet-{file_id}
+   * - ruby_importer_v2
+     - {file_id}
+   * - suse_importer_v2
+     - CVE ID of the record
+   * - ubuntu_osv_importer_v2
+     - ID of the OSV record
+   * - vulnrichment_importer_v2
+     - CVE ID of the record
+   * - xen_importer_v2
+     - XSA-<ID>

docs/source/SOURCES.rst

@@ -0,0 +1,112 @@
+.. _sources:
+
+Sources
+=======
+
+.. list-table:: Sources
+   :header-rows: 1
+   :widths: 20 50 30
+
+   * - Importer Name
+     - Data Source
+     - Ecosystems Covered
+
+   * - rust
+     - https://github.com/RustSec/advisory-db
+     - rust crates
+
+   * - alpine
+     - https://secdb.alpinelinux.org/
+     - alpine packages
+
+   * - archlinux
+     - https://security.archlinux.org/json
+     - arch packages
+
+   * - debian
+     - https://security-tracker.debian.org/tracker/data/json
+     - debian packages
+
+   * - npm
+     - https://github.com/nodejs/security-wg.git
+     - npm packages
+
+   * - ruby
+     - https://github.com/rubysec/ruby-advisory-db.git
+     - ruby gems
+
+   * - ubuntu
+     -
+     - ubuntu packages
+
+   * - retiredotnet
+     - https://github.com/RetireNet/Packages.git
+     - .NET packages
+
+   * - suse_backports
+     - http://ftp.suse.com/pub/projects/security/yaml/
+     - SUSE packages
+
+   * - debian_oval
+     - https://www.debian.org/security/oval/
+     - debian packages
+
+   * - redhat
+     - https://access.redhat.com/hydra/rest/securitydata/cve.json
+     - rpm packages
+
+   * - nvd
+     - https://nvd.nist.gov/vuln/data-feeds#JSON_FEED
+     - none
+
+   * - gentoo
+     - https://anongit.gentoo.org/git/data/glsa.git
+     - gentoo packages
+
+   * - openssl
+     - https://www.openssl.org/news/vulnerabilities.xml
+     - openssl
+
+   * - ubuntu_usn
+     - https://usn.ubuntu.com/usn-db/database-all.json.bz2
+     - ubuntu packages
+
+   * - github
+     - https://api.github.com/graphql
+     - maven, .NET, php-composer, pypi packages, ruby gems
+
+   * - msr2019
+     - https://raw.githubusercontent.com/SAP/project-kb/master/MSR2019/dataset/vulas_db_msr2019_release.csv
+     - maven packages
+
+   * - apache_httpd
+     - https://httpd.apache.org/security/json
+     - apache-httpd
+
+   * - kaybee
+     - https://github.com/SAP/project-kb.git
+     - maven packages
+
+   * - nginx
+     - http://nginx.org/en/security_advisories.html
+     - nginx
+
+   * - postgresql
+     - https://www.postgresql.org/support/security/
+     - postgresql
+
+   * - elixir_security
+     - https://github.com/dependabot/elixir-security-advisories
+     - hex packages
+
+   * - suse_scores
+     - https://ftp.suse.com/pub/projects/security/yaml/suse-cvss-scores.yaml
+     - vulnerability severity scores by SUSE
+
+   * - mozilla
+     - https://github.com/mozilla/foundation-security-advisories
+     - mozilla
+
+   * - mattermost
+     - https://mattermost.com/security-updates/
+     - mattermost server, desktop and mobile apps