Merge branch 'main' into cvss-v-display

Author: TG1999

requirements.txt

@@ -35,7 +35,7 @@ executing==0.8.3
 freezegun==1.2.1
 frozenlist==1.3.0
 gitdb==4.0.9
-GitPython==3.1.37
+GitPython==3.1.41
 gunicorn==20.1.0
 idna==3.3
 imagesize==1.3.0
@@ -44,7 +44,7 @@ iniconfig==1.1.1
 ipython==8.10.0
 isort==5.10.1
 jedi==0.18.1
-Jinja2==3.1.1
+Jinja2==3.1.3
 jsonschema==3.2.0
 license-expression==21.6.14
 lxml==4.9.1
@@ -106,7 +106,7 @@ toml==0.10.2
 tomli==2.0.1
 traitlets==5.1.1
 typing_extensions==4.1.1
-univers==30.10.0
+univers==30.11.0
 urllib3==1.26.18
 wcwidth==0.2.5
 websocket-client==0.59.0

setup.cfg

@@ -71,7 +71,7 @@ install_requires =
 
     #essentials
     packageurl-python>=0.10.5rc1
-    univers>=30.10.0
+    univers>=30.11.0
     license-expression>=21.6.14
 
     # file and data formats

vulnerabilities/importers/fireeye.py

@@ -44,7 +44,7 @@ def advisory_data(self) -> Iterable[AdvisoryData]:
                 if Path(file).stem == "README":
                     continue
                 try:
-                    with open(file) as f:
+                    with open(file, encoding="utf-8-sig") as f:
                         yield parse_advisory_data(raw_data=f.read(), file=file, base_path=base_path)
                 except UnicodeError:
                     logger.error(f"Invalid file {file}")

vulnerabilities/importers/github.py

@@ -92,6 +92,7 @@
 class GitHubAPIImporter(Importer):
     spdx_license_expression = "CC-BY-4.0"
     importer_name = "GHSA Importer"
+    license_url = "https://github.com/github/advisory-database/blob/main/LICENSE.md"
 
     def advisory_data(self) -> Iterable[AdvisoryData]:
         for ecosystem, package_type in PACKAGE_TYPE_BY_GITHUB_ECOSYSTEM.items():

vulnerabilities/migrations/0055_remove_changelogs_with_same_data_different_software_version.py

@@ -0,0 +1,52 @@
+#
+# Copyright (c) nexB Inc. and others. All rights reserved.
+# VulnerableCode is a trademark of nexB Inc.
+# SPDX-License-Identifier: Apache-2.0
+# See http://www.apache.org/licenses/LICENSE-2.0 for the license text.
+# See https://github.com/nexB/vulnerablecode for support or download.
+# See https://aboutcode.org for more information about nexB OSS projects.
+#
+
+from django.db import migrations
+from django.db import models
+
+
+class Migration(migrations.Migration):
+
+    def remove_duped_changelogs(apps, schema_editor):
+        PackageChangeLog = apps.get_model("vulnerabilities", "PackageChangeLog")
+        VulnerabilityChangeLog = apps.get_model("vulnerabilities", "VulnerabilityChangeLog")
+
+        models_list = [PackageChangeLog, VulnerabilityChangeLog]
+
+        for model in models_list:
+        # Identify duplicate records based on actor_name, action_type, and source_url
+            duplicate_records = model.objects.values('actor_name', 'action_type', 'source_url').annotate(count=models.Count('id')).filter(count__gt=1)
+
+            to_be_deleted = list()
+
+            for duplicate_set in duplicate_records:
+                # Get the records for the current duplicate set
+                records_to_delete = model.objects.filter(
+                    actor_name=duplicate_set['actor_name'],
+                    action_type=duplicate_set['action_type'],
+                    source_url=duplicate_set['source_url']
+                ).order_by('-software_version')
+
+                # Keep the record with the older software version
+                record_to_keep = records_to_delete.last()
+
+                # Delete the records with the newer software version
+                to_be_deleted.extend(records_to_delete.exclude(id=record_to_keep.id))
+
+            to_be_deleted = list(set(to_be_deleted))
+            to_be_deleted = [rec.id for rec in to_be_deleted]
+            model.objects.filter(id__in = to_be_deleted).delete()
+
+    dependencies = [
+        ("vulnerabilities", "0054_alter_packagechangelog_software_version_and_more"),
+    ]
+
+    operations = [
+        migrations.RunPython(remove_duped_changelogs, reverse_code=migrations.RunPython.noop),
+    ]

vulnerabilities/migrations/0056_alter_packagechangelog_unique_together_and_more.py

@@ -0,0 +1,21 @@
+# Generated by Django 4.1.13 on 2024-01-22 09:42
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("vulnerabilities", "0055_remove_changelogs_with_same_data_different_software_version"),
+    ]
+
+    operations = [
+        migrations.AlterUniqueTogether(
+            name="packagechangelog",
+            unique_together={("action_time", "actor_name", "action_type", "source_url")},
+        ),
+        migrations.AlterUniqueTogether(
+            name="vulnerabilitychangelog",
+            unique_together={("action_time", "actor_name", "action_type", "source_url")},
+        ),
+    ]

vulnerabilities/models.py

@@ -1145,6 +1145,7 @@ def get_iso_time(self):
     class Meta:
         abstract = True
         ordering = ("-action_time",)
+        unique_together = ("action_time", "actor_name", "action_type", "source_url")
 
 
 class VulnerabilityHistoryManager(models.Manager):

vulnerabilities/tests/test_data/fireeye/fireeye_test3.md

@@ -0,0 +1,31 @@
+# MNDT-2023-0017
+
+The IBM Personal Communications (PCOMM) application 13.0.0 and earlier caused a user's plaintext password to be written to the `C:\Temp\pcsnp_init.log` file when re-connection was made through a remote desktop protocol.
+
+## Common Weakness Enumeration
+CWE-312: Cleartext Storage of Sensitive Information
+
+## Impact
+High - An attacker with low-privilege access to a host with IBM PCOMM could recover the plaintext password of another user.
+
+## Exploitability
+Low - Exploitability varies depending on the environment in which IBM PCOMM is installed. Mandiant identified this vulnerability when conducting independent security research for a client that used Citrix to connect to shared Windows Server instances. In certain environments where remote desktop is used to connect to shared hosts with IBM PCOMM installed, the exploitability is greatly increased.
+
+## CVE Reference
+CVE-2016-0321 - scope expanded
+
+## Technical Details
+While conducting independent security research, Mandiant identified a plaintext Active Directory password stored within the `C:\Temp\pcsnp_init.log` file. The affected host had IBM PCOMM version 13.0.0 installed and was used by multiple users who connected with Citrix. Upon a user connecting, disconnecting, and connecting again, the user's plaintext password was stored in the `C:\Temp\pcsnp_init.log` file.
+
+## Discovery Credits
+- Adin Drabkin, Mandiant
+- Matthew Rotlevi, Mandiant
+
+## Disclosure Timeline
+- 2023-09-26 - Issue reported to the vendor.
+- 2023-11-03 - The vendor updated the security bulletin for CVE-2016-0321 to include all known affected and fixed versions.
+
+## References
+- [IBM Security Bulletin](https://www.ibm.com/support/pages/security-bulletin-ibm-personal-communications-could-allow-remote-user-obtain-sensitive-information-including-user-passwords-allowing-unauthorized-access-cve-2016-0321)
+- [IBM Personal Communications](https://www.ibm.com/support/pages/ibm-personal-communications)
+- [Mitre CVE-2016-0321](https://www.cve.org/CVERecord?id=CVE-2016-0321)

vulnerabilities/tests/test_data_migrations.py

@@ -610,3 +610,53 @@ def setUpBeforeMigration(self, apps):
     def test_removal_of_duped_purls(self):
         Package = apps.get_model("vulnerabilities", "Package")
         assert Package.objects.count() == 1
+
+
+class TestRemoveDupedChangeLogWithSameData(TestMigrations):
+    app_name = "vulnerabilities"
+    migrate_from = "0054_alter_packagechangelog_software_version_and_more"
+    migrate_to = "0055_remove_changelogs_with_same_data_different_software_version"
+
+    def setUpBeforeMigration(self, apps):
+        PackageChangeLog = apps.get_model("vulnerabilities", "PackageChangeLog")
+        VulnerabilityChangeLog = apps.get_model("vulnerabilities", "VulnerabilityChangeLog")
+        Package = apps.get_model("vulnerabilities", "Package")
+        Vulnerability = apps.get_model("vulnerabilities", "Vulnerability")
+        pkg1 = Package.objects.create(type="nginx", name="nginx", qualifiers={"os": "windows"})
+        vuln = Vulnerability.objects.create(summary="NEW")
+        PackageChangeLog.objects.create(
+            actor_name="Nginx",
+            action_type=1,
+            source_url="test",
+            software_version="1",
+            package=pkg1,
+            related_vulnerability=vuln,
+        )
+        PackageChangeLog.objects.create(
+            actor_name="Nginx",
+            action_type=1,
+            source_url="test",
+            software_version="2",
+            package=pkg1,
+            related_vulnerability=vuln,
+        )
+        VulnerabilityChangeLog.objects.create(
+            actor_name="Nginx",
+            action_type=1,
+            source_url="test",
+            software_version="2",
+            vulnerability=vuln,
+        )
+        VulnerabilityChangeLog.objects.create(
+            actor_name="Nginx",
+            action_type=1,
+            source_url="test",
+            software_version="1",
+            vulnerability=vuln,
+        )
+
+    def test_removal_of_changelog(self):
+        PackageChangeLog = apps.get_model("vulnerabilities", "PackageChangeLog")
+        VulnerabilityChangeLog = apps.get_model("vulnerabilities", "VulnerabilityChangeLog")
+        assert PackageChangeLog.objects.all().count() == 1
+        assert VulnerabilityChangeLog.objects.all().count() == 1

vulnerabilities/tests/test_fireeye.py

@@ -172,3 +172,48 @@ def test_parse_advisory_data_2(self):
         result = imported_data.to_dict()
 
         util_tests.check_results_against_json(result, expected_file)
+
+    def test_md_list_to_dict_2(self):
+        expected_output = {
+            "# MNDT-2023-0017\n": [
+                "\n",
+                "The IBM Personal Communications (PCOMM) application 13.0.0 and earlier caused a user's plaintext password to be written to the `C:\\Temp\\pcsnp_init.log` file when re-connection was made through a remote desktop protocol.\n",
+                "\n",
+            ],
+            "## Common Weakness Enumeration\n": [
+                "CWE-312: Cleartext Storage of Sensitive Information\n",
+                "\n",
+            ],
+            "## Impact\n": [
+                "High - An attacker with low-privilege access to a host with IBM PCOMM could recover the plaintext password of another user.\n",
+                "\n",
+            ],
+            "## Exploitability\n": [
+                "Low - Exploitability varies depending on the environment in which IBM PCOMM is installed. Mandiant identified this vulnerability when conducting independent security research for a client that used Citrix to connect to shared Windows Server instances. In certain environments where remote desktop is used to connect to shared hosts with IBM PCOMM installed, the exploitability is greatly increased.\n",
+                "\n",
+            ],
+            "## CVE Reference\n": ["CVE-2016-0321 - scope expanded\n", "\n"],
+            "## Technical Details\n": [
+                "While conducting independent security research, Mandiant identified a plaintext Active Directory password stored within the `C:\\Temp\\pcsnp_init.log` file. The affected host had IBM PCOMM version 13.0.0 installed and was used by multiple users who connected with Citrix. Upon a user connecting, disconnecting, and connecting again, the user's plaintext password was stored in the `C:\\Temp\\pcsnp_init.log` file.\n",
+                "\n",
+            ],
+            "## Discovery Credits\n": [
+                "- Adin Drabkin, Mandiant\n",
+                "- Matthew Rotlevi, Mandiant\n",
+                "\n",
+            ],
+            "## Disclosure Timeline\n": [
+                "- 2023-09-26 - Issue reported to the vendor.\n",
+                "- 2023-11-03 - The vendor updated the security bulletin for CVE-2016-0321 to include all known affected and fixed versions.\n",
+                "\n",
+            ],
+            "## References\n": [
+                "- [IBM Security Bulletin](https://www.ibm.com/support/pages/security-bulletin-ibm-personal-communications-could-allow-remote-user-obtain-sensitive-information-including-user-passwords-allowing-unauthorized-access-cve-2016-0321)\n",
+                "- [IBM Personal Communications](https://www.ibm.com/support/pages/ibm-personal-communications)\n",
+                "- [Mitre CVE-2016-0321](https://www.cve.org/CVERecord?id=CVE-2016-0321)\n",
+            ],
+        }
+        with open(os.path.join(TEST_DATA, "fireeye_test3.md"), encoding="utf-8-sig") as f:
+            md_list = f.readlines()
+            md_dict = md_list_to_dict(md_list)
+            assert md_dict == expected_output