Merge pull request #15 from nexB/models_db

Database Model #5

Author: tdruez

.travis.yml

@@ -1,12 +1,18 @@
 language: python
 python: 3.6
 
-install: pip install pycodestyle
+install:
+  - pip install -r requirements.txt
+  - pip install pycodestyle
 
 before_script:
-  - pycodestyle --exclude=migrations,settings.py --max-line-length=100 .
+  - pycodestyle --exclude=migrations,settings.py,lib,tests --max-line-length=100 .
+
+script:
+  - python3.6 -m pytest -v tests/
 
 notifications:
+  email: false
   webhooks:
     urls:
       - https://webhooks.gitter.im/e/b119fa557626081e1f36

README.md

@@ -24,5 +24,16 @@ Tests
 -----
 
 ```
-python3.6 -m pytest -v
+pycodestyle --exclude=migrations,settings.py,lib --max-line-length=100 .
+python3.6 -m pytest -v tests/
+```
+
+Scrape
+------
+
+```
+from scraper import debian, ubuntu
+
+debian.scrape_cves()
+ubuntu.scrape_cves()
 ```

api_data.py

@@ -52,4 +52,4 @@ def extract_fields(data, fields_names):
     cve-search' api. Takes as input data, fields requested
     """
     return [{name: item.get(name) for name in fields_names}
-        for item in data]
+            for item in data]

app/app/urls.py

@@ -13,7 +13,7 @@
     1. Import the include() function: from django.conf.urls import url, include
     2. Add a URL to urlpatterns:  url(r'^blog/', include('blog.urls'))
 """
-from django.conf.urls import url,include
+from django.conf.urls import url, include
 from django.contrib import admin
 
 urlpatterns = [

app/vulncode_app/models.py

@@ -26,38 +26,40 @@
 
 
 class Vulnerability(models.Model):
-	vulnerability_id = models.AutoField(primary_key=True)
-	summary = models.TextField(max_length=50)
-	cvss = models.FloatField(max_length=50)
+    summary = models.TextField(max_length=50, help_text="Summary of the vulnerability")
+    cvss = models.FloatField(max_length=50, help_text="CVSS Score")
 
 
 class VulnerabilityReference(models.Model):
-	vulnerability_id = models.ForeignKey('Vulnerability')
-	source = models.CharField(max_length=50)
-	reference_id = models.CharField(max_length=50)
-	url = models.URLField(max_length=50)
+    vulnerability = models.ForeignKey('Vulnerability')
+    source = models.CharField(max_length=50, help_text="Source's name eg:NVD")
+    reference_id = models.CharField(max_length=50, help_text="Reference ID, eg:CVE-ID")
+    url = models.URLField(max_length=1024, help_text="URL of Vulnerability data")
 
 
 class ImpactedPackage(models.Model):
-	vulnerability_id = models.ForeignKey('Vulnerability')
-	package_id = models.ForeignKey('Package')
+    vulnerability = models.ForeignKey('Vulnerability')
+    package = models.ForeignKey('Package')
 
 
 class ResolvedPackage(models.Model):
-	vulnerability_id = models.ForeignKey('Vulnerability')
-	package_id = models.ForeignKey('Package')
+    vulnerability = models.ForeignKey('Vulnerability')
+    package = models.ForeignKey('Package')
 
 
 class Package(models.Model):
-	package_id = models.AutoField(primary_key=True)
-	platform = models.CharField(max_length=50)
-	name = models.CharField(max_length=50)
-	version = models.FloatField(max_length=50)
+    platform = models.CharField(max_length=50, help_text="Package platform eg:maven")
+    name = models.CharField(max_length=50, help_text="Package name")
+    version = models.CharField(max_length=50, help_text="Pacakge version")
 
 
 class PackageReference(models.Model):
-	package_id = models.ForeignKey('Package')
-	repository = models.CharField(max_length=50)
-	platform = models.CharField(max_length=50)
-	name = models.CharField(max_length=50)
-	version = models.FloatField(max_length=50)
+    package = models.ForeignKey('Package')
+    repository = models.CharField(max_length=50,
+                                  help_text="Repository URL eg:http://central.maven.org")
+    platform = models.CharField(max_length=50,
+                                help_text="Platform eg:maven")
+    name = models.CharField(max_length=50,
+                            help_text="Package reference name eg:org.apache.commons.io")
+    version = models.CharField(max_length=50,
+                               help_text="Reference version")

requirements.txt

@@ -1,4 +1,5 @@
 beautifulsoup4==4.6.0
+lxml==3.8.0
 
 # Tests
 pytest==3.1.3

scraper/debian.py

@@ -0,0 +1,94 @@
+#
+# Copyright (c) 2017 nexB Inc. and others. All rights reserved.
+# http://nexb.com and https://github.com/nexB/vulnerablecode/
+# The VulnerableCode software is licensed under the Apache License version 2.0.
+# Data generated with VulnerableCode require an acknowledgment.
+#
+# You may not use this software except in compliance with the License.
+# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software distributed
+# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+# CONDITIONS OF ANY KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations under the License.
+#
+# When you publish or redistribute any data created with VulnerableCode or any VulnerableCode
+# derivative work, you must accompany this data with the following acknowledgment:
+#
+#  Generated with VulnerableCode and provided on an "AS IS" BASIS, WITHOUT WARRANTIES
+#  OR CONDITIONS OF ANY KIND, either express or implied. No content created from
+#  VulnerableCode should be considered or used as legal advice. Consult an Attorney
+#  for any legal advice.
+#  VulnerableCode is a free software code scanning tool from nexB Inc. and others.
+#  Visit https://github.com/nexB/vulnerablecode/ for support and download.
+
+import logging
+import re
+from urllib.request import urlopen
+
+import bs4
+
+
+DEBIAN_ROOT_URL = 'https://security-tracker.debian.org'
+
+
+def extract_tracker_paths(html):
+    """
+    Return a list of tracker URL paths extracted from the given `html` input.
+    """
+    soup = bs4.BeautifulSoup(html, 'lxml')
+    tracker_links = soup.findAll('a', href=re.compile('^/track+.*'))
+    return [link.get('href') for link in tracker_links]
+
+
+def extract_cves_from_tracker(html):
+    """
+    Return all CVEs extracted from the given `html` input.
+    """
+    cve_id = []
+    package_name = []
+    vulnerability_status = []
+    soup = bs4.BeautifulSoup(html, 'lxml')
+
+    for tag in soup.find_all('a'):
+        href = tag.get('href')
+
+        if re.search('/tracker/CVE-(.+)', href):
+            id = re.findall('(?<=/tracker/).*', href)
+            cve_id.append(id[0])
+
+        if re.search('^/tracker/TEMP-+.*', href):
+            id = re.findall('(?<=/tracker/).*', href)
+            cve_id.append(id[0])
+
+        if re.search('/tracker/source-package/(.+)', href):
+            pkg = re.findall('(?<=/tracker/source-package/).*', href)
+            package_name.append(pkg[0])
+
+        # if package name is empty, use the previous package name
+        if href == '/tracker/source-package/':
+            package_name.append(pkg)
+
+    for tag in soup.find_all('td'):
+        if 'medium' in tag or 'low' in tag or 'not yet assigned' in tag:
+            vulnerability_status.append(tag.text)
+        elif tag.find_all('span', {'class': 'red'}) and tag.text == 'high**' or tag.text == 'high':
+            vulnerability_status.append(tag.text)
+
+    return cve_id, package_name, vulnerability_status
+
+
+def scrape_cves():
+    """
+    Runs the full scraping process of Debian CVEs.
+    """
+    tracker_root_html = urlopen(f'{DEBIAN_ROOT_URL}/tracker/').read()
+    tracker_paths = extract_tracker_paths(tracker_root_html)
+
+    cves = []
+    for tracker_path in tracker_paths:
+        tracker_url = f'{DEBIAN_ROOT_URL}{tracker_path}/'
+        logging.info(f'Visiting: {tracker_url}')
+        html = urlopen(tracker_url).read()
+        cves.append(extract_cves_from_tracker(html))
+
+    return cves

scraper/scraper_debian.py

@@ -21,37 +21,44 @@
 #  VulnerableCode is a free software code scanning tool from nexB Inc. and others.
 #  Visit https://github.com/nexB/vulnerablecode/ for support and download.
 
-import bs4 as bs 
 import re
 from urllib.request import urlopen
 
+import bs4
 
-def ubuntu_data():
-	cve_id = []
-	package_name = []
-	vulnerability_status = []
-	
-	url = urlopen("https://people.canonical.com/~ubuntu-security/cve/main.html")
-	soup = bs.BeautifulSoup (url, "lxml")
-	
-	"""
-	Scrape vulnerability status.
-	Ubuntu provides a general vulnerability 
-	status of a package across all it's releases. 
-	"""
-	for tag in soup.find_all('tr'):
-		if re.match('<\w+\s\w+="(\w+)">', str(tag)):
-			status = re.findall('<\w+\s\w+="(\w+)">', str(tag))
-			vulnerability_status.append(status[0])
-
-	for tag in soup.find_all('a'):
-		href = tag.get ('href', None)
-
-		if re.findall ('^CVE.+', href): 
-			cve_id.append(href)
-		
-		if re.match('\pkg+.*', href):
-			pkg = re.findall ('pkg/(.+)\.html', href)
-			package_name.append(pkg[0])
-
-	return cve_id, package_name, vulnerability_status
+
+UBUNTU_ROOT_URL = 'https://people.canonical.com/~ubuntu-security/cve/main.html'
+
+
+def extract_cves(html):
+    soup = bs4.BeautifulSoup(html, 'lxml')
+
+    cve_id = []
+    package_name = []
+    vulnerability_status = []
+
+    for tag in soup.find_all('tr'):
+        if re.match('<\w+\s\w+="(\w+)">', str(tag)):
+            status = re.findall('<\w+\s\w+="(\w+)">', str(tag))
+            vulnerability_status.append(status[0])
+
+    for tag in soup.find_all('a'):
+        href = tag.get('href', None)
+
+        if re.findall('^CVE.+', href):
+            cve_id.append(href)
+
+        if re.match('pkg+.*', href):
+            pkg = re.findall('pkg/(.+)\.html', href)
+            package_name.append(pkg[0])
+
+    return cve_id, vulnerability_status, package_name
+
+
+def scrape_cves():
+    """
+    Runs the full scraping process of Ubuntu CVEs.
+    """
+    html = urlopen(UBUNTU_ROOT_URL).read()
+    cves = extract_cves(html)
+    return cves