Rename to . Utilise in RubyDataSource. Add caching in rubyAPI . Fix reference_urls in RubyDataSource

Signed-off-by: Shivam Sandbhor <shivam.sandbhor@gmail.com>

Author: sbs2001

vulnerabilities/importers/__init__.py

@@ -26,4 +26,4 @@
 from vulnerabilities.importers.npm import NpmDataSource
 from vulnerabilities.importers.rust import RustDataSource
 from vulnerabilities.importers.safety_db import SafetyDbDataSource
-from vulnerabilities.importers.ruby import rubyDataSource
+from vulnerabilities.importers.ruby import RubyDataSource

vulnerabilities/importers/ruby.py

@@ -1,3 +1,25 @@
+# Copyright (c) 2017 nexB Inc. and others. All rights reserved.
+# http://nexb.com and https://github.com/nexB/vulnerablecode/
+# The VulnerableCode software is licensed under the Apache License version 2.0.
+# Data generated with VulnerableCode require an acknowledgment.
+#
+# You may not use this software except in compliance with the License.
+# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software distributed
+# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+# CONDITIONS OF ANY KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations under the License.
+#
+# When you publish or redistribute any data created with VulnerableCode or any VulnerableCode
+# derivative work, you must accompany this data with the following acknowledgment:
+#
+#  Generated with VulnerableCode and provided on an "AS IS" BASIS, WITHOUT WARRANTIES
+#  OR CONDITIONS OF ANY KIND, either express or implied. No content created from
+#  VulnerableCode should be considered or used as legal advice. Consult an Attorney
+#  for any legal advice.
+#  VulnerableCode is a free software code scanning tool from nexB Inc. and others.
+#  Visit https://github.com/nexB/vulnerablecode/ for support and download.
+
 from json import JSONDecodeError
 from typing import Set
 from typing import List
@@ -12,17 +34,26 @@
 from vulnerabilities.data_source import GitDataSource
 
 
-class rubyDataSource(GitDataSource):
+class RubyDataSource(GitDataSource):
 
     def __enter__(self):
-        super(rubyDataSource, self).__enter__()
+        super(RubyDataSource, self).__enter__()
 
         if not getattr(self, '_added_files', None):
             self._added_files, self._updated_files = self.file_changes(
                 recursive=True, file_ext='yml', subdir='./gems')
 
     def updated_advisories(self) -> Set[Advisory]:
-        files = self._updated_files.union(self._added_files)
+        files = self._updated_files
+        advisories = []
+        for f in files:
+            processed_data = self.process_file(f)
+            if processed_data:
+                advisories.append(processed_data)
+        return self.batch_advisories(advisories)
+
+    def added_advisories(self) -> Set[Advisory]:
+        files = self._added_files
         advisories = []
         for f in files:
             processed_data = self.process_file(f)
@@ -77,7 +108,7 @@ def process_file(self, path) -> List[Advisory]:
                 summary=record.get('description', ''),
                 impacted_package_urls=impacted_purls,
                 resolved_package_urls=resolved_purls,
-                reference_urls=record.get('url', ''),
+                reference_urls=[record.get('url', '')],
                 cve_id=cve_id
             )
 
@@ -108,6 +139,7 @@ class rubyAPI:
 
     def __init__(self):
         self.client = requests.Session()
+        self.cache = {}
 
     def call_api(self, pkg_name) -> List:
         end_pt = self.base_endpt.format(pkg_name)
@@ -120,7 +152,11 @@ def call_api(self, pkg_name) -> List:
 
     def get_all_version_of_package(self, pkg_name) -> Set[str]:
         all_versions = set()
+        if self.cache.get(pkg_name):
+            return self.cache.get(pkg_name)
+
         json_resp = self.call_api(pkg_name)
         for release in json_resp:
             all_versions.add(release['number'])
+        self.cache[pkg_name] = all_versions
         return all_versions

vulnerabilities/migrations/0008_ruby_importer.py

@@ -1,3 +1,25 @@
+# Copyright (c) 2017 nexB Inc. and others. All rights reserved.
+# http://nexb.com and https://github.com/nexB/vulnerablecode/
+# The VulnerableCode software is licensed under the Apache License version 2.0.
+# Data generated with VulnerableCode require an acknowledgment.
+#
+# You may not use this software except in compliance with the License.
+# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software distributed
+# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+# CONDITIONS OF ANY KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations under the License.
+#
+# When you publish or redistribute any data created with VulnerableCode or any VulnerableCode
+# derivative work, you must accompany this data with the following acknowledgment:
+#
+#  Generated with VulnerableCode and provided on an "AS IS" BASIS, WITHOUT WARRANTIES
+#  OR CONDITIONS OF ANY KIND, either express or implied. No content created from
+#  VulnerableCode should be considered or used as legal advice. Consult an Attorney
+#  for any legal advice.
+#  VulnerableCode is a free software code scanning tool from nexB Inc. and others.
+#  Visit https://github.com/nexB/vulnerablecode/ for support and download.
+
 from django.db import migrations
 
 
@@ -8,7 +30,7 @@ def add_ruby_importer(apps, _):
         name='ruby',
         license='',
         last_run=None,
-        data_source='rubyDataSource',
+        data_source='RubyDataSource',
         data_source_cfg={
             'repository_url': 'https://github.com/rubysec/ruby-advisory-db.git',
         },

vulnerabilities/tests/test_ruby.py

@@ -1,3 +1,25 @@
+# Copyright (c) 2017 nexB Inc. and others. All rights reserved.
+# http://nexb.com and https://github.com/nexB/vulnerablecode/
+# The VulnerableCode software is licensed under the Apache License version 2.0.
+# Data generated with VulnerableCode require an acknowledgment.
+#
+# You may not use this software except in compliance with the License.
+# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software distributed
+# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+# CONDITIONS OF ANY KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations under the License.
+#
+# When you publish or redistribute any data created with VulnerableCode or any VulnerableCode
+# derivative work, you must accompany this data with the following acknowledgment:
+#
+#  Generated with VulnerableCode and provided on an "AS IS" BASIS, WITHOUT WARRANTIES
+#  OR CONDITIONS OF ANY KIND, either express or implied. No content created from
+#  VulnerableCode should be considered or used as legal advice. Consult an Attorney
+#  for any legal advice.
+#  VulnerableCode is a free software code scanning tool from nexB Inc. and others.
+#  Visit https://github.com/nexB/vulnerablecode/ for support and download.
+
 import os
 import pathlib
 from unittest.mock import patch
@@ -6,7 +28,7 @@
 
 from packageurl import PackageURL
 
-from vulnerabilities.importers.ruby import rubyDataSource
+from vulnerabilities.importers.ruby import RubyDataSource
 from vulnerabilities.data_source import GitDataSourceConfiguration
 from vulnerabilities.data_source import Advisory
 
@@ -20,13 +42,13 @@
     MOCK_ADDED_FILES.append(filepath.absolute())
 
 
-class rubyDataSourceTest(TestCase):
+class RubyDataSourceTest(TestCase):
 
     @classmethod
     def setUpClass(cls):
         data_source_cfg = {
             'repository_url': 'https://github.com/rubysec/ruby-advisory-db.git', }
-        cls.data_src = rubyDataSource(1, config=data_source_cfg)
+        cls.data_src = RubyDataSource(1, config=data_source_cfg)
 
     @patch('vulnerabilities.importers.ruby.rubyAPI.get_all_version_of_package',
            return_value={'1.0.0', '1.8.0', '2.0.3'})
@@ -60,7 +82,7 @@ def test_process_file(self, mock_write):
                         version='2.0.3',
                         qualifiers=OrderedDict(),
                         subpath=None)},
-                reference_urls='https://github.com/sinatra/sinatra/pull/1379',
+                reference_urls=['https://github.com/sinatra/sinatra/pull/1379'],
                 reference_ids=[],
                 cve_id='CVE-2018-7212'),
             Advisory(
@@ -89,7 +111,7 @@ def test_process_file(self, mock_write):
                         version='2.0.3',
                         qualifiers=OrderedDict(),
                         subpath=None)},
-                reference_urls='https://github.com/sinatra/sinatra/issues/1428',
+                reference_urls=['https://github.com/sinatra/sinatra/issues/1428'],
                 reference_ids=[],
                 cve_id='CVE-2018-11627'),
             None}