3838from vulnerabilities .references import XsaReference
3939from vulnerabilities .references import ZbxReference
4040
41+ BASE_URL = "https://secdb.alpinelinux.org/"
42+
4143
4244class AlpineImporter (Importer ):
4345 spdx_license_expression = "CC-BY-SA-4.0"
4446 license_url = "https://secdb.alpinelinux.org/license.txt"
4547
4648 def advisory_data (self ) -> Iterable [AdvisoryData ]:
4749 advisories = []
48- advisory_links = fetch_advisory_links ()
50+ page_response_content = fetch_response (BASE_URL ).content
51+ advisory_directory_links = fetch_advisory_directory_links (page_response_content )
52+ advisory_links = []
53+ for advisory_directory_link in advisory_directory_links :
54+ advisory_directory_page = fetch_response (advisory_directory_link ).content
55+ advisory_links .extend (
56+ fetch_advisory_links (advisory_directory_page , advisory_directory_link )
57+ )
4958 for link in advisory_links :
5059 record = fetch_response (link ).json ()
5160 if not record ["packages" ]:
@@ -64,34 +73,30 @@ def fetch_response(url):
6473 raise Exception ("Failed to fetch data from the URL" )
6574
6675
67- def fetch_advisory_links ( ):
76+ def fetch_advisory_directory_links ( page_response_content ):
6877 """
69- Fetch content from `BASE_URL` and scrape data from it
70- Take all the alpine versions starting with `v`
71- collect all the urls for json files in the directory
72- and return them as `advisory_links`
78+ Return `advisory_directory_links` present in `index_page`
7379 """
74- BASE_URL = "https://secdb.alpinelinux.org/"
75- page_response_content = fetch_response (BASE_URL ).content
7680 index_page = BeautifulSoup (page_response_content , features = "lxml" )
77-
7881 alpine_versions = [link .text for link in index_page .find_all ("a" ) if link .text .startswith ("v" )]
7982
8083 assert alpine_versions , f"No versions found in { BASE_URL }
8184
8285 advisory_directory_links = [f"{ BASE_URL } { version } for version in alpine_versions ]
8386
84- advisory_links = []
85- for advisory_directory_link in advisory_directory_links :
86- advisory_directory_page = fetch_response (advisory_directory_link ).content
87- advisory_directory_page = BeautifulSoup (advisory_directory_page , features = "lxml" )
88- anchor_tags = advisory_directory_page .find_all ("a" )
89- assert anchor_tags , f"No anchor tags found in { advisory_directory_link }
90- for anchor_tag in anchor_tags :
91- if anchor_tag .text .endswith ("json" ):
92- advisory_links .append (f"{ advisory_directory_link } { anchor_tag .text } )
93-
94- return advisory_links
87+ return advisory_directory_links
88+
89+
90+ def fetch_advisory_links (advisory_directory_page , advisory_directory_link ):
91+ """
92+ Yield json file urls present in `advisory_directory_page`
93+ """
94+ advisory_directory_page = BeautifulSoup (advisory_directory_page , features = "lxml" )
95+ anchor_tags = advisory_directory_page .find_all ("a" )
96+ assert anchor_tags , f"No anchor tags found in { advisory_directory_link }
97+ for anchor_tag in anchor_tags :
98+ if anchor_tag .text .endswith ("json" ):
99+ yield f"{ advisory_directory_link } { anchor_tag .text }
95100
96101
97102def process_record (record : dict ) -> List [AdvisoryData ]:
0 commit comments