fix: add User-Agent value for data collection HTTP requests (#574)

Signed-off-by: tdruez <tdruez@aboutcode.org>

Author: tdruez

component_catalog/tests/test_models.py

@@ -2473,6 +2473,21 @@ def test_collect_package_data(self, mock_get):
         }
         self.assertEqual(expected_data, collect_package_data(download_url))
 
+    @mock.patch("requests.get")
+    def test_collect_package_data_user_agent(self, mock_get):
+        mock_get.return_value = mock.Mock(
+            content=b"\x00",
+            headers={"content-length": 1},
+            status_code=200,
+            url="http://domain.com/a.zip",
+        )
+
+        collect_package_data("http://domain.com/a.zip")
+        self.assertEqual(
+            download.USER_AGENT,
+            mock_get.call_args.kwargs["headers"]["User-Agent"],
+        )
+
     def test_package_create_save_set_usage_policy_from_license(self):
         from policy.models import AssociatedPolicy
         from policy.models import UsagePolicy

dejacode_toolkit/download.py

@@ -26,14 +26,19 @@
 CONTENT_MAX_LENGTH = 536870912  # 512 MB
 DEFAULT_TIMEOUT = 5
 
+# Some sites (e.g. crates.io) reject requests without an identifying User-Agent.
+USER_AGENT = "DejaCode (https://github.com/aboutcode-org/dejacode)"
+
 
 class DataCollectionException(Exception):
     pass
 
 
 def collect_package_data(url):
+    headers = {"User-Agent": USER_AGENT}
+
     try:
-        response = requests.get(url, timeout=DEFAULT_TIMEOUT, stream=True)
+        response = requests.get(url, headers=headers, timeout=DEFAULT_TIMEOUT, stream=True)
     except (TimeoutError, requests.RequestException) as e:
         raise DataCollectionException(e)