Skip to content

Commit 6588009

Browse files
ziadhanyziadhany
committed
Move the implementation logic into the pipes
Fix a typo in the image documentation Signed-off-by: ziad hany <ziadhany2016@gmail.com>
1 parent 1d80f7a commit 6588009

4 files changed

Lines changed: 191 additions & 128 deletions

File tree

‎docs/images/tutorial-web-ui-project-form-repo-health.png‎

-460 Bytes
Loading

‎scanpipe/pipelines/scan_repo_health.py‎

Lines changed: 18 additions & 126 deletions
Original file line numberDiff line numberDiff line change
@@ -19,24 +19,19 @@
1919
#
2020
# ScanCode.io is a free software code scanning tool from nexB Inc. and others.
2121
# Visit https://github.com/aboutcode-org/scancode.io for support and download.
22-
import json
23-
import subprocess
24-
import urllib.parse
25-
from os import environ
2622

2723
from scanpipe.pipelines import Pipeline
28-
from scanpipe.pipes import run_command_safely
29-
30-
GRIMOIRELAB_METRICS_EXECUTABLE = environ.get("GRIMOIRELAB_METRICS_EXECUTABLE", "")
31-
GRIMOIRELAB_OPENSEARCH_INDEX = environ.get("GRIMOIRELAB_OPENSEARCH_INDEX", "")
32-
GRIMOIRELAB_OPENSEARCH_PASSWORD = environ.get("GRIMOIRELAB_OPENSEARCH_PASSWORD", "")
33-
GRIMOIRELAB_OPENSEARCH_URL = environ.get("GRIMOIRELAB_OPENSEARCH_URL", "")
34-
GRIMOIRELAB_OPENSEARCH_USERNAME = environ.get("GRIMOIRELAB_OPENSEARCH_USERNAME", "")
35-
GRIMOIRELAB_PASSWORD = environ.get("GRIMOIRELAB_PASSWORD", "")
36-
GRIMOIRELAB_URL = environ.get("GRIMOIRELAB_URL", "")
37-
GRIMOIRELAB_USERNAME = environ.get("GRIMOIRELAB_USERNAME", "")
38-
GRIMOIRELAB_ECOSYSTEM = environ.get("GRIMOIRELAB_ECOSYSTEM", "")
39-
GRIMOIRELAB_PROJECT = environ.get("GRIMOIRELAB_PROJECT", "")
24+
from scanpipe.pipes import repo_health
25+
from scanpipe.pipes.repo_health import GRIMOIRELAB_ECOSYSTEM
26+
from scanpipe.pipes.repo_health import GRIMOIRELAB_METRICS_EXECUTABLE
27+
from scanpipe.pipes.repo_health import GRIMOIRELAB_OPENSEARCH_INDEX
28+
from scanpipe.pipes.repo_health import GRIMOIRELAB_OPENSEARCH_PASSWORD
29+
from scanpipe.pipes.repo_health import GRIMOIRELAB_OPENSEARCH_URL
30+
from scanpipe.pipes.repo_health import GRIMOIRELAB_OPENSEARCH_USERNAME
31+
from scanpipe.pipes.repo_health import GRIMOIRELAB_PASSWORD
32+
from scanpipe.pipes.repo_health import GRIMOIRELAB_PROJECT
33+
from scanpipe.pipes.repo_health import GRIMOIRELAB_URL
34+
from scanpipe.pipes.repo_health import GRIMOIRELAB_USERNAME
4035

4136

4237
class ScanRepoHealth(Pipeline):
@@ -71,126 +66,23 @@ def get_availability(cls):
7166

7267
def get_repo_url_input(self):
7368
"""Validate and extract the repository URL from the project's input sources"""
74-
if len(self.project.input_sources) != 1:
75-
raise ValueError("Expected exactly one input source")
76-
77-
self.repo_url = self.project.input_sources[0]["download_url"]
78-
if not is_valid_vcs_url(self.repo_url):
79-
raise ValueError(
80-
"Invalid input source: the pipeline accepts only a valid repository URL"
81-
)
82-
83-
self.repo_url = self.repo_url.replace("git://", "https://")
84-
if not self.repo_url.endswith(".git"):
85-
self.repo_url += ".git"
69+
self.repo_url = repo_health.get_repo_url_input(project=self.project)
8670

8771
def collect_and_store_grimoire_metric(self):
8872
"""
8973
Run the grimoirelab-metrics command against the input source.
9074
Save the generated metrics JSON to the project output directory.
9175
"""
92-
self.metrics_output_path = self.project.get_output_file_path("metrics", "json")
93-
command_args = [
94-
GRIMOIRELAB_METRICS_EXECUTABLE,
95-
self.repo_url,
96-
"--grimoirelab-url",
97-
GRIMOIRELAB_URL,
98-
"--grimoirelab-user",
99-
GRIMOIRELAB_USERNAME,
100-
"--grimoirelab-password",
101-
GRIMOIRELAB_PASSWORD,
102-
"--grimoirelab-ecosystem",
103-
GRIMOIRELAB_ECOSYSTEM,
104-
"--grimoirelab-project",
105-
GRIMOIRELAB_PROJECT,
106-
"--opensearch-url",
107-
GRIMOIRELAB_OPENSEARCH_URL,
108-
"--opensearch-index",
109-
GRIMOIRELAB_OPENSEARCH_INDEX,
110-
"--opensearch-user",
111-
GRIMOIRELAB_OPENSEARCH_USERNAME,
112-
"--opensearch-password",
113-
GRIMOIRELAB_OPENSEARCH_PASSWORD,
114-
"--output",
115-
str(self.metrics_output_path),
116-
]
117-
118-
try:
119-
run_command_safely(command_args=command_args)
120-
self.log("GrimoireLab metrics pipeline completed successfully")
121-
except subprocess.SubprocessError:
122-
raise RuntimeError("Grimoirelab-metrics client failure")
123-
except FileNotFoundError:
124-
raise FileNotFoundError(
125-
"Grimoirelab-metrics not found. "
126-
"Please ensure grimoirelab-metrics is correctly configured."
127-
)
76+
self.metrics_output_path = repo_health.collect_and_store_grimoire_metric(
77+
project=self.project, repo_url=self.repo_url, logger=self.log
78+
)
12879

12980
def format_metrics_output(self):
13081
"""
13182
Format the GrimoireLab metrics output by extracting the repository URL,
13283
score, and metrics from the generated JSON and overwriting it with a
13384
simplified structure, and updating the project's extra data.
13485
"""
135-
if not self.metrics_output_path.exists():
136-
raise FileNotFoundError(
137-
"GrimoireLab client did not return a valid metrics JSON file"
138-
)
139-
140-
with open(self.metrics_output_path) as f:
141-
data = json.load(f)
142-
143-
if not isinstance(data, dict):
144-
raise ValueError("Invalid metrics JSON: Expected a JSON object.")
145-
146-
package_data = data.get("packages")
147-
if not package_data or not isinstance(package_data, dict):
148-
raise ValueError(
149-
"Invalid metrics JSON: Missing or malformed 'packages' section."
150-
)
151-
152-
packages = list(package_data.values())
153-
if not packages:
154-
raise ValueError("Invalid metrics JSON: 'packages' contains no data.")
155-
156-
target_package = packages[0]
157-
repository = target_package.get("repository")
158-
score = target_package.get("score")
159-
metrics = target_package.get("metrics")
160-
161-
if repository is None or score is None or metrics is None:
162-
raise ValueError(
163-
f"Invalid metrics JSON. missing or null field(s): "
164-
f"repository: {repository}, score: {score}, metrics: {metrics}"
165-
)
166-
167-
result = {
168-
"repository": repository,
169-
"score": score,
170-
"metrics": metrics,
171-
}
172-
173-
with open(self.metrics_output_path, "w") as f:
174-
json.dump(result, f)
175-
176-
self.project.update_extra_data(result)
177-
178-
179-
def is_valid_vcs_url(url):
180-
"""Determine whether the URL string has the expected syntax of a VCS repository."""
181-
if not isinstance(url, str) or not url:
182-
return False
183-
184-
if any(char.isspace() for char in url):
185-
return False
186-
187-
forbidden_chars = ["|", ";", "&", "`", "$(", ">", "<", "&&", "||"]
188-
if any(char in url for char in forbidden_chars):
189-
return False
190-
191-
parsed = urllib.parse.urlparse(url)
192-
valid_schemes = {"https", "http", "git"}
193-
if parsed.scheme in valid_schemes and parsed.netloc:
194-
return True
195-
196-
return False
86+
repo_health.format_metrics_output(
87+
project=self.project, metrics_output_path=self.metrics_output_path
88+
)

‎scanpipe/pipes/repo_health.py‎

Lines changed: 171 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,171 @@
1+
# SPDX-License-Identifier: Apache-2.0
2+
#
3+
# http://nexb.com and https://github.com/aboutcode-org/scancode.io
4+
# The ScanCode.io software is licensed under the Apache License version 2.0.
5+
# Data generated with ScanCode.io is provided as-is without warranties.
6+
# ScanCode is a trademark of nexB Inc.
7+
#
8+
# You may not use this software except in compliance with the License.
9+
# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0
10+
# Unless required by applicable law or agreed to in writing, software distributed
11+
# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
12+
# CONDITIONS OF ANY KIND, either express or implied. See the License for the
13+
# specific language governing permissions and limitations under the License.
14+
#
15+
# Data Generated with ScanCode.io is provided on an "AS IS" BASIS, WITHOUT WARRANTIES
16+
# OR CONDITIONS OF ANY KIND, either express or implied. No content created from
17+
# ScanCode.io should be considered or used as legal advice. Consult an Attorney
18+
# for any legal advice.
19+
#
20+
# ScanCode.io is a free software code scanning tool from nexB Inc. and others.
21+
# Visit https://github.com/aboutcode-org/scancode.io for support and download.
22+
23+
import json
24+
import subprocess
25+
import urllib.parse
26+
from os import environ
27+
28+
from scanpipe.pipes import run_command_safely
29+
30+
GRIMOIRELAB_METRICS_EXECUTABLE = environ.get("GRIMOIRELAB_METRICS_EXECUTABLE", "")
31+
GRIMOIRELAB_OPENSEARCH_INDEX = environ.get("GRIMOIRELAB_OPENSEARCH_INDEX", "")
32+
GRIMOIRELAB_OPENSEARCH_PASSWORD = environ.get("GRIMOIRELAB_OPENSEARCH_PASSWORD", "")
33+
GRIMOIRELAB_OPENSEARCH_URL = environ.get("GRIMOIRELAB_OPENSEARCH_URL", "")
34+
GRIMOIRELAB_OPENSEARCH_USERNAME = environ.get("GRIMOIRELAB_OPENSEARCH_USERNAME", "")
35+
GRIMOIRELAB_PASSWORD = environ.get("GRIMOIRELAB_PASSWORD", "")
36+
GRIMOIRELAB_URL = environ.get("GRIMOIRELAB_URL", "")
37+
GRIMOIRELAB_USERNAME = environ.get("GRIMOIRELAB_USERNAME", "")
38+
GRIMOIRELAB_ECOSYSTEM = environ.get("GRIMOIRELAB_ECOSYSTEM", "")
39+
GRIMOIRELAB_PROJECT = environ.get("GRIMOIRELAB_PROJECT", "")
40+
41+
42+
def get_repo_url_input(project):
43+
"""Validate and extract the repository URL from the project's input sources"""
44+
if len(project.input_sources) != 1:
45+
raise ValueError("Expected exactly one input source")
46+
47+
repo_url = project.input_sources[0]["download_url"]
48+
if not is_valid_vcs_url(repo_url):
49+
raise ValueError(
50+
"Invalid input source: the pipeline accepts only a valid repository URL"
51+
)
52+
53+
repo_url = repo_url.replace("git://", "https://")
54+
if not repo_url.endswith(".git"):
55+
repo_url += ".git"
56+
57+
return repo_url
58+
59+
60+
def collect_and_store_grimoire_metric(project, repo_url, logger=None):
61+
"""
62+
Run the grimoirelab-metrics command against the input source.
63+
Save the generated metrics JSON to the project output directory.
64+
"""
65+
metrics_output_path = project.get_output_file_path("metrics", "json")
66+
command_args = [
67+
GRIMOIRELAB_METRICS_EXECUTABLE,
68+
repo_url,
69+
"--grimoirelab-url",
70+
GRIMOIRELAB_URL,
71+
"--grimoirelab-user",
72+
GRIMOIRELAB_USERNAME,
73+
"--grimoirelab-password",
74+
GRIMOIRELAB_PASSWORD,
75+
"--grimoirelab-ecosystem",
76+
GRIMOIRELAB_ECOSYSTEM,
77+
"--grimoirelab-project",
78+
GRIMOIRELAB_PROJECT,
79+
"--opensearch-url",
80+
GRIMOIRELAB_OPENSEARCH_URL,
81+
"--opensearch-index",
82+
GRIMOIRELAB_OPENSEARCH_INDEX,
83+
"--opensearch-user",
84+
GRIMOIRELAB_OPENSEARCH_USERNAME,
85+
"--opensearch-password",
86+
GRIMOIRELAB_OPENSEARCH_PASSWORD,
87+
"--output",
88+
str(metrics_output_path),
89+
]
90+
91+
try:
92+
run_command_safely(command_args=command_args)
93+
logger("GrimoireLab metrics pipeline completed successfully")
94+
return metrics_output_path
95+
except subprocess.SubprocessError:
96+
raise RuntimeError("Grimoirelab-metrics client failure")
97+
except FileNotFoundError:
98+
raise FileNotFoundError(
99+
"Grimoirelab-metrics not found. "
100+
"Please ensure grimoirelab-metrics is correctly configured."
101+
)
102+
103+
104+
def format_metrics_output(project, metrics_output_path):
105+
"""
106+
Format the GrimoireLab metrics output by extracting the repository URL,
107+
score, and metrics from the generated JSON and overwriting it with a
108+
simplified structure, and updating the project's extra data.
109+
"""
110+
if not metrics_output_path.exists():
111+
raise FileNotFoundError(
112+
"GrimoireLab client did not return a valid metrics JSON file"
113+
)
114+
115+
with open(metrics_output_path) as f:
116+
data = json.load(f)
117+
118+
if not isinstance(data, dict):
119+
raise ValueError("Invalid metrics JSON: Expected a JSON object.")
120+
121+
package_data = data.get("packages")
122+
if not package_data or not isinstance(package_data, dict):
123+
raise ValueError(
124+
"Invalid metrics JSON: Missing or malformed 'packages' section."
125+
)
126+
127+
packages = list(package_data.values())
128+
if not packages:
129+
raise ValueError("Invalid metrics JSON: 'packages' contains no data.")
130+
131+
target_package = packages[0]
132+
repository = target_package.get("repository")
133+
score = target_package.get("score")
134+
metrics = target_package.get("metrics")
135+
136+
if repository is None or score is None or metrics is None:
137+
raise ValueError(
138+
f"Invalid metrics JSON. missing or null field(s): "
139+
f"repository: {repository}, score: {score}, metrics: {metrics}"
140+
)
141+
142+
result = {
143+
"repository": repository,
144+
"score": score,
145+
"metrics": metrics,
146+
}
147+
148+
with open(metrics_output_path, "w") as f:
149+
json.dump(result, f)
150+
151+
project.update_extra_data(result)
152+
153+
154+
def is_valid_vcs_url(url):
155+
"""Determine whether the URL string has the expected syntax of a VCS repository."""
156+
if not isinstance(url, str) or not url:
157+
return False
158+
159+
if any(char.isspace() for char in url):
160+
return False
161+
162+
forbidden_chars = ["|", ";", "&", "`", "$(", ">", "<", "&&", "||"]
163+
if any(char in url for char in forbidden_chars):
164+
return False
165+
166+
parsed = urllib.parse.urlparse(url)
167+
valid_schemes = {"https", "http", "git"}
168+
if parsed.scheme in valid_schemes and parsed.netloc:
169+
return True
170+
171+
return False

‎scanpipe/tests/pipes/test_scan_repo_health.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,7 @@
3131
from django.test import TestCase
3232

3333
from scanpipe.pipelines.scan_repo_health import ScanRepoHealth
34-
from scanpipe.pipelines.scan_repo_health import is_valid_vcs_url
34+
from scanpipe.pipes.repo_health import is_valid_vcs_url
3535

3636

3737
class ScanRepoGrimoirelabTest(TestCase):
@@ -47,7 +47,7 @@ def setUp(self):
4747
self.pipeline.project.get_output_file_path.return_value = "metrics.json"
4848
self.pipeline.log = MagicMock()
4949

50-
@patch("scanpipe.pipelines.scan_repo_health.run_command_safely")
50+
@patch("scanpipe.pipes.repo_health.run_command_safely")
5151
def test_collect_and_store_grimoire_metric_called_process_error(
5252
self, mock_run_command
5353
):

0 commit comments

Comments
 (0)