From 1ecb3ec15299d3f581e7691423d19cd337c6ef69 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Cavalcante?= Date: Mon, 16 Dec 2024 10:54:08 -0300 Subject: [PATCH] refact: Use same org_path as modulesrepo first (#22) * refact: Use same org_path as modulesrepo first * refact: Change current_remote to a modulesrepo instance * refact: Try using modulesrepo in get_comps * refact: Try using utility functions to grab the yml * fix: Restore old regex behaviour * refact: Use parent path instead of regex in comp_utils * refact: Use modulesrepo for current_repo in install * refact: Move constants to new file and use MRepo Moves constants from component_utils to constants.py so that ModulesRepo can be used from inside of component_utils, thereby avoiding a circular import * Revert "refact: Use modulesrepo for current_repo in install" This reverts commit f4cc0fe7ae8bdb3b7c857ec59cf6c63f74b14507. * Reapply "refact: Use modulesrepo for current_repo in install" This reverts commit c5bebcf418ee641b9f149dde4ca73c49f8580337. * Revert "refact: Move constants to new file and use MRepo" This reverts commit c0a7a007ae15597bfd60304628e90d9c2b372a60. * refact: Change constants to diff file, use MRepo * fix: Change import in test_update --- nf_core/__main__.py | 2 +- nf_core/components/components_utils.py | 24 ++++++------------------ nf_core/components/constants.py | 4 ++++ nf_core/components/info.py | 2 +- nf_core/components/install.py | 14 ++++++++------ nf_core/components/update.py | 17 +++++++---------- nf_core/modules/modules_json.py | 3 ++- nf_core/modules/modules_repo.py | 3 +-- nf_core/synced_repo.py | 2 +- tests/modules/test_modules_json.py | 2 +- tests/modules/test_update.py | 2 +- tests/subworkflows/test_update.py | 2 +- 12 files changed, 34 insertions(+), 43 deletions(-) create mode 100644 nf_core/components/constants.py diff --git a/nf_core/__main__.py b/nf_core/__main__.py index 9f16188e9..06d4be8f7 100644 --- a/nf_core/__main__.py +++ b/nf_core/__main__.py @@ -54,7 +54,7 @@ subworkflows_test, subworkflows_update, ) -from nf_core.components.components_utils import NF_CORE_MODULES_REMOTE +from nf_core.components.constants import NF_CORE_MODULES_REMOTE from nf_core.pipelines.download import DownloadError from nf_core.utils import check_if_outdated, nfcore_logo, rich_force_colors, setup_nfcore_dir diff --git a/nf_core/components/components_utils.py b/nf_core/components/components_utils.py index 3acacb4fe..be28a0d87 100644 --- a/nf_core/components/components_utils.py +++ b/nf_core/components/components_utils.py @@ -1,25 +1,18 @@ import logging import re from pathlib import Path -from typing import TYPE_CHECKING, Dict, List, Optional, Tuple, Union +from typing import Dict, List, Optional, Tuple, Union import questionary import requests import rich.prompt import yaml -if TYPE_CHECKING: - from nf_core.modules.modules_repo import ModulesRepo - import nf_core.utils +from nf_core.modules.modules_repo import ModulesRepo log = logging.getLogger(__name__) -# Constants for the nf-core/modules repo used throughout the module files -NF_CORE_MODULES_NAME = "nf-core" -NF_CORE_MODULES_REMOTE = "https://github.com/nf-core/modules.git" -NF_CORE_MODULES_DEFAULT_BRANCH = "master" - def get_repo_info(directory: Path, use_prompt: Optional[bool] = True) -> Tuple[Path, Optional[str], str]: """ @@ -181,20 +174,15 @@ def get_components_to_install( for component in components: if isinstance(component, dict): component_name = list(component.keys())[0].lower() + branch = component[component_name].get("branch") git_remote = component[component_name]["git_remote"] - org_path_match = re.search(r"(?:https://|git@)[\w\.]+[:/](.*?)/", git_remote) - if org_path_match: - org_path = org_path_match.group(1) - else: - raise UserWarning( - f"The organisation path of {component_name} could not be established from '{git_remote}'" - ) + modules_repo = ModulesRepo(git_remote, branch=branch) current_comp_dict = subworkflows if component_name in subworkflows else modules component_dict = { - "org_path": org_path, + "org_path": modules_repo.repo_path, "git_remote": git_remote, - "branch": component[component_name].get("branch"), + "branch": branch, } current_comp_dict[component_name].update(component_dict) diff --git a/nf_core/components/constants.py b/nf_core/components/constants.py new file mode 100644 index 000000000..cc155f3d5 --- /dev/null +++ b/nf_core/components/constants.py @@ -0,0 +1,4 @@ +# Constants for the nf-core/modules repo used throughout the module files +NF_CORE_MODULES_NAME = "nf-core" +NF_CORE_MODULES_REMOTE = "https://github.com/nf-core/modules.git" +NF_CORE_MODULES_DEFAULT_BRANCH = "master" diff --git a/nf_core/components/info.py b/nf_core/components/info.py index 31769785a..a8cb5a0fa 100644 --- a/nf_core/components/info.py +++ b/nf_core/components/info.py @@ -15,7 +15,7 @@ import nf_core.utils from nf_core.components.components_command import ComponentCommand -from nf_core.components.components_utils import NF_CORE_MODULES_REMOTE +from nf_core.components.constants import NF_CORE_MODULES_REMOTE from nf_core.modules.modules_json import ModulesJson log = logging.getLogger(__name__) diff --git a/nf_core/components/install.py b/nf_core/components/install.py index f40b0712b..d45b4d2c3 100644 --- a/nf_core/components/install.py +++ b/nf_core/components/install.py @@ -15,10 +15,12 @@ import nf_core.utils from nf_core.components.components_command import ComponentCommand from nf_core.components.components_utils import ( - NF_CORE_MODULES_NAME, get_components_to_install, prompt_component_version_sha, ) +from nf_core.components.constants import ( + NF_CORE_MODULES_NAME, +) from nf_core.modules.modules_json import ModulesJson from nf_core.modules.modules_repo import ModulesRepo @@ -39,7 +41,7 @@ def __init__( installed_by: Optional[List[str]] = None, ): super().__init__(component_type, pipeline_dir, remote_url, branch, no_pull) - self.current_remote = remote_url + self.current_remote = ModulesRepo(remote_url, branch) self.branch = branch self.force = force self.prompt = prompt @@ -53,15 +55,15 @@ def __init__( def install(self, component: Union[str, Dict[str, str]], silent: bool = False) -> bool: if isinstance(component, dict): # Override modules_repo when the component to install is a dependency from a subworkflow. - remote_url = component.get("git_remote", self.current_remote) + remote_url = component.get("git_remote", self.current_remote.remote_url) branch = component.get("branch", self.branch) self.modules_repo = ModulesRepo(remote_url, branch) component = component["name"] if self.current_remote is None: - self.current_remote = self.modules_repo.remote_url + self.current_remote = self.modules_repo - if self.current_remote == self.modules_repo.remote_url and self.sha is not None: + if self.current_remote.remote_url == self.modules_repo.remote_url and self.sha is not None: self.current_sha = self.sha else: self.current_sha = None @@ -244,7 +246,7 @@ def collect_and_verify_name( raise ValueError - if self.current_remote == modules_repo.remote_url: + if self.current_remote.remote_url == modules_repo.remote_url: if not modules_repo.component_exists(component, self.component_type, commit=self.current_sha): warn_msg = f"{self.component_type[:-1].title()} '{component}' not found in remote '{modules_repo.remote_url}' ({modules_repo.branch})" log.warning(warn_msg) diff --git a/nf_core/components/update.py b/nf_core/components/update.py index 1e80b05e3..7c61b6b00 100644 --- a/nf_core/components/update.py +++ b/nf_core/components/update.py @@ -1,6 +1,5 @@ import logging import os -import re import shutil import tempfile from pathlib import Path @@ -42,7 +41,7 @@ def __init__( limit_output=False, ): super().__init__(component_type, pipeline_dir, remote_url, branch, no_pull) - self.current_remote = remote_url + self.current_remote = ModulesRepo(remote_url, branch) self.branch = branch self.force = force self.prompt = prompt @@ -97,7 +96,7 @@ def update(self, component=None, silent=False, updated=None, check_diff_exist=Tr """ if isinstance(component, dict): # Override modules_repo when the component to install is a dependency from a subworkflow. - remote_url = component.get("git_remote", self.current_remote) + remote_url = component.get("git_remote", self.current_remote.remote_url) branch = component.get("branch", self.branch) self.modules_repo = ModulesRepo(remote_url, branch) component = component["name"] @@ -892,7 +891,7 @@ def get_components_to_update(self, component): if self.component_type == "modules": # All subworkflow names in the installed_by section of a module are subworkflows using this module # We need to update them too - git_remote = self.current_remote + git_remote = self.current_remote.remote_url for subworkflow in installed_by: if subworkflow != component: for remote_url, content in mods_json["repos"].items(): @@ -965,9 +964,7 @@ def update_linked_components( def manage_changes_in_linked_components(self, component, modules_to_update, subworkflows_to_update): """Check for linked components added or removed in the new subworkflow version""" if self.component_type == "subworkflows": - org_path_match = re.search(r"(?:https://|git@)[\w\.]+[:/](.*?)/", self.current_remote) - if org_path_match: - org_path = org_path_match.group(1) + org_path = self.current_remote.repo_path subworkflow_directory = Path(self.directory, self.component_type, org_path, component) included_modules, included_subworkflows = get_components_to_install(subworkflow_directory) @@ -991,7 +988,7 @@ def manage_changes_in_linked_components(self, component, modules_to_update, subw # If a new module/subworkflow is included in the subworklfow and wasn't included before for module in included_modules: module_name = module["name"] - module["git_remote"] = module.get("git_remote", self.current_remote) + module["git_remote"] = module.get("git_remote", self.current_remote.remote_url) module["branch"] = module.get("branch", self.branch) if module_name not in modules_to_update: log.info(f"Installing newly included module '{module_name}' for '{component}'") @@ -999,7 +996,7 @@ def manage_changes_in_linked_components(self, component, modules_to_update, subw install_module_object.install(module, silent=True) for subworkflow in included_subworkflows: subworkflow_name = subworkflow["name"] - subworkflow["git_remote"] = subworkflow.get("git_remote", self.current_remote) + subworkflow["git_remote"] = subworkflow.get("git_remote", self.current_remote.remote_url) subworkflow["branch"] = subworkflow.get("branch", self.branch) if subworkflow_name not in subworkflows_to_update: log.info(f"Installing newly included subworkflow '{subworkflow_name}' for '{component}'") @@ -1022,4 +1019,4 @@ def _reset_component_type(self, original_component_type, original_update_all): self.modules_json.pipeline_components = None self.update_all = original_update_all if self.current_remote is None: - self.current_remote = self.modules_repo.remote_url + self.current_remote = self.modules_repo diff --git a/nf_core/modules/modules_json.py b/nf_core/modules/modules_json.py index a9ba3b442..f53097935 100644 --- a/nf_core/modules/modules_json.py +++ b/nf_core/modules/modules_json.py @@ -15,7 +15,8 @@ from typing_extensions import NotRequired, TypedDict # for py<3.11 import nf_core.utils -from nf_core.components.components_utils import NF_CORE_MODULES_NAME, NF_CORE_MODULES_REMOTE, get_components_to_install +from nf_core.components.components_utils import get_components_to_install +from nf_core.components.constants import NF_CORE_MODULES_NAME, NF_CORE_MODULES_REMOTE from nf_core.modules.modules_repo import ModulesRepo from nf_core.pipelines.lint_utils import dump_json_with_prettier diff --git a/nf_core/modules/modules_repo.py b/nf_core/modules/modules_repo.py index 357fc49cc..30a724d73 100644 --- a/nf_core/modules/modules_repo.py +++ b/nf_core/modules/modules_repo.py @@ -10,9 +10,8 @@ import rich.prompt from git.exc import GitCommandError, InvalidGitRepositoryError -import nf_core.modules.modules_json import nf_core.modules.modules_utils -from nf_core.components.components_utils import NF_CORE_MODULES_NAME, NF_CORE_MODULES_REMOTE +from nf_core.components.constants import NF_CORE_MODULES_NAME, NF_CORE_MODULES_REMOTE from nf_core.synced_repo import RemoteProgressbar, SyncedRepo from nf_core.utils import NFCORE_CACHE_DIR, NFCORE_DIR, load_tools_config diff --git a/nf_core/synced_repo.py b/nf_core/synced_repo.py index dd61b72a2..efb7a8e03 100644 --- a/nf_core/synced_repo.py +++ b/nf_core/synced_repo.py @@ -9,7 +9,7 @@ import git from git.exc import GitCommandError -from nf_core.components.components_utils import ( +from nf_core.components.constants import ( NF_CORE_MODULES_NAME, NF_CORE_MODULES_REMOTE, ) diff --git a/tests/modules/test_modules_json.py b/tests/modules/test_modules_json.py index 325a8073b..029eb32cc 100644 --- a/tests/modules/test_modules_json.py +++ b/tests/modules/test_modules_json.py @@ -3,7 +3,7 @@ import shutil from pathlib import Path -from nf_core.components.components_utils import ( +from nf_core.components.constants import ( NF_CORE_MODULES_DEFAULT_BRANCH, NF_CORE_MODULES_NAME, NF_CORE_MODULES_REMOTE, diff --git a/tests/modules/test_update.py b/tests/modules/test_update.py index 6c8eacc66..807f67cb8 100644 --- a/tests/modules/test_update.py +++ b/tests/modules/test_update.py @@ -8,7 +8,7 @@ import yaml import nf_core.utils -from nf_core.components.components_utils import NF_CORE_MODULES_NAME, NF_CORE_MODULES_REMOTE +from nf_core.components.constants import NF_CORE_MODULES_NAME, NF_CORE_MODULES_REMOTE from nf_core.modules.install import ModuleInstall from nf_core.modules.modules_json import ModulesJson from nf_core.modules.patch import ModulePatch diff --git a/tests/subworkflows/test_update.py b/tests/subworkflows/test_update.py index 63cc7c0ef..b540d3555 100644 --- a/tests/subworkflows/test_update.py +++ b/tests/subworkflows/test_update.py @@ -8,7 +8,7 @@ import yaml import nf_core.utils -from nf_core.components.components_utils import NF_CORE_MODULES_NAME, NF_CORE_MODULES_REMOTE +from nf_core.components.constants import NF_CORE_MODULES_NAME, NF_CORE_MODULES_REMOTE from nf_core.modules.modules_json import ModulesJson from nf_core.modules.update import ModuleUpdate from nf_core.subworkflows.install import SubworkflowInstall