Profile Retrieval in Metecho (#3711)

[W-14007410](https://gus.lightning.force.com/lightning/r/ADM_Work__c/a07EE00001ZCxhyYAD/view)

Included functionality to retrieve complete profile in Metecho (this is
linked to
[W-8932343](https://gus.lightning.force.com/lightning/r/ADM_Work__c/a07B00000097PhOIAU/view))

---------

Co-authored-by: Naman Jain <[email protected]>
Co-authored-by: Jaipal Reddy Kasturi <[email protected]>

cumulusci/salesforce_api/retrieve_profile_api.py

@@ -87,7 +87,9 @@ def __init__(
 class RetrieveProfileApi(BaseSalesforceApiTask):
     def _init_task(self):
         super(RetrieveProfileApi, self)._init_task()
-        self.api_version = self.org_config.latest_api_version
+        self.api_version = self.project_config.config["project"]["package"][
+            "api_version"
+        ]
 
     def _retrieve_existing_profiles(self, profiles: List[str]):
         query = self._build_query(["Name"], "Profile", {"Name": profiles})
@@ -97,6 +99,10 @@ def _retrieve_existing_profiles(self, profiles: List[str]):
         for data in result["records"]:
             existing_profiles.append(data["Name"])
 
+        # Since System Administrator is named Admin in Metadata API
+        if "Admin" in profiles:
+            existing_profiles.extend(["Admin", "System Administrator"])
+
         return existing_profiles
 
     def _run_query(self, query):

cumulusci/salesforce_api/tests/test_retrieve_profile_api.py

@@ -18,7 +18,7 @@ def retrieve_profile_api_instance():
     project_config = MagicMock()
     task_config = MagicMock()
     org_config = MagicMock()
-    org_config.latest_api_version = "58.0"
+    project_config.config = {"project": {"package": {"api_version": "58.0"}}}
     sf_mock.query.return_value = {"records": []}
     api = RetrieveProfileApi(
         project_config=project_config, org_config=org_config, task_config=task_config
@@ -36,7 +36,7 @@ def test_init_task(retrieve_profile_api_instance):
 
 
 def test_retrieve_existing_profiles(retrieve_profile_api_instance):
-    profiles = ["Profile1", "Profile2"]
+    profiles = ["Profile1", "Profile2", "Admin"]
     result = {"records": [{"Name": "Profile1"}]}
     with patch.object(
         RetrieveProfileApi, "_build_query", return_value="some_query"
@@ -47,6 +47,8 @@ def test_retrieve_existing_profiles(retrieve_profile_api_instance):
 
     assert "Profile1" in existing_profiles
     assert "Profile2" not in existing_profiles
+    assert "Admin" in existing_profiles
+    assert "System Administrator" in existing_profiles
 
 
 def test_run_query_sf(retrieve_profile_api_instance):

cumulusci/tasks/salesforce/retrieve_profile.py

@@ -1,4 +1,4 @@
-import os
+from pathlib import Path
 
 from cumulusci.core.utils import process_bool_arg, process_list_arg
 from cumulusci.salesforce_api.metadata import ApiRetrieveUnpackaged
@@ -28,21 +28,29 @@ class RetrieveProfile(BaseSalesforceMetadataApiTask):
 
     def _init_options(self, kwargs):
         super(RetrieveProfile, self)._init_options(kwargs)
-        self.api_version = self.org_config.latest_api_version
+        self.api_version = self.project_config.config["project"]["package"][
+            "api_version"
+        ]
         self.profiles = process_list_arg(self.options["profiles"])
         if not self.profiles:
             raise ValueError("At least one profile must be specified.")
 
-        self.extract_dir = self.options.get("path", "force-app/default/main")
+        self.extract_dir = self.options.get("path", "force-app")
+        extract_path = Path(self.extract_dir)
 
-        if not os.path.exists(self.extract_dir):
+        if not extract_path.exists():
             raise FileNotFoundError(
                 f"The extract directory '{self.extract_dir}' does not exist."
             )
-
-        if not os.path.isdir(self.extract_dir):
+        if not extract_path.is_dir():
             raise NotADirectoryError(f"'{self.extract_dir}' is not a directory.")
 
+        # If extract_dir is force-app and main/default is not present
+        if self.extract_dir == "force-app":
+            if not (extract_path / "main" / "default").exists():
+                (extract_path / "main" / "default").mkdir(parents=True, exist_ok=True)
+            self.extract_dir = "force-app/main/default"
+
         self.strictMode = process_bool_arg(self.options.get("strict_mode", True))
 
     def _check_existing_profiles(self, retrieve_profile_api_task):
@@ -90,10 +98,25 @@ def add_flow_accesses(self, profile_content, flows):
         return profile_content
 
     def save_profile_file(self, extract_dir, filename, content):
-        profile_path = os.path.join(extract_dir, filename)
-        os.makedirs(os.path.dirname(profile_path), exist_ok=True)
-        with open(profile_path, "w", encoding="utf-8") as updated_profile_file:
-            updated_profile_file.write(content)
+        profile_path = Path(extract_dir) / filename
+        profile_meta_xml_path = Path(extract_dir) / f"{filename}-meta.xml"
+
+        # Check if either the profile file or metadata file exists
+        if profile_path.exists():
+            self.update_file_content(profile_path, content)
+        elif profile_meta_xml_path.exists():
+            self.update_file_content(profile_meta_xml_path, content)
+        else:
+            # Neither file exists, create the profile file
+            profile_meta_xml_path.parent.mkdir(parents=True, exist_ok=True)
+            with profile_meta_xml_path.open(
+                mode="w", encoding="utf-8"
+            ) as updated_profile_file:
+                updated_profile_file.write(content)
+
+    def update_file_content(self, file_path, content):
+        with open(file_path, "w", encoding="utf-8") as updated_file:
+            updated_file.write(content)
 
     def _run_task(self):
         self.retrieve_profile_api_task = RetrieveProfileApi(
@@ -126,9 +149,7 @@ def _run_task(self):
             ) and file_info.filename.endswith(".profile"):
                 with zip_result.open(file_info) as profile_file:
                     profile_content = profile_file.read().decode("utf-8")
-                    profile_name = os.path.splitext(
-                        os.path.basename(file_info.filename)
-                    )[0]
+                    profile_name = profile_name = Path(file_info.filename).stem
 
                     if profile_name in profile_flows:
                         profile_content = self.add_flow_accesses(
@@ -140,7 +161,9 @@ def _run_task(self):
                     )
 
         # zip_result.extractall('./unpackaged')
-
+        self.existing_profiles.remove(
+            "Admin"
+        ) if "Admin" in self.existing_profiles else None
         self.logger.info(
             f"Profiles {', '.join(self.existing_profiles)} unzipped into folder '{self.extract_dir}'"
         )

cumulusci/tasks/salesforce/sourcetracking.py

@@ -6,11 +6,13 @@
 import time
 from collections import defaultdict
 
-from cumulusci.core.config import ScratchOrgConfig
+from cumulusci.core.config import BaseProjectConfig, ScratchOrgConfig, TaskConfig
+from cumulusci.core.exceptions import ProjectConfigNotFound
 from cumulusci.core.sfdx import sfdx
 from cumulusci.core.utils import process_bool_arg, process_list_arg
 from cumulusci.tasks.metadata.package import PackageXmlGenerator
 from cumulusci.tasks.salesforce import BaseRetrieveMetadata, BaseSalesforceApiTask
+from cumulusci.tasks.salesforce.retrieve_profile import RetrieveProfile
 from cumulusci.utils import (
     inject_namespace,
     process_text_in_directory,
@@ -167,6 +169,12 @@ def _reset_sfdx_snapshot(self):
         + " Defaults to project__package__api_version"
     )
 }
+retrieve_changes_task_options["retrieve_complete_profile"] = {
+    "description": (
+        "If set to True, will use RetrieveProfile to retrieve"
+        + " the complete profile. Default is set to False"
+    )
+}
 retrieve_changes_task_options["namespace_tokenize"] = BaseRetrieveMetadata.task_options[
     "namespace_tokenize"
 ]
@@ -194,6 +202,19 @@ def _write_manifest(changes, path, api_version):
         f.write(package_xml)
 
 
+def separate_profiles(components):
+    """Separate the profiles from components"""
+    updated_components = []
+    profiles = []
+    for comp in components:
+        if comp["MemberType"] == "Profile":
+            profiles.append(comp["MemberName"])
+        else:
+            updated_components.append(comp)
+
+    return updated_components, profiles
+
+
 def retrieve_components(
     components,
     org_config,
@@ -202,6 +223,8 @@ def retrieve_components(
     extra_package_xml_opts: dict,
     namespace_tokenize: str,
     api_version: str,
+    project_config: BaseProjectConfig = None,
+    retrieve_complete_profile: bool = False,
 ):
     """Retrieve specified components from an org into a target folder.
 
@@ -215,6 +238,15 @@ def retrieve_components(
     """
 
     target = os.path.realpath(target)
+    profiles = []
+
+    # If retrieve_complete_profile and project_config is None, raise error
+    # This is because project_config is only required if retrieve_complete_profile is True
+    if retrieve_complete_profile and project_config is None:
+        raise ProjectConfigNotFound(
+            "Kindly provide project_config as part of retrieve_components"
+        )
+
     with contextlib.ExitStack() as stack:
         if md_format:
             # Create target if it doesn't exist
@@ -247,27 +279,47 @@ def retrieve_components(
                 check_return=True,
             )
 
-        # Construct package.xml with components to retrieve, in its own tempdir
-        package_xml_path = stack.enter_context(temporary_dir(chdir=False))
-        _write_manifest(components, package_xml_path, api_version)
-
-        # Retrieve specified components in DX format
-        sfdx(
-            "force:source:retrieve",
-            access_token=org_config.access_token,
-            log_note="Retrieving components",
-            args=[
-                "-a",
-                str(api_version),
-                "-x",
-                os.path.join(package_xml_path, "package.xml"),
-                "-w",
-                "5",
-            ],
-            capture_output=False,
-            check_return=True,
-            env={"SFDX_INSTANCE_URL": org_config.instance_url},
-        )
+        # If retrieve_complete_profile is True, separate the profiles from
+        # components to retrieve complete profile
+        if retrieve_complete_profile:
+            components, profiles = separate_profiles(components)
+
+        if components:
+            # Construct package.xml with components to retrieve, in its own tempdir
+            package_xml_path = stack.enter_context(temporary_dir(chdir=False))
+            _write_manifest(components, package_xml_path, api_version)
+
+            # Retrieve specified components in DX format
+            sfdx(
+                "force:source:retrieve",
+                access_token=org_config.access_token,
+                log_note="Retrieving components",
+                args=[
+                    "-a",
+                    str(api_version),
+                    "-x",
+                    os.path.join(package_xml_path, "package.xml"),
+                    "-w",
+                    "5",
+                ],
+                capture_output=False,
+                check_return=True,
+                env={"SFDX_INSTANCE_URL": org_config.instance_url},
+            )
+
+        # Extract Profiles
+        if profiles:
+            task_config = TaskConfig(
+                config={
+                    "options": {"profiles": ",".join(profiles), "path": "force-app"}
+                }
+            )
+            cls_retrieve_profile = RetrieveProfile(
+                org_config=org_config,
+                project_config=project_config,
+                task_config=task_config,
+            )
+            cls_retrieve_profile()
 
         if md_format:
             # Convert back to metadata format
@@ -304,6 +356,9 @@ class RetrieveChanges(ListChanges, BaseSalesforceApiTask):
     def _init_options(self, kwargs):
         super(RetrieveChanges, self)._init_options(kwargs)
         self.options["snapshot"] = process_bool_arg(kwargs.get("snapshot", True))
+        self.options["retrieve_complete_profile"] = process_bool_arg(
+            self.options.get("retrieve_complete_profile", False)
+        )
 
         # Check which directories are configured as dx packages
         package_directories = []
@@ -369,6 +424,8 @@ def _run_task(self):
             namespace_tokenize=self.options.get("namespace_tokenize"),
             api_version=self.options["api_version"],
             extra_package_xml_opts=package_xml_opts,
+            project_config=self.project_config,
+            retrieve_complete_profile=self.options["retrieve_complete_profile"],
         )
 
         if self.options["snapshot"]:

cumulusci/tasks/salesforce/tests/test_retrieve_profile.py

@@ -130,11 +130,12 @@ def create_temp_zip_file():
     return zipfile.ZipFile(temp_zipfile, "r")
 
 
-def test_save_profile_file(retrieve_profile_task, tmpdir):
+def test_save_profile_file_new(retrieve_profile_task, tmpdir):
     extract_dir = str(tmpdir)
     filename = "TestProfile.profile"
+    meta_filename = "TestProfile.profile-meta.xml"
     content = "Profile content"
-    expected_file_path = os.path.join(extract_dir, filename)
+    expected_file_path = os.path.join(extract_dir, meta_filename)
     retrieve_profile_task.save_profile_file(extract_dir, filename, content)
 
     assert os.path.exists(expected_file_path)
@@ -143,6 +144,39 @@ def test_save_profile_file(retrieve_profile_task, tmpdir):
     assert saved_content == content
 
 
+def test_save_profile_file_existing_meta_xml(retrieve_profile_task, tmpdir):
+    extract_dir = str(tmpdir)
+    filename = "TestProfile.profile"
+    meta_filename = "TestProfile.profile-meta.xml"
+    content = "Profile content"
+    existing_file_path = os.path.join(extract_dir, meta_filename)
+
+    with open(existing_file_path, "w", encoding="utf-8") as existing_file:
+        existing_file.write("Existing content")
+
+    retrieve_profile_task.save_profile_file(extract_dir, filename, content)
+
+    with open(existing_file_path, "r", encoding="utf-8") as profile_file:
+        saved_content = profile_file.read()
+    assert saved_content == content
+
+
+def test_save_profile_file_existing(retrieve_profile_task, tmpdir):
+    extract_dir = str(tmpdir)
+    filename = "TestProfile.profile"
+    content = "Profile content"
+    existing_file_path = os.path.join(extract_dir, filename)
+
+    with open(existing_file_path, "w", encoding="utf-8") as existing_file:
+        existing_file.write("Existing content")
+
+    retrieve_profile_task.save_profile_file(extract_dir, filename, content)
+
+    with open(existing_file_path, "r", encoding="utf-8") as profile_file:
+        saved_content = profile_file.read()
+    assert saved_content == content
+
+
 def test_add_flow_accesses(retrieve_profile_task):
     profile_content = "<Profile>\n" "    <some_tag>Hello</some_tag>\n" "</Profile>"
     flows = ["Flow1", "Flow2"]
@@ -186,7 +220,7 @@ def test_run_task(retrieve_profile_task, tmpdir, caplog):
         retrieve_profile_task._run_task()
 
     assert os.path.exists(tmpdir)
-    profile1_path = os.path.join(tmpdir, "profiles/Profile1.profile")
+    profile1_path = os.path.join(tmpdir, "profiles/Profile1.profile-meta.xml")
     assert os.path.exists(profile1_path)
 
     log_messages = [record.message for record in caplog.records]