Conversion test

.gitignore

@@ -134,3 +134,5 @@ dmypy.json
 #misc
 endpoint_schemas/
 tests/
+src/ibl_to_nwb/local/
+.vscode

src/ibl_to_nwb/_scripts/_convert_brainwide_map.py

@@ -0,0 +1,210 @@
+import os
+import shutil
+import sys
+from datetime import datetime
+from pathlib import Path
+
+import spikeglx
+
+# if running on SDSC, use the OneSdsc, else normal
+if "USE_SDSC_ONE" in os.environ:
+    print("using SDSC ONE")
+    from deploy.iblsdsc import OneSdsc as ONE
+else:
+    print("using regular ONE")
+    from one.api import ONE
+
+from ibl_to_nwb.converters import BrainwideMapConverter, IblSpikeGlxConverter
+from ibl_to_nwb.datainterfaces import (
+    BrainwideMapTrialsInterface,
+    IblPoseEstimationInterface,
+    IblSortingInterface,
+    LickInterface,
+    PupilTrackingInterface,
+    RawVideoInterface,
+    RoiMotionEnergyInterface,
+    WheelInterface,
+)
+
+
+def create_symlinks(source_dir, target_dir, remove_uuid=True):
+    """replicates the tree under source_dir at target dir in the form of symlinks"""
+    for root, dirs, files in os.walk(source_dir):
+        for dir in dirs:
+            folder = target_dir / (Path(root) / dir).relative_to(source_dir)
+            folder.mkdir(parents=True, exist_ok=True)
+
+    for root, dirs, files in os.walk(source_dir):
+        for file in files:
+            source_file_path = Path(root) / file
+            target_file_path = target_dir / source_file_path.relative_to(source_dir)
+            if remove_uuid:
+                parent, name = target_file_path.parent, target_file_path.name
+                name_parts = name.split(".")
+                name_parts.remove(name_parts[-2])
+                target_file_path = parent / ".".join(name_parts)
+            if not target_file_path.exists():
+                target_file_path.symlink_to(source_file_path)
+
+
+def get_last_before(eid: str, one: ONE, revision: str):
+    revisions = one.list_revisions(eid, revision="*")
+    revisions = [datetime.strptime(revision, "%Y-%m-%d") for revision in revisions]
+    revision = datetime.strptime(revision, "%Y-%m-%d")
+    revisions = sorted(revisions)
+    ix = sum([not (rev > revision) for rev in revisions]) - 1
+    return revisions[ix].strftime("%Y-%m-%d")
+
+
+def convert(eid: str, one: ONE, data_interfaces: list, revision: str, mode: str):
+    # Run conversion
+    session_converter = BrainwideMapConverter(one=one, session=eid, data_interfaces=data_interfaces, verbose=True)
+    metadata = session_converter.get_metadata()
+    metadata["NWBFile"]["session_id"] = f"{eid}:{revision}"  # FIXME this hack has to go
+    subject_id = metadata["Subject"]["subject_id"]
+
+    subject_folder_path = output_folder / f"sub-{subject_id}"
+    subject_folder_path.mkdir(exist_ok=True)
+    if mode == "raw":
+        fname = f"sub-{subject_id}_ses-{eid}_desc-raw_ecephys+image.nwb"
+    if mode == "processed":
+        fname = f"sub-{subject_id}_ses-{eid}_desc-processed_behavior+ecephys.nwb"
+
+    nwbfile_path = subject_folder_path / fname
+    session_converter.run_conversion(
+        nwbfile_path=nwbfile_path,
+        metadata=metadata,
+        overwrite=True,
+    )
+    return nwbfile_path
+
+
+cleanup = False
+
+if __name__ == "__main__":
+    if len(sys.argv) == 1:
+        eid = "caa5dddc-9290-4e27-9f5e-575ba3598614"
+        mode = "raw"
+    else:
+        eid = sys.argv[1]
+        mode = sys.argv[2]  # raw or processed
+
+    print(eid)
+    print(mode)
+
+    # path setup
+    base_path = Path.home() / "ibl_scratch"
+    output_folder = base_path / "nwbfiles"
+    output_folder.mkdir(exist_ok=True, parents=True)
+    local_scratch_folder = base_path / eid
+
+    # common
+    one_kwargs = dict(
+        base_url="https://openalyx.internationalbrainlab.org",
+        password="international",
+        mode="remote",
+    )
+
+    # if not running on SDSC adding the cache folder explicitly
+    if "USE_SDSC_ONE" in os.environ:
+        one_kwargs["cache_rest"] = None  # disables rest caching (write permission errors on popeye)
+    else:
+        # Initialize IBL (ONE) client to download processed data for this session
+        one_cache_folder_path = base_path / "ibl_conversion" / eid / "cache"
+        one_kwargs["cache_dir"] = one_cache_folder_path
+
+    # instantiate one
+    one = ONE(**one_kwargs)
+
+    # correct revision
+    revision = get_last_before(eid=eid, one=one, revision="2024-07-10")
+
+    # Initialize as many of each interface as we need across the streams
+    data_interfaces = []
+
+    if mode == "raw":
+        # ephys
+        session_folder = one.eid2path(eid)
+        spikeglx_source_folder_path = session_folder / "raw_ephys_data"
+
+        # create symlinks at local scratch
+        create_symlinks(spikeglx_source_folder_path, local_scratch_folder)
+
+        # check and decompress
+        cbin_paths = []
+        for root, dirs, files in os.walk(local_scratch_folder):
+            for file in files:
+                if file.endswith(".cbin"):
+                    cbin_paths.append(Path(root) / file)
+
+        for path in cbin_paths:
+            if not path.with_suffix(".bin").exists():
+                print(f"decompressing {path}")
+                spikeglx.Reader(path).decompress_to_scratch()
+
+        # Specify the path to the SpikeGLX files on the server but use ONE API for timestamps
+        spikeglx_subconverter = IblSpikeGlxConverter(folder_path=spikeglx_source_folder_path, one=one, eid=eid)
+        data_interfaces.append(spikeglx_subconverter)
+
+        # video
+        metadata_retrieval = BrainwideMapConverter(one=one, session=eid, data_interfaces=[], verbose=False)
+        subject_id = metadata_retrieval.get_metadata()["Subject"]["subject_id"]
+
+        pose_estimation_files = one.list_datasets(eid=eid, filename="*.dlc*")
+        for pose_estimation_file in pose_estimation_files:
+            camera_name = pose_estimation_file.replace("alf/_ibl_", "").replace(".dlc.pqt", "")
+
+            video_interface = RawVideoInterface(
+                nwbfiles_folder_path=output_folder,
+                subject_id=subject_id,
+                one=one,
+                session=eid,
+                camera_name=camera_name,
+            )
+            data_interfaces.append(video_interface)
+
+    if mode == "processed":
+        # These interfaces should always be present in source data
+        data_interfaces.append(IblSortingInterface(one=one, session=eid, revision=revision))
+        data_interfaces.append(BrainwideMapTrialsInterface(one=one, session=eid, revision=revision))
+        data_interfaces.append(WheelInterface(one=one, session=eid, revision=revision))
+
+        # # These interfaces may not be present; check if they are before adding to list
+        pose_estimation_files = one.list_datasets(eid=eid, filename="*.dlc*")
+        for pose_estimation_file in pose_estimation_files:
+            camera_name = pose_estimation_file.replace("alf/_ibl_", "").replace(".dlc.pqt", "")
+            data_interfaces.append(
+                IblPoseEstimationInterface(one=one, session=eid, camera_name=camera_name, revision=revision)
+            )
+
+        pupil_tracking_files = one.list_datasets(eid=eid, filename="*features*")
+        for pupil_tracking_file in pupil_tracking_files:
+            camera_name = pupil_tracking_file.replace("alf/_ibl_", "").replace(".features.pqt", "")
+            data_interfaces.append(
+                PupilTrackingInterface(one=one, session=eid, camera_name=camera_name, revision=revision)
+            )
+
+        roi_motion_energy_files = one.list_datasets(eid=eid, filename="*ROIMotionEnergy.npy*")
+        for roi_motion_energy_file in roi_motion_energy_files:
+            camera_name = roi_motion_energy_file.replace("alf/", "").replace(".ROIMotionEnergy.npy", "")
+            data_interfaces.append(
+                RoiMotionEnergyInterface(one=one, session=eid, camera_name=camera_name, revision=revision)
+            )
+
+        if one.list_datasets(eid=eid, collection="alf", filename="licks*"):
+            data_interfaces.append(LickInterface(one=one, session=eid, revision=revision))
+
+    # run the conversion
+    nwbfile_path = convert(
+        eid=eid,
+        one=one,
+        data_interfaces=data_interfaces,
+        revision=revision,
+        mode=mode,
+    )
+
+    # cleanup
+    if cleanup:
+        if mode == "raw":
+            os.system(f"find {local_scratch_folder} -type l -exec unlink {{}} \;")
+            shutil.rmtree(local_scratch_folder)

src/ibl_to_nwb/_scripts/_convert_brainwide_map_processed.py

@@ -0,0 +1,106 @@
+import os
+import sys
+from datetime import datetime
+from pathlib import Path
+
+if "USE_SDSC_ONE" in os.envion:
+    from deploy.iblsdsc import OneSdsc as ONE
+else:
+    from one.api import ONE
+
+from ibl_to_nwb.converters import BrainwideMapConverter
+from ibl_to_nwb.datainterfaces import (
+    BrainwideMapTrialsInterface,
+    IblPoseEstimationInterface,
+    IblSortingInterface,
+    LickInterface,
+    PupilTrackingInterface,
+    RoiMotionEnergyInterface,
+    WheelInterface,
+)
+
+
+def get_last_before(eid: str, one: ONE, revision: str):
+    revisions = one.list_revisions(eid, revision="*")
+    revisions = [datetime.strptime(revision, "%Y-%m-%d") for revision in revisions]
+    revision = datetime.strptime(revision, "%Y-%m-%d")
+    revisions = sorted(revisions)
+    ix = sum([not (rev > revision) for rev in revisions]) - 1
+    return revisions[ix].strftime("%Y-%m-%d")
+
+
+def convert(eid: str, one: ONE, data_interfaces: list, revision: str):
+    # Run conversion
+    session_converter = BrainwideMapConverter(one=one, session=eid, data_interfaces=data_interfaces, verbose=True)
+    metadata = session_converter.get_metadata()
+    metadata["NWBFile"]["session_id"] = f"{eid}:{revision}"  # FIXME this hack has to go
+    subject_id = metadata["Subject"]["subject_id"]
+
+    subject_folder_path = output_folder / f"sub-{subject_id}"
+    subject_folder_path.mkdir(exist_ok=True)
+    fname = f"sub-{subject_id}_ses-{eid}_desc-processed.nwb"
+
+    nwbfile_path = subject_folder_path / fname
+    session_converter.run_conversion(
+        nwbfile_path=nwbfile_path,
+        metadata=metadata,
+        overwrite=True,
+    )
+    return nwbfile_path
+
+
+if __name__ == "__main__":
+    if len(sys.argv) == 1:
+        eid = "caa5dddc-9290-4e27-9f5e-575ba3598614"
+    else:
+        eid = sys.argv[1]
+
+    # path setup
+    base_path = Path.home() / "ibl_scratch"
+    output_folder = base_path / "nwbfiles"
+    output_folder.mkdir(exist_ok=True, parents=True)
+
+    # Initialize IBL (ONE) client to download processed data for this session
+    one_cache_folder_path = base_path / "ibl_conversion" / eid / "cache"
+    one = ONE(
+        base_url="https://openalyx.internationalbrainlab.org",
+        password="international",
+        mode="remote",
+        # silent=True,
+        cache_dir=one_cache_folder_path,
+    )
+
+    revision = get_last_before(eid=eid, one=one, revision="2024-07-10")
+
+    # Initialize as many of each interface as we need across the streams
+    data_interfaces = list()
+
+    # These interfaces should always be present in source data
+    data_interfaces.append(IblSortingInterface(one=one, session=eid, revision=revision))
+    data_interfaces.append(BrainwideMapTrialsInterface(one=one, session=eid, revision=revision))
+    data_interfaces.append(WheelInterface(one=one, session=eid, revision=revision))
+
+    # # These interfaces may not be present; check if they are before adding to list
+    pose_estimation_files = one.list_datasets(eid=eid, filename="*.dlc*")
+    for pose_estimation_file in pose_estimation_files:
+        camera_name = pose_estimation_file.replace("alf/_ibl_", "").replace(".dlc.pqt", "")
+        data_interfaces.append(
+            IblPoseEstimationInterface(one=one, session=eid, camera_name=camera_name, revision=revision)
+        )
+
+    pupil_tracking_files = one.list_datasets(eid=eid, filename="*features*")
+    for pupil_tracking_file in pupil_tracking_files:
+        camera_name = pupil_tracking_file.replace("alf/_ibl_", "").replace(".features.pqt", "")
+        data_interfaces.append(PupilTrackingInterface(one=one, session=eid, camera_name=camera_name, revision=revision))
+
+    roi_motion_energy_files = one.list_datasets(eid=eid, filename="*ROIMotionEnergy.npy*")
+    for roi_motion_energy_file in roi_motion_energy_files:
+        camera_name = roi_motion_energy_file.replace("alf/", "").replace(".ROIMotionEnergy.npy", "")
+        data_interfaces.append(
+            RoiMotionEnergyInterface(one=one, session=eid, camera_name=camera_name, revision=revision)
+        )
+
+    if one.list_datasets(eid=eid, collection="alf", filename="licks*"):
+        data_interfaces.append(LickInterface(one=one, session=eid, revision=revision))
+
+    nwbfile_path = convert(eid=eid, one=one, data_interfaces=data_interfaces, revision=revision)