Restructure df columns (#66)

* restructure dataframe from DLC and add events column. rename references to pose_estimation_model_str to model_str. * Move addition of video file column outside DLC df restructuring. Extract subset of frames just after DLC df reformatting. Add options for reading a multianimal DLC dataframe. * updated mypy in precommits * fix some mypy errors but still ignoring some types * added comments * add ROI to dataframe * add shapely to pyproject * rename roi_names for ROI_tags * reorder fields in sample metadata file with ROIs * insert video_file column after model_str * put ROI assignment in a separate function. add option for defining a custom hierarchy of ROIs. * add custom ROI hierarchy option in project config * add option for buffer around the ROIs boundaries (not mandatory) * minor additions to docstring * add cases for when ROIs or Events are not defined for a video. * move initialisation of ROI column outside aux function. added comment to why initialising column for all video dataframes * added shapely to mypy override * add pose data for jwaspE_nectar-open-close_control.avi * adding co-authorship for svg_path_to_polygon auxiliary function Co-authored-by: Niko Sirmpilatze [email protected] * clarify hierarchy of ROIs definition. spell out slc. replace dot access in dataframe by square brackets access. remove metadata as input. * small changes from review feedback * revert mypy version
SainsburyWellcomeCentre · Mar 29, 2023 · cf601c0 · cf601c0
1 parent 220787d
commit cf601c0
Show file tree

Hide file tree

Showing 7 changed files with 364 additions and 118 deletions.
diff --git a/pyproject.toml b/pyproject.toml
@@ -22,7 +22,8 @@ dependencies = [
   "dash",
   "dash-bootstrap-components",
   "opencv-python",
-  "PyYAML"
+  "PyYAML",
+  "shapely",
 ]
 
 classifiers = [
@@ -131,5 +132,6 @@ module = [
   "matplotlib.*",
   "plotly.*",
   "cv2.*",
+  "shapely.*",
 ]
 ignore_missing_imports = true
diff --git a/sample_project/input_config.yaml b/sample_project/input_config.yaml
@@ -2,9 +2,9 @@ videos_dir_path: ./sample_project/videos
 metadata_fields_file_path: ./sample_project/metadata_fields.yaml
 metadata_key_field_str: File
 pose_estimation_results_path: ./sample_project/pose_estimation_results
-pose_estimation_model_str: DLC_resnet50_jwasp_femaleandmaleSep12shuffle1_1000000
+model_str: DLC_resnet50_jwasp_femaleandmaleSep12shuffle1_1000000
 dashboard_export_data_path: ./sample_project/output
-roi_names:
+ROI_tags:
   - enclosure
   - left_box
   - top_box
@@ -16,3 +16,5 @@ event_tags:
   - box_1_open
   - box_1_closed
   - experiment_end
+use_ROIs_order_as_hierarchy: False
+buffer_around_ROIs_boundaries: 1e-09
diff --git a/.../jwaspE_nectar-open-close_controlDLC_resnet50_jwasp_femaleandmaleSep12shuffle1_1000000.h5 b/.../jwaspE_nectar-open-close_controlDLC_resnet50_jwasp_femaleandmaleSep12shuffle1_1000000.h5
diff --git a/sample_project/videos/jwaspE_nectar-open-close_control.metadata.yaml b/sample_project/videos/jwaspE_nectar-open-close_control.metadata.yaml
@@ -1,10 +1,21 @@
+File: jwaspE_nectar-open-close_control.avi
+Species_name: Ampulex_compressa
 Common_name: jewel_wasp
-Date_end: 5/8/22
+Subject: E_male
+Treatment: nectar-open-close_control
+Treatment_description: Reference 'nectar-open-close.pdf' on ceph (in \zoo\raw\LondonZoo\Experiment-protocols\Jewel-wasp_Ampulex-compressa\)
+  or Google Drive (most up-to-date version if they are different - in Experiment Protocols\jewel-wasp_Ampulex-compressa\)
 Date_start: 5/8/22
-File: jwaspE_nectar-open-close_control.avi
-Further_description: '-'
+Time_start: '11:47:53'
+Date_end: 5/8/22
+Time_end: '12:19:29'
+Time_recorded: 00:31:36
+Video_length: 00:31:36
 Hardware_description: Reference 'nectar-open-close.pdf' on ceph (in \zoo\raw\LondonZoo\Experiment-protocols\Jewel-wasp_Ampulex-compressa\)
   or Google Drive (most up-to-date version if they are different - in Experiment Protocols\jewel-wasp_Ampulex-compressa\)
+Software_description: Reference 'nectar-open-close.pdf' on ceph (in \zoo\raw\LondonZoo\Experiment-protocols\Jewel-wasp_Ampulex-compressa\)
+  or Google Drive (most up-to-date version if they are different - in Experiment Protocols\jewel-wasp_Ampulex-compressa\)
+Further_description: '-'
 ROIs:
 - drawn_on_frame: 57000
   line_color: '#2E91E5'
@@ -30,17 +41,6 @@ ROIs:
   line_color: '#222A2A'
   name: tube
   path: M722.1672180890754,559.5676484911008L718.4197091286674,572.4757349102841L718.4197091286674,584.5510415604879L720.0852686666265,586.216601098447L731.7441854323404,588.2985505208959L741.3211527756054,592.0460594813039L755.0620189637682,594.1280089037529L768.3864952674413,597.4591279796712L778.7962423796859,600.3738571710996L785.4584805315224,602.8721964780383L799.1993467196852,605.7869256694668L814.1893825613174,610.7836042833442L816.6877218682562,609.1180447453851L820.0188409441744,602.0394167090587L823.7663499045824,596.2099583262018L825.0155195580518,589.1313302898755L825.4319094425416,586.216601098447Z
-Software_description: Reference 'nectar-open-close.pdf' on ceph (in \zoo\raw\LondonZoo\Experiment-protocols\Jewel-wasp_Ampulex-compressa\)
-  or Google Drive (most up-to-date version if they are different - in Experiment Protocols\jewel-wasp_Ampulex-compressa\)
-Species_name: Ampulex_compressa
-Subject: E_male
-Time_end: '12:19:29'
-Time_recorded: 00:31:36
-Time_start: '11:47:53'
-Treatment: nectar-open-close_control
-Treatment_description: Reference 'nectar-open-close.pdf' on ceph (in \zoo\raw\LondonZoo\Experiment-protocols\Jewel-wasp_Ampulex-compressa\)
-  or Google Drive (most up-to-date version if they are different - in Experiment Protocols\jewel-wasp_Ampulex-compressa\)
-Video_length: 00:31:36
 Events:
   experiment_start: 0
   box_1_open: 25123

diff --git a/wazp/callbacks/dashboard.py b/wazp/callbacks/dashboard.py
@@ -1,5 +1,4 @@
 import datetime
-import os
 import pathlib as pl
 import re
 
@@ -468,8 +467,6 @@ def modify_rows_selection(
             color of the export message
         """
 
-        # TODO: select all rows per page?
-        # TODO: split into smaller functions
         list_missing_pose_data_bool = [
             videos_table_data[r][POSE_DATA_STR] == FALSE_EMOJI
             for r in range(len(videos_table_data))
@@ -508,7 +505,7 @@ def modify_rows_selection(
         # ---------------------------
         # If export button is clicked
         if n_clicks_export > 0:
-            # if no rows selected show warning
+            # if no rows are selected show warning
             if not list_selected_rows:
                 n_clicks_export = 0
 
@@ -538,7 +535,7 @@ def modify_rows_selection(
 
                 # get list of dataframes to combine
                 # - one dataframe per selected video
-                # - we add 'File', 'event_tag' and 'ROI' data to each dataframe
+                # - we add the fields 'video_file', 'event_tag' and 'ROI'
                 # - we select only the frames within the interval
                 #   set by the slider
                 list_df_to_export = utils.get_dataframes_to_combine(
@@ -548,24 +545,30 @@ def modify_rows_selection(
                 )
 
                 # concatenate all dataframes
+                # NOTE: we explicitly initialise ROI_tags
+                # and event_tags columns for all video dataframes
+                # with empty strings (then ROIs and events are only assigned
+                # if defined for a video)
                 df = pd.concat(list_df_to_export)
 
                 # ---------
-                # Save df as h5
-                # get output path
+                # Export dataframe as h5
+                # TODO: provide an option to export as h5 or csv?
+
+                # Get path to output directory
                 # if not specified in config, use dir where
-                # 'start_wazp_server.sh' is at
+                # server was launched from
                 output_path = pl.Path(
                     app_storage["config"].get(
                         "dashboard_export_data_path", "."
                     )
                 )
 
-                # if output dir does not exist, create it
-                if not output_path.is_dir():
-                    os.mkdir(output_path)
+                # If output directory does not exist,
+                # create it
+                output_path.mkdir(parents=True, exist_ok=True)
 
-                # save combined df as h5 file
+                # Save combined dataframe as h5 file
                 h5_file_path = output_path / pl.Path(
                     "df_export_"
                     + datetime.datetime.now().strftime("%Y%m%d_%H%M%S")
@@ -578,7 +581,7 @@ def modify_rows_selection(
                 )
 
                 # ---------
-                # reset triggers and states
+                # Reset triggers and states
                 list_selected_rows = []
                 n_clicks_export = 0
                 export_message_children[0] = "".join(

diff --git a/wazp/callbacks/roi.py b/wazp/callbacks/roi.py
@@ -109,7 +109,7 @@ def update_roi_select_options(
         if "config" in app_storage.keys():
             # Get ROI names from stored config
             config = app_storage["config"]
-            roi_names = config["roi_names"]
+            roi_names = config["ROI_tags"]
             options = [{"label": r, "value": r} for r in roi_names]
             value = roi_names[0]