fixes of the day

src/scicat_communication.py

@@ -43,7 +43,7 @@ def create_scicat_dataset(
     """
     logger.info("Sending POST request to create new dataset")
     response = _post_to_scicat(
-        url=config.urls["datasets"],
+        url=config.urls.datasets,
         posting_obj=dataset,
         headers=config.headers,
         timeout=config.timeout,
@@ -75,7 +75,7 @@ def create_scicat_origdatablock(
     """
     logger.info("Sending POST request to create new origdatablock")
     response = _post_to_scicat(
-        url=config.urls["origdatablocks"],
+        url=config.urls.origdatablocks,
         posting_obj=origdatablock,
         headers=config.headers,
         timeout=config.timeout,
@@ -103,8 +103,8 @@ def render_full_url(
     config: SciCatOptions,
 ) -> str:
     if not url.startswith("http://") and not url.startswith("https://"):
-        for endpoint in config.urls.keys():
+        for endpoint in config.urls.__dict__.keys():
             if url.startswith(endpoint):
-                url = url.replace(endpoint, config.urls[endpoint])
+                url = url.replace(endpoint, config.urls.__getattribute__(endpoint))
                 break
     return url

src/scicat_configuration.py

@@ -204,6 +204,7 @@ class FileHandlingOptions:
     ingestor_files_directory: str = "../ingestor"
     message_to_file: bool = True
     message_file_extension: str = "message.json"
+    use_full_file_path: bool = False
 
 
 @dataclass(kw_only=True)

src/scicat_dataset.py

@@ -95,6 +95,7 @@ def convert_to_type(input_value: Any, dtype_desc: str) -> Any:
         ),
         "evaluate": lambda value: eval(value),
         "filename": lambda value: os.path.basename(value),
+        "dirname": lambda value: os.path.dirname(value),
         "dirname-2": lambda value: os.path.dirname(os.path.dirname(value)),
     }
 )
@@ -129,6 +130,7 @@ def extract_variables_values(
     config: OfflineIngestorConfig,
 ) -> dict:
     variable_map = {
+        "ingestor_run_id": str(uuid.uuid4()),
         "filepath": pathlib.Path(config.nexus_file),
         "now": datetime.datetime.now(tz=datetime.UTC).isoformat(),
     }
@@ -166,7 +168,7 @@ def extract_paths_from_h5_file(
     _path: list[str],
 ) -> list[str]:
     master_key = _path.pop(0)
-    output_paths = [master_key]
+    output_paths = []
     if "*" in master_key:
         temp_keys = [k2 for k2 in _h5_object.keys() if re.search(master_key, k2)]
         for key in temp_keys:
@@ -217,6 +219,8 @@ class ScicatDataset:
     proposalId: str | None = None
     ownerGroup: str | None = None
     accessGroups: list[str] | None = None
+    startTime: str | None = None
+    endTime: str | None = None
 
 
 @dataclass(kw_only=True)

src/scicat_metadata.py

@@ -110,6 +110,7 @@ class MetadataSchema:
     name: str
     instrument: str
     selector: str | dict
+    order: int
     variables: dict[str, MetadataSchemaVariable]
     schema: dict[str, MetadataItem]
 
@@ -158,11 +159,11 @@ def collect_schemas(dir_path: pathlib.Path) -> OrderedDict[str, MetadataSchema]:
             MetadataSchema.from_file(schema_file_path)
             for schema_file_path in list_schema_file_names(dir_path)
         ],
-        key=lambda schema: schema.name,
+        key=lambda schema: (schema.order, schema.name),
     )
     schemas = OrderedDict()
     for metadata_schema in metadata_schemas:
-        schemas[metadata_schema.name] = metadata_schema
+        schemas[metadata_schema.id] = metadata_schema
     return schemas
 
 

src/scicat_offline_ingestor.py

@@ -82,7 +82,7 @@ def main() -> None:
             nexus_file=nexus_file_path,
             ingestor_directory=ingestor_directory,
             config=fh_options,
-            source_folder=variable_map["source_folder"],
+            source_folder=variable_map["source_folder"] if config.ingestion.file_handling.use_full_file_path==False else "",
             logger=logger,
             # TODO: add done_writing_message_file and nexus_structure_file
         )