Merge branch 'main' of https://github.com/SciCatProject/scicat-filewr…

…iter-ingest into background-ingestor

resources/config.sample.json

@@ -17,6 +17,11 @@
     "username": "USERNAME",
     "password": "PASSWORD"
   },
+  "graylog": {
+    "host" : "",
+    "port" : "",
+    "facility" : "scicat.ingestor"
+  },
   "dataset": {
     "instrument_id" : "",
     "instrument" : "INSTRUMENT_NAME",
@@ -37,6 +42,7 @@
     "system_log_facility" : "mail",
     "log_message_prefix" : " SFI: ",
     "check_by_job_id" : true,
-    "pyscicat": null
+    "pyscicat": null,
+    "graylog" : false
   }
 }

resources/dream.configuration.json.example

@@ -0,0 +1,55 @@
+{
+ "proposal_id" : {
+  "field_type" : "high_level",
+  "machine_name" : "proposalId",
+  "source" : "NXS:/entry/experiment_identifier",
+  "type" : "string"
+ },
+ "principal_investigator" : {
+  "field_type" : "high_level",
+  "machine_name" : "principalInvestigator",
+  "source" : "UO:proposal:/proposer/firstname + UO:proposal:/proposer/lastname",
+  "type" : "string"
+ },
+ "dataset_name" : {
+  "field_type" : "high_level",
+  "machine_name" : "datasetName",
+  "source" : "NXS:/entry/title",
+  "type" : "string"
+ },
+ "techniques" : {
+  "field_type" : "high_level",
+  "machine_name" : "techniques",
+  "human_name" : "techniques",
+  "value" : "",
+  "type" : "object"
+ },
+ "start_time" : {
+   "field_type" : "scientific_metadata",
+   "machine_name" : "start_time",
+   "human_name" : "Start Time",
+   "source" : "WRDN:/metadata/start_time",
+   "transformation" : "timestamp_to_iso8601",
+   "type" : "date"
+ }
+ "end_time" : {
+   "field_type" : "scientific_metadata",
+   "machine_name" : "end_time",
+   "human_name" : "End Time",
+   "source" : "WRDN:/metadata/end_time",
+   "transformation" : "timestamp_to_iso8601",
+   "type" : "date"
+ },
+ "sample_id" : {
+  "field_type" : "high_level",
+  "machine_name" : "sampleId",
+  "value" : "",
+  "type" : "string"
+ },
+ "instrument_id" : {
+  "field_type" : "high_level",
+  "machine_name" : "instrumentId",
+  "value" : "DREAM",
+  "type" : "string"
+ },
+]

src/scicat_configuration.py

@@ -8,82 +8,89 @@
 def build_main_arg_parser() -> argparse.ArgumentParser:
     parser = argparse.ArgumentParser()
 
-    group = parser.add_argument_group('Scicat Ingestor Options')
+    group = parser.add_argument_group("Scicat Ingestor Options")
 
     group.add_argument(
-        '-c',
-        '--cf',
-        '--config',
-        '--config-file',
-        default='config.20240405.json',
-        dest='config_file',
-        help='Configuration file name. Default: config.20240405.json',
+        "-c",
+        "--cf",
+        "--config",
+        "--config-file",
+        default="config.20240405.json",
+        dest="config_file",
+        help="Configuration file name. Default: config.20240405.json",
         type=str,
     )
     group.add_argument(
-        '-v',
-        '--verbose',
-        dest='verbose',
-        help='Provide logging on stdout',
-        action='store_true',
+        "-v",
+        "--verbose",
+        dest="verbose",
+        help="Provide logging on stdout",
+        action="store_true",
         default=False,
     )
     group.add_argument(
-        '--file-log',
-        dest='file_log',
-        help='Provide logging on file',
-        action='store_true',
+        "--file-log",
+        dest="file_log",
+        help="Provide logging on file",
+        action="store_true",
         default=False,
     )
     group.add_argument(
-        '--log-filepath-prefix',
-        dest='log_filepath_prefix',
-        help='Prefix of the log file path',
-        default='.scicat_ingestor_log',
+        "--log-filepath-prefix",
+        dest="log_filepath_prefix",
+        help="Prefix of the log file path",
+        default=".scicat_ingestor_log",
     )
     group.add_argument(
-        '--file-log-timestamp',
-        dest='file_log_timestamp',
-        help='Provide logging on the system log',
-        action='store_true',
+        "--file-log-timestamp",
+        dest="file_log_timestamp",
+        help="Provide logging on the system log",
+        action="store_true",
         default=False,
     )
     group.add_argument(
-        '--system-log',
-        dest='system_log',
-        help='Provide logging on the system log',
-        action='store_true',
+        "--system-log",
+        dest="system_log",
+        help="Provide logging on the system log",
+        action="store_true",
         default=False,
     )
     group.add_argument(
-        '--system-log-facility',
-        dest='system_log_facility',
-        help='Facility for system log',
-        default='mail',
+        "--system-log-facility",
+        dest="system_log_facility",
+        help="Facility for system log",
+        default="mail",
     )
     group.add_argument(
-        '--log-message-prefix',
-        dest='log_message_prefix',
-        help='Prefix for log messages',
-        default=' SFI: ',
+        "--log-message-prefix",
+        dest="log_message_prefix",
+        help="Prefix for log messages",
+        default=" SFI: ",
     )
     group.add_argument(
-        '--log-level', dest='log_level', help='Logging level', default='INFO', type=str
+        "--log-level", dest="log_level", help="Logging level", default="INFO", type=str
     )
     group.add_argument(
-        '--check-by-job-id',
-        dest='check_by_job_id',
-        help='Check the status of a job by job_id',
-        action='store_true',
+        "--check-by-job-id",
+        dest="check_by_job_id",
+        help="Check the status of a job by job_id",
+        action="store_true",
         default=True,
     )
     group.add_argument(
-        '--pyscicat',
-        dest='pyscicat',
-        help='Location where a specific version of pyscicat is available',
+        "--pyscicat",
+        dest="pyscicat",
+        help="Location where a specific version of pyscicat is available",
         default=None,
         type=str,
     )
+    group.add_argument(
+        "--graylog",
+        dest="graylog",
+        help="Use graylog for additional logs",
+        action="store_true",
+        default=False,
+    )
     return parser
 
 
@@ -120,6 +127,13 @@ def build_background_ingestor_arg_parser() -> argparse.ArgumentParser:
     return parser
 
 
+@dataclass
+class GraylogOptions:
+    host: str = ""
+    port: str = ""
+    facility: str = "scicat.ingestor"
+
+
 @dataclass
 class RunOptions:
     """RunOptions dataclass to store the configuration options.
@@ -140,6 +154,7 @@ class RunOptions:
     check_by_job_id: bool
     system_log_facility: Optional[str] = None
     pyscicat: Optional[str] = None
+    graylog: bool = False
 
 
 @dataclass
@@ -172,6 +187,8 @@ class ScicatConfig:
     """Merged configuration dictionary with command line arguments."""
     kafka_options: kafkaOptions
     """Kafka configuration options read from files."""
+    graylog_options: GraylogOptions
+    """Graylog configuration options for streaming logs."""
 
     def to_dict(self) -> dict:
         """Return the configuration as a dictionary."""
@@ -183,7 +200,9 @@ def to_dict(self) -> dict:
             if isinstance(value, Mapping):
                 original_dict[key] = dict(value)
 
-        copied = ScicatConfig(original_dict, self.run_options, self.kafka_options)
+        copied = ScicatConfig(
+            original_dict, self.run_options, self.kafka_options, self.graylog_options
+        )
         return asdict(copied)
 
 
@@ -204,7 +223,7 @@ def build_scicat_config(input_args: argparse.Namespace) -> ScicatConfig:
         config_dict = dict()
 
     # Overwrite deep-copied options with command line arguments
-    run_option_dict: dict = copy.deepcopy(config_dict.setdefault('options', dict()))
+    run_option_dict: dict = copy.deepcopy(config_dict.setdefault("options", dict()))
     for arg_name, arg_value in vars(input_args).items():
         if arg_value is not None:
             run_option_dict[arg_name] = arg_value
@@ -217,5 +236,6 @@ def build_scicat_config(input_args: argparse.Namespace) -> ScicatConfig:
     return ScicatConfig(
         original_dict=MappingProxyType(config_dict),
         run_options=RunOptions(**run_option_dict),
-        kafka_options=kafkaOptions(**config_dict.setdefault('kafka', dict())),
+        kafka_options=kafkaOptions(**config_dict.setdefault("kafka", dict())),
+        graylog_options=GraylogOptions(**config_dict.setdefault("graylog", dict())),
     )

src/scicat_logging.py

@@ -4,6 +4,8 @@
 import logging.handlers
 from datetime import datetime
 
+import graypy
+
 from scicat_configuration import ScicatConfig
 
 
@@ -33,6 +35,16 @@ def build_logger(config: ScicatConfig) -> logging.Logger:
     if run_options.system_log:
         logger.addHandler(logging.handlers.SysLogHandler(address='/dev/log'))
 
+    # Add graylog handler
+    if run_options.graylog:
+        graylog_config = config.graylog_options
+        graylog_handler = graypy.GELFTCPHandler(
+            graylog_config.host,
+            int(graylog_config.port),
+            facility=graylog_config.facility,
+        )
+        logger.addHandler(graylog_handler)
+
     # Set the level and formatter for all handlers
     logger.setLevel(run_options.log_level)
     for handler in logger.handlers:

tests/test_logging.py

@@ -2,7 +2,7 @@
 
 import pytest
 
-from scicat_configuration import RunOptions, ScicatConfig, kafkaOptions
+from scicat_configuration import GraylogOptions, RunOptions, ScicatConfig, kafkaOptions
 
 
 @pytest.fixture
@@ -23,6 +23,7 @@ def scicat_config(tmp_path: pathlib.Path) -> ScicatConfig:
             pyscicat='test',
         ),
         kafka_options=kafkaOptions(),
+        graylog_options=GraylogOptions(),
     )