From 8cb8ec5962c9e3922c29a4964b3f9a277f151b04 Mon Sep 17 00:00:00 2001 From: dwest77 Date: Tue, 19 Nov 2024 15:48:37 +0000 Subject: [PATCH] Added various log messages and configuration --- facet_scanner/__init__.py | 9 +++++++++ facet_scanner/collection_handlers/base.py | 7 +++++++ facet_scanner/collection_handlers/cci.py | 13 ++++++++++++- facet_scanner/collection_handlers/cmip5.py | 7 +++++++ facet_scanner/core/elasticsearch_connection.py | 6 ++++++ facet_scanner/core/facet_scanner.py | 6 ++++++ facet_scanner/scripts/facet_scanner_cmd.py | 7 +++++++ facet_scanner/scripts/lotus_facet_scanner.py | 7 +++++++ facet_scanner/utils/snippets.py | 7 +++++++ 9 files changed, 68 insertions(+), 1 deletion(-) diff --git a/facet_scanner/__init__.py b/facet_scanner/__init__.py index d1c2687..d01e63c 100644 --- a/facet_scanner/__init__.py +++ b/facet_scanner/__init__.py @@ -5,3 +5,12 @@ __license__ = 'BSD - see LICENSE file in top-level package directory' __contact__ = 'daniel.westwood@stfc.ac.uk' +# Logger setup +import logging + +logging.basicConfig(level=logging.DEBUG) +logstream = logging.StreamHandler() + +formatter = logging.Formatter('%(levelname)s [%(name)s]: %(message)s') +logstream.setFormatter(formatter) + diff --git a/facet_scanner/collection_handlers/base.py b/facet_scanner/collection_handlers/base.py index 0c49935..99a7a48 100644 --- a/facet_scanner/collection_handlers/base.py +++ b/facet_scanner/collection_handlers/base.py @@ -17,6 +17,13 @@ from facet_scanner.utils import generator_grouper, Singleton import time +from facet_scanner import logstream +import logging + +logger = logging.getLogger(__name__) +logger.addHandler(logstream) +logger.propagate = False + class CollectionHandler(metaclass=Singleton): """ diff --git a/facet_scanner/collection_handlers/cci.py b/facet_scanner/collection_handlers/cci.py index 0b19c25..b0e2739 100644 --- a/facet_scanner/collection_handlers/cci.py +++ b/facet_scanner/collection_handlers/cci.py @@ -11,12 +11,19 @@ from tqdm import tqdm import hashlib import json +import logging from facet_scanner.collection_handlers.base import CollectionHandler from facet_scanner.collection_handlers.utils import CatalogueDatasets from facet_scanner.utils import parse_key from tag_scanner.tagger import ProcessDatasets +from facet_scanner import logstream + +logger = logging.getLogger(__name__) +logger.addHandler(logstream) +logger.propagate = False + def nested_get(key_list, input_dict, default=None): """ Takes an iterable of keys and returns none if not found or the value @@ -113,8 +120,11 @@ def get_facets(self, path): :return: Dict Facet:value pairs """ + logger.debug('Getting facets for CCI-type path') + tagged_dataset = self.pds.get_file_tags(path) + logger.debug('Translating tagging code to facet map') # Translate between output from tagging code to map to named facets mapped_facets = {} for tag_name, tag_value in tagged_dataset.labels.items(): @@ -128,6 +138,7 @@ def get_facets(self, path): if tag_name == facet and tag_name_mapping is None: mapped_facets[facet] = tag_value + logger.debug('Obtaining moles record metadata') # Get MOLES catalogue moles_info = self.catalogue.get_moles_record_metadata(path) @@ -137,6 +148,7 @@ def get_facets(self, path): if moles_info: mapped_facets['datasetId'] = moles_info['url'].split('uuid/')[-1] + logger.debug('Completed facet mapping') return mapped_facets @staticmethod @@ -256,7 +268,6 @@ def _get_collection_variables(self, results, file_index): if values: ids = [x['key'] for x in values] - print(ids) # Sample 1 netCDF file from each DRS for id in ids: diff --git a/facet_scanner/collection_handlers/cmip5.py b/facet_scanner/collection_handlers/cmip5.py index baeb936..3ac54bc 100644 --- a/facet_scanner/collection_handlers/cmip5.py +++ b/facet_scanner/collection_handlers/cmip5.py @@ -10,6 +10,13 @@ from facet_scanner.collection_handlers.base import CollectionHandler import os +import logging + +from facet_scanner import logstream + +logger = logging.getLogger(__name__) +logger.addHandler(logstream) +logger.propagate = False class CMIP5(CollectionHandler): project_name = 'opensearch' diff --git a/facet_scanner/core/elasticsearch_connection.py b/facet_scanner/core/elasticsearch_connection.py index 3ac3901..e78276c 100644 --- a/facet_scanner/core/elasticsearch_connection.py +++ b/facet_scanner/core/elasticsearch_connection.py @@ -11,6 +11,12 @@ from elasticsearch.helpers import scan, bulk from ceda_elasticsearch_tools.elasticsearch import CEDAElasticsearchClient +import logging +from facet_scanner import logstream + +logger = logging.getLogger(__name__) +logger.addHandler(logstream) +logger.propagate = False class ElasticsearchConnection: """ diff --git a/facet_scanner/core/facet_scanner.py b/facet_scanner/core/facet_scanner.py index 4376fae..0b91752 100644 --- a/facet_scanner/core/facet_scanner.py +++ b/facet_scanner/core/facet_scanner.py @@ -11,7 +11,11 @@ from facet_scanner.collection_handlers.utils import FacetFactory import logging +from facet_scanner import logstream + logger = logging.getLogger(__name__) +logger.addHandler(logstream) +logger.propagate = False class FacetScanner: @@ -33,8 +37,10 @@ def get_handler(self, path, **kwargs): :return: Mapped collection handler :rtype: CollectionHandler """ + logger.debug("Obtaining handler") handler, collection_root = self.handler_factory.get_handler(path) + logger.debug('Handler Obtainment complete') # Handle situation where handler not found if handler is None: logger.error(f'Unable to find a handler for: {path} in facet_scanner.collection_handlers.utils.collection_map.' diff --git a/facet_scanner/scripts/facet_scanner_cmd.py b/facet_scanner/scripts/facet_scanner_cmd.py index cfac021..3e945ed 100644 --- a/facet_scanner/scripts/facet_scanner_cmd.py +++ b/facet_scanner/scripts/facet_scanner_cmd.py @@ -24,6 +24,13 @@ from configparser import RawConfigParser from facet_scanner.utils import query_yes_no from facet_scanner.core.facet_scanner import FacetScanner +import logging + +from facet_scanner import logstream + +logger = logging.getLogger(__name__) +logger.addHandler(logstream) +logger.propagate = False class FacetExtractor(FacetScanner): diff --git a/facet_scanner/scripts/lotus_facet_scanner.py b/facet_scanner/scripts/lotus_facet_scanner.py index c1f03e0..d285f60 100644 --- a/facet_scanner/scripts/lotus_facet_scanner.py +++ b/facet_scanner/scripts/lotus_facet_scanner.py @@ -23,6 +23,13 @@ import os import json +import logging +from facet_scanner import logstream + +logger = logging.getLogger(__name__) +logger.addHandler(logstream) +logger.propagate = False + class LotusFacetScanner(FacetExtractor): diff --git a/facet_scanner/utils/snippets.py b/facet_scanner/utils/snippets.py index b4cfea7..26ebd8d 100644 --- a/facet_scanner/utils/snippets.py +++ b/facet_scanner/utils/snippets.py @@ -11,6 +11,13 @@ import sys import itertools from collections import OrderedDict +import logging + +from facet_scanner import logstream + +logger = logging.getLogger(__name__) +logger.addHandler(logstream) +logger.propagate = False def query_yes_no(question, default="yes"):