Merge pull request #32 from ISISComputingGroup/Ticket3243_WebDashboar…

…d_flag_an_error_if_instrument_time_and_webserver_time_are_different

Ticket3243 web dashboard flag an error if instrument time and webserver time are different

external_webpage/instrument_information_collator.py

@@ -148,7 +148,7 @@ def _get_inst_pvs(self, instrument_archive_blocks):
                         "1:1:LABEL", "2:1:LABEL", "3:1:LABEL", "1:2:LABEL", "2:2:LABEL", "3:2:LABEL",
                         "BANNER:LEFT:LABEL", "BANNER:MIDDLE:LABEL", "BANNER:RIGHT:LABEL", "1:1:VALUE", "2:1:VALUE",
                         "3:1:VALUE", "1:2:VALUE", "2:2:VALUE", "3:2:VALUE", "BANNER:LEFT:VALUE",
-                        "BANNER:MIDDLE:VALUE", "BANNER:RIGHT:VALUE"]
+                        "BANNER:MIDDLE:VALUE", "BANNER:RIGHT:VALUE", "TIME_OF_DAY"]
 
 
         for pv in required_pvs:

external_webpage/request_handler_utils.py

@@ -1,6 +1,10 @@
 from builtins import str
 import re
 from collections import OrderedDict
+import time
+import logging
+
+logger = logging.getLogger('JSON_bourne')
 
 
 def get_instrument_and_callback(path):
@@ -52,15 +56,88 @@ def get_summary_details_of_all_instruments(data):
     return inst_data
 
 
-def get_detailed_state_of_specific_instrument(instrument, data):
+def get_instrument_time_since_epoch(instrument_name, instrument_data):
+    """
+    Return the instrument time as seconds since epoch.
+
+    :param instrument_name: The name of the instrument
+    :param instrument_data: The data associated with the instrument
+    :return: the instrument time as seconds since epoch.
+
+    :raises: KeyError: instrument time cannot be parsed from instrument_data
+    :raises: ValueError: if instrument time has wrong time format
+    """
+
+    try:
+        channel = instrument_data['Channel']
+    except KeyError:
+        channel = "UNKNOWN"
+
+    time_format = '%m/%d/%Y %H:%M:%S'
+    try:
+        tod = 'TIME_OF_DAY'
+        inst_time_str = instrument_data['inst_pvs'][tod]['value']
+    except KeyError:
+        logger.exception(f"{instrument_name}: Cannot find {tod} in PV {channel}.")
+        raise
+
+    try:
+        inst_time_struct = time.strptime(inst_time_str, time_format)
+    except ValueError:
+        logger.exception(f"{instrument_name}: Value {inst_time_str} from PV {channel} does not match time format "
+                         f"{time_format}.")
+        raise
+
+    try:
+        inst_time = time.mktime(inst_time_struct)
+    except (ValueError, OverflowError):
+        inst_time = None
+        logger.error(f"{instrument_name}: Cannot parse value {inst_time} from PV {channel} as time")
+
+    return inst_time
+
+
+def set_time_shift(instrument_name, instrument_data, time_shift_threshold,
+                   extract_time_from_instrument_func=get_instrument_time_since_epoch,
+                   current_time_func=time.time):
+    """
+    Update the instrument data with the time shift to the webserver.
+
+    :param instrument_name: The name of the instrument
+    :param instrument_data: The data dictionary of the instrument
+    :param time_shift_threshold: If the time shift is greater than this value the data is considered outdated
+    """
+    try:
+        inst_time = extract_time_from_instrument_func(instrument_name, instrument_data)
+        current_time = current_time_func()
+        time_diff = int(round(abs(current_time - inst_time)))
+    except (ValueError, TypeError, KeyError):
+        time_diff = None
+
+    try:
+        instrument_data['time_diff'] = time_diff
+
+        if time_diff is not None and time_diff > time_shift_threshold:
+            instrument_data['out_of_sync'] = True
+            logger.warning(f"There is a time shift of {time_diff} seconds between {instrument_name} and web server")
+        else:
+            instrument_data['out_of_sync'] = False
+    except TypeError:
+        logger.error(f"Cannot set time shift information for {instrument_name}.")
+
+
+def get_detailed_state_of_specific_instrument(instrument, data, time_shift_threshold=5*60):
     """
     Gets the detailed state of a specific instrument, used to display the instrument's dataweb screen
     :param instrument: The instrument to get data for
     :param data: The data scraped from the archiver webpage
+    :param time_shift_threshold: The allowed time difference in seconds between the instrument and the webserver time
     :return: The data from the archiver webpage filtered to only contain data about the requested instrument
     """
     if instrument not in data.keys():
         raise ValueError(str(instrument) + " not known")
     if data[instrument] == "":
         raise ValueError("Instrument has become unavailable")
+    set_time_shift(instrument, data[instrument], time_shift_threshold)
+
     return data[instrument]

front_end/default.html

@@ -17,8 +17,15 @@
     <body>
 		<div id = "top_bar"></div>
         <div id="error_status"></div>
-        <div id="config_name"></div>
         <table cellpadding="30" style="width:100%">
+            <tr>
+                <td valign="top">
+		  <h2 id="config_name"></h2>
+                </td>
+                <td valign="top">
+		  <h2 id="time_diff"></h2>
+                </td>
+            </tr>
             <tr>
                 <td valign="top" rowspan="2">
                     <h2 id="run_information">Run Information</h2>

front_end/display_blocks.js

@@ -7,6 +7,7 @@ var nodeInstTitle = document.createElement("H2");
 var nodeConfigTitle = document.createElement("H2");
 var nodeErrorStatus = document.createElement("H3");
 nodeErrorStatus.style.color = "RED";
+var nodeTimeDiffTitle = document.createElement("H2");
 var instrumentState;
 var showHidden;
 var timeout = 4000;
@@ -38,7 +39,9 @@ dictInstPV = {
     MONITORTO: 'Monitor To',
     NUMTIMECHANNELS: 'Number of Time Channels',
     NUMSPECTRA: 'Number of Spectra',
-	SIM_MODE: 'DAE Simulation mode'
+    SHUTTER: 'Shutter Status',
+    SIM_MODE: 'DAE Simulation mode',
+    TIME_OF_DAY: 'Instrument time',
 };
 
 dictLongerInstPVs = {
@@ -190,6 +193,7 @@ function parseObject(obj) {
     showHidden = document.getElementById("showHidden").checked;
     clear(nodeInstTitle);
     clear(nodeConfigTitle);
+    clear(nodeTimeDiffTitle);
 
     // populate blocks
     var nodeGroups = document.getElementById("groups");
@@ -201,6 +205,8 @@ function parseObject(obj) {
 
     getDisplayRunInfo(nodeInstPVs, instrumentState.inst_pvs);
 
+    getDisplayTimeDiffInfo(instrumentState);
+
     nodeInstTitle.appendChild(document.createTextNode(instrument));
     nodeConfigTitle.appendChild(document.createTextNode("Configuration: " + instrumentState.config_name));
 
@@ -225,6 +231,14 @@ function get_inst_pv_value(inst_details, pv) {
     }
 }
 
+function getDisplayTimeDiffInfo(instrumentState){
+    if (instrumentState.out_of_sync)  {
+      nodeTimeDiffTitle.appendChild(document.createTextNode("There is a time shift of " + instrumentState.time_diff + " seconds between the instrument and the web server. Dataweb may not be updating correctly."));
+      document.getElementById("time_diff").appendChild(nodeTimeDiffTitle);
+      document.getElementById("time_diff").style.color = "RED";
+    }
+}
+
 /**
  * creates a Title at the top looking similar to the IBEX GUI
  */

tests/test_handler_utils.py

@@ -1,9 +1,11 @@
 from hamcrest import *
 
 from external_webpage.request_handler_utils import get_instrument_and_callback, \
-    get_summary_details_of_all_instruments, get_detailed_state_of_specific_instrument
+    get_summary_details_of_all_instruments, get_detailed_state_of_specific_instrument, \
+    get_instrument_time_since_epoch, set_time_shift
 import json
 import unittest
+import time
 
 
 CALLBACK_STR = "?callback={}&"
@@ -161,3 +163,70 @@ def test_GIVEN_instrument_with_data_WHEN_get_details_called_THEN_data_returned_f
         out = get_detailed_state_of_specific_instrument(inst, data_dict)
 
         self.assertEqual(out, inst_data)
+
+
+class TestHandlerUtils_InstrumentTime(unittest.TestCase):
+
+    def test_that_GIVEN_good_instrument_THEN_retrun_inst_time(self):
+        instrument_data = {'inst_pvs': {'TIME_OF_DAY': {'value': '01/01/1970 01:00:00'}}}
+        inst_time = get_instrument_time_since_epoch("", instrument_data)
+
+        assert_that(inst_time, equal_to(3600.0 + time.timezone))
+
+    def test_that_GIVEN_wrong_formated_instrument_THEN_raise_value_error(self):
+        instrument_data = {'inst_pvs': {'TIME_OF_DAY': {'value': 'this is no propper time format'}}}
+
+        assert_that(calling(get_instrument_time_since_epoch).with_args("", instrument_data), raises(ValueError))
+
+    def test_that_GIVEN_instrument_data_without_time_of_day_THEN_raise_value_error(self):
+        instrument_data = {'inst_pvs': {'this_is_not_TIME_OF_DAY': {'value': '01/01/1970 01:00:00'}}}
+
+        assert_that(calling(get_instrument_time_since_epoch).with_args("", instrument_data), raises(KeyError))
+
+
+class TestHandlerUtils_CheckOutOfSync(unittest.TestCase):
+
+    def test_that_GIVEN_time_difference_is_greater_than_threshold_THEN_set_out_of_sync_to_true(self):
+        instrument_data = {'inst_pvs': {'TIME_OF_DAY': {'value': 'does not matter'}}}
+
+        set_time_shift("", instrument_data, time_shift_threshold=2,
+                          extract_time_from_instrument_func=lambda _, __ : 5,
+                          current_time_func=lambda : 10)
+
+        assert_that(instrument_data['out_of_sync'], equal_to(True))
+
+    def test_that_GIVEN_time_difference_is_less_than_threshold_THEN_set_out_of_sync_to_false(self):
+        instrument_data = {'inst_pvs': {'TIME_OF_DAY': {'value': 'does not matter'}}}
+
+        set_time_shift("", instrument_data, time_shift_threshold=17,
+                          extract_time_from_instrument_func=lambda _, __ : 5,
+                          current_time_func=lambda : 10)
+
+        assert_that(instrument_data['out_of_sync'], equal_to(False))
+
+    def test_that_GIVEN_time_difference_of_five_THEN_set_time_diff_to_five(self):
+        instrument_data = {'inst_pvs': {'TIME_OF_DAY': {'value': 'does not matter'}}}
+
+        set_time_shift("", instrument_data, time_shift_threshold=17,
+                          extract_time_from_instrument_func=lambda _, __ : 5,
+                          current_time_func=lambda : 10)
+
+        assert_that(instrument_data['time_diff'], equal_to(5))
+
+    def test_that_GIVEN_invalid_time_THEN_set_time_diff_to_None(self):
+        instrument_data = {'inst_pvs': {'TIME_OF_DAY': {'value': 'does not matter'}}}
+
+        set_time_shift("", instrument_data, time_shift_threshold=17,
+                       extract_time_from_instrument_func=lambda _, __: 'foo',
+                       current_time_func=lambda: 10)
+
+        assert_that(instrument_data['time_diff'], equal_to(None))
+
+    def test_that_GIVEN_invalid_time_THEN_set_out_of_sync_to_false(self):
+        instrument_data = {'inst_pvs': {'TIME_OF_DAY': {'value': 'does not matter'}}}
+
+        set_time_shift("", instrument_data, time_shift_threshold=17,
+                       extract_time_from_instrument_func=lambda _, __: 'foo',
+                       current_time_func=lambda: 10)
+
+        assert_that(instrument_data['out_of_sync'], equal_to(False))

webserver.py

@@ -80,7 +80,7 @@ def log_message(self, format, *args):
 
 if __name__ == '__main__':
     # It can sometime be useful to define a local instrument list to add/override the instrument list do this here
-    # E.g. to add local instrument local_inst_list = {"localhost": "localhost"}
+    # E.g. to add local instrument local_inst_list = {"LOCALHOST": ("localhost", "MYPVPREFIX")}
     local_inst_list = {}
     web_manager = WebScrapperManager(local_inst_list=local_inst_list)
     web_manager.start()