Add test for ExtractionJobTracker machinery

tests/test_backend.py

@@ -9,6 +9,7 @@
 import subprocess
 import textwrap
 from time import sleep, time
+from uuid import uuid4
 from unittest.mock import MagicMock, patch
 
 import extra_data as ed
@@ -25,6 +26,7 @@
 from damnit.backend import backend_is_running, initialize_and_start_backend
 from damnit.backend.db import DamnitDB
 from damnit.backend.extract_data import Extractor, RunExtractor, add_to_db
+from damnit.backend.extraction_control import ExtractionJobTracker
 from damnit.backend.listener import (MAX_CONCURRENT_THREADS, EventProcessor,
                                      local_extraction_threads)
 from damnit.backend.supervisord import wait_until, write_supervisord_conf
@@ -903,3 +905,32 @@ def test_event_processor(mock_db, caplog):
 
         assert len(local_extraction_threads) == MAX_CONCURRENT_THREADS
         assert 'Too many events processing' in caplog.text
+
+
+def test_job_tracker():
+    tracker = ExtractionJobTracker()
+
+    d = {'proposal': 1234, 'data': 'all', 'hostname': '', 'username': '',
+         'slurm_cluster': '', 'slurm_job_id': ''}
+
+    prid1, prid2 = str(uuid4()), str(uuid4())
+
+    # Add two running jobs
+    tracker.on_processing_running(d | {'run': 1, 'processing_id': prid1})
+    tracker.on_processing_running(d | {
+        'run': 2, 'processing_id': prid2,
+        'slurm_cluster': 'maxwell', 'slurm_job_id': '321'
+    })
+    assert set(tracker.jobs) == {prid1, prid2}
+
+    # One job finishes normally
+    tracker.on_processing_finished({'processing_id': prid1})
+    assert set(tracker.jobs) == {prid2}
+
+    # The other one fails to send a finished message, so checking Slurm reveals
+    # that it failed.
+    with MockCommand.fixed_output('squeue', '') as fake_squeue:
+        tracker.check_slurm_jobs()
+
+    fake_squeue.assert_called()
+    assert set(tracker.jobs) == set()