Merge pull request #1852 from uwcirg/hotfix/unknown-observations

TN-666 Include status in observation handling for all APIs.

portal/models/user.py

@@ -621,7 +621,8 @@ def add_observation(self, fhir, audit):
 
         issued = fhir.get('issued') and\
             parser.parse(fhir.get('issued')) or None
-        observation = self.save_observation(cc, vq, audit, issued)
+        status = fhir.get('status')
+        observation = self.save_observation(cc, vq, audit, status, issued)
         if 'performer' in fhir:
             for p in fhir['performer']:
                 performer = Performer.from_fhir(p)
@@ -676,13 +677,30 @@ def add_service_account(self):
         self.add_relationship(service_user, RELATIONSHIP.SPONSOR)
         return service_user
 
-    def fetch_values_for_concept(self, codeable_concept):
-        """Return any matching ValueQuantities for this user"""
+    def fetch_value_status_for_concept(self, codeable_concept):
+        """Return matching ValueQuantity & status for this user
+
+        Expected to be used on constrained concepts, where a user
+        should have zero or one defined.  More than one will raise
+        a value error
+
+        :returns: (value_quantity, status) tuple for the observation
+         if found on the user, else (None, None)
+
+        """
         # User may not have persisted concept - do so now for match
         codeable_concept = codeable_concept.add_if_not_found()
 
-        return [obs.value_quantity for obs in self.observations if
-                obs.codeable_concept_id == codeable_concept.id]
+        matching_obs = [
+            obs for obs in self.observations if
+            obs.codeable_concept_id == codeable_concept.id]
+        if not matching_obs:
+            return None, None
+        if len(matching_obs) > 1:
+            raise ValueError(
+                "multiple observations for {} on constrianed {}".format(
+                    self, codeable_concept))
+        return matching_obs[0].value_quantity, matching_obs[0].status
 
     def fetch_datetime_for_concept(self, codeable_concept):
         """Return newest issued timestamp from matching observation"""
@@ -697,7 +715,8 @@ def fetch_datetime_for_concept(self, codeable_concept):
         return newest
 
     def save_constrained_observation(
-            self, codeable_concept, value_quantity, audit, issued=None):
+            self, codeable_concept, value_quantity, audit, status,
+            issued=None):
         """Add or update the value for given concept as observation
 
         We can store any number of observations for a patient, and
@@ -717,23 +736,28 @@ def save_constrained_observation(
 
         if existing:
             if existing[0].value_quantity_id == value_quantity.id:
-                # perfect match -- update audit info
+                # perfect match -- update audit info, setting status
+                # and issued as given
+                existing.status = status
+                if issued:
+                    existing.issued = issued
                 existing[0].audit = audit
                 return
             else:
                 # We don't want multiple observations for this concept
                 # with different values.  Delete old and add new
                 self.observations.remove(existing[0])
 
-        self.save_observation(codeable_concept, value_quantity, audit, issued)
+        self.save_observation(codeable_concept, value_quantity, audit, status, issued)
 
-    def save_observation(self, cc, vq, audit, issued):
+    def save_observation(self, cc, vq, audit, status, issued):
         """Helper method for creating new observations"""
         # avoid cyclical imports
         from .assessment_status import invalidate_assessment_status_cache
 
         observation = Observation(
             codeable_concept_id=cc.id,
+            status=status,
             issued=issued,
             value_quantity_id=vq.id).add_if_not_found(True)
         # The audit defines the acting user, to which the current

portal/views/clinical.py

@@ -152,6 +152,9 @@ def biopsy_set(patient_id):
             value:
               type: boolean
               description: has the patient undergone a biopsy
+            status:
+              type: string
+              description: optional status such as 'final' or 'unknown'
     responses:
       200:
         description: successful operation
@@ -207,6 +210,9 @@ def pca_diag_set(patient_id):
             value:
               type: boolean
               description: the patient's PCa diagnosis
+            status:
+              type: string
+              description: optional status such as 'final' or 'unknown'
     responses:
       200:
         description: successful operation
@@ -262,6 +268,9 @@ def pca_localized_set(patient_id):
             value:
               type: boolean
               description: the patient's PCaLocalized diagnosis
+            status:
+              type: string
+              description: optional status such as 'final' or 'unknown'
     responses:
       200:
         description: successful operation
@@ -488,10 +497,12 @@ def clinical_api_shortcut_set(patient_id, codeable_concept):
         abort(400, "Expecting boolean for 'value'")
 
     truthiness = ValueQuantity(value=value, units='boolean')
-    patient.save_constrained_observation(codeable_concept=codeable_concept,
-                                         value_quantity=truthiness,
-                                         audit=Audit(user_id=current_user().id,
-                                          subject_id=patient_id, context='observation'))
+    audit = Audit(user_id=current_user().id,
+                  subject_id=patient_id, context='observation')
+    patient.save_constrained_observation(
+        codeable_concept=codeable_concept, value_quantity=truthiness,
+        audit=audit, status=request.json.get('status'))
+
     db.session.commit()
     auditable_event("set {0} {1} on user {2}".format(
         codeable_concept, truthiness, patient_id), user_id=current_user().id,
@@ -505,9 +516,9 @@ def clinical_api_shortcut_get(patient_id, codeable_concept):
     patient = get_user(patient_id)
     if patient.deleted:
         abort(400, "deleted user - operation not permitted")
-    value_quantities = patient.fetch_values_for_concept(codeable_concept)
-    if value_quantities:
-        assert len(value_quantities) == 1
-        return jsonify(value=value_quantities[0].value)
+    value_quantity, status = patient.fetch_value_status_for_concept(
+        codeable_concept)
+    if value_quantity and status != 'unknown':
+        return jsonify(value=value_quantity.value)
 
     return jsonify(value='unknown')

tests/__init__.py

@@ -162,7 +162,7 @@ def add_required_clinical_data(self, backdate=None):
         for cc in CC.BIOPSY, CC.PCaDIAG, CC.PCaLocalized:
             get_user(TEST_USER_ID).save_constrained_observation(
                 codeable_concept=cc, value_quantity=CC.TRUE_VALUE,
-                audit=audit, issued=timestamp)
+                audit=audit, status='preliminary', issued=timestamp)
 
     def add_procedure(self, code='367336001', display='Chemotherapy',
                       system=SNOMED, setdate=None):

tests/test_assessment_status.py

@@ -647,6 +647,7 @@ def test_no_start_date(self):
         self.test_user = db.session.merge(self.test_user)
         self.test_user.save_constrained_observation(
             codeable_concept=CC.BIOPSY, value_quantity=CC.FALSE_VALUE,
-            audit=Audit(user_id=TEST_USER_ID, subject_id=TEST_USER_ID))
+            audit=Audit(user_id=TEST_USER_ID, subject_id=TEST_USER_ID),
+            status='final')
         self.assertFalse(
             QuestionnaireBank.qbs_for_user(self.test_user, 'baseline'))

tests/test_clinical.py

@@ -128,6 +128,25 @@ def test_empty_clinical_get(self):
         rv = self.client.get('/api/patient/%s/clinical' % TEST_USER_ID)
         self.assert200(rv)
 
+    def test_unknown_clinical_post(self):
+        self.login()
+        data = {
+            "resourceType": "Observation",
+            "code":{"coding":[{
+                "code":"121",
+                "display":"PCa diagnosis",
+                "system":"http://us.truenth.org/clinical-codes"}]},
+                "status":"unknown",
+                "valueQuantity":{"units":"boolean","value":"false"}}
+        rv = self.client.post('/api/patient/{}/clinical'.format(
+            TEST_USER_ID), content_type='application/json',
+            data=json.dumps(data))
+
+        # confirm status unknown sticks
+        self.assert200(rv)
+        rv = self.client.get('/api/patient/%s/clinical' % TEST_USER_ID)
+        self.assertEquals('unknown', rv.json['entry'][0]['content']['status'])
+
     def test_empty_biopsy_get(self):
         """Access biopsy on user w/o any clinical info"""
         self.login()
@@ -180,6 +199,53 @@ def test_clinical_biopsy_put(self):
         user = User.query.get(TEST_USER_ID)
         self.assertEquals(user.observations.count(), 1)
 
+    def test_clinical_biopsy_unknown(self):
+        """Shortcut API - biopsy data w status unknown"""
+        self.login()
+        rv = self.client.post(
+            '/api/patient/%s/clinical/biopsy' % TEST_USER_ID,
+            content_type='application/json',
+            data=json.dumps({'value': True, 'status': 'unknown'}))
+        self.assert200(rv)
+        result = json.loads(rv.data)
+        self.assertEquals(result['message'], 'ok')
+
+        # Can we get it back in FHIR?
+        rv = self.client.get('/api/patient/%s/clinical' % TEST_USER_ID)
+        data = json.loads(rv.data)
+        coding = data['entry'][0]['content']['code']['coding'][0]
+        vq = data['entry'][0]['content']['valueQuantity']
+        status = data['entry'][0]['content']['status']
+
+        self.assertEquals(coding['code'], '111')
+        self.assertEquals(coding['display'], 'biopsy')
+        self.assertEquals(coding['system'],
+                          'http://us.truenth.org/clinical-codes')
+        self.assertEquals(vq['value'], 'true')
+        self.assertEquals(status, 'unknown')
+
+        # Access the direct biopsy value
+        rv = self.client.get('/api/patient/%s/clinical/biopsy' % TEST_USER_ID)
+        data = json.loads(rv.data)
+        self.assertEquals(data['value'], 'unknown')
+
+        # Can we alter the value?
+        rv = self.client.post('/api/patient/%s/clinical/biopsy' % TEST_USER_ID,
+                              content_type='application/json',
+                              data=json.dumps({'value': False}))
+        self.assert200(rv)
+        result = json.loads(rv.data)
+        self.assertEquals(result['message'], 'ok')
+
+        # Confirm it's altered
+        rv = self.client.get('/api/patient/%s/clinical/biopsy' % TEST_USER_ID)
+        data = json.loads(rv.data)
+        self.assertEquals(data['value'], 'false')
+
+        # Confirm the db is clean
+        user = User.query.get(TEST_USER_ID)
+        self.assertEquals(user.observations.count(), 1)
+
     def test_clinical_pca_diag(self):
         """Shortcut API - just PCa diagnosis w/o FHIR overhead"""
         self.login()
@@ -209,6 +275,37 @@ def test_clinical_pca_diag(self):
         data = json.loads(rv.data)
         self.assertEquals(data['value'], 'true')
 
+    def test_clinical_pca_diag_unknown(self):
+        """Shortcut API - PCa diagnosis w/ status unknown"""
+        self.login()
+        rv = self.client.post(
+            '/api/patient/%s/clinical/pca_diag' % TEST_USER_ID,
+            content_type='application/json',
+            data=json.dumps({'value': True, 'status': 'unknown'}))
+        self.assert200(rv)
+        result = json.loads(rv.data)
+        self.assertEquals(result['message'], 'ok')
+
+        # Can we get it back in FHIR?
+        rv = self.client.get('/api/patient/%s/clinical' % TEST_USER_ID)
+        data = json.loads(rv.data)
+        coding = data['entry'][0]['content']['code']['coding'][0]
+        vq = data['entry'][0]['content']['valueQuantity']
+        status = data['entry'][0]['content']['status']
+
+        self.assertEquals(coding['code'], '121')
+        self.assertEquals(coding['display'], 'PCa diagnosis')
+        self.assertEquals(coding['system'],
+                          'http://us.truenth.org/clinical-codes')
+        self.assertEquals(vq['value'], 'true')
+        self.assertEquals(status, 'unknown')
+
+        # Access the direct pca_diag value
+        rv = self.client.get(
+            '/api/patient/%s/clinical/pca_diag' % TEST_USER_ID)
+        data = json.loads(rv.data)
+        self.assertEquals(data['value'], 'unknown')
+
     def test_clinical_pca_localized(self):
         """Shortcut API - just PCa localized diagnosis w/o FHIR overhead"""
         self.login()
@@ -237,6 +334,37 @@ def test_clinical_pca_localized(self):
         data = json.loads(rv.data)
         self.assertEquals(data['value'], 'true')
 
+    def test_clinical_pca_localized_unknown(self):
+        """Shortcut API - PCa localized diagnosis w status unknown"""
+        self.login()
+        rv = self.client.post(
+            '/api/patient/%s/clinical/pca_localized' % TEST_USER_ID,
+            content_type='application/json',
+            data=json.dumps({'value': False, 'status': 'unknown'}))
+        self.assert200(rv)
+        result = json.loads(rv.data)
+        self.assertEquals(result['message'], 'ok')
+
+        # Can we get it back in FHIR?
+        rv = self.client.get('/api/patient/%s/clinical' % TEST_USER_ID)
+        data = json.loads(rv.data)
+        coding = data['entry'][0]['content']['code']['coding'][0]
+        vq = data['entry'][0]['content']['valueQuantity']
+        status = data['entry'][0]['content']['status']
+
+        self.assertEquals(coding['code'], '141')
+        self.assertEquals(coding['display'], 'PCa localized diagnosis')
+        self.assertEquals(coding['system'],
+                          'http://us.truenth.org/clinical-codes')
+        self.assertEquals(vq['value'], 'false')
+        self.assertEquals(status, 'unknown')
+
+        # Access the direct pca_localized value
+        rv = self.client.get(
+            '/api/patient/%s/clinical/pca_localized' % TEST_USER_ID)
+        data = json.loads(rv.data)
+        self.assertEquals(data['value'], 'unknown')
+
     def test_weight(self):
         with open(os.path.join(os.path.dirname(__file__),
                                'weight_example.json'), 'r') as fhir_data:

tests/test_communication.py

@@ -382,7 +382,8 @@ def test_st_metastatic(self):
         self.test_user = db.session.merge(self.test_user)
         self.test_user.save_constrained_observation(
             codeable_concept=CC.PCaLocalized, value_quantity=CC.FALSE_VALUE,
-            audit=Audit(user_id=TEST_USER_ID, subject_id=TEST_USER_ID))
+            audit=Audit(user_id=TEST_USER_ID, subject_id=TEST_USER_ID),
+            status='final')
 
         # Confirm test user doesn't qualify for ST QB
         self.assertFalse(

tests/test_intervention.py

@@ -231,7 +231,8 @@ def test_diag_stategy(self):
         self.login()
         user.save_constrained_observation(
             codeable_concept=CC.PCaDIAG, value_quantity=CC.TRUE_VALUE,
-            audit=Audit(user_id=TEST_USER_ID, subject_id=TEST_USER_ID))
+            audit=Audit(user_id=TEST_USER_ID, subject_id=TEST_USER_ID),
+            status='registered')
         with SessionScope(db):
             db.session.commit()
         user, cp = map(db.session.merge, (user, cp))
@@ -746,7 +747,8 @@ def test_p3p_conditions(self):
         self.login()
         user.save_constrained_observation(
             codeable_concept=CC.PCaLocalized, value_quantity=CC.TRUE_VALUE,
-            audit=Audit(user_id=TEST_USER_ID, subject_id=TEST_USER_ID))
+            audit=Audit(user_id=TEST_USER_ID, subject_id=TEST_USER_ID),
+            status='preliminary')
         with SessionScope(db):
             db.session.commit()
         user, ds_p3p = map(db.session.merge, (user, ds_p3p))
@@ -875,7 +877,8 @@ def test_eproms_p3p_conditions(self):
         self.login()
         user.save_constrained_observation(
             codeable_concept=CC.PCaLocalized, value_quantity=CC.TRUE_VALUE,
-            audit=Audit(user_id=TEST_USER_ID, subject_id=TEST_USER_ID))
+            audit=Audit(user_id=TEST_USER_ID, subject_id=TEST_USER_ID),
+            status='final')
         with SessionScope(db):
             db.session.commit()
         user, ds_p3p = map(db.session.merge, (user, ds_p3p))
@@ -969,7 +972,8 @@ def test_truenth_st_conditions(self):
         self.login()
         user.save_constrained_observation(
             codeable_concept=CC.BIOPSY, value_quantity=CC.TRUE_VALUE,
-            audit=Audit(user_id=TEST_USER_ID, subject_id=TEST_USER_ID))
+            audit=Audit(user_id=TEST_USER_ID, subject_id=TEST_USER_ID),
+            status='unknown')
         with SessionScope(db):
             db.session.commit()
         user, sm = map(db.session.merge, (user, sm))

tests/test_questionnaire_bank.py

@@ -86,7 +86,8 @@ def test_intervention_trigger_date(self):
         self.login()
         self.test_user.save_constrained_observation(
             codeable_concept=CC.BIOPSY, value_quantity=CC.TRUE_VALUE,
-            audit=Audit(user_id=TEST_USER_ID, subject_id=TEST_USER_ID))
+            audit=Audit(user_id=TEST_USER_ID, subject_id=TEST_USER_ID),
+            status='')
         self.test_user = db.session.merge(self.test_user)
         obs = self.test_user.observations.first()
         self.assertEquals(obs.codeable_concept.codings[0].display, 'biopsy')

tests/test_site_persistence.py

@@ -85,7 +85,8 @@ def testP3Pstrategy(self):
             code='424313000', display='Started active surveillance')
         get_user(TEST_USER_ID).save_constrained_observation(
             codeable_concept=CC.PCaLocalized, value_quantity=CC.TRUE_VALUE,
-            audit=Audit(user_id=TEST_USER_ID, subject_id=TEST_USER_ID))
+            audit=Audit(user_id=TEST_USER_ID, subject_id=TEST_USER_ID),
+            status=None)
         self.promote_user(user, role_name=ROLE.PATIENT)
         with SessionScope(db):
             db.session.commit()