diff --git a/tanagra-indexing-1.html b/tanagra-indexing-1.html new file mode 100644 index 000000000..cb80bc124 --- /dev/null +++ b/tanagra-indexing-1.html @@ -0,0 +1,1282 @@ + + + + Indexing Jobs Report + + + +

Indexing Jobs Report

+

Code Version

+

Git Tag 0.0.660-1-ge5571c11

+

Git Hash e5571c116803162599435ba066387879d8adf86d

+

Build 0.0.660-SNAPSHOT

+
+

Command: tanagra index underlay --indexer-config=emerge/emerge_20240603

+

Job Runner: PARALLEL

+

Working Directory: /Users/subracr1/Projects/Tanagra/tanagra

+
+

Total # jobs failed: 1

+

Total # jobs run: 148

+

Elapsed Time (min.sec): 93.00

+
+

Summary

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Entity/Group# Jobs Failed# Jobs Run
bmi05
cpt409
cpt4Occurrence05
drug06
drugOccurrence05
height05
icd10cm09
icd10cmOccurrence05
icd10pcs09
icd10pcsOccurrence05
icd909
icd9Occurrence05
lab06
labOccurrence05
person05
phenotype09
phenotypeOccurrence05
phewas09
weight05
bmiPerson01
cpt4Person03
drugPerson02
heightPerson01
icd10cmPerson03
icd10pcsPerson03
icd9Person03
labPerson13
phenotypePerson03
phewasPerson04
weightPerson01
+



+

Job Details

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Entity/GroupJob NameElapsed time (min.sec)ThreadStatus
bmiCreateEntityMain-ENT_bmi0.01pool-27-thread-1SUCCEEDED
bmiValidateDataTypes-ENT_bmi0.02pool-3-thread-1SUCCEEDED
bmiValidateUniqueIds-ENT_bmi0.04pool-68-thread-1SUCCEEDED
bmiWriteEntityAttributes-ENT_bmi0.07pool-42-thread-1SUCCEEDED
bmiWriteEntityLevelDisplayHints-ELDH_bmi0.37pool-68-thread-2SUCCEEDED
cpt4CreateEntityMain-ENT_cpt40.01pool-38-thread-1SUCCEEDED
cpt4ValidateDataTypes-ENT_cpt40.02pool-11-thread-1SUCCEEDED
cpt4ValidateUniqueIds-ENT_cpt40.01pool-63-thread-1SUCCEEDED
cpt4WriteAncestorDescendant-HAD_cpt4_default5.42pool-78-thread-3SUCCEEDED
cpt4WriteChildParent-HCP_cpt4_default0.08pool-78-thread-2SUCCEEDED
cpt4WriteEntityAttributes-ENT_cpt40.04pool-53-thread-1SUCCEEDED
cpt4WriteEntityLevelDisplayHints-ELDH_cpt40.35pool-63-thread-2SUCCEEDED
cpt4WriteNumChildrenAndPaths-ENT_cpt43.49pool-1226-thread-1SUCCEEDED
cpt4WriteTextSearchField-ENT_cpt40.07pool-78-thread-1SUCCEEDED
cpt4OccurrenceCreateEntityMain-ENT_cpt4Occurrence0.01pool-24-thread-1SUCCEEDED
cpt4OccurrenceValidateDataTypes-ENT_cpt4Occurrence0.02pool-15-thread-1SUCCEEDED
cpt4OccurrenceValidateUniqueIds-ENT_cpt4Occurrence0.06pool-77-thread-1SUCCEEDED
cpt4OccurrenceWriteEntityAttributes-ENT_cpt4Occurrence0.22pool-51-thread-1SUCCEEDED
cpt4OccurrenceWriteEntityLevelDisplayHints-ELDH_cpt4Occurrence0.35pool-77-thread-2SUCCEEDED
drugCreateEntityMain-ENT_drug0.01pool-26-thread-1SUCCEEDED
drugValidateDataTypes-ENT_drug0.02pool-19-thread-1SUCCEEDED
drugValidateUniqueIds-ENT_drug0.01pool-61-thread-1SUCCEEDED
drugWriteEntityAttributes-ENT_drug0.04pool-40-thread-1SUCCEEDED
drugWriteEntityLevelDisplayHints-ELDH_drug0.36pool-61-thread-2SUCCEEDED
drugWriteTextSearchField-ENT_drug0.06pool-80-thread-1SUCCEEDED
drugOccurrenceCreateEntityMain-ENT_drugOccurrence0.01pool-25-thread-1SUCCEEDED
drugOccurrenceValidateDataTypes-ENT_drugOccurrence0.02pool-17-thread-1SUCCEEDED
drugOccurrenceValidateUniqueIds-ENT_drugOccurrence0.03pool-67-thread-1SUCCEEDED
drugOccurrenceWriteEntityAttributes-ENT_drugOccurrence0.06pool-49-thread-1SUCCEEDED
drugOccurrenceWriteEntityLevelDisplayHints-ELDH_drugOccurrence0.35pool-67-thread-2SUCCEEDED
heightCreateEntityMain-ENT_height0.01pool-22-thread-1SUCCEEDED
heightValidateDataTypes-ENT_height0.02pool-7-thread-1SUCCEEDED
heightValidateUniqueIds-ENT_height0.04pool-71-thread-1SUCCEEDED
heightWriteEntityAttributes-ENT_height0.07pool-50-thread-1SUCCEEDED
heightWriteEntityLevelDisplayHints-ELDH_height0.36pool-71-thread-2SUCCEEDED
icd10cmCreateEntityMain-ENT_icd10cm0.01pool-39-thread-1SUCCEEDED
icd10cmValidateDataTypes-ENT_icd10cm0.02pool-8-thread-1SUCCEEDED
icd10cmValidateUniqueIds-ENT_icd10cm0.01pool-70-thread-1SUCCEEDED
icd10cmWriteAncestorDescendant-HAD_icd10cm_default5.51pool-96-thread-3SUCCEEDED
icd10cmWriteChildParent-HCP_icd10cm_default0.07pool-96-thread-2SUCCEEDED
icd10cmWriteEntityAttributes-ENT_icd10cm0.07pool-48-thread-1SUCCEEDED
icd10cmWriteEntityLevelDisplayHints-ELDH_icd10cm0.36pool-70-thread-2SUCCEEDED
icd10cmWriteNumChildrenAndPaths-ENT_icd10cm4.05pool-1245-thread-1SUCCEEDED
icd10cmWriteTextSearchField-ENT_icd10cm0.08pool-96-thread-1SUCCEEDED
icd10cmOccurrenceCreateEntityMain-ENT_icd10cmOccurrence0.01pool-32-thread-1SUCCEEDED
icd10cmOccurrenceValidateDataTypes-ENT_icd10cmOccurrence0.02pool-2-thread-1SUCCEEDED
icd10cmOccurrenceValidateUniqueIds-ENT_icd10cmOccurrence0.05pool-74-thread-1SUCCEEDED
icd10cmOccurrenceWriteEntityAttributes-ENT_icd10cmOccurrence0.07pool-52-thread-1SUCCEEDED
icd10cmOccurrenceWriteEntityLevelDisplayHints-ELDH_icd10cmOccurrence0.35pool-74-thread-2SUCCEEDED
icd10pcsCreateEntityMain-ENT_icd10pcs0.01pool-31-thread-1SUCCEEDED
icd10pcsValidateDataTypes-ENT_icd10pcs0.02pool-20-thread-1SUCCEEDED
icd10pcsValidateUniqueIds-ENT_icd10pcs0.01pool-75-thread-1SUCCEEDED
icd10pcsWriteAncestorDescendant-HAD_icd10pcs_default6.23pool-97-thread-3SUCCEEDED
icd10pcsWriteChildParent-HCP_icd10pcs_default0.10pool-97-thread-2SUCCEEDED
icd10pcsWriteEntityAttributes-ENT_icd10pcs0.08pool-58-thread-1SUCCEEDED
icd10pcsWriteEntityLevelDisplayHints-ELDH_icd10pcs0.35pool-75-thread-2SUCCEEDED
icd10pcsWriteNumChildrenAndPaths-ENT_icd10pcs4.25pool-1288-thread-1SUCCEEDED
icd10pcsWriteTextSearchField-ENT_icd10pcs0.09pool-97-thread-1SUCCEEDED
icd10pcsOccurrenceCreateEntityMain-ENT_icd10pcsOccurrence0.01pool-33-thread-1SUCCEEDED
icd10pcsOccurrenceValidateDataTypes-ENT_icd10pcsOccurrence0.02pool-12-thread-1SUCCEEDED
icd10pcsOccurrenceValidateUniqueIds-ENT_icd10pcsOccurrence0.01pool-66-thread-1SUCCEEDED
icd10pcsOccurrenceWriteEntityAttributes-ENT_icd10pcsOccurrence0.05pool-56-thread-1SUCCEEDED
icd10pcsOccurrenceWriteEntityLevelDisplayHints-ELDH_icd10pcsOccurrence0.35pool-66-thread-2SUCCEEDED
icd9CreateEntityMain-ENT_icd90.01pool-21-thread-1SUCCEEDED
icd9ValidateDataTypes-ENT_icd90.02pool-5-thread-1SUCCEEDED
icd9ValidateUniqueIds-ENT_icd90.01pool-65-thread-1SUCCEEDED
icd9WriteAncestorDescendant-HAD_icd9_default5.51pool-83-thread-3SUCCEEDED
icd9WriteChildParent-HCP_icd9_default0.04pool-83-thread-2SUCCEEDED
icd9WriteEntityAttributes-ENT_icd90.05pool-57-thread-1SUCCEEDED
icd9WriteEntityLevelDisplayHints-ELDH_icd90.35pool-65-thread-2SUCCEEDED
icd9WriteNumChildrenAndPaths-ENT_icd94.17pool-1234-thread-1SUCCEEDED
icd9WriteTextSearchField-ENT_icd90.07pool-83-thread-1SUCCEEDED
icd9OccurrenceCreateEntityMain-ENT_icd9Occurrence0.01pool-28-thread-1SUCCEEDED
icd9OccurrenceValidateDataTypes-ENT_icd9Occurrence0.02pool-4-thread-1SUCCEEDED
icd9OccurrenceValidateUniqueIds-ENT_icd9Occurrence0.05pool-76-thread-1SUCCEEDED
icd9OccurrenceWriteEntityAttributes-ENT_icd9Occurrence0.09pool-54-thread-1SUCCEEDED
icd9OccurrenceWriteEntityLevelDisplayHints-ELDH_icd9Occurrence0.35pool-76-thread-2SUCCEEDED
labCreateEntityMain-ENT_lab0.01pool-29-thread-1SUCCEEDED
labValidateDataTypes-ENT_lab0.02pool-16-thread-1SUCCEEDED
labValidateUniqueIds-ENT_lab0.01pool-60-thread-1SUCCEEDED
labWriteEntityAttributes-ENT_lab0.04pool-41-thread-1SUCCEEDED
labWriteEntityLevelDisplayHints-ELDH_lab0.36pool-60-thread-2SUCCEEDED
labWriteTextSearchField-ENT_lab0.06pool-79-thread-1SUCCEEDED
labOccurrenceCreateEntityMain-ENT_labOccurrence0.01pool-35-thread-1SUCCEEDED
labOccurrenceValidateDataTypes-ENT_labOccurrence0.02pool-6-thread-1SUCCEEDED
labOccurrenceValidateUniqueIds-ENT_labOccurrence0.06pool-72-thread-1SUCCEEDED
labOccurrenceWriteEntityAttributes-ENT_labOccurrence0.07pool-45-thread-1SUCCEEDED
labOccurrenceWriteEntityLevelDisplayHints-ELDH_labOccurrence0.35pool-72-thread-2SUCCEEDED
personCreateEntityMain-ENT_person0.01pool-34-thread-1SUCCEEDED
personValidateDataTypes-ENT_person0.02pool-18-thread-1SUCCEEDED
personValidateUniqueIds-ENT_person0.01pool-64-thread-1SUCCEEDED
personWriteEntityAttributes-ENT_person0.04pool-44-thread-1SUCCEEDED
personWriteEntityLevelDisplayHints-ELDH_person0.40pool-64-thread-2SUCCEEDED
phenotypeCreateEntityMain-ENT_phenotype0.01pool-37-thread-1SUCCEEDED
phenotypeValidateDataTypes-ENT_phenotype0.02pool-9-thread-1SUCCEEDED
phenotypeValidateUniqueIds-ENT_phenotype0.01pool-62-thread-1SUCCEEDED
phenotypeWriteAncestorDescendant-HAD_phenotype_default5.38pool-82-thread-3SUCCEEDED
phenotypeWriteChildParent-HCP_phenotype_default0.04pool-82-thread-2SUCCEEDED
phenotypeWriteEntityAttributes-ENT_phenotype0.04pool-47-thread-1SUCCEEDED
phenotypeWriteEntityLevelDisplayHints-ELDH_phenotype0.36pool-62-thread-2SUCCEEDED
phenotypeWriteNumChildrenAndPaths-ENT_phenotype3.56pool-1222-thread-1SUCCEEDED
phenotypeWriteTextSearchField-ENT_phenotype0.06pool-82-thread-1SUCCEEDED
phenotypeOccurrenceCreateEntityMain-ENT_phenotypeOccurrence0.01pool-23-thread-1SUCCEEDED
phenotypeOccurrenceValidateDataTypes-ENT_phenotypeOccurrence0.02pool-14-thread-1SUCCEEDED
phenotypeOccurrenceValidateUniqueIds-ENT_phenotypeOccurrence0.01pool-73-thread-1SUCCEEDED
phenotypeOccurrenceWriteEntityAttributes-ENT_phenotypeOccurrence0.07pool-55-thread-1SUCCEEDED
phenotypeOccurrenceWriteEntityLevelDisplayHints-ELDH_phenotypeOccurrence0.35pool-73-thread-2SUCCEEDED
phewasCreateEntityMain-ENT_phewas0.01pool-30-thread-1SUCCEEDED
phewasValidateDataTypes-ENT_phewas0.02pool-10-thread-1SUCCEEDED
phewasValidateUniqueIds-ENT_phewas0.01pool-59-thread-1SUCCEEDED
phewasWriteAncestorDescendant-HAD_phewas_default5.51pool-81-thread-3SUCCEEDED
phewasWriteChildParent-HCP_phewas_default0.03pool-81-thread-2SUCCEEDED
phewasWriteEntityAttributes-ENT_phewas0.04pool-43-thread-1SUCCEEDED
phewasWriteEntityLevelDisplayHints-ELDH_phewas0.36pool-59-thread-2SUCCEEDED
phewasWriteNumChildrenAndPaths-ENT_phewas4.03pool-1236-thread-1SUCCEEDED
phewasWriteTextSearchField-ENT_phewas0.06pool-81-thread-1SUCCEEDED
weightCreateEntityMain-ENT_weight0.01pool-36-thread-1SUCCEEDED
weightValidateDataTypes-ENT_weight0.02pool-13-thread-1SUCCEEDED
weightValidateUniqueIds-ENT_weight0.05pool-69-thread-1SUCCEEDED
weightWriteEntityAttributes-ENT_weight0.07pool-46-thread-1SUCCEEDED
weightWriteEntityLevelDisplayHints-ELDH_weight0.37pool-69-thread-2SUCCEEDED
bmiPersonWriteRollupCounts-bmiPerson-ENT_person-NOHIER7.08pool-1305-thread-1SUCCEEDED
cpt4PersonWriteRelationshipIntermediateTable-RIDS_cpt4Person_person_cpt40.36pool-1308-thread-1SUCCEEDED
cpt4PersonWriteRollupCounts-cpt4Person-ENT_cpt4-NOHIER6.30pool-1439-thread-1SUCCEEDED
cpt4PersonWriteRollupCounts-cpt4Person-ENT_cpt4-default6.33pool-1439-thread-2SUCCEEDED
drugPersonWriteRelationshipIntermediateTable-RIDS_drugPerson_person_drug0.22pool-1313-thread-1SUCCEEDED
drugPersonWriteRollupCounts-drugPerson-ENT_drug-NOHIER6.55pool-1361-thread-1SUCCEEDED
heightPersonWriteRollupCounts-heightPerson-ENT_person-NOHIER7.11pool-1303-thread-1SUCCEEDED
icd10cmPersonWriteRelationshipIntermediateTable-RIDS_icd10cmPerson_person_icd10cm0.29pool-1306-thread-1SUCCEEDED
icd10cmPersonWriteRollupCounts-icd10cmPerson-ENT_icd10cm-NOHIER6.43pool-1397-thread-1SUCCEEDED
icd10cmPersonWriteRollupCounts-icd10cmPerson-ENT_icd10cm-default6.38pool-1397-thread-2SUCCEEDED
icd10pcsPersonWriteRelationshipIntermediateTable-RIDS_icd10pcsPerson_person_icd10pcs0.06pool-1307-thread-1SUCCEEDED
icd10pcsPersonWriteRollupCounts-icd10pcsPerson-ENT_icd10pcs-NOHIER7.03pool-1323-thread-1SUCCEEDED
icd10pcsPersonWriteRollupCounts-icd10pcsPerson-ENT_icd10pcs-default7.15pool-1323-thread-2SUCCEEDED
icd9PersonWriteRelationshipIntermediateTable-RIDS_icd9Person_person_icd90.33pool-1310-thread-1SUCCEEDED
icd9PersonWriteRollupCounts-icd9Person-ENT_icd9-NOHIER6.34pool-1413-thread-1SUCCEEDED
icd9PersonWriteRollupCounts-icd9Person-ENT_icd9-default6.36pool-1413-thread-2SUCCEEDED
labPersonWriteInstanceLevelDisplayHints-ILDH_labOccurrence_lab6.00pool-1378-thread-2FAILED
labPersonWriteRelationshipIntermediateTable-RIDS_labPerson_person_lab0.26pool-1311-thread-1SUCCEEDED
labPersonWriteRollupCounts-labPerson-ENT_lab-NOHIER6.40pool-1378-thread-1SUCCEEDED
phenotypePersonWriteRelationshipIntermediateTable-RIDS_phenotypePerson_person_phenotype0.23pool-1309-thread-1SUCCEEDED
phenotypePersonWriteRollupCounts-phenotypePerson-ENT_phenotype-NOHIER6.43pool-1362-thread-1SUCCEEDED
phenotypePersonWriteRollupCounts-phenotypePerson-ENT_phenotype-default6.45pool-1362-thread-2SUCCEEDED
phewasPersonWriteRelationshipIntermediateTable-RIDS_phewasPerson_icd9Occurrence_phewas0.35pool-1312-thread-1SUCCEEDED
phewasPersonWriteRelationshipIntermediateTable-RIDS_phewasPerson_person_phewas0.26pool-1312-thread-2SUCCEEDED
phewasPersonWriteRollupCounts-phewasPerson-ENT_phewas-NOHIER6.33pool-1435-thread-1SUCCEEDED
phewasPersonWriteRollupCounts-phewasPerson-ENT_phewas-default6.31pool-1435-thread-2SUCCEEDED
weightPersonWriteRollupCounts-weightPerson-ENT_person-NOHIER7.24pool-1304-thread-1SUCCEEDED
+



+

Stack Traces

+

WriteInstanceLevelDisplayHints-ILDH_labOccurrence_lab

+ 
java.lang.RuntimeException: java.net.UnknownHostException: dataflow.googleapis.com
+	at org.apache.beam.runners.dataflow.DataflowPipelineJob.waitUntilFinish(DataflowPipelineJob.java:220)
+	at org.apache.beam.runners.dataflow.DataflowPipelineJob.waitUntilFinish(DataflowPipelineJob.java:206)
+	at bio.terra.tanagra.indexing.job.dataflow.WriteInstanceLevelDisplayHints.run(WriteInstanceLevelDisplayHints.java:153)
+	at bio.terra.tanagra.indexing.jobexecutor.JobThread.execute(JobThread.java:59)
+	at bio.terra.tanagra.indexing.jobexecutor.JobThread.call(JobThread.java:35)
+	at bio.terra.tanagra.indexing.jobexecutor.JobThread.call(JobThread.java:9)
+	at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
+	at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
+	at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
+	at java.base/java.lang.Thread.run(Thread.java:840)
+Caused by: java.net.UnknownHostException: dataflow.googleapis.com
+	at java.base/sun.nio.ch.NioSocketImpl.connect(NioSocketImpl.java:572)
+	at java.base/java.net.SocksSocketImpl.connect(SocksSocketImpl.java:327)
+	at java.base/java.net.Socket.connect(Socket.java:633)
+	at java.base/sun.security.ssl.SSLSocketImpl.connect(SSLSocketImpl.java:304)
+	at java.base/sun.net.NetworkClient.doConnect(NetworkClient.java:178)
+	at java.base/sun.net.www.http.HttpClient.openServer(HttpClient.java:533)
+	at java.base/sun.net.www.http.HttpClient.openServer(HttpClient.java:638)
+	at java.base/sun.net.www.protocol.https.HttpsClient.(HttpsClient.java:266)
+	at java.base/sun.net.www.protocol.https.HttpsClient.New(HttpsClient.java:380)
+	at java.base/sun.net.www.protocol.https.AbstractDelegateHttpsURLConnection.getNewHttpClient(AbstractDelegateHttpsURLConnection.java:193)
+	at java.base/sun.net.www.protocol.http.HttpURLConnection.plainConnect0(HttpURLConnection.java:1241)
+	at java.base/sun.net.www.protocol.http.HttpURLConnection.plainConnect(HttpURLConnection.java:1127)
+	at java.base/sun.net.www.protocol.https.AbstractDelegateHttpsURLConnection.connect(AbstractDelegateHttpsURLConnection.java:179)
+	at java.base/sun.net.www.protocol.https.HttpsURLConnectionImpl.connect(HttpsURLConnectionImpl.java:142)
+	at com.google.api.client.http.javanet.NetHttpRequest.execute(NetHttpRequest.java:151)
+	at com.google.api.client.http.javanet.NetHttpRequest.execute(NetHttpRequest.java:84)
+	at com.google.api.client.http.HttpRequest.execute(HttpRequest.java:1012)
+	at com.google.api.client.googleapis.services.AbstractGoogleClientRequest.executeUnparsed(AbstractGoogleClientRequest.java:565)
+	at com.google.api.client.googleapis.services.AbstractGoogleClientRequest.executeUnparsed(AbstractGoogleClientRequest.java:506)
+	at com.google.api.client.googleapis.services.AbstractGoogleClientRequest.execute(AbstractGoogleClientRequest.java:616)
+	at org.apache.beam.runners.dataflow.DataflowClient.listJobMessages(DataflowClient.java:131)
+	at org.apache.beam.runners.dataflow.util.MonitoringUtil.getJobMessages(MonitoringUtil.java:137)
+	at org.apache.beam.runners.dataflow.DataflowPipelineJob.processJobMessages(DataflowPipelineJob.java:417)
+	at org.apache.beam.runners.dataflow.DataflowPipelineJob.waitUntilFinish(DataflowPipelineJob.java:334)
+	at org.apache.beam.runners.dataflow.DataflowPipelineJob.waitUntilFinish(DataflowPipelineJob.java:253)
+	at org.apache.beam.runners.dataflow.DataflowPipelineJob.waitUntilFinish(DataflowPipelineJob.java:212)
+	... 9 more
+
+ + diff --git a/underlay/src/main/resources/config/criteria/sd/criteriaselector/agdGenotypeResult/agdGenotypeResult.json b/underlay/src/main/resources/config/criteria/sd/criteriaselector/agdGenotypeResult/agdGenotypeResult.json new file mode 100644 index 000000000..81f0e0426 --- /dev/null +++ b/underlay/src/main/resources/config/criteria/sd/criteriaselector/agdGenotypeResult/agdGenotypeResult.json @@ -0,0 +1,3 @@ +{ + "attribute": "has_agd_genotype_result" +} diff --git a/underlay/src/main/resources/config/criteria/sd/criteriaselector/agdGenotypeResult/selector.json b/underlay/src/main/resources/config/criteria/sd/criteriaselector/agdGenotypeResult/selector.json new file mode 100644 index 000000000..e59676b76 --- /dev/null +++ b/underlay/src/main/resources/config/criteria/sd/criteriaselector/agdGenotypeResult/selector.json @@ -0,0 +1,15 @@ +{ + "name": "agdGenotypeResult", + "displayName": "AGD Data Availability", + "isEnabledForCohorts": true, + "isEnabledForDataFeatureSets": false, + "display": { + "category": "BioVU", + "tags": null + }, + "filterBuilder": "core.PrimaryEntityFilterBuilder", + "plugin": "attribute", + "pluginConfig": null, + "pluginConfigFile": "agdGenotypeResult.json", + "modifiers": null +} diff --git a/underlay/src/main/resources/config/criteria/sd/criteriaselector/deceased/deceased.json b/underlay/src/main/resources/config/criteria/sd/criteriaselector/deceased/deceased.json new file mode 100644 index 000000000..754c5e68a --- /dev/null +++ b/underlay/src/main/resources/config/criteria/sd/criteriaselector/deceased/deceased.json @@ -0,0 +1,3 @@ +{ + "attribute": "is_deceased" +} diff --git a/underlay/src/main/resources/config/criteria/sd/criteriaselector/deceased/selector.json b/underlay/src/main/resources/config/criteria/sd/criteriaselector/deceased/selector.json new file mode 100644 index 000000000..f198b5260 --- /dev/null +++ b/underlay/src/main/resources/config/criteria/sd/criteriaselector/deceased/selector.json @@ -0,0 +1,15 @@ +{ + "name": "tanagra-deceased", + "displayName": "Deceased", + "isEnabledForCohorts": true, + "isEnabledForDataFeatureSets": false, + "display": { + "category": "Demographics", + "tags": null + }, + "filterBuilder": "core.PrimaryEntityFilterBuilder", + "plugin": "attribute", + "pluginConfig": null, + "pluginConfigFile": "deceased.json", + "modifiers": null +} diff --git a/underlay/src/main/resources/config/datamapping/sd/entity/condition/all.sql b/underlay/src/main/resources/config/datamapping/sd/entity/condition/all.sql new file mode 100644 index 000000000..ffe331b95 --- /dev/null +++ b/underlay/src/main/resources/config/datamapping/sd/entity/condition/all.sql @@ -0,0 +1,10 @@ +SELECT + concept_id, + concept_name, + vocabulary_id, + concept_code, + (CASE WHEN standard_concept IS NULL THEN 'Source' WHEN standard_concept = 'S' THEN 'Standard' ELSE 'Unknown' END) AS standard_concept + +FROM `${omopDataset}.concept` + +WHERE domain_id = 'Condition' diff --git a/underlay/src/main/resources/config/datamapping/sd/entity/condition/childParent.sql b/underlay/src/main/resources/config/datamapping/sd/entity/condition/childParent.sql new file mode 100644 index 000000000..3d23a32d5 --- /dev/null +++ b/underlay/src/main/resources/config/datamapping/sd/entity/condition/childParent.sql @@ -0,0 +1,12 @@ +SELECT + cr.concept_id_1 AS parent, + cr.concept_id_2 AS child, +FROM `${omopDataset}.concept_relationship` cr +JOIN `${omopDataset}.concept` c1 ON c1.concept_id = cr.concept_id_1 +JOIN `${omopDataset}.concept` c2 ON c2.concept_id = cr.concept_id_2 +WHERE + cr.relationship_id = 'Subsumes' + AND c1.domain_id = c2.domain_id + AND c2.domain_id = 'Condition' + AND c1.vocabulary_id = c2.vocabulary_id + AND c2.vocabulary_id = 'SNOMED' diff --git a/underlay/src/main/resources/config/datamapping/sd/entity/condition/entity.json b/underlay/src/main/resources/config/datamapping/sd/entity/condition/entity.json new file mode 100644 index 000000000..214d60b94 --- /dev/null +++ b/underlay/src/main/resources/config/datamapping/sd/entity/condition/entity.json @@ -0,0 +1,25 @@ +{ + "name": "condition", + "allInstancesSqlFile": "all.sql", + "attributes": [ + { "name": "id", "dataType": "INT64", "valueFieldName": "concept_id" }, + { "name": "name", "dataType": "STRING", "valueFieldName": "concept_name" }, + { "name": "vocabulary", "dataType": "STRING", "valueFieldName": "vocabulary_id", "isComputeDisplayHint": true }, + { "name": "standard_concept", "dataType": "STRING", "isComputeDisplayHint": true }, + { "name": "concept_code", "dataType": "STRING" } + ], + "idAttribute": "id", + "textSearch": { + "attributes": [ "id", "name", "concept_code" ] + }, + "hierarchies": [ + { + "childParentIdPairsSqlFile": "childParent.sql", + "childIdFieldName": "child", + "parentIdFieldName": "parent", + "rootNodeIds": [ 441840 ], + "maxDepth": 20, + "keepOrphanNodes": false + } + ] +} diff --git a/underlay/src/main/resources/config/datamapping/sd/entity/conditionOccurrence/all.sql b/underlay/src/main/resources/config/datamapping/sd/entity/conditionOccurrence/all.sql index 064e09e3b..2710ad386 100644 --- a/underlay/src/main/resources/config/datamapping/sd/entity/conditionOccurrence/all.sql +++ b/underlay/src/main/resources/config/datamapping/sd/entity/conditionOccurrence/all.sql @@ -1,4 +1,4 @@ -SELECT +SELECT DISTINCT co.condition_occurrence_id, co.person_id, p.person_source_value, diff --git a/underlay/src/main/resources/config/datamapping/sd/entity/measurementOccurrence/all.sql b/underlay/src/main/resources/config/datamapping/sd/entity/measurementOccurrence/all.sql index af2ed5bf9..207f5f607 100644 --- a/underlay/src/main/resources/config/datamapping/sd/entity/measurementOccurrence/all.sql +++ b/underlay/src/main/resources/config/datamapping/sd/entity/measurementOccurrence/all.sql @@ -1,4 +1,4 @@ -SELECT +SELECT DISTINCT mo.measurement_id, mo.person_id, p.person_source_value, diff --git a/underlay/src/main/resources/config/datamapping/sd/entity/observationOccurrence/all.sql b/underlay/src/main/resources/config/datamapping/sd/entity/observationOccurrence/all.sql index 168a1bdfc..cdcb8e962 100644 --- a/underlay/src/main/resources/config/datamapping/sd/entity/observationOccurrence/all.sql +++ b/underlay/src/main/resources/config/datamapping/sd/entity/observationOccurrence/all.sql @@ -1,4 +1,4 @@ -SELECT +SELECT DISTINCT o.observation_id, o.person_id, p.person_source_value, diff --git a/underlay/src/main/resources/config/datamapping/sd/entity/person/all.sql b/underlay/src/main/resources/config/datamapping/sd/entity/person/all.sql index f86c5a7e9..995a5d585 100644 --- a/underlay/src/main/resources/config/datamapping/sd/entity/person/all.sql +++ b/underlay/src/main/resources/config/datamapping/sd/entity/person/all.sql @@ -15,7 +15,11 @@ SELECT CASE WHEN x.nonshippable_ind = '1' THEN true WHEN x.nonshippable_ind = '0' THEN false ELSE null END AS biovu_sample_is_nonshippable, CASE WHEN x.plasma_ind = '1' THEN true WHEN x.plasma_ind = '0' THEN false ELSE null END AS biovu_sample_has_plasma, EXISTS - (SELECT 1 FROM `${omopDataset}.x_agd_queue` aq WHERE p.person_id = aq.person_id) AS has_agd_queue + (SELECT 1 FROM `${omopDataset}.x_agd_queue` aq WHERE p.person_id = aq.person_id) AS has_agd_queue, + EXISTS + (SELECT 1 FROM `${omopDataset}.genotype_result` gr WHERE p.person_id = gr.person_id + AND gr.assay_name = 'agd whole genome sequencing' ) AS has_agd_genotype_result, + CASE WHEN d.death_date is null THEN true ELSE false END AS is_deceased FROM `${omopDataset}.person` p @@ -28,6 +32,10 @@ ON rc.concept_id = p.race_source_concept_id LEFT JOIN `${omopDataset}.concept` ec ON ec.concept_id = p.ethnicity_concept_id +LEFT JOIN (SELECT person_id, max(death_date) as death_date + FROM `${omopDataset}.death` GROUP BY person_id) d + ON (p.person_id = d.person_id) + LEFT OUTER JOIN ( /* Get rid of duplicate rows in x_biovu_sample_status. For example, person diff --git a/underlay/src/main/resources/config/datamapping/sd/entity/person/entity.json b/underlay/src/main/resources/config/datamapping/sd/entity/person/entity.json index f27e4a026..045f2eab4 100644 --- a/underlay/src/main/resources/config/datamapping/sd/entity/person/entity.json +++ b/underlay/src/main/resources/config/datamapping/sd/entity/person/entity.json @@ -17,7 +17,9 @@ { "name": "biovu_sample_is_compromised", "dataType": "BOOLEAN" }, { "name": "biovu_sample_is_nonshippable", "dataType": "BOOLEAN" }, { "name": "biovu_sample_has_plasma", "dataType": "BOOLEAN" }, - { "name": "has_agd_queue", "dataType": "BOOLEAN" } + { "name": "has_agd_queue", "dataType": "BOOLEAN" }, + { "name": "has_agd_genotype_result", "dataType": "BOOLEAN" }, + { "name": "is_deceased", "dataType": "BOOLEAN" } ], "idAttribute": "id", "optimizeGroupByAttributes": [ "gender", "race", "age" ] diff --git a/underlay/src/main/resources/config/indexer/emerge/emerge_20240603.json b/underlay/src/main/resources/config/indexer/emerge/emerge_20240603.json index cbeb498a6..5fa9c4c05 100644 --- a/underlay/src/main/resources/config/indexer/emerge/emerge_20240603.json +++ b/underlay/src/main/resources/config/indexer/emerge/emerge_20240603.json @@ -20,6 +20,7 @@ "serviceAccountEmail": "emerge-dev-dataflow-indexer@vumc-emerge-dev.iam.gserviceaccount.com", "gcsTempDirectory": "gs://dataflow-indexing-emerge-dev/temp/", "workerMachineType": "n1-standard-4", - "usePublicIps": false + "usePublicIps": false, + "vpcSubnetworkName": "default-dataflow" } } diff --git a/underlay/src/main/resources/config/indexer/sd/sd020240831.json b/underlay/src/main/resources/config/indexer/sd/sd020240831.json index 1332922fc..9ad8fcf47 100644 --- a/underlay/src/main/resources/config/indexer/sd/sd020240831.json +++ b/underlay/src/main/resources/config/indexer/sd/sd020240831.json @@ -11,7 +11,7 @@ }, "indexData": { "projectId": "vumc-sd-dev", - "datasetId": "indexed_sd_20240831_1" + "datasetId": "indexed_sd_20240831_2" }, "queryProjectId": "vumc-sd-dev", "dataLocation": "us-central1" @@ -20,6 +20,7 @@ "serviceAccountEmail": "sd-dev-dataflow-indexer@vumc-sd-dev.iam.gserviceaccount.com", "gcsTempDirectory": "gs://dataflow-indexing-sd-dev/temp/", "workerMachineType": "n1-standard-4", - "usePublicIps": false + "usePublicIps": false, + "vpcSubnetworkName": "us-central1" } } diff --git a/underlay/src/main/resources/config/service/sd/sd_local.json b/underlay/src/main/resources/config/service/sd/sd_local.json index 3759e5220..04b4d205b 100644 --- a/underlay/src/main/resources/config/service/sd/sd_local.json +++ b/underlay/src/main/resources/config/service/sd/sd_local.json @@ -2,16 +2,16 @@ "underlay": "sd", "bigQuery": { "sourceData": { - "projectId": "vumc-sd-test", - "datasetId": "sd_20230831", + "projectId": "vumc-sd-dev", + "datasetId": "sd_20240831", "sqlSubstitutions": { - "omopDataset": "vumc-sd-test.sd_20230831", - "staticTablesDataset": "vumc-sd-test.sd_20230831" + "omopDataset": "vumc-sd-test.sd_20240831", + "staticTablesDataset": "vumc-sd-test.sd_20240831" } }, "indexData": { - "projectId": "vumc-sd-test", - "datasetId": "indexed_sd_20230831" + "projectId": "vumc-sd-dev", + "datasetId": "indexed_sd_20240831_2" }, "queryProjectId": "vumc-sd-test", "dataLocation": "us-central1" diff --git a/underlay/src/main/resources/config/underlay/sd/underlay.json b/underlay/src/main/resources/config/underlay/sd/underlay.json index 099cb0b38..a76fd6b1b 100644 --- a/underlay/src/main/resources/config/underlay/sd/underlay.json +++ b/underlay/src/main/resources/config/underlay/sd/underlay.json @@ -4,7 +4,7 @@ "entities": [ "sd/person", - "omop/condition", + "sd/condition", "sd/conditionOccurrence", "omop/procedure", "sd/procedureOccurrence", @@ -87,9 +87,11 @@ "sd/gender", "sd/race", "sd/age", + "sd/deceased", "sd/bioVU", "sd/bioVUPlasma", "sd/agdSubjectQueue", + "sd/agdGenotypeResult", "sd/genotyping", "sd/measurement", "sd/documents",