-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request #17 from umccr/implement-psa-schema-spreadsheet-li…
…brary-tracking-metadata Implemented OrcaVault psa schema
- Loading branch information
Showing
8 changed files
with
276 additions
and
2 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,7 @@ | ||
| #!/usr/bin/env bash | ||
|
|
||
| PGPASSWORD=dev psql -h 0.0.0.0 -d orcavault -U dev -c 'SELECT tsa.truncate_tables();' | ||
|
|
||
| PGPASSWORD=dev psql -h 0.0.0.0 -d orcavault -U dev <<EOF | ||
| \copy tsa.spreadsheet_library_tracking_metadata from '/data/orcavault_tsa_spreadsheet_library_tracking_metadata.next.csv' with (format csv, header true, delimiter ','); | ||
| EOF |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,38 @@ | ||
| SELECT current_database(); | ||
|
|
||
| -- create psa schema | ||
| CREATE SCHEMA IF NOT EXISTS psa AUTHORIZATION dev; | ||
| SET search_path TO psa; | ||
|
|
||
| SELECT current_schema(); | ||
|
|
||
| CREATE TABLE IF NOT EXISTS orcavault.psa.spreadsheet_library_tracking_metadata | ||
| ( | ||
| assay varchar, | ||
| comments varchar, | ||
| coverage varchar, | ||
| experiment_id varchar, | ||
| external_sample_id varchar, | ||
| external_subject_id varchar, | ||
| library_id varchar, | ||
| override_cycles varchar, | ||
| phenotype varchar, | ||
| project_name varchar, | ||
| project_owner varchar, | ||
| qpcr_id varchar, | ||
| quality varchar, | ||
| run varchar, | ||
| sample_id varchar, | ||
| sample_name varchar, | ||
| samplesheet_sample_id varchar, | ||
| source varchar, | ||
| subject_id varchar, | ||
| truseq_index varchar, | ||
| type varchar, | ||
| workflow varchar, | ||
| r_rna varchar, | ||
| study varchar, | ||
| sheet_name varchar, | ||
| load_datetime timestamptz, | ||
| record_source varchar(255) | ||
| ); |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,60 @@ | ||
| version: 2 | ||
|
|
||
| models: | ||
|
|
||
| - name: spreadsheet_library_tracking_metadata | ||
| columns: | ||
| - name: assay | ||
| data_type: varchar | ||
| - name: comments | ||
| data_type: varchar | ||
| - name: coverage | ||
| data_type: varchar | ||
| - name: experiment_id | ||
| data_type: varchar | ||
| - name: external_sample_id | ||
| data_type: varchar | ||
| - name: external_subject_id | ||
| data_type: varchar | ||
| - name: library_id | ||
| data_type: varchar | ||
| - name: override_cycles | ||
| data_type: varchar | ||
| - name: phenotype | ||
| data_type: varchar | ||
| - name: project_name | ||
| data_type: varchar | ||
| - name: project_owner | ||
| data_type: varchar | ||
| - name: qpcr_id | ||
| data_type: varchar | ||
| - name: quality | ||
| data_type: varchar | ||
| - name: run | ||
| data_type: varchar | ||
| - name: sample_id | ||
| data_type: varchar | ||
| - name: sample_name | ||
| data_type: varchar | ||
| - name: samplesheet_sample_id | ||
| data_type: varchar | ||
| - name: source | ||
| data_type: varchar | ||
| - name: subject_id | ||
| data_type: varchar | ||
| - name: truseq_index | ||
| data_type: varchar | ||
| - name: type | ||
| data_type: varchar | ||
| - name: workflow | ||
| data_type: varchar | ||
| - name: r_rna | ||
| data_type: varchar | ||
| - name: study | ||
| data_type: varchar | ||
| - name: sheet_name | ||
| data_type: varchar | ||
| - name: load_datetime | ||
| data_type: timestamptz | ||
| - name: record_source | ||
| data_type: varchar(255) |
123 changes: 123 additions & 0 deletions
123
orcavault/models/psa/spreadsheet_library_tracking_metadata.sql
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,123 @@ | ||
| {{ | ||
| config( | ||
| materialized='incremental' | ||
| ) | ||
| }} | ||
|
|
||
| with source as ( | ||
|
|
||
| select * from {{ source('tsa', 'spreadsheet_library_tracking_metadata') }} | ||
|
|
||
| ), | ||
|
|
||
| cleaned as ( | ||
|
|
||
| select | ||
| trim(regexp_replace(assay, E'[\\n\\r]+', '', 'g')) as assay, | ||
| trim(regexp_replace(comments, E'[\\n\\r]+', '', 'g')) as comments, | ||
| trim(regexp_replace(coverage, E'[\\n\\r]+', '', 'g')) as coverage, | ||
| trim(regexp_replace(experiment_id, E'[\\n\\r]+', '', 'g')) as experiment_id, | ||
| trim(regexp_replace(external_sample_id, E'[\\n\\r]+', '', 'g')) as external_sample_id, | ||
| trim(regexp_replace(external_subject_id, E'[\\n\\r]+', '', 'g')) as external_subject_id, | ||
| trim(regexp_replace(library_id, E'[\\n\\r]+', '', 'g')) as library_id, | ||
| trim(regexp_replace(override_cycles, E'[\\n\\r]+', '', 'g')) as override_cycles, | ||
| trim(regexp_replace(phenotype, E'[\\n\\r]+', '', 'g')) as phenotype, | ||
| trim(regexp_replace(project_name, E'[\\n\\r]+', '', 'g')) as project_name, | ||
| trim(regexp_replace(project_owner, E'[\\n\\r]+', '', 'g')) as project_owner, | ||
| trim(regexp_replace(qpcr_id, E'[\\n\\r]+', '', 'g')) as qpcr_id, | ||
| trim(regexp_replace(quality, E'[\\n\\r]+', '', 'g')) as quality, | ||
| trim(regexp_replace(run, E'[\\n\\r]+', '', 'g')) as run, | ||
| trim(regexp_replace(sample_id, E'[\\n\\r]+', '', 'g')) as sample_id, | ||
| trim(regexp_replace(sample_name, E'[\\n\\r]+', '', 'g')) as sample_name, | ||
| trim(regexp_replace(samplesheet_sample_id, E'[\\n\\r]+', '', 'g')) as samplesheet_sample_id, | ||
| trim(regexp_replace(source, E'[\\n\\r]+', '', 'g')) as source, | ||
| trim(regexp_replace(subject_id, E'[\\n\\r]+', '', 'g')) as subject_id, | ||
| trim(regexp_replace(truseq_index, E'[\\n\\r]+', '', 'g')) as truseq_index, | ||
| trim(regexp_replace(type, E'[\\n\\r]+', '', 'g')) as type, | ||
| trim(regexp_replace(workflow, E'[\\n\\r]+', '', 'g')) as workflow, | ||
| trim(regexp_replace(r_rna, E'[\\n\\r]+', '', 'g')) as r_rna, | ||
| trim(regexp_replace(study, E'[\\n\\r]+', '', 'g')) as study, | ||
| trim(regexp_replace(sheet_name, E'[\\n\\r]+', '', 'g')) as sheet_name | ||
| from | ||
| source | ||
| where | ||
| coalesce | ||
| ( | ||
| nullif(assay, ''), | ||
| nullif(comments, ''), | ||
| nullif(coverage, ''), | ||
| nullif(experiment_id, ''), | ||
| nullif(external_sample_id, ''), | ||
| nullif(external_subject_id, ''), | ||
| nullif(library_id, ''), | ||
| nullif(override_cycles, ''), | ||
| nullif(phenotype, ''), | ||
| nullif(project_name, ''), | ||
| nullif(project_owner, ''), | ||
| nullif(qpcr_id, ''), | ||
| nullif(quality, ''), | ||
| nullif(run, ''), | ||
| nullif(sample_id, ''), | ||
| nullif(sample_name, ''), | ||
| nullif(samplesheet_sample_id, ''), | ||
| nullif(source, ''), | ||
| nullif(subject_id, ''), | ||
| nullif(truseq_index, ''), | ||
| nullif(type, ''), | ||
| nullif(workflow, ''), | ||
| nullif(r_rna, ''), | ||
| nullif(study, '') | ||
| ) is not null | ||
|
|
||
| ), | ||
|
|
||
| transformed as ( | ||
|
|
||
| select | ||
| * | ||
| from | ||
| cleaned | ||
| except | ||
| select | ||
| assay, | ||
| comments, | ||
| coverage, | ||
| experiment_id, | ||
| external_sample_id, | ||
| external_subject_id, | ||
| library_id, | ||
| override_cycles, | ||
| phenotype, | ||
| project_name, | ||
| project_owner, | ||
| qpcr_id, | ||
| quality, | ||
| run, | ||
| sample_id, | ||
| sample_name, | ||
| samplesheet_sample_id, | ||
| source, | ||
| subject_id, | ||
| truseq_index, | ||
| type, | ||
| workflow, | ||
| r_rna, | ||
| study, | ||
| sheet_name | ||
| from | ||
| {{ this }} | ||
|
|
||
| ), | ||
|
|
||
| final as ( | ||
|
|
||
| select | ||
| *, | ||
| cast('{{ run_started_at }}' as timestamptz) as load_datetime, | ||
| (select 'UMCCR_Library_Tracking_MetaData') as record_source | ||
| from | ||
| transformed | ||
|
|
||
| ) | ||
|
|
||
| select * from final |