Commit 61dd6b7d authored by Sebastian Mate's avatar Sebastian Mate
Browse files

GECCO: Update ETL for Observation

parent 3d3ebe0f
......@@ -17,20 +17,19 @@ truncate table i2b2miracum.FhirFields;
-- List all resource types:
select distinct resourcetype from i2b2miracum.FhirFields;
select distinct systempath from i2b2miracum.FhirFields;
order by systempath asc;;
select distinct systempath from i2b2miracum.FhirFields where resourcetype = 'Condition' order by systempath asc;
select distinct systempath from i2b2miracum.FhirFields where resourcetype = 'Procedure' order by systempath asc;
select distinct systempath from i2b2miracum.FhirFields where resourcetype = 'Observation' order by systempath asc;
select distinct systempath from i2b2miracum.FhirFields where resourcetype = 'MedicationStatement' order by systempath asc;
select distinct systempath from i2b2miracum.FhirFields where resourcetype = 'DiagnosticReport' order by systempath asc;
select distinct systempath from i2b2miracum.FhirFields where resourcetype = 'Consent' order by systempath asc;
select distinct systempath from i2b2miracum.FhirFields where resourcetype = 'Condition' order by systempath asc;
select distinct systempath from i2b2miracum.FhirFields where resourcetype = 'Immunization' order by systempath asc;
select distinct systempath from i2b2miracum.FhirFields where resourcetype = 'Observation' order by systempath asc;
select distinct systempath from i2b2miracum.FhirFields where resourcetype = 'Procedure' order by systempath asc;
with FhirTempTable (fhir2) as (select '{
"resourceType": "Procedure",
"id": "21d1509d40a698daddad8723861bf665",
"resourceType": "Observation",
"id": "354488c80a956289ad08c38e09dd55f9",
"meta": {
"profile": [ "https://www.netzwerk-universitaetsmedizin.de/fhir/StructureDefinition/respiratory-therapies" ]
"profile": [ "https://www.netzwerk-universitaetsmedizin.de/fhir/StructureDefinition/smoking-status" ]
},
"identifier": [ {
"system": "https://mein.krankenhaus.de/fhir/NamingSystem/procedureId",
"value": "2-GECCOVISIT.null-Form.therapie.1-therapie.beatmungstherapie_code.1-beatmungstherapie"
"system": "https://mein.krankenhaus.de/fhir/NamingSystem/observationId",
"value": "Option01-GECCOVISIT.null-Form.anamnese_risikofaktoren.1-anamnese_risikofaktoren.raucherstatus_code.1-raucherstatus_code"
} ],
"status": "in-progress",
"category": {
"status": "final",
"category": [ {
"coding": [ {
"system": "http://snomed.info/sct",
"code": "277132007",
"display": "Therapeutic procedure (procedure)"
"system": "http://terminology.hl7.org/CodeSystem/observation-category",
"code": "social-history",
"display": "Social History"
} ]
},
} ],
"code": {
"coding": [ {
"system": "http://snomed.info/sct",
"code": "40617009:425391005=129121000"
"system": "http://loinc.org",
"code": "72166-2"
} ]
},
"subject": {
"reference": "Patient/e7407eae9d4dc4cb8def9a0f96932c02"
"reference": "Patient/969ba73aa2f35a27d698cbbf6f3785fe"
},
"performedDateTime": "1970-01-01T00:00:00+00:00",
"_performedDateTime": {
"effectiveDateTime": "1970-01-01T00:00:00+00:00",
"_effectiveDateTime": {
"extension": [ {
"url": "http://hl7.org/fhir/StructureDefinition/data-absent-reason",
"valueCode": "unknown"
} ]
},
"usedCode": [ {
"valueCodeableConcept": {
"coding": [ {
"system": "http://snomed.info/sct",
"code": "129121000"
"system": "http://loinc.org",
"code": "LA18976-3"
} ]
} ]
}
}'::jsonb fhir)
select
(fhir2->>'id')::text as ID,
StartTimes, EndTimes,
MainSystem, MainCode,
ModifierType, ModifierSystem, ModifierCode
ModifierType, ModifierSystem, ModifierCode,
valueQuantity_value, valueQuantity_unit, valueQuantity_system, valueQuantity_code
from FhirTempTable
left join (select (fhir2->>'performedDateTime')::timestamp as StartTimes
from FhirTempTable
union
select (fhir2->>'recordedDate')::timestamp as StartTimes
from FhirTempTable
from FhirTempTable
union
select (fhir2->>'effectiveDateTime')::timestamp as StartTimes
from FhirTempTable
) as sq1 on true and StartTimes is not null
left join (select (fhir2->>'performedDateTime')::timestamp as EndTimes
from FhirTempTable
union
select (fhir2->>'recordedDate')::timestamp as EndTimes
from FhirTempTable
union
select (fhir2->>'effectiveDateTime')::timestamp as EndTimes
from FhirTempTable) as sq2 on true and EndTimes is not null
left join (select jsonb_array_elements(fhir2->'code'->'coding')->>'code' as MainCode,
jsonb_array_elements(fhir2->'code'->'coding')->>'system' as MainSystem
......@@ -64,16 +71,11 @@ with FhirTempTable (fhir2) as (select '{
jsonb_array_elements(jsonb_array_elements(fhir2->'bodySite')->'coding')->>'system' as ModifierSystem
from FhirTempTable
union
--select 'category' ModifierType, -- GECCO Condition
-- jsonb_array_elements(jsonb_array_elements(fhir2->'category')->'coding')->>'code' as ModifierCode,
-- jsonb_array_elements(jsonb_array_elements(fhir2->'category')->'coding')->>'system' as ModifierSystem
-- from FhirTempTable
--union
select 'category' ModifierType, -- GECCO Procedure
jsonb_array_elements(fhir2->'category'->'coding')->>'code' as ModifierCode,
jsonb_array_elements(fhir2->'category'->'coding')->>'system' as ModifierSystem
from FhirTempTable
union
select 'category' ModifierType, -- GECCO Condition, Observation
jsonb_array_elements(jsonb_array_elements(fhir2->'category')->'coding')->>'code' as ModifierCode,
jsonb_array_elements(jsonb_array_elements(fhir2->'category')->'coding')->>'system' as ModifierSystem
from FhirTempTable
union
select 'clinicalStatus' ModifierType, -- GECCO Condition
jsonb_array_elements(fhir2->'clinicalStatus'->'coding')->>'code' as ModifierCode,
jsonb_array_elements(fhir2->'clinicalStatus'->'coding')->>'system' as ModifierSystem
......@@ -112,6 +114,37 @@ with FhirTempTable (fhir2) as (select '{
select 'usedCode' ModifierType, -- GECCO Procedure
jsonb_array_elements(jsonb_array_elements(fhir2->'usedCode')->'coding')->>'code' as ModifierCode,
jsonb_array_elements(jsonb_array_elements(fhir2->'usedCode')->'coding')->>'system' as ModifierSystem
from FhirTempTable) as sq4 on true
from FhirTempTable
union
select 'component.code' ModifierType, -- GECCO Observation
jsonb_array_elements(jsonb_array_elements(fhir2->'component')->'code'->'coding')->>'code' as ModifierCode,
jsonb_array_elements(jsonb_array_elements(fhir2->'component')->'code'->'coding')->>'system' as ModifierSystem
from FhirTempTable
union
select 'component.valueCodeableConcept' ModifierType, -- GECCO Observation
jsonb_array_elements(jsonb_array_elements(fhir2->'component')->'valueCodeableConcept'->'coding')->>'code' as ModifierCode,
jsonb_array_elements(jsonb_array_elements(fhir2->'component')->'valueCodeableConcept'->'coding')->>'system' as ModifierSystem
from FhirTempTable
union
select 'dataAbsentReason' ModifierType, -- GECCO Observation
jsonb_array_elements(fhir2->'dataAbsentReason'->'coding')->>'code' as ModifierCode,
jsonb_array_elements(fhir2->'dataAbsentReason'->'coding')->>'system' as ModifierSystem
from FhirTempTable
union
select 'method' ModifierType, -- GECCO Observation
jsonb_array_elements(fhir2->'method'->'coding')->>'code' as ModifierCode,
jsonb_array_elements(fhir2->'method'->'coding')->>'system' as ModifierSystem
from FhirTempTable
union
select 'valueCodeableConcept' ModifierType, -- GECCO Observation
jsonb_array_elements(fhir2->'valueCodeableConcept'->'coding')->>'code' as ModifierCode,
jsonb_array_elements(fhir2->'valueCodeableConcept'->'coding')->>'system' as ModifierSystem
from FhirTempTable
) as sq4 on true
left join (select (fhir2->'valueQuantity'->>'value')::numeric(18,5) as valueQuantity_value, -- GECCO Observation
fhir2->'valueQuantity'->>'unit'::text as valueQuantity_unit,
fhir2->'valueQuantity'->>'system'::text as valueQuantity_system,
fhir2->'valueQuantity'->>'code'::text as valueQuantity_code
from FhirTempTable) as sq5 on true
;
\ No newline at end of file
......@@ -459,13 +459,13 @@ begin
insert into i2b2miracum.observation_fact (encounter_num, patient_num, concept_cd, provider_id, start_date, modifier_cd, instance_num, valtype_cd, tval_char, nval_num, valueflag_cd, quantity_num, units_cd, end_date, location_cd, observation_blob, confidence_num, update_date, download_date, import_date, sourcesystem_cd, upload_id, text_search_index)
select i2b2_encounter_identifier, -- encounter_num
i2b2_patient_identifier, -- patient_num
i2b2miracum.get_system_abbrv(fhir_data.code_coding_system) || ':' || fhir_data.code_coding_code, -- concept_cd
i2b2miracum.get_system_abbrv(fhir_data.MainSystem) || ':' || fhir_data.MainCode, -- concept_cd
'FHIR', -- provider_id
case when fhir_data.performedDateTime is not null then fhir_data.performedDateTime
when fhir_data.effectiveDateTime is not null then fhir_data.effectiveDateTime
else '-infinity'::timestamptz end, -- start_date
case when fhir_data.valueCodeableConcept_coding_system is not null
then i2b2miracum.get_system_abbrv(fhir_data.valueCodeableConcept_coding_system) || ':' || fhir_data.valueCodeableConcept_coding_code
case when fhir_data.StartTimes is not null then fhir_data.StartTimes
else '-infinity'::timestamptz
end, -- start_date
case when fhir_data.ModifierSystem is not null
then i2b2miracum.get_system_abbrv(fhir_data.ModifierSystem) || ':' || fhir_data.ModifierCode
else '@'
end, -- modifier_cd
nextval('instance_num_seq'), -- instance_num
......@@ -475,49 +475,82 @@ begin
'', -- valueflag_cd
1, -- quantity_num
fhir_data.valueQuantity_unit, -- units_cd
case when fhir_data.performedDateTime is not null then fhir_data.performedDateTime
when fhir_data.effectiveDateTime is not null then fhir_data.effectiveDateTime
else '-infinity'::timestamptz end, -- end_date
case when fhir_data.StartTimes is not null then fhir_data.StartTimes
else '-infinity'::timestamptz
end, -- start_date -- end_date
'', -- location_cd
FHIR_id, -- observation_blob
ID, -- observation_blob
0, -- confidence_num
now(), -- update_date
now(), -- download_date
now(), -- import_date
FHIR_sourcesystem, -- sourcesystem_cd
'Observation.' || ModifierType FHIR_sourcesystem, -- sourcesystem_cd
1, -- upload_id
nextval('instance_num_seq') -- text_search_index
from (
with FhirTempTable(fhir2) AS (select fhir)
select
--jsonb_path_query(fhir2, '$.id')->>0 as ID,
select
(fhir2->>'id')::text as ID,
performedDateTime,
code_coding_code,
code_coding_system,
valueQuantity_value,
valueQuantity_unit,
valueQuantity_system,
valueQuantity_code,
valueCodeableConcept_coding_code,
valueCodeableConcept_coding_system,
effectiveDateTime
StartTimes, EndTimes,
MainSystem, MainCode,
ModifierType, ModifierSystem, ModifierCode,
valueQuantity_value, valueQuantity_unit, valueQuantity_system, valueQuantity_code
from FhirTempTable
left join (select (fhir2->>'performedDateTime')::timestamp as performedDateTime
from FhirTempTable) as sq1 on true
left join (select jsonb_array_elements(fhir2->'code'->'coding')->>'code' as code_coding_code,
jsonb_array_elements(fhir2->'code'->'coding')->>'system' as code_coding_system
from FhirTempTable) as sq2 on true
left join (select (fhir2->'valueQuantity'->>'value')::numeric(18,5) as valueQuantity_value,
left join (select (fhir2->>'performedDateTime')::timestamp as StartTimes
from FhirTempTable
union
select (fhir2->>'recordedDate')::timestamp as StartTimes
from FhirTempTable
union
select (fhir2->>'effectiveDateTime')::timestamp as StartTimes
from FhirTempTable
) as sq1 on true and StartTimes is not null
left join (select (fhir2->>'performedDateTime')::timestamp as EndTimes
from FhirTempTable
union
select (fhir2->>'recordedDate')::timestamp as EndTimes
from FhirTempTable
union
select (fhir2->>'effectiveDateTime')::timestamp as EndTimes
from FhirTempTable) as sq2 on true and EndTimes is not null
left join (select jsonb_array_elements(fhir2->'code'->'coding')->>'code' as MainCode,
jsonb_array_elements(fhir2->'code'->'coding')->>'system' as MainSystem
from FhirTempTable) as sq3 on true
left join (select 'category' ModifierType, -- GECCO Condition, Observation
jsonb_array_elements(jsonb_array_elements(fhir2->'category')->'coding')->>'code' as ModifierCode,
jsonb_array_elements(jsonb_array_elements(fhir2->'category')->'coding')->>'system' as ModifierSystem
from FhirTempTable
union
select 'component.code' ModifierType, -- GECCO Observation
jsonb_array_elements(jsonb_array_elements(fhir2->'component')->'code'->'coding')->>'code' as ModifierCode,
jsonb_array_elements(jsonb_array_elements(fhir2->'component')->'code'->'coding')->>'system' as ModifierSystem
from FhirTempTable
union
select 'component.valueCodeableConcept' ModifierType, -- GECCO Observation
jsonb_array_elements(jsonb_array_elements(fhir2->'component')->'valueCodeableConcept'->'coding')->>'code' as ModifierCode,
jsonb_array_elements(jsonb_array_elements(fhir2->'component')->'valueCodeableConcept'->'coding')->>'system' as ModifierSystem
from FhirTempTable
union
select 'dataAbsentReason' ModifierType, -- GECCO Observation
jsonb_array_elements(fhir2->'dataAbsentReason'->'coding')->>'code' as ModifierCode,
jsonb_array_elements(fhir2->'dataAbsentReason'->'coding')->>'system' as ModifierSystem
from FhirTempTable
union
select 'method' ModifierType, -- GECCO Observation
jsonb_array_elements(fhir2->'method'->'coding')->>'code' as ModifierCode,
jsonb_array_elements(fhir2->'method'->'coding')->>'system' as ModifierSystem
from FhirTempTable
union
select 'valueCodeableConcept' ModifierType, -- GECCO Observation
jsonb_array_elements(fhir2->'valueCodeableConcept'->'coding')->>'code' as ModifierCode,
jsonb_array_elements(fhir2->'valueCodeableConcept'->'coding')->>'system' as ModifierSystem
from FhirTempTable
) as sq4 on true
left join (select (fhir2->'valueQuantity'->>'value')::numeric(18,5) as valueQuantity_value, -- GECCO Observation
fhir2->'valueQuantity'->>'unit'::text as valueQuantity_unit,
fhir2->'valueQuantity'->>'system'::text as valueQuantity_system,
fhir2->'valueQuantity'->>'code'::text as valueQuantity_code
from FhirTempTable) as sq3 on true
left join (select jsonb_array_elements(fhir2->'valueCodeableConcept'->'coding')->>'code' as valueCodeableConcept_coding_code,
jsonb_array_elements(fhir2->'valueCodeableConcept'->'coding')->>'system' as valueCodeableConcept_coding_system
from FhirTempTable) as sq4 on true
left join (select (fhir2->>'effectiveDateTime')::timestamp as effectiveDateTime
from FhirTempTable) as sq5 on true
from FhirTempTable) as sq5 on true
) fhir_data;
end if;
--
......@@ -650,12 +683,13 @@ begin
insert into i2b2miracum.observation_fact (encounter_num, patient_num, concept_cd, provider_id, start_date, modifier_cd, instance_num, valtype_cd, tval_char, nval_num, valueflag_cd, quantity_num, units_cd, end_date, location_cd, observation_blob, confidence_num, update_date, download_date, import_date, sourcesystem_cd, upload_id, text_search_index)
select i2b2_encounter_identifier, -- encounter_num
i2b2_patient_identifier, -- patient_num
i2b2miracum.get_system_abbrv(fhir_data.code_coding_system) || ':' || fhir_data.code_coding_code, -- concept_cd
i2b2miracum.get_system_abbrv(fhir_data.MainSystem) || ':' || fhir_data.MainCode, -- concept_cd
'FHIR', -- provider_id
case when fhir_data.performedDateTime is not null then fhir_data.performedDateTime
else '-infinity'::timestamptz end, -- start_date
case when fhir_data.bodySite_coding_system is not null
then i2b2miracum.get_system_abbrv(fhir_data.bodySite_coding_system) || ':' || fhir_data.bodySite_coding_code
case when fhir_data.StartTimes is not null then fhir_data.StartTimes
else '-infinity'::timestamptz
end, -- start_date
case when fhir_data.ModifierSystem is not null
then i2b2miracum.get_system_abbrv(fhir_data.ModifierSystem) || ':' || fhir_data.ModifierCode
else '@'
end, -- modifier_cd
nextval('instance_num_seq'), -- instance_num
......@@ -665,15 +699,16 @@ begin
null, -- valueflag_cd
null, -- quantity_num
null, -- units_cd
case when fhir_data.performedDateTime is not null then fhir_data.performedDateTime
else '-infinity'::timestamptz end, -- end_date
case when fhir_data.EndTimes is not null then fhir_data.EndTimes
else '-infinity'::timestamptz
end, -- end_date
'', -- location_cd
FHIR_id, -- observation_blob
ID, -- observation_blob
0, -- confidence_num
now(), -- update_date
now(), -- download_date
now(), -- import_date
FHIR_sourcesystem, -- sourcesystem_cd
'Procedure.' || ModifierType FHIR_sourcesystem, -- sourcesystem_cd
1, -- upload_id
nextval('instance_num_seq') -- text_search_index
from (
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment