Merge pull request #124 from medizininformatik-initiative/release-v3.…

…0.0-alpha.1 Release v3.0.0 alpha.1
medizininformatik-initiative · Nov 15, 2024 · 914c1c8 · 914c1c8
2 parents e3e6f7a + 4446190
commit 914c1c8
Show file tree

Hide file tree

Showing 91 changed files with 177,610 additions and 189 deletions.
diff --git a/.env b/.env
@@ -1,2 +1,3 @@
 PYTHONPATH="/Users/lorenz/Documents/Programming/Work/ontology-generator-new copy 2/fhir-ontology-generator"
-ONTOLOGY_SERVER_ADDRESS="https://ontoserver.imi.uni-luebeck.de/fhir/"
+ONTOLOGY_SERVER_ADDRESS="https://ontoserver.imi.uni-luebeck.de/fhir/"
+POSTGRES_VERSION="16"
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -0,0 +1,66 @@
+name: Generate and Release Ontology
+
+on:
+  push:
+    tags:
+    - v[0-9]+.[0-9]+.[0-9]+**
+
+jobs:
+
+  release:
+    if: ${{ startsWith(github.ref, 'refs/tags/v') }}
+    runs-on: ubuntu-latest
+    steps:
+    - name: Checkout repo
+      uses: actions/checkout@v4
+
+    - name: Set locale
+      run: |
+        sudo apt-get update
+        sudo apt-get install tzdata locales -y
+        sudo locale-gen de_DE.UTF-8
+        sudo localectl set-locale LANG="de_DE.UTF-8"
+        export LANG="de_DE.UTF-8"
+        sudo update-locale
+        locale -a
+        locale
+        locale -c -k LC_NUMERIC
+        localectl status
+
+    - name: Setup python 3 environment
+      uses: actions/setup-python@v5
+      with:
+        python-version: '3.13'
+        cache: 'pip'
+
+    - name: Install required python modules
+      run: pip3 install -r requirements.txt
+
+    - name: Save secret to file
+      id: certificates
+      run: |
+        echo "$PRIVATE_KEY" > private-key.pem
+        echo "$SERVER_CERTIFICATE" > certificate.pem
+        echo privateKey=$(readlink -f private-key.pem) >> "$GITHUB_OUTPUT"
+        echo certificate=$(readlink -f certificate.pem) >> "$GITHUB_OUTPUT"
+      env:
+        SERVER_CERTIFICATE: ${{ secrets.FDPGPLUS_ONTO_SERVER_CERT }}
+        PRIVATE_KEY: ${{ secrets.FDPGPLUS_ONTO_SERVER_KEY }}
+
+    - name: Run ontology generation
+      env:
+        ONTOLOGY_SERVER_ADDRESS: ${{ secrets.FDPGPLUS_ONTO_SERVER_URL }}
+        SERVER_CERTIFICATE: ${{ steps.certificates.outputs.certificate }}
+        PRIVATE_KEY: ${{ steps.certificates.outputs.privateKey }}
+      run: ./generate_full_ontology.sh --all
+
+
+    - name: Release
+      uses: softprops/action-gh-release@v2
+      with:
+        draft: true
+        generate_release_notes: true
+        files: |
+          example/fdpg-ontology/elastic.zip
+          example/fdpg-ontology/backend.zip
+          example/fdpg-ontology/mapping.zip
diff --git a/.gitignore b/.gitignore
@@ -13,3 +13,6 @@ availability/*.json
 
 /.commit
 .commit
+
+# FSH
+/fsh/*
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -13,6 +13,36 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 ### Fixed
 ### Security
 
+
+## [3.0.0-alpha.1] - 2024-11-15
+
+| Modul          | Version                                                               | Changelog                                       |
+|----------------|-----------------------------------------------------------------------|-------------------------------------------------|
+| MII_DIAGNOSE   | "de.medizininformatikinitiative.kerndatensatz.diagnose 2025.0.0-alpha1" | Added Orphanet                                  |
+| MII_LAB        | "de.medizininformatikinitiative.kerndatensatz.laborbefund 2025.0.0-alpha1" |                                                 |
+| MII_MEDICATION | "de.medizininformatikinitiative.kerndatensatz.medikation 2025.0.0-alpha3" | Added MedicationStatement and MedicationRequest |
+| MII_PERSON     | "de.medizininformatikinitiative.kerndatensatz.person 2025.0.0-alpha1" |                                                 |
+| MII_PROCEDURE  | "de.medizininformatikinitiative.kerndatensatz.prozedur 2025.0.0-alpha1" |                                                 |
+| MII_FALL       | "de.medizininformatikinitiative.kerndatensatz.fall 2025.0.0-alpha1"   |                                                 |
+| MII_SPECIMEN   | "de.medizininformatikinitiative.kerndatensatz.biobank 2025.0.0"       |                                                 |
+| MII_CONSENT    | "de.medizininformatikinitiative.kerndatensatz.consent 1.0.7"          | Added combined consent                          |
+
+### Added
+
+- Add required and recommended to profile details, include mustSupport,… by @juliangruendner in https://github.com/medizininformatik-initiative/fhir-ontology-generator/pull/107
+- Make psql version configurable @paulolaup in https://github.com/medizininformatik-initiative/fhir-ontology-generator/pull/111
+- Merge same ui tree subtrees into one @paulolaup in https://github.com/medizininformatik-initiative/fhir-ontology-generator/pull/114
+- Add field names of profile to profile tree @juliangruendner in https://github.com/medizininformatik-initiative/fhir-ontology-generator/pull/116
+- Make optional values attributes configurable @paulolaup https://github.com/medizininformatik-initiative/fhir-ontology-generator/pull/118
+- Resolve reference to create code filter by @juliangruendner in https://github.com/medizininformatik-initiative/fhir-ontology-generator/pull/99
+- Change attribute type CodeableConcept to Coding by @juliangruendner in https://github.com/medizininformatik-initiative/fhir-ontology-generator/pull/123
+- Added automatic CI build by @EmteZogaf in https://github.com/medizininformatik-initiative/fhir-ontology-generator/pull/123
+- Add fields for each profile comma separated to dse profile tree
+
+### Fixed
+
+- Fix composite generation by @BoehmDo in https://github.com/medizininformatik-initiative/fhir-ontology-generator/pull/97
+
 ## [3.0.0-alpha] - 2024-10-20
 
 | Modul          | Version | Changelog |

diff --git a/README.md b/README.md
@@ -138,11 +138,13 @@ The program goes through all snapshots and for each
 
 **Environment variables:**
 
-| Var | Description | Example |
-|--------|-------------|---------|
-|ONTOLOGY_SERVER_ADDRESS | Address of the Ontology server fhir api (with "/") | my_onto_server.com/fhir/
-|SERVER_CERTIFICATE | Path to the certificate of the Ontology server fhir api| C:\Users\Certs\certificate.pem
-|PRIVATE_KEY | Path to the private key for the Ontology server | C:\Users\Certs\private_key.pem
+| Var                     | Description                                                                                                           | Example                        | Default |
+|-------------------------|-----------------------------------------------------------------------------------------------------------------------|--------------------------------|---------|
+| ONTOLOGY_SERVER_ADDRESS | Address of the Ontology server fhir api (with "/")                                                                    | my_onto_server.com/fhir/       |         |
+| SERVER_CERTIFICATE      | Path to the certificate of the Ontology server fhir api                                                               | C:\Users\Certs\certificate.pem |         | 
+| PRIVATE_KEY             | Path to the private key for the Ontology server                                                                       | C:\Users\Certs\private_key.pem |         |
+| POSTGRES_VERSION        | Optional version of PostgreSQL (version part of Docker image tag, i.e. `postgres:<version>`) the generator should use | 16                             | lastest |
+| POSTGRES_BASE_IMAGE     | Optional base image of on which the PostgreSQL image is build (`postgres:<version>[-<base-img>]`)                     | alpine | NONE |                                                  
 
 **Script Options:**
 

diff --git a/TerminologService/valueSetToRoots.py b/TerminologService/valueSetToRoots.py
@@ -12,7 +12,7 @@
 from model.UiDataModel import TermCode
 from util.LoggingUtil import init_logger
 
-locale.setlocale(locale.LC_ALL, 'de_DE')
+locale.setlocale(locale.LC_ALL, 'de_DE.UTF-8')
 
 logger = init_logger("valueSetToRoots", logging.DEBUG)
 
@@ -99,7 +99,7 @@ def create_vs_tree_map(canonical_url: str) -> TreeMap:
                         treemap.entries[parent].children.append(node)
     except Exception as e:
         print(e)
-        
+
     return treemap
 
 

diff --git a/core/CQLMappingGenerator.py b/core/CQLMappingGenerator.py
@@ -8,7 +8,8 @@
 
 from core import StrucutureDefinitionParser as FHIRParser
 from core.ResourceQueryingMetaDataResolver import ResourceQueryingMetaDataResolver
-from core.StrucutureDefinitionParser import resolve_defining_id, extract_value_type, extract_reference_type
+from core.StrucutureDefinitionParser import resolve_defining_id, extract_value_type, extract_reference_type, \
+    CQL_TYPES_TO_VALUE_TYPES
 from helper import generate_attribute_key
 from model.MappingDataModel import CQLMapping, CQLAttributeSearchParameter
 from model.ResourceQueryingMetaData import ResourceQueryingMetaData
@@ -137,7 +138,8 @@ def generate_cql_mapping(self, profile_snapshot, querying_meta_data: ResourceQue
         if time_defining_id := querying_meta_data.time_restriction_defining_id:
             cql_mapping.timeRestrictionFhirPath = self.translate_element_id_to_fhir_path_expressions_time_restriction(
                 time_defining_id, profile_snapshot)
-        for attr_defining_id, attr_type in querying_meta_data.attribute_defining_id_type_map.items():
+        for attr_defining_id, attr_attributes in querying_meta_data.attribute_defining_id_type_map.items():
+            attr_type = attr_attributes.get("type", "")
             self.set_attribute_search_param(attr_defining_id, cql_mapping, attr_type, profile_snapshot)
 
         return cql_mapping
@@ -341,7 +343,10 @@ def get_attribute_type(self, profile_snapshot: dict, attribute_id: str) -> VALUE
         if " as ValueSet" in attribute_id:
             attribute_id = attribute_id.replace(" as ValueSet", "")
         attribute_element = resolve_defining_id(profile_snapshot, attribute_id, self.data_set_dir, self.module_dir)
-        return extract_value_type(attribute_element, profile_snapshot.get('name'))
+
+        attribute_type = extract_value_type(attribute_element, profile_snapshot.get('name'))
+
+        return CQL_TYPES_TO_VALUE_TYPES.get(attribute_type)
 
     def get_reference_type(self, profile_snapshot: dict, attr_defining_id):
         """

diff --git a/core/ElasticSearchBulkGenerator.py b/core/ElasticSearchBulkGenerator.py
@@ -46,13 +46,14 @@ def __build_crit_set_map(context_termcode_hash_to_crit_set, crit_set_dir, namesp
                         context_termcode_hash_to_crit_set[cont_term_hash].append(crit_set_url)
 
     @staticmethod
-    def __get_relation_crit_object(code, system, term_code_info_map, namespace_uuid_str):
+    def __get_relation_crit_object(code, system, context, term_code_info_map, namespace_uuid_str):
 
-        term_code_hash = ElasticSearchGenerator.__get_termcode_hash(
-            {"code": code, "system": system}, namespace_uuid_str=namespace_uuid_str)
+        term_code = {"code": code, "system": system}
+        context_term_code_hash = ElasticSearchGenerator.__get_contextualized_termcode_hash(
+                          context, term_code,
+                          namespace_uuid_str=namespace_uuid_str)
 
-
-        parent_term_code_info = term_code_info_map[term_code_hash]
+        parent_term_code_info = term_code_info_map[context_term_code_hash]
         parent_context = parent_term_code_info['context']
         parent_term_code = parent_term_code_info['term_code']
 
@@ -72,11 +73,18 @@ def __iterate_tree(ui_tree_list, term_code_info_map, target_elastic_crit_list, c
 
             for entry in ui_tree['entries']:
 
-                term_code_hash = ElasticSearchGenerator.__get_termcode_hash({"code": entry['key'], "system": ui_tree['system']},namespace_uuid_str)
-                term_code_info = term_code_info_map[term_code_hash]
+                cur_tree_context = ui_tree['context']
+                ui_tree_term_code = {
+                    "system": ui_tree['system'],
+                    "code": entry['key']
+                }
+
+                context_termcode_hash = ElasticSearchGenerator.__get_contextualized_termcode_hash(cur_tree_context, ui_tree_term_code,
+                                                                          namespace_uuid_str)
 
+                term_code_info = term_code_info_map[context_termcode_hash]
                 term_code = term_code_info['term_code']
-                selectable = True if term_code_info['children_count']  < children_cut_off else False
+                selectable = True if term_code_info['children_count'] < children_cut_off else False
                 obj = {
                'hash': ElasticSearchGenerator.__get_contextualized_termcode_hash(term_code_info['context'], term_code, namespace_uuid_str),
                'name': term_code['display'],
@@ -95,10 +103,10 @@ def __iterate_tree(ui_tree_list, term_code_info_map, target_elastic_crit_list, c
                }
 
                 for parent_code in entry['parents']:
-                    obj['parents'].append(ElasticSearchGenerator.__get_relation_crit_object(parent_code,ui_tree['system'],term_code_info_map,namespace_uuid_str))
+                    obj['parents'].append(ElasticSearchGenerator.__get_relation_crit_object(parent_code,ui_tree['system'], cur_tree_context, term_code_info_map,namespace_uuid_str))
 
                 for child_code in entry['children']:
-                    obj['children'].append(ElasticSearchGenerator.__get_relation_crit_object(child_code,ui_tree['system'],term_code_info_map,namespace_uuid_str))
+                    obj['children'].append(ElasticSearchGenerator.__get_relation_crit_object(child_code,ui_tree['system'], cur_tree_context, term_code_info_map,namespace_uuid_str))
 
                 # TODO - Siblings - needs to be considered as it is possible to have siblings from other ui_tree
                 #for sibling_code in term_code_info['siblings']:
@@ -112,6 +120,9 @@ def __iterate_tree(ui_tree_list, term_code_info_map, target_elastic_crit_list, c
     @staticmethod
     def __convert_value_set(value_set, termcode_to_valueset, namespace_uuid_str):
 
+        if "contains" not in value_set['expansion']:
+            return
+
         for termcode in value_set['expansion']['contains']:
 
             termcode_hash_input = f"{termcode['code']}{termcode['system']}"
@@ -215,8 +226,8 @@ def load_termcode_info(ontology_dir, tree_file_name, namespace_uuid_str):
 
             print(f"loaded termcode info map from file {filename_prefix}")
             for term_code_info in term_code_info_list:
-                term_code_hash = ElasticSearchGenerator.__get_termcode_hash(term_code_info['term_code'],namespace_uuid_str )
-                term_code_info_map[term_code_hash] = term_code_info
+                hash = ElasticSearchGenerator.__get_contextualized_termcode_hash(term_code_info['context'], term_code_info['term_code'], namespace_uuid_str)
+                term_code_info_map[hash] = term_code_info
 
         return term_code_info_map
 

diff --git a/core/FHIRSearchMappingGenerator.py b/core/FHIRSearchMappingGenerator.py
@@ -139,7 +139,8 @@ def generate_fhir_search_mapping(self, profile_snapshot: dict, querying_meta_dat
                                                                                   self.resolve_fhir_search_parameter(
                                                                                       querying_meta_data.time_restriction_defining_id,
                                                                                       profile_snapshot, "date"))
-        for attribute, predefined_type in querying_meta_data.attribute_defining_id_type_map.items():
+        for attribute, predefined_attributes in querying_meta_data.attribute_defining_id_type_map.items():
+            predefined_type = predefined_attributes.get("type", "")
             self.set_attribute_search_param(attribute, fhir_mapping, predefined_type, profile_snapshot)
         return fhir_mapping
 

diff --git a/core/PathlingMappingGenerator.py b/core/PathlingMappingGenerator.py
@@ -109,7 +109,8 @@ def generate_pathling_mapping(self, profile_snapshot, querying_meta_data: Resour
         if time_defining_id := querying_meta_data.time_restriction_defining_id:
             pathling_mapping.timeRestrictionFhirPath = self.translate_element_id_to_fhir_path_expressions(
                 time_defining_id, profile_snapshot)
-        for attr_defining_id, attr_type in querying_meta_data.attribute_defining_id_type_map.items():
+        for attr_defining_id, attr_attributes in querying_meta_data.attribute_defining_id_type_map.items():
+            attr_type = attr_attributes.get("type", "")
             attribute_key = generate_attribute_key(attr_defining_id)
             attribute_type = attr_type if attr_type else self.get_attribute_type(profile_snapshot,
                                                                                  attr_defining_id)
-Original file line number
+Diff line change
@@ Expand Up / @@ -13,3 +13,6 @@ availability/*.json @@
     /.commit
     .commit
+    # FSH
+    /fsh/*