From 653b957adb8cffd338f8eae4bf898d6ed6319f79 Mon Sep 17 00:00:00 2001 From: Darryn McGaw Date: Wed, 16 Oct 2024 15:12:48 +0100 Subject: [PATCH 01/13] Start 10.6.0-SNAPSHOT --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 3a4508b7c..c3ce9991b 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ SNOMED CT Terminology Server Using Elasticsearch snowstorm - 10.5.0 + 10.6.0-SNAPSHOT org.snomed snomed-parent-bom From 45214a9041acf28c9c4ecefcad2e6eb20c3794be Mon Sep 17 00:00:00 2001 From: Quyen Ly Date: Wed, 19 Jun 2024 15:34:13 +0700 Subject: [PATCH 02/13] MAINT-2597 Added ICD-11 module to SI module to support MDRS --- .../org/snomed/snowstorm/core/data/domain/Concepts.java | 1 + .../core/data/services/ModuleDependencyService.java | 7 +++++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/snomed/snowstorm/core/data/domain/Concepts.java b/src/main/java/org/snomed/snowstorm/core/data/domain/Concepts.java index 5fdb1fda1..7428c4e68 100644 --- a/src/main/java/org/snomed/snowstorm/core/data/domain/Concepts.java +++ b/src/main/java/org/snomed/snowstorm/core/data/domain/Concepts.java @@ -14,6 +14,7 @@ public class Concepts { public static final String CORE_MODULE = "900000000000207008"; public static final String MODEL_MODULE = "900000000000012004"; public static final String ICD10_MODULE = "449080006"; + public static final String ICD11_MODULE = "1204363008"; public static final String COMMON_FRENCH_MODULE = "11000241103"; public static final String MODULE = "900000000000443000"; diff --git a/src/main/java/org/snomed/snowstorm/core/data/services/ModuleDependencyService.java b/src/main/java/org/snomed/snowstorm/core/data/services/ModuleDependencyService.java index 3263e4c19..c6e6140c8 100644 --- a/src/main/java/org/snomed/snowstorm/core/data/services/ModuleDependencyService.java +++ b/src/main/java/org/snomed/snowstorm/core/data/services/ModuleDependencyService.java @@ -42,7 +42,7 @@ public class ModuleDependencyService extends ComponentService { public static final Set CORE_MODULES = Set.of(Concepts.CORE_MODULE, Concepts.MODEL_MODULE); - public Set SI_MODULES = new HashSet<>(Set.of(Concepts.CORE_MODULE, Concepts.MODEL_MODULE, Concepts.ICD10_MODULE)); + public Set SI_MODULES = new HashSet<>(Set.of(Concepts.CORE_MODULE, Concepts.MODEL_MODULE, Concepts.ICD10_MODULE, Concepts.ICD11_MODULE)); @Autowired private BranchService branchService; @@ -94,10 +94,13 @@ public synchronized void refreshCache() { cacheValidAt = currentTime; logger.info("MDR cache of International Modules refreshed for HEAD time: {}", currentTime); - //During unit tests, or in non-standard installations we might not see the ICD-10 Module + //During unit tests, or in non-standard installations we might not see the ICD-10 and ICD-11 Modules if (!cachedInternationalModules.contains(Concepts.ICD10_MODULE)) { SI_MODULES.remove(Concepts.ICD10_MODULE); } + if (!cachedInternationalModules.contains(Concepts.ICD11_MODULE)) { + SI_MODULES.remove(Concepts.ICD11_MODULE); + } derivativeModules = cachedInternationalModules.stream() .filter(m -> !SI_MODULES.contains(m)) From ffa99b3b89153c0c7d759b1bf292e6dd4c0e76e3 Mon Sep 17 00:00:00 2001 From: jonmdtn Date: Fri, 18 Oct 2024 14:59:42 +0200 Subject: [PATCH 03/13] Add reindex of branch-review to elasticsearch upgrade guide --- docs/elasticsearch8-upgrade.md | 133 ++++++++++++++++++++++++++++++++- 1 file changed, 132 insertions(+), 1 deletion(-) diff --git a/docs/elasticsearch8-upgrade.md b/docs/elasticsearch8-upgrade.md index b83bda791..fcb7aca3b 100644 --- a/docs/elasticsearch8-upgrade.md +++ b/docs/elasticsearch8-upgrade.md @@ -31,7 +31,7 @@ POST codesystem-version/_update_by_query } ``` -### Option 2 - Reindex codesystem-version to change the field type of importDate to long +### Option 2 - Reindex codesystem-version and branch-review to change the field type of importDate and lastUpdated to long #### In Kibana create a new index for codesystem-version-tmp with updated mapping ``` @@ -135,6 +135,137 @@ POST _reindex ``` DELETE codesystem-version-tmp ``` + +#### Create a new index for branch-review-tmp with updated mapping +``` +PUT branch-review-tmp +{ + "mappings":{ + "properties":{ + "_class":{ + "type":"text", + "fields":{ + "keyword":{ + "type":"keyword", + "ignore_above":256 + } + } + }, + "changedConcepts":{ + "type":"long" + }, + "id":{ + "type":"keyword" + }, + "lastUpdated":{ + "type":"long" + }, + "source":{ + "type":"nested", + "properties":{ + "baseTimestamp":{ + "type":"long" + }, + "headTimestamp":{ + "type":"long" + }, + "path":{ + "type":"text", + "fields":{ + "keyword":{ + "type":"keyword", + "ignore_above":256 + } + } + } + } + }, + "sourceIsParent":{ + "type":"boolean" + }, + "status":{ + "type":"keyword" + }, + "target":{ + "type":"nested", + "properties":{ + "baseTimestamp":{ + "type":"long" + }, + "headTimestamp":{ + "type":"long" + }, + "path":{ + "type":"text", + "fields":{ + "keyword":{ + "type":"keyword", + "ignore_above":256 + } + } + } + } + } + } + } +} +``` + +#### Reindex branch-review to branch-review-tmp +``` +POST _reindex +{ + "source": { + "index": "branch-review" + }, + "dest": { + "index": "branch-review-tmp" + }, + "script": { + "source": """ + if (ctx._source.containsKey('lastUpdated')) { + def value = ctx._source['lastUpdated']; + // Try parsing the value as a date and convert to millis + try { + if (value instanceof String) { + // Convert date to milliseconds + ZonedDateTime zdt = ZonedDateTime.parse(value); + long milliSinceEpoch = zdt.toInstant().toEpochMilli(); + ctx._source.lastUpdated = milliSinceEpoch; + } + } catch (Exception e) { + // If parsing fails, handle the failure (e.g., log an error, set a default value) + ctx._source.lastUpdated = 1000; // Set a default value for debug, adjust as needed + } + } + """, + "lang": "painless" + } +} +``` + +#### Delete branch-review +``` +DELETE branch-review +``` + +#### Reindex branch-review-tmp back to the original index name. +``` +POST _reindex +{ + "source": { + "index": "branch-review-tmp" + }, + "dest": { + "index": "branch-review" + } +} +``` + +#### Delete branch-review-tmp +``` +DELETE branch-review-tmp +``` Note: You can use curl for above operations if you don't have Kibana installed. See more details on [Reindex API](https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-reindex.html) ## Step two - Upgrade cluster to Elasticsearch 8 From 9e9e405a85d8700f1f562012e69342672a0451e6 Mon Sep 17 00:00:00 2001 From: Darryn McGaw Date: Tue, 3 Dec 2024 14:53:12 +0000 Subject: [PATCH 04/13] Start 10.7.0-SNAPSHOT --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 230d432be..f884b19cd 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ SNOMED CT Terminology Server Using Elasticsearch snowstorm - 10.6.1 + 10.7.0-SNAPSHOT org.snomed snomed-parent-bom From 0f22f29fd69d5d9e39371697165a4eda111f4f51 Mon Sep 17 00:00:00 2001 From: Quyen Ly Date: Wed, 19 Jun 2024 15:34:13 +0700 Subject: [PATCH 05/13] MAINT-2597 Added ICD-11 module to SI module to support MDRS --- .../org/snomed/snowstorm/core/data/domain/Concepts.java | 1 + .../core/data/services/ModuleDependencyService.java | 7 +++++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/snomed/snowstorm/core/data/domain/Concepts.java b/src/main/java/org/snomed/snowstorm/core/data/domain/Concepts.java index 5fdb1fda1..7428c4e68 100644 --- a/src/main/java/org/snomed/snowstorm/core/data/domain/Concepts.java +++ b/src/main/java/org/snomed/snowstorm/core/data/domain/Concepts.java @@ -14,6 +14,7 @@ public class Concepts { public static final String CORE_MODULE = "900000000000207008"; public static final String MODEL_MODULE = "900000000000012004"; public static final String ICD10_MODULE = "449080006"; + public static final String ICD11_MODULE = "1204363008"; public static final String COMMON_FRENCH_MODULE = "11000241103"; public static final String MODULE = "900000000000443000"; diff --git a/src/main/java/org/snomed/snowstorm/core/data/services/ModuleDependencyService.java b/src/main/java/org/snomed/snowstorm/core/data/services/ModuleDependencyService.java index 3263e4c19..c6e6140c8 100644 --- a/src/main/java/org/snomed/snowstorm/core/data/services/ModuleDependencyService.java +++ b/src/main/java/org/snomed/snowstorm/core/data/services/ModuleDependencyService.java @@ -42,7 +42,7 @@ public class ModuleDependencyService extends ComponentService { public static final Set CORE_MODULES = Set.of(Concepts.CORE_MODULE, Concepts.MODEL_MODULE); - public Set SI_MODULES = new HashSet<>(Set.of(Concepts.CORE_MODULE, Concepts.MODEL_MODULE, Concepts.ICD10_MODULE)); + public Set SI_MODULES = new HashSet<>(Set.of(Concepts.CORE_MODULE, Concepts.MODEL_MODULE, Concepts.ICD10_MODULE, Concepts.ICD11_MODULE)); @Autowired private BranchService branchService; @@ -94,10 +94,13 @@ public synchronized void refreshCache() { cacheValidAt = currentTime; logger.info("MDR cache of International Modules refreshed for HEAD time: {}", currentTime); - //During unit tests, or in non-standard installations we might not see the ICD-10 Module + //During unit tests, or in non-standard installations we might not see the ICD-10 and ICD-11 Modules if (!cachedInternationalModules.contains(Concepts.ICD10_MODULE)) { SI_MODULES.remove(Concepts.ICD10_MODULE); } + if (!cachedInternationalModules.contains(Concepts.ICD11_MODULE)) { + SI_MODULES.remove(Concepts.ICD11_MODULE); + } derivativeModules = cachedInternationalModules.stream() .filter(m -> !SI_MODULES.contains(m)) From d3c43bf6a967f45a4e6650c409bc5f4704935846 Mon Sep 17 00:00:00 2001 From: Quyen Ly Date: Fri, 25 Oct 2024 15:25:01 +0700 Subject: [PATCH 06/13] RAP-111 Send JMS topic after completing the extension upgrade --- .../services/CodeSystemUpgradeService.java | 27 ++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/snomed/snowstorm/core/data/services/CodeSystemUpgradeService.java b/src/main/java/org/snomed/snowstorm/core/data/services/CodeSystemUpgradeService.java index c1d2bf6a3..c07632030 100644 --- a/src/main/java/org/snomed/snowstorm/core/data/services/CodeSystemUpgradeService.java +++ b/src/main/java/org/snomed/snowstorm/core/data/services/CodeSystemUpgradeService.java @@ -4,6 +4,7 @@ import io.kaicode.elasticvc.api.PathUtil; import io.kaicode.elasticvc.domain.Branch; import io.kaicode.elasticvc.domain.Metadata; +import org.apache.activemq.command.ActiveMQTopic; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.snomed.snowstorm.core.data.domain.CodeSystem; @@ -14,6 +15,7 @@ import org.snomed.snowstorm.dailybuild.DailyBuildService; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; +import org.springframework.jms.core.JmsTemplate; import org.springframework.security.access.prepost.PreAuthorize; import org.springframework.security.core.context.SecurityContext; import org.springframework.security.core.context.SecurityContextHolder; @@ -24,7 +26,8 @@ import java.util.concurrent.ExecutorService; import static org.snomed.snowstorm.core.data.services.BranchMetadataHelper.INTERNAL_METADATA_KEY; -import static org.snomed.snowstorm.core.data.services.BranchMetadataKeys.*; +import static org.snomed.snowstorm.core.data.services.BranchMetadataKeys.DEPENDENCY_PACKAGE; +import static org.snomed.snowstorm.core.data.services.BranchMetadataKeys.DEPENDENCY_RELEASE; @Service public class CodeSystemUpgradeService { @@ -55,9 +58,18 @@ public class CodeSystemUpgradeService { @Autowired private ExecutorService executorService; + @Autowired + private JmsTemplate jmsTemplate; + @Value("${snowstorm.rest-api.readonly}") private boolean isReadOnly; + @Value("${snowstorm.codesystem-version.message.enabled}") + private boolean jmsMessageEnabled; + + @Value("${jms.queue.prefix}") + private String jmsQueuePrefix; + private static final Map upgradeJobMap = new HashMap<>(); private final Logger logger = LoggerFactory.getLogger(getClass()); @@ -202,6 +214,19 @@ public synchronized void upgrade(String id, CodeSystem codeSystem, Integer newDe if (job != null) { job.setStatus(CodeSystemUpgradeJob.UpgradeStatus.COMPLETED); } + + if (jmsMessageEnabled) { + Map payload = new HashMap<>(); + payload.put("codeSystemShortName", codeSystem.getShortName()); + payload.put("codeSystemBranchPath", codeSystem.getBranchPath()); + payload.put(DEPENDENCY_PACKAGE, newParentVersion.getReleasePackage()); + payload.put(DEPENDENCY_RELEASE, String.valueOf(newParentVersion.getEffectiveDate())); + + String topicDestination = jmsQueuePrefix + ".upgrade.complete"; + logger.info("Sending JMS Topic - destination {}, payload {}...", topicDestination, payload); + jmsTemplate.convertAndSend(new ActiveMQTopic(topicDestination), payload); + } + upgradedSuccessfully = true; } catch (Exception e) { logger.error("Upgrade on {} failed", branchPath, e); From 7db97666a635319cfc5659087084623c23af6a3f Mon Sep 17 00:00:00 2001 From: Darryn McGaw Date: Fri, 25 Oct 2024 16:47:10 +0100 Subject: [PATCH 07/13] MAINT-2309 Prevent copyConcepts from working against versioned branches MAINT-2309 Remove public modifier --- .../core/data/services/ConceptService.java | 11 ++++ .../data/services/ConceptServiceTest.java | 62 +++++++++++++++++++ 2 files changed, 73 insertions(+) diff --git a/src/main/java/org/snomed/snowstorm/core/data/services/ConceptService.java b/src/main/java/org/snomed/snowstorm/core/data/services/ConceptService.java index e1920ed10..3deb999bd 100644 --- a/src/main/java/org/snomed/snowstorm/core/data/services/ConceptService.java +++ b/src/main/java/org/snomed/snowstorm/core/data/services/ConceptService.java @@ -870,6 +870,17 @@ public List copyConcepts(String ecl, String sourceBranchPath, Strin final Branch sourceBranch = branchService.findBranchOrThrow(sourceBranchPath, true); final Branch destinationBranch = branchService.findBranchOrThrow(destinationBranchPath, true); + CodeSystem codeSystem = codeSystemService.findClosestCodeSystemUsingAnyBranch(destinationBranchPath, false); + if (codeSystem != null) { + List codeSystemVersions = codeSystemService.findAllVersions(codeSystem.getShortName(), true, true); + String branchPath = destinationBranch.getPath(); + for (CodeSystemVersion codeSystemVersion : codeSystemVersions) { + if (Objects.equals(branchPath, codeSystemVersion.getBranchPath())) { + throw new ServiceException("Cannot donate concepts from " + sourceBranchPath + " to versioned " + destinationBranchPath); + } + } + } + if (getDefaultModuleId(sourceBranch).equals(getDefaultModuleId(destinationBranch))) { throw new ServiceException("Cannot donate concepts from " + sourceBranchPath + " to " + destinationBranchPath + " as they are from the same module: " + getDefaultModuleId(sourceBranch)); } diff --git a/src/test/java/org/snomed/snowstorm/core/data/services/ConceptServiceTest.java b/src/test/java/org/snomed/snowstorm/core/data/services/ConceptServiceTest.java index 60466e974..65252d7b7 100644 --- a/src/test/java/org/snomed/snowstorm/core/data/services/ConceptServiceTest.java +++ b/src/test/java/org/snomed/snowstorm/core/data/services/ConceptServiceTest.java @@ -17,6 +17,8 @@ import org.snomed.snowstorm.AbstractTest; import org.snomed.snowstorm.config.Config; import org.snomed.snowstorm.core.data.domain.*; +import org.snomed.snowstorm.core.data.domain.review.MergeReview; +import org.snomed.snowstorm.core.data.domain.review.MergeReviewConceptVersions; import org.snomed.snowstorm.core.data.services.pojo.AsyncConceptChangeBatch; import org.snomed.snowstorm.core.data.services.pojo.DescriptionCriteria; import org.snomed.snowstorm.core.data.services.pojo.MemberSearchRequest; @@ -3011,6 +3013,66 @@ void testActivatingInvalidAxiomThrowsException() throws ServiceException { }); } + @Test + void copyConcepts_ShouldThrowException_WhenTargetIsVersionedBranch() throws ServiceException { + String intMain = "MAIN"; + String extMain = "MAIN/SNOMEDCT-XX"; + Map intPreferred = Map.of(US_EN_LANG_REFSET, descriptionAcceptabilityNames.get(PREFERRED), GB_EN_LANG_REFSET, descriptionAcceptabilityNames.get(PREFERRED)); + Map intAcceptable = Map.of(US_EN_LANG_REFSET, descriptionAcceptabilityNames.get(PREFERRED), GB_EN_LANG_REFSET, descriptionAcceptabilityNames.get(ACCEPTABLE)); + String ci = "CASE_INSENSITIVE"; + Concept concept; + Description description; + CodeSystem codeSystem; + + // Create International Concept + concept = new Concept() + .addDescription(new Description("Medicine (medicine)").setTypeId(FSN).setCaseSignificance(ci).setAcceptabilityMap(intPreferred)) + .addDescription(new Description("Medicine").setTypeId(SYNONYM).setCaseSignificance(ci).setAcceptabilityMap(intPreferred)) + .addAxiom(new Relationship(ISA, SNOMEDCT_ROOT)) + .addRelationship(new Relationship(ISA, SNOMEDCT_ROOT)); + concept = conceptService.create(concept, intMain); + String medicineId = concept.getConceptId(); + + // Version International + codeSystem = codeSystemService.find("SNOMEDCT"); + codeSystemService.createVersion(codeSystem, 20240101, "20240101"); + + // Create Extension + codeSystem = codeSystemService.createCodeSystem(new CodeSystem("SNOMEDCT-XX", extMain)); + concept = conceptService.create( + new Concept() + .addDescription(new Description("Extension module (module)").setTypeId(FSN).setCaseSignificance(ci).setAcceptabilityMap(intPreferred)) + .addDescription(new Description("Extension module").setTypeId(SYNONYM).setCaseSignificance(ci).setAcceptabilityMap(intPreferred)) + .addAxiom(new Relationship(ISA, MODULE)), + extMain + ); + String extModuleA = concept.getConceptId(); + branchService.updateMetadata(extMain, Map.of(Config.DEFAULT_MODULE_ID_KEY, extModuleA, Config.EXPECTED_EXTENSION_MODULES, List.of(extModuleA))); + + // Create Extension concept + concept = new Concept() + .setModuleId(extModuleA) + .addDescription(new Description("Paracetamol (medicine)").setTypeId(FSN).setCaseSignificance(ci).setAcceptabilityMap(intPreferred)) + .addDescription(new Description("Paracetamol").setTypeId(SYNONYM).setCaseSignificance(ci).setAcceptabilityMap(intPreferred)) + .addAxiom(new Relationship(ISA, medicineId)) + .addRelationship(new Relationship(ISA, medicineId)); + concept = conceptService.create(concept, extMain); + String paracetamolId = concept.getConceptId(); + + // Version Extension + codeSystem = codeSystemService.find("SNOMEDCT-XX"); + codeSystemService.createVersion(codeSystem, 20240102, "20240102"); + + // Copy Extension to International + assertThrows(ServiceException.class, () -> { + conceptService.copyConcepts("<< " + paracetamolId, extMain, "MAIN/2024-01-01", true); + }); + + // Assert copying failed + concept = conceptService.find(paracetamolId, "MAIN/2024-01-01"); + assertNull(concept); + } + private Description getDescriptionByTerm(Concept concept, String term) { if (concept == null) { return null; From 079cba9ad7b1cd287327a6efa83dc14676238a44 Mon Sep 17 00:00:00 2001 From: Quyen Ly Date: Fri, 1 Nov 2024 12:12:24 +0700 Subject: [PATCH 08/13] PIP-575 Update snomed-parent-pom from 3.7.1 to 3.8.0-SNAPSHOT --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index f884b19cd..d3fb21435 100644 --- a/pom.xml +++ b/pom.xml @@ -10,7 +10,7 @@ org.snomed snomed-parent-bom - 3.7.1 + 3.8.0-SNAPSHOT From 0d526e5da782dd6c85f7d01619ddda5337d28b6f Mon Sep 17 00:00:00 2001 From: Quyen Ly Date: Mon, 2 Dec 2024 10:46:30 +0700 Subject: [PATCH 09/13] PIP-584 Update snomed-parent-bom version from 3.8.0-SNAPSHOT to 3.9.0-SNAPSHOT --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index d3fb21435..de8509e9f 100644 --- a/pom.xml +++ b/pom.xml @@ -10,7 +10,7 @@ org.snomed snomed-parent-bom - 3.8.0-SNAPSHOT + 3.9.0-SNAPSHOT From b87decaf0c0169406f17f49ef46c61ade7aa86eb Mon Sep 17 00:00:00 2001 From: Kai Kewley Date: Wed, 13 Nov 2024 14:17:32 +0000 Subject: [PATCH 10/13] ISTO-125 Fixes #638 LOINC pagination fix. --- .../snowstorm/fhir/services/FHIRConceptService.java | 12 +++++++----- .../snowstorm/fhir/services/FHIRValueSetService.java | 10 ++++++---- 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/src/main/java/org/snomed/snowstorm/fhir/services/FHIRConceptService.java b/src/main/java/org/snomed/snowstorm/fhir/services/FHIRConceptService.java index 87477e4c7..989873335 100644 --- a/src/main/java/org/snomed/snowstorm/fhir/services/FHIRConceptService.java +++ b/src/main/java/org/snomed/snowstorm/fhir/services/FHIRConceptService.java @@ -4,7 +4,6 @@ import ca.uhn.fhir.jpa.entity.TermConcept; import co.elastic.clients.elasticsearch._types.query_dsl.BoolQuery; import com.google.common.collect.Iterables; - import org.hl7.fhir.r4.model.CodeSystem; import org.hl7.fhir.r4.model.CodeType; import org.slf4j.Logger; @@ -19,10 +18,11 @@ import org.springframework.data.domain.Page; import org.springframework.data.domain.PageRequest; import org.springframework.data.domain.Pageable; -import org.springframework.data.elasticsearch.core.ElasticsearchOperations; -import org.springframework.data.elasticsearch.core.SearchHits; +import org.springframework.data.domain.Sort; import org.springframework.data.elasticsearch.client.elc.NativeQuery; import org.springframework.data.elasticsearch.client.elc.NativeQueryBuilder; +import org.springframework.data.elasticsearch.core.ElasticsearchOperations; +import org.springframework.data.elasticsearch.core.SearchHits; import org.springframework.stereotype.Service; import java.util.*; @@ -182,6 +182,7 @@ public FHIRConcept findConcept(FHIRCodeSystemVersion systemVersion, String code) public Page findConcepts(BoolQuery.Builder fhirConceptQuery, PageRequest pageRequest) { NativeQuery searchQuery = new NativeQueryBuilder() .withQuery(fhirConceptQuery.build()._toQuery()) + .withSort(Sort.by(FHIRConcept.Fields.CODE)) .withPageable(pageRequest) .build(); searchQuery.setTrackTotalHits(true); @@ -189,9 +190,10 @@ public Page findConcepts(BoolQuery.Builder fhirConceptQuery, PageRe return toPage(elasticsearchOperations.search(searchQuery, FHIRConcept.class), pageRequest); } - public SearchAfterPage findConceptCodes(BoolQuery.Builder fhirConceptQuery, PageRequest pageRequest) { + public SearchAfterPage findConceptCodes(BoolQuery fhirConceptQuery, PageRequest pageRequest) { NativeQuery searchQuery = new NativeQueryBuilder() - .withQuery(fhirConceptQuery.build()._toQuery()) + .withQuery(fhirConceptQuery._toQuery()) + .withSort(Sort.by(FHIRConcept.Fields.CODE)) .withPageable(pageRequest) .build(); searchQuery.setTrackTotalHits(true); diff --git a/src/main/java/org/snomed/snowstorm/fhir/services/FHIRValueSetService.java b/src/main/java/org/snomed/snowstorm/fhir/services/FHIRValueSetService.java index 4d04b98a1..7e17db113 100644 --- a/src/main/java/org/snomed/snowstorm/fhir/services/FHIRValueSetService.java +++ b/src/main/java/org/snomed/snowstorm/fhir/services/FHIRValueSetService.java @@ -282,7 +282,7 @@ public ValueSet expand(final ValueSetExpansionParameters params, String displayL // FHIR Concept Expansion (non-SNOMED) String sortField = filter != null ? "displayLen" : "code"; pageRequest = PageRequest.of(pageRequest.getPageNumber(), pageRequest.getPageSize(), Sort.Direction.ASC, sortField); - BoolQuery.Builder fhirConceptQuery = getFhirConceptQuery(codeSelectionCriteria, filter); + BoolQuery fhirConceptQuery = getFhirConceptQuery(codeSelectionCriteria, filter).build(); int offsetRequested = (int) pageRequest.getOffset(); int limitRequested = (int) (pageRequest.getOffset() + pageRequest.getPageSize()); @@ -317,14 +317,16 @@ public ValueSet expand(final ValueSetExpansionParameters params, String displayL conceptsToLoad = new ArrayList<>(); } if (!conceptsToLoad.isEmpty()) { - fhirConceptQuery.must(termsQuery(FHIRConcept.Fields.CODE, conceptsToLoad)); - conceptsPage = conceptService.findConcepts(fhirConceptQuery, LARGE_PAGE); + BoolQuery.Builder conceptsToLoadQuery = bool() + .must(fhirConceptQuery._toQuery()) + .must(termsQuery(FHIRConcept.Fields.CODE, conceptsToLoad)); + conceptsPage = conceptService.findConcepts(conceptsToLoadQuery, LARGE_PAGE); conceptsPage = new PageImpl<>(conceptsPage.getContent(), pageRequest, totalResults); } else { conceptsPage = new PageImpl<>(new ArrayList<>(), pageRequest, totalResults); } } else { - conceptsPage = conceptService.findConcepts(fhirConceptQuery, pageRequest); + conceptsPage = conceptService.findConcepts(bool().must(fhirConceptQuery._toQuery()), pageRequest); } } From d747eb64a21c58c1c5bf67549e7b4d9915b29c58 Mon Sep 17 00:00:00 2001 From: Kai Kewley Date: Tue, 26 Nov 2024 10:02:56 +0000 Subject: [PATCH 11/13] ISTO-127 Fix #642 Realign Jena with HAPI FHIR version. --- pom.xml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/pom.xml b/pom.xml index de8509e9f..201d2ca34 100644 --- a/pom.xml +++ b/pom.xml @@ -15,6 +15,8 @@ ${project.artifactId} + + 4.9.0 + + org.apache.jena + jena-core + ${jena_version} + + + org.apache.jena + jena-arq + ${jena_version} + + io.kaicode From 30de66d58d52cc36c82d53c76695aa556fbb57e3 Mon Sep 17 00:00:00 2001 From: Kai Kewley Date: Mon, 25 Nov 2024 15:58:20 +0000 Subject: [PATCH 12/13] ISTO-127 Fix #641 Multi prefix search for other codesystems This fixes a bug when filtering FHIR ValueSets that are not using SNOMED CT. The bug fix enables searching with multiple words or prefixes. Before the fix only searching with one word or prefix was working. --- .../data/services/DescriptionService.java | 9 ++-- .../fhir/services/FHIRValueSetService.java | 20 ++++--- ...FHIRValueSetProviderExpandGenericTest.java | 54 ++++++++++++++++++- 3 files changed, 70 insertions(+), 13 deletions(-) diff --git a/src/main/java/org/snomed/snowstorm/core/data/services/DescriptionService.java b/src/main/java/org/snomed/snowstorm/core/data/services/DescriptionService.java index 289ddde6a..44474ec7f 100644 --- a/src/main/java/org/snomed/snowstorm/core/data/services/DescriptionService.java +++ b/src/main/java/org/snomed/snowstorm/core/data/services/DescriptionService.java @@ -943,7 +943,7 @@ private void addClause(Query queryClause, BoolQuery.Builder boolBuilder, boolean } } - private List analyze(String text, StandardAnalyzer analyzer) { + public static List analyze(String text, StandardAnalyzer analyzer) { List result = new ArrayList<>(); try { TokenStream tokenStream = analyzer.tokenStream("contents", text); @@ -953,12 +953,13 @@ private List analyze(String text, StandardAnalyzer analyzer) { result.add(attr.toString()); } } catch (IOException e) { - logger.error("Failed to analyze text {}", text, e); + LoggerFactory.getLogger(DescriptionService.class) + .error("Failed to analyze text {}", text, e); } return result; } - private String constructSimpleQueryString(String searchTerm) { + public static String constructSimpleQueryString(String searchTerm) { return (searchTerm.trim().replace(" ", "* ") + "*").replace("**", "*"); } @@ -1000,7 +1001,7 @@ private String constructRegexQuery(String term) { return regexBuilder.toString(); } - private String constructSearchTerm(List tokens) { + public static String constructSearchTerm(List tokens) { StringBuilder builder = new StringBuilder(); for (String token : tokens) { builder.append(token); diff --git a/src/main/java/org/snomed/snowstorm/fhir/services/FHIRValueSetService.java b/src/main/java/org/snomed/snowstorm/fhir/services/FHIRValueSetService.java index 7e17db113..76c4e875d 100644 --- a/src/main/java/org/snomed/snowstorm/fhir/services/FHIRValueSetService.java +++ b/src/main/java/org/snomed/snowstorm/fhir/services/FHIRValueSetService.java @@ -1,10 +1,10 @@ package org.snomed.snowstorm.fhir.services; import co.elastic.clients.elasticsearch._types.query_dsl.BoolQuery; -import co.elastic.clients.elasticsearch._types.query_dsl.PrefixQuery; +import co.elastic.clients.elasticsearch._types.query_dsl.Operator; import com.google.common.base.Strings; import it.unimi.dsi.fastutil.longs.LongArrayList; - +import org.apache.lucene.analysis.standard.StandardAnalyzer; import org.hl7.fhir.r4.model.*; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; @@ -15,6 +15,7 @@ import org.snomed.snowstorm.core.data.domain.QueryConcept; import org.snomed.snowstorm.core.data.domain.ReferenceSetMember; import org.snomed.snowstorm.core.data.services.ConceptService; +import org.snomed.snowstorm.core.data.services.DescriptionService; import org.snomed.snowstorm.core.data.services.QueryService; import org.snomed.snowstorm.core.data.services.ReferenceSetMemberService; import org.snomed.snowstorm.core.data.services.pojo.MemberSearchRequest; @@ -30,10 +31,11 @@ import org.snomed.snowstorm.rest.pojo.SearchAfterPageRequest; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.data.domain.*; -import org.springframework.data.elasticsearch.core.ElasticsearchOperations; -import org.springframework.data.elasticsearch.core.SearchHits; import org.springframework.data.elasticsearch.client.elc.NativeQuery; import org.springframework.data.elasticsearch.client.elc.NativeQueryBuilder; +import org.springframework.data.elasticsearch.client.elc.Queries; +import org.springframework.data.elasticsearch.core.ElasticsearchOperations; +import org.springframework.data.elasticsearch.core.SearchHits; import org.springframework.stereotype.Service; import java.net.URLDecoder; @@ -41,11 +43,12 @@ import java.util.*; import java.util.stream.Collectors; +import static co.elastic.clients.elasticsearch._types.query_dsl.QueryBuilders.bool; import static io.kaicode.elasticvc.api.ComponentService.LARGE_PAGE; +import static io.kaicode.elasticvc.helper.QueryHelper.termQuery; +import static io.kaicode.elasticvc.helper.QueryHelper.termsQuery; import static java.lang.Boolean.TRUE; import static java.lang.String.format; -import static co.elastic.clients.elasticsearch._types.query_dsl.QueryBuilders.*; -import static io.kaicode.elasticvc.helper.QueryHelper.*; import static org.snomed.snowstorm.core.data.services.ReferenceSetMemberService.AGGREGATION_MEMBER_COUNTS_BY_REFERENCE_SET; import static org.snomed.snowstorm.core.util.CollectionUtils.orEmpty; import static org.snomed.snowstorm.fhir.services.FHIRHelper.*; @@ -392,7 +395,10 @@ private BoolQuery.Builder getFhirConceptQuery(CodeSelectionCriteria codeSelectio BoolQuery.Builder masterQuery = bool(); masterQuery.must(contentQuery.build()._toQuery()); if (termFilter != null) { - masterQuery.must(PrefixQuery.of(pq -> pq.field(FHIRConcept.Fields.DISPLAY).value(termFilter.toLowerCase()))._toQuery()); + List elasticAnalyzedWords = DescriptionService.analyze(termFilter, new StandardAnalyzer()); + String searchTerm = DescriptionService.constructSearchTerm(elasticAnalyzedWords); + String query = DescriptionService.constructSimpleQueryString(searchTerm); + masterQuery.filter(Queries.queryStringQuery(FHIRConcept.Fields.DISPLAY, query, Operator.And, 2.0f)._toQuery()); } return masterQuery; } diff --git a/src/test/java/org/snomed/snowstorm/fhir/services/FHIRValueSetProviderExpandGenericTest.java b/src/test/java/org/snomed/snowstorm/fhir/services/FHIRValueSetProviderExpandGenericTest.java index 9d7812e81..7550efa01 100644 --- a/src/test/java/org/snomed/snowstorm/fhir/services/FHIRValueSetProviderExpandGenericTest.java +++ b/src/test/java/org/snomed/snowstorm/fhir/services/FHIRValueSetProviderExpandGenericTest.java @@ -17,9 +17,9 @@ import java.io.FileInputStream; import java.io.IOException; import java.nio.charset.StandardCharsets; +import java.util.List; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.*; public class FHIRValueSetProviderExpandGenericTest extends AbstractFHIRTest { @@ -196,4 +196,54 @@ public void testExpandIncludesOtherValueSet() { assertEquals(2, valueSet.getExpansion().getContains().size()); } + @Test + void testSearchMultipleWordsOrPrefixes() { + // "display": "additive, propagating", + System.out.println("----------"); + ResponseEntity response = restTemplate.exchange(baseUrl + "/ValueSet/$expand?url=http://terminology.hl7.org/CodeSystem/v3-ContextControl?fhir_vs", HttpMethod.GET, null, String.class); + System.out.println("Search no filter"); + assertExpand(response, 8, null); + + System.out.println("----------"); + response = restTemplate.exchange(baseUrl + "/ValueSet/$expand?url=http://terminology.hl7.org/CodeSystem/v3-ContextControl?fhir_vs&filter=additive", HttpMethod.GET, null, String.class); + System.out.println("Search additive"); + System.out.println(response.getBody()); + assertExpand(response, 2, "[AP|'additive, propagating', AN|'additive, non-propagating']"); + + System.out.println("----------"); + response = restTemplate.exchange(baseUrl + "/ValueSet/$expand?url=http://terminology.hl7.org/CodeSystem/v3-ContextControl?fhir_vs&filter=additive propagating", HttpMethod.GET, null, String.class); + System.out.println("Search additive propagating"); + System.out.println(response.getBody()); + assertExpand(response, 2, "[AP|'additive, propagating', AN|'additive, non-propagating']"); + + System.out.println("----------"); + response = restTemplate.exchange(baseUrl + "/ValueSet/$expand?url=http://terminology.hl7.org/CodeSystem/v3-ContextControl?fhir_vs&filter=add", HttpMethod.GET, null, String.class); + System.out.println("Search add"); + System.out.println(response.getBody()); + assertExpand(response, 2, "[AP|'additive, propagating', AN|'additive, non-propagating']"); + + System.out.println("----------"); + response = restTemplate.exchange(baseUrl + "/ValueSet/$expand?url=http://terminology.hl7.org/CodeSystem/v3-ContextControl?fhir_vs&filter=add prop non", HttpMethod.GET, null, String.class); + System.out.println("Search add prop non"); + System.out.println(response.getBody()); + assertExpand(response, 1, "[AN|'additive, non-propagating']"); + + System.out.println("----------"); + } + + private void assertExpand(ResponseEntity response, int expected, String expectedCodingString) { + assertEquals(HttpStatus.OK, response.getStatusCode()); + assertNotNull(response.getBody()); + ValueSet valueSet = fhirJsonParser.parseResource(ValueSet.class, response.getBody()); + List contains = valueSet.getExpansion().getContains(); + assertEquals(expected, contains.size()); + if (expectedCodingString != null) { + assertEquals(expectedCodingString, toCodingsString(contains)); + } + } + + private String toCodingsString(List contains) { + return contains.stream().map(coding -> "%s|'%s'".formatted(coding.getCode(), coding.getDisplay())).toList().toString(); + } + } From c81143d2d8cbc7977fe767f0a86903d5d54e0584 Mon Sep 17 00:00:00 2001 From: Kai Kewley Date: Thu, 19 Dec 2024 13:21:19 +0000 Subject: [PATCH 13/13] ISTO-125 Fix sorting after merge with ISTO-127. --- .../org/snomed/snowstorm/fhir/services/FHIRConceptService.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/snomed/snowstorm/fhir/services/FHIRConceptService.java b/src/main/java/org/snomed/snowstorm/fhir/services/FHIRConceptService.java index 989873335..82943b575 100644 --- a/src/main/java/org/snomed/snowstorm/fhir/services/FHIRConceptService.java +++ b/src/main/java/org/snomed/snowstorm/fhir/services/FHIRConceptService.java @@ -182,7 +182,7 @@ public FHIRConcept findConcept(FHIRCodeSystemVersion systemVersion, String code) public Page findConcepts(BoolQuery.Builder fhirConceptQuery, PageRequest pageRequest) { NativeQuery searchQuery = new NativeQueryBuilder() .withQuery(fhirConceptQuery.build()._toQuery()) - .withSort(Sort.by(FHIRConcept.Fields.CODE)) + .withSort(Sort.by(FHIRConcept.Fields.DISPLAY_LENGTH, FHIRConcept.Fields.CODE)) .withPageable(pageRequest) .build(); searchQuery.setTrackTotalHits(true);