Fix groups api calls

Dockerfile

@@ -3,7 +3,7 @@ COPY . /usr/src/app
 WORKDIR /usr/src/app
 RUN mvn clean install -DskipTests=true
 
-FROM adoptopenjdk/openjdk11:alpine
+FROM aehrc/jre:openjdk-17
 LABEL maintainer="SNOMED International <[email protected]>"
 
 ARG SUID=1042

azure-pipelines.yml

@@ -0,0 +1,48 @@
+name: rvf-$(Date:yyyyMMdd)$(Rev:.r)
+
+trigger:
+  branches:
+    include:
+      - 'groups-api-fix'
+      - 'support-stored-proc-file'
+pool:
+  vmImage: 'ubuntu-latest'
+
+variables:
+  mavenCache: $(Pipeline.Workspace)/.m2/repository
+  mavenOptions: '-Dmaven.repo.local=$(mavenCache)'
+
+stages:
+  - stage: build
+    displayName: Build
+    jobs:
+      - job: build
+        displayName: Build
+        steps:
+          - task: Cache@2
+            displayName: Cache Maven local repo
+            inputs:
+              key: 'maven | "$(Agent.OS)" | **/pom.xml'
+              restoreKeys: |
+                maven | "$(Agent.OS)"
+                maven
+              path: $(mavenCache)
+          - task: MavenAuthenticate@0
+            inputs:
+              artifactsFeeds: 'mavenbuild'
+          - task: Docker@2
+            inputs:
+              containerRegistry: $(acr_registry)
+              command: 'login'
+          - task: Maven@3
+            displayName: Build and verify
+            inputs:
+              mavenPomFile: 'pom.xml'
+              options: '-Ddocker.image.prefix=$(acr_repo)'
+              mavenOptions: '$(mavenOptions)'
+              javaHomeOption: 'JDKVersion'
+              jdkVersionOption: '1.17'
+              jdkArchitectureOption: 'x64'
+              publishJUnitResults: true
+              testResultsFiles: '**/surefire-reports/TEST-*.xml'
+              goals: 'clean install dockerfile:build dockerfile:push'

docker-compose.yml

@@ -7,7 +7,7 @@ volumes:
 
 services:
   db:
-    image: mysql:5.7
+    image: mysql:8.0.28
     restart: always
     environment:
       - MYSQL_ROOT_PASSWORD=snomed
@@ -21,7 +21,7 @@ services:
     volumes:
       - mysql:/var/lib/mysql
     command:
-      mysqld --sql_mode="NO_ENGINE_SUBSTITUTION,STRICT_TRANS_TABLES" --lower_case_table_names=1
+      mysqld --local-infile=ON --sql_mode="NO_ENGINE_SUBSTITUTION,STRICT_TRANS_TABLES" --lower_case_table_names=1
   rvf:
     image: snomedinternational/release-validation-framework:latest
     container_name: rvf
@@ -31,7 +31,7 @@ services:
     ports:
       - 8081:8081
     environment:
-      - SPRING_DATASOURCE_URL=jdbc:mysql://db:3306/?useSSL=false
+      - SPRING_DATASOURCE_URL=jdbc:mysql://db:3306/?useSSL=false&allowLoadLocalInfile=true&allowPublicKeyRetrieval=true
       - SPRING_DATASOURCE_USERNAME=root
       - SPRING_DATASOURCE_PASSWORD=snomed
       - rvf.assertion.resource.local.path=./snomed-release-validation-assertions/

pom.xml

@@ -351,6 +351,18 @@
                     </execution>
                 </executions>
             </plugin>
+
+            <plugin>
+                <groupId>org.owasp</groupId>
+                <artifactId>dependency-check-maven</artifactId>
+                <version>8.3.1</version>
+                <executions>
+                    <execution>
+                        <phase>none</phase>
+                    </execution>
+                </executions>
+            </plugin>
+
         </plugins>
     </build>
 

src/main/java/org/ihtsdo/rvf/core/data/model/ExecutionCommand.java

@@ -18,14 +18,14 @@ public class ExecutionCommand {
 	@GeneratedValue
 	Long id;
 
-	@Column(columnDefinition = "text")
+	@Column(columnDefinition = "longtext")
 	String template = null;
 	@JsonBackReference
 	@OneToOne(fetch = FetchType.EAGER, mappedBy = "command")
 	Test test;
 	@ElementCollection(fetch = FetchType.EAGER)
 	@CollectionTable(name="command_statements", joinColumns=@JoinColumn(name="command_id"))
-	@Column(name="statement", columnDefinition = "text")
+	@Column(name="statement", columnDefinition = "longtext")
 	@OrderColumn(name="statement_index")
 	List<String> statements = new ArrayList<>();
 

src/main/java/org/ihtsdo/rvf/core/service/AssertionExecutionService.java

@@ -1,8 +1,22 @@
 package org.ihtsdo.rvf.core.service;
 
+import java.sql.*;
+import java.util.Map;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.regex.Pattern;
+import java.util.stream.Collectors;
+
+import javax.annotation.Resource;
+import javax.naming.ConfigurationException;
+import javax.naming.ConfigurationException;
 import org.apache.commons.dbcp.BasicDataSource;
 import org.ihtsdo.rvf.core.data.model.*;
 import org.ihtsdo.rvf.core.service.config.MysqlExecutionConfig;
+import org.ihtsdo.rvf.core.service.util.MySqlQueryTransformer;
 import org.ihtsdo.rvf.importer.AssertionGroupImporter.ProductName;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -208,7 +222,7 @@ private void executeCommand(final Assertion assertion, final MysqlExecutionConfi
 			else {
 				if (sqlStatement.startsWith("create table")){
 					// only add engine if we do not create using a like statement
-					if (!(sqlStatement.contains("like") || sqlStatement.contains("as"))) {
+					if (!sqlStatement.toUpperCase().contains(" ENGINE") && !(sqlStatement.contains("like") || sqlStatement.contains("as"))) {
 						sqlStatement = sqlStatement + " ENGINE = MyISAM";
 					}
 				}
@@ -218,59 +232,10 @@ private void executeCommand(final Assertion assertion, final MysqlExecutionConfi
 	}
 
 	private List<String> transformSql(String[] parts, Assertion assertion, MysqlExecutionConfig config) throws ConfigurationException {
-		List<String> result = new ArrayList<>();
-		String defaultCatalog = dataSource.getDefaultCatalog();
-		String prospectiveSchema = config.getProspectiveVersion();
-		final String[] nameParts = config.getProspectiveVersion().split("_");
-		String defaultModuleId = StringUtils.hasLength(config.getDefaultModuleId()) ? config.getDefaultModuleId() : (nameParts.length >= 2 ? ProductName.toModuleId(nameParts[1]) : "NOT_SUPPLIED");
-		String includedModules = config.getIncludedModules().stream().collect(Collectors.joining(","));
-		String version = (nameParts.length >= 3 ? nameParts[2] : "NOT_SUPPLIED");
-
-		String previousReleaseSchema = config.getPreviousVersion();
-		String dependencyReleaseSchema = config.getExtensionDependencyVersion();
-
-		//We need both these schemas to exist
-		if (prospectiveSchema == null) {
-			throw new ConfigurationException (FAILED_TO_FIND_RVF_DB_SCHEMA + prospectiveSchema);
-		}
-
-		if (config.isReleaseValidation() && !config.isFirstTimeRelease() && previousReleaseSchema == null) {
-			throw new ConfigurationException (FAILED_TO_FIND_RVF_DB_SCHEMA + previousReleaseSchema);
-		}
-		for( String part : parts) {
-			if ((part.contains("<PREVIOUS>") && previousReleaseSchema == null)
-				|| (part.contains("<DEPENDENCY>") && dependencyReleaseSchema == null)) {
-				continue;
-			}
-
-			logger.debug("Original sql statement: {}", part);
-			// remove all SQL comments - //TODO might throw errors for -- style comments
-			final Pattern commentPattern = Pattern.compile("/\\*.*?\\*/", Pattern.DOTALL);
-			part = commentPattern.matcher(part).replaceAll("");
-			// replace all substitutions for exec
-			part = part.replaceAll("<RUNID>", String.valueOf(config.getExecutionId()));
-			part = part.replaceAll("<ASSERTIONUUID>", String.valueOf(assertion.getAssertionId()));
-			part = part.replaceAll("<MODULEID>", defaultModuleId);
-			part = part.replaceAll("<MODULEIDS>", includedModules);
-			part = part.replaceAll("<VERSION>", version);
-			// watch out for any 's that users might have introduced
-			part = part.replaceAll("qa_result", defaultCatalog+ "." + qaResulTableName);
-			part = part.replaceAll("<PROSPECTIVE>", prospectiveSchema);
-			part = part.replaceAll("<TEMP>", prospectiveSchema);
-			if (previousReleaseSchema != null) {
-				part = part.replaceAll("<PREVIOUS>", previousReleaseSchema);
-			}
-			if (dependencyReleaseSchema != null) {
-				part = part.replaceAll("<DEPENDENCY>", dependencyReleaseSchema);
-			}
-			part = part.replaceAll("<DELTA>", deltaTableSuffix);
-			part = part.replaceAll("<SNAPSHOT>", snapshotTableSuffix);
-			part = part.replaceAll("<FULL>", fullTableSuffix);
-			part.trim();
-			logger.debug("Transformed sql statement: {}", part);
-			result.add(part);
-		}
-		return result;
+		String qaResult = dataSource.getDefaultCatalog()+ "." + qaResulTableName;
+		MySqlQueryTransformer queryTransformer = new MySqlQueryTransformer();
+		Map configMap = Map.of("qa_result", qaResult, "<ASSERTIONUUID>", String.valueOf(assertion.getAssertionId()));
+		return queryTransformer.transformSql(parts, config, configMap);
 	}
 
 

src/main/java/org/ihtsdo/rvf/core/service/util/MySqlQueryTransformer.java

@@ -0,0 +1,129 @@
+package org.ihtsdo.rvf.core.service.util;
+
+import com.facebook.presto.sql.parser.StatementSplitter;
+import com.google.common.collect.ImmutableSet;
+import org.apache.commons.dbcp.BasicDataSource;
+import org.ihtsdo.otf.rest.exception.BusinessServiceException;
+import org.ihtsdo.rvf.core.data.model.Assertion;
+import org.ihtsdo.rvf.core.service.config.MysqlExecutionConfig;
+import org.ihtsdo.rvf.importer.AssertionGroupImporter;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.util.StringUtils;
+
+import javax.naming.ConfigurationException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.regex.Pattern;
+import java.util.stream.Collectors;
+
+public class MySqlQueryTransformer {
+    private final Logger logger = LoggerFactory.getLogger(MySqlQueryTransformer.class);
+
+    private static final String FAILED_TO_FIND_RVF_DB_SCHEMA = "Failed to find rvf db schema for ";
+
+    private String deltaTableSuffix = "d";
+    private String snapshotTableSuffix = "s";
+    private String fullTableSuffix = "f";
+    private static final String DEFAULT_DELIMITER = ";";
+    private static final String DELIMITER_REGEX_PATTERN = "^[ ]*(delimiter|DELIMITER)";
+
+    public List<String> transformSql(String[] parts, MysqlExecutionConfig config, final Map<String, String> configMap)
+            throws ConfigurationException {
+
+        logger.info("Config Map contains " + configMap.entrySet().stream().map(e -> e.getKey() + " : " + e.getValue()).collect(Collectors.joining(",")));
+
+        List<String> result = new ArrayList<>();
+        String prospectiveSchema = config.getProspectiveVersion();
+        String previousReleaseSchema = config.getPreviousVersion();
+        String dependencyReleaseSchema = config.getExtensionDependencyVersion();
+
+        //We need both these schemas to exist
+        if (prospectiveSchema == null) {
+            throw new ConfigurationException (FAILED_TO_FIND_RVF_DB_SCHEMA + prospectiveSchema);
+        }
+
+        if (config.isReleaseValidation() && !config.isFirstTimeRelease() && previousReleaseSchema == null) {
+            throw new ConfigurationException (FAILED_TO_FIND_RVF_DB_SCHEMA + previousReleaseSchema);
+        }
+
+        final String[] nameParts = config.getProspectiveVersion().split("_");
+        String version = (nameParts.length >= 3 ? nameParts[2] : "NOT_SUPPLIED");
+        String includedModules = config.getIncludedModules().stream().collect(Collectors.joining(","));
+        String defaultModuleId = StringUtils.hasLength(config.getDefaultModuleId()) ? config.getDefaultModuleId() : (nameParts.length >= 2 ? AssertionGroupImporter.ProductName.toModuleId(nameParts[1]) : "NOT_SUPPLIED");
+        for( String part : parts) {
+            if ((part.contains("<PREVIOUS>") && previousReleaseSchema == null)
+                    || (part.contains("<DEPENDENCY>") && dependencyReleaseSchema == null)) {
+                continue;
+            }
+
+            logger.debug("Original sql statement: {}", part);
+            // remove all SQL comments - //TODO might throw errors for -- style comments
+            final Pattern commentPattern = Pattern.compile("/\\*.*?\\*/", Pattern.DOTALL);
+            part = commentPattern.matcher(part).replaceAll("");
+            // replace all substitutions for exec
+            part = part.replaceAll("<RUNID>", String.valueOf(config.getExecutionId()));
+            part = part.replaceAll("<ASSERTIONUUID>", configMap.get("<ASSERTIONUUID>"));
+            part = part.replaceAll("<MODULEID>", defaultModuleId);
+            part = part.replaceAll("<MODULEIDS>", includedModules);
+            part = part.replaceAll("<VERSION>", version);
+            // watch out for any 's that users might have introduced
+            part = part.replaceAll("qa_result", configMap.get("qa_result"));
+            part = part.replaceAll("<PROSPECTIVE>", prospectiveSchema);
+            part = part.replaceAll("<TEMP>", prospectiveSchema);
+            if (previousReleaseSchema != null) {
+                part = part.replaceAll("<PREVIOUS>", previousReleaseSchema);
+            }
+            if (dependencyReleaseSchema != null) {
+                part = part.replaceAll("<DEPENDENCY>", dependencyReleaseSchema);
+            }
+            part = part.replaceAll("<DELTA>", deltaTableSuffix);
+            part = part.replaceAll("<SNAPSHOT>", snapshotTableSuffix);
+            part = part.replaceAll("<FULL>", fullTableSuffix);
+            part = part.replaceAll(Pattern.quote("[[:<:]]"),"\\\\b" );
+            part = part.replaceAll(Pattern.quote("[[:>:]]"),"\\\\b" );
+            for(Map.Entry<String, String> configMapEntry: configMap.entrySet()){
+                if (configMapEntry.getKey().matches("^<[^>]+>$")) {
+                    part = part.replaceAll(configMapEntry.getKey(), configMapEntry.getValue());
+                }
+            }
+            part.trim();
+            logger.debug("Transformed sql statement: {}", part);
+            result.add(part);
+        }
+        return result;
+    }
+    /**
+     * Convert given sql file content to multiple statements
+     * @param sqlFileContent
+     * @return
+     */
+    public List<String> transformToStatements(String sqlFileContent) throws BusinessServiceException {
+        String delimiter = DEFAULT_DELIMITER;
+        List<String> result = new ArrayList<>();
+        String[] sqlChunks = sqlFileContent.trim().split(DELIMITER_REGEX_PATTERN, Pattern.MULTILINE);
+        for (int i = 0; i < sqlChunks.length; i++) {
+            String sqlChunk = sqlChunks[i].trim();
+            if (!sqlChunk.isEmpty()) {
+                if (i > 0) {
+                    delimiter = sqlChunk.trim().replaceAll("(?s)^([^ \r\n]+).*$", "$1");
+                    sqlChunk = sqlChunk.trim().replaceAll("(?s)^[^ \r\n]+(.*)$", "$1").trim();
+                }
+                if (!sqlChunk.isEmpty()) {
+                    logger.debug("Executing pre-requisite SQL: " + sqlChunk);
+                    final StatementSplitter splitter = new StatementSplitter(sqlChunk, ImmutableSet.of(delimiter));
+                    if (splitter.getCompleteStatements() == null || splitter.getCompleteStatements().isEmpty()) {
+                        String errorMsg = String.format("SQL statements not ending with %s %s",delimiter, sqlChunk);
+                        logger.error( errorMsg);
+                        throw new BusinessServiceException(errorMsg);
+                    }
+                    result= splitter.getCompleteStatements().stream().map(s -> s.statement()).collect(Collectors.toList());
+
+                }
+            }
+
+        }
+        return result;
+    }
+}