Merge pull request #83 from nextflow-io/fix-non-list-samplesheets

Fix non list files validation

CHANGELOG.md

@@ -1,5 +1,12 @@
 # nextflow-io/nf-schema: Changelog
 
+# Version 2.3.0dev
+
+## Bug fixes
+
+1. The help message will now also be printed out when no functions of the plugin get included in the pipeline.
+2. JSON and YAML files that are not a list of values should now also be validated correctly. (Mind that samplesheets always have to be a list of values to work with `samplesheetToList`)
+
 # Version 2.2.1
 
 ## Bug fixes

plugins/nf-schema/src/main/nextflow/validation/CustomEvaluators/SchemaEvaluator.groovy

@@ -50,11 +50,11 @@ class SchemaEvaluator implements Evaluator {
         log.debug("Started validating ${file.toString()}")
 
         def String schemaFull = Utils.getSchemaPath(this.baseDir, this.schema)
-        def JSONArray arrayJSON = Utils.fileToJsonArray(file, Path.of(schemaFull))
+        def Object json = Utils.fileToJson(file, Path.of(schemaFull))
         def String schemaContents = Files.readString( Path.of(schemaFull) )
         def validator = new JsonSchemaValidator(config)
 
-        def List<String> validationErrors = validator.validate(arrayJSON, schemaContents)
+        def List<String> validationErrors = validator.validate(json, schemaContents)
         if (validationErrors) {
             def List<String> errors = ["Validation of file failed:"] + validationErrors.collect { "\t${it}" as String}
             return Evaluator.Result.failure(errors.join("\n"))

plugins/nf-schema/src/main/nextflow/validation/SamplesheetConverter.groovy

@@ -80,14 +80,21 @@ class SamplesheetConverter {
             throw new SchemaValidationException(msg)
         }
 
+        def LinkedHashMap schemaMap = new JsonSlurper().parseText(schemaFile.text) as LinkedHashMap
+        def List<String> schemaKeys = schemaMap.keySet() as List<String>
+        if(schemaKeys.contains("properties") || !schemaKeys.contains("items")) {
+            def msg = "${colors.red}The schema for '${samplesheetFile.toString()}' (${schemaFile.toString()}) is not valid. Please make sure that 'items' is the top level keyword and not 'properties'\n${colors.reset}\n"
+            throw new SchemaValidationException(msg)
+        }
+
         if(!samplesheetFile.exists()) {
             def msg = "${colors.red}Samplesheet file ${samplesheetFile.toString()} does not exist\n${colors.reset}\n"
             throw new SchemaValidationException(msg)
         }
 
         // Validate
         final validator = new JsonSchemaValidator(config)
-        def JSONArray samplesheet = Utils.fileToJsonArray(samplesheetFile, schemaFile)
+        def JSONArray samplesheet = Utils.fileToJson(samplesheetFile, schemaFile) as JSONArray
         def List<String> validationErrors = validator.validate(samplesheet, schemaFile.text)
         if (validationErrors) {
             def msg = "${colors.red}The following errors have been detected in ${samplesheetFile.toString()}:\n\n" + validationErrors.join('\n').trim() + "\n${colors.reset}\n"
@@ -96,8 +103,7 @@ class SamplesheetConverter {
         }
 
         // Convert
-        def LinkedHashMap schemaMap = new JsonSlurper().parseText(schemaFile.text) as LinkedHashMap
-        def List samplesheetList = Utils.fileToList(samplesheetFile, schemaFile)
+        def List samplesheetList = Utils.fileToObject(samplesheetFile, schemaFile) as List
 
         this.rows = []
 

plugins/nf-schema/src/main/nextflow/validation/Utils.groovy

@@ -52,13 +52,19 @@ public class Utils {
     }
 
     // Converts a given file to a List
-    public static List fileToList(Path file, Path schema) {
+    public static Object fileToObject(Path file, Path schema) {
         def String fileType = Utils.getFileType(file)
         def String delimiter = fileType == "csv" ? "," : fileType == "tsv" ? "\t" : null
+        def Map schemaMap = (Map) new JsonSlurper().parse( schema )
         def Map types = variableTypes(schema)
 
-        if (types.find{ it.value == "array" } as Boolean && fileType in ["csv", "tsv"]){
-            def msg = "Using \"type\": \"array\" in schema with a \".$fileType\" samplesheet is not supported\n"
+        if (schemaMap.type == "object" && fileType in ["csv", "tsv"]) {
+            def msg = "CSV or TSV files are not supported. Use a JSON or YAML file instead of ${file.toString()}. (Expected a non-list data structure, which is not supported in CSV or TSV)"
+            throw new SchemaValidationException(msg, [])
+        }
+
+        if ((types.find{ it.value == "array" || it.value == "object" } as Boolean) && fileType in ["csv", "tsv"]){
+            def msg = "Using \"type\": \"array\" or \"type\": \"object\" in schema with a \".$fileType\" samplesheet is not supported\n"
             log.error("ERROR: Validation of pipeline parameters failed!")
             throw new SchemaValidationException(msg, [])
         }
@@ -67,7 +73,7 @@ public class Utils {
             return new Yaml().load((file.text))
         }
         else if(fileType == "json"){
-            return new JsonSlurper().parseText(file.text) as List
+            return new JsonSlurper().parseText(file.text)
         }
         else {
             def Boolean header = getValueFromJson("#/items/properties", new JSONObject(schema.text)) ? true : false
@@ -82,13 +88,21 @@ public class Utils {
     }
 
     // Converts a given file to a JSONArray
-    public static JSONArray fileToJsonArray(Path file, Path schema) {
+    public static Object fileToJson(Path file, Path schema) {
         // Remove all null values from JSON object
         // and convert the groovy object to a JSONArray
         def jsonGenerator = new JsonGenerator.Options()
             .excludeNulls()
             .build()
-        return new JSONArray(jsonGenerator.toJson(fileToList(file, schema)))
+        def Object obj = fileToObject(file, schema)
+        if (obj instanceof List) {
+            return new JSONArray(jsonGenerator.toJson(obj))
+        } else if (obj instanceof Map) {
+            return new JSONObject(jsonGenerator.toJson(obj))
+        } else {
+            def msg = "Could not determine if the file is a list or map of values"
+            throw new SchemaValidationException(msg, [])
+        }
     }
 
     //
@@ -141,7 +155,7 @@ public class Utils {
         def Map parsed = (Map) slurper.parse( schema )
 
         // Obtain the type of each variable in the schema
-        def Map properties = (Map) parsed['items']['properties']
+        def Map properties = (Map) parsed['items'] ? parsed['items']['properties'] : parsed["properties"]
         for (p in properties) {
             def String key = (String) p.key
             def Map property = properties[key] as Map

plugins/nf-schema/src/test/nextflow/validation/ValidateParametersTest.groovy

@@ -1224,4 +1224,102 @@ class ValidateParametersTest extends Dsl2Spec{
         !stdout
     }
 
+    def 'should validate a map file - yaml' () {
+        given:
+        def schema = Path.of('src/testResources/nextflow_schema_with_map_file.json').toAbsolutePath().toString()
+        def SCRIPT = """
+            params.map_file = 'src/testResources/map_file.yaml'
+            include { validateParameters } from 'plugin/nf-schema'
+            
+            validateParameters(parameters_schema: '$schema')
+        """
+
+        when:
+        def config = [:]
+        def result = new MockScriptRunner(config).setScript(SCRIPT).execute()
+        def stdout = capture
+                .toString()
+                .readLines()
+                .findResults {it.contains('WARN nextflow.validation.SchemaValidator') || it.startsWith('* --') ? it : null }
+
+        then:
+        noExceptionThrown()
+        !stdout
+    }
+
+    def 'should validate a map file - json' () {
+        given:
+        def schema = Path.of('src/testResources/nextflow_schema_with_map_file.json').toAbsolutePath().toString()
+        def SCRIPT = """
+            params.map_file = 'src/testResources/map_file.json'
+            include { validateParameters } from 'plugin/nf-schema'
+            
+            validateParameters(parameters_schema: '$schema')
+        """
+
+        when:
+        def config = [:]
+        def result = new MockScriptRunner(config).setScript(SCRIPT).execute()
+        def stdout = capture
+                .toString()
+                .readLines()
+                .findResults {it.contains('WARN nextflow.validation.SchemaValidator') || it.startsWith('* --') ? it : null }
+
+        then:
+        noExceptionThrown()
+        !stdout
+    }
+
+    def 'should give an error when a map file is wrong - yaml' () {
+        given:
+        def schema = Path.of('src/testResources/nextflow_schema_with_map_file.json').toAbsolutePath().toString()
+        def SCRIPT = """
+            params.map_file = 'src/testResources/map_file_wrong.yaml'
+            include { validateParameters } from 'plugin/nf-schema'
+            
+            validateParameters(parameters_schema: '$schema')
+        """
+
+        when:
+        def config = [:]
+        def result = new MockScriptRunner(config).setScript(SCRIPT).execute()
+        def stdout = capture
+                .toString()
+                .readLines()
+                .findResults {it.contains('WARN nextflow.validation.SchemaValidator') || it.startsWith('* --') ? it : null }
+
+
+        then:
+        def error = thrown(SchemaValidationException)
+        error.message.contains("* --map_file (src/testResources/map_file_wrong.yaml): Validation of file failed:")
+        error.message.contains("	* --this.is.deep (hello): Value is [string] but should be [integer]")
+        !stdout
+    }
+
+    def 'should give an error when a map file is wrong - json' () {
+        given:
+        def schema = Path.of('src/testResources/nextflow_schema_with_map_file.json').toAbsolutePath().toString()
+        def SCRIPT = """
+            params.map_file = 'src/testResources/map_file_wrong.json'
+            include { validateParameters } from 'plugin/nf-schema'
+            
+            validateParameters(parameters_schema: '$schema')
+        """
+
+        when:
+        def config = [:]
+        def result = new MockScriptRunner(config).setScript(SCRIPT).execute()
+        def stdout = capture
+                .toString()
+                .readLines()
+                .findResults {it.contains('WARN nextflow.validation.SchemaValidator') || it.startsWith('* --') ? it : null }
+
+
+        then:
+        def error = thrown(SchemaValidationException)
+        error.message.contains("* --map_file (src/testResources/map_file_wrong.json): Validation of file failed:")
+        error.message.contains("	* --this.is.deep (hello): Value is [string] but should be [integer]")
+        !stdout
+    }
+
 }

plugins/nf-schema/src/testResources/map_file.json

@@ -0,0 +1,8 @@
+{
+    "test": "hello",
+    "this": {
+        "is": {
+            "deep": 20
+        }
+    }
+}

plugins/nf-schema/src/testResources/map_file.yaml

@@ -0,0 +1,4 @@
+test: hello
+this:
+  is:
+    deep: 20

plugins/nf-schema/src/testResources/map_file_wrong.json

@@ -0,0 +1,8 @@
+{
+    "test": "hello",
+    "this": {
+        "is": {
+            "deep": "hello"
+        }
+    }
+}

plugins/nf-schema/src/testResources/map_file_wrong.yaml

@@ -0,0 +1,4 @@
+test: hello
+this:
+  is:
+    deep: hello

plugins/nf-schema/src/testResources/nextflow_schema_with_map_file.json

@@ -0,0 +1,14 @@
+{
+    "$schema": "https://json-schema.org/draft/2020-12/schema",
+    "$id": "https://raw.githubusercontent.com/nf-core/testpipeline/master/nextflow_schema.json",
+    "title": "nf-core/testpipeline pipeline parameters",
+    "description": "this is a test",
+    "type": "object",
+    "properties": {
+        "map_file": {
+            "type": "string",
+            "format": "file-path",
+            "schema": "src/testResources/schema_map_file.json"
+        }
+    }
+}

plugins/nf-schema/src/testResources/schema_map_file.json

@@ -0,0 +1,25 @@
+{
+    "$schema": "https://json-schema.org/draft/2020-12/schema",
+    "$id": "https://raw.githubusercontent.com/test/test/master/assets/schema_input.json",
+    "title": "Test schema for samplesheets",
+    "description": "Schema for the file provided with params.input",
+    "type": "object",
+    "properties": {
+        "test": {
+            "type": "string"
+        },
+        "this": {
+            "type": "object",
+            "properties": {
+                "is": {
+                    "type": "object",
+                    "properties": {
+                        "deep": {
+                            "type": "integer"
+                        }
+                    }
+                }
+            }
+        }
+    }
+}