Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

validation: make environment property mandatory in serial steps #415

Merged
merged 3 commits into from
Nov 29, 2023
Merged

validation: make environment property mandatory in serial steps #415

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
8205862
schemas: update JSON Schema version to draft 7
giuseppe-steduto Nov 9, 2023
6167c0c
validation: add workflow-engine specific clauses to specification schema
giuseppe-steduto Nov 13, 2023
120559a
release: 0.9.4a3
giuseppe-steduto Nov 28, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions CHANGES.rst
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,10 @@ Changes
Version 0.9.4 (UNRELEASED)
--------------------------

- Changes the REANA specification schema to use the ``draft-07`` version of the JSON schema specification.
- Changes validation of REANA specification to expose functions for loading workflow input parameters and workflow specifications.
- Changes the validation schema of the REANA specification to make the ``environment`` property mandatory for the steps of serial workflows.
- Changes the validation schema of the REANA specification to raise a warning for unexpected properties for the steps of serial workflows.
- Changes CVMFS support to allow users to automatically mount any available repository.
- Fixes the mounting of CVMFS volumes for the REANA deployments that use non-default Kubernetes namespace.

Expand Down
183 changes: 152 additions & 31 deletions reana_commons/validation/schemas/reana_analysis_schema.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
{
"$schema": "http://json-schema.org/draft-04/schema#",
"definitions": {},
"id": "reana_spec",
"$schema": "http://json-schema.org/draft-07/schema",
"$defs": {},
"$id": "reana_spec",
"type": "object",
"title": "REANA analysis specification",
"description": "Full analysis specification including data, sofware, environment and workflow enabling reproducibility on a REANA cluster.",
Expand All @@ -11,72 +11,72 @@
"additionalProperties": false,
"properties": {
"version": {
"id": "/properties/version",
"$id": "/properties/version",
"type": "string",
"title": "REANA version.",
"description": "REANA platform version to which the analysis was written for."
},
"inputs": {
"id": "/properties/inputs",
"$id": "/properties/inputs",
"type": "object",
"title": "Analysis inputs.",
"additionalProperties": false,
"properties": {
"files": {
"id": "/properties/inputs/properties/files",
"$id": "/properties/inputs/properties/files",
"type": "array",
"title": "Analysis input files.",
"description": "List of files provided as input for a given analysis.",
"items": {
"id": "/properties/inputs/properties/files/items",
"$id": "/properties/inputs/properties/files/items",
"type": "string",
"title": "Relative path to the file."
}
},
"directories": {
"id": "/properties/inputs/properties/directories",
"$id": "/properties/inputs/properties/directories",
"type": "array",
"title": "Analysis input directories.",
"description": "List of directories provided as input for a given analysis.",
"items": {
"id": "/properties/inputs/properties/directories/items",
"$id": "/properties/inputs/properties/directories/items",
"type": "string",
"title": "Relative path to the directory."
}
},
"parameters": {
"id": "/properties/inputs/properties/parameters",
"$id": "/properties/inputs/properties/parameters",
"type": "object",
"title": "Analysis parameters.",
"description": "Key value data structure which represents the analysis parameters."
},
"options": {
"id": "/properties/workflow/properties/options",
"$id": "/properties/workflow/properties/options",
"type": "object",
"title": "Workflow operational options.",
"description": "Extra operational options accepted by workflow engines."
}
}
},
"workflow": {
"id": "/properties/workflow",
"$id": "/properties/workflow",
"type": "object",
"title": "Analysis workflow.",
"description": "Workflow which represents the steps that need to be run to reproduce an analysis.",
"additionalProperties": false,
"properties": {
"specification": {
"id": "/properties/workflow/properties/specification",
"$id": "/properties/workflow/properties/specification",
"type": "object",
"title": "Workflow specification in yaml format."
},
"file": {
"id": "/properties/workflow/properties/file",
"$id": "/properties/workflow/properties/file",
"type": "string",
"title": "Workflow file name."
},
"type": {
"id": "/properties/workflow/properties/type",
"$id": "/properties/workflow/properties/type",
"type": "string",
"title": "Workflow engine.",
"description": "Name which represents a supported workflow engine to be used to reproduce the analysis.",
Expand All @@ -88,12 +88,12 @@
]
},
"resources": {
"id": "/properties/workflow/properties/resources",
"$id": "/properties/workflow/properties/resources",
"type": "object",
"title": "Workflow resources in yaml format.",
"properties": {
"cvmfs": {
"id": "/properties/workflow/properties/resources/properties/cvmfs",
"$id": "/properties/workflow/properties/resources/properties/cvmfs",
"type": "array",
"items": {
"type": "string",
Expand All @@ -105,11 +105,12 @@
"uniqueItems": true
},
"kerberos": {
"id": "/properties/workflow/properties/resources/properties/kerberos",
"$id": "/properties/workflow/properties/resources/properties/kerberos",
"type": "boolean",
"title": "Kerberos authentication for the whole workflow."
}
}
},
"additionalProperties": false
}
},
"anyOf": [
Expand All @@ -128,68 +129,68 @@
]
},
"outputs": {
"id": "/properties/outputs",
"$id": "/properties/outputs",
"type": "object",
"title": "Analysis outputs.",
"additionalProperties": false,
"properties": {
"files": {
"id": "/properties/outputs/properties/files",
"$id": "/properties/outputs/properties/files",
"type": "array",
"title": "Analysis result files.",
"description": "Expected output from analysis represented by a set of files.",
"items": {
"id": "/properties/outputs/properties/files/items",
"$id": "/properties/outputs/properties/files/items",
"type": "string",
"title": "Relative path to the file."
}
},
"directories": {
"id": "/properties/outputs/properties/directories",
"$id": "/properties/outputs/properties/directories",
"type": "array",
"title": "Analysis result directories.",
"description": "Expected output from analysis represented by a set of directories.",
"items": {
"id": "/properties/outputs/properties/directories/items",
"$id": "/properties/outputs/properties/directories/items",
"type": "string",
"title": "Relative path to the directory."
}
}
}
},
"tests": {
"id": "/properties/tests",
"$id": "/properties/tests",
"type": "object",
"title": "Analysis tests.",
"additionalProperties": false,
"properties": {
"files": {
"id": "/properties/tests/properties/files",
"$id": "/properties/tests/properties/files",
"type": "array",
"title": "Analysis test files.",
"description": "Gherkin test feature files.",
"items": {
"id": "/properties/tests/properties/files/items",
"$id": "/properties/tests/properties/files/items",
"type": "string",
"title": "Relative path to the file."
}
}
}
},
"workspace": {
"id": "/properties/workspace",
"$id": "/properties/workspace",
"type": "object",
"title": "Workflow workspace.",
"description": "Full workspace path in which the workflow will run.",
"additionalProperties": false,
"properties": {
"root_path": {
"id": "/properties/workspace/properties/root_path",
"$id": "/properties/workspace/properties/root_path",
"type": "string",
"title": "Workspace root path."
},
"retention_days": {
"id": "/properties/workspace/properties/retention_days",
"$id": "/properties/workspace/properties/retention_days",
"type": "object",
"title": "Retention rules for the files in the workspace.",
"minProperties": 1,
Expand All @@ -200,5 +201,125 @@
}
}
}
},
"if": {
"properties": {
"workflow": {
"properties": {
"type": {
"const": "serial"
}
}
}
}
},
"then": {
"properties": {
"workflow": {
"properties": {
"specification": {
"type": "object",
"title": "Serial workflow specification.",
"description": "Serial workflow specification.",
"additionalProperties": false,
"properties": {
"steps": {
"type": "array",
"title": "Serial workflow steps.",
"description": "List of steps which represent the workflow.",
"items": {
"type": "object",
"title": "Serial workflow step.",
"description": "Serial workflow step.",
"additionalProperties": false,
"properties": {
"commands": {
"type": "array",
"items": {
"type": "string"
},
"minItems": 1,
"title": "Step commands",
"description": "List of commands to be run in the step."
},
"compute_backend": {
"type": "string",
"enum": [
"kubernetes",
"htcondorcern",
"slurmcern"
],
"title": "Compute backend"
},
"environment": {
"type": "string",
"title": "Container image for the step",
"description": "Image to be used by the container in which the step should be run."
},
"htcondor_accounting_group": {
"type": "string",
"title": "HTCondor accounting group"
},
"htcondor_max_runtime": {
"type": "string",
"title": "HTCondor maximum runtime"
},
"kerberos": {
"type": "boolean",
"title": "Use Kerberos authentication",
"description": "Whether to use Kerberos authentication for the step. This would require you to upload a valid Kerberos ticket as a REANA secret."
},
"kubernetes_job_timeout": {
"type": "integer",
"title": "Kubernetes job timeout",
"description": "Maximum time for the step to run (number of seconds)"
},
"kubernetes_memory_limit": {
"type": "string",
"title": "Kubernetes memory limit",
"description": "Kubernetes memory limit (e.g. 256Mi - read more about the expected memory values on the official Kubernetes documentation: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/#meaning-of-memory)"
},
"name": {
"type": "string",
"title": "Step name"
},
"rucio": {
"type": "boolean",
"title": "Rucio integration",
"description": "Whether to use Rucio integration."
},
"unpacked_img": {
"type": "boolean",
"title": "Unpacked container image",
"description": "Whether to use an unpacked container image. Useful for Singularity images stored on CVMFS"
},
"voms_proxy": {
"type": "boolean",
"title": "VOMS proxy",
"description": "Whether to use a VOMS proxy for the step. This would require you to upload a valid VOMS proxy as a REANA secret."
}
},
"required": [
"commands",
"environment"
]
}
}
}
}
}
}
}
},
"else": {
"properties": {
"workflow": {
"properties": {
"file": {
"type": "string"
}
}
}
}
}
}
}
Loading