add validation for config json (#389)

* add validation for config json

* fix flake8

* add a test for downloading the schemas

* convert from string to json

* schemas are on current site, not rest api

* <bot> update requirements-docs.txt

* <bot> update requirements-tests.txt

* <bot> update requirements.txt

* more better url

* ret vs response

* must use lowercase.  add more try/catch

* const is block scope, so use var as function scope

* actually print failed validation message

* validation errors is an array

---------

Co-authored-by: github-actions <[email protected]>

iceprod/core/data/dataset.schema.json

@@ -1,6 +1,6 @@
 {
   "$schema": "https://json-schema.org/draft/2020-12/schema",
-  "$id": "https://iceprod2.icecube.wisc.edu/static/dataset.schema.json",
+  "$id": "https://iceprod.icecube.aq/schemas/v3/dataset.schema.json",
   "title": "IceProd Dataset Config",
   "description": "The schema for an IceProd Dataset Config",
   "type": "object",

iceprod/server/data/config.schema.json

@@ -1,6 +1,6 @@
 {
   "$schema": "https://json-schema.org/draft/2020-12/schema",
-  "$id": "https://iceprod2.icecube.wisc.edu/static/config.schema.json",
+  "$id": "https://iceprod.icecube.aq/schemas/v3/config.schema.json",
   "title": "IceProd Server Config",
   "description": "The schema for an IceProd Server Config",
   "type": "object",

iceprod/website/data/www_templates/submit.html

@@ -71,16 +71,38 @@ <h4>$(task)</h4>
 {% block body_scripts %}
 <script src="/static/fetch.js"></script>
 <script type="text/javascript" src="/static/rest.js"></script>
-<!--<script type="text/javascript" src="/static/dataclasses.js"></script>
-<script type="text/javascript" src="/static/jsviews.min.js"></script>
-<script type="text/javascript" src="/static/documentation.js"></script>
-<script type="text/javascript" src="/static/jsonlint.js"></script>-->
+<script src="https://cdnjs.cloudflare.com/ajax/libs/ajv/8.17.1/ajv2020.bundle.min.js" integrity="sha512-mXlpzWzZB+t4DFFezQkpSiCbT8aW12t688aLsd6KGNbRWDOdCur5C6Fl0rDl75VruBy42GvWsd6F35VQcs3lwQ==" crossorigin="anonymous" referrerpolicy="no-referrer"></script>
 <script type="text/javascript">
-var rest_api = '{{ rest_api }}';
-var passkey = '{{ passkey }}';
+const rest_api = '{{ rest_api }}';
+const passkey = '{{ passkey }}';
 var orig_config = {{ json_encode(config) }};
 var edit = {{ json_encode(edit) }};
 var dataset_id = '{{ dataset_id }}';
+
+async function validator(config) {
+    try {
+        const ret = await fetch('/schemas/v3/dataset.schema.json');
+        if (!ret.ok) {
+            throw new Error('Response status: '+ret.status);
+        }
+        var schema = await ret.json();
+    } catch (error) {
+        $('#error').text('Error downloading json schema: '+error.message);
+        return false;
+    }
+    try {
+        const ajv = new ajv2020({useDefaults: true});
+        const validate = ajv.compile(schema);
+        const valid = validate(config)
+        if (!valid) {
+            throw new Error(validate.errors[0].instancePath+' - '+validate.errors[0].message);
+        }
+    } catch (error) {
+        $('#error').text('Error validating config: '+error);
+        return false;
+    }
+    return true;
+}
 function parse_json(config){
     try {
         return JSON.parse(config);
@@ -119,6 +141,12 @@ <h4>$(task)</h4>
     if (ntasks < 0)
         return;
 
+    // validate config
+    const ret = await validator(submit_data);
+    if (!ret) {
+        return;
+    }
+
     // create dataset
     var response = await fetch_json('POST', '/datasets',
         {'description': description,
@@ -141,6 +169,10 @@ <h4>$(task)</h4>
     );
     if ('error' in response) {
         $('#error').text('error uploading config: '+response['error']);
+        await fetch_json('PUT', '/datasets/' + dataset_id + '/status',
+            {'status': 'errors'},
+            passkey
+        );
         return;
     }
     window.location = '/dataset/' + dataset_id;
@@ -150,6 +182,13 @@ <h4>$(task)</h4>
     var submit_data = parse_json($("#submit_box").val());
     if (submit_data == null)
         return;
+
+    // validate config
+    const ret = await validator(submit_data);
+    if (!ret) {
+        return;
+    }
+
     var response = await fetch_json('PUT', '/config/' + dataset_id,
         submit_data, passkey
     );

iceprod/website/server.py

@@ -31,6 +31,8 @@
 
 import iceprod
 from iceprod.roles_groups import GROUPS
+from iceprod.core.config import CONFIG_SCHEMA as DATASET_SCHEMA
+from iceprod.server.config import CONFIG_SCHEMA as SERVER_SCHEMA
 import iceprod.core.functions
 from iceprod.server import documentation
 from iceprod.server.module import FakeStatsClient, StatsClientIgnoreErrors
@@ -260,6 +262,18 @@ async def get(self):
         self.render('main.html')
 
 
+class Schemas(PublicHandler):
+    """Handle /schemas/v3/(.*) urls"""
+    @catch_error
+    async def get(self, schema):
+        if schema == 'dataset.schema.json':
+            self.write(DATASET_SCHEMA)
+        elif schema == 'config.schema.json':
+            self.write(SERVER_SCHEMA)
+        else:
+            raise tornado.web.HTTPError(404, reason='unknown schema')
+
+
 class Submit(PublicHandler):
     """Handle /submit urls"""
     @authenticated
@@ -790,6 +804,7 @@ def __init__(self):
         server.add_route(r"/", Default, handler_args)
         server.add_route(r"/submit", Submit, handler_args)
         server.add_route(r"/config", Config, handler_args)
+        server.add_route(r"/schemas/v3/([\w\.]+)", Schemas, handler_args)
         server.add_route(r"/dataset", DatasetBrowse, handler_args)
         server.add_route(r"/dataset/(\w+)", Dataset, handler_args)
         server.add_route(r"/dataset/(\w+)/task", TaskBrowse, handler_args)

requirements-docs.txt

@@ -16,9 +16,9 @@ attrs==24.2.0
     #   referencing
 babel==2.16.0
     # via sphinx
-boto3==1.35.40
+boto3==1.35.41
     # via iceprod (setup.py)
-botocore==1.35.40
+botocore==1.35.41
     # via
     #   boto3
     #   s3transfer

requirements-tests.txt

@@ -14,11 +14,11 @@ attrs==24.2.0
     #   referencing
 beautifulsoup4==4.12.3
     # via iceprod (setup.py)
-boto3==1.35.40
+boto3==1.35.41
     # via
     #   iceprod (setup.py)
     #   moto
-botocore==1.35.40
+botocore==1.35.41
     # via
     #   boto3
     #   moto

requirements.txt

@@ -12,9 +12,9 @@ attrs==24.2.0
     # via
     #   jsonschema
     #   referencing
-boto3==1.35.40
+boto3==1.35.41
     # via iceprod (setup.py)
-botocore==1.35.40
+botocore==1.35.41
     # via
     #   boto3
     #   s3transfer

tests/website/test_server.py

@@ -1,5 +1,6 @@
 import asyncio
 import binascii
+import json
 import logging
 import random
 import re
@@ -122,11 +123,24 @@ async def test_website_root(server):
     client = server()
     await client.request('GET', '/')
 
+
+async def test_website_schemas(server):
+    client = server()
+    ret = await client.request('GET', '/schemas/v3/dataset.schema.json')
+    ret = json.loads(ret)
+    assert ret['title'] == 'IceProd Dataset Config'
+
+    ret = await client.request('GET', '/schemas/v3/config.schema.json')
+    ret = json.loads(ret)
+    assert ret['title'] == 'IceProd Server Config'
+
+
 async def test_website_submit(server, requests_mock):
     client = server(username='username', roles=['user'], groups=['users', 'simprod'])
 
     ret = await client.request('GET', '/submit')
 
+
 async def test_website_config(server):
     client = server(username='username', roles=['user'], groups=['users', 'simprod'])