merging main with gh-pages

CITATION.cff

@@ -22,6 +22,6 @@ maintainers:
     orcid: "https://orcid.org/0000-0001-9266-5146"
 
 repository-code: "https://github.com/caltechlibrary/dataset"
-version: 2.1.13
+version: 2.1.14
 license-url: "https://data.caltech.edu/license"
 keywords: [ "GitHub", "metadata", "data", "software", "json" ]

Makefile

@@ -40,6 +40,8 @@ ifeq ($(OS), Windows)
 	EXT = .exe
 endif
 
+EXT_WEB = .wasm
+
 DIST_FOLDERS = bin/* man/*
 
 build: version.go $(PROGRAMS) man CITATION.cff about.md installer.sh
@@ -193,6 +195,15 @@ dist/Linux-armv7l:
 	@cd dist && zip -r $(PROJECT)-v$(VERSION)-Linux-armv7l.zip LICENSE codemeta.json CITATION.cff *.md $(DIST_FOLDERS)
 	@rm -fR dist/bin
 
+# WASM code build is experimental, Python maybe able to load WASM code via wasmer-python, https://github.com/wasmerio/wasmer-python
+# This would let me avoid having at have seperate machines to build a libdataset C-shared library.
+dist/js-wasm:
+	@mkdir -p dist/bin
+	@for FNAME in $(PROGRAMS); do env GOOS=js GOARCH=wasm go build -o dist/bin/$$FNAME$(EXT_WEB) cmd/$$FNAME/*.go; done
+	@cd dist && zip -r $(PROJECT)-v$(VERSION)-js-wasm.zip LICENSE codemeta.json CITATION.cff *.md $(DIST_FOLDERS)
+	@rm -fR dist/bin
+
+
 distribute_docs:
 	if [ -d dist ]; then rm -fR dist; fi
 	mkdir -p dist

TODO.html

@@ -235,6 +235,13 @@ <h2 id="next-prep-for-v2.1.1">Next (prep for v2.1.1)</h2>
 </ul>
 <h2 id="someday-maybe">Someday, Maybe</h2>
 <ul class="task-list">
+<li><label><input type="checkbox" />Allow a WASM module to be used to
+validate objects in the collection. It needs to me integrate such that
+it “travels” will the dataset collection</label>
+<ul>
+<li>this would let our JSON collections support explicit JSON structures
+as well as ad-hoc JSON objects</li>
+</ul></li>
 <li><label><input type="checkbox" />Review <a
 href="https://go-app.dev/">Go-app</a> and see if this would be a way to
 create a local client UI for working with datasets and enabling LunrJS
@@ -260,7 +267,30 @@ <h2 id="someday-maybe">Someday, Maybe</h2>
 <li><label><input type="checkbox" />Implement version support in the web
 service</label></li>
 <li><label><input type="checkbox" />Implement an integrated UI for
-datasetd</label></li>
+datasetd</label>
+<ul class="task-list">
+<li><label><input type="checkbox" />Form pages could be expressed in
+Markdown+YAML for forms and embedded in the datasetd settings YAML
+file</label>
+<ul>
+<li>See my notes on my text oreinted web experiment,
+yaml2webform.go</li>
+<li>Forms could be render into the htdocs auto-magically saving
+development effort</li>
+<li>The same forms could then be used server side for validation based
+on descriptors and JavaScript converted to WASM code</li>
+</ul></li>
+<li><label><input type="checkbox" />A standard JavaScript library could
+be used to knit the forms to the datasetd web service (sort of a
+mini-newt) It would be nice if citesearch was defined by the
+citesearch.yaml file and some markdown documents taking a text oriented
+web approach to embedding forms in Markdown combined with some JS glue
+code to knit the two together</label></li>
+</ul></li>
+<li><label><input type="checkbox" />Consider updating datasetd to
+support urlencoded data submissions in additional to application/json,
+this might make it easier to quicklt develop browser side UI for
+datasetd web services</label></li>
 </ul>
 </section>
 

TODO.md

@@ -113,6 +113,8 @@ Next (prep for v2.1.1)
 Someday, Maybe
 --------------
 
+- [ ] Allow a WASM module to be used to validate objects in the collection. It needs to me integrate such that it "travels" will the dataset collection
+  - this would let our JSON collections support explicit JSON structures as well as ad-hoc JSON objects
 - [ ] Review [Go-app](https://go-app.dev/) and see if this would be a way to create a local client UI for working with datasets and enabling LunrJS for search
 - [ ] Document an example Python 3 http client support for web API implementing a drop in replacement for py_dataset using the web service or cli
 - [X] Missing tests for AttachStream()
@@ -129,3 +131,10 @@ Someday, Maybe
       content driven by a dataset collection
 - [ ] Implement version support in the web service
 - [ ] Implement an integrated UI for datasetd
+    - [ ] Form pages could be expressed in Markdown+YAML for forms and embedded in the datasetd settings YAML file
+        - See my notes on my text oreinted web experiment, yaml2webform.go
+        - Forms could be render into the htdocs auto-magically saving development effort
+        - The same forms could then be used server side for validation based on descriptors and JavaScript converted to WASM code
+    - [ ] A standard JavaScript library could be used to knit the forms to the datasetd web service (sort of a mini-newt)
+It would be nice if citesearch was defined by the citesearch.yaml file and some markdown documents taking a text oriented web approach to embedding forms in Markdown combined with some JS glue code to knit the two together
+- [ ] Consider updating datasetd to support urlencoded data submissions in additional to application/json, this might make it easier to quicklt develop browser side UI for datasetd web services

about.html

@@ -24,7 +24,7 @@
 
 <section>
 <h1 id="about-this-software">About this software</h1>
-<h2 id="dataset-2.1.13">dataset 2.1.13</h2>
+<h2 id="dataset-2.1.14">dataset 2.1.14</h2>
 <h3 id="authors">Authors</h3>
 <ul>
 <li>R. S. Doiel</li>

about.md

@@ -14,7 +14,7 @@ authors:
     orcid: "https://orcid.org/0000-0001-9266-5146"
 
 repository-code: "https://github.com/caltechlibrary/dataset"
-version: 2.1.13
+version: 2.1.14
 license-url: "https://data.caltech.edu/license"
 keywords: [ "GitHub", "metadata", "data", "software", "json" ]
 
@@ -23,7 +23,7 @@ keywords: [ "GitHub", "metadata", "data", "software", "json" ]
 About this software
 ===================
 
-## dataset 2.1.13
+## dataset 2.1.14
 
 ### Authors
 

api_test.go

@@ -955,8 +955,9 @@ Client testings starts in %s (s = seconds)
 
 	// Send our web service into the back ground so we can run
 	// a client test.
+	debug := false
 	go func() {
-		if err := RunAPI(appName, fName); err != nil {
+		if err := RunAPI(appName, fName, debug); err != nil {
 			t.Errorf("Expected API to setup and run, %s", err)
 		}
 	}()

cmd/datasetd/datasetd.go

@@ -47,11 +47,10 @@ const (
 
 # DESCRIPTION
 
-Runs a web service for one or more dataset collections. Requires
-the collections to exist (e.g. created previously with the dataset
-cli). It requires a settings JSON or YAML file that decribes the
-web service configuration and permissions per collection that are
-available via the web service.
+{app_name} provides a web service for one or more dataset collections. Requires the
+collections to exist (e.g. created previously with the dataset cli). It requires a
+settings JSON or YAML file that decribes the web service configuration and
+permissions per collection that are available via the web service.
 
 # OPTIONS
 
@@ -98,12 +97,12 @@ dataset
 
 query
 : (optional) is map of query name to SQL statement. A POST is used to access
-the query (i.e. a POST To the path "/api/<COLLECTION_NAME>/query/<QUERY_NAME>")
+the query (i.e. a POST To the path "`+"`"+`/api/<COLLECTION_NAME>/query/<QUERY_NAME>`+"`"+`")
 The parameters submitted in the post are passed to the SQL statement.
 NOTE: Only dataset collections using a SQL store are supported. The SQL
 needs to conform the SQL dialect of the store being used (e.g. MySQL, Postgres,
-SQLite3). The SQL statement functions with the same contrainst of dsquery SQL
-statements.
+SQLite3). The SQL statement functions with the same contraints of dsquery SQL
+statements. The SQL statement is defined as a YAML text blog.
 
 ## API Permissions
 

codemeta.json

@@ -9,7 +9,7 @@
     "codeRepository": "https://github.com/caltechlibrary/dataset",
     "issueTracker": "https://github.com/caltechlibrary/dataset/issues",
     "license": "https://data.caltech.edu/license",
-    "version": "2.1.13",
+    "version": "2.1.14",
     "author": [
         {
             "@type": "Person",

dataset.1.html

@@ -130,7 +130,7 @@ <h1 id="examples">EXAMPLES</h1>
    dataset delete my_objects.ds &quot;345&quot;
 
    dataset keys my_objects.ds</code></pre>
-<p>dataset 2.1.13</p>
+<p>dataset 2.1.12</p>
 </section>
 
 <footer>

dataset.1.md

@@ -1,4 +1,4 @@
-%dataset(1) user manual | version 2.1.13 6f79b21
+%dataset(1) user manual | version 2.1.12 856ffcd
 % R. S. Doiel and Tom Morrell
 % 2024-06-12
 
@@ -114,6 +114,6 @@ implements.
    dataset keys my_objects.ds
 ~~~
 
-dataset 2.1.13
+dataset 2.1.12
 
 

datasetd.1.html

@@ -28,11 +28,11 @@ <h1 id="name">NAME</h1>
 <h1 id="synopsis">SYNOPSIS</h1>
 <p>datasetd <a href="#options">OPTIONS</a> SETTINGS_FILE</p>
 <h1 id="description">DESCRIPTION</h1>
-<p>Runs a web service for one or more dataset collections. Requires the
-collections to exist (e.g. created previously with the dataset cli). It
-requires a settings JSON or YAML file that decribes the web service
-configuration and permissions per collection that are available via the
-web service.</p>
+<p>datasetd provides a web service for one or more dataset collections.
+Requires the collections to exist (e.g. created previously with the
+dataset cli). It requires a settings JSON or YAML file that decribes the
+web service configuration and permissions per collection that are
+available via the web service.</p>
 <h1 id="options">OPTIONS</h1>
 <dl>
 <dt>-help</dt>
@@ -91,11 +91,13 @@ <h1 id="settings_file">SETTINGS_FILE</h1>
 <dd>
 (optional) is map of query name to SQL statement. A POST is used to
 access the query (i.e. a POST To the path
-“/api/<COLLECTION_NAME>/query/<QUERY_NAME>”) The parameters submitted in
-the post are passed to the SQL statement. NOTE: Only dataset collections
-using a SQL store are supported. The SQL needs to conform the SQL
-dialect of the store being used (e.g. MySQL, Postgres, SQLite3). The SQL
-statement functions with the same contrainst of dsquery SQL statements.
+“<code>/api/&lt;COLLECTION_NAME&gt;/query/&lt;QUERY_NAME&gt;</code>”)
+The parameters submitted in the post are passed to the SQL statement.
+NOTE: Only dataset collections using a SQL store are supported. The SQL
+needs to conform the SQL dialect of the store being used (e.g. MySQL,
+Postgres, SQLite3). The SQL statement functions with the same contraints
+of dsquery SQL statements. The SQL statement is defined as a YAML text
+blog.
 </dd>
 </dl>
 <h2 id="api-permissions">API Permissions</h2>
@@ -231,7 +233,7 @@ <h1 id="examples">EXAMPLES</h1>
 <pre><code>    curl http://localhost:8485/api/t1.ds/keys</code></pre>
 <p>In the shell session where datasetd is running press “ctr-C” to
 terminate the service.</p>
-<p>datasetd 2.1.13</p>
+<p>datasetd 2.1.12</p>
 </section>
 
 <footer>

datasetd.1.md

@@ -1,4 +1,4 @@
-%datasetd (1) user manual | verion 2.1.13 {release_hash}"
+%datasetd (1) user manual | verion 2.1.12 {release_hash}"
 % R. S. Doiel
 % {release_date}
 
@@ -12,11 +12,10 @@ datasetd [OPTIONS] SETTINGS_FILE
 
 # DESCRIPTION
 
-Runs a web service for one or more dataset collections. Requires
-the collections to exist (e.g. created previously with the dataset
-cli). It requires a settings JSON or YAML file that decribes the
-web service configuration and permissions per collection that are
-available via the web service.
+datasetd provides a web service for one or more dataset collections. Requires the
+collections to exist (e.g. created previously with the dataset cli). It requires a
+settings JSON or YAML file that decribes the web service configuration and
+permissions per collection that are available via the web service.
 
 # OPTIONS
 
@@ -63,12 +62,12 @@ dataset
 
 query
 : (optional) is map of query name to SQL statement. A POST is used to access
-the query (i.e. a POST To the path "/api/<COLLECTION_NAME>/query/<QUERY_NAME>")
+the query (i.e. a POST To the path "`/api/<COLLECTION_NAME>/query/<QUERY_NAME>`")
 The parameters submitted in the post are passed to the SQL statement.
 NOTE: Only dataset collections using a SQL store are supported. The SQL
 needs to conform the SQL dialect of the store being used (e.g. MySQL, Postgres,
-SQLite3). The SQL statement functions with the same contrainst of dsquery SQL
-statements.
+SQLite3). The SQL statement functions with the same contraints of dsquery SQL
+statements. The SQL statement is defined as a YAML text blog.
 
 ## API Permissions
 
@@ -223,6 +222,6 @@ In the shell session where datasetd is running press "ctr-C"
 to terminate the service.
 
 
-datasetd 2.1.13
+datasetd 2.1.12
 
 

dsimporter.1.md

@@ -1,4 +1,4 @@
-%dsimporter(1) dataset user manual | version 2.1.13 6f79b21
+%dsimporter(1) dataset user manual | version 2.1.12 856ffcd
 % R. S. Doiel and Tom Morrell
 % 2024-06-12
 

dsquery.1.md

@@ -1,4 +1,4 @@
-%dsquery(1) dataset user manual | version 2.1.13 6f79b21
+%dsquery(1) dataset user manual | version 2.1.12 856ffcd
 % R. S. Doiel and Tom Morrell
 % 2024-06-12
 

go.mod

@@ -1,24 +1,26 @@
-module github.com/caltechlibrary/dataset/v2
+module github.com/caltechlibrary/dataset
 
-go 1.22
+go 1.22.5
 
 require (
+	github.com/caltechlibrary/dataset/v2 v2.1.13
 	github.com/caltechlibrary/dotpath v0.0.4
 	github.com/caltechlibrary/dsv1 v0.0.0-20220817192039-7c2741c5699d
 	github.com/caltechlibrary/pairtree v1.0.3
 	github.com/caltechlibrary/semver v0.0.0-20220817184719-a504da2d5c6a
 	github.com/glebarez/go-sqlite v1.22.0
-	github.com/go-sql-driver/mysql v1.6.0
+	github.com/go-sql-driver/mysql v1.8.1
 	github.com/lib/pq v1.10.9
+	gopkg.in/yaml.v3 v3.0.1
 )
 
 require (
+	filippo.io/edwards25519 v1.1.0 // indirect
 	github.com/dustin/go-humanize v1.0.1 // indirect
 	github.com/google/uuid v1.5.0 // indirect
 	github.com/mattn/go-isatty v0.0.20 // indirect
 	github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
 	golang.org/x/sys v0.15.0 // indirect
-	gopkg.in/yaml.v3 v3.0.1 // indirect
 	modernc.org/libc v1.37.6 // indirect
 	modernc.org/mathutil v1.6.0 // indirect
 	modernc.org/memory v1.7.2 // indirect