merging main with gh-pages

INSTALL.md

@@ -25,6 +25,13 @@ curl https://caltechlibrary.github.io/dataset/installer.sh | sh
 
 This will install dataset and datasetd in your `$HOME/bin` directory.
 
+If you are running Windows 10 or 11 there is a experimental
+installer you can run from Powershell.
+
+~~~
+irm https://caltechlibrary.github.io/dataset/installer.pwsh | iex
+~~~
+
 Compiled version
 ----------------
 

Makefile

@@ -44,7 +44,7 @@ EXT_WEB = .wasm
 
 DIST_FOLDERS = bin/* man/*
 
-build: version.go $(PROGRAMS) man CITATION.cff about.md installer.sh
+build: version.go $(PROGRAMS) man CITATION.cff about.md installer.sh installer.pwsh
 
 version.go: .FORCE
 	@echo '' | pandoc --from t2t --to plain \
@@ -87,10 +87,15 @@ about.md: codemeta.json .FORCE
 	echo "" | pandoc --metadata title="About $(PROGRAM)" --metadata-file=_codemeta.json --template=codemeta-md.tmpl >about.md
 
 installer.sh: .FORCE
-	@echo '' | pandoc --metadata title="Installer" --metadata git_org_or_person="$(GIT_GROUP)" --metadata-file codemeta.json --template codemeta-installer.tmpl >installer.sh
+	@echo '' | pandoc --metadata title="Installer" --metadata git_org_or_person="$(GIT_GROUP)" --metadata-file codemeta.json --template codemeta-bash-installer.tmpl >installer.sh
 	chmod 775 installer.sh
 	git add -f installer.sh
 
+installer.pwsh: .FORCE
+	@echo '' | pandoc --metadata title="Installer" --metadata git_org_or_person="$(GIT_GROUP)" --metadata-file codemeta.json --template codemeta-pwsh-installer.tmpl >installer.pwsh
+	chmod 775 installer.pwsh
+	git add -f installer.pwsh
+
 # NOTE: on macOS you must use "mv" instead of "cp" to avoid problems
 install: build
 	@if [ ! -d $(PREFIX)/bin ]; then mkdir -p $(PREFIX)/bin; fi
@@ -213,6 +218,7 @@ distribute_docs:
 	cp -v LICENSE dist/
 	cp -v INSTALL.md dist/
 	cp installer.sh dist/
+	cp installer.pwsh dist/
 	cp -vR man dist/
 
 update_version:

TODO.html

@@ -31,7 +31,7 @@ <h2 id="bugs">Bugs</h2>
 forming a dsn for SQL store dataset collections using SQLite3, MySQL and
 PostgreSQL from docs/init.md</label></li>
 </ul>
-<h2 id="next-prep-for-v2.1.1">Next (prep for v2.1.1)</h2>
+<h2 id="next-prep-for-v2.1.14">Next (prep for v2.1.14)</h2>
 <ul class="task-list">
 <li><label><input type="checkbox" />Need to add getting updated Man
 pages using the <code>dataset help ...</code> command</label></li>
@@ -92,12 +92,13 @@ <h2 id="next-prep-for-v2.1.1">Next (prep for v2.1.1)</h2>
 <li><label><input type="checkbox" />update</label>
 <ul>
 <li>replace the object in the collection for given key, return false is
-object does not to replace or replacement fails</li>
+object does not to replace or replacement fails
+<ul>
 <li>if collection has versioning turned on then version the object</li>
+</ul></li>
 <li><label><input type="checkbox" /><code>update()</code> update the
 current record respecting the version settings for
 collection</label></li>
-</ul></li>
 <li><label><input type="checkbox" checked="" />delete</label>
 <ul>
 <li>delete the object in the collection for given key, return true if
@@ -177,6 +178,7 @@ <h2 id="next-prep-for-v2.1.1">Next (prep for v2.1.1)</h2>
 <li><label><input type="checkbox" />has_frame</label></li>
 </ul></li>
 </ul></li>
+</ul></li>
 <li><label><input type="checkbox" />Verbs supported by cli only</label>
 <ul class="task-list">
 <li><label><input type="checkbox" />set_versioning (accepts ““,”patch”,
@@ -235,6 +237,9 @@ <h2 id="next-prep-for-v2.1.1">Next (prep for v2.1.1)</h2>
 </ul>
 <h2 id="someday-maybe">Someday, Maybe</h2>
 <ul class="task-list">
+<li><label><input type="checkbox" />Look at <a
+href="https://github.com/metacall/golang-typescript-example">Metacall</a>
+and consider TypeScrit integration into dataset/datasetd</label></li>
 <li><label><input type="checkbox" />Allow a WASM module to be used to
 validate objects in the collection. It needs to me integrate such that
 it “travels” will the dataset collection</label>

TODO.md

@@ -7,112 +7,113 @@ Bugs
 
 - [x] `dataset help init` should include examples of forming a dsn for SQL store dataset collections using SQLite3, MySQL and PostgreSQL from docs/init.md
 
-Next (prep for v2.1.1)
-----------------------
+Next (prep for v2.1.14)
+-----------------------
 
 - [ ] Need to add getting updated Man pages using the `dataset help ...` command
 - [ ] Need to map path parts to parameter sequence for calling sql functions in datasetd
 - [ ] My current approach to versioning is too confusing, causing issues in implementing py_dataset, versioning needs to be automatic with a minimum set of methods explicitly supporting it otherwise versioning should just happen in the back ground and only be supported at the package and libdataset levels.
-    - [ ] create, read, update, list operations should always reflect the "current" version (objects or attachments), delete should delete all versions of objects as should prune for attachments, this is because versioning suggests things never really get deleted, just replaced.
+  - [ ] create, read, update, list operations should always reflect the "current" version (objects or attachments), delete should delete all versions of objects as should prune for attachments, this is because versioning suggests things never really get deleted, just replaced.
 - [ ] Common dataset verbs (dataset/datasetd)
-    - [X] keys
-        - list the keys in a collection
-        - at the package level keys returns a list of keys and an error value
-    - [X] has_key
-        - return "true "(w/OS exit 0 in CLI) if key is in collection,
-          "false" otherwise (w/OS exit 1 in CLI)
-    - [ ] sample
-        - return a sample of keys from a collection
-        - [ ] the newly create collections should have versioning disabled by default
-    - [ ] create
-        - add an new object to the collection if key does not exist,
-          return false if object already exists or unable to create
-          the new object
-        - if versioning is enabled set the semver appropriately
-    - [ ] read
-        - return the object with nil error in the collection with the
-          provided key, nil object and error value if not found
-        - read always returns to the "current" object version
-        - [ ] `read_versions()`, list the versions available for JSON object
-        - [ ] `read_version()` list an JSON object for a specific version
-             - return the object with nil error in the collection with the
-               provided key and version, nil object and error value if not
-               found
-    - [ ] update
-        - replace the object in the collection for given key, return false
-          is object does not to replace or replacement fails
-        - if collection has versioning turned on then version the object
-        - [ ] `update()` update the current record respecting the version settings for collection
+  - [X] keys
+    - list the keys in a collection
+    - at the package level keys returns a list of keys and an error value
+  - [X] has_key
+    - return "true "(w/OS exit 0 in CLI) if key is in collection,
+      "false" otherwise (w/OS exit 1 in CLI)
+  - [ ] sample
+    - return a sample of keys from a collection
+    - [ ] the newly create collections should have versioning disabled by default
+  - [ ] create
+    - add an new object to the collection if key does not exist,
+      return false if object already exists or unable to create
+      the new object
+    - if versioning is enabled set the semver appropriately
+  - [ ] read
+    - return the object with nil error in the collection with the
+      provided key, nil object and error value if not found
+    - read always returns to the "current" object version
+    - [ ] `read_versions()`, list the versions available for JSON object
+    - [ ] `read_version()` list an JSON object for a specific version
+      - return the object with nil error in the collection with the
+        provided key and version, nil object and error value if not
+        found
+  - [ ] update
+    - replace the object in the collection for given key, return false
+      is object does not to replace or replacement fails
+      - if collection has versioning turned on then version the object
+    - [ ] `update()` update the current record respecting the version settings for collection
     - [X] delete
-        - delete the object in the collection for given key, return true
-          if deletion was successful, false if the object was not deleted
-          (e.g. key not found or the collection is read only)
-        - if collection has versioning turned on then delete **all objects**, if you want to revert you just update the object with the revised object values
-        - [ ] `delete()` delete all versions of an object
-        - If versioning is enabled the idea of "deleting" an object or attachment doesn't make sense, you only need to support Create, Read, Update and List, possibly with the ability to read versions available and retrieve the specific version, is this worth implementing in the CLI? Or is this just a lib dataset/package "feature"?
+      - delete the object in the collection for given key, return true
+        if deletion was successful, false if the object was not deleted
+        (e.g. key not found or the collection is read only)
+      - if collection has versioning turned on then delete **all objects**, if you want to revert you just update the object with the revised object values
+      - [ ] `delete()` delete all versions of an object
+      - If versioning is enabled the idea of "deleting" an object or attachment doesn't make sense, you only need to support Create, Read, Update and List, possibly with the ability to read versions available and retrieve the specific version, is this worth implementing in the CLI? Or is this just a lib dataset/package "feature"?
     - [ ] versioning, versioning is now set for the whole collection and effects JSON objects and their attachments (you're versioning both or neither), versioning will auto-increment for patch, minir and major semvere values if set
-        - [ ] `set_versioning()`, set the versioning on a collection, the following strings enable
+      - [ ] `set_versioning()`, set the versioning on a collection, the following strings enable
           versioning "major", "minor", "patch". Any other value disables
           versioning on the collection
-        - [ ] `get_versioning()` on a colleciton (should return "major", "minor", "patch" or "")
+      - [ ] `get_versioning()` on a colleciton (should return "major", "minor", "patch" or "")
     - [ ] Attachment support
-        - [ ] `attach()` will add a basename file to the JSON object record, if versioning is enabled then it needs to handle the appropraite versioning setting
-        - [ ] `attachments()` lists the attachments for a JSON object record
-        - [ ] `attachment_versions()` list versions of a specific attachment
-        - [ ] `detach()` retrieve "current" version of attachment
-        - [ ] `detach_version()` retrieve a specific version of attachment
-        - [ ] `prune()` remove all versions of attachments
+      - [ ] `attach()` will add a basename file to the JSON object record, if versioning is enabled then it needs to handle the appropraite versioning setting
+      - [ ] `attachments()` lists the attachments for a JSON object record
+      - [ ] `attachment_versions()` list versions of a specific attachment
+      - [ ] `detach()` retrieve "current" version of attachment
+      - [ ] `detach_version()` retrieve a specific version of attachment
+      - [ ] `prune()` remove all versions of attachments
     - [ ] Data Frame Support
-        - [ ] frame_names
-            - list the names of the frames currently defined in the collection
-        - [ ] frame
-            - define a new frame in the collection, if frame exists replace it
-        - [ ] frame_meta
-            - return the frame definition and metadata about the frame (e.g.  how many objects and attributes)
-        - [ ] frame_objects
-            - return the frame's list of objects
-        - [ ] refresh
-            - update all the objects in the frame based on current state of
-              the collection
-        - [ ] reframe
-            - replace the frame definition but using the existing frame's keys
-              refresh the frame with the new object describe
-        - [ ] delete_frame
-        - [ ] has_frame
+      - [ ] frame_names
+        - list the names of the frames currently defined in the collection
+      - [ ] frame
+        - define a new frame in the collection, if frame exists replace it
+      - [ ] frame_meta
+        - return the frame definition and metadata about the frame (e.g.  how many objects and attributes)
+      - [ ] frame_objects
+        - return the frame's list of objects
+      - [ ] refresh
+        - update all the objects in the frame based on current state of
+          the collection
+      - [ ] reframe
+        - replace the frame definition but using the existing frame's keys
+          refresh the frame with the new object describe
+      - [ ] delete_frame
+      - [ ] has_frame
 - [ ] Verbs supported by cli only
-    - [ ] set_versioning (accepts "", "patch", "minor", or "major" as values)
-    - [ ] get_versioning (returns collection's version setting)
-    - [ ] keys
-    - [ ] create (if versioning is enable then handle versioning)
-    - [ ] read (if versioning is enabled return the current version of an object)
-    - [ ] update (if versioning is enable then handle versioning)
-    - [ ] delete (if versioning is enable, delete all versions of object and attachments)
-    - [ ] sample
-    - [ ] clone
-    - [ ] clone-sample
-    - [ ] check
-    - [ ] repair
-    - [ ] frames (return a list of frames defined for collection)
-    - [ ] frame, define a new frame in a collection
-    - [ ] frame_objects, return the object list from a frame
-    - [ ] refresh, refresh all objects in a frame based on the current state of the collection
-    - [ ] reframe, replace the frame definition but using the existing frame's keys for the object listed in frame
-    - [ ] delete_frame, remove a frame from the collection
-    - [ ] has_frame return true if frame exists or false otherwise
-    - [ ] attachments, list the attachments for a JSON object in the collection
-    - [ ] attach, add an attachment to a JSON object in the collection, respect versioning if enabled
-    - [ ] detach, retrieve an attachment from the JSON object in the collection
-    - [ ] prune, remove attachments (including all versions) from an JSON object in the collection
+  - [ ] set_versioning (accepts "", "patch", "minor", or "major" as values)
+  - [ ] get_versioning (returns collection's version setting)
+  - [ ] keys
+  - [ ] create (if versioning is enable then handle versioning)
+  - [ ] read (if versioning is enabled return the current version of an object)
+  - [ ] update (if versioning is enable then handle versioning)
+  - [ ] delete (if versioning is enable, delete all versions of object and attachments)
+  - [ ] sample
+  - [ ] clone
+  - [ ] clone-sample
+  - [ ] check
+  - [ ] repair
+  - [ ] frames (return a list of frames defined for collection)
+  - [ ] frame, define a new frame in a collection
+  - [ ] frame_objects, return the object list from a frame
+  - [ ] refresh, refresh all objects in a frame based on the current state of the collection
+  - [ ] reframe, replace the frame definition but using the existing frame's keys for the object listed in frame
+  - [ ] delete_frame, remove a frame from the collection
+  - [ ] has_frame return true if frame exists or false otherwise
+  - [ ] attachments, list the attachments for a JSON object in the collection
+  - [ ] attach, add an attachment to a JSON object in the collection, respect versioning if enabled
+  - [ ] detach, retrieve an attachment from the JSON object in the collection
+  - [ ] prune, remove attachments (including all versions) from an JSON object in the collection
 - [ ] Document example Shell access to datasetd via cURL
 - [ ] take KeyMap out of collection.json so collection.json is smaller
-    - support for segmented key maps (to limit memory consumption for very
-      large collections)
+  - support for segmented key maps (to limit memory consumption for very
+    large collections)
 - [ ] Auto-version attachments by patch, minor or major release per
       settings in collection.json using keywords of patch, minor, major
 
 Someday, Maybe
 --------------
 
+- [ ] Look at [Metacall](https://github.com/metacall/golang-typescript-example) and consider TypeScrit integration into dataset/datasetd
 - [ ] Allow a WASM module to be used to validate objects in the collection. It needs to me integrate such that it "travels" will the dataset collection
   - this would let our JSON collections support explicit JSON structures as well as ad-hoc JSON objects
 - [ ] Review [Go-app](https://go-app.dev/) and see if this would be a way to create a local client UI for working with datasets and enabling LunrJS for search
@@ -131,10 +132,10 @@ Someday, Maybe
       content driven by a dataset collection
 - [ ] Implement version support in the web service
 - [ ] Implement an integrated UI for datasetd
-    - [ ] Form pages could be expressed in Markdown+YAML for forms and embedded in the datasetd settings YAML file
-        - See my notes on my text oreinted web experiment, yaml2webform.go
-        - Forms could be render into the htdocs auto-magically saving development effort
-        - The same forms could then be used server side for validation based on descriptors and JavaScript converted to WASM code
-    - [ ] A standard JavaScript library could be used to knit the forms to the datasetd web service (sort of a mini-newt)
+  - [ ] Form pages could be expressed in Markdown+YAML for forms and embedded in the datasetd settings YAML file
+    - See my notes on my text oreinted web experiment, yaml2webform.go
+    - Forms could be render into the htdocs auto-magically saving development effort
+    - The same forms could then be used server side for validation based on descriptors and JavaScript converted to WASM code
+  - [ ] A standard JavaScript library could be used to knit the forms to the datasetd web service (sort of a mini-newt)
 It would be nice if citesearch was defined by the citesearch.yaml file and some markdown documents taking a text oriented web approach to embedding forms in Markdown combined with some JS glue code to knit the two together
 - [ ] Consider updating datasetd to support urlencoded data submissions in additional to application/json, this might make it easier to quicklt develop browser side UI for datasetd web services

codemeta-pwsh-installer.tmpl

@@ -0,0 +1,53 @@
+#!/usr/bin/env pwsh
+# Generated with codemeta-pwsh-installer.tmpl, see https://github.com/caltechlibrary/codemeta-pandoc-examples
+
+#
+# Set the package name and version to install
+#
+$$PACKAGE = "$name$"
+$$VERSION = "$version$"
+$$GIT_GROUP = "$git_org_or_person$"
+$$RELEASE = "https://github.com/$${GIT_GROUP}/$${PACKAGE}/releases/tag/v$${VERSION}"
+$$SYSTEM_TYPE = Get-ComputerInfo -Property CsSystemType
+if ($$SYSTEM_TYPE.CsSystemType.Contains("ARM64")) {
+    $$MACHINE = "arm64"
+}
+if ($$SYSTEM_TYPE.CsSystemType.Contains("x64")) {
+    $$MACHINE = "x86_64"
+}
+
+# FIGURE OUT Install directory
+$$APP_INSTALL = $$env:$name$.ToUpper()
+$$BIN_DIR = if ($$APP_INSTALL) {
+  "$${APP_INSTALL}\bin"
+} else {
+  "$${Home}\\bin"
+}
+
+#
+# Figure out what the zip file is named
+#
+$$ZIPFILE = "$${PACKAGE}-v$${VERSION}-Windows-$${MACHINE}.zip"
+
+#
+# Check to see if this zip file has been downloaded.
+#
+$$DOWNLOAD_URL = "https://github.com/$${GIT_GROUP}/$${PACKAGE}/releases/download/v$${VERSION}/$${ZIPFILE}"
+
+if (!(Test-Path $$BIN_DIR)) {
+  New-Item $$BIN_DIR -ItemType Directory | Out-Null
+}
+curl.exe -Lo "$${ZIPFILE}" "$${DOWNLOAD_URL}"
+
+tar.exe xf "$${ZIPFILE}"" -C $$BIN_DIR
+
+Remove-Item $$ZIPFILE
+
+$$User = [System.EnvironmentVariableTarget]::User
+$$Path = [System.Environment]::GetEnvironmentVariable('Path', $$User)
+if (!(";$${Path};".ToLower() -like "*;$${BIN_DIR};*".ToLower())) {
+  [System.Environment]::SetEnvironmentVariable('Path', "$${Path};$${BIN_DIR}", $$User)
+  $$Env:Path += ";$${BIN_DIR}"
+}
+
+Write-Output "$${PACKAGE} was installed successfully to $${BIN_DIR}"