add schema tests to the shell pipeline (#318)

* add schema tests to the shell pipeline

I validated that the task works as expected with fly execute.  Once I
verify it is working as expected in the pipeline I'll add it as a passed
constraint to the publish job.  I'll also move the integ tests to the
test job, that way we will always auto run the tests and publish can be
executed with the latest version that passed the tests (or any selected
version that passed the tests)

concourse/pipeline.yml

@@ -8,6 +8,16 @@ resources:
       branch: main
 
 jobs:
+  - name: test
+    serial: true
+    trigger: true
+    plan:
+      - get: fauna-shell-repository
+      - task: schema-tests
+        file: fauna-shell-repository/concourse/tasks/schema-tests.yml
+        params:
+          FAUNA_SECRET: ((cli_test_secret))
+
   - name: release
     serial: true
     public: false

concourse/tasks/schema-tests.yml

@@ -0,0 +1,26 @@
+---
+platform: linux
+image_resource:
+  type: registry-image
+  source:
+    repository: node
+    tag: 18
+
+params:
+  FAUNA_SECRET:
+
+inputs:
+  - name: fauna-shell-repository
+
+run:
+  path: sh
+  args:
+    - -ec
+    - |
+      npm install -g zx
+      cd fauna-shell-repository
+      yarn install
+      yarn build
+      faunaCmd="$(pwd)/bin/dev"
+      cd fsl
+      ./test-script.mjs "$faunaCmd"

fsl/.fauna-project

@@ -0,0 +1,6 @@
+schema_directory=schema
+default=dev
+
+[stack.dev]
+endpoint=cli_test-us
+database=FaunaCLITest

fsl/schema/collections.fsl

@@ -0,0 +1,6 @@
+collection hi {
+
+}
+collection bye {
+
+}

fsl/schema/sayHelloFunction.fsl

@@ -0,0 +1,3 @@
+function sayHello(name: String): String {
+  "Hello " + name
+}

fsl/test-script.mjs

@@ -0,0 +1,99 @@
+#!/usr/bin/env zx
+
+/**
+  * This is provided in the pipeline to pass in the built version of the
+  * fauna CLI to be tested.
+  */
+let faunaCmd;
+if (argv._.length === 1) {
+  faunaCmd = `${argv._[0]}`
+} else {
+  faunaCmd = "fauna"
+}
+
+const expectedCollNames = ["bye", "hi"]
+const expectedFuncNames = ["sayHello"]
+const secretFlag = process.env.FAUNA_SECRET
+if (secretFlag) {
+  $.verbose = false
+  console.log("A secret was found for FAUNA_SECRET, it will be used for all fauna CLI commands")
+}
+
+
+await ensureClean()
+
+/**
+  * Test Push
+  */
+await execFaunaCmd(["schema", "push", "--force"])
+const collNames = await execPaginated("Collection.all().map(.name).order()")
+if (collNames.length != expectedCollNames.length || !collNames.every((elem, idx) => elem === expectedCollNames[idx])) {
+  throw new Error(`Schema collections do not match actual: ${collNames} expected: ${expectedCollNames}`)
+}
+const funcNames = await execPaginated("Function.all().map(.name)")
+if (funcNames.length != expectedFuncNames.length || !funcNames.every((elem, idx) => elem === expectedFuncNames[idx])) {
+  throw new Error(`Schema functions do not match actual: ${collNames} expected: ${expectedCollNames}`)
+}
+
+await $`rm schema/*`
+
+/**
+ * Test Pull
+ * When using execFaunaCmd writing to stdin in this way
+ * doesn't appear to work...
+ */
+let pullProcess;
+if (secretFlag) {
+  pullProcess = $`${faunaCmd} schema pull --secret ${secretFlag}`
+} else {
+  pullProcess = $`${faunaCmd} schema pull`
+}
+pullProcess.stdin.write('yes\n')
+pullProcess.stdin.end()
+await sleep(10000)
+const schemaFiles = (await $`ls schema`).stdout.trim().split('\n')
+if (schemaFiles.length !== 2 || schemaFiles[0] !== "collections.fsl" || schemaFiles[1] !== "sayHelloFunction.fsl") {
+  throw new Error(`Schema files after pull did not equal expected.  Expected: ['collections.fsl, 'sayHelloFunction.fsl'] Actual: ${schemaFiles}`)
+}
+console.log("Schema tests run successfully!")
+
+async function ensureClean() {
+  await execFQL(`
+    Collection.all().forEach(.delete())
+    Role.all().forEach(.delete())
+    Function.all().forEach(.delete())
+  `)
+  // await execFQL("Collection.create({ name: 'hi' })")
+  const respColls = await execPaginated("Collection.all() { name }")
+  if (!Array.isArray(respColls) || respColls.length != 0) {
+    throw new Error(`Expected empty collection set. ${respColls}`)
+  }
+  const respFunctions = await execPaginated("Function.all() { name }")
+  if (!Array.isArray(respFunctions) || respFunctions.length != 0) {
+    throw new Error(`Expected empty function set. ${resp}`)
+  }
+}
+
+/**
+  * The provided cmd must be an array where each string you want to show up
+  * in the shell is an element.
+  * ex: execFaunaCmd(["schema", "diff"])
+  */
+async function execFaunaCmd(cmd) {
+  if (secretFlag) {
+    return $`${faunaCmd} ${cmd} --secret ${secretFlag}`
+  } else {
+    return $`${faunaCmd} ${cmd}`
+  }
+}
+
+async function execPaginated(fql) {
+  const resp = await execFQL(fql)
+  return resp?.data
+}
+
+async function execFQL(fql) {
+  const resp = secretFlag ? await $`${faunaCmd} eval ${fql} --format=json --secret ${secretFlag}` : await $`${faunaCmd} eval ${fql} --format=json`
+  const respParsed = JSON.parse(resp._stdout)
+  return respParsed
+}