diff --git a/concourse/pipeline.yml b/concourse/pipeline.yml
index 25b496ee..bf53a767 100644
--- a/concourse/pipeline.yml
+++ b/concourse/pipeline.yml
@@ -8,6 +8,16 @@ resources:
       branch: main
 
 jobs:
+  - name: test
+    serial: true
+    trigger: true
+    plan:
+      - get: fauna-shell-repository
+      - task: schema-tests
+        file: fauna-shell-repository/concourse/tasks/schema-tests.yml
+        params:
+          FAUNA_SECRET: ((cli_test_secret))
+
   - name: release
     serial: true
     public: false
diff --git a/concourse/tasks/schema-tests.yml b/concourse/tasks/schema-tests.yml
new file mode 100644
index 00000000..5f897226
--- /dev/null
+++ b/concourse/tasks/schema-tests.yml
@@ -0,0 +1,26 @@
+---
+platform: linux
+image_resource:
+  type: registry-image
+  source:
+    repository: node
+    tag: 18
+
+params:
+  FAUNA_SECRET:
+
+inputs:
+  - name: fauna-shell-repository
+
+run:
+  path: sh
+  args:
+    - -ec
+    - |
+      npm install -g zx
+      cd fauna-shell-repository
+      yarn install
+      yarn build
+      faunaCmd="$(pwd)/bin/dev"
+      cd fsl
+      ./test-script.mjs "$faunaCmd"
diff --git a/fsl/.fauna-project b/fsl/.fauna-project
new file mode 100644
index 00000000..23443d22
--- /dev/null
+++ b/fsl/.fauna-project
@@ -0,0 +1,6 @@
+schema_directory=schema
+default=dev
+
+[stack.dev]
+endpoint=cli_test-us
+database=FaunaCLITest
diff --git a/fsl/schema/collections.fsl b/fsl/schema/collections.fsl
new file mode 100644
index 00000000..aeb79e34
--- /dev/null
+++ b/fsl/schema/collections.fsl
@@ -0,0 +1,6 @@
+collection hi {
+
+}
+collection bye {
+
+}
diff --git a/fsl/schema/sayHelloFunction.fsl b/fsl/schema/sayHelloFunction.fsl
new file mode 100644
index 00000000..52dafe73
--- /dev/null
+++ b/fsl/schema/sayHelloFunction.fsl
@@ -0,0 +1,3 @@
+function sayHello(name: String): String {
+  "Hello " + name
+}
diff --git a/fsl/test-script.mjs b/fsl/test-script.mjs
new file mode 100755
index 00000000..0a4a0362
--- /dev/null
+++ b/fsl/test-script.mjs
@@ -0,0 +1,99 @@
+#!/usr/bin/env zx
+
+/**
+  * This is provided in the pipeline to pass in the built version of the
+  * fauna CLI to be tested.
+  */
+let faunaCmd;
+if (argv._.length === 1) {
+  faunaCmd = `${argv._[0]}`
+} else {
+  faunaCmd = "fauna"
+}
+
+const expectedCollNames = ["bye", "hi"]
+const expectedFuncNames = ["sayHello"]
+const secretFlag = process.env.FAUNA_SECRET
+if (secretFlag) {
+  $.verbose = false
+  console.log("A secret was found for FAUNA_SECRET, it will be used for all fauna CLI commands")
+}
+
+
+await ensureClean()
+
+/**
+  * Test Push
+  */
+await execFaunaCmd(["schema", "push", "--force"])
+const collNames = await execPaginated("Collection.all().map(.name).order()")
+if (collNames.length != expectedCollNames.length || !collNames.every((elem, idx) => elem === expectedCollNames[idx])) {
+  throw new Error(`Schema collections do not match actual: ${collNames} expected: ${expectedCollNames}`)
+}
+const funcNames = await execPaginated("Function.all().map(.name)")
+if (funcNames.length != expectedFuncNames.length || !funcNames.every((elem, idx) => elem === expectedFuncNames[idx])) {
+  throw new Error(`Schema functions do not match actual: ${collNames} expected: ${expectedCollNames}`)
+}
+
+await $`rm schema/*`
+
+/**
+ * Test Pull
+ * When using execFaunaCmd writing to stdin in this way
+ * doesn't appear to work...
+ */
+let pullProcess;
+if (secretFlag) {
+  pullProcess = $`${faunaCmd} schema pull --secret ${secretFlag}`
+} else {
+  pullProcess = $`${faunaCmd} schema pull`
+}
+pullProcess.stdin.write('yes\n')
+pullProcess.stdin.end()
+await sleep(10000)
+const schemaFiles = (await $`ls schema`).stdout.trim().split('\n')
+if (schemaFiles.length !== 2 || schemaFiles[0] !== "collections.fsl" || schemaFiles[1] !== "sayHelloFunction.fsl") {
+  throw new Error(`Schema files after pull did not equal expected.  Expected: ['collections.fsl, 'sayHelloFunction.fsl'] Actual: ${schemaFiles}`)
+}
+console.log("Schema tests run successfully!")
+
+async function ensureClean() {
+  await execFQL(`
+    Collection.all().forEach(.delete())
+    Role.all().forEach(.delete())
+    Function.all().forEach(.delete())
+  `)
+  // await execFQL("Collection.create({ name: 'hi' })")
+  const respColls = await execPaginated("Collection.all() { name }")
+  if (!Array.isArray(respColls) || respColls.length != 0) {
+    throw new Error(`Expected empty collection set. ${respColls}`)
+  }
+  const respFunctions = await execPaginated("Function.all() { name }")
+  if (!Array.isArray(respFunctions) || respFunctions.length != 0) {
+    throw new Error(`Expected empty function set. ${resp}`)
+  }
+}
+
+/**
+  * The provided cmd must be an array where each string you want to show up
+  * in the shell is an element.
+  * ex: execFaunaCmd(["schema", "diff"])
+  */
+async function execFaunaCmd(cmd) {
+  if (secretFlag) {
+    return $`${faunaCmd} ${cmd} --secret ${secretFlag}`
+  } else {
+    return $`${faunaCmd} ${cmd}`
+  }
+}
+
+async function execPaginated(fql) {
+  const resp = await execFQL(fql)
+  return resp?.data
+}
+
+async function execFQL(fql) {
+  const resp = secretFlag ? await $`${faunaCmd} eval ${fql} --format=json --secret ${secretFlag}` : await $`${faunaCmd} eval ${fql} --format=json`
+  const respParsed = JSON.parse(resp._stdout)
+  return respParsed
+}