From f35e60484e785a6025f68aeaecc0397217070abd Mon Sep 17 00:00:00 2001
From: Dan Allan <dallan@bnl.gov>
Date: Tue, 17 Sep 2024 10:48:18 -0400
Subject: [PATCH 1/4] Add Hello Python and Jupyter

---
 docs/recipes/tutorials/example/demo.md        |  36 ---
 .../hello-python-and-jupyter.md               | 267 ++++++++++++++++++
 .../tutorials/getting-started/hello_world.py  |   2 +
 3 files changed, 269 insertions(+), 36 deletions(-)
 delete mode 100644 docs/recipes/tutorials/example/demo.md
 create mode 100644 docs/recipes/tutorials/getting-started/hello-python-and-jupyter.md
 create mode 100644 docs/recipes/tutorials/getting-started/hello_world.py

diff --git a/docs/recipes/tutorials/example/demo.md b/docs/recipes/tutorials/example/demo.md
deleted file mode 100644
index 1f783f7..0000000
--- a/docs/recipes/tutorials/example/demo.md
+++ /dev/null
@@ -1,36 +0,0 @@
----
-jupytext:
-  text_representation:
-    extension: .md
-    format_name: myst
-    format_version: 0.13
-    jupytext_version: 1.16.4
-kernelspec:
-  display_name: Python 3 (ipykernel)
-  language: python
-  name: python3
----
-
-# Executable Code
-
-## Basics
-
-```{code-cell} ipython3
-1 + 3
-```
-
-```{code-cell} ipython3
-a = 8**2
-```
-
-```{code-cell} ipython3
-a
-```
-
-This cell has an excepted error:
-
-```{code-cell} ipython3
-:tags: [raises-exception]
-
-1 / 0
-```
diff --git a/docs/recipes/tutorials/getting-started/hello-python-and-jupyter.md b/docs/recipes/tutorials/getting-started/hello-python-and-jupyter.md
new file mode 100644
index 0000000..de238a9
--- /dev/null
+++ b/docs/recipes/tutorials/getting-started/hello-python-and-jupyter.md
@@ -0,0 +1,267 @@
+---
+jupytext:
+  text_representation:
+    extension: .md
+    format_name: myst
+    format_version: 0.13
+    jupytext_version: 1.16.4
+kernelspec:
+  display_name: Python 3 (ipykernel)
+  language: python
+  name: python3
+---
+
+# Hello Python and Jupyter
+
+In this notebook you will:
+
+* Learn how to use a Jupyter notebook such as this one
+* Get a very quick tour of Python syntax and scientific libraries
+* Learn some IPython features that we will use in later tutorials
+
+
+## What is this?
+
+This is Jupyter notebook running in a personal "container" just for you, stocked with example data, demos, and tutorials that you can run and modify. All of the software you'll need is already installed and ready to use.
+
+
+## Run some Python code!
+
+To run the code below:
+
+1. Click on the cell to select it.
+2. Press `SHIFT+ENTER` on your keyboard or press the <button class='btn btn-default'><span class='fa fa-step-forward'></span > Run</button> button in the toolbar above (in the toolbar above!).
+
+```{code-cell} ipython3
+1 + 1
+```
+
+Notice that you can edit a cell and re-run it.
+
+
+The notebook document mixes executable code and narrative content. It supports text, links, embedded videos, and even typeset math: $\int_{-\infty}^\infty x\, d x = \frac{x^2}{2}$
+
+
+## Whirlwind Tour of Python Syntax
+
+### Lists
+
+```{code-cell} ipython3
+stuff = [4, 'a', 8.3, True]
+```
+
+```{code-cell} ipython3
+stuff[0]  # the first element
+```
+
+```{code-cell} ipython3
+stuff[-1]  # the last element
+```
+
+### Dictionaries (Mappings)
+
+```{code-cell} ipython3
+d = {'a': 1, 'b': 2}
+```
+
+```{code-cell} ipython3
+d
+```
+
+```{code-cell} ipython3
+d['b']
+```
+
+```{code-cell} ipython3
+d['c'] = 3
+```
+
+```{code-cell} ipython3
+d
+```
+
+**TIP:** For large or nested dictionaries, it is more convient to use `list()`.  Often custom python objects can be interrogated in the same manner.
+
+```{code-cell} ipython3
+list(d)
+```
+
+### Functions
+
+```{code-cell} ipython3
+def f(a, b):
+    return a + b
+```
+
+In IPython `f?` or `?f` display information about `f`, such as its arguments.
+
+```{code-cell} ipython3
+f?
+```
+
+If the function includes inline documentation (a "doc string") then `?` displays that as well.
+
+```{code-cell} ipython3
+def f(a, b):
+    "Add a and b."
+    return a + b
+```
+
+```{code-cell} ipython3
+f?
+```
+
+```{code-cell} ipython3
+f??
+```
+
+Arguments can have default values.
+
+```{code-cell} ipython3
+def f(a, b, c=1):
+    return (a + b) * c
+```
+
+```{code-cell} ipython3
+f(1, 2)
+```
+
+```{code-cell} ipython3
+f(1, 2, 3)
+```
+
+Any argument can be passed by keyword. This is slower to type but clearer to read later.
+
+```{code-cell} ipython3
+f(a=1, b=2, c=3)
+```
+
+If using keywords, you don't have to remember the argument order.
+
+```{code-cell} ipython3
+f(c=3, a=1, b=2)
+```
+
+## Fast numerical computation using numpy
+
+For numerical computing, a numpy array is more useful and performant than a plain list.
+
+```{code-cell} ipython3
+import numpy as np
+
+a = np.array([1, 2, 3, 4])
+```
+
+```{code-cell} ipython3
+a
+```
+
+```{code-cell} ipython3
+np.mean(a)
+```
+
+```{code-cell} ipython3
+np.sin(a)
+```
+
+We'll use the IPython `%%timeit` magic to measure the speed difference between built-in Python lists and numpy arrays.
+
+```{code-cell} ipython3
+%%timeit
+
+big = list(range(10000))  # setup line, not timed
+sum(big)  # timed
+```
+
+```{code-cell} ipython3
+%%timeit
+
+big = np.arange(10000)  # setup line, not timed
+np.sum(big)  # timed
+```
+
+If a single loops is desired for a longer computation, use `%time` on the desired line.
+
+```{code-cell} ipython3
+big = np.arange(10000)  # setup line, not timed
+%time np.sum(big)  # timed
+```
+
+## Plotting using matplotlib
+
+In an interactive setting, this will show a canvas that we can pan and zoom. (Keep reading for what we can do in a non-interactive setting, such as the static web page version of this tutorial.)
+
+```{code-cell} ipython3
+# We just have to do this line once, before we do any plotting.
+%matplotlib widget
+import matplotlib.pyplot as plt
+
+plt.figure()
+```
+
+We can plot some data like so. In an interactive setting, this will update the canvas above.
+
+```{code-cell} ipython3
+plt.plot([1, 1, 2, 3, 5, 8])
+```
+
+And we can show a noninteractive snapshot of the state of the figure at this point by display the figure itself.
+
+```{code-cell} ipython3
+plt.gcf()
+```
+
+Displaying `plt.gcf()` (or any `Figure`) shows a non-interactive snapshot of a figure. Displaying `plt.gcf().canvas` or any `Canvas` gives us another interactive, live-updating view of the figure.
+
+
+## Interrupting the IPython Kernel
+
+Run this cell, and then click the square 'stop' button in the notebook toolbar to interrupt the infinite loop.
+
+(This is equivalent to Ctrl+C in a terminal.)
+
+```{code-cell} ipython3
+# This runs forever -- hit the square 'stop' button in Jupyter to interrupt
+# The following is "commented out". Un-comment the lines below to run them.
+# while True:
+#     continue
+```
+
+## "Magics"
+
+
+The code entered here is interpreted by _IPython_, which extends Python by adding some conveniences that help you make the most out of using Python interactively. It was originally created by a physicist, Fernando Perez.
+
+"Magics" are special IPython syntax. They are not part of the Python language, and they should not be used in scripts or libraries; they are meant for interactive use.
+
+The `%run` magic executes a Python script.
+
+```{code-cell} ipython3
+%run hello_world.py
+```
+
+When the script completes, any variables defined in that script will be dumepd into our namespace. For example (as we will see below), this script happens to define a variable named `message`. Now that we have `%run` the script, `message` is in our namespace.
+
+```{code-cell} ipython3
+message
+```
+
+This behavior can be confusing, in the sense that the reader has to do some digging to figure out where ``message`` was defined and what it is, but it has its uses. Throughout this tutorial, we will use the `%run` magic as a shorthand for running boilerplate configuration code and defining variables representing hardware.
+
+
+The `%load` magic copies the contents of a file into a cell but does not run it.
+
+```{code-cell} ipython3
+%load hello_world.py
+```
+
+Execute the cell a second time to actually run the code. Throughout this tutorial, we use the `%load` magic to load solutions to exercises.
+
+
+## System Shell Access
+
+Any input line beginning with a `!` character is passed verbatim (minus the `!`, of course) to the underlying operating system.
+
+```{code-cell} ipython3
+!ls
+```
diff --git a/docs/recipes/tutorials/getting-started/hello_world.py b/docs/recipes/tutorials/getting-started/hello_world.py
new file mode 100644
index 0000000..72bfc47
--- /dev/null
+++ b/docs/recipes/tutorials/getting-started/hello_world.py
@@ -0,0 +1,2 @@
+message = 'hello world'
+print(message)

From e845bb7cc98fd15456a2781ea7b454a76afd5ab5 Mon Sep 17 00:00:00 2001
From: Dan Allan <dallan@bnl.gov>
Date: Wed, 18 Sep 2024 10:14:47 -0400
Subject: [PATCH 2/4] Remove recipes dir

---
 .github/workflows/cd.yml                      |  6 +--
 convert_all.sh                                |  4 +-
 docs/contributing.md                          | 38 +++++++++----------
 docs/{recipes => }/how-to.md                  |  2 +-
 docs/{recipes => }/how-to/example/demo.md     |  0
 docs/index.md                                 |  7 ++--
 docs/{recipes => }/tutorials.md               |  2 +-
 .../getting-started/hello-bluesky.md          | 16 ++++++++
 .../hello-python-and-jupyter.md               |  0
 .../tutorials/getting-started/hello_world.py  |  0
 test.sh                                       |  2 +-
 11 files changed, 46 insertions(+), 31 deletions(-)
 rename docs/{recipes => }/how-to.md (75%)
 rename docs/{recipes => }/how-to/example/demo.md (100%)
 rename docs/{recipes => }/tutorials.md (80%)
 create mode 100644 docs/tutorials/getting-started/hello-bluesky.md
 rename docs/{recipes => }/tutorials/getting-started/hello-python-and-jupyter.md (100%)
 rename docs/{recipes => }/tutorials/getting-started/hello_world.py (100%)

diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml
index ca93bce..689b269 100644
--- a/.github/workflows/cd.yml
+++ b/.github/workflows/cd.yml
@@ -40,17 +40,17 @@ jobs:
         uses: actions/upload-artifact@v4
         with:
           name: executed-notebooks
-          path: build/jupyter_execute/recipes
+          path: build/jupyter_execute/
           if-no-files-found: error
 
       - name: Prepare for Jupyter Lite
         run: |
-          cp README.md docs/recipes/
+          cp README.md docs/
           # Convert Jupytext to ipynb format.
           pixi run ./convert_all.sh ipynb
 
       - name: Build Jupyter Lite
-        run: pixi run -e jupyterlite jupyter lite build --contents docs/recipes --output-dir build/html/jupyterlite
+        run: pixi run -e jupyterlite jupyter lite build --contents docs/ --output-dir build/html/jupyterlite
 
       - name: Upload HTML as GitHub artifact
         uses: actions/upload-pages-artifact@v3
diff --git a/convert_all.sh b/convert_all.sh
index 0ac64c4..3844fe0 100755
--- a/convert_all.sh
+++ b/convert_all.sh
@@ -10,7 +10,7 @@ fi
 error_occurred=0
 
 if [ "$1" = "ipynb" ]; then
-    files=$(find docs/recipes/ -name "*.md" | grep -v .ipynb_checkpoints)
+    files=$(find docs/ -name "*.md" | grep -v .ipynb_checkpoints)
     for file in $files; do
         # Extract the kernel information from the Jupytext Markdown file
         kernel_info=$(grep -A 10 '^---$' "$file" | grep -E 'kernelspec')
@@ -27,7 +27,7 @@ if [ "$1" = "ipynb" ]; then
         fi
     done
 elif [ "$1" = "md" ]; then
-    files=$(find docs/recipes/ -name "*.ipynb" | grep -v .ipynb_checkpoints)
+    files=$(find docs/ -name "*.ipynb" | grep -v .ipynb_checkpoints)
     for file in $files; do
         jupytext --to markdown "$file" && rm "$file"
         if [ $? -ne 0 ]; then
diff --git a/docs/contributing.md b/docs/contributing.md
index ca6b2e7..dec5078 100644
--- a/docs/contributing.md
+++ b/docs/contributing.md
@@ -28,10 +28,10 @@ git clone git@github.com:bluesky/bluesky-cookbook
 
 ## Overview
 
-Each "recipe" is a directory under `docs/recipes/tutorials/` or
-`docs/recipes/how-to/`. It may contain one or more Markdown (`.md`) files with
-a mixture of narrative text and code. Each recipe directory may also contain
-supporting data files, scripts, illustrations, solutions to exercises, etc.
+Each "recipe" is a directory under `docs/tutorials/` or `docs/how-to/`. It may
+contain one or more Markdown (`.md`) files with a mixture of narrative text and
+code. Each recipe directory may also contain supporting data files, scripts,
+illustrations, solutions to exercises, etc.
 
 ```none
 $ tree docs/
@@ -39,13 +39,14 @@ docs/
 ├── conf.py
 ├── contributing.md
 ├── index.md
-├── recipes
-│   ├── tutorials
-│   │   ├── getting-started
-│   │   │   └── hello-bluesky.md
-│   │   ├── flyscanning
-│   │   │   └── basic-demo.md
-│   ├── how-to
+├── tutorials
+│   ├── getting-started
+│   │   └── hello-bluesky.md
+│   ├── flyscanning
+│   │   └── basic-demo.md
+├── how-to
+│   ├── deploy
+│   │   └── single-process.md
 ...
 ```
 
@@ -90,13 +91,12 @@ Open `build/html/index.html` in a web browser.
 pixi run jupyter lab
 ```
 
-In the file browser, locate one of the examples under `docs/recipes/`. Double
-click to open.
+In the file browser, locate one of the examples under `docs/tutorials/` or
+`docs/how-to/`. Double click to open the directory.
 
-Files like `docs/recipes/static/static.md` are plain Markdown files. Files like
-`docs/recipes/executable/basics.md` have a YAML header which enables the
-documentation build system to convert them to Jupyter notebooks and execute
-them:
+Files like `docs/tutorial/getting-started/hello-bluesky.md` have a YAML header
+which enables the documentation build system to convert them to Jupyter
+notebooks and execute them:
 
 ```yaml
 ---
@@ -132,7 +132,7 @@ The script `test.sh` runs files with executable code from top to bottom and
 prints any unexpected tracebacks.
 
 ```
-pixi run ./test.sh docs/recipes/executable/basics.md
+pixi run ./test.sh docs/tutorials/getting-started/hello-bluesky.md
 ```
 
 `````{note}
@@ -171,5 +171,5 @@ Once changes are merged to the `main` branch, the GitHub Actions [Publish workfl
   where users need to be handed Jupyter Notebook files directly, such as Google
   Colab.
 
-[notebooks-branch]: https://github.com/bluesky/bluesky-cookbook/tree/notebooks/docs/recipes
+[notebooks-branch]: https://github.com/bluesky/bluesky-cookbook/tree/notebooks/
 [Publish workflow]: https://github.com/bluesky/bluesky-cookbook/actions/workflows/cd.yml
diff --git a/docs/recipes/how-to.md b/docs/how-to.md
similarity index 75%
rename from docs/recipes/how-to.md
rename to docs/how-to.md
index 8eb9abd..9039e0c 100644
--- a/docs/recipes/how-to.md
+++ b/docs/how-to.md
@@ -1,4 +1,4 @@
-# How-To Guides
+# How-to Guides
 
 ```{toctree}
 :maxdepth: 2
diff --git a/docs/recipes/how-to/example/demo.md b/docs/how-to/example/demo.md
similarity index 100%
rename from docs/recipes/how-to/example/demo.md
rename to docs/how-to/example/demo.md
diff --git a/docs/index.md b/docs/index.md
index 7291917..154d5a6 100644
--- a/docs/index.md
+++ b/docs/index.md
@@ -1,10 +1,9 @@
 # Bluesky Cookbook
 
 ```{toctree}
-:maxdepth: 1
-:glob:
-recipes/tutorials.md
-recipes/how-to.md
+:maxdepth: 2
+tutorials
+how-to
 glossary
 contributing
 ```
diff --git a/docs/recipes/tutorials.md b/docs/tutorials.md
similarity index 80%
rename from docs/recipes/tutorials.md
rename to docs/tutorials.md
index 15cc312..010f386 100644
--- a/docs/recipes/tutorials.md
+++ b/docs/tutorials.md
@@ -1,7 +1,7 @@
 # Tutorials
 
 ```{toctree}
-:maxdepth: 2
+:maxdepth: 1
 :glob:
 tutorials/**/*
 ```
diff --git a/docs/tutorials/getting-started/hello-bluesky.md b/docs/tutorials/getting-started/hello-bluesky.md
new file mode 100644
index 0000000..08fce5e
--- /dev/null
+++ b/docs/tutorials/getting-started/hello-bluesky.md
@@ -0,0 +1,16 @@
+---
+jupytext:
+  text_representation:
+    extension: .md
+    format_name: myst
+    format_version: 0.13
+    jupytext_version: 1.16.4
+kernelspec:
+  display_name: Python 3 (ipykernel)
+  language: python
+  name: python3
+---
+
+# Hello Bluesky
+
+TODO
diff --git a/docs/recipes/tutorials/getting-started/hello-python-and-jupyter.md b/docs/tutorials/getting-started/hello-python-and-jupyter.md
similarity index 100%
rename from docs/recipes/tutorials/getting-started/hello-python-and-jupyter.md
rename to docs/tutorials/getting-started/hello-python-and-jupyter.md
diff --git a/docs/recipes/tutorials/getting-started/hello_world.py b/docs/tutorials/getting-started/hello_world.py
similarity index 100%
rename from docs/recipes/tutorials/getting-started/hello_world.py
rename to docs/tutorials/getting-started/hello_world.py
diff --git a/test.sh b/test.sh
index 35418d3..6b11e72 100755
--- a/test.sh
+++ b/test.sh
@@ -29,7 +29,7 @@ error_occurred=0
 
 # If --all is passed, locate eligible files and execute them all.
 if [ "$1" == "--all" ]; then
-    files=$(find docs/recipes/ -name "*.md" | grep -v .ipynb_checkpoints)
+    files=$(find docs/ -name "*.md" | grep -v .ipynb_checkpoints)
     for file in $files; do
         if [ -f "$file" ]; then
 	    # Extract the kernel information from the Jupytext Markdown file

From d82a708569fedd6f0d95f124110e9947741b629e Mon Sep 17 00:00:00 2001
From: Dan Allan <dallan@bnl.gov>
Date: Wed, 18 Sep 2024 11:37:23 -0400
Subject: [PATCH 3/4] Fix TOC structure

---
 docs/how-to.md                       |  3 +--
 docs/how-to/example.md               |  6 ++++++
 docs/tutorials.md                    |  7 +++++--
 docs/tutorials/flyscanning.md        |  6 ++++++
 docs/tutorials/flyscanning/basics.md | 16 ++++++++++++++++
 docs/tutorials/getting-started.md    |  6 ++++++
 6 files changed, 40 insertions(+), 4 deletions(-)
 create mode 100644 docs/how-to/example.md
 create mode 100644 docs/tutorials/flyscanning.md
 create mode 100644 docs/tutorials/flyscanning/basics.md
 create mode 100644 docs/tutorials/getting-started.md

diff --git a/docs/how-to.md b/docs/how-to.md
index 9039e0c..b0e3d8e 100644
--- a/docs/how-to.md
+++ b/docs/how-to.md
@@ -2,6 +2,5 @@
 
 ```{toctree}
 :maxdepth: 2
-:glob:
-how-to/**/*
+how-to/example/
 ```
diff --git a/docs/how-to/example.md b/docs/how-to/example.md
new file mode 100644
index 0000000..7b882e5
--- /dev/null
+++ b/docs/how-to/example.md
@@ -0,0 +1,6 @@
+# Example
+
+```{toctree}
+:glob:
+example/*
+```
diff --git a/docs/tutorials.md b/docs/tutorials.md
index 010f386..cdf67f2 100644
--- a/docs/tutorials.md
+++ b/docs/tutorials.md
@@ -1,7 +1,10 @@
 # Tutorials
 
 ```{toctree}
-:maxdepth: 1
+:maxdepth: 2
 :glob:
-tutorials/**/*
+tutorials/getting-started/
+tutorials/flyscanning/
 ```
+
+Test 4
diff --git a/docs/tutorials/flyscanning.md b/docs/tutorials/flyscanning.md
new file mode 100644
index 0000000..258d126
--- /dev/null
+++ b/docs/tutorials/flyscanning.md
@@ -0,0 +1,6 @@
+# Flyscanning
+
+```{toctree}
+:glob:
+flyscanning/*
+```
diff --git a/docs/tutorials/flyscanning/basics.md b/docs/tutorials/flyscanning/basics.md
new file mode 100644
index 0000000..627f754
--- /dev/null
+++ b/docs/tutorials/flyscanning/basics.md
@@ -0,0 +1,16 @@
+---
+jupytext:
+  text_representation:
+    extension: .md
+    format_name: myst
+    format_version: 0.13
+    jupytext_version: 1.16.4
+kernelspec:
+  display_name: Python 3 (ipykernel)
+  language: python
+  name: python3
+---
+
+# Flyscanning Basics
+
+TODO
diff --git a/docs/tutorials/getting-started.md b/docs/tutorials/getting-started.md
new file mode 100644
index 0000000..03c4f0b
--- /dev/null
+++ b/docs/tutorials/getting-started.md
@@ -0,0 +1,6 @@
+# Getting Started
+
+```{toctree}
+:glob:
+getting-started/*
+```

From a79edb3119dd3a646560e2c8ae2ce1b764b991bf Mon Sep 17 00:00:00 2001
From: Dan Allan <dallan@bnl.gov>
Date: Wed, 18 Sep 2024 11:43:38 -0400
Subject: [PATCH 4/4] Detect Jupytext header more robustly.

---
 convert_all.sh | 2 +-
 test.sh        | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/convert_all.sh b/convert_all.sh
index 3844fe0..6d1f329 100755
--- a/convert_all.sh
+++ b/convert_all.sh
@@ -13,7 +13,7 @@ if [ "$1" = "ipynb" ]; then
     files=$(find docs/ -name "*.md" | grep -v .ipynb_checkpoints)
     for file in $files; do
         # Extract the kernel information from the Jupytext Markdown file
-        kernel_info=$(grep -A 10 '^---$' "$file" | grep -E 'kernelspec')
+	kernel_info=$(head -n 15 "$file" | grep -A 10 '^---$' | grep -E 'kernelspec')
         # Skip if no kernel information was found
         if [ -z "$kernel_info" ]; then
             continue
diff --git a/test.sh b/test.sh
index 6b11e72..2f8c5ce 100755
--- a/test.sh
+++ b/test.sh
@@ -33,7 +33,7 @@ if [ "$1" == "--all" ]; then
     for file in $files; do
         if [ -f "$file" ]; then
 	    # Extract the kernel information from the Jupytext Markdown file
-	    kernel_info=$(grep -A 10 '^---$' "$file" | grep -E 'kernelspec')
+	    kernel_info=$(head -n 15 "$file" | grep -A 10 '^---$' | grep -E 'kernelspec')
             # Skip if no kernel information was found
             if [ -z "$kernel_info" ]; then
 		continue