diff --git a/.env.example b/.env.example new file mode 100644 index 00000000..9e0aaddc --- /dev/null +++ b/.env.example @@ -0,0 +1,7 @@ +## Default: 0.4.27 +UV_IMG_VERSION= +## Default: 3.12-slim +PYTHON_IMG_VERSION= + +## Default: 8000 +MKDOCS_HTTP_PORT=8002 diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..bece8b2e --- /dev/null +++ b/Dockerfile @@ -0,0 +1,63 @@ +## Add build args. +# You can pass different args for these values in a docker-compose.yml +# file's build: section +ARG UV_BASE=${UV_IMG_VER:-0.4.27} +ARG PYTHON_BASE=${PYTHON_IMG_VER:-3.12-slim} +ARG NGINX_BASE=${NGINX_IMG_VER:-alpine} + +FROM ghcr.io/astral-sh/uv:$UV_BASE as uv +FROM python:$PYTHON_BASE AS base +## Add astral.sh/uv to container's /bin/ path +COPY --from=uv /uv /bin/ + +## Set environment variables. These will be passed +# to stages that inherit from this layer +ENV PYTHONDONTWRITEBYTECODE 1 \ + PYTHONUNBUFFERED 1 + +## Set CWD in container +WORKDIR /project + +## Copy project files & install with uv +COPY pyproject.toml uv.lock ./ +RUN uv sync --all-extras --dev && \ + uv pip install . + +## Build layer to install system dependencies, copy scripts, +# setup container users, etc +FROM base AS build + +RUN apt-get update -y && \ + apt-get install -y --no-install-recommends git + +WORKDIR /project + +## Copy remaining project files, i.e. source code +COPY ./docs ./docs +COPY ./includes ./includes +COPY ./mkdocs.yml ./mkdocs.yml + +## Build the mkdocs site +RUN uv run mkdocs build + +## Runtime layer +FROM build AS run + +COPY --from=build /project /project + +WORKDIR /project + +## Expose a port from a service inside the container +EXPOSE 8000 + +## Run a command/script inside the container +ENTRYPOINT ["uv", "run", "mkdocs", "serve", "--dev-addr", "0.0.0.0:8000"] + +## Serve the static site with nginx +FROM nginx:${NGINX_BASE} AS serve + +COPY --from=build /project/site /usr/share/nginx/html + +EXPOSE 80 + +ENTRYPOINT ["nginx", "-g", "daemon off;"] diff --git a/dev.docker-compose.yml b/dev.docker-compose.yml new file mode 100644 index 00000000..ea565298 --- /dev/null +++ b/dev.docker-compose.yml @@ -0,0 +1,21 @@ +--- +services: + mkdocs: + container_name: redkb-mkdocs + restart: unless-stopped + build: + context: . + dockerfile: Dockerfile + args: + UV_BASE: ${UV_IMG_VERSION:-0.4.27} + PYTHON_BASE: ${PYTHON_IMG_VERSION:-3.12-slim} + ## Layer that runs mkdocs serve + target: run + volumes: + - ./docs:/project/docs + - ./includes:/project/includes + # expose: + # - 8000 + ports: + - ${MKDOCS_HTTP_PORT:-8000}:8000 + \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 00000000..e08cf82d --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,22 @@ +--- +services: + mkdocs: + container_name: redkb-mkdocs + restart: unless-stopped + build: + context: . + dockerfile: Dockerfile + args: + UV_BASE: ${UV_IMG_VERSION:-0.4.27} + PYTHON_BASE: ${PYTHON_IMG_VERSION:-3.12-slim} + ## Layer that serves site/ with nginx + target: serve + volumes: + - ./docs:/project/docs + - ./includes:/project/includes + - ./.git:/project/.git + # expose: + # - 80 + ports: + - ${NGINX_HTTP_PORT:-80}:80 + \ No newline at end of file diff --git a/docs/programming/docker/index.md b/docs/programming/docker/index.md deleted file mode 100644 index e96ab177..00000000 --- a/docs/programming/docker/index.md +++ /dev/null @@ -1,390 +0,0 @@ ---- -tags: - - docker ---- - -# Docker - -Notes, links, & reference code for Docker/Docker Compose. - -!!!warning - - In progress... - -!!!TODO - - - [x] Add sections for things that took me entirely too long to learn/understand - - [x] Multistage builds - - [x] How to target specific layers, i.e. `dev` vs `prod` - - [x] Common Docker commands, how to interpret/modify them - - [x] Docker build - - [x] Docker run - - [x] `ENV` vs `ARG` - - [x] `EXPOSE` - - [x] `CMD` vs `ENTRYPOINT` vs `RUN` - - [ ] Add section(s) for Docker Compose - - [ ] Add an example `docker-compose.yml` - - [ ] Detail required vs optional sections (i.e. `version` (required) and `volumes` (optional)) - - [ ] Links (with `depends_on`) - - [ ] Networking - - [ ] Internal & external networks - - [ ] Proxying - - [ ] Exposing ports (and when you don't need to/shouldn't) - -## How to use Docker build layers (multistage builds) - -You can take advantage of Docker's [BuildKit](https://docs.docker.com/build/buildkit/), which caches Docker layers so subsequent rebuilds with `docker build` are much faster. BuildKit works by keeping a cache of the "layers" in your Docker container, rebuilding a layer only if changes have been made. What this means in practice is that you can separate the steps you use to build your container into stages like `base`, `build`, and `run`, and if nothing in your `build` layer has changed (i.e. no new dependencies added), that layer will not be rebuilt. - -### Example: layered Dockerfile - -In this example, I am building a simple Python app inside a Docker container. The Python code itself does not matter for this example. - -To illustrate the differences in a multistage Dockerfile, let's start with a "flat" Dockerfile, and modify it with build layers. This is the basic Dockerfile: - -```dockerfile title="Example flat Dockerfile" linenums="1" - -## Start with a Python 3.11 Docker base -FROM python:3.11-slim as base - -## Set ENV variables to control Python/pip behavior inside container -ENV PYTHONDONTWRITEBYTECODE 1 \ - PYTHONUNBUFFERED 1 \ - ## Pip - PIP_NO_CACHE_DIR=off \ - PIP_DISABLE_PIP_VERSION_CHECK=on \ - PIP_DEFAULT_TIMEOUT=100 - -## Set the CWD inside the container -WORKDIR /app - -## Copy Python requirements.txt file into container & install dependencies -COPY requirements.txt requirements.txt -RUN pip install -r requirements.txt - -## Copy project source from host into container -COPY ./src . - -## Expose port 8000, which we can pretend the Python app uses to serve the application -EXPOSE 8000 -## Run the Python app -CMD ["python", "main.py"] - -``` - -In this example, any changes to the code or dependencies will cause the entire container to rebuild each time. This is slow & inefficient, and leads to a larger container image. We can break these stages into multiple build layers. In the example below, the container is built in 3 "stages": `base`, `build`, and `run`: - -```dockerfile title="Example multistage Dockerfile" linenums="1" - -## Start with the python:3.11-slim Docker image as your "base" layer -FROM python:3.11-slim as base - -## Set ENV variables to control Python/pip behavior inside container -ENV PYTHONDONTWRITEBYTECODE 1 \ - PYTHONUNBUFFERED 1 \ - ## Pip - PIP_NO_CACHE_DIR=off \ - PIP_DISABLE_PIP_VERSION_CHECK=on \ - PIP_DEFAULT_TIMEOUT=100 - -## Create a "build" layer, where you setup your Python environment -FROM base AS build - -## Set the CWD inside the container -WORKDIR /app - -## Copy Python requirements.txt file into container & install dependencies -COPY requirements.txt requirements.txt -RUN pip install -r requirements.txt - -## Inherit from the build layer -FROM build AS run - -## Set the CWD inside the container -WORKDIR /app - -## Copy project source from host into container -COPY ./src . - -## Expose port 8000, which we can pretend the Python app uses to serve the application -EXPOSE 8000 -## Run the Python app -CMD ["python", "main.py"] -``` - -Layers: - -- `base`: The base layer provides a common environment for the rest of the layers. - - In this example, we set `ENV` variables, which persist across layers - - In contrast, `ARG` lines can be set per-layer, and will need to be re-set for each new layer. This example does not use any `ARG` lines, but be aware that build arguments you set with `ARG` are only present for the layer they are declared in. If you create a new layer and want to access the same argument, you will need to set the `ARG` value again in the new layer -- `build`: The build layer is where you install your Python dependencies. - - You can also install system packages in this layer with `apt`/`apt-get` - - The `python:3.11-slim` base image is built on Debian. If you are using a different Dockerfile, i.e. `python:3.11-alpine`, use the appropriate package manager (i.e. `apk` for Alpine, `rpm` for Fedora/OpenSuSE, etc) to install packages in the `build` layer -- `run`: Finally, the run layer executes the code built in the previous `base` & `build` steps. It also exposes port `8000` inside the container to the host, which can be mapped with `docker run -p 1234:8000`, where `1234` is the port on your host you want to map to port `8000` inside the container. - -Using this method, each time you run `docker build` after the first, only layers that have changed in some way will trigger a rebuild. For example, if you add a Python dependency with `pip install ` and update the `requirements.txt` file with `pip freeze > requirements.txt`, the `build` layer will be rebuilt. If you make changes to your Python application, the `run` layer will be rebuilt. Each layer that does not need to be rebuilt reduces the overall build time of the container, and only the `run` layer will be saved as your image, leading to smaller Docker images. - -### Example: Targeting a specific Dockerfile build stage - -With multistage builds, you can also create a `dev` and `prod` layer, which you can target with `docker run` or a `docker-compose.yml` file. This allows you to build the development & production version of an application using the same Dockerfile. - -Let's modify the multistage Dockerfile example from above to add a `dev` and `prod` layer. Modifications to the multistage Dockerfile include adding an `ENV` variable for storing the app's environment (`dev`/`prod`). In my projects, I use [`Dynaconf`](https://dynaconf.com) to manage app configurations depending on my environment. Dynaconf allows you to set an `ENV` variable called `$ENV_FOR_DYNACONF` so you can control app configurations per-environment ([Dynaconf environment docs](https://www.dynaconf.com/envvars/)). - -```dockerfile title="Example multistage Dockerfile with dev/prod env" linenums="1" - -FROM python:3.11-slim as base - -## Set ENV variables to control Python/pip behavior inside container -ENV PYTHONDONTWRITEBYTECODE 1 \ - PYTHONUNBUFFERED 1 \ - ## Pip - PIP_NO_CACHE_DIR=off \ - PIP_DISABLE_PIP_VERSION_CHECK=on \ - PIP_DEFAULT_TIMEOUT=100 - -FROM base AS build - -WORKDIR /app - -COPY requirements.txt requirements.txt -RUN pip install -r requirements.txt - -## Use target: dev to build this step -FROM build AS dev - -## Set the Dynaconf env to dev -ENV ENV_FOR_DYNACONF=dev -## Tell Dynaconf to always load from the environment first while in the container -ENV DYNACONF_ALWAYS_LOAD_ENV_VARS=True - -WORKDIR /app -COPY ./src . - -############ -# Optional # -############ -# Export ports, set an entrypoint/CMD, etc -# Note: This is normally handled by your orchestrator (docker-compose, Azure Container App, etc). -# If you are buliding/running the container directly, uncomment the EXPOSE & COMMAND lines below - -# EXPOSE 5000 -# CMD ["python", "main.py"] - -## Use target: prod to build this step -FROM build AS prod - -## Set the Dynaconf env to prod -ENV ENV_FOR_DYNACONF=prod -## Tell Dynaconf to always load from the environment first while in the container -ENV DYNACONF_ALWAYS_LOAD_ENV_VARS=True - -WORKDIR /app -COPY ./src . - -############ -# Optional # -############ -# Export ports, set an entrypoint/CMD, etc -# Note: This is normally handled by your orchestrator (docker-compose, Azure Container App, etc) -# If you are buliding/running the container directly, uncomment the EXPOSE & COMMAND lines below - -# EXPOSE 5000 -# CMD ["python", "main.py"] - -``` - -With this multistage Dockerfile, you can target a specific layer with `docker built --target ` (i.e. `docker build --target dev`). This will run through the `base` and `build` layers, but skip the `prod` layer. - -You can also target a specific layer in a `docker-compose.yml` file: - -```yml title="Example multistage docker-compose file" linenums="1" - -version: "3.8" - -services: - - my_app: - container_name: my-application - restart: unless-stopped - build: - ## The "root" directory for Docker compose. If your Dockerfile/project - # are in a subdirectory, specify it here. - context: . - ## Set the name/path to the Dockerfile, keeping in mind the context you set above - dockerfile: Dockerfile - ## Target the "dev" layer of the Dockerfile - target: dev - ## Set the working directory inside the container to /app - working_dir: /app - ## Set the command to run inside the container. Equivalent to CMD in the Dockerfile - command: python main.py - volumes: - ## Mount the project's code directory in the container so changes don't require a rebuild - - ./src:/app - ports: - ## Expose port 8000 in the container, set to port 80 on the host - - 80:8000 - ... -``` - -The example `docker-compose.yml` file above demonstrates targeting the `dev` layer of the multistage Dockerfile above it. We also set the entrypoint (instead of using `CMD` in the Dockerfile), and expose port `8000` in the container. - -### ENV vs ARG in a Dockerfile - -The `ENV` and `ARG` commands in a Dockerfile can be used to control how an image is built and how it functions when live. The differences between an `ENV` and an `ARG` are outlined below. - -!!! note - - This list is **not** a complete comparison between `ENV` and `ARG`. For more information, please check the [`Docker build documentation`](https://docs.docker.com/reference/dockerfile/) guide. - -- `ENV` - - Define environment variables for the container. - - Can be accessed the same way you would on a host, with `$ENV_VAR_NAME`. - - Can be set/overridden with `docker build -e`, or the `environment:` stanza in a `docker-compose.yml` file. - - Available during both the `build` and `run` phases when building a container. - - When building a container (`docker build` or `docker compose build`), `ENV` variables will always use the value declared in the `Dockerfile`. - - At runtime (i.e. when running `docker run` or `docker compose up`), the values can be overridden with `docker run -e/--env` or the `environment:` stanza in a `docker-compose.yml` file. -- `ARG` - - Define environment variables that are only available at build time. - - Values may be overridden *while building*, i.e. between layers or after a command runs. - - Can be set/overridden with `docker build --build-arg ARG_NAME=value`, or the `build: args:` stanza in a `docker-compose.yml` file - -Example: - -```dockerfile title="ENV vs ARG" linenums="1" - -FROM python:3.11-slim AS base - -## This env variable will be available in the build layer -ENV PYTHONDONTWRITEBYTECODE 1 - -## Define a required ARG, without setting its value. The build will fail if this arg is not passed -ARG SOME_VAR -## Define an ARG and set a default value -ARG SOME_OTHER_VAR_ARG=1.0 - -## Set an ENV value, using an ARG's value, to make it available throughout the rest of the build -ENV SOME_OTHER_VAR $SOME_OTHER_VAR_ARG - -FROM base AS build - -## Re-define SOME_OTHER_VAR_ARG from the SOME_OTHER_VAR ENV variable. -# The ENV variable carries into the build layer, but the ARG defined -# in the base layer is not. -ARG SOME_OTHER_VAR_ARG=$SOME_OTHER_VAR - -``` - -Build `ARGS` are useful for setting things like a software version number, i.e. when downloading a specific software release from `Github`. You can set a build arg for the release version, i.e. `ARG RELEASE_VER`, and provide it at buildtime with `docker build --build-arg RELEASE_VER=1.2.3`, or in a `docker-compose.yml` file like: - -```yml title="Example build arg stanza" linenums="1" - -... - -services: - - service1: - build: - context: . - args: - RELEASE_VER: 1.2.3 - -... - -``` - -`ENV` variables, meanwhile, can store things like a database password or some other secret, or configurations for the app. - -```yml title="Example ENV vars stanza" linenums="1" - -... - -services: - - service1: - container_name: service1 - restart: unless-stopped - build: - ... - ... - environment: - ## Load $RELEASE_VERSION from the host's environment or a .env file - RELEASE_VER: ${RELEASE_VERSION:-1.2.3} - -... - -``` - -### Exposing container ports - -In previous examples you have seen the `EXPOSE` line in a Dockerfile. This command exposes a network port from within the container to the host. This is useful if your containerized application utilizes network ports (i.e. running a web frontend on port `8000`), and you are running the container directly with `docker run` instead of through an orchestrator like Docker Compose or Kubernetes. - -!!! note - - When using an orchestrator like `docker-compose`, `kubernetes`, `hashicorp nomad`, etc, it is not necessary (and often counterproductive) to - define `EXPOSE` lines in a Dockerfile. It is better to define port binds between the host and container using the orchestrator's capabilities, - i.e. the `ports:` stanza of a `docker-compose.yml` file. - - When building & running a container image locally or without an orchestrator, you can add these sections to a Dockerfile so when you run the built - container image, you can bind ports with `docker run -p $HOST_PORT:$CONTAINER_PORT`. - -Example: - -```dockerfile title="Example EXPOSE syntax" linenums="1" - -... - -FROM build AS run - -... - -## Expose port 8000 in the container to the host running this container image -EXPOSE 8000 - -## Start a Uvicorn server inside the container. The web server runs on port 8000 (by default) -CMD ["uvicorn", "main:app", "--reload"] - - -``` - -After building this container, you can run it and bind to a port on the host (i.e. port `80`) with `docker run -rm -p 80:8000 ...`, or by specifying the port binding in a `docker-compose.yml` file - -!!!warning - - If you are using Docker Compose, comment/remove the `EXPOSE` and `CMD` lines in your container and pass the values in through Docker Compose - -```yml title="Docker compose port binds" linenums="1" - -... - -services: - - service1: - ... - ## Set the container startup command here, instead of with RUN in the Dockerfile - command: uvicorn main:app --reload - ports: - ## Serve the container application running on port 8000 in the container - # over port 80 on the host. - - 80:8000 - -``` - -### CMD vs RUN vs ENTRYPOINT - -- `RUN` - - Execute when an image is built. The command defined with `RUN` is executed on top of the current base image. - - Example: Installing the `neovim` container inside of a Dockerfile built on top of `ubuntu:latest` image: `RUN apt-get update -y && apt-get install -y neovim` - - Commands defined with `RUN` show their output in the console as the container is built, but are not executed when the built container image is run with `docker run` or `docker compose up` -- `CMD` - - Execute when the container is starting. - - Commands defined with `CMD` execute when you run the container with `docker run` or `docker compose up` - - These commands *do not* execute if a different command is passed, i.e. with `docker run myimage cat log.txt` - - The `cat log.txt` command overrides the `CMD` defined in the container - - **IMPORTANT**: Only the last `CMD` defined in your image is executed. If you specify more than one, all but the last `CMD` will execute. - - The `CMD` command supersedes `ENTRYPOINT`, and should almost always be used instead of `ENTRYPOINT`. -- `ENTRYPOINT` - - `ENTRYPOINT` functions almost the same way as `CMD`, but should be used only when extending an existing image (i.e. `nginx`, `tomcat`, etc) - - Providing an `ENTRYPOINT` to an existing container will change the way that container executes, running the underlying Dockerfile logic with an ad-hoc command you provide. - - The entrypoint for a container can be overridden with `docker run --entrypoint` - - If you define a `CMD` and an `ENTRYPOINT`, the `CMD` line will be provided as arguments to the `ENTRYPOINT`, meaning you can do things like `cat` a file with a `CMD`, then "pipe" the command's output into an `ENTRYPOINT` - - In general, it is best practice to use `CMD` in your Dockerfiles, unless you are aware of and fully understand a reason to use `ENTRYPOINT` instead. diff --git a/docs/programming/docker/my_containers/automation/index.md b/docs/programming/docker/my_containers/automation/index.md deleted file mode 100644 index eb1ae458..00000000 --- a/docs/programming/docker/my_containers/automation/index.md +++ /dev/null @@ -1 +0,0 @@ -... diff --git a/docs/programming/index.md b/docs/programming/index.md index 4c44d8c3..2284d361 100644 --- a/docs/programming/index.md +++ b/docs/programming/index.md @@ -2,4 +2,4 @@ Notes & code snippets I want to remember/reference. -Use the sections along the left to read through my notes and see code examples. You are in the `Programming` section, but might want to check out my [`Standard project files` for Docker](standard-project-files/Docker/index.md) (as an example). +Use the sections along the left to read through my notes and see code examples. diff --git a/docs/programming/standard-project-files/Docker/index.md b/docs/programming/standard-project-files/Docker/index.md deleted file mode 100644 index 8129c883..00000000 --- a/docs/programming/standard-project-files/Docker/index.md +++ /dev/null @@ -1,11 +0,0 @@ ---- -tags: - - standard-project-files - - docker ---- - -# Standard Docker files - -I use Docker a lot. I'm frequently copying/pasting Dockerfiles from previous projects to modify for my current project. Here, I will add some of my more common Dockerfiles, with notes & code snippets for reference. - -This section covers some of the Dockerfiles I frequently re-use. diff --git a/docs/template/docker/automation/index.md b/docs/template/docker/automation/index.md new file mode 100644 index 00000000..ac7b3e0f --- /dev/null +++ b/docs/template/docker/automation/index.md @@ -0,0 +1,8 @@ +--- +tags: + - docker + - templates + - automation +--- + +# Automation templates diff --git a/docs/programming/docker/my_containers/automation/n8n/index.md b/docs/template/docker/automation/n8n/index.md similarity index 97% rename from docs/programming/docker/my_containers/automation/n8n/index.md rename to docs/template/docker/automation/n8n/index.md index 35344d0f..83e43a1c 100644 --- a/docs/programming/docker/my_containers/automation/n8n/index.md +++ b/docs/template/docker/automation/n8n/index.md @@ -1,3 +1,11 @@ +--- +tags: + - docker + - templates + - automation + - nocode +--- + # n8n [`n8n`](https://n8n.io) is a workflow automation tool. Similar to Zapier, `n8n` can integrate with many sources and provide a no/low code (javascript) interface for building your own automations. diff --git a/docs/programming/docker/my_containers/automation/prefect_server/index.md b/docs/template/docker/automation/prefect_server/index.md similarity index 97% rename from docs/programming/docker/my_containers/automation/prefect_server/index.md rename to docs/template/docker/automation/prefect_server/index.md index afd9b51a..e21e2f12 100644 --- a/docs/programming/docker/my_containers/automation/prefect_server/index.md +++ b/docs/template/docker/automation/prefect_server/index.md @@ -1,3 +1,12 @@ +--- +tags: + - docker + - templates + - automation + - python + - data +--- + # Prefect [`prefect`](https://www.prefect.io) is a Python library for automating data workflows/pipelines. A lighter, more beginner-friendly analogue to [Apache `airflow`](https://airflow.apache.org). diff --git a/docs/programming/docker/my_containers/databases/index.md b/docs/template/docker/databases/index.md similarity index 62% rename from docs/programming/docker/my_containers/databases/index.md rename to docs/template/docker/databases/index.md index 412613ba..8a0a64c9 100644 --- a/docs/programming/docker/my_containers/databases/index.md +++ b/docs/template/docker/databases/index.md @@ -1,3 +1,10 @@ +--- +tags: + - docker + - templates + - database +--- + # Databases Docker containers for databases like PostgreSQL, MySQL/MariaDB, InfluxDB, and more. diff --git a/docs/programming/docker/my_containers/databases/influxdb/index.md b/docs/template/docker/databases/influxdb/index.md similarity index 98% rename from docs/programming/docker/my_containers/databases/influxdb/index.md rename to docs/template/docker/databases/influxdb/index.md index 390e4198..f7c7bc88 100644 --- a/docs/programming/docker/my_containers/databases/influxdb/index.md +++ b/docs/template/docker/databases/influxdb/index.md @@ -1,3 +1,11 @@ +--- +tags: + - docker + - templates + - database + - influxdb +--- + # InfluxDB Dockerized InfluxDB database server. InfluxDB is a special purpose time-series database. diff --git a/docs/programming/docker/my_containers/databases/mariadb/index.md b/docs/template/docker/databases/mariadb/index.md similarity index 94% rename from docs/programming/docker/my_containers/databases/mariadb/index.md rename to docs/template/docker/databases/mariadb/index.md index ad5dd001..1264ec7f 100644 --- a/docs/programming/docker/my_containers/databases/mariadb/index.md +++ b/docs/template/docker/databases/mariadb/index.md @@ -1,3 +1,12 @@ +--- +tags: + - docker + - templates + - database + - mariadb + - mysql +--- + # MariaDB Dockerized MariaDB database server. diff --git a/docs/programming/docker/my_containers/databases/postgres/index.md b/docs/template/docker/databases/postgres/index.md similarity index 97% rename from docs/programming/docker/my_containers/databases/postgres/index.md rename to docs/template/docker/databases/postgres/index.md index 2decbc31..e6444db5 100644 --- a/docs/programming/docker/my_containers/databases/postgres/index.md +++ b/docs/template/docker/databases/postgres/index.md @@ -1,3 +1,11 @@ +--- +tags: + - docker + - templates + - database + - postgres +--- + # Postgresql Dockerized PostgreSQL database server. diff --git a/docs/programming/docker/my_containers/databases/redis/index.md b/docs/template/docker/databases/redis/index.md similarity index 96% rename from docs/programming/docker/my_containers/databases/redis/index.md rename to docs/template/docker/databases/redis/index.md index 72f27b1e..7cba9b09 100644 --- a/docs/programming/docker/my_containers/databases/redis/index.md +++ b/docs/template/docker/databases/redis/index.md @@ -1,3 +1,11 @@ +--- +tags: + - docker + - templates + - database + - redis +--- + # Redis Dockerized Redis database server. Redis is a key/value store that can act as a powerful cache or in-memory database. diff --git a/docs/programming/docker/my_containers/my_containers.md b/docs/template/docker/index.md similarity index 65% rename from docs/programming/docker/my_containers/my_containers.md rename to docs/template/docker/index.md index e83ccfb4..1ff21cbc 100644 --- a/docs/programming/docker/my_containers/my_containers.md +++ b/docs/template/docker/index.md @@ -1,8 +1,9 @@ --- tags: - docker + - templates --- -# My Containers +# Docker container templates Doc pages for specific containers/container stacks I use. diff --git a/docs/programming/docker/my_containers/portainer/portainer.md b/docs/template/docker/portainer/index.md similarity index 99% rename from docs/programming/docker/my_containers/portainer/portainer.md rename to docs/template/docker/portainer/index.md index e4e85ac8..4af6f1a4 100644 --- a/docs/programming/docker/my_containers/portainer/portainer.md +++ b/docs/template/docker/portainer/index.md @@ -2,6 +2,7 @@ tags: - docker - portainer + - template --- # Portainer diff --git a/docs/programming/standard-project-files/Docker/python_docker.md b/docs/template/docker/python/index.md similarity index 71% rename from docs/programming/standard-project-files/Docker/python_docker.md rename to docs/template/docker/python/index.md index bb228836..fd04a7e5 100644 --- a/docs/programming/standard-project-files/Docker/python_docker.md +++ b/docs/template/docker/python/index.md @@ -1,6 +1,6 @@ --- tags: - - standard-project-files + - templates - python - docker --- @@ -121,4 +121,73 @@ COPY ./src . # EXPOSE 5000 # CMD ["python", "main.py"] -``` \ No newline at end of file +``` + +## Multistage astral/uv Python Dockerfile + +The [Astral.sh `uv` Python project manager](https://docs.astral.sh/uv/guides/integration/docker/) can be used inside of a Dockerfile. In your "base" stage (or at the top of the Dockerfile, if you're not doing a multistage build), you can import `uv` by adding a `COPY` line to your Dockerfile. + +```Dockerfile title="Python uv Dockerfile" linenums="1" +FROM python:3.12-slim AS base +## Import uv from Astral's Docker container, add uv to /bin/ +COPY --from=ghcr.io/astral-sh/uv:latest /uv /bin/ + +... +``` + +After adding `uv` to your Dockerfile, it can be used in other stages. + +```Dockerfile title="Python multistage Dockerfile with uv" linenums="1" +ARG PYTHON_BASE=3.12-slim +ARG UV_BASE=0.4.27 +FROM python:$PYTHON_BASE AS base +## Add astral.sh/uv to container's /bin/ path +COPY --from=ghrc.io/astral-sh/uv:$UV_BASE /uv /bin/ + +## Set environment variables. These will be passed +# to stages that inherit from this layer +ENV PYTHONDONTWRITEBYTECODE 1 \ + PYTHONUNBUFFERED 1 + +## Set CWD in container +WORKDIR /project + +## Copy project files & install with uv +COPY pyproject.toml uv.lock ./ +RUN uv sync --all-extras --dev && \ + uv pip install . + +## Build layer to install system dependencies, copy scripts, +# setup container users, etc +FROM base AS build + +WORKDIR /project + +## Install system dependencies +RUN apt-get update -y && \ + apt-get install -y --no-install-recommends dos2unix + +## Copy an entrypoint script & set executable +COPY ./scripts/docker-entrypoint.sh ./entrypoint.sh +## Replace line endings +RUN sed -i 's/\r$//g' ./entrypoint.sh && \ + dos2unix ./entrypoint.sh && \ + chmod +x ./entrypoint.sh + +## Copy remaining project files, i.e. source code +COPY ./src ./src + +## Runtime layer +FROM build AS run + +COPY --from=build /project /project + +WORKDIR /project + +## Expose a port from a service inside the container +# EXPOSE 8000 + +## Run a command/script inside the container +ENTRYPOINT ["./entrypoint.sh"] + +``` diff --git a/docs/template/index.md b/docs/template/index.md new file mode 100644 index 00000000..e3e14a46 --- /dev/null +++ b/docs/template/index.md @@ -0,0 +1,9 @@ +--- +tags: + - docker + - templates +--- + +# Templates + +Copy/paste-able code for Docker containers, Python scripts, & more. diff --git a/docs/windows/index.md b/docs/windows/index.md new file mode 100644 index 00000000..5f5eaf4e --- /dev/null +++ b/docs/windows/index.md @@ -0,0 +1,9 @@ +--- +tags: + - windows + +--- + +# Windows + +Notes/guides for Windows troubleshooting, administration, and development. diff --git a/docs/windows/wsl/backup-and-restore.md b/docs/windows/wsl/backup-and-restore.md new file mode 100644 index 00000000..62a8907d --- /dev/null +++ b/docs/windows/wsl/backup-and-restore.md @@ -0,0 +1,40 @@ +--- +tags: + - windows + - wsl +--- + +# Backup & Restore WSL distributions + +WSL distributions can be backed up and restored using the `--export` and `--import` flags. This is useful for moving WSL distributions to new machines, creating backups before making modifications, and for restoring from "base" images. + +The WSL backup files are in `.tar` format. + +## Backup WSL + +Backing up a WSL distribution involves compressing the entire virtual disk to a `.tar` file. The general command format is: `wsl --export C:\path\to\.tar`. + +```powershell title="Backup distribution named 'debian' to C:\wsl_backups" linenums="1" +wsl --export debian C:\wsl_backups\debian.tar +``` + +## Restore WSL + +Restoring a WSL distribution involves decompressing an existing `.tar` file into a full clone of the source. The general command format is: `wsl --import C:\path\to\new-distribution-name C:\path\to\.tar` + +```powershell title="Create distribution named debian-new from C:\wsl_backups\debian.tar" linenums="1" +wsl --import debian-new C:\wsl\debian-new C:\wsl_backups\debian.tar +``` + +## Clone existing WSL into new image + +A useful practice is to create a "base" image that you can use to spawn off new WSL distributions as-needed. If you want a specific set of configurations, installed packages, etc, you can create a WSL distribution where you make all of these changes, then take a backup which you can repeatedly restore from for new distributions. + +For example, say you have a WSL distribution named `deb-base`. In this distribution, you have modified the `/etc/wsl.conf` file, installed `git`, `docker`, and `neovim`. You've modified the `~/.config/nvim` directory, setting up a customized environment for the `neovim` app. You've also installed `tmux` and modified `~/.tmux.config`. + +You do not want to repeat these steps each time you create a new WSL distribution. Pretend you now want to create a distribution named `deb-pydev`, where you will install [Astral's `uv` project manager](https://astral.sh/uv). + +- First, create a backup of `deb-base` + - `wsl --export deb-base C:\wsl_backups\deb-base.tar` +- Then, create a new distribution named `deb-pydev`, with the WSL VM's data stored in `C:\wsl\deb-pydev` + - `wsl --import deb-pydev C:\wsl\deb-pydev C:\wsl_backup\deb-base.tar` diff --git a/docs/windows/wsl/configure-wsl.md b/docs/windows/wsl/configure-wsl.md new file mode 100644 index 00000000..da1ae3f3 --- /dev/null +++ b/docs/windows/wsl/configure-wsl.md @@ -0,0 +1,191 @@ +--- +tags: + - windows + - wsl +--- + +# Configuring WSL + +WSL configuration is done in 1 of 2 places: + +- On the host side, editing `C:\Users\\.wslconfig` + - This file configures the WSL machine, and applies to all distributions installed. + - This is where you set options like `guiApplications`, `localhostForwarding`, etc. +- On the WSL side, by editing `/etc/wsl.conf` + - Configures options for the specific distribution you edit this file from. + - Set options like the default user, enable systemd, and more + +## Host configurations (C:\Users\\.wslconfig) + +Global configurations + +- [Microsoft Docs: Global WSL configuration](https://learn.microsoft.com/en-us/windows/wsl/wsl-config#wslconfig) + +### Enable GUI apps in WSL + +You can add support for graphical programs (instead of just a Bash CLI) by enabling `guiApplications` + +```conf title="Enable GUI support for WSL" linenums="1" +[wsl2] +guiApplications=true +``` + +### Enable/disable Windows firewall rules in WSL + +WSL can use the Windows Firewall rules when `firewall` is enabled. + +```conf title="Enable Windows Firewall rules in WSL" linenums="1" +[wsl2] +firewall=true +``` + +```conf title="Disable Windows Firewall rules in WSL" linenums="1" +[wsl2] +firewall=false +``` + +### Limit WSL memory + +```conf title="Limit global WSL memory" linenums="1" +[wsl2] +memory=4GB +``` + +### Set WSL swap amount + +```conf title="Set global WSL swap memory" linenums="1" +[wsl2] +swap=8GB +``` + +You can also set a swap file disk on the host. The default is `%USERPROFILE%\AppData\Local\Temp\swap.vhdx`. + +```conf title="Set swap file" linenums="1" +swapfile=C:\\temp\\wsl-swap.vhdx +``` + +### Disable WSL page reporting + +Disabling page reporting for WSL causes it to retain all allocated memory claimed from Windows, releasing none back when free. **NOT RECOMMENDED** + +```conf title="Disable page reporting" linenums="1" +[wsl2] +pageReporting=false +``` + +### Forward Windows host network connection to WSL + +Turn on default connection to bind WSL 2 localhost to Windows localhost. Setting is ignored when `networkingMode=mirrored` + +```conf title="Forward host localnet" linenums="1" +[wsl2] +localhostforwarding=true +``` + +### Enable/disable nested virtualization, i.e. Docker in WSL + +```conf title="Enable nested virtualization" linenums="1" +[wsl2] +nestedVirtualization=true +``` + +```conf title="Disable nested virtualization" linenums="1" +[wsl2] +nestedVirtualization=false +``` + +## WSL distribution configurations (/etc/wsl.conf) + +Configurations per-distribution. + +- [Microsoft Docs: wsl-config](https://learn.microsoft.com/en-us/windows/wsl/wsl-config#wslconf) + +### Disable joining Windows path + +WSL will attempt to join the Windows `PATH` variable with its own `$PATH`. This can lead to unexpected behavior, like if `pyenv` is installed in both Windows and WSL. + +To fix this, disable the `appendWindowsPath` flag in the `[interop]` section of `/etc/wsl.conf` + +```conf title="Disable Windows PATH join" linenums="1" +## /etc/wsl.conf + +... + +[interop] +appendWindowsPath = false + +... + +``` + +### Set default user + +!!! note + + To set a default user inside the WSL distribution, the user account must exist. When you first run a WSL container, you will be prompted to create a user account. + + You can create additional users with the `useradd` command: + + ```bash title="Create new Linux user in WSL container" linenums="1" + sudo adduser + ``` + +```conf title="Set default WSL user" linenums="1" +[user] +default= +``` + +### Enable systemd + +```conf title="Enable systemd in WSL" linenums="1" +[boot] +systemd=true +``` + +### Enable/disable automounting of Windows drives + +By default, Windows will mount all fixed drives (i.e. `C:\`, `D:\`, etc) in the container at `/mnt/`. This feature can be controlled with the `enabled` flag in `[automount]` + +```conf title="Enable automounting Windows drives in WSL" linenums="1" +[automount] +enabled=true +``` + +```conf title="Disable automounting Windows drives in WSL" linenums="1" +[automount] +enabled=false +``` + +### Control mounts from within WSL's /etc/fstab + +To mount extra paths inside the WSL container, i.e. an SMB share, you can modify the `/etc/fstab` the same way you would on a "full" Linux install, but you must also enable `mountFsTab`. + +```conf title="Enable auto-mount WSL's /etc/fstab" linenums="1" +[automount] +mountFsTab=true +``` + +```conf title="Disable auto-mount WSL's /etc/fstab" linenums="1" +[automount] +mountFsTab=false +``` + +### Control default root directory + +When starting up a WSL distribution, your terminal's CWD will be the path you ran `wsl` from in Windows. For example, if you are in `C:\Users\` and run `wsl`, the WSL distribution's prompt will be `/mnt/c/Users/`. + +The default WSL root directory is `/mnt`. To set a different path, edit the `root` flag in the `[automount]` section. + +```conf title="Change default WSL root directory" linenums="1" +[automount] +root=/home/ +``` + +### Set a hostname for WSL distribution + +By default, the WSL distribution's hostname will be the same as the Windows host. This can be modified by changing the `hostname` flag in the `[network]` section. + +```conf title="Set WSL hostname" linenums="1" +[network] +hostname="your-hostname" +``` diff --git a/docs/windows/wsl/index.md b/docs/windows/wsl/index.md new file mode 100644 index 00000000..7bec8ef9 --- /dev/null +++ b/docs/windows/wsl/index.md @@ -0,0 +1,12 @@ +--- +tags: + - windows + - linux + - wsl +--- + +# Windows Subsystem for Linux (WSL) + +Windows Subsystem for Linux (WSL) is a utility for running Linux under Windows. It's an impressive project that works very well, allowing you to have a nearly fully-featured Bash prompt (including systemd, if enabled) on a Windows machine. + +VSCode even supports developing inside a WSL machine, using the Remote Explorer plugin. diff --git a/docs/windows/wsl/install-wsl.md b/docs/windows/wsl/install-wsl.md new file mode 100644 index 00000000..22d4e572 --- /dev/null +++ b/docs/windows/wsl/install-wsl.md @@ -0,0 +1,66 @@ +--- +tags: + - windows + - wsl + - linux +--- + +# Install WSL on Windows + +- [📚 Microsoft Docs: Install WSL](https://learn.microsoft.com/en-us/windows/wsl/install) + +!!! note + + Most recent versions of Windows will come with `wsl` pre-installed. Run the `wsl --version` command to check if you already have WSL. + + If you see a version when you run `wsl --version`, you can simply run `wsl --install` to install an Ubuntu image. If you want to use a different version of Linux, you can run `wsl --install `. + + For example, to install Debian: + + ```powershell title="Install a WSL distribution" linenums="1" + wsl --install debian + ``` + + Run with no distribution name to install Ubuntu + + ```powershell title="Install Ubuntu in WSL" linenums="1" + wsl --install + ``` + + To change the default distribution that executes when you run `wsl` with no `-d `, use `--setdefault` + + ```powershell title="Set default distribution for WSL" linenums="1" + wsl --setdefault + ``` + +## Install on older versions of Windows + +On older versions of Windows, if this command fails, you can install `wsl` with the following steps: + +- Enable Windows Subsystem for Linux: + +```powershell title="Enable WSL" linenums="1" +dism.exe /online /enable-feature /featurename:Microsoft-Windows-Subsystem-Linux /all /norestart +``` + +- Enable virtual machine feature + +```powershell title="Enable virtual machine" linenums="1" +dism.exe /online /enable-feature /featurename:VirtualMachinePlatform /all /norestart +``` + +- Download and install the WSL2 Linux kernel update package + - [WSL2 Linux Kernel Download](https://wslstorestorage.blob.core.windows.net/wslblob/wsl_update_x64.msi) + - Run the installer after download + +- Set WSL version 2 as default + +```powershell title="Set WSL2 as default version" linenums="1" +wsl --set-default-version 2 +``` + +- Install a distribution, i.e. Debian + +```powershell title="Install Debian Linux in WSL" linenums="1" +wsl --install -d Debian +``` diff --git a/docs/windows/wsl/troubleshooting.md b/docs/windows/wsl/troubleshooting.md new file mode 100644 index 00000000..075a10a6 --- /dev/null +++ b/docs/windows/wsl/troubleshooting.md @@ -0,0 +1,94 @@ +--- +tags: + - windows + - wsl +--- + +# Troubleshooting WSL + +When I run into issues with WSL and solve them, I add the problem & solution to this page. Notes may be sparse, I normally only care about what I need to copy/paste to get things up and running. + +## Problem: Signature errors while working with Azure libraries + +While working with Azure libraries in WSL, you might see errors about signatures. Try the following. + +### Solution: Set ntpdate + +Edit `/etc/wsl.conf` and modify the `[boot]` section as follows. If you do not see a `[boot]` section, simply create it, and if any of the options below are already present, do not repeat them: + +```conf title="Modify /etc/wsl.conf to fix Azure tool signature errors" linenums="1" +## /etc/wsl.conf + +... + +[boot] +systemd=true +command="ntpdate ntp.ubuntu.com" + +... +``` + +After modifying this file you will need to restart WSL. You can do this with `wsl --shutdown`, then re-launching your WSL distribution. + +## Problem: Ping doesn't work in WSL + +When trying to run `ping` in a WSL distribution, you may see an error like this: + +```bash title="Ping error in WSL container" linenums="1" +ping: socktype: SOCK_RAW +ping: socket: Operation not permitted +ping: => missing cap_net_raw+p capability or setuid? +``` + +The problem here is the line `missing cap_net_raw+p capability`. + +### Fix: Add cap_net_raw permission + +In the WSL container, run this comand: + +```bash title="Add cap_net_raw capability to WSL distribution" linenums="1" +sudo setcap cap_net_raw+p /bin/ping +``` + +### Fix for all WSL2 distributions + +To fix this for all distributions, you can modify the `kernelCommandLine` flag in the `[wsl2]` section of `%USERPROFILE\.wslconfig`. + +```config title="Fix ping for all WSL2 distributions" linenums="1" +## %USERPROFILE\.wslconfig + +[wsl2] +kernelCommandLine = sysctl.net.ipv4.ping_group_range=\"0 2147483647\" +``` + +## Problem: WSL is completely frozen + +This happens sometimes when I'm using the VSCode remote extension to connect to a WSL distribution and the computer goes to sleep. WSL becomes completely responsive, ignoring all commands including `wsl --shutdown`. + +### Fix: Kill the wslservice.exe task + +```powershell title="Kill wslservice.exe" linenums="1" +taskkill /f /im wslservice.exe +``` + +## Problem: Git is not working in WSL + +When `git` is installed on both the Windows and WSL side, you will often run into errors, specifically around authentication. + +### Fix: Use Windows git credential manager in WSL + +Run the following 2 commands, the first on the Windows side and the second in a WSL distribution. + +```powershell title="Run on Windows host" linenums="1" +git config --global credential.helper wincred +``` + +```bash title="Run in WSL distribution" linenums="1" +git config --global credential.helper "/mnt/c/Program\ Files/Git/mingw64/bin/git-credential-manager.exe +``` + +If your Git remote is Azuree DevOps, you also need to run: + +```bash title="Enable support for Azure DevOps repositories" linenums="1" +git config --global credential.https://dev.azure.com.useHttpPath true +``` diff --git a/noxfile.py b/noxfile.py index fd05dc66..dc8e2939 100644 --- a/noxfile.py +++ b/noxfile.py @@ -276,7 +276,7 @@ def new_docker_template_page(session: nox.Session): log.info("Answer the prompts to create a new page in docs/programming/docker/my_containers") COOKIECUTTER_TEMPLATE_FILE: Path = Path("./templates/docs/containers/docker_template_page") - DOCKER_CONTAINER_DOCS_ROOT: Path = Path("./docs/programming/docker/my_containers") + DOCKER_CONTAINER_DOCS_ROOT: Path = Path("./docs/template/docker") if not COOKIECUTTER_TEMPLATE_FILE.exists(): log.warning(f"Could not find cookiecutter template at path '{COOKIECUTTER_TEMPLATE_FILE}'.") @@ -291,7 +291,14 @@ def new_docker_template_page(session: nox.Session): if not CONTAINER_SECTION.exists(): # mkdir_choice = input(f"[WARNING] Container directory section '{CONTAINER_SECTION}' does not exist. Create directory now? (Y/N): ") - log.warning(f"Could not find section '{CONTAINER_SECTION}'.") + log.warning(f"Could not find section '{CONTAINER_SECTION}'. Creating path '{CONTAINER_SECTION}'") + try: + CONTAINER_SECTION.mkdir(parents=True, exist_ok=True) + except Exception as exc: + msg = f"({type(exc)}) Error creating section '{CONTAINER_SECTION}'. Details: {exc}" + log.error(msg) + + raise exc break