improve devcontainer experience

CrunchyData · Nov 9, 2024 · e001d31 · e001d31
1 parent 451f347
commit e001d31
Show file tree

Hide file tree

Showing 14 changed files with 228 additions and 113 deletions.
diff --git a/.devcontainer/.env b/.devcontainer/.env
@@ -0,0 +1,11 @@
+# S3 tests
+ AWS_ACCESS_KEY_ID=minioadmin
+ AWS_SECRET_ACCESS_KEY=minioadmin
+ AWS_REGION=us-east-1
+ AWS_S3_TEST_BUCKET=testbucket
+ MINIO_ROOT_USER=minioadmin
+ MINIO_ROOT_PASSWORD=minioadmin
+
+ # Others
+ RUST_TEST_THREADS=1
+ PG_PARQUET_TEST=true
diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile
@@ -6,10 +6,11 @@ ENV TZ="Europe/Istanbul"
 ARG PG_MAJOR=17
 
 # install deps
-RUN apt-get update && apt-get -y install build-essential libreadline-dev zlib1g-dev    \
-                                         flex bison libxml2-dev libxslt-dev libssl-dev \
-                                         libxml2-utils xsltproc ccache pkg-config wget \
-                                         curl lsb-release sudo nano net-tools git awscli
+RUN apt-get update && apt-get -y install build-essential libreadline-dev zlib1g-dev      \
+                                         flex bison libxml2-dev libxslt-dev libssl-dev   \
+                                         libxml2-utils xsltproc ccache pkg-config wget   \
+                                         curl lsb-release ca-certificates gnupg sudo git \
+                                         nano net-tools awscli
 
 # install Postgres
 RUN sh -c 'echo "deb https://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list'
@@ -19,34 +20,20 @@ RUN apt-get update && apt-get -y install postgresql-${PG_MAJOR}-postgis-3  \
                                          postgresql-client-${PG_MAJOR}     \
                                          libpq-dev
 
-# download and install MinIO server and client
-RUN wget https://dl.min.io/server/minio/release/linux-amd64/minio
-RUN chmod +x minio
-RUN mv minio /usr/local/bin/minio
+# set up permissions so that rust user can create extensions
+RUN chmod a+rwx `pg_config --pkglibdir`          \
+                `pg_config --sharedir`/extension \
+                /var/run/postgresql/
 
-# download and install MinIO admin
-RUN wget https://dl.min.io/client/mc/release/linux-amd64/mc
-RUN chmod +x mc
-RUN mv mc /usr/local/bin/mc
-
-# set up pgrx with non-sudo user
+# initdb requires non-root user. This will also be the user that runs the container.
 ARG USERNAME=rust
-ARG USER_UID=501
-ARG USER_GID=$USER_UID
-RUN groupadd --gid $USER_GID $USERNAME \
-    && useradd --uid $USER_UID --gid $USER_GID -s /bin/bash -m $USERNAME
-
-RUN mkdir /workspaces && chown -R $USER_UID:$USER_GID /workspaces
+ARG USER_UID=1000
+ARG USER_GID=1000
+RUN groupadd --gid $USER_GID $USERNAME
+RUN useradd --uid $USER_UID --gid $USER_GID -s /bin/bash -m $USERNAME
 
-# set up permissions so that the user below can create extensions
-RUN chmod a+rwx `pg_config --pkglibdir` \
-    `pg_config --sharedir`/extension \
-    /var/run/postgresql/
+RUN echo "$USERNAME ALL=(ALL) NOPASSWD: ALL" > /etc/sudoers.d/$USERNAME
 
-# add it to sudoers
-RUN echo "$USERNAME ALL=(ALL) NOPASSWD: ALL" > /etc/sudoers.d/$USERNAME 
-
-# now it is time to switch to user
 USER $USERNAME
 
 # install Rust environment
@@ -59,10 +46,9 @@ RUN cargo install --locked cargo-pgrx@${PGRX_VERSION}
 RUN cargo pgrx init --pg${PG_MAJOR} $(which pg_config)
 RUN echo "shared_preload_libraries = 'pg_parquet'" >> $HOME/.pgrx/data-${PG_MAJOR}/postgresql.conf
 
-ENV MINIO_ROOT_USER=admin
-ENV MINIO_ROOT_PASSWORD=admin123
-ENV AWS_S3_TEST_BUCKET=testbucket
-ENV AWS_REGION=us-east-1
-ENV AWS_ACCESS_KEY_ID=admin
-ENV AWS_SECRET_ACCESS_KEY=admin123
-ENV PG_PARQUET_TEST=true
+# required for pgrx to work
+ENV USER=$USERNAME
+
+# git completion
+RUN curl -o ~/.git-completion.bash https://raw.githubusercontent.com/git/git/master/contrib/completion/git-completion.bash
+RUN echo "source ~/.git-completion.bash" >> ~/.bashrc
diff --git a/.devcontainer/create-test-buckets.sh b/.devcontainer/create-test-buckets.sh
@@ -0,0 +1,3 @@
+#!/bin/bash
+
+ aws --endpoint-url http://localhost:9000 s3 mb s3://$AWS_S3_TEST_BUCKET
diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json
@@ -1,7 +1,10 @@
 {
-    "build": {
-        "dockerfile": "Dockerfile"
-    },
+    "name": "pg_parquet Dev Environment",
+    "dockerComposeFile": "docker-compose.yml",
+    "service": "app",
+    "workspaceFolder": "/workspace",
+    "postStartCommand": "bash .devcontainer/create-test-buckets.sh",
+    "postAttachCommand": "sudo chown -R rust /workspace",
     "customizations": {
         "vscode": {
             "extensions": [
@@ -14,12 +17,5 @@
                 "henriiik.docker-linter"
             ]
         }
-    },
-    "postStartCommand": "bash .devcontainer/scripts/setup-minio.sh",
-    "forwardPorts": [
-        5432
-    ],
-    "capAdd": [
-        "SYS_PTRACE"
-    ]
+    }
 }
diff --git a/.devcontainer/docker-compose.yml b/.devcontainer/docker-compose.yml
@@ -0,0 +1,32 @@
+services:
+  app:
+    build:
+      context: .
+      dockerfile: Dockerfile
+    command: sleep infinity
+    network_mode: host
+    volumes:
+      - ..:/workspace
+      - ${USERPROFILE}${HOME}/.ssh:/home/rust/.ssh:ro
+      - ${USERPROFILE}${HOME}/.ssh/known_hosts:/home/rust/.ssh/known_hosts:rw
+      - ${USERPROFILE}${HOME}/.gitconfig:/home/rust/.gitconfig:ro
+      - ${USERPROFILE}${HOME}/.aws:/home/rust/.aws:ro
+    env_file:
+      - .env
+    cap_add:
+      - SYS_PTRACE
+    depends_on:
+      - minio
+
+  minio:
+    image: minio/minio
+    env_file:
+      - .env
+    network_mode: host
+    command: server /data
+    restart: unless-stopped
+    healthcheck:
+      test: ["CMD", "curl", "http://localhost:9000"]
+      interval: 6s
+      timeout: 2s
+      retries: 3
diff --git a/.devcontainer/scripts/setup-minio.sh b/.devcontainer/scripts/setup-minio.sh
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -65,28 +65,25 @@ jobs:
           path: ${{ env.SCCACHE_DIR }}
           key: pg_parquet-sccache-cache-${{ runner.os }}-${{ hashFiles('Cargo.lock', '.github/workflows/ci.yml') }}
 
+      - name: Export environment variables from .env file
+        uses: falti/dotenv-action@v1
+        with:
+          path: .devcontainer/.env
+          export_variables: true
+
       - name: Install PostgreSQL
         run: |
           sudo sh -c 'echo "deb https://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list'
           wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | sudo apt-key add -
           sudo apt-get update
-          sudo apt-get install build-essential libreadline-dev zlib1g-dev flex bison libxml2-dev libxslt-dev libssl-dev libxml2-utils xsltproc ccache pkg-config
+          sudo apt-get -y install build-essential libreadline-dev zlib1g-dev flex bison libxml2-dev \
+                                  libxslt-dev libssl-dev libxml2-utils xsltproc ccache pkg-config   \
+                                  gnupg ca-certificates
           sudo apt-get -y install postgresql-${{ env.PG_MAJOR }}-postgis-3 \
                                   postgresql-server-dev-${{ env.PG_MAJOR }} \
                                   postgresql-client-${{ env.PG_MAJOR }} \
                                   libpq-dev
 
-      - name: Install MinIO
-        run: |
-          # Download and install MinIO server and client
-          wget https://dl.min.io/server/minio/release/linux-amd64/minio
-          chmod +x minio
-          mv minio /usr/local/bin/minio
-
-          # Download and install MinIO admin
-          wget https://dl.min.io/client/mc/release/linux-amd64/mc
-          chmod +x mc
-          mv mc /usr/local/bin/mc
 
       - name: Install and configure pgrx
         run: |
@@ -101,48 +98,32 @@ jobs:
           cargo fmt --all -- --check
           cargo clippy --all-targets --features "pg${{ env.PG_MAJOR }}, pg_test" --no-default-features -- -D warnings
 
-      - name: Run tests
+      - name: Set up permissions for PostgreSQL
         run: |
-          # Set up permissions so that the current user below can create extensions
           sudo chmod a+rwx $(pg_config --pkglibdir)          \
                            $(pg_config --sharedir)/extension \
                            /var/run/postgresql/
 
-          # pgrx tests with runas argument ignores environment variables, so
-          # we read env vars from .env file in tests (https://github.com/pgcentralfoundation/pgrx/pull/1674)
-          touch /tmp/.env
-          echo AWS_ACCESS_KEY_ID=${{ env.AWS_ACCESS_KEY_ID }} >> /tmp/.env
-          echo AWS_SECRET_ACCESS_KEY=${{ env.AWS_SECRET_ACCESS_KEY }} >> /tmp/.env
-          echo AWS_S3_TEST_BUCKET=${{ env.AWS_S3_TEST_BUCKET }} >> /tmp/.env
-          echo AWS_REGION=${{ env.AWS_REGION }} >> /tmp/.env
-          echo PG_PARQUET_TEST=${{ env.PG_PARQUET_TEST }} >> /tmp/.env
+      - name: Start Minio for s3 emulator tests
+        run: |
+          docker run -p 9000:9000 minio/minio server /data
 
-          # Start MinIO server
-          export MINIO_ROOT_USER=${{ env.AWS_ACCESS_KEY_ID }}
-          export MINIO_ROOT_PASSWORD=${{ env.AWS_SECRET_ACCESS_KEY }}
-          minio server /tmp/minio-storage > /dev/null 2>&1 &
+          while ! nc -z localhost 9000; do
+              echo "Waiting for localhost:9000..."
+              sleep 1
+          done
 
-          # Set access key and create test bucket 
-          mc alias set local http://localhost:9000 ${{ env.AWS_ACCESS_KEY_ID }} ${{ env.AWS_SECRET_ACCESS_KEY }}
-          aws --endpoint-url http://localhost:9000 s3 mb s3://${{ env.AWS_S3_TEST_BUCKET }}
+          aws --endpoint-url http://localhost:9000 s3 mb s3://$AWS_S3_TEST_BUCKET
 
+      - name: Run tests
+        run: |
           # Run tests with coverage tool
           source <(cargo llvm-cov show-env --export-prefix)
           cargo llvm-cov clean
           cargo build --features "pg${{ env.PG_MAJOR }}, pg_test" --no-default-features
           cargo pgrx test pg${{ env.PG_MAJOR }} --no-default-features
           cargo llvm-cov report --lcov > lcov.info
 
-          # Stop MinIO server
-          pkill -9 minio
-        env:
-          RUST_TEST_THREADS: 1
-          AWS_ACCESS_KEY_ID: test_secret_access_key
-          AWS_SECRET_ACCESS_KEY: test_access_key_id
-          AWS_REGION: us-east-1
-          AWS_S3_TEST_BUCKET: testbucket
-          PG_PARQUET_TEST: true
-
       - name: Upload coverage report to Codecov
         if: ${{ env.PG_MAJOR }} == 17
         uses: codecov/codecov-action@v4

diff --git a/.gitignore b/.gitignore
@@ -12,5 +12,4 @@
 *.lcov
 *.xml
 lcov.info
-.env
 playground.rs
diff --git a/.vscode/settings.json b/.vscode/settings.json
@@ -4,4 +4,7 @@
     "rust-analyzer.check.command": "clippy",
     "rust-analyzer.checkOnSave": true,
     "editor.inlayHints.enabled": "offUnlessPressed",
-}
+    "files.watcherExclude": {
+         "**/target/**": true
+     }
+}
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,3 @@
		#!/bin/bash

		aws --endpoint-url http://localhost:9000 s3 mb s3://$AWS_S3_TEST_BUCKET
-Original file line number
+Diff line change
@@ Expand Up / @@ -12,5 +12,4 @@ @@
     *.lcov
     *.xml
     lcov.info
-    .env
     playground.rs