configure test endpoint via env var

CrunchyData · Dec 16, 2024 · f1b7114 · f1b7114
1 parent 7664102
commit f1b7114
Show file tree

Hide file tree

Showing 6 changed files with 22 additions and 35 deletions.
diff --git a/.devcontainer/.env b/.devcontainer/.env
@@ -12,6 +12,8 @@ MINIO_ROOT_PASSWORD=minioadmin
 AZURE_STORAGE_ACCOUNT=devstoreaccount1
 AZURE_STORAGE_KEY="Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw=="
 AZURE_STORAGE_CONNECTION_STRING="DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://localhost:10000/devstoreaccount1;"
+AZURE_STORAGE_ENDPOINT=http://localhost:10000/devstoreaccount1
+AZURE_ALLOW_HTTP=true
 AZURE_TEST_CONTAINER_NAME=testcontainer
 AZURE_TEST_READ_ONLY_SAS="se=2100-05-05&sp=r&sv=2022-11-02&sr=c&sig=YMPFnAHKe9y0o3hFegncbwQTXtAyvsJEgPB2Ne1b9CQ%3D"
 AZURE_TEST_READ_WRITE_SAS="se=2100-05-05&sp=rcw&sv=2022-11-02&sr=c&sig=TPz2jEz0t9L651t6rTCQr%2BOjmJHkM76tnCGdcyttnlA%3D"

diff --git a/.devcontainer/docker-compose.yml b/.devcontainer/docker-compose.yml
@@ -3,7 +3,7 @@ services:
     build:
       context: .
       dockerfile: Dockerfile
-    command: sleep infinity
+    entrypoint: "./entrypoint.sh"
     network_mode: host
     volumes:
       - ..:/workspace
@@ -12,7 +12,7 @@ services:
       - ${USERPROFILE}${HOME}/.gitconfig:/home/rust/.gitconfig:ro
       - ${USERPROFILE}${HOME}/.aws:/home/rust/.aws:rw
       - ${USERPROFILE}${HOME}/.azure:/home/rust/.azure:rw
-
+      - ./entrypoint.sh:/entrypoint.sh
     env_file:
       - .env
     cap_add:
@@ -33,8 +33,8 @@ services:
       interval: 6s
       timeout: 2s
       retries: 3
-  volumes:
-      - ./minio-entrypoint.sh:/entrypoint.sh
+    volumes:
+        - ./minio-entrypoint.sh:/entrypoint.sh
 
   azurite:
     image: mcr.microsoft.com/azure-storage/azurite
@@ -43,9 +43,7 @@ services:
     network_mode: host
     restart: unless-stopped
     healthcheck:
-      test: ["CMD", "curl", "http://localhost:10000"]
+      test: ["CMD", "nc", "-z", "localhost", "10000"]
       interval: 6s
       timeout: 2s
       retries: 3
-    volumes:
-      - ./azurite-entrypoint.sh:/entrypoint.sh
diff --git a/.devcontainer/azurite-entrypoint.sh → .devcontainer/entrypoint.sh b/.devcontainer/azurite-entrypoint.sh → .devcontainer/entrypoint.sh
@@ -2,16 +2,7 @@
 
 trap "echo 'Caught termination signal. Exiting...'; exit 0" SIGINT SIGTERM
 
-minio server /data &
-
-azurite_pid=$!
-
-while ! curl $AWS_ENDPOINT_URL; do
-    echo "Waiting for $AWS_ENDPOINT_URL..."
-    sleep 1
-done
-
-# create container
+# create azurite container
 az storage container create -n $AZURE_TEST_CONTAINER_NAME --connection-string $AZURE_STORAGE_CONNECTION_STRING
 
-wait $azurite_pid
+sleep infinity
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -117,7 +117,6 @@ jobs:
             -p 9000:9000 \
             --entrypoint "./entrypoint.sh" \
             --volume ./.devcontainer/minio-entrypoint.sh:/entrypoint.sh \
-            --name miniocontainer \
             minio/minio
 
           while ! curl $AWS_ENDPOINT_URL; do
@@ -127,13 +126,19 @@ jobs:
 
       - name: Start Azurite for Azure Blob Storage emulator tests
         run: |
-          docker run -d --env-file .devcontainer/.env -p 10000:10000 mcr.microsoft.com/azure-storage/azurite
+          docker run -d \
+            --env-file .devcontainer/.env \
+            -p 10000:10000 \
+            mcr.microsoft.com/azure-storage/azurite
 
-          while ! nc -z localhost 10000; do
-            echo "Waiting for localhost:10000..."
+          while ! curl $AZURITE_URL; do
+            echo "Waiting for $AZURITE_URL..."
             sleep 1
           done
 
+          # create container
+          az storage container create -n $AZURE_TEST_CONTAINER_NAME --connection-string $AZURE_STORAGE_CONNECTION_STRING
+
       - name: Run tests
         run: |
           # Run tests with coverage tool

diff --git a/README.md b/README.md
@@ -210,7 +210,9 @@ Alternatively, you can use the following environment variables when starting pos
 - `AZURE_STORAGE_ACCOUNT`: the storage account name of the Azure Blob
 - `AZURE_STORAGE_KEY`: the storage key of the Azure Blob
 - `AZURE_STORAGE_SAS_TOKEN`: the storage SAS token for the Azure Blob
-- `AZURE_CONFIG_FILE`: an alternative location for the config file
+- `AZURE_STORAGE_ENDPOINT`: the endpoint **(only via environment variables)**
+- `AZURE_CONFIG_FILE`: an alternative location for the config file **(only via environment variables)**
+- `AZURE_ALLOW_HTTP`: allows http endpoints **(only via environment variables)**
 
 Supported Azure Blob Storage uri formats are shown below:
 - az:// \<container\> / \<path\>

diff --git a/src/arrow_parquet/uri_utils.rs b/src/arrow_parquet/uri_utils.rs
@@ -185,14 +185,7 @@ async fn get_s3_object_store(bucket_name: &str) -> AmazonS3 {
 }
 
 async fn get_azure_object_store(container_name: &str) -> MicrosoftAzure {
-    let mut azure_builder = MicrosoftAzureBuilder::new().with_container_name(container_name);
-
-    if is_testing() {
-        // use azurite for testing
-        azure_builder =
-            azure_builder.with_endpoint("http://localhost:10000/devstoreaccount1".into());
-        azure_builder = azure_builder.with_allow_http(true);
-    }
+    let mut azure_builder = MicrosoftAzureBuilder::from_env().with_container_name(container_name);
 
     // ~/.azure/config
     let azure_config_file_path = std::env::var("AZURE_CONFIG_FILE").unwrap_or(
@@ -252,10 +245,6 @@ async fn get_azure_object_store(container_name: &str) -> MicrosoftAzure {
     azure_builder.build().unwrap_or_else(|e| panic!("{}", e))
 }
 
-fn is_testing() -> bool {
-    std::env::var("PG_PARQUET_TEST").is_ok()
-}
-
 pub(crate) fn parse_uri(uri: &str) -> Url {
     if !uri.contains("://") {
         // local file