0xProject · nicoalencar · Aug 16, 2024 · Aug 9, 2024 · Aug 16, 2024 · Aug 16, 2024
diff --git a/README.md b/README.md
@@ -14,9 +14,9 @@ Configure the scape **Mode**
 
 2. Start Postgres
 
-```sh
-docker-compose up -d # get postgres up
-```
+For local development, you will need to get the local databases running depending on which pipelines you are going to develop/test.
+You can setup them by following the instructions on the [0x-data-migrations](https://github.com/0xProject/0x-data-migrations) repository. We manage all db setup/changes there.
+It will guide you on how to get the dbs up and running the initial setup that is necessary to be used here.
 
 3. Build the Scraper images
 

diff --git a/build_local.sh b/build_local.sh
@@ -1,6 +1,5 @@
 #!/usr/bin/env sh
 
-docker-compose -f docker-compose-dev.yml up -d postgres
 docker-compose -f docker-compose-dev.yml build event-pipeline-ethereum
 docker-compose -f docker-compose-dev.yml build event-pipeline-base
 

diff --git a/docker-compose-dev.yml b/docker-compose-dev.yml
@@ -1,20 +1,5 @@
 version: '3'
 services:
-    postgres:
-        image: postgres:13.4
-        environment:
-          POSTGRES_USER: '${POSTGRES_USER}'
-          POSTGRES_PASSWORD: '${POSTGRES_PASSWORD}'
-          POSTGRES_DB: '${POSTGRES_DB}'
-          POSTGRES_PORT: ${POSTGRES_PORT}
-        # persist the postgres data to disk so we don't lose it
-        # on rebuilds.
-        volumes:
-            - ./mounts/postgres:/var/lib/postgresql/data
-            - ./postgres-init.sql:/docker-entrypoint-initdb.d/postgres-init.sql
-        ports:
-            - '5432:5432'
-        command: ["postgres", "-c", "log_statement=none", "-c", "log_destination=stderr"]
 
     event-pipeline-ethereum:
         depends_on:
@@ -166,7 +151,6 @@ services:
             FEAT_WRAP_UNWRAP_NATIVE_EVENT: "true"
             WRAP_UNWRAP_NATIVE_CONTRACT_ADDRESS: '0xb31f66aa3c1e785363f0875a1b74e27b85fd66c7'
 
-
     event-pipeline-fantom:
         depends_on:
             - postgres

diff --git a/docker-compose.yml b/docker-compose.yml
@@ -1,31 +1,14 @@
 version: '3'
 services:
-    postgres:
-        image: postgres:13.4
-        environment:
-          POSTGRES_USER: '${POSTGRES_USER}'
-          POSTGRES_PASSWORD: '${POSTGRES_PASSWORD}'
-          POSTGRES_DB: '${POSTGRES_DB}'
-          POSTGRES_PORT: ${POSTGRES_PORT}
-        # persist the postgres data to disk so we don't lose it
-        # on rebuilds.
-        volumes:
-            - ./mounts/postgres:/var/lib/postgresql/data
-        ports:
-            - '5432:5432'
-        command: ["postgres", "-c", "log_statement=all", "-c", "log_destination=stderr"]
-
     event-pipeline-ethereum:
-        depends_on:
-            - postgres
         build:
             context: .
             dockerfile: Dockerfile
         restart: always
         environment:
             EVM_RPC_URL: '${RPC_URL_ETHEREUM}'
             CHAIN_ID: '1'
-            POSTGRES_URI: 'postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@postgres/${POSTGRES_DB}'
+            POSTGRES_URI: 'postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@${POSTGRES_HOST}/${POSTGRES_DB}'
             SCHEMA: 'events'
             # FEAT_EXCLUSIVE_TOKENS_FROM_TRANSACTIONS: "true"
             # TOKENS_FROM_TRANSACTIONS_START_BLOCK: 9193266
@@ -85,16 +68,14 @@ services:
             FEAT_SETTLER_RFQ_ORDER_EVENT: "true"
 
     event-pipeline-bsc:
-        depends_on:
-            - postgres
         build:
             context: .
             dockerfile: Dockerfile
         restart: always
         environment:
             EVM_RPC_URL: '${RPC_URL_BSC}'
             CHAIN_ID: '56'
-            POSTGRES_URI: 'postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@postgres/${POSTGRES_DB}'
+            POSTGRES_URI: 'postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@${POSTGRES_HOST}/${POSTGRES_DB}'
             SCHEMA: 'events_bsc'
             SCRAPER_MODE: 'BLOCKS'
             MAX_BLOCKS_REORG: 300
@@ -123,16 +104,14 @@ services:
             FEAT_SETTLER_ERC721_TRANSFER_EVENT: "true"
 
     event-pipeline-polygon:
-        depends_on:
-            - postgres
         build:
             context: .
             dockerfile: Dockerfile
         restart: always
         environment:
             EVM_RPC_URL: '${RPC_URL_POLYGON}'
             CHAIN_ID: '137'
-            POSTGRES_URI: 'postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@postgres/${POSTGRES_DB}'
+            POSTGRES_URI: 'postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@${POSTGRES_HOST}/${POSTGRES_DB}'
             SCHEMA: 'events_polygon'
             SCRAPER_MODE: 'BLOCKS'
             ENABLE_PROMETHEUS_METRICS: "true"
@@ -159,16 +138,14 @@ services:
             FEAT_SETTLER_ERC721_TRANSFER_EVENT: "true"
 
     event-pipeline-avalanche:
-        depends_on:
-            - postgres
         build:
             context: .
             dockerfile: Dockerfile
         restart: always
         environment:
             EVM_RPC_URL: '${RPC_URL_AVALANCHE}'
             CHAIN_ID: '43114'
-            POSTGRES_URI: 'postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@postgres/${POSTGRES_DB}'
+            POSTGRES_URI: 'postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@${POSTGRES_HOST}/${POSTGRES_DB}'
             SCHEMA: 'events_avalanche'
             #KAFKA_BROKERS: '${KAFKA_BROKERS}'
             #KAFKA_SSL: '${KAFKA_SSL}'
@@ -196,16 +173,14 @@ services:
 
 
     event-pipeline-fantom:
-        depends_on:
-            - postgres
         build:
             context: .
             dockerfile: Dockerfile
         restart: always
         environment:
             EVM_RPC_URL: '${RPC_URL_FANTOM}'
             CHAIN_ID: '250'
-            POSTGRES_URI: 'postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@postgres/${POSTGRES_DB}'
+            POSTGRES_URI: 'postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@${POSTGRES_HOST}/${POSTGRES_DB}'
             SCHEMA: 'events_fantom'
             #KAFKA_BROKERS: '${KAFKA_BROKERS}'
             #KAFKA_SSL: '${KAFKA_SSL}'
@@ -228,16 +203,14 @@ services:
             WRAP_UNWRAP_NATIVE_CONTRACT_ADDRESS: '0x21be370d5312f44cb42ce377bc9b8a0cef1a4c83'
 
     event-pipeline-celo:
-        depends_on:
-            - postgres
         build:
             context: .
             dockerfile: Dockerfile
         restart: always
         environment:
             EVM_RPC_URL: '${RPC_URL_CELO}'
             CHAIN_ID: '42220'
-            POSTGRES_URI: 'postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@postgres/${POSTGRES_DB}'
+            POSTGRES_URI: 'postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@${POSTGRES_HOST}/${POSTGRES_DB}'
             SCHEMA: 'events_celo'
             #KAFKA_BROKERS: '${KAFKA_BROKERS}'
             #KAFKA_SSL: '${KAFKA_SSL}'
@@ -254,16 +227,14 @@ services:
             NFT_FEATURE_START_BLOCK: 11820000
 
     event-pipeline-optimism:
-        depends_on:
-            - postgres
         build:
             context: .
             dockerfile: Dockerfile
         restart: always
         environment:
             EVM_RPC_URL: '${RPC_URL_OPTIMISM}'
             CHAIN_ID: '10'
-            POSTGRES_URI: 'postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@postgres/${POSTGRES_DB}'
+            POSTGRES_URI: 'postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@${POSTGRES_HOST}/${POSTGRES_DB}'
             SCHEMA: 'events_optimism'
             #KAFKA_BROKERS: '${KAFKA_BROKERS}'
             #KAFKA_SSL: '${KAFKA_SSL}'
@@ -289,16 +260,14 @@ services:
             FEAT_SETTLER_ERC721_TRANSFER_EVENT: "true"
 
     event-pipeline-arbitrum:
-        depends_on:
-            - postgres
         build:
             context: .
             dockerfile: Dockerfile
         restart: always
         environment:
             EVM_RPC_URL: '${EVM_RPC_URL_ARBITRUM}'
             CHAIN_ID: '42161'
-            POSTGRES_URI: 'postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@postgres/${POSTGRES_DB}'
+            POSTGRES_URI: 'postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@${POSTGRES_HOST}/${POSTGRES_DB}'
             SCHEMA: 'events_arbitrum'
             #KAFKA_BROKERS: '${KAFKA_BROKERS}'
             #KAFKA_SSL: '${KAFKA_SSL}'
@@ -320,16 +289,14 @@ services:
             FEAT_SETTLER_ERC721_TRANSFER_EVENT: "true"
 
     event-pipeline-base:
-        depends_on:
-            - postgres
         build:
             context: .
             dockerfile: Dockerfile
         restart: always
         environment:
             EVM_RPC_URL: '${RPC_URL_BASE}'
             CHAIN_ID: '8453'
-            POSTGRES_URI: 'postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@postgres/${POSTGRES_DB}'
+            POSTGRES_URI: 'postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@${POSTGRES_HOST}/${POSTGRES_DB}'
             SCHEMA: 'events_base'
             SCRAPER_MODE: 'BLOCKS'
             FEAT_ZEROEX_EXCHANGE_PROXY: "true"
@@ -352,16 +319,14 @@ services:
             FEAT_SETTLER_RFQ_ORDER_EVENT: "true"
 
     token-scraper-ethereum:
-        depends_on:
-            - postgres
         build:
             context: .
             dockerfile: Dockerfile
         restart: always
         environment:
             EVM_RPC_URL: '${RPC_URL_ETHEREUM}'
             CHAIN_ID: '1'
-            POSTGRES_URI: 'postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@postgres/${POSTGRES_DB}'
+            POSTGRES_URI: 'postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@${POSTGRES_HOST}/${POSTGRES_DB}'
             SCHEMA: 'events'
             #KAFKA_BROKERS: '${KAFKA_BROKERS}'
             #KAFKA_SSL: '${KAFKA_SSL}'

diff --git a/.env.example → env.tmpl b/.env.example → env.tmpl
@@ -1,6 +1,7 @@
-POSTGRES_USER=scraper
-POSTGRES_PASSWORD=scraper
-POSTGRES_DB=events
+POSTGRES_HOST=172.17.0.1
+POSTGRES_USER=zeroex
+POSTGRES_PASSWORD=zeroex
+POSTGRES_DB=hashalytics
 POSTGRES_PORT=5432
 
 RPC_URL_ETHEREUM=
@@ -11,4 +12,3 @@ RPC_URL_FANTOM=
 RPC_URL_CELO=
 RPC_URL_OPTIMISM=
 RPC_URL_ARBITRUM=
-