ENVs changes and test script for validator (#1296)

* use SENTRY_CSP_REPORT_URI only in nextjs part of the app

* rename NEXT_PUBLIC_FAVICON_GENERATOR_API_KEY to FAVICON_GENERATOR_API_KEY

* tests presets for envs validator

* simplify logs

* refactor getExternalJsonContent function

* tweak contribution guide

* break validator

* debug workflow

* fix copying assets folder

* fix CI

.env.example

@@ -5,4 +5,4 @@ NEXT_PUBLIC_RE_CAPTCHA_APP_SITE_KEY=xxx
 NEXT_PUBLIC_GOOGLE_ANALYTICS_PROPERTY_ID=UA-XXXXXX-X
 NEXT_PUBLIC_MIXPANEL_PROJECT_TOKEN=xxx
 NEXT_PUBLIC_AUTH0_CLIENT_ID=xxx
-NEXT_PUBLIC_FAVICON_GENERATOR_API_KEY=xxx
+FAVICON_GENERATOR_API_KEY=xxx

.eslintrc.js

@@ -305,7 +305,7 @@ module.exports = {
       },
     },
     {
-      files: [ '*.config.ts', 'playwright/**', 'deploy/tools/**', 'middleware.ts' ],
+      files: [ '*.config.ts', 'playwright/**', 'deploy/tools/**', 'middleware.ts', 'nextjs/**' ],
       rules: {
         // for configs allow to consume env variables from process.env directly
         'no-restricted-properties': [ 0 ],

.github/workflows/checks.yml

@@ -52,7 +52,7 @@ jobs:
         run: yarn lint:tsc
 
   envs_validation:
-    name: ENV variables presets validation
+    name: ENV variables validation
     runs-on: ubuntu-latest
     needs: [ code_quality ]
     steps:
@@ -80,13 +80,10 @@ jobs:
       - name: Install script dependencies
         run: cd ./deploy/tools/envs-validator && yarn --frozen-lockfile --ignore-optional
 
-      - name: Copy secrets file
-        run: cp ./.env.example ./configs/envs/.env.secrets
-
-      - name: Run validation script
+      - name: Run validation tests
         run: |
           set +e
-          cd ./deploy/tools/envs-validator && yarn dev
+          cd ./deploy/tools/envs-validator && yarn test
           exitcode="$?"
           echo "exitcode=$exitcode" >> $GITHUB_OUTPUT
           exit "$exitcode"

configs/app/features/sentry.ts

@@ -14,25 +14,10 @@ const instance = (() => {
 })();
 const environment = getEnvValue('NEXT_PUBLIC_APP_ENV') || 'production';
 const release = getEnvValue('NEXT_PUBLIC_GIT_TAG');
-const cspReportUrl = (() => {
-  try {
-    const url = new URL(getEnvValue('SENTRY_CSP_REPORT_URI') || '');
-
-    // https://docs.sentry.io/product/security-policy-reporting/#additional-configuration
-    url.searchParams.set('sentry_environment', environment);
-    release && url.searchParams.set('sentry_release', release);
-
-    return url.toString();
-  } catch (error) {
-    return;
-  }
-})();
-
 const title = 'Sentry error monitoring';
 
 const config: Feature<{
   dsn: string;
-  cspReportUrl: string | undefined;
   instance: string;
   release: string | undefined;
   environment: string;
@@ -42,7 +27,6 @@ const config: Feature<{
       title,
       isEnabled: true,
       dsn,
-      cspReportUrl,
       instance,
       release,
       environment,

deploy/scripts/favicon_generator.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-master_url="${NEXT_PUBLIC_FAVICON_MASTER_URL:-$NEXT_PUBLIC_NETWORK_ICON}"
+master_url="${FAVICON_MASTER_URL:-$NEXT_PUBLIC_NETWORK_ICON}"
 export MASTER_URL="$master_url"
 
 cd ./deploy/tools/favicon-generator

deploy/tools/envs-validator/.gitignore

@@ -2,4 +2,5 @@
 /public
 .env
 .env.registry
+.env.secrets
 index.js

deploy/tools/envs-validator/index.ts

@@ -3,12 +3,15 @@ import fs from 'fs';
 import path from 'path';
 import type { ValidationError } from 'yup';
 
+import { buildExternalAssetFilePath } from '../../../configs/app/utils';
 import schema from './schema';
 
+const silent = process.argv.includes('--silent');
+
 run();
 
 async function run() {
-  console.log();
+  !silent && console.log();
   try {
     const appEnvs = Object.entries(process.env)
       .filter(([ key ]) => key.startsWith('NEXT_PUBLIC_'))
@@ -26,25 +29,22 @@ async function run() {
 }
 
 async function validateEnvs(appEnvs: Record<string, string>) {
-  console.log(`🌀 Validating ENV variables values...`);
+  !silent && console.log(`🌀 Validating ENV variables values...`);
 
   try {
     // replace ENVs with external JSON files content
-    appEnvs.NEXT_PUBLIC_FEATURED_NETWORKS = await getExternalJsonContent(
-      './public/assets/featured_networks.json',
-      appEnvs.NEXT_PUBLIC_FEATURED_NETWORKS,
-    ) || '[]';
-    appEnvs.NEXT_PUBLIC_MARKETPLACE_CONFIG_URL = await getExternalJsonContent(
-      './public/assets/marketplace_config.json',
-      appEnvs.NEXT_PUBLIC_MARKETPLACE_CONFIG_URL,
-    ) || '[]';
-    appEnvs.NEXT_PUBLIC_FOOTER_LINKS = await getExternalJsonContent(
-      './public/assets/footer_links.json',
-      appEnvs.NEXT_PUBLIC_FOOTER_LINKS,
-    ) || '[]';
+    const envsWithJsonConfig = [
+      'NEXT_PUBLIC_FEATURED_NETWORKS',
+      'NEXT_PUBLIC_MARKETPLACE_CONFIG_URL',
+      'NEXT_PUBLIC_FOOTER_LINKS',
+    ];
+
+    for await (const envName of envsWithJsonConfig) {
+      appEnvs[envName] = await(appEnvs[envName] ? getExternalJsonContent(envName) : Promise.resolve()) || '[]';
+    }
 
     await schema.validate(appEnvs, { stripUnknown: false, abortEarly: false });
-    console.log('👍 All good!');
+    !silent && console.log('👍 All good!');
   } catch (_error) {
     if (typeof _error === 'object' && _error !== null && 'errors' in _error) {
       console.log('🚨 ENVs validation failed with the following errors:');
@@ -59,15 +59,12 @@ async function validateEnvs(appEnvs: Record<string, string>) {
     throw _error;
   }
 
-  console.log();
+  !silent && console.log();
 }
 
-async function getExternalJsonContent(fileName: string, envValue: string): Promise<string | void> {
+async function getExternalJsonContent(envName: string): Promise<string | void> {
   return new Promise((resolve, reject) => {
-    if (!envValue) {
-      resolve();
-      return;
-    }
+    const fileName = `./public${ buildExternalAssetFilePath(envName, '.json') }`;
 
     fs.readFile(path.resolve(__dirname, fileName), 'utf8', (err, data) => {
       if (err) {
@@ -83,7 +80,7 @@ async function getExternalJsonContent(fileName: string, envValue: string): Promi
 
 async function checkPlaceholdersCongruity(envsMap: Record<string, string>) {
   try {
-    console.log(`🌀 Checking environment variables and their placeholders congruity...`);
+    !silent && console.log(`🌀 Checking environment variables and their placeholders congruity...`);
 
     const runTimeEnvs = await getEnvsPlaceholders(path.resolve(__dirname, '.env.registry'));
     const buildTimeEnvs = await getEnvsPlaceholders(path.resolve(__dirname, '.env'));
@@ -108,7 +105,7 @@ async function checkPlaceholdersCongruity(envsMap: Record<string, string>) {
       throw new Error();
     }
 
-    console.log('👍 All good!\n');
+    !silent && console.log('👍 All good!\n');
   } catch (error) {
     console.log('🚨 Congruity check failed.\n');
     throw error;

deploy/tools/envs-validator/package.json

@@ -6,7 +6,7 @@
   "scripts": {
     "build": "yarn webpack-cli -c ./webpack.config.js",
     "validate": "node ./index.js",
-    "dev": "./dev.sh"
+    "test": "./test.sh"
   },
   "dependencies": {
     "ts-loader": "^9.4.4",

deploy/tools/envs-validator/schema.ts

@@ -438,7 +438,6 @@ const schema = yup
     NEXT_PUBLIC_RE_CAPTCHA_APP_SITE_KEY: yup.string(),
     NEXT_PUBLIC_GOOGLE_ANALYTICS_PROPERTY_ID: yup.string(),
     NEXT_PUBLIC_MIXPANEL_PROJECT_TOKEN: yup.string(),
-    NEXT_PUBLIC_FAVICON_GENERATOR_API_KEY: yup.string(),
 
     // Misc
     NEXT_PUBLIC_USE_NEXT_JS_PROXY: yup.boolean(),

deploy/tools/envs-validator/test.sh

@@ -0,0 +1,49 @@
+#!/bin/bash
+
+secrets_file=".env.secrets"
+test_folder="./test"
+common_file="${test_folder}/.env.common"
+
+# Generate ENV registry file
+export NEXT_PUBLIC_GIT_COMMIT_SHA=$(git rev-parse --short HEAD)
+export NEXT_PUBLIC_GIT_TAG=$(git describe --tags --abbrev=0)
+../../scripts/collect_envs.sh ../../../docs/ENVS.md
+cp ../../../.env.example ${secrets_file}
+
+# Copy test assets
+mkdir -p "./public/assets"
+cp -r ${test_folder}/assets ./public/
+
+# Build validator script
+yarn build
+
+validate_file() {
+    local test_file="$1"
+
+    echo
+    echo "🧿 Validating file '$test_file'..."
+
+    dotenv \
+        -e $test_file \
+        -e $common_file \
+        -e $secrets_file \
+            yarn run validate -- --silent
+
+    if [ $? -eq 0 ]; then
+        echo "👍 All good!"
+        return 0
+    else
+        echo "🛑 The file is invalid. Please fix errors and run script again."
+        echo
+        return 1
+    fi
+}
+
+test_files=($(find "$test_folder" -maxdepth 1 -type f | grep -vE '\/\.env\.common$'))
+
+for file in "${test_files[@]}"; do
+    validate_file "$file"
+    if [ $? -eq 1 ]; then
+        exit 1
+    fi
+done

deploy/tools/envs-validator/test/.env.adbutler

@@ -0,0 +1,3 @@
+NEXT_PUBLIC_AD_BANNER_PROVIDER=adbutler
+NEXT_PUBLIC_AD_ADBUTLER_CONFIG_DESKTOP={'id':'123456','width':'728','height':'90'}
+NEXT_PUBLIC_AD_ADBUTLER_CONFIG_MOBILE={'id':'654321','width':'300','height':'100'}

deploy/tools/envs-validator/test/.env.base

@@ -0,0 +1,52 @@
+NEXT_PUBLIC_AD_TEXT_PROVIDER=coinzilla
+NEXT_PUBLIC_AD_BANNER_PROVIDER=slise
+NEXT_PUBLIC_ADMIN_SERVICE_API_HOST=https://example.com
+NEXT_PUBLIC_API_BASE_PATH=/
+NEXT_PUBLIC_API_SPEC_URL=https://example.com
+NEXT_PUBLIC_API_WEBSOCKET_PROTOCOL=ws
+NEXT_PUBLIC_APP_ENV=development
+NEXT_PUBLIC_APP_PORT=3000
+NEXT_PUBLIC_APP_PROTOCOL=http
+NEXT_PUBLIC_BRIDGED_TOKENS_CHAINS=[{'id':'1','title':'Ethereum','short_title':'ETH','base_url':'https://example.com'}]
+NEXT_PUBLIC_BRIDGED_TOKENS_BRIDGES=[{'type':'omni','title':'OmniBridge','short_title':'OMNI'}]
+NEXT_PUBLIC_CONTRACT_INFO_API_HOST=https://example.com
+NEXT_PUBLIC_FEATURED_NETWORKS=https://example.com
+NEXT_PUBLIC_FOOTER_LINKS=https://example.com
+NEXT_PUBLIC_GRAPHIQL_TRANSACTION=0xf7d4972356e6ae44ae948d0cf19ef2beaf0e574c180997e969a2837da15e349d
+NEXT_PUBLIC_HIDE_INDEXING_ALERT=false
+NEXT_PUBLIC_HOMEPAGE_CHARTS=['daily_txs']
+NEXT_PUBLIC_HOMEPAGE_PLATE_TEXT_COLOR='#fff'
+NEXT_PUBLIC_HOMEPAGE_PLATE_BACKGROUND='rgb(255, 145, 0)'
+NEXT_PUBLIC_HOMEPAGE_SHOW_GAS_TRACKER=true
+NEXT_PUBLIC_HOMEPAGE_SHOW_AVG_BLOCK_TIME=true
+NEXT_PUBLIC_IS_TESTNET=true
+NEXT_PUBLIC_MARKETPLACE_CONFIG_URL=https://example.com
+NEXT_PUBLIC_MARKETPLACE_SUBMIT_FORM=https://example.com
+NEXT_PUBLIC_MAINTENANCE_ALERT_MESSAGE='<a href="#">Hello</a>'
+NEXT_PUBLIC_NETWORK_CURRENCY_DECIMALS=18
+NEXT_PUBLIC_NETWORK_CURRENCY_NAME=Ether
+NEXT_PUBLIC_NETWORK_CURRENCY_SYMBOL=ETH
+NEXT_PUBLIC_NETWORK_EXPLORERS=[{'title':'Explorer','baseUrl':'https://example.com/','paths':{'tx':'/tx','address':'/address','token':'/token','block':'/block'}}]
+NEXT_PUBLIC_NETWORK_GOVERNANCE_TOKEN_SYMBOL=gETH
+NEXT_PUBLIC_NETWORK_ICON=https://example.com/icon.png
+NEXT_PUBLIC_NETWORK_ICON_DARK=https://example.com/icon.png
+NEXT_PUBLIC_NETWORK_LOGO=https://example.com/logo.png
+NEXT_PUBLIC_NETWORK_LOGO_DARK=https://example.com/logo.png
+NEXT_PUBLIC_NETWORK_RPC_URL=https://example.com
+NEXT_PUBLIC_NETWORK_SHORT_NAME=Test
+NEXT_PUBLIC_NETWORK_VERIFICATION_TYPE=validation
+NEXT_PUBLIC_OG_DESCRIPTION='Hello world!'
+NEXT_PUBLIC_OG_IMAGE_URL=https://example.com/image.png
+NEXT_PUBLIC_OTHER_LINKS=[{'url':'https://blockscout.com','text':'Blockscout'}]
+NEXT_PUBLIC_PROMOTE_BLOCKSCOUT_IN_TITLE=true
+NEXT_PUBLIC_STATS_API_HOST=https://example.com
+NEXT_PUBLIC_USE_NEXT_JS_PROXY=false
+NEXT_PUBLIC_VIEWS_ADDRESS_IDENTICON_TYPE=gradient_avatar
+NEXT_PUBLIC_VIEWS_ADDRESS_HIDDEN_VIEWS=['top_accounts']
+NEXT_PUBLIC_VIEWS_BLOCK_HIDDEN_FIELDS=['burnt_fees','total_reward']
+NEXT_PUBLIC_VIEWS_NFT_MARKETPLACES=[{'name':'NFT Marketplace','collection_url':'https://example.com/{hash}','instance_url':'https://example.com/{hash}/{id}','logo_url':'https://example.com/logo.png'}]
+NEXT_PUBLIC_VIEWS_TX_ADDITIONAL_FIELDS=['fee_per_gas']
+NEXT_PUBLIC_VIEWS_TX_HIDDEN_FIELDS=['value','fee_currency','gas_price','tx_fee','gas_fees','burnt_fees']
+NEXT_PUBLIC_VISUALIZE_API_HOST=https://example.com
+NEXT_PUBLIC_WEB3_DISABLE_ADD_TOKEN_TO_WALLET=false
+NEXT_PUBLIC_WEB3_WALLETS=['coinbase','metamask','token_pocket']

deploy/tools/envs-validator/test/.env.beacon_chain

@@ -0,0 +1,2 @@
+NEXT_PUBLIC_HAS_BEACON_CHAIN=true
+NEXT_PUBLIC_BEACON_CHAIN_CURRENCY_SYMBOL=aETH

deploy/tools/envs-validator/test/.env.common

@@ -0,0 +1,7 @@
+NEXT_PUBLIC_API_HOST=blockscout.com
+NEXT_PUBLIC_APP_HOST=localhost
+NEXT_PUBLIC_AUTH_URL=https://example.com
+NEXT_PUBLIC_IS_ACCOUNT_SUPPORTED=true
+NEXT_PUBLIC_LOGOUT_URL=https://example.com
+NEXT_PUBLIC_NETWORK_ID=1
+NEXT_PUBLIC_NETWORK_NAME=Testnet

deploy/tools/envs-validator/test/.env.rollup

@@ -0,0 +1,3 @@
+NEXT_PUBLIC_IS_L2_NETWORK=true
+NEXT_PUBLIC_L1_BASE_URL=https://example.com
+NEXT_PUBLIC_L2_WITHDRAWAL_URL=https://example.com