diff --git a/.github/workflows/ecr.yml b/.github/workflows/ecr.yml new file mode 100644 index 000000000..6c54ab837 --- /dev/null +++ b/.github/workflows/ecr.yml @@ -0,0 +1,55 @@ +name: AWS ECR + +on: + push: + branches: + - main + pull_request: + types: + - opened + - synchronize + - reopened + - ready_for_review + +permissions: + id-token: write + contents: read + +jobs: + build: + runs-on: ubuntu-latest + if: github.event.pull_request.draft == false + strategy: + matrix: + task: [api, data, pmtiles, hooks, events] + steps: + - uses: actions/checkout@v4 + with: + ref: ${{github.event.pull_request.head.sha || github.sha}} + + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: arn:aws-us-gov:iam::${{secrets.AWS_ACCOUNT_ID}}:role/Github-ECR-Builder + role-session-name: GithubECRBuilder + aws-region: ${{secrets.AWS_REGION}} + + - name: Docker Build Task + run: npm run build -- ${{matrix.task}} + env: + AWS_ACCOUNT_ID: ${{secrets.AWS_ACCOUNT_ID}} + AWS_REGION: ${{secrets.AWS_REGION}} + + - name: Configure AWS Credentials (STAGING) + uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: arn:aws:iam::${{secrets.AWS_STAGING_ACCOUNT_ID}}:role/Github-ECR-Builder + role-session-name: GithubECRBuilder + aws-region: ${{secrets.AWS_STAGING_REGION}} + + - name: Docker Build Task (STAGING) + run: npm run build -- ${{matrix.task}} + env: + AWS_ACCOUNT_ID: ${{secrets.AWS_STAGING_ACCOUNT_ID}} + AWS_REGION: ${{secrets.AWS_STAGING_REGION}} + diff --git a/.github/workflows/ecr_api.yml b/.github/workflows/ecr_api.yml deleted file mode 100644 index 44cc75bf6..000000000 --- a/.github/workflows/ecr_api.yml +++ /dev/null @@ -1,62 +0,0 @@ -name: AWS ECR API - -on: - push: - branches: - - main - pull_request: - types: - - opened - - synchronize - - reopened - - ready_for_review - -permissions: - id-token: write - contents: read - -jobs: - build: - runs-on: ubuntu-latest - if: github.event.pull_request.draft == false - steps: - - uses: actions/checkout@v4 - with: - ref: ${{github.event.pull_request.head.sha || github.sha}} - - - name: Docker Build API - run: docker compose build api - - - name: Configure AWS Credentials - uses: aws-actions/configure-aws-credentials@v4 - with: - role-to-assume: arn:aws-us-gov:iam::${{secrets.AWS_ACCOUNT_ID}}:role/Github-ECR-Builder - role-session-name: GithubECRBuilder - aws-region: ${{secrets.AWS_REGION}} - - - name: Login to Amazon ECR - id: login-ecr - uses: aws-actions/amazon-ecr-login@v2 - - - name: Docker Tag API - run: docker tag cloudtak-api:latest ${{secrets.AWS_ACCOUNT_ID}}.dkr.ecr.${{secrets.AWS_REGION}}.amazonaws.com/coe-ecr-etl:${{github.event.pull_request.head.sha || github.sha}} - - - name: Docker Push API - run: docker push ${{secrets.AWS_ACCOUNT_ID}}.dkr.ecr.${{secrets.AWS_REGION}}.amazonaws.com/coe-ecr-etl:${{github.event.pull_request.head.sha || github.sha}} - - - name: Configure AWS Credentials (STAGING) - uses: aws-actions/configure-aws-credentials@v4 - with: - role-to-assume: arn:aws:iam::${{secrets.AWS_STAGING_ACCOUNT_ID}}:role/Github-ECR-Builder - role-session-name: GithubECRBuilder - aws-region: ${{secrets.AWS_STAGING_REGION}} - - - name: Login to Amazon ECR (STAGING) - id: login-ecr-staging - uses: aws-actions/amazon-ecr-login@v2 - - - name: Docker Tag API (STAGING) - run: docker tag cloudtak-api:latest ${{secrets.AWS_STAGING_ACCOUNT_ID}}.dkr.ecr.${{secrets.AWS_STAGING_REGION}}.amazonaws.com/coe-ecr-etl:${{github.event.pull_request.head.sha || github.sha}} - - - name: Docker Push API (STAGING) - run: docker push ${{secrets.AWS_STAGING_ACCOUNT_ID}}.dkr.ecr.${{secrets.AWS_STAGING_REGION}}.amazonaws.com/coe-ecr-etl:${{github.event.pull_request.head.sha || github.sha}} diff --git a/.github/workflows/ecr_task.yml b/.github/workflows/ecr_task.yml deleted file mode 100644 index 233189566..000000000 --- a/.github/workflows/ecr_task.yml +++ /dev/null @@ -1,65 +0,0 @@ -name: AWS ECR Task - -on: - push: - branches: - - main - pull_request: - types: - - opened - - synchronize - - reopened - - ready_for_review - -permissions: - id-token: write - contents: read - -jobs: - build: - runs-on: ubuntu-latest - if: github.event.pull_request.draft == false - strategy: - matrix: - task: [data, pmtiles, hooks, events] - steps: - - uses: actions/checkout@v4 - with: - ref: ${{github.event.pull_request.head.sha || github.sha}} - - - name: Configure AWS Credentials - uses: aws-actions/configure-aws-credentials@v4 - with: - role-to-assume: arn:aws-us-gov:iam::${{secrets.AWS_ACCOUNT_ID}}:role/Github-ECR-Builder - role-session-name: GithubECRBuilder - aws-region: ${{secrets.AWS_REGION}} - - - name: Docker Build Task - run: docker buildx build ./tasks/${{ matrix.task }} -t cloudtak-${{ matrix.task }} - - - name: Login to Amazon ECR - id: login-ecr - uses: aws-actions/amazon-ecr-login@v2 - - - name: Docker Tag Task - run: docker tag cloudtak-${{matrix.task}}:latest ${{secrets.AWS_ACCOUNT_ID}}.dkr.ecr.${{secrets.AWS_REGION}}.amazonaws.com/coe-ecr-etl:${{matrix.task}}-${{github.event.pull_request.head.sha || github.sha}} - - - name: Docker Push task - run: docker push ${{secrets.AWS_ACCOUNT_ID}}.dkr.ecr.${{secrets.AWS_REGION}}.amazonaws.com/coe-ecr-etl:${{matrix.task}}-${{github.event.pull_request.head.sha || github.sha}} - - - name: Configure AWS Credentials (STAGING) - uses: aws-actions/configure-aws-credentials@v4 - with: - role-to-assume: arn:aws:iam::${{secrets.AWS_STAGING_ACCOUNT_ID}}:role/Github-ECR-Builder - role-session-name: GithubECRBuilder - aws-region: ${{secrets.AWS_STAGING_REGION}} - - - name: Login to Amazon ECR (STAGING) - id: login-ecr-staging - uses: aws-actions/amazon-ecr-login@v2 - - - name: Docker Tag Task (STAGING) - run: docker tag cloudtak-${{matrix.task}}:latest ${{secrets.AWS_STAGING_ACCOUNT_ID}}.dkr.ecr.${{secrets.AWS_STAGING_REGION}}.amazonaws.com/coe-ecr-etl:${{matrix.task}}-${{github.event.pull_request.head.sha || github.sha}} - - - name: Docker Push task (STAGING) - run: docker push ${{secrets.AWS_STAGING_ACCOUNT_ID}}.dkr.ecr.${{secrets.AWS_STAGING_REGION}}.amazonaws.com/coe-ecr-etl:${{matrix.task}}-${{github.event.pull_request.head.sha || github.sha}} diff --git a/build.js b/build.js new file mode 100755 index 000000000..d81d8a692 --- /dev/null +++ b/build.js @@ -0,0 +1,98 @@ +#! /usr/bin/env node + +import fs from 'node:fs/promises'; +import CP from 'child_process'; + +process.env.GITSHA = sha(); + +for (const env of ['GITSHA', 'AWS_REGION', 'AWS_ACCOUNT_ID']) { + if (!process.env[env]) { + console.error(`${env} Env Var must be set`); + process.exit(); + } +} + +await login(); + + +if (!process.argv[2]) { + console.error('ok - building all containers'); + + await api(); + + for (const dir of await fs.readdir(new URL('./tasks/', import.meta.url))) { + await task(dir); + } +} else { + if (process.argv[2] === 'api') { + await api(); + } else { + await task(process.argv[2]); + } +} + +function login() { + return new Promise((resolve, reject) => { + const $ = CP.exec(` + aws ecr get-login-password \ + --region $\{AWS_REGION\} \ + | docker login \ + --username AWS \ + --password-stdin "$\{AWS_ACCOUNT_ID\}.dkr.ecr.$\{AWS_REGION\}.amazonaws.com" + + `, (err) => { + if (err) return reject(err); + return resolve(); + }); + + $.stdout.pipe(process.stdout); + $.stderr.pipe(process.stderr); + }); + +} + +function api() { + return new Promise((resolve, reject) => { + const $ = CP.exec(` + docker compose build api \ + && docker tag cloudtak-api:latest "$\{AWS_ACCOUNT_ID\}.dkr.ecr.$\{AWS_REGION\}.amazonaws.com/coe-ecr-etl:$\{GITSHA\}" \ + && docker push "$\{AWS_ACCOUNT_ID\}.dkr.ecr.$\{AWS_REGION\}.amazonaws.com/coe-ecr-etl:$\{GITSHA\}" + `, (err) => { + if (err) return reject(err); + return resolve(); + }); + + $.stdout.pipe(process.stdout); + $.stderr.pipe(process.stderr); + }); +} + +async function task(task) { + process.env.TASK = task; + + return new Promise((resolve, reject) => { + const $ = CP.exec(` + docker buildx build ./tasks/$\{TASK\}/ -t cloudtak-$\{TASK\} \ + && docker tag cloudtak-$\{TASK\}:latest "$\{AWS_ACCOUNT_ID\}.dkr.ecr.$\{AWS_REGION\}.amazonaws.com/coe-ecr-etl:$\{TASK\}-$\{GITSHA\}" \ + && docker push "$\{AWS_ACCOUNT_ID\}.dkr.ecr.$\{AWS_REGION\}.amazonaws.com/coe-ecr-etl:$\{TASK\}-$\{GITSHA\}" + `, (err) => { + if (err) return reject(err); + return resolve(); + }); + + $.stdout.pipe(process.stdout); + $.stderr.pipe(process.stderr); + }); + +} + +function sha() { + const git = CP.spawnSync('git', [ + '--git-dir', new URL('.git', import.meta.url).pathname, + 'rev-parse', 'HEAD' + ]); + + if (!git.stdout) throw Error('Is this a git repo? Could not determine GitSha'); + return String(git.stdout).replace(/\n/g, ''); + +} diff --git a/docker-compose.yml b/docker-compose.yml index 124983678..20f6e9aee 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,5 +1,3 @@ -version: '3' - services: api: platform: linux/amd64 diff --git a/eslint.config.js b/eslint.config.js index d85f4a42a..ea5e88c63 100644 --- a/eslint.config.js +++ b/eslint.config.js @@ -7,7 +7,10 @@ export default [ { "rules": { "no-console": 0, + "no-useless-escape": "warn", "arrow-parens": [ "error", "always" ], + "n/no-process-exit": "warn", + "n/hashbang": "warn", "no-var": "error", "prefer-const": "error", "array-bracket-spacing": [ "error", "never" ], diff --git a/package-lock.json b/package-lock.json index 76fb90b63..31b340b11 100644 --- a/package-lock.json +++ b/package-lock.json @@ -16,7 +16,8 @@ "devDependencies": { "@types/xml2js": "^0.4.11", "eslint": "^9.0.0", - "eslint-plugin-n": "^17.2.1" + "eslint-plugin-n": "^17.2.1", + "minimist": "^1.2.8" }, "engines": { "node": ">= 18" diff --git a/package.json b/package.json index 320f5b4d7..9a8a3c5c9 100644 --- a/package.json +++ b/package.json @@ -4,7 +4,8 @@ "version": "4.8.0", "description": "Facilitate ETL operations to bring non-TAK sources into a TAK Server", "scripts": { - "lint": "eslint cloudformation/", + "lint": "eslint build.js cloudformation/", + "build": "./build.js", "test": "echo \"Error: no test specified\" && exit 1" }, "engines": { diff --git a/tasks/data/Dockerfile b/tasks/data/Dockerfile index a190eefed..397e0f840 100644 --- a/tasks/data/Dockerfile +++ b/tasks/data/Dockerfile @@ -8,12 +8,12 @@ RUN apt-get update \ && apt-get install -y build-essential curl cmake autoconf libtool software-properties-common git libsqlite3-dev \ && apt-get install -y python3 python3-pip python3-dev python3-setuptools -RUN export NODEV='22.8.0' \ +RUN export NODEV='22.9.0' \ && curl "https://nodejs.org/dist/v${NODEV}/node-v${NODEV}-linux-x64.tar.gz" | tar -xzv > /dev/null \ && cp ./node-v${NODEV}-linux-x64/bin/node /usr/bin/ \ && ./node-v${NODEV}-linux-x64/bin/npm install -g npm -RUN npm install --global aws-lambda-ric +# RUN npm install --global aws-lambda-ric # PMTiles: https://github.com/protomaps/go-pmtiles/releases RUN curl -L https://github.com/protomaps/go-pmtiles/releases/download/v1.20.0/go-pmtiles_1.20.0_Linux_x86_64.tar.gz > /tmp/pmtiles.tar.gz && \