Merge pull request #359 from dfpc-coe/plugins

Cold Start

.deploy

@@ -13,7 +13,7 @@
     "artifacts": {
         "docker": [
             "coe-ecr-etl:{{gitsha}}",
-            "coe-ecr-etl:task-{{gitsha}}",
+            "coe-ecr-etl:data-{{gitsha}}",
             "coe-ecr-etl:pmtiles-{{gitsha}}",
             "coe-ecr-etl:hooks-{{gitsha}}",
             "coe-ecr-etl:events-{{gitsha}}"

.github/workflows/ecr_task.yml

@@ -21,7 +21,7 @@ jobs:
     if: github.event.pull_request.draft == false
     strategy:
       matrix:
-        task: [task, pmtiles, hooks, events]
+        task: [data, pmtiles, hooks, events]
     steps:
       - uses: actions/checkout@v4
         with:
@@ -35,7 +35,7 @@ jobs:
           aws-region: ${{secrets.AWS_REGION}}
 
       - name: Docker Build Task
-        run: docker compose build ${{ matrix.task }}
+        run: docker buildx build ./tasks/${{ matrix.task }} -t cloudtak-${{ matrix.task }}
 
       - name: Login to Amazon ECR
         id: login-ecr

.github/workflows/test.yml

@@ -22,13 +22,13 @@ jobs:
           ref: ${{github.event.pull_request.head.sha || github.sha}}
 
       - name: Docker Data Task Build
-        run: docker compose build task
+        run: docker buildx build ./tasks/data -t cloudtak-data
 
       - name: Docker Data Task Lint
-        run: docker run cloudtak-task:latest npm run lint
+        run: docker run cloudtak-data:latest npm run lint
 
       - name: Docker Data Task Test
-        run: docker run cloudtak-task:latest npm test
+        run: docker run cloudtak-data:latest npm test
 
   pmtiles:
     runs-on: ubuntu-latest

api/lib/api/files.ts

@@ -16,6 +16,10 @@ export const Content = Type.Object({
   Name: Type.String()
 });
 
+export const Config = Type.Object({
+    uploadSizeLimit: Type.Integer()
+})
+
 export default class File {
     api: TAKAPI;
 
@@ -130,7 +134,7 @@ export default class File {
         });
     }
 
-    async config() {
+    async config(): Promise<Static<typeof Config>> {
         const url = new URL('/files/api/config', this.api.url);
 
         return await this.api.fetch(url, {

api/package.json

@@ -9,7 +9,8 @@
         "tstest": "test/test",
         "build": "tsc && cp .env* dist/ || true && cp ./lib/jobs dist/lib/jobs -r && cp ./icons dist/icons -r && cp package.json dist/ && cp migrations/ dist/ -r",
         "lint": "eslint index.ts lib/ routes/ test/",
-        "loaddb": "echo 'DROP DATABASE tak_ps_etl' | psql && echo 'CREATE DATABASE tak_ps_etl' | psql && psql tak_ps_etl < ",
+        "emptydb": "echo 'DROP DATABASE tak_ps_etl' | psql && echo 'CREATE DATABASE tak_ps_etl' | psql",
+        "loaddb": "npm run emptydb && psql tak_ps_etl < ",
         "migratedb": "npm run lint && git stash && tsc --outDir . && drizzle-kit generate && git add migrations/* && git commit -m \"DB Migration\" && git clean -f && git stash",
         "dev": "tsx watch --ignore web/ index.ts --noevents --nometrics --nosinks --unsafe",
         "prod": "NODE_OPTIONS='--max-old-space-size=6144' node dist/index.js"

api/routes/login.ts

@@ -23,33 +23,39 @@ export default async function router(schema: Schema, config: Config) {
         })
     }, async (req, res) => {
         try {
-            const email = await provider.login(req.body.username.toLowerCase(), req.body.password);
+            let profile;
 
-            if (config.server.provider_url) {
-                try {
-                    const response = await config.external.login(email);
+            if (config.server.auth.key && config.server.auth.cert) {
+                const email = await provider.login(req.body.username.toLowerCase(), req.body.password);
 
+                if (config.server.provider_url) {
+                    try {
+                        const response = await config.external.login(email);
+
+                        await config.models.Profile.commit(email, {
+                            ...response,
+                            last_login: new Date().toISOString()
+                        });
+                    } catch (err) {
+                        // If there are upstream errors the user is limited to WebTAK like functionality
+                        await config.models.Profile.commit(email, {
+                            system_admin: false,
+                            agency_admin: [],
+                            last_login: new Date().toISOString()
+                        });
+                        console.error(err);
+                    }
+                } else {
                     await config.models.Profile.commit(email, {
-                        ...response,
-                        last_login: new Date().toISOString()
-                    });
-                } catch (err) {
-                    // If there are upstream errors the user is limited to WebTAK like functionality
-                    await config.models.Profile.commit(email, {
-                        system_admin: false,
-                        agency_admin: [],
                         last_login: new Date().toISOString()
                     });
-                    console.error(err);
                 }
+
+                profile = await config.models.Profile.from(email);
             } else {
-                await config.models.Profile.commit(email, {
-                    last_login: new Date().toISOString()
-                });
+                throw new Err(400, null, 'Server has not been configured');
             }
 
-            const profile = await config.models.Profile.from(email);
-
             let access = AuthUserAccess.USER
             if (profile.system_admin) {
                 access = AuthUserAccess.ADMIN
@@ -59,8 +65,8 @@ export default async function router(schema: Schema, config: Config) {
 
             return res.json({
                 access,
-                email,
-                token: jwt.sign({ access, email }, config.SigningSecret, { expiresIn: '16h' })
+                email: profile.username,
+                token: jwt.sign({ access, email: profile.username }, config.SigningSecret, { expiresIn: '16h' })
             })
         } catch (err) {
             return Err.respond(err, res);
@@ -80,7 +86,10 @@ export default async function router(schema: Schema, config: Config) {
 
             const profile = await config.models.Profile.from(user.email);
 
-            await provider.valid(profile);
+            // If the server hasn't been configured the user won't have a valid cert
+            if (config.server.auth.key && config.server.auth.cert) {
+                await provider.valid(profile);
+            }
 
             return res.json({
                 email: user.email,

api/routes/server.ts

@@ -6,6 +6,10 @@ import Auth, { AuthUserAccess } from '../lib/auth.js';
 import { sql } from 'drizzle-orm';
 import Config from '../lib/config.js';
 import { ServerResponse } from '../lib/types.js';
+import TAKAPI, {
+    APIAuthCertificate,
+    APIAuthPassword
+} from '../lib/tak-api.js';
 
 export default async function router(schema: Schema, config: Config) {
     await schema.get('/server', {
@@ -15,9 +19,7 @@ export default async function router(schema: Schema, config: Config) {
         res: ServerResponse
     }, async (req, res) => {
         try {
-            const user = await Auth.as_user(config, req);
-
-            if (!config.server.auth) {
+            if (!config.server.auth.key || !config.server.auth.cert) {
                 return res.json({
                     id: 1,
                     status: 'unconfigured',
@@ -29,8 +31,12 @@ export default async function router(schema: Schema, config: Config) {
                     auth: false
                 });
             } else {
+                const user = await Auth.as_user(config, req);
+
                 let auth = false
-                if (config.server.auth.cert && config.server.auth.key) auth = true;
+                if (config.server.auth.cert && config.server.auth.key) {
+                    auth = true;
+                }
 
                 if (user.access === AuthUserAccess.ADMIN) {
                     const response: Static<typeof ServerResponse> = {
@@ -74,6 +80,11 @@ export default async function router(schema: Schema, config: Config) {
             provider_url: Type.Optional(Type.String()),
             provider_secret: Type.Optional(Type.String()),
             provider_client: Type.Optional(Type.String()),
+
+            // Used during initial server config to test connection & set system admin
+            username: Type.Optional(Type.String()),
+            password: Type.Optional(Type.String()),
+
             auth: Type.Optional(Type.Object({
                 cert: Type.String(),
                 key: Type.String(),
@@ -82,10 +93,40 @@ export default async function router(schema: Schema, config: Config) {
         res: ServerResponse
     }, async (req, res) => {
         try {
-            await Auth.as_user(config, req, { admin: true });
+            if (config.server.auth.key && config.server.auth.cert) {
+                await Auth.as_user(config, req, { admin: true });
+            }
 
             if (!config.server) throw new Err(400, null, 'Cannot patch a server that hasn\'t been created');
 
+            if (req.body.auth) {
+                const api = await TAKAPI.init(
+                    new URL(String(req.body.api)),
+                    new APIAuthCertificate(req.body.auth.cert, req.body.auth.key)
+                );
+
+                const config = await api.Files.config();
+                if (config.uploadSizeLimit === undefined) {
+                    throw new Err(400, null, 'Could not connect to TAK Server');
+                }
+            }
+
+            // An unconfigured server will set the first successful username/pass as a CloudTAK System Admin
+            if (!config.server.auth.key && !config.server.auth.cert && req.body.username && req.body.password) {
+                const auth = new APIAuthPassword(req.body.username, req.body.password)
+                const api = await TAKAPI.init(new URL(req.body.api), auth);
+
+                const certs = await api.Credentials.generate();
+
+                await config.models.Profile.generate({
+                    auth: certs,
+                    username: req.body.username,
+                    system_admin: true
+                });
+            } else if (!config.server.auth.key && !config.server.auth.cert && (!req.body.username || !req.body.password)) {
+                throw new Err(400, null, 'Initial configuration must include valid TAK Username & Password to set System Administrator');
+            }
+
             config.server = await config.models.Server.commit(config.server.id, {
                 ...req.body,
                 updated: sql`Now()`,
@@ -97,9 +138,9 @@ export default async function router(schema: Schema, config: Config) {
             if (config.server.auth.cert && config.server.auth.key) auth = true;
 
             const response: Static<typeof ServerResponse> = {
-                    status: 'configured',
-                    ...config.server,
-                    auth
+                status: 'configured',
+                ...config.server,
+                auth
             };
 
             if (config.server.auth.cert && config.server.auth.key) {

api/routes/users.ts

@@ -44,19 +44,19 @@ export default async function router(schema: Schema, config: Config) {
         }
     });
 
-    await schema.get('/user/:email', {
+    await schema.get('/user/:username', {
         name: 'Get User',
         group: 'User',
         description: 'Let Admins see a given user of the system',
         params: Type.Object({
-            email: Type.String(),
+            username: Type.String(),
         }),
         res: ProfileResponse
     }, async (req, res) => {
         try {
             await Auth.as_user(config, req, { admin: true });
 
-            const user = await config.models.Profile.from(req.params.email);
+            const user = await config.models.Profile.from(req.params.username);
 
             return res.json({
                 ...user,

api/start

@@ -3,12 +3,20 @@
 set -x
 set -euo pipefail
 
-export ROOT_URL=$(echo "${API_URL}" | grep -Po "[a-z]+\.[a-z]+$")
+echo "API_URL: ${API_URL}"
+if [[ ${API_URL} =~ "localhost" ]]; then
+    export ROOT_URL=${API_URL}
+    echo "ROOT_URL: ${ROOT_URL}"
+else
+    export ROOT_URL=$(echo "${API_URL}" | grep -Po "[a-z]+\.[a-z]+$")
+    echo "ROOT_URL: ${ROOT_URL}"
+fi
 
-memcached -d -u root
 sed -i "s/API_URL/${API_URL}/g" /etc/nginx/nginx.conf
 sed -i "s/ROOT_URL/${ROOT_URL}/g" /etc/nginx/nginx.conf
 
+memcached -d -u root
+
 nginx
 
 npm run prod