Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Cold Start #359

Merged
merged 20 commits into from
Oct 8, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .deploy
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@
"artifacts": {
"docker": [
"coe-ecr-etl:{{gitsha}}",
"coe-ecr-etl:task-{{gitsha}}",
"coe-ecr-etl:data-{{gitsha}}",
"coe-ecr-etl:pmtiles-{{gitsha}}",
"coe-ecr-etl:hooks-{{gitsha}}",
"coe-ecr-etl:events-{{gitsha}}"
Expand Down
4 changes: 2 additions & 2 deletions .github/workflows/ecr_task.yml
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ jobs:
if: github.event.pull_request.draft == false
strategy:
matrix:
task: [task, pmtiles, hooks, events]
task: [data, pmtiles, hooks, events]
steps:
- uses: actions/checkout@v4
with:
Expand All @@ -35,7 +35,7 @@ jobs:
aws-region: ${{secrets.AWS_REGION}}

- name: Docker Build Task
run: docker compose build ${{ matrix.task }}
run: docker buildx build ./tasks/${{ matrix.task }} -t cloudtak-${{ matrix.task }}

- name: Login to Amazon ECR
id: login-ecr
Expand Down
6 changes: 3 additions & 3 deletions .github/workflows/test.yml
Original file line number Diff line number Diff line change
Expand Up @@ -22,13 +22,13 @@ jobs:
ref: ${{github.event.pull_request.head.sha || github.sha}}

- name: Docker Data Task Build
run: docker compose build task
run: docker buildx build ./tasks/data -t cloudtak-data

- name: Docker Data Task Lint
run: docker run cloudtak-task:latest npm run lint
run: docker run cloudtak-data:latest npm run lint

- name: Docker Data Task Test
run: docker run cloudtak-task:latest npm test
run: docker run cloudtak-data:latest npm test

pmtiles:
runs-on: ubuntu-latest
Expand Down
6 changes: 5 additions & 1 deletion api/lib/api/files.ts
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,10 @@ export const Content = Type.Object({
Name: Type.String()
});

export const Config = Type.Object({
uploadSizeLimit: Type.Integer()
})

export default class File {
api: TAKAPI;

Expand Down Expand Up @@ -130,7 +134,7 @@ export default class File {
});
}

async config() {
async config(): Promise<Static<typeof Config>> {
const url = new URL('/files/api/config', this.api.url);

return await this.api.fetch(url, {
Expand Down
2,137 changes: 1,066 additions & 1,071 deletions api/package-lock.json

Large diffs are not rendered by default.

3 changes: 2 additions & 1 deletion api/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,8 @@
"tstest": "test/test",
"build": "tsc && cp .env* dist/ || true && cp ./lib/jobs dist/lib/jobs -r && cp ./icons dist/icons -r && cp package.json dist/ && cp migrations/ dist/ -r",
"lint": "eslint index.ts lib/ routes/ test/",
"loaddb": "echo 'DROP DATABASE tak_ps_etl' | psql && echo 'CREATE DATABASE tak_ps_etl' | psql && psql tak_ps_etl < ",
"emptydb": "echo 'DROP DATABASE tak_ps_etl' | psql && echo 'CREATE DATABASE tak_ps_etl' | psql",
"loaddb": "npm run emptydb && psql tak_ps_etl < ",
"migratedb": "npm run lint && git stash && tsc --outDir . && drizzle-kit generate && git add migrations/* && git commit -m \"DB Migration\" && git clean -f && git stash",
"dev": "tsx watch --ignore web/ index.ts --noevents --nometrics --nosinks --unsafe",
"prod": "NODE_OPTIONS='--max-old-space-size=6144' node dist/index.js"
Expand Down
51 changes: 30 additions & 21 deletions api/routes/login.ts
Original file line number Diff line number Diff line change
Expand Up @@ -23,33 +23,39 @@ export default async function router(schema: Schema, config: Config) {
})
}, async (req, res) => {
try {
const email = await provider.login(req.body.username.toLowerCase(), req.body.password);
let profile;

if (config.server.provider_url) {
try {
const response = await config.external.login(email);
if (config.server.auth.key && config.server.auth.cert) {
const email = await provider.login(req.body.username.toLowerCase(), req.body.password);

if (config.server.provider_url) {
try {
const response = await config.external.login(email);

await config.models.Profile.commit(email, {
...response,
last_login: new Date().toISOString()
});
} catch (err) {
// If there are upstream errors the user is limited to WebTAK like functionality
await config.models.Profile.commit(email, {
system_admin: false,
agency_admin: [],
last_login: new Date().toISOString()
});
console.error(err);
}
} else {
await config.models.Profile.commit(email, {
...response,
last_login: new Date().toISOString()
});
} catch (err) {
// If there are upstream errors the user is limited to WebTAK like functionality
await config.models.Profile.commit(email, {
system_admin: false,
agency_admin: [],
last_login: new Date().toISOString()
});
console.error(err);
}

profile = await config.models.Profile.from(email);
} else {
await config.models.Profile.commit(email, {
last_login: new Date().toISOString()
});
throw new Err(400, null, 'Server has not been configured');
}

const profile = await config.models.Profile.from(email);

let access = AuthUserAccess.USER
if (profile.system_admin) {
access = AuthUserAccess.ADMIN
Expand All @@ -59,8 +65,8 @@ export default async function router(schema: Schema, config: Config) {

return res.json({
access,
email,
token: jwt.sign({ access, email }, config.SigningSecret, { expiresIn: '16h' })
email: profile.username,
token: jwt.sign({ access, email: profile.username }, config.SigningSecret, { expiresIn: '16h' })
})
} catch (err) {
return Err.respond(err, res);
Expand All @@ -80,7 +86,10 @@ export default async function router(schema: Schema, config: Config) {

const profile = await config.models.Profile.from(user.email);

await provider.valid(profile);
// If the server hasn't been configured the user won't have a valid cert
if (config.server.auth.key && config.server.auth.cert) {
await provider.valid(profile);
}

return res.json({
email: user.email,
Expand Down
57 changes: 49 additions & 8 deletions api/routes/server.ts
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,10 @@ import Auth, { AuthUserAccess } from '../lib/auth.js';
import { sql } from 'drizzle-orm';
import Config from '../lib/config.js';
import { ServerResponse } from '../lib/types.js';
import TAKAPI, {
APIAuthCertificate,
APIAuthPassword
} from '../lib/tak-api.js';

export default async function router(schema: Schema, config: Config) {
await schema.get('/server', {
Expand All @@ -15,9 +19,7 @@ export default async function router(schema: Schema, config: Config) {
res: ServerResponse
}, async (req, res) => {
try {
const user = await Auth.as_user(config, req);

if (!config.server.auth) {
if (!config.server.auth.key || !config.server.auth.cert) {
return res.json({
id: 1,
status: 'unconfigured',
Expand All @@ -29,8 +31,12 @@ export default async function router(schema: Schema, config: Config) {
auth: false
});
} else {
const user = await Auth.as_user(config, req);

let auth = false
if (config.server.auth.cert && config.server.auth.key) auth = true;
if (config.server.auth.cert && config.server.auth.key) {
auth = true;
}

if (user.access === AuthUserAccess.ADMIN) {
const response: Static<typeof ServerResponse> = {
Expand Down Expand Up @@ -74,6 +80,11 @@ export default async function router(schema: Schema, config: Config) {
provider_url: Type.Optional(Type.String()),
provider_secret: Type.Optional(Type.String()),
provider_client: Type.Optional(Type.String()),

// Used during initial server config to test connection & set system admin
username: Type.Optional(Type.String()),
password: Type.Optional(Type.String()),

auth: Type.Optional(Type.Object({
cert: Type.String(),
key: Type.String(),
Expand All @@ -82,10 +93,40 @@ export default async function router(schema: Schema, config: Config) {
res: ServerResponse
}, async (req, res) => {
try {
await Auth.as_user(config, req, { admin: true });
if (config.server.auth.key && config.server.auth.cert) {
await Auth.as_user(config, req, { admin: true });
}

if (!config.server) throw new Err(400, null, 'Cannot patch a server that hasn\'t been created');

if (req.body.auth) {
const api = await TAKAPI.init(
new URL(String(req.body.api)),
new APIAuthCertificate(req.body.auth.cert, req.body.auth.key)
);

const config = await api.Files.config();
if (config.uploadSizeLimit === undefined) {
throw new Err(400, null, 'Could not connect to TAK Server');
}
}

// An unconfigured server will set the first successful username/pass as a CloudTAK System Admin
if (!config.server.auth.key && !config.server.auth.cert && req.body.username && req.body.password) {
const auth = new APIAuthPassword(req.body.username, req.body.password)
const api = await TAKAPI.init(new URL(req.body.api), auth);

const certs = await api.Credentials.generate();

await config.models.Profile.generate({
auth: certs,
username: req.body.username,
system_admin: true
});
} else if (!config.server.auth.key && !config.server.auth.cert && (!req.body.username || !req.body.password)) {
throw new Err(400, null, 'Initial configuration must include valid TAK Username & Password to set System Administrator');
}

config.server = await config.models.Server.commit(config.server.id, {
...req.body,
updated: sql`Now()`,
Expand All @@ -97,9 +138,9 @@ export default async function router(schema: Schema, config: Config) {
if (config.server.auth.cert && config.server.auth.key) auth = true;

const response: Static<typeof ServerResponse> = {
status: 'configured',
...config.server,
auth
status: 'configured',
...config.server,
auth
};

if (config.server.auth.cert && config.server.auth.key) {
Expand Down
6 changes: 3 additions & 3 deletions api/routes/users.ts
Original file line number Diff line number Diff line change
Expand Up @@ -44,19 +44,19 @@ export default async function router(schema: Schema, config: Config) {
}
});

await schema.get('/user/:email', {
await schema.get('/user/:username', {
name: 'Get User',
group: 'User',
description: 'Let Admins see a given user of the system',
params: Type.Object({
email: Type.String(),
username: Type.String(),
}),
res: ProfileResponse
}, async (req, res) => {
try {
await Auth.as_user(config, req, { admin: true });

const user = await config.models.Profile.from(req.params.email);
const user = await config.models.Profile.from(req.params.username);

return res.json({
...user,
Expand Down
12 changes: 10 additions & 2 deletions api/start
Original file line number Diff line number Diff line change
Expand Up @@ -3,12 +3,20 @@
set -x
set -euo pipefail

export ROOT_URL=$(echo "${API_URL}" | grep -Po "[a-z]+\.[a-z]+$")
echo "API_URL: ${API_URL}"
if [[ ${API_URL} =~ "localhost" ]]; then
export ROOT_URL=${API_URL}
echo "ROOT_URL: ${ROOT_URL}"
else
export ROOT_URL=$(echo "${API_URL}" | grep -Po "[a-z]+\.[a-z]+$")
echo "ROOT_URL: ${ROOT_URL}"
fi

memcached -d -u root
sed -i "s/API_URL/${API_URL}/g" /etc/nginx/nginx.conf
sed -i "s/ROOT_URL/${ROOT_URL}/g" /etc/nginx/nginx.conf

memcached -d -u root

nginx

npm run prod
Loading
Loading