From 38c6d78ed10baa20bd1179304fa55d7b2b04b9e3 Mon Sep 17 00:00:00 2001 From: ingalls Date: Sat, 26 Oct 2024 08:46:53 -0600 Subject: [PATCH 1/2] Autoload Basemaps --- api/Dockerfile | 14 +++++------ api/index.ts | 4 +++ api/{nginx.conf => nginx.conf.js} | 14 ++++++++--- api/package-lock.json | 41 +++++++++++++++++-------------- api/routes/basemap.ts | 22 +++++++++++++++++ api/routes/profile.ts | 2 ++ api/routes/users.ts | 3 +++ api/start | 3 +-- docker-compose.yml | 3 ++- 9 files changed, 73 insertions(+), 33 deletions(-) rename api/{nginx.conf => nginx.conf.js} (86%) diff --git a/api/Dockerfile b/api/Dockerfile index 12664fb96..6512bc612 100644 --- a/api/Dockerfile +++ b/api/Dockerfile @@ -5,12 +5,14 @@ EXPOSE 5000 ENV HOME=/home/etl WORKDIR $HOME -RUN apk add nodejs-current npm memcached python3 make bash g++ openssl postgresql-client grep +RUN apk add git nodejs-current npm memcached python3 make bash g++ openssl postgresql-client grep WORKDIR $HOME/api -COPY package.json ./ -COPY package-lock.json ./ +ADD package.json ./ +ADD package-lock.json ./ + +RUN git clone --branch v1.1.0 --depth 1 https://github.com/dfpc-coe/CloudTAK-Data.git data RUN npm install @@ -24,9 +26,7 @@ RUN cd web \ && cd .. RUN npm run lint \ - && npm run build - -COPY ./nginx.conf /etc/nginx/nginx.conf -COPY ./start $HOME/api + && npm run build \ + && mv data dist/ CMD ["./start"] diff --git a/api/index.ts b/api/index.ts index 8c527316f..9762954a0 100644 --- a/api/index.ts +++ b/api/index.ts @@ -47,6 +47,10 @@ try { const pkg = JSON.parse(String(fs.readFileSync(new URL('./package.json', import.meta.url)))); +process.on('uncaughtException', (exception) => { + console.error('Error', exception); +}); + if (import.meta.url === `file://${process.argv[1]}`) { const config = await Config.env({ silent: args.silent || false, diff --git a/api/nginx.conf b/api/nginx.conf.js similarity index 86% rename from api/nginx.conf rename to api/nginx.conf.js index 74a27272c..f74f7b0f7 100644 --- a/api/nginx.conf +++ b/api/nginx.conf.js @@ -1,3 +1,8 @@ + +let CSP = `add_header 'Content-Security-Policy' "default-src 'self' *.${process.env.API_URL}; $\{IMG}; $\{WORKER}; $\{CONNECT}; $\{STYLE_SRC_ATTR}; $\{STYLE_SRC_ELEM}; $\{FONT}; upgrade-insecure-requests;" always;` +if (process.env.API_URL.includes('localhost')) CSP = ''; + +console.log(` user nginx; worker_processes auto; error_log /var/log/nginx/error.log warn; @@ -31,13 +36,13 @@ http { add_header 'Strict-Transport-Security' 'max-age=31536000; includeSubDomains; preload' always; add_header 'Permissions-Policy' 'fullscreen=(self), geolocation=(self), clipboard-read=(self), clipboard-write=(self)' always; - set $IMG "img-src 'self' data: *.API_URL"; + set $IMG "img-src 'self' data: *.${process.env.API_URL}"; set $FONT "font-src 'self' data:"; set $WORKER "worker-src 'self' blob:"; set $STYLE_SRC_ELEM "style-src-elem 'self' 'unsafe-inline'"; set $STYLE_SRC_ATTR "style-src-attr 'unsafe-inline'"; - set $CONNECT "connect-src 'self' *.API_URL:* *.ROOT_URL:*"; - add_header 'Content-Security-Policy' "default-src 'self' *.API_URL; ${IMG}; ${WORKER}; ${CONNECT}; ${STYLE_SRC_ATTR}; ${STYLE_SRC_ELEM}; ${FONT}; upgrade-insecure-requests;" always; + set $CONNECT "connect-src 'self' *.${process.env.API_URL}:* *.ROOT_URL:*"; + ${CSP} location = / { add_header 'X-Content-Type-Options' 'nosniff' always; @@ -45,7 +50,7 @@ http { add_header 'Referrer-Policy' 'strict-origin-when-cross-origin' always; add_header 'Strict-Transport-Security' 'max-age=31536000; includeSubDomains; preload' always; add_header 'Permissions-Policy' 'fullscreen=(self), geolocation=(self), clipboard-read=(self), clipboard-write=(self)' always; - add_header 'Content-Security-Policy' "default-src 'self' *.API_URL; ${IMG}; ${WORKER}; ${CONNECT}; ${STYLE_SRC_ATTR}; ${STYLE_SRC_ELEM}; ${FONT}; upgrade-insecure-requests;" always; + ${CSP} add_header 'Cache-Control' 'no-store, no-cache, must-revalidate' always; add_header 'Expires' 0 always; @@ -93,3 +98,4 @@ http { } } +`) diff --git a/api/package-lock.json b/api/package-lock.json index e9e8d35b6..4b1179e81 100644 --- a/api/package-lock.json +++ b/api/package-lock.json @@ -470,9 +470,9 @@ } }, "node_modules/@aws-sdk/client-cloudwatch-logs": { - "version": "3.679.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/client-cloudwatch-logs/-/client-cloudwatch-logs-3.679.0.tgz", - "integrity": "sha512-A1qTVNX8KdpqvXgULd4Suo88uuNWPa8DiuBL8Qkw/WefYT7TSWsOpwuVK0oFkMpCfB0rQN9fXZh2DBiaz8YmZg==", + "version": "3.680.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-cloudwatch-logs/-/client-cloudwatch-logs-3.680.0.tgz", + "integrity": "sha512-Jb9njjO9ziLyjUIlGz+59+Ifo2sDSCLjphvO8+GL4A7KsxgpDLis6IeNQn4PguVDWusOGa4rpm11o0GfMHAQBw==", "license": "Apache-2.0", "dependencies": { "@aws-crypto/sha256-browser": "5.2.0", @@ -801,9 +801,9 @@ } }, "node_modules/@aws-sdk/client-lambda": { - "version": "3.679.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/client-lambda/-/client-lambda-3.679.0.tgz", - "integrity": "sha512-CSJ7FF1RSgABJRf9mRaH4JGUvfMKv42zO8hHsr/GLd7HL9uTTZ2iRQ7Hpx7TRLWo/Ju7WIyZbbImNLtQ9lpVeQ==", + "version": "3.680.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-lambda/-/client-lambda-3.680.0.tgz", + "integrity": "sha512-oABfvrJTVvlVnCEi6lMtxlC/ta5wxu79bB2ZACe/0AxFZrXgr+gscrUtSm1iNye5+6oeYvQPD7b+ft6UwFrSlw==", "license": "Apache-2.0", "dependencies": { "@aws-crypto/sha256-browser": "5.2.0", @@ -2666,17 +2666,20 @@ } }, "node_modules/@eslint-community/eslint-utils": { - "version": "4.4.0", - "resolved": "https://registry.npmjs.org/@eslint-community/eslint-utils/-/eslint-utils-4.4.0.tgz", - "integrity": "sha512-1/sA4dwrzBAyeUoQ6oxahHKmrZvsnLCg4RfxW3ZFGGmQkSNQPFNLV9CUEFQP1x9EYXHTo5p6xdhZM1Ne9p/AfA==", + "version": "4.4.1", + "resolved": "https://registry.npmjs.org/@eslint-community/eslint-utils/-/eslint-utils-4.4.1.tgz", + "integrity": "sha512-s3O3waFUrMV8P/XaF/+ZTp1X9XBZW1a4B97ZnjQF2KYWaFD2A8KyFBsrsfSjEmjn3RGWAIuvlneuZm3CUK3jbA==", "dev": true, "license": "MIT", "dependencies": { - "eslint-visitor-keys": "^3.3.0" + "eslint-visitor-keys": "^3.4.3" }, "engines": { "node": "^12.22.0 || ^14.17.0 || >=16.0.0" }, + "funding": { + "url": "https://opencollective.com/eslint" + }, "peerDependencies": { "eslint": "^6.0.0 || ^7.0.0 || >=8.0.0" } @@ -2695,9 +2698,9 @@ } }, "node_modules/@eslint-community/regexpp": { - "version": "4.11.1", - "resolved": "https://registry.npmjs.org/@eslint-community/regexpp/-/regexpp-4.11.1.tgz", - "integrity": "sha512-m4DVN9ZqskZoLU5GlWZadwDnYo3vAEydiUayB9widCl9ffWx2IvPnp6n3on5rJmziJSw9Bv+Z3ChDVdMwXCY8Q==", + "version": "4.11.2", + "resolved": "https://registry.npmjs.org/@eslint-community/regexpp/-/regexpp-4.11.2.tgz", + "integrity": "sha512-2WwyTYNVaMNUWPZTOJdkax9iqTdirrApgTbk+Qoq5EPX6myqZvG8QGFRgdKmkjKVG6/G/a565vpPauHk0+hpBA==", "dev": true, "license": "MIT", "engines": { @@ -4357,9 +4360,9 @@ } }, "node_modules/@tak-ps/node-cot": { - "version": "12.7.1", - "resolved": "https://registry.npmjs.org/@tak-ps/node-cot/-/node-cot-12.7.1.tgz", - "integrity": "sha512-3algHB/+7ntJAKOQjprsiv+hDvMA8VcfyOlkBJLUTKjrXBc5q5+/6XeapxFJ/04u1fS+ksPug8KplGBp6ReMlg==", + "version": "12.8.0", + "resolved": "https://registry.npmjs.org/@tak-ps/node-cot/-/node-cot-12.8.0.tgz", + "integrity": "sha512-1WOTlUUyjpICEXgiWMGs8AeT+77YtQQA4NOo/SXq1YZCOO8V88J2m1GM+Mt9S1unqF0o/F5YGK2JvI55EGOd3A==", "license": "MIT", "dependencies": { "@openaddresses/batch-error": "^2.4.0", @@ -4877,9 +4880,9 @@ } }, "node_modules/@types/node": { - "version": "22.8.0", - "resolved": "https://registry.npmjs.org/@types/node/-/node-22.8.0.tgz", - "integrity": "sha512-84rafSBHC/z1i1E3p0cJwKA+CfYDNSXX9WSZBRopjIzLET8oNt6ht2tei4C7izwDeEiLLfdeSVBv1egOH916hg==", + "version": "22.8.1", + "resolved": "https://registry.npmjs.org/@types/node/-/node-22.8.1.tgz", + "integrity": "sha512-k6Gi8Yyo8EtrNtkHXutUu2corfDf9su95VYVP10aGYMMROM6SAItZi0w1XszA6RtWTHSVp5OeFof37w0IEqCQg==", "license": "MIT", "dependencies": { "undici-types": "~6.19.8" diff --git a/api/routes/basemap.ts b/api/routes/basemap.ts index 9099f65b6..88fe1fe9b 100644 --- a/api/routes/basemap.ts +++ b/api/routes/basemap.ts @@ -1,5 +1,6 @@ import path from 'node:path'; import { bbox } from '@turf/bbox'; +import fs from 'node:fs/promises' import Err from '@openaddresses/batch-error'; import TileJSON, { TileJSONType } from '../lib/control/tilejson.js'; import Auth, { AuthUserAccess, ResourceCreationScope } from '../lib/auth.js'; @@ -30,6 +31,27 @@ const AugmentedBasemapResponse = Type.Composite([ ]) export default async function router(schema: Schema, config: Config) { + const count = await config.models.Basemap.count(); + if (count === 0) { + try { + await fs.access(new URL('../data/', import.meta.url)); + + for (const file of await fs.readdir(new URL('../data/basemaps/', import.meta.url))) { + console.error(`ok - loading basemap ${file}`); + const b = (await BasemapParser.parse(String(await fs.readFile(new URL(`../data/basemaps/${file}`, import.meta.url))))).to_json(); + + await config.models.Basemap.generate({ + name: b.name || 'Unknown', + url: b.url, + minzoom: b.minZoom || 0, + maxzoom: b.maxZoom || 16 + }) + } + } catch (err) { + console.log('Could not automatically load basemaps', err); + } + } + await schema.put('/basemap', { name: 'Import Basemaps', group: 'BaseMap', diff --git a/api/routes/profile.ts b/api/routes/profile.ts index d32299f85..bda9b8e91 100644 --- a/api/routes/profile.ts +++ b/api/routes/profile.ts @@ -19,6 +19,7 @@ export default async function router(schema: Schema, config: Config) { const user = await Auth.as_user(config, req); const profile = await config.models.Profile.from(user.email); + // @ts-expect-error Update Batch-Generic to specify actual geometry type (Point) instead of Geometry res.json(profile); } catch (err) { Err.respond(err, res); @@ -52,6 +53,7 @@ export default async function router(schema: Schema, config: Config) { updated: sql`Now()` }); + // @ts-expect-error Update Batch-Generic to specify actual geometry type (Point) instead of Geometry res.json(profile); } catch (err) { Err.respond(err, res); diff --git a/api/routes/users.ts b/api/routes/users.ts index 4fd65c1f9..2edaa0665 100644 --- a/api/routes/users.ts +++ b/api/routes/users.ts @@ -38,6 +38,7 @@ export default async function router(schema: Schema, config: Config) { ` }); + // @ts-expect-error Update Batch-Generic to specify actual geometry type (Point) instead of Geometry res.json(list); } catch (err) { Err.respond(err, res); @@ -61,6 +62,7 @@ export default async function router(schema: Schema, config: Config) { const user = await config.models.Profile.commit(req.params.username, req.body); + // @ts-expect-error Update Batch-Generic to specify actual geometry type (Point) instead of Geometry res.json({ ...user, agency_admin: user.agency_admin || [] @@ -84,6 +86,7 @@ export default async function router(schema: Schema, config: Config) { const user = await config.models.Profile.from(req.params.username); + // @ts-expect-error Update Batch-Generic to specify actual geometry type (Point) instead of Geometry res.json({ ...user, agency_admin: user.agency_admin || [] diff --git a/api/start b/api/start index 8ed805813..6af0ab478 100755 --- a/api/start +++ b/api/start @@ -12,8 +12,7 @@ else echo "ROOT_URL: ${ROOT_URL}" fi -sed -i "s/API_URL/${API_URL}/g" /etc/nginx/nginx.conf -sed -i "s/ROOT_URL/${ROOT_URL}/g" /etc/nginx/nginx.conf +node nginx.conf.js > /etc/nginx/nginx.conf memcached -d -u root diff --git a/docker-compose.yml b/docker-compose.yml index 20f6e9aee..7b52d4142 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -11,7 +11,8 @@ services: environment: - SigningSecret=coe-wildland-fire - POSTGRES=postgres://docker:docker@postgis:5432/gis - - API_URL=localhost:5001 + - API_URL=localhost:5000 + - ROOT_URL=localhost:5001 postgis: platform: linux/amd64 From 07551073386392338c146704219730d363909653 Mon Sep 17 00:00:00 2001 From: ingalls Date: Sat, 26 Oct 2024 08:49:17 -0600 Subject: [PATCH 2/2] Allow Port 5001 --- docker-compose.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/docker-compose.yml b/docker-compose.yml index 7b52d4142..60c8d8455 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -8,6 +8,7 @@ services: - store ports: - "5000:5000" + - "5001:5001" environment: - SigningSecret=coe-wildland-fire - POSTGRES=postgres://docker:docker@postgis:5432/gis