Added support for local azure storage emulation

census/README.md

@@ -1,4 +1,3 @@
-
 # Getting started
 
 ## Prerequisites
@@ -9,30 +8,13 @@
 4. Clone `./api/.env.example` and rename as `./api/.env` and configure it as follows.
 
 ### Twitch setup
+
 1. Go to the [Twitch Developer Dashboard](https://dev.twitch.tv/console).
 2. Create a new application and add the following as OAuth Redirect URLs.
-    - `http://localhost:35523/auth/redirect`
-    - `http://localhost:35523/admin/redirect`
+   - `http://localhost:35523/auth/redirect`
+   - `http://localhost:35523/admin/redirect`
 3. Copy the generated Client ID and Client Secret
- 	- Update the `./api/.env` file with `TWITCH_CLIENT_ID` and `TWITCH_CLIENT_SECRET`.
-
-### Azure setup
-
-Currently, we don't support the Azurite emulator for local development, so you'll need to have an Azure storage account and container that you can use.
-
-1. Go to the Azure Portal.
-2. Create a storage account: 
- 	- Under Azure services, select Storage accounts and create a new storage account.
- 	- Update the `./api/.env` file with `STORAGE_ACCOUNT_NAME`.
-3. Obtain the access key:
-    - From your storage account's page, navigate to Security + networking > Access keys.
- 	- Update the `./api/.env` file with `STORAGE_ACCOUNT_KEY` with either `key1` or `key2`.
-4. Create a storage container:
-    - From your storage account's page, go to Data storage > Containers and create a new container.
-    - Update the `./api/.env` file with `CONTAINER_NAME`.
-5. Enable public access for the container
-	- From your storage account's page, go to Configuration and set `Allow Blob anonymous access` to `Enabled`.
-	- From your container's page, select Change access level and set the `anonymous access level` as `Container` or `Blob`.
+   - Update the `./api/.env` file with `TWITCH_CLIENT_ID` and `TWITCH_CLIENT_SECRET`.
 
 ### `JWT_SECRET` setup
 
@@ -41,13 +23,33 @@ This is the secret that the API uses to sign the JWTs. For local development, yo
 ## Running the services
 
 1. Start the database.
-	- In the root of the repo, run `docker compose up`.
+   - In the root of the repo, run `docker compose up`.
 2. Start the API.
-	- `pnpm --filter=@alveusgg/census-api start`.
-	- `pnpm --filter=@alveusgg/census-api dev` to start in watch mode.
+   - `pnpm --filter=@alveusgg/census-api start`.
+   - `pnpm --filter=@alveusgg/census-api dev` to start in watch mode.
 3. Start the UI.
-	- `pnpm --filter=@alveusgg/census-ui start`.
+   - `pnpm --filter=@alveusgg/census-website start`.
 
 ## Seeding the database
 
-You will need to seed the database with the correct data. To add yourself as an admin, run `pnpm --filter=@alveusgg/census-api setup:api` and follow the prompts.
+You will need to seed the database with the correct data. To add yourself as an admin, run `pnpm --filter=@alveusgg/census-api setup:api` and follow the prompts.
+
+## Optional & additional setup
+
+### Azure setup
+
+If you want to actually host & store on the internet, you'll need to setup an Azure storage account.
+
+1. Go to the Azure Portal.
+2. Create a storage account:
+   - Under Azure services, select Storage accounts and create a new storage account.
+   - Update the `./api/.env` file with `STORAGE_ACCOUNT_NAME`.
+3. Obtain the access key:
+   - From your storage account's page, navigate to Security + networking > Access keys.
+   - Update the `./api/.env` file with `STORAGE_ACCOUNT_KEY` with either `key1` or `key2`.
+4. Create a storage container:
+   - From your storage account's page, go to Data storage > Containers and create a new container.
+   - Update the `./api/.env` file with `CONTAINER_NAME`.
+5. Enable public access for the container
+   - From your storage account's page, go to Configuration and set `Allow Blob anonymous access` to `Enabled`.
+   - From your container's page, select Change access level and set the `anonymous access level` as `Container` or `Blob`.

census/api/.env.example

@@ -12,8 +12,7 @@ POSTGRES_HOST=localhost
 UI_URL=http://localhost:5173
 JWT_SECRET=bed840003446895cfee2372afad1c9adace070b0b54cdfdc640d61b521b5a5aeb3af9519785d5935b8de0f25469b782701b2276a22df329a971c3065bd59a565
 
-STORAGE_ACCOUNT_NAME=
-STORAGE_ACCOUNT_KEY=
-CONTAINER_NAME=
+STORAGE_CONNECTION_STRING=DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://127.0.0.1:10000/devstoreaccount1;
+CONTAINER_NAME=azurite
 
 NODE_ENV=development

census/api/src/scripts/api.ts

@@ -2,7 +2,7 @@ import { config } from 'dotenv';
 config();
 
 import { input } from '@inquirer/prompts';
-import { users } from '../db/schema/index.js';
+import { feeds, users } from '../db/schema/index.js';
 import { createEnvironment } from '../utils/env/env.js';
 
 const seed = async () => {
@@ -15,7 +15,17 @@ const seed = async () => {
     role: 'admin',
     username
   });
+
   console.log(`${username} has been added to the admin role.`);
+  console.log(`Setting up default development feed...`);
+
+  await environment.db.insert(feeds).values({
+    id: 'pollinator',
+    key: 'pollinatorkey1',
+    status: 'healthy'
+  });
+
+  console.log(`Default development feed has been setup.`);
 
   process.exit(0);
 };

census/api/src/services/twitch/clips.ts

@@ -2,6 +2,7 @@ import { randomUUID } from 'crypto';
 import { useEnvironment } from '../../utils/env/env';
 import { TemporaryFile } from '../../utils/tmp';
 import { runTwitchDownloader } from './utils';
+
 export const downloadClip = async (id: string) => {
   const { storage } = useEnvironment();
   const file = await TemporaryFile.create(`${id}-${randomUUID()}.mp4`, 5 * 60, async file => {
@@ -10,6 +11,5 @@ export const downloadClip = async (id: string) => {
 
   const client = storage.getBlockBlobClient(file.name);
   await client.uploadFile(file.path);
-  const url = `https://${storage.accountName}.blob.core.windows.net/${storage.containerName}/${file.name}`;
-  return url;
+  return client.url;
 };

census/api/src/utils/env/config.ts

@@ -1,4 +1,4 @@
-import { ContainerClient, StorageSharedKeyCredential } from '@azure/storage-blob';
+import { ContainerClient } from '@azure/storage-blob';
 import { ApiClient } from '@twurple/api';
 import { AppTokenAuthProvider } from '@twurple/auth';
 import z from 'zod';
@@ -22,8 +22,7 @@ export const config = z.object({
   CONTAINER_APP_NAME: z.string().optional(),
   CONTAINER_APP_ENV_DNS_SUFFIX: z.string().optional(),
 
-  STORAGE_ACCOUNT_NAME: z.string(),
-  STORAGE_ACCOUNT_KEY: z.string(),
+  STORAGE_CONNECTION_STRING: z.string(),
   CONTAINER_NAME: z.string(),
 
   ASSETS_PATH: z.string().default('./assets'),
@@ -41,10 +40,8 @@ export const services = async (variables: z.infer<typeof config>) => {
     variables.POSTGRES_SSL
   );
 
-  const storage = new ContainerClient(
-    `https://${variables.STORAGE_ACCOUNT_NAME}.blob.core.windows.net/${variables.CONTAINER_NAME}`,
-    new StorageSharedKeyCredential(variables.STORAGE_ACCOUNT_NAME, variables.STORAGE_ACCOUNT_KEY)
-  );
+  const storage = new ContainerClient(variables.STORAGE_CONNECTION_STRING, variables.CONTAINER_NAME);
+  await storage.createIfNotExists({ access: 'blob' });
 
   const twitch = new ApiClient({
     authProvider: new AppTokenAuthProvider(variables.TWITCH_CLIENT_ID, variables.TWITCH_CLIENT_SECRET)

census/website/src/index.css

@@ -1,5 +1,6 @@
-@import url('https://fonts.googleapis.com/css2?family=Nunito:wght@400;500;600;700;800;&display=swap');
-@import url('https://fonts.googleapis.com/css2?family=Fira+Code:wght@400;500;600;700;&display=swap');
+@import url('https://fonts.googleapis.com/css2?family=Nunito:wght@400;500;600;700;800&display=swap');
+@import url('https://fonts.googleapis.com/css2?family=Fira+Code:wght@400;500;600;700&display=swap');
+
 @tailwind base;
 @tailwind components;
 @tailwind utilities;

census/website/tailwind.config.js → census/website/tailwind.config.mjs

@@ -1,4 +1,4 @@
-const theme = require('tailwindcss/defaultTheme');
+import theme from 'tailwindcss/defaultTheme';
 
 /** @type {import('tailwindcss').Config} */
 export default {

local/clip-services.yml

@@ -0,0 +1,26 @@
+services:
+  test:
+    image: bluenviron/mediamtx:latest-ffmpeg
+    restart: always
+    volumes:
+      - ./test-stream.mediamtx.yml:/mediamtx.yml
+
+  mux:
+    image: bluenviron/mediamtx:latest-ffmpeg
+    restart: always
+    environment:
+      - MTX_PROTOCOLS=tcp
+    ports:
+      - '8554:8554'
+      - '1935:1935'
+      - '8888:8888'
+      - '8889:8889'
+      - '8890:8890/udp'
+      - '8189:8189/udp'
+      - '9996:9996'
+    volumes:
+      - ./mux.dev.mediamtx.yml:/mediamtx.yml
+      - recordings:/recordings
+
+volumes:
+  recordings:

local/core-services.yml

@@ -0,0 +1,41 @@
+version: '3.8'
+
+services:
+  db:
+    image: postgres:latest
+    environment:
+      POSTGRES_USER: myuser
+      POSTGRES_PASSWORD: mypassword
+      POSTGRES_DB: db01
+    ports:
+      - '5432:5432'
+    volumes:
+      - pgdata:/var/lib/postgresql/data
+    command:
+      - postgres
+      - -c
+      - wal_level=logical
+
+  dragonfly:
+    image: 'docker.dragonflydb.io/dragonflydb/dragonfly'
+    ulimits:
+      memlock: -1
+    ports:
+      - '6379:6379'
+    volumes:
+      - dragonflydata:/data
+
+  azurite:
+    image: mcr.microsoft.com/azure-storage/azurite
+    command: 'azurite --loose --blobHost 0.0.0.0 --blobPort 10000 --location /workspace --debug /workspace/debug.log'
+    container_name: 'azurite'
+    hostname: azurite
+    restart: always
+    ports:
+      - '10000:10000'
+    volumes:
+      - azuritedata:/workspace
+volumes:
+  dragonflydata:
+  azuritedata:
+  pgdata:

local/mux.dev.mediamtx.yml

@@ -0,0 +1,12 @@
+playback: yes
+playbackAddress: :9996
+
+paths:
+  test:
+    source: srt://test:8890?streamid=read:test
+    record: yes
+    recordPath: ./recordings/%path/%Y-%m-%d_%H-%M-%S-%f
+    recordFormat: fmp4
+    recordPartDuration: 1s
+    recordSegmentDuration: 1h
+    recordDeleteAfter: 1h

local/test-stream.mediamtx.yml

@@ -0,0 +1,3 @@
+paths:
+  test:
+    runOnInit: ffmpeg -re -f lavfi -i testsrc2=size=3840x2160:rate=60 -f lavfi -i aevalsrc="sin(0*2*PI*t)" -c:v libx264 -r 30 -g 30 -preset fast -vb 8000k -pix_fmt rgb24 -pix_fmt yuv420p -f rtsp rtsp://localhost:$RTSP_PORT/$MTX_PATH

package.json

@@ -1,7 +1,12 @@
 {
   "name": "@alveusgg/census",
   "version": "1.0.0",
-  "scripts": {},
+  "scripts": {
+    "deps:up": "docker compose --file local/core-services.yml up",
+    "deps:down": "docker compose --file local/core-services.yml down",
+    "stream:up": "docker compose --file local/clip-services.yml up",
+    "stream:down": "docker compose --file local/clip-services.yml down"
+  },
   "keywords": [],
   "author": "",
   "license": "MIT",