Wizard

src/hooks.server.ts

@@ -56,12 +56,12 @@ export const authentication: Handle = async ({ event, resolve }) => {
 export const configuration: Handle = async ({event, resolve}) => {
 		const settings = await event.locals.pb.collection('settings').getList(1,1);
 
-		if (!settings.wizard && !event.url.pathname.endsWith('/wizard')) {
+		console.log(settings)
+
+		if (!settings.items[0].wizard && !event.url.pathname.endsWith('/wizard') && !event.url.pathname.startsWith('/api/')) {
 			// If the wizard is not completed, redirect to /wizard
 			console.log("Redirecting to wizard <-------------");
 			throw redirect(307, '/wizard');
-		} else {
-			console.log("Hardware has been configured. Opening app");
 		}
 
 		return resolve(event)

src/lib/components/DisplayPane.svelte

@@ -245,12 +245,14 @@
 											<div
 												class="flex items-center justify-center gap-3 text-xs text-carbongray-500"
 											>
-												<div
-													class="flex items-center gap-1 text-sm font-bold text-carbongray-800 dark:text-carbongray-50"
-												>
-													<Volume2 size={12} />
-													<div class="text-sm">{t.speaker}</div>
-												</div>
+												{#if t.speaker}
+													<div
+														class="flex items-center gap-1 text-sm font-bold text-carbongray-800 dark:text-carbongray-50"
+													>
+														<Volume2 size={12} />
+														<div class="text-sm">{t.speaker}</div>
+													</div>
+												{/if}
 												<div class="text-[0.8em]">
 													{t.timestamps.from.split(',')[0]}
 												</div>

src/lib/components/StatusSpinner.svelte

@@ -3,15 +3,27 @@
 	export let msg;
 	export let size = 13;
 	export let success = -1;
+	export let pos = 'end';
 </script>
 
-<div class="flex w-fit items-center justify-center gap-2 p-2">
-	<div class="text-base">{msg}</div>
-	{#if success < 0}
-		<Loader {size} class="animate-spin text-carbonblue-500" />
-	{:else if success > 0}
-		<CircleCheck {size} class="animate-pulse text-[#24a147]" />
+<div class="flex w-fit items-center justify-start gap-2 p-0">
+	{#if pos === 'end'}
+		<div class="text-base">{msg}</div>
+		{#if success < 0}
+			<Loader {size} class="animate-spin text-carbonblue-500" />
+		{:else if success > 0}
+			<CircleCheck {size} class="animate-pulse text-[#24a147]" />
+		{:else}
+			<CircleX {size} class="animate-pulse" />
+		{/if}
 	{:else}
-		<CircleX {size} class="animate-pulse" />
+		{#if success < 0}
+			<Loader {size} class="animate-spin text-carbonblue-500" />
+		{:else if success > 0}
+			<CircleCheck {size} class="animate-pulse text-[#24a147]" />
+		{:else}
+			<CircleX {size} class="animate-pulse" />
+		{/if}
+		<div class="text-base">{msg}</div>
 	{/if}
 </div>

src/lib/queue.ts

@@ -1,5 +1,6 @@
 import { Queue, Worker } from 'bullmq';
 import { exec } from 'child_process';
+import { wizardQueue } from './wizardQueue'; 
 import fs from 'fs';
 import path from 'path';
 import PocketBase from 'pocketbase';
@@ -24,7 +25,7 @@ const app = express();
 // Create Bull Board UI
 const serverAdapter = new ExpressAdapter();
 createBullBoard({
-	queues: [new BullMQAdapter(transcriptionQueue)],
+	queues: [new BullMQAdapter(transcriptionQueue), new BullMQAdapter(wizardQueue)],
 	serverAdapter: serverAdapter
 });
 
@@ -125,6 +126,11 @@ const execCommandWithLogging = (cmd: string, job: Job, progress: number) => {
 				reject(new Error(`Command failed with exit code ${code}`));
 			}
 		});
+
+
+		process.on('error', (err) => {
+			reject(new Error(`Failed to start process: ${err.message}`));
+		});
 	});
 };
 
@@ -178,23 +184,34 @@ const worker = new Worker(
 				if (err) throw err;
 			});
 
-			let whisperCmd = `whisper -m /models/ggml-${settings.model}.en.bin -f ${ffmpegPath} -oj -of ${transcriptPath} -t ${settings.threads} -p ${settings.processors} -pp`;
+			let whisperCmd;
+
+			if (env.DEV_MODE) {
+				whisperCmd = `./whisper.cpp/main -m ./whisper.cpp/models/ggml-${settings.model}.en.bin -f ${ffmpegPath} -oj -of ${transcriptPath} -t ${settings.threads} -p ${settings.processors} -pp`;
+			} else {
+				whisperCmd = `whisper -m /models/ggml-${settings.model}.en.bin -f ${ffmpegPath} -oj -of ${transcriptPath} -t ${settings.threads} -p ${settings.processors} -pp`;
+			}
+
 
 			let rttmContent;
 			let segments;
 
 			if (settings.diarize) {
 				job.updateProgress(12);
 				const rttmPath = path.resolve(baseUrl, `${recordId}.rttm`);
-				const diarizeCmd = `python3 ./diarize/local.py ${ffmpegPath} ${rttmPath}`;
+				const diarizeCmd = `python ./diarize/local.py ${ffmpegPath} ${rttmPath}`;
 				await execCommandWithLogging(diarizeCmd, job);
 				await job.log(`Diarization completed successfully`);
 				// Read and parse the RTTM file
 				rttmContent = fs.readFileSync(rttmPath, 'utf-8');
 				segments = parseRttm(rttmContent);
 				await job.log(`Parsed RTTM file for record ${recordId}`);
 
-				whisperCmd = `./whisper.cpp/main -m ./whisper.cpp/models/ggml-${settings.model}.en.bin -f ${ffmpegPath} -oj -of ${transcriptPath} -t ${settings.threads} -p ${settings.processors} -pp -ml 1`;
+				if (env.DEV_MODE) {
+					whisperCmd = `./whisper.cpp/main -m ./whisper.cpp/models/ggml-${settings.model}.en.bin -f ${ffmpegPath} -oj -of ${transcriptPath} -t ${settings.threads} -p ${settings.processors} -pp -ml 1`;
+				} else {
+					whisperCmd = `whisper -m /models/ggml-${settings.model}.en.bin -f ${ffmpegPath} -oj -of ${transcriptPath} -t ${settings.threads} -p ${settings.processors} -pp -ml 1`;
+				}
 			}
 
 			job.updateProgress(35);

src/lib/wizardQueue.ts

@@ -0,0 +1,221 @@
+import { Queue, Worker } from 'bullmq';
+import { execSync, spawn } from 'child_process';
+import { exec } from 'child_process';
+import fs from 'fs';
+import path from 'path';
+import PocketBase from 'pocketbase';
+import { env } from '$env/dynamic/private';
+
+// Create the queue
+export const wizardQueue = new Queue('wizardQueue', {
+	connection: { host: env.REDIS_HOST, port: env.REDIS_PORT }
+});
+const pb = new PocketBase('http://localhost:8080');
+pb.autoCancellation(false);
+await pb.admins.authWithPassword(env.POCKETBASE_ADMIN_EMAIL, env.POCKETBASE_ADMIN_PASSWORD);
+
+
+// Remove all jobs from the queue
+async function clearQueue() {
+	await wizardQueue.drain();
+	console.log('Queue cleared!');
+}
+
+async function cleanAllJobs() {
+	// Remove all completed jobs
+	await wizardQueue.clean(0, 0, 'completed');
+	console.log('All completed jobs have been removed');
+
+	// Remove all failed jobs
+	await wizardQueue.clean(0, 0, 'failed');
+	console.log('All failed jobs have been removed');
+
+	// Remove all waiting jobs
+	await wizardQueue.clean(0, 0, 'waiting');
+	console.log('All waiting jobs have been removed');
+
+	// Remove all active jobs (this may kill jobs that are actively being processed)
+	await wizardQueue.clean(0, 0, 'active');
+	console.log('All active jobs have been removed');
+
+	// Remove all delayed jobs
+	await wizardQueue.clean(0, 0, 'delayed');
+	console.log('All delayed jobs have been removed');
+}
+
+async function pauseAndCleanQueue() {
+	// Pause the queue to prevent new jobs from being processed
+	await wizardQueue.pause();
+	console.log('Queue paused');
+
+	// Clean all jobs as described above
+	await cleanAllJobs();
+
+	// Optionally resume the queue
+	await wizardQueue.resume();
+	console.log('Queue resumed');
+}
+
+// pauseAndCleanQueue();
+
+// clearQueue();
+
+// Helper function to execute shell commands and log output
+const execCommandWithLogging = (cmd, job) => {
+	return new Promise((resolve, reject) => {
+		const process = exec(cmd);
+
+		// Capture stdout
+		process.stdout.on('data', async (data) => {
+			console.log(`stdout: ${data}`);
+			await job.log(`stdout: ${data}`);
+		});
+
+		// Capture stderr (in case you want to log errors)
+		process.stderr.on('data', async (data) => {
+			console.error(`stderr: ${data}`);
+			await job.log(`stderr: ${data}`);
+		});
+
+		// Handle process close event (when the command finishes)
+		process.on('close', (code) => {
+			if (code === 0) {
+				resolve(true); // Command succeeded
+			} else {
+				reject(new Error(`Command failed with exit code ${code}`)); // Command failed
+			}
+		});
+
+		// Handle possible errors during execution
+		process.on('error', (err) => {
+			reject(new Error(`Failed to start process: ${err.message}`));
+		});
+	});
+};
+;
+
+export const execCommandWithLoggingSync = (cmd: string, job: any): Promise<boolean> => {
+    return new Promise((resolve, reject) => {
+        const process = exec(cmd, { shell: true, maxBuffer: 1024 * 1024 * 10 }); // Max buffer for larger outputs
+
+        process.stdout.on('data', async (data) => {
+            console.log(`stdout: ${data}`);
+            await job.log(`stdout: ${data}`);
+        });
+
+        process.stderr.on('data', async (data) => {
+            console.error(`stderr: ${data}`);
+            await job.log(`stderr: ${data}`);
+        });
+
+        process.on('close', (code) => {
+            if (code === 0) {
+                resolve(true);
+            } else {
+                reject(new Error(`Command failed with exit code ${code !== null ? code : 'unknown'}`));
+            }
+        });
+
+        process.on('error', (err) => {
+            reject(new Error(`Failed to start process: ${err.message}`));
+        });
+    });
+};
+
+// Set up the worker to process jobs automatically
+const worker = new Worker(
+	'wizardQueue',
+	async (job) => {
+		console.log("hello world from wizard")
+		let modelPath;
+		let cmd;
+		try {
+			const {settings }= job.data;
+			if (env.DEV_MODE) {
+				modelPath = path.resolve(env.SCRIBO_FILES, 'models/whisper.cpp')
+			} else {
+				modelPath = path.resolve('/models/whisper.cpp')
+			}
+			await job.log('starting job')
+			cmd = `make clean -C ${modelPath}`
+			await execCommandWithLogging(cmd, job);
+
+			cmd = `make -C ${modelPath}`;
+			await execCommandWithLogging(cmd, job);
+			await job.log('finished making whisper')
+			job.updateProgress(50)
+
+			const modToDownload = modelsToDownload(settings);
+			console.log(modToDownload)
+			await job.log(modToDownload)
+
+			modToDownload.forEach(async (m, idx) => {
+				let cmd2;
+
+				if (env.DEV_MODE) {
+				  cmd2 = `sh ${modelPath}/models/download-ggml-model.sh ${m}.en`;
+				} else {
+				  cmd2 = `sh ${modelPath}/models/download-ggml-model.sh ${m}.en /models`;
+				}
+
+			  await job.log(`Executing command: ${cmd2}`);
+			  execCommandWithLoggingSync(cmd2, job);
+			  const prg = 50 + (50 * (idx + 1) / modelsToDownload.length); // idx + 1 ensures progress increments
+			  await job.updateProgress(prg);
+			});
+
+			await job.log('finished job')
+		} catch(error) {
+			console.log(error);
+			job.log(error)
+		}
+
+
+		const settt = await pb.collection('settings').getList(1,1);
+
+		if (settt && settt.items.length > 0) {
+	    const record = settt.items[0]; // Get the first record (assuming one record is returned)
+
+	    // Update the 'wizard' field to true
+	    const updatedRecord = await pb.collection('settings').update(record.id, {
+	        wizard: true
+	    });
+
+	    console.log('Updated record:', updatedRecord);
+		} else {
+		    console.log('No records found in settings collection');
+		}
+
+		job.updateProgress(100)
+	},
+	{
+		connection: { host: env.REDIS_HOST, port: env.REDIS_PORT }, // Redis connection
+		concurrency: 1, // Allows multiple jobs to run concurrently
+		lockDuration: 500000, // Lock duration (in milliseconds), e.g., 5 minutes
+    lockRenewTime: 500000
+	}
+);
+
+function modelsToDownload(settings) {
+		let m = [];
+
+		const set = JSON.parse(settings)
+		console.log('Settings:', settings);
+	console.log('Models:', set.models);
+
+
+		if (set.models.small) {
+			m.push('small');
+		}
+		if (set.models.tiny) {
+			m.push('tiny');
+		}
+		if (set.models.medium) {
+			m.push('medium');
+		}
+		if (set.models.largev1) {
+			m.push('large-v1');
+		}
+
+		return m;
+	}

src/routes/+layout.svelte

@@ -43,7 +43,7 @@
 	</div>
 </div>
 <div
-	class="container mx-auto h-[704px] max-h-screen border-carbongray-100 text-black shadow transition-colors duration-300 ease-in-out dark:border-carbongray-700 dark:text-white lg:max-w-[1000px] lg:rounded-xl lg:border 2xl:mt-[50px] 2xl:h-[900px]"
+	class="container mx-auto h-[704px] max-h-screen border-carbongray-100 text-black shadow transition-colors duration-300 ease-in-out dark:border-carbongray-700 dark:text-white lg:max-w-[1000px] lg:rounded-xl lg:border 2xl:mt-[30px] 2xl:h-[900px]"
 >
 	<slot></slot>
 </div>

src/routes/api/jobs/configure/+server.ts

@@ -0,0 +1,28 @@
+import { RequestHandler } from '@sveltejs/kit';
+import { ensureCollectionExists } from '$lib/fileFuncs';
+import { wizardQueue } from '$lib/wizardQueue'; 
+
+export const POST: RequestHandler = async ({ request, locals }) => {
+	try {
+		// Get the form data (including file) from the request
+
+		const formData = await request.formData();
+		const settings = formData.get('settings');
+
+
+		const job = await wizardQueue.add('configWizard', {
+		  settings: settings
+		});
+		console.log('Created job:', job.id);
+
+		return new Response(JSON.stringify({ jobId: job.id}), {
+			status: 200
+		});
+	} catch (error: any) {
+		console.log(error.message);
+		return new Response(JSON.stringify({ message: 'Configuration wizard failed', error: error.message }), {
+			status: 500
+		});
+	}
+};
+