diff --git a/apps/easypid/metro.config.js b/apps/easypid/metro.config.js index 5cbcdb86..d456cb8a 100644 --- a/apps/easypid/metro.config.js +++ b/apps/easypid/metro.config.js @@ -15,7 +15,6 @@ config.resolver.nodeModulesPaths = [ path.resolve(workspaceRoot, 'node_modules'), ] config.resolver.sourceExts = [...config.resolver.sourceExts, 'js', 'json', 'ts', 'tsx', 'cjs', 'mjs'] -config.resolver.assetExts = [...config.resolver.assetExts, 'bin'] config.resolver.extraNodeModules = { // Needed for cosmjs trying to import node crypto crypto: require.resolve('./src/polyfills/crypto.ts'), diff --git a/apps/easypid/src/features/share/FunkePresentationNotificationScreen.tsx b/apps/easypid/src/features/share/FunkePresentationNotificationScreen.tsx index fa72b7a9..2d9b22a2 100644 --- a/apps/easypid/src/features/share/FunkePresentationNotificationScreen.tsx +++ b/apps/easypid/src/features/share/FunkePresentationNotificationScreen.tsx @@ -1,6 +1,6 @@ import type { DisplayImage, FormattedSubmission, TrustedEntity } from '@package/agent' -import type { OverAskingResponse, VerificationAnalysisResult } from '@easypid/use-cases/ValidateVerification' +import type { OverAskingResponse, VerificationAnalysisResult } from '@easypid/use-cases/OverAskingApi' import { type SlideStep, SlideWizard } from '@package/app' import { LoadingRequestSlide } from '../receive/slides/LoadingRequestSlide' import { VerifyPartySlide } from '../receive/slides/VerifyPartySlide' diff --git a/apps/easypid/src/features/share/components/RequestPurposeSection.tsx b/apps/easypid/src/features/share/components/RequestPurposeSection.tsx index e4c92b90..cb0d2005 100644 --- a/apps/easypid/src/features/share/components/RequestPurposeSection.tsx +++ b/apps/easypid/src/features/share/components/RequestPurposeSection.tsx @@ -1,4 +1,4 @@ -import type { OverAskingResponse } from '@easypid/use-cases/ValidateVerification' +import type { OverAskingResponse } from '@easypid/use-cases/OverAskingApi' import { AnimatedStack, Circle, diff --git a/apps/easypid/src/features/share/slides/ShareCredentialsSlide.tsx b/apps/easypid/src/features/share/slides/ShareCredentialsSlide.tsx index 428711ea..3a7282a6 100644 --- a/apps/easypid/src/features/share/slides/ShareCredentialsSlide.tsx +++ b/apps/easypid/src/features/share/slides/ShareCredentialsSlide.tsx @@ -1,4 +1,4 @@ -import type { OverAskingResponse, VerificationAnalysisResult } from '@easypid/use-cases/ValidateVerification' +import type { OverAskingResponse, VerificationAnalysisResult } from '@easypid/use-cases/OverAskingApi' import type { DisplayImage, FormattedSubmission } from '@package/agent' import { DualResponseButtons, usePushToWallet, useScrollViewPosition } from '@package/app' import { useWizard } from '@package/app' diff --git a/apps/easypid/src/hooks/useOverAskingAi.tsx b/apps/easypid/src/hooks/useOverAskingAi.tsx index 03608813..e8feb25b 100644 --- a/apps/easypid/src/hooks/useOverAskingAi.tsx +++ b/apps/easypid/src/hooks/useOverAskingAi.tsx @@ -1,10 +1,8 @@ import { useEffect, useState } from 'react' import { useLLM } from '@easypid/llm/useLLM' -import type { OverAskingResponse, VerificationAnalysisInput } from '@easypid/use-cases/ValidateVerification' -import { analyzeVerification as analyzeVerificationApi } from '@easypid/use-cases/ValidateVerification' - -// todos: add a timeout to both api and local calls +import type { OverAskingInput, OverAskingResponse } from '@easypid/use-cases/OverAskingApi' +import { checkForOverAskingApi as analyzeVerificationApi } from '@easypid/use-cases/OverAskingApi' const fallbackResponse: OverAskingResponse = { validRequest: 'could_not_determine', @@ -17,13 +15,8 @@ export function useOverAskingAi() { const { generate, response, error, isModelReady, isModelGenerating } = useLLM() - useEffect(() => { - console.log('response', response) - }, [response]) - useEffect(() => { if (error) { - console.error('Error generating using LLM:', error) setIsProcessingOverAsking(false) setOverAskingResponse(fallbackResponse) return @@ -32,7 +25,7 @@ export function useOverAskingAi() { if (!response || isModelGenerating) return try { - const result = formatResult(response) + const result = formatLocalResult(response) setOverAskingResponse(result) } catch (e) { console.error('Error parsing AI response:', e) @@ -41,18 +34,18 @@ export function useOverAskingAi() { } }, [response, isModelGenerating, error]) - const checkForOverAsking = async (input: VerificationAnalysisInput) => { + const checkForOverAsking = async (input: OverAskingInput) => { setIsProcessingOverAsking(true) if (isModelReady) { - console.log('Model ready, using local LLM') - const prompt = formatPrompt(input) + console.debug('Local LLM ready, using local LLM') + const prompt = formatLocalPrompt(input) await generate(prompt) } else { - console.log('Local LLM not ready, using API') + console.debug('Local LLM not ready, using API') await analyzeVerificationApi(input) .then(setOverAskingResponse) .catch((e) => { - console.error('Error analyzing verification:', e) + console.error('Error analyzing verification using API:', e) setOverAskingResponse(fallbackResponse) }) .finally(() => setIsProcessingOverAsking(false)) @@ -66,9 +59,15 @@ export function useOverAskingAi() { } } -const formatResult = (response: string) => { +// AI responds in XML format, so we need to parse it +// Expected format: +// +// Your concise reason for the assessment +// yes +// +const formatLocalResult = (response: string) => { const match = response.match(/([\s\S]*?)<\/response>/) - if (!match) return + if (!match) return fallbackResponse const responseContent = match[1] @@ -85,13 +84,11 @@ const formatResult = (response: string) => { return fallbackResponse } -const formatPrompt = (input: VerificationAnalysisInput) => { +const formatLocalPrompt = (input: OverAskingInput) => { const cards = input.cards - .map( - (credential) => - `${credential.name} - ${credential.subtitle}. Requested attributes: ${credential.requestedAttributes.join(', ')}` - ) + .map((credential) => `- ${credential.name}. Requested attributes: ${credential.requestedAttributes.join(', ')}`) .join('\n') + return ` You are an AI assistant specializing in data privacy analysis. Your task is to evaluate data verification requests and determine if they are asking for an appropriate amount of information or if they are overasking. @@ -105,10 +102,6 @@ ${input.verifier.name} ${input.verifier.domain} - -${input.name} - - ${input.purpose} @@ -118,20 +111,13 @@ ${cards} -Provide a small evaluation of the request, and provide your final response in the following XML structure: +Provide a short reason for your assessment of the request. Use the following XML structure: Your concise reason for the assessment yes -Example of a properly formatted response: - - -Request aligns with purpose. Information amount appropriate. Verifier seems legitimate. -yes - - -Remember: Provide a concise reason and use the correct XML structure in your response. Do not add any text outside of the specified tags. +Remember: DO NOT add any text outside of the specified tags. DO NOT bother responding with anything other than the XML structure. ` } diff --git a/apps/easypid/src/llm/useLLM.tsx b/apps/easypid/src/llm/useLLM.tsx index 3a8b1685..65e2fbe3 100644 --- a/apps/easypid/src/llm/useLLM.tsx +++ b/apps/easypid/src/llm/useLLM.tsx @@ -4,7 +4,7 @@ import { Platform } from 'react-native' import { LLAMA3_2_1B_QLORA_URL, LLAMA3_2_1B_TOKENIZER } from 'react-native-executorch' import { useMMKVBoolean } from 'react-native-mmkv' import RnExecutorch, { subscribeToDownloadProgress, subscribeToTokenGenerated } from './RnExecutorchModule' -import { DEFAULT_CONTEXT_WINDOW_LENGTH, DEFAULT_SYSTEM_PROMPT, EOT_TOKEN } from './constants' +import { DEFAULT_CONTEXT_WINDOW_LENGTH, EOT_TOKEN } from './constants' import type { Model, ResourceSource } from './types' const interrupt = () => { @@ -38,13 +38,9 @@ export function removeIsModelDownloading() { export const useLLM = ({ modelSource = LLAMA3_2_1B_QLORA_URL, tokenizerSource = LLAMA3_2_1B_TOKENIZER, - systemPrompt = DEFAULT_SYSTEM_PROMPT, - contextWindowLength = DEFAULT_CONTEXT_WINDOW_LENGTH, }: { modelSource?: ResourceSource tokenizerSource?: ResourceSource - systemPrompt?: string - contextWindowLength?: number } = {}): Model => { const [error, setError] = useState(null) const [isModelActivated, setIsModelActivated] = useIsModelActivated() @@ -55,6 +51,16 @@ export const useLLM = ({ const [downloadProgress, setDownloadProgress] = useState(0) const initialized = useRef(false) + useEffect(() => { + if (!response) return + console.debug('Local LLM response', response) + }, [response]) + + useEffect(() => { + if (!error) return + console.debug('Local LLM error', error) + }, [error]) + useEffect(() => { const unsubscribeDownloadProgress = subscribeToDownloadProgress((data) => { if (data) { @@ -77,7 +83,7 @@ export const useLLM = ({ try { try { setIsModelDownloading(true) - await RnExecutorch.loadLLM(modelSource, tokenizerSource, systemPrompt, contextWindowLength) + await RnExecutorch.loadLLM(modelSource, tokenizerSource, '', DEFAULT_CONTEXT_WINDOW_LENGTH) await RnExecutorch } catch (error) { console.log('ERROR LOADING MODEL', error) @@ -92,15 +98,7 @@ export const useLLM = ({ setError(message) initialized.current = false } - }, [ - contextWindowLength, - modelSource, - systemPrompt, - tokenizerSource, - setIsModelReady, - setIsModelActivated, - setIsModelDownloading, - ]) + }, [modelSource, tokenizerSource, setIsModelReady, setIsModelActivated, setIsModelDownloading]) const generate = useCallback( async (input: string): Promise => { diff --git a/apps/easypid/src/use-cases/ValidateVerification.ts b/apps/easypid/src/use-cases/OverAskingApi.ts similarity index 85% rename from apps/easypid/src/use-cases/ValidateVerification.ts rename to apps/easypid/src/use-cases/OverAskingApi.ts index e30834b1..0b1f07a7 100644 --- a/apps/easypid/src/use-cases/ValidateVerification.ts +++ b/apps/easypid/src/use-cases/OverAskingApi.ts @@ -2,7 +2,7 @@ const PLAYGROUND_URL = 'https://funke.animo.id' export const EXCLUDED_ATTRIBUTES_FOR_ANALYSIS = ['Issuing authority', 'Issuing country', 'Issued at', 'Expires at'] -export type VerificationAnalysisInput = { +export type OverAskingInput = { verifier: { name: string domain: string @@ -21,17 +21,12 @@ export type OverAskingResponse = { reason: string } -export type VerificationAnalysisResult = { - isLoading: boolean - result: OverAskingResponse | undefined -} - -export const analyzeVerification = async ({ +export const checkForOverAskingApi = async ({ verifier, name, purpose, cards, -}: VerificationAnalysisInput): Promise => { +}: OverAskingInput): Promise => { try { const cardsWithoutExcludedAttributes = cards.map((card) => ({ ...card,