From 5b7a2a5991844b3b6ea64de2d05b392dae8260b8 Mon Sep 17 00:00:00 2001 From: eduardruzga Date: Tue, 26 Nov 2024 10:18:46 +0200 Subject: [PATCH 1/3] Refinement of folder import --- app/components/chat/ImportFolderButton.tsx | 191 +++++++-------------- app/utils/fileUtils.ts | 97 +++++++++++ app/utils/folderImport.ts | 56 ++++++ 3 files changed, 216 insertions(+), 128 deletions(-) create mode 100644 app/utils/fileUtils.ts create mode 100644 app/utils/folderImport.ts diff --git a/app/components/chat/ImportFolderButton.tsx b/app/components/chat/ImportFolderButton.tsx index 5f822ee81..558049e0d 100644 --- a/app/components/chat/ImportFolderButton.tsx +++ b/app/components/chat/ImportFolderButton.tsx @@ -1,102 +1,74 @@ -import React from 'react'; +import React, { useState } from 'react'; import type { Message } from 'ai'; import { toast } from 'react-toastify'; -import ignore from 'ignore'; +import { MAX_FILES, isBinaryFile, shouldIncludeFile } from '../../utils/fileUtils'; +import { createChatFromFolder } from '../../utils/folderImport'; interface ImportFolderButtonProps { className?: string; importChat?: (description: string, messages: Message[]) => Promise; } -// Common patterns to ignore, similar to .gitignore -const IGNORE_PATTERNS = [ - 'node_modules/**', - '.git/**', - 'dist/**', - 'build/**', - '.next/**', - 'coverage/**', - '.cache/**', - '.vscode/**', - '.idea/**', - '**/*.log', - '**/.DS_Store', - '**/npm-debug.log*', - '**/yarn-debug.log*', - '**/yarn-error.log*', -]; - -const ig = ignore().add(IGNORE_PATTERNS); -const generateId = () => Math.random().toString(36).substring(2, 15); - -const isBinaryFile = async (file: File): Promise => { - const chunkSize = 1024; // Read the first 1 KB of the file - const buffer = new Uint8Array(await file.slice(0, chunkSize).arrayBuffer()); - - for (let i = 0; i < buffer.length; i++) { - const byte = buffer[i]; - - if (byte === 0 || (byte < 32 && byte !== 9 && byte !== 10 && byte !== 13)) { - return true; // Found a binary character - } - } - - return false; -}; - export const ImportFolderButton: React.FC = ({ className, importChat }) => { - const shouldIncludeFile = (path: string): boolean => { - return !ig.ignores(path); - }; - - const createChatFromFolder = async (files: File[], binaryFiles: string[]) => { - const fileArtifacts = await Promise.all( - files.map(async (file) => { - return new Promise((resolve, reject) => { - const reader = new FileReader(); - - reader.onload = () => { - const content = reader.result as string; - const relativePath = file.webkitRelativePath.split('/').slice(1).join('/'); - resolve( - ` -${content} -`, - ); - }; - reader.onerror = reject; - reader.readAsText(file); - }); - }), - ); - - const binaryFilesMessage = - binaryFiles.length > 0 - ? `\n\nSkipped ${binaryFiles.length} binary files:\n${binaryFiles.map((f) => `- ${f}`).join('\n')}` - : ''; + const [isLoading, setIsLoading] = useState(false); - const message: Message = { - role: 'assistant', - content: `I'll help you set up these files.${binaryFilesMessage} + const handleFileChange = async (e: React.ChangeEvent) => { + const allFiles = Array.from(e.target.files || []); - -${fileArtifacts.join('\n\n')} -`, - id: generateId(), - createdAt: new Date(), - }; - - const userMessage: Message = { - role: 'user', - id: generateId(), - content: 'Import my files', - createdAt: new Date(), - }; - - const description = `Folder Import: ${files[0].webkitRelativePath.split('/')[0]}`; + if (allFiles.length > MAX_FILES) { + toast.error( + `This folder contains ${allFiles.length.toLocaleString()} files. This product is not yet optimized for very large projects. Please select a folder with fewer than ${MAX_FILES.toLocaleString()} files.` + ); + return; + } - if (importChat) { - await importChat(description, [userMessage, message]); + const folderName = allFiles[0]?.webkitRelativePath.split('/')[0] || 'Unknown Folder'; + setIsLoading(true); + const loadingToast = toast.loading(`Importing ${folderName}...`); + + try { + const filteredFiles = allFiles.filter((file) => shouldIncludeFile(file.webkitRelativePath)); + + if (filteredFiles.length === 0) { + toast.error('No files found in the selected folder'); + return; + } + + const fileChecks = await Promise.all( + filteredFiles.map(async (file) => ({ + file, + isBinary: await isBinaryFile(file), + })), + ); + + const textFiles = fileChecks.filter((f) => !f.isBinary).map((f) => f.file); + const binaryFilePaths = fileChecks + .filter((f) => f.isBinary) + .map((f) => f.file.webkitRelativePath.split('/').slice(1).join('/')); + + if (textFiles.length === 0) { + toast.error('No text files found in the selected folder'); + return; + } + + if (binaryFilePaths.length > 0) { + toast.info(`Skipping ${binaryFilePaths.length} binary files`); + } + + const { userMessage, assistantMessage } = await createChatFromFolder(textFiles, binaryFilePaths, folderName); + + if (importChat) { + await importChat(folderName, [userMessage, assistantMessage]); + } + + toast.success('Folder imported successfully'); + } catch (error) { + console.error('Failed to import folder:', error); + toast.error('Failed to import folder'); + } finally { + setIsLoading(false); + toast.dismiss(loadingToast); + e.target.value = ''; // Reset file input } }; @@ -108,46 +80,8 @@ ${fileArtifacts.join('\n\n')} className="hidden" webkitdirectory="" directory="" - onChange={async (e) => { - const allFiles = Array.from(e.target.files || []); - const filteredFiles = allFiles.filter((file) => shouldIncludeFile(file.webkitRelativePath)); - - if (filteredFiles.length === 0) { - toast.error('No files found in the selected folder'); - return; - } - - try { - const fileChecks = await Promise.all( - filteredFiles.map(async (file) => ({ - file, - isBinary: await isBinaryFile(file), - })), - ); - - const textFiles = fileChecks.filter((f) => !f.isBinary).map((f) => f.file); - const binaryFilePaths = fileChecks - .filter((f) => f.isBinary) - .map((f) => f.file.webkitRelativePath.split('/').slice(1).join('/')); - - if (textFiles.length === 0) { - toast.error('No text files found in the selected folder'); - return; - } - - if (binaryFilePaths.length > 0) { - toast.info(`Skipping ${binaryFilePaths.length} binary files`); - } - - await createChatFromFolder(textFiles, binaryFilePaths); - } catch (error) { - console.error('Failed to import folder:', error); - toast.error('Failed to import folder'); - } - - e.target.value = ''; // Reset file input - }} - {...({} as any)} // if removed webkitdirectory will throw errors as unknow attribute + onChange={handleFileChange} + {...({} as any)} /> ); diff --git a/app/utils/fileUtils.ts b/app/utils/fileUtils.ts new file mode 100644 index 000000000..f6a52d909 --- /dev/null +++ b/app/utils/fileUtils.ts @@ -0,0 +1,97 @@ +import ignore from 'ignore'; + +// Common patterns to ignore, similar to .gitignore +export const IGNORE_PATTERNS = [ + 'node_modules/**', + '.git/**', + 'dist/**', + 'build/**', + '.next/**', + 'coverage/**', + '.cache/**', + '.vscode/**', + '.idea/**', + '**/*.log', + '**/.DS_Store', + '**/npm-debug.log*', + '**/yarn-debug.log*', + '**/yarn-error.log*', +]; + +export const MAX_FILES = 1000; +export const ig = ignore().add(IGNORE_PATTERNS); + +export const generateId = () => Math.random().toString(36).substring(2, 15); + +export const isBinaryFile = async (file: File): Promise => { + const chunkSize = 1024; + const buffer = new Uint8Array(await file.slice(0, chunkSize).arrayBuffer()); + + for (let i = 0; i < buffer.length; i++) { + const byte = buffer[i]; + if (byte === 0 || (byte < 32 && byte !== 9 && byte !== 10 && byte !== 13)) { + return true; + } + } + return false; +}; + +export const shouldIncludeFile = (path: string): boolean => { + return !ig.ignores(path); +}; + +const readPackageJson = async (files: File[]): Promise<{ scripts?: Record } | null> => { + const packageJsonFile = files.find(f => f.webkitRelativePath.endsWith('package.json')); + if (!packageJsonFile) return null; + + try { + const content = await new Promise((resolve, reject) => { + const reader = new FileReader(); + reader.onload = () => resolve(reader.result as string); + reader.onerror = reject; + reader.readAsText(packageJsonFile); + }); + + return JSON.parse(content); + } catch (error) { + console.error('Error reading package.json:', error); + return null; + } +}; + +export const detectProjectType = async (files: File[]): Promise<{ type: string; setupCommand: string; followupMessage: string }> => { + const hasFile = (name: string) => files.some(f => f.webkitRelativePath.endsWith(name)); + + if (hasFile('package.json')) { + const packageJson = await readPackageJson(files); + const scripts = packageJson?.scripts || {}; + + // Check for preferred commands in priority order + const preferredCommands = ['dev', 'start', 'preview']; + const availableCommand = preferredCommands.find(cmd => scripts[cmd]); + + if (availableCommand) { + return { + type: 'Node.js', + setupCommand: `npm install && npm run ${availableCommand}`, + followupMessage: `Found "${availableCommand}" script in package.json. Running "npm run ${availableCommand}" after installation.` + }; + } + + return { + type: 'Node.js', + setupCommand: 'npm install', + followupMessage: 'Would you like me to inspect package.json to determine the available scripts for running this project?' + }; + } + + if (hasFile('index.html')) { + return { + type: 'Static', + setupCommand: 'npx --yes serve', + followupMessage: '' + }; + } + + return { type: '', setupCommand: '', followupMessage: '' }; +}; diff --git a/app/utils/folderImport.ts b/app/utils/folderImport.ts new file mode 100644 index 000000000..2803286b2 --- /dev/null +++ b/app/utils/folderImport.ts @@ -0,0 +1,56 @@ +import type { Message } from 'ai'; +import { generateId, detectProjectType } from './fileUtils'; + +export const createChatFromFolder = async ( + files: File[], + binaryFiles: string[], + folderName: string +): Promise<{ userMessage: Message; assistantMessage: Message }> => { + const fileArtifacts = await Promise.all( + files.map(async (file) => { + return new Promise((resolve, reject) => { + const reader = new FileReader(); + reader.onload = () => { + const content = reader.result as string; + const relativePath = file.webkitRelativePath.split('/').slice(1).join('/'); + resolve( + ` +${content} +`, + ); + }; + reader.onerror = reject; + reader.readAsText(file); + }); + }), + ); + + const project = await detectProjectType(files); + const setupCommand = project.setupCommand ? `\n\n\n${project.setupCommand}\n` : ''; + const followupMessage = project.followupMessage ? `\n\n${project.followupMessage}` : ''; + + const binaryFilesMessage = binaryFiles.length > 0 + ? `\n\nSkipped ${binaryFiles.length} binary files:\n${binaryFiles.map((f) => `- ${f}`).join('\n')}` + : ''; + + const assistantMessage: Message = { + role: 'assistant', + content: `I've imported the contents of the "${folderName}" folder.${binaryFilesMessage} + + +${fileArtifacts.join('\n\n')} +${setupCommand} +${followupMessage}`, + id: generateId(), + createdAt: new Date(), + }; + + const userMessage: Message = { + role: 'user', + id: generateId(), + content: `Import the "${folderName}" folder`, + createdAt: new Date(), + }; + + return { userMessage, assistantMessage }; +}; From 64814d56c5aaf41791c9d835263ba3d599e27dd5 Mon Sep 17 00:00:00 2001 From: Anirban Kar Date: Sun, 8 Dec 2024 04:11:34 +0530 Subject: [PATCH 2/3] Update folderImport.ts updated function output --- app/utils/folderImport.ts | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/app/utils/folderImport.ts b/app/utils/folderImport.ts index 2803286b2..57cbe318b 100644 --- a/app/utils/folderImport.ts +++ b/app/utils/folderImport.ts @@ -5,7 +5,7 @@ export const createChatFromFolder = async ( files: File[], binaryFiles: string[], folderName: string -): Promise<{ userMessage: Message; assistantMessage: Message }> => { +): Promise => { const fileArtifacts = await Promise.all( files.map(async (file) => { return new Promise((resolve, reject) => { @@ -33,17 +33,24 @@ ${content} ? `\n\nSkipped ${binaryFiles.length} binary files:\n${binaryFiles.map((f) => `- ${f}`).join('\n')}` : ''; - const assistantMessage: Message = { + const assistantMessages: Message[] = [{ role: 'assistant', content: `I've imported the contents of the "${folderName}" folder.${binaryFilesMessage} ${fileArtifacts.join('\n\n')} +`, + id: generateId(), + createdAt: new Date(), + },{ + role: 'assistant', + content: ` + ${setupCommand} ${followupMessage}`, id: generateId(), createdAt: new Date(), - }; + }]; const userMessage: Message = { role: 'user', @@ -52,5 +59,5 @@ ${setupCommand} createdAt: new Date(), }; - return { userMessage, assistantMessage }; + return [ userMessage, ...assistantMessages ]; }; From 11f93a88004228d4977c36b3830fed83457ef1af Mon Sep 17 00:00:00 2001 From: Anirban Kar Date: Sun, 8 Dec 2024 04:12:29 +0530 Subject: [PATCH 3/3] Update ImportFolderButton.tsx added fix --- app/components/chat/ImportFolderButton.tsx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/components/chat/ImportFolderButton.tsx b/app/components/chat/ImportFolderButton.tsx index 70e9f1f44..e766a7167 100644 --- a/app/components/chat/ImportFolderButton.tsx +++ b/app/components/chat/ImportFolderButton.tsx @@ -54,10 +54,10 @@ export const ImportFolderButton: React.FC = ({ classNam toast.info(`Skipping ${binaryFilePaths.length} binary files`); } - const { userMessage, assistantMessage } = await createChatFromFolder(textFiles, binaryFilePaths, folderName); + const messages = await createChatFromFolder(textFiles, binaryFilePaths, folderName); if (importChat) { - await importChat(folderName, [userMessage, assistantMessage]); + await importChat(folderName, [...messages]); } toast.success('Folder imported successfully');