Skip to content

Commit

Permalink
Refinement of folder import
Browse files Browse the repository at this point in the history
  • Loading branch information
wonderwhy-er committed Nov 26, 2024
1 parent 9b62edd commit 5b7a2a5
Show file tree
Hide file tree
Showing 3 changed files with 216 additions and 128 deletions.
191 changes: 63 additions & 128 deletions app/components/chat/ImportFolderButton.tsx
Original file line number Diff line number Diff line change
@@ -1,102 +1,74 @@
import React from 'react';
import React, { useState } from 'react';
import type { Message } from 'ai';
import { toast } from 'react-toastify';
import ignore from 'ignore';
import { MAX_FILES, isBinaryFile, shouldIncludeFile } from '../../utils/fileUtils';
import { createChatFromFolder } from '../../utils/folderImport';

interface ImportFolderButtonProps {
className?: string;
importChat?: (description: string, messages: Message[]) => Promise<void>;
}

// Common patterns to ignore, similar to .gitignore
const IGNORE_PATTERNS = [
'node_modules/**',
'.git/**',
'dist/**',
'build/**',
'.next/**',
'coverage/**',
'.cache/**',
'.vscode/**',
'.idea/**',
'**/*.log',
'**/.DS_Store',
'**/npm-debug.log*',
'**/yarn-debug.log*',
'**/yarn-error.log*',
];

const ig = ignore().add(IGNORE_PATTERNS);
const generateId = () => Math.random().toString(36).substring(2, 15);

const isBinaryFile = async (file: File): Promise<boolean> => {
const chunkSize = 1024; // Read the first 1 KB of the file
const buffer = new Uint8Array(await file.slice(0, chunkSize).arrayBuffer());

for (let i = 0; i < buffer.length; i++) {
const byte = buffer[i];

if (byte === 0 || (byte < 32 && byte !== 9 && byte !== 10 && byte !== 13)) {
return true; // Found a binary character
}
}

return false;
};

export const ImportFolderButton: React.FC<ImportFolderButtonProps> = ({ className, importChat }) => {
const shouldIncludeFile = (path: string): boolean => {
return !ig.ignores(path);
};

const createChatFromFolder = async (files: File[], binaryFiles: string[]) => {
const fileArtifacts = await Promise.all(
files.map(async (file) => {
return new Promise<string>((resolve, reject) => {
const reader = new FileReader();

reader.onload = () => {
const content = reader.result as string;
const relativePath = file.webkitRelativePath.split('/').slice(1).join('/');
resolve(
`<boltAction type="file" filePath="${relativePath}">
${content}
</boltAction>`,
);
};
reader.onerror = reject;
reader.readAsText(file);
});
}),
);

const binaryFilesMessage =
binaryFiles.length > 0
? `\n\nSkipped ${binaryFiles.length} binary files:\n${binaryFiles.map((f) => `- ${f}`).join('\n')}`
: '';
const [isLoading, setIsLoading] = useState(false);

const message: Message = {
role: 'assistant',
content: `I'll help you set up these files.${binaryFilesMessage}
const handleFileChange = async (e: React.ChangeEvent<HTMLInputElement>) => {
const allFiles = Array.from(e.target.files || []);

<boltArtifact id="imported-files" title="Imported Files">
${fileArtifacts.join('\n\n')}
</boltArtifact>`,
id: generateId(),
createdAt: new Date(),
};

const userMessage: Message = {
role: 'user',
id: generateId(),
content: 'Import my files',
createdAt: new Date(),
};

const description = `Folder Import: ${files[0].webkitRelativePath.split('/')[0]}`;
if (allFiles.length > MAX_FILES) {
toast.error(
`This folder contains ${allFiles.length.toLocaleString()} files. This product is not yet optimized for very large projects. Please select a folder with fewer than ${MAX_FILES.toLocaleString()} files.`
);
return;
}

if (importChat) {
await importChat(description, [userMessage, message]);
const folderName = allFiles[0]?.webkitRelativePath.split('/')[0] || 'Unknown Folder';
setIsLoading(true);
const loadingToast = toast.loading(`Importing ${folderName}...`);

try {
const filteredFiles = allFiles.filter((file) => shouldIncludeFile(file.webkitRelativePath));

if (filteredFiles.length === 0) {
toast.error('No files found in the selected folder');
return;
}

const fileChecks = await Promise.all(
filteredFiles.map(async (file) => ({
file,
isBinary: await isBinaryFile(file),
})),
);

const textFiles = fileChecks.filter((f) => !f.isBinary).map((f) => f.file);
const binaryFilePaths = fileChecks
.filter((f) => f.isBinary)
.map((f) => f.file.webkitRelativePath.split('/').slice(1).join('/'));

if (textFiles.length === 0) {
toast.error('No text files found in the selected folder');
return;
}

if (binaryFilePaths.length > 0) {
toast.info(`Skipping ${binaryFilePaths.length} binary files`);
}

const { userMessage, assistantMessage } = await createChatFromFolder(textFiles, binaryFilePaths, folderName);

if (importChat) {
await importChat(folderName, [userMessage, assistantMessage]);
}

toast.success('Folder imported successfully');
} catch (error) {
console.error('Failed to import folder:', error);
toast.error('Failed to import folder');
} finally {
setIsLoading(false);
toast.dismiss(loadingToast);
e.target.value = ''; // Reset file input
}
};

Expand All @@ -108,56 +80,19 @@ ${fileArtifacts.join('\n\n')}
className="hidden"
webkitdirectory=""
directory=""
onChange={async (e) => {
const allFiles = Array.from(e.target.files || []);
const filteredFiles = allFiles.filter((file) => shouldIncludeFile(file.webkitRelativePath));

if (filteredFiles.length === 0) {
toast.error('No files found in the selected folder');
return;
}

try {
const fileChecks = await Promise.all(
filteredFiles.map(async (file) => ({
file,
isBinary: await isBinaryFile(file),
})),
);

const textFiles = fileChecks.filter((f) => !f.isBinary).map((f) => f.file);
const binaryFilePaths = fileChecks
.filter((f) => f.isBinary)
.map((f) => f.file.webkitRelativePath.split('/').slice(1).join('/'));

if (textFiles.length === 0) {
toast.error('No text files found in the selected folder');
return;
}

if (binaryFilePaths.length > 0) {
toast.info(`Skipping ${binaryFilePaths.length} binary files`);
}

await createChatFromFolder(textFiles, binaryFilePaths);
} catch (error) {
console.error('Failed to import folder:', error);
toast.error('Failed to import folder');
}

e.target.value = ''; // Reset file input
}}
{...({} as any)} // if removed webkitdirectory will throw errors as unknow attribute
onChange={handleFileChange}
{...({} as any)}
/>
<button
onClick={() => {
const input = document.getElementById('folder-import');
input?.click();
}}
className={className}
disabled={isLoading}
>
<div className="i-ph:folder-simple-upload" />
Import Folder
{isLoading ? 'Importing...' : 'Import Folder'}
</button>
</>
);
Expand Down
97 changes: 97 additions & 0 deletions app/utils/fileUtils.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,97 @@
import ignore from 'ignore';

// Common patterns to ignore, similar to .gitignore
export const IGNORE_PATTERNS = [
'node_modules/**',
'.git/**',
'dist/**',
'build/**',
'.next/**',
'coverage/**',
'.cache/**',
'.vscode/**',
'.idea/**',
'**/*.log',
'**/.DS_Store',
'**/npm-debug.log*',
'**/yarn-debug.log*',
'**/yarn-error.log*',
];

export const MAX_FILES = 1000;
export const ig = ignore().add(IGNORE_PATTERNS);

export const generateId = () => Math.random().toString(36).substring(2, 15);

export const isBinaryFile = async (file: File): Promise<boolean> => {
const chunkSize = 1024;
const buffer = new Uint8Array(await file.slice(0, chunkSize).arrayBuffer());

for (let i = 0; i < buffer.length; i++) {
const byte = buffer[i];
if (byte === 0 || (byte < 32 && byte !== 9 && byte !== 10 && byte !== 13)) {
return true;
}
}
return false;
};

export const shouldIncludeFile = (path: string): boolean => {
return !ig.ignores(path);
};

const readPackageJson = async (files: File[]): Promise<{ scripts?: Record<string, string> } | null> => {
const packageJsonFile = files.find(f => f.webkitRelativePath.endsWith('package.json'));
if (!packageJsonFile) return null;

try {
const content = await new Promise<string>((resolve, reject) => {
const reader = new FileReader();
reader.onload = () => resolve(reader.result as string);
reader.onerror = reject;
reader.readAsText(packageJsonFile);
});

return JSON.parse(content);
} catch (error) {
console.error('Error reading package.json:', error);
return null;
}
};

export const detectProjectType = async (files: File[]): Promise<{ type: string; setupCommand: string; followupMessage: string }> => {
const hasFile = (name: string) => files.some(f => f.webkitRelativePath.endsWith(name));

if (hasFile('package.json')) {
const packageJson = await readPackageJson(files);
const scripts = packageJson?.scripts || {};

// Check for preferred commands in priority order
const preferredCommands = ['dev', 'start', 'preview'];
const availableCommand = preferredCommands.find(cmd => scripts[cmd]);

if (availableCommand) {
return {
type: 'Node.js',
setupCommand: `npm install && npm run ${availableCommand}`,
followupMessage: `Found "${availableCommand}" script in package.json. Running "npm run ${availableCommand}" after installation.`
};
}

return {
type: 'Node.js',
setupCommand: 'npm install',
followupMessage: 'Would you like me to inspect package.json to determine the available scripts for running this project?'
};
}

if (hasFile('index.html')) {
return {
type: 'Static',
setupCommand: 'npx --yes serve',
followupMessage: ''
};
}

return { type: '', setupCommand: '', followupMessage: '' };
};
56 changes: 56 additions & 0 deletions app/utils/folderImport.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,56 @@
import type { Message } from 'ai';
import { generateId, detectProjectType } from './fileUtils';

export const createChatFromFolder = async (
files: File[],
binaryFiles: string[],
folderName: string
): Promise<{ userMessage: Message; assistantMessage: Message }> => {
const fileArtifacts = await Promise.all(
files.map(async (file) => {
return new Promise<string>((resolve, reject) => {
const reader = new FileReader();
reader.onload = () => {
const content = reader.result as string;
const relativePath = file.webkitRelativePath.split('/').slice(1).join('/');
resolve(
`<boltAction type="file" filePath="${relativePath}">
${content}
</boltAction>`,
);
};
reader.onerror = reject;
reader.readAsText(file);
});
}),
);

const project = await detectProjectType(files);
const setupCommand = project.setupCommand ? `\n\n<boltAction type="shell">\n${project.setupCommand}\n</boltAction>` : '';
const followupMessage = project.followupMessage ? `\n\n${project.followupMessage}` : '';

const binaryFilesMessage = binaryFiles.length > 0
? `\n\nSkipped ${binaryFiles.length} binary files:\n${binaryFiles.map((f) => `- ${f}`).join('\n')}`
: '';

const assistantMessage: Message = {
role: 'assistant',
content: `I've imported the contents of the "${folderName}" folder.${binaryFilesMessage}
<boltArtifact id="imported-files" title="Imported Files">
${fileArtifacts.join('\n\n')}
${setupCommand}
</boltArtifact>${followupMessage}`,
id: generateId(),
createdAt: new Date(),
};

const userMessage: Message = {
role: 'user',
id: generateId(),
content: `Import the "${folderName}" folder`,
createdAt: new Date(),
};

return { userMessage, assistantMessage };
};

0 comments on commit 5b7a2a5

Please sign in to comment.