From 0840b4b8b86a46ecf03858369b2b717911c629fc Mon Sep 17 00:00:00 2001 From: Kaloyan Raev Date: Fri, 26 Aug 2016 12:05:36 +0300 Subject: [PATCH] Fixes #142: Move indexing logic entirely to the server-side Indexing is now triggered as part of the onInitialize() request. Any code on the client related to indexing (except registered VS Code commands) is moved to the server. The user commands are just proxies - they make a request to the server. All the code related to building, querying and maintaining the index is moved to the index.ts file. Wherever possible, standard message request from the language server protocol are used instead of custom messages. For example onDidChangeContent() and onDidChangeWatchedFiles() instead of the custom "buildObjectTreeForDocument", "saveTreeCache", etc. --- client/package.json | 4 - client/src/crane.ts | 206 ++----------- client/src/extension.ts | 3 + client/src/utils/Cranefs.ts | 246 --------------- server/package.json | 6 +- server/src/index.ts | 424 ++++++++++++++++++++++++++ server/src/server.ts | 455 ++++++++++------------------ server/src/suggestionBuilder.ts | 26 +- server/src/{util => utils}/Debug.ts | 0 server/src/utils/Path.ts | 23 ++ 10 files changed, 643 insertions(+), 750 deletions(-) delete mode 100644 client/src/utils/Cranefs.ts create mode 100644 server/src/index.ts rename server/src/{util => utils}/Debug.ts (100%) create mode 100644 server/src/utils/Path.ts diff --git a/client/package.json b/client/package.json index 0545de4..4c088d8 100644 --- a/client/package.json +++ b/client/package.json @@ -111,11 +111,7 @@ "vscode": "^0.11.13" }, "dependencies": { - "fstream": "^1.0.9", - "mkdirp": "^0.5.1", "php-parser": "HvyIndustries/php-parser#bde9c58", - "rimraf": "^2.5.2", - "unzip": "^0.1.11", "open": "^0.0.5", "vscode-languageclient": "^2.4.2" } diff --git a/client/src/crane.ts b/client/src/crane.ts index ae4103e..41a3734 100644 --- a/client/src/crane.ts +++ b/client/src/crane.ts @@ -13,7 +13,6 @@ import { } from 'vscode'; import { LanguageClient, RequestType, NotificationType } from 'vscode-languageclient'; import { ThrottledDelayer } from './utils/async'; -import { Cranefs } from './utils/Cranefs'; import { Debug } from './utils/Debug'; import { Config } from './utils/Config'; @@ -22,85 +21,23 @@ const util = require('util'); let craneSettings = workspace.getConfiguration("crane"); -const cranefs: Cranefs = new Cranefs(); console.log(process.platform) export default class Crane { public static langClient: LanguageClient; - private disposable: Disposable; - private delayers: { [key: string]: ThrottledDelayer }; - public static statusBarItem: StatusBarItem; constructor(languageClient: LanguageClient) { - Crane.langClient = languageClient; - - this.delayers = Object.create(null); - - let subscriptions: Disposable[] = []; - - workspace.onDidChangeTextDocument((e) => this.onChangeTextHandler(e.document), null, subscriptions); - workspace.onDidCloseTextDocument((textDocument)=> { delete this.delayers[textDocument.uri.toString()]; }, null, subscriptions); - workspace.onDidSaveTextDocument((document) => this.handleFileSave()); + console.log("Crane Initialised..."); - this.disposable = Disposable.from(...subscriptions); + Crane.langClient = languageClient; if (!Crane.statusBarItem) { Crane.statusBarItem = window.createStatusBarItem(StatusBarAlignment.Left); Crane.statusBarItem.hide(); } - this.checkVersion().then(indexTriggered => { - this.doInit(indexTriggered); - }); - } - - private checkVersion(): Thenable - { - var self = this; - Debug.info('Checking the current version of Crane'); - return new Promise((resolve, reject) => { - cranefs.getVersionFile().then(result => { - if (result.err && result.err.code == "ENOENT") { - // New install - window.showInformationMessage(`Welcome to Crane v${Config.version}.`, "Getting Started Guide").then(data => { - if (data != null) { - open("https://github.com/HvyIndustries/crane/wiki/end-user-guide#getting-started"); - } - }); - cranefs.createOrUpdateVersionFile(false); - cranefs.deleteAllCaches().then(item => { - self.processAllFilesInWorkspace(); - resolve(true); - }); - } else { - // Strip newlines from data - result.data = result.data.replace("\n", ""); - result.data = result.data.replace("\r", ""); - if (result.data && result.data != Config.version) { - // Updated install - window.showInformationMessage(`You're been upgraded to Crane v${Config.version}.`, "View Release Notes").then(data => { - if (data == "View Release Notes") { - open("https://github.com/HvyIndustries/crane/releases"); - } - }); - cranefs.createOrUpdateVersionFile(true); - cranefs.deleteAllCaches().then(item => { - self.processAllFilesInWorkspace(); - resolve(true); - }); - } else { - resolve(false); - } - } - }); - }); - } - - public doInit(indexInProgress: boolean) { - console.log("Crane Initialised..."); - this.showIndexingStatusBarMessage(); var statusBarItem: StatusBarItem = window.createStatusBarItem(StatusBarAlignment.Right); @@ -118,7 +55,26 @@ export default class Crane } }); - var requestType: RequestType = { method: "workDone" }; + var openBrowserMessage: NotificationType<{ url: string }> = { method: "window/openBrowser" }; + Crane.langClient.onNotification(openBrowserMessage, message => { + open(message.url); + }); + + // Update the UI so the user knows the processing status + var fileProcessed: NotificationType<{ filename: string, count: number, total: number, error: any }> = { method: "index/fileProcessed" }; + Crane.langClient.onNotification(fileProcessed, data => { + // Get the percent complete + var percent: string = ((data.count / data.total) * 100).toFixed(1); + Crane.statusBarItem.text = `$(zap) Indexing PHP files (${data.count} of ${data.total} / ${percent}%)`; + if (data.error) { + Debug.error("There was a problem parsing PHP file: " + data.filename); + Debug.error(`${data.error}`); + } else { + Debug.info(`Parsed file ${data.count} of ${data.total} : ${data.filename}`); + } + }); + + var requestType: RequestType = { method: "index/workDone" }; Crane.langClient.onRequest(requestType, (tree) => { // this.projectBuilding = false; Crane.statusBarItem.text = '$(check) PHP File Indexing Complete!'; @@ -141,37 +97,15 @@ export default class Crane Debug.info(`Watching these files: {${types.include.join(',')}}`); var fsw: FileSystemWatcher = workspace.createFileSystemWatcher(`{${types.include.join(',')}}`); - fsw.onDidChange(e => { - workspace.openTextDocument(e).then(document => { - if (document.languageId != 'php') return; - Debug.info('File Changed: ' + e.fsPath); - Crane.langClient.sendRequest({ method: 'buildObjectTreeForDocument' }, { - path: e.fsPath, - text: document.getText() - }); - }); - }); fsw.onDidCreate(e => { - workspace.openTextDocument(e).then(document => { - if (document.languageId != 'php') return; - Debug.info('File Created: ' + e.fsPath); - Crane.langClient.sendRequest({ method: 'buildObjectTreeForDocument' }, { - path: e.fsPath, - text: document.getText() - }); - }); + Crane.langClient.notifyFileEvent({ uri: e.fsPath, type: 1}); + }); + fsw.onDidChange(e => { + Crane.langClient.notifyFileEvent({ uri: e.fsPath, type: 2}); }); fsw.onDidDelete(e => { - Debug.info('File Deleted: ' + e.fsPath); - Crane.langClient.sendRequest({ method: 'deleteFile' }, { - path: e.fsPath - }); + Crane.langClient.notifyFileEvent({ uri: e.fsPath, type: 3}); }); - - if (!indexInProgress) { - // Send request to server to build object tree for all workspace files - this.processAllFilesInWorkspace(); - } } private showIndexingStatusBarMessage() { @@ -184,97 +118,19 @@ export default class Crane open("https://github.com/HvyIndustries/crane/issues"); } - public handleFileSave() { - var editor = window.activeTextEditor; - if (editor == null) return; - - var document = editor.document; - - this.buildObjectTreeForDocument(document).then(() => { - Crane.langClient.sendRequest({ method: 'saveTreeCache' }, { projectDir: cranefs.getProjectDir(), projectTree: cranefs.getTreePath() }); - }).catch(error => { - Debug.error(util.inspect(error, false, null)); - }); - } - - public processAllFilesInWorkspace() { - cranefs.createProjectDir().then(data => { - var createTreeFile: boolean = false; - // Folder was created so there is no tree cache - if (data.folderCreated) { - this.processWorkspaceFiles(); - } else { - // Check for a tree file, if it exists load it; - // otherwise we need to process the files in the workspace - cranefs.doesProjectTreeExist().then(tree => { - if (!tree.exists) { - this.processWorkspaceFiles(); - } else { - this.processProject(); - } - }); - } - }).catch(error => { - Debug.error(util.inspect(error, false, null)); - }); - } - - public deleteCaches() { - var self = this; - cranefs.deleteAllCaches().then(success => { - window.showInformationMessage('All PHP file caches were successfully deleted'); - self.processAllFilesInWorkspace(); - }); + Crane.langClient.sendRequest({ method: "index/deleteAllCaches" }, {}); } public rebuildProject() { - cranefs.rebuildProject(); + Crane.langClient.sendRequest({ method: "index/rebuild" }, {}); } public downloadPHPLibraries() { - cranefs.downloadPHPLibraries(); - } - - public processWorkspaceFiles() { - cranefs.processWorkspaceFiles(); - } - - public processProject() { - cranefs.processProject(); - } - - private onChangeTextHandler(textDocument: TextDocument) { - // Only parse PHP files - if (textDocument.languageId != "php") return; - - let key = textDocument.uri.toString(); - let delayer = this.delayers[key]; - - if (!delayer) { - delayer = new ThrottledDelayer(250); - this.delayers[key] = delayer; - } - - delayer.trigger(() => this.buildObjectTreeForDocument(textDocument)); - } - - private buildObjectTreeForDocument(document: TextDocument): Promise - { - return new Promise((resolve, reject) => { - var path = document.fileName; - var text = document.getText(); - var projectDir = cranefs.getProjectDir(); - var projectTree = cranefs.getTreePath(); - - var requestType: RequestType = { method: "buildObjectTreeForDocument" }; - Crane.langClient.sendRequest(requestType, { path, text, projectDir, projectTree }).then(() => resolve() ); - }); + Crane.langClient.sendRequest({ method: "index/downloadStubs" }, { url: Config.phpstubsZipFile }); } - dispose() - { - this.disposable.dispose(); + dispose() { Crane.statusBarItem.dispose(); } } diff --git a/client/src/extension.ts b/client/src/extension.ts index 478f78e..d6c305e 100644 --- a/client/src/extension.ts +++ b/client/src/extension.ts @@ -36,6 +36,9 @@ export function activate(context: ExtensionContext) synchronize: { configurationSection: "languageServerExample", fileEvents: workspace.createFileSystemWatcher("**/.clientrc") + }, + initializationOptions: { + enableCache: Config.enableCache } } diff --git a/client/src/utils/Cranefs.ts b/client/src/utils/Cranefs.ts deleted file mode 100644 index 12841f2..0000000 --- a/client/src/utils/Cranefs.ts +++ /dev/null @@ -1,246 +0,0 @@ -import { workspace, window } from 'vscode'; -import { NotificationType, RequestType } from 'vscode-languageclient'; -import Crane from '../crane'; -import { Debug } from './Debug'; -import { Config } from './Config'; - -const crypto = require('crypto'); -const fs = require('fs'); -const fstream = require('fstream'); -const http = require('https'); -const unzip = require('unzip'); -const util = require('util'); -const mkdirp = require('mkdirp'); -const rmrf = require('rimraf'); - -let craneSettings = workspace.getConfiguration("crane"); - -interface result { - err; - data; -} - -export class Cranefs { - - public isCacheable(): boolean { - return Config.enableCache; - } - - public getCraneDir(): string { - if (process.env.APPDATA) { - return process.env.APPDATA + '/Crane'; - } - if (process.env.XDG_CONFIG_HOME) { - return process.env.XDG_CONFIG_HOME + '/Crane'; - } - if (process.platform == 'darwin') { - return process.env.HOME + '/Library/Preferences/Crane'; - } - if (process.platform == 'linux') { - return process.env.HOME + '/.config/Crane'; - } - } - - public getVersionFile(): Thenable { - return new Promise((resolve, reject) => { - var filePath = this.getCraneDir() + "/version"; - fs.readFile(filePath, "utf-8", (err, data) => { - resolve({ err, data }); - }); - }); - } - - public createOrUpdateVersionFile(fileExists: boolean) { - var filePath = this.getCraneDir() + "/version"; - - if (fileExists) { - // Delete the file - fs.unlinkSync(filePath); - } - - // Create the file + write Config.version into it - mkdirp(this.getCraneDir(), err => { - if (err) { - Debug.error(err); - return; - } - fs.writeFile(filePath, Config.version, "utf-8", err => { - if (err != null) { - Debug.error(err); - } - }); - }); - - } - - public deleteAllCaches(): Promise { - return new Promise((resolve, reject) => { - rmrf(this.getCraneDir() + '/projects/*', err => { - if (!err) { - Debug.info('Project caches were deleted'); - return resolve(true); - } - Debug.info('Project caches were not deleted'); - return resolve(false); - }); - }); - } - - public getProjectDir(): string { - var md5sum = crypto.createHash('md5'); - // Get the workspace location for the user - return this.getCraneDir() + '/projects/' + (md5sum.update(workspace.rootPath)).digest('hex'); - } - - public getStubsDir(): string { - return this.getCraneDir() + '/phpstubs'; - } - - public getTreePath(): string { - return this.getProjectDir() + '/tree.cache'; - } - - public createProjectDir(): Promise<{ folderExists: boolean, folderCreated: boolean, path: string }> { - return new Promise((resolve, reject) => { - if (this.isCacheable()) { - this.createProjectFolder().then(projectCreated => { - resolve(projectCreated); - }).catch(error => { - Debug.error(util.inspect(error, false, null)); - }); - } else { - resolve({folderExists: false, folderCreated: false, path: null}) - } - }); - } - - public doesProjectTreeExist(): Promise<{exists:boolean, path:string}> { - return new Promise((resolve, reject) => { - fs.stat(this.getTreePath(), (err, stat) => { - if (err === null) { - resolve({exists: true, path: this.getTreePath()}); - } else { - resolve({exists: false, path: null}); - } - }); - }); - } - - public processWorkspaceFiles(rebuild: boolean = false) { - if (workspace.rootPath == undefined) return; - var fileProcessCount = 0; - - // Get PHP files from 'files.associations' to be processed - var files = Config.phpFileTypes; - - // Find all the php files to process - workspace.findFiles(`{${files.include.join(',')}}`, `{${files.exclude.join(',')}}`).then(files => { - Debug.info(`Preparing to parse ${files.length} PHP source files...`); - - fileProcessCount = files.length; - var filePaths: string[] = []; - - // Get the objects path value for the current file system - files.forEach(file => { - filePaths.push(file.fsPath); - }); - - Crane.statusBarItem.text = "$(zap) Indexing PHP files"; - - // Send the array of paths to the language server - Crane.langClient.sendRequest({ method: "buildFromFiles" }, { - files: filePaths, - craneRoot: this.getCraneDir(), - projectPath: this.getProjectDir(), - treePath: this.getTreePath(), - enableCache: this.isCacheable(), - rebuild: rebuild - }); - - // Update the UI so the user knows the processing status - var fileProcessed: NotificationType = { method: "fileProcessed" }; - Crane.langClient.onNotification(fileProcessed, data => { - // Get the percent complete - var percent: string = ((data.total / fileProcessCount) * 100).toFixed(1); - Crane.statusBarItem.text = `$(zap) Indexing PHP files (${data.total} of ${fileProcessCount} / ${percent}%)`; - if (data.error) { - Debug.error("There was a problem parsing PHP file: " + data.filename); - Debug.error(`${data.error}`); - } else { - Debug.info(`Parsed file ${data.total} of ${fileProcessCount} : ${data.filename}`); - } - }); - }); - } - - public processProject(): void { - Debug.info('Building project from cache file: ' + this.getTreePath()); - Crane.langClient.sendRequest({ method: "buildFromProject" }, { - treePath: this.getTreePath(), - enableCache: this.isCacheable() - }); - } - - public rebuildProject(): void { - Debug.info('Rebuilding the project files'); - fs.unlink(this.getTreePath(), (err) => { - this.createProjectFolder().then(success => { - if (success) { - this.processWorkspaceFiles(true); - } - }); - }); - } - - public downloadPHPLibraries(): void { - var zip = Config.phpstubsZipFile; - var tmp = this.getCraneDir() + '/phpstubs.tmp.zip'; - Debug.info(`Downloading ${zip} to ${tmp}`); - this.createPhpStubsFolder().then(created => { - if (created) { - var file = fs.createWriteStream(tmp); - http.get(zip, (response) => { - response.pipe(file); - response.on('end', () => { - Debug.info('PHPStubs Download Complete'); - Debug.info(`Unzipping to ${this.getStubsDir()}`); - fs.createReadStream(tmp) - .pipe(unzip.Parse()) - .pipe(fstream.Writer(this.getStubsDir())); - window.showInformationMessage('PHP Library Stubs downloaded and installed. You may need to re-index the workspace for them to work correctly.', 'Rebuild Now').then(item => { - this.rebuildProject(); - }); - }); - }); - } - }).catch(error => { - Debug.error(util.inspect(error, false, null)); - }); - } - - private createProjectFolder(): Promise<{ folderExists: boolean, folderCreated: boolean, path: string }> { - return new Promise((resolve, reject) => { - mkdirp(this.getProjectDir(), (err) => { - if (err) { - resolve(false); - } else { - resolve(true); - } - }); - }); - } - - private createPhpStubsFolder(): Promise { - return new Promise((resolve, reject) => { - var craneDir: string = this.getCraneDir(); - mkdirp(craneDir + '/phpstubs', (err) => { - if (err) { - resolve(false); - } else { - resolve(true); - } - }); - }); - } - -} diff --git a/server/package.json b/server/package.json index d32397b..3217881 100644 --- a/server/package.json +++ b/server/package.json @@ -1,7 +1,7 @@ { "name": "crane-lang-server", "description": "The language server for Crane", - "version": "1.0.1", + "version": "0.2.0", "author": "HVY Industries", "license": "MIT", "engines": { @@ -9,8 +9,12 @@ }, "dependencies": { "filequeue": "^0.5.0", + "fstream": "^1.0.9", "glob": "7.0.3", + "mkdirp": "^0.5.1", "php-parser": "HvyIndustries/php-parser#bde9c58", + "rimraf": "^2.5.2", + "unzip": "^0.1.11", "vscode-languageserver": "^2.2.1" }, "devDependencies": { diff --git a/server/src/index.ts b/server/src/index.ts new file mode 100644 index 0000000..633a153 --- /dev/null +++ b/server/src/index.ts @@ -0,0 +1,424 @@ +import { IConnection, RequestType } from 'vscode-languageserver'; +import { workspaceRoot, enableCache, getCraneDir } from './server'; +import { TreeBuilder, FileNode } from "./hvy/treeBuilder"; +import { Debug } from './utils/Debug'; + +const crypto = require('crypto'); +const fs = require("fs"); +const fstream = require('fstream'); +const http = require('https'); +const mkdirp = require('mkdirp'); +const rmrf = require('rimraf'); +const util = require('util'); +const unzip = require('unzip'); +const zlib = require('zlib'); + +// Glob for file searching +const glob = require("glob"); + +// FileQueue for queuing files so we don't open too many +const FileQueue = require('filequeue'); +const fq = new FileQueue(200); + +export class Index { + + public tree: FileNode[] = []; + + private connection: IConnection; + private treeBuilder: TreeBuilder = new TreeBuilder(); + + public constructor(connection: IConnection) { + this.connection = connection; + + this.treeBuilder.SetConnection(connection); + + const downloadStubs: RequestType<{ url: string }, void, void> = { method: "index/downloadStubs" }; + connection.onRequest(downloadStubs, param => { + this.downloadStubs(param.url); + }); + + const deleteAllCaches: RequestType = { method: "index/deleteAllCaches" }; + connection.onRequest(deleteAllCaches, param => { + this.deleteAllCaches().then(result => { + this.connection.window.showInformationMessage('All PHP file caches were successfully deleted'); + this.rebuild(); + }); + }); + + const rebuild: RequestType = { method: "index/rebuild" }; + connection.onRequest(rebuild, param => { + this.rebuild(); + }); + } + + public build(): void { + this.createProjectDir().then(result => { + // Folder was created so there is no tree cache + if (result.folderCreated) { + this.processWorkspace(); + } else { + // Check for a tree file, if it exists load it; + // otherwise we need to process the files in the workspace + this.doesProjectTreeExist().then(tree => { + if (!tree.exists) { + this.processWorkspace(); + } else { + this.restoreFromCache(); + } + }); + } + }).catch(error => { + Debug.error(util.inspect(error, false, null)); + }); + } + + public rebuild(): void { + Debug.info('Rebuilding the project files'); + fs.unlink(this.getTreePath(), (err) => { + this.createProjectFolder().then(success => { + if (success) { + this.processWorkspace(true); + } + }); + }); + } + + public deleteAllCaches(): Promise { + return new Promise((resolve, reject) => { + rmrf(getCraneDir() + '/projects/*', err => { + if (!err) { + Debug.info('Project caches were deleted'); + return resolve(true); + } + Debug.info('Project caches were not deleted'); + return resolve(false); + }); + }); + } + + public addFile(path: string): void { + fq.readFile(path, { encoding: 'utf8' }, (err, data) => { + this.updateFile(path, data, true); + }); + } + + public updateFile(path: string, text: string, saveCache: boolean = false): void { + this.treeBuilder.Parse(text, path).then(result => { + this.addToWorkspaceTree(result.tree); + if (saveCache) { + this.saveCache(); + } + // notifyClientOfWorkComplete(); + return true; + }) + .catch(error => { + console.log(error); + this.notifyClientForWorkComplete(); + return false; + }); + } + + public removeFile(path: string): void { + var node = this.getFileNode(path); + this.removeFromWorkspaceTree(node); + this.saveCache(); + } + + public getFileNode(path: string): FileNode { + var returnNode = null; + + this.tree.forEach(fileNode => { + if (fileNode.path == path) { + returnNode = fileNode; + } + }); + + return returnNode; + } + + private saveCache() { + this.saveProjectTree(this.getProjectDir(), this.getTreePath()).then(saved => { + this.notifyClientForWorkComplete(); + }).catch(error => { + Debug.error(util.inspect(error, false, null)); + }); + } + + private getProjectDir(): string { + var md5sum = crypto.createHash('md5'); + // Get the workspace location for the user + return getCraneDir() + '/projects/' + (md5sum.update(workspaceRoot)).digest('hex'); + } + + private getStubsDir(): string { + return getCraneDir() + '/phpstubs'; + } + + private getTreePath(): string { + return this.getProjectDir() + '/tree.cache'; + } + + private downloadStubs(url: String): void { + var tmp = getCraneDir() + '/phpstubs.tmp.zip'; + Debug.info(`Downloading ${url} to ${tmp}`); + this.createStubsFolder().then(created => { + if (created) { + var file = fs.createWriteStream(tmp); + http.get(url, (response) => { + response.pipe(file); + response.on('end', () => { + Debug.info('PHPStubs Download Complete'); + + Debug.info(`Unzipping to ${this.getStubsDir()}`); + fs.createReadStream(tmp) + .pipe(unzip.Parse()) + .pipe(fstream.Writer(this.getStubsDir())); + this.connection.window.showInformationMessage('PHP Library Stubs downloaded and installed. You may need to re-index the workspace for them to work correctly.', { title: 'Rebuild Now' }).then(item => { + this.rebuild(); + }); + }); + }); + } + }).catch(error => { + Debug.error(util.inspect(error, false, null)); + }); + } + + private createStubsFolder(): Promise { + return new Promise((resolve, reject) => { + mkdirp(getCraneDir() + '/phpstubs', (err) => { + if (err) { + resolve(false); + } else { + resolve(true); + } + }); + }); + } + + private createProjectFolder(): Promise<{ folderExists: boolean, folderCreated: boolean, path: string }> { + return new Promise((resolve, reject) => { + mkdirp(this.getProjectDir(), (err) => { + if (err) { + resolve(false); + } else { + resolve(true); + } + }); + }); + } + + private createProjectDir(): Promise<{ folderExists: boolean, folderCreated: boolean, path: string }> { + return new Promise((resolve, reject) => { + if (enableCache) { + this.createProjectFolder().then(projectCreated => { + resolve(projectCreated); + }).catch(error => { + Debug.error(util.inspect(error, false, null)); + }); + } else { + resolve({folderExists: false, folderCreated: false, path: null}) + } + }); + } + + private doesProjectTreeExist(): Promise<{exists:boolean, path:string}> { + return new Promise((resolve, reject) => { + fs.stat(this.getTreePath(), (err, stat) => { + if (err === null) { + resolve({exists: true, path: this.getTreePath()}); + } else { + resolve({exists: false, path: null}); + } + }); + }); + } + + private processWorkspace(rebuild: boolean = false) { + glob(workspaceRoot + '/**/*.php', (err, fileNames) => { + this.buildIndexFromFiles(fileNames, rebuild); + }); + } + + + + private buildIndexFromFiles(files: string[], rebuild: boolean) { + if (rebuild) { + this.tree = []; + this.treeBuilder = new TreeBuilder(); + } + this.connection.console.log('starting work!'); + // Run asynchronously + setTimeout(() => { + var craneRoot = getCraneDir() + glob(craneRoot + '/phpstubs/*/*.php', (err, fileNames) => { + // Process the php stubs + var stubs: string[] = fileNames; + Debug.info(`Processing ${stubs.length} stubs from ${craneRoot}/phpstubs`); + this.connection.console.log(`Stub files to process: ${stubs.length}`); + this.processStub(stubs).then(data => { + this.connection.console.log('stubs done!'); + this.processWorkspaceFiles(files, this.getProjectDir(), this.getTreePath()); + }).catch(data => { + this.connection.console.log('No stubs found!'); + this.processWorkspaceFiles(files, this.getProjectDir(), this.getTreePath()); + }); + }); + }, 100); + } + + /** + * Processes the stub files + */ + private processStub(stubs: string[]) { + return new Promise((resolve, reject) => { + var offset: number = 0; + if (stubs.length == 0) { + reject(); + } + stubs.forEach(file => { + fq.readFile(file, { encoding: 'utf8' }, (err, data) => { + this.treeBuilder.Parse(data, file).then(result => { + this.addToWorkspaceTree(result.tree); + this.connection.console.log(`${offset} Stub Processed: ${file}`); + offset++; + if (offset == stubs.length) { + resolve(); + } + }).catch(err => { + this.connection.console.log(`${offset} Stub Error: ${file}`); + Debug.error((util.inspect(err, false, null))); + offset++; + if (offset == stubs.length) { + resolve(); + } + }); + }); + }); + }); + } + + /** + * Processes the users workspace files + */ + private processWorkspaceFiles(files: string[], projectPath: string, treePath: string) { + this.connection.console.log(`Workspace files to process: ${files.length}`); + var progress = 0; + files.forEach(file => { + fq.readFile(file, { encoding: 'utf8' }, (err, data) => { + this.treeBuilder.Parse(data, file).then(result => { + this.addToWorkspaceTree(result.tree); + progress++; + this.connection.console.log(`(${progress} of ${files.length}) File: ${file}`); + this.connection.sendNotification({ method: "index/fileProcessed" }, { filename: file, count: progress, total: files.length, error: null }); + if (files.length == progress) { + this.workspaceProcessed(projectPath, treePath); + } + }).catch(data => { + progress++; + if (files.length == progress) { + this.workspaceProcessed(projectPath, treePath); + } + this.connection.console.log(util.inspect(data, false, null)); + this.connection.console.log(`Issue processing ${file}`); + this.connection.sendNotification({ method: "index/fileProcessed" }, { filename: file, count: progress, total: files.length, error: util.inspect(data, false, null) }); + }); + }); + }); + } + + private workspaceProcessed(projectPath, treePath) { + Debug.info("Workspace files have processed"); + this.saveProjectTree(projectPath, treePath).then(savedTree => { + this.notifyClientForWorkComplete(); + if (savedTree) { + Debug.info('Project tree has been saved'); + } + }).catch(error => { + Debug.error(util.inspect(error, false, null)); + }); + } + + private restoreFromCache(): void { + Debug.info('Restoring index from cache file: ' + this.getTreePath()); + fs.readFile(this.getTreePath(), (err, data) => { + if (err) { + Debug.error('Could not read cache file'); + Debug.error((util.inspect(err, false, null))); + } else { + Debug.info('Unzipping the file'); + var treeStream = new Buffer(data); + zlib.gunzip(treeStream, (err, buffer) => { + if (err) { + Debug.error('Could not unzip cache file'); + Debug.error((util.inspect(err, false, null))); + } else { + Debug.info('Cache file successfully read'); + this.tree = JSON.parse(buffer.toString()); + Debug.info('Loaded'); + this.notifyClientForWorkComplete(); + } + }); + } + }); + } + + private notifyClientForWorkComplete() + { + var requestType: RequestType = { method: "index/workDone" }; + this.connection.sendRequest(requestType); + } + + private addToWorkspaceTree(tree: FileNode) { + // Loop through existing filenodes and replace if exists, otherwise add + + var fileNode = this.tree.filter((fileNode) => { + return fileNode.path == tree.path; + })[0]; + + var index = this.tree.indexOf(fileNode); + + if (index !== -1) { + this.tree[index] = tree; + } else { + this.tree.push(tree); + } + + // Debug + // connection.console.log("Parsed file: " + tree.path); + } + + private removeFromWorkspaceTree(tree: FileNode) { + var index: number = this.tree.indexOf(tree); + if (index > -1) { + this.tree.splice(index, 1); + } + } + + private saveProjectTree(projectPath: string, treeFile: string): Promise { + return new Promise((resolve, reject) => { + if (!enableCache) { + resolve(false); + } else { + Debug.info('Packing tree file: ' + treeFile); + fq.writeFile(`${projectPath}/tree.tmp`, JSON.stringify(this.tree), (err) => { + if (err) { + Debug.error('Could not write to cache file'); + Debug.error(util.inspect(err, false, null)); + resolve(false); + } else { + var gzip = zlib.createGzip(); + var inp = fs.createReadStream(`${projectPath}/tree.tmp`); + var out = fs.createWriteStream(treeFile); + inp.pipe(gzip).pipe(out).on('close', function () { + fs.unlinkSync(`${projectPath}/tree.tmp`); + }); + Debug.info('Cache file updated'); + resolve(true); + } + }); + } + }); + } + +} \ No newline at end of file diff --git a/server/src/server.ts b/server/src/server.ts index 91351c5..cc84767 100644 --- a/server/src/server.ts +++ b/server/src/server.ts @@ -9,25 +9,20 @@ import { IPCMessageReader, IPCMessageWriter, SymbolKind, createConnection, IConnection, TextDocumentSyncKind, - TextDocuments, Diagnostic, DiagnosticSeverity, - InitializeParams, InitializeResult, TextDocumentIdentifier, TextDocumentPositionParams, - CompletionList, CompletionItem, CompletionItemKind, RequestType, Position, - SignatureHelp, SignatureInformation, ParameterInformation + TextDocuments, InitializeResult, TextDocumentPositionParams, + CompletionList, CompletionItem, CompletionItemKind, RequestType, FileChangeType } from 'vscode-languageserver'; -import { TreeBuilder, FileNode, FileSymbolCache, SymbolType, AccessModifierNode, ClassNode } from "./hvy/treeBuilder"; -import { Debug } from './util/Debug'; +import { FileSymbolCache } from "./hvy/treeBuilder"; +import { Debug } from './utils/Debug'; +import { Path } from './utils/Path'; +import { Index } from './index'; import { SuggestionBuilder } from './suggestionBuilder'; const fs = require("fs"); -const util = require('util'); -const zlib = require('zlib'); +const mkdirp = require('mkdirp'); -// Glob for file searching -const glob = require("glob"); -// FileQueue for queuing files so we don't open too many -const FileQueue = require('filequeue'); -const fq = new FileQueue(200); +var pkg = require('./package.json'); let connection: IConnection = createConnection(new IPCMessageReader(process), new IPCMessageWriter(process)); @@ -35,17 +30,28 @@ let documents: TextDocuments = new TextDocuments(); documents.listen(connection); Debug.SetConnection(connection); -let treeBuilder: TreeBuilder = new TreeBuilder(); -treeBuilder.SetConnection(connection); -let workspaceTree: FileNode[] = []; +let index: Index = new Index(connection); + +export let workspaceRoot: string; +export let enableCache: boolean = true; -let workspaceRoot: string; -var craneProjectDir: string; -let enableCache: boolean = true; connection.onInitialize((params): InitializeResult => { workspaceRoot = params.rootPath; + // Read initialization options if provided by client + var opts = params.initializationOptions; + if (opts && opts.enableCache) { + enableCache = opts.enableCache; + } + + checkVersion().then(indexTriggered => { + if (!indexTriggered) { + // Send request to server to build object tree for all workspace files + index.build(); + } + }); + return { capabilities: { @@ -59,6 +65,101 @@ connection.onInitialize((params): InitializeResult => } }); +function checkVersion(): Thenable +{ + Debug.info('Checking the current version of Crane'); + return new Promise((resolve, reject) => { + getVersionFile().then(result => { + if (result.err && result.err.code == "ENOENT") { + // New install + connection.window.showInformationMessage(`Welcome to Crane v${pkg.version}.`, { title: "Getting Started Guide" }).then(data => { + if (data != null) { + connection.sendNotification({ method: "window/openBrowser" }, + { url: "https://github.com/HvyIndustries/crane/wiki/end-user-guide#getting-started" }); + } + }); + createOrUpdateVersionFile(false); + index.deleteAllCaches().then(item => { + index.build(); + resolve(true); + }); + } else { + // Strip newlines from data + result.data = result.data.replace("\n", ""); + result.data = result.data.replace("\r", ""); + if (result.data && result.data != pkg.version) { + // Updated install + connection.window.showInformationMessage(`You're been upgraded to Crane v${pkg.version}.`, { title: "View Release Notes" }).then(data => { + if (data != null) { + connection.sendNotification({ method: "window/openBrowser" }, + { url: "https://github.com/HvyIndustries/crane/releases" }); + } + }); + createOrUpdateVersionFile(true); + index.deleteAllCaches().then(item => { + index.build(); + resolve(true); + }); + } else { + resolve(false); + } + } + }); + }); +} + +export function getCraneDir(): string { + if (process.env.APPDATA) { + return process.env.APPDATA + '/Crane'; + } + if (process.env.XDG_CONFIG_HOME) { + return process.env.XDG_CONFIG_HOME + '/Crane'; + } + if (process.platform == 'darwin') { + return process.env.HOME + '/Library/Preferences/Crane'; + } + if (process.platform == 'linux') { + return process.env.HOME + '/.config/Crane'; + } +} + +interface result { + err; + data; +} + +function getVersionFile(): Thenable { + return new Promise((resolve, reject) => { + var filePath = getCraneDir() + "/version"; + fs.readFile(filePath, "utf-8", (err, data) => { + resolve({ err, data }); + }); + }); +} + +function createOrUpdateVersionFile(fileExists: boolean) { + var filePath = getCraneDir() + "/version"; + + if (fileExists) { + // Delete the file + fs.unlinkSync(filePath); + } + + // Create the file + write Config.version into it + mkdirp(getCraneDir(), err => { + if (err) { + Debug.error(err); + return; + } + fs.writeFile(filePath, pkg.version, "utf-8", err => { + if (err != null) { + Debug.error(err); + } + }); + }); + +} + // The settings interface describe the server relevant settings part interface Settings { languageServerExample: ExampleSettings; @@ -87,10 +188,35 @@ connection.onDidChangeConfiguration((change) => // https://github.com/Microsoft/vscode/blob/580d19ab2e1fd6488c3e515e27fe03dceaefb819/extensions/json/server/src/server.ts //connection.sendRequest() +documents.onDidChangeContent((event) => { + Debug.info('Document Changed: ' + event.document.uri); + index.updateFile(Path.fromURI(event.document.uri), event.document.getText()); +}); + connection.onDidChangeWatchedFiles((change) => { - // Monitored files have change in VSCode - connection.console.log('We recevied an file change event'); + change.changes.forEach(event => { + switch (event.type) { + case FileChangeType.Created: + Debug.info('File Created: ' + event.uri); + index.addFile(Path.fromURI(event.uri)); + break; + + case FileChangeType.Changed: + Debug.info('File Changed: ' + event.uri); + index.addFile(Path.fromURI(event.uri)); + break; + + case FileChangeType.Deleted: + Debug.info('File Deleted: ' + event.uri); + index.removeFile(Path.fromURI(event.uri)); + break; + + default: + Debug.error('Unknown FileChangeType: ' + event.type); + break; + } + }); }); // This handler provides the initial list of the completion items. @@ -99,7 +225,7 @@ connection.onCompletion((textDocumentPosition: TextDocumentPositionParams): Comp var doc = documents.get(textDocumentPosition.textDocument.uri); var suggestionBuilder = new SuggestionBuilder(); - suggestionBuilder.prepare(textDocumentPosition, doc, workspaceTree); + suggestionBuilder.prepare(textDocumentPosition, doc, index.tree); var toReturn: CompletionList = { isIncomplete: false, items: suggestionBuilder.build() }; @@ -121,28 +247,10 @@ connection.onCompletionResolve((item: CompletionItem): CompletionItem => return item; }); -var buildObjectTreeForDocument: RequestType<{path:string,text:string}, any, any> = { method: "buildObjectTreeForDocument" }; -connection.onRequest(buildObjectTreeForDocument, (requestObj) => +var findNode: RequestType<{path:string}, any, any> = { method: "findNode" }; +connection.onRequest(findNode, (requestObj) => { - var fileUri = requestObj.path; - var text = requestObj.text; - - treeBuilder.Parse(text, fileUri).then(result => { - addToWorkspaceTree(result.tree); - // notifyClientOfWorkComplete(); - return true; - }) - .catch(error => { - console.log(error); - notifyClientOfWorkComplete(); - return false; - }); -}); - -var deleteFile: RequestType<{path:string}, any, any> = { method: "findNode" }; -connection.onRequest(deleteFile, (requestObj) => -{ - var node = getFileNodeFromPath(requestObj.path); + var node = index.getFileNode(requestObj.path); // connection.console.log(node); @@ -153,7 +261,7 @@ connection.onRequest(deleteFile, (requestObj) => */ var findFileDocumentSymbols: RequestType<{path:string}, any, any> = { method: "findFileDocumentSymbols" }; connection.onRequest(findFileDocumentSymbols, (requestObj) => { - var node = getFileNodeFromPath(requestObj.path); + var node = index.getFileNode(requestObj.path); return { symbols: node.symbolCache }; }); // function getSymbolObject(node: any, query: string, path: string, usings: string[], parent: any = null): FileSymbolCache { @@ -191,7 +299,7 @@ connection.onRequest(findWorkspaceSymbols, (requestObj) => { connection.console.log(query); - workspaceTree.forEach(item => { + index.tree.forEach(item => { // Search The interfaces item.interfaces.forEach(interfaceNode => { let ns: string = interfaceNode.namespaceParts.join('\\'); @@ -369,7 +477,7 @@ connection.onRequest(findWorkspaceSymbols, (requestObj) => { * Finds the Usings in a file */ function getFileUsings(path: string): string[] { - var node = getFileNodeFromPath(path); + var node = index.getFileNode(path); var namespaces: string[] = []; node.classes.forEach(item => { @@ -396,259 +504,4 @@ function getFileUsings(path: string): string[] { return namespaces; }; -var deleteFile: RequestType<{path:string}, any, any> = { method: "deleteFile" }; -connection.onRequest(deleteFile, (requestObj) => -{ - var node = getFileNodeFromPath(requestObj.path); - if (node instanceof FileNode) { - removeFromWorkspaceTree(node); - } -}); - -var saveTreeCache: RequestType<{ projectDir: string, projectTree: string }, any, any> = { method: "saveTreeCache" }; -connection.onRequest(saveTreeCache, request => { - saveProjectTree(request.projectDir, request.projectTree).then(saved => { - notifyClientOfWorkComplete(); - }).catch(error => { - Debug.error(util.inspect(error, false, null)); - }); -}); - -let docsDoneCount = 0; -var docsToDo: string[] = []; -var stubsToDo: string[] = []; - -var buildFromFiles: RequestType<{ - files: string[], - craneRoot: string, - projectPath: string, - treePath: string, - enableCache: boolean, - rebuild: boolean -}, any, any> = { method: "buildFromFiles" }; -connection.onRequest(buildFromFiles, (project) => { - if (project.rebuild) { - workspaceTree = []; - treeBuilder = new TreeBuilder(); - } - enableCache = project.enableCache; - docsToDo = project.files; - docsDoneCount = 0; - connection.console.log('starting work!'); - // Run asynchronously - setTimeout(() => { - glob(project.craneRoot + '/phpstubs/*/*.php', (err, fileNames) => { - // Process the php stubs - stubsToDo = fileNames; - Debug.info(`Processing ${stubsToDo.length} stubs from ${project.craneRoot}/phpstubs`) - connection.console.log(`Stub files to process: ${stubsToDo.length}`); - processStub().then(data => { - connection.console.log('stubs done!'); - connection.console.log(`Workspace files to process: ${docsToDo.length}`); - processWorkspaceFiles(project.projectPath, project.treePath); - }).catch(data => { - connection.console.log('No stubs found!'); - connection.console.log(`Workspace files to process: ${docsToDo.length}`); - processWorkspaceFiles(project.projectPath, project.treePath); - }); - }); - }, 100); -}); - -var buildFromProject: RequestType<{treePath:string, enableCache:boolean}, any, any> = { method: "buildFromProject" }; -connection.onRequest(buildFromProject, (data) => { - enableCache = data.enableCache; - fs.readFile(data.treePath, (err, data) => { - if (err) { - Debug.error('Could not read cache file'); - Debug.error((util.inspect(err, false, null))); - } else { - Debug.info('Unzipping the file'); - var treeStream = new Buffer(data); - zlib.gunzip(treeStream, (err, buffer) => { - if (err) { - Debug.error('Could not unzip cache file'); - Debug.error((util.inspect(err, false, null))); - } else { - Debug.info('Cache file successfully read'); - workspaceTree = JSON.parse(buffer.toString()); - Debug.info('Loaded'); - notifyClientOfWorkComplete(); - } - }); - } - }); -}); - -/** - * Processes the stub files - */ -function processStub() { - return new Promise((resolve, reject) => { - var offset: number = 0; - if (stubsToDo.length == 0) { - reject(); - } - stubsToDo.forEach(file => { - fq.readFile(file, { encoding: 'utf8' }, (err, data) => { - treeBuilder.Parse(data, file).then(result => { - addToWorkspaceTree(result.tree); - connection.console.log(`${offset} Stub Processed: ${file}`); - offset++; - if (offset == stubsToDo.length) { - resolve(); - } - }).catch(err => { - connection.console.log(`${offset} Stub Error: ${file}`); - Debug.error((util.inspect(err, false, null))); - offset++; - if (offset == stubsToDo.length) { - resolve(); - } - }); - }); - }); - }); -} - -/** - * Processes the users workspace files - */ -function processWorkspaceFiles(projectPath: string, treePath: string) { - docsToDo.forEach(file => { - fq.readFile(file, { encoding: 'utf8' }, (err, data) => { - treeBuilder.Parse(data, file).then(result => { - addToWorkspaceTree(result.tree); - docsDoneCount++; - connection.console.log(`(${docsDoneCount} of ${docsToDo.length}) File: ${file}`); - connection.sendNotification({ method: "fileProcessed" }, { filename: file, total: docsDoneCount, error: null }); - if (docsToDo.length == docsDoneCount) { - workspaceProcessed(projectPath, treePath); - } - }).catch(data => { - docsDoneCount++; - if (docsToDo.length == docsDoneCount) { - workspaceProcessed(projectPath, treePath); - } - connection.console.log(util.inspect(data, false, null)); - connection.console.log(`Issue processing ${file}`); - connection.sendNotification({ method: "fileProcessed" }, { filename: file, total: docsDoneCount, error: util.inspect(data, false, null) }); - }); - }); - }); -} - -function workspaceProcessed(projectPath, treePath) { - Debug.info("Workspace files have processed"); - saveProjectTree(projectPath, treePath).then(savedTree => { - notifyClientOfWorkComplete(); - if (savedTree) { - Debug.info('Project tree has been saved'); - } - }).catch(error => { - Debug.error(util.inspect(error, false, null)); - }); -} - -function addToWorkspaceTree(tree:FileNode) -{ - // Loop through existing filenodes and replace if exists, otherwise add - - var fileNode = workspaceTree.filter((fileNode) => { - return fileNode.path == tree.path; - })[0]; - - var index = workspaceTree.indexOf(fileNode); - - if (index !== -1) { - workspaceTree[index] = tree; - } else { - workspaceTree.push(tree); - } - - // Debug - // connection.console.log("Parsed file: " + tree.path); -} - -function removeFromWorkspaceTree(tree: FileNode) { - var index: number = workspaceTree.indexOf(tree); - if (index > -1) { - workspaceTree.splice(index, 1); - } -} - -function getClassNodeFromTree(className:string): ClassNode -{ - var toReturn = null; - - var fileNode = workspaceTree.forEach((fileNode) => { - fileNode.classes.forEach((classNode) => { - if (classNode.name.toLowerCase() == className.toLowerCase()) { - toReturn = classNode; - } - }) - }); - - return toReturn; -} - -function getTraitNodeFromTree(traitName: string): ClassNode -{ - var toReturn = null; - - var fileNode = workspaceTree.forEach((fileNode) => { - fileNode.traits.forEach((traitNode) => { - if (traitNode.name.toLowerCase() == traitName.toLowerCase()) { - toReturn = traitNode; - } - }) - }); - - return toReturn; -} - -function getFileNodeFromPath(path: string): FileNode { - var returnNode = null; - - workspaceTree.forEach(fileNode => { - if (fileNode.path == path) { - returnNode = fileNode; - } - }); - - return returnNode; -} - -function notifyClientOfWorkComplete() -{ - var requestType: RequestType = { method: "workDone" }; - connection.sendRequest(requestType); -} - -function saveProjectTree(projectPath: string, treeFile: string): Promise { - return new Promise((resolve, reject) => { - if (!enableCache) { - resolve(false); - } else { - Debug.info('Packing tree file: ' + treeFile); - fq.writeFile(`${projectPath}/tree.tmp`, JSON.stringify(workspaceTree), (err) => { - if (err) { - Debug.error('Could not write to cache file'); - Debug.error(util.inspect(err, false, null)); - resolve(false); - } else { - var gzip = zlib.createGzip(); - var inp = fs.createReadStream(`${projectPath}/tree.tmp`); - var out = fs.createWriteStream(treeFile); - inp.pipe(gzip).pipe(out).on('close', function () { - fs.unlinkSync(`${projectPath}/tree.tmp`); - }); - Debug.info('Cache file updated'); - resolve(true); - } - }); - } - }); -} - connection.listen(); diff --git a/server/src/suggestionBuilder.ts b/server/src/suggestionBuilder.ts index c478044..6cadcf6 100644 --- a/server/src/suggestionBuilder.ts +++ b/server/src/suggestionBuilder.ts @@ -11,8 +11,8 @@ import { TreeBuilder, FileNode, FileSymbolCache, SymbolType, AccessModifierNode, ClassNode, TraitNode } from "./hvy/treeBuilder"; - -const fs = require('fs'); +import { Debug } from './utils/Debug'; +import { Path } from './utils/Path'; export class SuggestionBuilder { @@ -31,7 +31,7 @@ export class SuggestionBuilder { this.workspaceTree = workspaceTree; - this.filePath = this.buildDocumentPath(textDocumentPosition.textDocument.uri); + this.filePath = Path.fromURI(textDocumentPosition.textDocument.uri); this.lineIndex = textDocumentPosition.position.line; this.charIndex = textDocumentPosition.position.character; @@ -356,26 +356,6 @@ export class SuggestionBuilder return false; } - private buildDocumentPath(uri: string) : string - { - var path = uri; - path = path.replace("file:///", ""); - path = path.replace("%3A", ":"); - - // Handle Windows and Unix paths - switch (process.platform) { - case 'darwin': - case 'linux': - path = "/" + path; - break; - case 'win32': - path = path.replace(/\//g, "\\"); - break; - } - - return path; - } - private getClassNodeFromTree(className: string) : ClassNode { var toReturn = null; diff --git a/server/src/util/Debug.ts b/server/src/utils/Debug.ts similarity index 100% rename from server/src/util/Debug.ts rename to server/src/utils/Debug.ts diff --git a/server/src/utils/Path.ts b/server/src/utils/Path.ts new file mode 100644 index 0000000..409b2f1 --- /dev/null +++ b/server/src/utils/Path.ts @@ -0,0 +1,23 @@ +export class Path { + + public static fromURI(uri: string) : string + { + var path = uri; + path = path.replace("file:///", ""); + path = path.replace("%3A", ":"); + + // Handle Windows and Unix paths + switch (process.platform) { + case 'darwin': + case 'linux': + path = "/" + path; + break; + case 'win32': + path = path.replace(/\//g, "\\"); + break; + } + + return path; + } + +} \ No newline at end of file