Merge pull request #27 from jupyterlite/tab-complete-cmd

Improvements to command/alias tab completion

src/parse.ts

@@ -4,7 +4,10 @@ import { Token, tokenize } from './tokenize';
 const endOfCommand = ';&';
 //const ignore_trailing = ";"
 
-export abstract class Node {}
+export abstract class Node {
+  // Return last token and whether it is a command or not.
+  abstract lastToken(): [Token | null, boolean];
+}
 
 export class CommandNode extends Node {
   constructor(
@@ -14,13 +17,31 @@ export class CommandNode extends Node {
   ) {
     super();
   }
+
+  lastToken(): [Token | null, boolean] {
+    if (this.redirects && this.redirects.length > 0) {
+      return this.redirects[this.redirects.length - 1].lastToken();
+    } else if (this.suffix.length > 0) {
+      return [this.suffix[this.suffix.length - 1], false];
+    } else {
+      return [this.name, true];
+    }
+  }
 }
 
 export class PipeNode extends Node {
   // Must be at least 2 commands
   constructor(readonly commands: CommandNode[]) {
     super();
   }
+
+  lastToken(): [Token | null, boolean] {
+    if (this.commands.length > 0) {
+      return this.commands[this.commands.length - 1].lastToken();
+    } else {
+      return [null, false];
+    }
+  }
 }
 
 export class RedirectNode extends Node {
@@ -30,10 +51,14 @@ export class RedirectNode extends Node {
   ) {
     super();
   }
+
+  lastToken(): [Token | null, boolean] {
+    return [this.target, false];
+  }
 }
 
-export function parse(source: string, aliases?: Aliases): Node[] {
-  const tokens = tokenize(source, aliases);
+export function parse(source: string, throwErrors: boolean = true, aliases?: Aliases): Node[] {
+  const tokens = tokenize(source, throwErrors, aliases);
 
   const ret: Node[] = [];
   const stack: CommandNode[] = [];

src/shell.ts

@@ -7,6 +7,7 @@ import { IFileSystem } from './file_system';
 import { History } from './history';
 import { FileInput, FileOutput, IInput, IOutput, Pipe, TerminalInput, TerminalOutput } from './io';
 import { CommandNode, PipeNode, parse } from './parse';
+import { longestStartsWith, toColumns } from './utils';
 import * as FsModule from './wasm/fs';
 
 export namespace IShell {
@@ -62,22 +63,7 @@ export class Shell {
       }
     } else if (code === 9) {
       // Tab \t
-      const trimmedLine = this._currentLine.trimStart();
-      if (trimmedLine.length === 0) {
-        return;
-      }
-
-      // This tab complete needs to be improved.
-      const [offset, possibles] = await this._tabComplete(trimmedLine);
-      if (possibles.length === 1) {
-        const n = this._currentLine.length;
-        this._currentLine = this._currentLine + possibles[0].slice(offset) + ' ';
-        await this.output(this._currentLine.slice(n));
-      } else if (possibles.length > 1) {
-        const line = possibles.join('  ');
-        // Note keep leading whitespace on current line.
-        await this.output(`\r\n${line}\r\n${this._environment.getPrompt()}${this._currentLine}`);
-      }
+      await this._tabComplete(this._currentLine);
     } else if (code === 27) {
       // Escape following by 1+ more characters
       const remainder = char.slice(1);
@@ -165,7 +151,7 @@ export class Shell {
     const stdin = new TerminalInput(this._stdinCallback);
     const stdout = new TerminalOutput(this._outputCallback);
     try {
-      const nodes = parse(cmdText, this._aliases);
+      const nodes = parse(cmdText, true, this._aliases);
 
       for (const node of nodes) {
         if (node instanceof CommandNode) {
@@ -240,13 +226,48 @@ export class Shell {
     await context.flush();
   }
 
-  private async _tabComplete(text: string): Promise<[number, string[]]> {
-    // Assume tab completing command.
-    const commandMatches = CommandRegistry.instance().match(text);
-    const aliasMatches = this._aliases.match(text);
-    // Combine, removing duplicates, and sort.
-    const matches = [...new Set([...commandMatches, ...aliasMatches])];
-    return [text.length, matches];
+  private async _tabComplete(text: string): Promise<void> {
+    if (text.endsWith(' ') && text.trim().length > 0) {
+      return;
+    }
+
+    const parsed = parse(text, false);
+    const [lastToken, isCommand] =
+      parsed.length > 0 ? parsed[parsed.length - 1].lastToken() : [null, true];
+    const lookup = lastToken?.value ?? '';
+
+    let possibles: string[] = [];
+    if (isCommand) {
+      const commandMatches = CommandRegistry.instance().match(lookup);
+      const aliasMatches = this._aliases.match(lookup);
+      // Combine, removing duplicates, and sort.
+      possibles = [...new Set([...commandMatches, ...aliasMatches])].sort();
+    } else {
+      // Is filename, not yet implemented.
+    }
+
+    if (possibles.length === 0) {
+      return;
+    } else if (possibles.length === 1) {
+      const extra = possibles[0].slice(lookup.length) + ' ';
+      this._currentLine += extra;
+      await this.output(extra);
+      return;
+    }
+
+    // Multiple possibles.
+    const startsWith = longestStartsWith(possibles, lookup.length);
+    if (startsWith.length > lookup.length) {
+      // Complete up to the longest common startsWith.
+      const extra = startsWith.slice(lookup.length);
+      this._currentLine += extra;
+      await this.output(extra);
+    } else {
+      // Write all the possibles in columns across the terminal.
+      const lines = toColumns(possibles, this._environment.getNumber('COLUMNS') ?? 0);
+      const output = `\r\n${lines.join('\r\n')}\r\n${this._environment.getPrompt()}${this._currentLine}`;
+      await this.output(output);
+    }
   }
 
   private readonly _outputCallback: IOutputCallback;

src/tokenize.ts

@@ -9,8 +9,8 @@ export type Token = {
   value: string;
 };
 
-export function tokenize(source: string, aliases?: Aliases): Token[] {
-  const tokenizer = new Tokenizer(source, aliases);
+export function tokenize(source: string, throwErrors: boolean = true, aliases?: Aliases): Token[] {
+  const tokenizer = new Tokenizer(source, throwErrors, aliases);
   tokenizer.run();
   return tokenizer.tokens;
 }
@@ -27,6 +27,7 @@ enum CharType {
 class Tokenizer {
   constructor(
     source: string,
+    readonly throwErrors: boolean,
     readonly aliases?: Aliases
   ) {
     this._source = source;
@@ -38,8 +39,10 @@ class Tokenizer {
       this._next();
     }
 
-    if (this._endQuote !== '') {
-      throw new Error('Tokenize error, expected end quote ' + this._endQuote);
+    if (this.throwErrors) {
+      if (this._endQuote !== '') {
+        throw new Error('Tokenize error, expected end quote ' + this._endQuote);
+      }
     }
   }
 

src/utils.ts

@@ -0,0 +1,57 @@
+/**
+ * Find the longest string that starts all of the supplied strings.
+ * startIndex is the index to start at, if you already know that the first startIndex characters
+ * are identical.
+ */
+export function longestStartsWith(strings: string[], startIndex: number = 0): string {
+  if (strings.length < 1) {
+    return '';
+  }
+  const minLength = Math.min(...strings.map(str => str.length));
+  const toMatch = strings[0];
+  let index = startIndex;
+  while (index < minLength && strings.every(str => str[index] === toMatch[index])) {
+    index++;
+  }
+  return toMatch.slice(0, index);
+}
+
+/**
+ * Arrange an array of strings into columns that fit within a columnWidth.
+ * Each column is the same width.
+ * Return an array of lines.
+ */
+export function toColumns(strings: string[], columnWidth: number): string[] {
+  if (strings.length < 1) {
+    return [];
+  }
+
+  if (columnWidth < 1) {
+    // If don't know number of columns, use a single line which will be too long.
+    return [strings.join('  ')];
+  }
+
+  const nstrings = strings.length;
+  const gap = 2;
+  const maxLength = Math.max(...strings.map(str => str.length));
+  const ncols = Math.min(Math.floor(columnWidth / (maxLength + gap)), nstrings);
+  const nrows = Math.ceil(nstrings / ncols);
+
+  const lines = [];
+  for (let row = 0; row < nrows; ++row) {
+    let line = '';
+    for (let col = 0; col < ncols; ++col) {
+      const index = col * nrows + row;
+      if (index >= nstrings) {
+        continue;
+      }
+      const str = strings[index];
+      line += str;
+      if (index + nrows < nstrings) {
+        line += ' '.repeat(maxLength + gap - str.length);
+      }
+    }
+    lines.push(line);
+  }
+  return lines;
+}

tests/parse.test.ts

@@ -88,13 +88,13 @@ describe('parse', () => {
 
   it('should use aliases', () => {
     const aliases = new Aliases();
-    expect(parse('ll', aliases)).toEqual([
+    expect(parse('ll', true, aliases)).toEqual([
       new CommandNode({ offset: 0, value: 'ls' }, [
         { offset: 3, value: '--color=auto' },
         { offset: 16, value: '-lF' }
       ])
     ]);
-    expect(parse(' ll;', aliases)).toEqual([
+    expect(parse(' ll;', true, aliases)).toEqual([
       new CommandNode({ offset: 1, value: 'ls' }, [
         { offset: 4, value: '--color=auto' },
         { offset: 17, value: '-lF' }

tests/shell.test.ts

@@ -76,7 +76,7 @@ describe('Shell', () => {
     });
   });
 
-  describe('input tab complete', () => {
+  describe('input tab complete commands', () => {
     it('should complete ec', async () => {
       const { shell, output } = await shell_setup_empty();
       await shell.inputs(['e', 'c', '\t']);
@@ -107,10 +107,32 @@ describe('Shell', () => {
       expect(output.text).toEqual('unk');
     });
 
+    it('should arrange in columns', async () => {
+      const { shell, output } = await shell_setup_empty();
+      await shell.setSize(40, 10);
+      await shell.inputs(['t', '\t']);
+      expect(output.text).toMatch(/^t\r\ntail\r\ntouch\r\ntr\r\ntty\r\n/);
+      output.clear();
+
+      await shell.setSize(40, 20);
+      await shell.inputs(['\t']);
+      expect(output.text).toMatch(/^\r\ntail   tr\r\ntouch  tty\r\n/);
+    });
+
+    it('should add common startsWith', async () => {
+      const { shell, output } = await shell_setup_empty();
+      await shell.inputs(['s', 'h', '\t']);
+      expect(output.text).toEqual('sha');
+      output.clear();
+
+      await shell.inputs(['\t']);
+      expect(output.text).toMatch(/sha1sum  sha224sum  sha256sum  sha384sum  sha512sum/);
+    });
+
     it('should include aliases', async () => {
       const { shell, output } = await shell_setup_empty();
       await shell.inputs(['l', '\t']);
-      expect(output.text).toMatch(/^l\r\nln  logname  ls  ll/);
+      expect(output.text).toMatch(/^l\r\nll  ln  logname  ls/);
     });
   });
 

tests/tokenize.test.ts

@@ -131,30 +131,30 @@ describe('Tokenize', () => {
 
   it('should use aliases', () => {
     const aliases = new Aliases();
-    expect(tokenize('ll', aliases)).toEqual([
+    expect(tokenize('ll', true, aliases)).toEqual([
       { offset: 0, value: 'ls' },
       { offset: 3, value: '--color=auto' },
       { offset: 16, value: '-lF' }
     ]);
-    expect(tokenize('ll;', aliases)).toEqual([
+    expect(tokenize('ll;', true, aliases)).toEqual([
       { offset: 0, value: 'ls' },
       { offset: 3, value: '--color=auto' },
       { offset: 16, value: '-lF' },
       { offset: 19, value: ';' }
     ]);
-    expect(tokenize(' ll ', aliases)).toEqual([
+    expect(tokenize(' ll ', true, aliases)).toEqual([
       { offset: 1, value: 'ls' },
       { offset: 4, value: '--color=auto' },
       { offset: 17, value: '-lF' }
     ]);
-    expect(tokenize('cat; ll', aliases)).toEqual([
+    expect(tokenize('cat; ll', true, aliases)).toEqual([
       { offset: 0, value: 'cat' },
       { offset: 3, value: ';' },
       { offset: 5, value: 'ls' },
       { offset: 8, value: '--color=auto' },
       { offset: 21, value: '-lF' }
     ]);
-    expect(tokenize('ll; cat', aliases)).toEqual([
+    expect(tokenize('ll; cat', true, aliases)).toEqual([
       { offset: 0, value: 'ls' },
       { offset: 3, value: '--color=auto' },
       { offset: 16, value: '-lF' },