From f89b15e3dfa4f0c05ae84aefd5f1ed7372d769dc Mon Sep 17 00:00:00 2001 From: Ian Thomas Date: Mon, 22 Jul 2024 14:19:04 +0100 Subject: [PATCH] Support quotes in command line --- package-lock.json | 118 ++++++++++++++++++------------------- src/tokenize.ts | 107 +++++++++++++++++++++++---------- tests/commands/env.test.ts | 9 +++ tests/parse.test.ts | 15 +++++ tests/shell.test.ts | 6 ++ tests/tokenize.test.ts | 49 +++++++++++++++ 6 files changed, 213 insertions(+), 91 deletions(-) diff --git a/package-lock.json b/package-lock.json index 859089a..996c110 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1449,12 +1449,12 @@ } }, "node_modules/@playwright/test": { - "version": "1.45.2", - "resolved": "https://registry.npmjs.org/@playwright/test/-/test-1.45.2.tgz", - "integrity": "sha512-JxG9eq92ET75EbVi3s+4sYbcG7q72ECeZNbdBlaMkGcNbiDQ4cAi8U2QP5oKkOx+1gpaiL1LDStmzCaEM1Z6fQ==", + "version": "1.45.3", + "resolved": "https://registry.npmjs.org/@playwright/test/-/test-1.45.3.tgz", + "integrity": "sha512-UKF4XsBfy+u3MFWEH44hva1Q8Da28G6RFtR2+5saw+jgAFQV5yYnB1fu68Mz7fO+5GJF3wgwAIs0UelU8TxFrA==", "dev": true, "dependencies": { - "playwright": "1.45.2" + "playwright": "1.45.3" }, "bin": { "playwright": "cli.js" @@ -1673,16 +1673,16 @@ "dev": true }, "node_modules/@typescript-eslint/eslint-plugin": { - "version": "7.16.1", - "resolved": "https://registry.npmjs.org/@typescript-eslint/eslint-plugin/-/eslint-plugin-7.16.1.tgz", - "integrity": "sha512-SxdPak/5bO0EnGktV05+Hq8oatjAYVY3Zh2bye9pGZy6+jwyR3LG3YKkV4YatlsgqXP28BTeVm9pqwJM96vf2A==", + "version": "7.17.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/eslint-plugin/-/eslint-plugin-7.17.0.tgz", + "integrity": "sha512-pyiDhEuLM3PuANxH7uNYan1AaFs5XE0zw1hq69JBvGvE7gSuEoQl1ydtEe/XQeoC3GQxLXyOVa5kNOATgM638A==", "dev": true, "dependencies": { "@eslint-community/regexpp": "^4.10.0", - "@typescript-eslint/scope-manager": "7.16.1", - "@typescript-eslint/type-utils": "7.16.1", - "@typescript-eslint/utils": "7.16.1", - "@typescript-eslint/visitor-keys": "7.16.1", + "@typescript-eslint/scope-manager": "7.17.0", + "@typescript-eslint/type-utils": "7.17.0", + "@typescript-eslint/utils": "7.17.0", + "@typescript-eslint/visitor-keys": "7.17.0", "graphemer": "^1.4.0", "ignore": "^5.3.1", "natural-compare": "^1.4.0", @@ -1706,15 +1706,15 @@ } }, "node_modules/@typescript-eslint/parser": { - "version": "7.16.1", - "resolved": "https://registry.npmjs.org/@typescript-eslint/parser/-/parser-7.16.1.tgz", - "integrity": "sha512-u+1Qx86jfGQ5i4JjK33/FnawZRpsLxRnKzGE6EABZ40KxVT/vWsiZFEBBHjFOljmmV3MBYOHEKi0Jm9hbAOClA==", + "version": "7.17.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/parser/-/parser-7.17.0.tgz", + "integrity": "sha512-puiYfGeg5Ydop8eusb/Hy1k7QmOU6X3nvsqCgzrB2K4qMavK//21+PzNE8qeECgNOIoertJPUC1SpegHDI515A==", "dev": true, "dependencies": { - "@typescript-eslint/scope-manager": "7.16.1", - "@typescript-eslint/types": "7.16.1", - "@typescript-eslint/typescript-estree": "7.16.1", - "@typescript-eslint/visitor-keys": "7.16.1", + "@typescript-eslint/scope-manager": "7.17.0", + "@typescript-eslint/types": "7.17.0", + "@typescript-eslint/typescript-estree": "7.17.0", + "@typescript-eslint/visitor-keys": "7.17.0", "debug": "^4.3.4" }, "engines": { @@ -1734,13 +1734,13 @@ } }, "node_modules/@typescript-eslint/scope-manager": { - "version": "7.16.1", - "resolved": "https://registry.npmjs.org/@typescript-eslint/scope-manager/-/scope-manager-7.16.1.tgz", - "integrity": "sha512-nYpyv6ALte18gbMz323RM+vpFpTjfNdyakbf3nsLvF43uF9KeNC289SUEW3QLZ1xPtyINJ1dIsZOuWuSRIWygw==", + "version": "7.17.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/scope-manager/-/scope-manager-7.17.0.tgz", + "integrity": "sha512-0P2jTTqyxWp9HiKLu/Vemr2Rg1Xb5B7uHItdVZ6iAenXmPo4SZ86yOPCJwMqpCyaMiEHTNqizHfsbmCFT1x9SA==", "dev": true, "dependencies": { - "@typescript-eslint/types": "7.16.1", - "@typescript-eslint/visitor-keys": "7.16.1" + "@typescript-eslint/types": "7.17.0", + "@typescript-eslint/visitor-keys": "7.17.0" }, "engines": { "node": "^18.18.0 || >=20.0.0" @@ -1751,13 +1751,13 @@ } }, "node_modules/@typescript-eslint/type-utils": { - "version": "7.16.1", - "resolved": "https://registry.npmjs.org/@typescript-eslint/type-utils/-/type-utils-7.16.1.tgz", - "integrity": "sha512-rbu/H2MWXN4SkjIIyWcmYBjlp55VT+1G3duFOIukTNFxr9PI35pLc2ydwAfejCEitCv4uztA07q0QWanOHC7dA==", + "version": "7.17.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/type-utils/-/type-utils-7.17.0.tgz", + "integrity": "sha512-XD3aaBt+orgkM/7Cei0XNEm1vwUxQ958AOLALzPlbPqb8C1G8PZK85tND7Jpe69Wualri81PLU+Zc48GVKIMMA==", "dev": true, "dependencies": { - "@typescript-eslint/typescript-estree": "7.16.1", - "@typescript-eslint/utils": "7.16.1", + "@typescript-eslint/typescript-estree": "7.17.0", + "@typescript-eslint/utils": "7.17.0", "debug": "^4.3.4", "ts-api-utils": "^1.3.0" }, @@ -1778,9 +1778,9 @@ } }, "node_modules/@typescript-eslint/types": { - "version": "7.16.1", - "resolved": "https://registry.npmjs.org/@typescript-eslint/types/-/types-7.16.1.tgz", - "integrity": "sha512-AQn9XqCzUXd4bAVEsAXM/Izk11Wx2u4H3BAfQVhSfzfDOm/wAON9nP7J5rpkCxts7E5TELmN845xTUCQrD1xIQ==", + "version": "7.17.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/types/-/types-7.17.0.tgz", + "integrity": "sha512-a29Ir0EbyKTKHnZWbNsrc/gqfIBqYPwj3F2M+jWE/9bqfEHg0AMtXzkbUkOG6QgEScxh2+Pz9OXe11jHDnHR7A==", "dev": true, "engines": { "node": "^18.18.0 || >=20.0.0" @@ -1791,13 +1791,13 @@ } }, "node_modules/@typescript-eslint/typescript-estree": { - "version": "7.16.1", - "resolved": "https://registry.npmjs.org/@typescript-eslint/typescript-estree/-/typescript-estree-7.16.1.tgz", - "integrity": "sha512-0vFPk8tMjj6apaAZ1HlwM8w7jbghC8jc1aRNJG5vN8Ym5miyhTQGMqU++kuBFDNKe9NcPeZ6x0zfSzV8xC1UlQ==", + "version": "7.17.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/typescript-estree/-/typescript-estree-7.17.0.tgz", + "integrity": "sha512-72I3TGq93t2GoSBWI093wmKo0n6/b7O4j9o8U+f65TVD0FS6bI2180X5eGEr8MA8PhKMvYe9myZJquUT2JkCZw==", "dev": true, "dependencies": { - "@typescript-eslint/types": "7.16.1", - "@typescript-eslint/visitor-keys": "7.16.1", + "@typescript-eslint/types": "7.17.0", + "@typescript-eslint/visitor-keys": "7.17.0", "debug": "^4.3.4", "globby": "^11.1.0", "is-glob": "^4.0.3", @@ -1819,15 +1819,15 @@ } }, "node_modules/@typescript-eslint/utils": { - "version": "7.16.1", - "resolved": "https://registry.npmjs.org/@typescript-eslint/utils/-/utils-7.16.1.tgz", - "integrity": "sha512-WrFM8nzCowV0he0RlkotGDujx78xudsxnGMBHI88l5J8wEhED6yBwaSLP99ygfrzAjsQvcYQ94quDwI0d7E1fA==", + "version": "7.17.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/utils/-/utils-7.17.0.tgz", + "integrity": "sha512-r+JFlm5NdB+JXc7aWWZ3fKSm1gn0pkswEwIYsrGPdsT2GjsRATAKXiNtp3vgAAO1xZhX8alIOEQnNMl3kbTgJw==", "dev": true, "dependencies": { "@eslint-community/eslint-utils": "^4.4.0", - "@typescript-eslint/scope-manager": "7.16.1", - "@typescript-eslint/types": "7.16.1", - "@typescript-eslint/typescript-estree": "7.16.1" + "@typescript-eslint/scope-manager": "7.17.0", + "@typescript-eslint/types": "7.17.0", + "@typescript-eslint/typescript-estree": "7.17.0" }, "engines": { "node": "^18.18.0 || >=20.0.0" @@ -1841,12 +1841,12 @@ } }, "node_modules/@typescript-eslint/visitor-keys": { - "version": "7.16.1", - "resolved": "https://registry.npmjs.org/@typescript-eslint/visitor-keys/-/visitor-keys-7.16.1.tgz", - "integrity": "sha512-Qlzzx4sE4u3FsHTPQAAQFJFNOuqtuY0LFrZHwQ8IHK705XxBiWOFkfKRWu6niB7hwfgnwIpO4jTC75ozW1PHWg==", + "version": "7.17.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/visitor-keys/-/visitor-keys-7.17.0.tgz", + "integrity": "sha512-RVGC9UhPOCsfCdI9pU++K4nD7to+jTcMIbXTSOcrLqUEW6gF2pU1UUbYJKc9cvcRSK1UDeMJ7pdMxf4bhMpV/A==", "dev": true, "dependencies": { - "@typescript-eslint/types": "7.16.1", + "@typescript-eslint/types": "7.17.0", "eslint-visitor-keys": "^3.4.3" }, "engines": { @@ -2621,9 +2621,9 @@ } }, "node_modules/electron-to-chromium": { - "version": "1.4.832", - "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.4.832.tgz", - "integrity": "sha512-cTen3SB0H2SGU7x467NRe1eVcQgcuS6jckKfWJHia2eo0cHIGOqHoAxevIYZD4eRHcWjkvFzo93bi3vJ9W+1lA==", + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.5.0.tgz", + "integrity": "sha512-Vb3xHHYnLseK8vlMJQKJYXJ++t4u1/qJ3vykuVrVjvdiOEhYyT1AuP4x03G8EnPmYvYOhe9T+dADTmthjRQMkA==", "dev": true }, "node_modules/emittery": { @@ -5015,12 +5015,12 @@ } }, "node_modules/playwright": { - "version": "1.45.2", - "resolved": "https://registry.npmjs.org/playwright/-/playwright-1.45.2.tgz", - "integrity": "sha512-ReywF2t/0teRvNBpfIgh5e4wnrI/8Su8ssdo5XsQKpjxJj+jspm00jSoz9BTg91TT0c9HRjXO7LBNVrgYj9X0g==", + "version": "1.45.3", + "resolved": "https://registry.npmjs.org/playwright/-/playwright-1.45.3.tgz", + "integrity": "sha512-QhVaS+lpluxCaioejDZ95l4Y4jSFCsBvl2UZkpeXlzxmqS+aABr5c82YmfMHrL6x27nvrvykJAFpkzT2eWdJww==", "dev": true, "dependencies": { - "playwright-core": "1.45.2" + "playwright-core": "1.45.3" }, "bin": { "playwright": "cli.js" @@ -5033,9 +5033,9 @@ } }, "node_modules/playwright-core": { - "version": "1.45.2", - "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.45.2.tgz", - "integrity": "sha512-ha175tAWb0dTK0X4orvBIqi3jGEt701SMxMhyujxNrgd8K0Uy5wMSwwcQHtyB4om7INUkfndx02XnQ2p6dvLDw==", + "version": "1.45.3", + "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.45.3.tgz", + "integrity": "sha512-+ym0jNbcjikaOwwSZycFbwkWgfruWvYlJfThKYAlImbxUgdWFO2oW70ojPm4OpE4t6TAo2FY/smM+hpVTtkhDA==", "dev": true, "bin": { "playwright-core": "cli.js" @@ -5762,9 +5762,9 @@ } }, "node_modules/typescript": { - "version": "5.5.3", - "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.5.3.tgz", - "integrity": "sha512-/hreyEujaB0w76zKo6717l3L0o/qEUtRgdvUBvlkhoWeOVMjMuHNHk0BRBzikzuGDqNmPQbg5ifMEqsHLiIUcQ==", + "version": "5.5.4", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.5.4.tgz", + "integrity": "sha512-Mtq29sKDAEYP7aljRgtPOpTvOfbwRWlS6dPRzwjdE+C0R4brX/GUyhHSecbHMFLNBLcJIPt9nl9yG5TZ1weH+Q==", "dev": true, "bin": { "tsc": "bin/tsc", diff --git a/src/tokenize.ts b/src/tokenize.ts index 90bf91c..1e72f67 100644 --- a/src/tokenize.ts +++ b/src/tokenize.ts @@ -18,18 +18,12 @@ export function tokenize(source: string, aliases?: Aliases): Token[] { enum CharType { None, Delimiter, + DoubleQuote, + SingleQuote, Whitespace, Other } -class State { - prevChar: string = ''; - prevCharType: CharType = CharType.None; - index: number = -1; // Index into source string. - offset: number = -1; // Offset of start of current token, -1 if not in token. - aliasOffset: number = -1; -} - class Tokenizer { constructor( source: string, @@ -37,21 +31,27 @@ class Tokenizer { ) { this._source = source; this._tokens = []; - this._state = new State(); } run() { - while (this._state.index <= this._source.length) { + while (this._index <= this._source.length) { this._next(); } + + if (this._endQuote !== '') { + throw new Error('Tokenize error, expected end quote ' + this._endQuote); + } } get tokens(): Token[] { return this._tokens; } - private _addToken(offset: number, value: string): boolean { - if (this.aliases !== undefined && offset !== this._state.aliasOffset) { + private _addToken(): boolean { + const offset = this._offset; + const value = this._value; + + if (this.aliases !== undefined && offset !== this._aliasOffset) { const isCommand = this._tokens.length === 0 || ';&|'.includes(this._tokens.at(-1)!.value.at(-1)!); @@ -60,18 +60,21 @@ class Tokenizer { if (alias !== undefined) { // Replace token with its alias and set state to beginning of it to re-tokenize. const n = value.length; - this._state.offset = -1; - this._state.index = offset - 1; - this._state.aliasOffset = offset; // Do not attempt to alias this token again. + this._offset = -1; + this._index = offset - 1; + this._aliasOffset = offset; // Do not attempt to alias this token again. this._source = this._source.slice(0, offset) + alias + this._source.slice(offset + n); - this._state.prevChar = ''; - this._state.prevCharType = CharType.None; + this._prevChar = ''; + this._prevCharType = CharType.None; + this._value = ''; + this._endQuote = ''; return false; } } } this._tokens.push({ offset, value }); + this._endQuote = ''; return true; } @@ -80,44 +83,84 @@ class Tokenizer { return CharType.Whitespace; } else if (delimiters.includes(char)) { return CharType.Delimiter; + } else if (char === "'") { + return CharType.SingleQuote; + } else if (char === '"') { + return CharType.DoubleQuote; } else { return CharType.Other; } } - private _next() { - const i = ++this._state.index; + private _endQuoteFromCharType(charType: CharType): string { + if (charType === CharType.DoubleQuote) { + return '"'; + } else if (charType === CharType.SingleQuote) { + return "'"; + } else { + return ''; + } + } + private _next() { + const i = ++this._index; const char = i < this._source.length ? this._source[i] : ' '; - const charType = this._getCharType(char); - if (this._state.offset >= 0) { + let charType = this._getCharType(char); + const endQuote = this._endQuoteFromCharType(charType); + + if (this._offset >= 0) { // In token. - if (charType === CharType.Whitespace) { + if (this._endQuote) { + // In quoted section, continue until reach end quote. + if (char !== this._endQuote) { + this._value += char; + } else { + this._endQuote = ''; + charType = CharType.Other; + } + } else if (endQuote) { + // Start quoted section within current token. + this._endQuote = endQuote; + } else if (charType === CharType.Whitespace) { // Finish current token. - if (this._addToken(this._state.offset, this._source.slice(this._state.offset, i))) { - this._state.offset = -1; + if (this._addToken()) { + this._offset = -1; } } else if ( - charType !== this._state.prevCharType || - (charType === CharType.Delimiter && char !== this._state.prevChar) + charType !== this._prevCharType || + (charType === CharType.Delimiter && char !== this._prevChar) ) { // Finish current token and start new one. - if (this._addToken(this._state.offset, this._source.slice(this._state.offset, i))) { - this._state.offset = i; + if (this._addToken()) { + this._offset = i; + this._value = char; } + } else { + // Continue in current token. + this._value += char; } } else { // Not in token. if (charType !== CharType.Whitespace) { // Start new token. - this._state.offset = i; + this._offset = i; + this._endQuote = this._endQuoteFromCharType(charType); + this._value = this._endQuote === '' ? char : ''; } } - this._state.prevChar = char; - this._state.prevCharType = charType; + this._prevChar = char; + this._prevCharType = charType; } private _source: string; private _tokens: Token[]; - private _state: State; + + // Tokenizer state. + private _prevChar: string = ''; + private _prevCharType: CharType = CharType.None; + private _index: number = -1; // Index into source string. + private _offset: number = -1; // Offset of start of current token, -1 if not in token. + private _aliasOffset: number = -1; + private _value: string = ''; // Current token. + private _endQuote: string = ''; // End quote if in quoted section, otherwise emptry string. } diff --git a/tests/commands/env.test.ts b/tests/commands/env.test.ts index d209037..f9deb89 100644 --- a/tests/commands/env.test.ts +++ b/tests/commands/env.test.ts @@ -10,4 +10,13 @@ describe('env command', () => { expect(environment.get('MYENV')).toBeUndefined(); expect(output.text.trim().split('\r\n').at(-1)).toEqual('MYENV=23'); }); + + it('should support quotes', async () => { + const { shell, output } = await shell_setup_simple(); + const { environment } = shell; + + await shell._runCommands('env MYENV="ls -alF"'); + expect(environment.get('MYENV')).toBeUndefined(); + expect(output.text.trim().split('\r\n').at(-1)).toEqual('MYENV=ls -alF'); + }); }); diff --git a/tests/parse.test.ts b/tests/parse.test.ts index bde48d5..510b167 100644 --- a/tests/parse.test.ts +++ b/tests/parse.test.ts @@ -101,4 +101,19 @@ describe('parse', () => { ]) ]); }); + + it('should support quotes', () => { + expect(parse('alias ll="ls -lF"')).toEqual([ + new CommandNode({ offset: 0, value: 'alias' }, [{ offset: 6, value: 'll=ls -lF' }]) + ]); + expect(parse('alias ll="ls ""-lF"')).toEqual([ + new CommandNode({ offset: 0, value: 'alias' }, [{ offset: 6, value: 'll=ls -lF' }]) + ]); + expect(parse('lua -e "A=3;B=9"')).toEqual([ + new CommandNode({ offset: 0, value: 'lua' }, [ + { offset: 4, value: '-e' }, + { offset: 7, value: 'A=3;B=9' } + ]) + ]); + }); }); diff --git a/tests/shell.test.ts b/tests/shell.test.ts index f7703e7..746eb82 100644 --- a/tests/shell.test.ts +++ b/tests/shell.test.ts @@ -54,6 +54,12 @@ describe('Shell', () => { expect(mockStdin.enableCallCount).toEqual(1); expect(mockStdin.disableCallCount).toEqual(1); }); + + it('should support quotes', async () => { + const { shell, output } = await shell_setup_empty(); + await shell._runCommands('echo "Hello x; yz"'); + expect(output.text).toEqual('Hello x; yz\r\n'); + }); }); describe('input', () => { diff --git a/tests/tokenize.test.ts b/tests/tokenize.test.ts index 1b6ab86..5015fc4 100644 --- a/tests/tokenize.test.ts +++ b/tests/tokenize.test.ts @@ -162,4 +162,53 @@ describe('Tokenize', () => { { offset: 21, value: 'cat' } ]); }); + + describe('quote handling', () => { + it('should support matching single and double quotes', () => { + expect(tokenize("'ls -l'")).toEqual([{ offset: 0, value: 'ls -l' }]); + expect(tokenize('"ls -l"')).toEqual([{ offset: 0, value: 'ls -l' }]); + }); + + it('should throw if end quotes missing', () => { + expect(() => tokenize('"ls')).toThrow(); + expect(() => tokenize("'ls")).toThrow(); + }); + + it('should work next to whitespace', () => { + expect(tokenize('ls "a b"')).toEqual([ + { offset: 0, value: 'ls' }, + { offset: 3, value: 'a b' } + ]); + expect(tokenize('"a b" ls')).toEqual([ + { offset: 0, value: 'a b' }, + { offset: 6, value: 'ls' } + ]); + }); + + it('should support containing the other quote type', () => { + expect(tokenize('"xy\'s"')).toEqual([{ offset: 0, value: "xy's" }]); + expect(tokenize("'xy\"s'")).toEqual([{ offset: 0, value: 'xy"s' }]); + }); + + it('should join adjacent quoted sections', () => { + expect(tokenize('"ls -l""h"')).toEqual([{ offset: 0, value: 'ls -lh' }]); + }); + + it('should join a preceding non-quoted section', () => { + expect(tokenize('ABC="ab c"')).toEqual([{ offset: 0, value: 'ABC=ab c' }]); + }); + + it('should join a following non-quoted section', () => { + expect(tokenize('"ab c"d')).toEqual([{ offset: 0, value: 'ab cd' }]); + }); + + it('should support a complicated example', () => { + expect(tokenize('lua -e "A=3; B=7" -v')).toEqual([ + { offset: 0, value: 'lua' }, + { offset: 4, value: '-e' }, + { offset: 7, value: 'A=3; B=7' }, + { offset: 18, value: '-v' } + ]); + }); + }); });