Skip to content

Commit

Permalink
feat: variable string substitution functions (#148)
Browse files Browse the repository at this point in the history 
* start variable string substitution functions

* Minor fixes along with partial support for variable default to another expression

* Small fixes

* Added WordResult

* Added state modification

* Mutate state where needed

* Fix bugs with tests

* replaced variable with variable expansion in quoted words

* Add state modification in arithmetic ops

* Remove extra comment and remove test script

* Removed extra prints

* Added tests for default value expansion

* Added support for default value assign with tests

* Added support for alternate value expansion with tests

* Add tmate for debugging

* Updated behaviour of shell vars on Windows

* run fmt

* Remove tmate

* Added tests for lowercase shell vars

* Added support for arithmetic exprs and negative values in substring expansion

* Disable codecov comments

* Use miette::bail! more often

* Add double quoted tests

---------

Co-authored-by: prsabahrami <[email protected]>
Co-authored-by: Julian Hofer <[email protected]>
  • Loading branch information
@wolfv @prsabahrami @Hofer-Julian
3 people authored Oct 18, 2024
1 parent 9ecd2c3 commit cc84630
Show file tree
Hide file tree
Showing 6 changed files with 1,039 additions and 245 deletions.
91 changes: 67 additions & 24 deletions crates/deno_task_shell/src/grammar.pest
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ UNQUOTED_PENDING_WORD = ${
UNQUOTED_ESCAPE_CHAR |
"$" ~ ARITHMETIC_EXPRESSION |
SUB_COMMAND |
("$" ~ "{" ~ VARIABLE ~ "}" | "$" ~ VARIABLE) |
VARIABLE_EXPANSION |
UNQUOTED_CHAR |
QUOTED_WORD
))*)
Expand All @@ -25,55 +25,98 @@ UNQUOTED_PENDING_WORD = ${
UNQUOTED_ESCAPE_CHAR |
"$" ~ ARITHMETIC_EXPRESSION |
SUB_COMMAND |
("$" ~ "{" ~ VARIABLE ~ "}" | "$" ~ VARIABLE) |
VARIABLE_EXPANSION |
UNQUOTED_CHAR |
QUOTED_WORD
))+
}

TILDE_PREFIX = ${
"~" ~ (!(OPERATOR | WHITESPACE | NEWLINE | "/") ~ (
(!("\"" | "'" | "$" | "\\" | "/") ~ ANY)
))*
}
QUOTED_PENDING_WORD = ${ (
EXIT_STATUS |
QUOTED_ESCAPE_CHAR |
"$" ~ ARITHMETIC_EXPRESSION |
SUB_COMMAND |
VARIABLE_EXPANSION |
QUOTED_CHAR
)* }

ASSIGNMENT_TILDE_PREFIX = ${
"~" ~ (!(OPERATOR | WHITESPACE | NEWLINE | "/" | ":") ~
(!("\"" | "'" | "$" | "\\" | "/") ~ ANY)
)*
}
PARAMETER_PENDING_WORD = ${
TILDE_PREFIX ~ ( !"}" ~ !":" ~ (
EXIT_STATUS |
PARAMETER_ESCAPE_CHAR |
"$" ~ ARITHMETIC_EXPRESSION |
SUB_COMMAND |
VARIABLE_EXPANSION |
QUOTED_WORD |
QUOTED_CHAR
))* |
( !"}" ~ !":" ~ (
EXIT_STATUS |
PARAMETER_ESCAPE_CHAR |
"$" ~ ARITHMETIC_EXPRESSION |
SUB_COMMAND |
VARIABLE_EXPANSION |
QUOTED_WORD |
QUOTED_CHAR
))+
}

FILE_NAME_PENDING_WORD = ${
(TILDE_PREFIX ~ (!(WHITESPACE | OPERATOR | NEWLINE) ~ (
UNQUOTED_ESCAPE_CHAR |
("$" ~ VARIABLE) |
VARIABLE_EXPANSION |
UNQUOTED_CHAR |
QUOTED_WORD
))*)
|
(!(WHITESPACE | OPERATOR | NEWLINE) ~ (
UNQUOTED_ESCAPE_CHAR |
("$" ~ VARIABLE) |
VARIABLE_EXPANSION |
UNQUOTED_CHAR |
QUOTED_WORD
))+
}

QUOTED_PENDING_WORD = ${ (
EXIT_STATUS |
QUOTED_ESCAPE_CHAR |
SUB_COMMAND |
("$" ~ "{" ~ VARIABLE ~ "}" | "$" ~ VARIABLE) |
QUOTED_CHAR
)* }

UNQUOTED_ESCAPE_CHAR = ${ ("\\" ~ "$" | "$" ~ !"(" ~ !"{" ~ !VARIABLE) | "\\" ~ (" " | "`" | "\"" | "(" | ")")* }
QUOTED_ESCAPE_CHAR = ${ "\\" ~ "$" | "$" ~ !"(" ~ !"{" ~ !VARIABLE | "\\" ~ ("`" | "\"" | "(" | ")" | "'")* }
UNQUOTED_ESCAPE_CHAR = ${ ("\\" ~ "$" | "$" ~ !"(" ~ !"{" ~ !VARIABLE) | "\\" ~ (" " | "`" | "\"" | "(" | ")") }
QUOTED_ESCAPE_CHAR = ${ "\\" ~ "$" | "$" ~ !"(" ~ !"{" ~ !VARIABLE | "\\" ~ ("`" | "\"" | "(" | ")" | "'") }
PARAMETER_ESCAPE_CHAR = ${ "\\" ~ "$" | "$" ~ !"(" ~ !"{" ~ !VARIABLE | "\\" ~ "}" }

UNQUOTED_CHAR = ${ ("\\" ~ " ") | !("]]" | "[[" | "(" | ")" | "<" | ">" | "|" | "&" | ";" | "\"" | "'" | "$") ~ ANY }
QUOTED_CHAR = ${ !"\"" ~ ANY }

VARIABLE_EXPANSION = ${
"$" ~ (
"{" ~ VARIABLE ~ VARIABLE_MODIFIER? ~ "}" |
VARIABLE
)
}

VARIABLE = ${ (ASCII_ALPHA | "_") ~ (ASCII_ALPHANUMERIC | "_")* }

VARIABLE_MODIFIER = _{
VAR_DEFAULT_VALUE |
VAR_ASSIGN_DEFAULT |
VAR_ALTERNATE_VALUE |
VAR_SUBSTRING
}

VAR_DEFAULT_VALUE = !{ ":-" ~ PARAMETER_PENDING_WORD? }
VAR_ASSIGN_DEFAULT = !{ ":=" ~ PARAMETER_PENDING_WORD }
VAR_ALTERNATE_VALUE = !{ ":+" ~ PARAMETER_PENDING_WORD }
VAR_SUBSTRING = !{ ":" ~ PARAMETER_PENDING_WORD ~ (":" ~ PARAMETER_PENDING_WORD)? }

TILDE_PREFIX = ${
"~" ~ (!(OPERATOR | WHITESPACE | NEWLINE | "/") ~ (
(!("\"" | "'" | "$" | "\\" | "/") ~ ANY)
))*
}

ASSIGNMENT_TILDE_PREFIX = ${
"~" ~ (!(OPERATOR | WHITESPACE | NEWLINE | "/" | ":") ~
(!("\"" | "'" | "$" | "\\" | "/") ~ ANY)
)*
}

SUB_COMMAND = { "$(" ~ complete_command ~ ")"}

DOUBLE_QUOTED = @{ "\"" ~ QUOTED_PENDING_WORD ~ "\"" }
Expand Down
154 changes: 142 additions & 12 deletions crates/deno_task_shell/src/parser.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -365,6 +365,24 @@ impl Word {
}
}

#[cfg_attr(feature = "serialization", derive(serde::Serialize))]
#[cfg_attr(
feature = "serialization",
serde(rename_all = "camelCase", tag = "kind", content = "value")
)]
#[derive(Debug, PartialEq, Eq, Clone, Error)]
#[error("Invalid variable modifier")]
pub enum VariableModifier {
#[error("Invalid substring")]
Substring {
begin: Word,
length: Option<Word>,
},
DefaultValue(Word),
AssignDefault(Word),
AlternateValue(Word),
}

#[cfg_attr(feature = "serialization", derive(serde::Serialize))]
#[cfg_attr(
feature = "serialization",
Expand All @@ -375,7 +393,7 @@ pub enum WordPart {
#[error("Invalid text")]
Text(String),
#[error("Invalid variable")]
Variable(String),
Variable(String, Option<Box<VariableModifier>>),
#[error("Invalid command")]
Command(SequentialList),
#[error("Invalid quoted string")]
Expand Down Expand Up @@ -1203,16 +1221,12 @@ fn parse_word(pair: Pair<Rule>) -> Result<Word> {
Rule::UNQUOTED_ESCAPE_CHAR => {
let mut chars = part.as_str().chars();
let mut escaped_char = String::new();
while let Some(c) = chars.next() {
if let Some(c) = chars.next() {
match c {
'\\' => {
let next_char = chars.next().unwrap_or('\0');
escaped_char.push(next_char);
}
'$' => {
escaped_char.push(c);
break;
}
_ => {
escaped_char.push(c);
break;
Expand All @@ -1230,8 +1244,9 @@ fn parse_word(pair: Pair<Rule>) -> Result<Word> {
parse_complete_command(part.into_inner().next().unwrap())?;
parts.push(WordPart::Command(command));
}
Rule::VARIABLE => {
parts.push(WordPart::Variable(part.as_str().to_string()))
Rule::VARIABLE_EXPANSION => {
let variable_expansion = parse_variable_expansion(part)?;
parts.push(variable_expansion);
}
Rule::QUOTED_WORD => {
let quoted = parse_quoted_word(part)?;
Expand Down Expand Up @@ -1273,7 +1288,7 @@ fn parse_word(pair: Pair<Rule>) -> Result<Word> {
}
}
Rule::VARIABLE => {
parts.push(WordPart::Variable(part.as_str().to_string()))
parts.push(WordPart::Variable(part.as_str().to_string(), None))
}
Rule::UNQUOTED_CHAR => {
if let Some(WordPart::Text(ref mut text)) = parts.last_mut() {
Expand Down Expand Up @@ -1303,6 +1318,62 @@ fn parse_word(pair: Pair<Rule>) -> Result<Word> {
}
}
}
Rule::PARAMETER_PENDING_WORD => {
for part in pair.into_inner() {
match part.as_rule() {
Rule::PARAMETER_ESCAPE_CHAR => {
let mut chars = part.as_str().chars();
let mut escaped_char = String::new();
if let Some(c) = chars.next() {
match c {
'\\' => {
let next_char = chars.next().unwrap_or('\0');
escaped_char.push(next_char);
}
_ => {
escaped_char.push(c);
break;
}
}
}
if let Some(WordPart::Text(ref mut text)) = parts.last_mut() {
text.push_str(&escaped_char);
} else {
parts.push(WordPart::Text(escaped_char));
}
}
Rule::VARIABLE_EXPANSION => {
let variable_expansion = parse_variable_expansion(part)?;
parts.push(variable_expansion);
}
Rule::QUOTED_WORD => {
let quoted = parse_quoted_word(part)?;
parts.push(quoted);
}
Rule::TILDE_PREFIX => {
let tilde_prefix = parse_tilde_prefix(part)?;
parts.push(tilde_prefix);
}
Rule::ARITHMETIC_EXPRESSION => {
let arithmetic_expression = parse_arithmetic_expression(part)?;
parts.push(WordPart::Arithmetic(arithmetic_expression));
}
Rule::QUOTED_CHAR => {
if let Some(WordPart::Text(ref mut s)) = parts.last_mut() {
s.push_str(part.as_str());
} else {
parts.push(WordPart::Text(part.as_str().to_string()));
}
}
_ => {
return Err(miette!(
"Unexpected rule in PARAMETER_PENDING_WORD: {:?}",
part.as_rule()
));
}
}
}
}
_ => {
return Err(miette!("Unexpected rule in word: {:?}", pair.as_rule()));
}
Expand Down Expand Up @@ -1473,6 +1544,64 @@ fn parse_post_arithmetic_op(pair: Pair<Rule>) -> Result<PostArithmeticOp> {
}
}

fn parse_variable_expansion(part: Pair<Rule>) -> Result<WordPart> {
let mut inner = part.into_inner();
let variable = inner
.next()
.ok_or_else(|| miette!("Expected variable name"))?;
let variable_name = variable.as_str().to_string();

let modifier = inner.next();
let parsed_modifier = if let Some(modifier) = modifier {
match modifier.as_rule() {
Rule::VAR_SUBSTRING => {
let mut numbers = modifier.into_inner();
let begin: Word = if let Some(n) = numbers.next() {
parse_word(n)?
} else {
return Err(miette!("Expected a number for substring begin"));
};

let length = if let Some(len_word) = numbers.next() {
Some(parse_word(len_word)?)
} else {
None
};
Some(Box::new(VariableModifier::Substring { begin, length }))
}
Rule::VAR_DEFAULT_VALUE => {
let value = if let Some(val) = modifier.into_inner().next() {
parse_word(val)?
} else {
Word::new_empty()
};
Some(Box::new(VariableModifier::DefaultValue(value)))
}
Rule::VAR_ASSIGN_DEFAULT => {
let value = modifier.into_inner().next().unwrap();
Some(Box::new(VariableModifier::AssignDefault(parse_word(
value,
)?)))
}
Rule::VAR_ALTERNATE_VALUE => {
let value = modifier.into_inner().next().unwrap();
Some(Box::new(VariableModifier::AlternateValue(parse_word(
value,
)?)))
}
_ => {
return Err(miette!(
"Unexpected rule in variable expansion modifier: {:?}",
modifier.as_rule()
));
}
}
} else {
None
};
Ok(WordPart::Variable(variable_name, parsed_modifier))
}

fn parse_tilde_prefix(pair: Pair<Rule>) -> Result<WordPart> {
let tilde_prefix_str = pair.as_str();
let user = if tilde_prefix_str.len() > 1 {
Expand Down Expand Up @@ -1506,8 +1635,9 @@ fn parse_quoted_word(pair: Pair<Rule>) -> Result<WordPart> {
parse_complete_command(part.into_inner().next().unwrap())?;
parts.push(WordPart::Command(command));
}
Rule::VARIABLE => {
parts.push(WordPart::Variable(part.as_str().to_string()))
Rule::VARIABLE_EXPANSION => {
let variable_expansion = parse_variable_expansion(part)?;
parts.push(variable_expansion);
}
Rule::QUOTED_CHAR => {
if let Some(WordPart::Text(ref mut s)) = parts.last_mut() {
Expand Down Expand Up @@ -1944,7 +2074,7 @@ mod test {
env_vars: vec![],
args: vec![
Word::new_word("echo"),
Word(vec![WordPart::Variable("MY_ENV".to_string())]),
Word(vec![WordPart::Variable("MY_ENV".to_string(), None)]),
],
}
.into(),
Expand Down
11 changes: 6 additions & 5 deletions crates/deno_task_shell/src/shell/command.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,8 +28,9 @@ pub fn execute_unresolved_command_name(
mut context: ShellCommandContext,
) -> FutureExecuteResult {
async move {
let args = context.args.clone();
let command =
match resolve_command(&command_name, &context, &context.args).await {
match resolve_command(&command_name, &mut context, &args).await {
Ok(command_path) => command_path,
Err(ResolveCommandError::CommandPath(err)) => {
let _ = context.stderr.write_line(&format!("{}", err));
Expand Down Expand Up @@ -108,7 +109,7 @@ impl FailedShebangError {

async fn resolve_command<'a>(
command_name: &UnresolvedCommandName,
context: &ShellCommandContext,
context: &mut ShellCommandContext,
original_args: &'a Vec<String>,
) -> Result<ResolvedCommand<'a>, ResolveCommandError> {
let command_path = match resolve_command_path(
Expand Down Expand Up @@ -160,10 +161,10 @@ async fn resolve_command<'a>(

async fn parse_shebang_args(
text: &str,
context: &ShellCommandContext,
context: &mut ShellCommandContext,
) -> Result<Vec<String>> {
fn err_unsupported(text: &str) -> Result<Vec<String>> {
miette::bail!("unsupported shebang. Please report this as a bug (https://github.com/denoland/deno).\n\nShebang: {}", text)
miette::bail!("unsupported shebang. Please report this as a bug (https://github.com/prefix.dev/shell).\n\nShebang: {}", text)
}

let mut args = crate::parser::parse(text)?;
Expand Down Expand Up @@ -204,7 +205,7 @@ async fn parse_shebang_args(

let result = super::execute::evaluate_args(
cmd.args,
&context.state,
&mut context.state,
context.stdin.clone(),
context.stderr.clone(),
)
Expand Down
Loading

0 comments on commit cc84630

Please sign in to comment.