Source code
Revision control
Copy as Markdown
Other Tools
// Copyright 2015 Nicholas Allegra (comex).
// copied, modified, or distributed except according to those terms.
//! [`Shlex`] and friends for byte strings.
//!
//! This is used internally by the [outer module](crate), and may be more
//! convenient if you are working with byte slices (`[u8]`) or types that are
//! wrappers around bytes, such as [`OsStr`](std::ffi::OsStr):
//!
//! ```rust
//! #[cfg(unix)] {
//! use shlex::bytes::quote;
//! use std::ffi::OsStr;
//! use std::os::unix::ffi::OsStrExt;
//!
//! // `\x80` is invalid in UTF-8.
//! let os_str = OsStr::from_bytes(b"a\x80b c");
//! assert_eq!(quote(os_str.as_bytes()), &b"'a\x80b c'"[..]);
//! }
//! ```
//!
//! (On Windows, `OsStr` uses 16 bit wide characters so this will not work.)
extern crate alloc;
use alloc::vec::Vec;
use alloc::borrow::Cow;
#[cfg(test)]
use alloc::vec;
#[cfg(test)]
use alloc::borrow::ToOwned;
#[cfg(all(doc, not(doctest)))]
use crate::{self as shlex, quoting_warning};
use super::QuoteError;
/// An iterator that takes an input byte string and splits it into the words using the same syntax as
/// the POSIX shell.
pub struct Shlex<'a> {
in_iter: core::slice::Iter<'a, u8>,
/// The number of newlines read so far, plus one.
pub line_no: usize,
/// An input string is erroneous if it ends while inside a quotation or right after an
/// unescaped backslash. Since Iterator does not have a mechanism to return an error, if that
/// happens, Shlex just throws out the last token, ends the iteration, and sets 'had_error' to
/// true; best to check it after you're done iterating.
pub had_error: bool,
}
impl<'a> Shlex<'a> {
pub fn new(in_bytes: &'a [u8]) -> Self {
Shlex {
in_iter: in_bytes.iter(),
line_no: 1,
had_error: false,
}
}
fn parse_word(&mut self, mut ch: u8) -> Option<Vec<u8>> {
let mut result: Vec<u8> = Vec::new();
loop {
match ch as char {
'"' => if let Err(()) = self.parse_double(&mut result) {
self.had_error = true;
return None;
},
'\'' => if let Err(()) = self.parse_single(&mut result) {
self.had_error = true;
return None;
},
'\\' => if let Some(ch2) = self.next_char() {
if ch2 != '\n' as u8 { result.push(ch2); }
} else {
self.had_error = true;
return None;
},
' ' | '\t' | '\n' => { break; },
_ => { result.push(ch as u8); },
}
if let Some(ch2) = self.next_char() { ch = ch2; } else { break; }
}
Some(result)
}
fn parse_double(&mut self, result: &mut Vec<u8>) -> Result<(), ()> {
loop {
if let Some(ch2) = self.next_char() {
match ch2 as char {
'\\' => {
if let Some(ch3) = self.next_char() {
match ch3 as char {
// \$ => $
'$' | '`' | '"' | '\\' => { result.push(ch3); },
// \<newline> => nothing
'\n' => {},
// \x => =x
_ => { result.push('\\' as u8); result.push(ch3); }
}
} else {
return Err(());
}
},
'"' => { return Ok(()); },
_ => { result.push(ch2); },
}
} else {
return Err(());
}
}
}
fn parse_single(&mut self, result: &mut Vec<u8>) -> Result<(), ()> {
loop {
if let Some(ch2) = self.next_char() {
match ch2 as char {
'\'' => { return Ok(()); },
_ => { result.push(ch2); },
}
} else {
return Err(());
}
}
}
fn next_char(&mut self) -> Option<u8> {
let res = self.in_iter.next().copied();
if res == Some(b'\n') { self.line_no += 1; }
res
}
}
impl<'a> Iterator for Shlex<'a> {
type Item = Vec<u8>;
fn next(&mut self) -> Option<Self::Item> {
if let Some(mut ch) = self.next_char() {
// skip initial whitespace
loop {
match ch as char {
' ' | '\t' | '\n' => {},
'#' => {
while let Some(ch2) = self.next_char() {
if ch2 as char == '\n' { break; }
}
},
_ => { break; }
}
if let Some(ch2) = self.next_char() { ch = ch2; } else { return None; }
}
self.parse_word(ch)
} else { // no initial character
None
}
}
}
/// Convenience function that consumes the whole byte string at once. Returns None if the input was
/// erroneous.
pub fn split(in_bytes: &[u8]) -> Option<Vec<Vec<u8>>> {
let mut shl = Shlex::new(in_bytes);
let res = shl.by_ref().collect();
if shl.had_error { None } else { Some(res) }
}
/// A more configurable interface to quote strings. If you only want the default settings you can
/// use the convenience functions [`try_quote`] and [`try_join`].
///
/// The string equivalent is [`shlex::Quoter`].
#[derive(Default, Debug, Clone)]
pub struct Quoter {
allow_nul: bool,
// TODO: more options
}
impl Quoter {
/// Create a new [`Quoter`] with default settings.
#[inline]
pub fn new() -> Self {
Self::default()
}
/// Set whether to allow [nul bytes](quoting_warning#nul-bytes). By default they are not
/// allowed and will result in an error of [`QuoteError::Nul`].
#[inline]
pub fn allow_nul(mut self, allow: bool) -> Self {
self.allow_nul = allow;
self
}
/// Convenience function that consumes an iterable of words and turns it into a single byte string,
/// quoting words when necessary. Consecutive words will be separated by a single space.
pub fn join<'a, I: IntoIterator<Item = &'a [u8]>>(&self, words: I) -> Result<Vec<u8>, QuoteError> {
Ok(words.into_iter()
.map(|word| self.quote(word))
.collect::<Result<Vec<Cow<[u8]>>, QuoteError>>()?
.join(&b' '))
}
/// Given a single word, return a byte string suitable to encode it as a shell argument.
///
/// If given valid UTF-8, this will never produce invalid UTF-8. This is because it only
/// ever inserts valid ASCII characters before or after existing ASCII characters (or
/// returns two single quotes if the input was an empty string). It will never modify a
/// multibyte UTF-8 character.
pub fn quote<'a>(&self, mut in_bytes: &'a [u8]) -> Result<Cow<'a, [u8]>, QuoteError> {
if in_bytes.is_empty() {
// Empty string. Special case that isn't meaningful as only part of a word.
return Ok(b"''"[..].into());
}
if !self.allow_nul && in_bytes.iter().any(|&b| b == b'\0') {
return Err(QuoteError::Nul);
}
let mut out: Vec<u8> = Vec::new();
while !in_bytes.is_empty() {
// Pick a quoting strategy for some prefix of the input. Normally this will cover the
// entire input, but in some case we might need to divide the input into multiple chunks
// that are quoted differently.
let (cur_len, strategy) = quoting_strategy(in_bytes);
if cur_len == in_bytes.len() && strategy == QuotingStrategy::Unquoted && out.is_empty() {
// Entire string can be represented unquoted. Reuse the allocation.
return Ok(in_bytes.into());
}
let (cur_chunk, rest) = in_bytes.split_at(cur_len);
assert!(rest.len() < in_bytes.len()); // no infinite loop
in_bytes = rest;
append_quoted_chunk(&mut out, cur_chunk, strategy);
}
Ok(out.into())
}
}
#[derive(PartialEq)]
enum QuotingStrategy {
/// No quotes and no backslash escapes. (If backslash escapes would be necessary, we use a
/// different strategy instead.)
Unquoted,
/// Single quoted.
SingleQuoted,
/// Double quotes, potentially with backslash escapes.
DoubleQuoted,
// TODO: add $'xxx' and "$(printf 'xxx')" styles
}
/// Is this ASCII byte okay to emit unquoted?
const fn unquoted_ok(c: u8) -> bool {
match c as char {
// Allowed characters:
'+' | '-' | '.' | '/' | ':' | '@' | ']' | '_' |
'0'..='9' | 'A'..='Z' | 'a'..='z'
=> true,
// Non-allowed characters:
// "The application shall quote the following characters if they are to represent themselves:"
'|' | '&' | ';' | '<' | '>' | '(' | ')' | '$' | '`' | '\\' | '"' | '\'' | ' ' | '\t' | '\n' |
// "and the following may need to be quoted under certain circumstances[..]:"
'*' | '?' | '[' | '#' | '~' | '=' | '%' |
// Brace expansion. These ought to be in the POSIX list but aren't yet;
'{' | '}' |
// Also quote comma, just to be safe in the extremely odd case that the user of this crate
// is intentionally placing a quoted string inside a brace expansion, e.g.:
// format!("echo foo{{a,b,{}}}" | shlex::quote(some_str))
',' |
// '\r' is allowed in a word by all real shells I tested, but is treated as a word
// separator by Python `shlex` | and might be translated to '\n' in interactive mode.
'\r' |
// '!' and '^' are treated specially in interactive mode; see quoting_warning.
'!' | '^' |
// Nul bytes and control characters.
'\x00' ..= '\x1f' | '\x7f'
=> false,
'\u{80}' ..= '\u{10ffff}' => {
// This is unreachable since `unquoted_ok` is only called for 0..128.
// Non-ASCII bytes are handled separately in `quoting_strategy`.
// Can't call unreachable!() from `const fn` on old Rust, so...
unquoted_ok(c)
},
}
// Note: The logic cited above for quoting comma might suggest that `..` should also be quoted,
// it as a special case of brace expansion). But it's not necessary. There are three cases:
//
// 1. The user wants comma-based brace expansion, but the untrusted string being `quote`d
// contains `..`, so they get something like `{foo,bar,3..5}`.
// => That's safe; both Bash and Zsh expand this to `foo bar 3..5` rather than
// `foo bar 3 4 5`. The presence of commas disables sequence expression expansion.
//
// 2. The user wants comma-based brace expansion where the contents of the braces are a
// variable number of `quote`d strings and nothing else. There happens to be exactly
// one string and it contains `..`, so they get something like `{3..5}`.
// => Then this will expand as a sequence expression, which is unintended. But I don't mind,
// because any such code is already buggy. Suppose the untrusted string *didn't* contain
// `,` or `..`, resulting in shell input like `{foo}`. Then the shell would interpret it
// as the literal string `{foo}` rather than brace-expanding it into `foo`.
//
// 3. The user wants a sequence expression and wants to supply an untrusted string as one of
// the endpoints or the increment.
// => Well, that's just silly, since the endpoints can only be numbers or single letters.
}
/// Optimized version of `unquoted_ok`.
fn unquoted_ok_fast(c: u8) -> bool {
const UNQUOTED_OK_MASK: u128 = {
// Make a mask of all bytes in 0..<0x80 that pass.
let mut c = 0u8;
let mut mask = 0u128;
while c < 0x80 {
if unquoted_ok(c) {
mask |= 1u128 << c;
}
c += 1;
}
mask
};
((UNQUOTED_OK_MASK >> c) & 1) != 0
}
/// Is this ASCII byte okay to emit in single quotes?
fn single_quoted_ok(c: u8) -> bool {
match c {
// No single quotes in single quotes.
b'\'' => false,
// To work around a Bash bug, ^ is only allowed right after an opening single quote; see
// quoting_warning.
b'^' => false,
// Backslashes in single quotes are literal according to POSIX, but Fish treats them as an
// escape character. Ban them. Fish doesn't aim to be POSIX-compatible, but we *can*
// achieve Fish compatibility using double quotes, so we might as well.
b'\\' => false,
_ => true
}
}
/// Is this ASCII byte okay to emit in double quotes?
fn double_quoted_ok(c: u8) -> bool {
match c {
// Work around Python `shlex` bug where parsing "\`" and "\$" doesn't strip the
// backslash, even though POSIX requires it.
b'`' | b'$' => false,
// '!' and '^' are treated specially in interactive mode; see quoting_warning.
b'!' | b'^' => false,
_ => true
}
}
/// Given an input, return a quoting strategy that can cover some prefix of the string, along with
/// the size of that prefix.
///
/// Precondition: input size is nonzero. (Empty strings are handled by the caller.)
/// Postcondition: returned size is nonzero.
#[cfg_attr(manual_codegen_check, inline(never))]
fn quoting_strategy(in_bytes: &[u8]) -> (usize, QuotingStrategy) {
const UNQUOTED_OK: u8 = 1;
const SINGLE_QUOTED_OK: u8 = 2;
const DOUBLE_QUOTED_OK: u8 = 4;
let mut prev_ok = SINGLE_QUOTED_OK | DOUBLE_QUOTED_OK | UNQUOTED_OK;
let mut i = 0;
if in_bytes[0] == b'^' {
// To work around a Bash bug, ^ is only allowed right after an opening single quote; see
// quoting_warning.
prev_ok = SINGLE_QUOTED_OK;
i = 1;
}
while i < in_bytes.len() {
let c = in_bytes[i];
let mut cur_ok = prev_ok;
if c >= 0x80 {
// Normally, non-ASCII characters shouldn't require quoting, but see quoting_warning.md
// about \xa0. For now, just treat all non-ASCII characters as requiring quotes. This
// also ensures things are safe in the off-chance that you're in a legacy 8-bit locale that
// has additional characters satisfying `isblank`.
cur_ok &= !UNQUOTED_OK;
} else {
if !unquoted_ok_fast(c) {
cur_ok &= !UNQUOTED_OK;
}
if !single_quoted_ok(c){
cur_ok &= !SINGLE_QUOTED_OK;
}
if !double_quoted_ok(c) {
cur_ok &= !DOUBLE_QUOTED_OK;
}
}
if cur_ok == 0 {
// There are no quoting strategies that would work for both the previous characters and
// this one. So we have to end the chunk before this character. The caller will call
// `quoting_strategy` again to handle the rest of the string.
break;
}
prev_ok = cur_ok;
i += 1;
}
// Pick the best allowed strategy.
let strategy = if prev_ok & UNQUOTED_OK != 0 {
QuotingStrategy::Unquoted
} else if prev_ok & SINGLE_QUOTED_OK != 0 {
QuotingStrategy::SingleQuoted
} else if prev_ok & DOUBLE_QUOTED_OK != 0 {
QuotingStrategy::DoubleQuoted
} else {
unreachable!()
};
debug_assert!(i > 0);
(i, strategy)
}
fn append_quoted_chunk(out: &mut Vec<u8>, cur_chunk: &[u8], strategy: QuotingStrategy) {
match strategy {
QuotingStrategy::Unquoted => {
out.extend_from_slice(cur_chunk);
},
QuotingStrategy::SingleQuoted => {
out.reserve(cur_chunk.len() + 2);
out.push(b'\'');
out.extend_from_slice(cur_chunk);
out.push(b'\'');
},
QuotingStrategy::DoubleQuoted => {
out.reserve(cur_chunk.len() + 2);
out.push(b'"');
for &c in cur_chunk.into_iter() {
if let b'$' | b'`' | b'"' | b'\\' = c {
// Add a preceding backslash.
// Note: We shouldn't actually get here for $ and ` because they don't pass
// `double_quoted_ok`.
out.push(b'\\');
}
// Add the character itself.
out.push(c);
}
out.push(b'"');
},
}
}
/// Convenience function that consumes an iterable of words and turns it into a single byte string,
/// quoting words when necessary. Consecutive words will be separated by a single space.
///
/// Uses default settings except that nul bytes are passed through, which [may be
/// dangerous](quoting_warning#nul-bytes), leading to this function being deprecated.
///
/// Equivalent to [`Quoter::new().allow_nul(true).join(words).unwrap()`](Quoter).
///
/// (That configuration never returns `Err`, so this function does not panic.)
///
/// The string equivalent is [shlex::join].
#[deprecated(since = "1.3.0", note = "replace with `try_join(words)?` to avoid nul byte danger")]
pub fn join<'a, I: IntoIterator<Item = &'a [u8]>>(words: I) -> Vec<u8> {
Quoter::new().allow_nul(true).join(words).unwrap()
}
/// Convenience function that consumes an iterable of words and turns it into a single byte string,
/// quoting words when necessary. Consecutive words will be separated by a single space.
///
/// Uses default settings. The only error that can be returned is [`QuoteError::Nul`].
///
/// Equivalent to [`Quoter::new().join(words)`](Quoter).
///
/// The string equivalent is [shlex::try_join].
pub fn try_join<'a, I: IntoIterator<Item = &'a [u8]>>(words: I) -> Result<Vec<u8>, QuoteError> {
Quoter::new().join(words)
}
/// Given a single word, return a string suitable to encode it as a shell argument.
///
/// Uses default settings except that nul bytes are passed through, which [may be
/// dangerous](quoting_warning#nul-bytes), leading to this function being deprecated.
///
/// Equivalent to [`Quoter::new().allow_nul(true).quote(in_bytes).unwrap()`](Quoter).
///
/// (That configuration never returns `Err`, so this function does not panic.)
///
/// The string equivalent is [shlex::quote].
#[deprecated(since = "1.3.0", note = "replace with `try_quote(str)?` to avoid nul byte danger")]
pub fn quote(in_bytes: &[u8]) -> Cow<[u8]> {
Quoter::new().allow_nul(true).quote(in_bytes).unwrap()
}
/// Given a single word, return a string suitable to encode it as a shell argument.
///
/// Uses default settings. The only error that can be returned is [`QuoteError::Nul`].
///
/// Equivalent to [`Quoter::new().quote(in_bytes)`](Quoter).
///
/// (That configuration never returns `Err`, so this function does not panic.)
///
/// The string equivalent is [shlex::try_quote].
pub fn try_quote(in_bytes: &[u8]) -> Result<Cow<[u8]>, QuoteError> {
Quoter::new().quote(in_bytes)
}
#[cfg(test)]
const INVALID_UTF8: &[u8] = b"\xa1";
#[cfg(test)]
const INVALID_UTF8_SINGLEQUOTED: &[u8] = b"'\xa1'";
#[test]
#[allow(invalid_from_utf8)]
fn test_invalid_utf8() {
// Check that our test string is actually invalid UTF-8.
assert!(core::str::from_utf8(INVALID_UTF8).is_err());
}
#[cfg(test)]
static SPLIT_TEST_ITEMS: &'static [(&'static [u8], Option<&'static [&'static [u8]]>)] = &[
(b"foo$baz", Some(&[b"foo$baz"])),
(b"foo baz", Some(&[b"foo", b"baz"])),
(b"foo\"bar\"baz", Some(&[b"foobarbaz"])),
(b"foo \"bar\"baz", Some(&[b"foo", b"barbaz"])),
(b" foo \nbar", Some(&[b"foo", b"bar"])),
(b"foo\\\nbar", Some(&[b"foobar"])),
(b"\"foo\\\nbar\"", Some(&[b"foobar"])),
(b"'baz\\$b'", Some(&[b"baz\\$b"])),
(b"'baz\\\''", None),
(b"\\", None),
(b"\"\\", None),
(b"'\\", None),
(b"\"", None),
(b"'", None),
(b"foo #bar\nbaz", Some(&[b"foo", b"baz"])),
(b"foo #bar", Some(&[b"foo"])),
(b"foo#bar", Some(&[b"foo#bar"])),
(b"foo\"#bar", None),
(b"'\\n'", Some(&[b"\\n"])),
(b"'\\\\n'", Some(&[b"\\\\n"])),
(INVALID_UTF8, Some(&[INVALID_UTF8])),
];
#[test]
fn test_split() {
for &(input, output) in SPLIT_TEST_ITEMS {
assert_eq!(split(input), output.map(|o| o.iter().map(|&x| x.to_owned()).collect()));
}
}
#[test]
fn test_lineno() {
let mut sh = Shlex::new(b"\nfoo\nbar");
while let Some(word) = sh.next() {
if word == b"bar" {
assert_eq!(sh.line_no, 3);
}
}
}
#[test]
#[allow(deprecated)]
fn test_quote() {
// Validate behavior with invalid UTF-8:
assert_eq!(quote(INVALID_UTF8), INVALID_UTF8_SINGLEQUOTED);
// Replicate a few tests from lib.rs. No need to replicate all of them.
assert_eq!(quote(b""), &b"''"[..]);
assert_eq!(quote(b"foobar"), &b"foobar"[..]);
assert_eq!(quote(b"foo bar"), &b"'foo bar'"[..]);
assert_eq!(quote(b"'\""), &b"\"'\\\"\""[..]);
assert_eq!(quote(b""), &b"''"[..]);
}
#[test]
#[allow(deprecated)]
fn test_join() {
// Validate behavior with invalid UTF-8:
assert_eq!(join(vec![INVALID_UTF8]), INVALID_UTF8_SINGLEQUOTED);
// Replicate a few tests from lib.rs. No need to replicate all of them.
assert_eq!(join(vec![]), &b""[..]);
assert_eq!(join(vec![&b""[..]]), b"''");
}