Upgrade rust/crates/paste to 1.0.2
Test: make
Change-Id: I31f10999f2ea0443268237a7b5509752ab4fc39b
diff --git a/.cargo_vcs_info.json b/.cargo_vcs_info.json
index 18adbe1..8b0f57b 100644
--- a/.cargo_vcs_info.json
+++ b/.cargo_vcs_info.json
@@ -1,5 +1,5 @@
{
"git": {
- "sha1": "ead8998a76e6b28a0ade8574490e18f7bb52877b"
+ "sha1": "6a5265f7a937412fb1da72fb72fd32bbaffecebc"
}
}
diff --git a/Cargo.toml b/Cargo.toml
index c86fc87..86c30d9 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -13,7 +13,7 @@
[package]
edition = "2018"
name = "paste"
-version = "1.0.1"
+version = "1.0.2"
authors = ["David Tolnay <dtolnay@gmail.com>"]
description = "Macros for all your token pasting needs"
readme = "README.md"
diff --git a/Cargo.toml.orig b/Cargo.toml.orig
index 5365865..42aee90 100644
--- a/Cargo.toml.orig
+++ b/Cargo.toml.orig
@@ -1,6 +1,6 @@
[package]
name = "paste"
-version = "1.0.1"
+version = "1.0.2"
authors = ["David Tolnay <dtolnay@gmail.com>"]
edition = "2018"
license = "MIT OR Apache-2.0"
diff --git a/METADATA b/METADATA
index 74b7e73..4f5fb87 100644
--- a/METADATA
+++ b/METADATA
@@ -7,13 +7,13 @@
}
url {
type: ARCHIVE
- value: "https://static.crates.io/crates/paste/paste-1.0.1.crate"
+ value: "https://static.crates.io/crates/paste/paste-1.0.2.crate"
}
- version: "1.0.1"
+ version: "1.0.2"
license_type: NOTICE
last_upgrade_date {
year: 2020
- month: 9
- day: 15
+ month: 10
+ day: 26
}
}
diff --git a/src/doc.rs b/src/doc.rs
index 81d4184..5fa2ad9 100644
--- a/src/doc.rs
+++ b/src/doc.rs
@@ -1,5 +1,8 @@
+use crate::error::Result;
+use crate::segment::{self, Segment};
use proc_macro::{Delimiter, Span, TokenStream, TokenTree};
use std::iter;
+use std::mem;
use std::str::FromStr;
pub fn is_pasted_doc(input: &TokenStream) -> bool {
@@ -26,16 +29,33 @@
state == State::Rest
}
-pub fn do_paste_doc(attr: &TokenStream, span: Span) -> TokenStream {
+pub fn do_paste_doc(attr: &TokenStream, span: Span) -> Result<TokenStream> {
let mut expanded = TokenStream::new();
- let mut tokens = attr.clone().into_iter();
+ let mut tokens = attr.clone().into_iter().peekable();
expanded.extend(tokens.by_ref().take(2)); // `doc =`
- let mut lit = String::new();
- lit.push('"');
- for token in tokens {
- lit += &escaped_string_value(&token).unwrap();
+ let mut segments = segment::parse(&mut tokens)?;
+
+ for segment in &mut segments {
+ if let Segment::String(string) = segment {
+ if let Some(open_quote) = string.value.find('"') {
+ if open_quote == 0 {
+ string.value.truncate(string.value.len() - 1);
+ string.value.remove(0);
+ } else {
+ let begin = open_quote + 1;
+ let end = string.value.rfind('"').unwrap();
+ let raw_string = mem::replace(&mut string.value, String::new());
+ for ch in raw_string[begin..end].chars() {
+ string.value.extend(ch.escape_default());
+ }
+ }
+ }
+ }
}
+
+ let mut lit = segment::paste(&segments)?;
+ lit.insert(0, '"');
lit.push('"');
let mut lit = TokenStream::from_str(&lit)
@@ -45,48 +65,26 @@
.unwrap();
lit.set_span(span);
expanded.extend(iter::once(lit));
- expanded
+ Ok(expanded)
}
fn is_stringlike(token: &TokenTree) -> bool {
- escaped_string_value(token).is_some()
-}
-
-fn escaped_string_value(token: &TokenTree) -> Option<String> {
match token {
- TokenTree::Ident(ident) => Some(ident.to_string()),
+ TokenTree::Ident(_) => true,
TokenTree::Literal(literal) => {
- let mut repr = literal.to_string();
- if repr.starts_with('b') || repr.starts_with('\'') {
- None
- } else if repr.starts_with('"') {
- repr.truncate(repr.len() - 1);
- repr.remove(0);
- Some(repr)
- } else if repr.starts_with('r') {
- let begin = repr.find('"').unwrap() + 1;
- let end = repr.rfind('"').unwrap();
- let mut escaped = String::new();
- for ch in repr[begin..end].chars() {
- escaped.extend(ch.escape_default());
- }
- Some(escaped)
- } else {
- Some(repr)
- }
+ let repr = literal.to_string();
+ !repr.starts_with('b') && !repr.starts_with('\'')
}
TokenTree::Group(group) => {
if group.delimiter() != Delimiter::None {
- return None;
+ return false;
}
let mut inner = group.stream().into_iter();
- let first = inner.next()?;
- if inner.next().is_none() {
- escaped_string_value(&first)
- } else {
- None
+ match inner.next() {
+ Some(first) => inner.next().is_none() && is_stringlike(&first),
+ None => false,
}
}
- TokenTree::Punct(_) => None,
+ TokenTree::Punct(punct) => punct.as_char() == '\'' || punct.as_char() == ':',
}
}
diff --git a/src/lib.rs b/src/lib.rs
index 8fc755b..e342a33 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -142,13 +142,13 @@
mod doc;
mod error;
+mod segment;
use crate::doc::{do_paste_doc, is_pasted_doc};
use crate::error::{Error, Result};
-use proc_macro::{
- token_stream, Delimiter, Group, Ident, Punct, Spacing, Span, TokenStream, TokenTree,
-};
-use std::iter::{self, FromIterator, Peekable};
+use crate::segment::Segment;
+use proc_macro::{Delimiter, Group, Ident, Punct, Spacing, Span, TokenStream, TokenTree};
+use std::iter;
use std::panic;
#[proc_macro]
@@ -199,8 +199,9 @@
let span = group.span();
if delimiter == Delimiter::Bracket && is_paste_operation(&content) {
let segments = parse_bracket_as_segments(content, span)?;
- let pasted = paste_segments(span, &segments)?;
- expanded.extend(pasted);
+ let pasted = segment::paste(&segments)?;
+ let tokens = pasted_to_tokens(pasted, span)?;
+ expanded.extend(tokens);
*contains_paste = true;
} else if delimiter == Delimiter::None && is_flat_group(&content) {
expanded.extend(content);
@@ -209,7 +210,7 @@
&& (lookbehind == Lookbehind::Pound || lookbehind == Lookbehind::PoundBang)
&& is_pasted_doc(&content)
{
- let pasted = do_paste_doc(&content, span);
+ let pasted = do_paste_doc(&content, span)?;
let mut group = Group::new(delimiter, pasted);
group.set_span(span);
expanded.extend(iter::once(TokenTree::Group(group)));
@@ -302,22 +303,6 @@
state == State::Ident || state == State::Literal || state == State::Lifetime
}
-struct LitStr {
- value: String,
- span: Span,
-}
-
-struct Colon {
- span: Span,
-}
-
-enum Segment {
- String(String),
- Apostrophe(Span),
- Env(LitStr),
- Modifier(Colon, Ident),
-}
-
fn is_paste_operation(input: &TokenStream) -> bool {
let mut tokens = input.clone().into_iter();
@@ -347,7 +332,7 @@
None => return Err(Error::new(scope, "expected `[< ... >]`")),
}
- let segments = parse_segments(&mut tokens, scope)?;
+ let mut segments = segment::parse(&mut tokens)?;
match &tokens.next() {
Some(TokenTree::Punct(punct)) if punct.as_char() == '>' => {}
@@ -355,218 +340,39 @@
None => return Err(Error::new(scope, "expected `[< ... >]`")),
}
- match tokens.next() {
- Some(unexpected) => Err(Error::new(
+ if let Some(unexpected) = tokens.next() {
+ return Err(Error::new(
unexpected.span(),
"unexpected input, expected `[< ... >]`",
- )),
- None => Ok(segments),
+ ));
}
-}
-fn parse_segments(
- tokens: &mut Peekable<token_stream::IntoIter>,
- scope: Span,
-) -> Result<Vec<Segment>> {
- let mut segments = Vec::new();
- while match tokens.peek() {
- None => false,
- Some(TokenTree::Punct(punct)) => punct.as_char() != '>',
- Some(_) => true,
- } {
- match tokens.next().unwrap() {
- TokenTree::Ident(ident) => {
- let mut fragment = ident.to_string();
- if fragment.starts_with("r#") {
- fragment = fragment.split_off(2);
- }
- if fragment == "env"
- && match tokens.peek() {
- Some(TokenTree::Punct(punct)) => punct.as_char() == '!',
- _ => false,
- }
- {
- tokens.next().unwrap(); // `!`
- let expect_group = tokens.next();
- let parenthesized = match &expect_group {
- Some(TokenTree::Group(group))
- if group.delimiter() == Delimiter::Parenthesis =>
- {
- group
- }
- Some(wrong) => return Err(Error::new(wrong.span(), "expected `(`")),
- None => return Err(Error::new(scope, "expected `(` after `env!`")),
- };
- let mut inner = parenthesized.stream().into_iter();
- let lit = match inner.next() {
- Some(TokenTree::Literal(lit)) => lit,
- Some(wrong) => {
- return Err(Error::new(wrong.span(), "expected string literal"))
- }
- None => {
- return Err(Error::new2(
- ident.span(),
- parenthesized.span(),
- "expected string literal as argument to env! macro",
- ))
- }
- };
- let lit_string = lit.to_string();
- if lit_string.starts_with('"')
- && lit_string.ends_with('"')
- && lit_string.len() >= 2
- {
- // TODO: maybe handle escape sequences in the string if
- // someone has a use case.
- segments.push(Segment::Env(LitStr {
- value: lit_string[1..lit_string.len() - 1].to_owned(),
- span: lit.span(),
- }));
- } else {
- return Err(Error::new(lit.span(), "expected string literal"));
- }
- if let Some(unexpected) = inner.next() {
- return Err(Error::new(
- unexpected.span(),
- "unexpected token in env! macro",
- ));
- }
- } else {
- segments.push(Segment::String(fragment));
- }
+ for segment in &mut segments {
+ if let Segment::String(string) = segment {
+ if string.value.contains(&['#', '\\', '.', '+'][..]) {
+ return Err(Error::new(string.span, "unsupported literal"));
}
- TokenTree::Literal(lit) => {
- let mut lit_string = lit.to_string();
- if lit_string.contains(&['#', '\\', '.', '+'][..]) {
- return Err(Error::new(lit.span(), "unsupported literal"));
- }
- lit_string = lit_string
- .replace('"', "")
- .replace('\'', "")
- .replace('-', "_");
- segments.push(Segment::String(lit_string));
- }
- TokenTree::Punct(punct) => match punct.as_char() {
- '_' => segments.push(Segment::String("_".to_owned())),
- '\'' => segments.push(Segment::Apostrophe(punct.span())),
- ':' => {
- let colon = Colon { span: punct.span() };
- let ident = match tokens.next() {
- Some(TokenTree::Ident(ident)) => ident,
- wrong => {
- let span = wrong.as_ref().map_or(scope, TokenTree::span);
- return Err(Error::new(span, "expected identifier after `:`"));
- }
- };
- segments.push(Segment::Modifier(colon, ident));
- }
- _ => return Err(Error::new(punct.span(), "unexpected punct")),
- },
- TokenTree::Group(group) => {
- if group.delimiter() == Delimiter::None {
- let mut inner = group.stream().into_iter().peekable();
- let nested = parse_segments(&mut inner, group.span())?;
- if let Some(unexpected) = inner.next() {
- return Err(Error::new(unexpected.span(), "unexpected token"));
- }
- segments.extend(nested);
- } else {
- return Err(Error::new(group.span(), "unexpected token"));
- }
- }
+ string.value = string
+ .value
+ .replace('"', "")
+ .replace('\'', "")
+ .replace('-', "_");
}
}
+
Ok(segments)
}
-fn paste_segments(span: Span, segments: &[Segment]) -> Result<TokenStream> {
- let mut evaluated = Vec::new();
- let mut is_lifetime = false;
+fn pasted_to_tokens(mut pasted: String, span: Span) -> Result<TokenStream> {
+ let mut tokens = TokenStream::new();
- for segment in segments {
- match segment {
- Segment::String(segment) => {
- evaluated.push(segment.clone());
- }
- Segment::Apostrophe(span) => {
- if is_lifetime {
- return Err(Error::new(*span, "unexpected lifetime"));
- }
- is_lifetime = true;
- }
- Segment::Env(var) => {
- let resolved = match std::env::var(&var.value) {
- Ok(resolved) => resolved,
- Err(_) => {
- return Err(Error::new(
- var.span,
- &format!("no such env var: {:?}", var.value),
- ));
- }
- };
- let resolved = resolved.replace('-', "_");
- evaluated.push(resolved);
- }
- Segment::Modifier(colon, ident) => {
- let last = match evaluated.pop() {
- Some(last) => last,
- None => {
- return Err(Error::new2(colon.span, ident.span(), "unexpected modifier"))
- }
- };
- match ident.to_string().as_str() {
- "lower" => {
- evaluated.push(last.to_lowercase());
- }
- "upper" => {
- evaluated.push(last.to_uppercase());
- }
- "snake" => {
- let mut acc = String::new();
- let mut prev = '_';
- for ch in last.chars() {
- if ch.is_uppercase() && prev != '_' {
- acc.push('_');
- }
- acc.push(ch);
- prev = ch;
- }
- evaluated.push(acc.to_lowercase());
- }
- "camel" => {
- let mut acc = String::new();
- let mut prev = '_';
- for ch in last.chars() {
- if ch != '_' {
- if prev == '_' {
- for chu in ch.to_uppercase() {
- acc.push(chu);
- }
- } else if prev.is_uppercase() {
- for chl in ch.to_lowercase() {
- acc.push(chl);
- }
- } else {
- acc.push(ch);
- }
- }
- prev = ch;
- }
- evaluated.push(acc);
- }
- _ => {
- return Err(Error::new2(
- colon.span,
- ident.span(),
- "unsupported modifier",
- ));
- }
- }
- }
- }
+ if pasted.starts_with('\'') {
+ let mut apostrophe = TokenTree::Punct(Punct::new('\'', Spacing::Joint));
+ apostrophe.set_span(span);
+ tokens.extend(iter::once(apostrophe));
+ pasted.remove(0);
}
- let pasted = evaluated.into_iter().collect::<String>();
let ident = match panic::catch_unwind(|| Ident::new(&pasted, span)) {
Ok(ident) => TokenTree::Ident(ident),
Err(_) => {
@@ -576,11 +382,7 @@
));
}
};
- let tokens = if is_lifetime {
- let apostrophe = TokenTree::Punct(Punct::new('\'', Spacing::Joint));
- vec![apostrophe, ident]
- } else {
- vec![ident]
- };
- Ok(TokenStream::from_iter(tokens))
+
+ tokens.extend(iter::once(ident));
+ Ok(tokens)
}
diff --git a/src/segment.rs b/src/segment.rs
new file mode 100644
index 0000000..592a047
--- /dev/null
+++ b/src/segment.rs
@@ -0,0 +1,233 @@
+use crate::error::{Error, Result};
+use proc_macro::{token_stream, Delimiter, Ident, Span, TokenTree};
+use std::iter::Peekable;
+
+pub(crate) enum Segment {
+ String(LitStr),
+ Apostrophe(Span),
+ Env(LitStr),
+ Modifier(Colon, Ident),
+}
+
+pub(crate) struct LitStr {
+ pub value: String,
+ pub span: Span,
+}
+
+pub(crate) struct Colon {
+ pub span: Span,
+}
+
+pub(crate) fn parse(tokens: &mut Peekable<token_stream::IntoIter>) -> Result<Vec<Segment>> {
+ let mut segments = Vec::new();
+ while match tokens.peek() {
+ None => false,
+ Some(TokenTree::Punct(punct)) => punct.as_char() != '>',
+ Some(_) => true,
+ } {
+ match tokens.next().unwrap() {
+ TokenTree::Ident(ident) => {
+ let mut fragment = ident.to_string();
+ if fragment.starts_with("r#") {
+ fragment = fragment.split_off(2);
+ }
+ if fragment == "env"
+ && match tokens.peek() {
+ Some(TokenTree::Punct(punct)) => punct.as_char() == '!',
+ _ => false,
+ }
+ {
+ let bang = tokens.next().unwrap(); // `!`
+ let expect_group = tokens.next();
+ let parenthesized = match &expect_group {
+ Some(TokenTree::Group(group))
+ if group.delimiter() == Delimiter::Parenthesis =>
+ {
+ group
+ }
+ Some(wrong) => return Err(Error::new(wrong.span(), "expected `(`")),
+ None => {
+ return Err(Error::new2(
+ ident.span(),
+ bang.span(),
+ "expected `(` after `env!`",
+ ));
+ }
+ };
+ let mut inner = parenthesized.stream().into_iter();
+ let lit = match inner.next() {
+ Some(TokenTree::Literal(lit)) => lit,
+ Some(wrong) => {
+ return Err(Error::new(wrong.span(), "expected string literal"))
+ }
+ None => {
+ return Err(Error::new2(
+ ident.span(),
+ parenthesized.span(),
+ "expected string literal as argument to env! macro",
+ ))
+ }
+ };
+ let lit_string = lit.to_string();
+ if lit_string.starts_with('"')
+ && lit_string.ends_with('"')
+ && lit_string.len() >= 2
+ {
+ // TODO: maybe handle escape sequences in the string if
+ // someone has a use case.
+ segments.push(Segment::Env(LitStr {
+ value: lit_string[1..lit_string.len() - 1].to_owned(),
+ span: lit.span(),
+ }));
+ } else {
+ return Err(Error::new(lit.span(), "expected string literal"));
+ }
+ if let Some(unexpected) = inner.next() {
+ return Err(Error::new(
+ unexpected.span(),
+ "unexpected token in env! macro",
+ ));
+ }
+ } else {
+ segments.push(Segment::String(LitStr {
+ value: fragment,
+ span: ident.span(),
+ }));
+ }
+ }
+ TokenTree::Literal(lit) => {
+ segments.push(Segment::String(LitStr {
+ value: lit.to_string(),
+ span: lit.span(),
+ }));
+ }
+ TokenTree::Punct(punct) => match punct.as_char() {
+ '_' => segments.push(Segment::String(LitStr {
+ value: "_".to_owned(),
+ span: punct.span(),
+ })),
+ '\'' => segments.push(Segment::Apostrophe(punct.span())),
+ ':' => {
+ let colon_span = punct.span();
+ let colon = Colon { span: colon_span };
+ let ident = match tokens.next() {
+ Some(TokenTree::Ident(ident)) => ident,
+ wrong => {
+ let span = wrong.as_ref().map_or(colon_span, TokenTree::span);
+ return Err(Error::new(span, "expected identifier after `:`"));
+ }
+ };
+ segments.push(Segment::Modifier(colon, ident));
+ }
+ _ => return Err(Error::new(punct.span(), "unexpected punct")),
+ },
+ TokenTree::Group(group) => {
+ if group.delimiter() == Delimiter::None {
+ let mut inner = group.stream().into_iter().peekable();
+ let nested = parse(&mut inner)?;
+ if let Some(unexpected) = inner.next() {
+ return Err(Error::new(unexpected.span(), "unexpected token"));
+ }
+ segments.extend(nested);
+ } else {
+ return Err(Error::new(group.span(), "unexpected token"));
+ }
+ }
+ }
+ }
+ Ok(segments)
+}
+
+pub(crate) fn paste(segments: &[Segment]) -> Result<String> {
+ let mut evaluated = Vec::new();
+ let mut is_lifetime = false;
+
+ for segment in segments {
+ match segment {
+ Segment::String(segment) => {
+ evaluated.push(segment.value.clone());
+ }
+ Segment::Apostrophe(span) => {
+ if is_lifetime {
+ return Err(Error::new(*span, "unexpected lifetime"));
+ }
+ is_lifetime = true;
+ }
+ Segment::Env(var) => {
+ let resolved = match std::env::var(&var.value) {
+ Ok(resolved) => resolved,
+ Err(_) => {
+ return Err(Error::new(
+ var.span,
+ &format!("no such env var: {:?}", var.value),
+ ));
+ }
+ };
+ let resolved = resolved.replace('-', "_");
+ evaluated.push(resolved);
+ }
+ Segment::Modifier(colon, ident) => {
+ let last = match evaluated.pop() {
+ Some(last) => last,
+ None => {
+ return Err(Error::new2(colon.span, ident.span(), "unexpected modifier"))
+ }
+ };
+ match ident.to_string().as_str() {
+ "lower" => {
+ evaluated.push(last.to_lowercase());
+ }
+ "upper" => {
+ evaluated.push(last.to_uppercase());
+ }
+ "snake" => {
+ let mut acc = String::new();
+ let mut prev = '_';
+ for ch in last.chars() {
+ if ch.is_uppercase() && prev != '_' {
+ acc.push('_');
+ }
+ acc.push(ch);
+ prev = ch;
+ }
+ evaluated.push(acc.to_lowercase());
+ }
+ "camel" => {
+ let mut acc = String::new();
+ let mut prev = '_';
+ for ch in last.chars() {
+ if ch != '_' {
+ if prev == '_' {
+ for chu in ch.to_uppercase() {
+ acc.push(chu);
+ }
+ } else if prev.is_uppercase() {
+ for chl in ch.to_lowercase() {
+ acc.push(chl);
+ }
+ } else {
+ acc.push(ch);
+ }
+ }
+ prev = ch;
+ }
+ evaluated.push(acc);
+ }
+ _ => {
+ return Err(Error::new2(
+ colon.span,
+ ident.span(),
+ "unsupported modifier",
+ ));
+ }
+ }
+ }
+ }
+ }
+
+ let mut pasted = evaluated.into_iter().collect::<String>();
+ if is_lifetime {
+ pasted.insert(0, '\'');
+ }
+ Ok(pasted)
+}
diff --git a/tests/test_doc.rs b/tests/test_doc.rs
index 96fe3a0..1ceaf23 100644
--- a/tests/test_doc.rs
+++ b/tests/test_doc.rs
@@ -42,3 +42,13 @@
let expected = "int=0x1 bool=true float=0.01";
assert_eq!(doc, expected);
}
+
+#[test]
+fn test_case() {
+ let doc = paste! {
+ get_doc!(#[doc = "HTTP " get:upper "!"])
+ };
+
+ let expected = "HTTP GET!";
+ assert_eq!(doc, expected);
+}
