Update litrs to 0.4.1
Test: m
Change-Id: I9dd03a6483de174fbbefc37130bb4293592378c9
diff --git a/crates/litrs/.android-checksum.json b/crates/litrs/.android-checksum.json
index ac7656d..b9b03fb 100644
--- a/crates/litrs/.android-checksum.json
+++ b/crates/litrs/.android-checksum.json
@@ -1 +1 @@
-{"package":null,"files":{".cargo-checksum.json":"3d5b90bd6ce9063bb04fad37d7afd6f4d7a24db579f818972705943b1f4038fd","Android.bp":"34946635835e410125fdde874952a5185fc6ef6ed7eb1aea1cb5516f706088ba","CHANGELOG.md":"c561ce97f89d2c3e4d667ca03374dcd62138c2c506b8e8d22bfb214635212a8d","Cargo.toml":"429bd14ab71cb8e57c8c44ef278a7ceeaa8cbc0fb4a0de992c34a41309af4e5a","LICENSE":"50f827348bfd5def2df4f30cb41264072cf9c180bd9ed698437e8d85b2482754","LICENSE-APACHE":"50f827348bfd5def2df4f30cb41264072cf9c180bd9ed698437e8d85b2482754","LICENSE-MIT":"5431df8e7551a3ba0e6526fe532884000aa3cfe905fbf13ca66a7f8108561cb9","METADATA":"aedb495059560884f6e9fd993e189faf0d4bc93cab996974d6d618a8ee956379","MODULE_LICENSE_APACHE2":"0d6f8afa3940b7f06bebee651376d43bc8b0d5b437337be2696d30377451e93a","README.md":"9f9515dd556a269baffa43606eafffaa016b0fdd128b901b02cb90f2a6418a26","cargo_embargo.json":"e6e87d4bcbbf3f832b3f7823f0ddbc381fbe35c52e24d4a4fbf3aeb09b89c50e","src/bool/mod.rs":"f3bb8b87219ba2d5d8d059f43580bce3408275061a8bd82f1a0d666a7015685e","src/bool/tests.rs":"87c439a0f5b4cabedbcdec467225af1480ebe42f04eaf73a4f29e90932f293b4","src/byte/mod.rs":"c429c6d9d84c828b4f26a35b35ded7529226722a1764930c6f26021ea4ebdc55","src/byte/tests.rs":"455e83da47b0afc22bf85b8f19c89c7fd3f8e04b0f1004eff27f99e3cff11831","src/bytestr/mod.rs":"b1d73966b23de7607196d5cf5801addc51b2129f86570311b6a2803724429cc3","src/bytestr/tests.rs":"d0de0fa8c27f5ea04fc7ed70006ea6af9eaca65b208e7a11eabeb500bf5fc117","src/char/mod.rs":"2983dfb445a29d3fe2e7b8aa6a0e948c48e7aaa0a00ec61fa39b356a0fa1a3bd","src/char/tests.rs":"efb992b27824bc9c75fc54254ef6455d840075b1f7cfcd9da7f6d03282dbae43","src/err.rs":"b348412f6e2700bef152a19dcbe4836df992e7ad200bd178cbbed9711562dfe3","src/escape.rs":"0b65c042ccbd8ee350719bfa774c8c864cba30b3ff9a77f809ee0feead56bcf0","src/float/mod.rs":"6cf06c025efc91576e6f6cbb0907a4b7fe329c1900c00ce6ef7d2c40515e4738","src/float/tests.rs":"db9f8ceed54daa16a2e2429bc03966cacec83dcb4ea82736ed0ac3d80b56cd1b","src/impls.rs":"af2e13e91d8c7964a713e9d74f23f82015e4a2afa423f8e3638a82aa40fa512a","src/integer/mod.rs":"1765622faef27c5bacd2fc8e8dba8a68478972a7fdb20bfd3bee17d5622a15e7","src/integer/tests.rs":"2ebacd67502c16e373b717a7b99036b8ae34ae7f06f5453d260c97e4781b7beb","src/lib.rs":"04483e2593c446abe18943586fbfcbf701aa5ac1b5d6f89ab9b36385f34730bd","src/parse.rs":"025bdf3c402108011cac6c5094107f065d0db588dd65584b7ad94ccac32176fe","src/string/mod.rs":"7b3d4e8afac6a6432a82b32825b5cf552fc663d40d4481268934b0f94ab313de","src/string/tests.rs":"34576b9c8932d859ae68f1b08888b0ca3ab0c7e80dd5afc845b94b0db9c16386","src/test_util.rs":"f9bfe2a53a3480c3149b3264d68ccf76d493103de1d447579b3008d85b388a83","src/tests.rs":"1615f6634449ade712ad3e40a25e9504ebc0988cd4c735258787b31de27e407f"}}
\ No newline at end of file
+{"package":null,"files":{".cargo-checksum.json":"50fb0bbf2ab1fa41c81d92e0651fa50d7076eeaca866edca951ca95432d4bc2c","Android.bp":"81d21178cae96cc70509f5b5a317b2943cd372ef720d71d31c840be3fc830e8a","CHANGELOG.md":"57622d3aebf3ea223e1b92561e374e3ded461292fae9c6be0244c9367e2aedb1","Cargo.toml":"d19d08af5f35eb5793025f964a61c9d7ea9c36bbd0d269928314c305ab1a4525","LICENSE":"50f827348bfd5def2df4f30cb41264072cf9c180bd9ed698437e8d85b2482754","LICENSE-APACHE":"50f827348bfd5def2df4f30cb41264072cf9c180bd9ed698437e8d85b2482754","LICENSE-MIT":"5431df8e7551a3ba0e6526fe532884000aa3cfe905fbf13ca66a7f8108561cb9","METADATA":"2a7ebebaa896b165f41504d1a70155e7b86f3e706ca04751b0788047f258c22e","MODULE_LICENSE_APACHE2":"0d6f8afa3940b7f06bebee651376d43bc8b0d5b437337be2696d30377451e93a","README.md":"9f9515dd556a269baffa43606eafffaa016b0fdd128b901b02cb90f2a6418a26","cargo_embargo.json":"e6e87d4bcbbf3f832b3f7823f0ddbc381fbe35c52e24d4a4fbf3aeb09b89c50e","src/bool/mod.rs":"f3bb8b87219ba2d5d8d059f43580bce3408275061a8bd82f1a0d666a7015685e","src/bool/tests.rs":"87c439a0f5b4cabedbcdec467225af1480ebe42f04eaf73a4f29e90932f293b4","src/byte/mod.rs":"d9cb0a1a672f74380178c0307f4e91330b4a3b417d5397e43202dbfb95addbf2","src/byte/tests.rs":"34cc7732900a98e821c8cab370628b8990f9ced3fa90e353093beccaa36aea25","src/bytestr/mod.rs":"1d00041ac52052001f4b57561b1014578943d6bcc2b78bc0e47ac500c372a551","src/bytestr/tests.rs":"6261f1242d34bc479ae8fbbc29b49a72e3c8bb4969e14b0201ac4c6e7827a241","src/char/mod.rs":"1316ba70e7ea6eae0c98b4c49ee329f4c3bbba32b668ab7df55af867eac43f93","src/char/tests.rs":"c9ae0118a17e9735f25a78b80f3eefcdb1529a586755ee8440dab7e4ec42fa1d","src/err.rs":"f8ebb5e07ae94849b80d4b113deae04c5003bb1ebaffa0af3754b9861dae27ed","src/escape.rs":"b58e9d7b692b9180719e4c9ef42a197800307230c7d21ac14d2594b283c9cca9","src/float/mod.rs":"9c6c5e6d723fc4af01523ff28141c993f9b70e9448d84f016f8057aac9fc9636","src/float/tests.rs":"353b1f9ae589d21d145423a77360e082860094247dc72f97c86f43e495d054e9","src/impls.rs":"af2e13e91d8c7964a713e9d74f23f82015e4a2afa423f8e3638a82aa40fa512a","src/integer/mod.rs":"f27fce2145c73993313e1a65a8fd754b397756f1ba360d2be5ef9fdbc10e6f6e","src/integer/tests.rs":"d281932d124b89d41ed52fa240d8fdd61fa0fa6cffc841641a29630150c1f0db","src/lib.rs":"ac74e7b61e3330f4c28ea74379c2201a4e5b5fc6be558d538d3e55195eff5549","src/parse.rs":"aa2b773ff1a3f25cba8728415b951743fe0f4467f7a126603e20ccefeb7c85e5","src/string/mod.rs":"58c01076f8b59fdd19fef39daf5310a62d3c8b2d01967479c0e1b67716981741","src/string/tests.rs":"0c2fb0957858ee8661a7613926fbfd2aac3e9c391eec1d82c2dfcb004ebab7e0","src/test_util.rs":"f9bfe2a53a3480c3149b3264d68ccf76d493103de1d447579b3008d85b388a83","src/tests.rs":"498423bcb9e40531b3a6e9d0e1d6694b813b9b9554bb8ce7a80b9b83b4bbbc07"}}
\ No newline at end of file
diff --git a/crates/litrs/.cargo-checksum.json b/crates/litrs/.cargo-checksum.json
index 2ebdea4..d12ccff 100644
--- a/crates/litrs/.cargo-checksum.json
+++ b/crates/litrs/.cargo-checksum.json
@@ -1 +1 @@
-{"files":{"CHANGELOG.md":"75113cfc6e895ee8e72b86e34174fd34df2d5268df1d6df01963773412b9c35b","Cargo.toml":"60e17be214a6939e3db1371b40ddbd1389e3296bf371155713cbd770780c90a9","LICENSE-APACHE":"62c7a1e35f56406896d7aa7ca52d0cc0d272ac022b5d2796e7d6905db8a3636a","LICENSE-MIT":"7dc1552e88f49132cb358b1b962fc5e79fa42d70bcbb88c526d33e45b8e98036","README.md":"533d31adf3b4258b838cd6a1cdb58139e2cf761c3c38aa4654f66f34335c9073","src/bool/mod.rs":"53c6eedfd94552689e51233fffb8a99ce9321a32db0f08de8b18d48cda9b1877","src/bool/tests.rs":"a0e6d034036aa04aac6b847bb561bdba759d85c78d4cbb7fb93f4422efb83656","src/byte/mod.rs":"af5021c4f81e629177829fd0459ce3d164abeb23aa2d5528a2237e4e72c1fbff","src/byte/tests.rs":"cf0b4bf184a3376d5a90f8300fa51fee48849db18609dd00841084b78c4a8214","src/bytestr/mod.rs":"590d52ba2d61d10b7ad648eaac9a9ec1de7b86553c33c24c1cd7f730f8add161","src/bytestr/tests.rs":"cbc83534a52a3c909816dc00400de8f164917b2ab619f2e970cab8220ddfdd56","src/char/mod.rs":"16cb0d84945edaeff1cddc7f7a0520ab9b2afe19bec10548d4d88fb8eb5d36a6","src/char/tests.rs":"dc07b055f4c726cdcd4331aabfbc87b9f73c9f67b473c4200bc6ea15a9abe4b7","src/err.rs":"8678e3f962fcbeb01b0eb6c569912f5f3104e202c06bb3586d200019b7ff847a","src/escape.rs":"140738944527cbe552c93517cf101d4da4e75e3cf341d03825f32d790ab33600","src/float/mod.rs":"3eca51480da333dca7264223f00d80456b6fb7935eb0e799e755dfc769a66951","src/float/tests.rs":"042179ffbd7cfaba24282811207ea254f3711c2c3ebf2ae63dd8df40221b53e6","src/impls.rs":"c5dd37dd3ecd29c40a0ed243b907765a27729a1b1f73fa2c6762105feb6527bc","src/integer/mod.rs":"89cc7bc238a9f0298287fb7db3610fa69ae46fc149e40214df6aea0a3cf44782","src/integer/tests.rs":"fa7f68fdedcd53b2acaca2bd870a8d6c6bd590fead3cf81834401d7155d7b950","src/lib.rs":"6897158d3759d8c1ec19465234993a18731a0948ac33edb982bba448e4a87836","src/parse.rs":"92f8b3eb34f3b250232d104536d570c03235f79daf1d682a95504436c2482126","src/string/mod.rs":"6551733c782aa7f504f8d15ab6c5a8354d5ab9dfba64143dd5a36d24d30258ba","src/string/tests.rs":"90a4982fd7bc4e0eeca6bd41ddb71f0963af6226aeecffa4d57045f259d6e362","src/test_util.rs":"3badda83d7f256bb25b840820bc0d3a6523b4ded913555cbea5533b6ccad5654","src/tests.rs":"d13acd463be0dbb0ec67d21bd943c4eb05b918eec52a0eabe99dc7a40b3c3de3"},"package":"b487d13a3f4b465df87895a37b24e364907019afa12d943528df5b7abe0836f1"}
\ No newline at end of file
+{"files":{"CHANGELOG.md":"03cea7c394dd09087f6b2c7ba4b4641b5c2c50b32b7286cabd5be4850f62f170","Cargo.toml":"6ef884164a0139f0591a381ada2c99d850d38e5f3af3451efa12f808f8a799e0","LICENSE-APACHE":"62c7a1e35f56406896d7aa7ca52d0cc0d272ac022b5d2796e7d6905db8a3636a","LICENSE-MIT":"7dc1552e88f49132cb358b1b962fc5e79fa42d70bcbb88c526d33e45b8e98036","README.md":"533d31adf3b4258b838cd6a1cdb58139e2cf761c3c38aa4654f66f34335c9073","src/bool/mod.rs":"53c6eedfd94552689e51233fffb8a99ce9321a32db0f08de8b18d48cda9b1877","src/bool/tests.rs":"a0e6d034036aa04aac6b847bb561bdba759d85c78d4cbb7fb93f4422efb83656","src/byte/mod.rs":"ff2a3e6108a9b32ae0d925ec34735d20194d5c6b27af060516a46d21397c75be","src/byte/tests.rs":"ac36dace42cd151ac9d26cc35701bc8b65f8f1ed6ee1cfef4eeb6caa9dd702bc","src/bytestr/mod.rs":"8fd951374f7edc2077465cd4f97001eece46358f2bb0c45fddb2942aac6ee13b","src/bytestr/tests.rs":"194b28f157196260b1c2a612dfb36fb1dace491db2ed2bbb39227771ed6baf60","src/char/mod.rs":"2bb6f25da83670f18ec40f8a38565aa2294a4cdf81c8bbaf081531a32b6c6d0c","src/char/tests.rs":"9de497c8c7d7a139ff81f3d7bf8b5c682316d983bebb58c58d2af97f4cd26c35","src/err.rs":"54d000c4f37258c6886dd5b7069e2f5282e51aec3731feb77935582ae8c18908","src/escape.rs":"a944e95344df54c16bf4cc6a8fb01a81e2eac2aacd4758b938d3339212fce60c","src/float/mod.rs":"defaf83526acdc8f9b34c7d1ac17d866a93409dc392eb608160778d6bb4a1e25","src/float/tests.rs":"5875403f1a72104973ed83d0cf29d766e7b2fa5c23615c85a5f2eeed02b115c9","src/impls.rs":"c5dd37dd3ecd29c40a0ed243b907765a27729a1b1f73fa2c6762105feb6527bc","src/integer/mod.rs":"2b9109ddd34faf76fc9ce9dfb04bcc6aed4834231c74bd8a774bd256cc57c18a","src/integer/tests.rs":"01147ce9b6742bb1614cf863090699c54bf660b9f2c6a5eb529d67ae92230c0d","src/lib.rs":"2e79c8035d0fb77db9414b5569eeef13b6db8cde48ef2a45ffcf5f2492d02a4a","src/parse.rs":"e1fa4a76331d52f711e1b06cdba853a4f815281366f4f4f68b4c0a109f8a1734","src/string/mod.rs":"52a9cda38f7cd5b025bc5ec7edb8106487ba3d141789f5bc239c4561490cdc29","src/string/tests.rs":"1e0150ddd921a74ed5ebf6216708132d7768f3beb11a8c7bbfcf4ba01db40a5b","src/test_util.rs":"3badda83d7f256bb25b840820bc0d3a6523b4ded913555cbea5533b6ccad5654","src/tests.rs":"9f0dc2fe7a0eefb6575acd824767bb7d837a584dc7999ef59a457255a2cd7f3d"},"package":"b4ce301924b7887e9d637144fdade93f9dfff9b60981d4ac161db09720d39aa5"}
\ No newline at end of file
diff --git a/crates/litrs/Android.bp b/crates/litrs/Android.bp
index ca7e8b2..1adc935 100644
--- a/crates/litrs/Android.bp
+++ b/crates/litrs/Android.bp
@@ -18,7 +18,7 @@
host_cross_supported: false,
crate_name: "litrs",
cargo_env_compat: true,
- cargo_pkg_version: "0.3.0",
+ cargo_pkg_version: "0.4.1",
crate_root: "src/lib.rs",
edition: "2018",
features: [
diff --git a/crates/litrs/CHANGELOG.md b/crates/litrs/CHANGELOG.md
index 3d4ee99..e2927c2 100644
--- a/crates/litrs/CHANGELOG.md
+++ b/crates/litrs/CHANGELOG.md
@@ -5,6 +5,30 @@
## [Unreleased]
+## [0.4.1] - 2023-10-18
+- Fixed incorrectly labeling `27f32` a float literals in docs.
+- Added hint to integer literal docs about parsing as `u128`.
+
+## [0.4.0] - 2023-03-05
+### Added
+- Add ability to parse literals with arbitrary suffixes (e.g. `"foo"bla` or `23px`)
+- Add `suffix()` method to all literal types except `BoolLit`
+- Add `IntegerBase::value`
+- Add `from_suffix` and `suffix` methods to `FloatType` and `IntegerType`
+- Add `FromStr` and `Display` impls to `FloatType` and `IntegerType`
+
+### Changed
+- **Breaking**: Mark `FloatType` and `IntegerType` as `#[non_exhaustive]`
+- **Breaking**: Fix integer parsing for cases like `27f32`. `Literal::parse`
+ and `IntegerLit::parse` will both identify this as an integer literal.
+- **Breaking**: Fix float parsing by correctly rejecting inputs like `27f32`. A
+ float literal must have a period OR an exponent part, according to the spec.
+ Previously decimal integers were accepted in `FloatLit::parse`.
+- Improved some parts of the docs
+
+### Removed
+- **Breaking**: Remove `OwnedLiteral` and `SharedLiteral`
+
## [0.3.0] - 2022-12-19
### Breaking
- Bump MSRV (minimal supported Rust version) to 1.54
@@ -68,7 +92,9 @@
- Everything
-[Unreleased]: https://github.com/LukasKalbertodt/litrs/compare/v0.3.0...HEAD
+[Unreleased]: https://github.com/LukasKalbertodt/litrs/compare/v0.4.1...HEAD
+[0.4.1]: https://github.com/LukasKalbertodt/litrs/compare/v0.4.0...v0.4.1
+[0.4.0]: https://github.com/LukasKalbertodt/litrs/compare/v0.3.0...v0.4.0
[0.3.0]: https://github.com/LukasKalbertodt/litrs/compare/v0.2.3...v0.3.0
[0.2.3]: https://github.com/LukasKalbertodt/litrs/compare/v0.2.2...v0.2.3
[0.2.2]: https://github.com/LukasKalbertodt/litrs/compare/v0.2.1...v0.2.2
diff --git a/crates/litrs/Cargo.toml b/crates/litrs/Cargo.toml
index 62e2503..6e65403 100644
--- a/crates/litrs/Cargo.toml
+++ b/crates/litrs/Cargo.toml
@@ -13,7 +13,7 @@
edition = "2018"
rust-version = "1.54"
name = "litrs"
-version = "0.3.0"
+version = "0.4.1"
authors = ["Lukas Kalbertodt <lukas.kalbertodt@gmail.com>"]
exclude = [".github"]
description = """
@@ -42,5 +42,10 @@
version = "1"
optional = true
+[dependencies.unicode-xid]
+version = "0.2.4"
+optional = true
+
[features]
+check_suffix = ["unicode-xid"]
default = ["proc-macro2"]
diff --git a/crates/litrs/METADATA b/crates/litrs/METADATA
index fb25f66..e9503fd 100644
--- a/crates/litrs/METADATA
+++ b/crates/litrs/METADATA
@@ -1,17 +1,17 @@
name: "litrs"
description: "Parse and inspect Rust literals (i.e. tokens in the Rust programming language representing fixed values). Particularly useful for proc macros, but can also be used outside of a proc-macro context."
third_party {
- version: "0.3.0"
+ version: "0.4.1"
license_type: NOTICE
last_upgrade_date {
- year: 2023
- month: 2
- day: 3
+ year: 2024
+ month: 12
+ day: 21
}
homepage: "https://crates.io/crates/litrs"
identifier {
type: "Archive"
- value: "https://static.crates.io/crates/litrs/litrs-0.3.0.crate"
- version: "0.3.0"
+ value: "https://static.crates.io/crates/litrs/litrs-0.4.1.crate"
+ version: "0.4.1"
}
}
diff --git a/crates/litrs/src/byte/mod.rs b/crates/litrs/src/byte/mod.rs
index 7c64901..ffdff5d 100644
--- a/crates/litrs/src/byte/mod.rs
+++ b/crates/litrs/src/byte/mod.rs
@@ -4,6 +4,7 @@
Buffer, ParseError,
err::{perr, ParseErrorKind::*},
escape::unescape,
+ parse::check_suffix,
};
@@ -15,6 +16,8 @@
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct ByteLit<B: Buffer> {
raw: B,
+ /// Start index of the suffix or `raw.len()` if there is no suffix.
+ start_suffix: usize,
value: u8,
}
@@ -29,8 +32,8 @@
return Err(perr(None, InvalidByteLiteralStart));
}
- let value = parse_impl(&input)?;
- Ok(Self { raw: input, value })
+ let (value, start_suffix) = parse_impl(&input)?;
+ Ok(Self { raw: input, value, start_suffix })
}
/// Returns the byte value that this literal represents.
@@ -38,6 +41,11 @@
self.value
}
+ /// The optional suffix. Returns `""` if the suffix is empty/does not exist.
+ pub fn suffix(&self) -> &str {
+ &(*self.raw)[self.start_suffix..]
+ }
+
/// Returns the raw input that was passed to `parse`.
pub fn raw_input(&self) -> &str {
&self.raw
@@ -56,6 +64,7 @@
pub fn to_owned(&self) -> ByteLit<String> {
ByteLit {
raw: self.raw.to_owned(),
+ start_suffix: self.start_suffix,
value: self.value,
}
}
@@ -69,32 +78,29 @@
/// Precondition: must start with `b'`.
#[inline(never)]
-pub(crate) fn parse_impl(input: &str) -> Result<u8, ParseError> {
- if input.len() == 2 {
- return Err(perr(None, UnterminatedByteLiteral));
- }
- if *input.as_bytes().last().unwrap() != b'\'' {
- return Err(perr(None, UnterminatedByteLiteral));
- }
-
- let inner = &input[2..input.len() - 1];
- let first = inner.as_bytes().get(0).ok_or(perr(None, EmptyByteLiteral))?;
+pub(crate) fn parse_impl(input: &str) -> Result<(u8, usize), ParseError> {
+ let input_bytes = input.as_bytes();
+ let first = input_bytes.get(2).ok_or(perr(None, UnterminatedByteLiteral))?;
let (c, len) = match first {
- b'\'' => return Err(perr(2, UnescapedSingleQuote)),
- b'\n' | b'\t' | b'\r'
- => return Err(perr(2, UnescapedSpecialWhitespace)),
-
- b'\\' => unescape::<u8>(inner, 2)?,
+ b'\'' if input_bytes.get(3) == Some(&b'\'') => return Err(perr(2, UnescapedSingleQuote)),
+ b'\'' => return Err(perr(None, EmptyByteLiteral)),
+ b'\n' | b'\t' | b'\r' => return Err(perr(2, UnescapedSpecialWhitespace)),
+ b'\\' => unescape::<u8>(&input[2..], 2)?,
other if other.is_ascii() => (*other, 1),
_ => return Err(perr(2, NonAsciiInByteLiteral)),
};
- let rest = &inner[len..];
- if !rest.is_empty() {
- return Err(perr(len + 2..input.len() - 1, OverlongByteLiteral));
+ match input[2 + len..].find('\'') {
+ Some(0) => {}
+ Some(_) => return Err(perr(None, OverlongByteLiteral)),
+ None => return Err(perr(None, UnterminatedByteLiteral)),
}
- Ok(c)
+ let start_suffix = 2 + len + 1;
+ let suffix = &input[start_suffix..];
+ check_suffix(suffix).map_err(|kind| perr(start_suffix, kind))?;
+
+ Ok((c, start_suffix))
}
#[cfg(test)]
diff --git a/crates/litrs/src/byte/tests.rs b/crates/litrs/src/byte/tests.rs
index 08586b0..3cf16b5 100644
--- a/crates/litrs/src/byte/tests.rs
+++ b/crates/litrs/src/byte/tests.rs
@@ -3,16 +3,20 @@
// ===== Utility functions =======================================================================
macro_rules! check {
- ($lit:literal) => {
- let input = stringify!($lit);
+ ($lit:literal) => { check!($lit, stringify!($lit), "") };
+ ($lit:literal, $input:expr, $suffix:literal) => {
+ let input = $input;
let expected = ByteLit {
raw: input,
+ start_suffix: input.len() - $suffix.len(),
value: $lit,
};
assert_parse_ok_eq(input, ByteLit::parse(input), expected.clone(), "ByteLit::parse");
assert_parse_ok_eq(input, Literal::parse(input), Literal::Byte(expected), "Literal::parse");
- assert_eq!(ByteLit::parse(input).unwrap().value(), $lit);
+ let lit = ByteLit::parse(input).unwrap();
+ assert_eq!(lit.value(), $lit);
+ assert_eq!(lit.suffix(), $suffix);
assert_roundtrip(expected.to_owned(), input);
};
}
@@ -114,12 +118,22 @@
}
#[test]
+fn suffixes() {
+ check!(b'a', r##"b'a'peter"##, "peter");
+ check!(b'#', r##"b'#'peter"##, "peter");
+ check!(b'\n', r##"b'\n'peter"##, "peter");
+ check!(b'\'', r##"b'\''peter"##, "peter");
+ check!(b'\"', r##"b'\"'peter"##, "peter");
+ check!(b'\xFF', r##"b'\xFF'peter"##, "peter");
+}
+
+#[test]
fn invald_escapes() {
assert_err!(ByteLit, r"b'\a'", UnknownEscape, 2..4);
assert_err!(ByteLit, r"b'\y'", UnknownEscape, 2..4);
- assert_err!(ByteLit, r"b'\", UnterminatedByteLiteral, None);
- assert_err!(ByteLit, r"b'\x'", UnterminatedEscape, 2..4);
- assert_err!(ByteLit, r"b'\x1'", UnterminatedEscape, 2..5);
+ assert_err!(ByteLit, r"b'\", UnterminatedEscape, 2..3);
+ assert_err!(ByteLit, r"b'\x'", UnterminatedEscape, 2..5);
+ assert_err!(ByteLit, r"b'\x1'", InvalidXEscape, 2..6);
assert_err!(ByteLit, r"b'\xaj'", InvalidXEscape, 2..6);
assert_err!(ByteLit, r"b'\xjb'", InvalidXEscape, 2..6);
}
@@ -148,16 +162,16 @@
#[test]
fn parse_err() {
assert_err!(ByteLit, r"b''", EmptyByteLiteral, None);
- assert_err!(ByteLit, r"b' ''", OverlongByteLiteral, 3..4);
+ assert_err!(ByteLit, r"b' ''", UnexpectedChar, 4..5);
assert_err!(ByteLit, r"b'", UnterminatedByteLiteral, None);
assert_err!(ByteLit, r"b'a", UnterminatedByteLiteral, None);
assert_err!(ByteLit, r"b'\n", UnterminatedByteLiteral, None);
assert_err!(ByteLit, r"b'\x35", UnterminatedByteLiteral, None);
- assert_err!(ByteLit, r"b'ab'", OverlongByteLiteral, 3..4);
- assert_err!(ByteLit, r"b'a _'", OverlongByteLiteral, 3..5);
- assert_err!(ByteLit, r"b'\n3'", OverlongByteLiteral, 4..5);
+ assert_err!(ByteLit, r"b'ab'", OverlongByteLiteral, None);
+ assert_err!(ByteLit, r"b'a _'", OverlongByteLiteral, None);
+ assert_err!(ByteLit, r"b'\n3'", OverlongByteLiteral, None);
assert_err!(ByteLit, r"", Empty, None);
diff --git a/crates/litrs/src/bytestr/mod.rs b/crates/litrs/src/bytestr/mod.rs
index a2908b9..a0e0972 100644
--- a/crates/litrs/src/bytestr/mod.rs
+++ b/crates/litrs/src/bytestr/mod.rs
@@ -24,6 +24,9 @@
/// The number of hash signs in case of a raw string literal, or `None` if
/// it's not a raw string literal.
num_hashes: Option<u32>,
+
+ /// Start index of the suffix or `raw.len()` if there is no suffix.
+ start_suffix: usize,
}
impl<B: Buffer> ByteStringLit<B> {
@@ -37,7 +40,8 @@
return Err(perr(None, InvalidByteStringLiteralStart));
}
- Self::parse_impl(input)
+ let (value, num_hashes, start_suffix) = parse_impl(&input)?;
+ Ok(Self { raw: input, value, num_hashes, start_suffix })
}
/// Returns the string value this literal represents (where all escapes have
@@ -56,6 +60,11 @@
value.map(B::ByteCow::from).unwrap_or_else(|| raw.cut(inner_range).into_byte_cow())
}
+ /// The optional suffix. Returns `""` if the suffix is empty/does not exist.
+ pub fn suffix(&self) -> &str {
+ &(*self.raw)[self.start_suffix..]
+ }
+
/// Returns whether this literal is a raw string literal (starting with
/// `r`).
pub fn is_raw_byte_string(&self) -> bool {
@@ -75,27 +84,8 @@
/// The range within `self.raw` that excludes the quotes and potential `r#`.
fn inner_range(&self) -> Range<usize> {
match self.num_hashes {
- None => 2..self.raw.len() - 1,
- Some(n) => 2 + n as usize + 1..self.raw.len() - n as usize - 1,
- }
- }
-
- /// Precondition: input has to start with either `b"` or `br`.
- pub(crate) fn parse_impl(input: B) -> Result<Self, ParseError> {
- if input.starts_with(r"br") {
- let (value, num_hashes) = scan_raw_string::<u8>(&input, 2)?;
- Ok(Self {
- raw: input,
- value: value.map(|s| s.into_bytes()),
- num_hashes: Some(num_hashes),
- })
- } else {
- let value = unescape_string::<u8>(&input, 2)?.map(|s| s.into_bytes());
- Ok(Self {
- raw: input,
- value,
- num_hashes: None,
- })
+ None => 2..self.start_suffix - 1,
+ Some(n) => 2 + n as usize + 1..self.start_suffix - n as usize - 1,
}
}
}
@@ -108,6 +98,7 @@
raw: self.raw.to_owned(),
value: self.value,
num_hashes: self.num_hashes,
+ start_suffix: self.start_suffix,
}
}
}
@@ -119,5 +110,17 @@
}
+/// Precondition: input has to start with either `b"` or `br`.
+#[inline(never)]
+fn parse_impl(input: &str) -> Result<(Option<Vec<u8>>, Option<u32>, usize), ParseError> {
+ if input.starts_with("br") {
+ scan_raw_string::<u8>(&input, 2)
+ .map(|(v, num, start_suffix)| (v.map(String::into_bytes), Some(num), start_suffix))
+ } else {
+ unescape_string::<u8>(&input, 2)
+ .map(|(v, start_suffix)| (v.map(String::into_bytes), None, start_suffix))
+ }
+}
+
#[cfg(test)]
mod tests;
diff --git a/crates/litrs/src/bytestr/tests.rs b/crates/litrs/src/bytestr/tests.rs
index b0480fd..2afef5a 100644
--- a/crates/litrs/src/bytestr/tests.rs
+++ b/crates/litrs/src/bytestr/tests.rs
@@ -4,19 +4,25 @@
macro_rules! check {
($lit:literal, $has_escapes:expr, $num_hashes:expr) => {
- let input = stringify!($lit);
+ check!($lit, stringify!($lit), $has_escapes, $num_hashes, "")
+ };
+ ($lit:literal, $input:expr, $has_escapes:expr, $num_hashes:expr, $suffix:literal) => {
+ let input = $input;
let expected = ByteStringLit {
raw: input,
value: if $has_escapes { Some($lit.to_vec()) } else { None },
num_hashes: $num_hashes,
+ start_suffix: input.len() - $suffix.len(),
};
assert_parse_ok_eq(
input, ByteStringLit::parse(input), expected.clone(), "ByteStringLit::parse");
assert_parse_ok_eq(
input, Literal::parse(input), Literal::ByteString(expected.clone()), "Literal::parse");
- assert_eq!(ByteStringLit::parse(input).unwrap().value(), $lit);
- assert_eq!(ByteStringLit::parse(input).unwrap().into_value().as_ref(), $lit);
+ let lit = ByteStringLit::parse(input).unwrap();
+ assert_eq!(lit.value(), $lit);
+ assert_eq!(lit.suffix(), $suffix);
+ assert_eq!(lit.into_value().as_ref(), $lit);
assert_roundtrip(expected.into_owned(), input);
};
}
@@ -43,6 +49,7 @@
raw: &*input,
value: None,
num_hashes,
+ start_suffix: input.len(),
};
assert_parse_ok_eq(
&input, ByteStringLit::parse(&*input), expected.clone(), "ByteStringLit::parse");
@@ -148,16 +155,22 @@
}
#[test]
+fn suffixes() {
+ check!(b"hello", r###"b"hello"suffix"###, false, None, "suffix");
+ check!(b"fox", r#"b"fox"peter"#, false, None, "peter");
+ check!(b"a\x0cb\\", r#"b"a\x0cb\\"_jürgen"#, true, None, "_jürgen");
+ check!(br"a\x0cb\\", r###"br#"a\x0cb\\"#_jürgen"###, false, Some(1), "_jürgen");
+}
+
+#[test]
fn parse_err() {
assert_err!(ByteStringLit, r#"b""#, UnterminatedString, None);
assert_err!(ByteStringLit, r#"b"cat"#, UnterminatedString, None);
assert_err!(ByteStringLit, r#"b"Jurgen"#, UnterminatedString, None);
assert_err!(ByteStringLit, r#"b"foo bar baz"#, UnterminatedString, None);
- assert_err!(ByteStringLit, r#"b"fox"peter"#, UnexpectedChar, 6..11);
- assert_err!(ByteStringLit, r#"b"fox"peter""#, UnexpectedChar, 6..12);
- assert_err!(ByteStringLit, r#"b"fox"bar"#, UnexpectedChar, 6..9);
- assert_err!(ByteStringLit, r###"br#"foo "# bar"#"###, UnexpectedChar, 10..16);
+ assert_err!(ByteStringLit, r#"b"fox"peter""#, InvalidSuffix, 6);
+ assert_err!(ByteStringLit, r###"br#"foo "# bar"#"###, UnexpectedChar, 10);
assert_err!(ByteStringLit, "b\"\r\"", IsolatedCr, 2);
assert_err!(ByteStringLit, "b\"fo\rx\"", IsolatedCr, 4);
@@ -179,10 +192,10 @@
}
#[test]
-fn invald_escapes() {
+fn invalid_escapes() {
assert_err!(ByteStringLit, r#"b"\a""#, UnknownEscape, 2..4);
assert_err!(ByteStringLit, r#"b"foo\y""#, UnknownEscape, 5..7);
- assert_err!(ByteStringLit, r#"b"\"#, UnterminatedString, None);
+ assert_err!(ByteStringLit, r#"b"\"#, UnterminatedEscape, 2);
assert_err!(ByteStringLit, r#"b"\x""#, UnterminatedEscape, 2..4);
assert_err!(ByteStringLit, r#"b"foo\x1""#, UnterminatedEscape, 5..8);
assert_err!(ByteStringLit, r#"b" \xaj""#, InvalidXEscape, 3..7);
diff --git a/crates/litrs/src/char/mod.rs b/crates/litrs/src/char/mod.rs
index 96d5037..54f6f11 100644
--- a/crates/litrs/src/char/mod.rs
+++ b/crates/litrs/src/char/mod.rs
@@ -4,7 +4,7 @@
Buffer, ParseError,
err::{perr, ParseErrorKind::*},
escape::unescape,
- parse::first_byte_or_empty,
+ parse::{first_byte_or_empty, check_suffix},
};
@@ -16,6 +16,8 @@
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct CharLit<B: Buffer> {
raw: B,
+ /// Start index of the suffix or `raw.len()` if there is no suffix.
+ start_suffix: usize,
value: char,
}
@@ -25,8 +27,8 @@
pub fn parse(input: B) -> Result<Self, ParseError> {
match first_byte_or_empty(&input)? {
b'\'' => {
- let value = parse_impl(&input)?;
- Ok(Self { raw: input, value })
+ let (value, start_suffix) = parse_impl(&input)?;
+ Ok(Self { raw: input, value, start_suffix })
},
_ => Err(perr(0, DoesNotStartWithQuote)),
}
@@ -37,6 +39,11 @@
self.value
}
+ /// The optional suffix. Returns `""` if the suffix is empty/does not exist.
+ pub fn suffix(&self) -> &str {
+ &(*self.raw)[self.start_suffix..]
+ }
+
/// Returns the raw input that was passed to `parse`.
pub fn raw_input(&self) -> &str {
&self.raw
@@ -55,6 +62,7 @@
pub fn to_owned(&self) -> CharLit<String> {
CharLit {
raw: self.raw.to_owned(),
+ start_suffix: self.start_suffix,
value: self.value,
}
}
@@ -68,31 +76,29 @@
/// Precondition: first character in input must be `'`.
#[inline(never)]
-pub(crate) fn parse_impl(input: &str) -> Result<char, ParseError> {
- if input.len() == 1 {
- return Err(perr(None, UnterminatedCharLiteral));
- }
- if *input.as_bytes().last().unwrap() != b'\'' {
- return Err(perr(None, UnterminatedCharLiteral));
- }
-
- let inner = &input[1..input.len() - 1];
- let first = inner.chars().nth(0).ok_or(perr(None, EmptyCharLiteral))?;
+pub(crate) fn parse_impl(input: &str) -> Result<(char, usize), ParseError> {
+ let first = input.chars().nth(1).ok_or(perr(None, UnterminatedCharLiteral))?;
let (c, len) = match first {
- '\'' => return Err(perr(1, UnescapedSingleQuote)),
+ '\'' if input.chars().nth(2) == Some('\'') => return Err(perr(1, UnescapedSingleQuote)),
+ '\'' => return Err(perr(None, EmptyCharLiteral)),
'\n' | '\t' | '\r'
=> return Err(perr(1, UnescapedSpecialWhitespace)),
- '\\' => unescape::<char>(inner, 1)?,
+ '\\' => unescape::<char>(&input[1..], 1)?,
other => (other, other.len_utf8()),
};
- let rest = &inner[len..];
- if !rest.is_empty() {
- return Err(perr(len + 1..input.len() - 1, OverlongCharLiteral));
+ match input[1 + len..].find('\'') {
+ Some(0) => {}
+ Some(_) => return Err(perr(None, OverlongCharLiteral)),
+ None => return Err(perr(None, UnterminatedCharLiteral)),
}
- Ok(c)
+ let start_suffix = 1 + len + 1;
+ let suffix = &input[start_suffix..];
+ check_suffix(suffix).map_err(|kind| perr(start_suffix, kind))?;
+
+ Ok((c, start_suffix))
}
#[cfg(test)]
diff --git a/crates/litrs/src/char/tests.rs b/crates/litrs/src/char/tests.rs
index bfae5e4..19219db 100644
--- a/crates/litrs/src/char/tests.rs
+++ b/crates/litrs/src/char/tests.rs
@@ -4,16 +4,20 @@
// ===== Utility functions =======================================================================
macro_rules! check {
- ($lit:literal) => {
- let input = stringify!($lit);
+ ($lit:literal) => { check!($lit, stringify!($lit), "") };
+ ($lit:literal, $input:expr, $suffix:literal) => {
+ let input = $input;
let expected = CharLit {
raw: input,
+ start_suffix: input.len() - $suffix.len(),
value: $lit,
};
assert_parse_ok_eq(input, CharLit::parse(input), expected.clone(), "CharLit::parse");
assert_parse_ok_eq(input, Literal::parse(input), Literal::Char(expected), "Literal::parse");
- assert_eq!(CharLit::parse(input).unwrap().value(), $lit);
+ let lit = CharLit::parse(input).unwrap();
+ assert_eq!(lit.value(), $lit);
+ assert_eq!(lit.suffix(), $suffix);
assert_roundtrip(expected.to_owned(), input);
};
}
@@ -135,6 +139,15 @@
}
#[test]
+fn suffixes() {
+ check!('a', r##"'a'peter"##, "peter");
+ check!('#', r##"'#'peter"##, "peter");
+ check!('\n', r##"'\n'peter"##, "peter");
+ check!('\'', r##"'\''peter"##, "peter");
+ check!('\"', r##"'\"'peter"##, "peter");
+}
+
+#[test]
fn invald_ascii_escapes() {
assert_err!(CharLit, r"'\x80'", NonAsciiXEscape, 1..5);
assert_err!(CharLit, r"'\x81'", NonAsciiXEscape, 1..5);
@@ -151,12 +164,12 @@
}
#[test]
-fn invald_escapes() {
+fn invalid_escapes() {
assert_err!(CharLit, r"'\a'", UnknownEscape, 1..3);
assert_err!(CharLit, r"'\y'", UnknownEscape, 1..3);
- assert_err!(CharLit, r"'\", UnterminatedCharLiteral, None);
- assert_err!(CharLit, r"'\x'", UnterminatedEscape, 1..3);
- assert_err!(CharLit, r"'\x1'", UnterminatedEscape, 1..4);
+ assert_err!(CharLit, r"'\", UnterminatedEscape, 1);
+ assert_err!(CharLit, r"'\x'", UnterminatedEscape, 1..4);
+ assert_err!(CharLit, r"'\x1'", InvalidXEscape, 1..5);
assert_err!(CharLit, r"'\xaj'", InvalidXEscape, 1..5);
assert_err!(CharLit, r"'\xjb'", InvalidXEscape, 1..5);
}
@@ -167,10 +180,10 @@
assert_err!(CharLit, r"'\u '", UnicodeEscapeWithoutBrace, 1..3);
assert_err!(CharLit, r"'\u3'", UnicodeEscapeWithoutBrace, 1..3);
- assert_err!(CharLit, r"'\u{'", UnterminatedUnicodeEscape, 1..4);
- assert_err!(CharLit, r"'\u{12'", UnterminatedUnicodeEscape, 1..6);
- assert_err!(CharLit, r"'\u{a0b'", UnterminatedUnicodeEscape, 1..7);
- assert_err!(CharLit, r"'\u{a0_b '", UnterminatedUnicodeEscape, 1..10);
+ assert_err!(CharLit, r"'\u{'", UnterminatedUnicodeEscape, 1..5);
+ assert_err!(CharLit, r"'\u{12'", UnterminatedUnicodeEscape, 1..7);
+ assert_err!(CharLit, r"'\u{a0b'", UnterminatedUnicodeEscape, 1..8);
+ assert_err!(CharLit, r"'\u{a0_b '", UnterminatedUnicodeEscape, 1..11);
assert_err!(CharLit, r"'\u{_}'", InvalidStartOfUnicodeEscape, 4);
assert_err!(CharLit, r"'\u{_5f}'", InvalidStartOfUnicodeEscape, 4);
@@ -192,16 +205,16 @@
#[test]
fn parse_err() {
assert_err!(CharLit, r"''", EmptyCharLiteral, None);
- assert_err!(CharLit, r"' ''", OverlongCharLiteral, 2..3);
+ assert_err!(CharLit, r"' ''", UnexpectedChar, 3);
assert_err!(CharLit, r"'", UnterminatedCharLiteral, None);
assert_err!(CharLit, r"'a", UnterminatedCharLiteral, None);
assert_err!(CharLit, r"'\n", UnterminatedCharLiteral, None);
assert_err!(CharLit, r"'\x35", UnterminatedCharLiteral, None);
- assert_err!(CharLit, r"'ab'", OverlongCharLiteral, 2..3);
- assert_err!(CharLit, r"'a _'", OverlongCharLiteral, 2..4);
- assert_err!(CharLit, r"'\n3'", OverlongCharLiteral, 3..4);
+ assert_err!(CharLit, r"'ab'", OverlongCharLiteral, None);
+ assert_err!(CharLit, r"'a _'", OverlongCharLiteral, None);
+ assert_err!(CharLit, r"'\n3'", OverlongCharLiteral, None);
assert_err!(CharLit, r"", Empty, None);
diff --git a/crates/litrs/src/err.rs b/crates/litrs/src/err.rs
index 1011550..86d51dc 100644
--- a/crates/litrs/src/err.rs
+++ b/crates/litrs/src/err.rs
@@ -221,13 +221,6 @@
/// Integer literal does not contain any valid digits.
NoDigits,
- /// Found a integer type suffix that is invalid.
- InvalidIntegerTypeSuffix,
-
- /// Found a float type suffix that is invalid. Only `f32` and `f64` are
- /// valid.
- InvalidFloatTypeSuffix,
-
/// Exponent of a float literal does not contain any digits.
NoExponentDigits,
@@ -309,6 +302,17 @@
/// An literal `\r` character not followed by a `\n` character in a
/// (raw) string or byte string literal.
IsolatedCr,
+
+ /// Literal suffix is not a valid identifier.
+ InvalidSuffix,
+
+ /// Returned by `Float::parse` if an integer literal (no fractional nor
+ /// exponent part) is passed.
+ UnexpectedIntegerLit,
+
+ /// Integer suffixes cannot start with `e` or `E` as this conflicts with the
+ /// grammar for float literals.
+ IntegerSuffixStartingWithE,
}
impl std::error::Error for ParseError {}
@@ -324,8 +328,6 @@
DoesNotStartWithDigit => "number literal does not start with decimal digit",
InvalidDigit => "integer literal contains a digit invalid for its base",
NoDigits => "integer literal does not contain any digits",
- InvalidIntegerTypeSuffix => "invalid integer type suffix",
- InvalidFloatTypeSuffix => "invalid floating point type suffix",
NoExponentDigits => "exponent of floating point literal does not contain any digits",
UnknownEscape => "unknown escape",
UnterminatedEscape => "unterminated escape: input ended too soon",
@@ -354,6 +356,9 @@
InvalidByteLiteralStart => "invalid start for byte literal",
InvalidByteStringLiteralStart => "invalid start for byte string literal",
IsolatedCr => r"`\r` not immediately followed by `\n` in string",
+ InvalidSuffix => "literal suffix is not a valid identifier",
+ UnexpectedIntegerLit => "expected float literal, but found integer",
+ IntegerSuffixStartingWithE => "integer literal suffix must not start with 'e' or 'E'",
};
description.fmt(f)?;
diff --git a/crates/litrs/src/escape.rs b/crates/litrs/src/escape.rs
index 19b63a1..5eb8382 100644
--- a/crates/litrs/src/escape.rs
+++ b/crates/litrs/src/escape.rs
@@ -1,4 +1,4 @@
-use crate::{ParseError, err::{perr, ParseErrorKind::*}, parse::hex_digit_value};
+use crate::{ParseError, err::{perr, ParseErrorKind::*}, parse::{hex_digit_value, check_suffix}};
/// Must start with `\`
@@ -117,14 +117,15 @@
pub(crate) fn unescape_string<E: Escapee>(
input: &str,
offset: usize,
-) -> Result<Option<String>, ParseError> {
+) -> Result<(Option<String>, usize), ParseError> {
+ let mut closing_quote_pos = None;
let mut i = offset;
let mut end_last_escape = offset;
let mut value = String::new();
- while i < input.len() - 1 {
+ while i < input.len() {
match input.as_bytes()[i] {
// Handle "string continue".
- b'\\' if input.as_bytes()[i + 1] == b'\n' => {
+ b'\\' if input.as_bytes().get(i + 1) == Some(&b'\n') => {
value.push_str(&input[end_last_escape..i]);
// Find the first non-whitespace character.
@@ -143,7 +144,7 @@
end_last_escape = i;
}
b'\r' => {
- if input.as_bytes()[i + 1] == b'\n' {
+ if input.as_bytes().get(i + 1) == Some(&b'\n') {
value.push_str(&input[end_last_escape..i]);
value.push('\n');
i += 2;
@@ -152,16 +153,21 @@
return Err(perr(i, IsolatedCr))
}
}
- b'"' => return Err(perr(i + 1..input.len(), UnexpectedChar)),
+ b'"' => {
+ closing_quote_pos = Some(i);
+ break;
+ },
b if !E::SUPPORTS_UNICODE && !b.is_ascii()
=> return Err(perr(i, NonAsciiInByteLiteral)),
_ => i += 1,
}
}
- if input.as_bytes()[input.len() - 1] != b'"' || input.len() == offset {
- return Err(perr(None, UnterminatedString));
- }
+ let closing_quote_pos = closing_quote_pos.ok_or(perr(None, UnterminatedString))?;
+
+ let start_suffix = closing_quote_pos + 1;
+ let suffix = &input[start_suffix..];
+ check_suffix(suffix).map_err(|kind| perr(start_suffix, kind))?;
// `value` is only empty if there was no escape in the input string
// (with the special case of the input being empty). This means the
@@ -171,11 +177,11 @@
} else {
// There was an escape in the string, so we need to push the
// remaining unescaped part of the string still.
- value.push_str(&input[end_last_escape..input.len() - 1]);
+ value.push_str(&input[end_last_escape..closing_quote_pos]);
Some(value)
};
- Ok(value)
+ Ok((value, start_suffix))
}
/// Reads and checks a raw (byte) string literal, converting `\r\n` sequences to
@@ -185,7 +191,7 @@
pub(crate) fn scan_raw_string<E: Escapee>(
input: &str,
offset: usize,
-) -> Result<(Option<String>, u32), ParseError> {
+) -> Result<(Option<String>, u32, usize), ParseError> {
// Raw string literal
let num_hashes = input[offset..].bytes().position(|b| b != b'#')
.ok_or(perr(None, InvalidLiteral))?;
@@ -234,12 +240,11 @@
i += 1;
}
- let closing_quote_pos = closing_quote_pos
- .ok_or(perr(None, UnterminatedRawString))?;
+ let closing_quote_pos = closing_quote_pos.ok_or(perr(None, UnterminatedRawString))?;
- if closing_quote_pos + num_hashes != input.len() - 1 {
- return Err(perr(closing_quote_pos + num_hashes + 1..input.len(), UnexpectedChar));
- }
+ let start_suffix = closing_quote_pos + num_hashes + 1;
+ let suffix = &input[start_suffix..];
+ check_suffix(suffix).map_err(|kind| perr(start_suffix, kind))?;
// `value` is only empty if there was no \r\n in the input string (with the
// special case of the input being empty). This means the string value
@@ -253,5 +258,5 @@
Some(value)
};
- Ok((value, num_hashes as u32))
+ Ok((value, num_hashes as u32, start_suffix))
}
diff --git a/crates/litrs/src/float/mod.rs b/crates/litrs/src/float/mod.rs
index b196845..0518633 100644
--- a/crates/litrs/src/float/mod.rs
+++ b/crates/litrs/src/float/mod.rs
@@ -1,21 +1,22 @@
-use std::fmt;
+use std::{fmt, str::FromStr};
use crate::{
Buffer, ParseError,
err::{perr, ParseErrorKind::*},
- parse::{end_dec_digits, first_byte_or_empty},
+ parse::{end_dec_digits, first_byte_or_empty, check_suffix},
};
-/// A floating point literal, e.g. `3.14`, `8.`, `135e12`, `27f32` or `1.956e2f64`.
+/// A floating point literal, e.g. `3.14`, `8.`, `135e12`, or `1.956e2f64`.
///
/// This kind of literal has several forms, but generally consists of a main
/// number part, an optional exponent and an optional type suffix. See
/// [the reference][ref] for more information.
///
/// A leading minus sign `-` is not part of the literal grammar! `-3.14` are two
-/// tokens in the Rust grammar.
+/// tokens in the Rust grammar. Further, `27` and `27f32` are both not float,
+/// but integer literals! Consequently `FloatLit::parse` will reject them.
///
///
/// [ref]: https://doc.rust-lang.org/reference/tokens.html#floating-point-literals
@@ -52,21 +53,13 @@
/// The first index after the whole number part (everything except type suffix).
end_number_part: usize,
-
- /// Optional type suffix.
- type_suffix: Option<FloatType>,
-}
-
-/// All possible float type suffixes.
-#[derive(Debug, Clone, Copy, PartialEq, Eq)]
-pub enum FloatType {
- F32,
- F64,
}
impl<B: Buffer> FloatLit<B> {
/// Parses the input as a floating point literal. Returns an error if the
- /// input is invalid or represents a different kind of literal.
+ /// input is invalid or represents a different kind of literal. Will also
+ /// reject decimal integer literals like `23` or `17f32`, in accordance
+ /// with the spec.
pub fn parse(s: B) -> Result<Self, ParseError> {
match first_byte_or_empty(&s)? {
b'0'..=b'9' => {
@@ -75,26 +68,19 @@
end_integer_part,
end_fractional_part,
end_number_part,
- type_suffix,
..
} = parse_impl(&s)?;
- Ok(Self {
- raw: s,
- end_integer_part,
- end_fractional_part,
- end_number_part,
- type_suffix,
- })
+ Ok(Self { raw: s, end_integer_part, end_fractional_part, end_number_part })
},
_ => Err(perr(0, DoesNotStartWithDigit)),
}
}
- /// Returns the whole number part (including integer part, fractional part
- /// and exponent), but without the type suffix. If you want an actual
- /// floating point value, you need to parse this string, e.g. with
- /// `f32::from_str` or an external crate.
+ /// Returns the number part (including integer part, fractional part and
+ /// exponent), but without the suffix. If you want an actual floating
+ /// point value, you need to parse this string, e.g. with `f32::from_str`
+ /// or an external crate.
pub fn number_part(&self) -> &str {
&(*self.raw)[..self.end_number_part]
}
@@ -121,9 +107,9 @@
&(*self.raw)[self.end_fractional_part..self.end_number_part]
}
- /// The optional type suffix.
- pub fn type_suffix(&self) -> Option<FloatType> {
- self.type_suffix
+ /// The optional suffix. Returns `""` if the suffix is empty/does not exist.
+ pub fn suffix(&self) -> &str {
+ &(*self.raw)[self.end_number_part..]
}
/// Returns the raw input that was passed to `parse`.
@@ -146,7 +132,6 @@
end_integer_part: self.end_integer_part,
end_fractional_part: self.end_fractional_part,
end_number_part: self.end_number_part,
- type_suffix: self.type_suffix,
}
}
}
@@ -184,7 +169,6 @@
return Err(perr(end_integer_part + 1, UnexpectedChar));
}
-
// Optional exponent.
let end_number_part = if rest.starts_with('e') || rest.starts_with('E') {
// Strip single - or + sign at the beginning.
@@ -207,23 +191,67 @@
end_fractional_part
};
+ // Make sure the suffix is valid.
+ let suffix = &input[end_number_part..];
+ check_suffix(suffix).map_err(|kind| perr(end_number_part..input.len(), kind))?;
- // Type suffix
- let type_suffix = match &input[end_number_part..] {
- "" => None,
- "f32" => Some(FloatType::F32),
- "f64" => Some(FloatType::F64),
- _ => return Err(perr(end_number_part..input.len(), InvalidFloatTypeSuffix)),
- };
+ // A float literal needs either a fractional or exponent part, otherwise its
+ // an integer literal.
+ if end_integer_part == end_number_part {
+ return Err(perr(None, UnexpectedIntegerLit));
+ }
Ok(FloatLit {
raw: input,
end_integer_part,
end_fractional_part,
end_number_part,
- type_suffix,
})
}
+
+/// All possible float type suffixes.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+#[non_exhaustive]
+pub enum FloatType {
+ F32,
+ F64,
+}
+
+impl FloatType {
+ /// Returns the type corresponding to the given suffix (e.g. `"f32"` is
+ /// mapped to `Self::F32`). If the suffix is not a valid float type, `None`
+ /// is returned.
+ pub fn from_suffix(suffix: &str) -> Option<Self> {
+ match suffix {
+ "f32" => Some(FloatType::F32),
+ "f64" => Some(FloatType::F64),
+ _ => None,
+ }
+ }
+
+ /// Returns the suffix for this type, e.g. `"f32"` for `Self::F32`.
+ pub fn suffix(self) -> &'static str {
+ match self {
+ Self::F32 => "f32",
+ Self::F64 => "f64",
+ }
+ }
+}
+
+impl FromStr for FloatType {
+ type Err = ();
+ fn from_str(s: &str) -> Result<Self, Self::Err> {
+ Self::from_suffix(s).ok_or(())
+ }
+}
+
+impl fmt::Display for FloatType {
+ fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+ self.suffix().fmt(f)
+ }
+}
+
+
#[cfg(test)]
mod tests;
diff --git a/crates/litrs/src/float/tests.rs b/crates/litrs/src/float/tests.rs
index f15af05..f22443b 100644
--- a/crates/litrs/src/float/tests.rs
+++ b/crates/litrs/src/float/tests.rs
@@ -20,18 +20,18 @@
end_integer_part: $intpart.len(),
end_fractional_part: $intpart.len() + $fracpart.len(),
end_number_part: $intpart.len() + $fracpart.len() + $exppart.len(),
- type_suffix: check!(@ty $suffix),
};
assert_parse_ok_eq(
input, FloatLit::parse(input), expected_float.clone(), "FloatLit::parse");
assert_parse_ok_eq(
input, Literal::parse(input), Literal::Float(expected_float), "Literal::parse");
+ assert_eq!(FloatLit::parse(input).unwrap().suffix(), check!(@ty $suffix));
assert_roundtrip(expected_float.to_owned(), input);
};
- (@ty f32) => { Some(FloatType::F32) };
- (@ty f64) => { Some(FloatType::F64) };
- (@ty -) => { None };
+ (@ty f32) => { "f32" };
+ (@ty f64) => { "f64" };
+ (@ty -) => { "" };
(@stringify_suffix -) => { "" };
(@stringify_suffix $suffix:ident) => { stringify!($suffix) };
}
@@ -46,42 +46,42 @@
assert_eq!(f.integer_part(), "3");
assert_eq!(f.fractional_part(), Some("14"));
assert_eq!(f.exponent_part(), "");
- assert_eq!(f.type_suffix(), None);
+ assert_eq!(f.suffix(), "");
let f = FloatLit::parse("9.")?;
assert_eq!(f.number_part(), "9.");
assert_eq!(f.integer_part(), "9");
assert_eq!(f.fractional_part(), Some(""));
assert_eq!(f.exponent_part(), "");
- assert_eq!(f.type_suffix(), None);
+ assert_eq!(f.suffix(), "");
let f = FloatLit::parse("8e1")?;
assert_eq!(f.number_part(), "8e1");
assert_eq!(f.integer_part(), "8");
assert_eq!(f.fractional_part(), None);
assert_eq!(f.exponent_part(), "e1");
- assert_eq!(f.type_suffix(), None);
+ assert_eq!(f.suffix(), "");
let f = FloatLit::parse("8E3")?;
assert_eq!(f.number_part(), "8E3");
assert_eq!(f.integer_part(), "8");
assert_eq!(f.fractional_part(), None);
assert_eq!(f.exponent_part(), "E3");
- assert_eq!(f.type_suffix(), None);
+ assert_eq!(f.suffix(), "");
let f = FloatLit::parse("8_7_6.1_23e15")?;
assert_eq!(f.number_part(), "8_7_6.1_23e15");
assert_eq!(f.integer_part(), "8_7_6");
assert_eq!(f.fractional_part(), Some("1_23"));
assert_eq!(f.exponent_part(), "e15");
- assert_eq!(f.type_suffix(), None);
+ assert_eq!(f.suffix(), "");
let f = FloatLit::parse("8.2e-_04_9")?;
assert_eq!(f.number_part(), "8.2e-_04_9");
assert_eq!(f.integer_part(), "8");
assert_eq!(f.fractional_part(), Some("2"));
assert_eq!(f.exponent_part(), "e-_04_9");
- assert_eq!(f.type_suffix(), None);
+ assert_eq!(f.suffix(), "");
Ok(())
}
@@ -93,28 +93,28 @@
assert_eq!(f.integer_part(), "3");
assert_eq!(f.fractional_part(), Some("14"));
assert_eq!(f.exponent_part(), "");
- assert_eq!(f.type_suffix(), Some(FloatType::F32));
+ assert_eq!(FloatType::from_suffix(f.suffix()), Some(FloatType::F32));
let f = FloatLit::parse("8e1f64")?;
assert_eq!(f.number_part(), "8e1");
assert_eq!(f.integer_part(), "8");
assert_eq!(f.fractional_part(), None);
assert_eq!(f.exponent_part(), "e1");
- assert_eq!(f.type_suffix(), Some(FloatType::F64));
+ assert_eq!(FloatType::from_suffix(f.suffix()), Some(FloatType::F64));
let f = FloatLit::parse("8_7_6.1_23e15f32")?;
assert_eq!(f.number_part(), "8_7_6.1_23e15");
assert_eq!(f.integer_part(), "8_7_6");
assert_eq!(f.fractional_part(), Some("1_23"));
assert_eq!(f.exponent_part(), "e15");
- assert_eq!(f.type_suffix(), Some(FloatType::F32));
+ assert_eq!(FloatType::from_suffix(f.suffix()), Some(FloatType::F32));
let f = FloatLit::parse("8.2e-_04_9f64")?;
assert_eq!(f.number_part(), "8.2e-_04_9");
assert_eq!(f.integer_part(), "8");
assert_eq!(f.fractional_part(), Some("2"));
assert_eq!(f.exponent_part(), "e-_04_9");
- assert_eq!(f.type_suffix(), Some(FloatType::F64));
+ assert_eq!(FloatType::from_suffix(f.suffix()), Some(FloatType::F64));
Ok(())
}
@@ -125,7 +125,6 @@
check!("3" ".14" "" f32);
check!("3" ".14" "" f64);
- check!("3" "" "" f32);
check!("3" "" "e987654321" -);
check!("3" "" "e987654321" f64);
@@ -158,6 +157,47 @@
}
#[test]
+fn non_standard_suffixes() {
+ #[track_caller]
+ fn check_suffix(
+ input: &str,
+ integer_part: &str,
+ fractional_part: Option<&str>,
+ exponent_part: &str,
+ suffix: &str,
+ ) {
+ let lit = FloatLit::parse(input)
+ .unwrap_or_else(|e| panic!("expected to parse '{}' but got {}", input, e));
+ assert_eq!(lit.integer_part(), integer_part);
+ assert_eq!(lit.fractional_part(), fractional_part);
+ assert_eq!(lit.exponent_part(), exponent_part);
+ assert_eq!(lit.suffix(), suffix);
+
+ let lit = match Literal::parse(input) {
+ Ok(Literal::Float(f)) => f,
+ other => panic!("Expected float literal, but got {:?} for '{}'", other, input),
+ };
+ assert_eq!(lit.integer_part(), integer_part);
+ assert_eq!(lit.fractional_part(), fractional_part);
+ assert_eq!(lit.exponent_part(), exponent_part);
+ assert_eq!(lit.suffix(), suffix);
+ }
+
+ check_suffix("7.1f23", "7", Some("1"), "", "f23");
+ check_suffix("7.1f320", "7", Some("1"), "", "f320");
+ check_suffix("7.1f64_", "7", Some("1"), "", "f64_");
+ check_suffix("8.1f649", "8", Some("1"), "", "f649");
+ check_suffix("8.1f64f32", "8", Some("1"), "", "f64f32");
+ check_suffix("23e2_banana", "23", None, "e2_", "banana");
+ check_suffix("23.2_banana", "23", Some("2_"), "", "banana");
+ check_suffix("23e2pe55ter", "23", None, "e2", "pe55ter");
+ check_suffix("23e2p_e55ter", "23", None, "e2", "p_e55ter");
+ check_suffix("3.15Jürgen", "3", Some("15"), "", "Jürgen");
+ check_suffix("3e2e5", "3", None, "e2", "e5");
+ check_suffix("3e2e5f", "3", None, "e2", "e5f");
+}
+
+#[test]
fn parse_err() {
assert_err!(FloatLit, "", Empty, None);
assert_err_single!(FloatLit::parse("."), DoesNotStartWithDigit, 0);
@@ -176,10 +216,11 @@
assert_err_single!(FloatLit::parse("_2.7"), DoesNotStartWithDigit, 0);
assert_err_single!(FloatLit::parse(".5"), DoesNotStartWithDigit, 0);
- assert_err_single!(FloatLit::parse("0x44.5"), InvalidFloatTypeSuffix, 1..6);
assert_err!(FloatLit, "1e", NoExponentDigits, 1..2);
assert_err!(FloatLit, "1.e4", UnexpectedChar, 2);
assert_err!(FloatLit, "3._4", UnexpectedChar, 2);
+ assert_err!(FloatLit, "3.f32", UnexpectedChar, 2);
+ assert_err!(FloatLit, "3.e5", UnexpectedChar, 2);
assert_err!(FloatLit, "12345._987", UnexpectedChar, 6);
assert_err!(FloatLit, "46._", UnexpectedChar, 3);
assert_err!(FloatLit, "46.f32", UnexpectedChar, 3);
@@ -188,19 +229,25 @@
assert_err!(FloatLit, "46.e3f64", UnexpectedChar, 3);
assert_err!(FloatLit, "23.4e_", NoExponentDigits, 4..6);
assert_err!(FloatLit, "23E___f32", NoExponentDigits, 2..6);
- assert_err!(FloatLit, "7f23", InvalidFloatTypeSuffix, 1..4);
- assert_err!(FloatLit, "7f320", InvalidFloatTypeSuffix, 1..5);
- assert_err!(FloatLit, "7f64_", InvalidFloatTypeSuffix, 1..5);
- assert_err!(FloatLit, "8f649", InvalidFloatTypeSuffix, 1..5);
- assert_err!(FloatLit, "8f64f32", InvalidFloatTypeSuffix, 1..7);
- assert_err!(FloatLit, "55e3.1", InvalidFloatTypeSuffix, 4..6); // suboptimal
+ assert_err!(FloatLit, "55e3.1", UnexpectedChar, 4..6);
- assert_err!(FloatLit, "3.7+", InvalidFloatTypeSuffix, 3..4);
- assert_err!(FloatLit, "3.7+2", InvalidFloatTypeSuffix, 3..5);
- assert_err!(FloatLit, "3.7-", InvalidFloatTypeSuffix, 3..4);
- assert_err!(FloatLit, "3.7-2", InvalidFloatTypeSuffix, 3..5);
+ assert_err!(FloatLit, "3.7+", UnexpectedChar, 3..4);
+ assert_err!(FloatLit, "3.7+2", UnexpectedChar, 3..5);
+ assert_err!(FloatLit, "3.7-", UnexpectedChar, 3..4);
+ assert_err!(FloatLit, "3.7-2", UnexpectedChar, 3..5);
assert_err!(FloatLit, "3.7e+", NoExponentDigits, 3..5);
assert_err!(FloatLit, "3.7e-", NoExponentDigits, 3..5);
- assert_err!(FloatLit, "3.7e-+3", NoExponentDigits, 3..5); // suboptimal
- assert_err!(FloatLit, "3.7e+-3", NoExponentDigits, 3..5); // suboptimal
+ assert_err!(FloatLit, "3.7e-+3", NoExponentDigits, 3..5); // suboptimal error
+ assert_err!(FloatLit, "3.7e+-3", NoExponentDigits, 3..5); // suboptimal error
+ assert_err_single!(FloatLit::parse("0x44.5"), InvalidSuffix, 1..6);
+
+ assert_err_single!(FloatLit::parse("3"), UnexpectedIntegerLit, None);
+ assert_err_single!(FloatLit::parse("35_389"), UnexpectedIntegerLit, None);
+ assert_err_single!(FloatLit::parse("9_8_7f32"), UnexpectedIntegerLit, None);
+ assert_err_single!(FloatLit::parse("9_8_7banana"), UnexpectedIntegerLit, None);
+ assert_err_single!(FloatLit::parse("7f23"), UnexpectedIntegerLit, None);
+ assert_err_single!(FloatLit::parse("7f320"), UnexpectedIntegerLit, None);
+ assert_err_single!(FloatLit::parse("7f64_"), UnexpectedIntegerLit, None);
+ assert_err_single!(FloatLit::parse("8f649"), UnexpectedIntegerLit, None);
+ assert_err_single!(FloatLit::parse("8f64f32"), UnexpectedIntegerLit, None);
}
diff --git a/crates/litrs/src/integer/mod.rs b/crates/litrs/src/integer/mod.rs
index 79f7e55..cecd79d 100644
--- a/crates/litrs/src/integer/mod.rs
+++ b/crates/litrs/src/integer/mod.rs
@@ -1,9 +1,9 @@
-use std::fmt;
+use std::{fmt, str::FromStr};
use crate::{
Buffer, ParseError,
err::{perr, ParseErrorKind::*},
- parse::{first_byte_or_empty, hex_digit_value},
+ parse::{first_byte_or_empty, hex_digit_value, check_suffix},
};
@@ -25,52 +25,14 @@
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
#[non_exhaustive]
pub struct IntegerLit<B: Buffer> {
+ /// The raw literal. Grammar: `<prefix?><main part><suffix?>`.
raw: B,
- // First index of the main number part (after the base prefix).
+ /// First index of the main number part (after the base prefix).
start_main_part: usize,
- // First index not part of the main number part.
+ /// First index not part of the main number part.
end_main_part: usize,
+ /// Parsed `raw[..start_main_part]`.
base: IntegerBase,
- type_suffix: Option<IntegerType>,
-}
-
-/// The bases in which an integer can be specified.
-#[derive(Debug, Clone, Copy, PartialEq, Eq)]
-pub enum IntegerBase {
- Binary,
- Octal,
- Decimal,
- Hexadecimal,
-}
-
-/// All possible integer type suffixes.
-#[derive(Debug, Clone, Copy, PartialEq, Eq)]
-pub enum IntegerType {
- U8,
- U16,
- U32,
- U64,
- U128,
- Usize,
- I8,
- I16,
- I32,
- I64,
- I128,
- Isize,
-}
-
-impl IntegerBase {
- /// Returns the literal prefix that indicates this base, i.e. `"0b"`,
- /// `"0o"`, `""` and `"0x"`.
- pub fn prefix(self) -> &'static str {
- match self {
- Self::Binary => "0b",
- Self::Octal => "0o",
- Self::Decimal => "",
- Self::Hexadecimal => "0x",
- }
- }
}
impl<B: Buffer> IntegerLit<B> {
@@ -84,17 +46,10 @@
start_main_part,
end_main_part,
base,
- type_suffix,
..
} = parse_impl(&input, digit)?;
- Ok(Self {
- raw: input,
- start_main_part,
- end_main_part,
- base,
- type_suffix,
- })
+ Ok(Self { raw: input, start_main_part, end_main_part, base })
},
_ => Err(perr(0, DoesNotStartWithDigit)),
}
@@ -105,13 +60,15 @@
/// method**. This means `N` does not need to match the type suffix!
///
/// Returns `None` if the literal overflows `N`.
+ ///
+ /// Hint: `u128` can represent all possible values integer literal values,
+ /// as there are no negative literals (see type docs). Thus you can, for
+ /// example, safely use `lit.value::<u128>().to_string()` to get a decimal
+ /// string. (Technically, Rust integer literals can represent arbitrarily
+ /// large numbers, but those would be rejected at a later stage by the Rust
+ /// compiler).
pub fn value<N: FromIntegerLiteral>(&self) -> Option<N> {
- let base = match self.base {
- IntegerBase::Binary => N::from_small_number(2),
- IntegerBase::Octal => N::from_small_number(8),
- IntegerBase::Decimal => N::from_small_number(10),
- IntegerBase::Hexadecimal => N::from_small_number(16),
- };
+ let base = N::from_small_number(self.base.value());
let mut acc = N::from_small_number(0);
for digit in self.raw_main_part().bytes() {
@@ -142,9 +99,11 @@
&(*self.raw)[self.start_main_part..self.end_main_part]
}
- /// The type suffix, if specified.
- pub fn type_suffix(&self) -> Option<IntegerType> {
- self.type_suffix
+ /// The optional suffix. Returns `""` if the suffix is empty/does not exist.
+ ///
+ /// If you want the type, try `IntegerType::from_suffix(lit.suffix())`.
+ pub fn suffix(&self) -> &str {
+ &(*self.raw)[self.end_main_part..]
}
/// Returns the raw input that was passed to `parse`.
@@ -167,7 +126,6 @@
start_main_part: self.start_main_part,
end_main_part: self.end_main_part,
base: self.base,
- type_suffix: self.type_suffix,
}
}
}
@@ -248,59 +206,151 @@
};
let without_prefix = &input[end_prefix..];
- // Find end of main part.
- let end_main = without_prefix.bytes()
- .position(|b| !matches!(b, b'0'..=b'9' | b'a'..=b'f' | b'A'..=b'F' | b'_'))
- .unwrap_or(without_prefix.len());
- let (main_part, type_suffix) = without_prefix.split_at(end_main);
- // Check for invalid digits and make sure there is at least one valid digit.
- let invalid_digit_pos = match base {
- IntegerBase::Binary => main_part.bytes()
- .position(|b| !matches!(b, b'0' | b'1' | b'_')),
- IntegerBase::Octal => main_part.bytes()
- .position(|b| !matches!(b, b'0'..=b'7' | b'_')),
- IntegerBase::Decimal => main_part.bytes()
- .position(|b| !matches!(b, b'0'..=b'9' | b'_')),
- IntegerBase::Hexadecimal => None,
+ // Scan input to find the first character that's not a valid digit.
+ let is_valid_digit = match base {
+ IntegerBase::Binary => |b| matches!(b, b'0' | b'1' | b'_'),
+ IntegerBase::Octal => |b| matches!(b, b'0'..=b'7' | b'_'),
+ IntegerBase::Decimal => |b| matches!(b, b'0'..=b'9' | b'_'),
+ IntegerBase::Hexadecimal => |b| matches!(b, b'0'..=b'9' | b'a'..=b'f' | b'A'..=b'F' | b'_'),
};
+ let end_main = without_prefix.bytes()
+ .position(|b| !is_valid_digit(b))
+ .unwrap_or(without_prefix.len());
+ let (main_part, suffix) = without_prefix.split_at(end_main);
- if let Some(pos) = invalid_digit_pos {
- return Err(perr(end_prefix + pos, InvalidDigit));
+ check_suffix(suffix).map_err(|kind| {
+ // This is just to have a nicer error kind for this special case. If the
+ // suffix is invalid, it is non-empty -> unwrap ok.
+ let first = suffix.as_bytes()[0];
+ if !is_valid_digit(first) && first.is_ascii_digit() {
+ perr(end_main + end_prefix, InvalidDigit)
+ } else {
+ perr(end_main + end_prefix..input.len(), kind)
+ }
+ })?;
+ if suffix.starts_with('e') || suffix.starts_with('E') {
+ return Err(perr(end_main, IntegerSuffixStartingWithE));
}
+ // Make sure main number part is not empty.
if main_part.bytes().filter(|&b| b != b'_').count() == 0 {
return Err(perr(end_prefix..end_prefix + end_main, NoDigits));
}
-
- // Parse type suffix
- let type_suffix = match type_suffix {
- "" => None,
- "u8" => Some(IntegerType::U8),
- "u16" => Some(IntegerType::U16),
- "u32" => Some(IntegerType::U32),
- "u64" => Some(IntegerType::U64),
- "u128" => Some(IntegerType::U128),
- "usize" => Some(IntegerType::Usize),
- "i8" => Some(IntegerType::I8),
- "i16" => Some(IntegerType::I16),
- "i32" => Some(IntegerType::I32),
- "i64" => Some(IntegerType::I64),
- "i128" => Some(IntegerType::I128),
- "isize" => Some(IntegerType::Isize),
- _ => return Err(perr(end_main + end_prefix..input.len(), InvalidIntegerTypeSuffix)),
- };
-
Ok(IntegerLit {
raw: input,
start_main_part: end_prefix,
end_main_part: end_main + end_prefix,
base,
- type_suffix,
})
}
+/// The bases in which an integer can be specified.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum IntegerBase {
+ Binary,
+ Octal,
+ Decimal,
+ Hexadecimal,
+}
+
+impl IntegerBase {
+ /// Returns the literal prefix that indicates this base, i.e. `"0b"`,
+ /// `"0o"`, `""` and `"0x"`.
+ pub fn prefix(self) -> &'static str {
+ match self {
+ Self::Binary => "0b",
+ Self::Octal => "0o",
+ Self::Decimal => "",
+ Self::Hexadecimal => "0x",
+ }
+ }
+
+ /// Returns the base value, i.e. 2, 8, 10 or 16.
+ pub fn value(self) -> u8 {
+ match self {
+ Self::Binary => 2,
+ Self::Octal => 8,
+ Self::Decimal => 10,
+ Self::Hexadecimal => 16,
+ }
+ }
+}
+
+/// All possible integer type suffixes.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+#[non_exhaustive]
+pub enum IntegerType {
+ U8,
+ U16,
+ U32,
+ U64,
+ U128,
+ Usize,
+ I8,
+ I16,
+ I32,
+ I64,
+ I128,
+ Isize,
+}
+
+impl IntegerType {
+ /// Returns the type corresponding to the given suffix (e.g. `"u8"` is
+ /// mapped to `Self::U8`). If the suffix is not a valid integer type,
+ /// `None` is returned.
+ pub fn from_suffix(suffix: &str) -> Option<Self> {
+ match suffix {
+ "u8" => Some(Self::U8),
+ "u16" => Some(Self::U16),
+ "u32" => Some(Self::U32),
+ "u64" => Some(Self::U64),
+ "u128" => Some(Self::U128),
+ "usize" => Some(Self::Usize),
+ "i8" => Some(Self::I8),
+ "i16" => Some(Self::I16),
+ "i32" => Some(Self::I32),
+ "i64" => Some(Self::I64),
+ "i128" => Some(Self::I128),
+ "isize" => Some(Self::Isize),
+ _ => None,
+ }
+ }
+
+ /// Returns the suffix for this type, e.g. `"u8"` for `Self::U8`.
+ pub fn suffix(self) -> &'static str {
+ match self {
+ Self::U8 => "u8",
+ Self::U16 => "u16",
+ Self::U32 => "u32",
+ Self::U64 => "u64",
+ Self::U128 => "u128",
+ Self::Usize => "usize",
+ Self::I8 => "i8",
+ Self::I16 => "i16",
+ Self::I32 => "i32",
+ Self::I64 => "i64",
+ Self::I128 => "i128",
+ Self::Isize => "isize",
+ }
+ }
+}
+
+impl FromStr for IntegerType {
+ type Err = ();
+ fn from_str(s: &str) -> Result<Self, Self::Err> {
+ Self::from_suffix(s).ok_or(())
+ }
+}
+
+impl fmt::Display for IntegerType {
+ fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+ self.suffix().fmt(f)
+ }
+}
+
+
#[cfg(test)]
mod tests;
diff --git a/crates/litrs/src/integer/tests.rs b/crates/litrs/src/integer/tests.rs
index 1656345..e6dad3f 100644
--- a/crates/litrs/src/integer/tests.rs
+++ b/crates/litrs/src/integer/tests.rs
@@ -20,13 +20,13 @@
start_main_part: base.prefix().len(),
end_main_part: base.prefix().len() + main_part.len(),
base,
- type_suffix
};
assert_parse_ok_eq(
input, IntegerLit::parse(input), expected_integer.clone(), "IntegerLit::parse");
assert_parse_ok_eq(
input, Literal::parse(input), Literal::Integer(expected_integer), "Literal::parse");
assert_roundtrip(expected_integer.to_owned(), input);
+ assert_eq!(Ty::from_suffix(IntegerLit::parse(input).unwrap().suffix()), type_suffix);
let actual_value = IntegerLit::parse(input)
.unwrap()
@@ -101,7 +101,7 @@
check("0b10010u8", 0b10010u8, Binary, "10010", Some(Ty::U8));
check("0b10010i8", 0b10010u8, Binary, "10010", Some(Ty::I8));
check("0b10010u64", 0b10010u64, Binary, "10010", Some(Ty::U64));
- check("0b10010i64", 0b10010u64, Binary, "10010", Some(Ty::I64));
+ check("0b10010i64", 0b10010i64, Binary, "10010", Some(Ty::I64));
check(
"0b1011001_00110000_00101000_10100101u32",
0b1011001_00110000_00101000_10100101u32,
@@ -197,7 +197,7 @@
("123u64", Ty::U64),
("123u128", Ty::U128),
].iter().for_each(|&(s, ty)| {
- assert_eq!(IntegerLit::parse(s).unwrap().type_suffix(), Some(ty));
+ assert_eq!(Ty::from_suffix(IntegerLit::parse(s).unwrap().suffix()), Some(ty));
});
}
@@ -249,17 +249,15 @@
assert_err!(IntegerLit, "", Empty, None);
assert_err_single!(IntegerLit::parse("a"), DoesNotStartWithDigit, 0);
assert_err_single!(IntegerLit::parse(";"), DoesNotStartWithDigit, 0);
- assert_err_single!(IntegerLit::parse("0;"), InvalidIntegerTypeSuffix, 1..2);
- assert_err_single!(IntegerLit::parse("0a"), InvalidDigit, 1);
+ assert_err_single!(IntegerLit::parse("0;"), UnexpectedChar, 1..2);
assert_err!(IntegerLit, "0b", NoDigits, 2..2);
- assert_err_single!(IntegerLit::parse("0z"), InvalidIntegerTypeSuffix, 1..2);
assert_err_single!(IntegerLit::parse(" 0"), DoesNotStartWithDigit, 0);
- assert_err_single!(IntegerLit::parse("0 "), InvalidIntegerTypeSuffix, 1);
- assert_err_single!(IntegerLit::parse("0a3"), InvalidDigit, 1);
+ assert_err_single!(IntegerLit::parse("0 "), UnexpectedChar, 1);
assert_err!(IntegerLit, "0b3", InvalidDigit, 2);
- assert_err_single!(IntegerLit::parse("0z3"), InvalidIntegerTypeSuffix, 1..3);
assert_err_single!(IntegerLit::parse("_"), DoesNotStartWithDigit, 0);
assert_err_single!(IntegerLit::parse("_3"), DoesNotStartWithDigit, 0);
+ assert_err!(IntegerLit, "0x44.5", UnexpectedChar, 4..6);
+ assert_err_single!(IntegerLit::parse("123em"), IntegerSuffixStartingWithE, 3);
}
#[test]
@@ -267,30 +265,12 @@
assert_err!(IntegerLit, "0b10201", InvalidDigit, 4);
assert_err!(IntegerLit, "0b9", InvalidDigit, 2);
assert_err!(IntegerLit, "0b07", InvalidDigit, 3);
- assert_err!(IntegerLit, "0b0a", InvalidDigit, 3);
- assert_err!(IntegerLit, "0b0A", InvalidDigit, 3);
- assert_err!(IntegerLit, "0b01f", InvalidDigit, 4);
- assert_err!(IntegerLit, "0b01F", InvalidDigit, 4);
assert_err!(IntegerLit, "0o12380", InvalidDigit, 5);
assert_err!(IntegerLit, "0o192", InvalidDigit, 3);
- assert_err!(IntegerLit, "0o7a_", InvalidDigit, 3);
- assert_err!(IntegerLit, "0o7A_", InvalidDigit, 3);
- assert_err!(IntegerLit, "0o72f_0", InvalidDigit, 4);
- assert_err!(IntegerLit, "0o72F_0", InvalidDigit, 4);
- assert_err_single!(IntegerLit::parse("12a3"), InvalidDigit, 2);
- assert_err_single!(IntegerLit::parse("12f3"), InvalidDigit, 2);
- assert_err_single!(IntegerLit::parse("12f_"), InvalidDigit, 2);
- assert_err_single!(IntegerLit::parse("12F_"), InvalidDigit, 2);
assert_err_single!(IntegerLit::parse("a_123"), DoesNotStartWithDigit, 0);
assert_err_single!(IntegerLit::parse("B_123"), DoesNotStartWithDigit, 0);
-
- assert_err!(IntegerLit, "0x8cg", InvalidIntegerTypeSuffix, 4..5);
- assert_err!(IntegerLit, "0x8cG", InvalidIntegerTypeSuffix, 4..5);
- assert_err!(IntegerLit, "0x8c1h_", InvalidIntegerTypeSuffix, 5..7);
- assert_err!(IntegerLit, "0x8c1H_", InvalidIntegerTypeSuffix, 5..7);
- assert_err!(IntegerLit, "0x8czu16", InvalidIntegerTypeSuffix, 4..8);
}
#[test]
@@ -317,27 +297,61 @@
}
#[test]
-fn invalid_suffix() {
- assert_err!(IntegerLit, "5u7", InvalidIntegerTypeSuffix, 1..3);
- assert_err!(IntegerLit, "5u9", InvalidIntegerTypeSuffix, 1..3);
- assert_err!(IntegerLit, "5u0", InvalidIntegerTypeSuffix, 1..3);
- assert_err!(IntegerLit, "33u12", InvalidIntegerTypeSuffix, 2..5);
- assert_err!(IntegerLit, "84u17", InvalidIntegerTypeSuffix, 2..5);
- assert_err!(IntegerLit, "99u80", InvalidIntegerTypeSuffix, 2..5);
- assert_err!(IntegerLit, "1234uu16", InvalidIntegerTypeSuffix, 4..8);
+fn non_standard_suffixes() {
+ #[track_caller]
+ fn check_suffix<T: FromIntegerLiteral + PartialEq + Debug + Display>(
+ input: &str,
+ value: T,
+ base: IntegerBase,
+ main_part: &str,
+ suffix: &str,
+ ) {
+ check(input, value, base, main_part, None);
+ assert_eq!(IntegerLit::parse(input).unwrap().suffix(), suffix);
+ }
- assert_err!(IntegerLit, "5i7", InvalidIntegerTypeSuffix, 1..3);
- assert_err!(IntegerLit, "5i9", InvalidIntegerTypeSuffix, 1..3);
- assert_err!(IntegerLit, "5i0", InvalidIntegerTypeSuffix, 1..3);
- assert_err!(IntegerLit, "33i12", InvalidIntegerTypeSuffix, 2..5);
- assert_err!(IntegerLit, "84i17", InvalidIntegerTypeSuffix, 2..5);
- assert_err!(IntegerLit, "99i80", InvalidIntegerTypeSuffix, 2..5);
- assert_err!(IntegerLit, "1234ii16", InvalidIntegerTypeSuffix, 4..8);
+ check_suffix("5u7", 5, Decimal, "5", "u7");
+ check_suffix("5u7", 5, Decimal, "5", "u7");
+ check_suffix("5u9", 5, Decimal, "5", "u9");
+ check_suffix("5u0", 5, Decimal, "5", "u0");
+ check_suffix("33u12", 33, Decimal, "33", "u12");
+ check_suffix("84u17", 84, Decimal, "84", "u17");
+ check_suffix("99u80", 99, Decimal, "99", "u80");
+ check_suffix("1234uu16", 1234, Decimal, "1234", "uu16");
- assert_err!(IntegerLit, "0ui32", InvalidIntegerTypeSuffix, 1..5);
- assert_err!(IntegerLit, "1iu32", InvalidIntegerTypeSuffix, 1..5);
- assert_err_single!(IntegerLit::parse("54321a64"), InvalidDigit, 5);
- assert_err!(IntegerLit, "54321b64", InvalidDigit, 5);
- assert_err!(IntegerLit, "54321x64", InvalidIntegerTypeSuffix, 5..8);
- assert_err!(IntegerLit, "54321o64", InvalidIntegerTypeSuffix, 5..8);
+ check_suffix("5i7", 5, Decimal, "5", "i7");
+ check_suffix("5i9", 5, Decimal, "5", "i9");
+ check_suffix("5i0", 5, Decimal, "5", "i0");
+ check_suffix("33i12", 33, Decimal, "33", "i12");
+ check_suffix("84i17", 84, Decimal, "84", "i17");
+ check_suffix("99i80", 99, Decimal, "99", "i80");
+ check_suffix("1234ii16", 1234, Decimal, "1234", "ii16");
+
+ check_suffix("0ui32", 0, Decimal, "0", "ui32");
+ check_suffix("1iu32", 1, Decimal, "1", "iu32");
+ check_suffix("54321a64", 54321, Decimal, "54321", "a64");
+ check_suffix("54321b64", 54321, Decimal, "54321", "b64");
+ check_suffix("54321x64", 54321, Decimal, "54321", "x64");
+ check_suffix("54321o64", 54321, Decimal, "54321", "o64");
+
+ check_suffix("0a", 0, Decimal, "0", "a");
+ check_suffix("0a3", 0, Decimal, "0", "a3");
+ check_suffix("0z", 0, Decimal, "0", "z");
+ check_suffix("0z3", 0, Decimal, "0", "z3");
+ check_suffix("0b0a", 0, Binary, "0", "a");
+ check_suffix("0b0A", 0, Binary, "0", "A");
+ check_suffix("0b01f", 1, Binary, "01", "f");
+ check_suffix("0b01F", 1, Binary, "01", "F");
+ check_suffix("0o7a_", 7, Octal, "7", "a_");
+ check_suffix("0o7A_", 7, Octal, "7", "A_");
+ check_suffix("0o72f_0", 0o72, Octal, "72", "f_0");
+ check_suffix("0o72F_0", 0o72, Octal, "72", "F_0");
+
+ check_suffix("0x8cg", 0x8c, Hexadecimal, "8c", "g");
+ check_suffix("0x8cG", 0x8c, Hexadecimal, "8c", "G");
+ check_suffix("0x8c1h_", 0x8c1, Hexadecimal, "8c1", "h_");
+ check_suffix("0x8c1H_", 0x8c1, Hexadecimal, "8c1", "H_");
+ check_suffix("0x8czu16", 0x8c, Hexadecimal, "8c", "zu16");
+
+ check_suffix("123_foo", 123, Decimal, "123_", "foo");
}
diff --git a/crates/litrs/src/lib.rs b/crates/litrs/src/lib.rs
index bd81f56..64ed781 100644
--- a/crates/litrs/src/lib.rs
+++ b/crates/litrs/src/lib.rs
@@ -9,7 +9,35 @@
//! built. This crate also offers a bit more flexibility compared to `syn`
//! (only regarding literals, of course).
//!
-//! ---
+//!
+//! # Quick start
+//!
+//! | **`StringLit::try_from(tt)?.value()`** |
+//! | - |
+//!
+//! ... where `tt` is a `proc_macro::TokenTree` and where [`StringLit`] can be
+//! replaced with [`Literal`] or other types of literals (e.g. [`FloatLit`]).
+//! Calling `value()` returns the value that is represented by the literal.
+//!
+//! **Mini Example**
+//!
+//! ```ignore
+//! use proc_macro::TokenStream;
+//!
+//! #[proc_macro]
+//! pub fn foo(input: TokenStream) -> TokenStream {
+//! let first_token = input.into_iter().next().unwrap(); // Do proper error handling!
+//! let string_value = match litrs::StringLit::try_from(first_token) {
+//! Ok(string_lit) => string_lit.value(),
+//! Err(e) => return e.to_compile_error(),
+//! };
+//!
+//! // `string_value` is the string value with all escapes resolved.
+//! todo!()
+//! }
+//! ```
+//!
+//! # Overview
//!
//! The main types of this library are [`Literal`], representing any kind of
//! literal, and `*Lit`, like [`StringLit`] or [`FloatLit`], representing a
@@ -41,8 +69,8 @@
//!
//! **Note**: `true` and `false` are `Ident`s when passed to your proc macro.
//! The `TryFrom<TokenTree>` impls check for those two special idents and
-//! return a `BoolLit` appropriately. For that reason, there is also no
-//! `TryFrom<proc_macro::Literal>` impl for `BoolLit`. The `proc_macro::Literal`
+//! return a [`BoolLit`] appropriately. For that reason, there is also no
+//! `TryFrom<proc_macro::Literal>` impl for [`BoolLit`]. The `proc_macro::Literal`
//! simply cannot represent bool literals.
//!
//!
@@ -82,7 +110,7 @@
//! // Parse a specific kind of literal (float in this case):
//! let float_lit = FloatLit::parse("3.14f32");
//! assert!(float_lit.is_ok());
-//! assert_eq!(float_lit.unwrap().type_suffix(), Some(litrs::FloatType::F32));
+//! assert_eq!(float_lit.unwrap().suffix(), "f32");
//! assert!(FloatLit::parse("'c'").is_err());
//!
//! // Parse any kind of literal. After parsing, you can inspect the literal
@@ -105,6 +133,11 @@
//!
//! - `proc-macro2` (**default**): adds the dependency `proc_macro2`, a bunch of
//! `From` and `TryFrom` impls, and [`InvalidToken::to_compile_error2`].
+//! - `check_suffix`: if enabled, `parse` functions will exactly verify that the
+//! literal suffix is valid. Adds the dependency `unicode-xid`. If disabled,
+//! only an approximate check (only in ASCII range) is done. If you are
+//! writing a proc macro, you don't need to enable this as the suffix is
+//! already checked by the compiler.
//!
//!
//! [ref]: https://doc.rust-lang.org/reference/tokens.html#literals
@@ -152,17 +185,10 @@
// ===== `Literal` and type defs
// ==============================================================================================
-/// A literal which owns the underlying buffer.
-pub type OwnedLiteral = Literal<String>;
-
-/// A literal whose underlying buffer is borrowed.
-pub type SharedLiteral<'a> = Literal<&'a str>;
-
/// A literal. This is the main type of this library.
///
/// This type is generic over the underlying buffer `B`, which can be `&str` or
-/// `String`. There are two useful type aliases: [`OwnedLiteral`] and
-/// [`SharedLiteral`].
+/// `String`.
///
/// To create this type, you have to either call [`Literal::parse`] with an
/// input string or use the `From<_>` impls of this type. The impls are only
@@ -179,10 +205,66 @@
ByteString(ByteStringLit<B>),
}
+impl<B: Buffer> Literal<B> {
+ /// Parses the given input as a Rust literal.
+ pub fn parse(input: B) -> Result<Self, ParseError> {
+ parse::parse(input)
+ }
+
+ /// Returns the suffix of this literal or `""` if it doesn't have one.
+ ///
+ /// Rust token grammar actually allows suffixes for all kinds of tokens.
+ /// Most Rust programmer only know the type suffixes for integer and
+ /// floats, e.g. `0u32`. And in normal Rust code, everything else causes an
+ /// error. But it is possible to pass literals with arbitrary suffixes to
+ /// proc macros, for example:
+ ///
+ /// ```ignore
+ /// some_macro!(3.14f33 16px 'ðĶ'good_boy "toph"beifong);
+ /// ```
+ ///
+ /// Boolean literals, not actually being literals, but idents, cannot have
+ /// suffixes and this method always returns `""` for those.
+ ///
+ /// There are some edge cases to be aware of:
+ /// - Integer suffixes must not start with `e` or `E` as that conflicts with
+ /// the exponent grammar for floats. `0e1` is a float; `0eel` is also
+ /// parsed as a float and results in an error.
+ /// - Hexadecimal integers eagerly parse digits, so `0x5abcdefgh` has a
+ /// suffix von `gh`.
+ /// - Suffixes can contain and start with `_`, but for integer and number
+ /// literals, `_` is eagerly parsed as part of the number, so `1_x` has
+ /// the suffix `x`.
+ /// - The input `55f32` is regarded as integer literal with suffix `f32`.
+ ///
+ /// # Example
+ ///
+ /// ```
+ /// use litrs::Literal;
+ ///
+ /// assert_eq!(Literal::parse(r##"3.14f33"##).unwrap().suffix(), "f33");
+ /// assert_eq!(Literal::parse(r##"123hackerman"##).unwrap().suffix(), "hackerman");
+ /// assert_eq!(Literal::parse(r##"0x0fuck"##).unwrap().suffix(), "uck");
+ /// assert_eq!(Literal::parse(r##"'ðĶ'good_boy"##).unwrap().suffix(), "good_boy");
+ /// assert_eq!(Literal::parse(r##""toph"beifong"##).unwrap().suffix(), "beifong");
+ /// ```
+ pub fn suffix(&self) -> &str {
+ match self {
+ Literal::Bool(_) => "",
+ Literal::Integer(l) => l.suffix(),
+ Literal::Float(l) => l.suffix(),
+ Literal::Char(l) => l.suffix(),
+ Literal::String(l) => l.suffix(),
+ Literal::Byte(l) => l.suffix(),
+ Literal::ByteString(l) => l.suffix(),
+ }
+ }
+}
+
impl Literal<&str> {
/// Makes a copy of the underlying buffer and returns the owned version of
/// `Self`.
- pub fn into_owned(self) -> OwnedLiteral {
+ pub fn into_owned(self) -> Literal<String> {
match self {
Literal::Bool(l) => Literal::Bool(l.to_owned()),
Literal::Integer(l) => Literal::Integer(l.to_owned()),
@@ -218,7 +300,7 @@
///
/// This is trait is implementation detail of this library, cannot be
/// implemented in other crates and is not subject to semantic versioning.
-/// `litrs` only gurantees that this trait is implemented for `String` and
+/// `litrs` only guarantees that this trait is implemented for `String` and
/// `for<'a> &'a str`.
pub trait Buffer: sealed::Sealed + Deref<Target = str> {
/// This is `Cow<'static, str>` for `String`, and `Cow<'a, str>` for `&'a str`.
diff --git a/crates/litrs/src/parse.rs b/crates/litrs/src/parse.rs
index a0266da..efc6b87 100644
--- a/crates/litrs/src/parse.rs
+++ b/crates/litrs/src/parse.rs
@@ -9,52 +9,44 @@
IntegerLit,
Literal,
StringLit,
- err::{perr, ParseErrorKind::*},
+ err::{perr, ParseErrorKind::{*, self}},
};
-impl<B: Buffer> Literal<B> {
- /// Parses the given input as a Rust literal.
- pub fn parse(input: B) -> Result<Self, ParseError> {
- let (first, rest) = input.as_bytes().split_first().ok_or(perr(None, Empty))?;
- let second = input.as_bytes().get(1).copied();
+pub fn parse<B: Buffer>(input: B) -> Result<Literal<B>, ParseError> {
+ let (first, rest) = input.as_bytes().split_first().ok_or(perr(None, Empty))?;
+ let second = input.as_bytes().get(1).copied();
- match first {
- b'f' if &*input == "false" => Ok(Self::Bool(BoolLit::False)),
- b't' if &*input == "true" => Ok(Self::Bool(BoolLit::True)),
+ match first {
+ b'f' if &*input == "false" => Ok(Literal::Bool(BoolLit::False)),
+ b't' if &*input == "true" => Ok(Literal::Bool(BoolLit::True)),
- // A number literal (integer or float).
- b'0'..=b'9' => {
- // To figure out whether this is a float or integer, we do some
- // quick inspection here. Yes, this is technically duplicate
- // work with what is happening in the integer/float parse
- // methods, but it makes the code way easier for now and won't
- // be a huge performance loss.
- let end = 1 + end_dec_digits(rest);
- match input.as_bytes().get(end) {
- // Potential chars in integer literals: b, o, x for base; u
- // and i for type suffix.
- None | Some(b'b') | Some(b'o') | Some(b'x') | Some(b'u') | Some(b'i')
- => IntegerLit::parse(input).map(Literal::Integer),
+ // A number literal (integer or float).
+ b'0'..=b'9' => {
+ // To figure out whether this is a float or integer, we do some
+ // quick inspection here. Yes, this is technically duplicate
+ // work with what is happening in the integer/float parse
+ // methods, but it makes the code way easier for now and won't
+ // be a huge performance loss.
+ //
+ // The first non-decimal char in a float literal must
+ // be '.', 'e' or 'E'.
+ match input.as_bytes().get(1 + end_dec_digits(rest)) {
+ Some(b'.') | Some(b'e') | Some(b'E')
+ => FloatLit::parse(input).map(Literal::Float),
- // Potential chars for float literals: `.` as fractional
- // period, e and E as exponent start and f as type suffix.
- Some(b'.') | Some(b'e') | Some(b'E') | Some(b'f')
- => FloatLit::parse(input).map(Literal::Float),
+ _ => IntegerLit::parse(input).map(Literal::Integer),
+ }
+ },
- _ => Err(perr(end, UnexpectedChar)),
- }
- },
+ b'\'' => CharLit::parse(input).map(Literal::Char),
+ b'"' | b'r' => StringLit::parse(input).map(Literal::String),
- b'\'' => CharLit::parse(input).map(Literal::Char),
- b'"' | b'r' => StringLit::parse_impl(input).map(Literal::String),
+ b'b' if second == Some(b'\'') => ByteLit::parse(input).map(Literal::Byte),
+ b'b' if second == Some(b'r') || second == Some(b'"')
+ => ByteStringLit::parse(input).map(Literal::ByteString),
- b'b' if second == Some(b'\'') => ByteLit::parse(input).map(Literal::Byte),
- b'b' if second == Some(b'r') || second == Some(b'"')
- => ByteStringLit::parse_impl(input).map(Literal::ByteString),
-
- _ => Err(perr(None, InvalidLiteral)),
- }
+ _ => Err(perr(None, InvalidLiteral)),
}
}
@@ -79,3 +71,55 @@
_ => None,
}
}
+
+/// Makes sure that `s` is a valid literal suffix.
+pub(crate) fn check_suffix(s: &str) -> Result<(), ParseErrorKind> {
+ if s.is_empty() {
+ return Ok(());
+ }
+
+ let mut chars = s.chars();
+ let first = chars.next().unwrap();
+ let rest = chars.as_str();
+ if first == '_' && rest.is_empty() {
+ return Err(InvalidSuffix);
+ }
+
+ // This is just an extra check to improve the error message. If the first
+ // character of the "suffix" is already some invalid ASCII
+ // char, "unexpected character" seems like the more fitting error.
+ if first.is_ascii() && !(first.is_ascii_alphabetic() || first == '_') {
+ return Err(UnexpectedChar);
+ }
+
+ // Proper check is optional as it's not really necessary in proc macro
+ // context.
+ #[cfg(feature = "check_suffix")]
+ fn is_valid_suffix(first: char, rest: &str) -> bool {
+ use unicode_xid::UnicodeXID;
+
+ (first == '_' || first.is_xid_start())
+ && rest.chars().all(|c| c.is_xid_continue())
+ }
+
+ // When avoiding the dependency on `unicode_xid`, we just do a best effort
+ // to catch the most common errors.
+ #[cfg(not(feature = "check_suffix"))]
+ fn is_valid_suffix(first: char, rest: &str) -> bool {
+ if first.is_ascii() && !(first.is_ascii_alphabetic() || first == '_') {
+ return false;
+ }
+ for c in rest.chars() {
+ if c.is_ascii() && !(c.is_ascii_alphanumeric() || c == '_') {
+ return false;
+ }
+ }
+ true
+ }
+
+ if is_valid_suffix(first, rest) {
+ Ok(())
+ } else {
+ Err(InvalidSuffix)
+ }
+}
diff --git a/crates/litrs/src/string/mod.rs b/crates/litrs/src/string/mod.rs
index ab1cc3f..d2034a6 100644
--- a/crates/litrs/src/string/mod.rs
+++ b/crates/litrs/src/string/mod.rs
@@ -18,13 +18,16 @@
/// The raw input.
raw: B,
- /// The string value (with all escaped unescaped), or `None` if there were
- /// no escapes. In the latter case, `input` is the string value.
+ /// The string value (with all escapes unescaped), or `None` if there were
+ /// no escapes. In the latter case, the string value is in `raw`.
value: Option<String>,
/// The number of hash signs in case of a raw string literal, or `None` if
/// it's not a raw string literal.
num_hashes: Option<u32>,
+
+ /// Start index of the suffix or `raw.len()` if there is no suffix.
+ start_suffix: usize,
}
impl<B: Buffer> StringLit<B> {
@@ -32,7 +35,10 @@
/// input is invalid or represents a different kind of literal.
pub fn parse(input: B) -> Result<Self, ParseError> {
match first_byte_or_empty(&input)? {
- b'r' | b'"' => Self::parse_impl(input),
+ b'r' | b'"' => {
+ let (value, num_hashes, start_suffix) = parse_impl(&input)?;
+ Ok(Self { raw: input, value, num_hashes, start_suffix })
+ }
_ => Err(perr(0, InvalidStringLiteralStart)),
}
}
@@ -53,6 +59,11 @@
value.map(B::Cow::from).unwrap_or_else(|| raw.cut(inner_range).into_cow())
}
+ /// The optional suffix. Returns `""` if the suffix is empty/does not exist.
+ pub fn suffix(&self) -> &str {
+ &(*self.raw)[self.start_suffix..]
+ }
+
/// Returns whether this literal is a raw string literal (starting with
/// `r`).
pub fn is_raw_string(&self) -> bool {
@@ -72,27 +83,8 @@
/// The range within `self.raw` that excludes the quotes and potential `r#`.
fn inner_range(&self) -> Range<usize> {
match self.num_hashes {
- None => 1..self.raw.len() - 1,
- Some(n) => 1 + n as usize + 1..self.raw.len() - n as usize - 1,
- }
- }
-
- /// Precondition: input has to start with either `"` or `r`.
- pub(crate) fn parse_impl(input: B) -> Result<Self, ParseError> {
- if input.starts_with('r') {
- let (value, num_hashes) = scan_raw_string::<char>(&input, 1)?;
- Ok(Self {
- raw: input,
- value,
- num_hashes: Some(num_hashes),
- })
- } else {
- let value = unescape_string::<char>(&input, 1)?;
- Ok(Self {
- raw: input,
- value,
- num_hashes: None,
- })
+ None => 1..self.start_suffix - 1,
+ Some(n) => 1 + n as usize + 1..self.start_suffix - n as usize - 1,
}
}
}
@@ -105,6 +97,7 @@
raw: self.raw.to_owned(),
value: self.value,
num_hashes: self.num_hashes,
+ start_suffix: self.start_suffix,
}
}
}
@@ -115,6 +108,18 @@
}
}
+/// Precondition: input has to start with either `"` or `r`.
+#[inline(never)]
+pub(crate) fn parse_impl(input: &str) -> Result<(Option<String>, Option<u32>, usize), ParseError> {
+ if input.starts_with('r') {
+ scan_raw_string::<char>(&input, 1)
+ .map(|(v, hashes, start_suffix)| (v, Some(hashes), start_suffix))
+ } else {
+ unescape_string::<char>(&input, 1)
+ .map(|(v, start_suffix)| (v, None, start_suffix))
+ }
+}
+
#[cfg(test)]
mod tests;
diff --git a/crates/litrs/src/string/tests.rs b/crates/litrs/src/string/tests.rs
index 51519ab..1c0cb63 100644
--- a/crates/litrs/src/string/tests.rs
+++ b/crates/litrs/src/string/tests.rs
@@ -4,18 +4,24 @@
macro_rules! check {
($lit:literal, $has_escapes:expr, $num_hashes:expr) => {
- let input = stringify!($lit);
+ check!($lit, stringify!($lit), $has_escapes, $num_hashes, "")
+ };
+ ($lit:literal, $input:expr, $has_escapes:expr, $num_hashes:expr, $suffix:literal) => {
+ let input = $input;
let expected = StringLit {
raw: input,
value: if $has_escapes { Some($lit.to_string()) } else { None },
num_hashes: $num_hashes,
+ start_suffix: input.len() - $suffix.len(),
};
assert_parse_ok_eq(input, StringLit::parse(input), expected.clone(), "StringLit::parse");
assert_parse_ok_eq(
input, Literal::parse(input), Literal::String(expected.clone()), "Literal::parse");
- assert_eq!(StringLit::parse(input).unwrap().value(), $lit);
- assert_eq!(StringLit::parse(input).unwrap().into_value(), $lit);
+ let lit = StringLit::parse(input).unwrap();
+ assert_eq!(lit.value(), $lit);
+ assert_eq!(lit.suffix(), $suffix);
+ assert_eq!(lit.into_value(), $lit);
assert_roundtrip(expected.into_owned(), input);
};
}
@@ -47,6 +53,7 @@
raw: &*input,
value: None,
num_hashes,
+ start_suffix: input.len(),
};
assert_parse_ok_eq(
&input, StringLit::parse(&*input), expected.clone(), "StringLit::parse");
@@ -186,16 +193,23 @@
}
#[test]
+fn suffixes() {
+ check!("hello", r###""hello"suffix"###, false, None, "suffix");
+ check!(r"ãåãŊããæŧãã§ãã", r###"r"ãåãŊããæŧãã§ãã"_banana"###, false, Some(0), "_banana");
+ check!("fox", r#""fox"peter"#, false, None, "peter");
+ check!("ðĶ", r#""ðĶ"peter"#, false, None, "peter");
+ check!("ā°Ļā°āąā°\\\\u{0b10}", r###""ā°Ļā°āąā°\\\\u{0b10}"jü_rgen"###, true, None, "jü_rgen");
+}
+
+#[test]
fn parse_err() {
assert_err!(StringLit, r#"""#, UnterminatedString, None);
assert_err!(StringLit, r#""įŽ"#, UnterminatedString, None);
assert_err!(StringLit, r#""Jürgen"#, UnterminatedString, None);
assert_err!(StringLit, r#""foo bar baz"#, UnterminatedString, None);
- assert_err!(StringLit, r#""fox"peter"#, UnexpectedChar, 5..10);
- assert_err!(StringLit, r#""fox"peter""#, UnexpectedChar, 5..11);
- assert_err!(StringLit, r#""fox"ðĶ"#, UnexpectedChar, 5..9);
- assert_err!(StringLit, r###"r#"foo "# bar"#"###, UnexpectedChar, 9..15);
+ assert_err!(StringLit, r#""fox"peter""#, InvalidSuffix, 5);
+ assert_err!(StringLit, r###"r#"foo "# bar"#"###, UnexpectedChar, 9);
assert_err!(StringLit, "\"\r\"", IsolatedCr, 1);
assert_err!(StringLit, "\"fo\rx\"", IsolatedCr, 3);
@@ -225,10 +239,10 @@
}
#[test]
-fn invald_escapes() {
+fn invalid_escapes() {
assert_err!(StringLit, r#""\a""#, UnknownEscape, 1..3);
assert_err!(StringLit, r#""foo\y""#, UnknownEscape, 4..6);
- assert_err!(StringLit, r#""\"#, UnterminatedString, None);
+ assert_err!(StringLit, r#""\"#, UnterminatedEscape, 1);
assert_err!(StringLit, r#""\x""#, UnterminatedEscape, 1..3);
assert_err!(StringLit, r#""ðĶ\x1""#, UnterminatedEscape, 5..8);
assert_err!(StringLit, r#"" \xaj""#, InvalidXEscape, 2..6);
diff --git a/crates/litrs/src/tests.rs b/crates/litrs/src/tests.rs
index 526917e..613b429 100644
--- a/crates/litrs/src/tests.rs
+++ b/crates/litrs/src/tests.rs
@@ -25,25 +25,16 @@
#[test]
fn misc() {
- assert_err_single!(Literal::parse("0x44.5"), InvalidIntegerTypeSuffix, 4..6);
+ assert_err_single!(Literal::parse("0x44.5"), UnexpectedChar, 4..6);
assert_err_single!(Literal::parse("a"), InvalidLiteral, None);
assert_err_single!(Literal::parse(";"), InvalidLiteral, None);
assert_err_single!(Literal::parse("0;"), UnexpectedChar, 1);
- assert_err_single!(Literal::parse("0a"), UnexpectedChar, 1);
- assert_err_single!(Literal::parse("0z"), UnexpectedChar, 1);
assert_err_single!(Literal::parse(" 0"), InvalidLiteral, None);
assert_err_single!(Literal::parse("0 "), UnexpectedChar, 1);
- assert_err_single!(Literal::parse("0a3"), UnexpectedChar, 1);
- assert_err_single!(Literal::parse("0z3"), UnexpectedChar, 1);
assert_err_single!(Literal::parse("_"), InvalidLiteral, None);
assert_err_single!(Literal::parse("_3"), InvalidLiteral, None);
- assert_err_single!(Literal::parse("12a3"), UnexpectedChar, 2);
- assert_err_single!(Literal::parse("12f3"), InvalidFloatTypeSuffix, 2..4);
- assert_err_single!(Literal::parse("12f_"), InvalidFloatTypeSuffix, 2..4);
- assert_err_single!(Literal::parse("12F_"), UnexpectedChar, 2);
assert_err_single!(Literal::parse("a_123"), InvalidLiteral, None);
assert_err_single!(Literal::parse("B_123"), InvalidLiteral, None);
- assert_err_single!(Literal::parse("54321a64"), UnexpectedChar, 5);
}
macro_rules! assert_no_panic {
@@ -113,7 +104,11 @@
($input:expr, expected: $expected:path, actual: $actual:path $(,)?) => {
let err = $input.unwrap_err();
if err.expected != $expected {
- panic!("err.expected was expected to be {:?}, but is {:?}", $expected, err.expected);
+ panic!(
+ "err.expected was expected to be {:?}, but is {:?}",
+ $expected,
+ err.expected,
+ );
}
if err.actual != $actual {
panic!("err.actual was expected to be {:?}, but is {:?}", $actual, err.actual);
@@ -177,7 +172,10 @@
assert_eq!(Literal::from(FloatLit::try_from(pm_f32_lit.clone()).unwrap()), f32_lit);
assert_eq!(Literal::from(FloatLit::try_from(pm_f64_lit.clone()).unwrap()), f64_lit);
assert_eq!(Literal::from(StringLit::try_from(pm_string_lit.clone()).unwrap()), string_lit);
- assert_eq!(Literal::from(ByteStringLit::try_from(pm_bytestr_lit.clone()).unwrap()), bytestr_lit);
+ assert_eq!(
+ Literal::from(ByteStringLit::try_from(pm_bytestr_lit.clone()).unwrap()),
+ bytestr_lit,
+ );
assert_eq!(Literal::from(CharLit::try_from(pm_char_lit.clone()).unwrap()), char_lit);
assert_invalid_token!(
diff --git a/pseudo_crate/Cargo.lock b/pseudo_crate/Cargo.lock
index 39eaecd..286827b 100644
--- a/pseudo_crate/Cargo.lock
+++ b/pseudo_crate/Cargo.lock
@@ -260,7 +260,7 @@
"linked-hash-map",
"linkme",
"linkme-impl",
- "litrs 0.3.0",
+ "litrs",
"lock_api",
"log",
"lru-cache",
@@ -1724,7 +1724,7 @@
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cb6969eaabd2421f8a2775cfd2471a2b634372b4a25d41e3bd647b79912850a0"
dependencies = [
- "litrs 0.4.1",
+ "litrs",
]
[[package]]
@@ -3023,18 +3023,12 @@
[[package]]
name = "litrs"
-version = "0.3.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b487d13a3f4b465df87895a37b24e364907019afa12d943528df5b7abe0836f1"
-dependencies = [
- "proc-macro2 1.0.92",
-]
-
-[[package]]
-name = "litrs"
version = "0.4.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b4ce301924b7887e9d637144fdade93f9dfff9b60981d4ac161db09720d39aa5"
+dependencies = [
+ "proc-macro2 1.0.92",
+]
[[package]]
name = "lock_api"
diff --git a/pseudo_crate/Cargo.toml b/pseudo_crate/Cargo.toml
index 419e9da..9b62d56 100644
--- a/pseudo_crate/Cargo.toml
+++ b/pseudo_crate/Cargo.toml
@@ -175,7 +175,7 @@
linked-hash-map = "=0.5.6"
linkme = "=0.3.10"
linkme-impl = "=0.3.10"
-litrs = "=0.3.0"
+litrs = "=0.4.1"
lock_api = "=0.4.12"
log = "=0.4.22"
lru-cache = "=0.1.2"
