From e39c9a20bed6bc69d4efe9e2fe12d923900fcd67 Mon Sep 17 00:00:00 2001 From: Valentin Gosu <1454649+valenting@users.noreply.github.com> Date: Tue, 7 Nov 2023 22:32:21 +0100 Subject: [PATCH 1/7] Fix clippy (#878) --- url/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/url/src/lib.rs b/url/src/lib.rs index 58c7c07cd..6879d9f63 100644 --- a/url/src/lib.rs +++ b/url/src/lib.rs @@ -2697,7 +2697,7 @@ impl Ord for Url { impl PartialOrd for Url { #[inline] fn partial_cmp(&self, other: &Self) -> Option { - self.serialization.partial_cmp(&other.serialization) + Some(self.cmp(other)) } } From ae8d29e04a9a1d7f4e9cd799fdd0ac3e788a7b02 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bennet=20Ble=C3=9Fmann?= Date: Wed, 8 Nov 2023 09:29:28 +0100 Subject: [PATCH 2/7] use checked addition to not panic in debug build (#877) * commit always dirty file * add test to idna for bad punycode * use checked addition to not panic in debug build --- debug_metadata/README.md | 222 ++++++++++++++--------------- idna/src/punycode.rs | 5 +- idna/tests/bad_punycode_tests.json | 6 + idna/tests/punycode.rs | 37 +++++ 4 files changed, 155 insertions(+), 115 deletions(-) create mode 100644 idna/tests/bad_punycode_tests.json diff --git a/debug_metadata/README.md b/debug_metadata/README.md index ddf038d16..dab8de632 100644 --- a/debug_metadata/README.md +++ b/debug_metadata/README.md @@ -1,111 +1,111 @@ -## Debugger Visualizers - -Many languages and debuggers enable developers to control how a type is -displayed in a debugger. These are called "debugger visualizations" or "debugger -views". - -The Windows debuggers (WinDbg\CDB) support defining custom debugger visualizations using -the `Natvis` framework. To use Natvis, developers write XML documents using the natvis -schema that describe how debugger types should be displayed with the `.natvis` extension. -(See: https://docs.microsoft.com/en-us/visualstudio/debugger/create-custom-views-of-native-objects?view=vs-2019) -The Natvis files provide patterns which match type names a description of how to display -those types. - -The Natvis schema can be found either online (See: https://code.visualstudio.com/docs/cpp/natvis#_schema) -or locally at `\Xml\Schemas\1033\natvis.xsd`. - -The GNU debugger (GDB) supports defining custom debugger views using Pretty Printers. -Pretty printers are written as python scripts that describe how a type should be displayed -when loaded up in GDB/LLDB. (See: https://sourceware.org/gdb/onlinedocs/gdb/Pretty-Printing.html#Pretty-Printing) -The pretty printers provide patterns, which match type names, and for matching -types, descibe how to display those types. (For writing a pretty printer, see: https://sourceware.org/gdb/onlinedocs/gdb/Writing-a-Pretty_002dPrinter.html#Writing-a-Pretty_002dPrinter). - -### Embedding Visualizers - -Through the use of the `#[debugger_visualizer]` attribute, the `url` crate can embed -debugger visualizers into the crate metadata. - -Currently the two types of visualizers supported are Natvis and Pretty printers. - -For Natvis files, when linking an executable with a crate that includes Natvis files, -the MSVC linker will embed the contents of all Natvis files into the generated `PDB`. - -For pretty printers, the compiler will encode the contents of the pretty printer -in the `.debug_gdb_scripts` section of the `ELF` generated. - -### Testing Visualizers - -The `url` crate supports testing debugger visualizers defined for this crate. The entry point for -these tests are `tests/debugger_visualizer.rs`. These tests are defined using the `debugger_test` and -`debugger_test_parser` crates. The `debugger_test` crate is a proc macro crate which defines a -single proc macro attribute, `#[debugger_test]`. For more detailed information about this crate, -see https://crates.io/crates/debugger_test. The CI pipeline for the `url` crate has been updated -to run the debugger visualizer tests to ensure debugger visualizers do not become broken/stale. - -The `#[debugger_test]` proc macro attribute may only be used on test functions and will run the -function under the debugger specified by the `debugger` meta item. - -This proc macro attribute has 3 required values: - -1. The first required meta item, `debugger`, takes a string value which specifies the debugger to launch. -2. The second required meta item, `commands`, takes a string of new line (`\n`) separated list of debugger -commands to run. -3. The third required meta item, `expected_statements`, takes a string of new line (`\n`) separated list of -statements that must exist in the debugger output. Pattern matching through regular expressions is also -supported by using the `pattern:` prefix for each expected statement. - -#### Example: - -```rust -#[debugger_test( - debugger = "cdb", - commands = "command1\ncommand2\ncommand3", - expected_statements = "statement1\nstatement2\nstatement3")] -fn test() { - -} -``` - -Using a multiline string is also supported, with a single debugger command/expected statement per line: - -```rust -#[debugger_test( - debugger = "cdb", - commands = " -command1 -command2 -command3", - expected_statements = " -statement1 -pattern:statement[0-9]+ -statement3")] -fn test() { - -} -``` - -In the example above, the second expected statement uses pattern matching through a regular expression -by using the `pattern:` prefix. - -#### Testing Locally - -Currently, only Natvis visualizations have been defined for the `url` crate via `debug_metadata/url.natvis`, -which means the `tests/debugger_visualizer.rs` tests need to be run on Windows using the `*-pc-windows-msvc` targets. -To run these tests locally, first ensure the debugging tools for Windows are installed or install them following -the steps listed here, [Debugging Tools for Windows](https://docs.microsoft.com/en-us/windows-hardware/drivers/debugger/). -Once the debugging tools have been installed, the tests can be run in the same manner as they are in the CI -pipeline. - -#### Note - -When running the debugger visualizer tests, `tests/debugger_visualizer.rs`, they need to be run consecutively -and not in parallel. This can be achieved by passing the flag `--test-threads=1` to rustc. This is due to -how the debugger tests are run. Each test marked with the `#[debugger_test]` attribute launches a debugger -and attaches it to the current test process. If tests are running in parallel, the test will try to attach -a debugger to the current process which may already have a debugger attached causing the test to fail. - -For example: - -``` -cargo test --test debugger_visualizer --features debugger_visualizer -- --test-threads=1 -``` +## Debugger Visualizers + +Many languages and debuggers enable developers to control how a type is +displayed in a debugger. These are called "debugger visualizations" or "debugger +views". + +The Windows debuggers (WinDbg\CDB) support defining custom debugger visualizations using +the `Natvis` framework. To use Natvis, developers write XML documents using the natvis +schema that describe how debugger types should be displayed with the `.natvis` extension. +(See: https://docs.microsoft.com/en-us/visualstudio/debugger/create-custom-views-of-native-objects?view=vs-2019) +The Natvis files provide patterns which match type names a description of how to display +those types. + +The Natvis schema can be found either online (See: https://code.visualstudio.com/docs/cpp/natvis#_schema) +or locally at `\Xml\Schemas\1033\natvis.xsd`. + +The GNU debugger (GDB) supports defining custom debugger views using Pretty Printers. +Pretty printers are written as python scripts that describe how a type should be displayed +when loaded up in GDB/LLDB. (See: https://sourceware.org/gdb/onlinedocs/gdb/Pretty-Printing.html#Pretty-Printing) +The pretty printers provide patterns, which match type names, and for matching +types, descibe how to display those types. (For writing a pretty printer, see: https://sourceware.org/gdb/onlinedocs/gdb/Writing-a-Pretty_002dPrinter.html#Writing-a-Pretty_002dPrinter). + +### Embedding Visualizers + +Through the use of the `#[debugger_visualizer]` attribute, the `url` crate can embed +debugger visualizers into the crate metadata. + +Currently the two types of visualizers supported are Natvis and Pretty printers. + +For Natvis files, when linking an executable with a crate that includes Natvis files, +the MSVC linker will embed the contents of all Natvis files into the generated `PDB`. + +For pretty printers, the compiler will encode the contents of the pretty printer +in the `.debug_gdb_scripts` section of the `ELF` generated. + +### Testing Visualizers + +The `url` crate supports testing debugger visualizers defined for this crate. The entry point for +these tests are `tests/debugger_visualizer.rs`. These tests are defined using the `debugger_test` and +`debugger_test_parser` crates. The `debugger_test` crate is a proc macro crate which defines a +single proc macro attribute, `#[debugger_test]`. For more detailed information about this crate, +see https://crates.io/crates/debugger_test. The CI pipeline for the `url` crate has been updated +to run the debugger visualizer tests to ensure debugger visualizers do not become broken/stale. + +The `#[debugger_test]` proc macro attribute may only be used on test functions and will run the +function under the debugger specified by the `debugger` meta item. + +This proc macro attribute has 3 required values: + +1. The first required meta item, `debugger`, takes a string value which specifies the debugger to launch. +2. The second required meta item, `commands`, takes a string of new line (`\n`) separated list of debugger +commands to run. +3. The third required meta item, `expected_statements`, takes a string of new line (`\n`) separated list of +statements that must exist in the debugger output. Pattern matching through regular expressions is also +supported by using the `pattern:` prefix for each expected statement. + +#### Example: + +```rust +#[debugger_test( + debugger = "cdb", + commands = "command1\ncommand2\ncommand3", + expected_statements = "statement1\nstatement2\nstatement3")] +fn test() { + +} +``` + +Using a multiline string is also supported, with a single debugger command/expected statement per line: + +```rust +#[debugger_test( + debugger = "cdb", + commands = " +command1 +command2 +command3", + expected_statements = " +statement1 +pattern:statement[0-9]+ +statement3")] +fn test() { + +} +``` + +In the example above, the second expected statement uses pattern matching through a regular expression +by using the `pattern:` prefix. + +#### Testing Locally + +Currently, only Natvis visualizations have been defined for the `url` crate via `debug_metadata/url.natvis`, +which means the `tests/debugger_visualizer.rs` tests need to be run on Windows using the `*-pc-windows-msvc` targets. +To run these tests locally, first ensure the debugging tools for Windows are installed or install them following +the steps listed here, [Debugging Tools for Windows](https://docs.microsoft.com/en-us/windows-hardware/drivers/debugger/). +Once the debugging tools have been installed, the tests can be run in the same manner as they are in the CI +pipeline. + +#### Note + +When running the debugger visualizer tests, `tests/debugger_visualizer.rs`, they need to be run consecutively +and not in parallel. This can be achieved by passing the flag `--test-threads=1` to rustc. This is due to +how the debugger tests are run. Each test marked with the `#[debugger_test]` attribute launches a debugger +and attaches it to the current test process. If tests are running in parallel, the test will try to attach +a debugger to the current process which may already have a debugger attached causing the test to fail. + +For example: + +``` +cargo test --test debugger_visualizer --features debugger_visualizer -- --test-threads=1 +``` diff --git a/idna/src/punycode.rs b/idna/src/punycode.rs index faef379ca..8d26ce25e 100644 --- a/idna/src/punycode.rs +++ b/idna/src/punycode.rs @@ -269,10 +269,7 @@ where for c in input.clone() { let c = c as u32; if c < code_point { - delta += 1; - if delta == 0 { - return Err(()); // Overflow - } + delta = delta.checked_add(1).ok_or(())?; } if c == code_point { // Represent delta as a generalized variable-length integer: diff --git a/idna/tests/bad_punycode_tests.json b/idna/tests/bad_punycode_tests.json new file mode 100644 index 000000000..c0115290c --- /dev/null +++ b/idna/tests/bad_punycode_tests.json @@ -0,0 +1,6 @@ +[ + { + "description": "issue 870", + "decoded": " 񠀷 󠀷 󠀷 󠀷 󠀷 󠀷 󠀷 󠀷 󠀷 󠀷 󠀷 󠀷 󠀷 󠀷 21日 ᄢ 21日㩴 ᄢ 21日 ᄢ " + } +] diff --git a/idna/tests/punycode.rs b/idna/tests/punycode.rs index 95674137a..2867e4966 100644 --- a/idna/tests/punycode.rs +++ b/idna/tests/punycode.rs @@ -10,6 +10,7 @@ use crate::test::TestFn; use idna::punycode::{decode, encode_str}; use serde_json::map::Map; use serde_json::Value; +use std::panic::catch_unwind; use std::str::FromStr; fn one_test(decoded: &str, encoded: &str) { @@ -39,6 +40,15 @@ fn one_test(decoded: &str, encoded: &str) { } } +fn one_bad_test(encode: &str) { + let result = catch_unwind(|| encode_str(encode)); + assert!( + matches!(&result, Ok(None)), + "Should neither panic nor return Some result, but got {:?}", + result + ) +} + fn get_string<'a>(map: &'a Map, key: &str) -> &'a str { match map.get(&key.to_string()) { Some(Value::String(s)) => s, @@ -74,4 +84,31 @@ pub fn collect_tests(add_test: &mut F) { } other => panic!("{:?}", other), } + + match Value::from_str(include_str!("bad_punycode_tests.json")) { + Ok(Value::Array(tests)) => { + for (i, test) in tests.into_iter().enumerate() { + match test { + Value::Object(o) => { + let test_name = { + let desc = get_string(&o, "description"); + if desc.is_empty() { + format!("Bad Punycode {}", i + 1) + } else { + format!("Bad Punycode {}: {}", i + 1, desc) + } + }; + add_test( + test_name, + TestFn::DynTestFn(Box::new(move || { + one_bad_test(get_string(&o, "decoded")) + })), + ) + } + _ => panic!(), + } + } + } + other => panic!("{:?}", other), + } } From 67fc2730b6585ea60f7da6987459c554775967af Mon Sep 17 00:00:00 2001 From: edgul Date: Thu, 9 Nov 2023 10:28:18 -0500 Subject: [PATCH 3/7] Fix search setting for non-special urls with space, query and fragment (#879) --- url/src/lib.rs | 4 +++- url/tests/expected_failures.txt | 4 +--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/url/src/lib.rs b/url/src/lib.rs index 6879d9f63..febdb3609 100644 --- a/url/src/lib.rs +++ b/url/src/lib.rs @@ -1566,7 +1566,9 @@ impl Url { }); } else { self.query_start = None; - self.strip_trailing_spaces_from_opaque_path(); + if fragment.is_none() { + self.strip_trailing_spaces_from_opaque_path(); + } } self.restore_already_parsed_fragment(fragment); diff --git a/url/tests/expected_failures.txt b/url/tests/expected_failures.txt index 1c404a8b7..168a8327f 100644 --- a/url/tests/expected_failures.txt +++ b/url/tests/expected_failures.txt @@ -42,12 +42,10 @@ set pathname to <> set port to set href to - set search to <> - set search to <> set pathname to <\\\\> set pathname to set pathname to

set pathname to

set pathname to - set pathname to

\ No newline at end of file + set pathname to

From 5f454e20310f452dec9822aa0a42655c870e8c63 Mon Sep 17 00:00:00 2001 From: Redfire Date: Fri, 10 Nov 2023 07:56:24 +0800 Subject: [PATCH 4/7] Added #[must_use] Attributes for Configuration Options (#876) --- idna/src/uts46.rs | 1 + url/src/lib.rs | 1 + 2 files changed, 2 insertions(+) diff --git a/idna/src/uts46.rs b/idna/src/uts46.rs index 2ffb8414b..b082416c8 100644 --- a/idna/src/uts46.rs +++ b/idna/src/uts46.rs @@ -509,6 +509,7 @@ impl Idna { } #[derive(Clone, Copy)] +#[must_use] pub struct Config { use_std3_ascii_rules: bool, transitional_processing: bool, diff --git a/url/src/lib.rs b/url/src/lib.rs index febdb3609..71b12ff64 100644 --- a/url/src/lib.rs +++ b/url/src/lib.rs @@ -205,6 +205,7 @@ pub struct Url { /// Full configuration for the URL parser. #[derive(Copy, Clone)] +#[must_use] pub struct ParseOptions<'a> { base_url: Option<&'a Url>, encoding_override: EncodingOverride<'a>, From 912d716b04347ff35fa78dcd7b0567abc13d62dd Mon Sep 17 00:00:00 2001 From: sprock Date: Fri, 10 Nov 2023 02:53:46 -0700 Subject: [PATCH 5/7] Correct spelling mistake in `Position` docs (#875) The sentence in question was clearly intended to remark that the leading slash in the path is part of said path, but due to the missing "e" at the end of "Note", it (ungrammatically) says the opposite. Co-authored-by: Maeve Sproule --- url/src/slicing.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/url/src/slicing.rs b/url/src/slicing.rs index c061fee84..13829575d 100644 --- a/url/src/slicing.rs +++ b/url/src/slicing.rs @@ -93,7 +93,7 @@ fn test_count_digits() { /// /// The end of a component and the start of the next are either the same or separate /// by a delimiter. -/// (Not that the initial `/` of a path is considered part of the path here, not a delimiter.) +/// (Note that the initial `/` of a path is considered part of the path here, not a delimiter.) /// For example, `&url[..BeforeFragment]` would include a `#` delimiter (if present in `url`), /// so `&url[..AfterQuery]` might be desired instead. /// From 464b1f7d8fe9de117af184c1acb52b4f821a0cdc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bennet=20Ble=C3=9Fmann?= Date: Fri, 17 Nov 2023 12:01:09 +0100 Subject: [PATCH 6/7] Fix another overflow in punycode encode_into (#880) * add another overflowing test * fix overflow in case the input has more than u32::MAX characters * detect hugh length early --- idna/src/punycode.rs | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/idna/src/punycode.rs b/idna/src/punycode.rs index 8d26ce25e..b1afc96f7 100644 --- a/idna/src/punycode.rs +++ b/idna/src/punycode.rs @@ -215,6 +215,9 @@ impl<'a> ExactSizeIterator for Decode<'a> { /// This is a convenience wrapper around `encode`. #[inline] pub fn encode_str(input: &str) -> Option { + if input.len() > u32::MAX as usize { + return None; + } let mut buf = String::with_capacity(input.len()); encode_into(input.chars(), &mut buf).ok().map(|()| buf) } @@ -224,6 +227,9 @@ pub fn encode_str(input: &str) -> Option { /// Return None on overflow, which can only happen on inputs that would take more than /// 63 encoded bytes, the DNS limit on domain name labels. pub fn encode(input: &[char]) -> Option { + if input.len() > u32::MAX as usize { + return None; + } let mut buf = String::with_capacity(input.len()); encode_into(input.iter().copied(), &mut buf) .ok() @@ -235,9 +241,9 @@ where I: Iterator + Clone, { // Handle "basic" (ASCII) code points. They are encoded as-is. - let (mut input_length, mut basic_length) = (0, 0); + let (mut input_length, mut basic_length) = (0u32, 0); for c in input.clone() { - input_length += 1; + input_length = input_length.checked_add(1).ok_or(())?; if c.is_ascii() { output.push(c); basic_length += 1; @@ -311,3 +317,12 @@ fn value_to_digit(value: u32) -> char { _ => panic!(), } } + +#[test] +#[ignore = "slow"] +#[cfg(target_pointer_width = "64")] +fn huge_encode() { + let mut buf = String::new(); + assert!(encode_into(std::iter::repeat('ß').take(u32::MAX as usize + 1), &mut buf).is_err()); + assert_eq!(buf.len(), 0); +} From 00e9e18ce6504e0e9157816c593afec8e69cb9e3 Mon Sep 17 00:00:00 2001 From: Valentin Gosu <1454649+valenting@users.noreply.github.com> Date: Wed, 22 Nov 2023 19:56:54 +0100 Subject: [PATCH 7/7] Update url 2.5.0 (#885) * Update data-url to 0.3.1 * Update percent_encoding to 2.3.1 * Update form_urlencoded to 1.2.1 * Update idna to 0.5.0 * Update url to 2.5.0 --- data-url/Cargo.toml | 2 +- form_urlencoded/Cargo.toml | 2 +- idna/Cargo.toml | 2 +- percent_encoding/Cargo.toml | 2 +- url/Cargo.toml | 8 ++++---- url/src/lib.rs | 2 +- 6 files changed, 9 insertions(+), 9 deletions(-) diff --git a/data-url/Cargo.toml b/data-url/Cargo.toml index 0edfd281a..19a075383 100644 --- a/data-url/Cargo.toml +++ b/data-url/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "data-url" -version = "0.3.0" +version = "0.3.1" authors = ["Simon Sapin "] description = "Processing of data: URL according to WHATWG’s Fetch Standard" categories = ["no_std"] diff --git a/form_urlencoded/Cargo.toml b/form_urlencoded/Cargo.toml index 8e385d790..cf23948da 100644 --- a/form_urlencoded/Cargo.toml +++ b/form_urlencoded/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "form_urlencoded" -version = "1.2.0" +version = "1.2.1" authors = ["The rust-url developers"] description = "Parser and serializer for the application/x-www-form-urlencoded syntax, as used by HTML forms." categories = ["no_std"] diff --git a/idna/Cargo.toml b/idna/Cargo.toml index e5c9886a6..9ca92d688 100644 --- a/idna/Cargo.toml +++ b/idna/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "idna" -version = "0.4.0" +version = "0.5.0" authors = ["The rust-url developers"] description = "IDNA (Internationalizing Domain Names in Applications) and Punycode." categories = ["no_std"] diff --git a/percent_encoding/Cargo.toml b/percent_encoding/Cargo.toml index 4a3c08712..5f659ad50 100644 --- a/percent_encoding/Cargo.toml +++ b/percent_encoding/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "percent-encoding" -version = "2.3.0" +version = "2.3.1" authors = ["The rust-url developers"] description = "Percent encoding and decoding" categories = ["no_std"] diff --git a/url/Cargo.toml b/url/Cargo.toml index 969706d92..a50ab31f1 100644 --- a/url/Cargo.toml +++ b/url/Cargo.toml @@ -2,7 +2,7 @@ name = "url" # When updating version, also modify html_root_url in the lib.rs -version = "2.4.1" +version = "2.5.0" authors = ["The rust-url developers"] description = "URL library for Rust, based on the WHATWG URL Standard" @@ -22,9 +22,9 @@ serde_json = "1.0" bencher = "0.1" [dependencies] -form_urlencoded = { version = "1.2.0", path = "../form_urlencoded" } -idna = { version = "0.4.0", path = "../idna" } -percent-encoding = { version = "2.3.0", path = "../percent_encoding" } +form_urlencoded = { version = "1.2.1", path = "../form_urlencoded" } +idna = { version = "0.5.0", path = "../idna" } +percent-encoding = { version = "2.3.1", path = "../percent_encoding" } serde = { version = "1.0", optional = true, features = ["derive"] } [features] diff --git a/url/src/lib.rs b/url/src/lib.rs index 71b12ff64..b17b82573 100644 --- a/url/src/lib.rs +++ b/url/src/lib.rs @@ -134,7 +134,7 @@ url = { version = "2", features = ["debugger_visualizer"] } */ -#![doc(html_root_url = "https://docs.rs/url/2.4.1")] +#![doc(html_root_url = "https://docs.rs/url/2.5.0")] #![cfg_attr( feature = "debugger_visualizer", debugger_visualizer(natvis_file = "../../debug_metadata/url.natvis")