From 68ba8f6e62c7c0a12e3a62cffa2f1cdf7d1f5f30 Mon Sep 17 00:00:00 2001 From: Jonathan Giddy Date: Thu, 24 Aug 2023 08:49:40 +0100 Subject: [PATCH 1/6] Use explicit Default for GzHeaderState enum This allows the crate to build with Rust older than 1.62. --- src/gz/mod.rs | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/gz/mod.rs b/src/gz/mod.rs index e8e05c6e..31a69611 100644 --- a/src/gz/mod.rs +++ b/src/gz/mod.rs @@ -87,7 +87,7 @@ impl GzHeader { } } -#[derive(Debug, Default)] +#[derive(Debug)] pub enum GzHeaderState { Start(u8, [u8; 10]), Xlen(Option>, u8, [u8; 2]), @@ -95,10 +95,15 @@ pub enum GzHeaderState { Filename(Option>), Comment(Option>), Crc(Option>, u8, [u8; 2]), - #[default] Complete, } +impl Default for GzHeaderState { + fn default() -> Self { + Self::Complete + } +} + #[derive(Debug, Default)] pub struct GzHeaderParser { state: GzHeaderState, From 82e45fa8901824344cf636a37c03b2595478a4d1 Mon Sep 17 00:00:00 2001 From: Lukasz Anforowicz Date: Tue, 29 Aug 2023 20:19:24 +0000 Subject: [PATCH 2/6] Refactoring: Dedupe code into `write_to_spare_capacity_of_vec` helper. --- src/mem.rs | 64 +++++++++++++++++++++++++++++++++--------------------- 1 file changed, 39 insertions(+), 25 deletions(-) diff --git a/src/mem.rs b/src/mem.rs index 6313c220..6d311a6f 100644 --- a/src/mem.rs +++ b/src/mem.rs @@ -1,7 +1,6 @@ use std::error::Error; use std::fmt; use std::io; -use std::slice; use crate::ffi::{self, Backend, Deflate, DeflateBackend, ErrorMessage, Inflate, InflateBackend}; use crate::Compression; @@ -342,19 +341,12 @@ impl Compress { output: &mut Vec, flush: FlushCompress, ) -> Result { - let cap = output.capacity(); - let len = output.len(); - - unsafe { + write_to_spare_capacity_of_vec(output, |out| { let before = self.total_out(); - let ret = { - let ptr = output.as_mut_ptr().add(len); - let out = slice::from_raw_parts_mut(ptr, cap - len); - self.compress(input, out, flush) - }; - output.set_len((self.total_out() - before) as usize + len); - ret - } + let ret = self.compress(input, out, flush); + let bytes_written = self.total_out() - before; + (bytes_written as usize, ret) + }) } } @@ -473,19 +465,12 @@ impl Decompress { output: &mut Vec, flush: FlushDecompress, ) -> Result { - let cap = output.capacity(); - let len = output.len(); - - unsafe { + write_to_spare_capacity_of_vec(output, |out| { let before = self.total_out(); - let ret = { - let ptr = output.as_mut_ptr().add(len); - let out = slice::from_raw_parts_mut(ptr, cap - len); - self.decompress(input, out, flush) - }; - output.set_len((self.total_out() - before) as usize + len); - ret - } + let ret = self.decompress(input, out, flush); + let bytes_written = self.total_out() - before; + (bytes_written as usize, ret) + }) } /// Specifies the decompression dictionary to use. @@ -574,6 +559,35 @@ impl fmt::Display for CompressError { } } +/// Allows `writer` to write data into the spare capacity of the `output` vector. +/// This will not reallocate the vector provided or attempt to grow it, so space +/// for the `output` must be reserved by the caller before calling this +/// function. +/// +/// `writer` needs to return the number of bytes written (and can also return +/// another arbitrary return value). +fn write_to_spare_capacity_of_vec( + output: &mut Vec, + writer: impl FnOnce(&mut [u8]) -> (usize, T), +) -> T { + let cap = output.capacity(); + let len = output.len(); + + // FIXME: This is unsound - see https://github.com/rust-lang/flate2-rs/issues/220 + // (The code below reimplements `Vec::spare_capacity_mut`, but returns `&mut [u8]` + // instead of `&mut [MaybeUninit]`.) + unsafe { + let (bytes_written, ret) = { + let ptr = output.as_mut_ptr().add(len); + let out = slice::from_raw_parts_mut(ptr, cap - len); + writer(out) + }; + let new_len = core::cmp::min(len + bytes_written, cap); // Sanitizes `bytes_written`. + output.set_len(new_len); + ret + } +} + #[cfg(test)] mod tests { use std::io::Write; From 69972b8262fbe03c28450c4c18961b41ad92af6a Mon Sep 17 00:00:00 2001 From: Lukasz Anforowicz Date: Tue, 29 Aug 2023 20:52:45 +0000 Subject: [PATCH 3/6] Fix soundness of `write_to_spare_capacity_of_vec`. Fixes https://github.com/rust-lang/flate2-rs/issues/220 --- src/mem.rs | 20 +++++++------------- 1 file changed, 7 insertions(+), 13 deletions(-) diff --git a/src/mem.rs b/src/mem.rs index 6d311a6f..d4a50917 100644 --- a/src/mem.rs +++ b/src/mem.rs @@ -573,19 +573,13 @@ fn write_to_spare_capacity_of_vec( let cap = output.capacity(); let len = output.len(); - // FIXME: This is unsound - see https://github.com/rust-lang/flate2-rs/issues/220 - // (The code below reimplements `Vec::spare_capacity_mut`, but returns `&mut [u8]` - // instead of `&mut [MaybeUninit]`.) - unsafe { - let (bytes_written, ret) = { - let ptr = output.as_mut_ptr().add(len); - let out = slice::from_raw_parts_mut(ptr, cap - len); - writer(out) - }; - let new_len = core::cmp::min(len + bytes_written, cap); // Sanitizes `bytes_written`. - output.set_len(new_len); - ret - } + output.resize(output.capacity(), 0); + let (bytes_written, ret) = writer(&mut output[len..]); + + let new_len = core::cmp::min(len + bytes_written, cap); // Sanitizes `bytes_written`. + output.resize(new_len, 0 /* unused */); + + ret } #[cfg(test)] From 5b23cc91269c44165b07cbfee493d4aa353724fd Mon Sep 17 00:00:00 2001 From: Georeth Zhou Date: Sun, 10 Sep 2023 23:59:04 +0800 Subject: [PATCH 4/6] Fix and unify docs of `bufread` and `read` types. - fix wrong docs of `bufread::GzDecoder`, `bufread::GzEncoder`, `bufread::MultiGzDecoder` - make the types of inner reader and exported trait explicit. - explicit state the pull model: user initialize the read, then data is read from the underlying stream. --- src/deflate/bufread.rs | 10 ++++++---- src/deflate/read.rs | 8 ++++---- src/gz/bufread.rs | 16 +++++++++------- src/gz/read.rs | 14 +++++++------- src/zlib/bufread.rs | 10 ++++++---- src/zlib/read.rs | 8 ++++---- 6 files changed, 36 insertions(+), 30 deletions(-) diff --git a/src/deflate/bufread.rs b/src/deflate/bufread.rs index e375952d..c70a6303 100644 --- a/src/deflate/bufread.rs +++ b/src/deflate/bufread.rs @@ -7,9 +7,10 @@ use crate::{Compress, Decompress}; /// A DEFLATE encoder, or compressor. /// -/// This structure consumes a [`BufRead`] interface, reading uncompressed data -/// from the underlying reader, and emitting compressed data. +/// This structure implements a [`Read`] interface. When read from, it reads +/// uncompressed data from the underlying [`BufRead`] and provides the compressed data. /// +/// [`Read`]: https://doc.rust-lang.org/std/io/trait.Read.html /// [`BufRead`]: https://doc.rust-lang.org/std/io/trait.BufRead.html /// /// # Examples @@ -123,9 +124,10 @@ impl Write for DeflateEncoder { /// A DEFLATE decoder, or decompressor. /// -/// This structure consumes a [`BufRead`] interface, reading compressed data -/// from the underlying reader, and emitting uncompressed data. +/// This structure implements a [`Read`] interface. When read from, it reads +/// compressed data from the underlying [`BufRead`] and provides the uncompressed data. /// +/// [`Read`]: https://doc.rust-lang.org/std/io/trait.Read.html /// [`BufRead`]: https://doc.rust-lang.org/std/io/trait.BufRead.html /// /// # Examples diff --git a/src/deflate/read.rs b/src/deflate/read.rs index 5937e6f6..2b6b8f2f 100644 --- a/src/deflate/read.rs +++ b/src/deflate/read.rs @@ -6,8 +6,8 @@ use crate::bufreader::BufReader; /// A DEFLATE encoder, or compressor. /// -/// This structure implements a [`Read`] interface and will read uncompressed -/// data from an underlying stream and emit a stream of compressed data. +/// This structure implements a [`Read`] interface. When read from, it reads +/// uncompressed data from the underlying [`Read`] and provides the compressed data. /// /// [`Read`]: https://doc.rust-lang.org/std/io/trait.Read.html /// @@ -120,8 +120,8 @@ impl Write for DeflateEncoder { /// A DEFLATE decoder, or decompressor. /// -/// This structure implements a [`Read`] interface and takes a stream of -/// compressed data as input, providing the decompressed data when read from. +/// This structure implements a [`Read`] interface. When read from, it reads +/// compressed data from the underlying [`Read`] and provides the uncompressed data. /// /// [`Read`]: https://doc.rust-lang.org/std/io/trait.Read.html /// diff --git a/src/gz/bufread.rs b/src/gz/bufread.rs index 6fc48bcd..679b4a7d 100644 --- a/src/gz/bufread.rs +++ b/src/gz/bufread.rs @@ -19,10 +19,10 @@ fn copy(into: &mut [u8], from: &[u8], pos: &mut usize) -> usize { /// A gzip streaming encoder /// -/// This structure exposes a [`BufRead`] interface that will read uncompressed data -/// from the underlying reader and expose the compressed version as a [`BufRead`] -/// interface. +/// This structure implements a [`Read`] interface. When read from, it reads +/// uncompressed data from the underlying [`BufRead`] and provides the compressed data. /// +/// [`Read`]: https://doc.rust-lang.org/std/io/trait.Read.html /// [`BufRead`]: https://doc.rust-lang.org/std/io/trait.BufRead.html /// /// # Examples @@ -165,8 +165,8 @@ impl Write for GzEncoder { /// A decoder for a single member of a [gzip file]. /// -/// This structure exposes a [`BufRead`] interface, reading compressed data -/// from the underlying reader, and emitting uncompressed data. +/// This structure implements a [`Read`] interface. When read from, it reads +/// compressed data from the underlying [`BufRead`] and provides the uncompressed data. /// /// After reading a single member of the gzip data this reader will return /// Ok(0) even if there are more bytes available in the underlying reader. @@ -178,6 +178,7 @@ impl Write for GzEncoder { /// [in the introduction](../index.html#about-multi-member-gzip-files). /// /// [gzip file]: https://www.rfc-editor.org/rfc/rfc1952#page-5 +/// [`Read`]: https://doc.rust-lang.org/std/io/trait.Read.html /// [`BufRead`]: https://doc.rust-lang.org/std/io/trait.BufRead.html /// /// # Examples @@ -351,8 +352,8 @@ impl Write for GzDecoder { /// A gzip streaming decoder that decodes a [gzip file] that may have multiple members. /// -/// This structure exposes a [`BufRead`] interface that will consume compressed -/// data from the underlying reader and emit uncompressed data. +/// This structure implements a [`Read`] interface. When read from, it reads +/// compressed data from the underlying [`BufRead`] and provides the uncompressed data. /// /// A gzip file consists of a series of *members* concatenated one after another. /// MultiGzDecoder decodes all members from the data and only returns Ok(0) when the @@ -362,6 +363,7 @@ impl Write for GzDecoder { /// [in the introduction](../index.html#about-multi-member-gzip-files). /// /// [gzip file]: https://www.rfc-editor.org/rfc/rfc1952#page-5 +/// [`Read`]: https://doc.rust-lang.org/std/io/trait.Read.html /// [`BufRead`]: https://doc.rust-lang.org/std/io/trait.BufRead.html /// /// # Examples diff --git a/src/gz/read.rs b/src/gz/read.rs index 5a65526c..9dbadbda 100644 --- a/src/gz/read.rs +++ b/src/gz/read.rs @@ -8,9 +8,8 @@ use crate::Compression; /// A gzip streaming encoder /// -/// This structure exposes a [`Read`] interface that will read uncompressed data -/// from the underlying reader and expose the compressed version as a [`Read`] -/// interface. +/// This structure implements a [`Read`] interface. When read from, it reads +/// uncompressed data from the underlying [`Read`] and provides the compressed data. /// /// [`Read`]: https://doc.rust-lang.org/std/io/trait.Read.html /// @@ -92,8 +91,8 @@ impl Write for GzEncoder { /// A decoder for a single member of a [gzip file]. /// -/// This structure exposes a [`Read`] interface that will consume compressed -/// data from the underlying reader and emit uncompressed data. +/// This structure implements a [`Read`] interface. When read from, it reads +/// compressed data from the underlying [`Read`] and provides the uncompressed data. /// /// After reading a single member of the gzip data this reader will return /// Ok(0) even if there are more bytes available in the underlying reader. @@ -201,8 +200,9 @@ impl Write for GzDecoder { /// A gzip streaming decoder that decodes a [gzip file] that may have multiple members. /// -/// This structure exposes a [`Read`] interface that will consume compressed -/// data from the underlying reader and emit uncompressed data. +/// This structure implements a [`Read`] interface. When read from, it reads +/// compressed data from the underlying [`Read`] and provides the uncompressed +/// data. /// /// A gzip file consists of a series of *members* concatenated one after another. /// MultiGzDecoder decodes all members of a file and returns Ok(0) once the diff --git a/src/zlib/bufread.rs b/src/zlib/bufread.rs index aa8af64f..85bbd38a 100644 --- a/src/zlib/bufread.rs +++ b/src/zlib/bufread.rs @@ -7,9 +7,10 @@ use crate::{Compress, Decompress}; /// A ZLIB encoder, or compressor. /// -/// This structure consumes a [`BufRead`] interface, reading uncompressed data -/// from the underlying reader, and emitting compressed data. +/// This structure implements a [`Read`] interface. When read from, it reads +/// uncompressed data from the underlying [`BufRead`] and provides the compressed data. /// +/// [`Read`]: https://doc.rust-lang.org/std/io/trait.Read.html /// [`BufRead`]: https://doc.rust-lang.org/std/io/trait.BufRead.html /// /// # Examples @@ -128,9 +129,10 @@ impl Write for ZlibEncoder { /// A ZLIB decoder, or decompressor. /// -/// This structure consumes a [`BufRead`] interface, reading compressed data -/// from the underlying reader, and emitting uncompressed data. +/// This structure implements a [`Read`] interface. When read from, it reads +/// compressed data from the underlying [`BufRead`] and provides the uncompressed data. /// +/// [`Read`]: https://doc.rust-lang.org/std/io/trait.Read.html /// [`BufRead`]: https://doc.rust-lang.org/std/io/trait.BufRead.html /// /// # Examples diff --git a/src/zlib/read.rs b/src/zlib/read.rs index fbae7486..3b41ae6c 100644 --- a/src/zlib/read.rs +++ b/src/zlib/read.rs @@ -7,8 +7,8 @@ use crate::Decompress; /// A ZLIB encoder, or compressor. /// -/// This structure implements a [`Read`] interface and will read uncompressed -/// data from an underlying stream and emit a stream of compressed data. +/// This structure implements a [`Read`] interface. When read from, it reads +/// uncompressed data from the underlying [`Read`] and provides the compressed data. /// /// [`Read`]: https://doc.rust-lang.org/std/io/trait.Read.html /// @@ -126,8 +126,8 @@ impl Write for ZlibEncoder { /// A ZLIB decoder, or decompressor. /// -/// This structure implements a [`Read`] interface and takes a stream of -/// compressed data as input, providing the decompressed data when read from. +/// This structure implements a [`Read`] interface. When read from, it reads +/// compressed data from the underlying [`Read`] and provides the uncompressed data. /// /// [`Read`]: https://doc.rust-lang.org/std/io/trait.Read.html /// From 1260d3e88891672ad15c2d336537bdd8f607bb70 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Wed, 11 Oct 2023 08:25:32 +0200 Subject: [PATCH 5/6] prepare next patch-release --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index 1a481fb7..b59d2a6d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "flate2" authors = ["Alex Crichton ", "Josh Triplett "] -version = "1.0.27" +version = "1.0.28" edition = "2018" license = "MIT OR Apache-2.0" readme = "README.md" From 7a61ea527a2ed4657ba8c6b8e04360f5156dadd2 Mon Sep 17 00:00:00 2001 From: Manish Goregaokar Date: Thu, 12 Oct 2023 09:19:33 -0700 Subject: [PATCH 6/6] Reset StreamWrapper after calling mz_inflate / mz_deflate --- src/ffi/c.rs | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/ffi/c.rs b/src/ffi/c.rs index 48acd438..32864f8f 100644 --- a/src/ffi/c.rs +++ b/src/ffi/c.rs @@ -226,6 +226,12 @@ impl InflateBackend for Inflate { self.inner.total_in += (raw.next_in as usize - input.as_ptr() as usize) as u64; self.inner.total_out += (raw.next_out as usize - output.as_ptr() as usize) as u64; + // reset these pointers so we don't accidentally read them later + raw.next_in = ptr::null_mut(); + raw.avail_in = 0; + raw.next_out = ptr::null_mut(); + raw.avail_out = 0; + match rc { MZ_DATA_ERROR | MZ_STREAM_ERROR => mem::decompress_failed(self.inner.msg()), MZ_OK => Ok(Status::Ok), @@ -314,6 +320,12 @@ impl DeflateBackend for Deflate { self.inner.total_in += (raw.next_in as usize - input.as_ptr() as usize) as u64; self.inner.total_out += (raw.next_out as usize - output.as_ptr() as usize) as u64; + // reset these pointers so we don't accidentally read them later + raw.next_in = ptr::null_mut(); + raw.avail_in = 0; + raw.next_out = ptr::null_mut(); + raw.avail_out = 0; + match rc { MZ_OK => Ok(Status::Ok), MZ_BUF_ERROR => Ok(Status::BufError),