Thanks to visit codestin.com
Credit goes to github.com

Skip to content

Commit 50a3e25

Browse files
j178j178
authored
Preserve non-UTF8 filenames from git (#2023)
Preserve raw Git path bytes on Unix so non-UTF8 filenames no longer abort file collection. Glob filters now match paths directly; regex filters remain UTF-8-only. Closes #1701 Closes #649
1 parent 3268a83 commit 50a3e25

7 files changed

Lines changed: 143 additions & 63 deletions

File tree

‎crates/prek-identify/src/lib.rs‎

Lines changed: 22 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -18,13 +18,14 @@
1818
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
1919
// THE SOFTWARE.
2020

21-
use std::borrow::Cow;
2221
use std::io::{BufRead, Read};
2322
use std::ops::BitOrAssign;
2423
use std::path::Path;
2524

2625
#[cfg(feature = "serde")]
2726
use serde::de::{Error as DeError, SeqAccess, Visitor};
27+
#[cfg(any(feature = "serde", feature = "schemars"))]
28+
use std::borrow::Cow;
2829

2930
pub mod tags;
3031

@@ -300,19 +301,17 @@ pub fn tags_from_path(path: &Path) -> Result<TagSet, Error> {
300301

301302
fn tags_from_filename(filename: &Path) -> TagSet {
302303
let ext = filename.extension().and_then(|ext| ext.to_str());
303-
let filename = filename
304+
let mut result = filename
304305
.file_name()
305306
.and_then(|name| name.to_str())
306-
.expect("Invalid filename");
307-
308-
let mut result = tags::NAMES
309-
.get(filename)
310-
.or_else(|| {
311-
// Allow e.g. "Dockerfile.xenial" to match "Dockerfile".
312-
filename
313-
.split('.')
314-
.next()
315-
.and_then(|name| tags::NAMES.get(name))
307+
.and_then(|filename| {
308+
tags::NAMES.get(filename).or_else(|| {
309+
// Allow e.g. "Dockerfile.xenial" to match "Dockerfile".
310+
filename
311+
.split('.')
312+
.next()
313+
.and_then(|name| tags::NAMES.get(name))
314+
})
316315
})
317316
.copied()
318317
.unwrap_or_default();
@@ -597,6 +596,17 @@ mod tests {
597596
assert_tagset(&tags, &["text", "tiltfile"]);
598597
}
599598

599+
#[cfg(unix)]
600+
#[test]
601+
fn tags_from_non_utf8_filename_uses_utf8_extension_when_available() {
602+
use std::ffi::OsStr;
603+
use std::os::unix::ffi::OsStrExt as _;
604+
605+
let tags = super::tags_from_filename(Path::new(OsStr::from_bytes(b"bad-\xff.py")));
606+
607+
assert_tagset(&tags, &["python", "text"]);
608+
}
609+
600610
#[test]
601611
fn tags_from_interpreter() {
602612
let tags = super::tags_from_interpreter("/usr/bin/python3");

‎crates/prek/src/cli/auto_update/mod.rs‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -187,19 +187,19 @@ impl TagFilters {
187187
/// Repo-specific include filters override global include filters for that repo.
188188
fn is_included(&self, repo: &str, tag: &str) -> bool {
189189
if let Some(repo_include) = self.repo_include.get(repo) {
190-
return repo_include.is_empty() || repo_include.is_match(tag);
190+
return repo_include.is_empty() || repo_include.is_match(Path::new(tag));
191191
}
192192

193-
self.global_include.is_empty() || self.global_include.is_match(tag)
193+
self.global_include.is_empty() || self.global_include.is_match(Path::new(tag))
194194
}
195195

196196
/// Returns whether a tag matches any global or repo-specific exclude filter.
197197
fn is_excluded(&self, repo: &str, tag: &str) -> bool {
198-
self.global_exclude.is_match(tag)
198+
self.global_exclude.is_match(Path::new(tag))
199199
|| self
200200
.repo_exclude
201201
.get(repo)
202-
.is_some_and(|set| set.is_match(tag))
202+
.is_some_and(|set| set.is_match(Path::new(tag)))
203203
}
204204
}
205205

‎crates/prek/src/cli/run/filter.rs‎

Lines changed: 47 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -27,9 +27,6 @@ impl<'a> FilenameFilter<'a> {
2727
}
2828

2929
pub(crate) fn filter(&self, filename: &Path) -> bool {
30-
let Some(filename) = filename.to_str() else {
31-
return false;
32-
};
3330
if let Some(pattern) = &self.include {
3431
if !pattern.is_match(filename) {
3532
return false;
@@ -399,6 +396,10 @@ mod tests {
399396
FilePattern::Glob(GlobPatterns::new(vec![pattern.to_string()]).unwrap())
400397
}
401398

399+
fn regex_pattern(pattern: &str) -> FilePattern {
400+
FilePattern::regex(pattern).unwrap()
401+
}
402+
402403
#[test]
403404
fn filename_filter_supports_glob_include_and_exclude() {
404405
let include = glob_pattern("src/**/*.rs");
@@ -409,4 +410,47 @@ mod tests {
409410
assert!(!filter.filter(Path::new("src/lib/ignored.rs")));
410411
assert!(!filter.filter(Path::new("tests/main.rs")));
411412
}
413+
414+
#[cfg(unix)]
415+
#[test]
416+
fn filename_filter_allows_non_utf8_paths_without_patterns() {
417+
use std::ffi::OsStr;
418+
use std::os::unix::ffi::OsStrExt as _;
419+
420+
let path = Path::new(OsStr::from_bytes(b"bad-\xff.py"));
421+
let filter = FilenameFilter::new(None, None);
422+
423+
assert!(filter.filter(path));
424+
}
425+
426+
#[cfg(unix)]
427+
#[test]
428+
fn filename_filter_matches_non_utf8_paths_with_glob_patterns() {
429+
use std::ffi::OsStr;
430+
use std::os::unix::ffi::OsStrExt as _;
431+
432+
let include = glob_pattern("**/*.py");
433+
let exclude = glob_pattern("**/*.py");
434+
let path = Path::new(OsStr::from_bytes(b"bad-\xff.py"));
435+
let filter = FilenameFilter::new(Some(&include), None);
436+
437+
assert!(filter.filter(path));
438+
439+
let filter = FilenameFilter::new(None, Some(&exclude));
440+
441+
assert!(!filter.filter(path));
442+
}
443+
444+
#[cfg(unix)]
445+
#[test]
446+
fn filename_filter_skips_non_utf8_paths_with_regex_include() {
447+
use std::ffi::OsStr;
448+
use std::os::unix::ffi::OsStrExt as _;
449+
450+
let include = regex_pattern(r".*\.py$");
451+
let path = Path::new(OsStr::from_bytes(b"bad-\xff.py"));
452+
let filter = FilenameFilter::new(Some(&include), None);
453+
454+
assert!(!filter.filter(path));
455+
}
412456
}

‎crates/prek/src/config.rs‎

Lines changed: 30 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -40,8 +40,8 @@ impl GlobPatterns {
4040
self.patterns.is_empty()
4141
}
4242

43-
pub(crate) fn is_match(&self, value: &str) -> bool {
44-
self.set.is_match(Path::new(value))
43+
pub(crate) fn is_match(&self, path: &Path) -> bool {
44+
self.set.is_match(path)
4545
}
4646
}
4747

@@ -182,19 +182,22 @@ pub(crate) enum FilePattern {
182182
}
183183

184184
impl FilePattern {
185-
pub(crate) fn new_glob(patterns: Vec<String>) -> Result<Self, globset::Error> {
185+
pub(crate) fn glob(patterns: Vec<String>) -> Result<Self, globset::Error> {
186186
Ok(Self::Glob(GlobPatterns::new(patterns)?))
187187
}
188188

189-
pub(crate) fn new_regex(pattern: &str) -> Result<Self, fancy_regex::Error> {
189+
pub(crate) fn regex(pattern: &str) -> Result<Self, fancy_regex::Error> {
190190
Ok(Self::Regex(Regex::new(pattern)?))
191191
}
192192

193-
pub(crate) fn is_match(&self, str: &str) -> bool {
193+
pub(crate) fn is_match(&self, path: &Path) -> bool {
194194
match self {
195195
FilePattern::Never => false,
196-
FilePattern::Regex(regex) => regex.is_match(str).unwrap_or(false),
197-
FilePattern::Glob(globs) => globs.is_match(str),
196+
// Regex patterns are text matchers; globs can match OS paths directly.
197+
FilePattern::Regex(regex) => path
198+
.to_str()
199+
.is_some_and(|path| regex.is_match(path).unwrap_or(false)),
200+
FilePattern::Glob(globs) => globs.is_match(path),
198201
}
199202
}
200203
}
@@ -1460,9 +1463,9 @@ mod tests {
14601463
matches!(parsed.files, FilePattern::Regex(_)),
14611464
"expected regex pattern"
14621465
);
1463-
assert!(parsed.files.is_match("src/main.rs"));
1464-
assert!(!parsed.files.is_match("other/main.rs"));
1465-
assert!(parsed.exclude.is_match("target/debug/app"));
1466+
assert!(parsed.files.is_match(Path::new("src/main.rs")));
1467+
assert!(!parsed.files.is_match(Path::new("other/main.rs")));
1468+
assert!(parsed.exclude.is_match(Path::new("target/debug/app")));
14661469

14671470
let glob_yaml = indoc::indoc! {r"
14681471
files:
@@ -1476,10 +1479,10 @@ mod tests {
14761479
matches!(parsed.files, FilePattern::Glob(_)),
14771480
"expected glob pattern"
14781481
);
1479-
assert!(parsed.files.is_match("src/lib/main.rs"));
1480-
assert!(!parsed.files.is_match("src/lib/main.py"));
1481-
assert!(parsed.exclude.is_match("target/debug/app"));
1482-
assert!(!parsed.exclude.is_match("src/lib/main.rs"));
1482+
assert!(parsed.files.is_match(Path::new("src/lib/main.rs")));
1483+
assert!(!parsed.files.is_match(Path::new("src/lib/main.py")));
1484+
assert!(parsed.exclude.is_match(Path::new("target/debug/app")));
1485+
assert!(!parsed.exclude.is_match(Path::new("src/lib/main.rs")));
14831486

14841487
let glob_list_yaml = indoc::indoc! {r"
14851488
files:
@@ -1493,11 +1496,11 @@ mod tests {
14931496
"};
14941497
let parsed: Wrapper =
14951498
serde_saphyr::from_str(glob_list_yaml).expect("glob list patterns should parse");
1496-
assert!(parsed.files.is_match("src/lib/main.rs"));
1497-
assert!(parsed.files.is_match("crates/foo/src/lib.rs"));
1498-
assert!(!parsed.files.is_match("tests/main.rs"));
1499-
assert!(parsed.exclude.is_match("target/debug/app"));
1500-
assert!(parsed.exclude.is_match("dist/app"));
1499+
assert!(parsed.files.is_match(Path::new("src/lib/main.rs")));
1500+
assert!(parsed.files.is_match(Path::new("crates/foo/src/lib.rs")));
1501+
assert!(!parsed.files.is_match(Path::new("tests/main.rs")));
1502+
assert!(parsed.exclude.is_match(Path::new("target/debug/app")));
1503+
assert!(parsed.exclude.is_match(Path::new("dist/app")));
15011504
}
15021505

15031506
#[test]
@@ -1512,24 +1515,24 @@ mod tests {
15121515
pattern.to_string(),
15131516
"glob: [src/**/*.rs, crates/**/src/**/*.rs]"
15141517
);
1515-
assert!(pattern.is_match("src/main.rs"));
1516-
assert!(pattern.is_match("crates/foo/src/lib.rs"));
1517-
assert!(!pattern.is_match("tests/main.rs"));
1518+
assert!(pattern.is_match(Path::new("src/main.rs")));
1519+
assert!(pattern.is_match(Path::new("crates/foo/src/lib.rs")));
1520+
assert!(!pattern.is_match(Path::new("tests/main.rs")));
15181521
}
15191522

15201523
#[test]
15211524
fn file_pattern_never_matches() {
15221525
let pattern = FilePattern::Never;
1523-
assert!(!pattern.is_match(""));
1524-
assert!(!pattern.is_match("foo.txt"));
1525-
assert!(!pattern.is_match("nested/path.rs"));
1526+
assert!(!pattern.is_match(Path::new("")));
1527+
assert!(!pattern.is_match(Path::new("foo.txt")));
1528+
assert!(!pattern.is_match(Path::new("nested/path.rs")));
15261529
}
15271530

15281531
#[test]
15291532
fn empty_glob_list_matches_nothing() {
15301533
let pattern = serde_saphyr::from_str::<FilePattern>("glob: []").unwrap();
1531-
assert!(!pattern.is_match("any/file.rs"));
1532-
assert!(!pattern.is_match(""));
1534+
assert!(!pattern.is_match(Path::new("any/file.rs")));
1535+
assert!(!pattern.is_match(Path::new("")));
15331536
}
15341537

15351538
#[test]

‎crates/prek/src/git.rs‎

Lines changed: 29 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -86,10 +86,24 @@ pub(crate) fn git_cmd(summary: &str) -> Result<Cmd, Error> {
8686
fn zsplit(s: &[u8]) -> Result<Vec<PathBuf>, Utf8Error> {
8787
s.split(|&b| b == b'\0')
8888
.filter(|slice| !slice.is_empty())
89-
.map(|slice| str::from_utf8(slice).map(PathBuf::from))
89+
.map(path_from_git_bytes)
9090
.collect()
9191
}
9292

93+
#[cfg(unix)]
94+
#[expect(clippy::unnecessary_wraps)]
95+
fn path_from_git_bytes(bytes: &[u8]) -> Result<PathBuf, Utf8Error> {
96+
use std::ffi::OsStr;
97+
use std::os::unix::ffi::OsStrExt as _;
98+
99+
Ok(PathBuf::from(OsStr::from_bytes(bytes)))
100+
}
101+
102+
#[cfg(not(unix))]
103+
fn path_from_git_bytes(bytes: &[u8]) -> Result<PathBuf, Utf8Error> {
104+
str::from_utf8(bytes).map(PathBuf::from)
105+
}
106+
93107
pub(crate) async fn intent_to_add_files(root: &Path) -> Result<Vec<PathBuf>, Error> {
94108
let output = git_cmd("get intent to add files")?
95109
.arg("diff")
@@ -859,6 +873,20 @@ pub(crate) fn list_submodules(git_root: &Path) -> Result<Vec<PathBuf>, Error> {
859873
#[cfg(test)]
860874
mod tests {
861875
use super::shared_repository_file_mode;
876+
#[cfg(unix)]
877+
use super::zsplit;
878+
879+
#[cfg(unix)]
880+
#[test]
881+
fn zsplit_preserves_non_utf8_paths() {
882+
use std::os::unix::ffi::OsStrExt as _;
883+
884+
let paths = zsplit(b"normal.py\0bad-\xff.py\0").unwrap();
885+
886+
assert_eq!(paths.len(), 2);
887+
assert_eq!(paths[0].as_os_str().as_bytes(), b"normal.py");
888+
assert_eq!(paths[1].as_os_str().as_bytes(), b"bad-\xff.py");
889+
}
862890

863891
#[test]
864892
fn shared_repository_group_mode_matches_git_behavior() {

‎crates/prek/src/hooks/builtin_hooks/mod.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -164,7 +164,7 @@ impl BuiltinHook {
164164
"checks for filenames which cannot be created on Windows.".to_string(),
165165
),
166166
files: Some(
167-
FilePattern::new_regex(
167+
FilePattern::regex(
168168
check_illegal_windows_names::ILLEGAL_WINDOWS_PATTERN,
169169
)
170170
.expect("builtin files regex must be valid"),

‎crates/prek/src/hooks/meta_hooks.rs‎

Lines changed: 10 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -53,8 +53,7 @@ impl MetaHook {
5353
pub(crate) fn from_id(id: &str) -> Result<Self, ()> {
5454
let hook_id = MetaHooks::from_str(id).map_err(|_| ())?;
5555
let config_file_glob =
56-
FilePattern::new_glob(CONFIG_FILENAMES.iter().map(ToString::to_string).collect())
57-
.unwrap();
56+
FilePattern::glob(CONFIG_FILENAMES.iter().map(ToString::to_string).collect()).unwrap();
5857

5958
Ok(match hook_id {
6059
MetaHooks::CheckHooksApply => MetaHook {
@@ -146,17 +145,13 @@ fn excludes_any(
146145
}
147146

148147
files.iter().any(|f| {
149-
let Some(f) = f.as_ref().to_str() else {
150-
return false; // Skip files that cannot be converted to a string
151-
};
152-
153148
if let Some(pattern) = &include {
154-
if !pattern.is_match(f) {
149+
if !pattern.is_match(f.as_ref()) {
155150
return false;
156151
}
157152
}
158153
if let Some(pattern) = &exclude {
159-
if !pattern.is_match(f) {
154+
if !pattern.is_match(f.as_ref()) {
160155
return false;
161156
}
162157
}
@@ -271,7 +266,7 @@ mod tests {
271266
use prek_consts::{PRE_COMMIT_CONFIG_YAML, PRE_COMMIT_CONFIG_YML, PREK_TOML};
272267

273268
fn regex_pattern(pattern: &str) -> FilePattern {
274-
FilePattern::new_regex(pattern).unwrap()
269+
FilePattern::regex(pattern).unwrap()
275270
}
276271

277272
#[test]
@@ -299,15 +294,15 @@ mod tests {
299294
fn meta_hook_patterns_cover_config_files() {
300295
let apply = MetaHook::from_id("check-hooks-apply").expect("known meta hook");
301296
let apply_files = apply.options.files.as_ref().expect("files should be set");
302-
assert!(apply_files.is_match(PRE_COMMIT_CONFIG_YAML));
303-
assert!(apply_files.is_match(PRE_COMMIT_CONFIG_YML));
304-
assert!(apply_files.is_match(PREK_TOML));
297+
assert!(apply_files.is_match(Path::new(PRE_COMMIT_CONFIG_YAML)));
298+
assert!(apply_files.is_match(Path::new(PRE_COMMIT_CONFIG_YML)));
299+
assert!(apply_files.is_match(Path::new(PREK_TOML)));
305300

306301
let useless = MetaHook::from_id("check-useless-excludes").expect("known meta hook");
307302
let useless_files = useless.options.files.as_ref().expect("files should be set");
308-
assert!(useless_files.is_match(PRE_COMMIT_CONFIG_YAML));
309-
assert!(useless_files.is_match(PRE_COMMIT_CONFIG_YML));
310-
assert!(useless_files.is_match(PREK_TOML));
303+
assert!(useless_files.is_match(Path::new(PRE_COMMIT_CONFIG_YAML)));
304+
assert!(useless_files.is_match(Path::new(PRE_COMMIT_CONFIG_YML)));
305+
assert!(useless_files.is_match(Path::new(PREK_TOML)));
311306

312307
let identity = MetaHook::from_id("identity").expect("known meta hook");
313308
assert!(identity.options.files.is_none());

0 commit comments

Comments
 (0)