mirror of
https://github.com/storytold/filmcraft.git
synced 2026-11-19 08:53:21 +01:00
* Fix clippy expect_used under --all-features and the no-threads build (#2813) * xtask: root() falls back to the manifest dir instead of expect, so the whole-workspace expect_used gate passes (#2813) --------- Co-authored-by: skylar taylor-barrick <hi@skylar.cc>
This commit is contained in:
1 parent
1b0f9f8e7b
commit
b6a37d19db
2 files changed
+16
-2
No files matched your search
@@ -42,7 +42,8 @@ fn unicode_to_byte() -> HashMap<char, u8> {
|
||||
n += 1;
|
||||
}
|
||||
}
|
||||
bs.iter().zip(&cs).map(|(&b, &c)| (char::from_u32(c).expect("valid"), b as u8)).collect()
|
||||
// Every code point here is below 0x200, so `from_u32` always succeeds.
|
||||
bs.iter().zip(&cs).filter_map(|(&b, &c)| char::from_u32(c).map(|ch| (ch, b as u8))).collect()
|
||||
}
|
||||
|
||||
impl Tokenizer {
|
||||
@@ -176,6 +177,17 @@ pub fn group_words(tok: &Tokenizer, ids: &[u32]) -> Vec<(String, std::ops::Range
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn byte_table_covers_every_byte_once() {
|
||||
let t = unicode_to_byte();
|
||||
assert_eq!(t.len(), 256);
|
||||
let mut seen: Vec<u8> = t.values().copied().collect();
|
||||
seen.sort_unstable();
|
||||
assert!(seen.iter().copied().eq(0..=255u8));
|
||||
assert_eq!(t.get(&'!'), Some(&b'!'));
|
||||
assert_eq!(t.get(&'\u{120}'), Some(&b' '));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn sparse_huge_vocab_id_is_rejected_before_allocation() {
|
||||
let json = format!(r#"{{"model":{{"vocab":{{"x":{}}}}},"added_tokens":[]}}"#, MAX_VOCAB_ID + 1);
|
||||
|
||||
+3
-1
@@ -552,7 +552,9 @@ fn bench() -> Result<(), String> {
|
||||
|
||||
/// The workspace root (the parent of `xtask/`).
|
||||
fn root() -> std::path::PathBuf {
|
||||
std::path::Path::new(env!("CARGO_MANIFEST_DIR")).parent().expect("xtask lives in the workspace").to_path_buf()
|
||||
let manifest_dir = std::path::Path::new(env!("CARGO_MANIFEST_DIR"));
|
||||
// `xtask/` always has a parent; if it somehow did not, fall back to the manifest dir itself.
|
||||
manifest_dir.parent().unwrap_or(manifest_dir).to_path_buf()
|
||||
}
|
||||
|
||||
fn main() -> ExitCode {
|
||||
|
||||
Reference in new issue
Block a user