style: format Rust workspace

crates/xserv-tokenizer/src/bpe.rs

@@ -95,11 +95,15 @@ impl Tokenizer {
             let (a_str, b_str) = match entry {
                 MergeEntry::Str(s) => {
                     let parts: Vec<&str> = s.splitn(2, ' ').collect();
-                    if parts.len() != 2 { continue; }
+                    if parts.len() != 2 {
+                        continue;
+                    }
                     (parts[0].to_string(), parts[1].to_string())
                 }
                 MergeEntry::Pair(v) => {
-                    if v.len() != 2 { continue; }
+                    if v.len() != 2 {
+                        continue;
+                    }
                     (v[0].clone(), v[1].clone())
                 }
             };
@@ -174,7 +178,10 @@ impl Tokenizer {
                     if byte_fallback {
                         Regex::new(r"[\p{L}\p{N}]+|[^\s\p{L}\p{N}]|\s+").unwrap()
                     } else {
-                        Regex::new(r"'s|'t|'re|'ve|'m|'ll|'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+").unwrap()
+                        Regex::new(
+                            r"'s|'t|'re|'ve|'m|'ll|'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+",
+                        )
+                        .unwrap()
                     }
                 }
             }
@@ -262,7 +269,9 @@ impl Tokenizer {
 
             // BPE merges
             loop {
-                if token_ids.len() < 2 { break; }
+                if token_ids.len() < 2 {
+                    break;
+                }
                 let mut best_rank = usize::MAX;
                 let mut best_idx = 0;
                 for i in 0..token_ids.len() - 1 {
@@ -273,12 +282,15 @@ impl Tokenizer {
                         }
                     }
                 }
-                if best_rank == usize::MAX { break; }
+                if best_rank == usize::MAX {
+                    break;
+                }
 
                 let merged_bytes = [
                     self.decoder[token_ids[best_idx] as usize].as_slice(),
                     self.decoder[token_ids[best_idx + 1] as usize].as_slice(),
-                ].concat();
+                ]
+                .concat();
                 let merged_id = *self.encoder.get(&merged_bytes).unwrap_or_else(|| {
                     panic!("merged token not in vocab");
                 });
@@ -389,14 +401,13 @@ fn unicode_to_byte(c: char) -> u8 {
         m
     });
 
-    *map.get(&(c as u32)).unwrap_or_else(|| {
-        panic!("unmapped unicode char U+{:04X} in tokenizer", c as u32)
-    })
+    *map.get(&(c as u32))
+        .unwrap_or_else(|| panic!("unmapped unicode char U+{:04X} in tokenizer", c as u32))
 }
 
 #[cfg(test)]
 mod tests {
-    use super::{take_valid_utf8, Tokenizer};
+    use super::{Tokenizer, take_valid_utf8};
 
     #[test]
     fn qwen_added_tokens_are_indivisible_and_im_end_is_eos() {