this looks better but damn how can it be so bloated

Author: ArthurZucker

tokenizers/src/tokenizer/mod.rs

@@ -856,10 +856,25 @@ where
                         if skip_special_tokens && self.added_vocabulary.is_special_token(&token) {
                             None
                         } else if let Some(pre_tok) = &self.pre_tokenizer {
-                            let mut string = PreTokenizedString::from(token);
-                            pre_tok.pre_tokenize(&mut string);
-                            println!("Pre-tok String: {}", string.original);
-                            Some(string.original)
+                            let mut string = PreTokenizedString::from(token.clone());
+                            let _ = pre_tok.pre_tokenize(&mut string);
+                            println!(
+                                "Pre-tok String: {} vs token {} vs pret {:?}",
+                                string.original,
+                                token,
+                                string
+                                    .get_splits(OffsetReferential::Normalized, OffsetType::Byte)
+                                    .get(0)
+                                    .unwrap()
+                            );
+                            Some(
+                                string
+                                    .get_splits(OffsetReferential::Normalized, OffsetType::Byte)
+                                    .get(0)
+                                    .unwrap()
+                                    .0
+                                    .to_string(),
+                            )
                         } else {
                             println!("String: {}", token);
                             Some(token)
@@ -1334,6 +1349,8 @@ mod test {
 
         tokenizer.add_tokens(&[AddedToken::from("Bác", false)]);
         let encoded = tokenizer.encode("Hey Bác how are you?", false).unwrap();
+        println!("{:?}", encoded.get_tokens());
+        println!("{:?}", encoded.get_ids());
         println!("{}", tokenizer.decode(encoded.get_ids(), false).unwrap());
     }
 }