textkit: token counting and the two segmenters

Cargo.toml

@@ -11,6 +11,7 @@ repository = "https://git.pumpin.online/funkpower/catena"
 
 [workspace.dependencies]
 corpus-core = { path = "crates/corpus-core", version = "0.1.0" }
+textkit = { path = "crates/textkit", version = "0.1.0" }
 
 anyhow = "1"
 clap = { version = "4", features = ["derive"] }
@@ -18,6 +19,7 @@ schemars = "1"
 serde = { version = "1", features = ["derive"] }
 serde_json = "1"
 thiserror = "2"
+tiktoken-rs = "0.12"
 
 [profile.release]
 strip = true