Feature/analysis

Cargo.toml

@@ -1,11 +1,64 @@
+[package]
+name = "divvunspell"
+description = "Spell checking library for ZHFST/BHFST spellers, with case handling and tokenization support."
+version = "1.0.0-beta.5"
+authors = ["Brendan Molloy <brendan@bbqsrc.net>"]
+edition = "2021"
+license = "MIT OR Apache-2.0"
+repository = "https://github.com/divvun/divvunspell"
+
+[lib]
+name = "divvunspell"
+crate-type = ["rlib", "staticlib", "cdylib"]
+
+[dependencies]
+libc = "0.2"
+memmap2 = "0.9.4"
+byteorder = "1.3.4"
+serde = { version = "1.0.116", features = ["derive"] }
+serde_json = "1.0.57"
+serde-xml-rs = { version = "0.6.0", default-features = false }
+zip = { version = "0.5", default-features = false }
+unic-segment = "0.9.0"
+unic-char-range = "0.9.0"
+unic-char-property = "0.9.0"
+unic-ucd-category = "0.9.0"
+unic-emoji-char = "0.9.0"
+parking_lot = "0.11.2"
+hashbrown = { version = "0.11", features = ["serde"] }
+lifeguard = "0.6.1"
+smol_str = { version = "0.2.1", features = ["serde"] }
+box-format = { version = "0.3.2", features = ["reader"], default-features = false }
+itertools = "0.12.1"
+strsim = "0.11.0"
+log = "0.4.11"
+cffi = { git = "https://github.com/cffi-rs/cffi", optional = true }
+unic-ucd-common = "0.9.0"
+flatbuffers = { version = "0.6.1", optional = true }
+env_logger = { version = "0.11.2", optional = true }
+thiserror = "1.0.20"
+tempfile = "3.3.0"
+fs_extra = "1.2.0"
+eieio = "1.0.0"
+pathos = "0.3.0"
+language-tags = "0.3.2"
+globwalk = "0.9.1"
+
+[features]
+compression = ["zip/deflate"]
+logging = ["env_logger"]
+cargo-clippy = []
+
+# Internal features: unstable, not for external use!
+internal_convert = []
+internal_ffi = ["flatbuffers", "logging", "cffi"]
+
 [workspace]
 resolver = "2"
 members = [
-  "divvunspell",
-  "accuracy",
-  "divvunspell-bin",
-  "thfst-tools",
-  "regtest",
+  ".",
+  "cli",
+  "crates/*"
 ]
 
 [profile.dev]

LICENSE-MIT

@@ -1,6 +1,6 @@
-Copyright (c) 2017-2021  Brendan Molloy <brendan@bbqsrc.net>
-Copyright (c) 2018-2021  UiT The Arctic University of Norway
-Copyright (c) 2018-2021  Sámediggi 
+Copyright (c) 2017-2025  Brendan Molloy <brendan@bbqsrc.net>
+Copyright (c) 2018-2025  UiT The Arctic University of Norway
+Copyright (c) 2018-2025  Sámediggi 
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

README.md

@@ -17,17 +17,6 @@ cargo install thfst-tools
 cargo install --path .
 ```
 
-### Building with `gpt2` support on macOS aarch64
-
-(Skip this if you are not experimenting with gpt2 support. So skip. Now.)
-
-Clone this repo then:
-
-```bash
-brew install libtorch
-LIBTORCH=/opt/homebrew/opt/libtorch cargo build --features gpt2 --bin divvunspell
-```
-
 ### No Rust?
 
 ```sh
@@ -49,7 +38,6 @@ Optional arguments:
 Available subcommands:
   suggest   get suggestions for provided input
   tokenize  print input in word-separated tokenized form
-  predict   predict next words using GPT2 model
 
 $ divvunspell suggest -h
 Usage: divvunspell suggest [OPTIONS]

divvunspell-bin/Cargo.toml → cli/Cargo.toml

@@ -1,5 +1,5 @@
 [package]
-name = "divvunspell-bin"
+name = "divvunspell-cli"
 description = "Spellchecker for ZHFST/BHFST spellers, with case handling and tokenization support."
 version = "1.0.0"
 authors = ["Brendan Molloy <brendan@bbqsrc.net>"]
@@ -14,13 +14,9 @@ path = "src/main.rs"
 [dependencies]
 serde = { version = "1.0.116", features = ["derive"] }
 serde_json = "1.0.57"
-divvunspell = { version = "1.0.0-beta.5", features = ["internal_convert", "compression"], path = "../divvunspell" }
+divvunspell = { features = ["internal_convert", "compression"], path = ".." }
 box-format = { version = "0.3.2", features = ["reader"], default-features = false }
-pretty_env_logger = "0.4.0"
+pretty_env_logger = "0.5.0"
 gumdrop = "0.8.0"
 anyhow = "1.0.32"
 structopt = "0.3.17"
-
-[features]
-default = []
-gpt2 = ["divvunspell/gpt2"]