Add LRU example (#225)

phenylshima · web-flow · commit bbe827e5f792 · 2024-02-07T17:59:48.000+09:00
* add lru example

* fix text is not cleared

* refactor

* fmt

* fix for jlabel
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/examples/README.md b/examples/README.md
@@ -7,3 +7,9 @@ Simple example of binary executable using jpreprocess.
 ## example-wasm
 
 Example of using jpreprocess from WebAssembly.
+
+## example-lru
+
+Example of reducing memory usage by fetching from the file (with LRU cache) in jpreprocess.
+
+Also includes the example of `DictionaryFetcher`.
diff --git a/examples/example-lru/Cargo.toml b/examples/example-lru/Cargo.toml
@@ -0,0 +1,19 @@
+[package]
+name = "example-lru"
+edition = "2021"
+
+version.workspace = true
+description.workspace = true
+license.workspace = true
+repository.workspace = true
+keywords.workspace = true
+categories.workspace = true
+rust-version.workspace = true
+
+[dependencies]
+jpreprocess.workspace = true
+jpreprocess-core.workspace = true
+jpreprocess-dictionary.workspace = true
+lindera-tokenizer = { version = "0.27" }
+
+lru = "0.12.2"
diff --git a/examples/example-lru/README.md b/examples/example-lru/README.md
@@ -0,0 +1,5 @@
+# example-lru
+
+Example of reducing memory usage by fetching from the file (with LRU cache) in jpreprocess.
+
+Also includes the example of `DictionaryFetcher`.
diff --git a/examples/example-lru/src/main.rs b/examples/example-lru/src/main.rs
@@ -0,0 +1,36 @@
+#[cfg(not(target_family = "wasm"))]
+mod storage_fetcher;
+
+#[cfg(not(target_family = "wasm"))]
+fn main() -> Result<(), Box<dyn std::error::Error>> {
+    use jpreprocess::*;
+    use std::path::PathBuf;
+
+    use crate::storage_fetcher::StorageFetcher;
+
+    let path = match std::env::args().nth(1).map(PathBuf::from) {
+        Some(s) if s.is_dir() => s,
+        _ => {
+            eprintln!("Please specify a valid path to dictionary");
+            std::process::exit(-1);
+        }
+    };
+
+    let fetcher = StorageFetcher::new(&path)?;
+    let dictionary = SystemDictionaryConfig::File(path).load()?;
+
+    let jpreprocess = JPreprocess::with_dictionary_fetcher(fetcher, dictionary, None);
+
+    let mut text = String::new();
+    while std::io::stdin().read_line(&mut text).is_ok() {
+        let jpcommon_label = jpreprocess.extract_fullcontext(&text)?;
+        let string_labels: Vec<_> = jpcommon_label.iter().map(ToString::to_string).collect();
+        println!("{}", string_labels.join("\n"));
+        text.clear();
+    }
+
+    Ok(())
+}
+
+#[cfg(target_family = "wasm")]
+fn main() {}
diff --git a/examples/example-lru/src/storage_fetcher.rs b/examples/example-lru/src/storage_fetcher.rs
@@ -0,0 +1,116 @@
+use std::{
+    fs::File,
+    io::{Read, Seek},
+    num::NonZeroUsize,
+    path::Path,
+    sync::Mutex,
+};
+
+use jpreprocess_core::{error::JPreprocessErrorKind, word_entry::WordEntry, JPreprocessResult};
+use jpreprocess_dictionary::{default::WordDictionaryMode, DictionaryFetcher};
+use lindera_tokenizer::token::Token;
+use lru::LruCache;
+
+pub struct StorageFetcher {
+    inner: Mutex<CachedStorage>,
+}
+
+impl StorageFetcher {
+    pub fn new<P: AsRef<Path>>(dir: P) -> Result<Self, std::io::Error> {
+        Ok(Self {
+            inner: Mutex::new(CachedStorage::new(dir)?),
+        })
+    }
+}
+
+impl DictionaryFetcher for StorageFetcher {
+    fn get_word(&self, token: &Token) -> JPreprocessResult<WordEntry> {
+        if token.word_id.is_unknown() {
+            return Ok(WordEntry::default());
+        }
+
+        let mut g = self.inner.lock().unwrap();
+        g.get_word(token.word_id.0)
+    }
+}
+
+struct CachedStorage {
+    mode: WordDictionaryMode,
+    index_file: File,
+    words_file: File,
+    cache: LruCache<u32, WordEntry>,
+}
+
+impl CachedStorage {
+    pub fn new<P: AsRef<Path>>(dir: P) -> Result<Self, std::io::Error> {
+        let mut index = File::open(dir.as_ref().join("dict.wordsidx"))?;
+        let mut words = File::open(dir.as_ref().join("dict.words"))?;
+
+        let mut index_buf = vec![0u8; 4];
+        index.read_exact(&mut index_buf)?;
+        let start = u32::from_be_bytes([index_buf[0], index_buf[1], index_buf[2], index_buf[3]]);
+
+        let mut identifier_buf = vec![0u8; start as usize];
+        words.read_exact(&mut identifier_buf)?;
+
+        let mode = WordDictionaryMode::from_metadata(String::from_utf8(identifier_buf).ok());
+
+        Ok(Self {
+            mode,
+            index_file: index,
+            words_file: words,
+            cache: LruCache::new(NonZeroUsize::new(1000).unwrap()),
+        })
+    }
+
+    pub fn get_word(&mut self, index: u32) -> JPreprocessResult<WordEntry> {
+        if let Some(word) = self.cache.get(&index) {
+            println!("Word #{} found in cache", index);
+            return Ok(word.clone());
+        }
+        println!("Word #{} not found in cache", index);
+
+        let bytes = self
+            .get_bytes(index)
+            .map_err(|err| JPreprocessErrorKind::Io.with_error(err))?;
+        let entry = self.mode.into_serializer().deserialize(&bytes)?;
+        self.cache.push(index, entry.clone());
+
+        Ok(entry)
+    }
+
+    fn get_bytes(&mut self, index: u32) -> Result<Vec<u8>, std::io::Error> {
+        let (start, end) = self.read_u32_range(index)?;
+
+        self.words_file
+            .seek(std::io::SeekFrom::Start(start as u64))?;
+        if let Some(end) = end {
+            let mut word_buf = vec![0u8; (end - start) as usize];
+            self.words_file.read_exact(&mut word_buf)?;
+            Ok(word_buf)
+        } else {
+            let mut word_buf = Vec::new();
+            self.words_file.read_to_end(&mut word_buf)?;
+            Ok(word_buf)
+        }
+    }
+    fn read_u32_range(&mut self, index: u32) -> Result<(u32, Option<u32>), std::io::Error> {
+        self.index_file
+            .seek(std::io::SeekFrom::Start((index as u64) * 4))?;
+        let mut index_buf = vec![0u8; 8];
+        if self.index_file.read_exact(&mut index_buf).is_ok() {
+            let start =
+                u32::from_le_bytes([index_buf[0], index_buf[1], index_buf[2], index_buf[3]]);
+            let end = u32::from_le_bytes([index_buf[4], index_buf[5], index_buf[6], index_buf[7]]);
+            return Ok((start, Some(end)));
+        }
+
+        self.index_file
+            .seek(std::io::SeekFrom::Start((index as u64) * 4))?;
+        let mut index_buf = vec![0u8; 4];
+        self.index_file.read_exact(&mut index_buf)?;
+
+        let start = u32::from_le_bytes([index_buf[0], index_buf[1], index_buf[2], index_buf[3]]);
+        Ok((start, None))
+    }
+}
diff --git a/examples/example-simple/src/main.rs b/examples/example-simple/src/main.rs
@@ -19,7 +19,8 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
     while std::io::stdin().read_line(&mut text).is_ok() {
         let jpcommon_label = jpreprocess.extract_fullcontext(&text)?;
         let string_labels: Vec<_> = jpcommon_label.iter().map(ToString::to_string).collect();
-        println!("{}", string_labels.join("\n"))
+        println!("{}", string_labels.join("\n"));
+        text.clear();
     }
 
     Ok(())

Original file line number	Diff line number	Diff line change
`@@ -19,7 +19,8 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {`
`19`	`19`	`while std::io::stdin().read_line(&mut text).is_ok() {`
`20`	`20`	`let jpcommon_label = jpreprocess.extract_fullcontext(&text)?;`
`21`	`21`	`let string_labels: Vec<_> = jpcommon_label.iter().map(ToString::to_string).collect();`
`22`		`- println!("{}", string_labels.join("\n"))`
	`22`	`+ println!("{}", string_labels.join("\n"));`
	`23`	`+ text.clear();`
`23`	`24`	`}`
`24`	`25`
`25`	`26`	`Ok(())`