[ { "path": ".gitignore", "content": "/target\n" }, { "path": "CONTRIBUTING.md", "content": "I have very limited resources in terms of handling feedback on my projects, sorry. So here are the limitations to keep in mind:\n\n- I don't look into reported Issues.\n- I only look into small PRs that suggest\n - bug fixes,\n - documentation fixes.\n- I do not look into PRs that\n - implement new features,\n - refactor/cleanup the code.\n- What qualifies as a bug, a feature, or refactoring is entirely upon my interpretation.\n\nSorry for any inconveniences. If you want to stir the project in a particular direction in terms of features feel free to fork it, I don't mind. Just make sure you have fun while developing it! This is like the whole point!\n" }, { "path": "Cargo.toml", "content": "[package]\nname = \"seroost\"\nversion = \"0.1.0\"\nedition = \"2021\"\n\n# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html\n\n[dependencies]\npoppler-rs = \"0.21.0\"\nserde = { version = \"1.0.152\", features = [\"derive\"] }\nserde_json = \"1.0.91\"\ntiny_http = \"0.12.0\"\nxml-rs = \"0.8.4\"\n" }, { "path": "LICENSE", "content": "Copyright 2023 Alexey Kutepov \n\nPermission is hereby granted, free of charge, to any person obtaining\na copy of this software and associated documentation files (the\n\"Software\"), to deal in the Software without restriction, including\nwithout limitation the rights to use, copy, modify, merge, publish,\ndistribute, sublicense, and/or sell copies of the Software, and to\npermit persons to whom the Software is furnished to do so, subject to\nthe following conditions:\n\nThe above copyright notice and this permission notice shall be\nincluded in all copies or substantial portions of the Software.\n\nTHE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND,\nEXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF\nMERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND\nNONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE\nLIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION\nOF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION\nWITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE." }, { "path": "README.md", "content": "# Local Search Engine in Rust\n\n**THIS SOFTWARE IS UNFINISHED!!! Don't have any high expectations.**\n\n## Quick Start\n\n```console\n$ cargo run serve ./folder/\n$ iexplore.exe http://localhost:6969/\n```\n" }, { "path": "src/index.html", "content": "\n \n Seroost\n \n \n

Provide Your Query:

\n \n

\n \n \n\n" }, { "path": "src/index.js", "content": "// TODO: live update results as you type\nasync function search(prompt) {\n const results = document.getElementById(\"results\")\n results.innerHTML = \"\";\n const response = await fetch(\"/api/search\", {\n method: 'POST',\n headers: {'Content-Type': 'text/plain'},\n body: prompt,\n });\n const json = await response.json();\n results.innerHTML = \"\";\n for ([path, rank] of json) {\n let item = document.createElement(\"span\");\n item.appendChild(document.createTextNode(path));\n item.appendChild(document.createElement(\"br\"));\n results.appendChild(item);\n }\n}\n\nlet query = document.getElementById(\"query\");\nlet currentSearch = Promise.resolve()\n\nquery.addEventListener(\"keypress\", (e) => {\n if (e.key == \"Enter\") {\n currentSearch.then(() => search(query.value));\n }\n})\n" }, { "path": "src/lexer.rs", "content": "pub struct Lexer<'a> {\n content: &'a [char],\n}\n\nimpl<'a> Lexer<'a> {\n pub fn new(content: &'a [char]) -> Self {\n Self { content }\n }\n\n fn trim_left(&mut self) {\n while !self.content.is_empty() && self.content[0].is_whitespace() {\n self.content = &self.content[1..];\n }\n }\n\n fn chop(&mut self, n: usize) -> &'a [char] {\n let token = &self.content[0..n];\n self.content = &self.content[n..];\n token\n }\n\n fn chop_while

(&mut self, mut predicate: P) -> &'a [char] where P: FnMut(&char) -> bool {\n let mut n = 0;\n while n < self.content.len() && predicate(&self.content[n]) {\n n += 1;\n }\n self.chop(n)\n }\n\n pub fn next_token(&mut self) -> Option {\n self.trim_left();\n if self.content.is_empty() {\n return None\n }\n\n if self.content[0].is_numeric() {\n return Some(self.chop_while(|x| x.is_numeric()).iter().collect());\n }\n\n if self.content[0].is_alphabetic() {\n let term = self.chop_while(|x| x.is_alphanumeric()).iter().map(|x| x.to_ascii_lowercase()).collect::();\n let mut env = crate::snowball::SnowballEnv::create(&term);\n crate::snowball::algorithms::english_stemmer::stem(&mut env);\n let stemmed_term = env.get_current().to_string();\n return Some(stemmed_term);\n }\n\n return Some(self.chop(1).iter().collect());\n }\n}\n\nimpl<'a> Iterator for Lexer<'a> {\n type Item = String;\n\n fn next(&mut self) -> Option {\n self.next_token()\n }\n}\n" }, { "path": "src/main.rs", "content": "use std::fs::{self, File};\nuse std::path::{Path};\nuse xml::reader::{XmlEvent, EventReader};\nuse xml::common::{Position, TextPosition};\nuse std::env;\nuse std::result::Result;\nuse std::process::ExitCode;\nuse std::str;\nuse std::io::{BufReader, BufWriter};\nuse std::sync::{Arc, Mutex};\nuse std::thread;\n\nmod model;\nuse model::*;\nmod server;\nmod lexer;\npub mod snowball;\n\nfn parse_entire_txt_file(file_path: &Path) -> Result {\n fs::read_to_string(file_path).map_err(|err| {\n eprintln!(\"ERROR: coult not open file {file_path}: {err}\", file_path = file_path.display());\n })\n}\n\nfn parse_entire_pdf_file(file_path: &Path) -> Result {\n use poppler::Document;\n use std::io::Read;\n\n let mut content = Vec::new();\n File::open(file_path)\n .and_then(|mut file| file.read_to_end(&mut content))\n .map_err(|err| {\n eprintln!(\"ERROR: could not read file {file_path}: {err}\", file_path = file_path.display());\n })?;\n\n let pdf = Document::from_data(&content, None).map_err(|err| {\n eprintln!(\"ERROR: could not read file {file_path}: {err}\",\n file_path = file_path.display());\n })?;\n\n let mut result = String::new();\n\n let n = pdf.n_pages();\n for i in 0..n {\n let page = pdf.page(i).expect(&format!(\"{i} is within the bounds of the range of the page\"));\n if let Some(content) = page.text() {\n result.push_str(content.as_str());\n result.push(' ');\n }\n }\n\n Ok(result)\n}\n\nfn parse_entire_xml_file(file_path: &Path) -> Result {\n let file = File::open(file_path).map_err(|err| {\n eprintln!(\"ERROR: could not open file {file_path}: {err}\", file_path = file_path.display());\n })?;\n let er = EventReader::new(BufReader::new(file));\n let mut content = String::new();\n for event in er.into_iter() {\n let event = event.map_err(|err| {\n let TextPosition {row, column} = err.position();\n let msg = err.msg();\n eprintln!(\"{file_path}:{row}:{column}: ERROR: {msg}\", file_path = file_path.display());\n })?;\n\n if let XmlEvent::Characters(text) = event {\n content.push_str(&text);\n content.push(' ');\n }\n }\n Ok(content)\n}\n\nfn parse_entire_file_by_extension(file_path: &Path) -> Result {\n let extension = file_path.extension().ok_or_else(|| {\n eprintln!(\"ERROR: can't detect file type of {file_path} without extension\",\n file_path = file_path.display());\n })?.to_string_lossy();\n match extension.as_ref() {\n \"xhtml\" | \"xml\" => parse_entire_xml_file(file_path),\n // TODO: specialized parser for markdown files\n \"txt\" | \"md\" => parse_entire_txt_file(file_path),\n \"pdf\" => parse_entire_pdf_file(file_path),\n _ => {\n eprintln!(\"ERROR: can't detect file type of {file_path}: unsupported extension {extension}\",\n file_path = file_path.display(),\n extension = extension);\n Err(())\n }\n }\n}\n\nfn save_model_as_json(model: &Model, index_path: &Path) -> Result<(), ()> {\n println!(\"Saving {index_path}...\", index_path = index_path.display());\n\n let index_file = File::create(index_path).map_err(|err| {\n eprintln!(\"ERROR: could not create index file {index_path}: {err}\",\n index_path = index_path.display());\n })?;\n\n serde_json::to_writer(BufWriter::new(index_file), &model).map_err(|err| {\n eprintln!(\"ERROR: could not serialize index into file {index_path}: {err}\",\n index_path = index_path.display());\n })?;\n\n Ok(())\n}\n\nfn add_folder_to_model(dir_path: &Path, model: Arc>, processed: &mut usize) -> Result<(), ()> {\n let dir = fs::read_dir(dir_path).map_err(|err| {\n eprintln!(\"ERROR: could not open directory {dir_path} for indexing: {err}\",\n dir_path = dir_path.display());\n })?;\n\n 'next_file: for file in dir {\n let file = file.map_err(|err| {\n eprintln!(\"ERROR: could not read next file in directory {dir_path} during indexing: {err}\",\n dir_path = dir_path.display());\n })?;\n\n let file_path = file.path();\n\n let dot_file = file_path\n .file_name()\n .and_then(|s| s.to_str())\n .map(|s| s.starts_with(\".\"))\n .unwrap_or(false);\n\n if dot_file {\n continue 'next_file;\n }\n\n let file_type = file.file_type().map_err(|err| {\n eprintln!(\"ERROR: could not determine type of file {file_path}: {err}\",\n file_path = file_path.display());\n })?;\n let last_modified = file.metadata().map_err(|err| {\n eprintln!(\"ERROR: could not get the metadata of file {file_path}: {err}\",\n file_path = file_path.display());\n })?.modified().map_err(|err| {\n eprintln!(\"ERROR: could not get the last modification date of file {file_path}: {err}\",\n file_path = file_path.display())\n })?;\n\n if file_type.is_dir() {\n add_folder_to_model(&file_path, Arc::clone(&model), processed)?;\n continue 'next_file;\n }\n\n // TODO: how does this work with symlinks?\n\n let mut model = model.lock().unwrap();\n if model.requires_reindexing(&file_path, last_modified) {\n println!(\"Indexing {:?}...\", &file_path);\n\n let content = match parse_entire_file_by_extension(&file_path) {\n Ok(content) => content.chars().collect::>(),\n // TODO: still add the skipped files to the model to prevent their reindexing in the future\n Err(()) => continue 'next_file,\n };\n\n model.add_document(file_path, last_modified, &content);\n *processed += 1;\n }\n }\n\n Ok(())\n}\n\nfn usage(program: &str) {\n eprintln!(\"Usage: {program} [SUBCOMMAND] [OPTIONS]\");\n eprintln!(\"Subcommands:\");\n eprintln!(\" serve [address] start local HTTP server with Web Interface\");\n}\n\nfn entry() -> Result<(), ()> {\n let mut args = env::args();\n let program = args.next().expect(\"path to program is provided\");\n\n let subcommand = args.next().ok_or_else(|| {\n usage(&program);\n eprintln!(\"ERROR: no subcommand is provided\");\n })?;\n\n match subcommand.as_str() {\n \"serve\" => {\n let dir_path = args.next().ok_or_else(|| {\n usage(&program);\n eprintln!(\"ERROR: no directory is provided for {subcommand} subcommand\");\n })?;\n\n let mut index_path = Path::new(&dir_path).to_path_buf();\n index_path.push(\".seroost.json\");\n\n let address = args.next().unwrap_or(\"127.0.0.1:6969\".to_string());\n\n let exists = index_path.try_exists().map_err(|err| {\n eprintln!(\"ERROR: could not check the existence of file {index_path}: {err}\",\n index_path = index_path.display());\n })?;\n\n let model: Arc>;\n if exists {\n let index_file = File::open(&index_path).map_err(|err| {\n eprintln!(\"ERROR: could not open index file {index_path}: {err}\",\n index_path = index_path.display());\n })?;\n\n model = Arc::new(Mutex::new(serde_json::from_reader(index_file).map_err(|err| {\n eprintln!(\"ERROR: could not parse index file {index_path}: {err}\",\n index_path = index_path.display());\n })?));\n } else {\n model = Arc::new(Mutex::new(Default::default()));\n }\n\n {\n let model = Arc::clone(&model);\n thread::spawn(move || {\n let mut processed = 0;\n // TODO: what should we do in case indexing thread crashes\n add_folder_to_model(Path::new(&dir_path), Arc::clone(&model), &mut processed).unwrap();\n if processed > 0 {\n let model = model.lock().unwrap();\n save_model_as_json(&model, &index_path).unwrap();\n }\n println!(\"Finished indexing\");\n });\n }\n\n server::start(&address, Arc::clone(&model))\n }\n\n _ => {\n usage(&program);\n eprintln!(\"ERROR: unknown subcommand {subcommand}\");\n Err(())\n }\n }\n}\n\nfn main() -> ExitCode {\n match entry() {\n Ok(()) => ExitCode::SUCCESS,\n Err(()) => ExitCode::FAILURE,\n }\n}\n\n// TODO: search result must consist of clickable links\n// TODO: synonym terms\n" }, { "path": "src/model.rs", "content": "use std::collections::HashMap;\nuse std::path::{PathBuf, Path};\nuse serde::{Deserialize, Serialize};\nuse super::lexer::Lexer;\nuse std::time::SystemTime;\n\ntype DocFreq = HashMap;\ntype TermFreq = HashMap;\n#[derive(Deserialize, Serialize)]\npub struct Doc {\n tf: TermFreq,\n count: usize,\n // TODO: make sure that the serde serialization of SystemTime also work on other platforms\n last_modified: SystemTime,\n}\ntype Docs = HashMap;\n\n#[derive(Default, Deserialize, Serialize)]\npub struct Model {\n pub docs: Docs,\n pub df: DocFreq,\n}\n\nimpl Model {\n fn remove_document(&mut self, file_path: &Path) {\n if let Some(doc) = self.docs.remove(file_path) {\n for t in doc.tf.keys() {\n if let Some(f) = self.df.get_mut(t) {\n *f -= 1;\n }\n }\n }\n }\n\n pub fn requires_reindexing(&mut self, file_path: &Path, last_modified: SystemTime) -> bool {\n if let Some(doc) = self.docs.get(file_path) {\n return doc.last_modified < last_modified;\n }\n return true;\n }\n\n pub fn search_query(&self, query: &[char]) -> Vec<(PathBuf, f32)> {\n let mut result = Vec::new();\n let tokens = Lexer::new(&query).collect::>();\n for (path, doc) in &self.docs {\n let mut rank = 0f32;\n for token in &tokens {\n rank += compute_tf(token, doc) * compute_idf(&token, self.docs.len(), &self.df);\n }\n // TODO: investigate the sources of NaN\n if !rank.is_nan() {\n result.push((path.clone(), rank));\n }\n }\n result.sort_by(|(_, rank1), (_, rank2)| rank1.partial_cmp(rank2).expect(&format!(\"{rank1} and {rank2} are not comparable\")));\n result.reverse();\n result\n }\n\n pub fn add_document(&mut self, file_path: PathBuf, last_modified: SystemTime, content: &[char]) {\n self.remove_document(&file_path);\n\n let mut tf = TermFreq::new();\n\n let mut count = 0;\n for t in Lexer::new(content) {\n if let Some(f) = tf.get_mut(&t) {\n *f += 1;\n } else {\n tf.insert(t, 1);\n }\n count += 1;\n }\n\n for t in tf.keys() {\n if let Some(f) = self.df.get_mut(t) {\n *f += 1;\n } else {\n self.df.insert(t.to_string(), 1);\n }\n }\n\n self.docs.insert(file_path, Doc {count, tf, last_modified});\n }\n}\n\nfn compute_tf(t: &str, doc: &Doc) -> f32 {\n let n = doc.count as f32;\n let m = doc.tf.get(t).cloned().unwrap_or(0) as f32;\n m / n\n}\n\nfn compute_idf(t: &str, n: usize, df: &DocFreq) -> f32 {\n let n = n as f32;\n let m = df.get(t).cloned().unwrap_or(1) as f32;\n (n / m).log10()\n}\n" }, { "path": "src/server.rs", "content": "use std::str;\nuse std::io;\nuse std::sync::{Arc, Mutex};\n\nuse super::model::*;\n\nuse tiny_http::{Server, Request, Response, Header, Method, StatusCode};\n\nfn serve_404(request: Request) -> io::Result<()> {\n request.respond(Response::from_string(\"404\").with_status_code(StatusCode(404)))\n}\n\nfn serve_500(request: Request) -> io::Result<()> {\n request.respond(Response::from_string(\"500\").with_status_code(StatusCode(500)))\n}\n\nfn serve_400(request: Request, message: &str) -> io::Result<()> {\n request.respond(Response::from_string(format!(\"400: {message}\")).with_status_code(StatusCode(400)))\n}\n\nfn serve_bytes(request: Request, bytes: &[u8], content_type: &str) -> io::Result<()> {\n let content_type_header = Header::from_bytes(\"Content-Type\", content_type)\n .expect(\"That we didn't put any garbage in the headers\");\n request.respond(Response::from_data(bytes).with_header(content_type_header))\n}\n\n// TODO: the errors of serve_api_search should probably return JSON\n// 'Cause that's what expected from them.\nfn serve_api_search(model: Arc>, mut request: Request) -> io::Result<()> {\n let mut buf = Vec::new();\n if let Err(err) = request.as_reader().read_to_end(&mut buf) {\n eprintln!(\"ERROR: could not read the body of the request: {err}\");\n return serve_500(request);\n }\n\n let body = match str::from_utf8(&buf) {\n Ok(body) => body.chars().collect::>(),\n Err(err) => {\n eprintln!(\"ERROR: could not interpret body as UTF-8 string: {err}\");\n return serve_400(request, \"Body must be a valid UTF-8 string\");\n }\n };\n\n let model = model.lock().unwrap();\n let result = model.search_query(&body);\n\n let json = match serde_json::to_string(&result.iter().take(20).collect::>()) {\n Ok(json) => json,\n Err(err) => {\n eprintln!(\"ERROR: could not convert search results to JSON: {err}\");\n return serve_500(request)\n }\n };\n\n let content_type_header = Header::from_bytes(\"Content-Type\", \"application/json\")\n .expect(\"That we didn't put any garbage in the headers\");\n request.respond(Response::from_string(&json).with_header(content_type_header))\n}\n\nfn serve_api_stats(model: Arc>, request: Request) -> io::Result<()> {\n use serde::Serialize;\n\n #[derive(Default, Serialize)]\n struct Stats {\n docs_count: usize,\n terms_count: usize,\n }\n\n let mut stats: Stats = Default::default();\n {\n let model = model.lock().unwrap();\n stats.docs_count = model.docs.len();\n stats.terms_count = model.df.len();\n }\n\n let json = match serde_json::to_string(&stats) {\n Ok(json) => json,\n Err(err) => {\n eprintln!(\"ERROR: could not convert stats results to JSON: {err}\");\n return serve_500(request)\n }\n };\n\n let content_type_header = Header::from_bytes(\"Content-Type\", \"application/json\")\n .expect(\"That we didn't put any garbage in the headers\");\n request.respond(Response::from_string(&json).with_header(content_type_header))\n}\n\nfn serve_request(model: Arc>, request: Request) -> io::Result<()> {\n println!(\"INFO: received request! method: {:?}, url: {:?}\", request.method(), request.url());\n\n match (request.method(), request.url()) {\n (Method::Post, \"/api/search\") => {\n serve_api_search(model, request)\n }\n (Method::Get, \"/api/stats\") => {\n serve_api_stats(model, request)\n }\n (Method::Get, \"/index.js\") => {\n serve_bytes(request, include_bytes!(\"index.js\"), \"text/javascript; charset=utf-8\")\n }\n (Method::Get, \"/\") | (Method::Get, \"/index.html\") => {\n serve_bytes(request, include_bytes!(\"index.html\"), \"text/html; charset=utf-8\")\n }\n _ => {\n serve_404(request)\n }\n }\n}\n\npub fn start(address: &str, model: Arc>) -> Result<(), ()> {\n let server = Server::http(&address).map_err(|err| {\n eprintln!(\"ERROR: could not start HTTP server at {address}: {err}\");\n })?;\n\n println!(\"INFO: listening at http://{address}/\");\n\n for request in server.incoming_requests() {\n serve_request(Arc::clone(&model), request).map_err(|err| {\n eprintln!(\"ERROR: could not serve the response: {err}\");\n }).ok(); // <- don't stop on errors, keep serving\n }\n\n eprintln!(\"ERROR: the server socket has shutdown\");\n Err(())\n}\n" }, { "path": "src/snowball/algorithms/english_stemmer.rs", "content": "//! Generated by Snowball 2.2.0 - https://snowballstem.org/\n\n#![allow(non_snake_case)]\n#![allow(non_upper_case_globals)]\n#![allow(unused_mut)]\n#![allow(unused_parens)]\n#![allow(unused_variables)]\nuse crate::snowball::SnowballEnv;\nuse crate::snowball::Among;\n\nstatic A_0: &'static [Among; 3] = &[\n Among(\"arsen\", -1, -1, None),\n Among(\"commun\", -1, -1, None),\n Among(\"gener\", -1, -1, None),\n];\n\nstatic A_1: &'static [Among; 3] = &[\n Among(\"'\", -1, 1, None),\n Among(\"'s'\", 0, 1, None),\n Among(\"'s\", -1, 1, None),\n];\n\nstatic A_2: &'static [Among; 6] = &[\n Among(\"ied\", -1, 2, None),\n Among(\"s\", -1, 3, None),\n Among(\"ies\", 1, 2, None),\n Among(\"sses\", 1, 1, None),\n Among(\"ss\", 1, -1, None),\n Among(\"us\", 1, -1, None),\n];\n\nstatic A_3: &'static [Among; 13] = &[\n Among(\"\", -1, 3, None),\n Among(\"bb\", 0, 2, None),\n Among(\"dd\", 0, 2, None),\n Among(\"ff\", 0, 2, None),\n Among(\"gg\", 0, 2, None),\n Among(\"bl\", 0, 1, None),\n Among(\"mm\", 0, 2, None),\n Among(\"nn\", 0, 2, None),\n Among(\"pp\", 0, 2, None),\n Among(\"rr\", 0, 2, None),\n Among(\"at\", 0, 1, None),\n Among(\"tt\", 0, 2, None),\n Among(\"iz\", 0, 1, None),\n];\n\nstatic A_4: &'static [Among; 6] = &[\n Among(\"ed\", -1, 2, None),\n Among(\"eed\", 0, 1, None),\n Among(\"ing\", -1, 2, None),\n Among(\"edly\", -1, 2, None),\n Among(\"eedly\", 3, 1, None),\n Among(\"ingly\", -1, 2, None),\n];\n\nstatic A_5: &'static [Among; 24] = &[\n Among(\"anci\", -1, 3, None),\n Among(\"enci\", -1, 2, None),\n Among(\"ogi\", -1, 13, None),\n Among(\"li\", -1, 15, None),\n Among(\"bli\", 3, 12, None),\n Among(\"abli\", 4, 4, None),\n Among(\"alli\", 3, 8, None),\n Among(\"fulli\", 3, 9, None),\n Among(\"lessli\", 3, 14, None),\n Among(\"ousli\", 3, 10, None),\n Among(\"entli\", 3, 5, None),\n Among(\"aliti\", -1, 8, None),\n Among(\"biliti\", -1, 12, None),\n Among(\"iviti\", -1, 11, None),\n Among(\"tional\", -1, 1, None),\n Among(\"ational\", 14, 7, None),\n Among(\"alism\", -1, 8, None),\n Among(\"ation\", -1, 7, None),\n Among(\"ization\", 17, 6, None),\n Among(\"izer\", -1, 6, None),\n Among(\"ator\", -1, 7, None),\n Among(\"iveness\", -1, 11, None),\n Among(\"fulness\", -1, 9, None),\n Among(\"ousness\", -1, 10, None),\n];\n\nstatic A_6: &'static [Among; 9] = &[\n Among(\"icate\", -1, 4, None),\n Among(\"ative\", -1, 6, None),\n Among(\"alize\", -1, 3, None),\n Among(\"iciti\", -1, 4, None),\n Among(\"ical\", -1, 4, None),\n Among(\"tional\", -1, 1, None),\n Among(\"ational\", 5, 2, None),\n Among(\"ful\", -1, 5, None),\n Among(\"ness\", -1, 5, None),\n];\n\nstatic A_7: &'static [Among; 18] = &[\n Among(\"ic\", -1, 1, None),\n Among(\"ance\", -1, 1, None),\n Among(\"ence\", -1, 1, None),\n Among(\"able\", -1, 1, None),\n Among(\"ible\", -1, 1, None),\n Among(\"ate\", -1, 1, None),\n Among(\"ive\", -1, 1, None),\n Among(\"ize\", -1, 1, None),\n Among(\"iti\", -1, 1, None),\n Among(\"al\", -1, 1, None),\n Among(\"ism\", -1, 1, None),\n Among(\"ion\", -1, 2, None),\n Among(\"er\", -1, 1, None),\n Among(\"ous\", -1, 1, None),\n Among(\"ant\", -1, 1, None),\n Among(\"ent\", -1, 1, None),\n Among(\"ment\", 15, 1, None),\n Among(\"ement\", 16, 1, None),\n];\n\nstatic A_8: &'static [Among; 2] = &[\n Among(\"e\", -1, 1, None),\n Among(\"l\", -1, 2, None),\n];\n\nstatic A_9: &'static [Among; 8] = &[\n Among(\"succeed\", -1, -1, None),\n Among(\"proceed\", -1, -1, None),\n Among(\"exceed\", -1, -1, None),\n Among(\"canning\", -1, -1, None),\n Among(\"inning\", -1, -1, None),\n Among(\"earring\", -1, -1, None),\n Among(\"herring\", -1, -1, None),\n Among(\"outing\", -1, -1, None),\n];\n\nstatic A_10: &'static [Among; 18] = &[\n Among(\"andes\", -1, -1, None),\n Among(\"atlas\", -1, -1, None),\n Among(\"bias\", -1, -1, None),\n Among(\"cosmos\", -1, -1, None),\n Among(\"dying\", -1, 3, None),\n Among(\"early\", -1, 9, None),\n Among(\"gently\", -1, 7, None),\n Among(\"howe\", -1, -1, None),\n Among(\"idly\", -1, 6, None),\n Among(\"lying\", -1, 4, None),\n Among(\"news\", -1, -1, None),\n Among(\"only\", -1, 10, None),\n Among(\"singly\", -1, 11, None),\n Among(\"skies\", -1, 2, None),\n Among(\"skis\", -1, 1, None),\n Among(\"sky\", -1, -1, None),\n Among(\"tying\", -1, 5, None),\n Among(\"ugly\", -1, 8, None),\n];\n\nstatic G_v: &'static [u8; 4] = &[17, 65, 16, 1];\n\nstatic G_v_WXY: &'static [u8; 5] = &[1, 17, 65, 208, 1];\n\nstatic G_valid_LI: &'static [u8; 3] = &[55, 141, 2];\n\n#[derive(Clone)]\nstruct Context {\n b_Y_found: bool,\n i_p2: i32,\n i_p1: i32,\n}\n\nfn r_prelude(env: &mut SnowballEnv, context: &mut Context) -> bool {\n context.b_Y_found = false;\n let v_1 = env.cursor;\n 'lab0: loop {\n env.bra = env.cursor;\n if !env.eq_s(&\"'\") {\n break 'lab0;\n }\n env.ket = env.cursor;\n if !env.slice_del() {\n return false;\n }\n break 'lab0;\n }\n env.cursor = v_1;\n let v_2 = env.cursor;\n 'lab1: loop {\n env.bra = env.cursor;\n if !env.eq_s(&\"y\") {\n break 'lab1;\n }\n env.ket = env.cursor;\n if !env.slice_from(\"Y\") {\n return false;\n }\n context.b_Y_found = true;\n break 'lab1;\n }\n env.cursor = v_2;\n let v_3 = env.cursor;\n 'lab2: loop {\n 'replab3: loop{\n let v_4 = env.cursor;\n 'lab4: for _ in 0..1 {\n 'golab5: loop {\n let v_5 = env.cursor;\n 'lab6: loop {\n if !env.in_grouping(G_v, 97, 121) {\n break 'lab6;\n }\n env.bra = env.cursor;\n if !env.eq_s(&\"y\") {\n break 'lab6;\n }\n env.ket = env.cursor;\n env.cursor = v_5;\n break 'golab5;\n }\n env.cursor = v_5;\n if env.cursor >= env.limit {\n break 'lab4;\n }\n env.next_char();\n }\n if !env.slice_from(\"Y\") {\n return false;\n }\n context.b_Y_found = true;\n continue 'replab3;\n }\n env.cursor = v_4;\n break 'replab3;\n }\n break 'lab2;\n }\n env.cursor = v_3;\n return true;\n}\n\nfn r_mark_regions(env: &mut SnowballEnv, context: &mut Context) -> bool {\n context.i_p1 = env.limit;\n context.i_p2 = env.limit;\n let v_1 = env.cursor;\n 'lab0: loop {\n 'lab1: loop {\n let v_2 = env.cursor;\n 'lab2: loop {\n if env.find_among(A_0, context) == 0 {\n break 'lab2;\n }\n break 'lab1;\n }\n env.cursor = v_2;\n 'golab3: loop {\n 'lab4: loop {\n if !env.in_grouping(G_v, 97, 121) {\n break 'lab4;\n }\n break 'golab3;\n }\n if env.cursor >= env.limit {\n break 'lab0;\n }\n env.next_char();\n }\n 'golab5: loop {\n 'lab6: loop {\n if !env.out_grouping(G_v, 97, 121) {\n break 'lab6;\n }\n break 'golab5;\n }\n if env.cursor >= env.limit {\n break 'lab0;\n }\n env.next_char();\n }\n break 'lab1;\n }\n context.i_p1 = env.cursor;\n 'golab7: loop {\n 'lab8: loop {\n if !env.in_grouping(G_v, 97, 121) {\n break 'lab8;\n }\n break 'golab7;\n }\n if env.cursor >= env.limit {\n break 'lab0;\n }\n env.next_char();\n }\n 'golab9: loop {\n 'lab10: loop {\n if !env.out_grouping(G_v, 97, 121) {\n break 'lab10;\n }\n break 'golab9;\n }\n if env.cursor >= env.limit {\n break 'lab0;\n }\n env.next_char();\n }\n context.i_p2 = env.cursor;\n break 'lab0;\n }\n env.cursor = v_1;\n return true;\n}\n\nfn r_shortv(env: &mut SnowballEnv, context: &mut Context) -> bool {\n 'lab0: loop {\n let v_1 = env.limit - env.cursor;\n 'lab1: loop {\n if !env.out_grouping_b(G_v_WXY, 89, 121) {\n break 'lab1;\n }\n if !env.in_grouping_b(G_v, 97, 121) {\n break 'lab1;\n }\n if !env.out_grouping_b(G_v, 97, 121) {\n break 'lab1;\n }\n break 'lab0;\n }\n env.cursor = env.limit - v_1;\n if !env.out_grouping_b(G_v, 97, 121) {\n return false;\n }\n if !env.in_grouping_b(G_v, 97, 121) {\n return false;\n }\n if env.cursor > env.limit_backward {\n return false;\n }\n break 'lab0;\n }\n return true;\n}\n\nfn r_R1(env: &mut SnowballEnv, context: &mut Context) -> bool {\n if !(context.i_p1 <= env.cursor){\n return false;\n }\n return true;\n}\n\nfn r_R2(env: &mut SnowballEnv, context: &mut Context) -> bool {\n if !(context.i_p2 <= env.cursor){\n return false;\n }\n return true;\n}\n\nfn r_Step_1a(env: &mut SnowballEnv, context: &mut Context) -> bool {\n let mut among_var;\n let v_1 = env.limit - env.cursor;\n 'lab0: loop {\n env.ket = env.cursor;\n if env.find_among_b(A_1, context) == 0 {\n env.cursor = env.limit - v_1;\n break 'lab0;\n }\n env.bra = env.cursor;\n if !env.slice_del() {\n return false;\n }\n break 'lab0;\n }\n env.ket = env.cursor;\n among_var = env.find_among_b(A_2, context);\n if among_var == 0 {\n return false;\n }\n env.bra = env.cursor;\n if among_var == 1 {\n if !env.slice_from(\"ss\") {\n return false;\n }\n } else if among_var == 2 {\n 'lab1: loop {\n let v_2 = env.limit - env.cursor;\n 'lab2: loop {\n if !env.hop_back(2) {\n break 'lab2;\n }\n if !env.slice_from(\"i\") {\n return false;\n }\n break 'lab1;\n }\n env.cursor = env.limit - v_2;\n if !env.slice_from(\"ie\") {\n return false;\n }\n break 'lab1;\n }\n } else if among_var == 3 {\n if env.cursor <= env.limit_backward {\n return false;\n }\n env.previous_char();\n 'golab3: loop {\n 'lab4: loop {\n if !env.in_grouping_b(G_v, 97, 121) {\n break 'lab4;\n }\n break 'golab3;\n }\n if env.cursor <= env.limit_backward {\n return false;\n }\n env.previous_char();\n }\n if !env.slice_del() {\n return false;\n }\n }\n return true;\n}\n\nfn r_Step_1b(env: &mut SnowballEnv, context: &mut Context) -> bool {\n let mut among_var;\n env.ket = env.cursor;\n among_var = env.find_among_b(A_4, context);\n if among_var == 0 {\n return false;\n }\n env.bra = env.cursor;\n if among_var == 1 {\n if !r_R1(env, context) {\n return false;\n }\n if !env.slice_from(\"ee\") {\n return false;\n }\n } else if among_var == 2 {\n let v_1 = env.limit - env.cursor;\n 'golab0: loop {\n 'lab1: loop {\n if !env.in_grouping_b(G_v, 97, 121) {\n break 'lab1;\n }\n break 'golab0;\n }\n if env.cursor <= env.limit_backward {\n return false;\n }\n env.previous_char();\n }\n env.cursor = env.limit - v_1;\n if !env.slice_del() {\n return false;\n }\n let v_3 = env.limit - env.cursor;\n among_var = env.find_among_b(A_3, context);\n if among_var == 0 {\n return false;\n }\n env.cursor = env.limit - v_3;\n if among_var == 1 {\n let c = env.cursor;\n let (bra, ket) = (env.cursor, env.cursor);\n env.insert(bra, ket, \"e\");\n env.cursor = c;\n } else if among_var == 2 {\n env.ket = env.cursor;\n if env.cursor <= env.limit_backward {\n return false;\n }\n env.previous_char();\n env.bra = env.cursor;\n if !env.slice_del() {\n return false;\n }\n } else if among_var == 3 {\n if env.cursor != context.i_p1 {\n return false;\n }\n let v_4 = env.limit - env.cursor;\n if !r_shortv(env, context) {\n return false;\n }\n env.cursor = env.limit - v_4;\n let c = env.cursor;\n let (bra, ket) = (env.cursor, env.cursor);\n env.insert(bra, ket, \"e\");\n env.cursor = c;\n }\n }\n return true;\n}\n\nfn r_Step_1c(env: &mut SnowballEnv, context: &mut Context) -> bool {\n env.ket = env.cursor;\n 'lab0: loop {\n let v_1 = env.limit - env.cursor;\n 'lab1: loop {\n if !env.eq_s_b(&\"y\") {\n break 'lab1;\n }\n break 'lab0;\n }\n env.cursor = env.limit - v_1;\n if !env.eq_s_b(&\"Y\") {\n return false;\n }\n break 'lab0;\n }\n env.bra = env.cursor;\n if !env.out_grouping_b(G_v, 97, 121) {\n return false;\n }\n 'lab2: loop {\n if env.cursor > env.limit_backward {\n break 'lab2;\n }\n return false;\n }\n if !env.slice_from(\"i\") {\n return false;\n }\n return true;\n}\n\nfn r_Step_2(env: &mut SnowballEnv, context: &mut Context) -> bool {\n let mut among_var;\n env.ket = env.cursor;\n among_var = env.find_among_b(A_5, context);\n if among_var == 0 {\n return false;\n }\n env.bra = env.cursor;\n if !r_R1(env, context) {\n return false;\n }\n if among_var == 1 {\n if !env.slice_from(\"tion\") {\n return false;\n }\n } else if among_var == 2 {\n if !env.slice_from(\"ence\") {\n return false;\n }\n } else if among_var == 3 {\n if !env.slice_from(\"ance\") {\n return false;\n }\n } else if among_var == 4 {\n if !env.slice_from(\"able\") {\n return false;\n }\n } else if among_var == 5 {\n if !env.slice_from(\"ent\") {\n return false;\n }\n } else if among_var == 6 {\n if !env.slice_from(\"ize\") {\n return false;\n }\n } else if among_var == 7 {\n if !env.slice_from(\"ate\") {\n return false;\n }\n } else if among_var == 8 {\n if !env.slice_from(\"al\") {\n return false;\n }\n } else if among_var == 9 {\n if !env.slice_from(\"ful\") {\n return false;\n }\n } else if among_var == 10 {\n if !env.slice_from(\"ous\") {\n return false;\n }\n } else if among_var == 11 {\n if !env.slice_from(\"ive\") {\n return false;\n }\n } else if among_var == 12 {\n if !env.slice_from(\"ble\") {\n return false;\n }\n } else if among_var == 13 {\n if !env.eq_s_b(&\"l\") {\n return false;\n }\n if !env.slice_from(\"og\") {\n return false;\n }\n } else if among_var == 14 {\n if !env.slice_from(\"less\") {\n return false;\n }\n } else if among_var == 15 {\n if !env.in_grouping_b(G_valid_LI, 99, 116) {\n return false;\n }\n if !env.slice_del() {\n return false;\n }\n }\n return true;\n}\n\nfn r_Step_3(env: &mut SnowballEnv, context: &mut Context) -> bool {\n let mut among_var;\n env.ket = env.cursor;\n among_var = env.find_among_b(A_6, context);\n if among_var == 0 {\n return false;\n }\n env.bra = env.cursor;\n if !r_R1(env, context) {\n return false;\n }\n if among_var == 1 {\n if !env.slice_from(\"tion\") {\n return false;\n }\n } else if among_var == 2 {\n if !env.slice_from(\"ate\") {\n return false;\n }\n } else if among_var == 3 {\n if !env.slice_from(\"al\") {\n return false;\n }\n } else if among_var == 4 {\n if !env.slice_from(\"ic\") {\n return false;\n }\n } else if among_var == 5 {\n if !env.slice_del() {\n return false;\n }\n } else if among_var == 6 {\n if !r_R2(env, context) {\n return false;\n }\n if !env.slice_del() {\n return false;\n }\n }\n return true;\n}\n\nfn r_Step_4(env: &mut SnowballEnv, context: &mut Context) -> bool {\n let mut among_var;\n env.ket = env.cursor;\n among_var = env.find_among_b(A_7, context);\n if among_var == 0 {\n return false;\n }\n env.bra = env.cursor;\n if !r_R2(env, context) {\n return false;\n }\n if among_var == 1 {\n if !env.slice_del() {\n return false;\n }\n } else if among_var == 2 {\n 'lab0: loop {\n let v_1 = env.limit - env.cursor;\n 'lab1: loop {\n if !env.eq_s_b(&\"s\") {\n break 'lab1;\n }\n break 'lab0;\n }\n env.cursor = env.limit - v_1;\n if !env.eq_s_b(&\"t\") {\n return false;\n }\n break 'lab0;\n }\n if !env.slice_del() {\n return false;\n }\n }\n return true;\n}\n\nfn r_Step_5(env: &mut SnowballEnv, context: &mut Context) -> bool {\n let mut among_var;\n env.ket = env.cursor;\n among_var = env.find_among_b(A_8, context);\n if among_var == 0 {\n return false;\n }\n env.bra = env.cursor;\n if among_var == 1 {\n 'lab0: loop {\n let v_1 = env.limit - env.cursor;\n 'lab1: loop {\n if !r_R2(env, context) {\n break 'lab1;\n }\n break 'lab0;\n }\n env.cursor = env.limit - v_1;\n if !r_R1(env, context) {\n return false;\n }\n let v_2 = env.limit - env.cursor;\n 'lab2: loop {\n if !r_shortv(env, context) {\n break 'lab2;\n }\n return false;\n }\n env.cursor = env.limit - v_2;\n break 'lab0;\n }\n if !env.slice_del() {\n return false;\n }\n } else if among_var == 2 {\n if !r_R2(env, context) {\n return false;\n }\n if !env.eq_s_b(&\"l\") {\n return false;\n }\n if !env.slice_del() {\n return false;\n }\n }\n return true;\n}\n\nfn r_exception2(env: &mut SnowballEnv, context: &mut Context) -> bool {\n env.ket = env.cursor;\n if env.find_among_b(A_9, context) == 0 {\n return false;\n }\n env.bra = env.cursor;\n if env.cursor > env.limit_backward {\n return false;\n }\n return true;\n}\n\nfn r_exception1(env: &mut SnowballEnv, context: &mut Context) -> bool {\n let mut among_var;\n env.bra = env.cursor;\n among_var = env.find_among(A_10, context);\n if among_var == 0 {\n return false;\n }\n env.ket = env.cursor;\n if env.cursor < env.limit {\n return false;\n }\n if among_var == 1 {\n if !env.slice_from(\"ski\") {\n return false;\n }\n } else if among_var == 2 {\n if !env.slice_from(\"sky\") {\n return false;\n }\n } else if among_var == 3 {\n if !env.slice_from(\"die\") {\n return false;\n }\n } else if among_var == 4 {\n if !env.slice_from(\"lie\") {\n return false;\n }\n } else if among_var == 5 {\n if !env.slice_from(\"tie\") {\n return false;\n }\n } else if among_var == 6 {\n if !env.slice_from(\"idl\") {\n return false;\n }\n } else if among_var == 7 {\n if !env.slice_from(\"gentl\") {\n return false;\n }\n } else if among_var == 8 {\n if !env.slice_from(\"ugli\") {\n return false;\n }\n } else if among_var == 9 {\n if !env.slice_from(\"earli\") {\n return false;\n }\n } else if among_var == 10 {\n if !env.slice_from(\"onli\") {\n return false;\n }\n } else if among_var == 11 {\n if !env.slice_from(\"singl\") {\n return false;\n }\n }\n return true;\n}\n\nfn r_postlude(env: &mut SnowballEnv, context: &mut Context) -> bool {\n if !context.b_Y_found {\n return false;\n }\n 'replab0: loop{\n let v_1 = env.cursor;\n 'lab1: for _ in 0..1 {\n 'golab2: loop {\n let v_2 = env.cursor;\n 'lab3: loop {\n env.bra = env.cursor;\n if !env.eq_s(&\"Y\") {\n break 'lab3;\n }\n env.ket = env.cursor;\n env.cursor = v_2;\n break 'golab2;\n }\n env.cursor = v_2;\n if env.cursor >= env.limit {\n break 'lab1;\n }\n env.next_char();\n }\n if !env.slice_from(\"y\") {\n return false;\n }\n continue 'replab0;\n }\n env.cursor = v_1;\n break 'replab0;\n }\n return true;\n}\n\npub fn stem(env: &mut SnowballEnv) -> bool {\n let mut context = &mut Context {\n b_Y_found: false,\n i_p2: 0,\n i_p1: 0,\n };\n 'lab0: loop {\n let v_1 = env.cursor;\n 'lab1: loop {\n if !r_exception1(env, context) {\n break 'lab1;\n }\n break 'lab0;\n }\n env.cursor = v_1;\n 'lab2: loop {\n let v_2 = env.cursor;\n 'lab3: loop {\n if !env.hop(3) {\n break 'lab3;\n }\n break 'lab2;\n }\n env.cursor = v_2;\n break 'lab0;\n }\n env.cursor = v_1;\n r_prelude(env, context);\n r_mark_regions(env, context);\n env.limit_backward = env.cursor;\n env.cursor = env.limit;\n let v_5 = env.limit - env.cursor;\n r_Step_1a(env, context);\n env.cursor = env.limit - v_5;\n 'lab4: loop {\n let v_6 = env.limit - env.cursor;\n 'lab5: loop {\n if !r_exception2(env, context) {\n break 'lab5;\n }\n break 'lab4;\n }\n env.cursor = env.limit - v_6;\n let v_7 = env.limit - env.cursor;\n r_Step_1b(env, context);\n env.cursor = env.limit - v_7;\n let v_8 = env.limit - env.cursor;\n r_Step_1c(env, context);\n env.cursor = env.limit - v_8;\n let v_9 = env.limit - env.cursor;\n r_Step_2(env, context);\n env.cursor = env.limit - v_9;\n let v_10 = env.limit - env.cursor;\n r_Step_3(env, context);\n env.cursor = env.limit - v_10;\n let v_11 = env.limit - env.cursor;\n r_Step_4(env, context);\n env.cursor = env.limit - v_11;\n let v_12 = env.limit - env.cursor;\n r_Step_5(env, context);\n env.cursor = env.limit - v_12;\n break 'lab4;\n }\n env.cursor = env.limit_backward;\n let v_13 = env.cursor;\n r_postlude(env, context);\n env.cursor = v_13;\n break 'lab0;\n }\n return true;\n}\n" }, { "path": "src/snowball/algorithms/mod.rs", "content": "// Have a look at build.rs\n//include!(concat!(env!(\"OUT_DIR\"), \"/lang_include.rs\"));\npub mod english_stemmer;\n" }, { "path": "src/snowball/among.rs", "content": "use crate::snowball::SnowballEnv;\n\npub struct Among(pub &'static str,\n pub i32,\n pub i32,\n pub Option<&'static (dyn Fn(&mut SnowballEnv, &mut T) -> bool + Sync)>);\n" }, { "path": "src/snowball/mod.rs", "content": "// TODO: add Snowball license in here\npub mod algorithms;\nmod among;\nmod snowball_env;\n\n// TODO: why do we need this `crate::`?\npub use crate::snowball::among::Among;\npub use crate::snowball::snowball_env::SnowballEnv;\n" }, { "path": "src/snowball/snowball_env.rs", "content": "use std::borrow::Cow;\nuse crate::snowball::Among;\n\n#[derive(Debug, Clone)]\npub struct SnowballEnv<'a> {\n pub current: Cow<'a, str>,\n pub cursor: i32,\n pub limit: i32,\n pub limit_backward: i32,\n pub bra: i32,\n pub ket: i32,\n}\n\n\nimpl<'a> SnowballEnv<'a> {\n pub fn create(value: &'a str) -> Self {\n let len = value.len();\n SnowballEnv {\n current: Cow::from(value),\n cursor: 0,\n limit: len as i32,\n limit_backward: 0,\n bra: 0,\n ket: len as i32,\n }\n }\n\n pub fn get_current(self) -> Cow<'a, str> {\n self.current\n }\n\n pub fn set_current(&mut self, current: &'a str) {\n self.current = Cow::from(current);\n }\n\n pub fn set_current_s(&mut self, current: String) {\n self.current = Cow::from(current);\n }\n\n fn replace_s(&mut self, bra: i32, ket: i32, s: &str) -> i32 {\n let adjustment = s.len() as i32 - (ket - bra);\n let mut result = String::with_capacity(self.current.len());\n {\n let (lhs, _) = self.current.split_at(bra as usize);\n let (_, rhs) = self.current.split_at(ket as usize);\n result.push_str(lhs);\n result.push_str(s);\n result.push_str(rhs);\n }\n // ... not very nice...\n let new_lim = self.limit + adjustment;\n self.limit = new_lim;\n if self.cursor >= ket {\n let new_cur = self.cursor + adjustment;\n self.cursor = new_cur;\n } else if self.cursor > bra {\n self.cursor = bra\n }\n self.current = Cow::from(result);\n adjustment\n }\n\n /// Check if s is after cursor.\n /// If so, move cursor to the end of s\n pub fn eq_s(&mut self, s: &str) -> bool {\n if self.cursor >= self.limit {\n return false;\n }\n if self.current[(self.cursor as usize)..].starts_with(s) {\n self.cursor += s.len() as i32;\n while !self.current.is_char_boundary(self.cursor as usize) {\n self.cursor += 1;\n }\n true\n } else {\n false\n }\n }\n\n /// Check if 's' is before cursor\n /// If so, move cursor to the beginning of s\n pub fn eq_s_b(&mut self, s: &str) -> bool {\n if (self.cursor - self.limit_backward) < s.len() as i32 {\n false\n // Check if cursor -s.len is a char boundary. if not well... return false obv\n } else if !self.current.is_char_boundary(self.cursor as usize - s.len()) ||\n !self.current[self.cursor as usize - s.len()..].starts_with(s) {\n false\n } else {\n self.cursor -= s.len() as i32;\n true\n }\n }\n\n /// Replace string between `bra` and `ket` with s\n pub fn slice_from(&mut self, s: &str) -> bool {\n let (bra, ket) = (self.bra, self.ket);\n self.replace_s(bra, ket, s);\n true\n }\n\n /// Move cursor to next character\n pub fn next_char(&mut self) {\n self.cursor += 1;\n while !self.current.is_char_boundary(self.cursor as usize) {\n self.cursor += 1;\n }\n }\n\n /// Move cursor to previous character\n pub fn previous_char(&mut self) {\n self.cursor -= 1;\n while !self.current.is_char_boundary(self.cursor as usize) {\n self.cursor -= 1;\n }\n }\n\n pub fn hop(&mut self, mut delta: i32) -> bool {\n let mut res = self.cursor;\n while delta > 0 {\n delta -= 1;\n if res >= self.limit {\n return false;\n }\n res += 1;\n while res < self.limit && !self.current.is_char_boundary(res as usize) {\n res += 1;\n }\n }\n self.cursor = res;\n return true;\n }\n\n pub fn hop_checked(&mut self, delta: i32) -> bool {\n return delta >= 0 && self.hop(delta);\n }\n\n pub fn hop_back(&mut self, mut delta: i32) -> bool {\n let mut res = self.cursor;\n while delta > 0 {\n delta -= 1;\n if res <= self.limit_backward {\n return false;\n }\n res -= 1;\n while res > self.limit_backward && !self.current.is_char_boundary(res as usize) {\n res -= 1;\n }\n }\n self.cursor = res;\n return true;\n }\n\n pub fn hop_back_checked(&mut self, delta: i32) -> bool {\n return delta >= 0 && self.hop_back(delta);\n }\n\n // A grouping is represented by a minimum code point, a maximum code point,\n // and a bitfield of which code points in that range are in the grouping.\n // For example, in english.sbl, valid_LI is 'cdeghkmnrt'.\n // The minimum and maximum code points are 99 and 116,\n // so every time one of these grouping functions is called for g_valid_LI,\n // min must be 99 and max must be 116. There are 18 code points within that\n // range (inclusive) so the grouping is represented with 18 bits, plus 6 bits of padding:\n //\n // cdefghij klmnopqr st\n // 11101100 10110001 01000000\n //\n // The first bit is the least significant.\n // Those three bytes become &[0b00110111, 0b10001101, 0b00000010],\n // which is &[55, 141, 2], which is how g_valid_LI is defined in english.rs.\n /// Check if the char the cursor points to is in the grouping\n pub fn in_grouping(&mut self, chars: &[u8], min: u32, max: u32) -> bool {\n if self.cursor >= self.limit {\n return false;\n }\n if let Some(chr) = self.current[self.cursor as usize..].chars().next() {\n let mut ch = chr as u32; //codepoint as integer\n if ch > max || ch < min {\n return false;\n }\n ch -= min;\n if (chars[(ch >> 3) as usize] & (0x1 << (ch & 0x7))) == 0 {\n return false;\n }\n self.next_char();\n return true;\n }\n return false;\n }\n\n pub fn in_grouping_b(&mut self, chars: &[u8], min: u32, max: u32) -> bool {\n if self.cursor <= self.limit_backward {\n return false;\n }\n self.previous_char();\n if let Some(chr) = self.current[self.cursor as usize..].chars().next() {\n let mut ch = chr as u32; //codepoint as integer\n self.next_char();\n if ch > max || ch < min {\n return false;\n }\n ch -= min;\n if (chars[(ch >> 3) as usize] & (0x1 << (ch & 0x7))) == 0 {\n return false;\n }\n self.previous_char();\n return true;\n }\n return false;\n }\n\n pub fn out_grouping(&mut self, chars: &[u8], min: u32, max: u32) -> bool {\n if self.cursor >= self.limit {\n return false;\n }\n if let Some(chr) = self.current[self.cursor as usize..].chars().next() {\n let mut ch = chr as u32; //codepoint as integer\n if ch > max || ch < min {\n self.next_char();\n return true;\n }\n ch -= min;\n if (chars[(ch >> 3) as usize] & (0x1 << (ch & 0x7))) == 0 {\n self.next_char();\n return true;\n }\n }\n return false;\n }\n\n pub fn out_grouping_b(&mut self, chars: &[u8], min: u32, max: u32) -> bool {\n if self.cursor <= self.limit_backward {\n return false;\n }\n self.previous_char();\n if let Some(chr) = self.current[self.cursor as usize..].chars().next() {\n let mut ch = chr as u32; //codepoint as integer\n self.next_char();\n if ch > max || ch < min {\n self.previous_char();\n return true;\n }\n ch -= min;\n if (chars[(ch >> 3) as usize] & (0x1 << (ch & 0x7))) == 0 {\n self.previous_char();\n return true;\n }\n }\n return false;\n\n }\n\n\n /// Helper function that removes the string slice between `bra` and `ket`\n pub fn slice_del(&mut self) -> bool {\n self.slice_from(\"\")\n }\n\n pub fn insert(&mut self, bra: i32, ket: i32, s: &str) {\n let adjustment = self.replace_s(bra, ket, s);\n if bra <= self.bra {\n self.bra = self.bra + adjustment;\n }\n if bra <= self.ket {\n self.ket = self.ket + adjustment;\n }\n }\n\n pub fn assign_to(&mut self) -> String {\n self.current[0..self.limit as usize].to_string()\n }\n\n pub fn slice_to(&mut self) -> String {\n self.current[self.bra as usize..self.ket as usize].to_string()\n }\n\n pub fn find_among(&mut self, amongs: &[Among], context: &mut T) -> i32 {\n use std::cmp::min;\n let mut i: i32 = 0;\n let mut j: i32 = amongs.len() as i32;\n\n let c = self.cursor;\n let l = self.limit;\n\n let mut common_i = 0i32;\n let mut common_j = 0i32;\n\n let mut first_key_inspected = false;\n loop {\n let k = i + ((j - i) >> 1);\n let mut diff: i32 = 0;\n let mut common = min(common_i, common_j);\n let w = &amongs[k as usize];\n for lvar in common..w.0.len() as i32 {\n if c + common == l {\n diff = -1;\n break;\n }\n diff = self.current.as_bytes()[(c + common) as usize] as i32 - w.0.as_bytes()[lvar as usize] as i32;\n if diff != 0 {\n break;\n }\n common += 1;\n }\n if diff < 0 {\n j = k;\n common_j = common;\n } else {\n i = k;\n common_i = common;\n }\n if j - i <= 1 {\n if i > 0 {\n break;\n }\n if j == i {\n break;\n }\n if first_key_inspected {\n break;\n }\n first_key_inspected = true;\n }\n }\n\n loop {\n let w = &amongs[i as usize];\n if common_i >= w.0.len() as i32{\n self.cursor = c + w.0.len() as i32;\n if let Some(ref method) = w.3 {\n let res = method(self, context);\n self.cursor = c + w.0.len() as i32;\n if res {\n return w.2;\n }\n } else {\n return w.2;\n }\n }\n i = w.1;\n if i < 0 {\n return 0;\n }\n }\n }\n\n pub fn find_among_b(&mut self, amongs: &[Among], context: &mut T) -> i32 {\n let mut i: i32 = 0;\n let mut j: i32 = amongs.len() as i32;\n\n let c = self.cursor;\n let lb = self.limit_backward;\n\n let mut common_i = 0i32;\n let mut common_j = 0i32;\n\n let mut first_key_inspected = false;\n\n loop {\n let k = i + ((j - i) >> 1);\n let mut diff: i32 = 0;\n let mut common = if common_i < common_j {\n common_i\n } else {\n common_j\n };\n let w = &amongs[k as usize];\n for lvar in (0..w.0.len() - common as usize).rev() {\n if c - common == lb {\n diff = -1;\n break;\n }\n diff = self.current.as_bytes()[(c - common - 1) as usize] as i32 - w.0.as_bytes()[lvar] as i32;\n if diff != 0 {\n break;\n }\n // Count up commons. But not one character but the byte width of that char\n common += 1;\n }\n if diff < 0 {\n j = k;\n common_j = common;\n } else {\n i = k;\n common_i = common;\n }\n if j - i <= 1 {\n if i > 0 {\n break;\n }\n if j == i {\n break;\n }\n if first_key_inspected {\n break;\n }\n first_key_inspected = true;\n }\n }\n loop {\n let w = &amongs[i as usize];\n if common_i >= w.0.len() as i32 {\n self.cursor = c - w.0.len() as i32;\n if let Some(ref method) = w.3 {\n let res = method(self, context);\n self.cursor = c - w.0.len() as i32;\n if res {\n return w.2;\n }\n } else {\n return w.2;\n }\n }\n i = w.1;\n if i < 0 {\n return 0;\n }\n }\n }\n}\n" } ]