Repository: dalance/ptags Branch: master Commit: 24192b9ae17b Files: 21 Total size: 45.2 KB Directory structure: gitextract_rchttwyl/ ├── .cargo/ │ └── config ├── .gitattributes ├── .github/ │ ├── FUNDING.yml │ ├── dependabot.yml │ └── workflows/ │ ├── dependabot_merge.yml │ ├── periodic.yml │ ├── regression.yml │ └── release.yml ├── .gitignore ├── .gitmodules ├── Cargo.toml ├── LICENSE ├── Makefile ├── README.md ├── benches/ │ └── ptags_bench.rs ├── src/ │ ├── bin.rs │ ├── cmd_ctags.rs │ ├── cmd_git.rs │ ├── lib.rs │ └── main.rs └── test/ └── lfs.txt ================================================ FILE CONTENTS ================================================ ================================================ FILE: .cargo/config ================================================ [target.x86_64-pc-windows-gnu] linker = "x86_64-w64-mingw32-gcc" [target.i686-pc-windows-gnu] linker = "i686-w64-mingw32-gcc" ================================================ FILE: .gitattributes ================================================ test/lfs.txt filter=lfs diff=lfs merge=lfs -text ================================================ FILE: .github/FUNDING.yml ================================================ # These are supported funding model platforms github: dalance ================================================ FILE: .github/dependabot.yml ================================================ version: 2 updates: - package-ecosystem: cargo directory: "/" schedule: interval: daily time: "20:00" open-pull-requests-limit: 10 ================================================ FILE: .github/workflows/dependabot_merge.yml ================================================ name: Dependabot auto-merge on: pull_request_target permissions: pull-requests: write contents: write jobs: dependabot: runs-on: ubuntu-latest if: ${{ github.actor == 'dependabot[bot]' }} steps: - name: Dependabot metadata id: metadata uses: dependabot/fetch-metadata@v2.2.0 with: github-token: '${{ secrets.GITHUB_TOKEN }}' - name: Enable auto-merge for Dependabot PRs if: ${{ steps.metadata.outputs.update-type == 'version-update:semver-patch' || ( !startsWith( steps.metadata.outputs.new-version, '0.' ) && steps.metadata.outputs.update-type == 'version-update:semver-minor' ) }} run: gh pr merge --auto --merge "$PR_URL" env: PR_URL: ${{github.event.pull_request.html_url}} GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}} ================================================ FILE: .github/workflows/periodic.yml ================================================ name: Periodic on: schedule: - cron: 0 0 * * SUN jobs: build: strategy: matrix: os: [ubuntu-latest] rust: [stable, beta, nightly] runs-on: ${{ matrix.os }} steps: - name: Setup Rust uses: hecrj/setup-rust-action@v1 with: rust-version: ${{ matrix.rust }} - name: Install ctags on Linux if: matrix.os == 'ubuntu-latest' run: | sudo apt-get update sudo apt-get install universal-ctags - name: Checkout uses: actions/checkout@v1 - name: git submodule init run: | git submodule init git submodule update - name: Run tests run: cargo test -- --test-threads=1 ================================================ FILE: .github/workflows/regression.yml ================================================ name: Regression on: push: branches: - master pull_request: jobs: build: strategy: matrix: os: [ubuntu-latest, macOS-latest] rust: [stable] runs-on: ${{ matrix.os }} steps: - name: Setup Rust uses: hecrj/setup-rust-action@v1 with: rust-version: ${{ matrix.rust }} - name: Install ctags on Linux if: matrix.os == 'ubuntu-latest' run: | sudo apt-get update sudo apt-get install universal-ctags - name: Install ctags on macOS if: matrix.os == 'macOS-latest' run: | brew update brew install universal-ctags brew install git-lfs - name: Checkout uses: actions/checkout@v1 - name: git submodule init run: | git submodule init git submodule update - name: Run tests run: cargo test -- --test-threads=1 ================================================ FILE: .github/workflows/release.yml ================================================ name: Release on: workflow_dispatch: push: tags: - 'v*.*.*' jobs: build: strategy: matrix: os: [ubuntu-latest, macOS-latest, windows-latest] rust: [stable] runs-on: ${{ matrix.os }} steps: - name: Setup Rust uses: hecrj/setup-rust-action@v1 with: rust-version: ${{ matrix.rust }} - name: Checkout uses: actions/checkout@v1 - name: Setup MUSL if: matrix.os == 'ubuntu-latest' run: | rustup target add x86_64-unknown-linux-musl sudo apt-get -qq install musl-tools - name: Setup Target if: matrix.os == 'macOS-latest' run: | rustup target add aarch64-apple-darwin - name: Build for Linux if: matrix.os == 'ubuntu-latest' run: make release_lnx - name: Build for macOS if: matrix.os == 'macOS-latest' run: make release_mac - name: Build for Windows if: matrix.os == 'windows-latest' run: make release_win - name: Upload artifacts uses: actions/upload-artifact@v3 with: name: ptags path: '*.zip' - name: Release if: github.event_name == 'push' && github.ref_type == 'tag' uses: softprops/action-gh-release@v1 with: files: '*.zip' env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} ================================================ FILE: .gitignore ================================================ # Generated by Cargo # will have compiled files and executables /target/ # Remove Cargo.lock from gitignore if creating an executable, leave it for libraries # More information here http://doc.crates.io/guide.html#cargotoml-vs-cargolock #Cargo.lock # These are backup files generated by rustfmt **/*.rs.bk data/ tags *.zip *.gz ================================================ FILE: .gitmodules ================================================ [submodule "test/ptags_test"] path = test/ptags_test url = https://github.com/dalance/ptags_test.git ================================================ FILE: Cargo.toml ================================================ [package] name = "ptags" version = "0.3.5" authors = ["dalance@gmail.com"] repository = "https://github.com/dalance/ptags" keywords = ["ctags", "universal-ctags"] categories = ["command-line-utilities", "development-tools"] license = "MIT" readme = "README.md" description = "A parallel universal-ctags wrapper for git repository" edition = "2018" [badges] travis-ci = { repository = "dalance/ptags" } appveyor = { repository = "dalance/ptags", branch = "master", service = "github" } codecov = { repository = "dalance/ptags", branch = "master", service = "github" } [dependencies] anyhow = "1.0" dirs = "6" nix = { version = "0.31.3", features = ["fs"] } serde = "1" serde_derive = "1" structopt = "0.3" structopt-toml = "0.5" tempfile = "3" thiserror = "2.0" toml = "1.1" [dev-dependencies] bencher = "0.1" [lib] name = "ptagslib" path = "src/lib.rs" [[bin]] name = "ptags" path = "src/main.rs" [[bench]] name = "ptags_bench" harness = false [package.metadata.release] pre-release-commit-message = "Prepare to v{{version}}" post-release-commit-message = "Start next development iteration v{{version}}" tag-message = "Bump version to {{version}}" tag-prefix = "" ================================================ FILE: LICENSE ================================================ MIT License Copyright (c) 2018 dalance Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ================================================ FILE: Makefile ================================================ VERSION = $(patsubst "%",%, $(word 3, $(shell grep version Cargo.toml))) BUILD_TIME = $(shell date +"%Y/%m/%d %H:%M:%S") GIT_REVISION = $(shell git log -1 --format="%h") RUST_VERSION = $(word 2, $(shell rustc -V)) LONG_VERSION = "$(VERSION) ( rev: $(GIT_REVISION), rustc: $(RUST_VERSION), build at: $(BUILD_TIME) )" BIN_NAME = ptags export LONG_VERSION .PHONY: all test clean release_lnx release_win release_mac all: test test: cargo test -- --test-threads=1 watch: cargo watch "test -- --test-threads=1" clean: cargo clean release_lnx: cargo build --release --target=x86_64-unknown-linux-musl zip -j ${BIN_NAME}-v${VERSION}-x86_64-lnx.zip target/x86_64-unknown-linux-musl/release/${BIN_NAME} release_win: cargo build --release --target=x86_64-pc-windows-msvc 7z a ${BIN_NAME}-v${VERSION}-x86_64-win.zip target/x86_64-pc-windows-msvc/release/${BIN_NAME}.exe release_mac: cargo build --release --target=x86_64-apple-darwin zip -j ${BIN_NAME}-v${VERSION}-x86_64-mac.zip target/x86_64-apple-darwin/release/${BIN_NAME} cargo build --release --target=aarch64-apple-darwin zip -j ${BIN_NAME}-v${VERSION}-aarch64-mac.zip target/aarch64-apple-darwin/release/${BIN_NAME} ================================================ FILE: README.md ================================================ # ptags A parallel [universal-ctags](https://ctags.io) wrapper for git repository [![Actions Status](https://github.com/dalance/ptags/workflows/Regression/badge.svg)](https://github.com/dalance/ptags/actions) [![Crates.io](https://img.shields.io/crates/v/ptags.svg)](https://crates.io/crates/ptags) [![codecov](https://codecov.io/gh/dalance/ptags/branch/master/graph/badge.svg)](https://codecov.io/gh/dalance/ptags) ## Description **ptags** is a [universal-ctags](https://ctags.io) wrapper to have the following features. - Search git tracked files only ( `.gitignore` support ) - Call `ctags` command in parallel for acceleration - Up to x5 faster than universal-ctags ## Install ### Download binary Download from [release page](https://github.com/dalance/ptags/releases/latest), and extract to the directory in PATH. ### Arch Linux You can install from AUR. - https://aur.archlinux.org/packages/ptags/ - https://aur.archlinux.org/packages/ptags-git/ If you use `yay`, you can install like below: ``` yay -S ptags // latest tagged version yay -S ptags-git // current master of git repo ``` ### Cargo You can install by [cargo](https://crates.io). ``` cargo install ptags ``` ## Requirement **ptags** uses `ctags` and `git` command internally. The tested version is below. | Command | Version | | --------- | ----------------------------------------------------- | | `ctags` | Universal Ctags 0.0.0(f9e6e3c1) / Exuberant Ctags 5.8 | | `git` | git version 2.14.2 | | `git-lfs` | git-lfs/2.3.3 | ## Usage ``` ptags 0.1.12-pre dalance@gmail.com A parallel universal-ctags wrapper for git repository USAGE: ptags [FLAGS] [OPTIONS] [--] [DIR] FLAGS: --config Generate configuration sample file --exclude-lfs Exclude git-lfs tracked files -h, --help Prints help information --include-ignored Include ignored files --include-submodule Include submodule files --include-untracked Include untracked files -s, --stat Show statistics --unsorted Disable tags sort --validate-utf8 Validate UTF8 sequence of tag file -V, --version Prints version information -v, --verbose Verbose mode OPTIONS: --bin-ctags Path to ctags binary [default: ctags] --bin-git Path to git binary [default: git] --completion Generate shell completion file [possible values: bash, fish, zsh, powershell] -e, --exclude ... Glob pattern of exclude file ( ex. --exclude '*.rs' ) -c, --opt-ctags ... Options passed to ctags -g, --opt-git ... Options passed to git --opt-git-lfs ... Options passed to git-lfs -f, --file Output filename ( filename '-' means output to stdout ) [default: tags] -t, --thread Number of threads [default: 8] ARGS: Search directory [default: .] ``` You can pass options to `ctags` by`-c`/`--ctags_opt` option like below. ``` ptags -c --links=no -c --languages=Rust ``` Searched file types per options are below. `--include-submodule` and `--include_untracked` are exclusive. This is the restriction of `git ls-files`. Any include/exclude options without the above combination can be used simultaneously. | File type | Default | --exclude-lfs | --include-ignored | --include-submodule | --include-untracked | | ------------- | -------- | ------------- | ----------------- | ------------------- | ------------------- | | tracked | o | o | o | o | o | | untracked | x | x | x | x | o | | ignored | x | x | o | x | x | | lfs tracked | o | x | o | o | o | | in submodules | x | x | x | o | x | You can override any default option by `~/.ptags.toml` like below. The complete example of `~/.ptags.toml` can be generated by `--config` option. ```toml thread = 16 bin_ctags = "ctags2" bin_git = "git2" ``` ## Benchmark ### Environment - CPU: Ryzen Threadripper 1950X - MEM: 128GB - OS : CentOS 7.4.1708 ### Data | Name | Repository | Revision | Files | Size[GB] | | ------- | ------------------------------------ | ------------ | ------ | -------- | | source0 | https://github.com/neovim/neovim | f5b0f5e17 | 2370 | 0.1 | | source1 | https://github.com/llvm-mirror/llvm | ddf9edb4020 | 29670 | 1.2 | | source2 | https://github.com/torvalds/linux | 071e31e254e0 | 52998 | 2.2 | | source3 | https://github.com/chromium/chromium | d79c68510b7e | 293205 | 13 | ### Result **ptags** is up to x5 faster than universal-ctags. | Command | Version | source0 | source1 | source2 | source3 | | ------------- | ------------------------------- | --------------- | --------------- | ---------------- | --------------- | | `ctags -R` | Universal Ctags 0.0.0(f9e6e3c1) | 0.41s ( x1 ) | 3.42s ( x1 ) | 23.64s ( x1 ) | 32.23 ( x1 ) | | `ptags -t 16` | ptags 0.1.4 | 0.13s ( x3.15 ) | 0.58s ( x5.90 ) | 4.24s ( x5.58 ) | 7.27s ( x4.43 ) | ================================================ FILE: benches/ptags_bench.rs ================================================ #[macro_use] extern crate bencher; extern crate ptagslib; extern crate structopt; use bencher::Bencher; use ptagslib::bin::{run_opt, Opt}; use structopt::StructOpt; fn bench_default(bench: &mut Bencher) { bench.iter(|| { let args = vec!["ptags"]; let opt = Opt::from_iter(args.iter()); let _ = run_opt(&opt); }) } fn bench_unsorted(bench: &mut Bencher) { bench.iter(|| { let args = vec!["ptags", "--unsorted"]; let opt = Opt::from_iter(args.iter()); let _ = run_opt(&opt); }) } benchmark_group!(benches, bench_default, bench_unsorted); benchmark_main!(benches); ================================================ FILE: src/bin.rs ================================================ use crate::cmd_ctags::CmdCtags; use crate::cmd_git::CmdGit; use anyhow::{Context, Error}; use dirs; use serde_derive::{Deserialize, Serialize}; use std::fs; use std::io::BufRead; use std::io::{stdout, BufWriter, Read, Write}; use std::path::PathBuf; use std::process::Output; use std::str; use std::time::{Duration, Instant}; use structopt::{clap, StructOpt}; use structopt_toml::StructOptToml; use toml; // --------------------------------------------------------------------------------------------------------------------- // Options // --------------------------------------------------------------------------------------------------------------------- #[derive(Debug, Deserialize, Serialize, StructOpt, StructOptToml)] #[serde(default)] #[structopt(name = "ptags")] #[structopt(long_version = option_env!("LONG_VERSION").unwrap_or(env!("CARGO_PKG_VERSION")))] #[structopt(setting = clap::AppSettings::AllowLeadingHyphen)] #[structopt(setting = clap::AppSettings::ColoredHelp)] pub struct Opt { /// Number of threads #[structopt(short = "t", long = "thread", default_value = "8")] pub thread: usize, /// Output filename ( filename '-' means output to stdout ) #[structopt(short = "f", long = "file", default_value = "tags", parse(from_os_str))] pub output: PathBuf, /// Search directory #[structopt(name = "DIR", default_value = ".", parse(from_os_str))] pub dir: PathBuf, /// Show statistics #[structopt(short = "s", long = "stat")] pub stat: bool, /// Filename of input file list #[structopt(short = "L", long = "list")] pub list: Option, /// Path to ctags binary #[structopt(long = "bin-ctags", default_value = "ctags", parse(from_os_str))] pub bin_ctags: PathBuf, /// Path to git binary #[structopt(long = "bin-git", default_value = "git", parse(from_os_str))] pub bin_git: PathBuf, /// Options passed to ctags #[structopt(short = "c", long = "opt-ctags", number_of_values = 1)] pub opt_ctags: Vec, /// Options passed to git #[structopt(short = "g", long = "opt-git", number_of_values = 1)] pub opt_git: Vec, /// Options passed to git-lfs #[structopt(long = "opt-git-lfs", number_of_values = 1)] pub opt_git_lfs: Vec, /// Verbose mode #[structopt(short = "v", long = "verbose")] pub verbose: bool, /// Exclude git-lfs tracked files #[structopt(long = "exclude-lfs")] pub exclude_lfs: bool, /// Include untracked files #[structopt(long = "include-untracked")] pub include_untracked: bool, /// Include ignored files #[structopt(long = "include-ignored")] pub include_ignored: bool, /// Include submodule files #[structopt(long = "include-submodule")] pub include_submodule: bool, /// Validate UTF8 sequence of tag file #[structopt(long = "validate-utf8")] pub validate_utf8: bool, /// Disable tags sort #[structopt(long = "unsorted")] pub unsorted: bool, /// Glob pattern of exclude file ( ex. --exclude '*.rs' ) #[structopt(short = "e", long = "exclude", number_of_values = 1)] pub exclude: Vec, /// Generate shell completion file #[structopt( long = "completion", possible_values = &["bash", "fish", "zsh", "powershell"] )] pub completion: Option, /// Generate configuration sample file #[structopt(long = "config")] pub config: bool, } // --------------------------------------------------------------------------------------------------------------------- // Functions // --------------------------------------------------------------------------------------------------------------------- macro_rules! watch_time ( ( $func:block ) => ( { let beg = Instant::now(); $func; Instant::now() - beg } ); ); pub fn git_files(opt: &Opt) -> Result, Error> { let list = CmdGit::get_files(&opt)?; let mut files = vec![String::from(""); opt.thread]; for (i, f) in list.iter().enumerate() { files[i % opt.thread].push_str(f); files[i % opt.thread].push_str("\n"); } Ok(files) } pub fn input_files(file: &String, opt: &Opt) -> Result, Error> { let mut list = Vec::new(); if file == &String::from("-") { let stdin = std::io::stdin(); for line in stdin.lock().lines() { list.push(String::from(line?)); } } else { for line in fs::read_to_string(file)?.lines() { list.push(String::from(line)); } } let mut files = vec![String::from(""); opt.thread]; for (i, f) in list.iter().enumerate() { files[i % opt.thread].push_str(f); files[i % opt.thread].push_str("\n"); } Ok(files) } fn call_ctags(opt: &Opt, files: &[String]) -> Result, Error> { Ok(CmdCtags::call(&opt, &files)?) } fn get_tags_header(opt: &Opt) -> Result { Ok(CmdCtags::get_tags_header(&opt).context("failed to get ctags header")?) } fn write_tags(opt: &Opt, outputs: &[Output]) -> Result<(), Error> { let mut iters = Vec::new(); let mut lines = Vec::new(); for o in outputs { let mut iter = if opt.validate_utf8 { str::from_utf8(&o.stdout)?.lines() } else { unsafe { str::from_utf8_unchecked(&o.stdout).lines() } }; lines.push(iter.next()); iters.push(iter); } let mut f = if opt.output.to_str().unwrap_or("") == "-" { BufWriter::new(Box::new(stdout()) as Box) } else { let f = fs::File::create(&opt.output)?; BufWriter::new(Box::new(f) as Box) }; f.write(get_tags_header(&opt)?.as_bytes())?; while lines.iter().any(|x| x.is_some()) { let mut min = 0; for i in 1..lines.len() { if opt.unsorted { if !lines[i].is_none() && lines[min].is_none() { min = i; } } else { if !lines[i].is_none() && (lines[min].is_none() || lines[i].unwrap() < lines[min].unwrap()) { min = i; } } } f.write(lines[min].unwrap().as_bytes())?; f.write("\n".as_bytes())?; lines[min] = iters[min].next(); } Ok(()) } // --------------------------------------------------------------------------------------------------------------------- // Run // --------------------------------------------------------------------------------------------------------------------- pub fn run_opt(opt: &Opt) -> Result<(), Error> { if opt.config { let toml = toml::to_string(&opt)?; println!("{}", toml); return Ok(()); } match opt.completion { Some(ref x) => { let shell = match x.as_str() { "bash" => clap::Shell::Bash, "fish" => clap::Shell::Fish, "zsh" => clap::Shell::Zsh, "powershell" => clap::Shell::PowerShell, _ => clap::Shell::Bash, }; Opt::clap().gen_completions("ptags", shell, "./"); return Ok(()); } None => {} } let files; let time_git_files; if let Some(ref list) = opt.list { files = input_files(list, &opt).context("failed to get file list")?; time_git_files = Duration::from_secs(0); } else { time_git_files = watch_time!({ files = git_files(&opt).context("failed to get file list")?; }); } let outputs; let time_call_ctags = watch_time!({ outputs = call_ctags(&opt, &files).context("failed to call ctags")?; }); let time_write_tags = watch_time!({ let _ = write_tags(&opt, &outputs) .context(format!("failed to write file ({:?})", &opt.output))?; }); if opt.stat { let sum: usize = files.iter().map(|x| x.lines().count()).sum(); eprintln!("\nStatistics"); eprintln!("- Options"); eprintln!(" thread : {}\n", opt.thread); eprintln!("- Searched files"); eprintln!(" total : {}\n", sum); eprintln!("- Elapsed time[ms]"); eprintln!(" git_files : {}", time_git_files.as_millis()); eprintln!(" call_ctags: {}", time_call_ctags.as_millis()); eprintln!(" write_tags: {}", time_write_tags.as_millis()); } Ok(()) } pub fn run() -> Result<(), Error> { let cfg_path = match dirs::home_dir() { Some(mut path) => { path.push(".ptags.toml"); if path.exists() { Some(path) } else { None } } None => None, }; let opt = match cfg_path { Some(path) => { let mut f = fs::File::open(&path).context(format!("failed to open file ({:?})", path))?; let mut s = String::new(); let _ = f.read_to_string(&mut s); Opt::from_args_with_toml(&s).context(format!("failed to parse toml ({:?})", path))? } None => Opt::from_args(), }; run_opt(&opt) } // --------------------------------------------------------------------------------------------------------------------- // Test // --------------------------------------------------------------------------------------------------------------------- #[cfg(test)] mod tests { use super::*; use std::path::Path; #[test] fn test_run() { let args = vec!["ptags"]; let opt = Opt::from_iter(args.iter()); let ret = run_opt(&opt); assert!(ret.is_ok()); } #[test] fn test_run_opt() { let args = vec!["ptags", "-s", "-v", "--validate-utf8", "--unsorted"]; let opt = Opt::from_iter(args.iter()); let ret = run_opt(&opt); assert!(ret.is_ok()); } #[test] fn test_run_fail() { let args = vec!["ptags", "--bin-git", "aaa"]; let opt = Opt::from_iter(args.iter()); let ret = run_opt(&opt); assert_eq!( &format!("{:?}", ret)[0..42], "Err(failed to get file list\n\nCaused by:\n " ); } #[test] fn test_run_completion() { let args = vec!["ptags", "--completion", "bash"]; let opt = Opt::from_iter(args.iter()); let ret = run_opt(&opt); assert!(ret.is_ok()); let args = vec!["ptags", "--completion", "fish"]; let opt = Opt::from_iter(args.iter()); let ret = run_opt(&opt); assert!(ret.is_ok()); let args = vec!["ptags", "--completion", "zsh"]; let opt = Opt::from_iter(args.iter()); let ret = run_opt(&opt); assert!(ret.is_ok()); let args = vec!["ptags", "--completion", "powershell"]; let opt = Opt::from_iter(args.iter()); let ret = run_opt(&opt); assert!(ret.is_ok()); assert!(Path::new("ptags.bash").exists()); assert!(Path::new("ptags.fish").exists()); assert!(Path::new("_ptags").exists()); assert!(Path::new("_ptags.ps1").exists()); let _ = fs::remove_file("ptags.bash"); let _ = fs::remove_file("ptags.fish"); let _ = fs::remove_file("_ptags"); let _ = fs::remove_file("_ptags.ps1"); } #[test] fn test_run_config() { let args = vec!["ptags", "--config"]; let opt = Opt::from_iter(args.iter()); let ret = run_opt(&opt); assert!(ret.is_ok()); } } ================================================ FILE: src/cmd_ctags.rs ================================================ use crate::bin::Opt; use anyhow::{bail, Context, Error}; #[cfg(target_os = "linux")] use nix::fcntl::{fcntl, FcntlArg}; use std::fs; use std::fs::File; use std::io::{BufReader, Read, Write}; use std::path::PathBuf; use std::process::{ChildStdin, Command, Output, Stdio}; use std::str; use std::sync::mpsc; use std::thread; use tempfile::NamedTempFile; use thiserror::Error; // --------------------------------------------------------------------------------------------------------------------- // Error // --------------------------------------------------------------------------------------------------------------------- #[derive(Debug, Error)] enum CtagsError { #[error("failed to execute ctags command ({})\n{}", cmd, err)] ExecFailed { cmd: String, err: String }, #[error("failed to call ctags command ({})", cmd)] CallFailed { cmd: String }, #[error("failed to convert to UTF-8 ({:?})", s)] ConvFailed { s: Vec }, } // --------------------------------------------------------------------------------------------------------------------- // CmdCtags // --------------------------------------------------------------------------------------------------------------------- pub struct CmdCtags; impl CmdCtags { pub fn call(opt: &Opt, files: &[String]) -> Result, Error> { let mut args = Vec::new(); args.push(String::from("-L -")); args.push(String::from("-f -")); if opt.unsorted { args.push(String::from("--sort=no")); } for e in &opt.exclude { args.push(String::from(format!("--exclude={}", e))); } args.append(&mut opt.opt_ctags.clone()); let cmd = CmdCtags::get_cmd(&opt, &args); let (tx, rx) = mpsc::channel::>(); for i in 0..opt.thread { let tx = tx.clone(); let file = files[i].clone(); let dir = opt.dir.clone(); let bin_ctags = opt.bin_ctags.clone(); let args = args.clone(); let cmd = cmd.clone(); if opt.verbose { eprintln!("Call : {}", cmd); } thread::spawn(move || { let child = Command::new(bin_ctags.clone()) .args(args) .current_dir(dir) .stdin(Stdio::piped()) .stdout(Stdio::piped()) //.stderr(Stdio::piped()) // Stdio::piped is x2 slow to wait_with_output() completion .stderr(Stdio::null()) .spawn(); match child { Ok(mut x) => { { let stdin = x.stdin.as_mut().unwrap(); let pipe_size = std::cmp::min(file.len() as i32, 1048576); let _ = CmdCtags::set_pipe_size(&stdin, pipe_size) .or_else(|x| tx.send(Err(x.into()))); let _ = stdin.write_all(file.as_bytes()); } match x.wait_with_output() { Ok(x) => { let _ = tx.send(Ok(x)); } Err(x) => { let _ = tx.send(Err(x.into())); } } } Err(_) => { let _ = tx.send(Err(CtagsError::CallFailed { cmd }.into())); } } }); } let mut children = Vec::new(); for _ in 0..opt.thread { children.push(rx.recv()); } let mut outputs = Vec::new(); for child in children { let output = child??; if !output.status.success() { bail!(CtagsError::ExecFailed { cmd: cmd, err: String::from(str::from_utf8(&output.stderr).context( CtagsError::ConvFailed { s: output.stderr.to_vec(), } )?) }); } outputs.push(output); } Ok(outputs) } pub fn get_tags_header(opt: &Opt) -> Result { let tmp_empty = NamedTempFile::new()?; let tmp_tags = NamedTempFile::new()?; let tmp_tags_path: PathBuf = tmp_tags.path().into(); // In windiws environment, write access by ctags to the opened tmp_tags fails. // So the tmp_tags must be closed and deleted. tmp_tags.close()?; let _ = Command::new(&opt.bin_ctags) .arg(format!("-L {}", tmp_empty.path().to_string_lossy())) .arg(format!("-f {}", tmp_tags_path.to_string_lossy())) .args(&opt.opt_ctags) .current_dir(&opt.dir) .status(); let mut f = BufReader::new(File::open(&tmp_tags_path)?); let mut s = String::new(); f.read_to_string(&mut s)?; fs::remove_file(&tmp_tags_path)?; Ok(s) } fn get_cmd(opt: &Opt, args: &[String]) -> String { let mut cmd = format!( "cd {}; {}", opt.dir.to_string_lossy(), opt.bin_ctags.to_string_lossy() ); for arg in args { cmd = format!("{} {}", cmd, arg); } cmd } #[allow(dead_code)] fn is_exuberant_ctags(opt: &Opt) -> Result { let output = Command::new(&opt.bin_ctags) .arg("--version") .current_dir(&opt.dir) .output()?; Ok(str::from_utf8(&output.stdout)?.starts_with("Exuberant Ctags")) } #[cfg(target_os = "linux")] fn set_pipe_size(stdin: &ChildStdin, len: i32) -> Result<(), Error> { fcntl(stdin, FcntlArg::F_SETPIPE_SZ(len))?; Ok(()) } #[cfg(not(target_os = "linux"))] fn set_pipe_size(_stdin: &ChildStdin, _len: i32) -> Result<(), Error> { Ok(()) } } // --------------------------------------------------------------------------------------------------------------------- // Test // --------------------------------------------------------------------------------------------------------------------- #[cfg(test)] mod tests { use super::super::bin::{git_files, Opt}; use super::CmdCtags; use std::str; use structopt::StructOpt; #[test] fn test_call() { let args = vec!["ptags", "-t", "1", "--exclude=README.md"]; let opt = Opt::from_iter(args.iter()); let files = git_files(&opt).unwrap(); let outputs = CmdCtags::call(&opt, &files).unwrap(); let mut iter = str::from_utf8(&outputs[0].stdout).unwrap().lines(); assert_eq!( iter.next().unwrap_or(""), "BIN_NAME\tMakefile\t/^BIN_NAME = ptags$/;\"\tm" ); } #[test] fn test_call_with_opt() { let args = vec!["ptags", "-t", "1", "--opt-ctags=-u"]; let opt = Opt::from_iter(args.iter()); let files = git_files(&opt).unwrap(); let outputs = CmdCtags::call(&opt, &files).unwrap(); let mut iter = str::from_utf8(&outputs[0].stdout).unwrap().lines(); assert_eq!( iter.next().unwrap_or(""), "VERSION\tMakefile\t/^VERSION = $(patsubst \"%\",%, $(word 3, $(shell grep version Cargo.toml)))$/;\"\tm" ); } #[test] fn test_call_exclude() { let args = vec![ "ptags", "-t", "1", "--exclude=Make*", "--exclude=README.md", "-v", ]; let opt = Opt::from_iter(args.iter()); let files = git_files(&opt).unwrap(); let outputs = CmdCtags::call(&opt, &files).unwrap(); let mut iter = str::from_utf8(&outputs[0].stdout).unwrap().lines(); // Exuberant Ctags doesn't support Rust ( *.rs ). // So the result becomes empty when 'Makefile' is excluded. if CmdCtags::is_exuberant_ctags(&opt).unwrap() { assert_eq!(iter.next().unwrap_or(""), ""); } else { assert_eq!( iter.next().unwrap_or(""), "CallFailed\tsrc/cmd_ctags.rs\t/^ CallFailed { cmd: String },$/;\"\te\tenum:CtagsError" ); } } #[test] fn test_command_fail() { let args = vec!["ptags", "--bin-ctags", "aaa"]; let opt = Opt::from_iter(args.iter()); let files = git_files(&opt).unwrap(); let outputs = CmdCtags::call(&opt, &files); assert_eq!( &format!("{:?}", outputs), "Err(failed to call ctags command (cd .; aaa -L - -f -))" ); } #[test] fn test_ctags_fail() { let args = vec!["ptags", "--opt-ctags=--u"]; let opt = Opt::from_iter(args.iter()); let files = git_files(&opt).unwrap(); let outputs = CmdCtags::call(&opt, &files); assert_eq!( &format!("{:?}", outputs)[0..60], "Err(failed to execute ctags command (cd .; ctags -L - -f - -" ); } #[test] fn test_get_tags_header() { let args = vec!["ptags"]; let opt = Opt::from_iter(args.iter()); let output = CmdCtags::get_tags_header(&opt).unwrap(); let output = output.lines().next(); assert_eq!(&output.unwrap_or("")[0..5], "!_TAG"); } } ================================================ FILE: src/cmd_git.rs ================================================ use crate::bin::Opt; use anyhow::{bail, Context, Error}; use std::process::{Command, Output}; use std::str; use thiserror::Error; // --------------------------------------------------------------------------------------------------------------------- // Error // --------------------------------------------------------------------------------------------------------------------- #[derive(Debug, Error)] enum GitError { #[error("failed to execute git command ({})\n{}", cmd, err)] ExecFailed { cmd: String, err: String }, #[error("failed to call git command ({})", cmd)] CallFailed { cmd: String }, #[error("failed to convert to UTF-8 ({:?})", s)] ConvFailed { s: Vec }, } // --------------------------------------------------------------------------------------------------------------------- // CmdGit // --------------------------------------------------------------------------------------------------------------------- pub struct CmdGit; impl CmdGit { pub fn get_files(opt: &Opt) -> Result, Error> { let mut list = CmdGit::ls_files(&opt)?; if opt.exclude_lfs { let lfs_list = CmdGit::lfs_ls_files(&opt)?; let mut new_list = Vec::new(); for l in list { if !lfs_list.contains(&l) { new_list.push(l); } } list = new_list; } Ok(list) } fn call(opt: &Opt, args: &[String]) -> Result { let cmd = CmdGit::get_cmd(&opt, &args); if opt.verbose { eprintln!("Call : {}", cmd); } let output = Command::new(&opt.bin_git) .args(args) .current_dir(&opt.dir) .output() .context(GitError::CallFailed { cmd: cmd.clone() })?; if !output.status.success() { bail!(GitError::ExecFailed { cmd: cmd, err: String::from(str::from_utf8(&output.stderr).context( GitError::ConvFailed { s: output.stderr.to_vec(), } )?) }); } Ok(output) } fn ls_files(opt: &Opt) -> Result, Error> { let mut args = vec![String::from("ls-files")]; args.push(String::from("--cached")); args.push(String::from("--exclude-standard")); if opt.include_submodule { args.push(String::from("--recurse-submodules")); } else if opt.include_untracked { args.push(String::from("--other")); } else if opt.include_ignored { args.push(String::from("--ignored")); args.push(String::from("--other")); } args.append(&mut opt.opt_git.clone()); let output = CmdGit::call(&opt, &args)?; let list = str::from_utf8(&output.stdout) .context(GitError::ConvFailed { s: output.stdout.to_vec(), })? .lines(); let mut ret = Vec::new(); for l in list { ret.push(String::from(l)); } ret.sort(); if opt.verbose { eprintln!("Files: {}", ret.len()); } Ok(ret) } fn lfs_ls_files(opt: &Opt) -> Result, Error> { let mut args = vec![String::from("lfs"), String::from("ls-files")]; args.append(&mut opt.opt_git_lfs.clone()); let output = CmdGit::call(&opt, &args)?; let cdup = CmdGit::show_cdup(&opt)?; let prefix = CmdGit::show_prefix(&opt)?; let list = str::from_utf8(&output.stdout) .context(GitError::ConvFailed { s: output.stdout.to_vec(), })? .lines(); let mut ret = Vec::new(); for l in list { let mut path = String::from(l.split(' ').nth(2).unwrap_or("")); if path.starts_with(&prefix) { path = path.replace(&prefix, ""); } else { path = format!("{}{}", cdup, path); } ret.push(path); } ret.sort(); Ok(ret) } fn show_cdup(opt: &Opt) -> Result { let args = vec![String::from("rev-parse"), String::from("--show-cdup")]; let output = CmdGit::call(&opt, &args)?; let mut list = str::from_utf8(&output.stdout) .context(GitError::ConvFailed { s: output.stdout.to_vec(), })? .lines(); Ok(String::from(list.next().unwrap_or(""))) } fn show_prefix(opt: &Opt) -> Result { let args = vec![String::from("rev-parse"), String::from("--show-prefix")]; let output = CmdGit::call(&opt, &args)?; let mut list = str::from_utf8(&output.stdout) .context(GitError::ConvFailed { s: output.stdout.to_vec(), })? .lines(); Ok(String::from(list.next().unwrap_or(""))) } fn get_cmd(opt: &Opt, args: &[String]) -> String { let mut cmd = format!( "cd {}; {}", opt.dir.to_string_lossy(), opt.bin_git.to_string_lossy() ); for arg in args { cmd = format!("{} {}", cmd, arg); } cmd } } // --------------------------------------------------------------------------------------------------------------------- // Test // --------------------------------------------------------------------------------------------------------------------- #[cfg(test)] mod tests { use super::CmdGit; use crate::bin::Opt; use std::fs; use std::io::{BufWriter, Write}; use structopt::StructOpt; static TRACKED_FILES: [&'static str; 23] = [ ".cargo/config", ".gitattributes", ".github/FUNDING.yml", ".github/dependabot.yml", ".github/workflows/dependabot_merge.yml", ".github/workflows/periodic.yml", ".github/workflows/regression.yml", ".github/workflows/release.yml", ".gitignore", ".gitmodules", "Cargo.lock", "Cargo.toml", "LICENSE", "Makefile", "README.md", "benches/ptags_bench.rs", "src/bin.rs", "src/cmd_ctags.rs", "src/cmd_git.rs", "src/lib.rs", "src/main.rs", "test/lfs.txt", "test/ptags_test", ]; #[test] fn test_get_files() { let args = vec!["ptags"]; let opt = Opt::from_iter(args.iter()); let files = CmdGit::get_files(&opt).unwrap(); assert_eq!(files, TRACKED_FILES,); } #[test] fn test_get_files_exclude_lfs() { let args = vec!["ptags", "--exclude-lfs"]; let opt = Opt::from_iter(args.iter()); let files = CmdGit::get_files(&opt).unwrap(); let mut expect_files = Vec::new(); expect_files.extend_from_slice(&TRACKED_FILES); let idx = expect_files.binary_search(&"test/lfs.txt").unwrap(); expect_files.remove(idx); assert_eq!(files, expect_files,); } #[test] fn test_get_files_exclude_lfs_cd() { let args = vec!["ptags", "--exclude-lfs", "src"]; let opt = Opt::from_iter(args.iter()); let files = CmdGit::get_files(&opt).unwrap(); assert_eq!( files, vec!["bin.rs", "cmd_ctags.rs", "cmd_git.rs", "lib.rs", "main.rs"] ); } #[test] fn test_get_files_include_ignored() { { let mut f = BufWriter::new(fs::File::create("ignored.gz").unwrap()); let _ = f.write(b""); } let args = vec!["ptags", "--include-ignored"]; let opt = Opt::from_iter(args.iter()); let files: Vec = CmdGit::get_files(&opt) .unwrap() .into_iter() .filter(|f| !f.starts_with("target/")) .collect(); let _ = fs::remove_file("ignored.gz"); let mut expect_files = Vec::new(); expect_files.push("ignored.gz"); expect_files.push("tags"); assert_eq!(files, expect_files,); } #[test] fn test_get_files_include_submodule() { let args = vec!["ptags", "--include-submodule"]; let opt = Opt::from_iter(args.iter()); let files = CmdGit::get_files(&opt).unwrap(); let mut expect_files = Vec::new(); expect_files.extend_from_slice(&TRACKED_FILES); let idx = expect_files.binary_search(&"test/ptags_test").unwrap(); expect_files.remove(idx); expect_files.push("test/ptags_test/README.md"); assert_eq!(files, expect_files,); } #[test] fn test_get_files_include_untracked() { { let mut f = BufWriter::new(fs::File::create("tmp").unwrap()); let _ = f.write(b""); } let args = vec!["ptags", "--include-untracked"]; let opt = Opt::from_iter(args.iter()); let files = CmdGit::get_files(&opt).unwrap(); let _ = fs::remove_file("tmp"); let mut expect_files = Vec::new(); expect_files.extend_from_slice(&TRACKED_FILES); expect_files.push("tmp"); assert_eq!(files, expect_files,); } #[test] fn test_command_fail() { let args = vec!["ptags", "--bin-git", "aaa"]; let opt = Opt::from_iter(args.iter()); let files = CmdGit::ls_files(&opt); assert_eq!( &format!("{:?}", files)[0..42], "Err(failed to call git command (cd .; aaa " ); } #[test] fn test_git_fail() { let args = vec!["ptags", "--opt-git=-aaa"]; let opt = Opt::from_iter(args.iter()); let files = CmdGit::ls_files(&opt); assert_eq!( &format!("{:?}", files)[0..83], "Err(failed to execute git command (cd .; git ls-files --cached --exclude-standard -" ); } } ================================================ FILE: src/lib.rs ================================================ pub mod bin; pub mod cmd_ctags; pub mod cmd_git; ================================================ FILE: src/main.rs ================================================ use ptagslib::bin::run; // --------------------------------------------------------------------------------------------------------------------- // Main // --------------------------------------------------------------------------------------------------------------------- fn main() { match run() { Err(x) => { println!("{}", x); for x in x.chain() { println!("{}", x); } } _ => (), } } ================================================ FILE: test/lfs.txt ================================================ version https://git-lfs.github.com/spec/v1 oid sha256:17e682f060b5f8e47ea04c5c4855908b0a5ad612022260fe50e11ecb0cc0ab76 size 4