Repository: oldratlee/useful-scripts Branch: dev-3.x Commit: 8fc034deffc5 Files: 37 Total size: 166.8 KB Directory structure: gitextract_29uot6ky/ ├── .editorconfig ├── .github/ │ ├── dependabot.yml │ └── workflows/ │ ├── ci.yaml │ └── lint.yaml ├── .gitignore ├── .gitmodules ├── LICENSE ├── README.md ├── bin/ │ ├── a2l │ ├── ap │ ├── c │ ├── coat │ ├── cp-into-docker-run │ ├── echo-args │ ├── find-in-jars │ ├── rp │ ├── show-busy-java-threads │ ├── show-duplicate-java-classes │ ├── taoc │ ├── tcp-connection-state-counter │ ├── uq │ ├── xpf │ └── xpl ├── docs/ │ ├── developer-guide.md │ ├── install.md │ ├── java.md │ ├── logo.meta.txt │ └── shell.md └── test/ ├── chore/ │ ├── bump-scripts-version.sh │ ├── integration-test.sh │ └── lint.sh ├── my_unit_test_lib.sh ├── parseOpts_test.sh ├── self-installer.sh ├── uq_test.sh ├── uq_test_another_input └── uq_test_input ================================================ FILE CONTENTS ================================================ ================================================ FILE: .editorconfig ================================================ root = true [*] charset = utf-8 end_of_line = lf insert_final_newline = true tab_width = 4 indent_style = space indent_size = 2 trim_trailing_whitespace = true [*.{py,md,mkd,markdown}] indent_size = 4 [*.xml] indent_style = tab [*.{md,mkd,markdown}] trim_trailing_whitespace = false # python files without extension [show-duplicate-java-classes] indent_size = 4 ================================================ FILE: .github/dependabot.yml ================================================ # To get started with Dependabot version updates, you'll need to specify which # package ecosystems to update and where the package manifests are located. # Please see the documentation for all configuration options: # https://docs.github.com/github/administering-a-repository/configuration-options-for-dependency-updates version: 2 updates: - package-ecosystem: github-actions directory: / schedule: interval: daily ================================================ FILE: .github/workflows/ci.yaml ================================================ # Quickstart for GitHub Actions # https://docs.github.com/en/actions/quickstart name: CI on: [ push, pull_request, workflow_dispatch ] jobs: test: runs-on: ${{ matrix.os }} timeout-minutes: 5 strategy: matrix: # the OS supported by GitHub Actions # https://docs.github.com/en/actions/using-github-hosted-runners/using-github-hosted-runners/about-github-hosted-runners#supported-runners-and-hardware-resources os: [ ubuntu-latest, macos-latest ] fail-fast: false max-parallel: 64 name: Test on ${{ matrix.os }} steps: - uses: actions/checkout@v6 with: submodules: recursive - run: brew install coreutils gnu-sed # https://docs.github.com/en/actions/learn-github-actions/variables#detecting-the-operating-system # https://docs.github.com/en/actions/learn-github-actions/expressions if: runner.os == 'macOS' - run: test/chore/integration-test.sh # https://remarkablemark.org/blog/2017/10/12/check-git-dirty/ - name: Check git dirty run: | git status --short [ -z "$(git status --short)" ] ================================================ FILE: .github/workflows/lint.yaml ================================================ # Quickstart for GitHub Actions # https://docs.github.com/en/actions/quickstart name: Lint on: [ push, pull_request, workflow_dispatch ] jobs: test: runs-on: ubuntu-latest timeout-minutes: 5 name: Lint steps: - uses: actions/checkout@v6 with: submodules: recursive - run: test/chore/lint.sh # https://remarkablemark.org/blog/2017/10/12/check-git-dirty/ - name: Check git dirty run: | git status --short [ -z "$(git status --short)" ] ================================================ FILE: .gitmodules ================================================ [submodule "test/shunit2"] path = test/shunit2-lib url = https://github.com/kward/shunit2.git ================================================ FILE: LICENSE ================================================ Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 1. Definitions. "License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. "Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. "Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. "You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License. "Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. "Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. "Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). "Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. "Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as "Not a Contribution." "Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. 2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. 3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. 4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: (a) You must give any other recipients of the Work or Derivative Works a copy of this License; and (b) You must cause any modified files to carry prominent notices stating that You changed the files; and (c) You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and (d) If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. 5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. 6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. 7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. 8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. 9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. END OF TERMS AND CONDITIONS APPENDIX: How to apply the Apache License to your work. To apply the Apache License to your work, attach the following boilerplate notice, with the fields enclosed by brackets "[]" replaced with your own identifying information. (Don't include the brackets!) The text should be enclosed in the appropriate comment syntax for the file format. We also recommend that a file or class name and description of purpose be included on the same "printed page" as the copyright notice for easier identification within third-party archives. Copyright [yyyy] [name of copyright owner] Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. ================================================ FILE: README.md ================================================ #
🐌 useful-scripts

Github Workflow Build Status GitHub release License GitHub Stars GitHub Forks GitHub issues GitHub Contributors GitHub repo size

🐌 useful scripts for making developer's everyday life easier and happier, involved java, shell etc. 👉 平时有用的手动操作做成脚本，以便捷地使用，让开发的日常生活更轻松些。 💕 欢迎 👏 💖 - 提问，[提交 Issue](https://github.com/oldratlee/useful-scripts/issues/new) - 分享平时自己常用但没有写成脚本的功能（即需求、想法），[提交Issue](https://github.com/oldratlee/useful-scripts/issues/new) - 优化改进，[Fork 后提通过 Pull Request 贡献代码](https://github.com/oldratlee/useful-scripts/fork) - 提供的自己好用脚本实现，[Fork 后提通过 Pull Request 提供](https://github.com/oldratlee/useful-scripts/fork) 本仓库的脚本（如`Java`相关脚本）在阿里等公司（如随身云，见[`awesome-scripts`仓库](https://github.com/Suishenyun/awesome-scripts)说明）的线上生产环境部署使用。 如果你的公司有部署使用，欢迎使用通过 [Issue：who's using | 用户反馈收集](https://github.com/oldratlee/useful-scripts/issues/96) 告知，方便互相交流反馈～ 💗 repo-icon ---------------------- - [🔰 快速下载&使用](#-%E5%BF%AB%E9%80%9F%E4%B8%8B%E8%BD%BD%E4%BD%BF%E7%94%A8) - [📚 使用文档](#-%E4%BD%BF%E7%94%A8%E6%96%87%E6%A1%A3) - [☕ `Java`相关脚本](#-java%E7%9B%B8%E5%85%B3%E8%84%9A%E6%9C%AC) - [🐚 `Shell`相关脚本](#-shell%E7%9B%B8%E5%85%B3%E8%84%9A%E6%9C%AC) - [🎓 Developer Guide](#-developer-guide) - [🎯 面向开发者的目标](#-%E9%9D%A2%E5%90%91%E5%BC%80%E5%8F%91%E8%80%85%E7%9A%84%E7%9B%AE%E6%A0%87) - [关于`Shell`脚本](#%E5%85%B3%E4%BA%8Eshell%E8%84%9A%E6%9C%AC) - [🚦 开发约定](#-%E5%BC%80%E5%8F%91%E7%BA%A6%E5%AE%9A) - [📚 `Shell`学习与开发的资料](#-shell%E5%AD%A6%E4%B9%A0%E4%B8%8E%E5%BC%80%E5%8F%91%E7%9A%84%E8%B5%84%E6%96%99) ---------------------- 🔰 快速下载&使用 ---------------------- ```bash source <(curl -fsSL https://raw.githubusercontent.com/oldratlee/useful-scripts/release-3.x/test/self-installer.sh) ``` 更多下载&使用方式，参见[下载使用](docs/install.md)。 📚 使用文档 ---------------------- ### ☕ [`Java`相关脚本](docs/java.md) 1. [show-busy-java-threads](docs/java.md#-show-busy-java-threads) 用于快速排查`Java`的`CPU`性能问题(`top us`值过高)，自动查出运行的`Java`进程中消耗`CPU`多的线程，并打印出其线程栈，从而确定导致性能问题的方法调用。 1. [show-duplicate-java-classes](docs/java.md#-show-duplicate-java-classes) 找出`jar`文件和`class`目录中的重复类。用于排查`Java`类冲突问题。 1. [find-in-jars](docs/java.md#-find-in-jars) 在目录下所有`jar`文件里，查找类或资源文件。 ### 🐚 [`Shell`相关脚本](docs/shell.md) `Shell`使用加强： 1. [c](docs/shell.md#-c) 原样命令行输出，并拷贝标准输出到系统剪贴板，省去`CTRL+C`操作，优化命令行与其它应用之间的操作流。 1. [coat and taoc](docs/shell.md#-coat) 彩色`cat`/`tac`出文件行，方便人眼区分不同的行。 1. [a2l](docs/shell.md#-a2l) 按行彩色输出参数，方便人眼查看。 1. [uq](docs/shell.md#-uq) 不重排序输入完成整个输入行的去重。相比系统的`uniq`命令加强的是可以跨行去重，不需要排序输入。 1. [ap and rp](docs/shell.md#-ap-and-rp) 批量转换文件路径为绝对路径/相对路径，会自动跟踪链接并规范化路径。 1. [cp-into-docker-run](docs/shell.md#-cp-into-docker-run) 一个`Docker`使用的便利脚本。拷贝本机的执行文件到指定的`docker container`中并在`docker container`中执行。 1. [tcp-connection-state-counter](docs/shell.md#-tcp-connection-state-counter) 统计各个`TCP`连接状态的个数。用于方便排查系统连接负荷问题。 1. [xpl and xpf](docs/shell.md#-xpl-and-xpf) 在命令行中快速完成 在文件浏览器中 打开/选中 指定的文件或文件夹的操作，优化命令行与其它应用之间的操作流。 `Shell`开发/测试加强： 1. [echo-args](docs/shell.md#-echo-args) 输出脚本收到的参数，在控制台运行时，把参数值括起的括号显示成 **红色**，方便人眼查看。用于调试脚本参数输入。 1. [console-text-color-themes.sh](docs/shell.md#-console-text-color-themessh) 显示`Terminator`的全部文字彩色组合的效果及其打印方式，用于开发`Shell`的彩色输出。 1. [parseOpts.sh](docs/shell.md#-parseoptssh) 命令行选项解析库，加强支持选项有多个值（即数组）。 ## 🎓 Developer Guide 为用户提供有用的功能，当然是这个库的首要的价值体现和存在理由。 但作为一个**开源**项目，每个人都可以看到源码实现，这个库或许能做得更多。 ### 🎯 面向开发者的目标 - 将`Shell/Bash`作为线上生产环境使用的专业编程语言。 - 期望体现`Shell/Bash`脚本 生产环境级的严谨开发方式与最佳实践，进而有可能示例与改善在生产环境中`Shell`脚本的质量状况。 PS： - 虽然上面是自己期望的目标，但自己在`Shell`语言上一定会有很多理解和使用上的问题、在这些实现脚本中也会很多需要的改进，可以一起学习、讨论与实践～ 💕 - 这个库中脚本的实现也有使用`Python`。 #### 关于`Shell`脚本 命令行（`CLI`）几乎是每个程序员每天都在使用的工具。相比图形界面工具（`GUI`），命令行有着自己不可替代的便利性和优越性。 命令行里写出来其实就是`Shell`脚本，可以说每个开发者会写`Shell`脚本（或多或少）。在生产环境的功能实现中，也常会看到`Shell`脚本（虽然不如主流语言那么常见）。 可能正因为上面所说的`Shell`脚本的便利性和大众性： - `Shell`脚本有不少是顺手实现的（包括生产环境用的`Shell`脚本）； - `Shell`脚本的实现常常可能质量不高，会引发线上严重的故障。 ### 🚦 开发约定 在这个库中的`Shell`脚本： - 统一使用`Bash 3.2+`； - 面向生产环境，尽可能使用严谨安全的开发方式。 `Shell`用`Bash`的原因是： - 目前仍然是主流的`Shell`，并且在不同环境基本上都缺省部署了。 - 在[`Google`的`Shell`风格指南](https://zh-google-styleguide.readthedocs.io/en/latest/google-shell-styleguide/background.html#shell)中，明确说到了：`Bash`是**唯一**被允许执行的`shell`脚本语言。 - 统一用`Bash`，可以避免不同`Shell`之间差异所带来的风险与没有收益的复杂性。 - 有大量的`Shell`实现，`sh`、`bash`、`zsh`、`fish`、`csh`、`tcsh`、`ksh`、`ash`、`dash`…… - 不同的`Shell`有各种差异，深坑勿入。 - 个人系统学习过的是`Bash`，比较理解熟悉。 PS: 虽然交互`Shell`个人已经使用`Zsh` + [`oh-my-zsh`](https://ohmyz.sh/)，但在严谨的`Shell`脚本开发时还是使用`Bash`。 ### 📚 `Shell`学习与开发的资料 > 更多资料参见 [子文档](docs/developer-guide.md)。 - 🛠️ 开发规范与工具 - [`Google Shell Style Guide`](https://google.github.io/styleguide/shell.xml) | [中文版](https://zh-google-styleguide.readthedocs.io/en/latest/google-shell-styleguide/contents.html) - [`koalaman/shellcheck`](https://github.com/koalaman/shellcheck): `ShellCheck`, a static analysis tool for shell scripts - [`mvdan/sh(shfmt)`](https://github.com/mvdan/sh): `shfmt` formats shell programs - 👷 **`Bash/Shell`最佳实践与安全编程**文章 - [Use the Unofficial Bash Strict Mode (Unless You Looove Debugging)](http://redsymbol.net/articles/unofficial-bash-strict-mode/) - Bash Pitfalls: 编程易犯的错误 - 团子的小窝：[Part 1](http://kodango.com/bash-pitfalls-part-1) | [Part 2](http://kodango.com/bash-pitfalls-part-2) | [Part 3](http://kodango.com/bash-pitfalls-part-3) | [Part 4](http://kodango.com/bash-pitfalls-part-4) | [英文原文：Bash Pitfalls](http://mywiki.wooledge.org/BashPitfalls) - [不要自己去指定`sh`的方式去执行脚本](https://github.com/oldratlee/useful-scripts/issues/57#issuecomment-326485965) - 🎶 **Tips** - [让你提升命令行效率的 Bash 快捷键 【完整版】](https://linuxtoy.org/archives/bash-shortcuts.html) 补充：`ctrl + x, ctrl + e` 就地打开文本编辑器来编辑当前命令行，对于复杂命令行特别有用 - [应该知道的Linux技巧 | 酷 壳 - CoolShell](https://coolshell.cn/articles/8883.html) - 简洁的 Bash Programming 技巧 - 团子的小窝：[Part 1](http://kodango.com/simple-bash-programming-skills) | [Part 2](http://kodango.com/simple-bash-programming-skills-2) | [Part 3](http://kodango.com/simple-bash-programming-skills-3) - 💎 **系统学习** — 看文章、了解Tips完全不能替代系统学习才能真正理解并专业开发！ - [《Bash Pocket Reference》](https://book.douban.com/subject/26738258/) 力荐！说明简单直接结构体系的佳作，专业`Bash`编程必备！且16年的第二版更新到了新版的`Bash 4` - [《学习bash》](https://book.douban.com/subject/1241361/) 上面那本的展开版 - 官方资料 - [`bash man`](https://manned.org/bash) | [中文版](http://ahei.info/chinese-bash-man.htm) - [Bash Reference Manual - gnu.org](http://www.gnu.org/software/bash/manual/) | [中文版](https://yiyibooks.cn/Phiix/bash_reference_manual/bash%E5%8F%82%E8%80%83%E6%96%87%E6%A1%A3.html) Bash参考手册，讲得全面且有深度，比如会全面地讲解不同转义的区别、命令的解析过程，这有助统一深入的方式认识Bash整个执行方式和过程。这些内容在其它书中往往不会讲（因为复杂难于深入浅出的讲解），但却一通百通的关键。 - [Advanced Bash-Scripting Guide](https://hangar118.sdf.org/p/bash-scripting-guide/index.html): An in-depth exploration of the art of shell scripting. - [命令行的艺术 - `jlevy/the-art-of-command-line`](https://github.com/jlevy/the-art-of-command-line/blob/master/README-zh.md) - [`awesome-lists/awesome-bash`](https://github.com/awesome-lists/awesome-bash): A curated list of delightful Bash scripts and resources. - [`alebcay/awesome-shell`](https://github.com/alebcay/awesome-shell): A curated list of awesome command-line frameworks, toolkits, guides and gizmos. - 更多书籍参见个人整理的[书籍豆列 **_`Bash/Shell`_**](https://www.douban.com/doulist/1779379/) ================================================ FILE: bin/a2l ================================================ #!/usr/bin/env bash # @Function # print each arguments on one line colorfully. # # @Usage # $ ./a2l arg1 arg2 # $ ./a2l *.txt # # @online-doc https://github.com/oldratlee/useful-scripts/blob/dev-3.x/docs/shell.md#-a2l # @author Jerry Lee (oldratlee at gmail dot com) set -eEuo pipefail readonly PROG=${0##*/} readonly PROG_VERSION='3.x-dev' ################################################################################ # parse options ################################################################################ usage() { cat < 0)); do case "$1" in -h | --help) usage ;; -V | --version) progVersion ;; --) shift args=(${args[@]:+"${args[@]}"} "$@") break ;; -*) # if unrecognized option, treat it and all follow arguments as args args=(${args[@]:+"${args[@]}"} "$@") break ;; *) # if not option, treat it and all follow arguments as args args=(${args[@]:+"${args[@]}"} "$@") break ;; esac done readonly args ################################################################################ # biz logic ################################################################################ readonly -a ROTATE_COLORS=(33 35 36 31 32 37 34) COLOR_INDEX=0 rotateColorPrint() { local content=$* # - if stdout is a terminal, turn on color output. # '-t' check: is a terminal? # check isatty in bash https://stackoverflow.com/questions/10022323 # - skip color for white space if [[ ! -t 1 || $content =~ ^[[:space:]]*$ ]]; then printf '%s\n' "$content" else local color=${ROTATE_COLORS[COLOR_INDEX++ % ${#ROTATE_COLORS[@]}]} printf '\e[1;%sm%s\e[0m\n' "$color" "$content" fi } for a in ${args[@]:+"${args[@]}"}; do rotateColorPrint "$a" done ================================================ FILE: bin/ap ================================================ #!/usr/bin/env bash # @Function # convert to Absolute Path. # # @Usage # # print Absolute Path of current directory. # $ ./ap # # print Absolute Path of arguments. # $ ./ap a.txt ../dir1/b.txt # # @online-doc https://github.com/oldratlee/useful-scripts/blob/dev-3.x/docs/shell.md#-ap-and-rp # @author Jerry Lee (oldratlee at gmail dot com) set -eEuo pipefail readonly PROG=${0##*/} readonly PROG_VERSION='3.x-dev' ################################################################################ # util functions ################################################################################ errorMsgPrint() { local errorMsg="$PROG: $*" # if stdout is a terminal, turn on color output. # '-t' check: is a terminal? # check isatty in bash https://stackoverflow.com/questions/10022323 if [ -t 1 ]; then printf '\e[1;31m%s\e[0m\n' "$errorMsg" else printf '%s\n' "$errorMsg" fi } >&2 die() { local prompt_help=false exit_status=2 while (($# > 0)); do case "$1" in -h) prompt_help=true shift ;; -s) exit_status=$2 shift 2 ;; *) break ;; esac done (($# > 0)) && errorMsgPrint "$*" $prompt_help && echo "Try '$PROG --help' for more information." exit "$exit_status" } >&2 # `realpath` command exists on Linux and macOS, return resolved physical path # - realpath command on macOS do NOT support option `-e`; # combined `[ -e $file ]` to check file existence first. # - How can I get the behavior of GNU's readlink -f on a Mac? # https://stackoverflow.com/questions/1055671 realpath() { [ -e "$1" ] && command realpath -- "$1" } usage() { cat < 0)); do case "$1" in -h | --help) usage ;; -V | --version) progVersion ;; --) shift files=(${files[@]:+"${files[@]}"} "$@") break ;; -*) die -h "unrecognized option '$1'" ;; *) # if not option, treat all follow files as args files=(${files[@]:+"${files[@]}"} "$@") break ;; esac done # if files is empty, use "." readonly files=("${files[@]:-.}") ################################################################################ # biz logic ################################################################################ has_error=false for f in "${files[@]}"; do realpath "$f" || { has_error=true errorMsgPrint "$f: No such file or directory!" } done # set exit status ! $has_error ================================================ FILE: bin/c ================================================ #!/usr/bin/env bash # @Function # Run command and put output to system clipper. # # @Usage # $ c ls -l # $ ls -l | c # $ c -q < ~/.ssh/id_rsa.pub # # @online-doc https://github.com/oldratlee/useful-scripts/blob/dev-3.x/docs/shell.md#-c # @author Jerry Lee (oldratlee at gmail dot com) set -eEuo pipefail readonly PROG=${0##*/} readonly PROG_VERSION='3.x-dev' ################################################################################ # util functions ################################################################################ redPrint() { # if stdout is a terminal, turn on color output. # '-t' check: is a terminal? # check isatty in bash https://stackoverflow.com/questions/10022323 if [ -t 1 ]; then printf '\e[1;31m%s\e[0m\n' "$*" else printf '%s\n' "$*" fi } die() { local prompt_help=false exit_status=2 while (($# > 0)); do case "$1" in -h) prompt_help=true shift ;; -s) exit_status=$2 shift 2 ;; *) break ;; esac done (($# > 0)) && redPrint "$PROG: $*" $prompt_help && echo "Try '$PROG --help' for more information." exit "$exit_status" } >&2 usage() { cat < 0)); do case "$1" in -k | --keep-eol) keep_eol=true shift ;; -q | --quiet) quiet=true shift ;; -h | --help) usage ;; -V | --version) progVersion ;; --) shift target_command=(${target_command[@]:+"${target_command[@]}"} "$@") break ;; -*) die -h "unrecognized option '$1'" ;; *) # if not option, treat all follow arguments as command target_command=(${target_command[@]:+"${target_command[@]}"} "$@") break ;; esac done readonly keep_eol quiet target_command if ((${#target_command[@]} > 0)) && ! type -P "${target_command[0]}" &>/dev/null; then die "command '${target_command[0]}' not found on PATH" fi ################################################################################ # biz logic ################################################################################ systemClip() { case "$(uname)" in Darwin*) pbcopy ;; CYGWIN* | MINGW*) clip ;; *) xsel -b ;; esac } bufferCopy() { local content content=$(cat) if $keep_eol; then printf '%s\n' "$content" else printf %s "$content" fi | systemClip } teeAndCopy() { if $quiet; then bufferCopy else tee >(bufferCopy) fi } if ((${#target_command[@]} == 0)); then teeAndCopy else command "${target_command[@]}" | teeAndCopy fi ================================================ FILE: bin/coat ================================================ #!/usr/bin/env bash # @Function # cat lines colorfully. coat means *CO*lorful c*AT*. # # @Usage # $ echo -e 'Hello\nWorld' | coat # $ coat /path/to/file1 # $ coat /path/to/file1 /path/to/file2 # # @online-doc https://github.com/oldratlee/useful-scripts/blob/dev-3.x/docs/shell.md#-coat # @author Jerry Lee (oldratlee at gmail dot com) set -eEuo pipefail readonly PROG=${0##*/} readonly PROG_VERSION='3.x-dev' ################################################################################ # parse options ################################################################################ usage() { cat <= 0; --idx)); do [ "${args[idx]}" = --help ] && usage [ "${args[idx]}" = --version ] && progVersion done unset args idx ################################################################################ # biz logic ################################################################################ # if stdout is not a terminal, use `cat` directly. # '-t' check: is a terminal? # check isatty in bash https://stackoverflow.com/questions/10022323 [ -t 1 ] || exec cat "$@" readonly -a ROTATE_COLORS=(33 35 36 31 32 37 34) COLOR_INDEX=0 # CAUTION: print content WITHOUT new line rotateColorPrint() { local content=$* # skip color for white space if [[ $content =~ ^[[:space:]]*$ ]]; then printf %s "$content" else local color=${ROTATE_COLORS[COLOR_INDEX++ % ${#ROTATE_COLORS[@]}]} printf '\e[1;%sm%s\e[0m' "$color" "$content" fi } rotateColorPrintln() { # NOTE: $'foo' is the escape sequence syntax of bash rotateColorPrint "$*"$'\n' } colorLines() { local line # Bash read line does not read leading spaces # https://stackoverflow.com/questions/29689172 while IFS= read -r line; do rotateColorPrintln "$line" done # How to use `while read` (Bash) to read the last line in a file # if there’s no newline at the end of the file? # https://stackoverflow.com/questions/4165135 [ -z "$line" ] || rotateColorPrint "$line" } if (($# == 0)); then colorLines else cat "$@" | colorLines fi ================================================ FILE: bin/cp-into-docker-run ================================================ #!/usr/bin/env bash # @Function # Copy the command into docker container and run the command in container. # # Example: # cp-into-docker-run -c container_foo command_copied_into_container command_arg1 # # @online-doc https://github.com/oldratlee/useful-scripts/blob/dev-3.x/docs/shell.md#-cp-into-docker-run # @author Jerry Lee (oldratlee at gmail dot com) set -eEuo pipefail readonly PROG=${0##*/} readonly PROG_VERSION='3.x-dev' ################################################################################ # util functions ################################################################################ redPrint() { # if stdout is a terminal, turn on color output. # '-t' check: is a terminal? # check isatty in bash https://stackoverflow.com/questions/10022323 if [ -t 1 ]; then printf '\e[1;31m%s\e[0m\n' "$*" else printf '%s\n' "$*" fi } die() { local prompt_help=false exit_staus=2 while (($# > 0)); do case "$1" in -h) prompt_help=true shift ;; -s) exit_staus=$2 shift 2 ;; *) break ;; esac done (($# > 0)) && redPrint "$PROG: $*" $prompt_help && echo "Try '$PROG --help' for more information." exit "$exit_staus" } >&2 isAbsolutePath() { [[ "$1" =~ ^/ ]] } # `realpath` command exists on Linux and macOS, return resolved physical path # - realpath command on macOS do NOT support option `-e`; # combined `[ -e $file ]` to check file existence first. # - How can I get the behavior of GNU's readlink -f on a Mac? # https://stackoverflow.com/questions/1055671 realpath() { [ -e "$1" ] && command realpath -- "$1" } usage() { cat < 0)); do case "$1" in -c | --container) container_name=$2 shift 2 ;; -u | --docker-user) docker_user=$2 shift 2 ;; -w | --workdir) docker_workdir=$2 shift 2 ;; -t | --tmpdir) docker_tmpdir=$2 shift 2 ;; -p | --cp-path) docker_command_cp_path=$2 shift 2 ;; -v | --verbose) verbose=true shift ;; -h | --help) usage ;; -V | --version) progVersion ;; --) shift args=(${args[@]:+"${args[@]}"} "$@") break ;; -*) die -h "unrecognized option '$1'" ;; *) # if not option, treat all follow arguments as command args=(${args[@]:+"${args[@]}"} "$@") break ;; esac done readonly container_name docker_user docker_workdir docker_tmpdir docker_command_cp_path verbose args [ -n "$container_name" ] || die -h "requires destination docker container name, specified by option -c/--container!" if [ -n "$docker_workdir" ]; then isAbsolutePath "$docker_workdir" || die "docker workdir(-w/--workdir) must be absolute path: $docker_workdir" elif [ -n "$docker_command_cp_path" ]; then isAbsolutePath "$docker_command_cp_path" || die "when no docker workdir(-w/--workdir) is specified, the command path in docker to copy(-p/--cp-path) must be absolute path: $docker_command_cp_path" fi ################################################################################ # biz logic ################################################################################ ######################################## # check docker command existence ######################################## type -P docker &>/dev/null || die 'docker command not found!' ######################################## # prepare vars for docker operation ######################################## readonly specified_run_command=${args[0]} run_command=$specified_run_command if [ ! -f "$specified_run_command" ]; then type -P "$specified_run_command" &>/dev/null || die "specified command not exists and not found in PATH: $specified_run_command" run_command=$(type -P "$specified_run_command") fi run_command=$(realpath "$run_command") readonly run_command run_command_base_name=${run_command##*/} run_timestamp=$(date "+%Y%m%d_%H%M%S") readonly run_timestamp readonly uuid="${PROG}_${run_timestamp}_${$}_${RANDOM}" if [ -n "$docker_command_cp_path" ]; then if isAbsolutePath "$docker_command_cp_path"; then readonly run_command_in_docker=$docker_command_cp_path else readonly run_command_in_docker="${docker_workdir:+"$docker_workdir/"}$docker_command_cp_path" fi run_command_dir_in_docker=$(dirname -- "$run_command_in_docker") readonly run_command_dir_in_docker else readonly work_tmp_dir_in_docker=$docker_tmpdir/$uuid readonly run_command_in_docker="$work_tmp_dir_in_docker/$run_command_base_name" readonly run_command_dir_in_docker=$work_tmp_dir_in_docker fi cleanupWhenExit() { [ -n "${work_tmp_dir_in_docker:-}" ] || return 0 # remove tmp dir in docker by root user docker exec "$container_name" rm -rf -- "$work_tmp_dir_in_docker" &>/dev/null } trap cleanupWhenExit EXIT ######################################## # docker operations ######################################## logAndRun() { $verbose && printf '%s\n' "[$PROG] $*" >&2 "$@" } logAndRun docker exec ${docker_user:+"--user=$docker_user"} "$container_name" \ mkdir -p -- "$run_command_dir_in_docker" logAndRun docker cp "$run_command" "$container_name:$run_command_in_docker" logAndRun docker exec ${docker_user:+"--user=$docker_user"} "$container_name" \ chmod +x "$run_command_in_docker" logAndRun docker exec -i -t \ ${docker_user:+"--user=$docker_user"} \ ${docker_workdir:+"--workdir=$docker_workdir"} \ "$container_name" \ "$run_command_in_docker" "${args[@]:1:${#args[@]}}" ================================================ FILE: bin/echo-args ================================================ #!/usr/bin/env bash # @Function # print arguments in human and debugging friendly style. # # @online-doc https://github.com/oldratlee/useful-scripts/blob/dev-3.x/docs/shell.md#-echo-args # @author Jerry Lee (oldratlee at gmail dot com) set -eEuo pipefail digitCount() { # argument 1(num) is always a non-negative integer in this script usage, # so NO argument validation logic. local num=$1 count=0 while ((num != 0)); do ((++count)) ((num = num / 10)) done echo "$count" } digit_count=$(digitCount $#) readonly arg_count=$# digit_count readonly RED='\e[1;31m' BLUE='\e[1;36m' COLOR_RESET='\e[0m' printArg() { local idx=$1 value=$2 # if stdout is a terminal, turn on color output. # '-t' check: is a terminal? # check isatty in bash https://stackoverflow.com/questions/10022323 if [ -t 1 ]; then printf "%${digit_count}s/%s: ${RED}[${BLUE}%s${RED}]${COLOR_RESET}\n" "$idx" "$arg_count" "$value" else printf "%${digit_count}s/%s: [%s]\n" "$idx" "$arg_count" "$value" fi } printArg 0 "$0" idx=1 for a; do printArg $((idx++)) "$a" done ================================================ FILE: bin/find-in-jars ================================================ #!/usr/bin/env bash # @Function # Find files in the jar files under specified directory, search jar files recursively(include subdirectory). # # @Usage # $ find-in-jars 'log4j\.properties' # # search file log4j.properties/log4j.xml at zip root # $ find-in-jars '^log4j\.(properties|xml)$' # $ find-in-jars 'log4j\.properties$' -d /path/to/find/directory # $ find-in-jars '\.properties$' -d /path/to/find/dir1 -d path/to/find/dir2 # $ find-in-jars 'Service\.class$' -e jar -e zip # $ find-in-jars 'Mon[^$/]*Service\.class$' -s ' <-> ' # # @online-doc https://github.com/oldratlee/useful-scripts/blob/dev-3.x/docs/java.md#-find-in-jars # @author Jerry Lee (oldratlee at gmail dot com) set -eEuo pipefail readonly PROG=${0##*/} readonly PROG_VERSION='3.x-dev' ################################################################################ # util functions ################################################################################ readonly COLOR_RESET='\e[0m' redPrint() { # if stdout is a terminal, turn on color output. # '-t' check: is a terminal? # check isatty in bash https://stackoverflow.com/questions/10022323 if [ -t 1 ]; then printf "\e[1;31m%s$COLOR_RESET\n" "$*" else printf '%s\n' "$*" fi } # How to delete line with echo? # https://unix.stackexchange.com/questions/26576 # # terminal escapes: http://ascii-table.com/ansi-escape-sequences.php # In particular, to clear from the cursor position to the beginning of the line: # echo -e "\033[1K" # Or everything on the line, regardless of cursor position: # echo -e "\033[2K" readonly LINE_CLEAR='\e[2K\r' # Getting console width using a bash script # https://unix.stackexchange.com/questions/299067 [ -t 2 ] && COLUMNS=$(stty size | awk '{print $2}') printResponsiveMessage() { if ! $show_responsive || [ ! -t 2 ]; then return fi local content=$* # http://www.linuxforums.org/forum/red-hat-fedora-linux/142825-how-truncate-string-bash-script.html printf %b%s "$LINE_CLEAR" "${content:0:COLUMNS}" >&2 } clearResponsiveMessage() { if ! $show_responsive || [ ! -t 2 ]; then return fi printf %b "$LINE_CLEAR" >&2 } die() { local prompt_help=false exit_status=2 while (($# > 0)); do case "$1" in -h) prompt_help=true shift ;; -s) exit_status=$2 shift 2 ;; *) break ;; esac done clearResponsiveMessage (($# > 0)) && redPrint "$PROG: $*" $prompt_help && echo "Try '$PROG --help' for more information." exit "$exit_status" } >&2 usage() { cat < ' Find control: -d, --dir the directory that find jar files. default is current directory. this option can specify multiply times to find in multiply directories. -e, --extension set find file extension, default is jar. this option can specify multiply times to find multiply extension. -E, --extended-regexp PATTERN is an extended regular expression (*default*) -F, --fixed-strings PATTERN is a set of newline-separated strings -G, --basic-regexp PATTERN is a basic regular expression -P, --perl-regexp PATTERN is a Perl regular expression -i, --ignore-case ignore case distinctions Output control: -a, --absolute-path always print absolute path of jar file -s, --separator specify the separator between jar file and zip entry. default is \`!'. -L, --files-not-contained-found print only names of JAR FILEs NOT contained found -l, --files-contained-found print only names of JAR FILEs contained found -R, --no-find-progress do not display responsive find progress Miscellaneous: -h, --help display this help and exit -V, --version display version information and exit EOF exit } progVersion() { printf '%s\n' "$PROG $PROG_VERSION" exit } ################################################################################ # parse options ################################################################################ dirs=() extensions=() args=() separator='!' regex_mode=-E use_absolute_path=false show_responsive=true only_print_file_name=false while (($# > 0)); do case "$1" in -d | --dir) dirs=(${dirs[@]:+"${dirs[@]}"} "$2") shift 2 ;; -e | --extension) extensions=(${extensions[@]:+"${extensions[@]}"} "$2") shift 2 ;; -E | --extended-regexp) regex_mode=-E shift ;; -F | --fixed-strings) regex_mode=-F shift ;; -G | --basic-regexp) regex_mode=-G shift ;; -P | --perl-regexp) regex_mode=-P shift ;; -i | --ignore-case) ignore_case_option=-i shift ;; -a | --absolute-path) use_absolute_path=true shift ;; # support the legacy typo option name --seperator for compatibility -s | --separator | --seperator) separator=$2 shift 2 ;; -L | --files-not-contained-found) only_print_file_name=true print_matched_files=false shift ;; -l | --files-contained-found) only_print_file_name=true print_matched_files=true shift ;; -R | --no-find-progress) show_responsive=false shift ;; -h | --help) usage ;; -V | --version) progVersion ;; --) shift args=(${args[@]:+"${args[@]}"} "$@") break ;; -*) die -h "unrecognized option '$1'" ;; *) args=(${args[@]:+"${args[@]}"} "$1") shift ;; esac done readonly separator regex_mode ignore_case_option use_absolute_path only_print_file_name print_matched_files show_responsive args # shellcheck disable=SC2178 dirs=${dirs:-.} # shellcheck disable=SC2178 readonly extensions=${extensions:-jar} ((${#args[@]} == 0)) && die -h "requires file pattern!" ((${#args[@]} > 1)) && die -h "more than 1 file pattern: ${args[*]}" readonly pattern=${args[0]} tmp_dirs=() for d in "${dirs[@]}"; do [ -e "$d" ] || die "file $d(specified by option -d): No such file or directory!" [ -d "$d" ] || die "file $d(specified by option -d) exists but is not a directory!" [ -r "$d" ] || die "directory $d(specified by option -d) exists but is not readable!" # convert dirs to Absolute Path if has option -a, --absolute-path $use_absolute_path && tmp_dirs=(${tmp_dirs[@]:+"${tmp_dirs[@]}"} "$(cd "$d" && pwd)") done # set dirs to Absolute Path $use_absolute_path && dirs=("${tmp_dirs[@]}") readonly dirs unset d tmp_dirs # convert extensions to find -iname options find_iname_options=() for e in "${extensions[@]}"; do find_iname_options=(${find_iname_options[@]:+"${find_iname_options[@]}" -o} -iname "*.$e") done readonly find_iname_options unset e ################################################################################ # Check the existence of command for listing zip entry! ################################################################################ __prepareCommandToListZipEntries() { # `zipinfo -1`/`unzip -Z1` is ~25 times faster than `jar tf`, find zipinfo/unzip command first. # # How to list files in a zip without extra information in command line # https://unix.stackexchange.com/a/128304/136953 if type -P zipinfo &>/dev/null; then command_to_list_zip_entries=(zipinfo -1) is_use_zip_cmd_to_list_zip_entries=true elif type -P unzip &>/dev/null; then command_to_list_zip_entries=(unzip -Z1) is_use_zip_cmd_to_list_zip_entries=true elif [ -n "$JAVA_HOME" ]; then # search jar command under JAVA_HOME if [ -f "$JAVA_HOME/bin/jar" ]; then [ -x "$JAVA_HOME/bin/jar" ] || die "found \$JAVA_HOME/bin/jar($JAVA_HOME/bin/jar) is NOT executable!" command_to_list_zip_entries=("$JAVA_HOME/bin/jar" tf) elif [ -f "$JAVA_HOME/../bin/jar" ]; then [ -x "$JAVA_HOME/../bin/jar" ] || die "found \$JAVA_HOME/../bin/jar($JAVA_HOME/../bin/jar) is NOT executable!" command_to_list_zip_entries=("$JAVA_HOME/../bin/jar" tf) fi is_use_zip_cmd_to_list_zip_entries=false elif type -P jar &>/dev/null; then # search jar command under PATH command_to_list_zip_entries=(jar tf) is_use_zip_cmd_to_list_zip_entries=false else die "command to list zip entries NOT found : zipinfo, unzip or jar!" fi readonly command_to_list_zip_entries is_use_zip_cmd_to_list_zip_entries } __prepareCommandToListZipEntries listZipEntries() { local zip_file=$1 msg if $is_use_zip_cmd_to_list_zip_entries; then # How to check if zip file is empty in bash # https://superuser.com/questions/438878 msg=$("${command_to_list_zip_entries[@]}" -t "$zip_file" 2>&1) || { # NOTE: # if list emtpy zip file by zipinfo/unzip command, # exit code is 1, and print 'Empty zipfile.' if [ "$msg" != 'Empty zipfile.' ]; then clearResponsiveMessage redPrint "fail to list zip entries of $zip_file, ignored: $msg" >&2 fi return } fi "${command_to_list_zip_entries[@]}" "$zip_file" || { clearResponsiveMessage redPrint "fail to list zip entries of $zip_file, ignored!" >&2 } } ################################################################################ # find logic ################################################################################ searchJarFiles() { printResponsiveMessage "searching jars under dir ${dirs[*]} , ..." local jar_files total_jar_count jar_files=$(find "${dirs[@]}" "${find_iname_options[@]}" -type f) [ -n "$jar_files" ] || die "${extensions[*]} file NOT found!" total_jar_count=$(printf '%s\n' "$jar_files" | wc -l) # remove white space, because the `wc -l` output on mac contains white space! total_jar_count=${total_jar_count//[[:space:]]/} echo "$total_jar_count" printf '%s\n' "$jar_files" } readonly JAR_COLOR='\e[1;35m' SEP_COLOR='\e[1;32m' __outputResultOfJarFile() { local jar_file=$1 file # shellcheck disable=SC2206 local grep_opt_args=("$regex_mode" ${ignore_case_option:-} ${grep_color_option:-} -- "$pattern") if $only_print_file_name; then local matched=false # NOTE: Do NOT use -q flag with grep: # With the -q flag the grep program will stop immediately when the first line of data matches. # Normally you shouldn't use -q in a pipeline like this # unless you are sure the program at the other end can handle SIGPIPE. # more info see: # - https://stackoverflow.com/questions/19120263/why-exit-code-141-with-grep-q # - https://unix.stackexchange.com/questions/305547/broken-pipe-when-grepping-output-but-only-with-i-flag # - http://www.pixelbeat.org/programming/sigpipe_handling.html grep -c "${grep_opt_args[@]}" &>/dev/null && matched=true [ "$print_matched_files" != "$matched" ] && return clearResponsiveMessage if [ -t 1 ]; then printf "$JAR_COLOR%s$COLOR_RESET\n" "$jar_file" else printf '%s\n' "$jar_file" fi else { # Prevent grep from exiting in case of no match # https://unix.stackexchange.com/questions/330660 grep "${grep_opt_args[@]}" || true } | while IFS= read -r file; do clearResponsiveMessage if [ -t 1 ]; then printf "$JAR_COLOR%s$SEP_COLOR%s$COLOR_RESET%s\n" "$jar_file" "$separator" "$file" else printf '%s\n' "$jar_file$separator$file" fi done fi } findInJarFiles() { [ -t 1 ] && local -r grep_color_option='--color=always' local counter=1 total_jar_count jar_file read -r total_jar_count while IFS= read -r jar_file; do printResponsiveMessage "finding in jar($((counter++))/$total_jar_count): $jar_file" listZipEntries "$jar_file" | __outputResultOfJarFile "$jar_file" done clearResponsiveMessage } searchJarFiles | findInJarFiles ================================================ FILE: bin/rp ================================================ #!/usr/bin/env bash # @Function # convert to Relative Path. # # @Usage # # if 1 argument, print relative path to current dir. # $ ./rp /etc/apache2/httpd.conf # # if more than 1 argument, print relative path to last argument. # $ ./rp a.txt ../b.txt /etc/passwd /etc/apache2 # # @online-doc https://github.com/oldratlee/useful-scripts/blob/dev-3.x/docs/shell.md#-ap-and-rp # @author Jerry Lee (oldratlee at gmail dot com) set -eEuo pipefail readonly PROG=${0##*/} readonly PROG_VERSION='3.x-dev' ################################################################################ # util functions ################################################################################ redPrint() { # if stdout is a terminal, turn on color output. # '-t' check: is a terminal? # check isatty in bash https://stackoverflow.com/questions/10022323 if [ -t 1 ]; then printf '\e[1;31m%s\e[0m\n' "$*" else printf '%s\n' "$*" fi } die() { local prompt_help=false exit_status=2 while (($# > 0)); do case "$1" in -h) prompt_help=true shift ;; -s) exit_status=$2 shift 2 ;; *) break ;; esac done (($# > 0)) && redPrint "$PROG: $*" $prompt_help && echo "Try '$PROG --help' for more information." exit "$exit_status" } >&2 portableRelPath() { local file=$1 relTo=$2 uname uname=$(uname) case "$uname" in Linux* | CYGWIN* | MINGW*) realpath "$f" --relative-to="$relTo" ;; Darwin*) local py_args=(-c 'import os, sys; print(os.path.relpath(sys.argv[1], sys.argv[2]))' "$file" "$relTo") if type -P grealpath >/dev/null; then grealpath "$f" --relative-to="$relTo" elif type -P python3 >/dev/null; then python3 "${py_args[@]}" elif type -P python >/dev/null; then python "${py_args[@]}" else die "fail to find command(grealpath/python3/python) to get relative path!" fi ;; *) die "uname($uname) NOT support!" ;; esac } usage() { cat < 0)); do case "$1" in -h | --help) usage ;; -V | --version) progVersion ;; --) shift files=(${files[@]:+"${files[@]}"} "$@") break ;; -*) die -h "unrecognized option '$1'" ;; *) # if not option, treat all follow files as args files=(${files[@]:+"${files[@]}"} "$@") break ;; esac done ((${#files[@]} == 0)) && die -h "requires at least one argument!" if ((${#files[@]} == 1)); then relativeTo=. else argc=${#files[@]} # Get last argument relativeTo=${files[argc - 1]} files=("${files[@]:0:argc-1}") fi [ -f "$relativeTo" ] && relativeTo=$(dirname -- "$relativeTo") [ -e "$relativeTo" ] || die "relativeTo dir($relativeTo): No such file or directory!" readonly files relativeTo ################################################################################ # biz logic ################################################################################ has_error=false for f in "${files[@]}"; do if [ -e "$f" ]; then portableRelPath "$f" "$relativeTo" else redPrint "$PROG: $f: No such file or directory!" >&2 has_error=true fi done # set exit status ! $has_error ================================================ FILE: bin/show-busy-java-threads ================================================ #!/usr/bin/env bash # @Function # Find out the highest cpu consumed threads of java processes, and print the stack of these threads. # # @Usage # $ ./show-busy-java-threads # # @online-doc https://github.com/oldratlee/useful-scripts/blob/dev-3.x/docs/java.md#-show-busy-java-threads # @author Jerry Lee (oldratlee at gmail dot com) # @author superhj1987 (superhj1987 at 126 dot com) readonly PROG=${0##*/} readonly PROG_VERSION='3.x-dev' # choosing between $0 and BASH_SOURCE # https://stackoverflow.com/a/35006505/922688 # How can I get the source directory of a Bash script from within the script itself? # https://stackoverflow.com/questions/59895 # Will $0 always include the path to the script? # https://unix.stackexchange.com/questions/119929 readonly -a COMMAND_LINE=("${BASH_SOURCE[0]}" "$@") # CAUTION: env var $USER is not reliable! # $USER may be overwritten; if run command by `sudo -u`, may is not `root`. # more info see https://www.baeldung.com/linux/get-current-user # # DO NOT declare and assign var(as readonly) in ONE line! # more info see https://github.com/koalaman/shellcheck/wiki/SC2155 WHOAMI=$(whoami) UNAME=$(uname) readonly WHOAMI UNAME ################################################################################ # util functions ################################################################################ # NOTE: $'foo' is the escape sequence syntax of bash readonly NL=$'\n' # new line colorPrint() { local color=$1 shift # if stdout is a terminal, turn on color output. # '-t' check: is a terminal? # check isatty in bash https://stackoverflow.com/questions/10022323 if [ -t 1 ]; then printf '\e[1;%sm%s\e[0m\n' "$color" "$*" else printf '%s\n' "$*" fi } __appendToFile() { [[ -n "$append_file" && -w "$append_file" ]] && printf '%s\n' "$*" >>"$append_file" [[ -n "$store_dir" && -w "$store_dir" ]] && printf '%s\n' "$*" >>"$store_file_prefix$PROG" } colorOutput() { local color=$1 shift colorPrint "$color" "$*" __appendToFile "$*" } # shellcheck disable=SC2120 normalOutput() { printf '%s\n' "$*" __appendToFile "$*" } redOutput() { colorOutput 31 "$*" } greenOutput() { colorOutput 32 "$*" } yellowOutput() { colorOutput 33 "$*" } blueOutput() { colorOutput 36 "$*" } die() { local prompt_help=false exit_status=2 while (($# > 0)); do case "$1" in -h) prompt_help=true shift ;; -s) exit_status=$2 shift 2 ;; *) break ;; esac done (($# > 0)) && colorPrint "1;31" "$PROG: $*" $prompt_help && echo "Try '$PROG --help' for more information." exit "$exit_status" } >&2 logAndRun() { printf '%s\n' "$*" echo "$@" } logAndCat() { printf '%s\n' "$*" echo cat } # Bash RegEx to check floating point numbers from user input # https://stackoverflow.com/questions/13790763 isNonNegativeFloatNumber() { [[ "$1" =~ ^[+]?[0-9]+\.?[0-9]*$ ]] } isNaturalNumber() { [[ "$1" =~ ^[+]?[0-9]+$ ]] } isNaturalNumberList() { [[ "$1" =~ ^([0-9]+)(,[0-9]+)*$ ]] } # print calling(quoted) command line which is able to copy and paste to rerun safely # # How to get the complete calling command of a BASH script from inside the script (not just the arguments) # https://stackoverflow.com/questions/36625593 printCallingCommandLine() { local arg isFirst=true for arg in "${COMMAND_LINE[@]}"; do if $isFirst; then isFirst=false else printf ' ' fi printf '%q' "$arg" done echo } usage() { cat < find out the highest cpu consumed threads from the specified java process. support pid list(eg: 42,47). default from all java process. -c, --count set the thread count to show, default is 5. set count 0 to show all threads. -a, --append-file specifies the file to append output as log. -S, --store-dir specifies the directory for storing the intermediate files, and keep files. default store intermediate files at tmp dir, and auto remove after run. use this option to keep files so as to review jstack/top/ps output later. delay the delay between updates in seconds. count the number of updates. delay/count arguments imitates the style of vmstat command. jstack control: -s, --jstack-path specifies the path of jstack command. -F, --force set jstack to force a thread dump. use when jstack does not respond (process is hung). -m, --mix-native-frames set jstack to print both java and native frames (mixed mode). -l, --lock-info set jstack with long listing. prints additional information about locks. CPU usage calculation control: -i, --cpu-sample-interval specifies the delay between cpu samples to get thread cpu usage percentage during this interval. default is 0.5 (second). set interval 0 to get the percentage of time spent running during the *entire lifetime* of a process. Miscellaneous: -h, --help display this help and exit. -V, --version display version information and exit. EOF exit } progVersion() { printf '%s\n' "$PROG $PROG_VERSION" exit } ################################################################################ # check os support ################################################################################ [[ $UNAME = Linux* ]] || die "only support Linux, not support $UNAME yet!" ################################################################################ # parse options ################################################################################ # DO NOT declare and assign var ARGS(as readonly) in ONE line! ARGS=$( getopt -n "$PROG" -a -o c:p:a:s:S:i:Pd:FmlhV \ -l count:,pid:,append-file:,jstack-path:,store-dir:,cpu-sample-interval:,use-ps,top-delay:,force,mix-native-frames,lock-info,help,version \ -- "$@" ) || die -h eval set -- "$ARGS" unset ARGS count=5 cpu_sample_interval=0.5 while true; do case "$1" in -c | --count) count=$2 shift 2 ;; -p | --pid) pid_list=$2 shift 2 ;; -a | --append-file) append_file=$2 shift 2 ;; -s | --jstack-path) jstack_path=$2 shift 2 ;; -S | --store-dir) store_dir=$2 shift 2 ;; # support the legacy option name -P,--use-ps for compatibility -P | --use-ps) cpu_sample_interval=0 shift ;; # support the legacy option name -d,--top-delay for compatibility -i | --cpu-sample-interval | -d | --top-delay) cpu_sample_interval=$2 shift 2 ;; -F | --force) force=-F shift ;; -m | --mix-native-frames) mix_native_frames=-m shift ;; -l | --lock-info) lock_info=-l shift ;; -h | --help) usage ;; -V | --version) progVersion ;; --) shift break ;; esac done readonly count cpu_sample_interval force mix_native_frames lock_info readonly update_delay=${1:-0} isNonNegativeFloatNumber "$update_delay" || die "update delay($update_delay) is not a non-negative float number!" [ -z "$1" ] && update_count=1 || update_count=${2:-0} isNaturalNumber "$update_count" || die "update count($update_count) is not a natural number!" readonly update_count if [ -n "$pid_list" ]; then pid_list=${pid_list//[[:space:]]/} # delete white space isNaturalNumberList "$pid_list" || die "pid(s)($pid_list) is illegal! example: 42 or 42,99,67" fi readonly pid_list # check the directory of append-file(-a) mode, create if not existed. if [ -n "$append_file" ]; then if [ -e "$append_file" ]; then [ -f "$append_file" ] || die "$append_file(specified by option -a, for storing run output files) exists but is not a file!" [ -w "$append_file" ] || die "file $append_file(specified by option -a, for storing run output files) exists but is not writable!" else append_file_dir=$(dirname -- "$append_file") if [ -e "$append_file_dir" ]; then [ -d "$append_file_dir" ] || die "directory $append_file_dir(specified by option -a, for storing run output files) exists but is not a directory!" [ -w "$append_file_dir" ] || die "directory $append_file_dir(specified by option -a, for storing run output files) exists but is not writable!" else mkdir -p "$append_file_dir" || die "fail to create directory $append_file_dir(specified by option -a, for storing run output files)!" fi fi fi readonly append_file # check store directory(-S) mode, create directory if not existed. if [ -n "$store_dir" ]; then if [ -e "$store_dir" ]; then [ -d "$store_dir" ] || die "$store_dir(specified by option -S, for storing output files) exists but is not a directory!" [ -w "$store_dir" ] || die "directory $store_dir(specified by option -S, for storing output files) exists but is not writable!" else mkdir -p "$store_dir" || die "fail to create directory $store_dir(specified by option -S, for storing output files)!" fi fi readonly store_dir isNonNegativeFloatNumber "$cpu_sample_interval" || die "cpu sample interval($cpu_sample_interval) is not a non-negative float number!" ################################################################################ # search/check the existence of jstack command # # search order/priority: # 1. from -s option # 2. from under env var JAVA_HOME # 3. from under env var PATH ################################################################################ if [ -n "$jstack_path" ]; then # 1. check jstack_path set by -s option [ -f "$jstack_path" ] || die "$jstack_path (set by -s option) is NOT found!" [ -x "$jstack_path" ] || die "$jstack_path (set by -s option) is NOT executable!" elif [ -n "$JAVA_HOME" ]; then # 2. search jstack under JAVA_HOME if [ -f "$JAVA_HOME/bin/jstack" ]; then [ -x "$JAVA_HOME/bin/jstack" ] || die -h "found \$JAVA_HOME/bin/jstack($JAVA_HOME/bin/jstack) is NOT executable!${NL}Use -s option set jstack path manually." jstack_path="$JAVA_HOME/bin/jstack" elif [ -f "$JAVA_HOME/../bin/jstack" ]; then [ -x "$JAVA_HOME/../bin/jstack" ] || die -h "found \$JAVA_HOME/../bin/jstack($JAVA_HOME/../bin/jstack) is NOT executable!${NL}Use -s option set jstack path manually." jstack_path="$JAVA_HOME/../bin/jstack" fi elif type -P jstack &>/dev/null; then # 3. search jstack under PATH jstack_path=$(type -P jstack) [ -x "$jstack_path" ] || die -h "found $jstack_path from PATH is NOT executable!${NL}Use -s option set jstack path manually." else die -h "jstack NOT found by JAVA_HOME(${JAVA_HOME:-not set}) setting and PATH!${NL}Use -s option set jstack path manually." fi readonly jstack_path ################################################################################ # biz logic ################################################################################ # DO NOT declare and assign var run_timestamp(as readonly) in ONE line! run_timestamp=$(date "+%Y-%m-%d_%H:%M:%S.%N") readonly run_timestamp readonly uuid="${PROG}_${run_timestamp}_${$}_${RANDOM}" readonly tmp_store_dir="/tmp/$uuid" if [ -n "$store_dir" ]; then readonly store_file_prefix="$store_dir/${run_timestamp}_" else readonly store_file_prefix="$tmp_store_dir/${run_timestamp}_" fi mkdir -p "$tmp_store_dir" cleanupWhenExit() { rm -rf "$tmp_store_dir" &>/dev/null } trap cleanupWhenExit EXIT headInfo() { local timestamp=$1 colorPrint "0;34;42" ================================================================================ printf '%s\n' "$timestamp [$((update_round_num + 1))/$update_count]: $(printCallingCommandLine)" colorPrint "0;34;42" ================================================================================ echo } if [ -n "$pid_list" ]; then readonly ps_process_select_options="-p $pid_list" else readonly ps_process_select_options="-C java -C jsvc" fi __die_when_no_java_process_found() { if [ -n "$pid_list" ]; then die "process($pid_list) is not running, or not java process!" else die 'No java process found!' fi } # output field: pid, thread id(lwp), pcpu, user # order by pcpu(percentage of cpu usage) # # NOTE: # use ps command to find busy thread(cpu usage) # cpu usage of ps command is expressed as # the percentage of time spent running during the *entire lifetime* of a process, # this is not ideal in general. findBusyJavaThreadsByPs() { # 1. sort by %cpu by ps option `--sort -pcpu` # unfortunately, ps from `procps-ng 3.3.12`, `--sort` does not work properly with other options, # use # ps # combined # sort -k3,3nr # instead of # ps --sort -pcpu # 2. use wide output(unlimited width) by ps option `-ww` # avoid trunk user column to username_fo+ or $uid alike # shellcheck disable=SC2206 local -a ps_cmd_line=(ps $ps_process_select_options -wwLo 'pid,lwp,pcpu,user' --no-headers) # DO NOT combine var ps_out declaration and assignment in ONE line! # more info see https://github.com/koalaman/shellcheck/wiki/SC2155 local ps_out ps_out=$("${ps_cmd_line[@]}" | sort -k3,3nr) [ -n "$ps_out" ] || __die_when_no_java_process_found if [ -n "$store_dir" ]; then printf '%s\n' "$ps_out" | logAndCat "${ps_cmd_line[*]} | sort -k3,3nr" >"$store_file_prefix$((update_round_num + 1))_ps" fi if ((count > 0)); then printf '%s\n' "$ps_out" | head -n "$count" else printf '%s\n' "$ps_out" fi } # top with output field: thread id, %cpu __top_threadId_cpu() { # DO NOT combine var java_pid_list declaration and assignment in ONE line! local java_pid_list # shellcheck disable=SC2086 java_pid_list=$(ps $ps_process_select_options -o pid --no-headers) [ -n "$java_pid_list" ] || __die_when_no_java_process_found # shellcheck disable=SC2086 java_pid_list=$(echo $java_pid_list | tr ' ' ,) # join with , # 1. sort by %cpu by top option `-o %CPU` # unfortunately, top version 3.2 does not support -o option(supports from top version 3.3+), # use # HOME=$tmp_store_dir top -H -b -n 1 # combined # sort # instead of # HOME=$tmp_store_dir top -H -b -n 1 -o %CPU # 2. change HOME env var when run top, # so as to prevent top command output format being change by .toprc user config file unexpectedly # 3. use option `-d 0.5`(update interval 0.5 second) and `-n 2`(update 2 times), # and use second time update data to get cpu percentage of thread in 0.5 second interval # 4. top v3.3, there is 1 black line between 2 update; # but top v3.2, there is 2 blank lines between 2 update! local -a top_cmd_line=(top -H -b -d "$cpu_sample_interval" -n 2 -p "$java_pid_list") # DO NOT combine var top_out declaration and assignment in ONE line! local top_out top_out=$(HOME=$tmp_store_dir "${top_cmd_line[@]}") if [ -n "$store_dir" ]; then printf '%s\n' "$top_out" | logAndCat "${top_cmd_line[@]}" >"$store_file_prefix$((update_round_num + 1))_top" fi # DO NOT combine var result_threads_top_info declaration and assignment in ONE line! local result_threads_top_info result_threads_top_info=$(printf '%s\n' "$top_out" | awk '{ # from text line to empty line, increase block index if (previousLine && !$0) blockIndex++ # only print 4th text block(blockIndex == 3), aka. process info of second top update if (blockIndex == 3 && $1 ~ /^[0-9]+$/) print $1, $9 # $1 is thread id field, $9 is %cpu field previousLine = $0 }') [ -n "$result_threads_top_info" ] || __die_when_no_java_process_found printf '%s\n' "$result_threads_top_info" | sort -k2,2nr } __complete_pid_user_by_ps() { # ps output field: pid, thread id(lwp), user # shellcheck disable=SC2206 local -a ps_cmd_line=(ps $ps_process_select_options -wwLo 'pid,lwp,user' --no-headers) # DO NOT combine var ps_out declaration and assignment in ONE line! local ps_out ps_out=$("${ps_cmd_line[@]}") if [ -n "$store_dir" ]; then printf '%s\n' "$ps_out" | logAndCat "${ps_cmd_line[@]}" >"$store_file_prefix$((update_round_num + 1))_ps" fi local idx=0 threadId pcpu output_fields while read -r threadId pcpu; do ((count <= 0 || idx < count)) || break # output field: pid, threadId, pcpu, user output_fields=$(printf '%s\n' "$ps_out" | awk -v "threadId=$threadId" -v "pcpu=$pcpu" '$2==threadId { print $1, threadId, pcpu, $3; exit }') if [ -n "$output_fields" ]; then ((idx++)) printf '%s\n' "$output_fields" fi done } # output format is same as function findBusyJavaThreadsByPs findBusyJavaThreadsByTop() { __top_threadId_cpu | __complete_pid_user_by_ps } printStackOfThreads() { local idx=0 pid threadId pcpu user threadId0x while read -r pid threadId pcpu user; do printf -v threadId0x '%#x' "$threadId" ((idx++ > 0)) && normalOutput local jstackFile="$store_file_prefix$((update_round_num + 1))_jstack_$pid" [ -f "$jstackFile" ] || { # shellcheck disable=SC2206 local -a jstack_cmd_line=("$jstack_path" $force $mix_native_frames $lock_info $pid) if [ "$user" = "$WHOAMI" ]; then # run without sudo, when java process user is current user logAndRun "${jstack_cmd_line[@]}" >"$jstackFile" elif ((UID == 0)); then # if java process user is not current user, must run jstack with sudo logAndRun sudo -u "$user" "${jstack_cmd_line[@]}" >"$jstackFile" else # current user is not root user, so can not run with sudo; print error message and rerun suggestion redOutput "[$idx] Fail to jstack busy($pcpu%) thread($threadId/$threadId0x) stack of java process($pid) under user($user)." redOutput "User of java process($user) is not current user($WHOAMI), need sudo to rerun:" yellowOutput " sudo $(printCallingCommandLine)" continue fi || { redOutput "[$idx] Fail to jstack busy($pcpu%) thread($threadId/$threadId0x) stack of java process($pid) under user($user)." rm "$jstackFile" &>/dev/null continue } } blueOutput "[$idx] Busy($pcpu%) thread($threadId/$threadId0x) stack of java process($pid) under user($user):" if [ -n "$mix_native_frames" ]; then local sed_script="/--------------- $threadId ---------------/,/^---------------/ { /--------------- $threadId ---------------/b # skip first separator line /^---------------/d # delete second separator line p }" elif [ -n "$force" ]; then local sed_script="/^Thread $threadId:/,/^$/ { /^$/d; p # delete end separator line }" else local sed_script="/ nid=($threadId0x|$threadId) /,/^$/ { /^$/d; p # delete end separator line }" fi sed "$sed_script" -n -r "$jstackFile" | tee ${append_file:+-a "$append_file"} ${store_dir:+-a "$store_file_prefix$PROG"} done } main() { local update_round_num timestamp # if update_count <= 0, infinite loop till user interrupted (eg: CTRL+C) for ((update_round_num = 0; update_count <= 0 || update_round_num < update_count; ++update_round_num)); do ((update_round_num > 0)) && { sleep "$update_delay" normalOutput } timestamp=$(date "+%Y-%m-%d %H:%M:%S.%N") [[ -n "$append_file" || -n "$store_dir" ]] && headInfo "$timestamp" | tee ${append_file:+-a "$append_file"} ${store_dir:+-a "$store_file_prefix$PROG"} >/dev/null ((update_count != 1)) && headInfo "$timestamp" if [ "$cpu_sample_interval" = 0 ]; then findBusyJavaThreadsByPs else findBusyJavaThreadsByTop fi | printStackOfThreads done } main ================================================ FILE: bin/show-duplicate-java-classes ================================================ #!/usr/bin/env python3 # -*- coding: utf-8 -*- # @Function # Find duplicate classes among java lib dirs and class dirs. # # @Usage # $ show-duplicate-java-classes # search jars from current dir # $ show-duplicate-java-classes path/to/lib_dir1 /path/to/lib_dir2 # $ show-duplicate-java-classes -c path/to/class_dir1 -c /path/to/class_dir2 # $ show-duplicate-java-classes -c path/to/class_dir1 path/to/lib_dir1 # $ show-duplicate-java-classes -L path/to/lib_dir1 # search jars in the subdirectories of lib dir # $ show-duplicate-java-classes -J path/to/lib_dir1 # search jars in the jar file # # @online-doc https://github.com/oldratlee/useful-scripts/blob/dev-3.x/docs/java.md#-show-duplicate-java-classes # @author tg123 (farmer1992 at gmail dot com) # @author Jerry Lee (oldratlee at gmail dot com) __author__ = 'tg123' import os import re import sys from glob import glob from io import BytesIO from optparse import OptionParser from os import walk from os.path import exists, isdir, relpath from zipfile import BadZipfile, ZipFile ################################################################################ # utils functions ################################################################################ PROG_VERSION = '3.x-dev' # How to delete line with echo? # https://unix.stackexchange.com/questions/26576 # # terminal escapes: http://ascii-table.com/ansi-escape-sequences.php # In particular, to clear from the cursor position to the beginning of the line: # echo -e "\033[1K" # Or everything on the line, regardless of cursor position: # echo -e "\033[2K" __clear_line = '\033[2K\r' __show_responsive = True def __get_terminal_columns_of_stderr(): """ Rewritten for stderr from """ try: columns, _ = os.get_terminal_size(sys.stderr.fileno()) except (AttributeError, ValueError, OSError): columns = 0 return columns def print_responsive_message(msg): if not __show_responsive or not sys.stderr.isatty(): return columns = __get_terminal_columns_of_stderr() if columns <= 0: return print(__clear_line + msg[:columns], end='', file=sys.stderr) def clear_responsive_message(): if not __show_responsive or not sys.stderr.isatty(): return print(__clear_line, end='', file=sys.stderr) def print_error(msg): clear_responsive_message() print(msg, file=sys.stderr) def print_box_message(msg): print() print('=' * 80) print(msg) print('=' * 80) def str_len(x): return len(str(x)) # issue 32790: Keep trailing zeros in precision for string format option g - Python tracker # https://bugs.python.org/issue32790 def percent_str(num): """ Input => Output 1.4545 / 10 **-1 => 1455% 1.4545 / 10 ** 0 => 145% 1.4545 / 10 ** 1 => 14.5% 1.4545 / 10 ** 2 => 1.45% 1.4545 / 10 ** 3 => 0.145% 1.4545 / 10 ** 4 => 0.015% 1.4545 / 10 ** 5 => 0.001% 1.4545 / 10 ** 6 => 0.000% 1.4545 / 10 ** 7 => 0.000% """ num = num * 100 if num >= 100: return '%.0f%%' % num elif num >= 10: return '%.1f%%' % num elif num >= 1: return '%.2f%%' % num else: return '%.3f%%' % num def list_jar_file_under_lib_dirs(lib_dirs, recursive): jar_files = set() max_idx_str_len = str_len(len(lib_dirs)) for idx, lib_dir in enumerate(lib_dirs, start=1): print_responsive_message('list jar file under lib dir(%*s/%s): %s' % ( max_idx_str_len, idx, len(lib_dirs), lib_dir)) if not exists(lib_dir): print_error('WARN: lib dir %s not exists, ignored!' % lib_dir) continue if not isdir(lib_dir): jar_files.add(lib_dir) continue if not recursive: jar_files |= {p for p in glob(lib_dir + '/*.jar')} continue jar_files |= { dir_path + '/' + filename for dir_path, _, file_names in walk(lib_dir) for filename in file_names if filename.lower().endswith('.jar') } return jar_files def list_class_under_jar_file(jar_file, recursive, progress): """ :return: map: jar_jar_path('a.jar!b.jar!c.jar') -> classes """ index = 0 def list_zip_in_zip(jar_jar_path, zf): nonlocal index index += 1 index_marker = '' if recursive: index_marker = ' #%3s' % index print_responsive_message('list class under jar file(%*s/%s%s): %s' % ( str_len(progress[1]), progress[0], progress[1], index_marker, jar_jar_path)) ret = {} classes = {f for f in zf.namelist() if f.lower().endswith('.class')} ret[jar_jar_path] = classes if not recursive: return ret jars_in_jar = {f for f in zf.namelist() if f.lower().endswith('.jar')} for jar in jars_in_jar: next_jar_jar_path = jar_jar_path + '!' + jar try: with ZipFile(BytesIO(zf.read(jar))) as f: ret.update(list_zip_in_zip(next_jar_jar_path, f)) except BadZipfile as e: print_error('WARN: %s is bad zip file(%s), ignored!' % (next_jar_jar_path, e)) return ret try: with ZipFile(jar_file) as zip_file: return list_zip_in_zip(jar_file, zip_file) except BadZipfile as error: print_error('WARN: %s is bad zip file(%s), ignored!' % (jar_file, error)) return {} def list_class_under_class_dir(class_dir, progress): print_responsive_message('list class under class dir(%*s/%s): %s' % ( str_len(progress[1]), progress[0], progress[1], class_dir)) if not exists(class_dir): print_error('WARN: class dir %s not exists, ignored!' % class_dir) return {} if not isdir(class_dir): print_error('WARN: class dir %s is not dir, ignored!' % class_dir) return {} return {relpath(dir_path + '/' + filename, class_dir) for dir_path, _, file_names in walk(class_dir) for filename in file_names if filename.lower().endswith('.class')} def collect_class_path_to_classes(class_dirs, jar_files, recursive_jar): class_path_to_classes = {} total_count = len(jar_files) + len(class_dirs) index = 0 # list all classes in jar files for jar_file in jar_files: index += 1 class_path_to_classes.update( list_class_under_jar_file(jar_file, recursive=recursive_jar, progress=(index, total_count))) # list all classes in class dirs for class_dir in class_dirs: index += 1 class_path_to_classes[class_dir] = list_class_under_class_dir(class_dir, progress=(index, total_count)) return class_path_to_classes def invert_as_class_to_class_paths(class_path_to_classes): class_to_class_paths = {} for class_path, classes in class_path_to_classes.items(): for clazz in classes: class_to_class_paths.setdefault(clazz, set()).add(class_path) return class_to_class_paths ################################################################################ # biz functions ################################################################################ __java9_module_file_pattern = re.compile(r'(^|.*/)module-info\.class$') def find_duplicate_classes(class_to_class_paths): class_paths_to_duplicate_classes = {} for clazz, class_paths in class_to_class_paths.items(): # skip java 9 module-info files if len(class_paths) == 1 or __java9_module_file_pattern.match(clazz): continue classes = class_paths_to_duplicate_classes.setdefault(frozenset(class_paths), set()) classes.add(clazz) return class_paths_to_duplicate_classes def print_duplicate_classes_info(class_paths_to_duplicate_classes, class_path_to_classes): if not class_paths_to_duplicate_classes: print('COOL! No duplicate classes found!') return duplicate_classes_total_count = sum(len(dcs) for dcs in class_paths_to_duplicate_classes.values()) class_paths_total_count = sum(len(cps) for cps in class_paths_to_duplicate_classes) print('Found %s duplicate classes in %s class paths and %s class path sets:' % ( duplicate_classes_total_count, class_paths_total_count, len(class_paths_to_duplicate_classes))) # sort key(class_paths) and value(duplicate_classes) class_paths_to_duplicate_classes = [(sorted(cps), sorted(dcs)) for cps, dcs in class_paths_to_duplicate_classes.items()] # sort kv pairs # # sort by multiple keys: # 1. class paths count, *descending*; aka. sort by len(item[0]) *reverse=True* # 2. duplicate classes count, *descending*; aka. sort by len(item[1]) *reverse=True* # 3. class paths, ascending; aka. sort by item[0] # sort also ensure output consistent for same input. # # How to sort objects by multiple keys in Python? # https://stackoverflow.com/questions/1143671 # Sort a list by multiple attributes? # https://stackoverflow.com/questions/4233476 # # use - operator of number key for reverse sort key class_paths_to_duplicate_classes.sort(key=lambda item: (-len(item[0]), -len(item[1]), item[0])) max_idx_str_len = str_len(len(class_paths_to_duplicate_classes)) for idx, (class_paths, classes) in enumerate(class_paths_to_duplicate_classes, start=1): duplicate_ratio = len(classes) / min((len(class_path_to_classes[cp]) for cp in class_paths)) print('[%*s] found %s(%s) duplicate classes in %s class paths:' % ( max_idx_str_len, idx, len(classes), percent_str(duplicate_ratio), len(class_paths))) max_class_path_idx_str_len = str_len(len(class_paths)) max_classes_count_str_len = str_len(max(len(class_path_to_classes[cp]) for cp in class_paths)) for i, cp in enumerate(class_paths, start=1): print(' %*s: (contain %*s classes) %s' % ( max_class_path_idx_str_len, i, max_classes_count_str_len, len(class_path_to_classes[cp]), cp)) print_box_message('Duplicate classes detail info:') for idx, (class_paths, classes) in enumerate(class_paths_to_duplicate_classes, start=1): print('[%*s] found %s duplicate classes in %s class paths %s :' % ( max_idx_str_len, idx, len(classes), len(class_paths), ' '.join(class_paths))) max_class_idx_str_len = str_len(len(classes)) for i, c in enumerate(classes, start=1): print(' %*s: %s' % (max_class_idx_str_len, i, c)) def print_class_paths_info(class_path_to_classes): if not class_path_to_classes: return max_idx_str_len = str_len(len(class_path_to_classes)) max_classes_count_str_len = str_len(max(len(classes) for classes in class_path_to_classes.values())) class_path_to_classes = sorted(class_path_to_classes.items(), key=lambda item: item[0]) print_box_message('Find in %s class paths:' % len(class_path_to_classes)) for idx, (cp, classes) in enumerate(class_path_to_classes, start=1): print('%*s: (contain %*s classes) %s' % ( max_idx_str_len, idx, max_classes_count_str_len, len(classes), cp)) def main(): option_parser = OptionParser( usage='%prog [OPTION]...' ' [-c class-dir1 [-c class-dir2] ...]' ' [lib-dir1|jar-file1 [lib-dir2|jar-file2] ...]' '\nFind duplicate classes among java lib dirs and class dirs.' '\n\nExamples:' '\n %prog # search jars from current dir' '\n %prog path/to/lib_dir1 /path/to/lib_dir2' '\n %prog -c path/to/class_dir1 -c /path/to/class_dir2' '\n %prog -c path/to/class_dir1 path/to/lib_dir1' '\n %prog -L path/to/lib_dir1' '\n %prog -J path/to/lib_dir1', version='%prog ' + PROG_VERSION) option_parser.add_option('-L', '--recursive-lib', dest='recursive_lib', default=False, action='store_true', help='search jars in the sub-directories of lib dir') option_parser.add_option('-J', '--recursive-jar', dest='recursive_jar', default=False, action='store_true', help='search jars in the jar file') option_parser.add_option('-c', '--class-dir', dest='class_dirs', default=[], action='append', help='add class dir') option_parser.add_option('-R', '--no-find-progress', dest='show_responsive', default=True, action='store_false', help='do not display responsive find progress') options, lib_dirs = option_parser.parse_args() class_dirs = options.class_dirs if not lib_dirs and not class_dirs: lib_dirs = ['.'] global __show_responsive __show_responsive = options.show_responsive jar_files = list_jar_file_under_lib_dirs(lib_dirs, recursive=options.recursive_lib) if not jar_files and not class_dirs: clear_responsive_message() print('search no jar files under lib dirs, and class dirs is absent.') return 0 class_path_to_classes = collect_class_path_to_classes(class_dirs, jar_files, options.recursive_jar) if all(not classes for classes in class_path_to_classes.values()): clear_responsive_message() print('find no class files in jar files or class dirs.') return 0 print_responsive_message('find duplicate classes...') class_to_class_paths = invert_as_class_to_class_paths(class_path_to_classes) class_paths_to_duplicate_classes = find_duplicate_classes(class_to_class_paths) clear_responsive_message() print_duplicate_classes_info(class_paths_to_duplicate_classes, class_path_to_classes) print_class_paths_info(class_path_to_classes) return int(bool(class_paths_to_duplicate_classes)) if __name__ == '__main__': exit(main()) ================================================ FILE: bin/taoc ================================================ #!/usr/bin/env bash # @Function # tac lines colorfully. taoc means coat(*CO*lorful c*AT*) in reverse(last line first). # # @Usage # $ echo -e 'Hello\nWorld' | taoc # $ taoc /path/to/file1 # $ taoc /path/to/file1 /path/to/file2 # # @online-doc https://github.com/oldratlee/useful-scripts/blob/dev-3.x/docs/shell.md#-coat # @author Jerry Lee (oldratlee at gmail dot com) set -eEuo pipefail readonly PROG=${0##*/} readonly PROG_VERSION='3.x-dev' ################################################################################ # parse options ################################################################################ usage() { cat <= 0; --idx)); do [ "${args[idx]}" = --help ] && usage [ "${args[idx]}" = --version ] && progVersion done unset args idx ################################################################################ # biz logic ################################################################################ # if stdout is not a terminal, use `tac` directly. # '-t' check: is a terminal? # check isatty in bash https://stackoverflow.com/questions/10022323 [ -t 1 ] || exec tac "$@" readonly -a ROTATE_COLORS=(33 35 36 31 32 37 34) COLOR_INDEX=0 # CAUTION: print content WITHOUT new line rotateColorPrint() { local content=$* # skip color for white space if [[ $content =~ ^[[:space:]]*$ ]]; then printf %s "$content" else local color=${ROTATE_COLORS[COLOR_INDEX++ % ${#ROTATE_COLORS[@]}]} printf '\e[1;%sm%s\e[0m' "$color" "$content" fi } rotateColorPrintln() { # NOTE: $'foo' is the escape sequence syntax of bash rotateColorPrint "$*"$'\n' } colorLines() { local line # Bash read line does not read leading spaces # https://stackoverflow.com/questions/29689172 while IFS= read -r line; do rotateColorPrintln "$line" done # How to use `while read` (Bash) to read the last line in a file # if there’s no newline at the end of the file? # https://stackoverflow.com/questions/4165135 [ -z "$line" ] || rotateColorPrint "$line" } tac "$@" | colorLines ================================================ FILE: bin/tcp-connection-state-counter ================================================ #!/usr/bin/env bash # @Function # show count of tcp connection stat. # # @Usage # $ ./tcp-connection-state-counter # # @online-doc https://github.com/oldratlee/useful-scripts/blob/dev-3.x/docs/shell.md#-tcp-connection-state-counter # @author Jerry Lee (oldratlee at gmail dot com) # @author @sunuslee (sunuslee at gmail dot com) set -eEuo pipefail readonly PROG=${0##*/} readonly PROG_VERSION='3.x-dev' ################################################################################ # parse options ################################################################################ usage() { cat <= 0; --idx)); do [[ "${args[idx]}" = -h || "${args[idx]}" = --help ]] && usage [[ "${args[idx]}" = -V || "${args[idx]}" = --version ]] && progVersion done unset args idx ################################################################################ # biz logic ################################################################################ # On MacOS, netstat need to using -p tcp to get only tcp output. UNAME=$(uname) [[ $UNAME = Darwin* ]] && option_for_mac=-ptcp # shellcheck disable=SC2086 netstat -tna ${option_for_mac:-} | awk 'NR > 2 { ++s[$NF] } END { # get max length of stat and count for(v in s) { stat_len = length(v) if(stat_len > max_stat_len) max_stat_len = stat_len count_len = length(s[v]) if (count_len > max_count_len) max_count_len = count_len } for(v in s) { printf "%-" max_stat_len "s %" max_count_len "s\n", v, s[v] } }' | sort -nr -k2,2 ================================================ FILE: bin/uq ================================================ #!/usr/bin/env bash # @Function # Filter lines from INPUT (or standard input), writing to OUTPUT (or standard output). # same as `uniq` command in core utils, # but detect repeated lines that are not adjacent, no sorting required. # # @Usage # uq [OPTION]... [INPUT [OUTPUT]] # # @online-doc https://github.com/oldratlee/useful-scripts/blob/dev-3.x/docs/shell.md#-uq # @author Zava Xu (zava.kid at gmail dot com) # @author Jerry Lee (oldratlee at gmail dot com) set -eEuo pipefail readonly PROG=${0##*/} readonly PROG_VERSION='3.x-dev' ################################################################################ # util functions ################################################################################ # NOTE: $'foo' is the escape sequence syntax of bash readonly NL=$'\n' # new line redPrint() { # if stdout is a terminal, turn on color output. # '-t' check: is a terminal? # check isatty in bash https://stackoverflow.com/questions/10022323 if [ -t 1 ]; then printf '\e[1;31m%s\e[0m\n' "$*" else printf '%s\n' "$*" fi } yellowPrint() { if [ -t 1 ]; then printf '\e[1;33m%s\e[0m\n' "$*" else printf '%s\n' "$*" fi } die() { local prompt_help=false exit_status=2 while (($# > 0)); do case "$1" in -h) prompt_help=true shift ;; -s) exit_status=$2 shift 2 ;; *) break ;; esac done (($# > 0)) && redPrint "$PROG: $*" $prompt_help && echo "Try '$PROG --help' for more information." exit "$exit_status" } >&2 convertHumanReadableSizeToSize() { local human_readable_size=$1 [[ "$human_readable_size" =~ ^([0-9][0-9]*)([kmg]?)$ ]] || return 1 local size=${BASH_REMATCH[1]} unit=${BASH_REMATCH[2]} case "$unit" in k) ((size *= 1024)) ;; m) ((size *= 1024 ** 2)) ;; g) ((size *= 1024 ** 3)) ;; esac echo "$size" } usage() { cat < 0)); do case "$1" in -c | --count) uq_opt_count=1 shift ;; -d | --repeated) uq_opt_only_repeated=1 shift ;; -D) uq_opt_all_repeated=1 shift ;; --all-repeated=*) uq_opt_all_repeated=1 uq_opt_repeated_method=${1#--all-repeated=} [[ $uq_opt_repeated_method = 'none' || $uq_opt_repeated_method = 'prepend' || $uq_opt_repeated_method = 'separate' ]] || die -h "invalid argument ‘$uq_opt_repeated_method’ for ‘--all-repeated’${NL}Valid arguments are:$NL - ‘none’$NL - ‘prepend’$NL - ‘separate’" shift ;; -u | --unique) uq_opt_only_unique=1 shift ;; -i | --ignore-case) uq_opt_ignore_case=1 shift ;; -z | --zero-terminated) uq_opt_zero_terminated=1 shift ;; -XM | --max-input) uq_max_input_human_readable_size=$2 shift 2 ;; -h | --help) usage ;; -V | --version) progVersion ;; --) shift argv=(${argv[@]:+"${argv[@]}"} "$@") break ;; -) argv=(${argv[@]:+"${argv[@]}"} "$1") shift ;; -*) die -h "unrecognized option '$1'" ;; *) argv=(${argv[@]:+"${argv[@]}"} "$1") shift ;; esac done [[ $uq_opt_only_repeated = 1 && $uq_opt_only_unique = 1 ]] && die -h "printing duplicated lines(-d, --repeated) and unique lines(-u, --unique) is meaningless" [[ $uq_opt_all_repeated = 1 && $uq_opt_only_unique = 1 ]] && die -h "printing all duplicate lines(-D, --all-repeated) and unique lines(-u, --unique) is meaningless" [[ $uq_opt_all_repeated = 1 && $uq_opt_repeated_method = none && ($uq_opt_count = 0 && $uq_opt_only_repeated = 0) ]] && yellowPrint "WARN: -D/--all-repeated=none option without -c/-d option, just cat input simply!" >&2 # DO NOT declare and assign var uq_max_input_size(as readonly) in ONE line! # more info see https://github.com/koalaman/shellcheck/wiki/SC2155 uq_max_input_size=$(convertHumanReadableSizeToSize "$uq_max_input_human_readable_size") || die -h "illegal value of option -XM/--max-input: $uq_max_input_human_readable_size" readonly argc=${#argv[@]} argv uq_max_input_size if ((argc == 0)); then input_files=() output_file=/dev/stdout elif ((argc == 1)); then input_files=("${argv[0]}") output_file=/dev/stdout else input_files=("${argv[@]:0:argc-1}") output_file=${argv[argc - 1]} if [ "$output_file" = - ]; then output_file=/dev/stdout fi fi readonly output_file # Check input file for f in ${input_files[@]:+"${input_files[@]}"}; do # - is stdin, ok [ "$f" = - ] && continue [ -e "$f" ] || die "input file $f: No such file or directory!" [ ! -d "$f" ] || die "input file $f exists, but is a directory!" [ -f "$f" ] || die "input file $f exists, but is not a file!" [ -r "$f" ] || die "input file $f exists, but is not readable!" done unset f ################################################################################ # biz logic ################################################################################ # uq awk script # # edit in a separated file(eg: uq.awk) then copy here, # maybe more convenient(like good syntax highlight) # shellcheck disable=SC2016 readonly uq_awk_script=' function printResult(for_lines) { for (idx = 0; idx < length(for_lines); idx++) { line = for_lines[idx] count = line_count_array[caseAwareLine(line)] #printf "DEBUG: %s %s, index: %s, uq_opt_only_repeated: %s\n", count, line, idx, uq_opt_only_repeated if (uq_opt_only_unique) { if (count == 1) printLine(count, line) } else { if (uq_opt_only_repeated && count <= 1) continue if (uq_opt_repeated_method == "prepend" || uq_opt_repeated_method == "separate" && previous_output) { if (line != previous_output) print "" } printLine(count, line) previous_output = line } } } function printLine(count, line) { if (uq_opt_count) printf "%7s %s%s", count, line, ORS else print line } function caseAwareLine(line) { if (IGNORECASE) return tolower(line) else return line } BEGIN { if (uq_opt_zero_terminated) ORS = RS = "\0" } { total_input_size += length + 1 if (total_input_size > uq_max_input_size) { printf "%s: input size exceed max input size %s!\nuse option -XM/--max-input specify a REASONABLE larger value.\n", uq_PROG, uq_max_input_human_readable_size > "/dev/stderr" exit(1) } # use index to keep lines order original_lines[line_index++] = $0 case_aware_line = caseAwareLine($0) # line_count_array: line content -> count if (++line_count_array[case_aware_line] == 1) { # use index to keep lines order deduplicated_lines[deduplicated_line_index++] = case_aware_line } } END { if (uq_opt_all_repeated) printResult(original_lines) else printResult(deduplicated_lines) } ' awk \ -v "uq_opt_count=$uq_opt_count" \ -v "uq_opt_only_repeated=$uq_opt_only_repeated" \ -v "uq_opt_all_repeated=$uq_opt_all_repeated" \ -v "uq_opt_repeated_method=$uq_opt_repeated_method" \ -v "uq_opt_only_unique=$uq_opt_only_unique" \ -v "IGNORECASE=$uq_opt_ignore_case" \ -v "uq_opt_zero_terminated=$uq_opt_zero_terminated" \ -v "uq_max_input_human_readable_size=$uq_max_input_human_readable_size" \ -v "uq_max_input_size=$uq_max_input_size" \ -v "uq_PROG=$PROG" \ -f <(printf "%s" "$uq_awk_script") \ -- ${input_files[@]:+"${input_files[@]}"} \ >"$output_file" ================================================ FILE: bin/xpf ================================================ #!/usr/bin/env bash # @Function # Open file in file explorer, file is selected. # same as xpl --selected [file]... # # @Usage # $ ./xpf file # # @online-doc https://github.com/oldratlee/useful-scripts/blob/dev-3.x/docs/shell.md#-xpl-and-xpf # @author Jerry Lee (oldratlee at gmail dot com) set -eEuo pipefail ################################################################################ # util functions ################################################################################ # `realpath` command exists on Linux and macOS, return resolved physical path # - realpath command on macOS do NOT support option `-e`; # combined `[ -e $file ]` to check file existence first. # - How can I get the behavior of GNU's readlink -f on a Mac? # https://stackoverflow.com/questions/1055671 realpath() { [ -e "$1" ] && command realpath -- "$1" } ################################################################################ # biz logic ################################################################################ # DO NOT inline THIS_SCRIPT into BASE_DIR, because sub-shell: # BASE_DIR=$(dirname -- "$(realpath "${BASH_SOURCE[0]}")") THIS_SCRIPT=$(realpath "${BASH_SOURCE[0]}") BASE_DIR=$(dirname -- "$THIS_SCRIPT") # shellcheck disable=SC1091 source "$BASE_DIR/xpl" "$@" ================================================ FILE: bin/xpl ================================================ #!/usr/bin/env bash # @Function # Open file in file explorer. # # @Usage # $ ./xpf [file [file ...] ] # # @online-doc https://github.com/oldratlee/useful-scripts/blob/dev-3.x/docs/shell.md#-xpl-and-xpf # @author Jerry Lee (oldratlee at gmail dot com) set -eEuo pipefail readonly PROG=${0##*/} readonly PROG_VERSION='3.x-dev' ################################################################################ # util functions ################################################################################ redPrint() { # if stdout is a terminal, turn on color output. # '-t' check: is a terminal? # check isatty in bash https://stackoverflow.com/questions/10022323 if [ -t 1 ]; then printf '\e[1;31m%s\e[0m\n' "$*" else printf '%s\n' "$*" fi } die() { local prompt_help=false exit_status=2 while (($# > 0)); do case "$1" in -h) prompt_help=true shift ;; -s) exit_status=$2 shift 2 ;; *) break ;; esac done (($# > 0)) && redPrint "$PROG: $*" $prompt_help && echo "Try '$PROG --help' for more information." exit "$exit_status" } >&2 usage() { cat < 0)); do case "$1" in -s | --selected) selected=true shift ;; -h | --help) usage ;; -V | --version) progVersion ;; --) shift files=(${files[@]:+"${files[@]}"} "$@") break ;; -*) die -h "unrecognized option '$1'" ;; *) files=(${files[@]:+"${files[@]}"} "$1") shift ;; esac done # if files is empty, use one element "." files=("${files[@]:-.}") # if program name is xpf, set option selected! [ "xpf" = "$PROG" ] && selected=true readonly files selected ################################################################################ # biz logic ################################################################################ # open one file openOneFile() { local file=$1 slt=$selected case "$(uname)" in Darwin*) [ -f "$file" ] && slt=true if $slt; then open -R "$file" else open "$file" fi ;; CYGWIN*) [ -f "$file" ] && slt=true if $slt; then explorer /select, "$(cygpath -w "$file")" else explorer "$(cygpath -w "$file")" fi ;; *) if [ -d "$file" ]; then nautilus "$(dirname -- "$file")" else if $slt; then nautilus "$file" else nautilus "$(dirname -- "$file")" fi fi ;; esac local selected_msg $slt && selected_msg='with selection' printf 'open %14s: %s\n' "$selected_msg" "$file" } has_error=false for file in "${files[@]}"; do [ -e "$file" ] || { has_error=true redPrint "$PROG: $file: No such file or directory!" >&2 continue } openOneFile "$file" || has_error=true done # set exit status ! $has_error ================================================ FILE: docs/developer-guide.md ================================================ # 📚 `Shell`学习与开发的资料 - 🛠️ 开发规范与工具 - [`Google Shell Style Guide`](https://google.github.io/styleguide/shell.xml) | [中文版](https://zh-google-styleguide.readthedocs.io/en/latest/google-shell-styleguide/contents.html) - [`koalaman/shellcheck`](https://github.com/koalaman/shellcheck): `ShellCheck`, a static analysis tool for shell scripts - [`mvdan/sh(shfmt)`](https://github.com/mvdan/sh): `shfmt` formats shell programs - 👷 **`Bash/Shell`最佳实践与安全编程**文章 - [Use the Unofficial Bash Strict Mode (Unless You Looove Debugging)](http://redsymbol.net/articles/unofficial-bash-strict-mode/) - Bash Pitfalls: 编程易犯的错误 - 团子的小窝：[Part 1](http://kodango.com/bash-pitfalls-part-1) | [Part 2](http://kodango.com/bash-pitfalls-part-2) | [Part 3](http://kodango.com/bash-pitfalls-part-3) | [Part 4](http://kodango.com/bash-pitfalls-part-4) | [英文原文：Bash Pitfalls](http://mywiki.wooledge.org/BashPitfalls) - [编写可靠shell脚本的八个建议 - xshell.net](https://www.xshell.net/shell/1577.html) - [Shell 编码风格 - 团子的小窝](http://kodango.com/shell-script-style) - [Bash 优良编程实践](https://www.techug.com/post/bash-practice.html) - [不要自己去指定`sh`的方式去执行脚本](https://github.com/oldratlee/useful-scripts/issues/57#issuecomment-326485965) - 🎶 **Tips** - [让你提升命令行效率的 Bash 快捷键 【完整版】](https://linuxtoy.org/archives/bash-shortcuts.html) 补充：`ctrl + x, ctrl + e` 就地打开文本编辑器来编辑当前命令行，对于复杂命令行特别有用 - [应该知道的Linux技巧 | 酷 壳 - CoolShell](https://coolshell.cn/articles/8883.html) - 简洁的 Bash Programming 技巧 - 团子的小窝：[Part 1](http://kodango.com/simple-bash-programming-skills) | [Part 2](http://kodango.com/simple-bash-programming-skills-2) | [Part 3](http://kodango.com/simple-bash-programming-skills-3) - [Bash 测试和比较函数 — `test`、`[`、`[[`、`((`、和 `if-then-else` 解密](https://www.ibm.com/developerworks/cn/linux/l-bash-test.html) - [Filenames and Pathnames in Shell (bash, dash, ash, ksh, and so on): How to do it Correctly](https://dwheeler.com/essays/filenames-in-shell.html) - [理解 IFS - 团子的小窝](http://kodango.com/understand-ifs) - [shell中的IFS详解 – 笑遍世界](http://smilejay.com/2011/12/bash_ifs/) - [Bash脚本：怎样一行行地读文件（最好和最坏的方法）](http://blog.jobbole.com/72185/) - [Shell 脚本避免多次重复 source - 团子的小窝](http://kodango.com/avoid-repeated-source-in-shell) - [一个奇怪的 echo 结果 - 团子的小窝](http://kodango.com/a-strange-echo-result) - [浅谈 Shell 脚本配置文件格式 - 团子的小窝](http://kodango.com/config-file-format-in-shell) - [Bash function 还能这么玩 - 团子的小窝](http://kodango.com/bash-functions) - [Bash 获取当前函数名 - 团子的小窝](http://kodango.com/get-function-name-in-bash) - [Zsh和Bash，究竟有何不同 坑很深](https://www.xshell.net/shell/bash_zsh.html) - 💎 **系统学习** — 看文章、了解Tips完全不能替代系统学习才能真正理解并专业开发！ - [《Bash Pocket Reference》](https://book.douban.com/subject/26738258/) 力荐！说明简单直接结构体系的佳作，专业`Bash`编程必备！且16年的第二版更新到了新版的`Bash 4` - [《学习bash》](https://book.douban.com/subject/1241361/) 上面那本的展开版 - 官方资料 - [`bash man`](https://manned.org/bash) | [中文版](http://ahei.info/chinese-bash-man.htm) - [Bash Reference Manual - gnu.org](http://www.gnu.org/software/bash/manual/) | [中文版](https://yiyibooks.cn/Phiix/bash_reference_manual/bash%E5%8F%82%E8%80%83%E6%96%87%E6%A1%A3.html) Bash参考手册，讲得全面且有深度，比如会全面地讲解不同转义的区别、命令的解析过程，这有助统一深入的方式认识Bash整个执行方式和过程。这些内容在其它书中往往不会讲（因为复杂难于深入浅出的讲解），但却一通百通的关键。 - [Advanced Bash-Scripting Guide](https://hangar118.sdf.org/p/bash-scripting-guide/index.html): An in-depth exploration of the art of shell scripting. - [命令行的艺术 - `jlevy/the-art-of-command-line`](https://github.com/jlevy/the-art-of-command-line/blob/master/README-zh.md) - [`awesome-lists/awesome-bash`](https://github.com/awesome-lists/awesome-bash): A curated list of delightful Bash scripts and resources. - [`alebcay/awesome-shell`](https://github.com/alebcay/awesome-shell): A curated list of awesome command-line frameworks, toolkits, guides and gizmos. - [wzb56/13_questions_of_shell: shell十三问 - shell教程](https://github.com/wzb56/13_questions_of_shell) - [实用 Shell 文档 - 团子的小窝](http://kodango.com/useful-documents-about-shell) - 更多书籍参见个人整理的[书籍豆列 **_`Bash/Shell`_**](https://www.douban.com/doulist/1779379/) ================================================ FILE: docs/install.md ================================================ 🐌 下载使用 ==================================== 下载整个工程的脚本 ------------------- ### 直接clone工程 使用简单、方便更新，不过要安装有`git`。 ```bash git clone git://github.com/oldratlee/useful-scripts.git cd useful-scripts # 使用Release分支的内容 git checkout release-3.x # 更新脚本 git pull ``` 包含2个分支： - `dev-3.x`：开发分支 - `release-3.x`：发布分支，功能稳定的脚本 PS： 我的做法是把`useful-scripts` checkout到`$HOME/bin`目录下，再把`$HOME/bin/useful-scripts/bin`配置到`PATH`变量上，这样方便我本地使用所有的脚本。 ### 打包下载 下载文件[release-3.x.zip](https://github.com/oldratlee/useful-scripts/archive/release-3.x.zip)： ```bash wget --no-check-certificate https://github.com/oldratlee/useful-scripts/archive/release-3.x.zip unzip release-3.x.zip ``` 下载和运行单个文件 ------------------- 以[`show-busy-java-threads`](https://raw.github.com/oldratlee/useful-scripts/release-3.x/bin/show-busy-java-threads)为例。 ### `curl`文件直接用`bash`运行 ```bash curl -sLk 'https://raw.github.com/oldratlee/useful-scripts/release-3.x/bin/show-busy-java-threads' | bash ``` ### 下载单个文件 ```bash wget --no-check-certificate https://raw.github.com/oldratlee/useful-scripts/release-3.x/bin/show-busy-java-threads chmod +x show-busy-java-threads ./show-busy-java-threads ``` ================================================ FILE: docs/java.md ================================================ 🐌 `Java`相关脚本 ==================================== - [🍺 show-busy-java-threads](#-show-busy-java-threads) - [用法](#%E7%94%A8%E6%B3%95) - [示例](#%E7%A4%BA%E4%BE%8B) - [贡献者](#%E8%B4%A1%E7%8C%AE%E8%80%85) - [🍺 show-duplicate-java-classes](#-show-duplicate-java-classes) - [用法](#%E7%94%A8%E6%B3%95-1) - [`JDK`开发场景使用说明](#jdk%E5%BC%80%E5%8F%91%E5%9C%BA%E6%99%AF%E4%BD%BF%E7%94%A8%E8%AF%B4%E6%98%8E) - [对于一般的工程](#%E5%AF%B9%E4%BA%8E%E4%B8%80%E8%88%AC%E7%9A%84%E5%B7%A5%E7%A8%8B) - [对于`Web`工程](#%E5%AF%B9%E4%BA%8Eweb%E5%B7%A5%E7%A8%8B) - [`Android`开发场景使用说明](#android%E5%BC%80%E5%8F%91%E5%9C%BA%E6%99%AF%E4%BD%BF%E7%94%A8%E8%AF%B4%E6%98%8E) - [示例](#%E7%A4%BA%E4%BE%8B-1) - [贡献者](#%E8%B4%A1%E7%8C%AE%E8%80%85-1) - [🍺 find-in-jars](#-find-in-jars) - [用法](#%E7%94%A8%E6%B3%95-2) - [示例](#%E7%A4%BA%E4%BE%8B-2) - [运行效果](#%E8%BF%90%E8%A1%8C%E6%95%88%E6%9E%9C) - [参考资料](#%E5%8F%82%E8%80%83%E8%B5%84%E6%96%99) ------------------------------- 关于`Java`排错与诊断，力荐️`Arthas`： ❤️ - `Arthas`用户文档： https://arthas.aliyun.com/doc/quick-start.html - GitHub Repo： [alibaba/arthas: Alibaba Java诊断利器](https://github.com/alibaba/arthas) `Arthas`功能异常(😜)强劲，且在阿里巴巴线上支持使用多年。我自己也常用，一定要看看用用！ `Arthas`是通过`Agent`方式来连接运行的`Java`进程、主要通过交互式来完成功能，与之对应的脚本方式也有其优势，如： 1. 可以在进程不能启动的情况下完成诊断（如依赖中的重复类分析、`ClassPath`上的资源或类查找） 1. 开销少；简单少依赖（就纯文本的一个脚本文件） 1. 方便与（已有的）工具（如`awk`、`sed`、`cron`）、流程或设施集成，进一步编程/自动化 请按需按场景选用。 ------------------------------- 🍺 [show-busy-java-threads](../bin/show-busy-java-threads) ---------------------- 用于快速排查`Java`的`CPU`性能问题(`top us`值过高)，自动查出运行的`Java`进程中消耗`CPU`多的线程，并打印出其线程栈，从而确定导致性能问题的方法调用。 目前只支持`Linux`。原因是`Mac`、`Windows`的`ps`命令不支持列出进程的线程`id`，更多信息参见 [#33](https://github.com/oldratlee/useful-scripts/issues/33)，欢迎提供解法。 PS，如何操作可以参见[`@bluedavy`](http://weibo.com/bluedavy)的[《分布式Java应用》](https://book.douban.com/subject/4848587/)的【5.1.1 `CPU`消耗分析】一节，说得很详细： 1. `top`命令找出消耗`CPU`高的`Java`进程及其线程`id`： 1. 开启线程显示模式（`top -H`，或是打开`top`后按`H`） 1. 按`CPU`使用率排序（`top`缺省是按`CPU`使用降序，已经合要求；打开`top`后按`P`可以显式指定按`CPU`使用降序） 1. 记下`Java`进程`id`及其`CPU`高的线程`id` 1. 查看消耗`CPU`高的线程栈： 1. 用进程`id`作为参数，`jstack`出有问题的`Java`进程 1. 手动转换线程`id`成十六进制（可以用`printf %x 1234`） 1. 在`jstack`输出中查找十六进制的线程`id`（可以用`vim`的查找功能`/0x1234`，或是`grep 0x1234 -A 20`） 1. 查看对应的线程栈，分析问题 查问题时，会要多次上面的操作以分析确定问题，这个过程**太繁琐太慢了**。 期望整合上面的过程成一个脚本，这样一行命令就可以自动化地搞定。 ### 用法 ```bash show-busy-java-threads # 从所有运行的Java进程中找出最消耗CPU的线程（缺省5个），打印出其线程栈 # 缺省会自动从所有的Java进程中找出最消耗CPU的线程，这样用更方便 # 当然你可以通过 -p 选项 手动指定要分析的Java进程Id，以保证只会显示你关心的那个Java进程的信息 show-busy-java-threads -p <指定的Java进程Id> show-busy-java-threads -p 42 show-busy-java-threads -p 42,47 show-busy-java-threads -c <要展示示的线程栈个数> show-busy-java-threads <重复执行的间隔秒数> [<重复执行的次数>] # 多次执行；这2个参数的使用方式类似vmstat命令 show-busy-java-threads -a <运行输出的记录到的文件> # 记录到文件以方便回溯查看 show-busy-java-threads -S <存储jstack输出文件的目录> # 指定jstack输出文件的存储目录，方便记录以后续分析 ############################## # 注意： ############################## # 如果Java进程的用户 与 执行脚本的当前用户 不同，则jstack不了这个Java进程 # 为了能切换到Java进程的用户，需要加sudo来执行，即可以解决： sudo show-busy-java-threads show-busy-java-threads -s <指定jstack命令的全路径> # 对于sudo方式的运行，JAVA_HOME环境变量不能传递给root， # 而root用户往往没有配置JAVA_HOME且不方便配置，不能找到jstack命令。 # 这时显式指定jstack命令的路径就反而显得更方便了 # -m 选项：执行jstack命令时加上 -m 选项，显示上Native的栈帧，一般应用排查不需要使用 show-busy-java-threads -m # -F 选项：执行jstack命令时加上 -F 选项（如果直接jstack无响应时，用于强制jstack），一般情况不需要使用 show-busy-java-threads -F # -l 选项：执行jstack命令时加上 -l 选项，显示上更多相关锁的信息，一般情况不需要使用 # 注意：和 -m -F 选项一起使用时，可能会大大增加jstack操作的耗时 show-busy-java-threads -l # 帮助信息 $ show-busy-java-threads -h Usage: show-busy-java-threads [OPTION]... [delay [count]] Find out the highest cpu consumed threads of java processes, and print the stack of these threads. Example: show-busy-java-threads # show busy java threads info show-busy-java-threads 1 # update every 1 second, (stop by eg: CTRL+C) show-busy-java-threads 3 10 # update every 3 seconds, update 10 times Output control: -p, --pid find out the highest cpu consumed threads from the specified java process. support pid list(eg: 42,47). default from all java process. -c, --count set the thread count to show, default is 5. set count 0 to show all threads. -a, --append-file specifies the file to append output as log. -S, --store-dir specifies the directory for storing the intermediate files, and keep files. default store intermediate files at tmp dir, and auto remove after run. use this option to keep files so as to review jstack/top/ps output later. delay the delay between updates in seconds. count the number of updates. delay/count arguments imitates the style of vmstat command. jstack control: -s, --jstack-path specifies the path of jstack command. -F, --force set jstack to force a thread dump. use when jstack does not respond (process is hung). -m, --mix-native-frames set jstack to print both java and native frames (mixed mode). -l, --lock-info set jstack with long listing. prints additional information about locks. CPU usage calculation control: -i, --cpu-sample-interval specifies the delay between cpu samples to get thread cpu usage percentage during this interval. default is 0.5 (second). set interval 0 to get the percentage of time spent running during the *entire lifetime* of a process. Miscellaneous: -h, --help display this help and exit. -V, --version display version information and exit. ``` ### 示例 ```bash $ show-busy-java-threads [1] Busy(57.0%) thread(23355/0x5b3b) stack of java process(23269) under user(admin): "pool-1-thread-1" prio=10 tid=0x000000005b5c5000 nid=0x5b3b runnable [0x000000004062c000] java.lang.Thread.State: RUNNABLE at java.text.DateFormat.format(DateFormat.java:316) at com.xxx.foo.services.common.DateFormatUtil.format(DateFormatUtil.java:41) at com.xxx.foo.shared.monitor.schedule.AppMonitorDataAvgScheduler.run(AppMonitorDataAvgScheduler.java:127) at com.xxx.foo.services.common.utils.AliTimer$2.run(AliTimer.java:128) at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908) at java.lang.Thread.run(Thread.java:662) [2] Busy(26.1%) thread(24018/0x5dd2) stack of java process(23269) under user(admin): "pool-1-thread-2" prio=10 tid=0x000000005a968800 nid=0x5dd2 runnable [0x00000000420e9000] java.lang.Thread.State: RUNNABLE at java.util.Arrays.copyOf(Arrays.java:2882) at java.lang.AbstractStringBuilder.expandCapacity(AbstractStringBuilder.java:100) at java.lang.AbstractStringBuilder.append(AbstractStringBuilder.java:572) at java.lang.StringBuffer.append(StringBuffer.java:320) - locked <0x00000007908d0030> (a java.lang.StringBuffer) at java.text.SimpleDateFormat.format(SimpleDateFormat.java:890) at java.text.SimpleDateFormat.format(SimpleDateFormat.java:869) at java.text.DateFormat.format(DateFormat.java:316) at com.xxx.foo.services.common.DateFormatUtil.format(DateFormatUtil.java:41) at com.xxx.foo.shared.monitor.schedule.AppMonitorDataAvgScheduler.run(AppMonitorDataAvgScheduler.java:126) at com.xxx.foo.services.common.utils.AliTimer$2.run(AliTimer.java:128) at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908) at java.lang.Thread.run(Thread.java:662) ...... ``` 上面的线程栈可以看出，`CPU`消耗最高的2个线程都在执行`java.text.DateFormat.format`，业务代码对应的方法是`shared.monitor.schedule.AppMonitorDataAvgScheduler.run`。可以基本确定： - `AppMonitorDataAvgScheduler.run`调用`DateFormat.format`次数比较频繁。 - `DateFormat.format`比较慢。（这个可以由`DateFormat.format`的实现确定。） 多执行几次`show-busy-java-threads`，如果上面情况高概率出现，则可以确定上面的判定。 因为调用越少代码执行越快，则出现在线程栈的概率就越低。 脚本有自动多次执行的功能，指定 重复执行的间隔秒数/重复执行的次数 参数。 分析`shared.monitor.schedule.AppMonitorDataAvgScheduler.run`实现逻辑和调用方式，以优化实现解决问题。 ### 贡献者 - [silentforce](https://github.com/silentforce) 改进此脚本，增加对环境变量`JAVA_HOME`的判断。 [#15](https://github.com/oldratlee/useful-scripts/pull/15) - [liuyangc3](https://github.com/liuyangc3) - 发现并解决`jstack`非当前用户`Java`进程的问题。 [#50](https://github.com/oldratlee/useful-scripts/pull/50) - 优化性能，通过`read -a`简化反复的`awk`操作。 [#51](https://github.com/oldratlee/useful-scripts/pull/51) - [superhj1987](https://github.com/superhj1987) / [lirenzuo](https://github.com/lirenzuo) - 提出/实现了多次执行的功能 [superhj1987/awesome-scripts#1](https://github.com/superhj1987/awesome-scripts/issues/1) - [xiongchen2012](https://github.com/xiongchen2012) 提出并解决了长用户名截断的Bug [#62](https://github.com/oldratlee/useful-scripts/pull/62) - [qsLI](https://github.com/qsLI) / [sdslnmd](https://github.com/sdslnmd) - 发现并提交Issue：show-busy-java-threads支持top来获取cpu占用率，ps的cpu占用率非实时 [#67](https://github.com/oldratlee/useful-scripts/issues/67) - [geekMessi](https://github.com/geekMessi) - 发现并提交Issue：在`top v3.2`下提取不正确的Bug [#71](https://github.com/oldratlee/useful-scripts/issues/71) - 发现并提交Issue：support command name jsvc to find java process [#72](https://github.com/oldratlee/useful-scripts/issues/72) 🍺 [show-duplicate-java-classes](../bin/show-duplicate-java-classes) ---------------------- 找出`Java Lib`（`Java`库，即`Jar`文件）或`Class`目录（类目录）中的重复类。 全系统支持（`Python 3`实现，安装`Python 3`即可），如`Linux`、`Mac`、`Windows`。 `Java`开发的一个麻烦的问题是`Jar`冲突（即多个版本的`Jar`），或者说重复类。会出`NoSuchMethod`等的问题，还不见得当时出问题。找出有重复类的`Jar`，可以防患未然。 ### 用法 - 通过脚本参数 指定 `Libs`目录，查找目录下`Jar`文件，收集`Jar`文件中`Class`文件以分析重复类。可以指定多个`Libs`目录。 - 缺省只会查找指定`Lib`目录下`Jar`文件，不会收集`Lib`目录的子目录下`Jar`文件。 - 因为`Libs`目录一般不会用子目录再放`Jar`，也避免把去查找不期望的`Jar`文件。 - 可以通过 `-L`选项 设置 收集`Lib`子目录下的`Jar`文件；这样可以简化`Lib`目录的设置，不需要指定完整的`Lib`目录路径。 - 对于找到的`Jar`文件，缺省不会进一步收集包含在`Jar`文件中的`Jar`。 - 即`FatJar`/`UberJar`的场景，随着像`SpringBoot`的广泛使用，`FatJar`/`UberJar`也比较常见。 - 可以通过 `-J`选项 设置 收集包含在`Jar`文件中的`Jar`。 - 通过`-c`选项 指定 `Class`目录，直接收集这个目录下的`Class`文件以分析重复类。可以多次指定多个`Class`目录。 ```bash # 查找当前目录下所有Jar中的重复类 show-duplicate-java-classes # 查找多个指定目录下所有Jar中的重复类 show-duplicate-java-classes path/to/lib_dir1 /path/to/lib_dir2 # 通过 -L 选项，收集子目录中的Jar文件 show-duplicate-java-classes -L path/to/lib_dir1 # 通过 -J 选项，收集包含在Jar文件中的Jar文件（即 收集包含在FatJar/UberJar中的Jar） show-duplicate-java-classes -J path/to/lib_dir1 # 查找多个指定Class目录下的重复类。 Class目录 通过 -c 选项指定 show-duplicate-java-classes -c path/to/class_dir1 -c /path/to/class_dir2 # 查找指定Class目录和指定目录下所有Jar中的重复类的Jar show-duplicate-java-classes path/to/lib_dir1 /path/to/lib_dir2 -c path/to/class_dir1 -c path/to/class_dir2 # 帮助信息 $ show-duplicate-java-classes -h Usage: show-duplicate-java-classes [OPTION]... [-c class-dir1 [-c class-dir2] ...] [lib-dir1|jar-file1 [lib-dir2|jar-file2] ...] Find duplicate classes among java lib dirs and class dirs. Examples: show-duplicate-java-classes # search jars from current dir show-duplicate-java-classes path/to/lib_dir1 /path/to/lib_dir2 show-duplicate-java-classes -c path/to/class_dir1 -c /path/to/class_dir2 show-duplicate-java-classes -c path/to/class_dir1 path/to/lib_dir1 show-duplicate-java-classes -L path/to/lib_dir1 show-duplicate-java-classes -J path/to/lib_dir1 Options: --version show program's version number and exit -h, --help show this help message and exit -L, --recursive-lib search jars in the sub-directories of lib dir -J, --recursive-jar search jars in the jar file -c CLASS_DIRS, --class-dir=CLASS_DIRS add class dir -R, --no-find-progress do not display responsive find progress ``` #### `JDK`开发场景使用说明 以`Maven`作为构建工程示意过程。 ##### 对于一般的工程 ```sh # 在项目模块目录下执行，拷贝依赖Jar到目录target/dependency下 $ mvn dependency:copy-dependencies -DincludeScope=runtime ... # 检查重复类 $ show-duplicate-java-classes target/dependency ... ``` ##### 对于`Web`工程 对于`Web`工程，即`war` `maven`模块，会打包生成`war`文件。 ```sh # 在war模块目录下执行，生成war文件 $ mvn install ... # 解压war文件，war文件中包含了应用的依赖的Jar文件 $ unzip target/*.war -d target/war ... # 检查重复类 $ show-duplicate-java-classes -c target/war/WEB-INF/classes target/war/WEB-INF/lib ... ``` #### `Android`开发场景使用说明 `Android`开发，有重复类在编译打包时会报`[Dex Loader] Unable to execute dex: Multiple dex files define Lorg/foo/xxx/Yyy`。 但只会给出一个重复类名，如果重复类比较多时，上面打包/报错/排查会要进行多次，而`Android`的打包比较费时，这个过程比较麻烦，希望可以一次把所有重复类都列出来，一起排查掉。 以`Gradle`作为构建工程示意过程。 在`App`的`build.gradle`中添加拷贝库到目录`build/dependencies`下。 ```groovy task copyDependencies(type: Copy) { def dest = new File(buildDir, "dependencies") // clean dir dest.deleteDir() dest.mkdirs() // fill dir with dependencies from configurations.compile into dest } ``` ```sh # 拷贝依赖 $ ./gradlew app:copyDependencies ... # 检查重复类 $ show-duplicate-java-classes app/build/dependencies ... ``` ### 示例 ```bash $ show-duplicate-java-classes WEB-INF/lib COOL! No duplicate classes found! ================================================================================ Find in 150 class paths: ================================================================================ 1: (contain 9 classes) WEB-INF/lib/aopalliance-1.0.jar 2: (contain 25 classes) WEB-INF/lib/asm-5.0.4.jar 3: (contain 313 classes) WEB-INF/lib/aviator-5.0.0.jar 4: (contain 687 classes) WEB-INF/lib/cassandra-0.6.1.jar ... $ show-duplicate-java-classes -c WEB-INF/classes WEB-INF/lib Found 1272 duplicate classes in 345 class paths and 9 class path sets: [1] found 188(100%) duplicate classes in 3 class paths: 1: (contain 188 classes) WEB-INF/lib/jdom-2.0.2.jar 2: (contain 195 classes) WEB-INF/lib/jdom2-2.0.6.jar 3: (contain 195 classes) WEB-INF/lib/jdom2-2.0.8.jar [2] found 150(33.8%) duplicate classes in 2 class paths: 1: (contain 1385 classes) WEB-INF/lib/netty-all-4.0.35.Final.jar 2: (contain 444 classes) WEB-INF/lib/netty-common-4.1.31.Final.jar [3] found 148(55.4%) duplicate classes in 2 class paths: 1: (contain 1385 classes) WEB-INF/lib/netty-all-4.0.35.Final.jar 2: (contain 267 classes) WEB-INF/lib/netty-handler-4.1.31.Final.jar [4] found 103(82.4%) duplicate classes in 2 class paths: 1: (contain 125 classes) WEB-INF/lib/hessian-3.0.14.bugfix.jar 2: (contain 275 classes) WEB-INF/lib/hessian-4.0.38.jar ... ================================================================================ Duplicate classes detail info: ================================================================================ [1] found 188 duplicate classes in 3 class paths WEB-INF/lib/jdom-2.0.2.jar WEB-INF/lib/jdom2-2.0.6.jar WEB-INF/lib/jdom2-2.0.8.jar : 1: org/jdom2/Attribute.class 2: org/jdom2/AttributeList$1.class 3: org/jdom2/AttributeList$ALIterator.class 4: org/jdom2/AttributeList.class 5: org/jdom2/AttributeType.class ... [2] found 150 duplicate classes in 2 class paths WEB-INF/lib/netty-all-4.0.35.Final.jar WEB-INF/lib/netty-common-4.1.31.Final.jar : 1: io/netty/util/AbstractReferenceCounted.class 2: io/netty/util/Attribute.class 3: io/netty/util/AttributeKey.class 4: io/netty/util/AttributeMap.class 5: io/netty/util/CharsetUtil.class ... ... ================================================================================ Find in 232 class paths: ================================================================================ 1: (contain 42 classes) WEB-INF/classes 2: (contain 70 classes) WEB-INF/lib/HikariCP-2.7.8.jar 3: (contain 13 classes) WEB-INF/lib/accessors-smart-1.2.jar 4: (contain 9 classes) WEB-INF/lib/aopalliance-1.0.jar 5: (contain 25 classes) WEB-INF/lib/asm-5.0.4.jar 6: (contain 313 classes) WEB-INF/lib/aviator-5.0.0.jar ... ``` ### 贡献者 [tgic](https://github.com/tg123) 提供此脚本。友情贡献者的链接 [commandlinefu.cn](http://commandlinefu.cn/) | [微博linux命令行精选](http://weibo.com/u/2674868673) 🍺 [find-in-jars](../bin/find-in-jars) ---------------------- 在当前目录下所有`jar`文件里，查找类或资源文件。 支持`Linux`、`Mac`、`Windows`（`cygwin`、`MSSYS`）。 ### 用法 ```bash # 在当前目录下所有`jar`文件里，查找类或资源文件。 find-in-jars 'log4j\.properties' find-in-jars 'log4j\.xml$' find-in-jars log4j\\.xml$ # 和上面命令一样，Shell转义的不同写法而已 find-in-jars 'log4j\.(properties|xml)$' # -d选项 指定 查找目录（覆盖缺省的当前目录） find-in-jars 'log4j\.properties$' -d /path/to/find/directory # 支持多个查找目录，多次指定这个选项即可 find-in-jars 'log4j\.properties' -d /path/to/find/directory1 -d /path/to/find/directory2 # -e选项 指定 查找`zip`文件的扩展名，缺省是`jar` find-in-jars 'log4j\.properties' -e zip # 支持多种查找扩展名，多次指定这个选项即可 find-in-jars 'log4j\.properties' -e jar -e zip # -a选项 指定 查找结果中的Jar文件使用绝对路径 # 分享给别人时，Jar文件路径是完整的，方便别人找到文件 find-in-jars 'log4j\.properties' -a # -s选项 指定 查找结果中的Jar文件和Jar文件里的查找Entry间分隔符，缺省是『!』 # 方便你喜欢的人眼查看，或是与工具脚本如`awk`的处理 find-in-jars 'log4j\.properties' -s ' <-> ' find-in-jars 'log4j\.properties' -s ' ' | awk '{print $2}' # -l选项 指定 只列出Jar文件，不显示Jar文件内匹配的文件列表 # 列出 包含log4j.xml文件的Jar文件： find-in-jars -l 'log4j\.xml$' # 帮助信息 $ find-in-jars -h Usage: find-in-jars [OPTION]... PATTERN Find files in the jar files under specified directory, search jar files recursively(include subdirectory). The pattern default is *extended* regex. Example: find-in-jars 'log4j\.properties' # search file log4j.properties/log4j.xml at zip root find-in-jars '^log4j\.(properties|xml)$' find-in-jars 'log4j\.properties$' -d /path/to/find/directory find-in-jars '\.properties$' -d /path/to/find/dir1 -d path/to/find/dir2 find-in-jars 'Service\.class$' -e jar -e zip find-in-jars 'Mon[^$/]*Service\.class$' -s ' <-> ' Find control: -d, --dir the directory that find jar files. default is current directory. this option can specify multiply times to find in multiply directories. -e, --extension set find file extension, default is jar. this option can specify multiply times to find multiply extension. -E, --extended-regexp PATTERN is an extended regular expression (*default*) -F, --fixed-strings PATTERN is a set of newline-separated strings -G, --basic-regexp PATTERN is a basic regular expression -P, --perl-regexp PATTERN is a Perl regular expression -i, --ignore-case ignore case distinctions Output control: -a, --absolute-path always print absolute path of jar file -s, --separator specify the separator between jar file and zip entry. default is `!'. -L, --files-not-contained-found print only names of JAR FILEs NOT contained found -l, --files-contained-found print only names of JAR FILEs contained found -R, --no-find-progress do not display responsive find progress Miscellaneous: -h, --help display this help and exit -V, --version display version information and exit ``` 注意，Pattern缺省是`grep`的 **扩展**正则表达式。 ### 示例 ```bash # 在当前目录下的所有Jar文件中，查找出 log4j.properties文件 $ find-in-jars 'log4j\.properties$' ./hadoop-core-0.20.2-cdh3u3.jar!log4j.properties ...... # 查找出 以Service结尾的类，Jar文件路径输出成绝对路径 $ find-in-jars 'Service.class$' -a /home/foo/deploy/app/WEB-INF/libs/spring-2.5.6.SEC03.jar!org/springframework/stereotype/Service.class /home/foo/deploy/app/WEB-INF/libs/rpc-hello-0.0.1-SNAPSHOT.jar!com/taobao/biz/HelloService.class ...... # 在指定的多个目录的Jar文件中，查找出 properties文件 $ find-in-jars '\.properties$' -d WEB-INF/lib -d ../deploy/lib | grep -v '/pom\.properties$' WEB-INF/lib/aspectjtools-1.6.2.jar!org/aspectj/ajdt/ajc/messages.properties WEB-INF/lib/aspectjweaver-1.8.8.jar!org/aspectj/weaver/XlintDefault.properties ../deploy/lib/groovy-all-1.1-rc-1.jar!groovy/ui/InteractiveShell.properties ../deploy/lib/httpcore-4.3.3.jar!org/apache/http/version.properties ../deploy/lib/javax.servlet-api-3.0.1.jar!javax/servlet/http/LocalStrings_es.properties ...... # 列出 包含properties文件的Jar文件 $ find-in-jars '\.properties$' -l -d WEB-INF/lib WEB-INF/lib/aspectjtools-1.6.2.jar WEB-INF/lib/aspectjweaver-1.8.8.jar WEB-INF/lib/javax.servlet-api-3.0.1.jar ...... ``` ### 运行效果 支持彩色输出，文件名中的匹配部分以`grep`的高亮方式显示。 ![find-in-jar screenshot](https://user-images.githubusercontent.com/1063891/33545067-9eb66072-d8a2-11e7-8a77-d815c0979e5e.gif) ### 参考资料 [在多个Jar(Zip)文件查找Log4J配置文件的Shell命令行](http://oldratlee.github.io/458/tech/shell/find-file-in-jar-zip-files.html) ================================================ FILE: docs/logo.meta.txt ================================================ logo is created by https://www.logoly.pro font: Zilla Slab logo.fond-size: 60 logo-social.fond-size: 160 ================================================ FILE: docs/shell.md ================================================ 🐌 `Shell`相关脚本 ==================================== - [`Shell`使用加强](#shell%E4%BD%BF%E7%94%A8%E5%8A%A0%E5%BC%BA) - [🍺 c](#-c) - [用法/示例](#%E7%94%A8%E6%B3%95%E7%A4%BA%E4%BE%8B) - [参考资料](#%E5%8F%82%E8%80%83%E8%B5%84%E6%96%99) - [🍺 coat and taoc](#-coat-and-taoc) - [用法/示例](#%E7%94%A8%E6%B3%95%E7%A4%BA%E4%BE%8B-1) - [🍺 a2l](#-a2l) - [用法/示例](#%E7%94%A8%E6%B3%95%E7%A4%BA%E4%BE%8B-2) - [🍺 uq](#-uq) - [用法/示例](#%E7%94%A8%E6%B3%95%E7%A4%BA%E4%BE%8B-3) - [🍺 ap and rp](#-ap-and-rp) - [用法/示例](#%E7%94%A8%E6%B3%95%E7%A4%BA%E4%BE%8B-4) - [🍺 cp-into-docker-run](#-cp-into-docker-run) - [用法/示例](#%E7%94%A8%E6%B3%95%E7%A4%BA%E4%BE%8B-5) - [🍺 tcp-connection-state-counter](#-tcp-connection-state-counter) - [用法/示例](#%E7%94%A8%E6%B3%95%E7%A4%BA%E4%BE%8B-6) - [贡献者](#%E8%B4%A1%E7%8C%AE%E8%80%85) - [🍺 xpl and xpf](#-xpl-and-xpf) - [用法/示例](#%E7%94%A8%E6%B3%95%E7%A4%BA%E4%BE%8B-7) - [贡献者](#%E8%B4%A1%E7%8C%AE%E8%80%85-1) - [`Shell`开发/测试加强](#shell%E5%BC%80%E5%8F%91%E6%B5%8B%E8%AF%95%E5%8A%A0%E5%BC%BA) - [🍺 echo-args](#-echo-args) - [用法/示例](#%E7%94%A8%E6%B3%95%E7%A4%BA%E4%BE%8B-8) - [使用方式](#%E4%BD%BF%E7%94%A8%E6%96%B9%E5%BC%8F) - [🍺 console-text-color-themes.sh](#-console-text-color-themessh) - [用法](#%E7%94%A8%E6%B3%95) - [示例](#%E7%A4%BA%E4%BE%8B) - [运行效果](#%E8%BF%90%E8%A1%8C%E6%95%88%E6%9E%9C) - [贡献者](#%E8%B4%A1%E7%8C%AE%E8%80%85-2) - [参考资料](#%E5%8F%82%E8%80%83%E8%B5%84%E6%96%99-1) - [🍺 parseOpts.sh](#-parseoptssh) - [用法](#%E7%94%A8%E6%B3%95-1) - [示例](#%E7%A4%BA%E4%BE%8B-1) - [兼容性](#%E5%85%BC%E5%AE%B9%E6%80%A7) - [贡献者](#%E8%B4%A1%E7%8C%AE%E8%80%85-3) `Shell`使用加强 ==================================== 🍺 [c](../bin/c) ---------------------- 原样命令行输出，并拷贝标准输出到系统剪贴板，省去`CTRL+C`操作，优化命令行与其它应用之间的操作流。 支持`Linux`、`Mac`、`Windows`（`cygwin`、`MSSYS`）。 命令名`c`的意思是`Copy`，因为这个命令我平时非常常用，所以使用一个字符的命令名，方便快速键入。 更多说明参见[拷贝复制命令行输出放在系统剪贴板上](http://oldratlee.github.io/post/2012-12-23/command-output-to-clip)。 ### 用法/示例 有3种使用风格，根据需要或是你的偏好选取。 ```bash ############################################################ # 1. 前缀方式，后面跟上要运行的命令 ############################################################ $ c pwd /Users/jerry $ c echo -e 'a\nb' a b # 这种使用方式，后面跟的命令不能是别名（alias），对于别名可以用下面的使用方式。 ############################################################ # 2. 后缀方式，管道 ############################################################ $ echo -e 'a\nb' | nl | c 1 a 2 b # gb是oh-my-zsh的别名，列出git的分支，需要后缀的方式的使用。 $ gb | c ############################################################ # 3. 从标准输入读取内容。拷贝文件内容时这种方式最直接。 ############################################################ $ c < ~/.ssh/id_rsa.pub ssh-rsa EAAAABIwAAAQEAz+ETZEgoLeIiC0rjWewdDs0sbo8c...== a@b.com ############################################################ # -q选项：拷贝但不输出。 # 当输出内容比较多、又不关心输出内容和命令执行进展时，可以使用这个选项。 ############################################################ $ c -q < ~/.ssh/id_rsa.pub # 帮助信息 $ c --help Usage: c [OPTION]... [command [command_args ...]] Run command and put output to system clipper. If no command is specified, read from stdin(pipe). Example: c grep -i 'hello world' menu.h main.c set | c c -q < ~/.ssh/id_rsa.pub Options: -k, --keep-eol do not trim new line at end of file -q, --quiet suppress all normal output, default is false -h, --help display this help and exit -V, --version display version information and exit ``` ### 参考资料 - [拷贝复制命令行输出放在系统剪贴板上](http://oldratlee.github.io/post/2012-12-23/command-output-to-clip)，给出了不同系统可用命令。 - 关于文本文件最后的换行，参见[Why should text files end with a newline?](https://stackoverflow.com/questions/729692) 🍺 [coat](../bin/coat) and [taoc](../bin/taoc) ---------------------- 彩色`cat`/`tac`出文件行，方便人眼区分不同的行。 支持`Linux`、`Mac`、`Windows`（`cygwin`、`MSSYS`）。 命令支持选项、功能和使用方式与[`cat`](https://manned.org/cat)/[`tac`](https://manned.org/tac)命令完全一样。 文件操作在实现上完全代理给了`cat`/`tac`命令。 - 命令名`coat`的意思是`COlorful cAT`；同时单词`coat`是外套，而彩色的输出行就像件漂亮的外套～ 🌈 😆 - 命令名`taoc`是`coat`倒序拼写；命名方式就像`tac`之于`cat`。 🐈 ### 用法/示例 ```bash $ echo Hello world | coat Hello world $ echo -e 'Hello\nWorld' | coat Hello World $ echo -e 'Hello\nWorld' | taoc World Hello $ echo -e 'Hello\nWorld' | nl | coat 1 Hello 2 World $ coat file1 file2.txt line1 of file1 line2 of file1 ... line1 of file2 line2 of file2 ... # 帮助信息 # 可以看到本人机器上实现代理的`cat`/`tac`命令是GNU的实现。 $ coat --help Usage: coat [OPTION]... [FILE]... cat lines colorfully. Support options: --help display this help and exit --version output version information and exit All other options and arguments are delegated to command cat, more info see the help/man of command cat(e.g. cat --help). cat executable: /usr/local/opt/coreutils/libexec/gnubin/cat $ taoc --help Usage: taoc [OPTION]... [FILE]... tac lines colorfully. Support options: --help display this help and exit --version output version information and exit All other options and arguments are delegated to command tac, more info see the help/man of command tac(e.g. tac --help). tac executable: /usr/local/opt/coreutils/libexec/gnubin/tac ``` 注：上面示例中，没有彩色；在控制台上运行可以看出彩色效果，如下： ![coat screenshot](../docs/coat.png) 🍺 [a2l](../bin/a2l) ---------------------- 按行彩色输出参数，方便人眼查看。 支持`Linux`、`Mac`、`Windows`（`cygwin`、`MSSYS`）。 命令名`a2l`的意思是`Arguments to(2) Lines`。 ### 用法/示例 ```bash $ a2l *.java A.java B.java ... # zsh支持 **/* 跨目录glob，可以方便搜索，但是输出内容是空格分隔的不方便查看。 # 把参数按行输出方便查看 或是 grep $ a2l **/*.sh lib/console-text-color-themes.sh test/parseOpts_test.sh test/self-installer.sh ... ``` 注：上面示例中，没有彩色；在控制台上运行可以看出彩色效果，和上面的`coat`命令一样。 🍺 [uq](../bin/uq) ---------------------- 不重排序输入完成整个输入行的去重。相比系统的`uniq`命令加强的是可以跨行去重，不需要排序输入。 使用方式与支持的选项 模仿系统的`uniq`命令。支持`Linux`、`Mac`、`Windows`（`cygwin`、`MSSYS`）。 > ‼️ **_注意_**： 去重过程会在内存持有整个输入（因为全局去重）！ > > 对于输入大小较大的场景（如输入量有几G），需谨慎使用以避免占用过多内存；往往需要结合业务场景开发对应的优化实现。 > 虽然平时的大部分场景输入量非常有限（如几M），一个简单没有充分优化的实现是快速够用的。 > > `uq`处理的最大输入量缺省是 256m（字符数），超过了最大输入量则出错退出，以避免意外消耗了过大的内存； > 可以通过`-XM, --max-input`选项 设置 消耗更多内存可接受的合理最大输入量，如`uq --max-input 1g ...` 因为系统的`uniq`命令去重相邻的行，需要组合`sort`命令以对整个输入去重，并且有下面的问题： ```bash # 示例输入 $ cat foo.txt c c b a a c c $ uniq foo.txt c b a c # c输出了2次，原因是第二个c与第一个c不是相邻的重复行 # 可以通过 sort -u 来完成整个输入去重，但这样操作，顺序与输入行不一致 $ sort -u foo.txt a b c # 输入行重排序了！ # 另外一个经典的用法 sort 与 uniq -c，输出重复次数 $ sort foo.txt | uniq -c 2 a 1 b 4 c # 输入行重排序了！ ``` ### 用法/示例 ```bash $ uq foo.txt # 输入是文件 $ cat foo.txt | uq # 或是 标准输入/管道 c b a # 对整个输入行去重，且顺序与输入行一致（保留第一次出现的位置） # -c 选项：输出重复次数 $ uq -c foo.txt 4 c 1 b 2 a # -d, --repeated 选项：只输出 重复行 $ uq -d foo.txt c a # -u, --unique 选项：只输出 唯一行（即不重复的行） $ uq -u foo.txt b # -D 选项：重复行都输出，即重复了几次就输出几次 $ uq -D -c foo.txt 4 c 4 c 1 b 2 a 2 a 4 c 4 c # 有多个文件参数时，最后一个参数 是 输出文件 $ uq in1.txt in2.txt out.txt # 当有多个输入文件时，但要输出到控制台时，指定输出文件（最后一个文件参数）为 `-` 即可 $ uq in1.txt in2.txt - # 如果消耗更多内存可接受的合理的，可以通过 -XM, --max-input 选项设置更大的最大输入量（缺省是256m） $ uq -MI 768m large-file-input $ uq --max-input 10g huge-file-input # 帮助信息 $ uq -h Usage: uq [OPTION]... [INPUT [OUTPUT]] Filter lines from INPUT (or standard input), writing to OUTPUT (or standard output). Same as `uniq` command in core utils, but detect repeated lines that are not adjacent, no sorting required. Example: # only one file, output to stdout uq in.txt # more than 1 file, last file argument is output file uq in.txt out.txt # when use - as output file, output to stdout uq in1.txt in2.txt - Options: -c, --count prefix lines by the number of occurrences -d, --repeated only print duplicate lines, one for each group -D print all duplicate lines combined with -c/-d option usually --all-repeated[=METHOD] like -D, but allow separating groups with an empty line; METHOD={none(default),prepend,separate} -u, --unique Only output unique lines that are not repeated in the input -i, --ignore-case ignore differences in case when comparing -z, --zero-terminated line delimiter is NUL, not newline -XM, --max-input max input size(count by char), support k,m,g postfix default is 256m avoid consuming large memory unexpectedly -h, --help display this help and exit -V, --version display version information and exit ``` 🍺 [ap](../bin/ap) and [rp](../bin/rp) ---------------------- 批量转换文件路径为绝对路径/相对路径，会自动跟踪链接并规范化路径。 支持`Linux`、`Mac`、`Windows`（`cygwin`、`MSSYS`）。 命令名`ap`的意思是`Absolute Path`，`rp`是`Relative Path`。 ### 用法/示例 ```bash # ap缺省打印当前路径的绝对路径 $ ap /home/admin/useful-scripts/test $ ap .. /home/admin/useful-scripts # 支持多个参数 $ ap .. ../.. /etc /etc/../etc /home/admin/useful-scripts /home/admin /etc /etc # rp当一个参数时，打印相对于当前路径的相对路径 $ rp /home ../.. # 多于一个参数时，打印相对于最后一个参数的相对路径 $ rp /home /etc/../etc /home/admin .. ../../etc ``` 🍺 [cp-into-docker-run](../bin/cp-into-docker-run) ---------------------- 一个`Docker`使用的便利脚本。拷贝本机的执行文件到指定的`docker container`中并在`docker container`中执行。 支持`Linux`、`Mac`、`Windows`（`cygwin`、`MSSYS`）。 ### 用法/示例 ```bash # 通过 -c 选项 指定 docker container $ cp-into-docker-run -c container_foo /path/to/command command_args... # 如果 指定的command 不是一个路径，会从 PATH 中查找 $ cp-into-docker-run -c container_foo a2l command_arg1 command_arg2 # 帮助信息 $ cp-into-docker-run -h Usage: cp-into-docker-run [OPTION]... command [command-args]... Copy the command into docker container and run the command in container. Example: cp-into-docker-run -c container_foo command_copied_into_container command_arg1 docker options: -c, --container destination docker container -u, --docker-user docker username or UID to run command optional, docker default is (maybe) root user -w, --workdir absolute working directory inside the container optional, docker default is (maybe) root dir -t, --tmpdir tmp dir in docker to copy command optional, default is /tmp -p, --cp-path destination path in docker of the command(including file name) if specified, command will be kept when run finished optional, default is under tmp dir and deleted when run finished run options: -v, --verbose show operation step infos miscellaneous: -h, --help display this help and exit -V, --version display version information and exit ``` 🍺 [tcp-connection-state-counter](../bin/tcp-connection-state-counter) ---------------------- 统计各个`TCP`连接状态的个数。 支持`Linux`、`Mac`、`Windows`（`cygwin`、`MSSYS`）。 像`Nginx`、`Apache`的机器上需要查看，`TCP`连接的个数，以判定 - 连接数、负荷 - 是否有攻击，查看`SYN_RECV`数（`SYN`攻击） - `TIME_WAIT`数，太多会导致`TCP: time wait bucket table overflow`。 ### 用法/示例 ```bash $ tcp-connection-state-counter CLOSE_WAIT 584 ESTABLISHED 493 TIME_WAIT 112 LISTEN 27 SYN_SENT 7 ``` ### 贡献者 [sunuslee](https://github.com/sunuslee) 改进此脚本，增加对`MacOS`的支持。 [#56](https://github.com/oldratlee/useful-scripts/pull/56) 🍺 [xpl](../bin/xpl) and [xpf](../bin/xpf) ---------------------- 在命令行中快速完成 在文件浏览器中 打开/选中 指定的文件或文件夹的操作，优化命令行与其它应用之间的操作流。 支持`Linux`、`Mac`、`Windows`（`cygwin`、`MSSYS`）。 - `xpl`：在文件浏览器中打开指定的文件或文件夹。 `xpl`是`explorer`的缩写。 - `xpf`: 在文件浏览器中打开指定的文件或文件夹，并选中。 `xpf`是`EXplorer and select File`的缩写。 ### 用法/示例 ```bash xpl # 缺省打开当前目录 xpl <文件或是目录>... # 打开多个文件或目录 xpf # 缺省打开当前目录 xpf <文件或是目录>... # 打开多个文件或目录 # 示例 xpl /path/to/dir xpl /path/to/foo.txt xpl /path/to/dir1 /path/to/foo1.txt xpf /path/to/foo1.txt xpf /path/to/dir1 /path/to/foo1.txt ``` ### 贡献者 - [Linhua Tan](https://github.com/toolchainX) 修复Linux的选定Bug。 `Shell`开发/测试加强 ==================================== 🍺 [echo-args](../bin/echo-args) ---------------------- 在编写脚本时，常常要确认输入参数是否是期望的：参数个数，参数值（可能包含有人眼不容易发现的空格问题）。 支持`Linux`、`Mac`、`Windows`（`cygwin`、`MSSYS`）。 这个脚本输出脚本收到的参数。在控制台运行时，把参数值括起的括号显示成 **红色**，方便人眼查看。 ### 用法/示例 ```bash $ ./echo-args 1 " 2 foo " "3 3" 0/3: [./echo-args] 1/3: [1] 2/3: [ 2 foo ] 3/3: [3 3] ``` ### 使用方式 需要查看某个脚本（实际上也可以是其它的可执行程序）输出参数时，可以这么做： - 把要查看脚本重命名。 - 建一个`echo-args`脚本的符号链接到要查看参数的脚本的位置，名字和查看脚本一样。 这样可以不改其它的程序，查看到输入参数的信息。 🍺 [console-text-color-themes.sh](../lib/console-text-color-themes.sh) ---------------------- 显示`Terminator`的全部文字彩色组合的效果及其打印方式。 支持`Linux`、`Mac`、`Windows`（`cygwin`、`MSSYS`）。 脚本中，也给出了`colorEcho`和`colorEchoWithoutNewLine`函数更方便输出彩色文本 ### 用法 ```bash colorEcho <颜色样式> <要输出的文本>... colorEchoWithoutNewLine <颜色样式> <要输出的文本>... ``` ### 示例 ```bash source console-text-color-themes.sh # 输出红色文本 colorEcho "0;31;40" "Hello world!" # 输出黄色带下划线的文本 colorEchoWithoutNewLine "4;33;40" "Hello world!" "Hello Hell!" ``` ### 运行效果 ![console-text-color-themes.sh的运行效果图](console-colorful-text.png) ### 贡献者 [姜太公](https://github.com/jzwlqx) 提供循环输出彩色组合的脚本。 ### 参考资料 - [utensil](https://github.com/utensil) 的[在Bash下输出彩色的文本](http://utensil.github.io/tech/2007/09/10/colorful-bash.html)，这是篇很有信息量很钻研的文章！ 🍺 [parseOpts.sh](../lib/parseOpts.sh) ---------------------- 命令行选项解析库，加强支持选项有多个值（即数组）。 支持`Linux`、`Mac`、`Windows`（`cygwin`、`MSSYS`）。 自己写一个命令行选项解析函数，是因为[`bash`](https://manned.org/bash)的`builtin`命令[`getopts`](https://manned.org/man/getopts.1)和加强版本命令[`getopt`](https://manned.org/getopt)都不支持数组的值。 指定选项的多个值（即数组）的风格模仿[`find`](https://manned.org/find)命令的`-exec`选项： ```bash $ find . -name \*.txt -exec echo "find file: " {} \; find file: foo.txt find file: bar.txt ... ``` ### 用法 `parseOpts`函数的第一个参数是要解析的选项说明，后面跟实际要解析的输入参数。 选项说明可以长选项和短选项，用逗号分隔，如`a,a-long`。不同选项的说明间用竖号分隔，如`a,a-long|b,b-long:`。 选项说明最后可以有选项类型说明： - `-`： 无参数的选项。既有选项则把值设置成`true`。这是 ***缺省*** 的类型。 - `:`： 有参数的选项，值只有一个。 - `+`： 有多个参数值的选项。值列表要以`;`表示结束。 注意，`;`是`Bash`的元字符（用于一行中多个命令分隔），所以加上转义写成`\;`（当然也可以按你的喜好写成`";"`或`';'`）。 实际要解析的输入参数往往是你的脚本参数，这样`parseOpts`函数调用一般是： ```bash parseOpts "a,a-long|b,b-long:|c,c-long+" "$@" # "$@" 即是回放你的脚本参数 ``` 通过约定的全局变量来获取选项和参数： - 选项名为`a`，通过全局变量`_OPT_VALUE_a`来获取选项的值。 - 选项名为`a-long`，通过全局变量`_OPT_VALUE_a_long`来获取选项的值。 即，把选项名的`-`转`_`，再加上前缀`_OPT_VALUE_`对应的全局变量来获得选项值。 - 除了选项剩下的参数，通过全局变量`_OPT_ARGS`来获取。 按照惯例，输入参数中如果有`--`表示之后参数中不再有选项，即之后都是参数。 ### 示例 ```bash # 导入parseOpts.sh source /path/to/parseOpts.sh parseOpts "a,a-long|b,b-long:|c,c-long+" -a -b bv --c-long c.sh -p pv -q qv arg1 \; aa bb cc # 可以通过下面全局变量来获得解析的参数值： # _OPT_VALUE_a = true # _OPT_VALUE_a_long = true # _OPT_VALUE_b = bv # _OPT_VALUE_b_long = bv # _OPT_VALUE_c = (c.sh -p pv -q qv arg1) ，数组类型 # _OPT_VALUE_c_long = (c.sh -p pv -q qv arg1) ，数组类型 # _OPT_ARGS = (aa bb cc) ，数组类型 ``` `--`的使用效果示例： ```bash # 导入parseOpts.sh source /path/to/parseOpts.sh parseOpts "a,a-long|b,b-long:|c,c-long+" -a -b bv -- --c-long c.sh -p pv -q qv arg1 \; aa bb cc # 可以通过下面全局变量来获得解析的参数值： # _OPT_VALUE_a = true # _OPT_VALUE_a_long = true # _OPT_VALUE_b = bv # _OPT_VALUE_b_long = bv # _OPT_VALUE_c 没有设置过 # _OPT_VALUE_c_long 没有设置过 # _OPT_ARGS = (--c-long c.sh -p pv -q qv arg1 ';' aa bb cc) ，数组类型 ``` ### 兼容性 这个脚本比较复杂，测试过的环境有： 1. `bash --version` `GNU bash, version 4.1.5(1)-release (x86_64-pc-linux-gnu)` `uname -a` `Linux foo-host 2.6.32-41-generic #94-Ubuntu SMP Fri Jul 6 18:00:34 UTC 2012 x86_64 GNU/Linux` 1. `bash --version` `GNU bash, version 3.2.53(1)-release (x86_64-apple-darwin14)` `uname -a` `Darwin foo-host 14.0.0 Darwin Kernel Version 14.0.0: Fri Sep 19 00:26:44 PDT 2014; root:xnu-2782.1.97~2/RELEASE_X86_64 x86_64 i386 MacBookPro10,1 Darwin` 1. `bash --version` `GNU bash, version 3.00.15(1)-release (i386-redhat-linux-gnu)` `uname -a` `Linux foo-host 2.6.9-103.ELxenU #1 SMP Wed Mar 14 16:31:15 CST 2012 i686 i686 i386 GNU/Linux` ### 贡献者 - [Khotyn Huang](https://github.com/khotyn) 指出`bash` `3.0`下使用有问题，并提供`bash` `3.0`的测试机器。 ================================================ FILE: test/chore/bump-scripts-version.sh ================================================ #!/usr/bin/env bash set -eEuo pipefail ################################################################################ # util functions ################################################################################ # NOTE: $'foo' is the escape sequence syntax of bash readonly NL=$'\n' # new line colorPrint() { local color=$1 shift # if stdout is a terminal, turn on color output. # '-t' check: is a terminal? # check isatty in bash https://stackoverflow.com/questions/10022323 if [ -t 1 ]; then printf '\e[1;%sm%s\e[0m\n' "$color" "$*" else printf '%s\n' "$*" fi } redPrint() { colorPrint 31 "$@" } yellowPrint() { colorPrint 33 "$@" } bluePrint() { colorPrint 36 "$@" } logAndRun() { local simple_mode=false [ "$1" = "-s" ] && { simple_mode=true shift } if $simple_mode; then echo "Run under work directory $PWD : $*" "$@" else bluePrint "Run under work directory $PWD :$NL$*" time "$@" fi } die() { redPrint "Error: $*" >&2 exit 1 } ################################################################################ # biz logic ################################################################################ (($# != 1)) && die "need only 1 argument for version!$NL${NL}usage:$NL $0 3.x.y" readonly bump_version=$1 # adjust current dir to project dir # # Bash Pitfalls#5 # http://mywiki.wooledge.org/BashPitfalls#cd_.24.28dirname_.22.24f.22.29 cd -P -- "$(dirname -- "$0")"/.. # Bash Pitfalls#1 # http://mywiki.wooledge.org/BashPitfalls#for_f_in_.24.28ls_.2A.mp3.29 logAndRun find -D exec bin lib -type f -exec \ sed -ri "s/^(.*\bPROG_VERSION\s*=\s*')\S*('.*)$/\1$bump_version\2/" -- \ {} + ================================================ FILE: test/chore/integration-test.sh ================================================ #!/usr/bin/env bash set -eEuo pipefail realpath() { [ -e "$1" ] && command realpath -- "$1" } cd "$(dirname -- "$(realpath "${BASH_SOURCE[0]}")")"/.. ################################################################################ # common util functions ################################################################################ colorEcho() { local color=$1 shift # if stdout is a terminal, turn on color output. # '-t' check: is a terminal? # check isatty in bash https://stackoverflow.com/questions/10022323 if [[ -t 1 || "${GITHUB_ACTIONS:-}" = true ]]; then printf '\e[1;%sm%s\e[0m\n' "$color" "$*" else printf '%s\n' "$*" fi } redEcho() { colorEcho 31 "$@" } yellowEcho() { colorEcho 33 "$@" } blueEcho() { colorEcho 36 "$@" } logAndRun() { local simple_mode=false [ "$1" == "-s" ] && { simple_mode=true shift } if $simple_mode; then echo "Run under work directory $PWD : $*" "$@" else # NOTE: $'foo' is the escape sequence syntax of bash local nl=$'\n' # new line blueEcho "Run under work directory $PWD :$nl$*" time "$@" fi } ################################################################################ # run *_test.sh unit test cases ################################################################################ for test_case in *_test.sh; do logAndRun ./"$test_case" done ================================================ FILE: test/chore/lint.sh ================================================ #!/usr/bin/env bash set -eEuo pipefail realpath() { [ -e "$1" ] && command realpath -- "$1" } # cd to the root of the project cd "$(dirname -- "$(realpath "${BASH_SOURCE[0]}")")"/../.. find bin lib -type f | grep -Pv '/show-duplicate-java-classes$' | grep -Pv '/\.editorconfig$' | xargs --verbose shellcheck --shell=bash ================================================ FILE: test/my_unit_test_lib.sh ================================================ #!/usr/bin/env bash # unit test lib ################################################# # commons functions ################################################# __ut_colorEcho() { local color=$1 shift # if stdout is a terminal, turn on color output. # '-t' check: is a terminal? # check isatty in bash https://stackoverflow.com/questions/10022323 if [[ -t 1 || "${GITHUB_ACTIONS:-}" = true ]]; then printf '\e[1;%sm%s\e[0m\n' "$color" "$*" else printf '%s\n' "$*" fi } redEcho() { __ut_colorEcho 31 "$@" } greenEcho() { __ut_colorEcho 32 "$@" } yellowEcho() { __ut_colorEcho 33 "$@" } blueEcho() { __ut_colorEcho 34 "$@" } fail() { redEcho "TEST FAIL: $*" exit 1 } die() { redEcho "Error: $*" >&2 exit 1 } ################################################# # assertion functions ################################################# assertArrayEquals() { (($# == 2 || $# == 3)) || die "assertArrayEquals must 2 or 3 arguments!" local failMsg= (($# == 3)) && { failMsg=$1 shift } local a1PlaceHolder="$1[@]" local a2PlaceHolder="$2[@]" local a1=("${!a1PlaceHolder}") local a2=("${!a2PlaceHolder}") ((${#a1[@]} == ${#a2[@]})) || fail "assertArrayEquals array length [${#a1[@]}] != [${#a2[@]}]${failMsg:+: $failMsg}" local i for ((i = 0; i < ${#a1[@]}; i++)); do [ "${a1[$i]}" = "${a2[$i]}" ] || fail "assertArrayEquals fail element $i: [${a1[$i]}] != [${a2[$i]}]${failMsg:+: $failMsg}" done } assertEquals() { (($# == 2 || $# == 3)) || die "assertEqual must 2 or 3 arguments!" local failMsg="" (($# == 3)) && { failMsg=$1 shift } [ "$1" == "$2" ] || fail "assertEqual fail [$1] != [$2]${failMsg:+: $failMsg}" } readonly __ut_exclude_vars_builtin='^BASH_|^_=|^COLUMNS=|LINES=' readonly __ut_exclude_vars_ut_functions='^FUNCNAME=|^test_' assertAllVarsSame() { local test_afterVars test_afterVars=$(declare) diff \ <(echo "$test_beforeVars" | grep -Ev "$__ut_exclude_vars_builtin") \ <(echo "$test_afterVars" | grep -Ev "$__ut_exclude_vars_builtin|$__ut_exclude_vars_ut_functions") || fail "assertAllVarsSame: Unexpected extra global vars!" } assertAllVarsExcludeOptVarsSame() { local test_afterVars test_afterVars=$(declare) diff \ <(echo "$test_beforeVars" | grep -Ev "$__ut_exclude_vars_builtin") \ <(echo "$test_afterVars" | grep -Ev "$__ut_exclude_vars_builtin|$__ut_exclude_vars_ut_functions"'|^_OPT_|^_opts_index_name_') || fail "assertAllVarsExcludeOptVarsSame: Unexpected extra global vars!" } test_beforeVars=$(declare) ================================================ FILE: test/parseOpts_test.sh ================================================ #!/usr/bin/env bash BASE="$(dirname -- "${BASH_SOURCE[0]}")" source "$BASE/../lib/parseOpts.sh" source "$BASE/my_unit_test_lib.sh" ################################################# # Test ################################################# # ======================================== blueEcho "Test case: success parse" # ======================================== parseOpts "a,a-long|b,b-long:|c,c-long+|d,d-long+" aa -a -b bb -c c.sh -p pv -q qv cc \; bb --d-long d.sh -x xv d1 d2 d3 \; cc dd ee test_exitCode=$? _opts_showOptDescInfoList _opts_showOptValueInfoList ((test_exitCode == 0)) || fail "Wrong exit code!" ((${#_OPT_INFO_LIST_INDEX[@]} == 4)) || fail "Wrong _OPT_INFO_LIST_INDEX!" [[ $_OPT_VALUE_a == "true" && $_OPT_VALUE_a_long == "true" ]] || fail "Wrong option value of a!" [[ $_OPT_VALUE_b == "bb" && $_OPT_VALUE_b_long == "bb" ]] || fail "Wrong option value of b!" test_cArray=(c.sh -p pv -q qv cc) assertArrayEquals "Wrong option value of c!" test_cArray _OPT_VALUE_c assertArrayEquals "Wrong option value of c!" test_cArray _OPT_VALUE_c_long test_dArray=(d.sh -x xv d1 d2 d3) assertArrayEquals "Wrong option value of d!" test_dArray _OPT_VALUE_d assertArrayEquals "Wrong option value of d!" test_dArray _OPT_VALUE_d_long test_argArray=(aa bb cc dd ee) assertArrayEquals "Wrong args!" test_argArray _OPT_ARGS assertAllVarsExcludeOptVarsSame _opts_cleanOptValueInfoList assertAllVarsSame # ======================================== blueEcho "Test case: success parse with -- " # ======================================== parseOpts "a,a-long|b,b-long:|c,c-long+|d,d-long+" aa -a -b bb -c c.sh -p pv -q qv cc \; bb -- --d-long d.sh -x xv d1 d2 d3 \; cc dd ee test_exitCode=$? _opts_showOptDescInfoList _opts_showOptValueInfoList ((test_exitCode == 0)) || fail "Wrong exit code!" ((${#_OPT_INFO_LIST_INDEX[@]} == 4)) || fail "Wrong _OPT_INFO_LIST_INDEX!" [[ $_OPT_VALUE_a == "true" && $_OPT_VALUE_a_long == "true" ]] || fail "Wrong option value of a!" [[ $_OPT_VALUE_b == "bb" && $_OPT_VALUE_b_long == "bb" ]] || fail "Wrong option value of b!" test_cArray=(c.sh -p pv -q qv cc) assertArrayEquals "Wrong option value of c!" test_cArray _OPT_VALUE_c assertArrayEquals "Wrong option value of c!" test_cArray _OPT_VALUE_c_long [[ "$_OPT_VALUE_d" == "" && "$_OPT_VALUE_d_long" == "" ]] || fail "Wrong option value of d!" test_argArray=(aa bb --d-long d.sh -x xv d1 d2 d3 \; cc dd ee) assertArrayEquals "Wrong args!" test_argArray _OPT_ARGS assertAllVarsExcludeOptVarsSame _opts_cleanOptValueInfoList assertAllVarsSame # ======================================== blueEcho "Test case: illegal option x" # ======================================== parseOpts "a,a-long|b,b-long:|c,c-long+|d,d-long+" aa -a -b bb -x -c c.sh -p pv -q qv cc \; bb --d-long d.sh -x xv d1 d2 d3 \; cc -- dd ee test_exitCode=$? _opts_showOptDescInfoList _opts_showOptValueInfoList ((test_exitCode == 232)) || fail "Wrong exit code!" ((${#_OPT_INFO_LIST_INDEX[@]} == 0)) || fail "Wrong _OPT_INFO_LIST_INDEX!" [[ "$_OPT_VALUE_a" == "" && "$_OPT_VALUE_a_long" == "" ]] || fail "Wrong option value of a!" [[ "$_OPT_VALUE_b" == "" && "$_OPT_VALUE_b_long" == "" ]] || fail "Wrong option value of b!" [[ "$_OPT_VALUE_c" == "" && "$_OPT_VALUE_c_long" == "" ]] || fail "Wrong option value of c!" [[ "$_OPT_VALUE_d" == "" && "$_OPT_VALUE_d_long" == "" ]] || fail "Wrong option value of d!" [ "$_OPT_ARGS" == "" ] || fail "Wrong args!" # ======================================== blueEcho "Test case: empty options" # ======================================== parseOpts "a,a-long|b,b-long:|c,c-long+|d,d-long+" test_exitCode=$? _opts_showOptDescInfoList _opts_showOptValueInfoList ((test_exitCode == 0)) || fail "Wrong exit code!" ((${#_OPT_INFO_LIST_INDEX[@]} == 4)) || fail "Wrong _OPT_INFO_LIST_INDEX!" [[ "$_OPT_VALUE_a" == "" && "$_OPT_VALUE_a_long" == "" ]] || fail "Wrong option value of a!" [[ "$_OPT_VALUE_b" == "" && "$_OPT_VALUE_b_long" == "" ]] || fail "Wrong option value of b!" [[ "$_OPT_VALUE_c" == "" && "$_OPT_VALUE_c_long" == "" ]] || fail "Wrong option value of c!" [[ "$_OPT_VALUE_d" == "" && "$_OPT_VALUE_d_long" == "" ]] || fail "Wrong option value of d!" [ "$_OPT_ARGS" == "" ] || fail "Wrong args!" # ======================================== blueEcho "Test case: illegal option name" # ======================================== parseOpts "a,a-long|b,b-long:|c,c-long+|d,d-long+|#,z-long" aa -a -b bb -x -c c.sh -p pv -q qv cc \; bb -d d.sh -x xv d1 d2 d3 \; cc -- dd ee test_exitCode=$? _opts_showOptDescInfoList _opts_showOptValueInfoList ((test_exitCode == 221)) || fail "Wrong exit code!" ((${#_OPT_INFO_LIST_INDEX[@]} == 0)) || fail "Wrong _OPT_INFO_LIST_INDEX!" [[ "$_OPT_VALUE_a" == "" && "$_OPT_VALUE_a_long" == "" ]] || fail "Wrong option value of a!" [[ "$_OPT_VALUE_b" == "" && "$_OPT_VALUE_b_long" == "" ]] || fail "Wrong option value of b!" [[ "$_OPT_VALUE_c" == "" && "$_OPT_VALUE_c_long" == "" ]] || fail "Wrong option value of c!" [[ "$_OPT_VALUE_d" == "" && "$_OPT_VALUE_d_long" == "" ]] || fail "Wrong option value of d!" [ "$_OPT_ARGS" == "" ] || fail "Wrong args!" parseOpts "a,a-long|b,b-long:|c,c-long+|d,d-long+|z,z-#long" aa -a -b bb -x -c c.sh -p pv -q qv cc \; bb -d d.sh -x xv d1 d2 d3 \; cc -- dd ee test_exitCode=$? _opts_showOptDescInfoList _opts_showOptValueInfoList ((test_exitCode == 222)) || fail "Wrong exit code!" ((${#_OPT_INFO_LIST_INDEX[@]} == 0)) || fail "Wrong _OPT_INFO_LIST_INDEX!" [[ "$_OPT_VALUE_a" == "" && "$_OPT_VALUE_a_long" == "" ]] || fail "Wrong option value of a!" [[ "$_OPT_VALUE_b" == "" && "$_OPT_VALUE_b_long" == "" ]] || fail "Wrong option value of b!" [[ "$_OPT_VALUE_c" == "" && "$_OPT_VALUE_c_long" == "" ]] || fail "Wrong option value of c!" [[ "$_OPT_VALUE_d" == "" && "$_OPT_VALUE_d_long" == "" ]] || fail "Wrong option value of d!" [ "$_OPT_ARGS" == "" ] || fail "Wrong args!" assertAllVarsSame greenEcho "TEST SUCCESS!!!" ================================================ FILE: test/self-installer.sh ================================================ #!/usr/bin/env bash if [ ! -d "/tmp/useful-scripts-$USER" ]; then if type -P git &>/dev/null; then git clone https://github.com/oldratlee/useful-scripts.git "/tmp/useful-scripts-$USER" elif type -P svn &>/dev/null; then svn checkout https://github.com/oldratlee/useful-scripts/branches/release-3.x "/tmp/useful-scripts-$USER" else echo "fail to find command git/svn" return 1 fi fi export PATH="$PATH:/tmp/useful-scripts-$USER/bin" ================================================ FILE: test/uq_test.sh ================================================ #!/usr/bin/env bash set -eEuo pipefail realpath() { [ -e "$1" ] && command realpath -- "$1" } BASE=$(dirname -- "$(realpath "${BASH_SOURCE[0]}")") cd "$BASE" ################################################# # commons and test data ################################################# readonly uq="../bin/uq" # NOTE: $'foo' is the escape sequence syntax of bash readonly nl=$'\n' # new line test_input=$(cat uq_test_input) ################################################# # test cases ################################################# test_uq_simple() { assertEquals "c${nl}v${nl}a${nl}u" \ "$(echo "$test_input" | "$uq")" assertEquals "c${nl}v${nl}a${nl}u" \ "$("$uq" uq_test_input)" assertEquals "c${nl}a" \ "$(echo "$test_input" | "$uq" -d)" assertEquals "c${nl}a" \ "$("$uq" -d uq_test_input)" assertEquals "v${nl}u" "$(echo "$test_input" | "$uq" -u)" assertEquals "v${nl}u" "$("$uq" -u uq_test_input)" } readonly test_output_uq_count=' 4 c 1 v 2 a 1 u' readonly test_output_uq_D_count=' 4 c 4 c 1 v 2 a 2 a 4 c 4 c 1 u' test_uq_count() { assertEquals "$test_output_uq_count" "$(echo "$test_input" | "$uq" -c)" assertEquals "$test_output_uq_count" "$("$uq" -c uq_test_input)" assertEquals "$test_output_uq_D_count" "$(echo "$test_input" | "$uq" -D -c)" assertEquals "$test_output_uq_D_count" "$("$uq" -D -c uq_test_input)" } test_uq_only_D_option__same_as_cat() { assertEquals "$test_input" "$(echo "$test_input" | "$uq" -D)" assertEquals "$test_input" "$("$uq" -D uq_test_input)" } test_multi_input_files__output_file() { local output_file="$SHUNIT_TMPDIR/uq_output_file_${$}_${RANDOM}_${RANDOM}" "$uq" uq_test_input uq_test_another_input "$output_file" assertEquals "c${nl}v${nl}a${nl}u${nl}m${nl}x" \ "$(cat "$output_file")" local output_file="$SHUNIT_TMPDIR/uq_output_file_${$}_${RANDOM}_${RANDOM}" "$uq" -d uq_test_input uq_test_another_input "$output_file" assertEquals "c${nl}a${nl}m" \ "$(cat "$output_file")" local output_file="$SHUNIT_TMPDIR/uq_output_file_${$}_${RANDOM}_${RANDOM}" "$uq" -u uq_test_input uq_test_another_input "$output_file" assertEquals "v${nl}u${nl}x" \ "$(cat "$output_file")" } test_multi_input_files__output_stdout() { assertEquals "c${nl}v${nl}a${nl}u${nl}m${nl}x" "$("$uq" uq_test_input uq_test_another_input -)" assertEquals "c${nl}a${nl}m" "$("$uq" -d uq_test_input uq_test_another_input -)" assertEquals "v${nl}u${nl}x" "$("$uq" -u uq_test_input uq_test_another_input -)" } test_ignore_case() { local input="a${nl}b${nl}A" assertEquals "a${nl}b${nl}A" "$(echo "$input" | "$uq")" assertEquals "a${nl}b" "$(echo "$input" | "$uq" -i)" } test_ignore_case__count() { local input="a${nl}b${nl}A" assertEquals " 1 a${nl} 1 b${nl} 1 A" \ "$(echo "$input" | "$uq" -c)" assertEquals " 2 a${nl} 1 b" \ "$(echo "$input" | "$uq" -i -c)" assertEquals " 2 a${nl} 1 b${nl} 2 A" \ "$(echo "$input" | "$uq" -i -D -c)" } test_max_input_check() { # shellcheck disable=SC2016 assertTrue 'echo 123 | "$uq"' # shellcheck disable=SC2016 assertTrue 'echo 123 | "$uq" -XM 4' # shellcheck disable=SC2016 assertTrue 'echo 123 | "$uq" -XM 1k' # shellcheck disable=SC2016 assertTrue 'echo 123 | "$uq" --max-input 1042k' # shellcheck disable=SC2016 assertTrue 'echo 123 | "$uq" --max-input 1m' # shellcheck disable=SC2016 assertTrue 'echo 123 | "$uq" --max-input 10420g' # shellcheck disable=SC2016 assertTrue '"$uq" uq_test_input' # shellcheck disable=SC2016 assertTrue '"$uq" uq_test_input -XM 42m' # shellcheck disable=SC2016 assertTrue '"$uq" uq_test_input --max-input 1024000g' # shellcheck disable=SC2016 assertTrue '"$uq" uq_test_input --max-input 1234567890g' # shellcheck disable=SC2016 assertFalse 'should fail by -XM' 'echo -e 123 | "$uq" -XM 1' # shellcheck disable=SC2016 assertFalse 'should fail by -XM' 'echo -e 123 | "$uq" -XM 3' # shellcheck disable=SC2016 assertFalse 'should fail by --max-input' 'echo -e 123 | "$uq" --max-input 2' # shellcheck disable=SC2016 assertFalse 'should fail by --max-input' '"$uq" --max-input 2 uq_test_input' # shellcheck disable=SC2016 assertFalse 'should fail, number overflow!' '"$uq" uq_test_input --max-input 12345678901g' } ################################################# # Load and run shUnit2. ################################################# source "$BASE/shunit2-lib/shunit2" ================================================ FILE: test/uq_test_another_input ================================================ a m m x ================================================ FILE: test/uq_test_input ================================================ c c v a a c c u