Showing preview only (1,196K chars total). Download the full file or copy to clipboard to get everything.
Repository: dspinellis/dgsh
Branch: master
Commit: e51fb999119b
Files: 460
Total size: 1.1 MB
Directory structure:
gitextract_fo8a72ky/
├── .gitignore
├── .gitmodules
├── .travis/
│ ├── linux-ubuntu-trusty.install.sh
│ └── macosx-xcode8.3.install.sh
├── .travis.yml
├── CITATION.cff
├── LICENSE
├── Makefile
├── README.md
├── core-tools/
│ ├── .gitignore
│ ├── Makefile.am
│ ├── configure.ac
│ ├── src/
│ │ ├── .gitignore
│ │ ├── Makefile.am
│ │ ├── debug.h
│ │ ├── dgsh-conc.1
│ │ ├── dgsh-conc.c
│ │ ├── dgsh-debug.h
│ │ ├── dgsh-elf.s
│ │ ├── dgsh-enumerate.1
│ │ ├── dgsh-enumerate.c
│ │ ├── dgsh-fft-input.c
│ │ ├── dgsh-httpval.1
│ │ ├── dgsh-httpval.c
│ │ ├── dgsh-macho.s
│ │ ├── dgsh-merge-sum.1
│ │ ├── dgsh-merge-sum.pl
│ │ ├── dgsh-monitor.1
│ │ ├── dgsh-monitor.c
│ │ ├── dgsh-parallel.1
│ │ ├── dgsh-parallel.sh
│ │ ├── dgsh-pecho.c
│ │ ├── dgsh-readval.1
│ │ ├── dgsh-readval.c
│ │ ├── dgsh-tee.1
│ │ ├── dgsh-tee.c
│ │ ├── dgsh-w.c
│ │ ├── dgsh-wrap.1
│ │ ├── dgsh-wrap.c
│ │ ├── dgsh-writeval.1
│ │ ├── dgsh-writeval.c
│ │ ├── dgsh.1
│ │ ├── dgsh.h
│ │ ├── dgsh_negotiate.3
│ │ ├── kvstore.c
│ │ ├── kvstore.h
│ │ ├── minmax.h
│ │ ├── negotiate.c
│ │ ├── negotiate.h
│ │ ├── perm.1
│ │ ├── perm.sh
│ │ └── run-built-dgsh.sh
│ ├── tests/
│ │ ├── .gitignore
│ │ ├── Makefile.am
│ │ ├── Makefile.patch
│ │ ├── check.hack
│ │ └── check_negotiate.c
│ └── tests-regression/
│ ├── .gitignore
│ ├── author-compare/
│ │ └── out.ok
│ ├── bin/
│ │ └── gdate
│ ├── code-metrics/
│ │ ├── in/
│ │ │ └── date/
│ │ │ ├── date.c
│ │ │ ├── extern.h
│ │ │ ├── netdate.c
│ │ │ ├── vary.c
│ │ │ └── vary.h
│ │ └── out.ok
│ ├── commit-stats/
│ │ └── out.ok
│ ├── compress-compare/
│ │ └── out.ok
│ ├── dgsh-wrap/
│ │ ├── .gitignore
│ │ ├── dd-args.ok
│ │ ├── echo-deaf.ok
│ │ ├── echo-s.ok
│ │ ├── echo-s_caps.ok
│ │ ├── paste1.ok
│ │ ├── paste2.ok
│ │ ├── paste3.ok
│ │ ├── paste4.ok
│ │ ├── tee1.ok
│ │ └── tee2.ok
│ ├── duplicate-files/
│ │ ├── in/
│ │ │ ├── another-same-1
│ │ │ ├── another-same-2
│ │ │ ├── different-file-1
│ │ │ ├── different-file-2
│ │ │ ├── different-file-3
│ │ │ ├── same-file-1
│ │ │ ├── same-file-2
│ │ │ └── same-file-3
│ │ └── out.ok
│ ├── map-hierarchy/
│ │ ├── in/
│ │ │ ├── a/
│ │ │ │ └── date/
│ │ │ │ ├── date.c
│ │ │ │ ├── vary.c
│ │ │ │ └── vary.h
│ │ │ └── b/
│ │ │ └── bin/
│ │ │ └── date/
│ │ │ ├── date.c
│ │ │ ├── headers/
│ │ │ │ ├── extern.h
│ │ │ │ └── vary.h
│ │ │ ├── netdate.c
│ │ │ └── vary.c
│ │ └── out.ok/
│ │ └── map-hierarchy/
│ │ └── in/
│ │ └── b/
│ │ └── bin/
│ │ └── date/
│ │ ├── date.c
│ │ ├── headers/
│ │ │ └── vary.h
│ │ └── vary.c
│ ├── parallel-word-count/
│ │ └── out.ok
│ ├── regression/
│ │ ├── errors/
│ │ │ ├── stream-scatter-cycle.ok
│ │ │ ├── stream-scatter-cycle.sh
│ │ │ ├── unsafe-gather.ok
│ │ │ ├── unsafe-gather.sh
│ │ │ ├── unsafe-gather2.ok
│ │ │ └── unsafe-gather2.sh
│ │ ├── graphs/
│ │ │ ├── NMRPipe.ok
│ │ │ ├── code-metrics.ok
│ │ │ ├── commit-stats.ok
│ │ │ ├── committer-plot.ok
│ │ │ ├── compress-compare.ok
│ │ │ ├── dir.ok
│ │ │ ├── duplicate-files.ok
│ │ │ ├── ft2d.ok
│ │ │ ├── map-hierarchy.ok
│ │ │ ├── parallel-logresolve.ok
│ │ │ ├── spell-highlight.ok
│ │ │ ├── text-properties.ok
│ │ │ ├── web-log-report.ok
│ │ │ ├── web-log-stats.ok
│ │ │ └── word-properties.ok
│ │ ├── scripts/
│ │ │ ├── NMRPipe.ok
│ │ │ ├── code-metrics.ok
│ │ │ ├── commit-stats.ok
│ │ │ ├── committer-plot.ok
│ │ │ ├── compress-compare.ok
│ │ │ ├── dir.ok
│ │ │ ├── duplicate-files.ok
│ │ │ ├── ft2d.ok
│ │ │ ├── map-hierarchy.ok
│ │ │ ├── parallel-logresolve.ok
│ │ │ ├── spell-highlight.ok
│ │ │ ├── text-properties.ok
│ │ │ ├── web-log-report.ok
│ │ │ ├── web-log-stats.ok
│ │ │ └── word-properties.ok
│ │ └── warnings/
│ │ ├── single-target.ok
│ │ ├── single-target.sh
│ │ ├── unsafe-scatter.ok
│ │ └── unsafe-scatter.sh
│ ├── spell-highlight/
│ │ └── out.ok
│ ├── tee/
│ │ ├── oom.err
│ │ ├── perm.ok
│ │ ├── tee-fastout-I-l.ok
│ │ ├── tee-fastout-I-m 2k -f.ok
│ │ ├── tee-fastout-I-m 2k.ok
│ │ ├── tee-fastout-I.ok
│ │ ├── tee-fastout-l.ok
│ │ ├── tee-fastout-m 2k -f.ok
│ │ ├── tee-fastout-m 2k.ok
│ │ ├── tee-fastout.ok
│ │ ├── tee-lagout-I-l.ok
│ │ ├── tee-lagout-I-m 2k -f.ok
│ │ ├── tee-lagout-I-m 2k.ok
│ │ ├── tee-lagout-I.ok
│ │ ├── tee-lagout-l.ok
│ │ ├── tee-lagout-m 2k -f.ok
│ │ ├── tee-lagout-m 2k.ok
│ │ ├── tee-lagout.ok
│ │ └── tee-lahout.ok
│ ├── test-dgsh.sh
│ ├── test-kvstore.sh
│ ├── test-merge-sum.sh
│ ├── test-tee.sh
│ ├── test-wrap.sh
│ ├── text-properties/
│ │ └── out.ok/
│ │ ├── character.txt
│ │ ├── digram.txt
│ │ ├── trigram.txt
│ │ └── words.txt
│ ├── web-log-report/
│ │ ├── logfile
│ │ └── out.ok
│ └── word-properties/
│ ├── LostWorldChap1-3
│ └── out.ok
├── eval/
│ ├── .gitignore
│ ├── Makefile
│ ├── README.md
│ ├── SConstruct
│ ├── TextProperties.java
│ ├── WebStats.java
│ ├── eval-lib.sh
│ ├── ft2d.sh
│ ├── log-grow.pl
│ ├── perf-eval.sh
│ ├── web-log-report.pl
│ └── webeval.sh
├── example/
│ ├── NMRPipe.sh
│ ├── author-compare.sh
│ ├── code-metrics.sh
│ ├── commit-stats.sh
│ ├── committer-plot.sh
│ ├── compress-compare.sh
│ ├── dir.sh
│ ├── duplicate-files.sh
│ ├── fft-block8.sh
│ ├── ft2d.sh
│ ├── map-hierarchy.sh
│ ├── parallel-word-count.sh
│ ├── reorder-columns.sh
│ ├── spell-highlight.sh
│ ├── static-functions.sh
│ ├── text-properties.sh
│ ├── uniform-5x5.sh
│ ├── web-log-report.sh
│ └── word-properties.sh
├── png/
│ └── README
├── simple-shell/
│ ├── comm_paste.dgsh
│ ├── comm_paste.success
│ ├── comm_paste_join_diff.dgsh
│ ├── comm_paste_join_diff.success
│ ├── comm_sort.dgsh
│ ├── comm_sort.success
│ ├── dir-plain.dgsh
│ ├── grep_comm.dgsh
│ ├── grep_comm.success
│ ├── grep_comm.success-bash
│ ├── grep_diff.dgsh
│ ├── grep_diff.success
│ ├── grep_diff.success-bash
│ ├── grep_diff_comm.dgsh
│ ├── grep_diff_comm.success1
│ ├── grep_diff_comm.success1-bash
│ ├── grep_diff_comm.success2
│ ├── grep_diff_comm.success2-bash
│ ├── grep_diff_comm.success3
│ ├── grep_diff_comm.success3-bash
│ ├── grep_diff_comm.success4
│ ├── grep_diff_comm.success4-bash
│ ├── join_sort.dgsh
│ ├── join_sort.success
│ ├── join_sort_diff.dgsh
│ ├── join_sort_diff.success
│ ├── ls_wc.dgsh
│ ├── paste_diff.dgsh
│ ├── paste_diff.success
│ ├── secho_paste.dgsh
│ ├── secho_paste.success
│ ├── secho_secho_fgrep.dgsh
│ ├── secho_secho_fgrep.success
│ ├── simple-shell.py
│ ├── sort_sort_comm.dgsh
│ ├── sort_sort_comm.success
│ ├── sort_sort_comm_paste_join_diff.dgsh
│ ├── sort_sort_comm_paste_join_diff.success
│ ├── tee-copy_diff_comm.dgsh
│ ├── tee-copy_diff_comm.success
│ ├── tee-scatter_diff_comm.dgsh
│ ├── tee-scatter_diff_comm.success1
│ ├── tee-scatter_diff_comm.success2
│ ├── wrap-cat_comm_sort.dgsh
│ └── wrap-cat_comm_sort.success
├── test-data/
│ ├── .gitignore
│ ├── F
│ ├── access.log
│ ├── cmp0.success
│ ├── cmp1-same1.success
│ ├── cmp1-same2.success
│ ├── cmp2-diff.success
│ ├── cmp2-same.success
│ ├── comm_paste.success
│ ├── comm_paste_join_diff.success
│ ├── comm_sort.success
│ ├── d3
│ ├── data.csv
│ ├── diff0-noin.success
│ ├── diff0-stdin1.success
│ ├── diff0-stdin2.success
│ ├── diff0.success
│ ├── diff1-stdin.success
│ ├── diff1.success
│ ├── diff2.success
│ ├── diff3-0.success
│ ├── diff3-1.success
│ ├── diff3-2-stdin1.success
│ ├── diff3-2-stdin2.success
│ ├── diff3-2.success
│ ├── diff3-3.success
│ ├── diff4.success
│ ├── dir-plain.sh
│ ├── f1s
│ ├── f2s
│ ├── f3s
│ ├── f4s
│ ├── f4ss
│ ├── f5s
│ ├── f5ss
│ ├── ff
│ ├── function_bash_tools.success
│ ├── function_dgsh_tools.success
│ ├── grep-Lcap-c-l-matching-lines-nomatch.success
│ ├── grep-Lcap-c-l-matching-lines.success
│ ├── grep-Lcap-c-l-nomatch.success
│ ├── grep-Lcap-c-l.success
│ ├── grep-Lcap-c-matching-lines-l-nomatch.success
│ ├── grep-Lcap-c-matching-lines-l.success
│ ├── grep-Lcap-c-matching-lines-nomatch.success
│ ├── grep-Lcap-c-matching-lines.success
│ ├── grep-Lcap-c-nomatch.success
│ ├── grep-Lcap-c.success
│ ├── grep-Lcap-cat-nomatch.success
│ ├── grep-Lcap-cat.success
│ ├── grep-Lcap-l-c-matching-lines-nomatch.success
│ ├── grep-Lcap-l-c-matching-lines.success
│ ├── grep-Lcap-l-c-nomatch.success
│ ├── grep-Lcap-l-c.success
│ ├── grep-Lcap-l-matching-lines-c-nomatch.success
│ ├── grep-Lcap-l-matching-lines-c.success
│ ├── grep-Lcap-l-matching-lines-nomatch.success
│ ├── grep-Lcap-l-matching-lines.success
│ ├── grep-Lcap-l-nomatch.success
│ ├── grep-Lcap-l.success
│ ├── grep-Lcap-matching-lines-c-l-nomatch.success
│ ├── grep-Lcap-matching-lines-c-l.success
│ ├── grep-Lcap-matching-lines-c-nomatch.success
│ ├── grep-Lcap-matching-lines-c.success
│ ├── grep-Lcap-matching-lines-l-c-nomatch.success
│ ├── grep-Lcap-matching-lines-l-c.success
│ ├── grep-Lcap-matching-lines-l-nomatch.success
│ ├── grep-Lcap-matching-lines-l.success
│ ├── grep-Lcap-matching-lines-nomatch.success
│ ├── grep-Lcap-matching-lines.success
│ ├── grep-Lcap-nomatch.success
│ ├── grep-Lcap.success
│ ├── grep-c-Lcap-l-matching-lines-nomatch.success
│ ├── grep-c-Lcap-l-matching-lines.success
│ ├── grep-c-Lcap-l-nomatch.success
│ ├── grep-c-Lcap-l.success
│ ├── grep-c-Lcap-matching-lines-l-nomatch.success
│ ├── grep-c-Lcap-matching-lines-l.success
│ ├── grep-c-Lcap-matching-lines-nomatch.success
│ ├── grep-c-Lcap-matching-lines.success
│ ├── grep-c-Lcap-nomatch.success
│ ├── grep-c-Lcap.success
│ ├── grep-c-cat.success
│ ├── grep-c-l-Lcap-matching-lines-nomatch.success
│ ├── grep-c-l-Lcap-matching-lines.success
│ ├── grep-c-l-Lcap-nomatch.success
│ ├── grep-c-l-Lcap.success
│ ├── grep-c-l-matching-lines-Lcap-nomatch.success
│ ├── grep-c-l-matching-lines-Lcap.success
│ ├── grep-c-l-matching-lines.success
│ ├── grep-c-l.success
│ ├── grep-c-matching-lines-Lcap-l-nomatch.success
│ ├── grep-c-matching-lines-Lcap-l.success
│ ├── grep-c-matching-lines-Lcap-nomatch.success
│ ├── grep-c-matching-lines-Lcap.success
│ ├── grep-c-matching-lines-l-Lcap-nomatch.success
│ ├── grep-c-matching-lines-l-Lcap.success
│ ├── grep-c-matching-lines-l.success
│ ├── grep-c-matching-lines.success
│ ├── grep-c.success
│ ├── grep-f-cat.success
│ ├── grep-f.success
│ ├── grep-l-Lcap-c-matching-lines-nomatch.success
│ ├── grep-l-Lcap-c-matching-lines.success
│ ├── grep-l-Lcap-c-nomatch.success
│ ├── grep-l-Lcap-c.success
│ ├── grep-l-Lcap-matching-lines-c-nomatch.success
│ ├── grep-l-Lcap-matching-lines-c.success
│ ├── grep-l-Lcap-matching-lines-nomatch.success
│ ├── grep-l-Lcap-matching-lines.success
│ ├── grep-l-Lcap-nomatch.success
│ ├── grep-l-Lcap.success
│ ├── grep-l-c-Lcap-matching-lines-nomatch.success
│ ├── grep-l-c-Lcap-matching-lines.success
│ ├── grep-l-c-Lcap-nomatch.success
│ ├── grep-l-c-Lcap.success
│ ├── grep-l-c-matching-lines-Lcap-nomatch.success
│ ├── grep-l-c-matching-lines-Lcap.success
│ ├── grep-l-c-matching-lines.success
│ ├── grep-l-c.success
│ ├── grep-l-cat.success
│ ├── grep-l-matching-lines-Lcap-c-nomatch.success
│ ├── grep-l-matching-lines-Lcap-c.success
│ ├── grep-l-matching-lines-Lcap-nomatch.success
│ ├── grep-l-matching-lines-Lcap.success
│ ├── grep-l-matching-lines-c-Lcap-nomatch.success
│ ├── grep-l-matching-lines-c-Lcap.success
│ ├── grep-l-matching-lines-c.success
│ ├── grep-l-matching-lines.success
│ ├── grep-l.success
│ ├── grep-matching-lines-Lcap-c-l-nomatch.success
│ ├── grep-matching-lines-Lcap-c-l.success
│ ├── grep-matching-lines-Lcap-c-nomatch.success
│ ├── grep-matching-lines-Lcap-c.success
│ ├── grep-matching-lines-Lcap-l-c-nomatch.success
│ ├── grep-matching-lines-Lcap-l-c.success
│ ├── grep-matching-lines-Lcap-l-nomatch.success
│ ├── grep-matching-lines-Lcap-l.success
│ ├── grep-matching-lines-Lcap-nomatch.success
│ ├── grep-matching-lines-Lcap.success
│ ├── grep-matching-lines-c-Lcap-l-nomatch.success
│ ├── grep-matching-lines-c-Lcap-l.success
│ ├── grep-matching-lines-c-Lcap-nomatch.success
│ ├── grep-matching-lines-c-Lcap.success
│ ├── grep-matching-lines-c-l-Lcap-nomatch.success
│ ├── grep-matching-lines-c-l-Lcap.success
│ ├── grep-matching-lines-c-l.success
│ ├── grep-matching-lines-c.success
│ ├── grep-matching-lines-cat.success
│ ├── grep-matching-lines-l-Lcap-c-nomatch.success
│ ├── grep-matching-lines-l-Lcap-c.success
│ ├── grep-matching-lines-l-Lcap-nomatch.success
│ ├── grep-matching-lines-l-Lcap.success
│ ├── grep-matching-lines-l-c-Lcap-nomatch.success
│ ├── grep-matching-lines-l-c-Lcap.success
│ ├── grep-matching-lines-l-c.success
│ ├── grep-matching-lines-l.success
│ ├── grep-matching-lines.success
│ ├── grep-noargs-cat.success
│ ├── grep-noargs.success
│ ├── grep-o-cat.success
│ ├── grep-o.success
│ ├── grep-v-cat.success
│ ├── grep-v.success
│ ├── grep_comm.success
│ ├── group.success
│ ├── hello
│ ├── j2
│ ├── join_sort.success
│ ├── join_sort_diff.success
│ ├── last
│ ├── multipipe_one_last.success
│ ├── multipipe_one_start.success
│ ├── nondgsh.success
│ ├── p1
│ ├── paste_diff.success
│ ├── read_while.success
│ ├── recursive_multipipe_oneline_end.success
│ ├── recursive_multipipe_oneline_start.success
│ ├── results
│ ├── secho_paste.success
│ ├── secho_secho_fgrep.success
│ ├── sort_sort_comm.success
│ ├── sort_sort_comm_paste_join_diff.success
│ ├── subshell.success
│ ├── tee-copy_diff_comm.success
│ ├── top
│ ├── world
│ └── wrap-cat_comm_sort.success
├── unix-tools/
│ ├── .gitignore
│ ├── Makefile
│ ├── Readme.md
│ ├── cat.sh
│ ├── cmp.sh
│ ├── cpow.c
│ ├── cygwin-sys-select-patch.sh
│ ├── diff.sh
│ ├── diff3.sh
│ ├── echo_echo_dgsh-tee.sh
│ ├── install-wrapped.sh
│ ├── run_all_simple_tests.sh
│ ├── run_simple_test.sh
│ ├── run_test.sh
│ ├── tee.sh
│ ├── test-compat.sh
│ ├── wrapped-commands-posix
│ └── wrapped-commands-tests
└── web/
├── format-eg.sh
├── format-syntax.sh
└── index.html
================================================
FILE CONTENTS
================================================
================================================
FILE: .gitignore
================================================
*.dot
*.exe
*.o
*.out
*.outb
*.pdf
*.swp
*.test
.deps
.config
.gdb_history
/build
/png/*.png
tags
unix-tools/sf*
================================================
FILE: .gitmodules
================================================
[submodule "unix-tools/coreutils"]
path = unix-tools/coreutils
url = https://github.com/mfragkoulis/coreutils
branch = master
ignore=dirty
[submodule "unix-tools/grep"]
path = unix-tools/grep
url = https://github.com/mfragkoulis/grep
branch = master
ignore=dirty
[submodule "unix-tools/bash"]
path = unix-tools/bash
url = https://github.com/mfragkoulis/bash.git
branch = master
ignore=dirty
================================================
FILE: .travis/linux-ubuntu-trusty.install.sh
================================================
#!/bin/bash
sudo apt-get -qq update
# For installation
sudo apt-get install -y make automake gcc libtool pkg-config texinfo help2man autopoint bison check gperf
# For testing
sudo apt-get install -y wbritish wamerican libfftw3-dev csh
wget http://ftp.gnu.org/gnu/gettext/gettext-0.19.5.tar.xz
tar Jxvf gettext-0.19.5.tar.xz
cd gettext-0.19.5 && ./configure && make && sudo make install
cd ..
git clone --depth=1 -b madagascar-devel-2016 https://github.com/ahay/src.git madagascar
cd madagascar && rm -rf trip
sudo ./configure --prefix=/usr/local && sudo make && sudo make install
cd ..
mkdir nmrpipe && cd nmrpipe
wget https://www.ibbr.umd.edu/nmrpipe/install.com
wget https://www.ibbr.umd.edu/nmrpipe/binval.com
wget https://www.ibbr.umd.edu/nmrpipe/NMRPipeX.tZ
wget https://www.ibbr.umd.edu/nmrpipe/s.tZ
wget https://www.ibbr.umd.edu/nmrpipe/dyn.tZ
wget https://www.ibbr.umd.edu/nmrpipe/talos.tZ
wget http://spin.niddk.nih.gov/bax/software/smile/plugin.smile.tZ
chmod a+rx *.com && ./install.com
sudo install nmrbin.linux212_64/var2pipe nmrbin.linux212_64/nmrPipe /usr/local/bin
sudo install nmrbin.linux212_64/addNMR /usr/bin
cd ..
make config && make
================================================
FILE: .travis/macosx-xcode8.3.install.sh
================================================
#!/bin/bash
set -x
brew update
brew install autoconf check xz texinfo help2man gettext libelf
export PATH="/usr/local/opt/texinfo/bin:$PATH"
brew link gettext --force
#wget http://ftp.gnu.org/gnu/gettext/gettext-0.19.5.tar.xz
#tar Jxvf gettext-0.19.5.tar.xz >/dev/null
#cd gettext-0.19.5 && ./configure && make && sudo make install >/dev/null
#cd ..
#git clone --depth=1 -b madagascar-devel-2016 https://github.com/ahay/src.git madagascar
#cd madagascar && rm -rf trip
#sudo ./configure --prefix=/usr/local && sudo make && sudo make install
#cd ..
#mkdir nmrpipe && cd nmrpipe
#wget https://www.ibbr.umd.edu/nmrpipe/install.com
#wget https://www.ibbr.umd.edu/nmrpipe/binval.com
#wget https://www.ibbr.umd.edu/nmrpipe/NMRPipeX.tZ
#wget https://www.ibbr.umd.edu/nmrpipe/s.tZ
#wget https://www.ibbr.umd.edu/nmrpipe/dyn.tZ
#wget https://www.ibbr.umd.edu/nmrpipe/talos.tZ
#wget http://spin.niddk.nih.gov/bax/software/smile/plugin.smile.tZ
#chmod a+rx *.com && ./install.com >/dev/null
#sudo install nmrbin.linux212_64/var2pipe nmrbin.linux212_64/nmrPipe /usr/local/bin >/dev/null
#sudo install nmrbin.linux212_64/addNMR /usr/bin >/dev/null
#cd ..
make config && make
================================================
FILE: .travis.yml
================================================
matrix:
include:
- os: linux
dist: trusty
install:
- ./.travis/linux-ubuntu-trusty.install.sh
- os: osx
osx_image: xcode8.3
install:
- ./.travis/macosx-xcode8.3.install.sh
language: C
compiler: gcc
script: make test && sudo make install
================================================
FILE: CITATION.cff
================================================
cff-version: 1.2.0
title: dgsh: The Directed Graph Shell
message: >-
If you use this software, please cite, using the metadata from this file,
both the article from preferred-citation and the software itself.
preferred-citation:
date-released: "2017-09-01"
doi: "10.1109/TC.2017.2695447"
title: "Extending Unix Pipelines to DAGs"
authors:
- family-names: "Spinellis"
given-names: "Diomidis"
- family-names: "Fragkoulis"
given-names: "Marios"
type: "article"
volume: 66
issue: 9
journal: "IEEE Transactions on Computers"
start: 1547
end: 1561
type: software
authors:
- given-names: Diomidis
family-names: Spinellis
email: dds@aueb.gr
affiliation: Athens University of Economics and Business
orcid: 'https://orcid.org/0000-0003-4231-1897'
- given-names: "Marios"
family-names: "Fragkoulis"
repository-code: 'https://github.com/dspinellis/dgsh'
keywords:
- Unix
- shell
- dataflow programming
license: Apache-2.0
================================================
FILE: LICENSE
================================================
Copyright 2012-2013 Diomidis Spinellis
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
------------------------------------------------------------------------
jquery.js is licensed as follows.
Copyright 2013 jQuery Foundation and other contributors
http://jquery.com/
Permission is hereby granted, free of charge, to any person obtaining
a copy of this software and associated documentation files (the
"Software"), to deal in the Software without restriction, including
without limitation the rights to use, copy, modify, merge, publish,
distribute, sublicense, and/or sell copies of the Software, and to
permit persons to whom the Software is furnished to do so, subject to
the following conditions:
The above copyright notice and this permission notice shall be
included in all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
------------------------------------------------------------------------
dgsh-httpval is based on micro_httpd, which is licensed as follows.
micro_httpd:
Copyright (c) 1999,2005 by Jef Poskanzer <jef@mail.acme.com>.
All rights reserved.
Dgsh modifications:
Copyright 2013 Diomidis Spinellis
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions
are met:
1. Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
2. Redistributions in binary form must reproduce the above copyright
notice, this list of conditions and the following disclaimer in the
documentation and/or other materials provided with the distribution.
THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
SUCH DAMAGE.
================================================
FILE: Makefile
================================================
#
# Copyright 2012-2013 Diomidis Spinellis
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
-include .config
export PREFIX?=/usr/local
ifdef DEBUG
CPPFLAGS=-DDEBUG
CXXFLAGS=-g -Wall -O0
else
CXXFLAGS=-O -Wall
endif
ifdef TIME
CFLAGS+=-DTIME
endif
DOTFLAGS=-Nfontname=Arial -Ngradientangle=90 -Nstyle=filled -Nshape=ellipse -Nfillcolor=yellow:white
# Manual pages
MAN1SRC=$(wildcard core-tools/src/*.1)
MANPDF=$(patsubst %.1,%.pdf,$(MAN1SRC)) core-tools/src/dgsh_negotiate.pdf
MANHTML=$(patsubst %.1,%.html,$(MAN1SRC)) core-tools/src/dgsh_negotiate.html
# Web files
EXAMPLES=$(patsubst example/%,%,$(wildcard example/*.sh))
EGPNG=$(patsubst %.sh,png/%-pretty.png,$(EXAMPLES))
EGDOT=$(patsubst %.sh,graphdot/%.dot,$(EXAMPLES))
WEBPNG=$(EGPNG)
WEBDIST=../../../pubs/web/home/sw/dgsh/
png/%-pretty.png: graphdot/%.dot
gvpr 'BEG_G { graph_t L = cloneG($$G,"last")} END {write(L)}' $< | \
dot $(DOTFLAGS) -Tpng >$@
%.pdf: %.1
groff -man -Tps $< | ps2pdf - $@
%.pdf: %.3
groff -man -Tps $< | ps2pdf - $@
%.html: %.1
groff -man -Thtml $< >$@
%.html: %.3
groff -man -Thtml $< >$@
graphdot/%.dot: example/%.sh
mkdir -p graphdot
rm -f graphdot/$*.dot
DGSH_DRAW_EXIT=1 DGSH_DOT_DRAW=graphdot/$* ./unix-tools/bash/bash --dgsh $<
.PHONY: all tools core-tools unix-tools export-prefix \
config config-core-tools \
test test-dgsh test-merge-sum test-tee test-negotiate \
test-unix-tools test-wrap test-kvstore \
clean install webfiles dist pull commit uninstall dotfiles
all: tools
tools: core-tools unix-tools
core-tools:
$(MAKE) -C core-tools CFLAGS="$(CFLAGS)"
cd core-tools/src && $(MAKE) build-install
unix-tools: core-tools
$(MAKE) -C unix-tools make MAKEFLAGS=
$(MAKE) -C unix-tools build-install
export-prefix:
echo "export PREFIX?=$(PREFIX)" >.config
config: export-prefix config-core-tools
$(MAKE) -C unix-tools configure
config-core-tools: core-tools/configure.ac core-tools/Makefile.am core-tools/src/Makefile.am core-tools/tests/Makefile.am
-mkdir core-tools/m4
cd core-tools && \
autoreconf --install && \
./configure --prefix=$(PREFIX) \
--bindir=$(PREFIX)/bin && \
cd tests && \
patch Makefile <Makefile.patch
test: test-negotiate test-tee test-kvstore test-unix-tools test-merge-sum test-wrap test-dgsh
test-dgsh: tools
cd core-tools/tests-regression && ./test-dgsh.sh
test-wrap: tools
cd core-tools/tests-regression && ./test-wrap.sh
test-merge-sum:
cd core-tools/tests-regression && ./test-merge-sum.sh
test-tee: tools
cd core-tools/tests-regression && ./test-tee.sh
test-negotiate: tools
cd core-tools/tests && \
$(MAKE) && \
$(MAKE) check
test-unix-tools: tools
$(MAKE) -C unix-tools -s test
test-kvstore: core-tools
cd core-tools/tests-regression && ./test-kvstore.sh
clean:
rm -rf build $(MANPDF) $(MANHTML) $(EGPNG)
$(MAKE) -C core-tools clean
$(MAKE) -C unix-tools clean
install:
-rm -r build
$(MAKE) -C core-tools install
$(MAKE) -C unix-tools install
webfiles: $(MANPDF) $(MANHTML) $(WEBPNG)
# Create web page
dist: $(MANPDF) $(MANHTML) $(WEBPNG)
perl -n -e 'if (/^<!-- #!(.*) -->/) { system("$$1"); } else { print; }' web/index.html >$(WEBDIST)/index.html
cp $(MANHTML) $(MANPDF) $(WEBDIST)
cp $(WEBPNG) $(WEBDIST)
# Obtain dot files from Unix host
rsync-graphdot:
ssh stereo 'cd src/dgsh && git pull && make webfiles'
rsync -a stereo:src/dgsh/graphdot/ graphdot/
rsync -a stereo:src/dgsh/example/ example/
dotfiles: $(EGDOT)
pull:
git pull
# Pull master on all sub-repositories.
# Note that the gnulib ones get detached by by builds specifying
# a specific gnulib version. Through this pull repos on master
# stay on master; detached repos (gnulib) stay in the version they
# were detached.
git submodule status --recursive | awk '{print $$2}' | sort -r | while read d ; do ( echo "Pulling $$d" && cd $$d && old=$$(if [ $$(git rev-parse master) = $$(git rev-parse HEAD) ] ; then echo master ; else git rev-parse HEAD ; fi) && git checkout master && git pull && git checkout -q $$old ) ; done
push:
git push --recurse-submodules=on-demand
commit:
# Commit -a including submodules
printf '\n\n# Please enter the commit message for your changes.\n#\n' >.git/COMMIT_EDITMSG
git status --ignore-submodules=untracked | sed '/./s/^/# /;s/^$$/#/' >>.git/COMMIT_EDITMSG
$${VISUAL-vi} .git/COMMIT_EDITMSG
for i in $$(echo unix-tools/*/.git | sed 's/\.git//g') . ; do grep -v '^#' .git/COMMIT_EDITMSG | (cd $$i && git commit -a -F -) ; done
rm -f .git/COMMIT_EDITMSG
zip:
cd .. && zip -r dgsh.zip dgsh -x *.git*
# Rough uninstall rule to verify that tests pick up correct files
uninstall:
rm -rf $(PREFIX)/bin/dgsh-* $(PREFIX)/libexec/dgsh \
$(PREFIX)/lib/libdgsh.a
================================================
FILE: README.md
================================================
## dgsh: The Directed Graph Shell
[](https://travis-ci.org/dspinellis/dgsh)
The directed graph shell, *dgsh*, allows the expressive expression of efficient big data set and streams processing pipelines using existing Unix tools as well as custom-built components. It is a Unix-style shell allowing the specification of pipelines with non-linear scatter-gather operations. These form a directed acyclic process graph, which is typically executed by multiple processor cores, thus increasing the operation's processing throughput.
You can find a complete introduction, reference documentation,
and illustrated examples in the suite's
[web site](http://www.spinellis.gr/sw/dgsh/).
See also,
a [quick video overview](https://youtu.be/crqzO4YanwA) and
the associated (open access) paper,
[Extending Unix pipelines to DAGs](http://dx.doi.org/10.1109/TC.2017.2695447),
published in the *IEEE Transactions on Computers*, 66(9):1547–1561, 2017.
================================================
FILE: core-tools/.gitignore
================================================
Makefile
Makefile.in
aclocal.m4
autom4te.cache/
build-aux/
config.h
config.h.in
config.log
config.status
configure
libtool
m4/
stamp-h1
================================================
FILE: core-tools/Makefile.am
================================================
## Process this file with automake to produce Makefile.in
ACLOCAL_AMFLAGS = -I m4
SUBDIRS = src . tests
================================================
FILE: core-tools/configure.ac
================================================
# Process this file with autoconf to produce a configure script.
# Prelude.
AC_PREREQ([2.59])
AC_INIT([DGSH Negotiate], [1.0], [check-devel AT lists.sourceforge.net])
# unique source file --- primitive safety check
AC_CONFIG_SRCDIR([src/negotiate.c])
AC_CONFIG_MACRO_DIR([m4])
# place to put some extra build scripts installed
AC_CONFIG_AUX_DIR([build-aux])
# fairly severe build strictness
# change foreign to gnu or gnits to comply with gnu standards
AM_INIT_AUTOMAKE([-Wall -Werror subdir-objects foreign 1.9.6])
# Checks for programs.
AM_PROG_AR
AM_PROG_AS
AC_PROG_CC
AC_PROG_LIBTOOL
# Checks for libraries.
# This macro is defined in check.m4 and tests if check.h and
# libcheck.a are installed in your system. It sets CHECK_CFLAGS and
# CHECK_LIBS accordingly.
# AM_PATH_CHECK([MINIMUM-VERSION,
# [ACTION-IF-FOUND [, ACTION-IF-NOT-FOUND]]])
PKG_CHECK_MODULES([CHECK], [check >= 0.9.4])
# Checks for header files.
AC_HEADER_STDC
AC_CHECK_HEADERS([stdlib.h])
# Checks for typedefs, structures, and compiler characteristics.
# Checks for library functions.
AC_FUNC_MALLOC
AC_CHECK_FUNCS(cpow)
# Output files
AC_CONFIG_HEADERS([config.h])
AC_CONFIG_FILES([Makefile
src/Makefile
tests/Makefile])
AM_CONDITIONAL([LINUX], [test $(uname) = "Linux"])
AM_CONDITIONAL([DARWIN], [test $(uname) = "Darwin"])
AC_OUTPUT
================================================
FILE: core-tools/src/.gitignore
================================================
dgsh-conc
dgsh-conc.html
dgsh-enumerate
dgsh-enumerate.html
dgsh-fft-input
dgsh.html
dgsh-httpval
dgsh-httpval.html
dgsh-merge-sum
dgsh-merge-sum.html
dgsh-monitor
dgsh-monitor.html
dgsh_negotiate.html
dgsh-parallel
dgsh-parallel.html
dgsh-pecho
dgsh-ps
dgsh-readval
dgsh-readval.html
dgsh-tee
dgsh-tee.html
dgsh-w
dgsh-wrap
dgsh-wrap.html
dgsh-writeval
dgsh-writeval.html
libdgsh.a
negotiate
perm
perm.html
================================================
FILE: core-tools/src/Makefile.am
================================================
include ../../.config
if LINUX
DGSH_ASSEMBLY_FILE=dgsh-elf.s
else
if DARWIN
DGSH_ASSEMBLY_FILE=dgsh-macho.s
endif
endif
lib_LIBRARIES = libdgsh.a
libdgsh_a_SOURCES = negotiate.c $(DGSH_ASSEMBLY_FILE)
include_HEADERS = dgsh.h
bin_PROGRAMS = dgsh-monitor dgsh-httpval dgsh-readval
bin_SCRIPTS = dgsh-merge-sum
man1_MANS = dgsh.1 dgsh-conc.1 dgsh-enumerate.1 dgsh-httpval.1 \
dgsh-merge-sum.1 dgsh-monitor.1 \
dgsh-parallel.1 dgsh-readval.1 dgsh-tee.1 dgsh-wrap.1 \
dgsh-writeval.1 perm.1
man3_MANS = dgsh_negotiate.3
libexec_PROGRAMS = dgsh-tee dgsh-writeval dgsh-readval dgsh-monitor \
dgsh-conc dgsh-wrap dgsh-enumerate dgsh-pecho \
dgsh-fft-input dgsh-w
libexec_SCRIPTS = dgsh-parallel perm
libexecdir = $(prefix)/libexec/dgsh
dgsh_monitor_SOURCES = dgsh-monitor.c
dgsh_httpval_SOURCES = dgsh-httpval.c kvstore.c
dgsh_readval_SOURCES = dgsh-readval.c kvstore.c
dgsh_tee_SOURCES = dgsh-tee.c
dgsh_writeval_SOURCES = dgsh-writeval.c
dgsh_conc_SOURCES = dgsh-conc.c
dgsh_wrap_SOURCES = dgsh-wrap.c
dgsh_enumerate_SOURCES = dgsh-enumerate.c
dgsh_pecho_SOURCES = dgsh-pecho.c
dgsh_fft_input_SOURCES = dgsh-fft-input.c
dgsh_w_SOURCES = dgsh-w.c $(CPOW)
dgsh_readval_LDADD = libdgsh.a
dgsh_writeval_LDADD = libdgsh.a
dgsh_conc_LDADD = libdgsh.a
dgsh_wrap_LDADD = libdgsh.a
dgsh_tee_LDADD = libdgsh.a
dgsh_enumerate_LDADD = libdgsh.a
dgsh_pecho_LDADD = libdgsh.a
dgsh_fft_input_LDADD = libdgsh.a
dgsh_w_LDADD = libdgsh.a -lm
dgsh-parallel: dgsh-parallel.sh
install $? $@
perm: perm.sh
install $? $@
dgsh-merge-sum: dgsh-merge-sum.pl
install $? $@
clean-local:
-rm -rf dgsh-parallel perm degsh-merge-sum
build-install:
mkdir -p ../../build/bin ../../build/libexec/dgsh
cp $(bin_PROGRAMS) $(bin_SCRIPTS) ../../build/bin/
cp $(libexec_PROGRAMS) $(libexec_SCRIPTS) ../../build/libexec/dgsh/
================================================
FILE: core-tools/src/debug.h
================================================
/*
* Copyright 2013-2017 Diomidis Spinellis
*
* Debug macros
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
#ifndef DEBUG_H
#define DEBUG_H
#ifdef DEBUG
/* ## is a gcc extension that removes trailing comma if no args */
#define DPRINTF(fmt, ...) fprintf(stderr, "%d: " fmt " \n", (int)getpid(), ##__VA_ARGS__)
#else
#define DPRINTF(fmt, ...)
#endif
#endif /* DEBUG_H */
================================================
FILE: core-tools/src/dgsh-conc.1
================================================
.TH DGSH-HTTPVAL 1 "14 July 2016"
.\"
.\" (C) Copyright 2016 Diomidis Spinellis. All rights reserved.
.\"
.\" Licensed under the Apache License, Version 2.0 (the "License");
.\" you may not use this file except in compliance with the License.
.\" You may obtain a copy of the License at
.\"
.\" http://www.apache.org/licenses/LICENSE-2.0
.\"
.\" Unless required by applicable law or agreed to in writing, software
.\" distributed under the License is distributed on an "AS IS" BASIS,
.\" WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
.\" See the License for the specific language governing permissions and
.\" limitations under the License.
.\"
.SH NAME
dgsh-conc \- input or output pipe concentrator for dgsh negotiation
.SH SYNOPSIS
\fBdgsh-conc\fP \fB\-i\fP | \fB-o\fP \fInprog\fP
.SH DESCRIPTION
\fIdgsh-conc\fP is a helper program used in the \fIdgsh\fP negotiation
phase.
It is used to allow multiple output processes to negotiate with
a single input process, or to allow a single output process to
negotiate with multple input ones.
Once the negotiation is complete, it passes around the generated
pipe file descriptor and exits.
The two obligatory arguments specify whether the command will
act as an input or output concentrator, and the number of
input or output programs to concentrate.
.SH OPTIONS
.IP "\fB\-i\fP
Act as an input concentrator by concentrating multiple inputs to single output.
.IP "\fB\-o\fP
Act as an output concentrator by concentrating multiple outputs to
a single input.
.SH "SEE ALSO"
\fIdgsh\fP(1),
.SH AUTHOR
Diomidis Spinellis \(em <http://www.spinellis.gr>.
================================================
FILE: core-tools/src/dgsh-conc.c
================================================
/*
* Copyright 2016, 2017 Diomidis Spinellis
*
* A passive component that aids the dgsh negotiation by passing
* message blocks among participating processes.
* When the negotiation is finished and the processes get connected by
* pipes, it exits.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
#include <assert.h>
#include <stdbool.h>
#include <stdio.h>
#include <stdlib.h>
#include <err.h>
#include <errno.h>
#include <limits.h>
#include <string.h>
#include <unistd.h> /* getpid(), alarm() */
#include <sys/select.h>
#include <signal.h> /* sig_atomic_t */
#include "negotiate.h" /* read/write_message_block(),
set_negotiation_complete() */
#include "dgsh-debug.h" /* DPRINTF */
#define DGSH_TIMEOUT 5
/* Alarm mechanism and on_exit handling */
extern volatile sig_atomic_t negotiation_completed;
#ifdef TIME
#include <time.h>
static struct timespec tstart={0,0}, tend={0,0};
#endif
static const char *program_name;
static pid_t pid;
static void
usage(void)
{
fprintf(stderr, "Usage: %s -i|-o [-n] nprog\n"
"-i" "\tInput concentrator: multiple inputs to single output\n"
"-o" "\tOutput concentrator: single input to multiple outputs\n"
"-n" "\tDo not consider standard input (used with -o)\n",
program_name);
exit(1);
}
/*
* Information for each I/O file descriptor on which
* the concentrator operates.
*/
static struct portinfo {
pid_t pid; // The id of the process talking to this port
bool seen; // True when the pid was seen
bool written; // True when we wrote to pid
bool run_ready; // True when the associated process can run
struct dgsh_negotiation *to_write; // Block pending a write
} *pi;
/*
* True when we're concentrating inputs, i.e. gathering 0, 3, 4, ... to 1
* Otherwise we scatter 0 to 1, 3, 4 ...
*/
STATIC bool multiple_inputs;
/* True when input concentrator is the gather endpoint
* of a scatter-first-gather-then block.
* In this case using the default route of the input
* concentrator results in complex cycles and ruins
* the algorithms that decide when negotiation should
* end both for concentrators and participating processes.
* The favored scheme in this case is:
* stdin -> stdout
* stdout -> stdin
* fd -> fd
*/
//STATIC bool pass_origin;
STATIC bool noinput;
/*
* Total number of file descriptors on which the process performs I/O
* (including stderr). The last fd used in nfd - 1.
*/
STATIC int nfd;
#define FREE_FILENO (STDERR_FILENO + 1)
/**
* Return the next fd where a read block should be passed
* Return whether we should restore the origin of the block
*/
STATIC int
next_fd(int fd, bool *ro)
{
if (multiple_inputs)
switch (fd) {
case STDIN_FILENO:
return STDOUT_FILENO;
case STDOUT_FILENO:
return STDIN_FILENO;
default:
*ro = true;
return fd;
}
else
switch (fd) {
case STDIN_FILENO:
if (!noinput)
return STDOUT_FILENO;
case STDOUT_FILENO:
if (nfd > 2) { // if ==2, treat in default case
if (!noinput)
*ro = true;
return FREE_FILENO;
}
default:
if (fd == nfd - 1)
if (!noinput)
return STDIN_FILENO;
else
return STDOUT_FILENO;
else {
if (!noinput)
*ro = true;
return fd + 1;
}
}
}
/**
* Return whether the process at port i
* is ready to run.
* Check whether this is the process whose
* pid is set and prepare for exit.
*/
STATIC bool
is_ready(int i, struct dgsh_negotiation *mb)
{
bool ready = false;
if (pi[i].seen && pi[i].written)
ready = true;
DPRINTF(4, "pi[%d].pid: %d %s?: %d\n",
i, pi[i].pid, __func__, ready);
return ready;
}
/**
* Register current concentrator to message block's
* concentrator array
*/
STATIC int
set_io_channels(struct dgsh_negotiation *mb)
{
if (find_conc(mb, pid))
return 0;
struct dgsh_conc c;
c.pid = pid;
c.input_fds = -1;
c.output_fds = -1;
c.n_proc_pids = (nfd > 2 ? nfd - 2 : 1);
c.multiple_inputs = multiple_inputs;
c.proc_pids = (int *)malloc(sizeof(int) * c.n_proc_pids);
int j = 0, i;
DPRINTF(4, "%s: n_proc_pids: %d", __func__, c.n_proc_pids);
if (multiple_inputs) {
c.endpoint_pid = pi[STDOUT_FILENO].pid;
if (c.endpoint_pid == 0)
return 1;
for (i = STDIN_FILENO; i < nfd; i == STDIN_FILENO ? i = FREE_FILENO : i++)
if (pi[i].pid == 0) {
free(c.proc_pids);
return 1;
} else
c.proc_pids[j++] = pi[i].pid;
} else {
bool ignore;
c.endpoint_pid = pi[STDIN_FILENO].pid;
if (c.endpoint_pid == 0)
return 1;
for (i = STDOUT_FILENO; i != STDIN_FILENO; i = next_fd(i, &ignore))
if (pi[i].pid == 0) {
free(c.proc_pids);
return 1;
} else
c.proc_pids[j++] = pi[i].pid;
}
if (!mb->conc_array) {
mb->conc_array = (struct dgsh_conc *)malloc(sizeof(struct dgsh_conc));
mb->n_concs = 1;
} else {
mb->n_concs++;
mb->conc_array = (struct dgsh_conc *)realloc(mb->conc_array,
sizeof(struct dgsh_conc) * mb->n_concs);
}
memcpy(&mb->conc_array[mb->n_concs - 1], &c, sizeof(struct dgsh_conc));
DPRINTF(4, "%s(): Added conc with pid: %d, now n_concs: %d",
__func__, mb->conc_array[mb->n_concs - 1].pid, mb->n_concs);
return 0;
}
STATIC void
print_state(int i, int var, int pcase)
{
switch (pcase) {
case 1:
DPRINTF(4, "%s(): pi[%d].pid: %d",
__func__, i, (int)pi[i].pid);
DPRINTF(4, " initiator pid: %d",
var);
DPRINTF(4, " pi[%d].seen: %d",
i, pi[i].seen);
DPRINTF(4, " write: %d", pi[i].written);
case 2:
DPRINTF(4, "%s(): pi[%d].pid: %d",
__func__, i, pi[i].pid);
DPRINTF(4, " run ready?: %d, seen times: %d",
(int)pi[i].run_ready, pi[i].seen);
DPRINTF(4, " written: %d, nfds: %d",
pi[i].written, var);
}
}
#define max(a, b) ((a) > (b) ? (a) : (b))
/*
* Pass around the message blocks so that they reach all processes
* connected through the concentrator.
*/
STATIC int
pass_message_blocks(void)
{
fd_set readfds, writefds;
int nfds = 0;
int i;
int oi = -1; /* scatter/gather block's origin index */
int ofd = -1; /* ... origin fd direction */
bool ro = false; /* Whether the read block's origin should
* be restored
*/
bool iswrite = false;
if (noinput) {
#ifdef TIME
clock_gettime(CLOCK_MONOTONIC, &tstart);
#endif
construct_message_block("dgsh-conc", pid);
chosen_mb->origin_fd_direction = STDOUT_FILENO;
chosen_mb->is_origin_conc = true;
chosen_mb->conc_pid = pid;
pi[STDOUT_FILENO].to_write = chosen_mb;
}
for (;;) {
// Create select(2) masks
FD_ZERO(&readfds);
FD_ZERO(&writefds);
for (i = 0; i < nfd; i++) {
if (noinput && i == STDIN_FILENO)
continue;
if (i == STDERR_FILENO)
continue;
if (!pi[i].seen) {
FD_SET(i, &readfds);
nfds = max(i + 1, nfds);
}
if (pi[i].to_write && !pi[i].written) {
FD_SET(i, &writefds);
nfds = max(i + 1, nfds);
pi[i].to_write->is_origin_conc = true;
pi[i].to_write->conc_pid = pid;
DPRINTF(4, "Actual origin: conc with pid %d", pid);
}
}
again:
if (select(nfds, &readfds, &writefds, NULL, NULL) < 0) {
if (errno == EINTR)
goto again;
/* All other cases are internal errors. */
err(1, "select");
}
// Read/write what we can
for (i = 0; i < nfd; i++) {
if (FD_ISSET(i, &writefds)) {
iswrite = true;
assert(pi[i].to_write);
chosen_mb = pi[i].to_write;
DPRINTF(4, "**fd i: %d set for writing to tool with pid %d", i, pi[i].pid);
write_message_block(i); // XXX check return
if (pi[i].to_write->state == PS_RUN ||
pi[i].to_write->state == PS_DRAW_EXIT ||
(pi[i].to_write->state == PS_ERROR &&
pi[i].to_write->is_error_confirmed))
pi[i].written = true;
// Write side exit
if (is_ready(i, pi[i].to_write)) {
pi[i].run_ready = true;
DPRINTF(4, "**%s(): pi[%d] is run ready",
__func__, i);
}
pi[i].to_write = NULL;
}
if (FD_ISSET(i, &readfds)) {
struct dgsh_negotiation *rb;
ro = false;
int next = next_fd(i, &ro);
assert(!pi[i].run_ready);
assert(pi[next].to_write == NULL);
if (read_message_block(i, &pi[next].to_write) ==
OP_ERROR) {
chosen_mb->state = PS_ERROR;
if (noinput)
chosen_mb->is_error_confirmed = true;
pi[next].to_write = chosen_mb;
continue;
}
rb = pi[next].to_write;
DPRINTF(4, "%s(): next write via fd %d to pid %d",
__func__, next, pi[next].pid);
if (oi == -1) {
if ((multiple_inputs && i == 1) ||
(!multiple_inputs &&
i == 0)) {
oi = rb->origin_index;
ofd = rb->origin_fd_direction;
DPRINTF(4, "**Store origin: %d, fd: %s",
oi, ofd ? "stdout" : "stdin");
}
}
/* If conc talks to conc, set conc's pid
* Required in order to allocate fds correctly
* in the end
*/
if (rb->is_origin_conc)
pi[i].pid = rb->conc_pid;
else
pi[i].pid = get_origin_pid(rb);
/* If needed, re-set origin.
* Don't move this block before get_origin_pid()
*/
if (ro) {
DPRINTF(4, "**Restore origin: %d, fd: %s",
oi, ofd ? "stdout" : "stdin");
pi[next].to_write->origin_index = oi;
pi[next].to_write->origin_fd_direction = ofd;
} else if (noinput) {
pi[next].to_write->origin_index = -1;
pi[next].to_write->origin_fd_direction = STDOUT_FILENO;
}
/* Set a conc's required/provided IO in mb */
if (!noinput)
set_io_channels(pi[next].to_write);
if (rb->state == PS_NEGOTIATION &&
noinput) {
int j, seen = 0;
pi[i].seen = true;
for (j = 1; j < nfd; j++)
if (pi[j].seen)
seen++;
if ((nfd > 2 && seen == nfd - 2) ||
seen == nfd - 1) {
chosen_mb = rb;
DPRINTF(1, "%s(): Gathered I/O requirements.", __func__);
int state = solve_graph();
if (state == OP_ERROR) {
pi[next].to_write->state = PS_ERROR;
pi[next].to_write->is_error_confirmed = true;
} else if (state == OP_DRAW_EXIT)
pi[next].to_write->state = PS_DRAW_EXIT;
else {
DPRINTF(1, "%s(): Computed solution", __func__);
pi[next].to_write->state = PS_RUN;
}
for (j = 1; j < nfd; j++)
pi[j].seen = false;
// Don't free
chosen_mb = NULL;
}
} else if (rb->state == PS_RUN ||
rb->state == PS_DRAW_EXIT ||
(rb->state == PS_ERROR &&
rb->is_error_confirmed))
pi[i].seen = true;
else if (rb->state == PS_ERROR)
rb->is_error_confirmed = true;
print_state(i, (int)rb->initiator_pid, 1);
if (pi[i].seen && pi[i].written) {
chosen_mb = pi[next].to_write;
pi[i].run_ready = true;
DPRINTF(4, "**%s(): pi[%d] is run ready",
__func__, i);
}
}
}
// See if all processes are run-ready
nfds = 0;
for (i = 0; i < nfd; i++) {
if (pi[i].run_ready)
nfds++;
print_state(i, nfds, 2);
}
if ((nfd > 2 && (nfds == nfd - 1 ||
(noinput && nfds == nfd - 2))) ||
(nfds == nfd || (noinput && nfds == nfd - 1))) {
assert(chosen_mb != NULL);
DPRINTF(4, "%s(): conc leaves negotiation", __func__);
return chosen_mb->state;
} else if (chosen_mb != NULL && iswrite) { // Free if we have written
DPRINTF(4, "chosen_mb: %lx, i: %d, next: %d, pi[next].to_write: %lx\n",
(long)chosen_mb, i, next_fd(i, &ro), (long)pi[next_fd(i, &ro)].to_write);
free_mb(chosen_mb);
chosen_mb = NULL;
iswrite = false;
}
}
}
/*
* Scatter the fds read from the input process to multiple outputs.
*/
STATIC void
scatter_input_fds(struct dgsh_negotiation *mb)
{
struct dgsh_conc *this_conc = find_conc(mb, pid);
if (!this_conc) {
printf("%s(): Concentrator with pid %d not registered",
__func__, pid);
exit(1); // XXX
}
int n_to_read = this_conc->input_fds;
int *read_fds = (int *)malloc(n_to_read * sizeof(int));
int i, j, write_index = 0;
bool ignore = false;
DPRINTF(4, "%s(): fds to read: %d", __func__, n_to_read);
for (i = 0; i < n_to_read; i++)
read_fds[i] = read_fd(STDIN_FILENO);
for (i = STDOUT_FILENO; i != STDIN_FILENO; i = next_fd(i, &ignore)) {
int n_to_write = get_expected_fds_n(mb, pi[i].pid);
DPRINTF(4, "%s(): fds to write for p[%d].pid %d: %d",
__func__, i, pi[i].pid, n_to_write);
for (j = write_index; j < write_index + n_to_write; j++) {
write_fd(i, read_fds[j]);
DPRINTF(4, "%s(): Write fd: %d to output channel: %d",
__func__, read_fds[j], i);
}
write_index += n_to_write;
}
assert(write_index == n_to_read);
}
/*
* Gather the fds read from input processes to a single output.
*/
STATIC void
gather_input_fds(struct dgsh_negotiation *mb)
{
struct dgsh_conc *this_conc = find_conc(mb, pid);
if (!this_conc) {
printf("%s(): Concentrator with pid %d not registered",
__func__, pid);
exit(1); // XXX
}
int n_to_write = this_conc->output_fds;
int *read_fds = (int *)malloc(n_to_write * sizeof(int));
int i, j, read_index;
DPRINTF(4, "%s(): fds to write: %d", __func__, n_to_write);
read_index = 0;
for (i = STDIN_FILENO; i < nfd; i == STDIN_FILENO ? i = FREE_FILENO : i++) {
int n_to_read = get_provided_fds_n(mb, pi[i].pid);
DPRINTF(4, "%s(): fds to read for p[%d].pid %d: %d",
__func__, i, pi[i].pid, n_to_read);
for (j = read_index; j < read_index + n_to_read; j++) {
read_fds[j] = read_fd(i);
DPRINTF(4, "%s(): Read fd: %d from input channel: %d",
__func__, read_fds[j], i);
}
read_index += n_to_read;
}
assert(read_index == n_to_write);
for (i = 0; i < n_to_write; i++)
write_fd(STDOUT_FILENO, read_fds[i]);
}
#ifndef UNIT_TESTING
int
main(int argc, char *argv[])
{
int ch;
int exit;
char *debug_level = NULL;
char *timeout;
program_name = argv[0];
pid = getpid();
noinput = false;
while ((ch = getopt(argc, argv, "ion")) != -1) {
switch (ch) {
case 'i':
multiple_inputs = true;
break;
case 'o':
multiple_inputs = false;
break;
case 'n': // special output conc that takes no input
if (!multiple_inputs)
noinput = true;
else
usage();
break;
case '?':
default:
usage();
}
}
argc -= optind;
argv += optind;
if (argc != 1)
usage();
debug_level = getenv("DGSH_DEBUG_LEVEL");
if (debug_level != NULL)
dgsh_debug_level = atoi(debug_level);
signal(SIGALRM, dgsh_alarm_handler);
if ((timeout = getenv("DGSH_TIMEOUT")) != NULL)
alarm(atoi(timeout));
else
alarm(DGSH_TIMEOUT);
/* +1 for stdin when scatter/stdout when gather
* +1 for stderr which is not used
*/
if (atoi(argv[0]) == 1)
nfd = 2;
else
nfd = atoi(argv[0]) + 2;
pi = (struct portinfo *)calloc(nfd, sizeof(struct portinfo));
chosen_mb = NULL;
exit = pass_message_blocks();
if (exit == PS_RUN) {
if (noinput)
DPRINTF(1, "%s(): Special (no-input) conc communicated the solution", __func__);
if (multiple_inputs)
gather_input_fds(chosen_mb);
else if (!noinput) // Output noinput conc has no job here
scatter_input_fds(chosen_mb);
exit = PS_COMPLETE;
}
free_mb(chosen_mb);
free(pi);
DPRINTF(3, "conc with pid %d terminates %s",
pid, exit == PS_COMPLETE ? "normally" : "with error");
#ifdef DEBUG
fflush(stderr);
#endif
#ifdef TIME
if (noinput) {
clock_gettime(CLOCK_MONOTONIC, &tend);
fprintf(stderr, "The dgsh negotiation procedure took about %.5f seconds\n",
((double)tend.tv_sec + 1.0e-9*tend.tv_nsec) -
((double)tstart.tv_sec + 1.0e-9*tstart.tv_nsec));
fflush(stderr);
}
#endif
set_negotiation_complete();
alarm(0); // Cancel alarm
signal(SIGALRM, SIG_IGN); // Do not handle the signal
return exit;
}
#endif
================================================
FILE: core-tools/src/dgsh-debug.h
================================================
/*
* Copyright 2013-2017 Diomidis Spinellis
*
* Debug macros
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
#ifndef DEBUG_H
#define DEBUG_H
extern int dgsh_debug_level;
/* ## is a gcc extension that removes trailing comma if no args */
#define DPRINTF(debug_level, fmt, ...) ((debug_level) <= dgsh_debug_level ? fprintf(stderr, "%d: " fmt "\n", (int)getpid(), ##__VA_ARGS__) : 0)
#endif /* DEBUG_H */
================================================
FILE: core-tools/src/dgsh-elf.s
================================================
/*
* ELF note header to mark dgsh-compatible programs
* See http://www.netbsd.org/docs/kernel/elf-notes.html
* Don't use line comments as these are not portable between
* different CPU architectures.
* https://en.wikipedia.org/wiki/GNU_Assembler#Single-Line_comments
*/
.comm dgsh_force_include,4,4
.section ".note.ident", "a"
.p2align 2
.long 1f - 0f /* name size (not including padding) */
.long 3f - 2f /* desc size (not including padding) */
.long 1 /* type */
0: .asciz "DSpinellis/dgsh" /* name */
1: .p2align 2
2: .long 0x00000001 /* desc */
.long 0x00000000
3: .p2align 2
================================================
FILE: core-tools/src/dgsh-enumerate.1
================================================
.TH DGSH-ENUMERATE 1 "27 January 2017"
.\"
.\" (C) Copyright 2017 Diomidis Spinellis. All rights reserved.
.\"
.\" Licensed under the Apache License, Version 2.0 (the "License");
.\" you may not use this file except in compliance with the License.
.\" You may obtain a copy of the License at
.\"
.\" http://www.apache.org/licenses/LICENSE-2.0
.\"
.\" Unless required by applicable law or agreed to in writing, software
.\" distributed under the License is distributed on an "AS IS" BASIS,
.\" WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
.\" See the License for the specific language governing permissions and
.\" limitations under the License.
.\"
.SH NAME
dgsh-enumerate \- enumerate an arbitrary number of output channels
.SH SYNOPSIS
\fBdgsh-enumerate\fP [\fIn\fP]
.SH DESCRIPTION
\fIdgsh-enumerate\fP will output a single newline-terminated ascending
integer on each one of its output channels.
If the number of channels is not specified, the command will
allow its downstream processes to specify the number and use that one.
.PP
The command demonstrates the \fIdgsh\fP negotiation API.
It can also be used as a debug tool.
.SH EXAMPLES
.PP
Enumerate the specified four output streams.
.ft C
.ps -1
.nf
$ dgsh -c 'dgsh-enumerate 4 | cat'
0
1
2
3
.fi
.ps +1
.ft P
.PP
Enumerate the two output streams required by the downstream multipipe block.
.ft C
.ps -1
.nf
$ dgsh -c 'dgsh-enumerate | {{ sed "s/^/A /" & sed "s/^/B /" & }} | cat'
A 0
B 1
.fi
.ps +1
.ft P
.SH "SEE ALSO"
.IR dgsh (1),
.IR dgsh_negotiate (3).
.SH AUTHOR
Diomidis Spinellis \(em <http://www.spinellis.gr>
================================================
FILE: core-tools/src/dgsh-enumerate.c
================================================
/*
* Copyright 2017 Diomidis Spinellis
*
* Enumerate an arbitrary number of output channels.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
#include <assert.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <err.h>
#include <unistd.h>
#include "dgsh.h"
int
main(int argc, char *argv[])
{
int n_input_fds = 0, n_output_fds;
int *output_fds;
int i;
switch (argc) {
case 1:
n_output_fds = -1;
break;
case 2:
n_output_fds = atoi(argv[1]);
break;
default:
errx(1, "usage: %s [n]", argv[0]);
}
dgsh_negotiate(DGSH_HANDLE_ERROR, argv[0], &n_input_fds,
&n_output_fds, NULL, &output_fds);
for (i = 0; i < n_output_fds; i++) {
char buff[10];
snprintf(buff, sizeof(buff), "%d\n", i);
write(output_fds[i], buff, strlen(buff));
close(output_fds[i]);
}
return 0;
}
================================================
FILE: core-tools/src/dgsh-fft-input.c
================================================
#include <assert.h> // assert()
#include <stdio.h> // printf
#include <complex.h> // double complex
#include <unistd.h> // read(), write()
#include <stdlib.h> // free()
#include <err.h> // errx()
#include "dgsh.h"
#include "debug.h"
int main(int argc, char **argv)
{
char *input_file;
FILE *f;
int ninput = 4, nlines = 0, i;
int ninputfds = 0, noutputfds;
int *inputfds = NULL, *outputfds = NULL;
size_t len = sizeof(long double), wsize;
char line[len + 1];
long double *input = (long double *)malloc(sizeof(long double) * ninput);
if (argc == 1) {
noutputfds = 8;
goto negotiate;
}
input_file = argv[1];
f = fopen(input_file, "r");
if (!f)
errx(2, "Open file %s failed", input_file);
DPRINTF(4, "Opened input file: %s", input_file);
while (fgets(line, len, f)) {
assert(len == sizeof(input[nlines - 1]));
nlines++;
if (nlines == ninput) {
ninput *= 2;
input = (long double *)realloc(input,
sizeof(long double) * ninput);
if (!input)
errx(2, "Realloc for input numbers failed");
}
input[nlines - 1] = atof(line);
DPRINTF(4, "Retrieved input %.10Lf\n", input[nlines - 1]);
}
noutputfds = nlines;
negotiate:
dgsh_negotiate(DGSH_HANDLE_ERROR, "fft-input", &ninputfds, &noutputfds,
&inputfds, &outputfds);
DPRINTF(4, "Read %d inputs, received %d fds", nlines, noutputfds);
assert(ninputfds == 0);
assert(noutputfds == nlines);
for (i = 0; i < noutputfds; i++) {
DPRINTF(4, "Write input %.10Lf to fd %d", input[i], outputfds[i]);
wsize = write(outputfds[i], &input[i],
sizeof(long double));
if (wsize == -1)
err(1, "write failed");
}
fclose(f);
free(input);
return 0;
}
================================================
FILE: core-tools/src/dgsh-httpval.1
================================================
.TH DGSH-HTTPVAL 1 "14 July 2013"
.\"
.\" (C) Copyright 2013 Diomidis Spinellis. All rights reserved.
.\"
.\" Licensed under the Apache License, Version 2.0 (the "License");
.\" you may not use this file except in compliance with the License.
.\" You may obtain a copy of the License at
.\"
.\" http://www.apache.org/licenses/LICENSE-2.0
.\"
.\" Unless required by applicable law or agreed to in writing, software
.\" distributed under the License is distributed on an "AS IS" BASIS,
.\" WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
.\" See the License for the specific language governing permissions and
.\" limitations under the License.
.\"
.SH NAME
dgsh-httpval \- data store HTTP server
.SH SYNOPSIS
\fBdgsh-httpval\fP
[\fB\-a\fP]
[\fB\-b\fP \fIquery:command\fP]
[\fB\-m\fP \fIMIME-type\fP]
[\fB\-n\fP]
[\fB\-p\fP \fIport\fP]
.SH DESCRIPTION
\fIdgsh-httpval\fP allows other programs to access \fIdgsh\fP
data stores through the HTTP protocol.
This simplifies the interfacing between web-based front-ends and
\fIdgsh\fP programs.
When \fIdgsh-httpval\fP receives a REST request with the name of a data store
whose endpoint is located in the directory where \fIdgsh-httpval\fP
was launched (e.g. \fChttp://localhost:8081/mystore\fP),
it will establish a connection with the store specified in the request,
send a command to read the store's value,
obtain the value,
and respond with it as the document sent with the HTTP response.
.PP
Requests for files located in the directory where \fIdgsh-httpval\fP
was launched will also be satisfied.
The correct MIME type will be sent for files with a suffix of
\fChtml\fP,
\fCjs\fP,
\fCjson\fP,
\fCpng\fP, and
\fCcss\fP.
.PP
A request for the resource \fC.server?quit\fP, will cause the server
to terminate processing and exit.
.PP
\fIdgsh-httpval\fP is normally executed from within \fIdgsh\fP-generated
scripts, rather than through end-user commands.
This manual page serves mainly to document its operation and
the flags that can be passed to \fIdgsh\fP for modifying its behavior.
.SH OPTIONS
.IP "\fB\-a\fP
Allow any Internet host to obtain a value from the server.
By default the server will only respond to requests arriving from the local
host's loop-back IP address (127.0.0.1).
.IP "\fB\-b\fP \fIquery:command\fP"
The colon-separated pair specifies a dynamic query
than can be sent to the server,
so that it will execute the specified command and return its output.
The query and the command can contain up to ten matching
\fIscanf(3)\fP and \fIprintf(3)\fP specifications for C integer-sized
arguments, which can be used to pass data from the query to the command.
An unlimited number of dynamic queries can be specified through multiple
.B -b
options.
The type of the data returned is specified using the
.B -m
option.
.IP "\fB\-m\fP \fIMIME-type\fP"
Specify the MIME-type that the server will provide on the \fCContent-type\fP
HTTP header for data coming from data stores and dynamic queries.
By default this value is \fCtext/plain\fP.
Other reasonable types are
\fCapplication/json\fP,
\fCtext/CSV\fP,
\fCtext/xml\fP, or
\fCapplication/octet-stream\fP.
.IP "\fB\-n\fP
Read values from stores using a non-blocking read command.
This means that the server will return an empty record,
if no complete record is available.
.IP "\fB\-p\fP \fIport\fP"
Specify the TCP port on which the server will listen for incoming HTTP
requests.
If no port is specified, then the server will listen on an arbitrary,
system-assigned, port,
and will print that port's number on its standard output.
That value can be conveniently piped into \fIdgsh-writeval\fP
to be made available to other processes.
.SH EXAMPLES
.PP
Specify that a query, such as
\fChttp://localhost:63001/server-bin/pstatus?id=4892\fP,
will run the \fIps(1)\fP command for the specified process-id.
.ft C
.nf
dgsh-httpval -b 'server-bin/pstatus?id=%d:ps -p %d'
.ft P
.fi
.SH "SEE ALSO"
\fIdgsh\fP(1),
\fIdgsh-writeval\fP(1),
\fIdgsh-readval\fP(1)
.SH BUGS
The server is single-threaded and will block if a value is not available
on a specified store.
.PP
The server only supports IPv4 and the HTTP 1.0 protocols.
Some clients may require special configuration to connect to it.
For instance, \fIcurl\fP(1) requires the specification of the \fC--ipv4\fP
and \fC--http1.0\fP flags.
.SH AUTHOR
Diomidis Spinellis \(em <http://www.spinellis.gr>.
Jef Poskanzer \(em <jef@mail.acme.com> \(em wrote micro_httpd on which
this server is based.
.SH BUGS
The possibilities for mallicious attacks through code injection and buffer
overflows offered by the dynamic query option are too numerous to list.
Use this feature only in setups where you restrict and control what is being
sent to the server.
================================================
FILE: core-tools/src/dgsh-httpval.c
================================================
/*-
*
* Provide HTTP access to the dgsh key-value store.
*
* Based on micro_httpd - really small HTTP server heavily modified by
* Diomidis Spinellis to use IP sockets, instead of depending on inetd,
* and to serve dgsh key-value store data, instead of files.
*
* micro_httpd:
* Copyright (c) 1999,2005 by Jef Poskanzer <jef@mail.acme.com>.
* All rights reserved.
*
* Dgsh modifications:
* Copyright 2013 Diomidis Spinellis
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*/
#include <sys/types.h>
#include <sys/socket.h>
#include <sys/stat.h>
#include <errno.h>
#include <unistd.h>
#include <stdlib.h>
#include <stdio.h>
#include <string.h>
#include <ctype.h>
#include <err.h>
#include <time.h>
#include <netinet/in.h>
#include <arpa/inet.h>
#include <stdint.h>
#include "kvstore.h"
#define SERVER_NAME "dgsh-httpval"
#define SERVER_URL "http://www.spinellis.gr/sw/dgsh"
#define PROTOCOL "HTTP/1.0"
#define RFC1123FMT "%a, %d %b %Y %H:%M:%S GMT"
/* Forwards. */
static void send_error(FILE *out, int status, char *title, char *extra_header,
char *text);
static void send_headers(FILE *out, int status, char *title, char *extra_header,
const char *mime_type, off_t length, time_t mod);
static char * get_mime_type(char *name);
static void strdecode(char *to, char *from);
static int hexit(char c);
static void http_serve(FILE *in, FILE *out, const char *mime_type);
#define c_isxdigit(x) isxdigit((unsigned char)(x))
static const char *program_name;
static void
usage(void)
{
fprintf(stderr, "Usage: %s [-a] [-b query:cmd] [-p port]\n"
"-a\t" "\tAllow non-localhost access\n"
"-b query:cmd" "\tSpecify a command for a given HTTP query\n"
"-m MIME-type" "\tSpecify the store Content-type header value\n"
"-n" "\tNon-blocking read from stores\n"
"-p port" "\tSpecify the port to listen to\n",
program_name);
exit(1);
}
static struct query {
const char *query;
const char *cmd;
int narg;
struct query *next;
} *query_list;
/* Command to read from stores: blocking read current record */
static char read_cmd = 'C';
int
main(int argc, char *argv[])
{
int sockfd, newsockfd;
struct sockaddr_in cli_addr, serv_addr;
int ch, port = 0;
bool localhost_access = true;
const char *mime_type = "text/plain";
int so_reuseaddr = 1;
struct linger so_linger;
program_name = argv[0];
while ((ch = getopt(argc, argv, "ab:m:np:")) != -1) {
char *p;
struct query *q;
switch (ch) {
case 'a':
localhost_access = false;
break;
case 'b':
if ((p = strchr(optarg, ':')) == NULL)
usage();
/* Save query */
*p = 0;
q = malloc(sizeof(struct query));
q->query = strdup(optarg);
q->cmd = strdup(p + 1);
/* Count number of query arguments */
q->narg = 0;
for (p = optarg; *p; p++)
if (*p == '%') {
if (p[1] == '%') {
p++;
continue;
}
q->narg++;
}
if (q->narg > 10) {
fprintf(stderr, "%s: More than ten query arguments specified.\n", program_name);
exit(1);
}
/* Insert query into the linked list */
q->next = query_list;
query_list = q;
break;
case 'm':
mime_type = optarg;
break;
case 'n':
/* Non-blocking read */
read_cmd = 'c';
break;
case 'p':
port = atoi(optarg);
break;
case '?':
default:
usage();
}
}
argc -= optind;
argv += optind;
if (argc != 0)
usage();
if ((sockfd = socket(AF_INET, SOCK_STREAM, 0)) < 0)
err(2, "socket");
if (setsockopt(sockfd, SOL_SOCKET, SO_REUSEADDR, &so_reuseaddr,
sizeof (so_reuseaddr)) < 0)
err(2, "setsockopt SO_REUSEADDR");
so_linger.l_onoff = 1;
so_linger.l_linger = 1;
if (setsockopt(sockfd, SOL_SOCKET, SO_LINGER, &so_linger,
sizeof (so_linger)) < 0)
err(2, "setsockopt SO_LINGER");
memset((char *)&serv_addr, 0, sizeof(serv_addr));
serv_addr.sin_family = AF_INET;
serv_addr.sin_addr.s_addr = htonl(INADDR_ANY);
serv_addr.sin_port = htons(port);
if (bind(sockfd, (struct sockaddr *)&serv_addr, sizeof(serv_addr)) < 0)
err(2, "bind");
if (port == 0) {
socklen_t len = sizeof(serv_addr);
serv_addr.sin_port = 0;
if (getsockname(sockfd, (struct sockaddr *)&serv_addr, &len) < 0)
err(2, "getsockname");
printf("%d\n", ntohs(serv_addr.sin_port));
fflush(stdout);
}
listen(sockfd, 5);
for (;;) {
socklen_t cli_len = sizeof(cli_addr);
FILE *in, *out;
if ((newsockfd = accept(sockfd, (struct sockaddr *)&cli_addr, &cli_len)) < 0)
err(2, "accept");
if (localhost_access && memcmp(inet_ntoa(cli_addr.sin_addr), "127.", 4)) {
warnx("Non-localhost access: %s", inet_ntoa(cli_addr.sin_addr));
close(newsockfd);
continue;
}
if ((in = fdopen(newsockfd, "r")) == NULL)
err(2, "fdopen for input");
setvbuf(in, NULL, _IOLBF, 4096);
/* Must dup(2) so that fclose will work correctly */
if ((out = fdopen(dup(newsockfd), "w")) == NULL)
err(2, "fdopen for output");
http_serve(in, out, mime_type);
(void)fclose(in);
(void)fclose(out);
}
}
/* Serve a single HTTP request */
static void
http_serve(FILE *in, FILE *out, const char *mime_type)
{
char line[10000], method[10000], path[10000], protocol[10000];
char *file;
size_t len;
struct stat sb;
struct query *q;
if (fgets(line, sizeof(line), in) == (char *) 0) {
send_error(out, 400, "Bad Request", (char *) 0,
"No request found.");
return;
}
if (sscanf(line, "%[^ ] %[^ ] %[^ ]", method, path, protocol) != 3) {
send_error(out, 400, "Bad Request", (char *) 0,
"Can't parse request.");
return;
}
while (fgets(line, sizeof(line), in) != (char *) 0) {
if (strcmp(line, "\n") == 0 || strcmp(line, "\r\n") == 0)
break;
}
if (strcasecmp(method, "get") != 0) {
send_error(out, 501, "Not Implemented", (char *) 0,
"That method is not implemented.");
return;
}
if (path[0] != '/') {
send_error(out, 400, "Bad Request", (char *) 0, "Bad filename.");
return;
}
file = &(path[1]);
strdecode(file, file);
if (strcmp(file, ".server?quit") == 0) {
send_error(out, 200, "OK", (char *) 0,
"Quitting.");
exit(0);
}
len = strlen(file);
/* Guard against attempts to move outside our directory */
if (file[0] == '/' || strcmp(file, "..") == 0
|| strncmp(file, "../", 3) == 0
|| strstr(file, "/../") != (char *) 0
|| strcmp(&(file[len - 3]), "/..") == 0) {
send_error(out, 400, "Bad Request", (char *) 0,
"Illegal filename.");
return;
}
/* User-specified query name space */
for (q = query_list; q; q = q->next) {
int v0, v1, v2, v3, v4, v5, v6, v7, v8, v9;
char cmd[11000];
*cmd = 0;
if (q->narg == 0 && strcmp(file, q->query) == 0)
strncpy(cmd, q->cmd, sizeof(cmd));
else if (q->narg && sscanf(file, q->query, &v0, &v1, &v2, &v3, &v4, &v5, &v6, &v7, &v8, &v9) == q->narg)
snprintf(cmd, sizeof(cmd), q->cmd, v0, v1, v2, v3, v4, v5, v6, v7, v8, v9);
if (*cmd) {
int ich;
FILE *fp;
fp = popen(cmd, "r");
if (fp == NULL) {
send_error(out, 502, "Bad Gateway", NULL, "Error in executing command.");
return;
}
send_headers(out, 200, "Ok", NULL, mime_type, -1,
time(NULL));
while ((ich = getc(fp)) != EOF)
putc(ich, out);
fclose(fp);
return;
}
}
/* File system name space */
if (stat(file, &sb) < 0) {
send_error(out, 404, "Not Found", NULL, strerror(errno));
return;
}
if (S_ISSOCK(sb.st_mode)) {
/* Value store */
send_headers(out, 200, "Ok", NULL, mime_type,
-1, (time_t)-1);
(void)fflush(out);
dgsh_send_command(file, read_cmd, true, false, fileno(out));
} else if (S_ISREG(sb.st_mode)) {
/* Regular file */
int ich;
FILE *fp;
fp = fopen(file, "r");
if (fp == NULL) {
send_error(out, 403, "Forbidden", NULL, "File is protected.");
return;
}
send_headers(out, 200, "Ok", NULL, get_mime_type(file),
sb.st_size, sb.st_mtime);
while ((ich = getc(fp)) != EOF)
putc(ich, out);
fclose(fp);
} else {
send_error(out, 403, "Forbidden", (char *) 0,
"File is not a regular file or a Unix domain socket.");
}
}
static void
send_error(FILE *out, int status, char *title, char *extra_header, char *text)
{
send_headers(out, status, title, extra_header, "text/html", -1, -1);
(void)fprintf(out,
"<html><head><title>%d %s</title></head>\n<body><h4>%d %s</h4>\n",
status, title, status, title);
(void)fprintf(out, "%s\n", text);
(void)fprintf(out,
"<hr />\n<address><a href=\"%s\">%s</a></address>\n</body></html>\n",
SERVER_URL, SERVER_NAME);
(void)fflush(out);
}
static void
send_headers(FILE *out, int status, char *title, char *extra_header,
const char *mime_type, off_t length, time_t mod)
{
time_t now;
char timebuf[100];
(void)fprintf(out, "%s %d %s\015\012", PROTOCOL, status, title);
(void)fprintf(out, "Server: %s\015\012", SERVER_NAME);
now = time((time_t *) 0);
(void)strftime(timebuf, sizeof(timebuf), RFC1123FMT, gmtime(&now));
(void)fprintf(out, "Date: %s\015\012", timebuf);
if (extra_header != (char *) 0)
(void)fprintf(out, "%s\015\012", extra_header);
if (mime_type != (char *) 0)
(void)fprintf(out, "Content-Type: %s\015\012", mime_type);
if (length >= 0)
#if __STDC_VERSION__ >= 199901L
(void)fprintf(out, "Content-Length: %jd\015\012",
(intmax_t) length);
#else
(void)fprintf(out, "Content-Length: %lld\015\012",
(long long) length);
#endif
if (mod != (time_t) - 1) {
(void)strftime(timebuf, sizeof(timebuf), RFC1123FMT,
gmtime(&mod));
(void)fprintf(out, "Last-Modified: %s\015\012", timebuf);
}
(void)fprintf(out, "Connection: close\015\012");
(void)fprintf(out, "\015\012");
}
static char *
get_mime_type(char *name)
{
char *dot;
dot = strrchr(name, '.');
if (dot == NULL)
return "text/plain";
if (strcmp(dot, ".json") == 0)
return "application/json";
if (strcmp(dot, ".html") == 0)
return "text/html";
if (strcmp(dot, ".js") == 0)
return "text/javascript";
if (strcmp(dot, ".png") == 0)
return "image/png";
if (strcmp(dot, ".css") == 0)
return "text/css";
return "text/plain; charset=iso-8859-1";
}
static void
strdecode(char *to, char *from)
{
for (; *from != '\0'; ++to, ++from) {
if (from[0] == '%' && c_isxdigit(from[1]) && c_isxdigit(from[2])) {
*to = hexit(from[1]) * 16 + hexit(from[2]);
from += 2;
} else
*to = *from;
}
*to = '\0';
}
static int
hexit(char c)
{
if (c >= '0' && c <= '9')
return c - '0';
if (c >= 'a' && c <= 'f')
return c - 'a' + 10;
if (c >= 'A' && c <= 'F')
return c - 'A' + 10;
/* Shouldn't happen, we're guarded by isxdigit() */
return 0;
}
================================================
FILE: core-tools/src/dgsh-macho.s
================================================
# MACHO note header to mark dgsh-compatible programs
.section ".note.ident", "a"
.asciz "DSpinellis/dgsh"
.section __TEXT,__text,regular,pure_instructions
.macosx_version_min 10, 13
.globl _dgsh_force_include ## @dgsh_force_include
.zerofill __DATA,__common,_dgsh_force_include,4,2
.subsections_via_symbols
================================================
FILE: core-tools/src/dgsh-merge-sum.1
================================================
.TH DGSH-MERGE-SUM 1 "10 September 2014"
.\"
.\" (C) Copyright 2014 Diomidis Spinellis. All rights reserved.
.\"
.\" Licensed under the Apache License, Version 2.0 (the "License");
.\" you may not use this file except in compliance with the License.
.\" You may obtain a copy of the License at
.\"
.\" http://www.apache.org/licenses/LICENSE-2.0
.\"
.\" Unless required by applicable law or agreed to in writing, software
.\" distributed under the License is distributed on an "AS IS" BASIS,
.\" WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
.\" See the License for the specific language governing permissions and
.\" limitations under the License.
.\"
.SH NAME
dgsh-merge-sum \- merge key value pairs, summing the values
.SH SYNOPSIS
\fBdgsh-merge-sum\fP \fIfile ...\fP
.SH DESCRIPTION
\fIdgsh-merge-sum\fP will read \fIkey\fP, \fIvalue\fP pairs from the files
specified in its standard input,
and print the input records merged together according to the value of the key.
The input files should be sorted according to the key's value.
Records with the same key will have their values summed, and a single
corresponding record will be printed.
Whitespace is used as the separator.
Leading whitespace is not taken into account when parsing fields.
Thus \fIdgsh-merge-sum\fP can process multiple files
generated by \fIuniq -c\fP,
and merge them into one.
.SH "SEE ALSO"
\fIuniq\fP(1),
\fIdgsh\fP(1)
.SH AUTHOR
Diomidis Spinellis \(em <http://www.spinellis.gr>
================================================
FILE: core-tools/src/dgsh-merge-sum.pl
================================================
#!/usr/bin/env perl
#
# Merge sorted (value, key) pairs, summing the values of equal keys
#
# Copyright 2014 Diomidis Spinellis
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
use strict;
use warnings;
# Read a record from the specified file reference
sub
read_record
{
my ($fr) = @_;
my $f = $fr->{file};
my $line = <$f>;
if (!defined($line)) {
$fr->{key} = undef;
return;
}
($fr->{value}, $fr->{key}) = ($line =~ m/^\s*(\d+)\s+(.*)/);
}
# Open input files; opening before reading prevents pipe writers from blocking
my @file;
# First file is always stdin
binmode(STDIN, ":utf8");
$file[0]->{file} = \*STDIN;
my $i = 1;
for my $name (@ARGV) {
open($file[$i]->{file}, '<:encoding(utf8)', $name) || die "Unable to open $name: $!\n";
$i++;
}
# Read first record from all files
for my $f (@file) {
read_record($f);
}
# Previous key printed
my $prev;
for (;;) {
# Find smallest key
my $smallest;
for my $r (@file) {
#print "Check $r->{value}, $r->{key}\n";
$smallest = $r if (!defined($smallest->{key}) ||
(defined($r->{key}) && $r->{key} lt $smallest->{key}));
}
exit 0 unless defined($smallest->{key});
#print "Smallest $smallest->{value}, $smallest->{key}\n";
# Sum up and renew all smallest keys
my $sum = 0;
my $key = $smallest->{key};
for my $r (@file) {
if (defined($r->{key}) && ($r->{key} cmp $key) == 0) {
$sum += $r->{value};
read_record($r);
}
}
# Verify that input is sorted
if (defined($prev) && ($key cmp $prev) < 0) {
print STDERR "Input is not sorted: [$key] came after [$prev]\n";
exit 1;
}
$prev = $key;
print "$sum $key\n";
}
================================================
FILE: core-tools/src/dgsh-monitor.1
================================================
.TH DGSH-MONITOR 1 "11 December 2016"
.\"
.\" (C) Copyright 2013 Diomidis Spinellis. All rights reserved.
.\"
.\" Licensed under the Apache License, Version 2.0 (the "License");
.\" you may not use this file except in compliance with the License.
.\" You may obtain a copy of the License at
.\"
.\" http://www.apache.org/licenses/LICENSE-2.0
.\"
.\" Unless required by applicable law or agreed to in writing, software
.\" distributed under the License is distributed on an "AS IS" BASIS,
.\" WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
.\" See the License for the specific language governing permissions and
.\" limitations under the License.
.\"
.SH NAME
dgsh-monitor \- monitor data on a pipe
.SH SYNOPSIS
\fBdgsh-monitor\fP
.SH DESCRIPTION
\fIdgsh-monitor\fP is a filter that reads lines from its standard input
and for each line writes a JSON record containing
the absolute timestamp (\fCatime\fP),
the relative timestamp (\fCrtime\fP),
the number of lines read (\fCnlines\fP),
the number of bytes read (\fCnbytes\fP),
and the actual data (\fCdata\fP).
The values printed are those that were in effect before the line's
first character was read (i.e. the number of bytes and lines starts at 0).
The timestamps are printed as a decimal number of seconds
(since epoch, January 1 1970, for the absolute one)
with microsecond precision.
.PP
The command can be used in conjunction with \fIdgsh-writeval\fP
for providing pipeline monitoring ports as a debugging aid.
.SH "SEE ALSO"
\fIdgsh\fP(1),
\fIdgsh-writeval\fP(1)
.SH BUGS
When providing data regarding a single record,
such as that written to a store,
it can be confusing to see all the numerical values as zero.
.SH AUTHOR
Diomidis Spinellis \(em <http://www.spinellis.gr>
================================================
FILE: core-tools/src/dgsh-monitor.c
================================================
/*
* Copyright 2013 Diomidis Spinellis
*
* Prepend lines read with timestamp, number of lines, number of bytes.
* Used for providing dgsh monitoring ports.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
#include <sys/types.h>
#include <sys/time.h>
#include <stdbool.h>
#include <stdio.h>
#include <stdlib.h>
#include <err.h>
#include <errno.h>
#include <limits.h>
#include <string.h>
#include <unistd.h>
#include "dgsh.h"
static const char *program_name;
static void
usage(void)
{
fprintf(stderr, "Usage: %s\n", program_name);
exit(1);
}
/* Print c with JSON escaping */
static void
escape(int c)
{
switch (c) {
case '\\': fputs("\\\\", stdout); break;
case '"': fputs("\\\"", stdout); break;
case '/': fputs("\\/", stdout); break;
case '\b': fputs("\\b", stdout); break;
case '\f': fputs("\\f", stdout); break;
case '\n': fputs("\\n", stdout); break;
case '\r': fputs("\\r", stdout); break;
case '\t': fputs("\\t", stdout); break;
default:
if (c < 0x1f)
printf("\\u%04x", c);
else
putchar(c);
}
}
int
main(int argc, char *argv[])
{
int c;
unsigned long nlines, nbytes;
struct timeval start, t;
bool write_header = true;
bool wrote_header = false;
program_name = argv[0];
/* Default if nothing else is specified */
if (argc != 1)
usage();
nbytes = nlines = 0;
gettimeofday(&start, NULL);
while ((c = getchar()) != EOF) {
if (write_header) {
gettimeofday(&t, NULL);
if (wrote_header)
printf("\" }\n");
printf("{ "
// Absolute time (s)
"\"atime\": %lld.%06d, "
// Relative time (s, from program start)
"\"rtime\": %.06lf, "
"\"nlines\": %lu, "
"\"nbytes\": %lu, "
"\"data\": \"",
(long long)t.tv_sec,
(int)t.tv_usec,
(t.tv_sec - start.tv_sec) +
(t.tv_usec - start.tv_usec) / 1e6,
nlines,
nbytes);
wrote_header = true;
write_header = false;
}
escape(c);
nbytes++;
if (c == '\n') {
nlines++;
write_header = true;
}
}
if (wrote_header)
printf("\" }\n");
return 0;
}
================================================
FILE: core-tools/src/dgsh-parallel.1
================================================
.TH DGSH-PARALLEL 1 "15 December 2016"
.\"
.\" (C) Copyright 2016 Diomidis Spinellis. All rights reserved.
.\"
.\" Licensed under the Apache License, Version 2.0 (the "License");
.\" you may not use this file except in compliance with the License.
.\" You may obtain a copy of the License at
.\"
.\" http://www.apache.org/licenses/LICENSE-2.0
.\"
.\" Unless required by applicable law or agreed to in writing, software
.\" distributed under the License is distributed on an "AS IS" BASIS,
.\" WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
.\" See the License for the specific language governing permissions and
.\" limitations under the License.
.\"
.SH NAME
dgsh-parallel \- Create a semi-homogeneous dgsh parallel processing block
.SH SYNOPSIS
\fBdgsh-parallel\fP
[\fB\-d\fP]
\fB\-f\fP \fIfile\fP |
\fB\-l\fP \fIlist\fP |
\fB\-n\fP \fIn\fP
\fIcommand ...\fP
.SH DESCRIPTION
\fIdgsh-parallel\fP creates and executes a \fIdgsh\fP block
that invokes multiple times the specified command and its optional arguments.
If the command or its options include the \fI{}\fP string,
this is replaced by the numeric or string identifier associated with
each invocation.
.SH OPTIONS
.IP "\fB\-d\fP
Allows the debugging of the generated script, by leaving it in the
temporary directory and echoing its path on the standard error.
.IP "\fB\-f\fP \fIfile\fP"
Obtain string arguments from the specified file: one argument per line.
One command will be generated for each line in the file.
Each command will have \fI{}\fP strings replaced with the contents of
the corresponding line.
.IP "\fB\-l\fP \fIlist\fP"
Obtain string arguments from the specified comma-separated list.
One command will be generated for each list element.
Each command will have \fI{}\fP strings replaced with the corresponding
element.
.IP "\fB\-n\fP \fIn\fP"
Run \fIn\fP instances of the command.
Each command will have \fI{}\fP strings replaced with the command's
ordinal number, starting from 1.
.SH EXAMPLES
.PP
Count in parallel the number of times each word appears in the specified
input file(s).
This sequence mirrors Hadoop's WordCount example.
.ft C
.nf
# Scatter input
dgsh-tee -s |
# Run four instances of the command
# Emulate Java's default StringTokenizer, sort, count
dgsh-parallel -n 4 "tr -s ' \\t\\n\\r\\f' '\\n' | sort | uniq -c" |
# Merge the four sorted counts
dgsh-merge-sum '<|' '<|' '<|'
.ft P
.fi
.SH "SEE ALSO"
\fIdgsh\fP(1),
\fIdgsh-tee\fP(1),
.SH BUGS
The interface between the generated script and its invokers is currently
(December 2016) being polished.
.SH AUTHOR
Diomidis Spinellis \(em <http://www.spinellis.gr>.
================================================
FILE: core-tools/src/dgsh-parallel.sh
================================================
#!/usr/bin/env bash
#!dgsh
#
# Create and execute a semi-homongeneous dgsh parallel processing block
#
# Remove dgsh from path, so that commands aren't wrapped here
# See http://stackoverflow.com/a/2108540/20520
# PATH => /bin:.../libexec/dgsh:/sbin
OPATH="$PATH"
WORK=:$PATH:
# WORK => :/bin:.../libexec/dgsh:/sbin:
REMOVE='[^:]*/libexec/dgsh'
WORK=${WORK/:$REMOVE:/:}
# WORK => :/bin:/sbin:
WORK=${WORK%:}
WORK=${WORK#:}
PATH=$WORK
# PATH => /bin:/sbin
# Remove DGSH_IN, OUT so that commands don't negotiate
test "$DGSH_IN" && ODGSH_IN="$DGSH_IN"
test "$DGSH_OUT" && ODGSH_OUT="$DGSH_OUT"
unset DGSH_IN
unset DGSH_OUT
usage()
{
echo 'Usage: dgsh-parallel [-d] -n n|-f file|-l list command ...' 1>&2
exit 2
}
# Process flags
while getopts 'df:l:n:' o; do
case "$o" in
d)
DEBUG=1
;;
n)
n="$OPTARG"
nspec=X$nspec
;;
f)
file="$OPTARG"
nspec=X$nspec
;;
-l)
list=$(echo "$OPTARG" | sed 's/,/ /g')
nspec=X$nspec
;;
*)
usage
;;
esac
done
shift $((OPTIND-1))
# Ensure commands is specified
if [ ! "$1" ] ; then
usage
fi
# Ensure exactly one sharding target is specified
if [ ! "$nspec" ] || expr $nspec : .. >/dev/null ; then
usage
fi
# Ensure generated script is always removed
SCRIPT="${TMP:-/tmp}/dgsh-parallel-$$"
if [ "$DEBUG" ] ; then
echo "Script is $SCRIPT" 1>&2
else
trap 'rm -rf "$SCRIPT"' 0
trap 'exit 2' 1 2 15
fi
cat >$SCRIPT <<EOF
#!/usr/bin/env dgsh
#
# Automatically generated file from:
# $0 $*
#
{{
EOF
# Generate list of nodes
if [ "$n" ] ; then
for i in $(seq "$n") ; do
echo $i
done
elif [ "$list" ] ; then
for i in $list ; do
echo "$i"
done
elif [ "$file" ] ; then
cat "$file"
else
echo Internal error 1>&2
exit 2
fi |
# Escape sed(1) special characters
sed 's/[&/\\]/\\&/g' |
# Replace {} with the name of each node
while IFS='' read -r node ; do
echo " $@" | sed "s/{}/$node/"
done >>$SCRIPT
cat >>$SCRIPT <<EOF
}}
EOF
# Restore dgsh settings
PATH="$OPATH"
# Remove DGSH_IN, OUT so that commands don't negotiate
test "$ODGSH_IN" && export DGSH_IN="$ODGSH_IN"
test "$ODGSH_OUT" && export DGSH_OUT="$ODGSH_OUT"
dgsh $SCRIPT
================================================
FILE: core-tools/src/dgsh-pecho.c
================================================
#include <stdio.h> /* printf() */
#include <stdlib.h> /* exit() */
#include <unistd.h> /* getpagesize() */
#include "dgsh.h"
int
main(int argc, char *argv[])
{
int ps = getpagesize();
char buf[ps];
int n = 1;
int ninputs = -1;
dgsh_negotiate(DGSH_HANDLE_ERROR, "dgsh-pecho", &ninputs,
NULL, NULL, NULL);
if (ninputs == 1) {
n = read(STDIN_FILENO, buf, ps);
while (n) {
printf("%s", buf);
n = read(STDIN_FILENO, buf, ps);
if (n < 0)
exit(1);
}
}
++argv;
while (*argv) {
(void)printf("%s", *argv);
if (*++argv)
putchar(' ');
}
putchar('\n');
return 0;
}
================================================
FILE: core-tools/src/dgsh-readval.1
================================================
.TH DGSH-READVAL 1 "21 March 2013"
.\"
.\" (C) Copyright 2013 Diomidis Spinellis. All rights reserved.
.\"
.\" Licensed under the Apache License, Version 2.0 (the "License");
.\" you may not use this file except in compliance with the License.
.\" You may obtain a copy of the License at
.\"
.\" http://www.apache.org/licenses/LICENSE-2.0
.\"
.\" Unless required by applicable law or agreed to in writing, software
.\" distributed under the License is distributed on an "AS IS" BASIS,
.\" WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
.\" See the License for the specific language governing permissions and
.\" limitations under the License.
.\"
.SH NAME
dgsh-readval \- data store client
.SH SYNOPSIS
\fBdgsh-readval\fP
[\fB\-c\fP | \fB-e\fP | \fB-l\fP]
[\fB\-nq\fP]
[\fB\-x\fP]
\fB\-s\fP \fIpath\fP
.SH DESCRIPTION
\fIdgsh-readval\fP is a data store client.
By default it will communicate with the store specified through
the path to a Unix domain socket,
ask to read the last (final) record written to that store,
and write the value on its standard output.
.PP
\fIdgsh-readval\fP is normally executed from within \fIdgsh\fP-generated scripts,
rather than through end-user commands.
This manual page serves mainly to document its operation and
the flags that can be used in \fIdgsh\fP scripts when reading from stores.
.SH OPTIONS
.IP "\fB\-c\fP
Read the current (rather than the last) value from the store.
If no complete record has been written into the store,
the operation will block until such a record is available.
.IP "\fB\-e\fP
Read the current or an empty value from the store.
If no complete record has been written into the store,
the operation will return an empty record, rather than block.
.IP "\fB\-l\fP
Read the last value from the store.
This is the default behavior of \fIdgsh-readval\fP.
The operation will block until the store's server (\fIdgsh-writeval\fP)
determines that it has read the last record
by detecting an end-of-file condition on its standard input.
.IP "\fB\-n\fP
Do not retry a failed connection to the store.
By default \fIdgsh-readval\fP will try to establish a connection to the
store every one second.
This behavior is designed to avoid failures due to race conditions between write stores
that are started asynchronously (in the background) and subsequent read
operations from them.
.IP "\fB\-q\fP
Ask the write store (the corresponding \fIdgsh-writeval\fP process)
to terminate its operation.
No value is read.
.IP "\fB\-x\fP
Do not participate in dgsh negotiation.
.IP "\fB\-s\fP \fIpath\fP"
This mandatory option must be used to specify the path of the Unix-domain socket
\fIdgsh-readval\fP will connect to communicate with the store.
This is specified as a normal Unix file path,
e.g. \fC/tmp/myvalue\fP.
.SH "SEE ALSO"
\fIdgsh\fP(1),
\fIdgsh-writeval\fP(1)
.SH AUTHOR
Diomidis Spinellis \(em <http://www.spinellis.gr>
================================================
FILE: core-tools/src/dgsh-readval.c
================================================
/*
* Copyright 2013 Diomidis Spinellis
*
* Communicate with the data store specified as a Unix-domain socket.
* (User interface)
* By default the command will read a value.
* Calling it with the -q flag will send the data store a termination
* command.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
#include <stdbool.h>
#include <stdio.h>
#include <stdlib.h>
#include <err.h>
#include <errno.h>
#include <limits.h>
#include <string.h>
#include <unistd.h>
#include "dgsh.h"
#include "negotiate.h"
#include "kvstore.h"
#include "dgsh-debug.h"
static const char *program_name;
static void
usage(void)
{
fprintf(stderr, "Usage: %s [-c|e|l] [-n] [-q] [-x] -s path\n"
"-c" "\tRead the current value from the store\n"
"-e" "\tRead current value or empty from the store\n"
"-l" "\tRead the last (before EOF) value from the store (default)\n"
"-n" "\tDo not retry failed connection to write store\n"
"-q" "\tAsk the write-end to quit\n"
"-x" "\tDo not participate in dgsh negotiation\n"
"-s path" "\tSpecify the socket to connect to\n",
program_name);
exit(1);
}
int
main(int argc, char *argv[])
{
int ch;
bool quit = false;
char cmd = 0;
const char *socket_path = NULL;
bool retry_connection = true;
bool should_negotiate = true;
int ninputs = 0;
int noutputs = 1;
program_name = argv[0];
/* Default if nothing else is specified */
if (argc == 3)
cmd = 'L';
while ((ch = getopt(argc, argv, "celnqxs:")) != -1) {
switch (ch) {
case 'c': /* Read current value */
cmd = 'C';
break;
case 'e': /* Read current or empty value */
cmd = 'c';
break;
case 'l': /* Read last value */
cmd = 'L';
break;
case 'n':
retry_connection = false;
break;
case 'q':
quit = true;
break;
case 's':
socket_path = optarg;
break;
case 'x':
should_negotiate = false;
break;
case '?':
default:
usage();
}
}
argc -= optind;
argv += optind;
if (argc != 0 || socket_path == NULL)
usage();
if (should_negotiate)
dgsh_negotiate(DGSH_HANDLE_ERROR, program_name, &ninputs, &noutputs, NULL, NULL);
else
set_negotiation_complete();
dgsh_send_command(socket_path, cmd, retry_connection, quit, STDOUT_FILENO);
return 0;
}
================================================
FILE: core-tools/src/dgsh-tee.1
================================================
.TH DGSH-TEE 1 "13 April 2017"
.\"
.\" (C) Copyright 2013-2017 Diomidis Spinellis. All rights reserved.
.\"
.\" Licensed under the Apache License, Version 2.0 (the "License");
.\" you may not use this file except in compliance with the License.
.\" You may obtain a copy of the License at
.\"
.\" http://www.apache.org/licenses/LICENSE-2.0
.\"
.\" Unless required by applicable law or agreed to in writing, software
.\" distributed under the License is distributed on an "AS IS" BASIS,
.\" WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
.\" See the License for the specific language governing permissions and
.\" limitations under the License.
.\"
.SH NAME
dgsh-tee \- buffer, copy, permute, or distribute data from input sources to output sinks
.SH SYNOPSIS
\fBdgsh-tee\fP
[\fB\-b\fP \fIbuffer-size\fP]
[\fB\-afIMs\fP]
[\fB\-i\fP \fIinput-file\fP]
[\fB\-o\fP \fIoutput-file\fP]
[\fB\-m\fP \fImemory-size\fP]
[\fB\-p\fP \fIo1,o2 ...\fP]
[\fB\-T\fP \fIdirectory\fP]
[\fB\-t\fP \fIcharacter\fP]
.SH DESCRIPTION
\fIdgsh-tee\fP will read data from the specified sources and copy or distribute
it to the specified sinks.
It resembles in its operation \fItee\fP(1) and \fIcat\fP(1),
but offers additional capabilities required for the operation of \fIdgsh\fP(1).
In contrast to these programs, \fIdgsh-tee\fP will buffer the data it handles,
so it will never cause deadlock or starvation when one or more sources
are unable to provide data or if sinks are unable to receive them.
Furthermore, \fIdgsh-tee\fP can copy data from multiple sources to
multiple sinks, permute the data between sources and sinks, and
also distribute the data among the sinks.
.PP
When copying data from a few sources to a multiple of their number sinks,
the first input tuple will appear in the first sinks, and so on.
As an example, two sources \fIa, b\fP will appear in six sinks as
\fIa, b, a, b, a, b\fP.
When copying data from many sources to a fraction of their number sinks,
a tuple of sources equal to the number of sinks is output first,
followed by a tuple of the next sinks, and so on.
As an example, six sources \fa, b, c, d, e, f\fP will appear in two
sinks as \fIa, c, e\fP in the first sink and \fIb, d, f\fP in the
second one.
In effect, the group of few sources or sinks is treated as a single
unit to be scattered or sequentially concatenated.
.PP
\fIdgsh-tee\fP is normally executed within \fIdgsh\fP through wrappers
that replace the system-provided \fItee\fP and \fIcat\fP commands.
This manual page serves mainly to document its operation,
to how it can be used in less common use cases, and
to allow the creation of plug-compatible replacements
implementing different record types.
.SH OPTIONS
.IP "\fB\-a\fP
Open files subsequently specified with the \fB-o\fP option for appending.
.IP "\fB\-b\fP \fIbuffer-size\fP"
Specify the size of the buffer to use.
This is by default 1MB.
Buffers are chained together when more space is required,
so the main utility of this option is to decrease the buffer
size in memory-constrained environments.
The specified number can be suffixed with
\fBk\fI, \fBM\fI, or \fBG\fI to specify the corresponding unit.
The specified buffer size must be less than the program's maximum memory size.
.IP "\fB\-f\fP
When the allocated memory size reaches the maximum memory threshold,
start using a temporary file for buffering the data.
This extends the amount of data that can be buffered to the
space available on disk.
The location of the temporary file follows the
\fItempnam\fP(3) rules, and can be overridden through the
.B -T
option.
.IP "\fB\-I\fP"
Implement input-side buffering.
By default \fIdgsh-tee\fP will buffer only as much input data,
as is needed to avoid starving one of the specified sinks.
In doing so it may cause its input source to block
while having data to write.
When this option is enabled
\fIdgsh-tee\fP will always read data if they are available
on the standard input,
and will write that data to any (including zero) sinks that
can read it.
This can be useful in cases where a command with insufficient input
buffering,
like \fIjoin\fP(1), \fIsort\fP(1), or \fIpaste\fP(1),
is gathering input from commands executing in parallel.
In such a case adding \fIdgsh-tee\fP with input side buffering
enabled at the end of each data pipeline,
will increase the number of processes that can operate concurrently.
.IP "\fB\-i\fP \fIinput-file\fP"
Read input from the specified source file, rather than the standard input.
The option can be provided multiple times to specify multiple files that
will be read sequentially.
All input files will be opened at the beginning of the program's operation
in order to unblock the execution of asynchronous shell commands
that redirect their output to the corresponding named pipes.
Furthermore, when input-side buffering is specified \fB-I\fP
data is read asynchronously from all specified input files.
.IP "\fB\-M\fP"
Provide memory use statistics on termination.
This is mainly used for testing,
to check against leaks of buffers.
.IP "\fB\-o\fP \fIoutput-file\fP"
Write copies of the input data to the specified sink file,
rather than the standard output.
The option can be provided multiple times to specify multiple files
where input data will be copied.
.IP "\fB\-m\fP \fImemory-size\fP"
Specify the maximum size of memory to allocate for buffers.
This is by default 256MB.
When \fIdgsh-tee\fP exhausts this memory, it enters a state where it
waits for its output buffers drain, thus freeing allocated memory.
The specified number can be suffixed with
\fBk\fI, \fBM\fI, or \fBG\fI to specify the corresponding unit.
The specified maximum memory size must be larger than the program's buffer size.
.IP "\fB\-p\fP \fIo1,o2 ...\fP"
Permute the inputs to the specified outputs.
The comma-separated arguments \fIo1,o2, ...\fP
specify the number of the output channel (starting from 1)
where the corresponding \fIn\fPth input channel will be sent.
Thus,
input 1 goes to output \fIo1\fP,
input 2 goes to output \fIo2\fP,
and so on.
As an example a cross-permutation is specified with the argument \fI-p 2,1\fP.
.IP "\fB\-s\fP"
Scatter the input fairly across the sinks, rather than copying it to all.
When this option is in effect,
the input data are divided into chunks of one or more lines,
and each chunk is written only to a single sink.
This is useful for dividing the work among multiple processes operating
in parallel.
.IP "\fB\-T\fP \fIdirectory\fP"
Specify the directory to use for storing the temporary file,
when the specified maximum buffer memory size is exceeded.
.IP "\fB\-t\fP \fIchar\fP"
Use \fIchar\fP as the record separator,
By default the record separator is a newline,
An empty (not missing) argument for the record separator
will make the record separator be the null character.
.SH "SEE ALSO"
\fIdgsh\fP(1)
\fItempnam\fP(3)
.SH AUTHOR
Diomidis Spinellis \(em <http://www.spinellis.gr>
================================================
FILE: core-tools/src/dgsh-tee.c
================================================
/*
* Copyright 2013 Diomidis Spinellis
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
#ifdef __linux__
#define _XOPEN_SOURCE 500 // pread pwrite
#endif
#include <sys/types.h>
#include <sys/stat.h>
#include <sys/select.h>
#include <assert.h>
#include <err.h>
#include <errno.h>
#include <fcntl.h>
#include <signal.h>
#include <stdbool.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <unistd.h>
#include "dgsh.h"
#include "dgsh-debug.h"
#include "minmax.h"
#if defined(DEBUG_DATA)
#define DATA_DUMP 1
#else
#define DATA_DUMP 0
#endif
/*
* Data that can't be written is stored in a sequential pool of buffers,
* each buffer_size long.
* As more data is read the buffer pool with the pointers (buffers)
* is continuously increased; there is no round-robin mechanism.
* However, as data is written out to all sinks, the actual buffers are
* freed, thus keeping memory consumption reasonable.
*/
static int buffer_size = 1024 * 1024;
/*
* A buffer in the memory pool.
*/
struct pool_buffer {
void *p; /* Memory allocated for it (b_memory) */
enum {
s_none, /* Stored nowhere */
s_memory, /* Stored in memory */
s_memory_backed,/* Stored in memory and backed to temporary file */
s_file /* Stored in temporary file */
} s; /* Where it is stored */
};
/*
* A pool of buffers
*/
struct buffer_pool {
struct pool_buffer *buffers; /* A dynamically adjusted vector of buffers */
int pool_size; /* Size of allocated pool_buffers vectors */
int allocated_pool_end; /* The first buffer in the above pool that has not been allocated */
/* Allocated bufffer information */
int buffers_allocated, buffers_freed, max_buffers_allocated;
/* Paging information */
int buffers_paged_out, buffers_paged_in, pages_freed;
int page_out_ptr; /* Pointer to first buffer to page out */
int page_file_fd; /* File descriptor of temporary file used for paging buffer pool */
int free_pool_begin; /* Start of freed area */
};
/* Construct a new buffer pool object */
static struct buffer_pool *
new_buffer_pool(void)
{
struct buffer_pool *bp;
if ((bp = (struct buffer_pool *)malloc(sizeof(struct buffer_pool))) == NULL)
err(1, NULL);
bp->buffers = NULL;
bp->pool_size = 0;
bp->page_out_ptr = 0;
bp->page_file_fd = -1;
bp->free_pool_begin = 0;
bp->allocated_pool_end = 0;
bp->buffers_allocated = bp->buffers_freed = bp->max_buffers_allocated =
bp->buffers_paged_out = bp->buffers_paged_in = bp->pages_freed = 0;
return bp;
}
/*
* A buffer that is used for I/O.
* It points to a part of a pool buffer.
*/
struct io_buffer {
void *p; /* Memory pointer */
size_t size; /* Buffer size */
};
/* Maximum amount of memory to allocate. (Set through -S) */
static unsigned long max_mem = 256 * 1024 * 1204;
/* Scatter the output across the files, rather than copying it. */
static bool opt_scatter = false;
/*
* When set, permute the inputs to the specified outputs
* Ordinals and number of the destination outputs
*/
static int *permute_dest = NULL;
static int permute_n = 0;
/* Use a temporary file for overflowing buffered data */
static bool use_tmp_file = false;
/* User-specified temporary directory */
static char *opt_tmp_dir = NULL;
/*
* Split scattered data on blocks of specified size; otherwise on line boundaries
* Currently there is no support for this option; a -l option should be added.
*/
static bool block_len = 0;
/* Set to true when we reach EOF on input */
static bool reached_eof = false;
/* Record terminator */
static char rt = '\n';
/* Linked list of files we write to */
struct sink_info {
struct sink_info *next; /* Next list element */
char *name; /* Output file name */
int fd; /* Output file descriptor */
off_t pos_written; /* Position up to which written */
off_t pos_to_write; /* Position up to which to write */
bool active; /* True if this sink is still active */
struct source_info *ifp;/* Input file we read from */
bool chain_last; /* True if last element in a group; Writing (copy or scatter)
should not continue to next element */
};
/* Construct a new sink_info object */
static struct sink_info *
new_sink_info(const char *name)
{
struct sink_info *ofp;
if ((ofp = (struct sink_info *)malloc(sizeof(struct sink_info))) == NULL)
err(1, NULL);
ofp->name = name ? strdup(name) : NULL;
ofp->active = true;
ofp->pos_written = ofp->pos_to_write = 0;
ofp->next = NULL;
return ofp;
}
/* Linked list of files we read from */
struct source_info {
struct source_info *next; /* Next list element */
char *name; /* Input file name */
int fd; /* Input file descriptor */
struct buffer_pool *bp; /* Buffers where pending input is stored */
off_t source_pos_read; /* The position up to which all sinks have read data */
bool reached_eof; /* True if we reached EOF for this source */
off_t read_min_pos; /* Minimum position read by all sinks */
bool active; /* True if this is a source that should be currently
read (rather than chained later on) */
bool is_read; /* True if an active sink reads it */
bool chain_last; /* True if reading should stop at this element rather
than continue to the next element */
};
/* Return the name of a source or sink */
#define fp_name(fp) ((fp)->name ? (fp)->name : fd_name((fp)->fd))
static char *
fd_name(int fd)
{
static char buff[40];
sprintf(buff, "fd(%d)", fd);
return buff;
}
/* Construct a new source_info object */
static struct source_info *
new_source_info(const char *name)
{
struct source_info *ifp;
if ((ifp = (struct source_info *)malloc(sizeof(struct source_info))) == NULL)
err(1, NULL);
ifp->name = name ? strdup(name) : NULL;
ifp->bp = new_buffer_pool();
ifp->source_pos_read = 0;
ifp->reached_eof = false;
ifp->next = NULL;
return ifp;
}
/*
* States for the copying engine.
* Two disjunct sets:
* input side buffering (ib) and output side buffering (ob)
* Input side buffering will always read input if it is available,
* presenting an infinite output buffer to the upstream process.
* The output-side buffering will read input only if at least one
* active output buffer is empty.
* The setting in effect is determined by the program's -I flag.
*
* States read_ib and read_ob have select return:
* - if data is available for reading,
* - if the process can write out data already read,
* - not if the process can write to other fds
*
* States drain_ib and write_ob have select return
* if the process can write to any fd.
* Waiting on all output buffers (not only those with data)
* is needed to avoid starvation of downstream processes
* when no output is available.
* If a program can accept data this process will then transition to
* read_* to read more data.
*
* State drain_ob has select return only if the process can write out
* data already read.
*
* See also the diagram tee-state.dot
*/
enum state {
read_ib, /* Must read input; write if data available */
read_ob, /* As above, but don't transition to write */
drain_ib, /* Don't read input; write if possible */
drain_ob, /* Empty data buffers by writing */
write_ob, /* Write data, before reading */
};
/*
* Return the total number of bytes required for storing all buffers
* up to the specified memory pool
*/
static unsigned long
memory_pool_size(struct buffer_pool *bp, int pool)
{
return ((bp->buffers_allocated - bp->buffers_freed) + (pool - bp->allocated_pool_end + 1)) * buffer_size;
}
/* Write half of the allocated buffer pool to the temporary file */
static void
page_out(struct buffer_pool *bp)
{
if (bp->page_file_fd == -1) {
char *template;
/*
* Create a temporary file that will be deleted on exit.
* The location follows tempnam rules (argument, TMPDIR,
* P_tmpdir, /tmp), while the creation through mkstemp
* avoids race conditions.
*/
if ((template = tempnam(opt_tmp_dir, "sg-")) == NULL)
err(1, "Unable to obtain temporary file name");
if ((template = realloc(template, strlen(template) + 7)) == NULL)
err(1, "Error obtaining temporary file name space");
strcat(template, "XXXXXX");
if ((bp->page_file_fd = mkstemp(template)) == -1)
err(1, "Unable to create temporary file %s", template);
}
/*
* Page-out memory buffers from the pool, round-robin fashion,
* starting from the oldest buffers.
* This is good enough for the simple common case where one output fd is blocked.
*/
while (memory_pool_size(bp, bp->allocated_pool_end - 1) > max_mem / 2) {
switch (bp->buffers[bp->page_out_ptr].s) {
case s_memory:
if (pwrite(bp->page_file_fd, bp->buffers[bp->page_out_ptr].p, buffer_size, (off_t)bp->page_out_ptr * buffer_size) != buffer_size)
err(1, "Write to temporary file failed");
/* FALLTHROUGH */
case s_memory_backed:
DPRINTF(4, "Page out buffer %d %p", bp->page_out_ptr, bp->buffers[bp->page_out_ptr].p);
bp->buffers[bp->page_out_ptr].s = s_file;
free(bp->buffers[bp->page_out_ptr].p);
bp->buffers_freed++;
bp->buffers_paged_out++;
DPRINTF(4, "Paged out buffer %d %p", bp->page_out_ptr, bp->buffers[bp->page_out_ptr].p);
break;
case s_file:
case s_none:
break;
default:
assert(false);
}
if (++bp->page_out_ptr == bp->allocated_pool_end)
bp->page_out_ptr = 0;
}
}
/*
* Allocate memory for the specified pool member.
* Return false if no such memory is available.
*/
static bool
allocate_pool_buffer(struct buffer_pool *bp, int pool)
{
struct pool_buffer *b = &bp->buffers[pool];
if ((b->p = malloc(buffer_size)) == NULL) {
DPRINTF(4, "Unable to allocate %d bytes for buffer %ld", buffer_size, b - bp->buffers);
bp->max_buffers_allocated = MAX(bp->buffers_allocated - bp->buffers_freed, bp->max_buffers_allocated);
return false;
}
b->s = s_memory;
DPRINTF(4, "Allocated buffer %ld to %p", b - bp->buffers, b->p);
bp->buffers_allocated++;
bp->max_buffers_allocated = MAX(bp->buffers_allocated - bp->buffers_freed, bp->max_buffers_allocated);
return true;
}
/*
* Ensure that the specified pool buffer is in memory
*/
static void
page_in(struct buffer_pool *bp, int pool)
{
struct pool_buffer *b = &bp->buffers[pool];
switch (b->s) {
case s_memory_backed:
case s_memory:
break;
case s_file:
/* Good time to ensure that there will be page-in memory available */
if (memory_pool_size(bp, bp->allocated_pool_end - 1) > max_mem)
page_out(bp);
if (!allocate_pool_buffer(bp, pool))
err(1, "Out of memory paging-in buffer");
if (pread(bp->page_file_fd, b->p, buffer_size, (off_t)pool * buffer_size) != buffer_size)
err(1, "Read from temporary file failed");
bp->buffers_paged_in++;
b->s = s_memory_backed;
DPRINTF(4, "Page in buffer %d", pool);
break;
case s_none:
default:
DPRINTF(4, "Buffer %d has invalid storage %d", pool, b->s);
assert(false);
break;
}
}
/*
* Allocate memory for the specified pool
* If we're out of memory by reaching the user-specified limit
* or a system's hard limit return false.
* If sufficient memory is available return true.
* buffers[pool] will then point to a buffer_size block of available memory.
*/
static bool
memory_allocate(struct buffer_pool *bp, int pool)
{
int i, orig_pool_size;
struct pool_buffer *orig_buffers;
if (pool < bp->allocated_pool_end)
return true;
DPRINTF(4, "Buffers allocated: %d Freed: %d", bp->buffers_allocated, bp->buffers_freed);
/* Check soft memory limit through allocated plus requested memory. */
if (memory_pool_size(bp, pool) > max_mem) {
if (use_tmp_file)
page_out(bp);
else
return false;
}
/* Keep original values to undo on failure. */
orig_pool_size = bp->pool_size;
orig_buffers = bp->buffers;
/* Resize bank, if needed. One iteration should suffice. */
while (pool >= bp->pool_size) {
if (bp->pool_size == 0)
bp->pool_size = 1;
else
bp->pool_size *= 2;
if ((bp->buffers = realloc(bp->buffers, bp->pool_size * sizeof(struct pool_buffer))) == NULL) {
DPRINTF(4, "Unable to reallocate buffer pool bank");
bp->pool_size = orig_pool_size;
bp->buffers = orig_buffers;
return false;
}
}
/* Allocate buffer memory [allocated_pool_end, pool]. */
for (i = bp->allocated_pool_end; i <= pool; i++)
if (!allocate_pool_buffer(bp, i)) {
bp->allocated_pool_end = i;
return false;
}
bp->allocated_pool_end = pool + 1;
return true;
}
/*
* Free a file-backed buffer at the specified pool location
* by punching a hole to the file. This is a best effort
* operation, as it is only supported on Linux.
*/
static void
buffer_file_free(struct buffer_pool *bp, int pool)
{
#ifdef FALLOC_FL_PUNCH_HOLE
static bool warned = false;
if (fallocate(bp->page_file_fd, FALLOC_FL_PUNCH_HOLE, pool * buffer_size, buffer_size) < 0 &&
!warned) {
warn("Failed to free temporary buffer space");
warned = true;
}
#endif
bp->pages_freed++;
}
/*
* Ensure that pool buffers from [0,pos) are free.
*/
static void
memory_free(struct buffer_pool *bp, off_t pos)
{
int pool_end = pos / buffer_size;
int i;
DPRINTF(4, "memory_free: pool=%p pos = %ld, begin=%d end=%d",
bp, (long)pos, bp->free_pool_begin, pool_end);
for (i = bp->free_pool_begin; i < pool_end; i++) {
switch (bp->buffers[i].s) {
case s_memory:
free(bp->buffers[i].p);
bp->buffers_freed++;
break;
case s_file:
buffer_file_free(bp, i);
break;
case s_memory_backed:
buffer_file_free(bp, i);
free(bp->buffers[i].p);
bp->buffers_freed++;
break;
case s_none:
break;
default:
assert(false);
break;
}
bp->buffers[i].s = s_none;
DPRINTF(4, "Freed buffer %d %p (pos = %ld, begin=%d end=%d)",
i, bp->buffers[i].p, (long)pos, bp->free_pool_begin, pool_end);
#ifdef DEBUG
bp->buffers[i].p = NULL;
#endif
}
bp->free_pool_begin = pool_end;
}
/*
* Set the buffer to write to for reading from a file from
* position onward, ensuring that sufficient memory is allocated.
* Return false if no memory is available.
*/
static bool
source_buffer(struct source_info *ifp, /* OUT */ struct io_buffer *b)
{
int pool = ifp->source_pos_read / buffer_size;
size_t pool_offset = ifp->source_pos_read % buffer_size;
if (!memory_allocate(ifp->bp, pool))
return false;
if (ifp->bp->buffers[pool].s != s_memory)
DPRINTF(4, "ifp->bp->buffers[pool].s = 0x%x, pool=%d\n", ifp->bp->buffers[pool].s, pool);
assert(ifp->bp->buffers[pool].s == s_memory);
b->p = ifp->bp->buffers[pool].p + pool_offset;
b->size = buffer_size - pool_offset;
DPRINTF(4, "Source buffer(%ld) returns pool %d(%p) o=%ld l=%ld a=%p",
(long)ifp->source_pos_read, pool, ifp->bp->buffers[pool].p, (long)pool_offset, (long)b->size, b->p);
return true;
}
/*
* Return a buffer to read from for writing to a file from a position onward
* When processing lines, b.size can be 0
*/
static struct io_buffer
sink_buffer(struct sink_info *ofp)
{
struct io_buffer b;
int pool = ofp->pos_written / buffer_size;
size_t pool_offset = ofp->pos_written % buffer_size;
size_t source_bytes = ofp->pos_to_write - ofp->pos_written;
b.size = MIN(buffer_size - pool_offset, source_bytes);
if (b.size == 0)
b.p = NULL;
else {
if (ofp->ifp->bp->page_file_fd != -1)
page_in(ofp->ifp->bp, pool);
b.p = ofp->ifp->bp->buffers[pool].p + pool_offset;
}
DPRINTF(4, "Sink buffer(%ld-%ld) returns pool %d(%p) o=%ld l=%ld a=%p for input fd: %s",
(long)ofp->pos_written, (long)ofp->pos_to_write, pool, b.size ? ofp->ifp->bp->buffers[pool].p : NULL, (long)pool_offset, (long)b.size, b.p, fp_name(ofp->ifp));
return b;
}
/*
* Return a pointer to read from for writing to a file from a position onward
*/
static char *
sink_pointer(struct buffer_pool *bp, off_t pos_written)
{
int pool = pos_written / buffer_size;
size_t pool_offset = pos_written % buffer_size;
if (bp->page_file_fd != -1)
page_in(bp, pool);
return bp->buffers[pool].p + pool_offset;
}
/*
* Return the size of a buffer region that can be read for the specified endpoints
*/
static size_t
sink_buffer_length(off_t start, off_t end)
{
size_t pool_offset = start % buffer_size;
size_t source_bytes = end - start;
DPRINTF(4, "sink_buffer_length(%ld, %ld) = %ld",
(long)start, (long)end, (long)MIN(buffer_size - pool_offset, source_bytes));
return MIN(buffer_size - pool_offset, source_bytes);
}
/* The result of the following read operation. */
enum read_result {
read_ok, /* Normal read */
read_oom, /* Out of buffer memory */
read_again, /* EAGAIN */
read_eof, /* EOF (0 bytes read) */
};
/*
* Read from the source into the memory buffer
* Return the number of bytes read, or -1 on end of file.
*/
static enum read_result
source_read(struct source_info *ifp)
{
int n;
struct io_buffer b;
if (!source_buffer(ifp, &b)) {
DPRINTF(4, "Memory full");
/* Provide some time for the output to drain. */
return read_oom;
}
if ((n = read(ifp->fd, b.p, b.size)) == -1)
switch (errno) {
case EAGAIN:
DPRINTF(4, "EAGAIN on %s", fp_name(ifp));
return read_again;
default:
err(3, "Read from %s", fp_name(ifp));
}
ifp->source_pos_read += n;
DPRINTF(4, "Read %d out of %zu bytes from %s data=[%.*s]", n, b.size, fp_name(ifp),
(int)n * DATA_DUMP, (char *)b.p);
/* Return -1 on EOF */
return n ? read_ok : read_eof;
}
/*
* Allocate available read data to empty sinks that can be written to,
* by adjusting their ifp, pos_written, and pos_to_write pointers.
*/
static void
allocate_data_to_sinks(fd_set *sink_fds, struct sink_info *files)
{
struct sink_info *ofp;
int available_sinks = 0;
off_t pos_assigned = 0;
size_t available_data, data_per_sink;
size_t data_to_assign = 0;
bool use_reliable = false;
/* Easy case: distribute to all files. */
if (!opt_scatter) {
for (ofp = files; ofp; ofp = ofp->next) {
/* Advance to next input file, if required */
if (ofp->pos_written == ofp->ifp->source_pos_read &&
ofp->ifp->reached_eof &&
!ofp->ifp->chain_last) {
DPRINTF(4, "%s(): advance to input file %s\n",
__func__, fp_name(ofp->ifp));
ofp->ifp = ofp->ifp->next;
ofp->ifp->active = true;
ofp->pos_written = 0;
}
ofp->pos_to_write = ofp->ifp->source_pos_read;
}
return;
}
/*
* Difficult case: fair scattering across available sinks
* Thankfully here we only have a single input file
*/
/* Determine amount of fresh data to write and number of available sinks. */
for (ofp = files; ofp; ofp = ofp->next) {
pos_assigned = MAX(pos_assigned, ofp->pos_to_write);
if (ofp->pos_written == ofp->pos_to_write && FD_ISSET(ofp->fd, sink_fds))
available_sinks++;
}
/*
* Ensure we operate in a continuous memory region by clamping
* the length of the available data to terminate at the end of
* the buffer.
*/
available_data = sink_buffer_length(pos_assigned, files->ifp->source_pos_read);
if (available_sinks == 0)
return;
/* Assign data to sinks. */
data_per_sink = available_data / available_sinks;
for (ofp = files; ofp; ofp = ofp->next) {
/* Move to next file if this has data to write, or isn't ready. */
if (ofp->pos_written != ofp->pos_to_write || !FD_ISSET(ofp->fd, sink_fds))
continue;
DPRINTF(4, "pos_assigned=%ld source_pos_read=%ld available_data=%ld available_sinks=%d data_per_sink=%ld",
(long)pos_assigned, (long)ofp->ifp->source_pos_read, (long)available_data, available_sinks, (long)data_per_sink);
/* First file also gets the remainder bytes. */
if (data_to_assign == 0)
data_to_assign = sink_buffer_length(pos_assigned,
pos_assigned + data_per_sink + available_data % available_sinks);
else
data_to_assign = data_per_sink;
/*
* Assign data_to_assign to *ofp (pos_written, pos_to_write),
* and advance pos_assigned.
*/
ofp->pos_written = pos_assigned; /* Initially nothing has been written. */
if (block_len == 0) { /* Write whole lines */
if (available_data > buffer_size / 2 && !use_reliable) {
/*
* Efficient algorithm:
* Assume that multiple lines appear in data_per_sink.
* Go to a calculated boundary and scan backward to find
* a new line.
*/
off_t data_end = pos_assigned + data_to_assign - 1;
for (;;) {
if (data_end <= pos_assigned) {
/*
* If no newline was found with backward scanning
* degenerate to the efficient algorithm. This will
* scan further forward, and can defer writing the
* last chunk, until more data is read.
*/
use_reliable = true;
goto reliable;
}
if (*sink_pointer(ofp->ifp->bp, data_end) == rt) {
pos_assigned = data_end + 1;
break;
}
data_end--;
}
} else {
/*
* Reliable algorithm:
* Scan forward for new lines until at least
* data_per_sink are covered, or we reach the end of available data.
* Keep a record of the last encountered newline.
* This is used to backtrack when we scan past the end of the
* available data.
*/
off_t data_end, last_nl;
reliable:
last_nl = -1;
data_end = pos_assigned;
for (;;) {
if (data_end >= ofp->ifp->source_pos_read) {
if (last_nl != -1) {
pos_assigned = last_nl + 1;
break;
} else {
/* No newline found in buffer; defer writing. */
ofp->pos_to_write = pos_assigned;
DPRINTF(4, "scatter to file[%s] no newline from %ld to %ld",
fp_name(ofp), (long)pos_assigned, (long)data_end);
return;
}
}
if (*sink_pointer(ofp->ifp->bp, data_end) == rt) {
last_nl = data_end;
if (data_end - pos_assigned > data_per_sink) {
pos_assigned = data_end + 1;
break;
}
}
data_end++;
}
}
} else
pos_assigned += data_to_assign;
ofp->pos_to_write = pos_assigned;
DPRINTF(4, "scatter to file[%s] pos_written=%ld pos_to_write=%ld data=[%.*s]",
fp_name(ofp), (long)ofp->pos_written, (long)ofp->pos_to_write,
(int)(ofp->pos_to_write - ofp->pos_written) * DATA_DUMP, sink_pointer(ofp->ifp->bp, ofp->pos_written));
}
}
/*
* Write out from the memory buffer to the sinks where write will not block.
* Free memory no more needed even by the write pointer farthest behind.
* Return the number of bytes written.
*/
static size_t
sink_write(struct source_info *ifiles, fd_set *sink_fds, struct sink_info *ofiles)
{
struct sink_info *ofp;
struct source_info *ifp;
size_t written = 0;
for (ifp = ifiles; ifp; ifp = ifp->next) {
ifp->read_min_pos = ifp->source_pos_read;
ifp->is_read = false;
}
allocate_data_to_sinks(sink_fds, ofiles);
for (ofp = ofiles; ofp; ofp = ofp->next) {
DPRINTF(4, "\n%s(): try write to file %s", __func__, fp_name(ofp));
if (ofp->active && FD_ISSET(ofp->fd, sink_fds)) {
int n;
struct io_buffer b;
b = sink_buffer(ofp);
DPRINTF(4, "\n%s(): sink buffer returned %d bytes to write",
__func__, (int)b.size);
if (b.size == 0)
/* Can happen when a line spans a buffer */
n = 0;
else {
n = write(ofp->fd, b.p, b.size);
if (n < 0)
switch (errno) {
/* EPIPE is acceptable, for the sink's reader can terminate early. */
case EPIPE:
ofp->active = false;
(void)close(ofp->fd);
DPRINTF(4, "EPIPE for %s", fp_name(ofp));
break;
case EAGAIN:
DPRINTF(4, "EAGAIN for %s", fp_name(ofp));
n = 0;
break;
default:
err(2, "Error writing to %s", fp_name(ofp));
}
else {
ofp->pos_written += n;
written += n;
}
}
DPRINTF(4, "Wrote %d out of %zu bytes for file %s pos_written=%lu data=[%.*s]",
n, b.size, fp_name(ofp), (unsigned long)ofp->pos_written, (int)n * DATA_DUMP, (char *)b.p);
}
if (ofp->active) {
ofp->ifp->read_min_pos = MIN(ofp->ifp->read_min_pos, ofp->pos_written);
ofp->ifp->is_read = true;
}
}
/* Free buffers all sinks have read */
for (ifp = ifiles; ifp; ifp = ifp->next) {
memory_free(ifp->bp, ifp->read_min_pos);
/*
* We are reading this source, so don't even think freeing
* sources after this.
*/
if (ifp->is_read)
break;
}
DPRINTF(4, "Wrote %zu total bytes", written);
return written;
}
static void
usage(const char *name)
{
fprintf(stderr, "Usage %s [-b size] [-i file] [-IMs] [-o file] [-m size] [-t char]\n"
"-a" "\tOpen output file(s) for appending\n"
"-b size" "\tSpecify the size of the buffer to use (used for stress testing)\n"
"-f" "\tOverflow buffered data into a temporary file\n"
"-I" "\tInput-side buffering\n"
"-i file" "\tGather input from specified file\n"
"-m size[k|M|G]""\tSpecify the maximum buffer memory size\n"
"-M" "\tProvide memory use statistics on termination\n"
"-o file" "\tScatter output to specified file\n"
"-p d1[,d2...]" "\tPermute inputs to specified outputs\n"
"-s" "\tScatter the input across the files, rather than copying it to all\n"
"-T dir" "\tSpecify directory for storing temporary file\n"
"-t char" "\tProcess char-terminated records (newline default)\n",
name);
exit(1);
}
/*
* Set the specified file descriptor to operate in non-blocking
* mode.
* It seems that even if select returns for a specified file
* descriptor, performing I/O to it may block depending on the
* amount of data specified.
* See See http://pubs.opengroup.org/onlinepubs/009695399/functions/write.html#tag_03_866
*/
static void
non_block(int fd, const char *name)
{
int flags = fcntl(fd, F_GETFL, 0);
if (flags < 0)
err(2, "Error getting flags for %s", name);
if (fcntl(fd, F_SETFL, flags | O_NONBLOCK) < 0)
err(2, "Error setting %s to non-blocking mode", name);
}
/*
* Show the arguments passed to select(2) in human-readable form
* If check is true, abort the program if no bit is on
*/
static void
show_select_args(const char *msg, fd_set *source_fds, struct source_info *ifiles, fd_set *sink_fds, struct sink_info *ofiles, bool check)
{
#ifdef DEBUG
struct sink_info *ofp;
struct source_info *ifp;
int nbits = 0;
fprintf(stderr, "%s: ", msg);
for (ifp = ifiles; ifp; ifp = ifp->next)
if (FD_ISSET(ifp->fd, source_fds)) {
fprintf(stderr, "%s ", fp_name(ifp));
nbits++;
}
for (ofp = ofiles; ofp; ofp = ofp->next)
if (FD_ISSET(ofp->fd, sink_fds)) {
fprintf(stderr, "%s ", fp_name(ofp));
nbits++;
}
fputc('\n', stderr);
if (check && nbits == 0)
abort();
#endif
}
static void
show_state(enum state state)
{
#ifdef DEBUG
char *s;
switch (state) {
case read_ib:
s = "read_ib";
break;
case read_ob:
s = "read_ob";
break;
case drain_ib:
s = "drain_ib";
break;
case drain_ob:
s = "drain_ob";
break;
case write_ob:
s = "write_ob";
break;
}
fprintf(stderr, "State: %s\n", s);
#endif
}
/* Parse the specified option as a size with a suffix and return its value. */
static unsigned long
parse_size(const char *progname, const char *opt)
{
char size;
unsigned long n;
size = 'b';
if (sscanf(opt, "%lu%c", &n, &size) < 1)
usage(progname);
switch (size) {
case 'B' : case 'b':
return n;
case 'K' : case 'k':
return n * 1024;
case 'M' : case 'm':
return n * 1024 * 1024;
case 'G' : case 'g':
return n * 1024 * 1024 * 1024;
default:
fprintf(stderr, "Unknown size suffix: %c\n", size);
usage(progname);
}
/* NOTREACHED */
return 0;
}
/*
* Parse and validate a comma-separated list of integers setting the
* variables permute_dest and permute_n.
*/
static void
parse_permute(char *s)
{
char *p;
char *copy = strdup(s);
int i;
if (copy == NULL)
errx(1, "Out of memory for destination string");
DPRINTF(4, "In parse_permute [%s]", s);
for (p = strtok(copy, ","); p != NULL; p = strtok(NULL, ","))
permute_n++;
free(copy);
if ((permute_dest = (int *)malloc(sizeof(int) * permute_n)) == NULL)
errx(1, "Out of memory for permutation destination");
for (p = strtok(s, ","), i= 0; p != NULL; p = strtok(NULL, ","), i++) {
permute_dest[i] = atoi(p) - 1;
if (permute_dest[i] < 0 || permute_dest[i] >= permute_n)
errx(1, "Illegal permutation destination [%s]", s);
}
for (i = 0; i < permute_n; i++)
DPRINTF(4, "%d = %d", i, permute_dest[i]);
DPRINTF(4, "permute_n=%d", permute_n);
}
/*
* Return the input file corresponding to the specified
* permuted output file number.
*/
static struct source_info *
output_source(struct source_info *ifiles, int output_n)
{
int i, input_n = -1;
struct source_info *ifp;
/* Find input file number */
for (i = 0; i < permute_n; i++)
if (permute_dest[i] == output_n) {
input_n = i;
break;
}
if (input_n == -1)
errx(1, "Unspecified output %d", output_n + 1);
/* Find input file pointer */
for (ifp = ifiles, i = 0; ifp; ifp = ifp->next, i++)
if (i == input_n)
return ifp;
assert(0);
return NULL;
}
static void
memory_stats(struct source_info *ifiles)
{
struct source_info *ifp;
for (ifp = ifiles; ifp; ifp = ifp->next) {
fprintf(stderr, "Input file: %s\n", fp_name(ifp));
fprintf(stderr, "Buffers allocated: %d Freed: %d Maximum allocated: %d\n",
ifp->bp->buffers_allocated, ifp->bp->buffers_freed, ifp->bp->max_buffers_allocated);
fprintf(stderr, "Page out: %d In: %d Pages freed: %d\n",
ifp->bp->buffers_paged_out, ifp->bp->buffers_paged_in, ifp->bp->pages_freed);
}
}
/*
* Return true if an element with ordinal number n,
* is the first element of a group in a series of groups
* of group_size each.
* Example: elements 0 and 3 in groups of size 3.
*/
static bool
first_in_group(int group_size, int n)
{
return n % group_size == 0;
}
/*
* Return true if an element with ordinal number n,
* is the last element of a group in a series of groups
* of group_size each.
* Example: elements 2 and 5 in groups of size 3.
*/
static bool
last_in_group(int group_size, int n)
{
return (n + 1) % group_size == 0;
}
struct list {
struct list *next;
};
/*
* Transpose the elements of th specified linked list given
* a notional new row length. As an example, a list of 12 elements
* with a row size of 3, would be transposed as follows.
* 0 -> 4 -> 8 ->
* 1 -> 5 -> 9 ->
* 2 -> 6 -> 10 ->
* 3 -> 7 -> 11
* This function can handle arbitrary lists, as long
* as the list's first element is the pointer to the
* next one.
*/
static void
list_transpose(struct list *lst, int row_length)
{
int i, count = 0;
struct list **vector, *p;
/* Create a vector of pointers to list elements */
for (p = lst; p; p = p->next)
count++;
vector = (struct list **)malloc(count * sizeof(struct list *));
if (vector == NULL)
err(1, NULL);
for (p = lst, i = 0; p; p = p->next, i++)
vector[i] = p;
/* Transpose notional rows into columns */
for (i = 0; i < count - row_length; i++)
vector[i]->next = vector[i + row_length];
for (i = count - row_length; i < count - 1; i++)
vector[i]->next = vector[(i + 1) % row_length];
free(vector);
}
/*
* Chain input and output files into groups
* by setting the chain_last of all I/O files
* and the input file (ifp) field of all output
* files.
*
* These are the possible cases and the corresponding
* group chains.
*
* Read from many, output to one (cat)
* A->B->C > a
* All input files are chained together
*
* Read from one, output to many (tee)
* A > b->c->d
* All output files are chained together
*
* Read from many output to permuted many (perm)
* A > d
* B > c
* C > b
* D > a
* No files are chained
*
* Read from few, output to more (multipipe tee)
* A > a->d->g
* B > b->e->h
* C > c->f->i
* Output files are chained into groups
*
* Read from many, output to few (multipipe cat)
* A->D->G > a
* B->E->H > b
* C->F->I > c
* Input files are chained into groups
*
* Note that cat, tee, and perm are special cases of the multipipe ones
* are are implemented as such.
*/
static void
chain_io_files(struct source_info *ifiles, struct sink_info *ofiles, bool permute)
{
int nin = 0, nout = 0;
int group_size, n, i;
struct source_info *ifp;
struct sink_info *ofp;
for (ifp = ifiles; ifp; ifp = ifp->next)
nin++;
for (ofp = ofiles; ofp; ofp = ofp->next)
nout++;
if (nin >= nout) {
/*
* Read from many output to few.
* First input element in group is active.
* Chain all but the last element of each input chain.
* None of the outputs are chained.
*/
if (nin % nout)
errx(1, "The number of inputs %d is not an exact multiple of the number of outputs %d", nin, nout);
group_size = nin / nout;
list_transpose((struct list *)ifiles, group_size);
for (ifp = ifiles, n = 0; ifp; ifp = ifp->next, n++) {
ifp->active = first_in_group(group_size, n);
ifp->chain_last = last_in_group(group_size, n);
}
for (ofp = ofiles, ifp = ifiles, n = 0; ofp; ofp = ofp->next, n++) {
ofp->chain_last = true;
ofp->ifp = permute ? output_source(ifiles, n) : ifp;
for (i = 0; i < group_size; i++)
ifp = ifp->next;
}
} else {
/*
* Read from few output to many.
* All inputs are active, none is chained.
* Chain all but the last element in the output chain.
*/
if (nout % nin)
errx(1, "The number of outputs %d is not an exact multiple of the number of inputs %d", nin, nout);
group_size = nout / nin;
assert(!permute);
list_transpose((struct list *)ofiles, group_size);
for (ifp = ifiles, n = 0; ifp; ifp = ifp->next, n++) {
ifp->active = true;
ifp->chain_last = true;
}
for (ofp = ofiles, ifp = ifiles, n = 0; ofp; ofp = ofp->next, n++) {
ofp->ifp = ifp;
if (last_in_group(group_size, n)) {
ifp = ifp->next;
ofp->chain_last = true;
} else
ofp->chain_last = false;
}
}
assert(ifp == NULL);
DPRINTF(3, "Input files");
for (ifp = ifiles; ifp; ifp = ifp->next)
DPRINTF(3, "%p: chain_last=%d (%s)", ifp, ifp->chain_last, ifp->name);
DPRINTF(3, "Output files");
for (ofp = ofiles; ofp; ofp = ofp->next)
DPRINTF(3, "%p: chain_last=%d ifp=%p (%s)", ofp, ofp->chain_last, ofp->ifp, ofp->name);
}
int
main(int argc, char *argv[])
{
int max_fd = 0;
struct sink_info *ofiles = NULL, *ofp;
struct sink_info **oend = &ofiles;
struct source_info *ifiles = NULL, *ifp;
struct source_info **iend = &ifiles;
struct source_info *front_ifp; /* To keep output sequential, never output past this one */
int ch;
const char *progname = argv[0];
enum state state = read_ob;
bool opt_memory_stats = false;
bool opt_append = false;
while ((ch = getopt(argc, argv, "ab:fIi:Mm:o:p:S:sTt:")) != -1) {
switch (ch) {
case 'a':
opt_append = true;
break;
case 'b':
buffer_size = (int)parse_size(progname, optarg);
break;
case 'f':
use_tmp_file = true;
break;
case 'I':
state = read_ib;
break;
case 'i': /* Specify input file */
ifp = new_source_info(optarg);
if ((ifp->fd = open(optarg, O_RDONLY)) < 0)
err(2, "Error opening %s", optarg);
max_fd = MAX(ifp->fd, max_fd);
non_block(ifp->fd, fp_name(ifp));
/* Add file at the end of the linked list */
*iend = ifp;
iend = &ifp->next;
break;
case 'm':
max_mem = parse_size(progname, optarg);
break;
case 'M': /* Provide memory use statistics on termination */
opt_memory_stats = true;
break;
case 'o': /* Specify output file */
ofp = new_sink_info(optarg);
if ((ofp->fd = open(optarg,
(opt_append ? O_APPEND : 0) |
O_WRONLY | O_CREAT | O_TRUNC, 0666)) < 0)
err(2, "Error opening %s", optarg);
max_fd = MAX(ofp->fd, max_fd);
non_block(ofp->fd, fp_name(ofp));
/* Add file at the end of the linked list */
*oend = ofp;
oend = &ofp->next;
break;
case 'p':
parse_permute(optarg);
break;
case 's':
opt_scatter = true;
break;
case 'T':
opt_tmp_dir = optarg;
break;
case 't': /* Record terminator */
/* We allow \0 as rt */
if (strlen(optarg) > 1)
usage(progname);
rt = *optarg;
break;
case '?':
default:
usage(progname);
}
}
argc -= optind;
argv += optind;
if (argc)
usage(progname);
/* dgsh */
int j = 0;
int noutputfds;
int *outputfds;
int ninputfds;
int *inputfds;
char *name;
if (permute_n) {
ninputfds = noutputfds = permute_n;
name = "perm";
} else {
char *in, *out;
/* No stdin or stdout, if these have been specified via args */
ninputfds = ifiles ? 0 : -1;
noutputfds = ofiles ? 0 : -1;
/* Heuristic to determine name */
in = getenv("DGSH_IN");
if (in && *in == '0')
in = NULL;
out = getenv("DGSH_OUT");
if (out && *out == '0')
out = NULL;
if (in && !out)
name = "cat";
else if (!in && out)
name = "tee";
else
name = "dgsh-tee";
}
DPRINTF(3, "Calling negotiate in=%d out=%d", ninputfds, noutputfds);
dgsh_negotiate(DGSH_HANDLE_ERROR, name, &ninputfds, &noutputfds, &inputfds, &outputfds);
DPRINTF(3, "nin=%d nout=%d", ninputfds, noutputfds);
assert(noutputfds >= 0);
assert(ninputfds >= 0);
if (permute_n && permute_n != ninputfds)
errx(1, "The number of inputs %d is not equal to the specified permuted outputs %d", ninputfds, permute_n);
if (permute_n && permute_n != noutputfds)
errx(1, "The number of outputs %d is not equal to the specified permuted outputs %d", noutputfds, permute_n);
for (j = 0; j < noutputfds; j++) {
DPRINTF(3, "New ofp assigned fd %d", outputfds[j]);
if (j == 0) {
ofp = new_sink_info("standard output");
ofp->fd = STDOUT_FILENO;
} else {
ofp = new_sink_info(NULL);
ofp->fd = outputfds[j];
}
max_fd = MAX(ofp->fd, max_fd);
non_block(ofp->fd, fp_name(ofp));
/* Add file at the end of the linked list */
*oend = ofp;
oend = &ofp->next;
}
for (j = 0; j < ninputfds; j++) {
DPRINTF(3, "New ifp assigned fd %d", inputfds[j]);
if (j == 0) {
ifp = new_source_info("standard input");
ifp->fd = STDIN_FILENO;
} else {
ifp = new_source_info(NULL);
ifp->fd = inputfds[j];
}
max_fd = MAX(ifp->fd, max_fd);
non_block(ifp->fd, fp_name(ifp));
/* Add file at the end of the linked list */
*iend = ifp;
iend = &ifp->next;
}
if (buffer_size > max_mem)
errx(1, "Buffer size %d is larger than the program's maximum memory limit %lu", buffer_size, max_mem);
if (opt_scatter && ifiles && ifiles->next)
errx(1, "Scattering not supported with more than one input file");
if (opt_scatter && permute_n)
errx(1, "Scattering and permutation cannot be used together");
if (ofiles == NULL) {
/* Output to stdout */
ofp = new_sink_info("standard output");
ofp->fd = STDOUT_FILENO;
max_fd = MAX(ofp->fd, max_fd);
non_block(ofp->fd, fp_name(ofp));
ofp->next = ofiles;
ofiles = ofp;
}
if (ifiles == NULL) {
/* Input from stdin */
ifp = new_source_info("standard input");
ifp->fd = STDIN_FILENO;
max_fd = MAX(ifp->fd, max_fd);
non_block(ifp->fd, fp_name(ifp));
ifp->next = ifiles;
ifiles = ifp;
}
/* We will handle SIGPIPE explicitly when calling write(2). */
signal(SIGPIPE, SIG_IGN);
front_ifp = ifiles;
chain_io_files(ifiles, ofiles, permute_n != 0);
/* Copy source to sink without allowing any single file to block us. */
for (;;) {
fd_set source_fds;
fd_set sink_fds;
int fd_set_count = 0;
show_state(state);
/* Set the fd's we're interested to read/write; close unneeded ones. */
FD_ZERO(&source_fds);
FD_ZERO(&sink_fds);
if (!reached_eof)
switch (state) {
case read_ib:
for (ifp = front_ifp; ifp; ifp = ifp->next)
if (!ifp->reached_eof) {
FD_SET(ifp->fd, &source_fds);
fd_set_count += 1;
}
break;
case read_ob:
for (ifp = front_ifp; ifp; ifp = ifp->next)
if (ifp->active && !ifp->reached_eof) {
FD_SET(ifp->fd, &source_fds);
fd_set_count += 1;
}
break;
default:
break;
}
for (ofp = ofiles; ofp; ofp = ofp->next)
if (ofp->active) {
switch (state) {
case read_ib:
case read_ob:
case drain_ob:
DPRINTF(4, "Check active file[%s] pos_written=%ld pos_to_write=%ld",
fp_name(ofp), (long)ofp->pos_written, (long)ofp->pos_to_write);
if (ofp->pos_written < ofp->pos_to_write) {
FD_SET(ofp->fd, &sink_fds);
fd_set_count += 1;
}
break;
case drain_ib:
case write_ob:
FD_SET(ofp->fd, &sink_fds);
fd_set_count += 1;
break;
}
}
if (fd_set_count != 0) {
/* Block until we can read or write. */
show_select_args("Entering select", &source_fds, ifiles, &sink_fds, ofiles, true);
if (select(max_fd + 1, &source_fds, &sink_fds, NULL, NULL) < 0)
err(3, "select");
show_select_args("Select returned", &source_fds, ifiles, &sink_fds, ofiles, false);
/* Write to all file descriptors that accept writes. */
if (sink_write(ifiles, &sink_fds, ofiles) > 0) {
/*
* If we wrote something, we made progress on the
* downstream end. Loop without reading to avoid
* allocating excessive buffer memory.
*/
if (state == drain_ob)
state = write_ob;
continue;
}
}
if (reached_eof) {
int active_fds = 0;
for (ofp = ofiles; ofp; ofp = ofp->next)
if (ofp->active) {
if (ofp->pos_written < ofp->pos_to_write)
active_fds++;
else {
DPRINTF(3, "Retiring file %s pos_written=pos_to_write=%ld source_pos_read=%ld",
fp_name(ofp), (long)ofp->pos_written, (long)ofp->ifp->source_pos_read);
/* No more data to write; close fd to avoid deadlocks downstream. */
if (close(ofp->fd) == -1)
err(2, "Error closing %s", fp_name(ofp));
ofp->active = false;
}
}
if (active_fds == 0) {
/* If no read possible, and no writes pending, terminate. */
if (opt_memory_stats)
memory_stats(ifiles);
return 0;
}
}
/*
* Note that we never reach this point after a successful write.
* See the continue statement above.
*/
switch (state) {
case read_ib:
/* Read, if possible; set global reached_eof if all have reached it */
reached_eof = true;
for (ifp = front_ifp; ifp; ifp = ifp->next) {
if (FD_ISSET(ifp->fd, &source_fds))
switch (source_read(ifp)) {
case read_eof:
ifp->reached_eof = true;
break;
case read_oom: /* Cannot fullfill promise to never block source, so bail out. */
errx(1, "Out of memory with input-side buffering specified");
break;
case read_ok:
case read_again:
break;
}
if (!ifp->reached_eof)
reached_eof = false;
}
if (reached_eof)
state = drain_ib;
break;
case read_ob:
/* Read, from possible sources; set global reached_eof if all have reached it */
reached_eof = true;
for (ifp = front_ifp; ifp; ifp = ifp->next) {
if (!ifp->active)
continue;
if (FD_ISSET(ifp->fd, &source_fds))
switch (source_read(ifp)) {
case read_eof:
ifp->reached_eof = true;
ifp->active = false;
if (!ifp->chain_last)
ifp->next->active = true;
break;
case read_again:
break;
case read_oom: /* Allow buffers to empty. */
state = drain_ob;
break;
case read_ok:
state = write_ob;
break;
}
if (!ifp->reached_eof)
reached_eof = false;
}
if (reached_eof)
state = drain_ib;
break;
case drain_ib:
break;
case drain_ob:
if (reached_eof)
state = write_ob;
else
state = read_ob;
break;
case write_ob:
if (!reached_eof)
state = read_ob;
break;
}
}
}
================================================
FILE: core-tools/src/dgsh-w.c
================================================
#include <assert.h> // assert()
#include <math.h> // M_PI, pow()
#include <complex.h> // I, cexp(), cpow()
#include <stdio.h> // DPRINTF
#include <stdlib.h> // atoi()
#include <err.h> // errx()
#include <unistd.h> // read(), write()
#include <string.h> // memcpy()
#include "dgsh.h"
#include "dgsh-debug.h"
#if !defined(HAVE_CPOW)
#include "../../unix-tools/cpow.c"
#endif
void
read_number(int fd, long double *x, long double complex *xc)
{
char buf[sizeof(long double complex) + 5]; // \n\0
char real[sizeof(long double)];
char imag[sizeof(long double)];
int rd_size;
// Read input: 2 float values
rd_size = read(fd, buf, sizeof(buf));
if (rd_size == -1)
err(1, "write failed");
DPRINTF(4, "Read %zu characters, long double size: %zu, long double complex size: %zu",
rd_size, sizeof(long double), sizeof(long double complex));
if (rd_size == sizeof(long double)) {
memcpy(x, buf, sizeof(*x));
DPRINTF(4, "Read input x: %.10Lf", *x);
} else {
sscanf(buf, "%s %s", real, imag);
*xc = atof(real) + atof(imag)*I;
DPRINTF(4, "##xc: %.10f + %.10fi (read %zu characters)\n",
creal(*xc), cimag(*xc), rd_size);
}
}
void
write_number(int fd, long double complex y)
{
char buf[sizeof(long double complex)];
int wr_size;
// Write output: 2 float values
memset(buf, 0, sizeof(buf));
snprintf(buf, sizeof(buf), "%.10f %.10fi", creal(y), cimag(y));
DPRINTF(4, "##buf(y): %s, len: %d", buf, strlen(buf));
wr_size = write(fd, buf, sizeof(buf));
if (wr_size == -1)
err(1, "write failed");
DPRINTF(4, "##y: %.10f + %.10fi (wrote %zu characters)\n",
creal(y), cimag(y), wr_size);
}
int
main(int argc, char** argv)
{
int noutputfds = 2;
int *outputfds = NULL;
int ninputfds = 2;
int *inputfds = NULL;
long double x1 = -1.0, x2 = -1.0;
long double complex xc1, xc2;
long double complex y1, y2, w, wmn;
size_t wr_size;
char negotiation_title[10];
int s; // stage (stage=1,2,3)
int m; // 2^stage
int n; // nth root of unity
assert(argc == 3);
s = atoi(argv[1]);
m = pow(2, s);
n = atoi(argv[2]);
snprintf(negotiation_title, sizeof(negotiation_title),
"%s %s %s", argv[0], argv[1], argv[2]);
dgsh_negotiate(DGSH_HANDLE_ERROR, negotiation_title, &ninputfds,
&noutputfds, &inputfds, &outputfds);
assert(ninputfds == 2);
assert(noutputfds == 2);
read_number(inputfds[0], &x1, &xc1);
read_number(inputfds[1], &x2, &xc2);
// Calculate
w = 2 * M_PI * I / m;
wmn = cpow(cexp(w), n);
DPRINTF(4, "w: %.10f + %.10fi", creal(w), cimag(w));
DPRINTF(4, "m: %d, n: %d, wmn: %.10f + %.10fi\n",
m, n, creal(wmn), cimag(wmn));
if (x1 == -1.0 && x2 == -1.0) {
y1 = xc1 + wmn * xc2;
y2 = xc1 - wmn * xc2;
} else {
y1 = x1 + wmn * x2;
y2 = x1 - wmn * x2;
}
write_number(outputfds[0], y1);
write_number(outputfds[1], y2);
return 0;
}
================================================
FILE: core-tools/src/dgsh-wrap.1
================================================
.TH DGSH-WRAP 1 "18 August 2017"
.\"
.\" (C) Copyright 2016-2017 Diomidis Spinellis. All rights reserved.
.\"
.\" Licensed under the Apache License, Version 2.0 (the "License");
.\" you may not use this file except in compliance with the License.
.\" You may obtain a copy of the License at
.\"
.\" http://www.apache.org/licenses/LICENSE-2.0
.\"
.\" Unless required by applicable law or agreed to in writing, software
.\" distributed under the License is distributed on an "AS IS" BASIS,
.\" WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
.\" See the License for the specific language governing permissions and
.\" limitations under the License.
.\"
.SH NAME
dgsh-wrap \- allow any program to participate in an dgsh pipeline
.SH SYNOPSIS
\fBdgsh-wrap\fP
[\fB-S\fP]
[\fB-i\fP \fB0\fP|\fBa\fP]
[\fB-o\fP \fB0\fP|\fBa\fP]
[\fB-eIO\fP]
\fIprogram\fP [\fIprogram-arguments\fP ...]
#!/usr/libexec/dgsh/\fBdgsh-wrap\fP
\fB-s\fP
[\fB-i\fP \fB0\fP|\fBa\fP]
[\fB-o\fP \fB0\fP|\fBa\fP]
[\fB-eIO\fP] [\fIprogram-arguments\fP ...]
\fBdgsh-wrap\fP
[\fB-Ss\fP]
\fB-x\fP
\fIprogram\fP [\fIprogram-arguments\fP ...]
.SH DESCRIPTION
\fIdgsh-wrap\fP takes as arguments an absolute path or the name
of a program to execute and its arguments.
It will participate in the \fIdgsh\fP negotiation process,
and then execute the specified program connected to the negotiated
input and output pipes.
If the program is not specified through an absolute path,
it will be executed by searching the existing path,
excluding from it elements ending in \fIdgsh\fP
(where programs already wrapped with \fIdgsh-wrap\fP may reside).
.PP
Arguments specified as \fI<|\fP are presented as additional
input channels and
arguments specified as \fI>|\fP are presented as additional
output channels.
Both are suitably replaced by named file descriptor paths
when the command is invoked.
.PP
In the context of a \fIdgsh\fP process graph, \fIdgsh(1)\fP automatically
invokes \fIdgsh-wrap\fP to allow non-dgsh compatible commands to participate
in the negotiation procedure.
Furthermore, the \fIdgsh\fP installation process sets up many POSIX programs
wrapped with \fIdgsh-wrap\fP in order to communicate their particular
I/O requirements.
.SH OPTIONS
.IP "\fB\-e\fP
Replace \fI<|\fP and \fI>|\fP strings embedded within arguments,
with names of input file descriptor paths.
By default, only standalone arguments are thus replaced.
.IP "\fB\-i\fP \fB0\fP|\fBa\fP
Specify the wrapped program's number of input channels.
The \fB0\fP character specifies that the program does not read any input.
The \fBa\fP character specifies that the program can read an arbitrary
number of input streams;
these will be automatically supplied by \fIdgsh-wrap\fP as file descriptor
command line arguments.
.IP "\fB\-I\fP
Do not include the standard input to the command line arguments,
when replacing \fI<|\fP arguments with names of input file descriptor paths.
This will result in the standard input becoming available to the
command as its standard input, rather than as a command line argument.
Without this option, the first path with be \fI/dev/fd/0\fP.
When this option is given, the program will require one input channel
more than those specified by the \fI<|\fP arguments.
.IP "\fB\-o\fP \fB0\fP|\fBa\fP
Specify the wrapped program's number of output channels.
The \fB0\fP character specifies that the program does not produce any output.
The \fBa\fP character specifies that the program can write to an arbitrary
number of output streams;
these will be automatically supplied by \fIdgsh-wrap\fP as file descriptor
command line arguments.
.IP "\fB\-O\fP
Do not include the standard output to the command line arguments,
when replacing \fI>|\fP arguments with names of output file descriptor paths.
This will result in the standard output becoming available to the
command as its standard output, rather than as a command line argument.
Without this option the first path with be \fI/dev/fd/1\fP.
When this option is given, the program will require one output channel
more than those specified by the \fI>|\fP arguments.
.IP "\fB\-S\fP
Process flags as a shebang-invoked (\fI#!\fP) interpreter using
an invocation-supplied program name.
This will change the argument processing in two ways.
First, it will cause the arguments being specified on the shebang line to
be properly parsed as separate arguments by splitting them on whitespace.
(Most operating systems pass any of the line's arguments as a single
string to the process.)
Second, it will remove from the arguments the kernel-supplied path
to the script that invoked \fIdgsh-wrap\fP.
(The script path is not needed,
because the program to wrap is specified as an argument.)
If this flag is specified, it must be the first command-line argument.
.IP "\fB\-s\fP
Process flags as a shebang-invoked (\fI#!\fP) interpreter using an
operating system-supplied program name.
This will change the argument processing in two ways.
First, it will split arguments in the shebang line, in the same
way as the \fI-S\fP flag.
Second, it will convert the kernel-supplied absolute path
to the script that invoked \fIdgsh-wrap\fP, into a command name.
If the name of the script is the same as the name of a command to
be wrapped, this path can be used to derive the name of the program to execute,
thus removing the need to supply the name of the program in the
invocation line.
.IP "\fB\-x\fP
Execute the specified command and arguments, without performing \fIdgsh\fP
negotiation on its behalf.
This is useful when
the specified command, \fIA\fP, is not \fIdgsh\fP-compatible,
but it will in turn execute, \fIB\fP, a \fIdgsh\fP-compatible command.
With this flag the \fIdgsh\fP will recognize the invocation of \fIA\fP
as an invocation of a \fIdgsh\fP-compatible command (due to the wrapping),
and will not attempt to autowrap it as a filter and thereby override a
more sophisticated \fIdgsh\fP interface that \fIB\fP would present.
.SH EXAMPLES
.PP
The following examples are given only to illustrate the command's functionality.
Note that most of the wrappings shown here are either performed automatically
or are not required,
because corresponding commands with built-in \fIdgsh\fP support
are already provided.
.PP
Wrap the \fIecho\fP command, specifying that it accepts no input.
.ft C
.ps -1
.nf
dgsh-wrap -i 0 echo hi
.fi
.ps +1
.ft P
.PP
Wrap the \fIcp\fP command, specifying that it does not perform any I/O.
.ft C
.ps -1
.nf
dgsh-wrap -i 0 -o 0 cp src-file dest-file
.fi
.ps +1
.ft P
.PP
Wrap the \fIpaste\fP command, supplying its two arguments.
.ft C
.ps -1
.nf
dgsh-wrap /usr/bin/paste "<|" "<|"
.fi
.ps +1
.ft P
.PP
Wrap the \fIpaste\fP command, with an invocation that processes the standard
input and uses an additional input argument.
.ft C
.ps -1
.nf
dgsh-wrap -I /usr/bin/paste - "<|"
.fi
.ps +1
.ft P
.PP
Wrap the \fIpaste\fP command, so that it will process all input channels
provided to it.
.ft C
.ps -1
.nf
dgsh-wrap -i a /usr/bin/paste
.fi
.ps +1
.ft P
.PP
A wrapped version of the \fIuname\fP command can be created with an
interpreter invocation file containing just the following line.
In contrast to a shell script, no shell is ever launched.
.ft C
.ps -1
.nf
#!/usr/libexec/dgsh/dgsh-wrap -S -d /bin/uname
.fi
.ps +1
.ft P
.PP
Even simpler, a wrapped version of the \fIuname\fP command can be created
if
a) the interpreter invocation file is named \fIuname\fP,
b) it resides in a directory named \fIdgsh\fP (so that it will be excluded
from the subsequently used search path), and
c) it contains the following line.
.ft C
.ps -1
.nf
#!/usr/libexec/dgsh/dgsh-wrap -s -d
.fi
.ps +1
.ft P
.PP
Trace the invocation of the \fItee\fP command,
presenting to the shell \fIsrtace\fP as a \fIdgsh\fP-compatible command
with the capabilities of \fItee\fP.
.ft C
.ps -1
.nf
dgsh-wrap -x strace tee
.fi
.ps +1
.ft P
.SH "SEE ALSO"
\fIdgsh\fP(1),
\fIdgsh-negotiate\fP(3)
.SH AUTHOR
Diomidis Spinellis \(em <http://www.spinellis.gr>
================================================
FILE: core-tools/src/dgsh-wrap.c
================================================
/*
* Copyright 2016-2017 Diomidis Spinellis
*
* Wrap any command to participate in the dgsh negotiation
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
/*
* Examples:
* dgsh-wrap -i 0 yes | fsck
* tar cf - / | dgsh-wrap -o 0 dd of=/dev/st0
* ls | dgsh-wrap /usr/bin/sort -k5n | more
*/
#define _GNU_SOURCE /* For asprintf() */
#include <assert.h>
#include <ctype.h>
#include <stdbool.h>
#include <stdio.h>
#include <stdlib.h>
#include <unistd.h>
#include <string.h>
#include <err.h>
#include "dgsh.h"
#include "dgsh-debug.h" /* DPRINTF(4, ) */
/* Determine if the OS splits shebang argument or not */
#if __APPLE__
#include "TargetConditionals.h"
#if TARGET_OS_MAC
#define OS_SPLITS_SHEBANG_ARGS 1
#endif
#endif
static void
usage(void)
{
fputs("Usage:\tdgsh-wrap [-S] [-i 0|a] [-o 0|a] [-eIO] program [program-arguments ...]\n"
"\tdgsh-wrap -s [-i 0|a] [-o 0|a] [-eIO] [program-arguments ...]\n"
"-e\t" "Process <| and >| embedded in arguments\n"
"-i 0|a\t" "Process no (0) or arbitrary (a) input channels\n"
"-I\t" "Do not provide standard input as a <| arg\n"
"-o 0|a\t" "Process no (0) or arbitrary (a) output channels\n"
"-O\t" "Do not provide standard output as a >| arg\n"
"-S\t" "Process flags and program as a #! interpreter\n"
"-s\t" "Process flags as a #! interpreter\n"
"\t" "(-S or -s must be the first flag of shebang line)\n"
"-x\t" "Wrap a non-dgsh command that will exec a dgsh one\n",
stderr);
exit(1);
}
static void *
xmalloc(size_t size)
{
void *r = malloc(size);
if (r == NULL)
err(1, "malloc out of memory");
return r;
}
char *
xstrdup(const char *s)
{
void *r = strdup(s);
if (r == NULL)
err(1, "stdup out of memory");
return r;
}
static void *
xrealloc(void *ptr, size_t size)
{
void *r = realloc(ptr, size);
if (r == NULL)
err(1, "realloc out of memory");
return r;
}
/*
* Remove from the PATH environment variable an entry with the specified string
*/
static void
remove_from_path(const char *string)
{
char *start, *end, *path, *strptr;
path = getenv("PATH");
if (!path)
return;
path = xstrdup(path);
if (!path)
err(1, "Error allocating path copy");
strptr = strstr(path, string);
if (!strptr)
return;
/* Find start of this path element */
for (start = strptr; start != path && *start != ':'; start--)
;
/* Find end of this path element */
for (end = strptr; *end && *end != ':'; end++)
;
/*
* At this point:
* start can point to : or path,
* end can point to : or \0.
* Work through all cases.
*/
if (*end == '\0')
*start = '\0';
else if (*start == ':')
memmove(start, end, strlen(end));
else /* first element, followed by another */
memmove(start, end + 1, strlen(end + 1));
if (setenv("PATH", path, 1) != 0)
err(1, "Setting path");
free(path);
}
static void
dump_args(int argc, char *argv[])
{
int i;
for (i = 0; i <= argc; i++)
DPRINTF(4, "argv[%d]: [%s]", i, argv[i]);
}
/*
* On operating systems that pass unsplit all the #! line arguments
* to the interpreter, process the arguments of a #! invocation to make
* them equivalent to a command-line one.
* This entails tokenizing argv[1], which contains all the #! line
* after the name of the interpreter.
*
* Example:
* Input arguments:
* argv[0]: /usr/local/libexec/dgsh/dgsh-wrap
* argv[1]: -S -d /bin/uname
* argv[2]: /usr/local/libexec/dgsh/uname
* argv[3]: -s
* argv[4]: NULL
* Output arguments:
* argv[0]: /usr/local/libexec/dgsh/dgsh-wrap
* argv[1]: -S
* argv[2]: -d
* argv[3]: /bin/uname
* argv[4]: /usr/local/libexec/dgsh/uname
* argv[5]: -s
* argv[6]: NULL
*/
static void
split_argv(int *argcp, char ***argvp)
{
int argc = *argcp;
char **argv = *argvp;
/* Tokenize argv[1] into multiple arguments */
argv = xmalloc((argc + 1) * sizeof(char *));
memcpy(argv, *argvp, (argc + 1) * sizeof(char *));
int i = 1;
const char *delim = " \t";
char *p = strtok(xstrdup(argv[1]), delim);
assert(p != NULL); /* One arg (-s or -S) is guaranteed to exist */
for (;;) {
argv[i] = p;
p = strtok(NULL, delim);
if (p == NULL)
break;
/* Make room for new string */
argc += 1;
argv = xrealloc(argv, (argc + 1) * sizeof(char *));
memmove(argv + i + 2, argv + i + 1,
(argc - i - 1) * sizeof(char *));
i++;
}
*argvp = argv;
*argcp = argc;
DPRINTF(4, "Arguments after split_argv");
dump_args(*argcp, *argvp);
}
/*
* -S: remove OS-supplied script path name
* Input arguments:
* argv[0]: /usr/local/libexec/dgsh/dgsh-wrap
* argv[1]: -S -d /bin/uname
* argv[2]: /usr/local/libexec/dgsh/uname
* argv[3]: -s
* Arguments at this point:
* argv[0]: /usr/local/libexec/dgsh/dgsh-wrap
* argv[1]: -S
* argv[2]: -d
* argv[3]: /bin/uname <- argv[optind]
* argv[4]: /usr/local/libexec/dgsh/uname
* argv[5]: -s
* Result arguments:
* argv[0]: /usr/local/libexec/dgsh/dgsh-wrap
* argv[1]: -S
* argv[2]: -d
* argv[3]: /bin/uname
* argv[4]: -s
*/
static void
remove_os_script_path(char **argv, int *argc, int optind)
{
memmove(argv + optind + 1, argv + optind + 2, (*argc - optind) * sizeof(char *));
*argc -= 1;
argv[*argc] = NULL;
}
/* Remove absolute path from specified string
* Example:
* -s: Remove absolute path from argv[optind]
* Input arguments:
* argv[0]: /usr/local/libexec/dgsh/dgsh-wrap
* argv[1]: -s -d
* argv[2]: /usr/local/libexec/dgsh/uname
* argv[3]: -s
* Arguments at this point:
* argv[0]: /usr/local/libexec/dgsh/dgsh-wrap
* argv[1]: -s
* argv[2]: -d
* argv[3]: /usr/local/libexec/dgsh/uname <- argv[optind]
* argv[4]: -s
* Result arguments:
* argv[0]: /usr/local/libexec/dgsh/dgsh-wrap
* argv[1]: -s
* argv[2]: -d
* argv[3]: uname
* argv[4]: -s
*/
static void
remove_absolute_path(char *s)
{
char *p = strrchr(s, '/');
if (p)
memmove(s, p + 1, strlen(p) + 1);
}
/*
* Replace an instance of the string special (e.g. "<|") embedded in arg
* with /dev/fd/N, where N is the integer pointed
* by fdptr, and increase fdptr to point to the next integer.
* Return true if a replacement was made, false if not.
*/
static bool
process_embedded_io_arg(char **arg, const char *special, int **fdptr)
{
char *p = strstr(*arg, special);
if (p == NULL)
return false;
*p = 0;
char *before = *arg;
char *after = p + strlen(special);
if (asprintf(arg, "%s/dev/fd/%d%s", before, **fdptr, after) == -1)
err(1, "asprintf out of memory");
(*fdptr)++;
return true;
}
/*
* Replace an instance of the string special (e.g. "<|") matching an arg
* with /dev/fd/N, where N is the integer pointed
* by fdptr, and increase fdptr to point to the next integer.
* If special is null, then the replacement is always made.
*/
static void
process_standalone_io_arg(char **arg, const char *special, int **fdptr)
{
if (special != NULL && strcmp(*arg, special) != 0)
return;
if (asprintf(arg, "/dev/fd/%d", **fdptr) == -1)
err(1, "asprintf out of memory");
(*fdptr)++;
}
/*
* Increment the channels specified by the given variable.
* Ensure that the corresponding variable is not
* already set to an arbitrary number of channels.
*/
static void
increment_channels(int *var)
{
if (*var == -1) {
fputs("I/O channel arguments cannot be combined with an arbitrary I/O file specification\n",
stderr);
exit(1);
}
(*var)++;
}
int
main(int argc, char *argv[])
{
int nflags = 0;
bool negotiation_flags = false;
int ninputs = 1, noutputs = 1;
bool xflag = false;
int ch, i;
char *p;
char *debug_level;
char *guest_program_name;
/* Option-dependent flags */
bool program_from_os = false, program_supplied = false;
bool embedded_args = false;
/* Pass stdin/stdout as a command-line argument */
bool stdin_as_arg = true, stdout_as_arg = true;
bool supply_input_args = false, supply_output_args = false;
debug_level = getenv("DGSH_DEBUG_LEVEL");
if (debug_level)
dgsh_debug_level = atoi(debug_level);
/* Preclude recursive wrapping */
DPRINTF(4, "PATH before: [%s]", getenv("PATH"));
remove_from_path("libexec/dgsh");
DPRINTF(4, "PATH after: [%s]", getenv("PATH"));
DPRINTF(4, "Initial arguments");
dump_args(argc, argv);
/* Check for #! (shebang) interpreter argument processing */
#if !defined(OS_SPLITS_SHEBANG_ARGS)
if (argc >= 2 && argv[1][0] == '-' && tolower(argv[1][1]) == 's')
split_argv(&argc, &argv);
#endif
/*
* The + argument to getopt causes it on glibc to stop processing on
* first non-flag argument.
* Therefore, adjust argc, argv on entry and optind on exit.
*/
while ((ch = getopt(argc, argv, "+ei:Io:OSsx")) != -1) {
DPRINTF(4, "getopt switch=%c", ch);
switch (ch) {
case 'i':
nflags++;
negotiation_flags = true;
if (strcmp(optarg, "0") == 0)
ninputs = 0;
else if (strcmp(optarg, "a") == 0) {
ninputs = -1;
supply_input_args = true;
} else
usage();
break;
case 'e':
embedded_args = true;
negotiation_flags = true;
nflags++;
break;
case 'I':
stdin_as_arg = false;
negotiation_flags = true;
nflags++;
break;
case 'o':
nflags++;
negotiation_flags = true;
if (strcmp(optarg, "0") == 0)
noutputs = 0;
else if (strcmp(optarg, "a") == 0) {
noutputs = -1;
supply_output_args = true;
} else
usage();
break;
case 'O':
stdout_as_arg = false;
negotiation_flags = true;
nflags++;
break;
case 'S':
/* Complain this is not the first flag */
if (nflags) {
fputs("-S must be the first provided flag\n",
stderr);
usage();
}
nflags++;
program_supplied = true;
break;
case 's':
/* Complain this is not the first flag */
if (nflags) {
fputs("-s must be the first provided flag\n",
stderr);
usage();
}
nflags++;
program_from_os = true;
break;
case 'x':
xflag = true;
break;
case '?':
default:
usage();
}
}
DPRINTF(3, "After getopt: ninputs=%d, noutputs=%d optind=%d argv[optind]=%s",
ninputs, noutputs, optind, argv[optind]);
DPRINTF(3, "program_supplied=%d", program_supplied);
if (optind >= argc)
usage();
if (xflag && negotiation_flags) {
fputs("-x cannot be combined with I/O specifications\n",
stderr);
usage();
}
/*
* Process argv[2], which is the name of the script
* supplied by the kernel to the interpreter, i.e.
* the name of the program we are being executed as.
*/
if (program_supplied && argc > optind)
remove_os_script_path(argv, &argc, optind);
else if (program_from_os)
remove_absolute_path(argv[optind]);
DPRINTF(4, "Arguments after processing program name (optind=%d)", optind);
dump_args(argc, argv);
if (xflag) {
/*
* Execute (non-dgsh) command, which will execute a dgsh
* command, which will negotiate on our behalf
*/
execvp(argv[optind], argv + optind);
err(1, "Unable to execute %s", argv[optind]);
return 1;
}
/* Obtain guest program name (without path) */
guest_program_name = xstrdup(argv[optind]);
remove_absolute_path(guest_program_name);
DPRINTF(4, "guest_program_name: %s", guest_program_name);
/*
* Adjust ninputs and noutputs by special arguments
* "<|" and ">|", which mean input from or output to
* /dev/fd/N
*/
DPRINTF(4, "embedded_args=%d", embedded_args);
for (i = optind + 1; i < argc; i++) {
if (embedded_args) {
for (p = argv[i]; p = strstr(p, "<|"); p += 2)
increment_channels(&ninputs);
for (p = argv[i]; p = strstr(p, ">|"); p += 2)
increment_channels(&noutputs);
} else {
if (strcmp(argv[i], "<|") == 0)
increment_channels(&ninputs);
if (strcmp(argv[i], ">|") == 0)
increment_channels(&noutputs);
}
}
/*
* Adjust for the default implicit I/O channel.
* E.g. if two <| are specified, ninputs will be 3 at this point,
* whereas we want it to be 2.
*/
if (stdin_as_arg && ninputs > 1)
ninputs--;
if (stdout_as_arg && noutputs > 1)
noutputs--;
/* Participate in negotiation */
DPRINTF(3, "calling negotiate with ninputs=%d noutputs=%d", ninputs, noutputs);
int *input_fds = NULL, *output_fds = NULL;
dgsh_negotiate(DGSH_HANDLE_ERROR, guest_program_name,
&ninputs, &noutputs,
&input_fds, &output_fds);
/*
* Substitute special arguments "<|" and ">|" with or add file descriptor
* paths /dev/fd/N using the fds received from negotiation.
*/
int *inptr = stdin_as_arg ? input_fds : input_fds + 1;
if (supply_input_args) {
if (!stdin_as_arg)
ninputs--;
/* Create space for arguments to add */
char **nargv = xmalloc((argc + ninputs + 1) * sizeof(char *));
memcpy(nargv, argv, argc * sizeof(char *));
memset(argv + argc, 0, (ninputs + 1) * sizeof(char *));
argv = nargv;
/* Add arguments */
for (i = argc; i < argc + ninputs; i++)
process_standalone_io_arg(&argv[i], NULL, &inptr);
argc += ninputs;
argv[argc] = NULL;
} else {
for (i = optind + 1; i < argc; i++) {
if (embedded_args)
while (process_embedded_io_arg(&argv[i], "<|", &inptr))
;
else
process_standalone_io_arg(&argv[i], "<|", &inptr);
}
}
int *outptr = stdout_as_arg ? output_fds : output_fds + 1;
if (supply_output_args) {
if (!stdout_as_arg)
noutputs--;
/* Create space for arguments to add */
char **nargv = xmalloc((argc + noutputs + 1) * sizeof(char *));
memcpy(nargv, argv, argc * sizeof(char *));
memset(argv + argc, 0, (noutputs + 1) * sizeof(char *));
argv = nargv;
/* Add arguments */
for (i = argc; i < argc + noutputs; i++)
process_standalone_io_arg(&argv[i], NULL, &outptr);
argc += noutputs;
argv[argc] = NULL;
} else {
for (i = optind + 1; i < argc; i++) {
if (embedded_args)
while (process_embedded_io_arg(&argv[i], ">|", &outptr))
;
else
process_standalone_io_arg(&argv[i], ">|", &outptr);
}
}
DPRINTF(4, "Arguments to execvp after substitung <| and >|");
dump_args(argc - optind, argv + optind);
/* Execute command */
execvp(argv[optind], argv + optind);
err(1, "Unable to execute %s", argv[optind]);
return 1;
}
================================================
FILE: core-tools/src/dgsh-writeval.1
================================================
.TH DGSH-WRITEVAL 1 "21 March 2013"
.\"
.\" (C) Copyright 2013 Diomidis Spinellis. All rights reserved.
.\"
.\" Licensed under the Apache License, Version 2.0 (the "License");
.\" you may not use this file except in compliance with the License.
.\" You may obtain a copy of the License at
.\"
.\" http://www.apache.org/licenses/LICENSE-2.0
.\"
.\" Unless required by applicable law or agreed to in writing, software
.\" distributed under the License is distributed on an "AS IS" BASIS,
.\" WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
.\" See the License for the specific language governing permissions and
.\" limitations under the License.
.\"
.SH NAME
dgsh-writeval \- write values to a data store
.SH SYNOPSIS
\fBdgsh-writeval\fP
[\fB\-l\fP \fIlength\fP | \fB-t\fP \fIcharacter\fP ]
[\fB\-b\fP \fIn\fP]
[\fB\-e\fP \fIn\fP]
[\fB\-u\fP \fIunit\fP]
\fB\-s\fP \fIpath\fP
.SH DESCRIPTION
\fIdgsh-writeval\fP will read values from its standard input and make them available
to other processes for reading through the specified Unix domain socket.
Thus this process acts as a data store:
it reads a series of values (think of them as assignments),
and provides a way to read the store's current value (from the socket).
By default \fIdgsh-writeval\fP will store the last value (line or data block)
it reads.
However, the default behavior can be modified through options
so that it stores a specified window of the stream it processes.
.PP
\fIdgsh-writeval\fP is normally executed from within \fIdgsh\fP-generated scripts,
rather than through end-user commands.
This manual page serves mainly to document its operation and
the flags that can be used in \fIdgsh\fP scripts when writing into stores.
.SH OPTIONS
.IP "\fB\-b\fP \fIn\fP"
Store records beginning in a window \fIn\fP units away from
the input's end.
By default this value is 1.
.IP "\fB\-e\fP \fIn\fP"
Store records ending in a window \fIn\fP units away from
the input's end.
By default this value is 0.
.IP "\fB\-l\fP \fIlen\fP"
Process fixed-width \fIlen\fP-sized records.
By default \fIdgsh-writeval\fP will process newline-terminated
records.
.IP "\fB\-s\fP \fIpath\fP"
This mandatory option must be used to specify the path of the Unix-domain socket
\fIdgsh-writeval\fP will create.
This is specified as a normal Unix file path,
e.g. \fC/tmp/myvalue\fP.
.IP "\fB\-t\fP \fIchar\fP"
Specify the record termination character to be \fIchar\fP.
This is the newline by default.
.IP "\fB\-u\fP \fIunit\fP"
Specify the unit of the window boundaries given in the
\fC-b\fP and \fC-e\fP options.
The following units can be specified, using single-character
identifiers.
.RS
.IP "\fBs\fP
seconds
.IP "\fBm\fP
minutes
.IP "\fBh\fP
hours
.IP "\fBd\fP
days
.IP "\fBr\fP
records (this is the default value)
.RE
.SH "SEE ALSO"
\fIdgsh\fP(1),
\fIdgsh-readval\fP(1)
.SH AUTHOR
Diomidis Spinellis \(em <http://www.spinellis.gr>
================================================
FILE: core-tools/src/dgsh-writeval.c
================================================
/*
* Copyright 2013 Diomidis Spinellis
*
* Read values from its standard input and make them available to other
* processes for reading through the specified Unix domain socket.
* Thus, this process acts in effect as a data store: it reads a series of
* values (think of them as assignements) and provides a way to read the
* store's current value (from the socket).
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
#include <sys/types.h>
#include <sys/stat.h>
#include <sys/select.h>
#include <sys/socket.h>
#include <sys/time.h>
#include <sys/uio.h>
#include <sys/un.h>
#include <assert.h>
#include <err.h>
#include <errno.h>
#include <fcntl.h>
#include <limits.h>
#include <stdbool.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <unistd.h>
#include "dgsh.h"
#include "kvstore.h"
#include "dgsh-debug.h"
#include "minmax.h"
#ifdef DEBUG
/* Small buffer size to catch errors with data spanning buffers */
#define BUFFER_SIZE 5
#else
/* PIPE_BUF is a reasonable size heuristic. */
#define BUFFER_SIZE PIPE_BUF
#endif
/* User options start here */
/* Record terminator */
static char rt = '\n';
/* Record length; 0 if we use a record terminator */
static int rl = 0;
/* True if the begin and end are specified using a time window */
static bool time_window;
/*
* Specified response record.
* This is specified using reverse iterators (counted from the end of the stream).
* The _rbegin is inclusive, _rend is exclusive
* Examples:
* To get the last record use the range rbegin = 0 rend = 1
* To get 5 records starting 10 records away from the end
* use the range rbegin = 10 rend = 15
*/
static union {
struct timeval t; /* Used if time_window is true */
int r; /* Used if time_window is false */
double d; /* Used when parsing */
} record_rbegin, record_rend;
/* User options end here */
/* True once we reach the end of file on standard input */
static bool reached_eof;
/* True if a complete record (ending in rt) is available */
static bool have_record;
/* Queue (doubly linked list) of buffers used for storing the last read record */
struct buffer {
struct buffer *next;
struct buffer *prev;
int size; /* Actual number of bytes stored */
struct timeval timestamp; /* Time the buffer was read */
long long record_count; /* Total number of complete records read (including this buffer)
(0-based ordinal of first record not in buffer) */
long long byte_count; /* Total number of bytes read (including this buffer) */
char data[BUFFER_SIZE];
};
static struct buffer *head, *tail;
/* The oldest buffer whose contents are still being written to a socket. */
static struct buffer *oldest_buffer_being_written;
/* A pointer to a character stored in a buffer */
struct dpointer {
struct buffer *b; /* The buffer */
int pos; /* The position within the buffer */
};
/* The last complete record read */
static struct dpointer current_record_begin, current_record_end;
/* The clients we're talking to */
struct client {
int fd;
struct dpointer write_begin; /* Start of data for next write */
struct dpointer write_end; /* End of data to write */
enum {
s_inactive, /* Free (unused or closed) */
s_read_command, /* Waiting for a command (Q or R) to be read */
s_send_current, /* Waiting for the current value to be written */
s_send_current_nblk, /* Non-blocking: waiting for the current or empty value to be written */
s_send_last, /* Waiting for the last (before EOF) value to be written */
s_sending_response, /* A response is being written */
s_wait_close, /* Wait for the client to close the connection */
} state;
};
#define MAX_CLIENTS 64
static struct client clients[MAX_CLIENTS];
static const char *program_name;
static const char *socket_path;
/*
* Increment dp by one byte.
* If no more bytes are available return false
* leaving pos to point one byte past the last available one
*/
static bool
dpointer_increment(struct dpointer *dp)
{
DPRINTF(4, "%p pos=%d", dp->b, dp->pos);
dp->pos++;
if (dp->pos == dp->b->size) {
if (!dp->b->next)
return false;
dp->b = dp->b->next;
dp->pos = 0;
}
DPRINTF(4, "return %p pos=%d", dp->b, dp->pos);
return true;
}
/* Decrement dp by one byte. Return false if no more bytes are available */
static bool
dpointer_decrement(struct dpointer *dp)
{
DPRINTF(4, "%p pos=%d", dp->b, dp->pos);
dp->pos--;
if (dp->pos == -1) {
if (!dp->b->prev) {
dp->pos++;
return false;
}
dp->b = dp->b->prev;
dp->pos = dp->b->size - 1;
}
DPRINTF(4, "return %p pos=%d", dp->b, dp->pos);
return true;
}
/*
* Add to dp the specified number of bytes.
* Return true of OK.
* If not enough bytes are available return false
* and set dp to point to the last byte in the buffer.
*/
static bool
dpointer_add(struct dpointer *dp, int n)
{
DPRINTF(4, "%p pos=%d n=%d", dp->b, dp->pos, n);
while (n > 0) {
int add = MIN(dp->b->size - dp->pos, n);
n -= add;
dp->pos += add;
if (dp->pos == dp->b->size) {
if (!dp->b->next)
return false;
dp->b = dp->b->next;
dp->pos = 0;
}
}
DPRINTF(4, "return %p pos=%d", dp->b, dp->pos);
return true;
}
/*
* Subtract from dp the specified number of bytes.
* Return true of OK.
* If not enough bytes are available return false
* and set dp to point beyond the first available byte.
*/
static bool
dpointer_subtract(struct dpointer *dp, int n)
{
DPRINTF(4, "%p pos=%d n=%d", dp->b, dp->pos, n);
while (n > 0) {
int subtract = MIN((dp->pos + 1) - 0, n);
n -= subtract;
dp->pos -= subtract;
if (dp->pos == -1) {
if (!dp->b->prev)
return false;
dp->b = dp->b->prev;
dp->pos = dp->b->size - 1;
}
}
DPRINTF(4, "return %p pos=%d", dp->b, dp->pos);
return true;
}
/*
* Move back dp the specified number of rt-terminated records.
* Postcondition: dp will point at the beginning of
* a record.
* Return true if OK, false if not enough records are available
* Example: to move back over one complete record, the function will
* encounter two rts, and return with pos set immediately after the
* second one.
*/
static bool
dpointer_move_back(struct dpointer *dp, int n)
{
DPRINTF(4, "%p pos=%d (size=%d, prev=%p) n=%d",
dp->b, dp->pos, dp->b->size, dp->b->prev, n);
for (;;) {
if (dpointer_decrement(dp)) {
if (dp->b->data[dp->pos] == rt && --n == -1) {
dpointer_increment(dp);
DPRINTF(4, "return %p pos=%d", dp->b, dp->pos);
return true;
}
} else {
if (--n == -1) {
DPRINTF(4, "(at begin) returns: %p pos=%d", dp->b, dp->pos);
return true;
} else
return false; /* Not enough records available */
}
}
}
/*
* Move forward dp the specified number of rt-terminated records.
* Postcondition: dp will point past the end of a record.
* Return true if OK, false if not enough records are available
*/
static bool
dpointer_move_forward(struct dpointer *dp, int n)
{
DPRINTF(4, "%p pos=%d (size=%d, next=%p) n=%d",
dp->b, dp->pos, dp->b->size, dp->b->next, n);
/* Cover the case where we are already at the beginning of the record */
if (!dpointer_decrement(dp)) {
DPRINTF(4, "return %p pos=%d (at head)", dp->b, dp->pos);
return true;
}
for (;;) {
if (dp->b->data[dp->pos] == rt && --n == -1) {
dpointer_increment(dp);
DPRINTF(4, "return %p pos=%d", dp->b, dp->pos);
return true;
}
if (!dpointer_increment(dp))
return false; /* Not enough records available */
}
}
/* Return the oldest of the two buffers (the one that comes first in the list) */
static struct buffer *
oldest_buffer(struct buffer *a, struct buffer *b)
{
struct buffer *bp;
if (a == NULL)
return b;
else if (b == NULL)
return a;
for (bp = head; bp ; bp = bp->next)
if (bp == a)
return a;
else if (bp == b)
return b;
assert(0);
}
/*
* Update oldest_buffer_being_written according to the
* buffers used by all clients sending a response.
*/
static void
update_oldest_buffer(void)
{
int i;
oldest_buffer_being_written = NULL;
for (i = 0; i < MAX_CLIENTS; i++)
if (clients[i].state == s_sending_response)
oldest_buffer_being_written =
oldest_buffer(oldest_buffer_being_written, clients[i].write_begin.b);
DPRINTF(4, "Oldest buffer beeing written is %p", oldest_buffer_being_written);
}
/* Free buffers preceding in position the used buffer */
static void
free_unused_buffers_by_position(struct buffer *used)
{
struct buffer *b, *bnext;
for (b = head; b; b = bnext) {
if (b == used || b == oldest_buffer_being_written) {
head = b;
b->prev = NULL;
DPRINTF(4, "After freeing buffer(s) head=%p tail=%p", head, tail);
return;
}
bnext = b->next;
DPRINTF(4, "Freeing buffer %p prev=%p next=%p", b, b->prev, b->next);
free(b);
}
/* Should have encountered used along the way. */
assert(0);
}
/* Free buffers preceding in time (older than) the used buffer */
static void
free_unused_buffers_by_time(struct timeval *used)
{
struct buffer *b;
DPRINTF(4, "Free buffers older than %lld.%06d",
(long long)used->tv_sec, (int)used->tv_usec);
/* Find first useful record */
for (b = head; b; b = b->next)
if (timercmp(&b->timestamp, used, >=) || b == oldest_buffer_being_written)
break;
assert(b); /* Should have encountered used along the way. */
DPRINTF(4, "First used buffer is %p", b);
/* Must now leave another record in case a record extends backward */
if (rl) {
int n = rl;
do {
b = b->prev;
} while (b && (n -= b->size) > 0);
} else {
do {
b = b->prev;
} while (b && !memchr(b->data, rt, b->size));
}
DPRINTF(4, "After extending back %p", b);
if (b)
free_unused_buffers_by_position(b);
}
/* Return the content length of the client's buffer */
static unsigned int
content_length(struct client *c)
{
struct buffer *bp;
unsigned int length;
if (c->write_begin.b == c->write_end.b)
length = c->write_end.pos - c->write_begin.pos;
else {
length = c->write_begin.b->size - c->write_begin.pos;
for (bp = c->write_begin.b->next; bp && bp != c->write_end.b; bp = bp->next)
length += bp->size;
length += c->write_end.pos;
}
DPRINTF(4, "return %u", length);
return length;
}
/*
* Update the pointers to the current response record based on the defined
* record terminator.
*/
static void
update_current_record_by_rt_number(void)
{
bool ret;
/* Point to the end of read data */
current_record_end.b = tail;
current_record_end.pos = tail->size;
/* Remove data that forms an incomplete record */
ret = dpointer_move_back(¤t_record_end, 0);
assert(ret);
/* Go back to the end of the specified record */
ret = dpointer_move_back(¤t_record_end, record_rbegin.r);
assert(ret);
/* Go further back to the begin of the specified record */
current_record_begin = current_record_end;
ret = dpointer_move_back(¤t_record_begin, record_rend.r - record_rbegin.r);
assert(ret);
}
/*
* Update the pointers to the current response record based on the defined
* record length.
*/
static void
update_current_record_by_rl_number(void)
{
bool ret;
/* Point to the end of read data */
current_record_end.b = tail;
current_record_end.pos = tail->size;
/* Remove data that forms an incomplete record */
ret = dpointer_subtract(¤t_record_end, tail->byte_count % rl);
assert(ret);
/* Go back to the end of the specified record */
ret = dpointer_subtract(¤t_record_end, record_rbegin.r * rl);
assert(ret);
/* Go further back to the begin of the specified record */
current_record_begin = current_record_end;
ret = dpointer_subtract(¤t_record_begin, (record_rend.r - record_rbegin.r) * rl);
assert(ret);
}
/*
* Update the pointers to the current response record to include the first
* record terminated record beginning in or after the begin buffer and
* the last record beginning in the end buffer.
* Set have_record to true if the corresponding data range exists
*/
static void
update_current_record_by_rt_time(struct buffer *begin, struct buffer *end)
{
/* Point to the begin of the data window */
current_record_begin.b = begin;
current_record_begin.pos = 0;
/* Go to the begin of a record starting at or after the buffer */
if (!dpointer_move_forward(¤t_record_begin, 0))
return;
/* Point to the end of the data window */
current_record_end.b = end;
current_record_end.pos = end->size;
dpointer_decrement(¤t_record_end);
/* Adjust data that forms an incomplete record */
if (!dpointer_move_forward(¤t_record_end, 0)) {
current_record_end.b = end;
current_record_end.pos = end->size;
if (!dpointer_move_back(¤t_record_end, 0))
return;
if (memcmp(¤t_record_begin, ¤t_record_end, sizeof(struct dpointer)) == 0)
return;
}
have_record = true;
}
/*
* Update the pointers to the current response record to include the first
* fixed length record beginning in or after the begin buffer and
* the last record beginning in the end buffer.
* Set have_record to true if the corresponding data range exists
*/
static void
update_current_record_by_rl_time(struct buffer *begin, struct buffer *end)
{
int mod;
DPRINTF(4, "Adjusting begin");
current_record_begin.b = begin;
current_record_begin.pos = 0;
if (begin->prev && (mod = begin->prev->byte_count % rl) != 0)
/*
* Example: rl == 10, prev->byte_count == 53
* mod = 3, dpointer_add(..., 7)
*/
if (!dpointer_add(¤t_record_begin, rl - mod))
return; /* Next record not there */
DPRINTF(4, "Adjusting end");
current_record_end.b = end;
current_record_end.pos = end->size;
if ((mod = end->byte_count % rl) != 0) {
/*
* Example: rl == 10, end->byte_count == 82
* mod = 2, dpointer_add(..., 8)
* Decrement and increment to convert between an iterator
* pointing beyond the range, and valid positions that dpointer_add
* can handle correctly.
*/
if (!dpointer_decrement(¤t_record_end) ||
!dpointer_add(¤t_record_end, rl - mod)) {
DPRINTF(4, "incomplete last record");
/* Try going back */
current_record_end.b = end;
current_record_end.pos = end->size;
if (!dpointer_subtract(¤t_record_end, mod))
return;
} else
(void)dpointer_increment(¤t_record_end);
}
if (memcmp(¤t_record_begin, ¤t_record_end, sizeof(struct dpointer)) == 0)
return;
have_record = true;
}
#ifdef DEBUG
/* Dump the buffer list using relative timestamps */
static void
dump_buffer_times(void)
{
struct buffer *bp;
struct timeval now, t;
gettimeofday(&now, NULL);
DPRINTF(4, "update_current_record: now=%lld.%06d rend=%lld.%06d rbegin=%lld.%06d",
(long long)now.tv_sec, (int)now.tv_usec,
(long long)record_rend.t.tv_sec, (int)record_rend.t.tv_usec,
(long long)record_rbegin.t.tv_sec, (int)record_rbegin.t.tv_usec);
for (bp = head; bp != NULL; bp = bp->next) {
timersub(&now, &bp->timestamp, &t);
DPRINTF(4, "\t%p size=%3d byte_count=%5lld Tr=%3lld.%06d Ta=%3lld.%06d [%.*s]",
bp, bp->size, bp->byte_count,
(long long)t.tv_sec, (int)t.tv_usec,
(long long)bp->timestamp.tv_sec, (int)bp->timestamp.tv_usec,
bp->size, bp->data);
}
}
static void
timestamp(const char *msg)
{
struct timeval now;
gettimeofday(&now, NULL);
DPRINTF(4, "%lld.%06d %s", (long long)now.tv_sec, (int)now.tv_usec, msg);
}
#define TIMESTAMP(x) timestamp(x)
#define DUMP_BUFFER_TIMES() dump_buffer_times()
#else
#define DUMP_BUFFER_TIMES()
#define TIMESTAMP(x)
#endif
/*
* Update the pointers to the current response record.
* Set have_record if a record is available.
*/
static void
update_current_record(void)
{
assert(head && tail);
if (time_window) {
struct timeval now, tbegin, tend; /* In absolute time units */
struct buffer *bbegin, *bend, *begin_candidate = NULL;
DUMP_BUFFER_TIMES();
have_record = false; /* Records in the window come and go */
/* Convert to absolute time */
gettimeofday(&now, NULL);
timersub(&now, &record_rend.t, &tbegin);
DPRINTF(4, "tail->timestamp=%lld.%06d tbegin=%lld.%06d",
(long long)tail->timestamp.tv_sec, (int)tail->timestamp.tv_usec,
(long long)tbegin.tv_sec, (int)tbegin.tv_usec);
if (timercmp(&tail->timestamp, &tbegin, <)) {
free_unused_buffers_by_position(tail);
return; /* No records fresh enough */
}
timersub(&now, &record_rbegin.t, &tend);
DPRINTF(4, "head->timestamp=%lld.%06d tend=%lld.%06d",
(long long)head->timestamp.tv_sec, (int)head->timestamp.tv_usec,
(long long)tend.tv_sec, (int)tend.tv_usec);
if (timercmp(&head->timestamp, &tend, >))
return; /* No records old enough */
/* Find the record range */
DPRINTF(4, "Looking for record range");
for (bend = tail; timercmp(&bend->timestamp, &tend, >); bend = bend->prev)
;
DPRINTF(4, "bend=%p %lld.%06d", bend, (long long)bend->timestamp.tv_sec, (int)bend->timestamp.tv_usec);
for (bbegin = bend; bbegin && timercmp(&bbegin->timestamp, &tbegin, >); bbegin = bbegin->prev)
begin_candidate = bbegin;
if (!begin_candidate) {
free_unused_buffers_by_time(&tbegin);
return; /* No records within the window */
}
bbegin = begin_candidate;
DPRINTF(4, "bbegin=%p %lld.%06d", bbegin, (long long)bbegin->timestamp.tv_sec, (int)bbegin->timestamp.tv_usec);
if (rl)
update_current_record_by_rl_time(bbegin, bend);
else
update_current_record_by_rt_time(bbegin, bend);
free_unused_buffers_by_time(&tbegin);
} else {
DPRINTF(4, "tail->record_count=%lld record_rend.r=%d",
tail->record_count, record_rend.r);
if (tail->record_count - record_rend.r < 0)
/* Not enough records */
return;
if (rl)
update_current_record_by_rl_number();
else
update_current_record_by_rt_number();
have_record = true;
free_unused_buffers_by_position(current_record_begin.b);
}
DPRINTF(4, "have_record=%d", have_record);
DPRINTF(4, "begin b=%p pos=%d", current_record_begin.b, current_record_begin.pos);
DPRINTF(4, "end b=%p pos=%d", current_record_end.b, current_record_end.pos);
}
/*
* Read a one character command from the specifid client and act on it
* The following commands are supported:
* R: Read value (the client wants to read our current store value)
* Q: Quit (Terminate the operation of this data store)
*/
static void
read_command(struct client *c)
{
char cmd;
int n;
switch (n = read(c->fd, &cmd, 1)) {
case -1: /* Error */
switch (errno) {
case EAGAIN:
DPRINTF(4, "EAGAIN on client socket read");
break;
default:
err(3, "Read from socket");
}
break;
case 0: /* EOF */
close(c->fd);
c->state = s_inactive;
DPRINTF(4, "Done with client %p", c);
update_oldest_buffer();
break;
default: /* Have data. Insert buffer at the end of the queue. */
DPRINTF(4, "Read command %c from client %p", cmd, c);
switch (cmd) {
case 'L':
c->state = s_send_last;
break;
case 'Q':
(void)unlink(socket_path);
exit(0);
case 'c':
c->state = s_send_current_nblk;
break;
case 'C':
c->state = s_send_current;
if (time_window && head)
update_current_record(); /* Refresh have_record */
break;
default:
errx(5, "Unknown command [%c]", cmd);
}
}
}
/*
* Write a single record to the specified client
* Update the write_begin pointer
* Close the connection and clean the client if done
* If write_length is true, precede the record with
* CONTENT_LENGTH_DIGITS digits representing the record's
* length.
*/
static void
write_record(struct client *c, bool write_length)
{
int n;
int towrite;
struct iovec iov[2], *iovptr;
char length[CONTENT_LENGTH_DIGITS + 2];
DPRINTF(4, "Write %srecord for client %p", write_length ? "first " : "", c);
if (c->write_begin.b == c->write_end.b) {
towrite = c->write_end.pos - c->write_begin.pos;
DPRINTF(4, "Single buffer %p: writing %d bytes. write_end.pos=%d write_begin.pos=%d",
c->write_begin.b, towrite, c->write_end.pos, c->write_begin.pos);
} else {
towrite = c->write_begin.b->size - c->write_begin.pos;
DPRINTF(4, "Multiple buffers %p %p: writing %d bytes. write_begin.b->size=%d write_begin.pos=%d",
c->write_begin.b, c->write_end.b,
towrite, c->write_begin.b->size, c->write_begin.pos);
}
iov[1].iov_base = c->write_begin.b->data + c->write_begin.pos;
iov[1].iov_len = towrite;
DPRINTF(4, "Writing [%.*s]", (int)iov[1].iov_len, (char *)iov[1].iov_base);
if (write_length) {
snprintf(length, sizeof(length), CONTENT_LENGTH_FORMAT, content_length(c));
iov[0].iov_base = length;
iov[0].iov_len = CONTENT_LENGTH_DIGITS;
iovptr = iov;
} else
iovptr = iov + 1;
if ((n = writev(c->fd, iovptr, write_length ? 2 : 1)) == -1)
switch (errno) {
case EAGAIN:
DPRINTF(4, "EAGAIN on client socket write");
return;
default:
err(3, "Write to socket");
}
if (write_length) {
if (n < CONTENT_LENGTH_DIGITS)
errx(5, "Short content length record write: %d", n);
n -= CONTENT_LENGTH_DIGITS;
}
c->write_begin.pos += n;
DPRINTF(4, "Wrote %u data bytes. Current buffer position=%d", n, c->write_begin.pos);
/*
* More data to write from this buffer?
* Yes, if there is still more data in the buffer
* and either the end is in another buffer, or we haven't
* reached it.
*/
if (c->write_begin.pos < c->write_begin.b->size &&
(c->write_begin.b != c->write_end.b || c->write_begin.pos < c->write_end.pos)) {
DPRINTF(4, "Continuing with same buffer");
return;
}
/* More buffers to write from? */
if (c->write_begin.b != c->write_end.b) {
c->write_begin.b = c->write_begin.b->next;
c->write_begin.pos = 0;
DPRINTF(4, "Moving to next buffer %p with size %u", c->write_begin.b, c->write_begin.b->size);
return;
}
/* Done with this client */
DPRINTF(4, "No more data to write for client %p", c);
c->state = s_wait_close;
}
/* Set the buffer's counters */
void
set_buffer_counters(struct buffer *b)
{
if (time_window)
gettimeofday(&b->timestamp, NULL);
if (rl == 0) {
/* Count records using RS */
int i;
b->record_count = b->prev ? b->prev->record_count : 0;
for (i = 0; i < b->size; i++)
if (b->data[i] == rt)
b->record_count++;
} else {
/* Count records using RL */
b->byte_count = b->prev ? b->prev->byte_count : 0;
b->byte_count += b->size;
b->record_count = b->byte_count / rl;
}
}
#if __GNUC__ == 4 && __GNUC_MINOR__ >= 2 && __GNUC_MINOR__ < 6
#pragma GCC diagnostic ignored "-Wuninitialized"
#endif
/* Read data from STDIN into a new buffer */
static void
buffer_read(void)
{
struct buffer *b;
struct timeval now, abs_rend_time;
if ((b = malloc(sizeof(struct buffer))) == NULL)
err(1, "Unable to allocate read buffer");
DPRINTF(4, "Calling read on stdin for buffer %p", b);
switch (b->size = read(STDIN_FILENO, b->data, sizeof(b->data))) {
case -1: /* Error */
switch (errno) {
case EAGAIN:
DPRINTF(4, "EAGAIN on standard input");
free(b);
break;
default:
err(3, "Read from standard input");
}
break;
case 0: /* EOF */
reached_eof = true;
if (time_window) {
/* Make abs_rend_time the latest absolute time that interests us */
gettimeofday(&now, NULL);
timeradd(&now, &record_rend.t, &abs_rend_time);
}
if (have_record) {
free(b);
#if __GNUC__ >= 4 && __GNUC_MINOR__ >= 6
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wmaybe-uninitialized"
#endif
} else if (!time_window || !tail ||
timercmp(&tail->timestamp, &abs_rend_time, >)) {
#if __GNUC__ >= 4 && __GNUC_MINOR__ >= 6
#pragma GCC diagnostic pop
#endif
/* Setup an empty record, if there will never be a record to send */
b->size = 0;
b->prev = b->next = NULL;
head = tail = b;
current_record_begin.b = current_record_end.b = b;
current_record_begin.pos = current_record_end.pos = 0;
have_record = true;
}
break;
default: /* Have data. Insert buffer at the end of the queue. */
b->prev = tail;
b->next = NULL;
if (tail)
tail->next = b;
tail = b;
if (!head)
head = b;
DPRINTF(4, "Read %d bytes into %p prev=%p next=%p head=%p tail=%p",
b->size, b, b->prev, b->next, head, tail);
set_buffer_counters(b);
update_current_record();
break;
}
}
/*
* Set the specified file descriptor to operate in non-blocking
* mode.
* It seems that even if select returns for a specified file
* descriptor, performing I/O to it may block depending on the
* amount of data specified.
* See See http://pubs.opengroup.org/onlinepubs/009695399/functions/write.html#tag_03_866
*/
static void
non_block(int fd)
{
int flags = fcntl(fd, F_GETFL, 0);
if (flags < 0)
err(2, "Error getting flags for socket");
if (fcntl(fd, F_SETFL, flags | O_NONBLOCK) < 0)
err(2, "Error setting socket to non-blocking mode");
}
/* Return an initialized free client entry, or exit with an error. */
static struct client *
get_free_client(void)
{
int i;
for (i = 0; i < MAX_CLIENTS; i++)
if (clients[i].state == s_inactive)
return &clients[i];
errx(5, "Maximum number of clients exceeded for socket %s", socket_path);
}
static void
usage(void)
{
fprintf(stderr, "Usage: %s [-l len|-t char] [-b n] [-e n] [-u s|m|h|d|r] -s path\n"
"-b n" "\tStore records beginning in a window n away from the end (default 1)\n"
"-e n" "\tStore records ending in a window n away from the end (default 0)\n"
"-l len" "\tProcess fixed-width len-sized records\n"
"-s path" "\tSpecify the socket to create\n"
"-t char" "\tProcess char-terminated records (newline default)\n"
"-u unit" "\tSpecify the unit of window boundaries\n"
"" "\ts: seconds\n"
"" "\tm: minutes\n"
"" "\th: hours\n"
"" "\td: days\n"
"" "\tr: records (default)\n",
program_name);
exit(1);
}
/*
* Parse a number >= 0
* Exit with an error if an error occurs
*/
static double
parse_double(const char *s)
{
char *endptr;
double d;
errno = 0;
d = strtod(s, &endptr);
if (endptr - s != strlen(s) || *s == 0)
errx(6, "Error in parsing [%s] as a number", s);
if (errno != 0)
err(6, "[%s]", s);
if (d < 0)
errx(6, "Argument [%s] cannot be negative", s);
return d;
}
/* Return the passed double as a timeval */
struct timeval
double_to_timeval(double d)
{
struct timeval t;
t.tv_sec = (time_t)d;
t.tv_usec = (int)((d - t.tv_sec) * 1e6);
return t;
}
/* Parse the program's arguments */
static void
parse_arguments(int argc, char *argv[])
{
int ch;
char unit = 'r';
program_name = argv[0];
/* By default return the last record read */
record_rbegin.d = 0;
record_rend.d = 1;
while ((ch = getopt(argc, argv, "b:e:l:s:t:u:")) != -1) {
switch (ch) {
case 'b': /* Begin record, measured from the end (0) */
record_rend.d = parse_double(optarg);
break;
case 'e': /* End record, measured from the end (0) */
record_rbegin.d = parse_double(optarg);
break;
case 'l': /* Fixed record length */
rl = atoi(optarg);
if (rl <= 0)
usage();
break;
case 's':
socket_path = optarg;
break;
case 't': /* Record terminator */
/* We allow \0 as rt */
if (strlen(optarg) > 1)
usage();
rt = *optarg;
break;
case 'u': /* Measurement unit */
if (strlen(optarg) != 1 || strchr("smhdr", *optarg) == NULL)
usage();
unit = *optarg;
break;
case '?':
default:
usage();
}
}
argc -= optind;
argv += optind;
if (argc != 0 || socket_path == NULL)
usage();
switch (unit) {
case 'r':
if (record_rbegin.d != (int)record_rbegin.d ||
record_rend.d != (int)record_rend.d)
errx(6, "Record numbers must be integers");
record_rbegin.r = (int)record_rbegin.d;
record_rend.r = (int)record_rend.d;
time_window = false;
break;
case 'd':
record_rbegin.d *= 24;
record_rend.d *= 24;
/* FALLTHROUGH */
case 'h':
record_rbegin.d *= 60;
record_rend.d *= 60;
/* FALLTHROUGH */
case 'm':
record_rbegin.d *= 60;
record_rend.d *= 60;
/* FALLTHROUGH */
case 's':
record_rbegin.t = double_to_timeval(record_rbegin.d);
record_rend.t = double_to_timeval(record_rend.d);
if (!timercmp(&record_rbegin.t, &record_rend.t, <))
errx(6, "Begin time must be older than end time");
time_window = true;
break;
}
}
/*
* Handle the events associated with the following elements
* The passed socket
* Standard input
* Communicating clients
* Elapsed time values
* This is called in an endless loop to do the following things:
* Setup select(2) arguments
* Call select(2)
* Process events that can be processed
*/
static void
handle_events(int sock)
{
fd_set source_fds;
fd_set sink_fds;
struct timeval wait_time, *waitptr;
bool set_waitptr;
int i, max_fd, nfds;
socklen_t len;
struct sockaddr_un remote;
/* Set the fds that interest us */
FD_ZERO(&source_fds);
FD_ZERO(&sink_fds);
waitptr = NULL;
max_fd = -1;
/* Read from standard input */
if (!reached_eof) {
FD_SET(STDIN_FILENO, &source_fds);
max_fd = STDIN_FILENO;
}
/* Accept incoming connection */
FD_SET(sock, &source_fds);
max_fd = MAX(sock, max_fd);
/* I/O with a client */
set_waitptr = false;
for (i = 0; i < MAX_CLIENTS; i++)
switch (clients[i].state) {
case s_inactive: /* Free (unused or closed) */
break;
case s_wait_close: /* Wait for the client to close the connection */
case s_read_command: /* Waiting for a command (Q or R) to be read */
FD_SET(clients[i].fd, &source_fds);
max_fd = MAX(clients[i].fd, max_fd);
break;
case s_send_last: /* Waiting for the last (before EOF) value to be written */
if (reached_eof) {
FD_SET(clients[i].fd, &sink_fds);
max_fd = MAX(clients[i].fd, max_fd);
}
break;
case s_send_current: /* Waiting for a response to be written */
if (have_record) {
FD_SET(clients[i].fd, &sink_fds);
max_fd = MAX(clients[i].fd, max_fd);
} else if (time_window)
set_waitptr = true;
break;
case s_send_current_nblk: /* Waiting for a response to be written */
FD_SET(clients[i].fd, &sink_fds);
max_fd = MAX(clients[i].fd, max_fd);
break;
case s_sending_response: /* A response is being sent */
FD_SET(clients[i].fd, &sink_fds);
max_fd = MAX(clients[i].fd, max_fd);
break;
}
if (set_waitptr) {
/*
* Find the oldest buffer that hasn't yet entered the time
* window and arrange for select(2) to wait for it to enter.
*/
struct buffer *bp, *candidate_buffer = NULL;
struct timeval now, abs_rbegin_time;
gettimeofday(&now, NULL);
timersub(&now, &record_rbegin.t, &abs_rbegin_time);
DPRINTF(4, "have to wait for a buffer to enter window %lld.%06d",
(long long)abs_rbegin_time.tv_sec, (int)abs_rbegin_time.tv_usec);
/*
* rbegin = 10
* 13 19 20 21 23
* abs_rbegin ... ... tail now
*/
for (bp = tail; bp && timercmp(&bp->timestamp, &abs_rbegin_time, >); bp = bp->prev)
candidate_buffer = bp;
if (candidate_buffer) {
/* There is a buffer worth waiting for */
waitptr = &wait_time;
timersub(&candidate_buffer->timestamp, &abs_rbegin_time, waitptr);
DPRINTF(4, "waiting %lld.%06d for %p %lld.%06d to enter the window",
(long long)wait_time.tv_sec, (int)wait_time.tv_usec,
candidate_buffer,
(long long)candidate_buffer->timestamp.tv_sec,
(int)candidate_buffer->timestamp.tv_usec);
} else
DPRINTF(4, "No candidate buffer found");
}
TIMESTAMP("Calling select");
if ((nfds = select(max_fd + 1, &source_fds, &sink_fds, NULL, waitptr)) < 0)
err(3, "select");
TIMESTAMP("Select returns");
if (FD_ISSET(STDIN_FILENO, &source_fds))
buffer_read();
if (waitptr && nfds == 0)
/* Expired timer; records may have entered the window */
update_current_record();
for (i = 0; i < MAX_CLIENTS; i++)
switch (clients[i].state) {
case s_inactive: /* Free (unused or closed) */
break;
case s_read_command: /* Waiting for a command (Q or R) to be read */
case s_wait_close: /* Wait for the client to close the connection */
if (FD_ISSET(clients[i].fd, &source_fds))
read_command(&clients[i]);
break;
case s_send_last: /* Waiting for the last (before EOF) value to be written */
/* FALLTHROUGH */
case s_send_current: /* Waiting for a response to be written */
if (FD_ISSET(clients[i].fd, &sink_fds)) {
assert(have_record);
/* Start writing the most fresh last record */
clients[i].write_begin = current_record_begin;
clients[i].write_end = current_record_end;
clients[i].state = s_sending_response;
oldest_buffer_being_written =
oldest_buffer(oldest_buffer_being_written, clients[i].write_begin.b);
write_record(&clients[i], true);
}
break;
case s_send_current_nblk: /* Waiting for a response (even empty) to be written */
if (FD_ISSET(clients[i].fd, &sink_fds)) {
if (have_record) {
/* Start writing the most fresh last record */
clients[i].write_begin = current_record_begin;
clients[i].write_end = current_record_end;
oldest_buffer_being_written =
oldest_buffer(oldest_buffer_being_written, clients[i].write_begin.b);
} else {
static struct buffer empty;
/* Write an empty record */
clients[i].write_begin.b = clients[i].write_end.b = ∅
clients[i].write_begin.pos = clients[i].write_end.pos = 0;
}
clients[i].state = s_sending_response;
write_record(&clients[i], true);
}
break;
case s_sending_response: /* A response is being written */
if (FD_ISSET(clients[i].fd, &sink_fds))
write_record(&clients[i], false);
break;
}
if (FD_ISSET(sock, &source_fds)) {
int rsock;
struct client *c;
len = sizeof(remote);
rsock = accept(sock, (struct sockaddr *)&remote, &len);
if (rsock == -1 && errno != EAGAIN)
err(5, "accept");
c = get_free_client();
non_block(rsock);
c->fd = rsock;
c->state = s_read_command;
}
}
int
main(int argc, char *argv[])
{
int sock;
socklen_t len;
struct sockaddr_un local;
int ninputs = 1;
int noutputs = 0;
parse_arguments(argc, argv);
dgsh_negotiate(DGSH_HANDLE_ERROR, program_name, &ninputs, &noutputs,
NULL, NULL);
if (strlen(socket_path) >= sizeof(local.sun_path) - 1)
errx(6, "Socket name [%s] must be shorter than %lu characters",
socket_path, (long unsigned)sizeof(local.sun_path));
(void)unlink(socket_path);
if ((sock = socket(AF_UNIX, SOCK_STREAM, 0)) == -1)
err(2, "Error creating socket");
local.sun_family = AF_UNIX;
strcpy(local.sun_path, socket_path);
len = strlen(local.sun_path) + 1 + sizeof(local.sun_family);
if (bind(sock, (struct sockaddr *)&local, len) == -1)
err(3, "Error binding socket to Unix domain address %s", argv[1]);
if (listen(sock, 5) == -1)
err(4, "listen");
non_block(sock);
reached_eof = false;
for (;;)
handle_events(sock);
}
================================================
FILE: core-tools/src/dgsh.1
================================================
.TH DGSH 1 "10 August 2017"
.\"
.\" (C) Copyright 2016-2017 Diomidis Spinellis. All rights reserved.
.\"
.\" Licensed under the Apache License, Version 2.0 (the "License");
.\" you may not use this file except in compliance with the License.
.\" You may obtain a copy of the License at
.\"
.\" http://www.apache.org/licenses/LICENSE-2.0
.\"
.\" Unless required by applicable law or agreed to in writing, software
.\" distributed under the License is distributed on an "AS IS" BASIS,
.\" WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
.\" See the License for the specific language governing permissions and
.\" limitations under the License.
.\"
.SH NAME
dgsh \- directed graph shell
.SH SYNOPSIS
\fBdgsh\fP
[\fIbash_options\fP]
[command_string | file]
.SH DESCRIPTION
\fIdgsh\fP is a modified version of \fIbash\fP
that allows the specification of pipelines with non-linear non-uniform
operations.
These form a directed acyclic process graph, which is
typically executed by multiple processor cores, thus increasing the
operation's processing throughput.
The \fIdgsh\fP command is equivalent to invoking the modified version of \fIbash\fP
with the \fI--dgsh\fP argument in order to enable the \fIdgsh\fP-specific
inter-process communication functionality.
.SH INTER-PROCESS COMMUNICATION
\fIDgsh\fP provides three new ways
for expressing inter-process communication.
.PP
\fBMultipipes\fP are expressed as usual Unix pipelines,
but can connect commands with more than one output or input channel.
As an example, the \fIcomm\fP command supplied with \fIdgsh\fP
expects two input channels and produces on its output three
output channels: the lines appearing only in first (sorted) channel,
the lines appearing only in the second channel,
and the lines appearing in both.
Connecting the output of the \fIcomm\fP command to the
\fIcat\fP command supplied with \fIdgsh\fP
will make the three outputs appear in sequence,
while connecting it to the
\fIpaste\fP command supplied with \fIdgsh\fP
will make the output appear in its customary format.
\fIDgsh\fP handles the following programs
as being multipipe compatible:
a) those that are linked with the \fIdgsh\fP library;
b) scripts that include in their first line one of the strings
\fCdgsh-wrap\fP, \fCenv dgsh\fP, or \fC--dgsh\fP;
c) scripts whose second line starts with \fC#!dgsh\fP.
.PP
\fBMultipipe blocks\fP are enclosed within double braces: {{ ... }}.
These
a) send the input received on their input side to the asynchronously-running
processes that reside within the block, and, b) pass the output
produced by the processes within the block to their output side.
Multipipe blocks typically receive input from more than one channel
and produce more than one output channel.
For example, a multipipe block that runs \fImd5sum\fP and \fIwc -c\fP
receives two inputs and produces two outputs:
the MD5 hash of its input and the input's size.
gitextract_fo8a72ky/
├── .gitignore
├── .gitmodules
├── .travis/
│ ├── linux-ubuntu-trusty.install.sh
│ └── macosx-xcode8.3.install.sh
├── .travis.yml
├── CITATION.cff
├── LICENSE
├── Makefile
├── README.md
├── core-tools/
│ ├── .gitignore
│ ├── Makefile.am
│ ├── configure.ac
│ ├── src/
│ │ ├── .gitignore
│ │ ├── Makefile.am
│ │ ├── debug.h
│ │ ├── dgsh-conc.1
│ │ ├── dgsh-conc.c
│ │ ├── dgsh-debug.h
│ │ ├── dgsh-elf.s
│ │ ├── dgsh-enumerate.1
│ │ ├── dgsh-enumerate.c
│ │ ├── dgsh-fft-input.c
│ │ ├── dgsh-httpval.1
│ │ ├── dgsh-httpval.c
│ │ ├── dgsh-macho.s
│ │ ├── dgsh-merge-sum.1
│ │ ├── dgsh-merge-sum.pl
│ │ ├── dgsh-monitor.1
│ │ ├── dgsh-monitor.c
│ │ ├── dgsh-parallel.1
│ │ ├── dgsh-parallel.sh
│ │ ├── dgsh-pecho.c
│ │ ├── dgsh-readval.1
│ │ ├── dgsh-readval.c
│ │ ├── dgsh-tee.1
│ │ ├── dgsh-tee.c
│ │ ├── dgsh-w.c
│ │ ├── dgsh-wrap.1
│ │ ├── dgsh-wrap.c
│ │ ├── dgsh-writeval.1
│ │ ├── dgsh-writeval.c
│ │ ├── dgsh.1
│ │ ├── dgsh.h
│ │ ├── dgsh_negotiate.3
│ │ ├── kvstore.c
│ │ ├── kvstore.h
│ │ ├── minmax.h
│ │ ├── negotiate.c
│ │ ├── negotiate.h
│ │ ├── perm.1
│ │ ├── perm.sh
│ │ └── run-built-dgsh.sh
│ ├── tests/
│ │ ├── .gitignore
│ │ ├── Makefile.am
│ │ ├── Makefile.patch
│ │ ├── check.hack
│ │ └── check_negotiate.c
│ └── tests-regression/
│ ├── .gitignore
│ ├── author-compare/
│ │ └── out.ok
│ ├── bin/
│ │ └── gdate
│ ├── code-metrics/
│ │ ├── in/
│ │ │ └── date/
│ │ │ ├── date.c
│ │ │ ├── extern.h
│ │ │ ├── netdate.c
│ │ │ ├── vary.c
│ │ │ └── vary.h
│ │ └── out.ok
│ ├── commit-stats/
│ │ └── out.ok
│ ├── compress-compare/
│ │ └── out.ok
│ ├── dgsh-wrap/
│ │ ├── .gitignore
│ │ ├── dd-args.ok
│ │ ├── echo-deaf.ok
│ │ ├── echo-s.ok
│ │ ├── echo-s_caps.ok
│ │ ├── paste1.ok
│ │ ├── paste2.ok
│ │ ├── paste3.ok
│ │ ├── paste4.ok
│ │ ├── tee1.ok
│ │ └── tee2.ok
│ ├── duplicate-files/
│ │ ├── in/
│ │ │ ├── another-same-1
│ │ │ ├── another-same-2
│ │ │ ├── different-file-1
│ │ │ ├── different-file-2
│ │ │ ├── different-file-3
│ │ │ ├── same-file-1
│ │ │ ├── same-file-2
│ │ │ └── same-file-3
│ │ └── out.ok
│ ├── map-hierarchy/
│ │ ├── in/
│ │ │ ├── a/
│ │ │ │ └── date/
│ │ │ │ ├── date.c
│ │ │ │ ├── vary.c
│ │ │ │ └── vary.h
│ │ │ └── b/
│ │ │ └── bin/
│ │ │ └── date/
│ │ │ ├── date.c
│ │ │ ├── headers/
│ │ │ │ ├── extern.h
│ │ │ │ └── vary.h
│ │ │ ├── netdate.c
│ │ │ └── vary.c
│ │ └── out.ok/
│ │ └── map-hierarchy/
│ │ └── in/
│ │ └── b/
│ │ └── bin/
│ │ └── date/
│ │ ├── date.c
│ │ ├── headers/
│ │ │ └── vary.h
│ │ └── vary.c
│ ├── parallel-word-count/
│ │ └── out.ok
│ ├── regression/
│ │ ├── errors/
│ │ │ ├── stream-scatter-cycle.ok
│ │ │ ├── stream-scatter-cycle.sh
│ │ │ ├── unsafe-gather.ok
│ │ │ ├── unsafe-gather.sh
│ │ │ ├── unsafe-gather2.ok
│ │ │ └── unsafe-gather2.sh
│ │ ├── graphs/
│ │ │ ├── NMRPipe.ok
│ │ │ ├── code-metrics.ok
│ │ │ ├── commit-stats.ok
│ │ │ ├── committer-plot.ok
│ │ │ ├── compress-compare.ok
│ │ │ ├── dir.ok
│ │ │ ├── duplicate-files.ok
│ │ │ ├── ft2d.ok
│ │ │ ├── map-hierarchy.ok
│ │ │ ├── parallel-logresolve.ok
│ │ │ ├── spell-highlight.ok
│ │ │ ├── text-properties.ok
│ │ │ ├── web-log-report.ok
│ │ │ ├── web-log-stats.ok
│ │ │ └── word-properties.ok
│ │ ├── scripts/
│ │ │ ├── NMRPipe.ok
│ │ │ ├── code-metrics.ok
│ │ │ ├── commit-stats.ok
│ │ │ ├── committer-plot.ok
│ │ │ ├── compress-compare.ok
│ │ │ ├── dir.ok
│ │ │ ├── duplicate-files.ok
│ │ │ ├── ft2d.ok
│ │ │ ├── map-hierarchy.ok
│ │ │ ├── parallel-logresolve.ok
│ │ │ ├── spell-highlight.ok
│ │ │ ├── text-properties.ok
│ │ │ ├── web-log-report.ok
│ │ │ ├── web-log-stats.ok
│ │ │ └── word-properties.ok
│ │ └── warnings/
│ │ ├── single-target.ok
│ │ ├── single-target.sh
│ │ ├── unsafe-scatter.ok
│ │ └── unsafe-scatter.sh
│ ├── spell-highlight/
│ │ └── out.ok
│ ├── tee/
│ │ ├── oom.err
│ │ ├── perm.ok
│ │ ├── tee-fastout-I-l.ok
│ │ ├── tee-fastout-I-m 2k -f.ok
│ │ ├── tee-fastout-I-m 2k.ok
│ │ ├── tee-fastout-I.ok
│ │ ├── tee-fastout-l.ok
│ │ ├── tee-fastout-m 2k -f.ok
│ │ ├── tee-fastout-m 2k.ok
│ │ ├── tee-fastout.ok
│ │ ├── tee-lagout-I-l.ok
│ │ ├── tee-lagout-I-m 2k -f.ok
│ │ ├── tee-lagout-I-m 2k.ok
│ │ ├── tee-lagout-I.ok
│ │ ├── tee-lagout-l.ok
│ │ ├── tee-lagout-m 2k -f.ok
│ │ ├── tee-lagout-m 2k.ok
│ │ ├── tee-lagout.ok
│ │ └── tee-lahout.ok
│ ├── test-dgsh.sh
│ ├── test-kvstore.sh
│ ├── test-merge-sum.sh
│ ├── test-tee.sh
│ ├── test-wrap.sh
│ ├── text-properties/
│ │ └── out.ok/
│ │ ├── character.txt
│ │ ├── digram.txt
│ │ ├── trigram.txt
│ │ └── words.txt
│ ├── web-log-report/
│ │ ├── logfile
│ │ └── out.ok
│ └── word-properties/
│ ├── LostWorldChap1-3
│ └── out.ok
├── eval/
│ ├── .gitignore
│ ├── Makefile
│ ├── README.md
│ ├── SConstruct
│ ├── TextProperties.java
│ ├── WebStats.java
│ ├── eval-lib.sh
│ ├── ft2d.sh
│ ├── log-grow.pl
│ ├── perf-eval.sh
│ ├── web-log-report.pl
│ └── webeval.sh
├── example/
│ ├── NMRPipe.sh
│ ├── author-compare.sh
│ ├── code-metrics.sh
│ ├── commit-stats.sh
│ ├── committer-plot.sh
│ ├── compress-compare.sh
│ ├── dir.sh
│ ├── duplicate-files.sh
│ ├── fft-block8.sh
│ ├── ft2d.sh
│ ├── map-hierarchy.sh
│ ├── parallel-word-count.sh
│ ├── reorder-columns.sh
│ ├── spell-highlight.sh
│ ├── static-functions.sh
│ ├── text-properties.sh
│ ├── uniform-5x5.sh
│ ├── web-log-report.sh
│ └── word-properties.sh
├── png/
│ └── README
├── simple-shell/
│ ├── comm_paste.dgsh
│ ├── comm_paste.success
│ ├── comm_paste_join_diff.dgsh
│ ├── comm_paste_join_diff.success
│ ├── comm_sort.dgsh
│ ├── comm_sort.success
│ ├── dir-plain.dgsh
│ ├── grep_comm.dgsh
│ ├── grep_comm.success
│ ├── grep_comm.success-bash
│ ├── grep_diff.dgsh
│ ├── grep_diff.success
│ ├── grep_diff.success-bash
│ ├── grep_diff_comm.dgsh
│ ├── grep_diff_comm.success1
│ ├── grep_diff_comm.success1-bash
│ ├── grep_diff_comm.success2
│ ├── grep_diff_comm.success2-bash
│ ├── grep_diff_comm.success3
│ ├── grep_diff_comm.success3-bash
│ ├── grep_diff_comm.success4
│ ├── grep_diff_comm.success4-bash
│ ├── join_sort.dgsh
│ ├── join_sort.success
│ ├── join_sort_diff.dgsh
│ ├── join_sort_diff.success
│ ├── ls_wc.dgsh
│ ├── paste_diff.dgsh
│ ├── paste_diff.success
│ ├── secho_paste.dgsh
│ ├── secho_paste.success
│ ├── secho_secho_fgrep.dgsh
│ ├── secho_secho_fgrep.success
│ ├── simple-shell.py
│ ├── sort_sort_comm.dgsh
│ ├── sort_sort_comm.success
│ ├── sort_sort_comm_paste_join_diff.dgsh
│ ├── sort_sort_comm_paste_join_diff.success
│ ├── tee-copy_diff_comm.dgsh
│ ├── tee-copy_diff_comm.success
│ ├── tee-scatter_diff_comm.dgsh
│ ├── tee-scatter_diff_comm.success1
│ ├── tee-scatter_diff_comm.success2
│ ├── wrap-cat_comm_sort.dgsh
│ └── wrap-cat_comm_sort.success
├── test-data/
│ ├── .gitignore
│ ├── F
│ ├── access.log
│ ├── cmp0.success
│ ├── cmp1-same1.success
│ ├── cmp1-same2.success
│ ├── cmp2-diff.success
│ ├── cmp2-same.success
│ ├── comm_paste.success
│ ├── comm_paste_join_diff.success
│ ├── comm_sort.success
│ ├── d3
│ ├── data.csv
│ ├── diff0-noin.success
│ ├── diff0-stdin1.success
│ ├── diff0-stdin2.success
│ ├── diff0.success
│ ├── diff1-stdin.success
│ ├── diff1.success
│ ├── diff2.success
│ ├── diff3-0.success
│ ├── diff3-1.success
│ ├── diff3-2-stdin1.success
│ ├── diff3-2-stdin2.success
│ ├── diff3-2.success
│ ├── diff3-3.success
│ ├── diff4.success
│ ├── dir-plain.sh
│ ├── f1s
│ ├── f2s
│ ├── f3s
│ ├── f4s
│ ├── f4ss
│ ├── f5s
│ ├── f5ss
│ ├── ff
│ ├── function_bash_tools.success
│ ├── function_dgsh_tools.success
│ ├── grep-Lcap-c-l-matching-lines-nomatch.success
│ ├── grep-Lcap-c-l-matching-lines.success
│ ├── grep-Lcap-c-l-nomatch.success
│ ├── grep-Lcap-c-l.success
│ ├── grep-Lcap-c-matching-lines-l-nomatch.success
│ ├── grep-Lcap-c-matching-lines-l.success
│ ├── grep-Lcap-c-matching-lines-nomatch.success
│ ├── grep-Lcap-c-matching-lines.success
│ ├── grep-Lcap-c-nomatch.success
│ ├── grep-Lcap-c.success
│ ├── grep-Lcap-cat-nomatch.success
│ ├── grep-Lcap-cat.success
│ ├── grep-Lcap-l-c-matching-lines-nomatch.success
│ ├── grep-Lcap-l-c-matching-lines.success
│ ├── grep-Lcap-l-c-nomatch.success
│ ├── grep-Lcap-l-c.success
│ ├── grep-Lcap-l-matching-lines-c-nomatch.success
│ ├── grep-Lcap-l-matching-lines-c.success
│ ├── grep-Lcap-l-matching-lines-nomatch.success
│ ├── grep-Lcap-l-matching-lines.success
│ ├── grep-Lcap-l-nomatch.success
│ ├── grep-Lcap-l.success
│ ├── grep-Lcap-matching-lines-c-l-nomatch.success
│ ├── grep-Lcap-matching-lines-c-l.success
│ ├── grep-Lcap-matching-lines-c-nomatch.success
│ ├── grep-Lcap-matching-lines-c.success
│ ├── grep-Lcap-matching-lines-l-c-nomatch.success
│ ├── grep-Lcap-matching-lines-l-c.success
│ ├── grep-Lcap-matching-lines-l-nomatch.success
│ ├── grep-Lcap-matching-lines-l.success
│ ├── grep-Lcap-matching-lines-nomatch.success
│ ├── grep-Lcap-matching-lines.success
│ ├── grep-Lcap-nomatch.success
│ ├── grep-Lcap.success
│ ├── grep-c-Lcap-l-matching-lines-nomatch.success
│ ├── grep-c-Lcap-l-matching-lines.success
│ ├── grep-c-Lcap-l-nomatch.success
│ ├── grep-c-Lcap-l.success
│ ├── grep-c-Lcap-matching-lines-l-nomatch.success
│ ├── grep-c-Lcap-matching-lines-l.success
│ ├── grep-c-Lcap-matching-lines-nomatch.success
│ ├── grep-c-Lcap-matching-lines.success
│ ├── grep-c-Lcap-nomatch.success
│ ├── grep-c-Lcap.success
│ ├── grep-c-cat.success
│ ├── grep-c-l-Lcap-matching-lines-nomatch.success
│ ├── grep-c-l-Lcap-matching-lines.success
│ ├── grep-c-l-Lcap-nomatch.success
│ ├── grep-c-l-Lcap.success
│ ├── grep-c-l-matching-lines-Lcap-nomatch.success
│ ├── grep-c-l-matching-lines-Lcap.success
│ ├── grep-c-l-matching-lines.success
│ ├── grep-c-l.success
│ ├── grep-c-matching-lines-Lcap-l-nomatch.success
│ ├── grep-c-matching-lines-Lcap-l.success
│ ├── grep-c-matching-lines-Lcap-nomatch.success
│ ├── grep-c-matching-lines-Lcap.success
│ ├── grep-c-matching-lines-l-Lcap-nomatch.success
│ ├── grep-c-matching-lines-l-Lcap.success
│ ├── grep-c-matching-lines-l.success
│ ├── grep-c-matching-lines.success
│ ├── grep-c.success
│ ├── grep-f-cat.success
│ ├── grep-f.success
│ ├── grep-l-Lcap-c-matching-lines-nomatch.success
│ ├── grep-l-Lcap-c-matching-lines.success
│ ├── grep-l-Lcap-c-nomatch.success
│ ├── grep-l-Lcap-c.success
│ ├── grep-l-Lcap-matching-lines-c-nomatch.success
│ ├── grep-l-Lcap-matching-lines-c.success
│ ├── grep-l-Lcap-matching-lines-nomatch.success
│ ├── grep-l-Lcap-matching-lines.success
│ ├── grep-l-Lcap-nomatch.success
│ ├── grep-l-Lcap.success
│ ├── grep-l-c-Lcap-matching-lines-nomatch.success
│ ├── grep-l-c-Lcap-matching-lines.success
│ ├── grep-l-c-Lcap-nomatch.success
│ ├── grep-l-c-Lcap.success
│ ├── grep-l-c-matching-lines-Lcap-nomatch.success
│ ├── grep-l-c-matching-lines-Lcap.success
│ ├── grep-l-c-matching-lines.success
│ ├── grep-l-c.success
│ ├── grep-l-cat.success
│ ├── grep-l-matching-lines-Lcap-c-nomatch.success
│ ├── grep-l-matching-lines-Lcap-c.success
│ ├── grep-l-matching-lines-Lcap-nomatch.success
│ ├── grep-l-matching-lines-Lcap.success
│ ├── grep-l-matching-lines-c-Lcap-nomatch.success
│ ├── grep-l-matching-lines-c-Lcap.success
│ ├── grep-l-matching-lines-c.success
│ ├── grep-l-matching-lines.success
│ ├── grep-l.success
│ ├── grep-matching-lines-Lcap-c-l-nomatch.success
│ ├── grep-matching-lines-Lcap-c-l.success
│ ├── grep-matching-lines-Lcap-c-nomatch.success
│ ├── grep-matching-lines-Lcap-c.success
│ ├── grep-matching-lines-Lcap-l-c-nomatch.success
│ ├── grep-matching-lines-Lcap-l-c.success
│ ├── grep-matching-lines-Lcap-l-nomatch.success
│ ├── grep-matching-lines-Lcap-l.success
│ ├── grep-matching-lines-Lcap-nomatch.success
│ ├── grep-matching-lines-Lcap.success
│ ├── grep-matching-lines-c-Lcap-l-nomatch.success
│ ├── grep-matching-lines-c-Lcap-l.success
│ ├── grep-matching-lines-c-Lcap-nomatch.success
│ ├── grep-matching-lines-c-Lcap.success
│ ├── grep-matching-lines-c-l-Lcap-nomatch.success
│ ├── grep-matching-lines-c-l-Lcap.success
│ ├── grep-matching-lines-c-l.success
│ ├── grep-matching-lines-c.success
│ ├── grep-matching-lines-cat.success
│ ├── grep-matching-lines-l-Lcap-c-nomatch.success
│ ├── grep-matching-lines-l-Lcap-c.success
│ ├── grep-matching-lines-l-Lcap-nomatch.success
│ ├── grep-matching-lines-l-Lcap.success
│ ├── grep-matching-lines-l-c-Lcap-nomatch.success
│ ├── grep-matching-lines-l-c-Lcap.success
│ ├── grep-matching-lines-l-c.success
│ ├── grep-matching-lines-l.success
│ ├── grep-matching-lines.success
│ ├── grep-noargs-cat.success
│ ├── grep-noargs.success
│ ├── grep-o-cat.success
│ ├── grep-o.success
│ ├── grep-v-cat.success
│ ├── grep-v.success
│ ├── grep_comm.success
│ ├── group.success
│ ├── hello
│ ├── j2
│ ├── join_sort.success
│ ├── join_sort_diff.success
│ ├── last
│ ├── multipipe_one_last.success
│ ├── multipipe_one_start.success
│ ├── nondgsh.success
│ ├── p1
│ ├── paste_diff.success
│ ├── read_while.success
│ ├── recursive_multipipe_oneline_end.success
│ ├── recursive_multipipe_oneline_start.success
│ ├── results
│ ├── secho_paste.success
│ ├── secho_secho_fgrep.success
│ ├── sort_sort_comm.success
│ ├── sort_sort_comm_paste_join_diff.success
│ ├── subshell.success
│ ├── tee-copy_diff_comm.success
│ ├── top
│ ├── world
│ └── wrap-cat_comm_sort.success
├── unix-tools/
│ ├── .gitignore
│ ├── Makefile
│ ├── Readme.md
│ ├── cat.sh
│ ├── cmp.sh
│ ├── cpow.c
│ ├── cygwin-sys-select-patch.sh
│ ├── diff.sh
│ ├── diff3.sh
│ ├── echo_echo_dgsh-tee.sh
│ ├── install-wrapped.sh
│ ├── run_all_simple_tests.sh
│ ├── run_simple_test.sh
│ ├── run_test.sh
│ ├── tee.sh
│ ├── test-compat.sh
│ ├── wrapped-commands-posix
│ └── wrapped-commands-tests
└── web/
├── format-eg.sh
├── format-syntax.sh
└── index.html
SYMBOL INDEX (562 symbols across 33 files)
FILE: core-tools/src/dgsh-conc.c
type timespec (line 46) | struct timespec
function usage (line 52) | static void
type portinfo (line 67) | struct portinfo {
function STATIC (line 107) | STATIC int
function STATIC (line 151) | STATIC bool
function STATIC (line 166) | STATIC int
function STATIC (line 221) | STATIC void
function STATIC (line 249) | STATIC int
function STATIC (line 455) | STATIC void
function STATIC (line 490) | STATIC void
function main (line 525) | int
FILE: core-tools/src/dgsh-enumerate.c
function main (line 29) | int
FILE: core-tools/src/dgsh-fft-input.c
function main (line 11) | int main(int argc, char **argv)
FILE: core-tools/src/dgsh-httpval.c
function usage (line 75) | static void
type query (line 88) | struct query {
function main (line 98) | int
function http_serve (line 230) | static void
function send_error (line 343) | static void
function send_headers (line 357) | static void
function strdecode (line 411) | static void
function hexit (line 424) | static int
FILE: core-tools/src/dgsh-monitor.c
function usage (line 36) | static void
function escape (line 44) | static void
function main (line 64) | int
FILE: core-tools/src/dgsh-pecho.c
function main (line 6) | int
FILE: core-tools/src/dgsh-readval.c
function usage (line 40) | static void
function main (line 55) | int
FILE: core-tools/src/dgsh-tee.c
type pool_buffer (line 60) | struct pool_buffer {
type buffer_pool (line 73) | struct buffer_pool {
type buffer_pool (line 91) | struct buffer_pool
type buffer_pool (line 94) | struct buffer_pool
type buffer_pool (line 96) | struct buffer_pool
type buffer_pool (line 96) | struct buffer_pool
type io_buffer (line 116) | struct io_buffer {
type sink_info (line 154) | struct sink_info {
type sink_info (line 167) | struct sink_info
type sink_info (line 170) | struct sink_info
type sink_info (line 172) | struct sink_info
type sink_info (line 172) | struct sink_info
type source_info (line 182) | struct source_info {
type source_info (line 211) | struct source_info
type source_info (line 214) | struct source_info
type source_info (line 216) | struct source_info
type source_info (line 216) | struct source_info
type state (line 254) | enum state {
function memory_pool_size (line 266) | static unsigned long
function page_out (line 273) | static void
function allocate_pool_buffer (line 328) | static bool
function page_in (line 349) | static void
function memory_allocate (line 386) | static bool
function buffer_file_free (line 436) | static void
function memory_free (line 454) | static void
function source_buffer (line 497) | static bool
function sink_buffer (line 519) | static struct io_buffer
type buffer_pool (line 544) | struct buffer_pool
function sink_buffer_length (line 557) | static size_t
type read_result (line 570) | enum read_result {
function source_read (line 581) | static enum read_result
function allocate_data_to_sinks (line 611) | static void
function sink_write (line 761) | static size_t
function usage (line 832) | static void
function non_block (line 860) | static void
function show_select_args (line 874) | static void
function show_state (line 899) | static void
function parse_size (line 927) | static unsigned long
function parse_permute (line 957) | static void
type source_info (line 986) | struct source_info
type source_info (line 987) | struct source_info
type source_info (line 990) | struct source_info
function memory_stats (line 1009) | static void
function first_in_group (line 1029) | static bool
function last_in_group (line 1041) | static bool
type list (line 1047) | struct list {
function list_transpose (line 1063) | static void
function chain_io_files (line 1126) | static void
function main (line 1194) | int
FILE: core-tools/src/dgsh-w.c
function read_number (line 17) | void
function write_number (line 42) | void
function main (line 59) | int
FILE: core-tools/src/dgsh-wrap.c
function usage (line 48) | static void
function remove_from_path (line 97) | static void
function dump_args (line 136) | static void
function split_argv (line 167) | static void
function remove_os_script_path (line 219) | static void
function remove_absolute_path (line 248) | static void
function process_embedded_io_arg (line 262) | static bool
function process_standalone_io_arg (line 283) | static void
function increment_channels (line 298) | static void
function main (line 309) | int
FILE: core-tools/src/dgsh-writeval.c
type timeval (line 75) | struct timeval
type buffer (line 89) | struct buffer {
type buffer (line 100) | struct buffer
type buffer (line 103) | struct buffer
type dpointer (line 106) | struct dpointer {
type dpointer (line 112) | struct dpointer
type client (line 115) | struct client {
type client (line 131) | struct client
function dpointer_increment (line 141) | static bool
function dpointer_decrement (line 157) | static bool
function dpointer_add (line 180) | static bool
function dpointer_subtract (line 205) | static bool
function dpointer_move_back (line 233) | static bool
function dpointer_move_forward (line 260) | static bool
type buffer (line 282) | struct buffer
type buffer (line 283) | struct buffer
type buffer (line 283) | struct buffer
type buffer (line 285) | struct buffer
function update_oldest_buffer (line 303) | static void
function free_unused_buffers_by_position (line 317) | static void
function free_unused_buffers_by_time (line 338) | static void
function content_length (line 371) | static unsigned int
function update_current_record_by_rt_number (line 393) | static void
function update_current_record_by_rl_number (line 420) | static void
function update_current_record_by_rt_time (line 449) | static void
function update_current_record_by_rl_time (line 484) | static void
function dump_buffer_times (line 530) | static void
function timestamp (line 553) | static void
function update_current_record (line 573) | static void
function read_command (line 656) | static void
function write_record (line 709) | static void
function set_buffer_counters (line 784) | void
function buffer_read (line 812) | static void
function non_block (line 884) | static void
type client (line 895) | struct client
function usage (line 906) | static void
function parse_double (line 929) | static double
function double_to_timeval (line 947) | struct timeval
function parse_arguments (line 958) | static void
function handle_events (line 1049) | static void
function main (line 1218) | int
FILE: core-tools/src/kvstore.c
function write_command (line 42) | static int
function dgsh_send_command (line 85) | void
FILE: core-tools/src/negotiate.c
type timespec (line 47) | struct timespec
type dgsh_edge (line 56) | struct dgsh_edge {
type dgsh_node (line 75) | struct dgsh_node {
type dgsh_node_connections (line 90) | struct dgsh_node_connections {
type dgsh_node_pipe_fds (line 116) | struct dgsh_node_pipe_fds {
type dgsh_negotiation (line 133) | struct dgsh_negotiation
type dgsh_node (line 134) | struct dgsh_node
type node_io_side (line 138) | struct node_io_side
type dgsh_node_pipe_fds (line 139) | struct dgsh_node_pipe_fds
function dgsh_force_include_function (line 151) | void
function dgsh_exit_handler (line 159) | static void
function dgsh_alarm_handler (line 176) | void
function install_exit_handler (line 191) | __attribute__((constructor))
function setup_iov_max (line 202) | __attribute__((constructor))
function STATIC (line 220) | STATIC void
function op_result (line 272) | op_result
function op_result (line 369) | op_result
function op_result (line 402) | op_result
function op_result (line 452) | op_result
function satisfy_io_constraints (line 486) | static enum op_result
function dry_match_io_constraints (line 535) | static enum op_result
function free_graph_solution (line 595) | static enum op_result
function record_move_flexible (line 618) | static enum op_result
function record_move_unbalanced (line 640) | static enum op_result
function move (line 667) | static enum op_result
function cross_match_io_constraints (line 743) | static enum op_result
type dgsh_conc (line 854) | struct dgsh_conc
type dgsh_negotiation (line 855) | struct dgsh_negotiation
type dgsh_conc (line 858) | struct dgsh_conc
function calculate_conc_fds (line 870) | static enum op_result
function prepare_solution (line 965) | static enum op_result
function print_solution_error (line 1014) | static void
function check_constraints_matched (line 1040) | static void
function cross_match_constraints (line 1073) | static enum op_result
function node_match_constraints (line 1158) | static enum op_result
function solve_graph (line 1221) | enum op_result
function establish_io_connections (line 1307) | static enum op_result
function write_output_fds (line 1380) | static enum op_result
function write_piece (line 1441) | static int
function get_struct_size (line 1456) | static int
function do_write (line 1472) | static int
function write_concs (line 1537) | static enum op_result
function write_graph_solution (line 1570) | static enum op_result
function set_dispatcher (line 1625) | static void
function write_message_block (line 1643) | enum op_result
function add_node (line 1713) | static enum op_result
function lookup_dgsh_edge (line 1737) | static enum op_result
function fill_dgsh_edge (line 1758) | static enum op_result
function add_edge (line 1807) | static enum op_result
function try_add_dgsh_edge (line 1828) | static enum op_result
function fill_node (line 1847) | static void
function try_add_dgsh_node (line 1882) | static enum op_result
function free_conc_array (line 1905) | static void
function free_mb (line 1916) | void
function register_node_edge (line 1931) | static enum op_result
function analyse_read (line 1953) | static enum op_result
function check_read (line 1988) | static enum op_result
function alloc_copy_proc_pids (line 2003) | static enum op_result
function alloc_copy_concs (line 2015) | static enum op_result
function alloc_copy_graph_solution (line 2028) | static enum op_result
function alloc_copy_edges (line 2042) | static enum op_result
function alloc_copy_nodes (line 2055) | static enum op_result
function alloc_copy_mb (line 2069) | static enum op_result
function call_read (line 2089) | static enum op_result
function read_chunk (line 2106) | static enum op_result
function alloc_fds (line 2167) | static enum op_result
function alloc_io_fds (line 2179) | static enum op_result
function pid_t (line 2213) | pid_t
function get_expected_fds_n (line 2230) | int
function get_provided_fds_n (line 2258) | int
function write_fd (line 2286) | void
function read_fd (line 2313) | int
function read_input_fds (line 2349) | static enum op_result
function read_concs (line 2391) | static enum op_result
function read_graph_solution (line 2426) | static enum op_result
function read_message_block (line 2503) | enum op_result
function construct_message_block (line 2567) | enum op_result
function get_env_var (line 2598) | static void
function get_environment_vars (line 2617) | static void
function op_result (line 2632) | op_result
function set_fds (line 2650) | static int
function set_negotiation_complete (line 2694) | void
function setup_file_descriptors (line 2700) | static int
function dgsh_exit (line 2729) | static int
type prot_state (line 2753) | enum prot_state
function dgsh_negotiate (line 2782) | int
FILE: core-tools/src/negotiate.h
type prot_state (line 31) | enum prot_state {
type cmsghdr (line 41) | struct cmsghdr
type op_result (line 49) | enum op_result {
type dgsh_edge (line 78) | struct dgsh_edge {
type dgsh_negotiation (line 98) | struct dgsh_negotiation
type node_io_side (line 101) | struct node_io_side {
type dgsh_conc (line 109) | struct dgsh_conc {
type dgsh_negotiation (line 120) | struct dgsh_negotiation {
type op_result (line 166) | enum op_result
type op_result (line 167) | enum op_result
type dgsh_conc (line 168) | struct dgsh_conc
type dgsh_negotiation (line 168) | struct dgsh_negotiation
type dgsh_negotiation (line 169) | struct dgsh_negotiation
type dgsh_negotiation (line 170) | struct dgsh_negotiation
type dgsh_negotiation (line 171) | struct dgsh_negotiation
type op_result (line 172) | enum op_result
type dgsh_negotiation (line 173) | struct dgsh_negotiation
type op_result (line 174) | enum op_result
type dgsh_negotiation (line 175) | struct dgsh_negotiation
FILE: core-tools/tests-regression/code-metrics/in/date/date.c
function main (line 72) | int
function setthetime (line 181) | static void
function badformat (line 288) | static void
function usage (line 295) | static void
FILE: core-tools/tests-regression/code-metrics/in/date/netdate.c
function netsettime (line 67) | int
FILE: core-tools/tests-regression/code-metrics/in/date/vary.c
type trans (line 36) | struct trans {
type trans (line 41) | struct trans
type trans (line 48) | struct trans
type tm (line 55) | struct tm
function domktime (line 57) | static int
function trans (line 69) | static int
type vary (line 82) | struct vary
type vary (line 83) | struct vary
type vary (line 85) | struct vary
type vary (line 95) | struct vary
type vary (line 95) | struct vary
function daysinmonth (line 104) | static int
function adjyear (line 127) | static int
function adjmon (line 148) | static int
function adjday (line 208) | static int
function adjwday (line 252) | static int
function adjhour (line 288) | static int
function adjmin (line 333) | static int
function adjsec (line 374) | static int
type vary (line 415) | struct vary
type vary (line 416) | struct vary
type tm (line 416) | struct tm
function vary_destroy (line 496) | void
FILE: core-tools/tests-regression/code-metrics/in/date/vary.h
type vary (line 29) | struct vary {
type vary (line 34) | struct vary
type vary (line 34) | struct vary
type vary (line 35) | struct vary
type vary (line 35) | struct vary
type tm (line 35) | struct tm
type vary (line 36) | struct vary
FILE: core-tools/tests-regression/map-hierarchy/in/a/date/date.c
function main (line 63) | int
function setthetime (line 169) | static void
function badformat (line 268) | static void
function usage (line 275) | static void
FILE: core-tools/tests-regression/map-hierarchy/in/a/date/vary.c
type trans (line 34) | struct trans {
type trans (line 39) | struct trans
type trans (line 46) | struct trans
type tm (line 53) | struct tm
function domktime (line 55) | static int
function trans (line 67) | static int
type vary (line 80) | struct vary
type vary (line 81) | struct vary
type vary (line 83) | struct vary
type vary (line 93) | struct vary
type vary (line 93) | struct vary
function daysinmonth (line 102) | static int
function adjyear (line 125) | static int
function adjmon (line 146) | static int
function adjday (line 206) | static int
function adjwday (line 250) | static int
function adjhour (line 286) | static int
function adjmin (line 331) | static int
function adjsec (line 372) | static int
type vary (line 413) | struct vary
type vary (line 414) | struct vary
type tm (line 414) | struct tm
function vary_destroy (line 494) | void
FILE: core-tools/tests-regression/map-hierarchy/in/a/date/vary.h
type vary (line 29) | struct vary {
type vary (line 34) | struct vary
type vary (line 34) | struct vary
type vary (line 35) | struct vary
type vary (line 35) | struct vary
type tm (line 35) | struct tm
type vary (line 36) | struct vary
FILE: core-tools/tests-regression/map-hierarchy/in/b/bin/date/date.c
function main (line 72) | int
function setthetime (line 181) | static void
function badformat (line 288) | static void
function usage (line 295) | static void
FILE: core-tools/tests-regression/map-hierarchy/in/b/bin/date/headers/vary.h
type vary (line 29) | struct vary {
type vary (line 34) | struct vary
type vary (line 34) | struct vary
type vary (line 35) | struct vary
type vary (line 35) | struct vary
type tm (line 35) | struct tm
type vary (line 36) | struct vary
FILE: core-tools/tests-regression/map-hierarchy/in/b/bin/date/netdate.c
function netsettime (line 67) | int
FILE: core-tools/tests-regression/map-hierarchy/in/b/bin/date/vary.c
type trans (line 36) | struct trans {
type trans (line 41) | struct trans
type trans (line 48) | struct trans
type tm (line 55) | struct tm
function domktime (line 57) | static int
function trans (line 69) | static int
type vary (line 82) | struct vary
type vary (line 83) | struct vary
type vary (line 85) | struct vary
type vary (line 95) | struct vary
type vary (line 95) | struct vary
function daysinmonth (line 104) | static int
function adjyear (line 127) | static int
function adjmon (line 148) | static int
function adjday (line 208) | static int
function adjwday (line 252) | static int
function adjhour (line 288) | static int
function adjmin (line 333) | static int
function adjsec (line 374) | static int
type vary (line 415) | struct vary
type vary (line 416) | struct vary
type tm (line 416) | struct tm
function vary_destroy (line 496) | void
FILE: core-tools/tests-regression/map-hierarchy/out.ok/map-hierarchy/in/b/bin/date/date.c
function main (line 63) | int
function setthetime (line 169) | static void
function badformat (line 268) | static void
function usage (line 275) | static void
FILE: core-tools/tests-regression/map-hierarchy/out.ok/map-hierarchy/in/b/bin/date/headers/vary.h
type vary (line 29) | struct vary {
type vary (line 34) | struct vary
type vary (line 34) | struct vary
type vary (line 35) | struct vary
type vary (line 35) | struct vary
type tm (line 35) | struct tm
type vary (line 36) | struct vary
FILE: core-tools/tests-regression/map-hierarchy/out.ok/map-hierarchy/in/b/bin/date/vary.c
type trans (line 34) | struct trans {
type trans (line 39) | struct trans
type trans (line 46) | struct trans
type tm (line 53) | struct tm
function domktime (line 55) | static int
function trans (line 67) | static int
type vary (line 80) | struct vary
type vary (line 81) | struct vary
type vary (line 83) | struct vary
type vary (line 93) | struct vary
type vary (line 93) | struct vary
function daysinmonth (line 102) | static int
function adjyear (line 125) | static int
function adjmon (line 146) | static int
function adjday (line 206) | static int
function adjwday (line 250) | static int
function adjhour (line 286) | static int
function adjmin (line 331) | static int
function adjsec (line 372) | static int
type vary (line 413) | struct vary
type vary (line 414) | struct vary
type tm (line 414) | struct tm
function vary_destroy (line 494) | void
FILE: core-tools/tests/check_negotiate.c
type dgsh_negotiation (line 18) | struct dgsh_negotiation
type dgsh_edge (line 19) | struct dgsh_edge
type dgsh_edge (line 20) | struct dgsh_edge
function setup_concs (line 30) | void
function setup_graph_solution (line 57) | void
function setup_chosen_mb (line 117) | void
function setup_mb (line 216) | void
function setup_pointers_to_edges (line 309) | void
function setup_self_node (line 325) | void
function setup_self_node_io_side (line 332) | void
function setup_pipe_fds (line 340) | void
function setup_args (line 353) | void setup_args(void)
function setup (line 361) | void
function setup_test_set_fds (line 371) | void
function setup_test_add_node (line 378) | void
function setup_test_lookup_dgsh_edge (line 386) | void
function setup_test_fill_dgsh_edge (line 392) | void
function setup_test_add_edge (line 400) | void
function setup_test_try_add_dgsh_edge (line 406) | void
function setup_test_try_add_dgsh_node (line 414) | void
function setup_test_fill_node (line 421) | void
function setup_test_free_mb (line 429) | void
function setup_test_analyse_read (line 435) | void
function setup_test_alloc_copy_graph_solution (line 452) | void
function setup_test_alloc_copy_concs (line 458) | void
function setup_test_alloc_copy_edges (line 464) | void
function setup_test_alloc_copy_nodes (line 470) | void
function setup_test_read_chunk (line 476) | void
function setup_test_alloc_io_fds (line 482) | void
function setup_test_get_provided_fds_n (line 490) | void
function setup_test_get_expected_fds_n (line 497) | void
function setup_test_get_origin_pid (line 504) | void
function setup_test_read_input_fds (line 510) | void
function setup_test_read_graph_solution (line 518) | void
function setup_test_read_concs (line 526) | void
function setup_test_write_graph_solution (line 535) | void
function setup_test_write_concs (line 543) | void
function setup_test_read_message_block (line 551) | void
function setup_test_write_message_block (line 558) | void
function setup_test_make_compact_edge_array (line 565) | void
function setup_test_reallocate_edge_pointer_array (line 571) | void
function setup_test_move (line 591) | void
function setup_test_satisfy_io_constraints (line 597) | void
function setup_test_dry_match_io_constraints (line 605) | void
function setup_test_node_match_constraints (line 614) | void
function setup_test_free_graph_solution (line 620) | void
function setup_test_solve_graph (line 627) | void
function setup_test_calculate_conc_fds (line 636) | void
function setup_test_write_output_fds (line 644) | void
function setup_test_set_dispatcher (line 652) | void
function setup_test_establish_io_connections (line 659) | void
function setup_pi (line 667) | void setup_pi(void)
function setup_test_is_ready (line 681) | void
function setup_test_set_io_channels (line 688) | void
function retire_pointers_to_edges (line 695) | void
function retire_graph_solution (line 704) | void
function retire_concs (line 718) | void retire_concs(struct dgsh_negotiation *mb)
function retire_chosen_mb (line 726) | void
function retire_mb (line 734) | void
function retire_pipe_fds (line 743) | void
function retire_args (line 753) | void
function retire (line 759) | void
function retire_test_set_fds (line 768) | void
function retire_test_construct_message_block (line 774) | void
function retire_test_add_node (line 780) | void
function retire_test_lookup_dgsh_edge (line 786) | void
function retire_test_fill_dgsh_edge (line 792) | void
function retire_test_add_edge (line 798) | void
function retire_test_try_add_dgsh_edge (line 804) | void
function retire_test_try_add_dgsh_node (line 810) | void
function retire_test_analyse_read (line 816) | void
function retire_test_alloc_copy_graph_solution (line 831) | void
function retire_test_alloc_copy_concs (line 837) | void
function retire_test_alloc_copy_edges (line 843) | void
function retire_test_alloc_copy_nodes (line 849) | void
function retire_test_alloc_io_fds (line 855) | void
function retire_test_get_provided_fds_n (line 864) | void
function retire_test_get_expected_fds_n (line 872) | void
function retire_test_read_input_fds (line 880) | void
function retire_test_get_origin_pid (line 888) | void
function retire_test_read_message_block (line 894) | void
function retire_test_write_message_block (line 900) | void
function retire_test_read_graph_solution (line 906) | void
function retire_test_read_concs (line 913) | void
function retire_test_write_graph_solution (line 921) | void
function retire_test_write_concs (line 929) | void
function retire_test_make_compact_edge_array (line 936) | void
function retire_test_reallocate_edge_pointer_array (line 943) | void
function retire_test_move (line 963) | void
function retire_test_satisfy_io_constraints (line 969) | void
function retire_test_dry_match_io_constraints (line 977) | void
function retire_test_node_match_constraints (line 987) | void
function retire_test_free_graph_solution (line 995) | void
function retire_test_solve_graph (line 1001) | void
function retire_test_calculate_conc_fds (line 1016) | void
function retire_test_write_output_fds (line 1025) | void
function retire_test_set_dispatcher (line 1032) | void
function retire_test_establish_io_connections (line 1038) | void
function retire_pi (line 1046) | void
function retire_test_is_ready (line 1052) | void
function retire_test_set_io_channels (line 1059) | void
function START_TEST (line 1067) | START_TEST(test_solve_graph)
function END_TEST (line 1130) | END_TEST
function END_TEST (line 1154) | END_TEST
function END_TEST (line 1161) | END_TEST
type dgsh_edge (line 1211) | struct dgsh_edge
function retire_dmic (line 1214) | void
function START_TEST (line 1224) | START_TEST(test_node_match_constraints)
function END_TEST (line 1257) | END_TEST
function END_TEST (line 1299) | END_TEST
function END_TEST (line 1355) | END_TEST
function END_TEST (line 1373) | END_TEST
function END_TEST (line 1426) | END_TEST
function END_TEST (line 1470) | END_TEST
function END_TEST (line 1483) | END_TEST
function END_TEST (line 1499) | END_TEST
function END_TEST (line 1520) | END_TEST
function END_TEST (line 1528) | END_TEST
function END_TEST (line 1544) | END_TEST
function END_TEST (line 1586) | END_TEST
function END_TEST (line 1658) | END_TEST
function END_TEST (line 1726) | END_TEST
function END_TEST (line 1803) | END_TEST
function END_TEST (line 1888) | END_TEST
function END_TEST (line 1928) | END_TEST
function END_TEST (line 1942) | END_TEST
function END_TEST (line 1953) | END_TEST
function END_TEST (line 1963) | END_TEST
function END_TEST (line 1978) | END_TEST
function END_TEST (line 1994) | END_TEST
function END_TEST (line 2052) | END_TEST
function END_TEST (line 2148) | END_TEST
function END_TEST (line 2187) | END_TEST
function END_TEST (line 2215) | END_TEST
function END_TEST (line 2230) | END_TEST
function END_TEST (line 2251) | END_TEST
function END_TEST (line 2270) | END_TEST
function END_TEST (line 2284) | END_TEST
function END_TEST (line 2298) | END_TEST
function END_TEST (line 2320) | END_TEST
function END_TEST (line 2328) | END_TEST
function END_TEST (line 2562) | END_TEST
function END_TEST (line 2568) | END_TEST
function END_TEST (line 2585) | END_TEST
function END_TEST (line 2603) | END_TEST
function END_TEST (line 2647) | END_TEST
function END_TEST (line 2658) | END_TEST
function END_TEST (line 2684) | END_TEST
function END_TEST (line 2694) | END_TEST
function END_TEST (line 2709) | END_TEST
function END_TEST (line 2726) | END_TEST
function END_TEST (line 2741) | END_TEST
function END_TEST (line 2755) | END_TEST
function END_TEST (line 2785) | END_TEST
function END_TEST (line 2802) | END_TEST
function END_TEST (line 2813) | END_TEST
function END_TEST (line 2828) | END_TEST
function END_TEST (line 2868) | END_TEST
function END_TEST (line 2906) | END_TEST
function Suite (line 2980) | Suite *
function Suite (line 3088) | Suite *
function Suite (line 3245) | Suite *
function run_suite (line 3269) | int run_suite(Suite *s)
function run_suite_connect (line 3279) | int
function run_suite_solve (line 3286) | int
function run_suite_broadcast (line 3293) | int
function run_suite_conc (line 3300) | int
function main (line 3308) | int main()
FILE: eval/TextProperties.java
class TextProperties (line 34) | class TextProperties {
method increment (line 37) | static void increment(Map<String, Integer> map, String member) {
method ngramIncrement (line 47) | static void ngramIncrement(Map<String, Integer> map, StringBuffer word...
method sortedList (line 53) | static void sortedList(Map<String, Integer> map, String fileName) {
method main (line 84) | public static void main(String args[]) {
FILE: eval/WebStats.java
class WebStats (line 37) | class WebStats {
method header (line 40) | static void header(String s) {
method add (line 49) | static void add(Map<String, Integer> map, String member, int value) {
method list (line 59) | static void list(String title, Map<String, Integer> map) {
method sortedList (line 66) | static void sortedList(String title, Map<String, Integer> map, int n) {
method main (line 93) | public static void main(String args[]) {
FILE: simple-shell/simple-shell.py
function debug (line 12) | def debug(s):
class Process (line 17) | class Process:
method __init__ (line 20) | def __init__(self, command):
method selectInputFileDescriptor (line 26) | def selectInputFileDescriptor(self):
method selectOutputFileDescriptor (line 39) | def selectOutputFileDescriptor(self):
function setupProcess (line 52) | def setupProcess(index, channel, connector):
function parse (line 65) | def parse(command):
FILE: unix-tools/cpow.c
function cpow (line 35) | double complex
Condensed preview — 460 files, each showing path, character count, and a content snippet. Download the .json file or copy for the full structured content (1,218K chars).
[
{
"path": ".gitignore",
"chars": 113,
"preview": "*.dot\n*.exe\n*.o\n*.out\n*.outb\n*.pdf\n*.swp\n*.test\n.deps\n.config\n.gdb_history\n/build\n/png/*.png\ntags\nunix-tools/sf*\n"
},
{
"path": ".gitmodules",
"chars": 403,
"preview": "[submodule \"unix-tools/coreutils\"]\n\tpath = unix-tools/coreutils\n\turl = https://github.com/mfragkoulis/coreutils\n\tbranch "
},
{
"path": ".travis/linux-ubuntu-trusty.install.sh",
"chars": 1161,
"preview": "#!/bin/bash\n\nsudo apt-get -qq update\n # For installation\nsudo apt-get install -y make automake gcc libtool pkg-config t"
},
{
"path": ".travis/macosx-xcode8.3.install.sh",
"chars": 1164,
"preview": "#!/bin/bash\n\nset -x\n\nbrew update\nbrew install autoconf check xz texinfo help2man gettext libelf\nexport PATH=\"/usr/local/"
},
{
"path": ".travis.yml",
"chars": 402,
"preview": "matrix:\n include:\n - os: linux\n dist: trusty\n install:\n "
},
{
"path": "CITATION.cff",
"chars": 1019,
"preview": "cff-version: 1.2.0\r\ntitle: dgsh: The Directed Graph Shell\r\nmessage: >-\r\n If you use this software, please cite, using t"
},
{
"path": "LICENSE",
"chars": 3324,
"preview": " Copyright 2012-2013 Diomidis Spinellis\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may"
},
{
"path": "Makefile",
"chars": 5183,
"preview": "#\n# Copyright 2012-2013 Diomidis Spinellis\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you "
},
{
"path": "README.md",
"chars": 1012,
"preview": "## dgsh: The Directed Graph Shell\n\n[](https://tr"
},
{
"path": "core-tools/.gitignore",
"chars": 136,
"preview": "Makefile\nMakefile.in\naclocal.m4\nautom4te.cache/\nbuild-aux/\nconfig.h\nconfig.h.in\nconfig.log\nconfig.status\nconfigure\nlibto"
},
{
"path": "core-tools/Makefile.am",
"chars": 104,
"preview": "## Process this file with automake to produce Makefile.in\nACLOCAL_AMFLAGS = -I m4\nSUBDIRS = src . tests\n"
},
{
"path": "core-tools/configure.ac",
"chars": 1380,
"preview": "# Process this file with autoconf to produce a configure script.\n\n# Prelude.\nAC_PREREQ([2.59])\nAC_INIT([DGSH Negotiate],"
},
{
"path": "core-tools/src/.gitignore",
"chars": 408,
"preview": "dgsh-conc\ndgsh-conc.html\ndgsh-enumerate\ndgsh-enumerate.html\ndgsh-fft-input\ndgsh.html\ndgsh-httpval\ndgsh-httpval.html\ndgsh"
},
{
"path": "core-tools/src/Makefile.am",
"chars": 1822,
"preview": "include ../../.config\n\nif LINUX\nDGSH_ASSEMBLY_FILE=dgsh-elf.s\nelse\nif DARWIN\nDGSH_ASSEMBLY_FILE=dgsh-macho.s\nendif\nendif"
},
{
"path": "core-tools/src/debug.h",
"chars": 895,
"preview": "/*\n * Copyright 2013-2017 Diomidis Spinellis\n *\n * Debug macros\n *\n * Licensed under the Apache License, Version 2.0 (th"
},
{
"path": "core-tools/src/dgsh-conc.1",
"chars": 1639,
"preview": ".TH DGSH-HTTPVAL 1 \"14 July 2016\"\n.\\\"\n.\\\" (C) Copyright 2016 Diomidis Spinellis. All rights reserved.\n.\\\"\n.\\\" Licensed"
},
{
"path": "core-tools/src/dgsh-conc.c",
"chars": 15929,
"preview": "/*\n * Copyright 2016, 2017 Diomidis Spinellis\n *\n * A passive component that aids the dgsh negotiation by passing\n * mes"
},
{
"path": "core-tools/src/dgsh-debug.h",
"chars": 926,
"preview": "/*\n * Copyright 2013-2017 Diomidis Spinellis\n *\n * Debug macros\n *\n * Licensed under the Apache License, Version 2.0 (th"
},
{
"path": "core-tools/src/dgsh-elf.s",
"chars": 625,
"preview": "/*\n * ELF note header to mark dgsh-compatible programs\n * See http://www.netbsd.org/docs/kernel/elf-notes.html\n * Don't "
},
{
"path": "core-tools/src/dgsh-enumerate.1",
"chars": 1621,
"preview": ".TH DGSH-ENUMERATE 1 \"27 January 2017\"\n.\\\"\n.\\\" (C) Copyright 2017 Diomidis Spinellis. All rights reserved.\n.\\\"\n.\\\" Lic"
},
{
"path": "core-tools/src/dgsh-enumerate.c",
"chars": 1336,
"preview": "/*\n * Copyright 2017 Diomidis Spinellis\n *\n * Enumerate an arbitrary number of output channels.\n *\n * Licensed under the"
},
{
"path": "core-tools/src/dgsh-fft-input.c",
"chars": 1655,
"preview": "#include <assert.h>\t// assert()\n#include <stdio.h>\t// printf\n#include <complex.h>\t// double complex\n#include <unistd.h>\t"
},
{
"path": "core-tools/src/dgsh-httpval.1",
"chars": 4748,
"preview": ".TH DGSH-HTTPVAL 1 \"14 July 2013\"\n.\\\"\n.\\\" (C) Copyright 2013 Diomidis Spinellis. All rights reserved.\n.\\\"\n.\\\" Licensed"
},
{
"path": "core-tools/src/dgsh-httpval.c",
"chars": 11710,
"preview": "/*-\n *\n * Provide HTTP access to the dgsh key-value store.\n *\n * Based on micro_httpd - really small HTTP server heavily"
},
{
"path": "core-tools/src/dgsh-macho.s",
"chars": 340,
"preview": "# MACHO note header to mark dgsh-compatible programs\n .section \".note.ident\", \"a\"\n .asciz \"DSpinellis/dgsh\"\n .s"
},
{
"path": "core-tools/src/dgsh-merge-sum.1",
"chars": 1501,
"preview": ".TH DGSH-MERGE-SUM 1 \"10 September 2014\"\n.\\\"\n.\\\" (C) Copyright 2014 Diomidis Spinellis. All rights reserved.\n.\\\"\n.\\\" L"
},
{
"path": "core-tools/src/dgsh-merge-sum.pl",
"chars": 2115,
"preview": "#!/usr/bin/env perl\n#\n# Merge sorted (value, key) pairs, summing the values of equal keys\n#\n# Copyright 2014 Diomidis S"
},
{
"path": "core-tools/src/dgsh-monitor.1",
"chars": 1775,
"preview": ".TH DGSH-MONITOR 1 \"11 December 2016\"\n.\\\"\n.\\\" (C) Copyright 2013 Diomidis Spinellis. All rights reserved.\n.\\\"\n.\\\" Lice"
},
{
"path": "core-tools/src/dgsh-monitor.c",
"chars": 2525,
"preview": "/*\n * Copyright 2013 Diomidis Spinellis\n *\n * Prepend lines read with timestamp, number of lines, number of bytes.\n * Us"
},
{
"path": "core-tools/src/dgsh-parallel.1",
"chars": 2646,
"preview": ".TH DGSH-PARALLEL 1 \"15 December 2016\"\n.\\\"\n.\\\" (C) Copyright 2016 Diomidis Spinellis. All rights reserved.\n.\\\"\n.\\\" Lic"
},
{
"path": "core-tools/src/dgsh-parallel.sh",
"chars": 2197,
"preview": "#!/usr/bin/env bash\n#!dgsh\n#\n# Create and execute a semi-homongeneous dgsh parallel processing block\n#\n\n# Remove dgsh fr"
},
{
"path": "core-tools/src/dgsh-pecho.c",
"chars": 601,
"preview": "#include <stdio.h>\t\t/* printf() */\n#include <stdlib.h>\t\t/* exit() */\n#include <unistd.h>\t\t/* getpagesize() */\n#include \""
},
{
"path": "core-tools/src/dgsh-readval.1",
"chars": 2918,
"preview": ".TH DGSH-READVAL 1 \"21 March 2013\"\n.\\\"\n.\\\" (C) Copyright 2013 Diomidis Spinellis. All rights reserved.\n.\\\"\n.\\\" License"
},
{
"path": "core-tools/src/dgsh-readval.c",
"chars": 2731,
"preview": "/*\n * Copyright 2013 Diomidis Spinellis\n *\n * Communicate with the data store specified as a Unix-domain socket.\n * (Use"
},
{
"path": "core-tools/src/dgsh-tee.1",
"chars": 6945,
"preview": ".TH DGSH-TEE 1 \"13 April 2017\"\n.\\\"\n.\\\" (C) Copyright 2013-2017 Diomidis Spinellis. All rights reserved.\n.\\\"\n.\\\" Licens"
},
{
"path": "core-tools/src/dgsh-tee.c",
"chars": 43519,
"preview": "/*\n * Copyright 2013 Diomidis Spinellis\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may"
},
{
"path": "core-tools/src/dgsh-w.c",
"chars": 2796,
"preview": "#include <assert.h>\t// assert()\n#include <math.h>\t// M_PI, pow()\n#include <complex.h>\t// I, cexp(), cpow()\n#include <std"
},
{
"path": "core-tools/src/dgsh-wrap.1",
"chars": 7991,
"preview": ".TH DGSH-WRAP 1 \"18 August 2017\"\n.\\\"\n.\\\" (C) Copyright 2016-2017 Diomidis Spinellis. All rights reserved.\n.\\\"\n.\\\" Lice"
},
{
"path": "core-tools/src/dgsh-wrap.c",
"chars": 14368,
"preview": "/*\n * Copyright 2016-2017 Diomidis Spinellis\n *\n * Wrap any command to participate in the dgsh negotiation\n *\n * License"
},
{
"path": "core-tools/src/dgsh-writeval.1",
"chars": 2920,
"preview": ".TH DGSH-WRITEVAL 1 \"21 March 2013\"\n.\\\"\n.\\\" (C) Copyright 2013 Diomidis Spinellis. All rights reserved.\n.\\\"\n.\\\" Licens"
},
{
"path": "core-tools/src/dgsh-writeval.c",
"chars": 34936,
"preview": "/*\n * Copyright 2013 Diomidis Spinellis\n *\n * Read values from its standard input and make them available to other\n * pr"
},
{
"path": "core-tools/src/dgsh.1",
"chars": 5996,
"preview": ".TH DGSH 1 \"10 August 2017\"\n.\\\"\n.\\\" (C) Copyright 2016-2017 Diomidis Spinellis. All rights reserved.\n.\\\"\n.\\\" Licensed "
},
{
"path": "core-tools/src/dgsh.h",
"chars": 855,
"preview": "/*\n * Copyright 2016-2017 Diomidis Spinellis and Marios Fragkoulis\n *\n * Dgsh public API\n *\n * Licensed under the Apache"
},
{
"path": "core-tools/src/dgsh_negotiate.3",
"chars": 7787,
"preview": ".TH DGSH_NEGOTIATE 3 \"16 February 2017\"\n.\\\"\n.\\\" (C) Copyright 2017 Diomidis Spinellis. All rights reserved.\n.\\\"\n.\\\" Li"
},
{
"path": "core-tools/src/kvstore.c",
"chars": 3675,
"preview": "/*\n * Copyright 2013 Diomidis Spinellis\n *\n * Communicate with the data store specified as a Unix-domain socket.\n * (API"
},
{
"path": "core-tools/src/kvstore.h",
"chars": 1316,
"preview": "/*\n * Copyright 2013 Diomidis Spinellis\n *\n * Communicate with the data store specified as a Unix-domain socket.\n *\n * L"
},
{
"path": "core-tools/src/minmax.h",
"chars": 778,
"preview": "/*\n * Copyright 2013-2017 Diomidis Spinellis\n *\n * MIN and MAX macros\n *\n * Licensed under the Apache License, Version 2"
},
{
"path": "core-tools/src/negotiate.c",
"chars": 90953,
"preview": "/*\n * Copyright 2016, 2017 Marios Fragkoulis\n *\n * A passive component that aids the dgsh negotiation by passing\n * mess"
},
{
"path": "core-tools/src/negotiate.h",
"chars": 5925,
"preview": "/*\n * Copyright 2016-2017 Marios Fragkoulis\n *\n * Dgsh private negotiation API\n *\n * Licensed under the Apache License, "
},
{
"path": "core-tools/src/perm.1",
"chars": 1455,
"preview": ".TH PERM 1 \"13 December 2016\"\n.\\\"\n.\\\" (C) Copyright 2016 Diomidis Spinellis. All rights reserved.\n.\\\"\n.\\\" Licensed und"
},
{
"path": "core-tools/src/perm.sh",
"chars": 316,
"preview": "#!/bin/sh\n#!dgsh\n#\n# Permute inputs to outputs by invoking dgsh-tee -p\n#\n\nusage()\n{\n echo 'Usage: perm n1,n2[, ...]'\n "
},
{
"path": "core-tools/src/run-built-dgsh.sh",
"chars": 385,
"preview": "#!/bin/sh\n#\n# Run any dgsh script from the built (rather than the installed)\n# version of dgsh.\n# The first argument mus"
},
{
"path": "core-tools/tests/.gitignore",
"chars": 100,
"preview": "Makefile.orig\ncheck_negotiate\ncheck_negotiate.log\ncheck_negotiate.trs\ntest-suite.log\nunit-test-dgsh\n"
},
{
"path": "core-tools/tests/Makefile.am",
"chars": 298,
"preview": "## Process this file with automake to produce Makefile.in\n\nTESTS = check_negotiate\ncheck_PROGRAMS = check_negotiate\n\nche"
},
{
"path": "core-tools/tests/Makefile.patch",
"chars": 1582,
"preview": "--- Makefile.original\t2016-06-20 18:17:16.661787072 +0300\n+++ Makefile\t2016-06-20 18:18:39.577783863 +0300\n@@ -686,17 +6"
},
{
"path": "core-tools/tests/check.hack",
"chars": 809,
"preview": "\t# MFG: Hack for identifying and aggregating test results in log.\n\t# Goes in test/negotiate/tests/Makefile\n\t# Grep for '"
},
{
"path": "core-tools/tests/check_negotiate.c",
"chars": 92707,
"preview": "#include <check.h> /* Check unit test framework API. */\n#include <stdlib.h> /* EXIT_SUCCESS, EXIT_FAILURE */\n#include <"
},
{
"path": "core-tools/tests-regression/.gitignore",
"chars": 6,
"preview": "test/\n"
},
{
"path": "core-tools/tests-regression/author-compare/out.ok",
"chars": 160,
"preview": "conf/icse/ papers: 219\njournals/software/ papers: 182\nAuthors only in conf/icse/: 547\nAuthors only in journals/software/"
},
{
"path": "core-tools/tests-regression/bin/gdate",
"chars": 339,
"preview": "#!/bin/sh\n#\n# Poor man's GNU date\n# This implements the GNU date -f - option for converting the date as\n# represented in"
},
{
"path": "core-tools/tests-regression/code-metrics/in/date/date.c",
"chars": 7556,
"preview": "/*-\n * Copyright (c) 1985, 1987, 1988, 1993\n *\tThe Regents of the University of California. All rights reserved.\n *\n * "
},
{
"path": "core-tools/tests-regression/code-metrics/in/date/extern.h",
"chars": 1739,
"preview": "/*-\n * Copyright (c) 1993\n *\tThe Regents of the University of California. All rights reserved.\n *\n * Redistribution and"
},
{
"path": "core-tools/tests-regression/code-metrics/in/date/netdate.c",
"chars": 5357,
"preview": "/*-\n * Copyright (c) 1990, 1993\n *\tThe Regents of the University of California. All rights reserved.\n *\n * Redistributi"
},
{
"path": "core-tools/tests-regression/code-metrics/in/date/vary.c",
"chars": 11246,
"preview": "/*-\n * Copyright (c) 1997 Brian Somers <brian@Awfulhak.org>\n * All rights reserved.\n *\n * Redistribution and use in sour"
},
{
"path": "core-tools/tests-regression/code-metrics/in/date/vary.h",
"chars": 1688,
"preview": "/*-\n * Copyright (c) 1997 Brian Somers <brian@Awfulhak.org>\n * All rights reserved.\n *\n * Redistribution and use in sour"
},
{
"path": "core-tools/tests-regression/code-metrics/out.ok",
"chars": 252,
"preview": "FNAMELEN: 7\nNSTRUCT: 2\nNTYPEDEF: 0\nNVOID: 18\nNGETS: 0\nIDLEN: 6.2008\nCHLINESCHAR: 1062:27586\nNCCHAR: 18\nNCOMMENT: 53\nNCOP"
},
{
"path": "core-tools/tests-regression/commit-stats/out.ok",
"chars": 823,
"preview": "Authors ordered by number of commits\n 557 Jim Meyering\n 252 Paul Eggert\n 146 Paolo Bonzini\n 79 Norihiro Tanaka\n 9 Bru"
},
{
"path": "core-tools/tests-regression/compress-compare/out.ok",
"chars": 69,
"preview": "File type: ASCII\nOriginal size:40735\nxz:16728\nbzip2:15556\ngzip:17730\n"
},
{
"path": "core-tools/tests-regression/dgsh-wrap/.gitignore",
"chars": 12,
"preview": "echo\necho-S\n"
},
{
"path": "core-tools/tests-regression/dgsh-wrap/dd-args.ok",
"chars": 2,
"preview": "0\n"
},
{
"path": "core-tools/tests-regression/dgsh-wrap/echo-deaf.ok",
"chars": 5,
"preview": "hi\n0\n"
},
{
"path": "core-tools/tests-regression/dgsh-wrap/echo-s.ok",
"chars": 5,
"preview": "hi\n0\n"
},
{
"path": "core-tools/tests-regression/dgsh-wrap/echo-s_caps.ok",
"chars": 5,
"preview": "hi\n0\n"
},
{
"path": "core-tools/tests-regression/dgsh-wrap/paste1.ok",
"chars": 4,
"preview": "0\t1\n"
},
{
"path": "core-tools/tests-regression/dgsh-wrap/paste2.ok",
"chars": 4,
"preview": "0\t1\n"
},
{
"path": "core-tools/tests-regression/dgsh-wrap/paste3.ok",
"chars": 4,
"preview": "0\t1\n"
},
{
"path": "core-tools/tests-regression/dgsh-wrap/paste4.ok",
"chars": 4,
"preview": "0\t1\n"
},
{
"path": "core-tools/tests-regression/dgsh-wrap/tee1.ok",
"chars": 8,
"preview": "ahi\nbhi\n"
},
{
"path": "core-tools/tests-regression/dgsh-wrap/tee2.ok",
"chars": 8,
"preview": "ahi\nbhi\n"
},
{
"path": "core-tools/tests-regression/duplicate-files/in/another-same-1",
"chars": 13,
"preview": "Another same\n"
},
{
"path": "core-tools/tests-regression/duplicate-files/in/another-same-2",
"chars": 13,
"preview": "Another same\n"
},
{
"path": "core-tools/tests-regression/duplicate-files/in/different-file-1",
"chars": 9,
"preview": "hi there\n"
},
{
"path": "core-tools/tests-regression/duplicate-files/in/different-file-2",
"chars": 15,
"preview": "Some more text\n"
},
{
"path": "core-tools/tests-regression/duplicate-files/in/different-file-3",
"chars": 10,
"preview": "More text\n"
},
{
"path": "core-tools/tests-regression/duplicate-files/in/same-file-1",
"chars": 13,
"preview": "hello, world\n"
},
{
"path": "core-tools/tests-regression/duplicate-files/in/same-file-2",
"chars": 13,
"preview": "hello, world\n"
},
{
"path": "core-tools/tests-regression/duplicate-files/in/same-file-3",
"chars": 13,
"preview": "hello, world\n"
},
{
"path": "core-tools/tests-regression/duplicate-files/out.ok",
"chars": 162,
"preview": "duplicate-files/in/same-file-1 duplicate-files/in/same-file-2 duplicate-files/in/same-file-3\n duplicate-files/in/another"
},
{
"path": "core-tools/tests-regression/map-hierarchy/in/a/date/date.c",
"chars": 7015,
"preview": "/*-\n * Copyright (c) 1985, 1987, 1988, 1993\n *\tThe Regents of the University of California. All rights reserved.\n *\n * "
},
{
"path": "core-tools/tests-regression/map-hierarchy/in/a/date/vary.c",
"chars": 11153,
"preview": "/*-\n * Copyright (c) 1997 Brian Somers <brian@Awfulhak.org>\n * All rights reserved.\n *\n * Redistribution and use in sour"
},
{
"path": "core-tools/tests-regression/map-hierarchy/in/a/date/vary.h",
"chars": 1688,
"preview": "/*-\n * Copyright (c) 1997 Brian Somers <brian@Awfulhak.org>\n * All rights reserved.\n *\n * Redistribution and use in sour"
},
{
"path": "core-tools/tests-regression/map-hierarchy/in/b/bin/date/date.c",
"chars": 7556,
"preview": "/*-\n * Copyright (c) 1985, 1987, 1988, 1993\n *\tThe Regents of the University of California. All rights reserved.\n *\n * "
},
{
"path": "core-tools/tests-regression/map-hierarchy/in/b/bin/date/headers/extern.h",
"chars": 1739,
"preview": "/*-\n * Copyright (c) 1993\n *\tThe Regents of the University of California. All rights reserved.\n *\n * Redistribution and"
},
{
"path": "core-tools/tests-regression/map-hierarchy/in/b/bin/date/headers/vary.h",
"chars": 1688,
"preview": "/*-\n * Copyright (c) 1997 Brian Somers <brian@Awfulhak.org>\n * All rights reserved.\n *\n * Redistribution and use in sour"
},
{
"path": "core-tools/tests-regression/map-hierarchy/in/b/bin/date/netdate.c",
"chars": 5357,
"preview": "/*-\n * Copyright (c) 1990, 1993\n *\tThe Regents of the University of California. All rights reserved.\n *\n * Redistributi"
},
{
"path": "core-tools/tests-regression/map-hierarchy/in/b/bin/date/vary.c",
"chars": 11246,
"preview": "/*-\n * Copyright (c) 1997 Brian Somers <brian@Awfulhak.org>\n * All rights reserved.\n *\n * Redistribution and use in sour"
},
{
"path": "core-tools/tests-regression/map-hierarchy/out.ok/map-hierarchy/in/b/bin/date/date.c",
"chars": 7015,
"preview": "/*-\n * Copyright (c) 1985, 1987, 1988, 1993\n *\tThe Regents of the University of California. All rights reserved.\n *\n * "
},
{
"path": "core-tools/tests-regression/map-hierarchy/out.ok/map-hierarchy/in/b/bin/date/headers/vary.h",
"chars": 1688,
"preview": "/*-\n * Copyright (c) 1997 Brian Somers <brian@Awfulhak.org>\n * All rights reserved.\n *\n * Redistribution and use in sour"
},
{
"path": "core-tools/tests-regression/map-hierarchy/out.ok/map-hierarchy/in/b/bin/date/vary.c",
"chars": 11153,
"preview": "/*-\n * Copyright (c) 1997 Brian Somers <brian@Awfulhak.org>\n * All rights reserved.\n *\n * Redistribution and use in sour"
},
{
"path": "core-tools/tests-regression/parallel-word-count/out.ok",
"chars": 25387,
"preview": "1 \"'Challenger,\n2 \"A\n1 \"Ah,\n1 \"An\n3 \"And\n1 \"Anything\n1 \"Anything--anywhere--I\n1 \"As\n6 \"But\n1 \"Can't\n1 \"Challenger\n1 \"Cha"
},
{
"path": "core-tools/tests-regression/regression/errors/stream-scatter-cycle.ok",
"chars": 88,
"preview": "The following dependencies across streams form a cycle:\n head /stream/d\n tail /stream/b\n"
},
{
"path": "core-tools/tests-regression/regression/errors/stream-scatter-cycle.sh",
"chars": 706,
"preview": "#!/usr/bin/env sgsh\n#\n# Copyright 2013 Diomidis Spinellis\n#\n# Licensed under the Apache License, Version 2.0 (the \"Lic"
},
{
"path": "core-tools/tests-regression/regression/errors/unsafe-gather.ok",
"chars": 213,
"preview": "test/regression/errors/unsafe-gather.sh(29): warning: Unsafe use of pass-through /stream/b in the gather section\ntest/re"
},
{
"path": "core-tools/tests-regression/regression/errors/unsafe-gather.sh",
"chars": 872,
"preview": "#!/usr/bin/env sgsh\n#\n# SYNOPSIS Highlight misspelled words\n# DESCRIPTION\n# Highlight the words that are misspelled in t"
},
{
"path": "core-tools/tests-regression/regression/errors/unsafe-gather2.ok",
"chars": 215,
"preview": "test/regression/errors/unsafe-gather2.sh(24): warning: Unsafe use of pass-through /stream/b in the gather section\ntest/r"
},
{
"path": "core-tools/tests-regression/regression/errors/unsafe-gather2.sh",
"chars": 765,
"preview": "#!/usr/bin/env sgsh\n#\n# Copyright 2013 Diomidis Spinellis\n#\n# Licensed under the Apache License, Version 2.0 (the \"Lic"
},
{
"path": "core-tools/tests-regression/regression/graphs/NMRPipe.ok",
"chars": 1231,
"preview": "\n\tdigraph \"\" {\n\t\trankdir = LR;\n\t\tnode [fontname=\"Courier\"];\n\t\tedge [];\n\t\tnode_tee_0 [id=\"node_tee_0\", label=\"sgsh-tee \","
},
{
"path": "core-tools/tests-regression/regression/graphs/code-metrics.ok",
"chars": 7709,
"preview": "\n\tdigraph \"\" {\n\t\trankdir = LR;\n\t\tnode [fontname=\"Courier\"];\n\t\tedge [];\n\t\t\"FNAMELEN\" [id=\"store:FNAMELEN\", shape=\"box\"];\n"
},
{
"path": "core-tools/tests-regression/regression/graphs/commit-stats.ok",
"chars": 721,
"preview": "\n\tdigraph \"\" {\n\t\trankdir = LR;\n\t\tnode [fontname=\"Courier\"];\n\t\tedge [];\n\t\tnode_tee_0 [id=\"node_tee_0\", label=\"sgsh-tee \","
},
{
"path": "core-tools/tests-regression/regression/graphs/committer-plot.ok",
"chars": 2820,
"preview": "\n\tdigraph \"\" {\n\t\trankdir = LR;\n\t\tnode [fontname=\"Courier\"];\n\t\tedge [];\n\t\tnode_tee_0 [id=\"node_tee_0\", label=\"sgsh-tee \","
},
{
"path": "core-tools/tests-regression/regression/graphs/compress-compare.ok",
"chars": 1651,
"preview": "\n\tdigraph \"\" {\n\t\trankdir = LR;\n\t\tnode [fontname=\"Courier\"];\n\t\tedge [];\n\t\tnode_tee_0 [id=\"node_tee_0\", label=\"sgsh-tee \","
},
{
"path": "core-tools/tests-regression/regression/graphs/dir.ok",
"chars": 1512,
"preview": "\n\tdigraph \"\" {\n\t\trankdir = LR;\n\t\tnode [fontname=\"Courier\"];\n\t\tedge [];\n\t\tnode_tee_0 [id=\"node_tee_0\", label=\"sgsh-tee \","
},
{
"path": "core-tools/tests-regression/regression/graphs/duplicate-files.ok",
"chars": 537,
"preview": "\n\tdigraph \"\" {\n\t\trankdir = LR;\n\t\tnode [fontname=\"Courier\"];\n\t\tedge [];\n\t\tnode_tee_0 [id=\"node_tee_0\", label=\"sgsh-tee \","
},
{
"path": "core-tools/tests-regression/regression/graphs/ft2d.ok",
"chars": 3652,
"preview": "\n\tdigraph \"\" {\n\t\trankdir = LR;\n\t\tnode [fontname=\"Courier\"];\n\t\tedge [];\n\t\tnode_cmd_0_0_0 [id=\"node_cmd_0_0_0\", label=\"sfs"
},
{
"path": "core-tools/tests-regression/regression/graphs/map-hierarchy.ok",
"chars": 1170,
"preview": "\n\tdigraph \"\" {\n\t\trankdir = LR;\n\t\tnode [fontname=\"Courier\"];\n\t\tedge [];\n\t\tnode_cmd_0_0_0 [id=\"node_cmd_0_0_0\", label=\"lin"
},
{
"path": "core-tools/tests-regression/regression/graphs/parallel-logresolve.ok",
"chars": 2114,
"preview": "\n\tdigraph \"\" {\n\t\trankdir = LR;\n\t\tnode [fontname=\"Courier\"];\n\t\tedge [];\n\t\tnode_tee_0 [id=\"node_tee_0\", label=\"sgsh-tee -"
},
{
"path": "core-tools/tests-regression/regression/graphs/spell-highlight.ok",
"chars": 824,
"preview": "\n\tdigraph \"\" {\n\t\trankdir = LR;\n\t\tnode [fontname=\"Courier\"];\n\t\tedge [];\n\t\tnode_tee_0 [id=\"node_tee_0\", label=\"sgsh-tee \","
},
{
"path": "core-tools/tests-regression/regression/graphs/text-properties.ok",
"chars": 1244,
"preview": "\n\tdigraph \"\" {\n\t\trankdir = LR;\n\t\tnode [fontname=\"Courier\"];\n\t\tedge [];\n\t\tnode_tee_0 [id=\"node_tee_0\", label=\"sgsh-tee \","
},
{
"path": "core-tools/tests-regression/regression/graphs/web-log-report.ok",
"chars": 6708,
"preview": "\n\tdigraph \"\" {\n\t\trankdir = LR;\n\t\tnode [fontname=\"Courier\"];\n\t\tedge [];\n\t\tnode_tee_0 [id=\"node_tee_0\", label=\"sgsh-tee \","
},
{
"path": "core-tools/tests-regression/regression/graphs/web-log-stats.ok",
"chars": 2760,
"preview": "\n\tdigraph \"\" {\n\t\trankdir = LR;\n\t\tnode [fontname=\"Courier\"];\n\t\tedge [];\n\t\tnode_tee_0 [id=\"node_tee_0\", label=\"sgsh-tee \","
},
{
"path": "core-tools/tests-regression/regression/graphs/word-properties.ok",
"chars": 1085,
"preview": "\n\tdigraph \"\" {\n\t\trankdir = LR;\n\t\tnode [fontname=\"Courier\"];\n\t\tedge [];\n\t\tnode_tee_0 [id=\"node_tee_0\", label=\"sgsh-tee \","
},
{
"path": "core-tools/tests-regression/regression/scripts/NMRPipe.ok",
"chars": 3431,
"preview": "#!/bin/sh\n# Automatically generated file\n# Source file example/NMRPipe.sh\n#!/usr/bin/env sgsh\n#\n# SYNOPSIS Nuclear magne"
},
{
"path": "core-tools/tests-regression/regression/scripts/code-metrics.ok",
"chars": 7411,
"preview": "#!/bin/sh\n# Automatically generated file\n# Source file example/code-metrics.sh\n#!/usr/bin/env sgsh\n#\n# SYNOPSIS C code m"
},
{
"path": "core-tools/tests-regression/regression/scripts/commit-stats.ok",
"chars": 2179,
"preview": "#!/bin/sh\n# Automatically generated file\n# Source file example/commit-stats.sh\n#!/usr/bin/env sgsh\n#\n# SYNOPSIS Git comm"
},
{
"path": "core-tools/tests-regression/regression/scripts/committer-plot.ok",
"chars": 4279,
"preview": "#!/bin/sh\n# Automatically generated file\n# Source file example/committer-plot.sh\n#!/usr/bin/env sgsh -s /bin/bash\n#\n# SY"
},
{
"path": "core-tools/tests-regression/regression/scripts/compress-compare.ok",
"chars": 2726,
"preview": "#!/bin/sh\n# Automatically generated file\n# Source file example/compress-compare.sh\n#!/usr/bin/env sgsh\n#\n# SYNOPSIS Comp"
},
{
"path": "core-tools/tests-regression/regression/scripts/dir.ok",
"chars": 2576,
"preview": "#!/bin/sh\n# Automatically generated file\n# Source file example/dir.sh\n#!/usr/bin/env sgsh\n#\n# SYNOPSIS Directory listing"
},
{
"path": "core-tools/tests-regression/regression/scripts/duplicate-files.ok",
"chars": 2336,
"preview": "#!/bin/sh\n# Automatically generated file\n# Source file example/duplicate-files.sh\n#!/usr/bin/env sgsh\n#\n# SYNOPSIS Find "
},
{
"path": "core-tools/tests-regression/regression/scripts/ft2d.ok",
"chars": 5372,
"preview": "#!/bin/sh\n# Automatically generated file\n# Source file example/ft2d.sh\n#!/usr/bin/env sgsh\n#\n# SYNOPSIS Waves: 2D Fourie"
},
{
"path": "core-tools/tests-regression/regression/scripts/map-hierarchy.ok",
"chars": 3244,
"preview": "#!/bin/sh\n# Automatically generated file\n# Source file example/map-hierarchy.sh\n#!/usr/bin/env sgsh\n#\n# SYNOPSIS Hierarc"
},
{
"path": "core-tools/tests-regression/regression/scripts/parallel-logresolve.ok",
"chars": 3565,
"preview": "#!/bin/sh\n# Automatically generated file\n# Source file example/parallel-logresolve.sh\n#!/usr/bin/env sgsh\n#\n# SYNOPSIS P"
},
{
"path": "core-tools/tests-regression/regression/scripts/spell-highlight.ok",
"chars": 2138,
"preview": "#!/bin/sh\n# Automatically generated file\n# Source file example/spell-highlight.sh\n#!/usr/bin/env sgsh\n#\n# SYNOPSIS Highl"
},
{
"path": "core-tools/tests-regression/regression/scripts/text-properties.ok",
"chars": 3370,
"preview": "#!/bin/sh\n# Automatically generated file\n# Source file example/text-properties.sh\n#!/usr/bin/env sgsh\n#\n# SYNOPSIS Text "
},
{
"path": "core-tools/tests-regression/regression/scripts/web-log-report.ok",
"chars": 7647,
"preview": "#!/bin/sh\n# Automatically generated file\n# Source file example/web-log-report.sh\n#!/usr/bin/env sgsh -s /bin/bash\n#\n# SY"
},
{
"path": "core-tools/tests-regression/regression/scripts/web-log-stats.ok",
"chars": 4271,
"preview": "#!/bin/sh\n# Automatically generated file\n# Source file example/web-log-stats.sh\n#!/usr/bin/env sgsh -s /bin/bash\n#\n# SYN"
},
{
"path": "core-tools/tests-regression/regression/scripts/word-properties.ok",
"chars": 2759,
"preview": "#!/bin/sh\n# Automatically generated file\n# Source file example/word-properties.sh\n#!/usr/bin/env sgsh\n#\n# SYNOPSIS Word "
},
{
"path": "core-tools/tests-regression/regression/warnings/single-target.ok",
"chars": 83,
"preview": "test/regression/warnings/single-target.sh(26): warning: Scatter to a single target\n"
},
{
"path": "core-tools/tests-regression/regression/warnings/single-target.sh",
"chars": 754,
"preview": "#!/usr/bin/env sgsh\n#\n# Copyright 2013 Diomidis Spinellis\n#\n# Licensed under the Apache License, Version 2.0 (the \"Lic"
},
{
"path": "core-tools/tests-regression/regression/warnings/unsafe-scatter.ok",
"chars": 222,
"preview": "test/regression/warnings/unsafe-scatter.sh(21): warning: Unsafe use of pass-through /stream/b in the scatter section\ntes"
},
{
"path": "core-tools/tests-regression/regression/warnings/unsafe-scatter.sh",
"chars": 734,
"preview": "#!/usr/bin/env sgsh\n#\n# Copyright 2013 Diomidis Spinellis\n#\n# Licensed under the Apache License, Version 2.0 (the \"Lic"
},
{
"path": "core-tools/tests-regression/spell-highlight/out.ok",
"chars": 18,
"preview": "hello cruwl world\n"
},
{
"path": "core-tools/tests-regression/tee/oom.err",
"chars": 60,
"preview": "dgsh-tee: Out of memory with input-side buffering specified\n"
},
{
"path": "core-tools/tests-regression/tee/perm.ok",
"chars": 8,
"preview": "3\n1\n2\n0\n"
},
{
"path": "core-tools/tests-regression/tee/tee-fastout-I-l.ok",
"chars": 57,
"preview": "Buffers allocated: 1025 Freed: 1024 Maximum allocated: 2\n"
},
{
"path": "core-tools/tests-regression/tee/tee-fastout-I-m 2k -f.ok",
"chars": 90,
"preview": "Buffers allocated: 1025 Freed: 1024 Maximum allocated: 2\nPage out: 0 In: 0 Pages freed: 0\n"
},
{
"path": "core-tools/tests-regression/tee/tee-fastout-I-m 2k.ok",
"chars": 90,
"preview": "Buffers allocated: 1025 Freed: 1024 Maximum allocated: 2\nPage out: 0 In: 0 Pages freed: 0\n"
},
{
"path": "core-tools/tests-regression/tee/tee-fastout-I.ok",
"chars": 90,
"preview": "Buffers allocated: 1025 Freed: 1024 Maximum allocated: 2\nPage out: 0 In: 0 Pages freed: 0\n"
},
{
"path": "core-tools/tests-regression/tee/tee-fastout-l.ok",
"chars": 57,
"preview": "Buffers allocated: 1025 Freed: 1024 Maximum allocated: 1\n"
},
{
"path": "core-tools/tests-regression/tee/tee-fastout-m 2k -f.ok",
"chars": 90,
"preview": "Buffers allocated: 1025 Freed: 1024 Maximum allocated: 1\nPage out: 0 In: 0 Pages freed: 0\n"
},
{
"path": "core-tools/tests-regression/tee/tee-fastout-m 2k.ok",
"chars": 90,
"preview": "Buffers allocated: 1025 Freed: 1024 Maximum allocated: 1\nPage out: 0 In: 0 Pages freed: 0\n"
},
{
"path": "core-tools/tests-regression/tee/tee-fastout.ok",
"chars": 90,
"preview": "Buffers allocated: 1025 Freed: 1024 Maximum allocated: 1\nPage out: 0 In: 0 Pages freed: 0\n"
},
{
"path": "core-tools/tests-regression/tee/tee-lagout-I-l.ok",
"chars": 59,
"preview": "Buffers allocated: 1025 Freed: 1024 Maximum allocated: 960\n"
},
{
"path": "core-tools/tests-regression/tee/tee-lagout-I-m 2k -f.ok",
"chars": 96,
"preview": "Buffers allocated: 1987 Freed: 1986 Maximum allocated: 3\nPage out: 962 In: 962 Pages freed: 962\n"
},
{
"path": "core-tools/tests-regression/tee/tee-lagout-I-m 2k.ok",
"chars": 60,
"preview": "sgsh-tee: Out of memory with input-side buffering specified\n"
},
{
"path": "core-tools/tests-regression/tee/tee-lagout-I.ok",
"chars": 92,
"preview": "Buffers allocated: 1025 Freed: 1024 Maximum allocated: 964\nPage out: 0 In: 0 Pages freed: 0\n"
},
{
"path": "core-tools/tests-regression/tee/tee-lagout-l.ok",
"chars": 57,
"preview": "Buffers allocated: 1025 Freed: 1024 Maximum allocated: 1\n"
},
{
"path": "core-tools/tests-regression/tee/tee-lagout-m 2k -f.ok",
"chars": 90,
"preview": "Buffers allocated: 1025 Freed: 1024 Maximum allocated: 1\nPage out: 0 In: 0 Pages freed: 0\n"
},
{
"path": "core-tools/tests-regression/tee/tee-lagout-m 2k.ok",
"chars": 90,
"preview": "Buffers allocated: 1025 Freed: 1024 Maximum allocated: 1\nPage out: 0 In: 0 Pages freed: 0\n"
},
{
"path": "core-tools/tests-regression/tee/tee-lagout.ok",
"chars": 90,
"preview": "Buffers allocated: 1025 Freed: 1024 Maximum allocated: 1\nPage out: 0 In: 0 Pages freed: 0\n"
},
{
"path": "core-tools/tests-regression/tee/tee-lahout.ok",
"chars": 57,
"preview": "Buffers allocated: 1025 Freed: 1024 Maximum allocated: 1\n"
},
{
"path": "core-tools/tests-regression/test-dgsh.sh",
"chars": 3411,
"preview": "#!/bin/sh\n#\n# Regression testing of the provided examples\n#\n\nTOP=$(cd ../.. ; pwd)\nDGSH=\"$TOP/build/bin/dgsh\"\nPATH=\"$TOP"
},
{
"path": "core-tools/tests-regression/test-kvstore.sh",
"chars": 15525,
"preview": "#!/bin/sh\n\nDGSH_READVAL=../src/dgsh-readval\nDGSH_WRITEVAL=../src/dgsh-writeval\nDGSH_HTTPVAL=../src/dgsh-httpval\n\n# Helpe"
},
{
"path": "core-tools/tests-regression/test-merge-sum.sh",
"chars": 1186,
"preview": "#!/usr/bin/env bash\n#\n# Tests for dgsh-merge-sum\n#\n\nMERGE_SUM=../src/dgsh-merge-sum.pl\n\n# Shortcut\ntestcase()\n{\n\tlocal n"
},
{
"path": "core-tools/tests-regression/test-tee.sh",
"chars": 6362,
"preview": "#!/bin/sh\n#\n# Regression tests for dgsh-tee\n#\n\nTOP=$(cd ../.. ; pwd)\nDGSH_TEE=$TOP/build/libexec/dgsh/dgsh-tee\nDGSH_ENUM"
},
{
"path": "core-tools/tests-regression/test-wrap.sh",
"chars": 3447,
"preview": "#!/bin/sh\n#\n# Regression testing of the provided examples\n#\n\nTOP=$(cd ../.. ; pwd)\nDGSH=\"$TOP/build/bin/dgsh\"\nPATH=\"$TOP"
},
{
"path": "core-tools/tests-regression/text-properties/out.ok/character.txt",
"chars": 856,
"preview": "8380 21%\n3546 e 8.7%\n2535 t 6.2%\n2368 a 5.8%\n2345 o 5.8%\n1942 n 4.8%\n1850 i 4.5%\n1746 s 4.3%\n1700 h 4.2%\n1610 r 4%\n1273"
},
{
"path": "core-tools/tests-regression/text-properties/out.ok/digram.txt",
"chars": 7238,
"preview": " 689 th\n 602 he\n 473 er\n 469 ou\n 468 in\n 454 an\n 392 ha\n 371 re\n 346 at\n 331 en\n 310 on"
},
{
"path": "core-tools/tests-regression/text-properties/out.ok/trigram.txt",
"chars": 29592,
"preview": " 350 the\n 176 and\n 173 you\n 168 ing\n 155 hat\n 112 tha\n 107 ent\n 95 her\n 94 his\n 90 ver\n"
},
{
"path": "core-tools/tests-regression/text-properties/out.ok/words.txt",
"chars": 30962,
"preview": " 275 the\n 258 I\n 175 a\n 166 to\n 156 of\n 146 and\n 127 you\n 113 it\n 99 that\n 99 in\n 87"
},
{
"path": "core-tools/tests-regression/web-log-report/logfile",
"chars": 171553,
"preview": "204.249.225.59 - - [28/Aug/1995:00:00:34 -0400] \"GET /pub/rmharris/catalogs/dawsocat/intro.html HTTP/1.0\" 200 3542\npc23-"
},
{
"path": "core-tools/tests-regression/web-log-report/out.ok",
"chars": 3148,
"preview": "\t\t\tWWW server statistics\n\t\t\t=====================\n\nSummary\n-------\nNumber of accesses: 1655\nNumber of Gbytes transferred"
},
{
"path": "core-tools/tests-regression/word-properties/LostWorldChap1-3",
"chars": 40735,
"preview": "The Project Gutenberg EBook of The Lost World, by Arthur Conan Doyle\n\nThis eBook is for the use of anyone anywhere at no"
},
{
"path": "core-tools/tests-regression/word-properties/out.ok",
"chars": 1447,
"preview": "Anthropology\t\tc: nthr\t\nFollowing\tp: ol-lo\t\t\nFrenchman\t\tc: nchm\t\nGazette\tp: et-te\t\t\nHTML\t\tc: HTML\t\nMORROW\tp: OR-RO\t\t\nPala"
},
{
"path": "eval/.gitignore",
"chars": 129,
"preview": "*.class\naccess-small.log\naccess.log\nbooks.txt\nbooks1.txt\ncharacter.txt\nemptydir\nemptygit\nhier\nlinux.new\nlinux.old\nlinux\n"
},
{
"path": "eval/Makefile",
"chars": 1601,
"preview": "include ../.config\n\nall: data WebStats.class TextProperties.class\n\tsh perf-eval.sh\n\ndata: books.txt linux access.log lin"
},
{
"path": "eval/README.md",
"chars": 313,
"preview": "The evaluation method for the classic shell performance is based\non an earlier version of *dgsh*, *sgsh*,\nwhich can tran"
},
{
"path": "eval/SConstruct",
"chars": 1409,
"preview": "# ft2d example souped up to compare with dgsh\n\nfrom rsf.proj import *\n\nFlow('pulse',None,\n '''\n spike n1=10000 n"
},
{
"path": "eval/TextProperties.java",
"chars": 3671,
"preview": "/*-\n *\n * Collect and print text frequency statistics from the text\n * read from the standard input.\n *\n * Copyright 201"
},
{
"path": "eval/WebStats.java",
"chars": 8527,
"preview": "/*-\n *\n * Collect and print Web statistics from common log format data\n * read from the standard input.\n *\n * Copyright "
},
{
"path": "eval/eval-lib.sh",
"chars": 1328,
"preview": "#!/bin/sh\n#\n# Portable time function\n#\n# Copyright 2013 Diomidis Spinellis\n#\n# Licensed under the Apache License, Vers"
},
{
"path": "eval/ft2d.sh",
"chars": 1869,
"preview": "#!/usr/local/bin/dgsh\n#\n# Modified version of the example with the same name, to test performance\n# on larger data sets\n"
},
{
"path": "eval/log-grow.pl",
"chars": 1454,
"preview": "#!/usr/bin/perl\n#\n# Grow the size of a web server log file by a factor of N,\n# which is specified as the first argument,"
},
{
"path": "eval/perf-eval.sh",
"chars": 3044,
"preview": "#!/bin/sh\n#\n# Benchamrk the performance of alternative implementations of the\n# provided example files\n#\n# Copyright 20"
},
{
"path": "eval/web-log-report.pl",
"chars": 5527,
"preview": "#!/usr/bin/perl\n#\n# Copyright 1995-2013 Diomidis Spinellis\n#\n# Licensed under the Apache License, Version 2.0 (the \"Li"
},
{
"path": "eval/webeval.sh",
"chars": 1806,
"preview": "#!/bin/sh\n#\n# Run performance evaluations\n#\n# Copyright 2013 Diomidis Spinellis\n#\n# Licensed under the Apache License,"
},
{
"path": "example/NMRPipe.sh",
"chars": 2443,
"preview": "#!/usr/bin/env dgsh\n#\n# SYNOPSIS Nuclear magnetic resonance processing\n# DESCRIPTION\n# Nuclear magnetic resonance in-pha"
},
{
"path": "example/author-compare.sh",
"chars": 2458,
"preview": "#!/usr/bin/env dgsh\n#\n# SYNOPSIS Venue author compare\n# DESCRIPTION\n# Given the specification of two publication venues,"
},
{
"path": "example/code-metrics.sh",
"chars": 3750,
"preview": "#!/usr/bin/env dgsh\n#\n# SYNOPSIS C code metrics\n# DESCRIPTION\n# Process a directory containing C source code, and produc"
},
{
"path": "example/commit-stats.sh",
"chars": 1156,
"preview": "#!/usr/bin/env dgsh\n#\n# SYNOPSIS Git commit statistics\n# DESCRIPTION\n# Process the Git history, and list the authors and"
},
{
"path": "example/committer-plot.sh",
"chars": 3319,
"preview": "#!/usr/bin/env dgsh\n#\n# SYNOPSIS Plot Git committer activity over time\n# DESCRIPTION\n# Process the Git history, and crea"
},
{
"path": "example/compress-compare.sh",
"chars": 1337,
"preview": "#!/usr/bin/env dgsh\n#\n# SYNOPSIS Compression benchmark\n# DESCRIPTION\n# Report file type, length, and compression perform"
},
{
"path": "example/dir.sh",
"chars": 1326,
"preview": "#!/usr/bin/env dgsh\n#\n# SYNOPSIS Directory listing\n# DESCRIPTION\n# Windows-like DIR command for the current directory.\n#"
},
{
"path": "example/duplicate-files.sh",
"chars": 1551,
"preview": "#!/usr/bin/env dgsh\n#\n# SYNOPSIS Find duplicate files\n# DESCRIPTION\n# List the names of duplicate files in the specified"
},
{
"path": "example/fft-block8.sh",
"chars": 1106,
"preview": "#!/usr/bin/env dgsh\n#\n# SYNOPSIS FFT calculation\n# DESCRIPTION\n# Calculate the iterative FFT for n = 8 in parallel.\n# De"
},
{
"path": "example/ft2d.sh",
"chars": 2090,
"preview": "#!/usr/bin/env dgsh\n#\n# SYNOPSIS Waves: 2D Fourier transforms\n# DESCRIPTION\n# Create two graphs:\n# 1) a broadened pulse "
},
{
"path": "example/map-hierarchy.sh",
"chars": 2315,
"preview": "#!/usr/bin/env dgsh\n#\n# SYNOPSIS Hierarchy map\n# DESCRIPTION\n# Given two directory hierarchies A and B passed as input a"
},
{
"path": "example/parallel-word-count.sh",
"chars": 1382,
"preview": "#!/usr/bin/env dgsh\n#\n# SYNOPSIS Parallel word count\n# DESCRIPTION\n# Count number of times each word appears in the spec"
},
{
"path": "example/reorder-columns.sh",
"chars": 816,
"preview": "#!/usr/bin/env dgsh\n#\n# SYNOPSIS Reorder columns\n# DESCRIPTION\n# Reorder columns in a CSV document.\n# Demonstrates the c"
},
{
"path": "example/spell-highlight.sh",
"chars": 1222,
"preview": "#!/usr/bin/env dgsh\n#\n# SYNOPSIS Highlight misspelled words\n# DESCRIPTION\n# Highlight the words that are misspelled in t"
},
{
"path": "example/static-functions.sh",
"chars": 1271,
"preview": "#!/usr/bin/env dgsh\n#\n# SYNOPSIS C/C++ symbols that should be static\n# DESCRIPTION\n# Given as an argument a directory co"
}
]
// ... and 260 more files (download for full content)
About this extraction
This page contains the full source code of the dspinellis/dgsh GitHub repository, extracted and formatted as plain text for AI agents and large language models (LLMs). The extraction includes 460 files (1.1 MB), approximately 396.7k tokens, and a symbol index with 562 extracted functions, classes, methods, constants, and types. Use this with OpenClaw, Claude, ChatGPT, Cursor, Windsurf, or any other AI tool that accepts text input. You can copy the full output to your clipboard or download it as a .txt file.
Extracted by GitExtract — free GitHub repo to text converter for AI. Built by Nikandr Surkov.