[ { "path": ".github/workflows/test.yml", "content": "name: Test\n\non:\n - push\n - pull_request\n\njobs:\n test:\n strategy:\n fail-fast: false\n matrix:\n os: [ubuntu-latest]\n mode: [tests]\n python:\n - \"3.11\"\n include:\n - os: ubuntu-latest\n mode: coverage\n python: \"3.8\"\n - os: ubuntu-latest\n mode: checks\n python: \"3.8\"\n - os: ubuntu-latest\n mode: check-images\n python: \"3.8\"\n runs-on: ${{ matrix.os }}\n env:\n TEST_MODE: ${{ matrix.mode }}\n REPROZIP_USAGE_STATS: \"off\"\n REPROZIP_PARAMETERS: https://stats.reprozip.org/parameters/travis/\n steps:\n - uses: actions/checkout@v4\n with:\n fetch-depth: 20\n - uses: actions/setup-python@v5\n with:\n python-version: ${{ matrix.python }}\n - name: Install dependencies\n run: |\n if [ -z \"${XDG_CACHE_HOME-}\" ]; then\n mkdir -p ~/.cache/reprozip\n else\n mkdir -p \"$XDG_CACHE_HOME/reprozip\"\n fi\n\n case \"$TEST_MODE\"\n in\n tests|coverage|check-images)\n if [ \"$TEST_MODE\" = \"coverage\" ]; then\n export CFLAGS=\"-fprofile-arcs -ftest-coverage\"\n fi\n PKGS=\"libc6-dev-i386 gcc-multilib libsqlite3-dev\"\n if [ \"$TEST_MODE\" = \"coverage\" ]; then PKGS=\"$PKGS lcov\"; fi\n sudo apt-get update -qq\n sudo apt-get install -qq $PKGS\n if [ $TEST_MODE = \"coverage\" ]; then\n pip install 'coverage<5'\n # `--config-settings editable_mode=compat` works around https://github.com/pypa/setuptools/issues/3557\n pip install -e ./reprozip -e ./reprounzip -e ./reprounzip-docker -e ./reprounzip-vagrant -e ./reprounzip-vistrails -e ./reprounzip-qt -e ./reprozip-jupyter --config-settings editable_mode=compat\n else\n pip install ./reprozip ./reprounzip ./reprounzip-docker ./reprounzip-vagrant ./reprounzip-vistrails ./reprounzip-qt ./reprozip-jupyter\n fi\n ;;\n checks)\n pip install flake8 readme_renderer\n ;;\n *)\n exit 1\n ;;\n esac\n - name: Test\n run: |\n export LANG=C\n export LC_ALL=C\n export REPROZIP_TEST_PYTHON=\"$(which python) -Wd\"\n case \"$TEST_MODE\"\n in\n coverage)\n export PYTHONUNBUFFERED=1\n export COVER=\"coverage run --append --source=$PWD/reprozip/reprozip,$PWD/reprounzip/reprounzip,$PWD/reprounzip-docker/reprounzip,$PWD/reprounzip-vagrant/reprounzip,$PWD/reprounzip-vistrails/reprounzip,$PWD/tests --branch\"\n python -Wd -m $COVER -m tests --run-docker\n ;;\n tests)\n export PYTHONUNBUFFERED=1\n python -Wd tests --run-docker\n ;;\n check-images)\n python -Wd tests --check-vagrant-images --check-docker-images\n ;;\n checks)\n flake8 --ignore=E731,W503,W504\n diff -q reprozip/reprozip/common.py reprounzip/reprounzip/common.py\n diff -q reprozip/reprozip/utils.py reprounzip/reprounzip/utils.py\n find reprozip reprounzip reprozip-* reprounzip-* -name '*.py' -or -name '*.sh' -or -name '*.h' -or -name '*.c' | (set +x; while read i; do\n T=$(file -b --mime \"$i\")\n if ! ( echo \"$T\" | grep -q ascii || echo \"$T\" | grep -q empty ) ; then\n echo \"$i is not ASCII\"\n exit 1\n fi\n done)\n find reprozip reprounzip reprozip-* reprounzip-* -name '*.py' -exec sh -c \"grep 'logging\\\\.\\\\(debug\\\\|warning\\\\|critical\\\\|error\\\\|info\\\\)' \\\"\\$@\\\" && exit 1; exit 0\" {} +\n for pkg in reprozip reprounzip reprozip-* reprounzip-*; do\n (cd $pkg && python setup.py check -r -s)\n done\n ;;\n *)\n exit 1\n ;;\n esac\n - name: Upload coverage\n if: matrix.mode == 'coverage'\n run: |\n # Python\n if [ -f .coverage ]; then mv .coverage .coverage.orig; fi # FIXME: useless?\n coverage combine\n\n # C\n # Find the coverage file (in distutils's build directory)\n OBJDIR=$(dirname \"$(find . -name pytracer.gcno | head -n 1)\")\n (cd reprozip/native && lcov --directory ../../$OBJDIR -c -o reprozip.lcov)\n\n curl -s -o - https://codecov.io/bash | bash -s - -X gcov\n\n test-container:\n strategy:\n fail-fast: false\n matrix:\n image:\n - python:2.7\n install-python:\n - false\n python-cmd:\n - python\n include:\n - image: ubuntu:20.04\n install-python: true\n python-cmd: python3\n runs-on: ubuntu-latest\n container:\n image: ${{ matrix.image }}\n options: \"--privileged\"\n env:\n TEST_MODE: tests\n REPROZIP_USAGE_STATS: \"off\"\n REPROZIP_PARAMETERS: https://stats.reprozip.org/parameters/travis/\n steps:\n - uses: actions/checkout@v4\n with:\n fetch-depth: 20\n - name: Install Docker client\n run: |\n apt-get update\n apt-get install -yy curl\n curl -Lo /tmp/docker.tgz https://download.docker.com/linux/static/stable/x86_64/docker-23.0.6.tgz\n tar -xf /tmp/docker.tgz -C /usr/local/bin --strip-components=1\n rm /tmp/docker.tgz\n - name: Install Python\n if: matrix.install-python == true\n run: |\n apt-get install -yy python3 python3-pip\n - name: Install dependencies\n run: |\n if [ -z \"${XDG_CACHE_HOME-}\" ]; then\n mkdir -p ~/.cache/reprozip\n else\n mkdir -p \"$XDG_CACHE_HOME/reprozip\"\n fi\n\n apt-get update -qq\n apt-get install -qq libc6-dev-i386 gcc-multilib libsqlite3-dev\n cat > pip.constraints.txt <<'EOF'\n pyelftools<0.30\n EOF\n pip install -c pip.constraints.txt ./reprozip ./reprounzip ./reprounzip-docker ./reprounzip-vagrant ./reprounzip-vistrails ./reprounzip-qt ./reprozip-jupyter\n - name: Test\n run: |\n export LANG=C\n export LC_ALL=C\n export REPROZIP_TEST_PYTHON=\"$(which ${{ matrix.python-cmd }}) -Wd\"\n export PYTHONUNBUFFERED=1\n ${{ matrix.python-cmd }} -Wd tests --run-docker\n" }, { "path": ".gitignore", "content": "*.py[co]\n.DS_Store\n\n# Packages\n*.egg\n*.egg-info\ndist\nbuild\neggs\nparts\nbin\nvar\nsdist\ndevelop-eggs\n.installed.cfg\nlib\nlib64\n\n# Installer logs\npip-log.txt\n\n# Unit test / coverage reports\n.coverage\n.tox\nnosetests.xml\n\n# Object files\n*.o\n*.ko\n*.obj\n*.elf\n\n# Libraries\n*.lib\n*.a\n\n# Shared objects (inc. Windows DLLs)\n*.dll\n*.so\n*.so.*\n*.dylib\n\n# Executables\n*.exe\n*.out\n*.i*86\n*.x86_64\n*.hex\n\n# Eclipse PyDev\n.project\n.pydevproject\n\n# PyCharm\n.idea\n\n# Qt Creator\n*.config\n*.creator\n*.creator.user\n*.files\n*.includes\n\n# Vagrant\n.vagrant\n\n/tests/vagrant\n/docs/_build\n" }, { "path": ".readthedocs.yaml", "content": "version: 2\n\nbuild:\n os: ubuntu-22.04\n tools:\n python: \"3.11\"\n\nsphinx:\n configuration: docs/conf.py\n" }, { "path": "CHANGELOG.md", "content": "Changelog\n=========\n\n1.3.2 (2026-01-18)\n------------------\n\n(reprozip and reprounzip only)\n\nEnhancement:\n* Don't depend on deprecated Python `pkg_resources` package. Might fix installation errors on some systems, mainly done for downstream packagers\n\n\n1.3.1 (2025-12-08)\n------------------\n\n(reprozip only)\n\nBugfixes:\n* Fix build on GCC 14\n* Remove fixed-size buffer for reaading paths from `/proc`\n\n1.3 (2023-12-07)\n----------------\n\n(reprozip and reprounzip only)\n\nEnhancements:\n* Store used UNIX sockets in the trace\n* Display a warning if the process connect to the systemd socket (https://docs.reprozip.org/s/systemd.html)\n* Print trace warnings in red (files read then written, systemd socket)\n* Improve message on interrupt, making clearer what happens if pressing twice\n* Change \"(d)elete\" option to \"(o)verwrite\" in prompt when a trace exists\n\n1.2.1 (2023-02-06)\n------------------\n\n(reprounzip and reprounzip-qt only)\n\nBugfixes:\n* Fix typo in reprounzip, reprounzip-qt: import from reprounzip, not reprozip\n\n1.2 (2023-02-06)\n----------------\n\nBugfixes:\n* Don't mark symlinks as input files\n* Fix reprounzip-vagrant not terminating after it says that it can't install packages\n* Add defense for CVE-2007-4559\n* Fix OrderedSet for Python 3.10+ compatibility\n\nEnhancements:\n* Recognize Ruby gems and apps and gather the whole environment\n* Don't mark Python .pth files as input files\n* Accept ZIP files in addition to TAR for RPZ files (reprozip doesn't currently create ZIP files)\n* Handle more Linux system calls: faccessat2, statx, execveat, clone3, openat2, fchownat, fchmodat, accept4, renameat2\n\n1.1 (2021-07-06)\n----------------\n\n(reprounzip-vistrails didn't change)\n\nBugfixes:\n* Fix possible crash reading `docker inspect` output\n* Fix reprozip-jupyter on more recent Tornado\n* Fix failure in reprounzip-docker upload if `/bin/sh` is missing\n* Have `reprounzip directory` execute with the correct interpreter (e.g. packed `ld-linux.so`)\n* Fix \"invalid cross-device link\" errors in vagrant download if the destination is not on the same device as the unpacked directory\n\nEnhancements:\n* Add PyQt5 compatibility to reprounzip-qt\n* reprounzip-docker: Keep ownership of uploaded files\n* Add a new certificate to download parameters, valid through 2121 (previous expires 2024)\n* Always show full path of executed files in `reprozip testrun`\n* Improved merging of pack on top of base image in reprounzip-docker\n* Made sure the root is a mountpoint in reprounzip-vagrant, as some applications expect it (e.g. Elasticsearch)\n* Disable unpacker options that won't work in reprounzip-qt (e.g. 'directory' and 'chroot' on non-Linux)\n\n1.0.16 (2019-02-06)\n-------------------\n\n(reprozip-jupyter and reprounzip-vagrant didn't change)\n\nBugfixes:\n* Fixed input/output file filter on Python 3 (to omit `.so`, `.pyc` etc files)\n* Fixed fetching updated parameters on Python 3 (to get the correct Docker and Vagrant base images, a small JSON file is downloaded from reprozip.org)\n* Fixed `--port` option of reprounzip-docker\n\nEnhancements:\n* Use the [distro](https://distro.readthedocs.io/) module instead of the deprecated `platform.linux_distribution()` function to detect the distribution (the latter will be removed in Python 3.8).\n* Use dpkg-query to identify Linux packages instead of reading `dpkg/info/*.list`\n\n1.0.15 (2018-07-31)\n-------------------\n\n(reprounzip-qt only)\n\nBugfixes:\n* Fixed running command from reprounzip-qt on Windows\n* Fixed using Jupyter from reprounzip-qt\n\n1.0.14 (2018-07-30)\n-------------------\n\n(reprozip, reprounzip-qt and reprozip-jupyter only)\n\nBugfixes:\n* Fixed reprounzip-qt refusing to close when an experiment is still unpacked, even after the user provided confirmation\n* Fixed reprozip-jupyter on Python 3\n* Fixed running gnome-terminal from reprounzip-qt, made it preferred over xterm\n* Don't duplicate the latest run in the config file when the trace didn't add a run (for example because the command does not exist)\n\nEnhancements:\n* Uniformized logos and icons\n* Native terminal opened by reprounzip-qt waits for a key after success before closing\n* Officially support reprounzip-qt and reprozip-jupyter on Python 3\n\n1.0.13 (2018-05-15)\n-------------------\n\nBugfixes:\n* Fix uninitialized return value making some xxx_at() calls abort the trace\n* Fix some other warnings via static analysis\n\nEnhancements:\n* Show a warning when executing a file that has the set-uid or set-gid bit set, since Linux will not give it its privileges, making it confusing for users why their run failed\n* Make reprounzip-docker run even without a TTY\n* Correctly handle experiment returning non-0 in Docker\n* The C extension now logs through Python's logging facilities ('reprozip' logger)\n* Collect usage information from reprounzip-qt as well\n\n1.0.12 (2018-03-30)\n-------------------\n\n(reprozip, reprounzip-qt and reprozip-jupyter only)\n\nBugfixes:\n* Fix some kernel/libc issuing unrecognized `openat()` calls, resulting in files missing in the trace\n* Fix `openat()` calls recording read-write as simply write\n* Fix double-click on .RPZ file\n\n1.0.11 (2017-11-05)\n-------------------\n\nBugfixes:\n* Write timestamp in config with timezone offset\n* Always fix up PATH on MacOS (to pick up Docker)\n* Don't have reprounzip-qt choke on MacOS's `-psn_x_xxxx` arguments\n* Fix \"Couldn't find reprounzip command\" in the GUI on Windows if installed from the installer and opened a package by double-click\n\nEnhancements:\n* Improve DOT graph layout\n* Don't silently overwrite output with `reprounzip graph`\n* Add `--regex-include` to `reprounzip graph`\n* `reprozip trace` now returns 125 for tracing errors (previously 1), and returns the traced program's exit code otherwise (previously always 0)\n* Manpages are available\n\n1.0.10 (2017-07-10)\n-------------------\n\nBugfixes:\n* Correctly escape shell commands containing backticks\n* Overwrite tty prompt works correctly on Python 3\n* Fix /proc in vagrant-chroot and chroot having outside mounts\n* Fix ANSI escapes showing in Qt terminal dialog\n* Fix reprozip combine crash on Python 3.6 (patch from James Clarke)\n* Using `graph --packages drop` together with `--json` no longer crashes\n\nEnhancements:\n* New reprozip-jupyter tool to trace and reproduce Jupyter notebooks\n* reprozip_jupyter can be registered as a Jupyter extension to trace notebooks from the Jupyter web interface\n* The Qt GUI knows to run packages with reprozip-jupyter if they are notebook environments (kernels) unpacked with Docker (and reprozip-jupyter is installed)\n* Add `--docker-cmd` to reprounzip-docker to select the Docker command (for example `--docker-cmd=\"sudo docker\"`)\n* Implement `--expose-port` option for Vagrant and Docker (no need for `--docker-option=-p...`)\n* Add docker-machine support to GUI (select which machine to use)\n* Better binaries for MacOS\n* Automatically register reprounzip-qt to open .RPZ files on Linux\n* Register ReproUnzip with Windows from installer\n* Add icon from @heng2j\n\n1.0.9 (2017-01-10)\n------------------\n\nBugfixes:\n* Fix CentOS Docker image versions\n* Remove Fedora Docker images, they don't have tar\n* Do include .pyc files in packages, so reproduction take same code path\n* Don't use the experiment's resolv.conf file\n* Fix handling of files opened in read-write mode\n\n1.0.8 (2016-10-07)\n------------------\n\nBehavior changes:\n* No longer default to overwriting trace directories. ReproZip will ask what to do or exit with an error if one of --continue/--overwrite is not provided\n\nBugfixes:\n* Fix an issue identifying Debian packages when a file's in two packages\n* Fix Python error `Mixing iteration and read methods would lose data`\n* Fix reprounzip info showing some numbers as 0 instead of hiding them in non-verbose mode\n* Another fix to X server IP determination for Docker\n\nEnhancements:\n* New GUI for reprounzip, allowing one to unpack without using the command-line\n* Add filters to remove some common files types from packed files (.pyc) or detected input files (.py, .so, ...)\n* Add JSON output format to `reprounzip info`\n* Allow using the Virtualbox display to reproduce X11-enabled experiments\n\n1.0.7 (2016-08-22)\n------------------\n\nBugfixes:\n* Correctly show an error message if ptrace is unavailable\n* Make Docker & Vagrant setup much faster\n\nEnhancements:\n* Add support for RPM-based distributions, in addition to Debian-based\n\n1.0.6 (2016-06-25)\n------------------\n\n(reprounzip-vistrails didn't change)\n\nBugfixes:\n* Fixes error using Docker with `--enable-x11` on Python 3\n\nEnhancements:\n* `docker run` gets a `--detach` command, to keep the container running (useful for starting servers on remote machines)\n* Restrictions on upload and download commands have been relaxed, in particular it is possible to download input files as well as output files\n* Don't compress outer tar (data is still compressed); this should make some operations (like `reprounzip info`) faster\n* Add `reprozip combine`, useful to combine multiple traces into one (as different runs). Handy if running distributed experiments on shared filesystem (MPI)\n\n1.0.5 (2016-04-07)\n------------------\n\n(reprounzip-vagrant didn't change)\n\nBugfixes:\n* Correctly download parameters from server\n* More reliable way of determining X server IP without using /bin/ip\n\n1.0.4 (2016-03-10)\n------------------\n\nBehavior changes:\n* reprounzip will no longer run on Python 2.6\n\nBugfixes:\n* Fixes file download not using cache if URL is HTTPS\n* Fixes unpacking with directory or chroot for some multi-step packages\n\nFeatures:\n* Add `--docker-option` to pass raw options to Docker\n* You can use `run` or `run -` to run every run, regardless of their number\n* Allow `download `, shortcut for `download :./` (downloads to current directory, keep name)\n* Allow `download --all`\n* Add `--input` and `--output` to showfiles\n* Implement `vagrant suspend` command\n* Do file downloads with [requests](http://python-requests.org/)\n* Download a parameter file to update URLs and image names without waiting for the next ReproZip version\n\n1.0.3 (2015-12-02)\n------------------\n\nBugfixes:\n* You could get a traceback with some unpackers (not Vagrant) on some packages that explicitely pack the / directory\n* Some environment variables prevented running, such as bash exported functions.\n\nFeatures:\n* Remove setup directory if setup fails, so you can run setup again without gettin `target directory exists`\n* Add `--set-env` and `--pass-env` to run\n\n1.0.2 (2015-10-26)\n------------------\n\nBugfixes:\n* You can now use X11 forwarding even with a remote Docker daemon\n* reprounzip-vagrant now works in paths containing spaces\n\n1.0.1 (2015-10-12)\n------------------\n\nBugfixes:\n* Files opened through a shebang were stored with a wrong process number\n* Running with Docker on non-Linux machine didn't work (e.g. docker-machine); now only X11 doesn't work.\n* Some fixes to the graph command\n\nFeatures:\n* `--memory` option for `reprounzip vagrant setup`, to set the VM's RAM.\n\n1.0.0 (2015-09-30)\n------------------\n\nBehavior change:\n* .rpz pack format changed (version 1 -> 2). Pack is now uncompressed, data is in a nested TGZ archive; allows faster retrieval of metadata (config & trace).\n* reprozip trace warnings are now info messages; won't show up without -v\n\nBugfixes:\n* After restarting a Vagrant machine, /dev and /proc wouldn't be mounted anymore\n* Files or links referenced in a shebang could be missed by the tracer\n\nFeatures:\n* Runs in the configuration file now have an 'id' field, that will be shown by 'reprounzip info' and can be selected when running\n* Reworked `reprounzip graph`: level of details, regex filters & replace, JSON output\n* Added *run* argument to `reprounzip showfiles`, to show inputs & outputs of a single run\n\n0.7.2 (2015-08-24)\n------------------\n\nBehavior change:\n* reprounzip-docker will now re-use the resulting image from the previous run when running again, instead of starting from scratch; a 'reset' command has been added to undo runs and uploads.\n\nBugfixes:\n* Couldn't reset an input file to the original (packed) file on Python 3\n* Don't show a warning about network connections when they didn't succeed\n* Hide traceback when failing because Vagrant is not installed\n* Fix input/output file detection assigning files to the same run\n* Fix selecting multiple runs in 'docker run'\n\nFeatures:\n* Display the relative portion of the path when unhandled xxx_at() syscalls are used, to give an idea of what's been missed\n* Add --dont-find-inputs-outputs to reprozip trace and reset, so you can clear that out if too many files would be selected (or if you don't use the feature)\n* Rewrote reprounzip-vistrails plugin; uses a proper VisTrails package that now lives in the VisTrails distribution.\n* Check pack format in unpackers; won't try to unpack version 2\n* It is now possible to select multiple runs with `unpackername run 1-4`\n\n0.7.1 (2015-07-14)\n------------------\n\n(reprozip only)\n\nBugfixes:\n* Files (or links) created with rename, link or symlink then read will no longer be packed.\n* A buffer overflow could happen in the log module, for instance when the experiment passes a very long argument to execve (over 4kB in a single argument) and running in debug mode (-v -v)\n\n\n0.7 (2015-07-07)\n----------------\n\nBehavior change:\n* No longer accept passing `-v` after the subcommand; use `reprozip -v testrun ...`, not `reprozip testrun -v`.\n* Rely on `PTHREAD_EVENT_EXEC` to handle `execve()`. Makes tracing more reliable, and enable it to behave correctly on weird kernels (like UML).\n* Rely on `PTRACE_EVENT_FORK` to handle `fork`/`vfork`/`clone`. Fixes vfork() deadlocking under trace.\n* Completely changed the structure of input and output files (old packs will still be loaded, but new packs are not retro-compatible).\n* Using one of the `run` commands without specifying a number will no longer default to running all of them; it will error out if there are multiple runs.\n\nBugfixes:\n* Fix insertion speed in SQLite3 database\n\nFeatures:\n* Makes VMs (Vagrant or Docker) more resilient to massive breakage of system libraries (obliterating /lib or /usr, when using very different operating systems) by putting busybox in / and using [rpzsudo](https://github.com/remram44/static-sudo).\n* No longer use `dpkg -S` to identify packages, do a single pass over internal dpkg database (this is considerably faster).\n\n0.6.4 (2015-06-07)\n------------------\n\n(reprounzip-vistrails didn't change)\n\nBugfixes:\n* Tracer: correctly handle `chdir()` in multi-threaded processes\n* Fix leaked file descriptors, eventually making SQLite3 fail\n* No longer exceed cmdline length in Dockerfile in big .RPZ pack\n* Fixes `check_output` call when running Docker (not available in Python 2.6)\n* Fixes installation of `sudo` failing if original machine wasn't Debian\n* Don't make TAR error status fatal in Dockerfile (might not run; this is needed because Docker mount some files in the container that can't be overwritten)\n\n0.6.3 (2015-05-06)\n------------------\n\n(reprounzip and plugins only)\n\nBugfixes:\n* Fixes reprounzip-vistrails failing because of reporting\n* Fixes reprounzip-vistrails not escaping correctly in XML in some conditions\n* Fixes docker run failing to read Docker's JSON output on Python 3\n* Fixes reprounzip chroot mounting too many filesystems\n* Fixes typo stopping reprounzip from running on unsupported distribs\n\nFeatures:\n* Adds Debian 8 'Jessie' to Vagrant boxes & Docker images\n* Adds Ubuntu 15.04 'Vivid' to Vagrant boxes & Docker images\n\n0.6.2 (2015-03-16)\n------------------\n\n(reprozip only)\n\nBugfixes:\n* Fixes installation on Python 3 with 7-bit locale\n* Fixes reprozip not showing traceback on exception\n* Fixes bug with multiple runs (`trace --continue`)\n\n0.6.1 (2015-02-17)\n------------------\n\n(reprozip only)\n\nBugfixes:\n* Fixes an overflow in _pytracer for some syscalls.\n\n0.6 (2015-02-11)\n----------------\n\n(reprounzip-vistrails didn't change)\n\nBugfixes:\n* Fixes `debug` log messages not being printed\n* Pressing Ctrl+C wouldn't stop package identification with KeyboardInterrupt\n* Fixes an error message while destroying a docker experiment\n* Fixes docker not installing packages if they were packed\n* Fixes docker always reporting exit status 0\n\nFeatures:\n* Adds `--install-pkgs` options to `docker setup`, to prefer installing\nfrom package manager over unpacking the packed files\n* With vagrant or docker, original machine hostname is restored\n* X11 support for chroot, vagrant and docker unpackers\n\n0.5.1 (2014-12-18)\n------------------\n\n(reprounzip-vistrails didn't change)\n\nBugfixes:\n* Goes back to pack format 0.4: generates `.rpz` files readable by older reprounzip versions\n* Fixes experiment not having a PTY in some conditions\n* Rewrite absolute paths on command-line for directory unpacker\n\nFeatures:\n* Python 2.6 support for reprounzip (except 'graph')\n* Makes error messages more readable\n* Default trace directory renamed from `.reprozip` to `.reprozip-trace`\n* Adds a log file under $HOME/.reprozip/log\n* reprounzip-vagrant will use 'ssh' executable if it's available; should be more reliable, especially on Windows\n* Automatically collects usage information. Nothing will be sent before you opt-in, and we made sure to only collect general details\n\n0.5 (2014-11-24)\n----------------\n\nFeatures:\n* All reprounzip plugins can be installed with `pip install reprounzip[all]`\n* Various improvements to interactive vagrant console\n* Adds support for generic plugins (alongside unpackers)\n* Adds reprounzip-vistrails plugin\n* Pressing Ctrl+C while tracing won't abort anymore; press it twice for SIGKILL\n\n0.4.1 (2014-10-06)\n------------------\n\nBugfixes:\n* reprounzip showed duplicated logging messages\n* Makes 'run' commands not fail if the command returns an error code\n* Unicode issues with Vagrant on Windows Python 3\n* Warning for files read then written didn't show the filenames\n* Fixes resetted input files breaking 'showfiles'\n\nFeatures:\n* 'docker upload' command\n* Adds signals (currently unused, needed for future plugins)\n\n0.4 (2014-09-15)\n----------------\n\nBugfixes:\n* Copying files from host to chroot when some packages where not packed\n* Don't use the full command path in directory's script\n* Fixes socketcall() handling\n\nFeatures:\n* Displays a warning for READ_THEN_WRITTEN files\n* chroot restores files' owner/group\n* Adds 'reprounzip info' command\n* Better error messages when trying to unpack on incompatible system\n* Identifies input files, which can be replaced ('upload' operation)\n* Identifies output files, which can be retrieved ('download' operation)\n* New command-line interface for unpackers, with setup/run/destroy; you can now do everything through reprounzip\n* Vagrant now defaults to --use-chroot`, added --no-use-chroot\n* Adds --summary and --missing to installpkgs\n* Adds Docker unpacker (no uploading support yet)\n\n0.3.2 (2014-08-28)\n------------------\n\n(reprounzip only)\n\nBugfixes:\n* Once busybox was in the local cache, setting it up could crash\n* 'script.sh' files were not set as executable\n\n0.3.1 (2014-08-26)\n------------------\n\n(reprozip only)\n\nBugfixes:\n* Don't crash when packing an experiment that wrote in temporary directories\n\n0.3 (2014-07-28)\n----------------\n\nBugfixes:\n* Handles Linux changing thread id to thread leader's on `execve()`\n* Correctly handles processes dying from signals (e.g. SEGV)\n* Fixes case of rt_sigreturn\n\nFeatures:\n* Database stores `is_directory` field\n* Handles `mkdir()`, `symlink()`\n* Forces pack to have a `.rpz` extension\n* Displays a warning when the process uses `connect()` or `accept()`\n* Improved logging\n* Handles i386 compatibility mode on x86_64\n* Handles *at() variants of system calls with AT_FDCWD\n\n0.2.1 (2014-07-11)\n------------------\n\nBugfixes:\n* 'pack' no longer stop if a file is missing\n* Do not attempt to pack files from /proc or /dev\n* Stops vagrant without --use-chroot from overwriting files\n* Downloads busybox instead of using the host's /bin/sh\n* Correctly packs the dynamic linkers from the original machine\n* The tracer no longer considers `execve()` to always happen before other accesses\n* Fixes pytracer forking the Python process if the executable cannot be found\n* Improves signal handling (but bugs might still exist with `SIGSTOP`)\n* Fixes a bug if a process resumed before its creator (race condition)\n\nFeatures:\n* -v flag also controls C tracer's verbosity\n* Detects (but doesn't handle yet) i386 compatibility and x32 mode on x86_64\n* Stores working directories and exit codes of all processes in database\n* Added `reprozip reset [-d dir]` to reset the configuration file from the database\n\n0.2 (2014-06-18)\n----------------\n\nFirst version of the rewritten ReproZip, that uses ptrace. Basic functionality.\n\n0.1 (2013-06-25)\n----------------\n\nSystemTap-based version of ReproZip.\n" }, { "path": "CITATION.cff", "content": "cff-version: 1.2.0\nmessage: If you use this software, please cite it as below.\nauthors:\n - family-names: Rampin\n given-names: Remi\n affiliation: New York University\n orcid: https://orcid.org/0000-0002-0524-2282\n website: https://remi.rampin.org/\n - family-names: Freire\n given-names: Juliana\n affiliation: New York University\n orcid: https://orcid.org/0000-0003-3915-7075\n website: https://vgc.poly.edu/~juliana/\n - family-names: Chirigati\n given-names: Fernando\n affiliation: New York University\n orcid: https://orcid.org/0000-0002-9566-5835\n website: http://fchirigati.com/\n - family-names: Shasha\n given-names: Dennis\n affiliation: New York University\n orcid: https://orcid.org/0000-0002-7036-3312\n website: http://cs.nyu.edu/shasha/\n - family-names: Rampin\n given-names: Vicky\n affiliation: New York University\n orcid: https://orcid.org/0000-0003-4298-168X\n website: https://vicky.rampin.org/\nlicense: BSD-3-Clause\nurl: https://www.reprozip.org/\nrepository-code: https://github.com/VIDA-NYU/reprozip\ntitle: ReproZip\nabstract: |\n ReproZip is a tool aimed at simplifying the process of creating reproducible experiments from command-line executions, a frequently-used common denominator in computational science. It tracks operating system calls and creates a package that contains all the binaries, files and dependencies required to run a given command on the author's computational environment (packing step).\nkeywords: [python, linux, docker, reproducibility, provenance, reproducible-research, reproducible-science]\nreferences:\n - type: proceedings\n doi: 10.1145/2882903.2899401\n conference:\n name: \"SIGMOD '16\"\n website: https://www.sigmod2016.org/\n city: San Francisco\n country: US\n authors:\n - family-names: Rampin\n given-names: Remi\n affiliation: New York University\n orcid: https://orcid.org/0000-0002-0524-2282\n website: https://remi.rampin.org/\n - family-names: Freire\n given-names: Juliana\n affiliation: New York University\n orcid: https://orcid.org/0000-0003-3915-7075\n website: https://vgc.poly.edu/~juliana/\n - family-names: Chirigati\n given-names: Fernando\n affiliation: New York University\n orcid: https://orcid.org/0000-0002-9566-5835\n website: http://fchirigati.com/\n - family-names: Shasha\n given-names: Dennis\n affiliation: New York University\n orcid: https://orcid.org/0000-0002-7036-3312\n website: http://cs.nyu.edu/shasha/\n date-published: 2016-06-26\n year: 2016\n month: 6\n title: \"ReproZip: Computational Reproducibility With Ease\"\n abstract: |\n We present ReproZip, the recommended packaging tool for the SIGMOD Reproducibility Review. ReproZip was designed to simplify the process of making an existing computational experiment reproducible across platforms, even when the experiment was put together without reproducibility in mind. The tool creates a self-contained package for an experiment by automatically tracking and identifying all its required dependencies. The researcher can share the package with others, who can then use ReproZip to unpack the experiment, reproduce the findings on their favorite operating system, as well as modify the original experiment for reuse in new research, all with little effort. The demo will consist of examples of non-trivial experiments, showing how these can be packed in a Linux machine and reproduced on different machines and operating systems. Demo visitors will also be able to pack and reproduce their own experiments.\n - type: proceedings\n doi: 10.21105/joss.00107\n journal: Journal of Open Source Software\n authors:\n - family-names: Rampin\n given-names: Remi\n affiliation: New York University\n orcid: https://orcid.org/0000-0002-0524-2282\n website: https://remi.rampin.org/\n - family-names: Freire\n given-names: Juliana\n affiliation: New York University\n orcid: https://orcid.org/0000-0003-3915-7075\n website: https://vgc.poly.edu/~juliana/\n - family-names: Chirigati\n given-names: Fernando\n affiliation: New York University\n orcid: https://orcid.org/0000-0002-9566-5835\n website: http://fchirigati.com/\n - family-names: Shasha\n given-names: Dennis\n affiliation: New York University\n orcid: https://orcid.org/0000-0002-7036-3312\n website: http://cs.nyu.edu/shasha/\n - family-names: Rampin\n given-names: Vicky\n affiliation: New York University\n orcid: https://orcid.org/0000-0003-4298-168X\n website: https://vicky.rampin.org/\n date-published: 2016-12-01\n year: 2016\n month: 12\n title: \"ReproZip: The Reproducibility Packer\"\npreferred-citation:\n type: proceedings\n doi: 10.1145/2882903.2899401\n conference:\n name: \"SIGMOD '16\"\n website: https://www.sigmod2016.org/\n city: San Francisco\n country: US\n authors:\n - family-names: Rampin\n given-names: Remi\n affiliation: New York University\n orcid: https://orcid.org/0000-0002-0524-2282\n website: https://remi.rampin.org/\n - family-names: Freire\n given-names: Juliana\n affiliation: New York University\n orcid: https://orcid.org/0000-0003-3915-7075\n website: https://vgc.poly.edu/~juliana/\n - family-names: Chirigati\n given-names: Fernando\n affiliation: New York University\n orcid: https://orcid.org/0000-0002-9566-5835\n website: http://fchirigati.com/\n - family-names: Shasha\n given-names: Dennis\n affiliation: New York University\n orcid: https://orcid.org/0000-0002-7036-3312\n website: http://cs.nyu.edu/shasha/\n date-published: 2016-06-26\n year: 2016\n month: 6\n title: \"ReproZip: Computational Reproducibility With Ease\"\n abstract: |\n We present ReproZip, the recommended packaging tool for the SIGMOD Reproducibility Review. ReproZip was designed to simplify the process of making an existing computational experiment reproducible across platforms, even when the experiment was put together without reproducibility in mind. The tool creates a self-contained package for an experiment by automatically tracking and identifying all its required dependencies. The researcher can share the package with others, who can then use ReproZip to unpack the experiment, reproduce the findings on their favorite operating system, as well as modify the original experiment for reuse in new research, all with little effort. The demo will consist of examples of non-trivial experiments, showing how these can be packed in a Linux machine and reproduced on different machines and operating systems. Demo visitors will also be able to pack and reproduce their own experiments.\nversion: \"1.1\"\ndate-released: 2021-07-06\ndoi: 10.5281/zenodo.5081097\n" }, { "path": "CITATION.txt", "content": "To reference ReproZip in publications, please cite the following:\n\nReproZip: Computational Reproducibility With Ease, F. Chirigati, R. Rampin, D. Shasha, and J. Freire. In Proceedings of the 2016 ACM SIGMOD International Conference on Management of Data (SIGMOD), pp. 2085-2088, 2016\n\nBibTeX:\n\n@inproceedings{ChirigatiRSF16,\n title = {ReproZip: Computational Reproducibility With Ease},\n author = {Chirigati, Fernando and Rampin, R{\\'e}mi and Shasha, Dennis and Freire, Juliana},\n year = {2016},\n isbn = {978-1-4503-3531-7},\n location = {San Francisco, California, USA},\n doi = {10.1145/2882903.2899401},\n booktitle = {Proceedings of the 2016 International Conference on Management of Data},\n series = {SIGMOD '16},\n pages = {2085--2088},\n numpages = {4},\n publisher = {ACM},\n keywords = {computational reproducibility, provenance, reprozip},\n}\n" }, { "path": "CONTRIBUTING.md", "content": "# Contents\n* [Notes](#notes)\n* [Contributing](#contributing)\n* [Resolving Merge Conflicts](#resolving-merge-conflicts)\n* [Best Practices for Contributing](#best-practices-for-contributing)\n* [Attribution](#attribution)\n\n# Notes\n\nAny contributions received are assumed to be covered by the BSD 3-Clause license. We might ask you to sign a Contributor License Agreement before accepting a larger contribution. To learn more about ReproZip, see:\n* [ReproZip Examples](https://examples.reprozip.org/)\n* [ReproZip Documentation](https://docs.reprozip.org/)\n* [ReproZip Demo Video](https://www.youtube.com/watch?v=-zLPuwCHXo0)\n\n# Contributing\n\nPlease follow the [GitHub Community Guidelines](https://docs.github.com/en/github/site-policy/github-community-guidelines) in all your interactions with the project. If you would like to contribute to this project by modifying/adding to the code, please read the [Best Practices for Contributing](#best-practices-for-contributing) below and feel free to follow the standard GitHub workflow:\n\n1. Fork the project.\n2. Clone your fork to your computer.\n * From the command line: `git clone https://github.com//reprozip.git`\n3. Change into your new project folder.\n * From the command line: `cd reprozip`\n4. [optional] Add the upstream repository to your list of remotes.\n * From the command line: `git remote add upstream https://github.com/ViDA-NYU/reprozip.git`\n5. Create a branch for your new feature.\n * From the command line: `git checkout -b my-feature-branch-name`\n6. Make your changes.\n * Avoid making changes to more files than necessary for your feature (i.e. refrain from combining your \"real\" pull request with incidental bug fixes). This will simplify the merging process and make your changes clearer.\n7. Commit your changes. From the command line:\n * `git add `\n * `git commit -m \"A descriptive commit message\"`\n8. While you were working some other changes might have gone in and break your stuff or vice versa. This can be a *merge conflict* but also conflicting behavior or code. Before you test, merge with upstream.\n * `git fetch upstream`\n * `git merge upstream/1.x`\n9. Test. Run the program and do something related to your feature/fix.\n10. Push the branch, uploading it to GitHub.\n * `git push origin my-feature-branch-name`\n11. Make a \"Pull Request\" from your branch here on GitHub.\n\n# Resolving Merge Conflicts\n\nDepending on the order that Pull Requests get processed, your PR may result in a conflict and become un-mergable. To correct this, do the following from the command line:\n\nSwitch to your branch: `git checkout my-feature-branch-name`\nPull in the latest upstream changes: `git pull upstream 1.x`\nFind out what files have a conflict: `git status`\n\nEdit the conflicting file(s) and look for a block that looks like this:\n```\n<<<<<<< HEAD\nmy awesome change\n=======\nsome other person's less awesome change\n>>>>>>> some-branch\n```\n\nReplace all five (or more) lines with the correct version (yours, theirs, or\na combination of the two). ONLY the correct content should remain (none of\nthat `<<<<< HEAD` stuff.)\n\nThen re-commit and re-push the file.\n\n```\ngit add the-changed-file.cs\ngit commit -m \"Resolved conflict between this and PR #123\"\ngit push origin my-feature-branch-name\n```\n\nThe pull request should automatically update to reflect your changes.\n\n## Best Practices for Contributing\n\n* Before you start coding, open an issue so that the community can discuss your change to ensure it is in line with the goals of the project and not being worked on by someone else. This allows for discussion and fine tuning of your feature and results in a more succent and focused additions.\n * If you are fixing a small glitch or bug, you may make a PR without opening an issue.\n * If you are adding a large feature, create an issue so that we may give you feedback and agree on what makes the most sense for the project before making your change and submitting a PR (this will make sure you don't have to do major changes down the line).\n\n* Pull Requests are eventually merged into the codebase. Please ensure they are:\n * Well tested by the author. It is the author's job to ensure their code works as expected.\n * Free of unnecessary log calls. Logging important for debugging, but when a PR is made, log calls should only be present when there is an actual error or to record some important piece of information or progress.\n\n* If your code is untested, log heavy, or incomplete, prefix your PR with \"[WIP]\", so others know it is still being tested and shouldn't be considered for merging yet. This way we can still give you feedback or help finalize the feature even if it's not ready for prime time.\n\nThat's it! Following these guidelines will ensure that your additions are approved quickly and integrated into the project. Thanks for your contribution!\n\n# Attribution\n\nThis CONTRIBUTING.md was adapted from [ProjectPorcupine's](https://github.com/TeamPorcupine/ProjectPorcupine)'s [CONTRIBUTING.md](https://github.com/TeamPorcupine/ProjectPorcupine/blob/master/CONTRIBUTING.md)\n\n# Contact info\n\nYou are welcome to [subscribe to](https://groups.google.com/a/nyu.edu/g/reprozip) or contact our user mailing list [reprozip@nyu.edu](mailto:reprozip@nyu.edu) for questions, suggestions and discussions about using ReproZip.\n" }, { "path": "LICENSE.txt", "content": "Copyright (C) 2014, New York University\nAll rights reserved.\n\nRedistribution and use in source and binary forms, with or without\nmodification, are permitted provided that the following conditions are met:\n\n1. Redistributions of source code must retain the above copyright notice, this\n list of conditions and the following disclaimer.\n\n2. Redistributions in binary form must reproduce the above copyright notice,\n this list of conditions and the following disclaimer in the documentation\n and/or other materials provided with the distribution.\n\n3. Neither the name of the copyright holder nor the names of its contributors\n may be used to endorse or promote products derived from this software\n without specific prior written permission.\n\nTHIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \"AS IS\" AND\nANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED\nWARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE\nDISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE\nFOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL\nDAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR\nSERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER\nCAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,\nOR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE\nOF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n" }, { "path": "README.md", "content": "[![Build Status](https://github.com/VIDA-NYU/reprozip/workflows/Test/badge.svg)](https://github.com/VIDA-NYU/reprozip/actions)\n[![Coverage Status](https://codecov.io/github/VIDA-NYU/reprozip/coverage.svg?branch=1.x)](https://codecov.io/github/VIDA-NYU/reprozip?branch=1.x)\n[![Documentation Status](https://readthedocs.org/projects/reprozip/badge/?version=1.x)](https://docs.reprozip.org/en/1.x/)\n[![Matrix](https://img.shields.io/badge/chat-matrix.org-blue.svg)](https://riot.im/app/#/room/#reprozip:matrix.org)\n[![paper](https://img.shields.io/badge/JOSS-10.21105%2Fjoss.00107-green.svg)](http://joss.theoj.org/papers/b578b171263c73f64dfb9d040ca80fe0)\n[![DOI](https://img.shields.io/badge/DOI-10.5281%2Fzenodo.10291844-green.svg)](https://doi.org/10.5281/zenodo.10291844)\n\nReproZip\n========\n\n[ReproZip][web] is a tool aimed at simplifying the process of creating reproducible experiments from command-line executions, a frequently-used common denominator in computational science.\n\nIt tracks operating system calls and creates a package that contains all the binaries, files and dependencies required to run a given command on the author's computational environment (packing step). A reviewer can then extract the experiment in his environment to reproduce the results (unpacking step).\n\nQuickstart\n----------\n\nWe have an [example repository](https://github.com/VIDA-NYU/reprozip-examples) with a variety of different software. Don't hesitate to check it out, and contribute your own example if use ReproZip for something new!\n\n### Packing\n\nPacking experiments is only available for Linux distributions. In the environment where the experiment is originally executed, first install reprozip:\n\n $ pip install reprozip\n\nThen, run your experiment with reprozip. Suppose you execute your experiment by originally running the following command:\n\n $ ./myexperiment -my --options inputs/somefile.csv other_file_here.bin\n\nTo run it with reprozip, you just need to use the prefix *reprozip trace*:\n\n $ reprozip trace ./myexperiment -my --options inputs/somefile.csv other_file_here.bin\n\nThis command creates a *.reprozip-trace* directory, in which you'll find the configuration file, named *config.yml*. You can edit the command line and environment variables, and choose which files to pack.\n\nIf you are using Debian or Ubuntu, most of these files (library dependencies) are organized by package. You can add or remove files, or choose not to include a package by changing option *packfiles* from true to false. In this way, smaller packs can be created with reprozip (if space is an issue), and reprounzip can download these files from the package manager; however, note this is only available for Debian and Ubuntu for now, and also be aware that package versions might differ. Choosing which files to pack is also important to remove sensitive information and third-party software that is not open source and should not be distributed.\n\nOnce done editing the configuration file (or even if you did not change anything), run the following command to create a ReproZip package named *my_experiment*:\n\n $ reprozip pack my_experiment.rpz\n\nVoilà! Now your experiment has been packed, and you can send it to your collaborators, reviewers, and researchers around the world!\n\nNote that you can open the help message for any reprozip command by using the flag *-h*.\n\n### Unpacking\n\nDo you need to unpack an experiment in a Linux machine? Easy! First, install reprounzip:\n\n $ pip install reprounzip\n\nThen, if you want to unpack everything in a single directory named *mydirectory* and execute the experiment from there, use the prefix *reprounzip directory*:\n\n $ reprounzip directory setup my_experiment.rpz mydirectory\n $ reprounzip directory run mydirectory\n\nIn case you prefer to build a chroot environment under *mychroot*, use the prefix *reprounzip chroot*:\n\n $ reprounzip chroot setup my_experiment.rpz mychroot\n $ reprounzip chroot run mychroot\n\nNote that the previous options do not interfere with the original configuration of the environment, so don't worry! If you are using Debian or Ubuntu, reprounzip also has an option to install all the library dependencies directly on the machine using package managers (rather than just copying the files from the .rpz package). Be aware that this will interfere in your environment and it may update your library packages, so use it at your own risk! For this option, just use the prefix *reprounzip installpkgs*:\n\n $ reprounzip installpkgs my_experiment.rpz\n\nWhat if you want to reproduce the experiment in Windows or Mac OS X? You can build a virtual machine with the experiment! Easy as well! First, install the plugin reprounzip-vagrant:\n\n $ pip install reprounzip-vagrant\n\nNote that (i) you must install reprounzip first, and (ii) the plugin requires having [Vagrant][vagrant] installed. Then, use the prefix *reprounzip vagrant* to create and start a virtual machine under directory *mytemplate*:\n\n $ reprounzip vagrant setup my_experiment.rpz mytemplate\n\nTo execute the experiment, simply run:\n\n $ reprounzip vagrant run mytemplate\n\nAlternatively, you may use [Docker][docker] containers to reproduce the experiment, which also works under Linux, Mac OS X, and Windows! First, install the plugin reprounzip-docker:\n\n $ pip install reprounzip-docker\n\nThen, assuming that you want to create the container under directory *mytemplate*, simply use the prefix *reprounzip docker*:\n\n $ reprounzip docker setup my_experiment.rpz mytemplate\n $ reprounzip docker run mytemplate\n\nRemember that you can open the help message and learn more about other available flags and options by using the flag *-h* for any reprounzip command.\n\nCiting ReproZip\n---------------\n\nPlease use the following when citing ReproZip ([BibTeX](CITATION.txt)):\n\n ReproZip: Computational Reproducibility With Ease\n F. Chirigati, R. Rampin, D. Shasha, and J. Freire.\n In Proceedings of the 2016 ACM SIGMOD International Conference on Management of Data (SIGMOD), pp. 2085-2088, 2016\n\nContribute\n----------\n\nPlease subscribe to and contact the [reprozip@nyu.edu](https://groups.google.com/a/nyu.edu/g/reprozip) mailing list for questions, suggestions and discussions about using reprozip.\n\nBugs and feature plannings are tracked in the [GitHub issues](https://github.com/VIDA-NYU/reprozip/issues). Feel free to add an issue!\n\nTo suggest changes to this source code, feel free to raise a [GitHub pull request](https://github.com/VIDA-NYU/reprozip/pulls). Any contributions received are assumed to be covered by the [BSD 3-Clause license](LICENSE.txt). We might ask you to sign a _Contributor License Agreement_ before accepting a larger contribution.\n\nLicense\n-------\n\n* Copyright (C) 2014, New York University\n\nLicensed under a **BSD 3-Clause license**. See the file [LICENSE.txt](LICENSE.txt) for details.\n\nLinks and References\n--------------------\n\nFor more detailed information, please refer to our [website][web], as well as to our [documentation][docs].\n\nReproZip is currently being developed at [NYU][nyu]. The team includes:\n\n* [Fernando Chirigati][fc]\n* [Juliana Freire][jf]\n* [Rémi Rampin][rr]\n* [Dennis Shasha][ds]\n* [Vicky Rampin][vr]\n\n[vagrant]: https://www.vagrantup.com/\n[docker]: https://www.docker.com/\n[docs]: https://docs.reprozip.org/\n[web]: https://www.reprozip.org/\n[pz]: https://pypi.python.org/pypi/reprozip\n[puz]: https://pypi.python.org/pypi/reprounzip\n[puzd]: https://pypi.python.org/pypi/reprounzip-docker\n[puzv]: https://pypi.python.org/pypi/reprounzip-vagrant\n[fc]: http://fchirigati.com/\n[jf]: https://vgc.poly.edu/~juliana/\n[rr]: https://remi.rampin.org/\n[ds]: http://cs.nyu.edu/shasha/\n[vr]: https://vicky.rampin.org/\n[nyu]: http://engineering.nyu.edu/\n" }, { "path": "Vagrantfile", "content": "# -*- mode: ruby -*-\n# vi: set ft=ruby :\n\nVAGRANTFILE_API_VERSION = \"2\"\n\nVagrant.configure(VAGRANTFILE_API_VERSION) do |config|\n config.vm.provision \"shell\",\n inline: <\" >&2\n exit 1\nfi\n" }, { "path": "docs/Makefile", "content": "# Makefile for Sphinx documentation\n#\n\n# You can set these variables from the command line.\nSPHINXOPTS =\nSPHINXBUILD = sphinx-build\nPAPER =\nBUILDDIR = _build\n\n# Internal variables.\nPAPEROPT_a4 = -D latex_paper_size=a4\nPAPEROPT_letter = -D latex_paper_size=letter\nALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) .\n# the i18n builder cannot share the environment and doctrees with the others\nI18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) .\n\n.PHONY: help\nhelp:\n\t@echo \"Please use \\`make ' where is one of\"\n\t@echo \" html to make standalone HTML files\"\n\t@echo \" dirhtml to make HTML files named index.html in directories\"\n\t@echo \" singlehtml to make a single large HTML file\"\n\t@echo \" pickle to make pickle files\"\n\t@echo \" json to make JSON files\"\n\t@echo \" htmlhelp to make HTML files and a HTML help project\"\n\t@echo \" qthelp to make HTML files and a qthelp project\"\n\t@echo \" applehelp to make an Apple Help Book\"\n\t@echo \" devhelp to make HTML files and a Devhelp project\"\n\t@echo \" epub to make an epub\"\n\t@echo \" epub3 to make an epub3\"\n\t@echo \" latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter\"\n\t@echo \" latexpdf to make LaTeX files and run them through pdflatex\"\n\t@echo \" latexpdfja to make LaTeX files and run them through platex/dvipdfmx\"\n\t@echo \" text to make text files\"\n\t@echo \" man to make manual pages\"\n\t@echo \" texinfo to make Texinfo files\"\n\t@echo \" info to make Texinfo files and run them through makeinfo\"\n\t@echo \" gettext to make PO message catalogs\"\n\t@echo \" changes to make an overview of all changed/added/deprecated items\"\n\t@echo \" xml to make Docutils-native XML files\"\n\t@echo \" pseudoxml to make pseudoxml-XML files for display purposes\"\n\t@echo \" linkcheck to check all external links for integrity\"\n\t@echo \" doctest to run all doctests embedded in the documentation (if enabled)\"\n\t@echo \" coverage to run coverage check of the documentation (if enabled)\"\n\t@echo \" dummy to check syntax errors of document sources\"\n\n.PHONY: clean\nclean:\n\trm -rf $(BUILDDIR)/*\n\n.PHONY: html\nhtml:\n\t$(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html\n\t@echo\n\t@echo \"Build finished. The HTML pages are in $(BUILDDIR)/html.\"\n\n.PHONY: dirhtml\ndirhtml:\n\t$(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml\n\t@echo\n\t@echo \"Build finished. The HTML pages are in $(BUILDDIR)/dirhtml.\"\n\n.PHONY: singlehtml\nsinglehtml:\n\t$(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml\n\t@echo\n\t@echo \"Build finished. The HTML page is in $(BUILDDIR)/singlehtml.\"\n\n.PHONY: pickle\npickle:\n\t$(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle\n\t@echo\n\t@echo \"Build finished; now you can process the pickle files.\"\n\n.PHONY: json\njson:\n\t$(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json\n\t@echo\n\t@echo \"Build finished; now you can process the JSON files.\"\n\n.PHONY: htmlhelp\nhtmlhelp:\n\t$(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp\n\t@echo\n\t@echo \"Build finished; now you can run HTML Help Workshop with the\" \\\n\t \".hhp project file in $(BUILDDIR)/htmlhelp.\"\n\n.PHONY: qthelp\nqthelp:\n\t$(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp\n\t@echo\n\t@echo \"Build finished; now you can run \"qcollectiongenerator\" with the\" \\\n\t \".qhcp project file in $(BUILDDIR)/qthelp, like this:\"\n\t@echo \"# qcollectiongenerator $(BUILDDIR)/qthelp/ReproZip.qhcp\"\n\t@echo \"To view the help file:\"\n\t@echo \"# assistant -collectionFile $(BUILDDIR)/qthelp/ReproZip.qhc\"\n\n.PHONY: applehelp\napplehelp:\n\t$(SPHINXBUILD) -b applehelp $(ALLSPHINXOPTS) $(BUILDDIR)/applehelp\n\t@echo\n\t@echo \"Build finished. The help book is in $(BUILDDIR)/applehelp.\"\n\t@echo \"N.B. You won't be able to view it unless you put it in\" \\\n\t \"~/Library/Documentation/Help or install it in your application\" \\\n\t \"bundle.\"\n\n.PHONY: devhelp\ndevhelp:\n\t$(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp\n\t@echo\n\t@echo \"Build finished.\"\n\t@echo \"To view the help file:\"\n\t@echo \"# mkdir -p $$HOME/.local/share/devhelp/ReproZip\"\n\t@echo \"# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/ReproZip\"\n\t@echo \"# devhelp\"\n\n.PHONY: epub\nepub:\n\t$(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub\n\t@echo\n\t@echo \"Build finished. The epub file is in $(BUILDDIR)/epub.\"\n\n.PHONY: epub3\nepub3:\n\t$(SPHINXBUILD) -b epub3 $(ALLSPHINXOPTS) $(BUILDDIR)/epub3\n\t@echo\n\t@echo \"Build finished. The epub3 file is in $(BUILDDIR)/epub3.\"\n\n.PHONY: latex\nlatex:\n\t$(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex\n\t@echo\n\t@echo \"Build finished; the LaTeX files are in $(BUILDDIR)/latex.\"\n\t@echo \"Run \\`make' in that directory to run these through (pdf)latex\" \\\n\t \"(use \\`make latexpdf' here to do that automatically).\"\n\n.PHONY: latexpdf\nlatexpdf:\n\t$(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex\n\t@echo \"Running LaTeX files through pdflatex...\"\n\t$(MAKE) -C $(BUILDDIR)/latex all-pdf\n\t@echo \"pdflatex finished; the PDF files are in $(BUILDDIR)/latex.\"\n\n.PHONY: latexpdfja\nlatexpdfja:\n\t$(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex\n\t@echo \"Running LaTeX files through platex and dvipdfmx...\"\n\t$(MAKE) -C $(BUILDDIR)/latex all-pdf-ja\n\t@echo \"pdflatex finished; the PDF files are in $(BUILDDIR)/latex.\"\n\n.PHONY: text\ntext:\n\t$(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text\n\t@echo\n\t@echo \"Build finished. The text files are in $(BUILDDIR)/text.\"\n\n.PHONY: man\nman:\n\t$(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man\n\t@echo\n\t@echo \"Build finished. The manual pages are in $(BUILDDIR)/man.\"\n\n.PHONY: texinfo\ntexinfo:\n\t$(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo\n\t@echo\n\t@echo \"Build finished. The Texinfo files are in $(BUILDDIR)/texinfo.\"\n\t@echo \"Run \\`make' in that directory to run these through makeinfo\" \\\n\t \"(use \\`make info' here to do that automatically).\"\n\n.PHONY: info\ninfo:\n\t$(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo\n\t@echo \"Running Texinfo files through makeinfo...\"\n\tmake -C $(BUILDDIR)/texinfo info\n\t@echo \"makeinfo finished; the Info files are in $(BUILDDIR)/texinfo.\"\n\n.PHONY: gettext\ngettext:\n\t$(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale\n\t@echo\n\t@echo \"Build finished. The message catalogs are in $(BUILDDIR)/locale.\"\n\n.PHONY: changes\nchanges:\n\t$(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes\n\t@echo\n\t@echo \"The overview file is in $(BUILDDIR)/changes.\"\n\n.PHONY: linkcheck\nlinkcheck:\n\t$(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck\n\t@echo\n\t@echo \"Link check complete; look for any errors in the above output \" \\\n\t \"or in $(BUILDDIR)/linkcheck/output.txt.\"\n\n.PHONY: doctest\ndoctest:\n\t$(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest\n\t@echo \"Testing of doctests in the sources finished, look at the \" \\\n\t \"results in $(BUILDDIR)/doctest/output.txt.\"\n\n.PHONY: coverage\ncoverage:\n\t$(SPHINXBUILD) -b coverage $(ALLSPHINXOPTS) $(BUILDDIR)/coverage\n\t@echo \"Testing of coverage in the sources finished, look at the \" \\\n\t \"results in $(BUILDDIR)/coverage/python.txt.\"\n\n.PHONY: xml\nxml:\n\t$(SPHINXBUILD) -b xml $(ALLSPHINXOPTS) $(BUILDDIR)/xml\n\t@echo\n\t@echo \"Build finished. The XML files are in $(BUILDDIR)/xml.\"\n\n.PHONY: pseudoxml\npseudoxml:\n\t$(SPHINXBUILD) -b pseudoxml $(ALLSPHINXOPTS) $(BUILDDIR)/pseudoxml\n\t@echo\n\t@echo \"Build finished. The pseudo-XML files are in $(BUILDDIR)/pseudoxml.\"\n\n.PHONY: dummy\ndummy:\n\t$(SPHINXBUILD) -b dummy $(ALLSPHINXOPTS) $(BUILDDIR)/dummy\n\t@echo\n\t@echo \"Build finished. Dummy builder generates no files.\"\n" }, { "path": "docs/conf.py", "content": "# -*- coding: utf-8 -*-\n#\n# ReproZip documentation build configuration file, created by\n# sphinx-quickstart on Tue May 31 15:43:00 2016.\n#\n# This file is execfile()d with the current directory set to its\n# containing dir.\n#\n# Note that not all possible configuration values are present in this\n# autogenerated file.\n#\n# All configuration values have a default; values that are commented out\n# serve to show the default.\n\n# If extensions (or modules to document with autodoc) are in another directory,\n# add these directories to sys.path here. If the directory is relative to the\n# documentation root, use os.path.abspath to make it absolute, like shown here.\nimport os\nimport sys\nsys.path.insert(0, os.path.abspath('..'))\n\n# -- General configuration ------------------------------------------------\n\n# If your documentation needs a minimal Sphinx version, state it here.\n#\n# needs_sphinx = '1.0'\n\n# Add any Sphinx extension module names here, as strings. They can be\n# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom\n# ones.\nextensions = [\n 'sphinx.ext.autodoc',\n 'sphinx.ext.intersphinx',\n]\n\n# Add any paths that contain templates here, relative to this directory.\ntemplates_path = ['_templates']\n\n# The suffix(es) of source filenames.\n# You can specify multiple suffix as a list of string:\n#\n# source_suffix = ['.rst', '.md']\nsource_suffix = '.rst'\n\n# The encoding of source files.\n#\n# source_encoding = 'utf-8-sig'\n\n# The master toctree document.\nmaster_doc = 'index'\n\n# General information about the project.\nproject = u'ReproZip'\ncopyright = u'2014, New York University'\n\nauthors = [u'Fernando Chirigati', u'Remi Rampin',\n u'Juliana Freire', u'Dennis Shasha']\nif len(authors) <= 1:\n authors_str = u', '.join(authors)\nelse:\n authors_str = u', '.join(authors[:-1]) + u', and ' + authors[-1]\n\n# The version info for the project you're documenting, acts as replacement for\n# |version| and |release|, also used in various other places throughout the\n# built documents.\n#\n# The short X.Y version.\nversion = u'1.3'\n# The full version, including alpha/beta/rc tags.\nrelease = u'1.3'\n\n# The language for content autogenerated by Sphinx. Refer to documentation\n# for a list of supported languages.\n#\n# This is also used if you do content translation via gettext catalogs.\n# Usually you set \"language\" from the command line for these cases.\nlanguage = 'en'\n\n# There are two options for replacing |today|: either, you set today to some\n# non-false value, then it is used:\n#\n# today = ''\n#\n# Else, today_fmt is used as the format for a strftime call.\n#\n# today_fmt = '%B %d, %Y'\n\n# List of patterns, relative to source directory, that match files and\n# directories to ignore when looking for source files.\n# This patterns also effect to html_static_path and html_extra_path\nexclude_patterns = ['_build', 'Thumbs.db', '.DS_Store']\n\n# The reST default role (used for this markup: `text`) to use for all\n# documents.\n#\n# default_role = None\n\n# If true, '()' will be appended to :func: etc. cross-reference text.\n#\n# add_function_parentheses = True\n\n# If true, the current module name will be prepended to all description\n# unit titles (such as .. function::).\n#\n# add_module_names = True\n\n# If true, sectionauthor and moduleauthor directives will be shown in the\n# output. They are ignored by default.\n#\n# show_authors = False\n\n# The name of the Pygments (syntax highlighting) style to use.\npygments_style = 'sphinx'\n\n# A list of ignored prefixes for module index sorting.\n# modindex_common_prefix = []\n\n# If true, keep warnings as \"system message\" paragraphs in the built documents.\n# keep_warnings = False\n\n# If true, `todo` and `todoList` produce output, else they produce nothing.\ntodo_include_todos = False\n\n\n# -- Options for HTML output ----------------------------------------------\n\n# The theme to use for HTML and HTML Help pages. See the documentation for\n# a list of builtin themes.\n#\nhtml_theme = 'default'\n\n# Theme options are theme-specific and customize the look and feel of a theme\n# further. For a list of options available for each theme, see the\n# documentation.\n#\n# html_theme_options = {}\n\n# Add any paths that contain custom themes here, relative to this directory.\n# html_theme_path = []\n\n# The name for this set of Sphinx documents.\n# \" v documentation\" by default.\n#\n# html_title = u'ReproZip'\n\n# A shorter title for the navigation bar. Default is the same as html_title.\n#\n# html_short_title = None\n\n# The name of an image file (relative to this directory) to place at the top\n# of the sidebar.\n#\n# html_logo = None\n\n# The name of an image file (relative to this directory) to use as a favicon of\n# the docs. This file should be a Windows icon file (.ico) being 16x16 or\n# 32x32 pixels large.\n#\n# html_favicon = None\n\n# Add any paths that contain custom static files (such as style sheets) here,\n# relative to this directory. They are copied after the builtin static files,\n# so a file named \"default.css\" will overwrite the builtin \"default.css\".\nhtml_static_path = ['_static']\n\n# Add any extra paths that contain custom files (such as robots.txt or\n# .htaccess) here, relative to this directory. These files are copied\n# directly to the root of the documentation.\n#\n# html_extra_path = []\n\n# If not None, a 'Last updated on:' timestamp is inserted at every page\n# bottom, using the given strftime format.\n# The empty string is equivalent to '%b %d, %Y'.\n#\n# html_last_updated_fmt = None\n\n# If true, SmartyPants will be used to convert quotes and dashes to\n# typographically correct entities.\n#\n# html_use_smartypants = True\n\n# Custom sidebar templates, maps document names to template names.\n#\n# html_sidebars = {}\n\n# Additional templates that should be rendered to pages, maps page names to\n# template names.\n#\n# html_additional_pages = {}\n\n# If false, no module index is generated.\n#\n# html_domain_indices = True\n\n# If false, no index is generated.\n#\n# html_use_index = True\n\n# If true, the index is split into individual pages for each letter.\n#\n# html_split_index = False\n\n# If true, links to the reST sources are added to the pages.\n#\n# html_show_sourcelink = True\n\n# If true, \"Created using Sphinx\" is shown in the HTML footer. Default is True.\n#\n# html_show_sphinx = True\n\n# If true, \"(C) Copyright ...\" is shown in the HTML footer. Default is True.\n#\n# html_show_copyright = True\n\n# If true, an OpenSearch description file will be output, and all pages will\n# contain a tag referring to it. The value of this option must be the\n# base URL from which the finished HTML is served.\n#\n# html_use_opensearch = ''\n\n# This is the file name suffix for HTML files (e.g. \".xhtml\").\n# html_file_suffix = None\n\n# Language to be used for generating the HTML full-text search index.\n# Sphinx supports the following languages:\n# 'da', 'de', 'en', 'es', 'fi', 'fr', 'hu', 'it', 'ja'\n# 'nl', 'no', 'pt', 'ro', 'ru', 'sv', 'tr', 'zh'\n#\n# html_search_language = 'en'\n\n# A dictionary with options for the search language support, empty by default.\n# 'ja' uses this config value.\n# 'zh' user can custom change `jieba` dictionary path.\n#\n# html_search_options = {'type': 'default'}\n\n# The name of a javascript file (relative to the configuration directory) that\n# implements a search results scorer. If empty, the default will be used.\n#\n# html_search_scorer = 'scorer.js'\n\n# Output file base name for HTML help builder.\nhtmlhelp_basename = 'ReproZip'\n\n# -- Options for LaTeX output ---------------------------------------------\n\nlatex_elements = {\n # The paper size ('letterpaper' or 'a4paper').\n #\n # 'papersize': 'letterpaper',\n\n # The font size ('10pt', '11pt' or '12pt').\n #\n # 'pointsize': '10pt',\n\n # Additional stuff for the LaTeX preamble.\n #\n # 'preamble': '',\n\n # Latex figure (float) alignment\n #\n # 'figure_align': 'htbp',\n}\n\n# Grouping the document tree into LaTeX files. List of tuples\n# (source start file, target name, title,\n# author, documentclass [howto, manual, or own class]).\nlatex_documents = [\n (master_doc, 'ReproZip.tex', u'ReproZip Documentation',\n authors_str, 'manual'),\n]\n\n# The name of an image file (relative to this directory) to place at the top of\n# the title page.\n#\n# latex_logo = None\n\n# For \"manual\" documents, if this is true, then toplevel headings are parts,\n# not chapters.\n#\n# latex_use_parts = False\n\n# If true, show page references after internal links.\n#\n# latex_show_pagerefs = False\n\n# If true, show URL addresses after external links.\n#\n# latex_show_urls = False\n\n# Documents to append as an appendix to all manuals.\n#\n# latex_appendices = []\n\n# If false, no module index is generated.\n#\n# latex_domain_indices = True\n\n\n# -- Options for manual page output ---------------------------------------\n\n# One entry per manual page. List of tuples\n# (source start file, name, description, authors, manual section).\nman_pages = []\n\n# If true, show URL addresses after external links.\n#\n# man_show_urls = False\n\n\n# -- Options for Texinfo output -------------------------------------------\n\n# Grouping the document tree into Texinfo files. List of tuples\n# (source start file, target name, title, author,\n# dir menu entry, description, category)\ntexinfo_documents = [\n (master_doc, 'ReproZip', u'ReproZip Documentation',\n authors_str, 'ReproZip',\n u\"Allows the reproducibility of command-line experiments in a different \"\n \"environment\",\n 'Miscellaneous'),\n]\n\n# Documents to append as an appendix to all manuals.\n#\n# texinfo_appendices = []\n\n# If false, no module index is generated.\n#\n# texinfo_domain_indices = True\n\n# How to display URL addresses: 'footnote', 'no', or 'inline'.\n#\n# texinfo_show_urls = 'footnote'\n\n# If true, do not generate a @detailmenu in the \"Top\" node's menu.\n#\n# texinfo_no_detailmenu = False\n\n\n# Example configuration for intersphinx: refer to the Python standard library.\nintersphinx_mapping = {\n 'python': ('https://docs.python.org/3/', None),\n 'rpaths': ('http://rpaths.remram.fr/en/latest/', None),\n}\n" }, { "path": "docs/developerguide.rst", "content": ".. _develop-plugins:\n\nDeveloper's Guide\n*****************\n\nGeneral Development Information\n-------------------------------\n\nDevelopment happens on `GitHub `__; bug reports and feature requests are welcome. If you are interested in giving us a hand, please do not hesitate to submit a pull request there.\n\nContinuous testing is provided by `GitHub Actions `__. Note that ReproZip still tries to support Python 2 as well as Python 3. Test coverage is not very high because there are a lot of operations that are difficult to cover on CI (for instance, Vagrant VMs cannot be used over there).\n\nIf you have any questions or need help with the development of an unpacker or plugin, please use our development mailing-list at `reprozip@nyu.edu `__.\n\nIntroduction to ReproZip\n------------------------\n\nReproZip works in two steps: tracing and packing. Under the hood, tracing is two separate steps, leading to the following workflow:\n\n* Running the experiment under trace. During this part, the experiment is running, and the ``_pytracer`` C extension watches it through the `ptrace` mechanism, recording information in the trace SQLite3 database (``.reprozip-trace/trace.sqlite3``). This database contains raw information as it is recorded and does little else, leaving that to the next step. This part is referred to as the \"C tracer\".\n* After the experiment is done, some additional information is computed by the Python code to generate the configuration file, by looking at the trace database and the filesystem. For example, all accesses to a file are aggregated to decide if it is read or written by the overall experiment, if it is an input or output file, resolve symlinks, etc. Additional information is written such as OS information and which distribution package each file comes from.\n* Packing reads the configuration file to create the ``.rpz`` bundle, which includes a configuration file (re-written into a \"canonical\" version), the trace database (though it is not read at this step), and the files listed in the configuration which was possibly altered by the user.\n\nTherefore it is important to note that the configuration file and the trace database contain distinct information, and although the configuration is inferred from the database, it contains some additional details that was obtained from the original machine afterwards.\n\nOnly the configuration file should be necessary to run unpackers. The trace database is included for information, and to support additional commands like ``reprounzip graph`` (:ref:`graph`).\n\nWriting Unpackers\n-----------------\n\nReproZip is divided into two steps. The first is packing, which gives a generic package containing the trace SQLite database, the YAML configuration file (which lists the paths, packages, and metadata such as command line, environment variables, and input/output files), and actual files. In the second step, a package can be run using *reprounzip*. This decoupling allows the reproducer to select the unpacker of his/her desire, and also means that when a new unpacker is released, users will be able to use it on their old packages.\n\nCurrently, different unpackers are maintained: the defaults ones (``directory`` and ``chroot``), ``vagrant`` (distributed as `reprounzip-vagrant `__) and ``docker`` (distributed as `reprounzip-docker `__). However, the interface is such that new unpackers can be easily added. While taking a look at the \"official\" unpackers' source is probably a good idea, this page gives some useful information about how they work.\n\nReproZip Bundle Format (``.rpz``)\n'''''''''''''''''''''''''''''''''\n\nAn ``.rpz`` file is a ``tar.gz`` archive that contains a directory ``METADATA``, which contains meta-information from *reprozip*, and an archive ``DATA.tar.gz``, which contains the actual files that were packed and that will be unpacked to the target directory for reproducing the experiment.\n\nThe ``METADATA/version`` file marks the file as a ReproZip bundle. It always contains the string ``REPROZIP VERSION 2``. It previously contained ``REPROZIP VERSION 1`` before version 0.8 (2015), where ``DATA`` was a directory instead of being a tar.gz file.\n\nThe ``METADATA/config.yml`` file is in the same format as the configuration file generated by *reprozip*, but without the ``additional_patterns`` section (at this point, it has already been expanded to the actual list of files while packing).\n\nThe ``METADATA/trace.sqlite3`` file is the original trace generated by the C tracer and maintained in a SQLite database; it contains all the information about the experiment, in case the configuration file is insufficient in some aspect. This file is used, for instance, by the *graph* unpacker, so that it can recover the exact hierarchy of processes, together with the executable images they execute and the files they access (with the time and mode of these accesses).\n\n.. seealso:: :ref:`Trace Database Schema `\n\nStructure\n'''''''''\n\nAn unpacker is a Python module. It can be distributed separately or be a part of a bigger distribution, given that it is declared in that distribution's ``setup.py`` as an `entry_point` to be registered with `importlib` (see `setuptools' advertising behavior section `__). You should declare a function as `entry_point` ``reprounzip.unpackers``. The name of the entry_point (before ``=``) will be the *reprounzip* subcommand, and the value is a callable that will get called with the :class:`argparse.ArgumentParser` object for that subcommand.\n\nThe package :mod:`reprounzip.unpackers` is a namespace package, so you should be able to add your own unpackers there if you want to. Please remember to put the correct code in the ``__init__.py`` file (which you can copy from `here `__) so that namespace packages work correctly.\n\nThe modules :mod:`reprounzip.common`, :mod:`reprounzip.utils`, and :mod:`reprounzip.unpackers.common` contain utilities that you might want to use (make sure to list *reprounzip* as a requirement in your ``setup.py``).\n\nExample of ``setup.py``::\n\n setup(name='reprounzip-vagrant',\n namespace_packages=['reprounzip', 'reprounzip.unpackers'],\n install_requires=['reprounzip>=0.4'],\n entry_points={\n 'reprounzip.unpackers': [\n 'vagrant = reprounzip.unpackers.vagrant:setup'\n # The setup() function sets up the parser for reprounzip vagrant\n ]\n }\n # ...\n )\n\nUsual Commands\n''''''''''''''\n\nIf possible, you should try to follow the same command names that the official unpackers use, which are:\n\n* ``setup``: to create the experiment directory and set everything for execution;\n* ``run``: to reproduce the experiment;\n* ``destroy``: to bring down all that setup and to prepare and delete the experiment directory safely;\n* ``upload`` and ``download``: to replace input files in the experiment, and to get the output files for further examination, respectively.\n\nIf these commands can be broken down into different steps that you want to expose to the user, or if you provide completely different actions from these defaults, you can add them to the parser as well. For instance, the *vagrant* unpacker exposes ``setup/start``, which starts or resumes the virtual machine, and ``destroy/vm``, which stops and deallocates the virtual machine but leaves the template for possible reuse.\n\nA Note on File Paths\n''''''''''''''''''''\n\nReproZip supports Python 2 and 3, is portable to different operating systems, and is meant to accept a wide variety of configurations so that it is compatible with most experiments out there. Even trickier, `reprounzip-vagrant` needs to manipulate POSIX filenames on Windows, e.g.: in the unpacker.\nTherefore, the `rpaths `__ library is used everywhere internally. You should make sure to use the correct type of path (either :class:`~rpaths.PosixPath` or :class:`~rpaths.Path`) and to cast these to the type that Python functions expect, keeping in mind 2/3 differences (most certainly either ``filename.path`` or ``str(filename)``).\n\nExperiment Directory Format\n'''''''''''''''''''''''''''\n\nUnpackers usually create a directory with everything necessary to later run the experiment. This directory is created by the ``setup`` operation, cleaned up by ``destroy``, and is the argument to every command. For example, with `reprounzip-vagrant`::\n\n $ reprounzip vagrant setup someexperiment.rpz mydirectory\n $ reprounzip vagrant upload mydirectory /tmp/replace.txt:input_text\n\nUnpackers unpack the config.yml file to the root of that directory, and keep status information in a ``.reprounzip`` file, which is a dict in :mod:`pickle` format. Following the same structure will allow the ``showfiles`` command, as well as :class:`~reprounzip.unpackers.common.FileUploader` and :class:`~reprounzip.unpackers.common.FileDownloader` classes, to work correctly. Please try to follow this structure.\n\nSignals\n'''''''\n\nSince version 0.4.1, `reprounzip` has signals that can be used to hook in plugins, although no such plugin has been released at this time. To ensure that these work correctly when using your unpacker, you should emit them when appropriate. The complete list of signals is available in `signal.py `__.\n\nFinal Observations\n------------------\n\nAfter reading this page, reading the source code of one of the \"official\" unpackers is probably the best way of understanding how to write your own. They should be short enough to be easy to grasp. Should you have additional questions, do not hesitate to use our mailing-list: `reprozip@nyu.edu`.\n" }, { "path": "docs/faq.rst", "content": ".. _faq:\n\nFrequently Asked Questions\n**************************\n\nCan ReproZip pack a client-server scenario?\n===========================================\n\nYes! However, note that only tracing the client will not capture the full story: reproducibility is better achieved (and guaranteed) if the server is traced as well.\nHaving said that, currently, ReproZip can only trace local servers: if the server is remote (i.e., running in another machine), ReproZip cannot capture it. In this case, you can trace the client, and the experiment can only be reproduced if the remote server is still running at the moment of the reproduction.\n\nThe easiest way to pack a local client-server experiment is to write a script that starts the server, runs your client(s), and then shuts down the server; ReproZip can then trace this script. See :ref:`Capturing Connections to Servers ` for more information.\n\nCan ReproZip pack a database?\n=============================\n\nReproZip can trace a database server; however, because of the format it uses to store data (and also because different databases work differently), it might be hard for you to control exactly what data will be packed. You probably want to pack all the data from the databases/tables that your experiment uses and not just the pages that were touched while tracing the execution. This can be done by inspecting the configuration file and adding the relevant patterns that cover the data, e.g.: for MySQL::\n\n additional_patterns:\n - /var/lib/mysql/**\n\nSee :ref:`Capturing Connections to Servers ` for an example using a local database and additional information.\n\nNote that ReproZip does not currently save the state of the files. Therefore, if your experiment modifies a database, ReproZip will pack the already modified data (not the one before tracing the experiment execution).\n\nCan ReproZip pack interactive tools?\n====================================\n\nYes! The `reprounzip` component should have no problems with experiments that interact with the user through the terminal. If your experiment runs until it receives a Ctrl+C signal, that is fine as well: ReproZip will not interfere unless you press Ctrl+C twice, stopping the experiment.\n\nGUI tools are also supported; see :ref:`gui-tools` for more information.\n\n.. _gui-tools:\n\nCan ReproZip pack graphical tools?\n==================================\n\nYes!\nOn Linux, graphical display is handled by the X server. Applications can connect to it as clients to display their windows and components, and to get user input.\nMost unpackers now support forwarding the X connection from the experiment to the X server running on the unpacking machine. You will need a running X server to make this work, such as `Xming `__ for Windows or `XQuartz `__ for Mac OS X. If you are running Linux, chances are that an X server is already configured and running.\n\nX support is **not** enabled by default; to enable it, use the flag ``--enable-x11`` in the ``run`` command of your preferred unpacker.\n\n.. warning::\n\n While displaying a UI through the X protocol works fine, applications using direct rendering (DRI) to access dedicated graphic hardware might not be reproducible: the libGL library packed with the experiment is often specific to the driver of the original machine, and therefore the machine where the experiment is being reproduced would need to use the exact same hardware and driver.\n\n Please refrain from requiring direct rendering in applications that you intend to pack with ReproZip.\n\nIf using Vagrant, you can also use the virtual machine's native display directly, by supplying the ``--use-gui`` option to ``reprounzip vagrant setup``.\n\nHow can I access the generated system or virtual machine directly?\n==================================================================\n\nIf you are running ``reprounzip vagrant``, you can connect to the Vagrant virtual machine by running ``vagrant ssh`` or connecting via SSH to the destination listed by ``vagrant ssh-config`` (often ``localhost:2222``). These commands should be run from inside the directory created by the unpacker.\n\nIf you are running ``reprounzip docker``, you can inspect the Docker container directly by using ``docker``, or start a new one based on the image created by `reprounzip`; all images generated by ReproZip are named with the ``reprounzip_`` prefix. For more information on how to inspect and create Docker containers, please refer to the `Docker documentation `__.\n\nFor ``reprounzip chroot`` and ``reprounzip directory``, the filesystem is in the ``root`` subdirectory under the experiment path.\n\nSee :ref:`Structure of Unpacked Experiments ` for more details.\n\n.. warning::\n\n Note that, in the generated system, only the files needed for running the unpacked experiment are guaranteed to work correctly. This means that you may have only parts of a software distribution (required to run the experiment), but not the software in its entirety (unless the complete software was included in the configuration file while packing). For example, you may only have a few Python files that the experiment needs, but not the ones required to run Python interactively or install new libraries. Therefore, do not expect that all the software components will run smoothly when acessing the system.\n\n The utilities from the base system might also not work correctly (if they are not part of the experiment) because `reprounzip` overwrites the libraries with the ones from the original environment. In the worst-case scenario, the dynamic linker or the shell may not be usable. Note that some unpackers install ``/bin/busybox``, which you may find helpful.\n\nWhat if my experiment runs on a distributed environment?\n========================================================\n\nReproZip cannot trace across multiple machines. You could trace each component separately, but ReproZip cannot connect these multiple ``.rpz`` files to setup the multiple machines the right way. In particular, you will probably need to set up the same network for the components to talk to each other.\n" }, { "path": "docs/glossary.rst", "content": ".. _glossary:\n\nGlossary\n********\n\n.. glossary::\n\n configuration (file)\n A YAML file generated by ``reprozip trace`` and read by ``reprozip pack``. It can be edited before creating the package to control which files are to be included. It also contain other metadata used during reproduction. See :ref:`packing-config`.\n\n distribution package\n A software component installed by the Linux distribution's package manager. ReproZip tries to identify from which distribution package each file comes; this allows the reproducer to install the software from his distribution's package manager instead of extracting the files from the ``.rpz`` file.\n\n bundle (or pack)\n A ``.rpz`` file generated by ``reprozip pack``, containing all the files and metadata required to reproduce the experiment on another machine. See :ref:`packing`.\n\n run\n A single command line traced by ``reprozip trace [--continue]``. Multiple commands can be traced successively before creating the bundle; the reproducer will be able to run them separately using ``reprounzip run ``.\n\n software package\n The same as a distribution package.\n\n unpacker\n A plugin for the `reprounzip` component that reproduces an experiment from a ``.rpz`` bundle. The unpackers `chroot`, `directory`, and `installpkgs` are distributed with `reprounzip`; others come in separate packages (`reprounzip-docker` and `reprounzip-vagrant`). See :ref:`unpack-unpackers`.\n" }, { "path": "docs/graph.rst", "content": ".. _graph:\n\nVisualizing the Provenance Graph\n********************************\n\n.. note:: If you are using a Python version older than 2.7.3, this feature will not be available due to `Python bug 57885 `__ related to sqlite3.\n\nTo generate a *provenance graph* related to the experiment execution, the ``reprounzip graph`` command should be used::\n\n $ reprounzip graph graphfile.dot mypackfile.rpz\n\nwhere `graphfile.dot` corresponds to the graph, and `mypackfile.rpz` corresponds to the experiment bundle.\n\nAlternatively, you can generate the graph after running ``reprozip trace`` without creating a ``.rpz`` bundle::\n\n $ reprounzip graph [-d tracedirectory] graphfile.dot\n\nThe graph is outputted in the `DOT `__ language. You can use `Graphviz `__ to load and visualize the graph::\n\n $ dot -Tpng graphfile.dot -o graph.png\n\nIt is also possible to output a JSON file with the flag ``--json``.\n\nCommand-Line Options\n====================\n\nSince an experiment may involve a significantly large number of file dependencies, ``reprounzip graph`` offers several command-line options to control what will be shown in the provenance graph, as described below. By default it includes all information available, which is often unreadable (see :numref:`fig-toobig`).\n\nFiltering Out Files\n+++++++++++++++++++\n\nFiles can be filtered out using a regular expression [#re]_ with the flag ``--regex-filter``. For example:\n\n* ``--regex-filter /~[^/]*$``` will filter out files whose name begins with a tilde\n* ``--regex-filter ^/usr/share`` will filter out ``/usr/share`` recursively\n* ``--regex-filter \\.bin$`` will filter out files with a ``.bin`` extension\n\nThese flags can be passed multiple times.\n\nReplacing Filenames\n+++++++++++++++++++\n\nUsers can remap filenames using regular expressions [#re]_ with the flag ``--regex-replace``. This can be used to:\n\n* simplify the graph by making filenames shorter,\n* aggregate multiple files to a single node by mapping them to the same name, or\n* fix programs that are using some type of cache or for which the wrong access was logged, such as Python's ``.pyc`` files.\n\nExample:\n\n* ``--regex-replace .pyc$ \\.py`` will replace accesses to bytecode cache files (.pyc) to the original source (.py)\n* ``--regex-replace ^/dev(/.*)?$ /dev`` will aggregate all device files as a single path `/dev`\n* ``--regex-replace ^/home/vagrant/experiment/data/(.*)\\.bin data:\\1`` will simplify the paths to some data files\n\nThe flag ``--aggregate`` is a shortcut allowing users to aggregate all files beginning with a given prefix. For instance, ``--aggregate /usr/somepath`` will collapse all files under ``/usr/somepath`` (this is equivalent to ``--regex-replace '^/usr/somepath' '/usr/somepath'``).\n\nBoth flags can be passed multiple times.\n\nControlling Levels of Detail\n++++++++++++++++++++++++++++\n\nUsers can control the levels of detail for each category of items in the provenance graph.\n\nSoftware Packages\n.................\n\n* ``--packages file`` will show all the files belonging to a package grouped under that package's name\n* ``--packages package`` will show the package as a single item, not detailing the individual files that it contains\n* ``--packages drop`` will entirely hide the packages, removing all their files from the graph\n* ``--packages ignore`` will ignore the package identification, handling their files as if they had not been detected as being part of a package\n\nNote that regex filters and replacements are applied beforehand, so files that are remapped to a package will be shown under that package name.\n\nProcesses\n.........\n\n* ``--processes thread`` will show every process and thread\n* ``--processes process`` will show every process and hide threads\n* ``--processes run`` will show only one node for an experiment run, even if the run is composed by multiple processes and threads\n\nOther Files\n...........\n\nFor files that are not part of a software package, or if ``--packages ignore`` is being used:\n\n* ``--otherfiles all`` will show every file (unless filtered by ``--regex-filter``)\n* ``--otherfiles io`` will show only the input and output files, as identified in the configuration file\n* ``--otherfiles no`` will ignore all the files\n\n.. [#re] Anchoring regular expressions with ``^`` and ``$`` and escaping dots (``\\.``) is recommended. For more information about regular expressions, please see `here `__.\n\nCommon Recipes\n==============\n\n* Full provenance graph (likely to be unreadable for most experiments, due to the large amount of information to be presented)::\n\n $ reprounzip graph graph.dot myexperiment.rpz\n\n.. _fig-toobig:\n\n.. figure:: figures/toobig.png\n :width: 10in\n :align: center\n\n Provenance graph showing all the information available (full graph). This represents the default configuration.\n\n* Mapping Python bytecode cache files to their corresponding source file (this may help attribute file accesses to software packages)::\n\n $ reprounzip graph --regex-replace '\\.pyc$' '\\.py' graph.dot myexperiment.rpz\n\n* Dataflow of the experiment, showing the runs and their corresponding input and output files::\n\n $ reprounzip graph --packages drop --otherfiles io --processes run graph.dot myexperiment.rpz\n\n.. _fig-digits-io:\n\n.. figure:: figures/digits-io.png\n :width: 10in\n :align: center\n\n Provenance graph showing input and output files for an experiment with 4 runs.\n\n* Provenance graph showing only processes and threads (no file accesses)::\n\n $ reprounzip graph --packages drop --otherfiles drop --processes thread graph.dot myexperiment.rpz\n\n.. _fig-processes:\n\n.. figure:: figures/ache-processes.png\n :width: 10in\n :align: center\n\n Provenance graph showing only processes and threads.\n" }, { "path": "docs/gui.rst", "content": ".. _unpacking-gui:\n\nReproUnzip GUI\n**************\n\n**reprounzip-qt** is a graphical interface (GUI) for reprounzip, allowing you to unpack and reproduce experiments from ``.rpz`` files without having to use the command-line. You can also set it as the default handler for the ``.rpz`` file extension so you can open them via double-click.\n\nInstallation\n============\n\n*reprounzip-qt* comes with the installer on `Windows `_ and `Mac `_. If you used one of these, you will be able to double click on any ``.rpz`` file to boot up the GUI.\n\nIf you are using Anaconda, you can install *reprounzip-qt* from anaconda.org::\n\n $ conda install --channel conda-forge reprounzip-qt\n\nOtherwise, you will need to `install PyQt5 `__ (or PyQt4 ) before you can install *reprounzip-qt* from pip (on Debian or Ubuntu, you can use ``apt-get install pyqt5-dev``).\n\nOn Linux, you will need to run the application one time so that it registers\nitself as the handler for ``.rpz`` files.\n\n.. image:: figures/reprounzip-qt-linux-register.png\n\nUsage\n=====\n\nThe first tab in the window that appears is for you to set up the experiment. This will allow you to choose which `unpacker `_ you'd like to use to reproduce the experiment, and in which directory you'd like to unpack it.\n\n.. image:: figures/reprounzip-qt.png\n\nAfter successfully unpacking, you'll be prompted to run the experiment in the second tab. You can choose which run you want to execute, though the default is to have all runs selected. ReproUnzip will detect the order of the runs and reproduce the experiment accordingly.\n\n.. image:: figures/reprounzip-qt-1.png\n\nClicking \"Manage Files\" will bring up options to download the input and output data of the original experiment, and upload your own data to use it in the same experiment. You'll notice input files are marked ``[I]`` and output files are marked ``[O]``. ``[IO]`` are both input and output files.\n\n.. image:: figures/reprounzip-qt-manageFiles.png\n" }, { "path": "docs/index.rst", "content": "ReproZip's Documentation\n************************\n\nWelcome to ReproZip's documentation!\n\n`ReproZip `__ is a tool aimed at simplifying the process of creating reproducible experiments from *command-line executions*. It tracks operating system calls and creates a bundle that contains all the binaries, files, and dependencies required to run a given command on the author's computational environment. A reviewer can then extract the experiment in his own environment to reproduce the results, even if the environment has a different operating system from the original one.\n\nCurrently, ReproZip can only pack experiments that originally run on Linux.\n\nConcretely, ReproZip has two main steps:\n\n- The :ref:`packing step ` happens in the original environment, and generates a compendium of the experiment so as to make it reproducible. ReproZip tracks operating system calls while executing the experiment, and creates a ``.rpz`` file, which contains all the necessary information and components for the experiment.\n- The :ref:`unpacking step ` reproduces the experiment from the ``.rpz`` file. ReproZip offers different unpacking methods, from simply decompressing the files in a directory to starting a full virtual machine, and they can be used interchangeably from the same packed experiment. It is also possible to automatically replace input files and command-line arguments. Note that this step is also available on Windows and Mac OS X, since ReproZip can unpack the experiment in a virtual machine for further reproduction.\n\nContents\n--------\n\n.. toctree::\n :maxdepth: 2\n\n reprozip\n install\n packing\n unpacking\n graph\n jupyter\n gui\n vistrails\n faq\n troubleshooting\n unpacked-format\n developerguide\n glossary\n\nLinks\n-----\n\n* `Project website `__\n* `GitHub repository `__\n* Mailing list: `reprozip@nyu.edu `__\n" }, { "path": "docs/install.rst", "content": ".. _install:\n\nInstallation\n************\n\nReproZip is available as open source, released under the Revised BSD License. The tool is comprised of two components: **reprozip** (for the packing step) and **reprounzip** (for the unpack step). Additional components and plugins are also provided for *reprounzip*: **reprounzip-vagrant**, which unpacks the experiment in a Vagrant virtual machine; **reprounzip-docker**, which unpacks the experiment in a Docker container; and **reprounzip-vistrails**, which creates a VisTrails workflow to reproduce the experiment. More plugins may be developed in the future (and, of course, you are free to :ref:`roll your own `).\nIn our `website `__, you can find links to our PyPI packages and our `GitHub repository `__.\n\nIn the following, you will find installation instructions for :ref:`linux`, :ref:`mac`, and :ref:`windows`. ReproZip is also available for the :ref:`conda` Python distribution.\n\n.. _linux:\n\nLinux\n=====\n\nFor Linux distributions, both *reprozip* and *reprounzip* components are available.\n\nRequired Software Packages\n--------------------------\n\nPython 2.7.3 or greater, or 3.3 or greater is required to run ReproZip [#bug]_. If you don't have Python on your machine, you can get it from `python.org `__. You will also need the `pip `__ installer.\n\nBesides Python and pip, each component or plugin to be used may have additional dependencies that you need to install (if you do not have them already installed in your environment), as described below:\n\n+------------------------+-------------------------------------------------+\n| Component / Plugin | Required Software Packages |\n+========================+=================================================+\n| *reprozip* | `SQLite `__, |\n| | Python headers, |\n| | a working C compiler |\n+------------------------+-------------------------------------------------+\n| *reprounzip* | None |\n+------------------------+-------------------------------------------------+\n| *reprounzip-vagrant* | Python headers, |\n| | a working C compiler, |\n| | `Vagrant v1.1+ `__, |\n| | `VirtualBox `__ |\n+------------------------+-------------------------------------------------+\n| *reprounzip-docker* | `Docker `__ |\n+------------------------+-------------------------------------------------+\n| *reprounzip-vistrails* | None [#vis1]_ |\n+------------------------+-------------------------------------------------+\n\nDebian and Ubuntu\n`````````````````\n\nYou can get all the required dependencies using APT::\n\n apt-get install python3 python3-dev python3-pip gcc libsqlite3-dev libssl-dev libffi-dev\n\nFedora & CentOS\n```````````````\n\nYou can get the dependencies using the Yum packaging manager::\n\n yum install python3 python3-devel gcc sqlite-devel openssl-devel libffi-devel\n\n.. [#bug] ``reprozip`` and ``reprounzip graph`` will not work before 2.7.3 due to `Python bug 13676 `__ related to sqlite3. Python 2.6 is ancient and unsupported.\n.. [#vis1] `VisTrails v2.2.3+ `__ is required to run the workflow generated by the plugin.\n\nInstalling *reprozip*\n---------------------\n\nTo install or update the *reprozip* component, simply run the following command::\n\n $ pip install -U reprozip\n\nInstalling *reprounzip*\n-----------------------\n\nYou can install or update *reprounzip* with all the available components and plugins using::\n\n $ pip install -U reprounzip[all]\n\nOr you can install *reprounzip* and choose components manually::\n\n # Example, this installs all the components\n $ pip install -U reprounzip reprounzip-docker reprounzip-vagrant reprounzip-vistrails\n\n.. _mac:\n\nMac OS X\n========\n\nFor Mac OS X, only the *reprounzip* component is available.\n\nBinaries\n--------\n\nAn installer containing Python 2.7, *reprounzip*, and all the plugins can be `downloaded from GitHub `__.\n\nRequired Software Packages\n--------------------------\n\nPython 2.7.3 or greater, or 3.3 or greater is required to run ReproZip [#bug2]_. If you don't have Python on your machine, you can get it from `python.org `__; you should prefer a 2.x release to a 3.x one. You will also need the `pip `__ installer.\n\nBesides Python and pip, each component or plugin to be used may have additional dependencies that you need to install (if you do not have them already installed in your environment), as described below:\n\n+------------------------+-------------------------------------------------+\n| Component / Plugin | Required Software Packages |\n+========================+=================================================+\n| *reprounzip* | None |\n+------------------------+-------------------------------------------------+\n| *reprounzip-vagrant* | Python headers, |\n| | `Vagrant v1.1+ `__, |\n| | `VirtualBox `__ |\n+------------------------+-------------------------------------------------+\n| *reprounzip-docker* | `Docker `__ |\n+------------------------+-------------------------------------------------+\n| *reprounzip-vistrails* | None [#vis2]_ |\n+------------------------+-------------------------------------------------+\n\nYou will need Xcode installed, which you can get from the Mac App Store, and the Command Line Developer Tools; instrucions on installing the latter may depend on your Mac OS X version (some information on StackOverflow `here `__).\n\n.. seealso:: :ref:`Why does reprounzip-vagrant installation fail with error \"unknown argument: -mno-fused-madd\" on Mac OS X? `\n\n.. [#bug2] ``reprozip`` and ``reprounzip graph`` will not work before 2.7.3 due to `Python bug 13676 `__ related to sqlite3. Python 2.6 is ancient and unsupported.\n.. [#vis2] `VisTrails v2.2.3+ `__ is required to run the workflow generated by the plugin.\n\nInstalling *reprounzip*\n-----------------------\n\nFirst, be sure to upgrade `setuptools`::\n\n $ pip install -U setuptools\n\nYou can install or update *reprounzip* with all the available components and plugins using::\n\n $ pip install -U reprounzip[all]\n\nOr you can install *reprounzip* and choose components manually::\n\n # Example, this installs all the components\n $ pip install -U reprounzip reprounzip-docker reprounzip-vagrant reprounzip-vistrails\n\n.. _windows:\n\nWindows\n=======\n\nFor Windows, only the *reprounzip* component is available.\n\nBinaries\n--------\n\nA 32-bit installer containing Python 2.7, *reprounzip*, and all the plugins can be `downloaded from GitHub `__.\n\nRequired Software Packages\n--------------------------\n\nPython 2.7.3 or greater, or 3.3 or greater is required to run ReproZip [#bug3]_. If you don't have Python on your machine, you can get it from `python.org `__; you should prefer a 2.x release to a 3.x one. You will also need the `pip `__ installer.\n\nBesides Python and pip, each component or plugin to be used may have additional dependencies that you need to install (if you do not have them already installed in your environment), as described below:\n\n+------------------------+------------------------------------------------------------------------+\n| Component / Plugin | Required Software Packages |\n+========================+========================================================================+\n| *reprounzip* | None |\n+------------------------+------------------------------------------------------------------------+\n| *reprounzip-vagrant* | `Vagrant v1.1+ `__, |\n| | `VirtualBox `__ |\n+------------------------+------------------------------------------------------------------------+\n| *reprounzip-docker* | `Docker `__ [#windowshome]_ |\n+------------------------+------------------------------------------------------------------------+\n| *reprounzip-vistrails* | None [#vis3]_ |\n+------------------------+------------------------------------------------------------------------+\n\n.. [#bug3] ``reprozip`` and ``reprounzip graph`` will not work before 2.7.3 due to `Python bug 13676 `__ related to sqlite3. Python 2.6 is ancient and unsupported.\n.. [#vis3] `VisTrails v2.2.3+ `__ is required to run the workflow generated by the plugin.\n.. [#windowshome] Windows 10 Professional Edition is required for Docker, it will not work on Windows 10 Home Edition because it requires the Windows Hyper-V features.\n\nInstalling *reprounzip*\n-----------------------\n\nYou can install or update *reprounzip* with all the available components and plugins using::\n\n $ pip install -U reprounzip[all]\n\nOr you can install *reprounzip* and choose components manually::\n\n # Example, this installs all the components\n $ pip install -U reprounzip reprounzip-docker reprounzip-vagrant reprounzip-vistrails\n\n.. _conda:\n\nAnaconda\n========\n\n*reprozip* and *reprounzip* can also be installed on the `Anaconda `__ Python distribution, from anaconda.org::\n\n $ conda install --channel conda-forge reprozip reprounzip reprounzip-docker reprounzip-vagrant reprounzip-vistrails\n\nNote, however, that *reprozip* is only available for Linux.\n" }, { "path": "docs/jupyter.rst", "content": ".. _reprozip-jupyter:\n\nMaking Jupyter Notebooks Reproducible with ReproZip\n***************************************************\n\n**reprozip-jupyter** is a plugin for `Jupyter Notebooks `__, a popular open-source web application that allows you to create and share documents that contain live code, equations, visualizations and narrative text. These are valuable documents for data cleaning, analysis, writing executable papers/articles, and more. However, Jupyter Notebooks are subject to dependency hell like any other application -- just the Notebook is not enough for full reproducibility. We have written a ReproZip plugin for Jupyter Notebooks to help users automatically capture dependencies (including data, environment variables, etc.) of Notebooks and also automatically set up those dependencies in another computing environment.\n\nInstallation\n============\nYou can install *reprozip-jupyter* with pip::\n\n\t $ pip install reprozip-jupyter\n\nOr Anaconda::\n\n\t\t$ conda install --channel conda-forge reprozip-jupyter\n\nOnce successfully installed, you should then enable the plugin for both the client and server side of Jupyter Notebooks::\n\n\t\t$ jupyter nbextension install --py reprozip_jupyter --user\n\t\t$ jupyter nbextension enable --py reprozip_jupyter --user\n\t\t$ jupyter serverextension enable --py reprozip_jupyter --user\n\nOnce these steps are completed, when you start a Jupyter Notebook server, you should be able to see the ReproZip button in your notebook's toolbar.\n\nPacking\n=======\n\nOnce you have a notebook that executes the way you want, you can trace and pack all the dependencies, data, and provenance with *reprozip-jupyter* by simply clicking the button on the notebook's toolbar:\n\n.. image:: figures/rzj-button.png\n\nThe notebook will execute from top-to-bottom and *reprozip-jupyter* traces that execution. If there are no errors in the execution, you'll see two pop-ups like this one after the other:\n\n.. image:: figures/rzj-running.png\n\n*reprozip-jupyter* will name the resulting ReproZip bundle (*.rpz*) as ``notebookname_datetime.rpz`` and save it to the same working directory the notebook is in:\n\n.. image:: figures/rzj-pkg.png\n\nNote that the notebook file itself (``.ipynb``) is not included in the bundle, so you should share or archive both of those files. The reason is that a lot of services can render notebooks (GitHub, OSF...), and they wouldn't be able to if it was in the RPZ file.\n\nUnpacking\n=========\n\nNow, anyone can rerun the Jupyter notebook, with all dependencies automatically configured. First, they would need to install *reprounzip* and the *reprounzip-docker* plugin (see the :ref:`installation steps `). Second, they need to download or otherwise acquire the ``.rpz`` file and original ``.ipynb`` notebook they'd like to reproduce.\n\nTo reproduce the notebook using the GUI, follow these steps:\n\n1. Double-click the .rpz file.\n2. The first tab in the window that appears is for you to set up how you'd like ReproUnzip to unpack and configure the contents of the *.rpz*. Choose docker as your unpacker, and choose the directory you'd like to unpack into.\n3. Make sure the Jupyter Integration is checked, and click Run experiment:\n\n.. image:: figures/rzj-setup.png\n\n4. This second table allows you to interact with and rerun the notebook. All you need to do is click 'Run Experiment' and the Jupyter Notebook home file list should pop up in your default browser (if not, navigate to ``localhost:8888``). Open the notebook, and rerun with every dependency configured for you!\n\n.. image:: figures/rzj-run.png\n\nOn the command line, you would:\n\n1. Set up the experiment using *reprounzip-docker*::\n\n\t\t$ reprounzip docker setup \n\n2. Rerun the notebook using *reprozip-jupyter*::\n\n\t\t$ reprozip-jupyter run \n\n3. The Jupyter Notebook home file list should pop up in your default browser (if not, navigate to ``localhost:8888``).\n4. Open the notebook, and rerun with every dependency configured for you!\n\n" }, { "path": "docs/make.bat", "content": "@ECHO OFF\r\n\r\nREM Command file for Sphinx documentation\r\n\r\nif \"%SPHINXBUILD%\" == \"\" (\r\n\tset SPHINXBUILD=sphinx-build\r\n)\r\nset BUILDDIR=_build\r\nset ALLSPHINXOPTS=-d %BUILDDIR%/doctrees %SPHINXOPTS% .\r\nset I18NSPHINXOPTS=%SPHINXOPTS% .\r\nif NOT \"%PAPER%\" == \"\" (\r\n\tset ALLSPHINXOPTS=-D latex_paper_size=%PAPER% %ALLSPHINXOPTS%\r\n\tset I18NSPHINXOPTS=-D latex_paper_size=%PAPER% %I18NSPHINXOPTS%\r\n)\r\n\r\nif \"%1\" == \"\" goto help\r\n\r\nif \"%1\" == \"help\" (\r\n\t:help\r\n\techo.Please use `make ^` where ^ is one of\r\n\techo. html to make standalone HTML files\r\n\techo. dirhtml to make HTML files named index.html in directories\r\n\techo. singlehtml to make a single large HTML file\r\n\techo. pickle to make pickle files\r\n\techo. json to make JSON files\r\n\techo. htmlhelp to make HTML files and a HTML help project\r\n\techo. qthelp to make HTML files and a qthelp project\r\n\techo. devhelp to make HTML files and a Devhelp project\r\n\techo. epub to make an epub\r\n\techo. epub3 to make an epub3\r\n\techo. latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter\r\n\techo. text to make text files\r\n\techo. man to make manual pages\r\n\techo. texinfo to make Texinfo files\r\n\techo. gettext to make PO message catalogs\r\n\techo. changes to make an overview over all changed/added/deprecated items\r\n\techo. xml to make Docutils-native XML files\r\n\techo. pseudoxml to make pseudoxml-XML files for display purposes\r\n\techo. linkcheck to check all external links for integrity\r\n\techo. doctest to run all doctests embedded in the documentation if enabled\r\n\techo. coverage to run coverage check of the documentation if enabled\r\n\techo. dummy to check syntax errors of document sources\r\n\tgoto end\r\n)\r\n\r\nif \"%1\" == \"clean\" (\r\n\tfor /d %%i in (%BUILDDIR%\\*) do rmdir /q /s %%i\r\n\tdel /q /s %BUILDDIR%\\*\r\n\tgoto end\r\n)\r\n\r\n\r\nREM Check if sphinx-build is available and fallback to Python version if any\r\n%SPHINXBUILD% 1>NUL 2>NUL\r\nif errorlevel 9009 goto sphinx_python\r\ngoto sphinx_ok\r\n\r\n:sphinx_python\r\n\r\nset SPHINXBUILD=python -m sphinx.__init__\r\n%SPHINXBUILD% 2> nul\r\nif errorlevel 9009 (\r\n\techo.\r\n\techo.The 'sphinx-build' command was not found. Make sure you have Sphinx\r\n\techo.installed, then set the SPHINXBUILD environment variable to point\r\n\techo.to the full path of the 'sphinx-build' executable. Alternatively you\r\n\techo.may add the Sphinx directory to PATH.\r\n\techo.\r\n\techo.If you don't have Sphinx installed, grab it from\r\n\techo.http://sphinx-doc.org/\r\n\texit /b 1\r\n)\r\n\r\n:sphinx_ok\r\n\r\n\r\nif \"%1\" == \"html\" (\r\n\t%SPHINXBUILD% -b html %ALLSPHINXOPTS% %BUILDDIR%/html\r\n\tif errorlevel 1 exit /b 1\r\n\techo.\r\n\techo.Build finished. The HTML pages are in %BUILDDIR%/html.\r\n\tgoto end\r\n)\r\n\r\nif \"%1\" == \"dirhtml\" (\r\n\t%SPHINXBUILD% -b dirhtml %ALLSPHINXOPTS% %BUILDDIR%/dirhtml\r\n\tif errorlevel 1 exit /b 1\r\n\techo.\r\n\techo.Build finished. The HTML pages are in %BUILDDIR%/dirhtml.\r\n\tgoto end\r\n)\r\n\r\nif \"%1\" == \"singlehtml\" (\r\n\t%SPHINXBUILD% -b singlehtml %ALLSPHINXOPTS% %BUILDDIR%/singlehtml\r\n\tif errorlevel 1 exit /b 1\r\n\techo.\r\n\techo.Build finished. The HTML pages are in %BUILDDIR%/singlehtml.\r\n\tgoto end\r\n)\r\n\r\nif \"%1\" == \"pickle\" (\r\n\t%SPHINXBUILD% -b pickle %ALLSPHINXOPTS% %BUILDDIR%/pickle\r\n\tif errorlevel 1 exit /b 1\r\n\techo.\r\n\techo.Build finished; now you can process the pickle files.\r\n\tgoto end\r\n)\r\n\r\nif \"%1\" == \"json\" (\r\n\t%SPHINXBUILD% -b json %ALLSPHINXOPTS% %BUILDDIR%/json\r\n\tif errorlevel 1 exit /b 1\r\n\techo.\r\n\techo.Build finished; now you can process the JSON files.\r\n\tgoto end\r\n)\r\n\r\nif \"%1\" == \"htmlhelp\" (\r\n\t%SPHINXBUILD% -b htmlhelp %ALLSPHINXOPTS% %BUILDDIR%/htmlhelp\r\n\tif errorlevel 1 exit /b 1\r\n\techo.\r\n\techo.Build finished; now you can run HTML Help Workshop with the ^\r\n.hhp project file in %BUILDDIR%/htmlhelp.\r\n\tgoto end\r\n)\r\n\r\nif \"%1\" == \"qthelp\" (\r\n\t%SPHINXBUILD% -b qthelp %ALLSPHINXOPTS% %BUILDDIR%/qthelp\r\n\tif errorlevel 1 exit /b 1\r\n\techo.\r\n\techo.Build finished; now you can run \"qcollectiongenerator\" with the ^\r\n.qhcp project file in %BUILDDIR%/qthelp, like this:\r\n\techo.^> qcollectiongenerator %BUILDDIR%\\qthelp\\ReproZip.qhcp\r\n\techo.To view the help file:\r\n\techo.^> assistant -collectionFile %BUILDDIR%\\qthelp\\ReproZip.ghc\r\n\tgoto end\r\n)\r\n\r\nif \"%1\" == \"devhelp\" (\r\n\t%SPHINXBUILD% -b devhelp %ALLSPHINXOPTS% %BUILDDIR%/devhelp\r\n\tif errorlevel 1 exit /b 1\r\n\techo.\r\n\techo.Build finished.\r\n\tgoto end\r\n)\r\n\r\nif \"%1\" == \"epub\" (\r\n\t%SPHINXBUILD% -b epub %ALLSPHINXOPTS% %BUILDDIR%/epub\r\n\tif errorlevel 1 exit /b 1\r\n\techo.\r\n\techo.Build finished. The epub file is in %BUILDDIR%/epub.\r\n\tgoto end\r\n)\r\n\r\nif \"%1\" == \"epub3\" (\r\n\t%SPHINXBUILD% -b epub3 %ALLSPHINXOPTS% %BUILDDIR%/epub3\r\n\tif errorlevel 1 exit /b 1\r\n\techo.\r\n\techo.Build finished. The epub3 file is in %BUILDDIR%/epub3.\r\n\tgoto end\r\n)\r\n\r\nif \"%1\" == \"latex\" (\r\n\t%SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex\r\n\tif errorlevel 1 exit /b 1\r\n\techo.\r\n\techo.Build finished; the LaTeX files are in %BUILDDIR%/latex.\r\n\tgoto end\r\n)\r\n\r\nif \"%1\" == \"latexpdf\" (\r\n\t%SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex\r\n\tcd %BUILDDIR%/latex\r\n\tmake all-pdf\r\n\tcd %~dp0\r\n\techo.\r\n\techo.Build finished; the PDF files are in %BUILDDIR%/latex.\r\n\tgoto end\r\n)\r\n\r\nif \"%1\" == \"latexpdfja\" (\r\n\t%SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex\r\n\tcd %BUILDDIR%/latex\r\n\tmake all-pdf-ja\r\n\tcd %~dp0\r\n\techo.\r\n\techo.Build finished; the PDF files are in %BUILDDIR%/latex.\r\n\tgoto end\r\n)\r\n\r\nif \"%1\" == \"text\" (\r\n\t%SPHINXBUILD% -b text %ALLSPHINXOPTS% %BUILDDIR%/text\r\n\tif errorlevel 1 exit /b 1\r\n\techo.\r\n\techo.Build finished. The text files are in %BUILDDIR%/text.\r\n\tgoto end\r\n)\r\n\r\nif \"%1\" == \"man\" (\r\n\t%SPHINXBUILD% -b man %ALLSPHINXOPTS% %BUILDDIR%/man\r\n\tif errorlevel 1 exit /b 1\r\n\techo.\r\n\techo.Build finished. The manual pages are in %BUILDDIR%/man.\r\n\tgoto end\r\n)\r\n\r\nif \"%1\" == \"texinfo\" (\r\n\t%SPHINXBUILD% -b texinfo %ALLSPHINXOPTS% %BUILDDIR%/texinfo\r\n\tif errorlevel 1 exit /b 1\r\n\techo.\r\n\techo.Build finished. The Texinfo files are in %BUILDDIR%/texinfo.\r\n\tgoto end\r\n)\r\n\r\nif \"%1\" == \"gettext\" (\r\n\t%SPHINXBUILD% -b gettext %I18NSPHINXOPTS% %BUILDDIR%/locale\r\n\tif errorlevel 1 exit /b 1\r\n\techo.\r\n\techo.Build finished. The message catalogs are in %BUILDDIR%/locale.\r\n\tgoto end\r\n)\r\n\r\nif \"%1\" == \"changes\" (\r\n\t%SPHINXBUILD% -b changes %ALLSPHINXOPTS% %BUILDDIR%/changes\r\n\tif errorlevel 1 exit /b 1\r\n\techo.\r\n\techo.The overview file is in %BUILDDIR%/changes.\r\n\tgoto end\r\n)\r\n\r\nif \"%1\" == \"linkcheck\" (\r\n\t%SPHINXBUILD% -b linkcheck %ALLSPHINXOPTS% %BUILDDIR%/linkcheck\r\n\tif errorlevel 1 exit /b 1\r\n\techo.\r\n\techo.Link check complete; look for any errors in the above output ^\r\nor in %BUILDDIR%/linkcheck/output.txt.\r\n\tgoto end\r\n)\r\n\r\nif \"%1\" == \"doctest\" (\r\n\t%SPHINXBUILD% -b doctest %ALLSPHINXOPTS% %BUILDDIR%/doctest\r\n\tif errorlevel 1 exit /b 1\r\n\techo.\r\n\techo.Testing of doctests in the sources finished, look at the ^\r\nresults in %BUILDDIR%/doctest/output.txt.\r\n\tgoto end\r\n)\r\n\r\nif \"%1\" == \"coverage\" (\r\n\t%SPHINXBUILD% -b coverage %ALLSPHINXOPTS% %BUILDDIR%/coverage\r\n\tif errorlevel 1 exit /b 1\r\n\techo.\r\n\techo.Testing of coverage in the sources finished, look at the ^\r\nresults in %BUILDDIR%/coverage/python.txt.\r\n\tgoto end\r\n)\r\n\r\nif \"%1\" == \"xml\" (\r\n\t%SPHINXBUILD% -b xml %ALLSPHINXOPTS% %BUILDDIR%/xml\r\n\tif errorlevel 1 exit /b 1\r\n\techo.\r\n\techo.Build finished. The XML files are in %BUILDDIR%/xml.\r\n\tgoto end\r\n)\r\n\r\nif \"%1\" == \"pseudoxml\" (\r\n\t%SPHINXBUILD% -b pseudoxml %ALLSPHINXOPTS% %BUILDDIR%/pseudoxml\r\n\tif errorlevel 1 exit /b 1\r\n\techo.\r\n\techo.Build finished. The pseudo-XML files are in %BUILDDIR%/pseudoxml.\r\n\tgoto end\r\n)\r\n\r\nif \"%1\" == \"dummy\" (\r\n\t%SPHINXBUILD% -b dummy %ALLSPHINXOPTS% %BUILDDIR%/dummy\r\n\tif errorlevel 1 exit /b 1\r\n\techo.\r\n\techo.Build finished. Dummy builder generates no files.\r\n\tgoto end\r\n)\r\n\r\n:end\r\n" }, { "path": "docs/man/reprounzip-docker.1", "content": ".\\\" Manpage for reprounzip\n.TH man 1 \"4 November 2017\" \"1.0.10\" \"reprounzip\\-docker man page\"\n.SH NAME\nreprounzip\\-docker \\- Docker unpacker for reprounzip\n.SH SYNOPSIS\nreprounzip [\\-v] docker [\\-h] [\\-\\-version] [\\-\\-docker\\-cmd DOCKER] ...\n.SH DESCRIPTION\nreprounzip\\-docker is the unpacker plugin for reprounzip that allows reproducing packed experiments on different environments by using a Docker container. You will need Docker to be installed on your machine to use it.\n.SH OPTIONS\n.SS General options:\n.TP\n.B \\-h, \\-\\-help\nshow this help message and exit\n.TP\n.B \\-\\-version\nshow program's version number and exit\n.TP\n.B \\-v, \\-\\-verbose\naugments verbosity level\n.TP\n.B \\-\\-docker\\-cmd DOCKER\nchanges the location or command used to run the Docker client. This option is split on spaces, for example you can use\n\\-\\-docker\\-cmd 'sudo /opt/bin/docker'\n\n.SS Commands:\n.TP\n.BI setup \" [\\-\\-base\\-image IMAGE] [\\-\\-distribution DISTRIBUTION] [\\-\\-install\\-pkgs] [\\-\\-image\\-name NAME] [\\-\\-docker\\-option OPTION] \"\nUnpacks an experiment and sets it up for reproduction in the specified directory. A Docker image\n.I NAME\nwill be created from the specified base\n.I IMAGE\nby adding the experiment's content on top of it. You can use \\-\\-docker\\-option to pass additional options to docker build for example \\-\\-docker\\-option=\\-\\-squash.\n.TP\n.BI run \" [\\-\\-expose\\-port PORT] [\\-\\-enable\\-x11] [\\-\\-x11\\-display DISPLAY] [\\-\\-tunneled\\-x11] [\\-d] [\\-\\-docker\\-option OPTION] [\\-\\-pass\\-env VAR] [\\-\\-set\\-env VAR[=val]] [run] [\\-\\-cmdline ...]\"\nRuns an experiment that has been unpacked to a directory.\n.TP\n.BI upload \" [localfile:experimentfile] [...]\"\nReplaces a file inside the experiment image with your own. This allows you to reproduce an experiment in its original environment, but with modified data or configuration. Running this command with only the\n.I directory\nwill list all the named files that you can replace with this command.\n.TP\n.BI download \" [experimentfile[:localfile]] [...]\"\nDownload a file from the experiment container to your filesystem. If\n.I localfile\nis not specified, a file of the same name will be written in the current directory. Running this command with only the\n.I directory\nwill list all the named files that you can download with this command.\n.TP\n.BI reset \" \"\nReset the experiment to the state it was when it was first unpacked. This undoes the effects of\n.BR upload \" and \" run\ncommands on the experiment's environment.\n.TP\n.BI destroy \" \"\nRemoves an unpacked experiment directory, and the associated Docker images.\n.SH SEE ALSO\n.TP\nThe ReproZip website\nhttps://www.reprozip.org/\n.TP\nReproZip's GitHub repository\nhttps://github.com/ViDA\\-NYU/reprozip\n.TP\nThe Docker website\nhttps://www.docker.com/\n.SH BUGS\nPlease report bugs on our mailing-list at reprozip@nyu.edu or on GitHub at https://github.com/VIDA\\-NYU/reprozip/issues.\n.SH AUTHOR\n.RB \"ReproZip is being developed at\" \" New York University\" .\n\nThe team includes:\n.RS\n.nf\nRemi Rampin\nFernando Chirigati\nVicky Rampin\nJuliana Freire\nDennis Shasha\n.fi\n.RE\n.SH COPYRIGHT\nCopyright 2014 New York University.\n\n.RB \"Licensed under a\" \" BSD 3-Clause license.\" \" See the LICENSE file included with the software for details.\"\n" }, { "path": "docs/man/reprounzip-vagrant.1", "content": ".\\\" Manpage for reprounzip\n.TH man 1 \"4 November 2017\" \"1.0.10\" \"reprounzip\\-vagrant man page\"\n.SH NAME\nreprounzip\\-vagrant \\- Vagrant unpacker for reprounzip\n.SH SYNOPSIS\nreprounzip [\\-v] vagrant [\\-h] [\\-\\-version] ...\n.SH DESCRIPTION\nreprounzip\\-vagrant is the unpacker plugin for reprounzip that allows reproducing packed experiments on different environments by using Vagrant to create a virtual machine. You will need Vagrant and VirtualBox to be installed on your machine to use it.\n.SH OPTIONS\n.SS General options:\n.TP\n.B \\-h, \\-\\-help\nshow this help message and exit\n.TP\n.B \\-\\-version\nshow program's version number and exit\n.TP\n.B \\-v, \\-\\-verbose\naugments verbosity level\n\n.SS Commands:\n.TP\n.BI setup \" [\\-\\-dont\\-use\\-chroot] [\\-\\-dont\\-bind\\-magic\\-dirs] [\\-\\-base\\-image IMAGE] [\\-\\-distribution DISTRIBUTION] [\\-\\-memory MEMORY] [\\-\\-use\\-gui] [\\-\\-expose\\-port PORT] \"\nUnpacks an experiment and sets it up for reproduction in the specified directory. A virtual machine will be created from the specified base\n.I IMAGE\nby adding the experiment's content on top of it. The default is to extract the experiment's files in a chroot inside the machine, and binding \"magic dirs\" into it\n.RI ( /proc \", \" /dev \", ...);\"\nthis can be disabled using \\-\\-dont\\-use\\-chroot.\n.TP\n.BI run \" [\\-\\-no\\-stdin] [\\-\\-no\\-pty] [\\-\\-expose\\-port PORT] [\\-\\-enable\\-x11] [\\-\\-x11\\-display DISPLAY] [\\-\\-pass\\-env VAR] [\\-\\-set\\-env VAR[=val]] [run] [\\-\\-cmdline ...]\"\nRuns an experiment that has been unpacked to a directory.\n.TP\n.BI upload \" [localfile:experimentfile] [...]\"\nReplaces a file inside the virtual machine with your own. This allows you to reproduce an experiment in its original environment, but with modified data or configuration. Running this command with only the\n.I directory\nwill list all the named files that you can replace with this command.\n.TP\n.BI download \" [experimentfile[:localfile]] [...]\"\nDownload a file from the virtual machine to your filesystem. If\n.I localfile\nis not specified, a file of the same name will be written in the current directory. Running this command with only the\n.I directory\nwill list all the named files that you can download with this command.\n.TP\n.BI destroy \" \"\nRemoves an unpacked experiment directory, and the associated virtual machine.\n.SH SEE ALSO\n.TP\nThe ReproZip website\nhttps://www.reprozip.org/\n.TP\nReproZip's GitHub repository\nhttps://github.com/ViDA\\-NYU/reprozip\n.TP\nThe Vagrant website\nhttps://www.vagrantup.com/\n.SH BUGS\nPlease report bugs on our mailing-list at reprozip@nyu.edu or on GitHub at https://github.com/VIDA\\-NYU/reprozip/issues.\n.SH AUTHOR\n.RB \"ReproZip is being developed at\" \" New York University\" .\n\nThe team includes:\n.RS\n.nf\nRemi Rampin\nFernando Chirigati\nVicky Rampin\nJuliana Freire\nDennis Shasha\n.fi\n.RE\n.SH COPYRIGHT\nCopyright 2014 New York University.\n\n.RB \"Licensed under a\" \" BSD 3-Clause license.\" \" See the LICENSE file included with the software for details.\"\n" }, { "path": "docs/man/reprounzip.1", "content": ".\\\" Manpage for reprounzip\n.TH man 1 \"4 November 2017\" \"1.0.11\" \"reprounzip man page\"\n.SH NAME\nreprounzip \\- the reproducibility unpacker\n.SH SYNOPSIS\nreprounzip [\\-h] [\\-\\-version] [\\-v] ...\n.SH DESCRIPTION\nreprounzip inspects or unpacks a reproducible experiment package (.rpz file) created by reprozip.\n.SH OPTIONS\n.SS General options:\n.TP\n.B \\-h, \\-\\-help\nshow this help message and exit\n.TP\n.B \\-\\-version\nshow program's version number and exit\n.TP\n.B \\-v, \\-\\-verbose\naugments verbosity level\n\n.SS Commands:\n.TP\n.B usage_report\nEnables or disables anonymous usage reports\n.TP\n.B info\nPrints out some information about a pack\n.TP\n.BI graph \" [package.rpz]\"\nGenerates a provenance graph from the trace data\n.TP\n.BI showfiles \" \"\nPrints out input and output file names\n.TP\n.B installpkgs \" \"\nInstalls the required packages on this system\n.TP\n.BR directory \" ...\"\nUnpacks the files in a directory and runs with PATH and LD_LIBRARY_PATH\n.TP\n.B chroot\nUnpacks the files and run with chroot\n.TP\n.B docker\nRuns the experiment in a Docker container\n.TP\n.B vagrant\nRuns the experiment in a virtual machine created through Vagrant\n\n.SS Unpacker commands:\nEach unpacker (such as\n.BR directory \", \" chroot \", \" docker \", or \" vagrant )\nhas a similar set of commands, detailed below. Please refer to\n.BR reprounzip\\-\\f(BIunpacker (1)\nfor detailed information about a specific unpacker.\n.TP\n.BI setup \" \"\nUnpacks an experiment and sets it up for reproduction in the specified directory.\n.TP\n.BI run \" [run] [\\-\\-cmdline ...]\"\nRuns an experiment that has been unpacked to a directory.\n.TP\n.BI upload \" [localfile:experimentfile] [...]\"\nReplaces a file inside the experiment with your own. This allows you to reproduce an experiment in its original environment, but with modified data or configuration. Running this command with only the\n.I directory\nwill list all the named files that you can replace with this command.\n.TP\n.BI download \" [experimentfile[:localfile]] [...]\"\nDownload a file from the experiment container/virtual machine/directory to your filesystem. If\n.I localfile\nis not specified, a file of the same name will be written in the current directory. Running this command with only the\n.I directory\nwill list all the named files that you can download with this command.\n.TP\n.BI destroy \" \"\nRemoves an unpacked experiment directory, and associated resources such as containers, virtual machines, or mount points.\n.B Always use this command\nto delete an unpacked experiment, otherwise resources might get left behind, or with chroot you might end up\n.B deleting system files\nthrough the mount point that has been created.\n.SH FILES\n.TP\n.B ~/.reprozip/log\nLog file where reprozip and reprounzip write messages during execution. Useful to report issues to the developers, even if you were running without\n.IR \\-\\-verbose .\n.SH ENVIRONMENT\n.TP\n.B REPROZIP_USAGE_STATS\nIf this variable is set to\n.I \\*(lqoff\\*(rq\nthen usage statistics will be completely disabled (won't be reported, recorded, and won't ask you about it).\n.TP\n.B REPROZIP_PARAMETERS\nIf this variable is set to a URL, it specifies an alternate location from which to download runtime parameters. If it is set to\n.IR \\*(lqoff\\*(rq ,\nnothing will be downloaded and the bundled parameters will be used instead.\n.SH EXAMPLES\n.P\nGet information on a package:\n.IP\n.nf\n.RB \"$\" \" reprounzip \\-v info\" \" foobar0.4\\-python2.rpz\"\n\\-\\-\\-\\-\\- Pack information \\-\\-\\-\\-\\-\nCompressed size: 1.64 MB\nUnpacked size: 4.98 MB\n[...]\n.fi\n\n.P\nUnpack the experiment using\n.BR reprounzip\\-docker (1)\nunder\n.IR /tmp :\n.IP\n.nf\n.RB \"$\" \" reprounzip docker setup\" \" foobar0.4\\-python2.rpz /tmp/foobar\\-docker\"\n.RB \"$\" \" reprounzip docker run\" \" /tmp/foobar\\-docker\"\n.RB \"$\" \" reprounzip docker destroy\" \" /tmp/foobar\\-docker\"\n.fi\n\n.SH SEE ALSO\n.TP\nThe ReproZip website\nhttps://www.reprozip.org/\n.TP\nReproZip's GitHub repository\nhttps://github.com/ViDA\\-NYU/reprozip\n.SH BUGS\nPlease report bugs on our mailing-list at reprozip@nyu.edu or on GitHub at https://github.com/VIDA\\-NYU/reprozip/issues.\n.SH AUTHOR\n.RB \"ReproZip is being developed at\" \" New York University\" .\n\nThe team includes:\n.RS\n.nf\nRemi Rampin\nFernando Chirigati\nVicky Rampin\nJuliana Freire\nDennis Shasha\n.fi\n.RE\n.SH COPYRIGHT\nCopyright 2014 New York University.\n\n.RB \"Licensed under a\" \" BSD 3-Clause license.\" \" See the LICENSE file included with the software for details.\"\n" }, { "path": "docs/man/reprozip.1", "content": ".\\\" Manpage for reprozip\n.TH man 1 \"4 November 2017\" \"1.0.11\" \"reprozip man page\"\n.SH NAME\nreprozip \\- the reproducibility packer\n.SH SYNOPSIS\nreprozip [\\-h] [\\-\\-version] [\\-d DIR] [\\-\\-dont\\-identify\\-packages]\n [\\-\\-dont\\-find\\-inputs\\-outputs] [\\-v] ...\n.SH DESCRIPTION\nreprozip traces and packs an experiment or program into a reproducible package (.rpz file). Those packages can be archived, and reproduced later using reprounzip(1).\n.SH OPTIONS\n.SS General options:\n.TP\n.B \\-h, \\-\\-help\nshow this help message and exit\n.TP\n.B \\-\\-version\nshow program's version number and exit\n.TP\n.B \\-d DIR, \\-\\-dir DIR\nwhere to store database and configuration file (default: ./.reprozip\\-trace)\n.TP\n.B \\-\\-dont\\-identify\\-packages\ndo not try identify which package each file comes from\n.TP\n.B \\-\\-dont\\-find\\-inputs\\-outputs\ndo not try to identify input and output files\n.TP\n.B \\-v, \\-\\-verbose\naugments verbosity level\n\n.SS Commands:\n.TP\n.B usage_report\nEnables or disables anonymous usage reports\n.TP\n.B trace\nRuns the program and writes out database and configuration file\n.TP\n.B testrun\nRuns the program and writes out the database contents\n.TP\n.B reset\nResets the configuration file\n.TP\n.B pack\nPacks the experiment according to the current configuration\n.TP\n.B combine\nCombine multiple traces into one (possibly as subsequent runs)\n.SH EXIT STATUS\nreprozip returns 1 in case of internal error, otherwise it returns the same result as the traced command.\n.SH FILES\n.TP\n.B ~/.reprozip/log\nLog file where reprozip and reprounzip write messages during execution. Useful to report issues to the developers, even if you were running without\n.IR \\-\\-verbose .\n.TP\n.B .reprozip\\-trace/trace.sqlite3, .reprozip\\-trace/config.yml\nWhen tracing, the trace database and configuration file get written to a directory\n.I .reprozip\\-trace\nin the current directory. This location can be overridden with\n.IR \\-\\-dir .\nIt can be useful to edit the configuration file before packing, to make sure no useless or private file is to be included in the package, and to correctly label input files, output files, and run steps.\n.SH ENVIRONMENT\n.TP\n.B REPROZIP_USAGE_STATS\nIf this variable is set to\n.I \\*(lqoff\\*(rq\nthen usage statistics will be completely disabled (won't be reported, recorded, and won't ask you about it).\n.SH EXAMPLES\n.P\nTrace and pack a simple Python script:\n.IP\n.nf\n.RB \"$\" \" reprozip trace\" \" python foobar.py\"\n.RB \"$\" \" reprozip pack\" \" foobar0.4\\-python2.rpz\"\n.fi\n.P\nUse testrun to show the files used by a command:\n.IP\n.nf\n.RB \"$\" \" reprozip testrun\" \" /usr/bin/id\"\n.fi\n\n.SH SEE ALSO\n.TP\nThe ReproZip website\nhttps://www.reprozip.org/\n.TP\nReproZip's GitHub repository\nhttps://github.com/ViDA\\-NYU/reprozip\n.SH BUGS\nPlease report bugs on our mailing-list at reprozip@nyu.edu or on GitHub at https://github.com/VIDA\\-NYU/reprozip/issues.\n.SH AUTHOR\n.RB \"ReproZip is being developed at\" \" New York University\" .\n\nThe team includes:\n.RS\n.nf\nRemi Rampin\nFernando Chirigati\nVicky Rampin\nJuliana Freire\nDennis Shasha\n.fi\n.RE\n.SH COPYRIGHT\nCopyright 2014 New York University.\n\n.RB \"Licensed under a\" \" BSD 3-Clause license.\" \" See the LICENSE file included with the software for details.\"\n" }, { "path": "docs/packing.rst", "content": ".. _packing:\n\nUsing *reprozip*\n****************\n\nThe *reprozip* component is responsible for packing an experiment, which is done in three steps: :ref:`tracing the experiment `, :ref:`editing the configuration file ` (if necessary), and :ref:`creating the reproducible package `. Each of these steps is explained in more details below. Please note that *reprozip* is only available for Linux distributions.\n\n.. _packing-trace:\n\nTracing an Experiment\n=====================\n\nFirst, *reprozip* needs to trace the operating system calls used by the experiment, so as to identify all the necessary information for its future re-execution, such as binaries, files, library dependencies, and environment variables.\n\nThe following command is used to trace a command line, or a `run`, used by the experiment::\n\n $ reprozip trace \n\nwhere `` is the command line. By running this command, *reprozip* executes `` and uses ``ptrace`` to trace all the system calls issued, storing them in an SQLite database.\n\nIf you run the command multiple times, *reprozip* might ask you if you want to continue with your current trace (append the new command-line to it) or replace it (throw away the previous command-line you traced). You can skip this prompt by using either the ``--continue`` or ``--overwrite`` flag, like this::\n\n $ reprozip trace --continue \n\nNote that the final bundle will be able to reproduce any of the runs, and files shared by multiple runs are only stored once.\n\nBy default, if the operating system is based on Debian or RPM packages (e.g.: Ubuntu, CentOS, Fedora, ...), *reprozip* will also try to automatically identify the distribution packages from which the files come, using the available package manager of the system. This is useful to provide more detailed information about the dependencies, as well as to further help when reproducing the experiment. However, note that the ``trace`` command can take some time doing that after the experiment finishes, depending on the number of file dependencies that the experiment has. To disable this feature, users may use the flag ``--dont-identify-packages``::\n\n $ reprozip trace --dont-identify-packages \n\nThe database, together with a *configuration file* (see below), are placed in a directory named ``.reprozip-trace``, created under the path where the ``reprozip trace`` command was issued.\n\n.. _packing-config:\n\nEditing the Configuration File\n==============================\n\nThe configuration file, which can be found in ``.reprozip-trace/config.yml``, contains all the information necessary for creating the experiment bundle. This file is generated by the tracer and drives the packing step.\n\nIt is very likely that you won't need to modify this file, as the automatically-generated one should be sufficient to create a working bundle. However, in some cases, you may want to edit it prior to the creation of the package to add or remove files used by your experiment. This can be particularly useful, for instance, to remove big files that can be obtained elsewhere when reproducing the experiment, to keep the size of package small, and also to remove sensitive information that the experiment may use. The configuration file can also be used to edit the main command line, to add or remove environment variables, and to edit information regarding input/output files.\n\n.. _packing-config-general:\n\nThe first part of the configuration file gives general information with respect to the experiment and its runs, including command lines, environment variables, working directory, and machine information. Also, each run has a unique identifier (given by ``id``) that is consistently used for packing and unpacking purposes::\n\n # Run info\n version: \n runs:\n # Run 0\n - id: \n architecture: \n argv: \n binary: \n distribution: \n environ: \n exitcode: \n gid: \n hostname: \n system: \n uid: \n workingdir: \n\n # Run 1\n - id: ...\n ...\n\nIf necessary, users may change command line parameters by editing ``argv``, and add or remove environment variables by editing ``environ``. Users may also give a more meaningful and user-friendly identifier for a run by changing ``id``. Other attributes should not be changed in general.\n\n.. _packing-config-inputoutput:\n\nThe next section brings information about input and output files, including their original paths and which runs read and/or wrote them. These are the files that `reprozip` identified as the main input or result of the experiment, which `reprounzip` will later be able to replace and extract from the experiment when reproducing it. You may add, remove, or edit these files in case `reprozip` fails in recognizing any important information, as well as give meaningful names to them by editing ``name``::\n\n # Inputs are files that are only read by a run; reprounzip can replace these\n # files on demand to run the experiment with custom data.\n # Outputs are files that are generated by a run; reprounzip can extract these\n # files from the experiment on demand, for the user to examine.\n # The name field is the identifier the user will use to access these files.\n inputs_outputs:\n - name: \n path: \n read_by_runs: \n written_by_runs: \n - name: ...\n ...\n\nNote that you can prevent `reprozip` from identifying which files are input or output by using the ``--dont-find-inputs-outputs`` flag in the ``reprozip trace`` command.\n\n.. note:: To visualize the dataflow of the experiment, pleaser refer to :ref:`graph`.\n\n.. seealso:: :ref:`Why doesn’t 'reprozip' identify my input/output file? `\n\n.. _packing-config-files:\n\nThe next section in the configuration file lists all the files to be packed. If the software dependencies were identified by the package manager of the system during the ``reprozip trace`` command, they will be organized in software packages and listed under ``packages``; otherwise, file dependencies will be listed under ``other_files``::\n\n packages:\n - name: \n version: \n size: \n packfiles: \n files:\n # Total files used: \n # Installed package size: \n \n - name: ...\n ...\n\n other_files:\n \n\nThe attribute ``packfiles`` can be used to control whether a software package will be packed: its default value is `true`, but users may change it to `false` to inform *reprozip* that the corresponding software package should not be included. To remove a file that was not identified as part of a package, users can simply remove it from the list under ``other_files``.\n\n.. warning::\n\n Note that if a software package is requested not to be included, the `reprounzip` component will try to install it from a package manager when unpacking the experiment. If the software version from the package manager is different from (and incompatible with) the one used by the experiment, the experiment may not be reproduced correctly.\n\n.. seealso:: :ref:`Why does 'reprounzip run' fail with \"no such file or directory\" or similar? `\n\n.. _packing-config-patterns:\n\nLast, users may add file patterns under ``additional_patterns`` to include other files that they think it will be useful for a future reproduction. As an example, the following would add everything under ``/etc/apache2/`` and all the Python files of all users from LXC containers (contrived example)::\n\n additional_patterns:\n - /etc/apache2/**\n - /var/lib/lxc/*/rootfs/home/**/*.py\n\nNote that users can always reset the configuration file to its initial state by running the following command::\n\n $ reprozip reset\n\n.. warning::\n\n When editing a configuration file, make sure your changes are as restrictive as possible, modifying only the necessary information. Removing important information and changing the structure of the file may cause issues while creating the bundle or unpacking the experiment.\n\n.. _packing-pack:\n\nCreating a Bundle\n=================\n\nAfter tracing all the runs from the experiment and optionally editing the configuration file, the experiment bundle can be created by using the following command::\n\n $ reprozip pack \n\nwhere `` is the name given to the package. This command generates a ``.rpz`` file in the current directory, which can then be sent to others so that the experiment can be reproduced. For more information regarding the unpacking step, please see :ref:`unpacking`.\n\nNote that, by using ``reprozip pack``, files will be copied from your environment to the package; as such, you should not change any file that the experiment used before packing it, otherwise the package will contain different files from the ones the experiment used when it was originally traced.\n\n.. warning::\n\n Before sending your bundle to others, it is advisable to test it and ensure that the reproduction of the experiment works.\n\n.. _packing-further:\n\nFurther Considerations\n======================\n\nPacking Multiple Command Lines\n++++++++++++++++++++++++++++++\n\nAs mentioned before, ReproZip allows multiple runs (i.e., command lines) to be traced and included in the same bundle. Alternatively, users can create a simple **script** that runs all the command lines, and pass *that* to ``reprozip trace``. However, in this case, there will be no flexibility in choosing a single run to be reproduced, since the entire script will be re-executed.\n\nNote that this flexibility has the caveat that users may reproduce the runs in a different order than the one originally used while tracing. If the order is important for the reproduction (e.g.: each run represents a step in a dataflow), please make sure to inform the correct reproduction order to whoever wants to replicate the experiment. This can also be obtained by running ``reprounzip graph``; please refer to :ref:`provenance-graph` for more information.\n\nReproZip can also combine multiple traces into a single one, in order to create a single bundle, using the ``reprozip combine`` command. The runs of each subsequent trace are simply appended in order.\n\nPacking GUI and Interactive Tools\n+++++++++++++++++++++++++++++++++\n\nReproZip is able to pack GUI tools. Additionally, there is no restriction in packing interactive experiments (i.e., experiments that require input from users). Note, however, that if entering something different can make the experiment load additional dependencies, the experiment will probably fail when reproduced on a different machine.\n\n.. _packing-clientserv:\n\nCapturing Connections to Servers\n++++++++++++++++++++++++++++++++\n\nWhen reproducing an experiment that communicates with a server, the experiment will try to connect to the same server, which may or may not fail depending on the status of the server at the moment of the reproduction. However, if the experiment uses a local server (e.g.: database) that the user has control over, this server can also be captured, together with the experiment, to ensure that the connection will succeed. Users should create a script to:\n\n* start the server,\n* execute the experiment, and\n* stop the server,\n\nand use *reprozip* to trace the script execution, rather than the experiment itself. In this way, ReproZip is able to capture the local server as well, which ensures that the server will be alive at the time of the reproduction.\n\nFor example, if you have an web app that uses MySQL and that runs until ``Ctrl+C`` is received, you can use the following script::\n\n #!/bin/sh\n\n if [ \"$(id -u)\" != 0 ]; then echo \"This script needs to run as root so that it can execute MySQL\" >&2; exit 1; fi\n\n # Start MySQL\n sudo -u mysql /usr/sbin/mysqld --pid-file=/run/mysqld/mysqld.pid &\n sleep 5\n\n # Don't exit the whole script on Ctrl+C\n trap ' ' INT\n\n # Execute actual experiment that uses the database\n ./manage.py runserver 0.0.0.0:8000\n\n trap - INT\n\n # Graceful shutdown\n /usr/bin/mysqladmin shutdown\n\nNote the use of ``trap`` to avoid exiting the entire script when pressing ``Ctrl+C``, to make sure that the database gets shutdown via the next command.\n\nExcluding Sensitive and Third-Party Information\n+++++++++++++++++++++++++++++++++++++++++++++++\n\nReproZip automatically tries to identify log and temporary files, removing them from the bundle, but the configuration file should be edited to remove any sensitive information that the experiment uses, or any third-party file/software that should not be distributed. Note that the ReproZip team is **not responsible** for personal and non-authorized files that may get distributed in a package; users should double-check the configuration file and their package before sending it to others.\n\nIdentifying Output Files\n++++++++++++++++++++++++\n\nThe `reprozip` component tries to automatically identify the main output files generated by the experiment during the ``trace`` command to provide useful interfaces for users during the unpacking step. However, if the experiment creates unique names for its outputs every time it is executed (e.g.: names with current date and time), the *reprounzip* component will not be able to correctly detect these; it assumes that input and output files do not have their path names changed between different executions. In this case, handling output files will fail. It is recommended that users modify their experiment (or use a wrapper script) to generate a symbolic link (with a fixed name) that always points to the latest result, and use that as the output file's path in the configuration file (under the ``inputs_outputs`` section).\n" }, { "path": "docs/reprozip.rst", "content": "Why ReproZip?\n*************\n\nReproducibility is a core component of the scientific process: it helps researchers all around the world to verify the results and to also build on them, allowing science to move forward. In natural science, long tradition requires experiments to be described in enough detail so that they can be reproduced by researchers around the world. The same standard, however, has not been widely applied to computational science, where researchers often have to rely on plots, tables, and figures included in papers, which loosely describe the obtained results.\n\nThe truth is computational reproducibility can be very painful to achieve for a number of reasons. Take the author-reviewer scenario of a scientific paper as an example. Authors must generate a compendium that encapsulates all the inputs needed to correctly reproduce their experiments: the data, a complete specification of the experiment and its steps, and information about the originating computational environment (OS, hardware architecture, and library dependencies). Keeping track of this information manually is rarely feasible: it is both time-consuming and error-prone. First, computational environments are complex, consisting of many layers of hardware and software, and the configuration of the OS is often hidden. Second, tracking library dependencies is challenging, especially for large experiments. If authors did not plan for reproducibility since the beginning of the project, reproducibility is drastically hampered.\n\nFor reviewers, even with a compendium in their hands, it may be hard to reproduce the results. There may be no instructions about how to execute the code and explore it further; the experiment may not run on his operating system; there may be missing libraries; library versions may be different; and several issues may arise while trying to install all the required dependencies, a problem colloquially known as `dependency hell `__.\n\nReproZip helps alleviate these problems by allowing the user to easily capture all the necessary components in a single, distributable bundle. Also, the tool makes it easier to reproduce an experiment by providing different unpacking methods and interfaces that avoids the need to install all the required dependencies and that makes it possible to run the experiment under different inputs.\n" }, { "path": "docs/traceschema.rst", "content": ".. _trace-schema:\n\nTrace Database Schema\n*********************\n\nThe database contains three tables: ``processes``, ``opened_files``, and ``executed_files``.\n\n``processes``\n'''''''''''''\n\nThis table contains information about all the processes. A process is identified by Linux as a *pid* (process id), and is either a thread or a full-fledged process.\n\nNote that processes are different from programs, and there is no one-to-one relationship with executions. A process is created by `clone(2) `__ or `fork(2) `__ and not necessarily followed by `execve(2) `__. By contrast, a program can change its image by calling execve(2) without creating new processes (i.e., without changing *pid*).\n\nEach entry in the ``processes`` table has the id of its parent, i.e. the process that created it by calling clone(2) or fork(2), except the original process that *reprozip* created, for which parent is NULL. There is thus exactly one process with a NULL parent per run stored in the bundle.\n\n::\n\n CREATE TABLE processes(\n id INTEGER NOT NULL PRIMARY KEY,\n run_id INTEGER NOT NULL,\n parent INTEGER,\n timestamp INTEGER NOT NULL,\n is_thread BOOLEAN NOT NULL,\n exitcode INTEGER\n );\n\n``opened_files``\n''''''''''''''''\n\nThis table contains information regarding the files accessed by the processes. Note that a failed access (e.g.: trying to read a non-existing file, permission denied, etc.) is not logged. A single path might appear several times, even if accessed by the same process.\n\nEach file has a numerical id, the canonical path name, the process that accessed it (from which you can get the executable by cross-referencing ``processes``, also using the timestamp), and the mode.\n\n::\n\n CREATE TABLE opened_files(\n id INTEGER NOT NULL PRIMARY KEY,\n run_id INTEGER NOT NULL,\n name TEXT NOT NULL,\n timestamp INTEGER NOT NULL,\n mode INTEGER NOT NULL,\n is_directory BOOLEAN NOT NULL,\n process INTEGER NOT NULL\n );\n\nThe *mode* attribute is a binary OR of the following values (accessible from ``reprounzip.common``)::\n\n FILE_READ = 0x01\n FILE_WRITE = 0x02\n FILE_WDIR = 0x04\n FILE_STAT = 0x08\n FILE_LINK = 0x10\n\n``executed_files``\n''''''''''''''''''\n\nThis is a variant of ``opened_files`` for file executions, i.e. `execve(2) `__ calls. There is no mode here (file is opened for reading by the call) and they are never directories; however, *workingdir*, *argv* (command-line arguments) and *envp* (environment variables) are added. *argv* is a list of arguments separated by null bytes (``0x00``) [#nullbytes]_, and *envp* is a list of ``VAR=value`` pairs separated by null (``0x00``) bytes [#nullbytes]_. Note that, again, failed executions (execve returns) are not logged.\n\n::\n\n CREATE TABLE executed_files(\n id INTEGER NOT NULL PRIMARY KEY,\n name TEXT NOT NULL,\n run_id INTEGER NOT NULL,\n timestamp INTEGER NOT NULL,\n process INTEGER NOT NULL,\n argv TEXT NOT NULL,\n envp TEXT NOT NULL,\n workingdir TEXT NOT NULL\n );\n\n.. [#nullbytes] Note that Python's sqlite3 lib is affected by `bug 13676 `__ up to Python 2.7.3, which prevents it from reading text or blob fields with embedded null bytes.\n" }, { "path": "docs/troubleshooting.rst", "content": ".. _troubleshooting:\n\nTroubleshooting\n***************\n\nThe best way to start solving an issue in ReproZip is probably to look at the log messages. Some messages are not displayed by default when running ReproZip, but you can use the ``--verbose`` (or ``-v``) flag to display them. In addition, all the log messages are stored under ``$HOME/.reprozip/log``.\n\nPlease feel free to contact us at reprozip@nyu.edu if you encounter issues while using ReproZip.\n\n------------\n\n.. _file_id:\n\n:Issue: **\"** `reprozip` **does not identify my input/output file.\"**\n:Diagnosis: ReproZip uses some heuristics to identify an input or output file. However, this is only intended to be a starting point, since these heuristics may fail.\n:Solution: You should check the configuration file and edit the ``inputs_outputs`` section if necessary; giving readable names to input/output files also helps during reproduction. Please refer to :ref:`packing-config` for more information.\n\n------------\n\n.. _systemd:\n\n:Issue: **\"None of my files are packed when tracing a daemon.\"**\n:Diagnosis: If you are starting the daemon via the `service` or `systemctl` tool, it might be calling `init` over a client/server connection. For example, this is the case if you are using SystemD. In this situation, ReproZip will successfully pack the client, but anything the server (`init`) does will not be captured, leaving out this entire daemon.\n:Solution: You can still trace the binary or a non-systemd `init` script directly.\n\n For example, instead of::\n\n $ reprozip trace service mysql start\n\n or::\n\n $ reprozip trace systemctl start mysql\n\n you can trace the binary::\n\n $ reprozip trace /usr/bin/mysqld\n\n Note that, if you choose to trace the binary, you need to figure out the right command line options to use.\n Also, note that running the init script in ``/etc/init.d/...`` is not enough, since those scripts get subverted to call `systemctl` when systemd is installed.\n\n------------\n\n.. _ptrace:\n\n:Issue: **\"** `reprozip` **fails with** ``couldn't use ptrace`` **\"**\n:Diagnosis: ``ptrace`` is the mechanism that ReproZip uses to attach to another process and follow its system calls. Because it is so powerful, some security policies, environments or isolation mechanism may disable it.\n:Solution:\n\n * If you are using Docker, you can use the Docker option ``--cap-add=SYS_PTRACE`` (or provide your own seccomp profile that allows ptrace, by adding ``\"ptrace\"`` to the `default profile `__; see `the Docker documentation on seccomp `__).\n\n------------\n\n.. _moving-outputs:\n\n:Issue: **\"** `reprounzip` **cannot get an output file using** ``download`` **after reproducing the experiment.\"**\n:Diagnosis: This is probably the case where this output file does not have a fixed path name. It is common for experiments to dynamically choose where the outputs should be written, e.g.: by putting the date and time in the filename. However, ReproZip uses filenames in the ``inputs_outputs`` section of the configuration file to detect those when reproducing the experiment: if the name of the output file when reproducing is different from when it was originally packed, ReproZip cannot detect these as output files, and therefore, cannot get them through the ``download`` command.\n:Solution: The easiest way to solve this issue is to re-pack the experiment: write a simple bash script that runs the experiment and either renames outputs or creates symbolic links to them with known filenames; then, trace this script (instead of the actual entry-point of your experiment) and specify these fixed path names in the ``inputs_outputs`` section of the configuration file.\n\n------------\n\n.. _compiler_mac:\n\n:Issue: **\"** `reprounzip-vagrant` **installation fails with error** ``unknown argument: '-mno-fused-madd'`` **on Mac OS X.\"**\n:Diagnosis: This is an issue with the Apple LLVM compiler, which treats unrecognized command-line options as errors.\n:Solution: As a workaround, before installing `reprounzip-vagrant`, run the following::\n\n $ export CFLAGS=\"-Wno-error=unused-command-line-argument-hard-error-in-future\"\n\n Then, re-install `reprounzip-vagrant`::\n\n $ pip install -I reprounzip-vagrant\n\n Or use the following command in case you want all the available plugins::\n\n $ pip install -I reprounzip[all]\n\n------------\n\n:Issue: **\"The experiment fails with** ``Error: Can't open display: :0`` **when trying to reproduce it.\"**\n:Diagnosis: The experiment probably involves running a GUI tool.\n:Solution: The `reprounzip` component supports GUI tools, but it is not enabled by default; add the flag ``--enable-x11`` to the ``run`` command to enable it. See :ref:`gui-tools` for more information.\n\n------------\n\n.. _directory_error:\n\n:Issue: **\"The experiment run with** `reprounzip directory` **fails to find a file that has been packed.\"**\n:Diagnosis: The `directory` unpacker does not provide any isolation from the filesystem: if the experiment being reproduced use absolute paths, these will point outside the experiment directory, and files may not be found.\n:Solution: Make sure that the experiment does not use any absolute paths: if only relative paths are used internally and in the command line, ``reprounzip directory`` should work. As an alternative, you can use other unpackers (e.g.: ``reprounzip chroot`` and ``reprounzip vagrant``) that work in the presence of hardcoded absolute paths.\n\n------------\n\n.. _distribnotfound:\n\n:Issue: **\"** `reprounzip` **fails with** ``DistributionNotFound`` **errors.\"**\n:Diagnosis: You probably have some plugins left over from a previous installation.\n:Solution: Be sure to upgrade or remove outdated plugins when you upgrade `reprounzip`. The following command may help::\n\n $ pip install -U reprounzip[all]\n\n------------\n\n:Issue: **\"** `reprounzip` **shows** ``running in chroot, ignoring request`` **.\"**\n:Diagnosis: This message comes from the systemd client, which will probably not work with ReproZip.\n:Solution: In this case, the experiment should be re-packed without using systemd (see :ref:`this issue ` for more information).\n\n------------\n\n:Issue: **\"** ``reprounzip vagrant setup`` **fails to resolve a host address.\"**\n:Diagnosis: When running ``reprounzip vagrant setup``, if you get an error similar to this::\n\n ==> default: failed: Temporary failure in name resolution.\n ==> default: wget: unable to resolve host address ...\n\n there is probably a firewall blocking the Vagrant VM to have Internet connection; the VM needs Internet connection to download required software for setting up the experiment for you.\n:Solution: Make sure that your anti-virus/firewall is not causing this issue.\n\n------------\n\n.. _vagrant-memory:\n\n:Issue: **\"The experiment fails because of insufficient memory in Vagrant.\"**\n:Diagnosis: It is possible that the default amount of memory allocated to the VM is insufficient for the experiment. You can see a lot of different messages there, including:\n\n * ``Out of memory``\n * ``Could not allocate memory``\n * ``Killed``\n\n:Solution: From VirtualBox, stop the machine and allocate more memory under `Settings > System > Motherboard > Memory`.\n\n You can also use the ``--memory`` option when you run ``reprounzip vagrant setup`` to specify the amount of memory (in megabytes) at that time.\n\n------------\n\n.. _nosuchfile:\n\n:Issue: **\"** ``reprounzip run`` **fails with** ``no such file or directory`` **or similar.\"**\n:Diagnosis: This error message may have different reasons, but it often means that a specific version of a library or a dynamic linker is missing:\n\n 1. If you are requesting `reprounzip` to install software using the package manager (by running ``reprounzip installpkgs``), it is possible that the software packages from the package manager are not compatible with the ones required by the experiment.\n 2. If, while packing, the user chose not to include some packages, `reprounzip` will try to install the ones from the package manager, which may not be compatible.\n 3. If you are using ``reprounzip vagrant`` or ``reprounzip docker``, ReproZip may be failing to detect the closest base system for unpacking the experiment.\n:Solution:\n 1. Use the files inside the experiment bundle to ensure compatibility.\n 2. Contact the author of the ReproZip bundle to ask for a new package with all software packages included.\n 3. Try a different base system that you think it is closer to the original one by using the option ``--base-image`` when running these unpackers.\n\n------------\n\n:Issue: **\"There are warnings from requests/urllib3 when running ReproZip.\"**\n ::\n\n /usr/local/lib/python2.7/dist-packages/requests/packages/urllib3/util/ssl_.py:79:\n InsecurePlatformWarning: A true SSLContext object is not available. This\n prevents urllib3 from configuring SSL appropriately and may cause certain SSL\n connections to fail. For more information, see\n https://urllib3.readthedocs.io/en/latest/security.html#insecureplatformwarning.\n\n:Diagnosis: Most Python versions are insecure, because they do not validate SSL certificates, thus generating these warnings.\n:Solution: If you are using Python 2.7.9 and later, you shouldn't be affected, but if you see ``InsecurePlatformWarning``, you can run ``pip install requests[security]``, which should bring in the missing components.\n" }, { "path": "docs/unpacked-format.rst", "content": ".. _unpacked-format:\n\nStructure of Unpacked Experiments\n*********************************\n\nWhile *reprounzip* is designed to allow users to reproduce an experiment without having to master the tool used to run it (e.g.: `Vagrant `__ and `Docker `__), in some situations it might be useful to go behind the scenes and interact with the unpacked experiments directly.\n\nThis page describes in more details how the unpackers operate.\n\n.. note:: Future versions of unpackers might work in a different way. No attempt is made to make unpacked experiments compatible across different versions of *reprounzip*. Bundles will always be compatible though.\n\n.. _unpacked-common:\n\nCommon Files across Unpackers\n=============================\n\nThe unpacked directory contains the original configuration file as ``config.yml``. In fact, the VisTrails integration relies on it.\n\nA file named ``.reprounzip`` also marks the directory as an unpacked experiment. This is a Python pickle file containing a dictionary with various types of information:\n\n* ``unpacker`` maps to the unpacker's name.\n* ``input_files`` is used by the uploader/downloader machinery to keep the state of the input files inside the experiment, as they may be replaced by the user or overwritten by runs.\n* Other information specific to the unpacker, as described next.\n\n.. _unpacked-directory:\n\nThe `directory` Unpacker\n========================\n\nThe experiment directory contains:\n\n* The original configuration file ``config.yml``.\n* The pickle file ``.reprounzip``.\n* The tarball ``inputs.tar.gz``, which contains the original files that were identifies as input files. This tarball is used for file restoration using ``upload :`` (see :ref:`unpacker-input-output`).\n* A directory called ``root``, which contains all the bundled files in their original path, with symbolic links to absolute paths rewritten to prepend the path to ``root``.\n\n::\n\n unpacked-directory/\n .reprounzip\n config.yml\n inputs.tar.gz\n root/\n ...\n\nWhen running the ``run`` command, the unpacker sets ``LD_LIBRARY_PATH`` and ``PATH`` to point inside ``root``, and optionally ``DISPLAY`` and ``XAUTHORITY`` to the host's ones.\n\n.. _unpacked-chroot:\n\nThe `chroot` Unpacker\n=====================\n\nThe experiment directory contains:\n\n* The original configuration file ``config.yml``.\n* The pickle file ``.reprounzip``, which stores whether magic directories are mounted, as explained below.\n* The tarball ``inputs.tar.gz``, which contains the original files that were identifies as input files. This tarball is used for file restoration using ``upload :`` (see :ref:`unpacker-input-output`).\n* A directory called ``root``, which contains all the bundled files in their original path, with no symbolic links rewritten and file ownership restored.\n\n::\n\n unpacked-directory/\n .reprounzip\n config.yml\n inputs.tar.gz\n root/\n dev/\n dev/pts/\n proc/\n ...\n\nIf a file is listed in the configuration file but wasn't packed (i.e.: ``pack_files`` was set to ``false`` for a software package), such file is copied from the host; if this file does not exist on the host, a warning is shown when unpacking.\n\nUnless ``--dont-bind-magic-dirs`` is specified when unpacking, the special directories ``/dev``, ``/dev/pts``, and ``/proc`` are mounted with ``mount -o bind`` from the host.\nAlso, if ``/bin/sh`` or ``/usr/bin/env`` weren't both packed, a static build of `busybox `__ is downloaded and put under ``/bin/busybox``, and the missing binaries are created as symbolic links pointing to busybox.\n\nShould you require a shell inside the experiment environment, you can use::\n\n chroot root/ /bin/sh\n\n.. _unpacked-vagrant:\n\nThe `vagrant` Unpacker\n======================\n\nThe experiment directory contains:\n\n* The original configuration file ``config.yml``.\n* The pickle file ``.reprounzip``, which stores whether a chroot is used, as explained below.\n* The tarball ``data.tgz``, which is part of the ``.rpz`` file and used to populate the virtual machine (VM) when it gets created.\n* The setup script ``setup.sh``.\n* The file ``rpz-files.list``, which contains the list of files to unpack. This list is passed to ``tar -T`` while unpacking.\n* A ``Vagrantfile``, which is used to build the VM.\n\n::\n\n unpacked-directory/\n .reprounzip\n config.yml\n data.tgz\n busybox\n Vagrantfile\n setup.sh\n rpz-files.list\n\nOnce ``vagrant up`` has been run by the ``setup/start`` command, a ``.vagrant`` subdirectory is created, and its content is managed by Vagrant (and appears to vary among different platforms).\n\nNote that Vagrant drives VirtualBox or a similar virtualization software to run the VM. These will maintain state outside of the experiment directory. If you need to reconfigure or otherwise interact with the VM, you should do it from that virtualization software (e.g.: VirtualBox). The VM is named as the experiment directory with an additional suffix.\n\nThere are two modes for the virtual machine, controlled through command-line flags:\n\n* The default mode, ``--use-chroot``, creates a chroot environment inside the VM at ``/experimentroot``. This allows ReproZip to unpack very different file system hierarchies without breaking the base system of the VM (in particular, ``ssh`` needs to keep working for the VM to be usable). In this mode, software packages that were not packed (i.e.: ``pack_files`` was set to ``false``) are installed in the VM and their required files are copied to the ``/experimentroot`` hierarchy. The software packages that were packed are simply copied over without any interaction with the VM's system.\n* If ``--dont-use-chroot`` is used, no chroot environment is created. Files from software packages are never copied from the ``.rpz`` file; instead, they get installed from the package manager. Other files are simply unpacked in the VM system, possibly overwriting existing files. As long as *reprounzip-vagrant* manages to find a VM image with the same operating system as the original one, reproduction is expected to work reliably.\n\nIn the ``--use-chroot`` mode, a static build of `busybox `__ is downloaded and put under ``/experimentroot/busybox``, and if ``/bin/sh`` wasn't packed, it is created as a symbolic link pointing to busybox.\n\nUploading and downloading files from the environment is done via the shared directory ``/vagrant``, which is the experiment directory mounted in the VM by Vagrant.\n\nShould you require a shell inside the experiment environment, you can use::\n\n vagrant ssh\n\nPlease be aware of whether ``--use-chroot`` is in use when accessing the experiment environment: in this case, the experiment's files are located under ``/experimentroot``.\n\n.. _unpacked-docker:\n\nThe `docker` Unpacker\n=====================\n\nThe experiment directory contains:\n\n* The original configuration file ``config.yml``.\n* The pickle file ``.reprounzip``, which stores the name of the images built by the unpacker, as explained below.\n* The tarball ``data.tgz``, which is part of the ``.rpz`` file and used to populate the Docker container.\n* The file ``rpz-files.list``, which contains the list of files to unpack. This list is passed to ``tar -T`` while unpacking.\n* A ``Dockerfile``, which is used to build the original image.\n\n::\n\n unpacked-directory/\n .reprounzip\n config.yml\n data.tgz\n busybox\n rpzsudo\n Dockerfile\n rpz-files.list\n\nStatic builds of `busybox `__ and `rpzsudo `__ are always downloaded and put into the Docker image as ``/busybox`` and ``/rpzsudo``, respectively.\n\nNote that the ``docker`` command connects to a Docker daemon over a socket and that state will be changed there. The daemon might not be local; in particular, ``docker-machine`` might be used, which allows `reprounzip-docker` to be used on non-Linux machines, and the daemon might be in a virtual machine, on another host, or in the cloud. The `docker` unpacker will keep the environment variables set when calling Docker, notably ``DOCKER_HOST``, so these can be set accordingly before running the unpacker.\n\nImages and containers built by the unpacker are given a random name with the prefixes ``reprounzip_image_`` and ``reprounzip_run_``, respectively; they are cleaned up when the ``destroy`` command is invoked. There are two images of which `reprounzip-docker` keeps track in the ``.reprounzip`` pickle file: the initial image, i.e., the one built by ``setup/build`` by calling ``docker build``, and the current image (initially the same as the initial image), which has been affected by a number of ``run`` and ``upload`` calls. Running the ``reset`` command returns to the initial image without having to rebuild. After each ``run`` invocation, the container is committed to a new current image so that state is kept.\n\nA ``--detach`` option allows to start container and forget about them. reprounzip-docker leaves the container running and doesn't wait for it; this means that you can start a service on a remote machine, but note that because that container won't be committed to a new image, the side-effects of running it won't affect later executions on the same unpacked folder.\n\nUploading files to the environment is done by running a simple Dockerfile that builds a new image. Downloading files is done via the ``docker cp`` command.\n" }, { "path": "docs/unpacking.rst", "content": ".. _unpacking:\n\nUsing *reprounzip*\n******************\n\nWhile *reprozip* is responsible for tracing and packing an experiment, *reprounzip* is the component used for the unpacking step. *reprounzip* is distributed with three **unpackers** for Linux (:ref:`reprounzip directory `, :ref:`reprounzip chroot `, and :ref:`reprounzip installpkgs `), but more unpackers are supported by installing additional plugins; some of these plugins are compatible with different environments as well (e.g.: :ref:`reprounzip-vagrant ` and :ref:`reprounzip-docker `).\n\n.. _unpack-info:\n\nInspecting a Bundle\n===================\n\nShowing Bundle Information\n++++++++++++++++++++++++++\n\nBefore unpacking an experiment, it is often useful to have further information with respect to its bundle. The ``reprounzip info`` command allows users to do so::\n\n $ reprounzip info \n\nwhere `` corresponds to the experiment bundle (i.e., the ``.rpz`` file).\n\nThe output of this command has three sections. The first section, `Pack information`, contains general information about the experiment bundle, including size and total number of files::\n\n ----- Pack information -----\n Compressed size: \n Unpacked size: \n Total packed paths: \n\nThe next section, `Metadata`, contains information about dependencies (i.e., software packages), machine architecture from the packing environment, and experiment runs::\n\n ----- Metadata -----\n Total software packages: \n Packed software packages: \n Architecture: (current: )\n Distribution: (current: )\n Runs:\n : \n : \n ...\n\nNote that, for `Architecture` and `Distribution`, the command shows information with respect to both the original environment (i.e., the environment where the experiment was packed) and the current one (i.e., the environment where the experiment is to be unpacked). This helps users understand the differences between the environments in order to provide a better guidance in choosing the most appropriate unpacker.\n\nIf the verbose mode is used, more detailed information on the runs is provided::\n\n $ reprounzip -v info \n ...\n ----- Metadata -----\n ...\n Runs:\n : \n wd: \n exitcode: \n : \n wd: \n exitcode: \n ...\n\nLast, the section `Unpackers` shows which of the installed *reprounzip* unpackers can be successfully used in the current environment::\n\n ----- Unpackers -----\n Compatible:\n ...\n Incompatible:\n ...\n Unknown:\n ...\n\n`Compatible` lists the unpackers that can be used in the current environment, while `Incompatible` lists the unpackers that are not supported in the current environment. When using the verbose mode, an additional `Unknown` list shows the installed unpackers that may not work. As an example, for an experiment originally packed on Ubuntu and a user reproducing it on Windows, the `vagrant` unpacker (available through the :ref:`reprounzip-vagrant ` plugin) is compatible, but :ref:`installpkgs ` is not; `vagrant` may also be listed under `Unknown` if ``vagrant`` is not in PATH (e.g.: if `Vagrant `__ is not installed).\n\n.. _showfiles:\n\nShowing Input and Output Files\n++++++++++++++++++++++++++++++\n\nThe ``reprounzip showfiles`` command can be used to list the input and output files defined for the experiment. These files are identified by an id, which is either chosen by ReproZip or set in the configuration file before creating the ``.rpz`` file::\n\n $ reprounzip showfiles bundle.rpz\n Input files:\n program_config\n ipython_config\n input_data\n Output files:\n rendered_image\n logfile\n\nUsing the flag ``-v`` shows the complete path of each of these files in the experiment environment::\n\n $ reprounzip -v showfiles bundle.rpz\n Input files:\n program_config (/home/user/.progrc)\n ipython_config (/home/user/.ipython/profile_default/ipython_config.py)\n input_data (/home/user/experiment/input.bin)\n Output files:\n rendered_image (/home/user/experiment/output.png)\n logfile (/home/user/experiment/log.txt)\n\nYou can use the ``--input`` or ``--output`` flags to show only files that are inputs or outputs. If the bundle contains multiple runs, you can also filter files for a specific run::\n\n $ reprounzip -v showfiles bundle.rpz preprocessing-step\n Input files:\n input_data (/home/user/experiment/input.bin)\n Output files:\n logfile (/home/user/experiment/log.txt)\n\nwhere `preprocessing-step` is the run id. To see the dataflow of the experiment, please refer to :ref:`graph`.\n\nThe ``reprounzip showfiles`` command is particularly useful if you want to replace an input file with your own, or to get and save an output file for further examination. Please refer to :ref:`unpacker-input-output` for more information.\n\n.. versionadded:: 1.0.4\n The ``--input`` and ``--output`` flags.\n\n.. _provenance-graph:\n\nCreating a Provenance Graph\n+++++++++++++++++++++++++++\n\nReproZip also allows users to generate a *provenance graph* related to the experiment execution by reading the metadata available in the ``.rpz`` bundle. This graph shows the experiment runs as well as the files and other dependencies they access during execution; this is particularly useful to visualize and understand the dataflow of the experiment.\n\nSee :ref:`graph` for details.\n\n.. _unpack-unpackers:\n\nUnpackers\n=========\n\nFrom the same ``.rpz`` bundle, `reprounzip` allows users to set up the experiment for reproduction in several ways by the use of different `unpackers`. Unpackers are plugins that have general interface and commands, but can also provide their own command-line syntax and options. Thanks to the decoupling between packing and unpacking steps, ``.rpz`` files from older versions of ReproZip can be used with new unpackers.\n\nThe `reprounzip` tool comes with three unpackers that are only compatible with Linux (``reprounzip directory``, ``reprounzip chroot``, and ``reprounzip installpkgs``). Additional unpackers, such as ``reprounzip vagrant`` and ``reprounzip docker``, can be installed separately. Next, each unpacker is described in more details; for more information on how to use an unpacker, please refer to :ref:`unpacker-commands`.\n\n.. _unpack-directory:\n\nThe `directory` Unpacker: Unpacking as a Plain Directory\n++++++++++++++++++++++++++++++++++++++++++++++++++++++++\n\nThe *directory* unpacker (``reprounzip directory``) allows users to unpack the entire experiment (including library dependencies) in a single directory, and to reproduce the experiment directly from that directory. It does so by automatically setting up environment variables (e.g.: PATH, HOME, and LD_LIBRARY_PATH) that point the experiment execution to the created directory, which has the same structure as in the packing environment.\n\nPlease note that, although this unpacker is easy to use and does not require any privilege on the reproducing machine, it is **unreliable** since the directory is not isolated in any way from the remainder of the system. In particular, should the experiment use absolute paths, they will hit the host system instead. However, if the system has all the required packages (see :ref:`unpack-installpkgs`), and the experiment's files are addressed with relative paths, the use of this unpacker should not cause any problems.\n\n.. warning:: ``reprounzip directory`` provides no isolation of the filesystem, as mentioned before. If the experiment uses absolute paths, either provided by you or hardcoded in the experiment, **they will point outside the unpacked directory**. Please be careful to use relative paths in the configuration and command line if you want this unpacker to work with your experiment. Other unpackers are more reliable in this regard.\n\n.. note:: ``reprounzip directory`` is automatically distributed with `reprounzip`.\n\n.. seealso:: :ref:`Why does 'reprounzip directory' fail with \"IOError\"? `\n\n.. _unpack-chroot:\n\nThe `chroot` Unpacker: Providing Isolation with the *chroot* Mechanism\n++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++\n\nIn the *chroot* unpacker (``reprounzip chroot``), similar to ``reprounzip directory``, a directory is created from the experiment bundle; however, a full system environment is also built, which can then be run with ``chroot(2)``, a Linux mechanism that changes the root directory ``/`` for the experiment to the experiment directory. Therefore, this unpacker addresses the limitation of the *directory* unpacker and does not fail in the presence of hardcoded absolute paths. Note as well that it **does not interfere with the current environment** since the experiment is isolated in that single directory.\n\n.. warning:: Do **not** try to delete the experiment directory manually; **always** use ``reprounzip chroot destroy``. If ``/dev`` is mounted inside, you will also delete your system's device pseudo-files (these can be restored by rebooting or running the ``MAKEDEV`` script).\n\n.. note:: Although *chroot* offers pretty good isolation, it is not considered completely safe: it is possible for processes owned by root to \"escape\" to the outer system. We recommend not running untrusted programs with this plugin.\n\n.. note:: ``reprounzip chroot`` is automatically distributed with `reprounzip`.\n\n.. _unpack-installpkgs:\n\nThe `installpkgs` Unpacker: Installing Software Packages\n++++++++++++++++++++++++++++++++++++++++++++++++++++++++\n\nBy default, ReproZip identifies if the current environment already has the required software packages for the experiment, then using the installed ones for reproduction. For the non-installed software packages, it uses the dependencies packed in the original environment and extracted under the experiment directory.\n\nUsers may also let ReproZip try and install all the dependencies of the experiment on their machine by using the *installpkgs* unpacker (``reprounzip installpkgs``). This unpacker currently works for distribution based on Debian or RPM packages (e.g.: Ubuntu, CentOS, Fedora, ...), and uses the package manager to automatically install all the required software packages directly on the current machine, thus **interfering with your environment**.\n\nTo install the required dependencies, the following command should be used::\n\n $ reprounzip installpkgs \n\nUsers may use flag *y* or *assume-yes* to automatically confirm all the questions from the package manager; flag *missing* to install only the software packages that were not originally included in the experiment package (i.e.: software packages excluded in the configuration file); and flag *summary* to simply provide a summary of which software packages are installed or not in the current environment **without installing any dependency**.\n\n.. warning:: Note that the package manager may not install the same software version as required for running the experiment, and if the versions are incompatible, the reproduction may fail.\n\n.. note:: This unpacker is only used to install software packages. Users still need to use either ``reprounzip directory`` or ``reprounzip chroot`` to extract the experiment and execute it.\n\n.. note:: ``reprounzip installpkgs`` is automatically distributed with `reprounzip`.\n\n.. _unpackers:\n\n.. _unpack-vagrant:\n\nThe `vagrant` Unpacker: Building a Virtual Machine\n++++++++++++++++++++++++++++++++++++++++++++++++++\n\nThe *vagrant* unpacker (``reprounzip vagrant``) allows an experiment to be unpacked into a Virtual Machine and reproduced in that emulated environment, by automatically using `Vagrant `__. Therefore, the experiment can be reproduced in any environment supported by this tool, i.e., Linux, Mac OS X, and Windows. Note that the plugin assumes that Vagrant and VirtualBox are installed on your machine.\n\nIn addition to the commands listed in :ref:`unpacker-commands`, you can use ``suspend`` to save the virtual machine state to disk, and ``setup/start`` to restart a previously-created machine::\n\n $ reprounzip vagrant suspend \n $ reprounzip vagrant setup/start \n\nThe ``setup`` command also takes a ``--memory`` argument to explicitely select how many megabytes of RAM to allocate to the virtual machine.\n\n.. note:: This unpacker is **not** distributed with `reprounzip`; it is a separate package that should be installed before use (see :ref:`install`).\n\n.. versionadded:: 1.0.1\n The ``--memory`` option.\n\n.. versionadded:: 1.0.4\n The ``suspend`` command.\n\n.. _docker-plugin:\n\nThe `docker` Unpacker: Building a Docker Container\n++++++++++++++++++++++++++++++++++++++++++++++++++\n\nReproZip can also extract and reproduce experiments as `Docker `__ containers. The *docker* unpacker (``reprounzip docker``) is responsible for such integration and it assumes that Docker is already installed in the current environment.\n\nYou can pass arguments to the ``docker(1)`` program by using the ``--docker-option`` option to the ``setup`` or ``run`` commands.\n\nThanks to Docker's image layers feature, you can easily go back to the initial image after having run commands in the environment or replaced input files. To do that, use the ``reset`` command::\n\n $ reprounzip docker reset \n\n.. note:: This unpacker is **not** distributed with `reprounzip`; it is a separate package that should be installed before use (see :ref:`install`).\n\n.. _unpacker-commands:\n\nUsing an Unpacker\n=================\n\nOnce you have chosen (and installed) an unpacker for your machine, you can use it to setup and run a packed experiment. An unpacker creates an **experiment directory** in which the working files are placed; these can be either the full filesystem (for *directory* or *chroot* unpackers) or other content (e.g.: a handle on a virtual machine for the *vagrant* unpacker); for the *chroot* unpacker, it might have mount points. To make sure that you free all resources and that you do not damage your environment, you should **always use the destroy command** to delete the experiment directory, not just merely delete it manually. See more information about this command below.\n\nAll the following commands need to state which unpacker is being used (i.e., ``reprounzip directory`` for the `directory` unpacker, ``reprounzip chroot`` for the `chroot` unpacker, ``reprounzip vagrant`` for the `vagrant` unpacker, and ``reprounzip docker`` for the `docker` unpacker). For the purpose of this documentation, we will use the `docker` unpacker; to use a different one, just replace ``docker`` in the following with the unpacker of your interest.\n\n.. seealso:: :ref:`unpacked-format` provides further detailed information on unpackers.\n\nSetting Up an Experiment Directory\n++++++++++++++++++++++++++++++++++\n\n.. note:: Some unpackers require an Internet connection during the ``setup`` command, to download some of the support software or the packages that were not packed. Make sure that you have an Internet connection, and that there is no firewall blocking the access.\n\nTo create the directory where the execution will take place, the ``setup`` command should be used::\n\n $ reprounzip docker setup \n\nwhere `` is the directory where the experiment will be unpacked, i.e., the experiment directory.\n\nNote that, once this is done, you should only remove `` with the `destroy` command described below: deleting this directory manually might leave files behind, or even damage your system through bound filesystems.\n\nThe other unpacker commands take the `` argument; they do not need the original bundle for the reproduction.\n\nReproducing the Experiment\n++++++++++++++++++++++++++\n\nAfter creating the directory, the experiment can be reproduced by issuing the ``run`` command::\n\n $ reprounzip docker run \n\nwhich will execute the experiment inside the experiment directory. Users may also change the command line of the experiment by using ``--cmdline``::\n\n $ reprounzip docker run --cmdline \n\nwhere `` is the modified command line. This is particularly useful to reproduce and test the experiment under different input parameter values. Using ``--cmdline`` without an argument only prints the original command line.\n\nIf the bundle contains multiple `runs` (separate commands that were packed together), all the runs are reproduced. You can also provide the id of the run or runs to be used::\n\n $ reprounzip docker run \n $ reprounzip docker run --cmdline \n\nFor example::\n\n $ reprounzip docker run unpacked-experiment 0-1,3 # First, second, and fourth runs\n $ reprounzip docker run unpacked-experiment 2- # Third run and up\n $ reprounzip docker run unpacked-experiment compile,test # Runs named 'compile' and 'test', in this order\n\nIf the experiment involves running a GUI tool, the graphical interface can be enable by using ``--enable-x11``::\n\n $ reprounzip docker run --enable-x11\n\nwhich will forward the X connection from the experiment to the X server running on your machine. In this case, make sure you have a running X server.\n\nIf the experiment is a server, for example a website, a database management system, etc, you can expose ports from the experiment on your local machines. This is not required for the `directory` and `chroot` unpackers, since they offer no isolation of the network; for the `docker` and `vagrant` unpackers, use the ``--expose-port`` option::\n\n $ reprounzip docker run --expose-port 8000:80 unpacked-experiment # Expose TCP port 80 (HTTP) of the experiment at http://localhost:8000/\n $ reprounzip docker run --expose-port 3000 unpacked-experiment # Expose TCP port 3000 of the experiment at localhost:3000\n $ reprounzip docker run --expose-port 5553:53/udp unpacked-experiment # Expose UDP port 53 of the experiment at localhost:5553\n\nNote that in some situations, you might want to pass specific environment variables to the experiment, for example to set execution limits or parameters (such as OpenMPI information). To that effect, you can use the ``--pass-env VARNAME`` option to pass variables from the current machine, overriding the value from the original packing machine (`VARNAME` can be a regex). You can also set a variable to any value using ``--set-env VARNAME=value``. For example::\n\n $ reprounzip docker run unpacked-experiment --pass-env 'OMPI_.*' --pass-env LANG --set-env DATA_SERVER_ADDRESS=localhost\n\n.. versionadded:: 1.0.3\n The ``--pass-env`` and ``-set-env`` options.\n\nRemoving the Experiment Directory\n+++++++++++++++++++++++++++++++++\n\nThe ``destroy`` command will unmount mounted paths, destroy virtual machines, free container images, and delete the experiment directory::\n\n $ reprounzip docker destroy \n\nMake sure you always use this command instead of simply deleting the directory manually.\n\n.. _unpacker-input-output:\n\nManaging Input and Output Files\n+++++++++++++++++++++++++++++++\n\nWhen tracing an experiment, ReproZip tries to identify which are the input and output files of the experiment. This can also be adjusted in the configuration file before packing.\nIf the unpacked experiment has such files, ReproZip provides some commands to manipulate them.\n\nFirst, you can list these files using the ``showfiles`` command::\n\n $ reprounzip showfiles \n Input files:\n program_config\n ipython_config\n input_data\n Output files:\n rendered_image\n logfile\n\nTo replace an input file with your own, `reprounzip`, you can use the ``upload`` command::\n\n $ reprounzip docker upload :\n\nwhere `` is the new file's path and `` is the input file to be replaced (from ``showfiles``). This command overwrites the original path in the environment with the file you provided from your system. To restore the original input file, the same command, but in the following format, should be used::\n\n $ reprounzip docker upload :\n\nRunning the ``showfiles`` command shows what the input files are currently set to::\n\n $ reprounzip showfiles --input\n Input files:\n program_config\n (original)\n ipython_config\n C:\\Users\\Remi\\Documents\\ipython-config\n\nIn this example, the input `program_config` has not been changed (the one bundled in the ``.rpz`` file will be used), while the input `ipython_config` has been replaced.\n\nAfter running the experiment, all the generated output files will be located under the experiment directory. To copy an output file from this directory to another desired location, use the ``download`` command::\n\n $ reprounzip docker download :\n\nwhere `` is the output file to be copied (from ``showfiles``) and `` is the desired destination of the file. If an empty destination is specified, the file will be printed to stdout::\n\n $ reprounzip docker download :\n\nYou can also omit the colon ``:`` altogether to download the file to the current directory under its original name::\n\n $ reprounzip docker download \n\nor even use ``--all`` to download every output file to the current directory under their original names.\n\nNote that the ``upload`` command takes the file id on the right side of the colon (meaning that the path is the origin, and the id is the destination), while the ``download`` command takes it on the left side (meaning that the id is the origin, and the path is the destination). Both commands move data from left to right.\n\n.. versionadded:: 1.0.4\n Allow ``download `` (no explicit destination), and add ``--all``.\n\n.. seealso:: :ref:`Why can’t 'reprounzip' get my output files after reproducing an experiment? `\n\nRunning the Experiment in VisTrails\n+++++++++++++++++++++++++++++++++++\n\nIn addition to reproducing the experiment, you may want to edit its dataflow by inserting your own processes between and around the experiment steps, or even by connecting multiple ReproZip'd experiments. However, manually managing the experiment workflow (with the help of ``reprounzip upload/download`` commands) can quickly become painful.\n\nTo allow users to easily manage these workflows, `reprounzip` provides a plugin for the `VisTrails `__ scientific workflow management system, which has easy-to-use interfaces to run and modify a dataflow. See :ref:`vistrails` for more information.\n\nFurther Considerations\n======================\n\nReproducing Multiple Execution Paths\n++++++++++++++++++++++++++++++++++++\n\nThe *reprozip* component can only guarantee that *reprounzip* will successfully reproduce the same execution path that the original experiment followed. There is no guarantee that the experiment won't need a different set of files if you use a different configuration; if some of these files were not packed into the ``.rpz`` package, the reproduction may fail.\n" }, { "path": "docs/vistrails.rst", "content": ".. _vistrails:\n\nVisTrails Plugin\n****************\n\nThe `reprounzip-vistrails` plugin is a component that interacts with the existing unpackers to generate and execute a `VisTrails `__ workflow from the packed experiment. By using VisTrails, you can better manage the experiment workflow: it allows you to run unpacked ReproZip experiments, replace input files, visualize and retrieve output files, and modify the dataflow to re-use steps of the original experiment. For more information about VisTrails, please see their `user's guide `__.\n\n.. note:: This plugin is **not** distributed with `reprounzip`; it is a separate component that should be installed beforehand (see :ref:`install` for more details).\n\nOnce the plugin is installed, a VisTrails workflow will be generated every time you unpack an experiment; note that this process does not require VisTrails. The workflow file is named ``vistrails.vt`` and is generated under the unpacked directory.\n\nVisTrails Setup\n===============\n\nTo run the workflow, you need VisTrails installed on your machine and the reprounzip package, which is included in VisTrails 2.2.3 and up. If you used an installer for either VisTrails or *reprounzip*, you need to set the path to reprounzip's Python interpreter in VisTrails's package configuration dialog:\n\n.. figure:: figures/vistrails-config.png\n :align: center\n\nFor example, this will be ``/opt/reprounzip/python27/bin/python`` if you used the Mac OS X installer, and something similar to ``C:\\Program Files (x86)\\ReproUnzip\\python2.7\\python.exe`` if you used the Windows installer.\n\nUsage\n=====\n\nThe workflow generated by `reprounzip-vistrails` contains a reference to the unpacked directory and VisTrails modules calling each run in a dataflow; running the workflow is thus the same as running the experiment through *reprounzip run*, except that VisTrails provides caching.\n\nYou can open VisTrails and then open the workflow file, which is auto-generated when you unpack an experiment.\n\n.. figure:: figures/vistrails-gene.png\n :align: center\n\nThe ``Directory`` module refers to the experiment, while the ``Run`` module refers to an experiment run. ``Directory`` is passed from module to module to represent the changes in the environment, since each ``Run`` will change the internal state of the machine. Note that, if you send the workflow file to another machine, the workflow needs to be updated with the correct path to the unpacked experiment by editing the input port of the ``Directory`` module.\n\nIn a ``Run`` module, the ports (except the ``Directory`` one) represent the input and output files that are used by the corresponding run. The module also exposes the command line, should you want to change a parameter or tweak flags.\n\nNote that a file exposed as an output port in one ``Run`` module may be the input port of the next ``Run`` module, and yet these are not connected. The dataflow, however, still works since the entire machine state is carried to the next execution. Connecting these ports would work, but would also make *reprounzip* download the file to VisTrails and then upload it again in the same location. You can speed up the workflow by not connecting the files that you do not want to examine or change, since downloading and uploading may take time.\n\nYou are encouraged to go through the `VisTrails documentation `__ to get familiar with the system.\n" }, { "path": "file-format.md", "content": "ReproZip pack file format\n=========================\n\nThe pack command creates an archive that contains the necessary files and environment information to reproduce the experiment elsewhere. Because reproducibility is the objective, it is meant to be a stable format, and should it evolve, we intend to keep the previous ones.\n\nGeneral structure\n-----------------\n\nPacked experiments have the .rpz extension by default. They are an (optionally gzipped) tar archive containing at least a METADATA/version file, which indicates the file format. Currently, it always contains either \"`REPROZIP VERSION 1\\n`\" or \"`REPROZIP VERSION 2\\n`\" (without quotes of any kind).\n\nFormat version 1\n----------------\n\n* `METADATA/trace.sqlite3` is the original trace file generated by the C tracer.\n* `METADATA/config.yml` is the configuration file that the pack command processed to make this pack. It contains information about the files that were included, organized by distribution package.\n* `DATA/` contains the files listed in the configuration (except packages with `packfiles: false`).\n\nFormat version 2\n----------------\n\nIn this version, the outer archive is usually uncompressed, and the `DATA/` directory is placed in a separate archive named `DATA.tar.gz`. This has the added benefit or keeping the metadata uncompressed, allowing for faster access.\n" }, { "path": "reprounzip/LICENSE.txt", "content": "Copyright (C) 2014, New York University\nAll rights reserved.\n\nRedistribution and use in source and binary forms, with or without\nmodification, are permitted provided that the following conditions are met:\n\n1. Redistributions of source code must retain the above copyright notice, this\n list of conditions and the following disclaimer.\n\n2. Redistributions in binary form must reproduce the above copyright notice,\n this list of conditions and the following disclaimer in the documentation\n and/or other materials provided with the distribution.\n\n3. Neither the name of the copyright holder nor the names of its contributors\n may be used to endorse or promote products derived from this software\n without specific prior written permission.\n\nTHIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \"AS IS\" AND\nANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED\nWARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE\nDISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE\nFOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL\nDAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR\nSERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER\nCAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,\nOR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE\nOF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n" }, { "path": "reprounzip/MANIFEST.in", "content": "include README.rst\ninclude LICENSE.txt\n" }, { "path": "reprounzip/README.rst", "content": "ReproZip project\n================\n\n`ReproZip `__ is a tool aimed at simplifying the process of creating reproducible experiments from command-line executions, a frequently-used common denominator in computational science. It tracks operating system calls and creates a bundle that contains all the binaries, files and dependencies required to run a given command on the author's computational environment (packing step). A reviewer can then extract the experiment in his environment to reproduce the results (unpacking step).\n\nreprounzip\n----------\n\nThis is the component responsible for the unpacking step on Linux distributions.\n\nPlease refer to `reprozip `__, `reprounzip-vagrant `_, and `reprounzip-docker `_ for other components and plugins.\n\nA GUI is available at `reprounzip-qt `_.\n\nAdditional Information\n----------------------\n\nFor more detailed information, please refer to our `website `_, as well as to our `documentation `_.\n\nReproZip is currently being developed at `NYU `_. The team includes:\n\n* `Fernando Chirigati `_\n* `Juliana Freire `_\n* `Remi Rampin `_\n* `Dennis Shasha `_\n* `Vicky Rampin `_\n" }, { "path": "reprounzip/reprounzip/__init__.py", "content": "try: # pragma: no cover\n __import__('pkg_resources').declare_namespace(__name__)\nexcept ImportError: # pragma: no cover\n from pkgutil import extend_path\n __path__ = extend_path(__path__, __name__)\n" }, { "path": "reprounzip/reprounzip/common.py", "content": "# Copyright (C) 2014 New York University\n# This file is part of ReproZip which is released under the Revised BSD License\n# See file LICENSE for full license details.\n\n# This file is shared:\n# reprozip/reprozip/common.py\n# reprounzip/reprounzip/common.py\n\n\"\"\"Common functions between reprozip and reprounzip.\n\nThis module contains functions that are specific to the reprozip software and\nits data formats, but that are shared between the reprozip and reprounzip\npackages. Because the packages can be installed separately, these functions are\nin a separate module which is duplicated between the packages.\n\nAs long as these are small in number, they are not worth putting in a separate\npackage that reprozip and reprounzip would both depend on.\n\n\"\"\"\n\nfrom __future__ import division, print_function, unicode_literals\n\nimport atexit\nimport contextlib\nimport copy\nfrom datetime import datetime\nfrom distutils.version import LooseVersion\nimport functools\nimport gzip\nimport logging\nimport logging.handlers\nimport os\nfrom rpaths import PosixPath, Path\nimport sys\nimport tarfile\nimport usagestats\nimport yaml\nimport zipfile\n\nfrom .utils import iteritems, itervalues, unicode_, stderr, UniqueNames, \\\n escape, optional_return_type, isodatetime, hsize, join_root, copyfile\n\n\nlogger = logging.getLogger(__name__.split('.', 1)[0])\n\n\nFILE_READ = 0x01\nFILE_WRITE = 0x02\nFILE_WDIR = 0x04\nFILE_STAT = 0x08\nFILE_LINK = 0x10\nFILE_SOCKET = 0x20\n\n\nclass File(object):\n \"\"\"A file, used at some point during the experiment.\n \"\"\"\n comment = None\n\n def __init__(self, path, size=None):\n self.path = path\n self.size = size\n\n def __eq__(self, other):\n return (isinstance(other, File) and\n self.path == other.path)\n\n def __ne__(self, other):\n return not self.__eq__(other)\n\n def __hash__(self):\n return hash(self.path)\n\n\nclass Package(object):\n \"\"\"A distribution package, containing a set of files.\n \"\"\"\n def __init__(self, name, version, files=None, packfiles=True, size=None):\n self.name = name\n self.version = version\n self.files = list(files) if files is not None else []\n self.packfiles = packfiles\n self.size = size\n\n def __eq__(self, other):\n return (isinstance(other, Package) and\n self.name == other.name and\n self.version == other.version)\n\n def __ne__(self, other):\n return not self.__eq__(other)\n\n def add_file(self, file_):\n self.files.append(file_)\n\n def __unicode__(self):\n return '%s (%s)' % (self.name, self.version)\n __str__ = __unicode__\n\n\n# Pack format history:\n# 1: used by reprozip 0.2 through 0.7. Single tar.gz file, metadata under\n# METADATA/, data under DATA/\n# 2: pack is usually not compressed, metadata under METADATA/, data in another\n# DATA.tar.gz (files inside it still have the DATA/ prefix for ease-of-use\n# in unpackers)\n#\n# Pack metadata history:\n# 0.2: used by reprozip 0.2\n# 0.2.1:\n# config: comments directories as such in config\n# trace database: adds executed_files.workingdir, adds processes.exitcode\n# data: packs dynamic linkers\n# 0.3:\n# config: don't list missing (unpacked) files in config\n# trace database: adds opened_files.is_directory\n# 0.3.1: no change\n# 0.3.2: no change\n# 0.4:\n# config: adds input_files, output_files, lists parent directories\n# 0.4.1: no change\n# 0.5: no change\n# 0.6: no change\n# 0.7:\n# moves input_files and output_files from run to global scope\n# adds processes.is_thread column to trace database\n# 0.8: adds 'id' field to run\n\n\nclass RPZPack(object):\n \"\"\"Encapsulates operations on the RPZ pack format.\n \"\"\"\n data = zip = tar = None\n\n def __init__(self, pack):\n self.pack = Path(pack)\n\n if self._open_tar():\n pass\n elif self._open_zip():\n pass\n else:\n raise ValueError(\"File doesn't appear to be an RPZ pack\")\n\n def _open_tar(self):\n try:\n self.tar = tarfile.open(str(self.pack), 'r:*')\n except tarfile.TarError:\n return False\n try:\n f = self.tar.extractfile('METADATA/version')\n except KeyError:\n raise ValueError(\"Invalid ReproZip file\")\n version = f.read()\n f.close()\n if version.startswith(b'REPROZIP VERSION '):\n try:\n version = int(version[17:].rstrip())\n except ValueError:\n version = None\n if version in (1, 2):\n self.version = version\n self.data_prefix = PosixPath(b'DATA')\n else:\n raise ValueError(\n \"Unknown format version %r (maybe you should upgrade \"\n \"reprounzip? I only know versions 1 and 2\" % version)\n else:\n raise ValueError(\"File doesn't appear to be an RPZ pack\")\n\n if self.version == 1:\n self.data = self.tar\n elif version == 2:\n self.data = tarfile.open(\n fileobj=self.tar.extractfile('DATA.tar.gz'),\n mode='r:*')\n else:\n assert False\n return True\n\n def _open_zip(self):\n try:\n self.zip = zipfile.ZipFile(str(self.pack))\n except zipfile.BadZipfile:\n return False\n try:\n f = self.zip.open('METADATA/version')\n except KeyError:\n raise ValueError(\"Invalid ReproZip file\")\n version = f.read()\n f.close()\n if version.startswith(b'REPROZIP VERSION '):\n try:\n version = int(version[17:].rstrip())\n except ValueError:\n version = None\n if version == 1:\n raise ValueError(\"Format version 1 is not accepted for ZIP\")\n elif version == 2:\n self.version = 2\n self.data_prefix = PosixPath(b'DATA')\n else:\n raise ValueError(\n \"Unknown format version %r (maybe you should upgrade \"\n \"reprounzip? I only know versions 1 and 2\" % version)\n else:\n raise ValueError(\"File doesn't appear to be an RPZ pack\")\n\n if sys.version_info < (3, 7):\n # zip.open() doesn't return a seekable file object before 3.6\n # Extract to a temporary file instead\n fd, temporary_data = Path.tempfile(\n prefix='reprounzip_data_',\n suffix='.zip',\n )\n os.close(fd)\n self._extract_file('DATA.tar.gz', temporary_data)\n self.data = tarfile.open(str(temporary_data), mode='r:*')\n atexit.register(os.remove, temporary_data.path)\n else:\n self.data = tarfile.open(fileobj=self.zip.open('DATA.tar.gz'),\n mode='r:*')\n return True\n\n def remove_data_prefix(self, path):\n if not isinstance(path, PosixPath):\n path = PosixPath(path)\n components = path.components[1:]\n if not components:\n return path.__class__('')\n return path.__class__(*components)\n\n def open_config(self):\n \"\"\"Gets the configuration file.\n \"\"\"\n if self.tar is not None:\n return self.tar.extractfile('METADATA/config.yml')\n else:\n return self.zip.open('METADATA/config.yml')\n\n def extract_config(self, target):\n \"\"\"Extracts the config to the specified path.\n\n It is up to the caller to remove that file once done.\n \"\"\"\n self._extract_file('METADATA/config.yml', target)\n\n def _extract_file(self, name, target):\n if self.tar is not None:\n member = copy.copy(self.tar.getmember(name))\n member.name = str(target.components[-1])\n self.tar.extract(member, path=str(Path.cwd() / target.parent))\n else:\n member = copy.copy(self.zip.getinfo(name))\n member.filename = str(target.components[-1])\n self.zip.extract(member, path=str(Path.cwd() / target.parent))\n target.chmod(0o644)\n assert target.is_file()\n\n def _extract_file_gz(self, name, target):\n if self.tar is not None:\n f_in = self.tar.extractfile(name)\n else:\n f_in = self.zip.open(name)\n f_in_gz = gzip.open(f_in)\n f_out = target.open('wb')\n try:\n chunk = f_in_gz.read(4096)\n while len(chunk) == 4096:\n f_out.write(chunk)\n chunk = f_in_gz.read(4096)\n if chunk:\n f_out.write(chunk)\n finally:\n f_out.close()\n f_in_gz.close()\n f_in.close()\n target.chmod(0o644)\n\n @contextlib.contextmanager\n def with_config(self):\n \"\"\"Context manager that extracts the config to a temporary file.\n \"\"\"\n fd, tmp = Path.tempfile(prefix='reprounzip_')\n os.close(fd)\n self.extract_config(tmp)\n yield tmp\n tmp.remove()\n\n def extract_trace(self, target):\n \"\"\"Extracts the trace database to the specified path.\n\n It is up to the caller to remove that file once done.\n \"\"\"\n target = Path(target)\n if self.version == 2:\n try:\n if self.tar is not None:\n self.tar.getmember('METADATA/trace.sqlite3.gz')\n else:\n self.zip.getinfo('METADATA/trace.sqlite3.gz')\n except KeyError:\n pass\n else:\n self._extract_file_gz('METADATA/trace.sqlite3.gz', target)\n return\n elif self.version != 2:\n assert False\n self._extract_file('METADATA/trace.sqlite3', target)\n\n @contextlib.contextmanager\n def with_trace(self):\n \"\"\"Context manager extracting the trace database to a temporary file.\n \"\"\"\n fd, tmp = Path.tempfile(prefix='reprounzip_')\n os.close(fd)\n self.extract_trace(tmp)\n yield tmp\n tmp.remove()\n\n def list_data(self):\n \"\"\"Returns tarfile.TarInfo objects for all the data paths.\n \"\"\"\n return [copy.copy(m)\n for m in self.data.getmembers()\n if m.name.startswith('DATA/')]\n\n def data_filenames(self):\n \"\"\"Returns a set of filenames for all the data paths.\n\n Those paths begin with a slash / and the 'DATA' prefix has been\n removed.\n \"\"\"\n return set(PosixPath(m.name[4:])\n for m in self.data.getmembers()\n if m.name.startswith('DATA/'))\n\n def get_data(self, path):\n \"\"\"Returns a tarfile.TarInfo object for the data path.\n\n Raises KeyError if no such path exists.\n \"\"\"\n path = PosixPath(path)\n path = join_root(PosixPath(b'DATA'), path)\n return copy.copy(self.data.getmember(path))\n\n def extract_data(self, root, members):\n \"\"\"Extracts the given members from the data tarball.\n\n The members must come from get_data().\n \"\"\"\n # Check for CVE-2007-4559\n abs_root = root.absolute()\n for member in members:\n member_path = (root / member.name).absolute()\n if not member_path.lies_under(abs_root):\n raise ValueError(\"Invalid path in data tar\")\n\n self.data.extractall(str(root), members)\n\n def copy_data_tar(self, target):\n \"\"\"Copies the file in which the data lies to the specified destination.\n \"\"\"\n if self.tar is not None:\n if self.version == 1:\n self.pack.copyfile(target)\n elif self.version == 2:\n with target.open('wb') as fp:\n data = self.tar.extractfile('DATA.tar.gz')\n copyfile(data, fp)\n data.close()\n else:\n with target.open('wb') as fp:\n data = self.zip.open('DATA.tar.gz')\n copyfile(data, fp)\n data.close()\n\n def extensions(self):\n \"\"\"Get a list of extensions present in this pack.\n \"\"\"\n extensions = set()\n if self.tar is not None:\n for m in self.tar.getmembers():\n if m.name.startswith('EXTENSIONS/'):\n name = m.name[11:]\n if '/' in name:\n name = name[:name.index('/')]\n if name:\n extensions.add(name)\n else:\n for m in self.zip.infolist():\n if m.filename.startswith('EXTENSIONS/'):\n name = m.filename[11:]\n if '/' in name:\n name = name[:name.index('/')]\n if name:\n extensions.add(name)\n return extensions\n\n def close(self):\n if self.data is not self.tar:\n self.data.close()\n if self.tar is not None:\n self.tar.close()\n elif self.zip is not None:\n self.zip.close()\n self.data = self.zip = self.tar = None\n\n\nclass InvalidConfig(ValueError):\n \"\"\"Configuration file is invalid.\n \"\"\"\n\n\ndef read_files(files, File=File):\n if files is None:\n return []\n return [File(PosixPath(f)) for f in files]\n\n\ndef read_packages(packages, File=File, Package=Package):\n if packages is None:\n return []\n new_pkgs = []\n for pkg in packages:\n pkg['files'] = read_files(pkg['files'], File)\n new_pkgs.append(Package(**pkg))\n return new_pkgs\n\n\nConfig = optional_return_type(['runs', 'packages', 'other_files'],\n ['inputs_outputs', 'additional_patterns',\n 'format_version'])\n\n\n@functools.total_ordering\nclass InputOutputFile(object):\n def __init__(self, path, read_runs, write_runs):\n self.path = path\n self.read_runs = read_runs\n self.write_runs = write_runs\n\n def __eq__(self, other):\n return ((self.path, self.read_runs, self.write_runs) ==\n (other.path, other.read_runs, other.write_runs))\n\n def __lt__(self, other):\n return self.path < other.path\n\n def __repr__(self):\n return \"\" % (\n self.path, self.read_runs, self.write_runs)\n\n\ndef load_iofiles(config, runs):\n \"\"\"Loads the inputs_outputs part of the configuration.\n\n This tests for duplicates, merge the lists of executions, and optionally\n loads from the runs for reprozip < 0.7 compatibility.\n \"\"\"\n files_list = config.get('inputs_outputs') or []\n\n # reprozip < 0.7 compatibility: read input_files and output_files from runs\n if 'inputs_outputs' not in config:\n for i, run in enumerate(runs):\n for rkey, wkey in (('input_files', 'read_by_runs'),\n ('output_files', 'written_by_runs')):\n for k, p in iteritems(run.pop(rkey, {})):\n files_list.append({'name': k,\n 'path': p,\n wkey: [i]})\n\n files = {} # name:str: InputOutputFile\n paths = {} # path:PosixPath: name:str\n required_keys = set(['name', 'path'])\n optional_keys = set(['read_by_runs', 'written_by_runs'])\n uniquenames = UniqueNames()\n for i, f in enumerate(files_list):\n keys = set(f)\n if (not keys.issubset(required_keys | optional_keys) or\n not keys.issuperset(required_keys)):\n raise InvalidConfig(\"File #%d has invalid keys\")\n name = f['name']\n path = PosixPath(f['path'])\n readers = sorted(f.get('read_by_runs', []))\n writers = sorted(f.get('written_by_runs', []))\n if (\n not isinstance(readers, (tuple, list))\n or not all(isinstance(e, int) for e in readers)\n ):\n raise InvalidConfig(\"read_by_runs should be a list of integers\")\n if (\n not isinstance(writers, (tuple, list))\n or not all(isinstance(e, int) for e in writers)\n ):\n raise InvalidConfig(\"written_by_runs should be a list of integers\")\n if name in files:\n if files[name].path != path:\n old_name, name = name, uniquenames(name)\n logger.warning(\"File name appears multiple times: %s\\n\"\n \"Using name %s instead\",\n old_name, name)\n else:\n uniquenames.insert(name)\n if path in paths:\n if paths[path] == name:\n logger.warning(\"File appears multiple times: %s\", name)\n else:\n logger.warning(\"Two files have the same path (but different \"\n \"names): %s, %s\\nUsing name %s\",\n name, paths[path], paths[path])\n name = paths[path]\n files[name].read_runs.update(readers)\n files[name].write_runs.update(writers)\n else:\n paths[path] = name\n files[name] = InputOutputFile(path, readers, writers)\n\n return files\n\n\ndef load_config(filename, canonical, File=File, Package=Package):\n \"\"\"Loads a YAML configuration file.\n\n `File` and `Package` parameters can be used to override the classes that\n will be used to hold files and distribution packages; useful during the\n packing step.\n\n `canonical` indicates whether a canonical configuration file is expected\n (in which case the ``additional_patterns`` section is not accepted). Note\n that this changes the number of returned values of this function.\n \"\"\"\n with filename.open(encoding='utf-8') as fp:\n config = yaml.safe_load(fp)\n\n ver = LooseVersion(config['version'])\n\n keys_ = set(config)\n if 'version' not in keys_:\n raise InvalidConfig(\"Missing version\")\n # Accepts versions from 0.2 to 0.8 inclusive\n elif not LooseVersion('0.2') <= ver < LooseVersion('0.9'):\n pkgname = (__package__ or __name__).split('.', 1)[0]\n raise InvalidConfig(\"Loading configuration file in unknown format %s; \"\n \"this probably means that you should upgrade \"\n \"%s\" % (ver, pkgname))\n unknown_keys = keys_ - set(['pack_id', 'version', 'runs',\n 'inputs_outputs',\n 'packages', 'other_files',\n 'additional_patterns',\n # Deprecated\n 'input_files', 'output_files'])\n if unknown_keys:\n logger.warning(\"Unrecognized sections in configuration: %s\",\n ', '.join(unknown_keys))\n\n runs = config.get('runs') or []\n packages = read_packages(config.get('packages'), File, Package)\n other_files = read_files(config.get('other_files'), File)\n\n inputs_outputs = load_iofiles(config, runs)\n\n # reprozip < 0.7 compatibility: set inputs/outputs on runs (for plugins)\n for i, run in enumerate(runs):\n run['input_files'] = dict((n, f.path)\n for n, f in iteritems(inputs_outputs)\n if i in f.read_runs)\n run['output_files'] = dict((n, f.path)\n for n, f in iteritems(inputs_outputs)\n if i in f.write_runs)\n\n # reprozip < 0.8 compatibility: assign IDs to runs\n for i, run in enumerate(runs):\n if run.get('id') is None:\n run['id'] = \"run%d\" % i\n\n record_usage_package(runs, packages, other_files,\n inputs_outputs,\n pack_id=config.get('pack_id'))\n\n kwargs = {'format_version': ver,\n 'inputs_outputs': inputs_outputs}\n\n if canonical:\n if 'additional_patterns' in config:\n raise InvalidConfig(\"Canonical configuration file shouldn't have \"\n \"additional_patterns key anymore\")\n else:\n kwargs['additional_patterns'] = config.get('additional_patterns') or []\n\n return Config(runs, packages, other_files,\n **kwargs)\n\n\ndef write_file(fp, fi, indent=0):\n fp.write(\"%s - \\\"%s\\\"%s\\n\" % (\n \" \" * indent,\n escape(unicode_(fi.path)),\n ' # %s' % fi.comment if fi.comment is not None else ''))\n\n\ndef write_package(fp, pkg, indent=0):\n indent_str = \" \" * indent\n fp.write(\"%s - name: \\\"%s\\\"\\n\" % (indent_str, escape(pkg.name)))\n fp.write(\"%s version: \\\"%s\\\"\\n\" % (indent_str, escape(pkg.version)))\n if pkg.size is not None:\n fp.write(\"%s size: %d\\n\" % (indent_str, pkg.size))\n fp.write(\"%s packfiles: %s\\n\" % (indent_str, 'true' if pkg.packfiles\n else 'false'))\n fp.write(\"%s files:\\n\"\n \"%s # Total files used: %s\\n\" % (\n indent_str, indent_str,\n hsize(sum(fi.size\n for fi in pkg.files\n if fi.size is not None))))\n if pkg.size is not None:\n fp.write(\"%s # Installed package size: %s\\n\" % (\n indent_str, hsize(pkg.size)))\n for fi in sorted(pkg.files, key=lambda fi_: fi_.path):\n write_file(fp, fi, indent + 1)\n\n\ndef save_config(filename, runs, packages, other_files, reprozip_version,\n inputs_outputs=None,\n canonical=False, pack_id=None):\n \"\"\"Saves the configuration to a YAML file.\n\n `canonical` indicates whether this is a canonical configuration file\n (no ``additional_patterns`` section).\n \"\"\"\n dump = lambda x: yaml.safe_dump(x, encoding='utf-8', allow_unicode=True)\n with filename.open('w', encoding='utf-8', newline='\\n') as fp:\n # Writes preamble\n fp.write(\"\"\"\\\n# ReproZip configuration file\n# This file was generated by reprozip {version} at {date}\n\n{what}\n\n# Run info{pack_id}\nversion: \"{format!s}\"\n\"\"\".format(pack_id=(('\\npack_id: \"%s\"' % pack_id) if pack_id is not None\n else ''),\n version=escape(reprozip_version),\n format='0.8',\n date=isodatetime(),\n what=(\"# It was generated by the packer and you shouldn't need to \"\n \"edit it\" if canonical\n else \"# You might want to edit this file before running the \"\n \"packer\\n# See 'reprozip pack -h' for help\")))\n\n fp.write(\"runs:\\n\")\n for i, run in enumerate(runs):\n # Remove reprozip < 0.7 compatibility fields\n run = dict((k, v) for k, v in iteritems(run)\n if k not in ('input_files', 'output_files'))\n fp.write(\"# Run %d\\n\" % i)\n fp.write(dump([run]).decode('utf-8'))\n fp.write(\"\\n\")\n\n fp.write(\"\"\"\\\n# Input and output files\n\n# Inputs are files that are only read by a run; reprounzip can replace these\n# files on demand to run the experiment with custom data.\n# Outputs are files that are generated by a run; reprounzip can extract these\n# files from the experiment on demand, for the user to examine.\n# The name field is the identifier the user will use to access these files.\ninputs_outputs:\"\"\")\n for n, f in iteritems(inputs_outputs):\n fp.write(\"\"\"\\\n\n- name: {name}\n path: {path}\n written_by_runs: {writers}\n read_by_runs: {readers}\"\"\".format(name=n, path=unicode_(f.path),\n readers=repr(f.read_runs),\n writers=repr(f.write_runs)))\n\n fp.write(\"\"\"\\\n\n\n# Files to pack\n# All the files below were used by the program; they will be included in the\n# generated package\n\n# These files come from packages; we can thus choose not to include them, as it\n# will simply be possible to install that package on the destination system\n# They are included anyway by default\npackages:\n\"\"\")\n\n # Writes files\n for pkg in sorted(packages, key=lambda p: p.name):\n write_package(fp, pkg)\n\n fp.write(\"\"\"\\\n\n# These files do not appear to come with an installed package -- you probably\n# want them packed\nother_files:\n\"\"\")\n for f in sorted(other_files, key=lambda fi: fi.path):\n write_file(fp, f)\n\n if not canonical:\n fp.write(\"\"\"\\\n\n# If you want to include additional files in the pack, you can list additional\n# patterns of files that will be included\nadditional_patterns:\n# Examples:\n# - /etc/apache2/** # Everything under apache2/\n# - /var/log/apache2/*.log # Log files directly under apache2/\n# - /var/lib/lxc/*/rootfs/home/**/*.py # All Python files of all users in\n# # that container\n\"\"\")\n\n\nclass LoggingDateFormatter(logging.Formatter):\n \"\"\"Formatter that puts milliseconds in the timestamp.\n \"\"\"\n converter = datetime.fromtimestamp\n\n def formatTime(self, record, datefmt=None):\n ct = self.converter(record.created)\n t = ct.strftime(\"%H:%M:%S\")\n s = \"%s.%03d\" % (t, record.msecs)\n return s\n\n\ndef setup_logging(tag, verbosity):\n \"\"\"Sets up the logging module.\n \"\"\"\n levels = [logging.CRITICAL, logging.WARNING, logging.INFO, logging.DEBUG]\n console_level = levels[min(verbosity, 3)]\n file_level = logging.INFO\n min_level = min(console_level, file_level)\n\n # Create formatter\n fmt = \"[%s] %%(asctime)s %%(levelname)s: %%(message)s\" % tag\n formatter = LoggingDateFormatter(fmt)\n\n # Console logger\n handler = logging.StreamHandler()\n handler.setLevel(console_level)\n handler.setFormatter(formatter)\n\n # Set up logger\n rootlogger = logging.root\n rootlogger.setLevel(min_level)\n rootlogger.addHandler(handler)\n\n # File logger\n if os.environ.get('REPROZIP_NO_LOGFILE', '').lower() in ('', 'false',\n '0', 'off'):\n dotrpz = Path('~/.reprozip').expand_user()\n try:\n if not dotrpz.is_dir():\n dotrpz.mkdir()\n filehandler = logging.handlers.RotatingFileHandler(\n str(dotrpz / 'log'), mode='a',\n delay=False, maxBytes=400000, backupCount=5)\n except (IOError, OSError):\n logger.warning(\"Couldn't create log file %s\", dotrpz / 'log')\n else:\n filehandler.setFormatter(formatter)\n filehandler.setLevel(file_level)\n rootlogger.addHandler(filehandler)\n\n filehandler.emit(logging.root.makeRecord(\n __name__.split('.', 1)[0],\n logging.INFO,\n \"(log start)\", 0,\n \"Log opened %s %s\",\n (datetime.now().strftime(\"%Y-%m-%d\"), sys.argv),\n None))\n\n logging.getLogger('urllib3').setLevel(logging.INFO)\n\n\n_usage_report = None\n\n\ndef setup_usage_report(name, version):\n \"\"\"Sets up the usagestats module.\n \"\"\"\n global _usage_report\n\n certificate_file = get_reprozip_ca_certificate()\n\n _usage_report = usagestats.Stats(\n '~/.reprozip/usage_stats',\n usagestats.Prompt(enable='%s usage_report --enable' % name,\n disable='%s usage_report --disable' % name),\n os.environ.get('REPROZIP_USAGE_URL',\n 'https://stats.reprozip.org/'),\n version='%s %s' % (name, version),\n unique_user_id=True,\n env_var='REPROZIP_USAGE_STATS',\n ssl_verify=certificate_file.path)\n try:\n os.getcwd().encode('ascii')\n except (UnicodeEncodeError, UnicodeDecodeError):\n record_usage(cwd_ascii=False)\n else:\n record_usage(cwd_ascii=True)\n\n\ndef enable_usage_report(enable):\n \"\"\"Enables or disables usage reporting.\n \"\"\"\n if enable:\n _usage_report.enable_reporting()\n stderr.write(\"Thank you, usage reports will be sent automatically \"\n \"from now on.\\n\")\n else:\n _usage_report.disable_reporting()\n stderr.write(\"Usage reports will not be collected nor sent.\\n\")\n\n\ndef record_usage(**kwargs):\n \"\"\"Records some info in the current usage report.\n \"\"\"\n if _usage_report is not None:\n _usage_report.note(kwargs)\n\n\ndef record_usage_package(runs, packages, other_files,\n inputs_outputs,\n pack_id=None):\n \"\"\"Records the info on some pack file into the current usage report.\n \"\"\"\n if _usage_report is None:\n return\n for run in runs:\n record_usage(argv0=run['argv'][0])\n record_usage(pack_id=pack_id or '',\n nb_packages=len(packages),\n nb_package_files=sum(len(pkg.files)\n for pkg in packages),\n packed_packages=sum(1 for pkg in packages\n if pkg.packfiles),\n nb_other_files=len(other_files),\n nb_input_outputs_files=len(inputs_outputs),\n nb_input_files=sum(1 for f in itervalues(inputs_outputs)\n if f.read_runs),\n nb_output_files=sum(1 for f in itervalues(inputs_outputs)\n if f.write_runs))\n\n\ndef submit_usage_report(**kwargs):\n \"\"\"Submits the current usage report to the usagestats server.\n \"\"\"\n _usage_report.submit(kwargs,\n usagestats.OPERATING_SYSTEM,\n usagestats.SESSION_TIME,\n usagestats.PYTHON_VERSION)\n\n\ndef get_reprozip_ca_certificate():\n \"\"\"Gets the ReproZip CA certificate filename.\n \"\"\"\n fd, certificate_file = Path.tempfile(prefix='rpz_stats_ca_', suffix='.pem')\n with certificate_file.open('wb') as fp:\n fp.write(usage_report_ca)\n os.close(fd)\n atexit.register(os.remove, certificate_file.path)\n return certificate_file\n\n\nusage_report_ca = b'''\\\n-----BEGIN CERTIFICATE-----\nMIIDzzCCAregAwIBAgIJAMmlcDnTidBEMA0GCSqGSIb3DQEBCwUAMH4xCzAJBgNV\nBAYTAlVTMREwDwYDVQQIDAhOZXcgWW9yazERMA8GA1UEBwwITmV3IFlvcmsxDDAK\nBgNVBAoMA05ZVTERMA8GA1UEAwwIUmVwcm9aaXAxKDAmBgkqhkiG9w0BCQEWGXJl\ncHJvemlwLWRldkB2Z2MucG9seS5lZHUwHhcNMTQxMTA3MDUxOTA5WhcNMjQxMTA0\nMDUxOTA5WjB+MQswCQYDVQQGEwJVUzERMA8GA1UECAwITmV3IFlvcmsxETAPBgNV\nBAcMCE5ldyBZb3JrMQwwCgYDVQQKDANOWVUxETAPBgNVBAMMCFJlcHJvWmlwMSgw\nJgYJKoZIhvcNAQkBFhlyZXByb3ppcC1kZXZAdmdjLnBvbHkuZWR1MIIBIjANBgkq\nhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA1fuTW2snrVji51vGVl9hXAAZbNJ+dxG+\n/LOOxZrF2f1RRNy8YWpeCfGbsZqiIEjorBv8lvdd9P+tD3M5sh9L0zQPU9dFvDb+\nOOrV0jx59hbK3QcCQju3YFuAtD1lu8TBIPgGEab0eJhLVIX+XU5cYXrfoBmwCpN/\n1wXWkUhN91ZVMA0ylATAxTpnoNuMKzfTxT8pyOWajiTskYkKmVBAxgYJQe1YDFA8\nfglBNkQuHqP8jgYAniEBCAPZRMMq8WpOtyFx+L9LX9/WcHtAQyDPPb9M81KKgPQq\nurtCqtuDKxuqcX9zg4/O8l4nZ50pwaJjbH4kMW/wnLzTPvzZCPtJYQIDAQABo1Aw\nTjAdBgNVHQ4EFgQUJjhDDOup4P0cdrAVq1F9ap3yTj8wHwYDVR0jBBgwFoAUJjhD\nDOup4P0cdrAVq1F9ap3yTj8wDAYDVR0TBAUwAwEB/zANBgkqhkiG9w0BAQsFAAOC\nAQEAeKpTiy2WYPqevHseTCJDIL44zghDJ9w5JmECOhFgPXR9Hl5Nh9S1j4qHBs4G\ncn8d1p2+8tgcJpNAysjuSl4/MM6hQNecW0QVqvJDQGPn33bruMB4DYRT5du1Zpz1\nYIKRjGU7Of3CycOCbaT50VZHhEd5GS2Lvg41ngxtsE8JKnvPuim92dnCutD0beV+\n4TEvoleIi/K4AZWIaekIyqazd0c7eQjgSclNGgePcdbaxIo0u6tmdTYk3RNzo99t\nDCfXxuMMg3wo5pbqG+MvTdECaLwt14zWU259z8JX0BoeVG32kHlt2eUpm5PCfxqc\ndYuwZmAXksp0T0cWo0DnjJKRGQ==\n-----END CERTIFICATE-----\n-----BEGIN CERTIFICATE-----\nMIIDRDCCAiygAwIBAgIUXaa8P7qR4c0P51hCDIqj4GUbG/owDQYJKoZIhvcNAQEL\nBQAwEzERMA8GA1UEAwwIUmVwcm9aaXAwIBcNMjEwNDI5MjEwNTUzWhgPMjEyMTA0\nMjkyMTA1NTNaMBMxETAPBgNVBAMMCFJlcHJvWmlwMIIBIjANBgkqhkiG9w0BAQEF\nAAOCAQ8AMIIBCgKCAQEA3udPriZ8kziQE+OyLVozJFDSZTe8RLlpFsu/ZQjSnIh1\nTsENMMu1lwv0GVEpT/EbtD5ORtZzwYQ7Vuh+IO4TQDhA5KvyJD2gZW8hE4txkkQd\nyI5vSj0iiViA80tKB7FSDLsvz9iiDxShYHJI947gswbaLmampHIXD/Rjjs7+hmL5\nRRS5NL8vCp2/2QVj5wnJupa5O2l2T1M6S/SyFcAgBMM/FhDsaA/yf4NPcOG6gFuO\nb5mYz2ERSf4v9mRL+G3r6YULYGZWS5ThY0QoZ0lYt2nlthzwfftazrJ9+yfYBkoJ\nK6Ug8UGtyOb5m3mK00c4wS7/wzuGgLMszkE0nE9SfwIDAQABo4GNMIGKMB0GA1Ud\nDgQWBBSqrIPVnO5vkHj9ImGvOr38r4rcNjBOBgNVHSMERzBFgBSqrIPVnO5vkHj9\nImGvOr38r4rcNqEXpBUwEzERMA8GA1UEAwwIUmVwcm9aaXCCFF2mvD+6keHND+dY\nQgyKo+BlGxv6MAwGA1UdEwQFMAMBAf8wCwYDVR0PBAQDAgEGMA0GCSqGSIb3DQEB\nCwUAA4IBAQC2g8yX1c5JutH/qAKUVvqSwP2KBm3IyOjdbN7cvnwn0gMkwEj88j7p\ndKhfO0Kfp/N4iKj1YK7PBXfrdlYhxINSbfPSVS3A9XWi8pJwiwgBfjrrACRMhBsv\nHAQPnkXnaEJrQm//k8s4etT25JYaPgXS8t+dgVS0iqonYlpCB1XkE0gw1kLNCW5F\n1SimUehJ5bZrJYGgo6kp44F12kPMzNHvk50Sf2p3nm2d9/BNbbJQxUBKt9n+i6Ir\nxGGDWfg5F+BLKURWkoGBnnLPqkRxlkaGvk6QpIAfD1h99fTyuWUno3+NoQ7hS952\nyWbmqwbavTIyG+D+kfhbGFEdRxLF5BeK\n-----END CERTIFICATE-----\n'''\n" }, { "path": "reprounzip/reprounzip/main.py", "content": "# Copyright (C) 2014 New York University\n# This file is part of ReproZip which is released under the Revised BSD License\n# See file LICENSE for full license details.\n\n\"\"\"Entry point for the reprounzip utility.\n\nThis contains :func:`~reprounzip.reprounzip.main`, which is the entry point\ndeclared to setuptools. It is also callable directly.\n\nIt dispatchs to plugins registered through importlib.metadata as entry point\n``reprounzip.unpackers``.\n\"\"\"\n\nfrom __future__ import division, print_function, unicode_literals\n\nif __name__ == '__main__': # noqa\n from reprounzip.main import main\n main()\n\nimport argparse\nfrom importlib_metadata import entry_points\nimport locale\nimport logging\nimport sys\nimport traceback\n\nfrom reprounzip.common import setup_logging, \\\n setup_usage_report, enable_usage_report, \\\n submit_usage_report, record_usage\nfrom reprounzip import signals\nfrom reprounzip.unpackers.common import UsageError\nfrom reprounzip.utils import stderr\n\n\n__version__ = '1.3.2'\n\n\nlogger = logging.getLogger('reprounzip')\n\n\nunpackers = {}\n\n\ndef get_plugins(entry_point_name):\n for entry_point in entry_points().select(group=entry_point_name):\n try:\n func = entry_point.load()\n except Exception:\n print(\"Plugin %s from %s %s failed to initialize!\" % (\n entry_point.name,\n entry_point.dist.project_name, entry_point.dist.version),\n file=sys.stderr)\n traceback.print_exc(file=sys.stderr)\n continue\n name = entry_point.name\n # Docstring is used as description (used for detailed help)\n descr = func.__doc__.strip()\n # First line of docstring is the help (used for general help)\n descr_1 = descr.split('\\n', 1)[0]\n\n yield name, func, descr, descr_1\n\n\nclass RPUZArgumentParser(argparse.ArgumentParser):\n def error(self, message):\n sys.stderr.write('error: %s\\n' % message)\n self.print_help(sys.stderr)\n sys.exit(2)\n\n\ndef usage_report(args):\n if bool(args.enable) == bool(args.disable):\n logger.critical(\"What do you want to do?\")\n raise UsageError\n enable_usage_report(args.enable)\n sys.exit(0)\n\n\ndef main():\n \"\"\"Entry point when called on the command-line.\n \"\"\"\n # Locale\n try:\n locale.setlocale(locale.LC_ALL, '')\n except locale.Error as e:\n stderr.write(\"Couldn't set locale: %s\\n\" % e)\n\n # Parses command-line\n\n # General options\n def add_options(opts):\n opts.add_argument('--version', action='version',\n version=\"reprounzip version %s\" % __version__)\n\n # Loads plugins\n for name, func, descr, descr_1 in get_plugins('reprounzip.plugins'):\n func()\n\n parser = RPUZArgumentParser(\n description=\"reprounzip is the ReproZip component responsible for \"\n \"unpacking and reproducing an experiment previously \"\n \"packed with reprozip\",\n epilog=\"Please report issues to reprozip@nyu.edu\")\n add_options(parser)\n parser.add_argument('-v', '--verbose', action='count', default=1,\n dest='verbosity',\n help=\"augments verbosity level\")\n subparsers = parser.add_subparsers(title=\"subcommands\", metavar='')\n\n # usage_report subcommand\n parser_stats = subparsers.add_parser(\n 'usage_report',\n help=\"Enables or disables anonymous usage reports\")\n add_options(parser_stats)\n parser_stats.add_argument('--enable', action='store_true')\n parser_stats.add_argument('--disable', action='store_true')\n parser_stats.set_defaults(func=usage_report)\n\n # Loads unpackers\n for name, func, descr, descr_1 in get_plugins('reprounzip.unpackers'):\n plugin_parser = subparsers.add_parser(\n name, help=descr_1, description=descr,\n formatter_class=argparse.RawDescriptionHelpFormatter)\n add_options(plugin_parser)\n info = func(plugin_parser)\n plugin_parser.set_defaults(selected_unpacker=name)\n if info is None:\n info = {}\n unpackers[name] = info\n\n signals.pre_parse_args(parser=parser, subparsers=subparsers)\n args = parser.parse_args()\n signals.post_parse_args(args=args)\n if getattr(args, 'func', None) is None:\n parser.print_help(sys.stderr)\n sys.exit(2)\n signals.unpacker = getattr(args, 'selected_unpacker', None)\n setup_logging('REPROUNZIP', args.verbosity)\n\n setup_usage_report('reprounzip', __version__)\n if hasattr(args, 'selected_unpacker'):\n record_usage(unpacker=args.selected_unpacker)\n signals.pre_setup.subscribe(lambda **kw: record_usage(setup=True))\n signals.pre_run.subscribe(lambda **kw: record_usage(run=True))\n\n try:\n try:\n args.func(args)\n except UsageError:\n raise\n except Exception as e:\n signals.application_finishing(reason=e)\n submit_usage_report(result=type(e).__name__)\n raise\n else:\n signals.application_finishing(reason=None)\n except UsageError:\n parser.print_help(sys.stderr)\n sys.exit(2)\n submit_usage_report(result='success')\n sys.exit(0)\n" }, { "path": "reprounzip/reprounzip/orderedset.py", "content": "# From http://code.activestate.com/recipes/576694/\n# With added update()\n\n# Copyright (C) 2009 Raymond Hettinger\n#\n# Permission is hereby granted, free of charge, to any person obtaining a copy\n# of this software and associated documentation files (the \"Software\"), to deal\n# in the Software without restriction, including without limitation the rights\n# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell\n# copies of the Software, and to permit persons to whom the Software is\n# furnished to do so, subject to the following conditions:\n#\n# The above copyright notice and this permission notice shall be included in\n# all copies or substantial portions of the Software.\n#\n# THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\n# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\n# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN\n# THE SOFTWARE.\n\ntry:\n from collections.abc import MutableSet\nexcept ImportError:\n from collections import MutableSet\n\n\nclass OrderedSet(MutableSet):\n def __init__(self, iterable=None):\n self.end = end = []\n end += [None, end, end] # sentinel node for doubly linked list\n self.map = {} # key --> [key, prev, next_]\n if iterable is not None:\n self |= iterable\n\n def __len__(self):\n return len(self.map)\n\n def __contains__(self, key):\n return key in self.map\n\n def add(self, key):\n if key not in self.map:\n end = self.end\n curr = end[1]\n curr[2] = end[1] = self.map[key] = [key, curr, end]\n\n def discard(self, key):\n if key in self.map:\n key, prev, next_ = self.map.pop(key)\n prev[2] = next_\n next_[1] = prev\n\n def __iter__(self):\n end = self.end\n curr = end[2]\n while curr is not end:\n yield curr[0]\n curr = curr[2]\n\n def __reversed__(self):\n end = self.end\n curr = end[1]\n while curr is not end:\n yield curr[0]\n curr = curr[1]\n\n def pop(self, last=True):\n if not self:\n raise KeyError('set is empty')\n key = self.end[1][0] if last else self.end[2][0]\n self.discard(key)\n return key\n\n def __repr__(self):\n if not self:\n return '%s()' % (self.__class__.__name__,)\n return '%s(%r)' % (self.__class__.__name__, list(self))\n\n def __eq__(self, other):\n if isinstance(other, OrderedSet):\n return len(self) == len(other) and list(self) == list(other)\n return set(self) == set(other)\n\n def update(self, iterable):\n for key in iterable:\n self.add(key)\n" }, { "path": "reprounzip/reprounzip/pack_info.py", "content": "# Copyright (C) 2014 New York University\n# This file is part of ReproZip which is released under the Revised BSD License\n# See file LICENSE for full license details.\n\nfrom __future__ import division, print_function, unicode_literals\n\nimport argparse\nimport distro\nimport json\nimport logging\nimport platform\nfrom rpaths import Path\nimport sys\n\nfrom reprounzip.common import RPZPack, load_config as load_config_file\nfrom reprounzip.main import unpackers\nfrom reprounzip.unpackers.common import load_config, COMPAT_OK, COMPAT_MAYBE, \\\n COMPAT_NO, UsageError, shell_escape, metadata_read\nfrom reprounzip.utils import iteritems, itervalues, unicode_, hsize\n\n\nlogger = logging.getLogger('reprounzip')\n\n\ndef get_package_info(pack, read_data=False):\n \"\"\"Get information about a package.\n \"\"\"\n runs, packages, other_files = config = load_config(pack)\n inputs_outputs = config.inputs_outputs\n\n information = {}\n\n if read_data:\n total_size = 0\n total_paths = 0\n files = 0\n dirs = 0\n symlinks = 0\n hardlinks = 0\n others = 0\n\n rpz_pack = RPZPack(pack)\n for m in rpz_pack.list_data():\n total_size += m.size\n total_paths += 1\n if m.isfile():\n files += 1\n elif m.isdir():\n dirs += 1\n elif m.issym():\n symlinks += 1\n elif hasattr(m, 'islnk') and m.islnk():\n hardlinks += 1\n else:\n others += 1\n rpz_pack.close()\n\n information['pack'] = {\n 'total_size': total_size,\n 'total_paths': total_paths,\n 'files': files,\n 'dirs': dirs,\n 'symlinks': symlinks,\n 'hardlinks': hardlinks,\n 'others': others,\n }\n\n total_paths = 0\n packed_packages_files = 0\n unpacked_packages_files = 0\n packed_packages = 0\n for package in packages:\n nb = len(package.files)\n total_paths += nb\n if package.packfiles:\n packed_packages_files += nb\n packed_packages += 1\n else:\n unpacked_packages_files += nb\n nb = len(other_files)\n total_paths += nb\n\n information['meta'] = {\n 'total_paths': total_paths,\n 'packed_packages_files': packed_packages_files,\n 'unpacked_packages_files': unpacked_packages_files,\n 'packages': len(packages),\n 'packed_packages': packed_packages,\n 'packed_paths': packed_packages_files + nb,\n }\n\n if runs:\n architecture = runs[0]['architecture']\n if any(r['architecture'] != architecture\n for r in runs):\n logger.warning(\"Runs have different architectures\")\n information['meta']['architecture'] = architecture\n distribution = runs[0]['distribution']\n if any(r['distribution'] != distribution\n for r in runs):\n logger.warning(\"Runs have different distributions\")\n information['meta']['distribution'] = distribution\n\n information['runs'] = [\n dict((k, run[k])\n for k in ['id', 'binary', 'argv', 'environ',\n 'workingdir', 'signal', 'exitcode']\n if k in run)\n for run in runs]\n\n information['inputs_outputs'] = {\n name: {'path': str(iofile.path),\n 'read_runs': iofile.read_runs,\n 'write_runs': iofile.write_runs}\n for name, iofile in iteritems(inputs_outputs)}\n\n rpz_pack = RPZPack(pack)\n information['extensions'] = sorted(rpz_pack.extensions())\n\n # Unpacker compatibility\n unpacker_status = {}\n for name, upk in iteritems(unpackers):\n if 'test_compatibility' in upk:\n compat = upk['test_compatibility']\n if callable(compat):\n compat = compat(pack, config=config)\n if isinstance(compat, (tuple, list)):\n compat, msg = compat\n else:\n msg = None\n unpacker_status.setdefault(compat, []).append((name, msg))\n else:\n unpacker_status.setdefault(None, []).append((name, None))\n information['unpacker_status'] = unpacker_status\n\n return information\n\n\ndef _print_package_info(pack, info, verbosity=1):\n print(\"Pack file: %s\" % pack)\n print(\"\\n----- Pack information -----\")\n print(\"Compressed size: %s\" % hsize(pack.size()))\n\n info_pack = info.get('pack')\n if info_pack:\n if 'total_size' in info_pack:\n print(\"Unpacked size: %s\" % hsize(info_pack['total_size']))\n if 'total_paths' in info_pack:\n print(\"Total packed paths: %d\" % info_pack['total_paths'])\n if verbosity >= 3:\n print(\" Files: %d\" % info_pack['files'])\n print(\" Directories: %d\" % info_pack['dirs'])\n if info_pack.get('symlinks'):\n print(\" Symbolic links: %d\" % info_pack['symlinks'])\n if info_pack.get('hardlinks'):\n print(\" Hard links: %d\" % info_pack['hardlinks'])\n if info_pack.get('others'):\n print(\" Unknown (what!?): %d\" % info_pack['others'])\n print(\"\\n----- Metadata -----\")\n info_meta = info['meta']\n if verbosity >= 3:\n print(\"Total paths: %d\" % info_meta['total_paths'])\n print(\"Listed packed paths: %d\" % info_meta['packed_paths'])\n if info_meta.get('packages'):\n print(\"Total software packages: %d\" % info_meta['packages'])\n print(\"Packed software packages: %d\" % info_meta['packed_packages'])\n if verbosity >= 3:\n print(\"Files from packed software packages: %d\" %\n info_meta['packed_packages_files'])\n print(\"Files from unpacked software packages: %d\" %\n info_meta['unpacked_packages_files'])\n if 'architecture' in info_meta:\n print(\"Architecture: %s (current: %s)\" % (info_meta['architecture'],\n platform.machine().lower()))\n if 'distribution' in info_meta:\n distribution = ' '.join(t for t in info_meta['distribution'] if t)\n current_distribution = [distro.id(), distro.version()]\n current_distribution = ' '.join(t for t in current_distribution if t)\n print(\"Distribution: %s (current: %s)\" % (\n distribution, current_distribution or \"(not Linux)\"))\n if 'runs' in info:\n runs = info['runs']\n print(\"Runs (%d):\" % len(runs))\n for run in runs:\n cmdline = ' '.join(shell_escape(a) for a in run['argv'])\n if len(runs) == 1 and run['id'] == \"run0\":\n print(\" %s\" % cmdline)\n else:\n print(\" %s: %s\" % (run['id'], cmdline))\n if verbosity >= 2:\n print(\" wd: %s\" % run['workingdir'])\n if 'signal' in run:\n print(\" signal: %d\" % run['signal'])\n else:\n print(\" exitcode: %d\" % run['exitcode'])\n\n inputs_outputs = info.get('inputs_outputs')\n if inputs_outputs:\n if verbosity < 2:\n print(\"Inputs/outputs files (%d): %s\" % (\n len(inputs_outputs), \", \".join(sorted(inputs_outputs))))\n else:\n print(\"Inputs/outputs files (%d):\" % len(inputs_outputs))\n for name, f in sorted(iteritems(inputs_outputs)):\n t = []\n if f['read_runs']:\n t.append(\"in\")\n if f['write_runs']:\n t.append(\"out\")\n print(\" %s (%s): %s\" % (name, ' '.join(t), f['path']))\n\n extensions = info.get('extensions')\n if extensions:\n print(\"\\n----- Extensions -----\")\n for name in extensions:\n print(name)\n\n unpacker_status = info.get('unpacker_status')\n if unpacker_status:\n print(\"\\n----- Unpackers -----\")\n for s, n in [(COMPAT_OK, \"Compatible\"), (COMPAT_MAYBE, \"Unknown\"),\n (COMPAT_NO, \"Incompatible\")]:\n if s != COMPAT_OK and verbosity < 2:\n continue\n if s not in unpacker_status:\n continue\n upks = unpacker_status[s]\n print(\"%s (%d):\" % (n, len(upks)))\n for upk_name, msg in upks:\n if msg is not None:\n print(\" %s (%s)\" % (upk_name, msg))\n else:\n print(\" %s\" % upk_name)\n\n\ndef print_info(args):\n \"\"\"Writes out some information about a pack file.\n \"\"\"\n pack = Path(args.pack[0])\n\n info = get_package_info(pack, read_data=args.json or args.verbosity >= 2)\n if args.json:\n json.dump(info, sys.stdout, indent=2)\n sys.stdout.write('\\n')\n else:\n _print_package_info(pack, info, args.verbosity)\n\n\ndef showfiles(args):\n \"\"\"Writes out the input and output files.\n\n Works both for a pack file and for an extracted directory.\n \"\"\"\n def parse_run(runs, s):\n for i, run in enumerate(runs):\n if run['id'] == s:\n return i\n try:\n r = int(s)\n except ValueError:\n logger.critical(\"Error: Unknown run %s\", s)\n raise UsageError\n if r < 0 or r >= len(runs):\n logger.critical(\"Error: Expected 0 <= run <= %d, got %d\",\n len(runs) - 1, r)\n sys.exit(1)\n return r\n\n show_inputs = args.input or not args.output\n show_outputs = args.output or not args.input\n\n def file_filter(fio):\n if file_filter.run is None:\n return ((show_inputs and fio.read_runs) or\n (show_outputs and fio.write_runs))\n else:\n return ((show_inputs and file_filter.run in fio.read_runs) or\n (show_outputs and file_filter.run in fio.write_runs))\n\n file_filter.run = None\n\n pack = Path(args.pack[0])\n\n if not pack.exists():\n logger.critical(\"Pack or directory %s does not exist\", pack)\n sys.exit(1)\n\n if pack.is_dir():\n # Reads info from an unpacked directory\n config = load_config_file(pack / 'config.yml',\n canonical=True)\n\n # Filter files by run\n if args.run is not None:\n file_filter.run = parse_run(config.runs, args.run)\n\n # The '.reprounzip' file is a pickled dictionary, it contains the name\n # of the files that replaced each input file (if upload was used)\n unpacked_info = metadata_read(pack, None)\n assigned_input_files = unpacked_info.get('input_files', {})\n\n if show_inputs:\n shown = False\n for input_name, f in sorted(iteritems(config.inputs_outputs)):\n if f.read_runs and file_filter(f):\n if not shown:\n print(\"Input files:\")\n shown = True\n if args.verbosity >= 2:\n print(\" %s (%s)\" % (input_name, f.path))\n else:\n print(\" %s\" % input_name)\n\n assigned = assigned_input_files.get(input_name)\n if assigned is None:\n assigned = \"(original)\"\n elif assigned is False:\n assigned = \"(not created)\"\n elif assigned is True:\n assigned = \"(generated)\"\n else:\n assert isinstance(assigned, (bytes, unicode_))\n print(\" %s\" % assigned)\n if not shown:\n print(\"Input files: none\")\n\n if show_outputs:\n shown = False\n for output_name, f in sorted(iteritems(config.inputs_outputs)):\n if f.write_runs and file_filter(f):\n if not shown:\n print(\"Output files:\")\n shown = True\n if args.verbosity >= 2:\n print(\" %s (%s)\" % (output_name, f.path))\n else:\n print(\" %s\" % output_name)\n if not shown:\n print(\"Output files: none\")\n\n else: # pack.is_file()\n # Reads info from a pack file\n config = load_config(pack)\n\n # Filter files by run\n if args.run is not None:\n file_filter.run = parse_run(config.runs, args.run)\n\n if any(f.read_runs for f in itervalues(config.inputs_outputs)):\n print(\"Input files:\")\n for input_name, f in sorted(iteritems(config.inputs_outputs)):\n if f.read_runs and file_filter(f):\n if args.verbosity >= 2:\n print(\" %s (%s)\" % (input_name, f.path))\n else:\n print(\" %s\" % input_name)\n else:\n print(\"Input files: none\")\n\n if any(f.write_runs for f in itervalues(config.inputs_outputs)):\n print(\"Output files:\")\n for output_name, f in sorted(iteritems(config.inputs_outputs)):\n if f.write_runs and file_filter(f):\n if args.verbosity >= 2:\n print(\" %s (%s)\" % (output_name, f.path))\n else:\n print(\" %s\" % output_name)\n else:\n print(\"Output files: none\")\n\n\ndef setup_info(parser, **kwargs):\n \"\"\"Prints out some information about a pack\n \"\"\"\n parser.add_argument('pack', nargs=1,\n help=\"Pack to read\")\n parser.add_argument('--json', action='store_true', default=False)\n parser.set_defaults(func=print_info)\n\n\ndef setup_showfiles(parser, **kwargs):\n \"\"\"Prints out input and output file names\n \"\"\"\n parser.add_argument('pack', nargs=1,\n help=\"Pack or directory to read from\")\n parser.add_argument('run', nargs=argparse.OPTIONAL,\n help=\"Run whose input and output files will be listed\")\n parser.add_argument('--input', action='store_true',\n help=\"Only show input files\")\n parser.add_argument('--output', action='store_true',\n help=\"Only show output files\")\n parser.set_defaults(func=showfiles)\n" }, { "path": "reprounzip/reprounzip/parameters.py", "content": "# Copyright (C) 2014 New York University\n# This file is part of ReproZip which is released under the Revised BSD License\n# See file LICENSE for full license details.\n\n\"\"\"Retrieve parameters from online source.\n\nMost unpackers require some parameters that are likely to change on a different\nschedule from ReproZip's releases. To account for that, ReproZip downloads a\n\"parameter file\", which is just a JSON with a bunch of parameters.\n\nIn there you will find things like the address of some binaries that are\ndownloaded from the web (rpzsudo and busybox), and the name of Vagrant boxes\nand Docker images for various operating systems.\n\"\"\"\n\nfrom __future__ import division, print_function, unicode_literals\n\nfrom distutils.version import LooseVersion\nimport json\nimport logging\nimport os\n\nfrom reprounzip.common import get_reprozip_ca_certificate\nfrom reprounzip.utils import download_file\n\n\nlogger = logging.getLogger('reprounzip')\n\n\nparameters = None\n\n\ndef update_parameters():\n \"\"\"Try to download a new version of the parameter file.\n \"\"\"\n global parameters\n if parameters is not None:\n return\n\n url = 'https://stats.reprozip.org/parameters/'\n env_var = os.environ.get('REPROZIP_PARAMETERS')\n if env_var and (\n env_var.startswith('http://') or env_var.startswith('https://')):\n # This is only used for testing\n # Note that this still expects the ReproZip CA\n url = env_var\n elif env_var not in (None, '', '1', 'on', 'enabled', 'yes', 'true'):\n parameters = _bundled_parameters\n return\n\n try:\n from reprounzip.main import __version__ as version\n filename = download_file(\n '%s%s' % (url, version),\n None,\n cachename='parameters.json',\n ssl_verify=str(get_reprozip_ca_certificate()))\n except Exception:\n logger.warning(\"Can't download parameters.json, using bundled \"\n \"parameters\")\n else:\n try:\n with filename.open() as fp:\n parameters = json.load(fp)\n except ValueError:\n logger.info(\"Downloaded parameters.json doesn't load, using \"\n \"bundled parameters\")\n try:\n filename.remove()\n except OSError:\n pass\n else:\n ver = LooseVersion(parameters.get('version', '1.0'))\n if LooseVersion('1.0') <= ver < LooseVersion('1.1'):\n return\n else:\n logger.info(\"parameters.json has incompatible version %s, \"\n \"using bundled parameters\", ver)\n\n parameters = _bundled_parameters\n\n\ndef get_parameter(section):\n \"\"\"Get a parameter from the downloaded or default parameter file.\n \"\"\"\n if parameters is None:\n update_parameters()\n\n return parameters.get(section, None)\n\n\n_bundled_parameters = {\n \"busybox_url\": {\n \"x86_64\": \"https://s3.amazonaws.com/reprozip-files/busybox-x86_64\",\n \"i686\": \"https://s3.amazonaws.com/reprozip-files/busybox-i686\"\n },\n \"rpzsudo_url\": {\n \"x86_64\": \"https://github.com/remram44/static-sudo/releases/download/\"\n \"current/rpzsudo-x86_64\",\n \"i686\": \"https://github.com/remram44/static-sudo/releases/download/\"\n \"current/rpzsudo-i686\"\n },\n \"rpztar_url\": {\n \"x86_64\": \"https://github.com/remram44/rpztar/releases/download/\"\n \"v1/rpztar-x86_64\",\n \"i686\": \"https://github.com/remram44/rpztar/releases/download/\"\n \"v1/rpztar-i686\"\n },\n \"docker_images\": {\n \"default\": \"debian\",\n \"images\": {\n \"ubuntu\": {\n \"versions\": [\n {\n \"version\": \"^12\\\\.04$\",\n \"distribution\": \"ubuntu\",\n \"image\": \"ubuntu:12.04\",\n \"name\": \"Ubuntu 12.04 'Precise'\"\n },\n {\n \"version\": \"^14\\\\.04$\",\n \"distribution\": \"ubuntu\",\n \"image\": \"ubuntu:14.04\",\n \"name\": \"Ubuntu 14.04 'Trusty'\"\n },\n {\n \"version\": \"^14\\\\.10$\",\n \"distribution\": \"ubuntu\",\n \"image\": \"ubuntu:14.10\",\n \"name\": \"Ubuntu 14.10 'Utopic'\"\n },\n {\n \"version\": \"^15\\\\.04$\",\n \"distribution\": \"ubuntu\",\n \"image\": \"ubuntu:15.04\",\n \"name\": \"Ubuntu 15.04 'Vivid'\"\n },\n {\n \"version\": \"^15\\\\.10$\",\n \"distribution\": \"ubuntu\",\n \"image\": \"ubuntu:15.10\",\n \"name\": \"Ubuntu 15.10 'Wily'\"\n },\n {\n \"version\": \"^16\\\\.04$\",\n \"distribution\": \"ubuntu\",\n \"image\": \"ubuntu:16.04\",\n \"name\": \"Ubuntu 16.04 'Xenial'\"\n },\n {\n \"version\": \"^16\\\\.10$\",\n \"distribution\": \"ubuntu\",\n \"image\": \"ubuntu:16.10\",\n \"name\": \"Ubuntu 16.10 'Yakkety'\"\n },\n {\n \"version\": \"^17\\\\.04$\",\n \"distribution\": \"ubuntu\",\n \"image\": \"ubuntu:17.04\",\n \"name\": \"Ubuntu 17.04 'Zesty'\"\n },\n {\n \"version\": \"^17\\\\.10$\",\n \"distribution\": \"ubuntu\",\n \"image\": \"ubuntu:17.10\",\n \"name\": \"Ubuntu 17.10 'Artful'\"\n },\n {\n \"version\": \"^18\\\\.04$\",\n \"distribution\": \"ubuntu\",\n \"image\": \"ubuntu:18.04\",\n \"name\": \"Ubuntu 18.04 'Bionic'\"\n },\n {\n \"version\": \"^18\\\\.10$\",\n \"distribution\": \"ubuntu\",\n \"image\": \"ubuntu:18.10\",\n \"name\": \"Ubuntu 18.10 'Cosmic'\"\n },\n {\n \"version\": \"^19\\\\.04$\",\n \"distribution\": \"ubuntu\",\n \"image\": \"ubuntu:19.04\",\n \"name\": \"Ubuntu 19.04 'Disco'\"\n }\n ],\n \"default\": {\n \"distribution\": \"ubuntu\",\n \"image\": \"ubuntu:19.04\",\n \"name\": \"Ubuntu 19.04 'Disco'\"\n }\n },\n \"debian\": {\n \"versions\": [\n {\n \"version\": \"^(6(\\\\.|$))|(squeeze)\",\n \"distribution\": \"debian\",\n \"image\": \"debian:squeeze\",\n \"name\": \"Debian 6 'Squeeze'\"\n },\n {\n \"version\": \"^(7(\\\\.|$))|(wheezy)\",\n \"distribution\": \"debian\",\n \"image\": \"debian:wheezy\",\n \"name\": \"Debian 7 'Wheezy'\"\n },\n {\n \"version\": \"^(8(\\\\.|$))|(jessie)\",\n \"distribution\": \"debian\",\n \"image\": \"debian:jessie\",\n \"name\": \"Debian 8 'Jessie'\"\n },\n {\n \"version\": \"^(9(\\\\.|$))|(stretch)\",\n \"distribution\": \"debian\",\n \"image\": \"debian:stretch\",\n \"name\": \"Debian 9 'Stretch'\"\n },\n {\n \"version\": \"^(10(\\\\.|$))|(buster)\",\n \"distribution\": \"debian\",\n \"image\": \"debian:buster\",\n \"name\": \"Debian 10 'Buster'\"\n }\n ],\n \"default\": {\n \"distribution\": \"debian\",\n \"image\": \"debian:stretch\",\n \"name\": \"Debian 9 'Stretch'\"\n }\n },\n \"centos\": {\n \"versions\": [\n {\n \"version\": \"^5(\\\\.|$)\",\n \"distribution\": \"centos\",\n \"image\": \"centos:centos5\",\n \"name\": \"CentOS 5\"\n },\n {\n \"version\": \"^6(\\\\.|$)\",\n \"distribution\": \"centos\",\n \"image\": \"centos:centos6\",\n \"name\": \"CentOS 6\"\n },\n {\n \"version\": \"^7(\\\\.|$)\",\n \"distribution\": \"centos\",\n \"image\": \"centos:centos7\",\n \"name\": \"CentOS 7\"\n }\n ],\n \"default\": {\n \"distribution\": \"centos\",\n \"image\": \"centos:centos7\",\n \"name\": \"CentOS 7\"\n }\n },\n \"centos linux\": {\n \"versions\": [\n {\n \"version\": \"^5(\\\\.|$)\",\n \"distribution\": \"centos\",\n \"image\": \"centos:centos5\",\n \"name\": \"CentOS 5\"\n },\n {\n \"version\": \"^6(\\\\.|$)\",\n \"distribution\": \"centos\",\n \"image\": \"centos:centos6\",\n \"name\": \"CentOS 6\"\n },\n {\n \"version\": \"^7(\\\\.|$)\",\n \"distribution\": \"centos\",\n \"image\": \"centos:centos7\",\n \"name\": \"CentOS 7\"\n }\n ],\n \"default\": {\n \"distribution\": \"centos\",\n \"image\": \"centos:centos7\",\n \"name\": \"CentOS 7\"\n }\n },\n \"fedora\": {\n \"versions\": [\n {\n \"version\": \"^20$\",\n \"distribution\": \"fedora\",\n \"image\": \"fedora:20\",\n \"name\": \"Fedora 20\"\n },\n # Fedora 21-24 omitted because they don't include tar\n {\n \"version\": \"^25$\",\n \"distribution\": \"fedora\",\n \"image\": \"fedora:25\",\n \"name\": \"Fedora 25\"\n },\n {\n \"version\": \"^26$\",\n \"distribution\": \"fedora\",\n \"image\": \"fedora:26\",\n \"name\": \"Fedora 26\"\n },\n {\n \"version\": \"^27$\",\n \"distribution\": \"fedora\",\n \"image\": \"fedora:27\",\n \"name\": \"Fedora 27\"\n },\n {\n \"version\": \"^28$\",\n \"distribution\": \"fedora\",\n \"image\": \"fedora:28\",\n \"name\": \"Fedora 28\"\n },\n {\n \"version\": \"^29$\",\n \"distribution\": \"fedora\",\n \"image\": \"fedora:29\",\n \"name\": \"Fedora 29\"\n }\n ],\n \"default\": {\n \"distribution\": \"fedora\",\n \"image\": \"fedora:29\",\n \"name\": \"Fedora 29\"\n }\n }\n }\n },\n \"vagrant_boxes\": {\n \"default\": \"debian\",\n \"boxes\": {\n \"ubuntu\": {\n \"versions\": [\n {\n \"version\": \"^12\\\\.04$\",\n \"distribution\": \"ubuntu\",\n \"architectures\": {\n \"i686\": \"hashicorp/precise32\",\n \"x86_64\": \"hashicorp/precise64\"\n },\n \"name\": \"Ubuntu 12.04 'Precise'\"\n },\n {\n \"version\": \"^14\\\\.04$\",\n \"distribution\": \"ubuntu\",\n \"architectures\": {\n \"i686\": \"ubuntu/trusty32\",\n \"x86_64\": \"ubuntu/trusty64\"\n },\n \"name\": \"Ubuntu 14.04 'Trusty'\"\n },\n {\n \"version\": \"^15\\\\.04$\",\n \"distribution\": \"ubuntu\",\n \"architectures\": {\n \"i686\": \"bento/ubuntu-15.04-i386\",\n \"x86_64\": \"bento/ubuntu-15.04\"\n },\n \"name\": \"Ubuntu 15.04 'Vivid'\"\n },\n {\n \"version\": \"^15\\\\.10$\",\n \"distribution\": \"ubuntu\",\n \"architectures\": {\n \"i686\": \"bento/ubuntu-15.10-i386\",\n \"x86_64\": \"bento/ubuntu-15.10\"\n },\n \"name\": \"Ubuntu 15.10 'Wily'\"\n },\n {\n \"version\": \"^16\\\\.04$\",\n \"distribution\": \"ubuntu\",\n \"architectures\": {\n \"i686\": \"bento/ubuntu-16.04-i386\",\n \"x86_64\": \"bento/ubuntu-16.04\"\n },\n \"name\": \"Ubuntu 16.04 'Xenial'\"\n },\n {\n \"version\": \"^16\\\\.10$\",\n \"distribution\": \"ubuntu\",\n \"architectures\": {\n \"i686\": \"bento/ubuntu-16.10-i386\",\n \"x86_64\": \"bento/ubuntu-16.10\"\n },\n \"name\": \"Ubuntu 16.10 'Yakkety'\"\n },\n {\n \"version\": \"^17\\\\.04$\",\n \"distribution\": \"ubuntu\",\n \"architectures\": {\n \"i686\": \"bento/ubuntu-17.04-i386\",\n \"x86_64\": \"bento/ubuntu-17.04\"\n },\n \"name\": \"Ubuntu 17.04 'Zesty'\"\n },\n {\n \"version\": \"^17\\\\.10$\",\n \"distribution\": \"ubuntu\",\n \"architectures\": {\n \"i686\": \"bento/ubuntu-17.10-i386\",\n \"x86_64\": \"bento/ubuntu-17.10\"\n },\n \"name\": \"Ubuntu 17.10 'Artful'\"\n },\n {\n \"version\": \"^18\\\\.04$\",\n \"distribution\": \"ubuntu\",\n \"architectures\": {\n \"x86_64\": \"bento/ubuntu-18.04\"\n },\n \"name\": \"Ubuntu 18.04 'Bionic'\"\n },\n {\n \"version\": \"^18\\\\.10$\",\n \"distribution\": \"ubuntu\",\n \"architectures\": {\n \"x86_64\": \"bento/ubuntu-18.10\"\n },\n \"name\": \"Ubuntu 18.10 'Cosmic'\"\n },\n {\n \"version\": \"^19\\\\.04$\",\n \"distribution\": \"ubuntu\",\n \"architectures\": {\n \"x86_64\": \"bento/ubuntu-19.04\"\n },\n \"name\": \"Ubuntu 19.04 'Disco'\"\n },\n {\n \"version\": \"^19\\\\.10\",\n \"distribution\": \"ubuntu\",\n \"architectures\": {\n \"x86_64\": \"bento/ubuntu-19.10\"\n },\n \"name\": \"Ubuntu 19.10 'Eoan'\"\n },\n {\n \"version\": \"^20\\\\.04$\",\n \"distribution\": \"ubuntu\",\n \"architectures\": {\n \"x86_64\": \"bento/ubuntu-20.04\"\n },\n \"name\": \"Ubuntu 20.04 'Focal'\"\n },\n {\n \"version\": \"^20\\\\.10\",\n \"distribution\": \"ubuntu\",\n \"architectures\": {\n \"x86_64\": \"bento/ubuntu-20.10\"\n },\n \"name\": \"Ubuntu 20.10 'Groovy'\"\n }\n ],\n \"default\": {\n \"distribution\": \"ubuntu\",\n \"architectures\": {\n \"i686\": \"bento/ubuntu-17.04-i386\",\n \"x86_64\": \"bento/ubuntu-20.04\"\n },\n \"name\": \"Ubuntu\"\n }\n },\n \"debian\": {\n \"versions\": [\n {\n \"version\": \"^(7(\\\\.|$))|(wheezy)\",\n \"distribution\": \"debian\",\n \"architectures\": {\n \"i686\": \"bento/debian-7.11-i386\",\n \"x86_64\": \"bento/debian-7\"\n },\n \"name\": \"Debian 7 'Wheezy'\"\n },\n {\n \"version\": \"^(8(\\\\.|$))|(jessie)\",\n \"distribution\": \"debian\",\n \"architectures\": {\n \"i686\": \"remram/debian-8-i386\",\n \"x86_64\": \"remram/debian-8-amd64\"\n },\n \"name\": \"Debian 8 'Jessie'\"\n },\n {\n \"version\": \"^(9(\\\\.|$))|(stretch)\",\n \"distribution\": \"debian\",\n \"architectures\": {\n \"i686\": \"remram/debian-9-i386\",\n \"x86_64\": \"remram/debian-9-amd64\"\n },\n \"name\": \"Debian 9 'Stretch'\"\n },\n {\n \"version\": \"^(10(\\\\.|$))|(buster)\",\n \"distribution\": \"debian\",\n \"architectures\": {\n \"i686\": \"remram/debian-10-i386\",\n \"x86_64\": \"bento/debian-10\"\n },\n \"name\": \"Debian 10 'Buster'\"\n }\n ],\n \"default\": {\n \"distribution\": \"debian\",\n \"architectures\": {\n \"i686\": \"remram/debian-10-i386\",\n \"x86_64\": \"bento/debian-10\"\n },\n \"name\": \"Debian 10 'Buster'\"\n }\n },\n \"centos\": {\n \"versions\": [\n {\n \"version\": \"^5\\\\.\",\n \"distribution\": \"centos\",\n \"architectures\": {\n \"i686\": \"bento/centos-5.11-i386\",\n \"x86_64\": \"bento/centos-5.11\"\n },\n \"name\": \"CentOS 5.11\"\n },\n {\n \"version\": \"^6\\\\.\",\n \"distribution\": \"centos\",\n \"architectures\": {\n \"i686\": \"bento/centos-6.10-i386\",\n \"x86_64\": \"bento/centos-6.10\"\n },\n \"name\": \"CentOS 6.10\"\n },\n {\n \"version\": \"^7\\\\.\",\n \"distribution\": \"centos\",\n \"architectures\": {\n \"i686\": \"remram/centos-7-i386\",\n \"x86_64\": \"bento/centos-7.6\"\n },\n \"name\": \"CentOS 7.6\"\n },\n {\n \"version\": \"^8\\\\.\",\n \"distribution\": \"centos\",\n \"architectures\": {\n \"x86_64\": \"bento/centos-8\"\n },\n \"name\": \"CentOS 8\"\n }\n ],\n \"default\": {\n \"distribution\": \"centos\",\n \"architectures\": {\n \"i686\": \"remram/centos-7-i386\",\n \"x86_64\": \"bento/centos-8\"\n },\n \"name\": \"CentOS\"\n }\n },\n \"centos linux\": {\n \"versions\": [\n {\n \"version\": \"^5\\\\.\",\n \"distribution\": \"centos\",\n \"architectures\": {\n \"i686\": \"bento/centos-5.11-i386\",\n \"x86_64\": \"bento/centos-5.11\"\n },\n \"name\": \"CentOS 5.11\"\n },\n {\n \"version\": \"^6\\\\.\",\n \"distribution\": \"centos\",\n \"architectures\": {\n \"i686\": \"bento/centos-6.10-i386\",\n \"x86_64\": \"bento/centos-6.10\"\n },\n \"name\": \"CentOS 6.10\"\n },\n {\n \"version\": \"^7\\\\.\",\n \"distribution\": \"centos\",\n \"architectures\": {\n \"x86_64\": \"bento/centos-7.6\"\n },\n \"name\": \"CentOS 7.6\"\n },\n {\n \"version\": \"^8\\\\.\",\n \"distribution\": \"centos\",\n \"architectures\": {\n \"x86_64\": \"bento/centos-8\"\n },\n \"name\": \"CentOS 8\"\n }\n ],\n \"default\": {\n \"distribution\": \"centos\",\n \"architectures\": {\n \"i686\": \"remram/centos-7-i386\",\n \"x86_64\": \"bento/centos-8\"\n },\n \"name\": \"CentOS\"\n }\n },\n \"fedora\": {\n \"versions\": [\n {\n \"version\": \"^22$\",\n \"distribution\": \"fedora\",\n \"architectures\": {\n \"i686\": \"remram/fedora-22-i386\",\n \"x86_64\": \"remram/fedora-22-amd64\"\n },\n \"name\": \"Fedora 22\"\n },\n {\n \"version\": \"^23$\",\n \"distribution\": \"fedora\",\n \"architectures\": {\n \"i686\": \"remram/fedora-23-i386\",\n \"x86_64\": \"remram/fedora-23-amd64\"\n },\n \"name\": \"Fedora 23\"\n },\n {\n \"version\": \"^24$\",\n \"distribution\": \"fedora\",\n \"architectures\": {\n \"i686\": \"remram/fedora-24-i386\",\n \"x86_64\": \"remram/fedora-24-amd64\"\n },\n \"name\": \"Fedora 24\"\n },\n {\n \"version\": \"^25$\",\n \"distribution\": \"fedora\",\n \"architectures\": {\n \"x86_64\": \"bento/fedora-25\"\n },\n \"name\": \"Fedora 25\"\n },\n {\n \"version\": \"^26$\",\n \"distribution\": \"fedora\",\n \"architectures\": {\n \"x86_64\": \"bento/fedora-26\"\n },\n \"name\": \"Fedora 26\"\n },\n {\n \"version\": \"^27$\",\n \"distribution\": \"fedora\",\n \"architectures\": {\n \"x86_64\": \"bento/fedora-27\"\n },\n \"name\": \"Fedora 27\"\n },\n {\n \"version\": \"^28$\",\n \"distribution\": \"fedora\",\n \"architectures\": {\n \"x86_64\": \"bento/fedora-28\"\n },\n \"name\": \"Fedora 28\"\n },\n {\n \"version\": \"^29$\",\n \"distribution\": \"fedora\",\n \"architectures\": {\n \"x86_64\": \"bento/fedora-29\"\n },\n \"name\": \"Fedora 29\"\n },\n {\n \"version\": \"^30\",\n \"distribution\": \"fedora\",\n \"architectures\": {\n \"x86_64\": \"bento/fedora-30\"\n },\n \"name\": \"Fedora 30\"\n },\n {\n \"version\": \"^31\",\n \"distribution\": \"fedora\",\n \"architectures\": {\n \"x86_64\": \"bento/fedora-31\"\n },\n \"name\": \"Fedora 31\"\n },\n {\n \"version\": \"^32\",\n \"distribution\": \"fedora\",\n \"architectures\": {\n \"x86_64\": \"bento/fedora-32\"\n },\n \"name\": \"Fedora 32\"\n },\n {\n \"version\": \"^33\",\n \"distribution\": \"fedora\",\n \"architectures\": {\n \"x86_64\": \"bento/fedora-33\"\n },\n \"name\": \"Fedora 33\"\n }\n ],\n \"default\": {\n \"distribution\": \"fedora\",\n \"architectures\": {\n \"i686\": \"remram/fedora-24-i386\",\n \"x86_64\": \"bento/fedora-33\"\n },\n \"name\": \"Fedora\"\n }\n }\n }\n },\n \"vagrant_boxes_x\": {\n \"default\": \"debian\",\n \"boxes\": {\n \"ubuntu\": {\n \"versions\": [\n {\n \"version\": \"^16\\\\.04$\",\n \"distribution\": \"ubuntu\",\n \"architectures\": {\n \"i686\": \"remram/ubuntu-1604-amd64-x\",\n \"x86_64\": \"remram/ubuntu-1604-amd64-x\"\n },\n \"name\": \"Ubuntu 16.04 'Xenial'\"\n }\n ],\n \"default\": {\n \"distribution\": \"ubuntu\",\n \"architectures\": {\n \"i686\": \"remram/ubuntu-1604-amd64-x\",\n \"x86_64\": \"remram/ubuntu-1604-amd64-x\"\n },\n \"name\": \"Ubuntu 16.04 'Xenial'\"\n }\n },\n \"debian\": {\n \"versions\": [\n {\n \"version\": \"^(8(\\\\.|$))|(jessie)\",\n \"distribution\": \"debian\",\n \"architectures\": {\n \"i686\": \"remram/debian-8-amd64-x\",\n \"x86_64\": \"remram/debian-8-amd64-x\"\n },\n \"name\": \"Debian 8 'Jessie'\"\n }\n ],\n \"default\": {\n \"distribution\": \"debian\",\n \"architectures\": {\n \"i686\": \"remram/debian-8-amd64-x\",\n \"x86_64\": \"remram/debian-8-amd64-x\"\n },\n \"name\": \"Debian 8 'Jessie'\"\n }\n }\n }\n }\n}\n" }, { "path": "reprounzip/reprounzip/plugins/__init__.py", "content": "try: # pragma: no cover\n __import__('pkg_resources').declare_namespace(__name__)\nexcept ImportError: # pragma: no cover\n from pkgutil import extend_path\n __path__ = extend_path(__path__, __name__)\n" }, { "path": "reprounzip/reprounzip/signals.py", "content": "# Copyright (C) 2014 New York University\n# This file is part of ReproZip which is released under the Revised BSD License\n# See file LICENSE for full license details.\n\n\"\"\"Signal system.\n\nEmitting and subscribing to these signals is the framework for the plugin\ninfrastructure.\n\"\"\"\n\nfrom __future__ import division, print_function, unicode_literals\n\nimport traceback\nimport warnings\n\nfrom reprounzip.utils import irange, iteritems\n\n\nclass SignalWarning(UserWarning):\n \"\"\"Warning from the Signal class.\n\n Mainly useful for testing (to turn these to errors), however a 'signal:'\n prefix is actually used in the messages because of Python bug 22543\n http://bugs.python.org/issue22543\n \"\"\"\n\n\nclass Signal(object):\n \"\"\"A signal, with its set of arguments.\n\n This holds the expected parameters that the signal expects, in several\n categories:\n * `expected_args` are the arguments of the signals that must be set. Trying\n to emit the signal without these will show a warning and won't touch the\n listeners. Listeners can rely on these being set.\n * `new_args` are new arguments that listeners cannot yet rely on but that\n emitters should try to pass in. Missing arguments doesn't show a warning\n yet but might in the future.\n * `old_args` are arguments that you might still pass in but that you should\n move away from; they will show a warning stating their deprecation.\n\n Listeners can subscribe to a signal, and may be any callable hashable\n object.\n \"\"\"\n REQUIRED, OPTIONAL, DEPRECATED = irange(3)\n\n def __init__(self, expected_args=[], new_args=[], old_args=[]):\n self._args = {}\n self._args.update((arg, Signal.REQUIRED) for arg in expected_args)\n self._args.update((arg, Signal.OPTIONAL) for arg in new_args)\n self._args.update((arg, Signal.DEPRECATED) for arg in old_args)\n if (len(expected_args) + len(new_args) + len(old_args) !=\n len(self._args)):\n raise ValueError(\"Repeated argument names\")\n self._listeners = set()\n\n def __call__(self, **kwargs):\n info = {}\n for arg, argtype in iteritems(self._args):\n if argtype == Signal.REQUIRED:\n try:\n info[arg] = kwargs.pop(arg)\n except KeyError:\n warnings.warn(\"signal: Missing required argument %s; \"\n \"signal ignored\" % arg,\n category=SignalWarning,\n stacklevel=2)\n return\n else:\n if arg in kwargs:\n info[arg] = kwargs.pop(arg)\n if argtype == Signal.DEPRECATED:\n warnings.warn(\n \"signal: Argument %s is deprecated\" % arg,\n category=SignalWarning,\n stacklevel=2)\n if kwargs:\n arg = next(iter(kwargs))\n warnings.warn(\n \"signal: Unexpected argument %s; signal ignored\" % arg,\n category=SignalWarning,\n stacklevel=2)\n return\n\n for listener in self._listeners:\n try:\n listener(**info)\n except Exception:\n traceback.print_exc()\n warnings.warn(\"signal: Got an exception calling a signal\",\n category=SignalWarning)\n\n def subscribe(self, func):\n \"\"\"Adds the given callable to the listeners.\n\n It must be callable and hashable (it will be put in a set).\n\n It will be called with the signals' arguments as keywords. Because new\n parameters might be introduced, it should accept these by using::\n\n def my_listener(param1, param2, **kwargs_):\n \"\"\"\n if not callable(func):\n raise TypeError(\"%r object is not callable\" % type(func))\n self._listeners.add(func)\n\n def unsubscribe(self, func):\n \"\"\"Removes the given callable from the listeners.\n\n If the listener wasn't subscribed, does nothing.\n \"\"\"\n self._listeners.discard(func)\n\n\npre_setup = Signal(['target', 'pack'])\npost_setup = Signal(['target'], ['pack'])\npre_destroy = Signal(['target'])\npost_destroy = Signal(['target'])\npre_run = Signal(['target'])\npost_run = Signal(['target', 'retcode'])\npre_parse_args = Signal(['parser', 'subparsers'])\npost_parse_args = Signal(['args'])\napplication_finishing = Signal(['reason'])\n\n\nunpacker = None\n" }, { "path": "reprounzip/reprounzip/unpackers/__init__.py", "content": "try: # pragma: no cover\n __import__('pkg_resources').declare_namespace(__name__)\nexcept ImportError: # pragma: no cover\n from pkgutil import extend_path\n __path__ = extend_path(__path__, __name__)\n" }, { "path": "reprounzip/reprounzip/unpackers/common/__init__.py", "content": "# Copyright (C) 2014 New York University\n# This file is part of ReproZip which is released under the Revised BSD License\n# See file LICENSE for full license details.\n\n\"\"\"Utility functions for unpacker plugins.\n\nThis contains functions related to shell scripts, package managers, and the\npack files.\n\"\"\"\n\nfrom __future__ import division, print_function, unicode_literals\n\nfrom reprounzip.utils import join_root\nfrom reprounzip.unpackers.common.misc import UsageError, \\\n COMPAT_OK, COMPAT_NO, COMPAT_MAYBE, \\\n composite_action, target_must_exist, unique_names, \\\n make_unique_name, shell_escape, load_config, busybox_url, sudo_url, \\\n rpztar_url, \\\n FileUploader, FileDownloader, get_runs, add_environment_options, \\\n fixup_environment, interruptible_call, \\\n metadata_read, metadata_write, metadata_initial_iofiles, \\\n metadata_update_run, parse_ports\nfrom reprounzip.unpackers.common.packages import THIS_DISTRIBUTION, \\\n PKG_NOT_INSTALLED, CantFindInstaller, select_installer\n\n\n__all__ = ['THIS_DISTRIBUTION', 'PKG_NOT_INSTALLED', 'select_installer',\n 'COMPAT_OK', 'COMPAT_NO', 'COMPAT_MAYBE',\n 'UsageError', 'CantFindInstaller',\n 'composite_action', 'target_must_exist', 'unique_names',\n 'make_unique_name', 'shell_escape', 'load_config', 'busybox_url',\n 'sudo_url', 'rpztar_url',\n 'join_root', 'FileUploader', 'FileDownloader', 'get_runs',\n 'add_environment_options', 'fixup_environment',\n 'interruptible_call', 'metadata_read', 'metadata_write',\n 'metadata_initial_iofiles', 'metadata_update_run',\n 'parse_ports']\n" }, { "path": "reprounzip/reprounzip/unpackers/common/misc.py", "content": "# Copyright (C) 2014 New York University\n# This file is part of ReproZip which is released under the Revised BSD License\n# See file LICENSE for full license details.\n\n\"\"\"Miscellaneous utilities for unpacker plugins.\n\"\"\"\n\nfrom __future__ import division, print_function, unicode_literals\n\nimport copy\nimport functools\nimport logging\nimport itertools\nimport os\nimport pickle\nimport random\nimport re\nimport warnings\n\nfrom rpaths import PosixPath, Path\nimport signal\nimport subprocess\nimport sys\nimport tarfile\n\nimport reprounzip.common\nfrom reprounzip.common import RPZPack\nfrom reprounzip.parameters import get_parameter\nfrom reprounzip.utils import PY3, irange, iteritems, itervalues, \\\n stdout_bytes, unicode_, join_root, copyfile\n\n\nlogger = logging.getLogger('reprounzip')\n\n\nCOMPAT_OK = 0\nCOMPAT_NO = 1\nCOMPAT_MAYBE = 2\n\n\nclass UsageError(Exception):\n def __init__(self, msg=\"Invalid command-line\"):\n Exception.__init__(self, msg)\n\n\ndef composite_action(*functions):\n \"\"\"Makes an action that just calls several other actions in sequence.\n\n Useful to implement ``myplugin setup`` in terms of ``myplugin setup/part1``\n and ``myplugin setup/part2``: simply use\n ``act1n2 = composite_action(act1, act2)``.\n \"\"\"\n def wrapper(args):\n for function in functions:\n function(args)\n return wrapper\n\n\ndef target_must_exist(func):\n \"\"\"Decorator that checks that ``args.target`` exists.\n \"\"\"\n @functools.wraps(func)\n def wrapper(args):\n target = Path(args.target[0])\n if not target.is_dir():\n logger.critical(\"Error: Target directory doesn't exist\")\n raise UsageError\n return func(args)\n return wrapper\n\n\ndef unique_names():\n \"\"\"Generates unique sequences of bytes.\n \"\"\"\n characters = (b\"abcdefghijklmnopqrstuvwxyz\"\n b\"0123456789\")\n characters = [characters[i:i + 1] for i in irange(len(characters))]\n rng = random.Random()\n while True:\n letters = [rng.choice(characters) for i in irange(10)]\n yield b''.join(letters)\n\n\nunique_names = unique_names()\n\n\ndef make_unique_name(prefix):\n \"\"\"Makes a unique (random) bytestring name, starting with the given prefix.\n \"\"\"\n assert isinstance(prefix, bytes)\n return prefix + next(unique_names)\n\n\nsafe_shell_chars = set(\"ABCDEFGHIJKLMNOPQRSTUVWXYZ\"\n \"abcdefghijklmnopqrstuvwxyz\"\n \"0123456789\"\n \"-+=/:.,%_\")\n\n\ndef shell_escape(s):\n r\"\"\"Given bl\"a, returns \"bl\\\\\"a\".\n \"\"\"\n if isinstance(s, bytes):\n s = s.decode('utf-8')\n if not s or any(c not in safe_shell_chars for c in s):\n return '\"%s\"' % (s.replace('\\\\', '\\\\\\\\')\n .replace('\"', '\\\\\"')\n .replace('`', '\\\\`')\n .replace('$', '\\\\$'))\n else:\n return s\n\n\ndef load_config(pack):\n \"\"\"Utility method loading the YAML configuration from inside a pack file.\n\n Decompresses the config.yml file from the tarball to a temporary file then\n loads it. Note that decompressing a single file is inefficient, thus\n calling this method can be slow.\n \"\"\"\n rpz_pack = RPZPack(pack)\n with rpz_pack.with_config() as configfile:\n return reprounzip.common.load_config(configfile, canonical=True)\n\n\ndef busybox_url(arch):\n \"\"\"Gets the correct URL for the busybox binary given the architecture.\n \"\"\"\n return get_parameter('busybox_url')[arch]\n\n\ndef sudo_url(arch):\n \"\"\"Gets the correct URL for the rpzsudo binary given the architecture.\n \"\"\"\n return get_parameter('rpzsudo_url')[arch]\n\n\ndef rpztar_url(arch):\n \"\"\"Gets the correct URL for the rpztar binary given the architecture.\n \"\"\"\n return get_parameter('rpztar_url')[arch]\n\n\nclass FileUploader(object):\n \"\"\"Common logic for 'upload' commands.\n \"\"\"\n data_tgz = 'data.tgz'\n\n def __init__(self, target, input_files, files):\n self.target = target\n self.input_files = input_files\n self.run(files)\n\n def run(self, files):\n reprounzip.common.record_usage(upload_files=len(files))\n inputs_outputs = self.get_config().inputs_outputs\n\n # No argument: list all the input files and exit\n if not files:\n print(\"Input files:\")\n for input_name in sorted(n for n, f in iteritems(inputs_outputs)\n if f.read_runs):\n assigned = self.input_files.get(input_name)\n if assigned is None:\n assigned = \"(original)\"\n elif assigned is False:\n assigned = \"(not created)\"\n elif assigned is True:\n assigned = \"(generated)\"\n else:\n assert isinstance(assigned, (bytes, unicode_))\n print(\" %s: %s\" % (input_name, assigned))\n return\n\n self.prepare_upload(files)\n\n try:\n # Upload files\n for filespec in files:\n filespec_split = filespec.rsplit(':', 1)\n if len(filespec_split) != 2:\n logger.critical(\"Invalid file specification: %r\",\n filespec)\n sys.exit(1)\n local_path, input_name = filespec_split\n\n try:\n input_path = inputs_outputs[input_name].path\n except KeyError:\n logger.critical(\"Invalid input file: %r\", input_name)\n sys.exit(1)\n\n temp = None\n\n if not local_path:\n # Restore original file from pack\n logger.debug(\"Restoring input file %s\", input_path)\n fd, temp = Path.tempfile(prefix='reprozip_input_')\n os.close(fd)\n local_path = self.extract_original_input(input_name,\n input_path,\n temp)\n if local_path is None:\n temp.remove()\n logger.warning(\"No original packed, can't restore \"\n \"input file %s\", input_name)\n continue\n else:\n local_path = Path(local_path)\n logger.debug(\"Uploading file %s to %s\",\n local_path, input_path)\n if not local_path.exists():\n logger.critical(\"Local file %s doesn't exist\",\n local_path)\n sys.exit(1)\n\n self.upload_file(local_path, input_path)\n\n if temp is not None:\n temp.remove()\n self.input_files.pop(input_name, None)\n else:\n self.input_files[input_name] = local_path.absolute().path\n finally:\n self.finalize()\n\n def get_config(self):\n return reprounzip.common.load_config(self.target / 'config.yml',\n canonical=True)\n\n def prepare_upload(self, files):\n pass\n\n def extract_original_input(self, input_name, input_path, temp):\n tar = tarfile.open(str(self.target / self.data_tgz), 'r:*')\n try:\n member = tar.getmember(str(join_root(PosixPath('DATA'),\n input_path)))\n except KeyError:\n return None\n member = copy.copy(member)\n member.name = str(temp.components[-1])\n tar.extract(member, str(temp.parent))\n tar.close()\n return temp\n\n def upload_file(self, local_path, input_path):\n raise NotImplementedError\n\n def finalize(self):\n pass\n\n\nclass FileDownloader(object):\n \"\"\"Common logic for 'download' commands.\n \"\"\"\n def __init__(self, target, files, all_=False):\n self.target = target\n self.run(files, all_)\n\n def run(self, files, all_):\n reprounzip.common.record_usage(download_files=len(files))\n inputs_outputs = self.get_config().inputs_outputs\n\n # No argument: list all the output files and exit\n if not (all_ or files):\n print(\"Output files:\")\n for output_name in sorted(n for n, f in iteritems(inputs_outputs)\n if f.write_runs):\n print(\" %s\" % output_name)\n return\n\n # Parse the name[:path] syntax\n resolved_files = []\n all_files = set(n for n, f in iteritems(inputs_outputs)\n if f.write_runs)\n for filespec in files:\n filespec_split = filespec.split(':', 1)\n if len(filespec_split) == 1:\n output_name = local_path = filespec\n elif len(filespec_split) == 2:\n output_name, local_path = filespec_split\n else:\n logger.critical(\"Invalid file specification: %r\",\n filespec)\n sys.exit(1)\n local_path = Path(local_path) if local_path else None\n all_files.discard(output_name)\n resolved_files.append((output_name, local_path))\n\n # If all_ is set, add all the files that weren't explicitely named\n if all_:\n for output_name in all_files:\n resolved_files.append((output_name, Path(output_name)))\n\n self.prepare_download(resolved_files)\n\n success = True\n try:\n # Download files\n for output_name, local_path in resolved_files:\n try:\n remote_path = inputs_outputs[output_name].path\n except KeyError:\n logger.critical(\"Invalid output file: %r\", output_name)\n sys.exit(1)\n\n logger.debug(\"Downloading file %s\", remote_path)\n if local_path is None:\n ret = self.download_and_print(remote_path)\n else:\n ret = self.download(remote_path, local_path)\n if ret is None:\n ret = True\n warnings.warn(\"download() returned None instead of \"\n \"True/False, assuming True\",\n category=DeprecationWarning)\n if not ret:\n success = False\n if not success:\n sys.exit(1)\n finally:\n self.finalize()\n\n def get_config(self):\n return reprounzip.common.load_config(self.target / 'config.yml',\n canonical=True)\n\n def prepare_download(self, files):\n pass\n\n def download_and_print(self, remote_path):\n # Download to temporary file\n fd, temp = Path.tempfile(prefix='reprozip_output_')\n os.close(fd)\n download_status = self.download(remote_path, temp)\n if download_status is not None and not download_status:\n return False\n # Output to stdout\n with temp.open('rb') as fp:\n copyfile(fp, stdout_bytes)\n temp.remove()\n return True\n\n def download(self, remote_path, local_path):\n raise NotImplementedError\n\n def finalize(self):\n pass\n\n\ndef get_runs(runs, selected_runs, cmdline):\n \"\"\"Selects which run(s) to execute based on parts of the command-line.\n\n Will return an iterable of run numbers. Might also fail loudly or exit\n after printing the original command-line.\n \"\"\"\n name_map = dict((r['id'], i) for i, r in enumerate(runs) if 'id' in r)\n run_list = []\n\n def parse_run(s):\n try:\n r = int(s)\n except ValueError:\n logger.critical(\"Error: Unknown run %s\", s)\n raise UsageError\n if r < 0 or r >= len(runs):\n logger.critical(\"Error: Expected 0 <= run <= %d, got %d\",\n len(runs) - 1, r)\n sys.exit(1)\n return r\n\n if selected_runs is None:\n run_list = list(irange(len(runs)))\n else:\n for run_item in selected_runs.split(','):\n run_item = run_item.strip()\n if run_item in name_map:\n run_list.append(name_map[run_item])\n continue\n\n sep = run_item.find('-')\n if sep == -1:\n run_list.append(parse_run(run_item))\n else:\n if sep > 0:\n first = parse_run(run_item[:sep])\n else:\n first = 0\n if sep + 1 < len(run_item):\n last = parse_run(run_item[sep + 1:])\n else:\n last = len(runs) - 1\n if last < first:\n logger.critical(\"Error: Last run number should be \"\n \"greater than the first\")\n sys.exit(1)\n run_list.extend(irange(first, last + 1))\n\n # --cmdline without arguments: display the original command-line\n if cmdline == []:\n print(\"Original command-lines:\")\n for run in run_list:\n print(' '.join(shell_escape(arg)\n for arg in runs[run]['argv']))\n sys.exit(0)\n\n return run_list\n\n\ndef add_environment_options(parser):\n parser.add_argument('--pass-env', action='append', default=[],\n help=\"Environment variable to pass through from the \"\n \"host (value from the original machine will be \"\n \"overridden; can be passed multiple times)\")\n parser.add_argument('--set-env', action='append', default=[],\n help=\"Environment variable to set (value from the \"\n \"original machine will be ignored; can be passed \"\n \"multiple times)\")\n\n\ndef fixup_environment(environ, args):\n if not (args.pass_env or args.set_env):\n return environ\n environ = dict(environ)\n\n regexes = [re.compile(pattern + '$') for pattern in args.pass_env]\n for var in os.environ:\n if any(regex.match(var) for regex in regexes):\n environ[var] = os.environ[var]\n\n for var in args.set_env:\n if '=' in var:\n var, value = var.split('=', 1)\n environ[var] = value\n else:\n environ.pop(var, None)\n\n return environ\n\n\nif PY3:\n def pty_spawn(*args, **kwargs):\n import pty\n\n return pty.spawn(*args, **kwargs)\nelse:\n def pty_spawn(argv):\n \"\"\"Version of pty.spawn() for PY2, that returns the exit code.\n\n This works around https://bugs.python.org/issue2489.\n \"\"\"\n logger.info(\"Using builtin pty.spawn()\")\n\n import pty\n import tty\n\n if isinstance(argv, bytes):\n argv = (argv,)\n pid, master_fd = pty.fork()\n if pid == pty.CHILD:\n os.execlp(argv[0], *argv)\n try:\n mode = tty.tcgetattr(pty.STDIN_FILENO)\n tty.setraw(pty.STDIN_FILENO)\n restore = 1\n except tty.error: # This is the same as termios.error\n restore = 0\n try:\n pty._copy(master_fd, pty._read, pty._read)\n except (IOError, OSError):\n if restore:\n tty.tcsetattr(pty.STDIN_FILENO, tty.TCSAFLUSH, mode)\n\n os.close(master_fd)\n return os.waitpid(pid, 0)[1]\n\n\ndef interruptible_call(cmd, **kwargs):\n assert signal.getsignal(signal.SIGINT) == signal.default_int_handler\n proc = [None]\n\n def _sigint_handler(signum, frame):\n if proc[0] is not None:\n try:\n proc[0].send_signal(signum)\n except OSError:\n pass\n\n signal.signal(signal.SIGINT, _sigint_handler)\n\n try:\n if kwargs.pop('request_tty', False):\n try:\n import pty # noqa: F401\n except ImportError:\n pass\n else:\n if hasattr(sys.stdin, 'isatty') and not sys.stdin.isatty():\n logger.info(\"We need a tty and we are not attached to \"\n \"one. Opening pty...\")\n if kwargs.pop('shell', False):\n if not isinstance(cmd, (str, unicode_)):\n raise TypeError(\"shell=True but cmd is not a \"\n \"string\")\n cmd = ['/bin/sh', '-c', cmd]\n res = pty_spawn(cmd)\n return res >> 8 - (res & 0xFF)\n proc[0] = subprocess.Popen(cmd, **kwargs)\n return proc[0].wait()\n finally:\n signal.signal(signal.SIGINT, signal.default_int_handler)\n\n\ndef metadata_read(path, type_):\n \"\"\"Read the unpacker-specific metadata from an unpacked directory.\n\n :param path: The unpacked directory; `.reprounzip` will be appended to get\n the name of the pickle file.\n :param type_: The name of the unpacker, to check for consistency.\n\n Unpackers need to store some specific information, along with the status of\n the input files. This is done in a consistent way so that showfiles can\n access it (and because duplicating code is not necessary here).\n\n It's a simple pickled dictionary under path / '.reprounzip'. The\n 'input_files' key stores the status of the input files.\n\n If you change it, don't forget to call `metadata_write` to write it to disk\n again.\n \"\"\"\n filename = path / '.reprounzip'\n\n if not filename.exists():\n logger.critical(\"Required metadata missing, did you point this \"\n \"command at the directory you created using the \"\n \"'setup' command?\")\n raise UsageError\n with filename.open('rb') as fp:\n dct = pickle.load(fp)\n if type_ is not None and dct['unpacker'] != type_:\n logger.critical(\"Wrong unpacker used: %s != %s\",\n dct['unpacker'], type_)\n raise UsageError\n return dct\n\n\ndef metadata_write(path, dct, type_):\n \"\"\"Write the unpacker-specific metadata in an unpacked directory.\n\n :param path: The unpacked directory; `.reprounzip` will be appended to get\n the name of the pickle file.\n :param type_: The name of the unpacker, that is written to the pickle file\n under the key 'unpacker'.\n :param dct: The dictionary with the info to write to the file.\n \"\"\"\n filename = path / '.reprounzip'\n\n to_write = {'unpacker': type_}\n to_write.update(dct)\n with filename.open('wb') as fp:\n pickle.dump(to_write, fp, 2)\n\n\ndef metadata_initial_iofiles(config, dct=None):\n \"\"\"Add the initial state of the {in/out}put files to the unpacker metadata.\n\n :param config: The configuration as returned by `load_config()`, which will\n be used to list the input and output files and to determine which ones have\n been packed (and therefore exist initially).\n\n The `input_files` key contains a dict mapping the name to either:\n * None (or inexistent): original file and exists\n * False: doesn't exist (wasn't packed)\n * True: has been generated by one of the run since the experiment was\n unpacked\n * basestring: the user uploaded a file with this path, and no run has\n overwritten it yet\n \"\"\"\n if dct is None:\n dct = {}\n\n path2iofile = {f.path: n\n for n, f in iteritems(config.inputs_outputs)}\n\n def packed_files():\n yield config.other_files\n for pkg in config.packages:\n if pkg.packfiles:\n yield pkg.files\n\n for f in itertools.chain.from_iterable(packed_files()):\n f = f.path\n path2iofile.pop(f, None)\n\n dct['input_files'] = dict((n, False) for n in itervalues(path2iofile))\n\n return dct\n\n\ndef metadata_update_run(config, dct, runs):\n \"\"\"Update the unpacker metadata after some runs have executed.\n\n :param runs: An iterable of run numbers that were probably executed.\n\n This maintains a crude idea of the status of input and output files by\n updating the files that are outputs of the runs that were just executed.\n This means that files that were uploaded by the user will no longer be\n shown as uploaded (they have been overwritten by the experiment) and files\n that weren't packed exist from now on.\n\n This is not very reliable because a run might have created a file that is\n not designated as its output anyway, or might have failed and thus not\n created the output (or a bad output).\n \"\"\"\n runs = set(runs)\n input_files = dct.setdefault('input_files', {})\n\n for name, fi in iteritems(config.inputs_outputs):\n if any(r in runs for r in fi.write_runs):\n input_files[name] = True\n\n\n_port_re = re.compile('^(?:([0-9]+):)?([0-9]+)(?:/([a-z]+))?$')\n\n\ndef parse_ports(specifications):\n ports = []\n\n for port in specifications:\n m = _port_re.match(port)\n if m is None:\n logger.critical(\"Invalid port specification: '%s'\", port)\n sys.exit(1)\n host, experiment, proto = m.groups()\n if not host:\n host = experiment\n if not proto:\n proto = 'tcp'\n ports.append((int(host), int(experiment), proto))\n\n return ports\n" }, { "path": "reprounzip/reprounzip/unpackers/common/packages.py", "content": "# Copyright (C) 2014 New York University\n# This file is part of ReproZip which is released under the Revised BSD License\n# See file LICENSE for full license details.\n\n\"\"\"Utility functions dealing with package managers.\n\"\"\"\n\nfrom __future__ import division, print_function, unicode_literals\n\nimport distro\nimport logging\nimport subprocess\n\nfrom reprounzip.unpackers.common.misc import UsageError\nfrom reprounzip.utils import itervalues\n\n\nlogger = logging.getLogger('reprounzip')\n\n\nTHIS_DISTRIBUTION = distro.id()\n\n\nPKG_NOT_INSTALLED = \"(not installed)\"\n\n\nclass CantFindInstaller(UsageError):\n def __init__(self, msg=\"Can't select a package installer\"):\n UsageError.__init__(self, msg)\n\n\nclass AptInstaller(object):\n \"\"\"Installer for deb-based systems (Debian, Ubuntu).\n \"\"\"\n def __init__(self, binary):\n self.bin = binary\n\n def install(self, packages, assume_yes=False):\n # Installs\n options = []\n if assume_yes:\n options.append('-y')\n required_pkgs = set(pkg.name for pkg in packages)\n r = subprocess.call([self.bin, 'install'] +\n options + list(required_pkgs))\n\n # Checks on packages\n pkgs_status = self.get_packages_info(packages)\n for pkg, status in itervalues(pkgs_status):\n if status is not None:\n required_pkgs.discard(pkg.name)\n if required_pkgs:\n logger.error(\"Error: some packages could not be installed:%s\",\n ''.join(\"\\n %s\" % pkg for pkg in required_pkgs))\n\n return r, pkgs_status\n\n @staticmethod\n def get_packages_info(packages):\n if not packages:\n return {}\n\n p = subprocess.Popen(['dpkg-query',\n '--showformat=${Package;-50}\\t${Version}\\n',\n '-W'] +\n [pkg.name for pkg in packages],\n stdout=subprocess.PIPE)\n # name -> (pkg, installed_version)\n pkgs_dict = dict((pkg.name, (pkg, PKG_NOT_INSTALLED))\n for pkg in packages)\n try:\n for line in p.stdout:\n fields = line.split()\n if len(fields) == 2:\n name = fields[0].decode('ascii')\n status = fields[1].decode('ascii')\n pkg, _ = pkgs_dict[name]\n pkgs_dict[name] = pkg, status\n finally:\n p.wait()\n\n return pkgs_dict\n\n def update_script(self):\n return '%s update' % self.bin\n\n def install_script(self, packages):\n return '%s install -y %s' % (self.bin,\n ' '.join(pkg.name for pkg in packages))\n\n\nclass YumInstaller(object):\n \"\"\"Installer for systems using RPM and Yum (Fedora, CentOS, Red-Hat).\n \"\"\"\n @classmethod\n def install(cls, packages, assume_yes=False):\n options = []\n if assume_yes:\n options.append('-y')\n required_pkgs = set(pkg.name for pkg in packages)\n r = subprocess.call(['yum', 'install'] + options + list(required_pkgs))\n\n # Checks on packages\n pkgs_status = cls.get_packages_info(packages)\n for pkg, status in itervalues(pkgs_status):\n if status is not None:\n required_pkgs.discard(pkg.name)\n if required_pkgs:\n logger.error(\"Error: some packages could not be installed:%s\",\n ''.join(\"\\n %s\" % pkg for pkg in required_pkgs))\n\n return r, pkgs_status\n\n @staticmethod\n def get_packages_info(packages):\n if not packages:\n return {}\n\n p = subprocess.Popen(['rpm', '-q'] +\n [pkg.name for pkg in packages] +\n ['--qf', '+%{NAME} %{VERSION}-%{RELEASE}\\\\n'],\n stdout=subprocess.PIPE)\n # name -> {pkg, installed_version}\n pkgs_dict = dict((pkg.name, (pkg, PKG_NOT_INSTALLED))\n for pkg in packages)\n try:\n for line in p.stdout:\n if line[0] == b'+':\n fields = line[1:].split()\n if len(fields) == 2:\n name = fields[0].decode('ascii')\n status = fields[1].decode('ascii')\n pkg, _ = pkgs_dict[name]\n pkgs_dict[name] = pkg, status\n finally:\n p.wait()\n\n return pkgs_dict\n\n @staticmethod\n def update_script():\n return ''\n\n @staticmethod\n def install_script(packages):\n return 'yum install -y %s' % ' '.join(pkg.name for pkg in packages)\n\n\ndef select_installer(pack, runs, target_distribution=THIS_DISTRIBUTION,\n check_distrib_compat=True):\n \"\"\"Selects the right package installer for a Linux distribution.\n \"\"\"\n orig_distribution = runs[0]['distribution'][0].lower()\n\n # Checks that the distributions match\n if not check_distrib_compat:\n pass\n elif (set([orig_distribution, target_distribution]) ==\n set(['ubuntu', 'debian'])):\n # Packages are more or less the same on Debian and Ubuntu\n logger.warning(\"Installing on %s but pack was generated on %s\",\n target_distribution.capitalize(),\n orig_distribution.capitalize())\n elif target_distribution is None:\n raise CantFindInstaller(\"Target distribution is unknown; try using \"\n \"--distribution\")\n elif orig_distribution != target_distribution:\n raise CantFindInstaller(\n \"Installing on %s but pack was generated on %s\" % (\n target_distribution.capitalize(),\n orig_distribution.capitalize()))\n\n # Selects installation method\n if target_distribution == 'ubuntu':\n installer = AptInstaller('apt-get')\n elif target_distribution == 'debian':\n # aptitude is not installed by default, so use apt-get here too\n installer = AptInstaller('apt-get')\n elif (target_distribution in ('centos', 'centos linux',\n 'fedora', 'scientific linux') or\n target_distribution.startswith('red hat')):\n installer = YumInstaller()\n else:\n raise CantFindInstaller(\"This distribution, \\\"%s\\\", is not supported\" %\n target_distribution.capitalize())\n\n return installer\n" }, { "path": "reprounzip/reprounzip/unpackers/common/x11.py", "content": "# Copyright (C) 2014 New York University\n# This file is part of ReproZip which is released under the Revised BSD License\n# See file LICENSE for full license details.\n\n\"\"\"Utility functions dealing with X servers.\n\"\"\"\n\nfrom __future__ import division, print_function, unicode_literals\n\nimport contextlib\nimport logging\nimport os\nfrom rpaths import Path, PosixPath\nimport select\nimport socket\nimport struct\nimport threading\n\nfrom reprounzip.utils import irange, iteritems\n\n\nlogger = logging.getLogger('reprounzip')\n\n\n# #include \n#\n# typedef struct xauth {\n# unsigned short family;\n# unsigned short address_length;\n# char *address;\n# unsigned short number_length;\n# char *number;\n# unsigned short name_length;\n# char *name;\n# unsigned short data_length;\n# char *data;\n# } Xauth;\n\n\n_read_short = lambda fp: struct.unpack('>H', fp.read(2))[0]\n_write_short = lambda i: struct.pack('>H', i)\n\n\ndef ascii(s):\n if isinstance(s, bytes):\n return s\n else:\n return s.encode('ascii')\n\n\nclass Xauth(object):\n \"\"\"A record in an Xauthority file.\n \"\"\"\n FAMILY_LOCAL = 256\n FAMILY_INTERNET = 0\n FAMILY_DECNET = 1\n FAMILY_CHAOS = 2\n FAMILY_INTERNET6 = 6\n FAMILY_SERVERINTERPRETED = 5\n\n def __init__(self, family, address, number, name, data):\n self.family = family\n self.address = address\n self.number = number\n self.name = name\n self.data = data\n\n @classmethod\n def from_file(cls, fp):\n family = _read_short(fp)\n address_length = _read_short(fp)\n address = fp.read(address_length)\n number_length = _read_short(fp)\n number = int(fp.read(number_length)) if number_length else None\n name_length = _read_short(fp)\n name = fp.read(name_length)\n data_length = _read_short(fp)\n data = fp.read(data_length)\n\n return cls(family, address, number, name, data)\n\n def as_bytes(self):\n number = ('%d' % self.number).encode('ascii')\n return (_write_short(self.family) +\n _write_short(len(self.address)) +\n ascii(self.address) +\n _write_short(len(number)) +\n number +\n _write_short(len(self.name)) +\n ascii(self.name) +\n _write_short(len(self.data)) +\n ascii(self.data))\n\n\nclass BaseX11Handler(object):\n \"\"\"X11 handler.\n\n This selects a way to connect to the local X server and an authentication\n mechanism. If provides `fix_env()` to set the X environment variable for\n the experiment, `init_cmds` to setup X before running the experiment's main\n commands, and `port_forward` which describes the reverse port tunnels from\n the experiment to the local X server.\n \"\"\"\n\n\nclass X11Handler(BaseX11Handler):\n \"\"\"X11 handler that will connect to a server outside on the host.\n\n This connects out of the created environment using the network. It is used\n by Vagrant (through SSH) and Docker (TCP connection), and may have\n significant latency.\n \"\"\"\n DISPLAY_NUMBER = 15\n\n SOCK2X = {socket.AF_INET: Xauth.FAMILY_INTERNET,\n socket.AF_INET6: Xauth.FAMILY_INTERNET6}\n X2SOCK = dict((v, k) for k, v in iteritems(SOCK2X))\n\n def __init__(self, enabled, target, display=None):\n self.enabled = enabled\n if not self.enabled:\n return\n\n self.target = target\n\n self.xauth = PosixPath('/.reprounzip_xauthority')\n self.display = (int(display) if display is not None\n else self.DISPLAY_NUMBER)\n logger.debug(\"X11 support enabled; will create Xauthority file %s \"\n \"for experiment. Display number is %d\", self.xauth,\n self.display)\n\n # List of addresses that match the $DISPLAY variable\n possible, local_display = self._locate_display()\n tcp_portnum = ((6000 + local_display) if local_display is not None\n else None)\n\n if ('XAUTHORITY' in os.environ and\n Path(os.environ['XAUTHORITY']).is_file()):\n xauthority = Path(os.environ['XAUTHORITY'])\n # Note: I'm assuming here that Xauthority has no XDG support\n else:\n xauthority = Path('~').expand_user() / '.Xauthority'\n\n # Read Xauthority file\n xauth_entries = {}\n if xauthority.is_file():\n with xauthority.open('rb') as fp:\n fp.seek(0, os.SEEK_END)\n size = fp.tell()\n fp.seek(0, os.SEEK_SET)\n while fp.tell() < size:\n entry = Xauth.from_file(fp)\n if (entry.name == 'MIT-MAGIC-COOKIE-1' and\n entry.number == local_display):\n if entry.family == Xauth.FAMILY_LOCAL:\n logger.debug(\"Found cookie for local connection\")\n xauth_entries[(entry.family, None)] = entry\n elif (entry.family == Xauth.FAMILY_INTERNET or\n entry.family == Xauth.FAMILY_INTERNET6):\n logger.debug(\"Found cookie for %r\",\n (entry.family, entry.address))\n xauth_entries[(entry.family,\n entry.address)] = entry\n else:\n logger.debug(\"No Xauthority file\")\n\n logger.debug(\"Possible X endpoints: %s\", possible)\n\n # Select socket and authentication cookie\n self.xauth_record = None\n self.connection_info = None\n for family, address in possible:\n # Checks that we have a cookie\n entry = family, (None if family is Xauth.FAMILY_LOCAL else address)\n if entry not in xauth_entries:\n continue\n if family == Xauth.FAMILY_LOCAL and hasattr(socket, 'AF_UNIX'):\n # Checks that the socket exists\n if not Path(address).exists():\n continue\n self.connection_info = (socket.AF_UNIX, socket.SOCK_STREAM,\n address)\n self.xauth_record = xauth_entries[(family, None)]\n logger.debug(\"Will connect to local X display via UNIX \"\n \"socket %s\", address)\n break\n else:\n # Checks that we have a cookie\n family = self.X2SOCK[family]\n self.connection_info = (family, socket.SOCK_STREAM,\n (address, tcp_portnum))\n self.xauth_record = xauth_entries[(family, address)]\n logger.debug(\"Will connect to X display %s:%d via %s/TCP\",\n address, tcp_portnum,\n \"IPv6\" if family == socket.AF_INET6 else \"IPv4\")\n break\n\n # Didn't find an Xauthority record -- assume no authentication is\n # needed, but still set self.connection_info\n if self.connection_info is None:\n logger.debug(\"Didn't find any matching Xauthority entry\")\n for family, address in possible:\n # Only try UNIX sockets, we'll use 127.0.0.1 otherwise\n if family == Xauth.FAMILY_LOCAL:\n if not hasattr(socket, 'AF_UNIX'):\n continue\n self.connection_info = (socket.AF_UNIX, socket.SOCK_STREAM,\n address)\n logger.debug(\"Will connect to X display via UNIX socket \"\n \"%s, no authentication\", address)\n break\n else:\n self.connection_info = (socket.AF_INET, socket.SOCK_STREAM,\n ('127.0.0.1', tcp_portnum))\n logger.debug(\"Will connect to X display 127.0.0.1:%d via \"\n \"IPv4/TCP, no authentication\",\n tcp_portnum)\n\n if self.connection_info is None:\n raise RuntimeError(\"Couldn't determine how to connect to local X \"\n \"server, DISPLAY is %s\" % (\n repr(os.environ['DISPLAY'])\n if 'DISPLAY' in os.environ\n else 'not set'))\n\n @classmethod\n def _locate_display(cls):\n \"\"\"Reads $DISPLAY and figures out possible sockets.\n \"\"\"\n # We default to \":0\", Xming for instance doesn't set $DISPLAY\n display = os.environ.get('DISPLAY', ':0')\n\n # It might be the full path to a UNIX socket\n if display.startswith('/'):\n return [(Xauth.FAMILY_LOCAL, display)], None\n\n local_addr, local_display = display.rsplit(':', 1)\n local_display = int(local_display.split('.', 1)[0])\n\n # Let's order the socket families: IPv4 first, then v6, then others\n def sort_families(gai, order={socket.AF_INET: 0, socket.AF_INET6: 1}):\n return sorted(gai, key=lambda x: order.get(x[0], 999999))\n\n # Network addresses of the local machine\n local_addresses = []\n for family, socktype, proto, canonname, sockaddr in \\\n sort_families(socket.getaddrinfo(socket.gethostname(), 6000)):\n try:\n family = cls.SOCK2X[family]\n except KeyError:\n continue\n local_addresses.append((family, sockaddr[0]))\n\n logger.debug(\"Local addresses: %s\", (local_addresses,))\n\n # Determine possible addresses for $DISPLAY\n if not local_addr:\n possible = [(Xauth.FAMILY_LOCAL,\n '/tmp/.X11-unix/X%d' % local_display)]\n possible += local_addresses\n else:\n local_possible = False\n possible = []\n for family, socktype, proto, canonname, sockaddr in \\\n sort_families(socket.getaddrinfo(local_addr, 6000)):\n try:\n family = cls.SOCK2X[family]\n except KeyError:\n continue\n if (family, sockaddr[0]) in local_addresses:\n local_possible = True\n possible.append((family, sockaddr[0]))\n if local_possible:\n possible = [(Xauth.FAMILY_LOCAL,\n '/tmp/.X11-unix/X%d' % local_display)] + possible\n\n return possible, local_display\n\n @property\n def port_forward(self):\n \"\"\"Builds the port forwarding info, for `run_interactive()`.\n\n Just requests port 6015 on the remote host to be forwarded to the X\n socket identified by `self.connection_info`.\n \"\"\"\n if not self.enabled:\n return []\n\n @contextlib.contextmanager\n def connect(src_addr):\n logger.info(\"Got remote X connection from %s\", (src_addr,))\n logger.debug(\"Connecting to X server: %s\",\n (self.connection_info,))\n sock = socket.socket(*self.connection_info[:2])\n sock.connect(self.connection_info[2])\n yield sock\n sock.close()\n logger.info(\"X connection from %s closed\", (src_addr,))\n\n return [(6000 + self.display, connect)]\n\n def fix_env(self, env):\n \"\"\"Sets ``$XAUTHORITY`` and ``$DISPLAY`` in the environment.\n \"\"\"\n if not self.enabled:\n return env\n new_env = dict(env)\n new_env['XAUTHORITY'] = str(self.xauth)\n if self.target[0] == 'local':\n new_env['DISPLAY'] = '127.0.0.1:%d' % self.display\n elif self.target[0] == 'internet':\n new_env['DISPLAY'] = '%s:%d' % (self.target[1], self.display)\n return new_env\n\n @property\n def init_cmds(self):\n \"\"\"Gets the commands to setup X on the server before the experiment.\n \"\"\"\n if not self.enabled or self.xauth_record is None:\n return []\n\n if self.target[0] == 'local':\n xauth_record = Xauth(Xauth.FAMILY_LOCAL,\n self.target[1],\n self.display,\n self.xauth_record.name,\n self.xauth_record.data)\n elif self.target[0] == 'internet':\n xauth_record = Xauth(Xauth.FAMILY_INTERNET,\n socket.inet_aton(self.target[1]),\n self.display,\n self.xauth_record.name,\n self.xauth_record.data)\n else:\n raise RuntimeError(\"Invalid target display type\")\n buf = xauth_record.as_bytes()\n xauth = ''.join(('\\\\x%02x' % ord(buf[i:i + 1]))\n for i in irange(len(buf)))\n return ['echo -ne \"%s\" > %s' % (xauth, self.xauth)]\n\n\nclass BaseForwarder(object):\n \"\"\"Accepts connections and forwards to the given connector object.\n\n The `connector` is a function which takes the address of remote process\n connecting on this ends, and gives out a socket object that is the second\n endpoint of the tunnel. The socket object must provide ``recv()``,\n ``sendall()`` and ``close()``.\n\n Abstract class, implementations will provide actual ways to accept\n connections.\n \"\"\"\n def __init__(self, connector):\n self.connector = connector\n\n def _forward(self, client, src_addr):\n try:\n with self.connector(src_addr) as local_connection:\n local_fd = local_connection.fileno()\n client_fd = client.fileno()\n while True:\n r, w, x = select.select([local_fd, client_fd], [], [])\n if local_fd in r:\n data = local_connection.recv(4096)\n if not data:\n break\n client.sendall(data)\n elif client_fd in r:\n data = client.recv(4096)\n if not data:\n break\n local_connection.sendall(data)\n finally:\n client.close()\n\n\nclass LocalForwarder(BaseForwarder):\n \"\"\"Listens on a random port and forwards to the given connector object.\n\n The `connector` is a function which takes the address of remote process\n connecting on this ends, and gives out a socket object that is the second\n endpoint of the tunnel. The socket object must provide ``recv()``,\n ``sendall()`` and ``close()``.\n \"\"\"\n def __init__(self, connector, local_port=None):\n BaseForwarder.__init__(self, connector)\n server = socket.socket(socket.AF_INET, socket.SOCK_STREAM)\n server.bind(('', local_port or 0))\n self.local_port = server.getsockname()[1]\n server.listen(5)\n\n t = threading.Thread(target=self._accept, args=(server,))\n t.setDaemon(True)\n t.start()\n\n def _accept(self, server):\n while True:\n client, src_addr = server.accept()\n t = threading.Thread(target=self._forward,\n args=(client, src_addr))\n t.setDaemon(True)\n t.start()\n" }, { "path": "reprounzip/reprounzip/unpackers/default.py", "content": "# Copyright (C) 2014 New York University\n# This file is part of ReproZip which is released under the Revised BSD License\n# See file LICENSE for full license details.\n\n\"\"\"Default unpackers for reprounzip.\n\nThis file contains the default plugins that come with reprounzip:\n- ``directory`` puts all the files in a simple directory. This is simple but\n can be unreliable.\n- ``chroot`` creates a chroot environment. This is more reliable as you get a\n harder isolation from the host system.\n- ``installpkgs`` installs on your distribution the packages that were used by\n the experiment on the original machine. This is useful if some of them were\n not packed and you do not have them installed.\n\"\"\"\n\nfrom __future__ import division, print_function, unicode_literals\n\nimport argparse\nimport copy\nfrom elftools.common.exceptions import ELFError\nfrom elftools.elf.elffile import ELFFile\nfrom elftools.elf.segments import InterpSegment\nimport logging\nimport os\nimport platform\nfrom rpaths import PosixPath, DefaultAbstractPath, Path\nimport socket\nimport subprocess\nimport sys\nimport tarfile\n\nfrom reprounzip.common import RPZPack, load_config as load_config_file, \\\n record_usage\nfrom reprounzip import signals\nfrom reprounzip.unpackers.common import THIS_DISTRIBUTION, PKG_NOT_INSTALLED, \\\n COMPAT_OK, COMPAT_NO, CantFindInstaller, target_must_exist, shell_escape, \\\n load_config, select_installer, busybox_url, join_root, FileUploader, \\\n FileDownloader, get_runs, add_environment_options, fixup_environment, \\\n interruptible_call, metadata_read, metadata_write, \\\n metadata_initial_iofiles, metadata_update_run\nfrom reprounzip.unpackers.common.x11 import X11Handler, LocalForwarder\nfrom reprounzip.utils import unicode_, irange, iteritems, itervalues, \\\n stdout_bytes, stderr, make_dir_writable, rmtree_fixed, copyfile, \\\n download_file\n\n\nlogger = logging.getLogger('reprounzip')\n\n\ndef get_elf_interpreter(file):\n try:\n elf = ELFFile(file)\n for segment in elf.iter_segments():\n if isinstance(segment, InterpSegment):\n return segment.get_interp_name()\n return None\n except ELFError:\n return None\n\n\ndef installpkgs(args):\n \"\"\"Installs the necessary packages on the current machine.\n \"\"\"\n if not THIS_DISTRIBUTION:\n logger.critical(\"Not running on Linux\")\n sys.exit(1)\n\n pack = args.pack[0]\n missing = args.missing\n\n # Loads config\n runs, packages, other_files = load_config(pack)\n\n try:\n installer = select_installer(pack, runs)\n except CantFindInstaller as e:\n logger.error(\"Couldn't select a package installer: %s\", e)\n sys.exit(1)\n\n if args.summary:\n # Print out a list of packages with their status\n if missing:\n print(\"Packages not present in pack:\")\n packages = [pkg for pkg in packages if not pkg.packfiles]\n else:\n print(\"All packages:\")\n pkgs = installer.get_packages_info(packages)\n for pkg in packages:\n print(\" %s (required version: %s, status: %s)\" % (\n pkg.name, pkg.version, pkgs[pkg.name][1]))\n else:\n if missing:\n # With --missing, ignore packages whose files were packed\n packages = [pkg for pkg in packages if not pkg.packfiles]\n\n # Installs packages\n record_usage(installpkgs_installing=len(packages))\n r, pkgs = installer.install(packages, assume_yes=args.assume_yes)\n for pkg in packages:\n req = pkg.version\n real = pkgs[pkg.name][1]\n if real == PKG_NOT_INSTALLED:\n logger.warning(\"package %s was not installed\", pkg.name)\n else:\n logger.warning(\"version %s of %s was installed, instead of \"\n \"%s\", real, pkg.name, req)\n if r != 0:\n logger.critical(\"Installer exited with %d\", r)\n sys.exit(r)\n\n\ndef directory_create(args):\n \"\"\"Unpacks the experiment in a folder.\n\n Only the files that are not part of a package are copied (unless they are\n missing from the system and were packed).\n\n In addition, input files are put in a tar.gz (so they can be put back after\n an upload) and the configuration file is extracted.\n \"\"\"\n if not args.pack:\n logger.critical(\"setup needs the pack filename\")\n sys.exit(1)\n\n pack = Path(args.pack[0])\n target = Path(args.target[0])\n if target.exists():\n logger.critical(\"Target directory exists\")\n sys.exit(1)\n\n if not issubclass(DefaultAbstractPath, PosixPath):\n logger.critical(\"Not unpacking on POSIX system\")\n sys.exit(1)\n\n signals.pre_setup(target=target, pack=pack)\n\n # Unpacks configuration file\n rpz_pack = RPZPack(pack)\n rpz_pack.extract_config(target / 'config.yml')\n\n # Loads config\n config = load_config_file(target / 'config.yml', True)\n packages = config.packages\n\n target.mkdir()\n root = (target / 'root').absolute()\n\n # Checks packages\n missing_files = False\n for pkg in packages:\n if pkg.packfiles:\n continue\n for f in pkg.files:\n if not Path(f.path).exists():\n logger.error(\n \"Missing file %s (from package %s that wasn't packed) \"\n \"on host, experiment will probably miss it.\",\n f, pkg.name)\n missing_files = True\n if missing_files:\n record_usage(directory_missing_pkgs=True)\n logger.error(\"Some packages are missing, you should probably install \"\n \"them.\\nUse 'reprounzip installpkgs -h' for help\")\n\n root.mkdir()\n try:\n # Unpacks files\n members = rpz_pack.list_data()\n for m in members:\n # Remove 'DATA/' prefix\n m.name = str(rpz_pack.remove_data_prefix(m.name))\n # Makes symlink targets relative\n if m.issym():\n linkname = PosixPath(m.linkname)\n if linkname.is_absolute:\n m.linkname = join_root(root, PosixPath(m.linkname)).path\n logger.info(\"Extracting files...\")\n rpz_pack.extract_data(root, members)\n rpz_pack.close()\n\n # Original input files, so upload can restore them\n input_files = [f.path for f in itervalues(config.inputs_outputs)\n if f.read_runs]\n if input_files:\n logger.info(\"Packing up original input files...\")\n inputtar = tarfile.open(str(target / 'inputs.tar.gz'), 'w:gz')\n for ifile in input_files:\n filename = join_root(root, ifile)\n if filename.exists():\n inputtar.add(str(filename), str(ifile))\n inputtar.close()\n\n # Meta-data for reprounzip\n metadata_write(target, metadata_initial_iofiles(config), 'directory')\n\n signals.post_setup(target=target, pack=pack)\n except Exception:\n rmtree_fixed(root)\n raise\n\n\n@target_must_exist\ndef directory_run(args):\n \"\"\"Runs the command in the directory.\n \"\"\"\n target = Path(args.target[0])\n unpacked_info = metadata_read(target, 'directory')\n cmdline = args.cmdline\n\n # Loads config\n config = load_config_file(target / 'config.yml', True)\n runs = config.runs\n\n selected_runs = get_runs(runs, args.run, cmdline)\n\n root = (target / 'root').absolute()\n\n # Gets library paths\n lib_dirs = []\n logger.debug(\"Running: %s\", \"/sbin/ldconfig -v -N\")\n p = subprocess.Popen(['/sbin/ldconfig', '-v', '-N'],\n stdin=subprocess.PIPE,\n stdout=subprocess.PIPE,\n stderr=subprocess.PIPE)\n stdout, _ = p.communicate()\n try:\n for line in stdout.splitlines():\n if len(line) < 2 or line[0] in (b' ', b'\\t'):\n continue\n if line.endswith(b':'):\n lib_dirs.append(Path(line[:-1]))\n finally:\n if p.returncode != 0:\n raise subprocess.CalledProcessError(p.returncode,\n ['/sbin/ldconfig', '-v', '-N'])\n\n cmds = []\n for run_number in selected_runs:\n run = runs[run_number]\n cmd = 'cd %s && ' % shell_escape(\n unicode_(join_root(root,\n Path(run['workingdir']))))\n cmd += '/usr/bin/env -i '\n cmd += 'LD_LIBRARY_PATH=%s ' % ':'.join(\n shell_escape(unicode_(join_root(root, d)))\n for d in lib_dirs\n )\n environ = run['environ']\n environ = fixup_environment(environ, args)\n if args.x11:\n environ = dict(environ)\n if 'DISPLAY' in os.environ:\n environ['DISPLAY'] = os.environ['DISPLAY']\n if 'XAUTHORITY' in os.environ:\n environ['XAUTHORITY'] = os.environ['XAUTHORITY']\n cmd += ' '.join('%s=%s' % (shell_escape(k), shell_escape(v))\n for k, v in iteritems(environ)\n if k != 'PATH')\n cmd += ' '\n\n # PATH\n # Get the original PATH components\n path = [PosixPath(d)\n for d in run['environ'].get('PATH', '').split(':')]\n # The same paths but in the directory\n dir_path = [join_root(root, d)\n for d in path\n if d.root == '/']\n # Rebuild string\n path = ':'.join(unicode_(d) for d in dir_path + path)\n cmd += 'PATH=%s ' % shell_escape(path)\n\n interpreter = get_elf_interpreter(\n join_root(root, PosixPath(run['binary'])).open('rb'),\n )\n if interpreter is not None:\n interpreter = Path(interpreter)\n if interpreter.exists():\n cmd += '%s ' % shell_escape(str(join_root(root, interpreter)))\n\n # FIXME : Use exec -a or something if binary != argv[0]\n if cmdline is None:\n argv = run['argv']\n\n # If the command is not a path, use the path instead\n if '/' not in argv[0]:\n argv = [run['binary']] + argv[1:]\n\n # Rewrites command-line arguments that are absolute filenames\n rewritten = False\n for i in irange(len(argv)):\n try:\n p = Path(argv[i])\n except UnicodeEncodeError:\n continue\n if p.is_absolute:\n rp = join_root(root, p)\n if (rp.exists() or\n (len(rp.components) > 3 and rp.parent.exists())):\n argv[i] = str(rp)\n rewritten = True\n if rewritten:\n logger.warning(\"Rewrote command-line as: %s\",\n ' '.join(shell_escape(a) for a in argv))\n else:\n argv = cmdline\n cmd += ' '.join(shell_escape(a) for a in argv)\n cmds.append(cmd)\n cmds = ' && '.join(cmds)\n\n signals.pre_run(target=target)\n logger.debug(\"Running: %s\", cmds)\n retcode = interruptible_call(cmds, shell=True)\n stderr.write(\"\\n*** Command finished, status: %d\\n\" % retcode)\n signals.post_run(target=target, retcode=retcode)\n\n # Update input file status\n metadata_update_run(config, unpacked_info, selected_runs)\n metadata_write(target, unpacked_info, 'directory')\n\n\n@target_must_exist\ndef directory_destroy(args):\n \"\"\"Destroys the directory.\n \"\"\"\n target = Path(args.target[0])\n metadata_read(target, 'directory')\n\n logger.info(\"Removing directory %s...\", target)\n signals.pre_destroy(target=target)\n rmtree_fixed(target)\n signals.post_destroy(target=target)\n\n\ndef should_restore_owner(param):\n \"\"\"Computes whether to restore original files' owners.\n \"\"\"\n if os.getuid() != 0:\n if param is True:\n # Restoring the owner was explicitely requested\n logger.critical(\"Not running as root, cannot restore files' \"\n \"owner/group as requested\")\n sys.exit(1)\n elif param is None:\n # Nothing was requested\n logger.warning(\"Not running as root, won't restore files' \"\n \"owner/group\")\n ret = False\n else:\n # If False: skip warning\n ret = False\n else:\n if param is None:\n # Nothing was requested\n logger.info(\"Running as root, we will restore files' \"\n \"owner/group\")\n ret = True\n elif param is True:\n ret = True\n else:\n # If False: skip warning\n ret = False\n record_usage(restore_owner=ret)\n return ret\n\n\ndef should_mount_magic_dirs(param):\n \"\"\"Computes whether to mount directories inside the chroot.\n \"\"\"\n if os.getuid() != 0:\n if param is True:\n # Restoring the owner was explicitely requested\n logger.critical(\"Not running as root, cannot mount /dev and \"\n \"/proc\")\n sys.exit(1)\n elif param is None:\n # Nothing was requested\n logger.warning(\"Not running as root, won't mount /dev and /proc\")\n ret = False\n else:\n # If False: skip warning\n ret = False\n else:\n if param is None:\n # Nothing was requested\n logger.info(\"Running as root, will mount /dev and /proc\")\n ret = True\n elif param is True:\n ret = True\n else:\n # If False: skip warning\n ret = False\n record_usage(mount_magic_dirs=ret)\n return ret\n\n\ndef chroot_create(args):\n \"\"\"Unpacks the experiment in a folder so it can be run with chroot.\n\n All the files in the pack are unpacked; system files are copied only if\n they were not packed, and busybox is installed if /bin/sh wasn't packed.\n\n In addition, input files are put in a tar.gz (so they can be put back after\n an upload) and the configuration file is extracted.\n \"\"\"\n if not args.pack:\n logger.critical(\"setup/create needs the pack filename\")\n sys.exit(1)\n\n pack = Path(args.pack[0])\n target = Path(args.target[0])\n if target.exists():\n logger.critical(\"Target directory exists\")\n sys.exit(1)\n\n if not issubclass(DefaultAbstractPath, PosixPath):\n logger.critical(\"Not unpacking on POSIX system\")\n sys.exit(1)\n\n signals.pre_setup(target=target, pack=pack)\n\n # We can only restore owner/group of files if running as root\n restore_owner = should_restore_owner(args.restore_owner)\n\n # Unpacks configuration file\n rpz_pack = RPZPack(pack)\n rpz_pack.extract_config(target / 'config.yml')\n\n # Loads config\n config = load_config_file(target / 'config.yml', True)\n packages = config.packages\n\n target.mkdir()\n root = (target / 'root').absolute()\n\n root.mkdir()\n try:\n # Checks that everything was packed\n packages_not_packed = [pkg for pkg in packages if not pkg.packfiles]\n if packages_not_packed:\n record_usage(chroot_missing_pkgs=True)\n logger.warning(\"According to configuration, some files were left \"\n \"out because they belong to the following \"\n \"packages:%s\\nWill copy files from HOST SYSTEM\",\n ''.join('\\n %s' % pkg\n for pkg in packages_not_packed))\n missing_files = False\n for pkg in packages_not_packed:\n for f in pkg.files:\n path = Path(f.path)\n if not path.exists():\n logger.error(\n \"Missing file %s (from package %s) on host, \"\n \"experiment will probably miss it\",\n path, pkg.name)\n missing_files = True\n continue\n dest = join_root(root, path)\n dest.parent.mkdir(parents=True)\n if path.is_link():\n dest.symlink(path.read_link())\n else:\n path.copy(dest)\n if restore_owner:\n stat = path.stat()\n dest.chown(stat.st_uid, stat.st_gid)\n if missing_files:\n record_usage(chroot_mising_files=True)\n\n # Unpacks files\n members = rpz_pack.list_data()\n for m in members:\n # Remove 'DATA/' prefix\n m.name = str(rpz_pack.remove_data_prefix(m.name))\n if not restore_owner:\n uid = os.getuid()\n gid = os.getgid()\n for m in members:\n m.uid = uid\n m.gid = gid\n logger.info(\"Extracting files...\")\n rpz_pack.extract_data(root, members)\n rpz_pack.close()\n\n resolvconf_src = Path('/etc/resolv.conf')\n if resolvconf_src.exists():\n try:\n resolvconf_src.copy(root / 'etc/resolv.conf')\n except IOError:\n pass\n\n # Sets up /bin/sh and /usr/bin/env, downloading busybox if necessary\n sh_path = join_root(root, Path('/bin/sh'))\n env_path = join_root(root, Path('/usr/bin/env'))\n if not sh_path.lexists() or not env_path.lexists():\n logger.info(\"Setting up busybox...\")\n busybox_path = join_root(root, Path('/bin/busybox'))\n busybox_path.parent.mkdir(parents=True)\n with make_dir_writable(join_root(root, Path('/bin'))):\n download_file(busybox_url(config.runs[0]['architecture']),\n busybox_path,\n 'busybox-%s' % config.runs[0]['architecture'])\n busybox_path.chmod(0o755)\n if not sh_path.lexists():\n sh_path.parent.mkdir(parents=True)\n sh_path.symlink('/bin/busybox')\n if not env_path.lexists():\n env_path.parent.mkdir(parents=True)\n env_path.symlink('/bin/busybox')\n\n # Original input files, so upload can restore them\n input_files = [f.path for f in itervalues(config.inputs_outputs)\n if f.read_runs]\n if input_files:\n logger.info(\"Packing up original input files...\")\n inputtar = tarfile.open(str(target / 'inputs.tar.gz'), 'w:gz')\n for ifile in input_files:\n filename = join_root(root, ifile)\n if filename.exists():\n inputtar.add(str(filename), str(ifile))\n inputtar.close()\n\n # Meta-data for reprounzip\n metadata_write(target, metadata_initial_iofiles(config), 'chroot')\n\n signals.post_setup(target=target, pack=pack)\n except Exception:\n rmtree_fixed(root)\n raise\n\n\n@target_must_exist\ndef chroot_mount(args):\n \"\"\"Mounts /dev and /proc inside the chroot directory.\n \"\"\"\n target = Path(args.target[0])\n unpacked_info = metadata_read(target, 'chroot')\n\n # Create proc mount\n d = target / 'root/proc'\n d.mkdir(parents=True)\n subprocess.check_call(['mount', '-t', 'proc', 'none', str(d)])\n\n # Bind /dev from host\n for m in ('/dev', '/dev/pts'):\n d = join_root(target / 'root', Path(m))\n d.mkdir(parents=True)\n logger.info(\"Mounting %s on %s...\", m, d)\n subprocess.check_call(['mount', '-o', 'bind', m, str(d)])\n\n unpacked_info['mounted'] = True\n metadata_write(target, unpacked_info, 'chroot')\n\n logger.warning(\"The host's /dev and /proc have been mounted into the \"\n \"chroot. Do NOT remove the unpacked directory with \"\n \"rm -rf, it WILL WIPE the host's /dev directory.\")\n\n\n@target_must_exist\ndef chroot_run(args):\n \"\"\"Runs the command in the chroot.\n \"\"\"\n target = Path(args.target[0])\n unpacked_info = metadata_read(target, 'chroot')\n cmdline = args.cmdline\n\n # Loads config\n config = load_config_file(target / 'config.yml', True)\n runs = config.runs\n\n selected_runs = get_runs(runs, args.run, cmdline)\n\n root = target / 'root'\n\n # X11 handler\n x11 = X11Handler(args.x11, ('local', socket.gethostname()),\n args.x11_display)\n\n cmds = []\n for run_number in selected_runs:\n run = runs[run_number]\n cmd = 'cd %s && ' % shell_escape(run['workingdir'])\n cmd += '/usr/bin/env -i '\n environ = x11.fix_env(run['environ'])\n environ = fixup_environment(environ, args)\n cmd += ' '.join('%s=%s' % (shell_escape(k), shell_escape(v))\n for k, v in iteritems(environ))\n cmd += ' '\n # FIXME : Use exec -a or something if binary != argv[0]\n if cmdline is None:\n argv = [run['binary']] + run['argv'][1:]\n else:\n argv = cmdline\n cmd += ' '.join(shell_escape(a) for a in argv)\n userspec = '%s:%s' % (run.get('uid', 1000),\n run.get('gid', 1000))\n cmd = 'chroot --userspec=%s %s /bin/sh -c %s' % (\n userspec,\n shell_escape(unicode_(root)),\n shell_escape(cmd))\n cmds.append(cmd)\n cmds = ['chroot %s /bin/sh -c %s' % (shell_escape(unicode_(root)),\n shell_escape(c))\n for c in x11.init_cmds] + cmds\n cmds = ' && '.join(cmds)\n\n # Starts forwarding\n forwarders = []\n for portnum, connector in x11.port_forward:\n fwd = LocalForwarder(connector, portnum)\n forwarders.append(fwd)\n\n signals.pre_run(target=target)\n logger.debug(\"Running: %s\", cmds)\n retcode = interruptible_call(cmds, shell=True)\n stderr.write(\"\\n*** Command finished, status: %d\\n\" % retcode)\n signals.post_run(target=target, retcode=retcode)\n\n # Update input file status\n metadata_update_run(config, unpacked_info, selected_runs)\n metadata_write(target, unpacked_info, 'chroot')\n\n\ndef chroot_unmount(target):\n \"\"\"Unmount magic directories, if they are mounted.\n \"\"\"\n unpacked_info = metadata_read(target, 'chroot')\n mounted = unpacked_info.get('mounted', False)\n\n if not mounted:\n return False\n\n target = target.resolve()\n for m in ('/dev', '/proc'):\n d = join_root(target / 'root', Path(m))\n if d.exists():\n logger.info(\"Unmounting %s...\", d)\n # Unmounts recursively\n subprocess.check_call(\n 'grep %s /proc/mounts | '\n 'cut -f2 -d\" \" | '\n 'sort -r | '\n 'xargs umount' % d,\n shell=True)\n\n unpacked_info['mounted'] = False\n metadata_write(target, unpacked_info, 'chroot')\n\n return True\n\n\n@target_must_exist\ndef chroot_destroy_unmount(args):\n \"\"\"Unmounts the bound magic dirs.\n \"\"\"\n target = Path(args.target[0])\n\n if not chroot_unmount(target):\n logger.critical(\"Magic directories were not mounted\")\n sys.exit(1)\n\n\n@target_must_exist\ndef chroot_destroy_dir(args):\n \"\"\"Destroys the directory.\n \"\"\"\n target = Path(args.target[0])\n mounted = metadata_read(target, 'chroot').get('mounted', False)\n\n if mounted:\n logger.critical(\"Magic directories might still be mounted\")\n sys.exit(1)\n\n logger.info(\"Removing directory %s...\", target)\n signals.pre_destroy(target=target)\n rmtree_fixed(target)\n signals.post_destroy(target=target)\n\n\n@target_must_exist\ndef chroot_destroy(args):\n \"\"\"Destroys the directory, unmounting first if necessary.\n \"\"\"\n target = Path(args.target[0])\n\n chroot_unmount(target)\n\n logger.info(\"Removing directory %s...\", target)\n signals.pre_destroy(target=target)\n rmtree_fixed(target)\n signals.post_destroy(target=target)\n\n\nclass LocalUploader(FileUploader):\n def __init__(self, target, input_files, files, type_, param_restore_owner):\n self.type = type_\n self.param_restore_owner = param_restore_owner\n FileUploader.__init__(self, target, input_files, files)\n\n def prepare_upload(self, files):\n self.restore_owner = (self.type == 'chroot' and\n should_restore_owner(self.param_restore_owner))\n self.root = (self.target / 'root').absolute()\n\n def extract_original_input(self, input_name, input_path, temp):\n tar = tarfile.open(str(self.target / 'inputs.tar.gz'), 'r:*')\n member = tar.getmember(str(join_root(PosixPath(''), input_path)))\n member = copy.copy(member)\n member.name = str(temp.components[-1])\n tar.extract(member, str(temp.parent))\n tar.close()\n return temp\n\n def upload_file(self, local_path, input_path):\n remote_path = join_root(self.root, input_path)\n\n # Copy\n orig_stat = remote_path.stat()\n with make_dir_writable(remote_path.parent):\n local_path.copyfile(remote_path)\n remote_path.chmod(orig_stat.st_mode & 0o7777)\n if self.restore_owner:\n remote_path.chown(orig_stat.st_uid, orig_stat.st_gid)\n\n\n@target_must_exist\ndef upload(args):\n \"\"\"Replaces an input file in the directory.\n \"\"\"\n target = Path(args.target[0])\n files = args.file\n unpacked_info = metadata_read(target, args.type)\n input_files = unpacked_info.setdefault('input_files', {})\n\n try:\n LocalUploader(target, input_files, files,\n args.type, args.type == 'chroot' and args.restore_owner)\n finally:\n metadata_write(target, unpacked_info, args.type)\n\n\nclass LocalDownloader(FileDownloader):\n def __init__(self, target, files, type_, all_=False):\n self.type = type_\n FileDownloader.__init__(self, target, files, all_=all_)\n\n def prepare_download(self, files):\n self.root = (self.target / 'root').absolute()\n\n def download_and_print(self, remote_path):\n remote_path = join_root(self.root, remote_path)\n\n # Output to stdout\n if not remote_path.exists():\n logger.critical(\"Can't get output file (doesn't exist): %s\",\n remote_path)\n return False\n with remote_path.open('rb') as fp:\n copyfile(fp, stdout_bytes)\n return True\n\n def download(self, remote_path, local_path):\n remote_path = join_root(self.root, remote_path)\n\n # Copy\n if not remote_path.exists():\n logger.critical(\"Can't get output file (doesn't exist): %s\",\n remote_path)\n return False\n remote_path.copyfile(local_path)\n remote_path.copymode(local_path)\n return True\n\n\n@target_must_exist\ndef download(args):\n \"\"\"Gets an output file from the directory.\n \"\"\"\n target = Path(args.target[0])\n files = args.file\n metadata_read(target, args.type)\n\n LocalDownloader(target, files, args.type, all_=args.all)\n\n\ndef test_same_pkgmngr(pack, config, **kwargs):\n \"\"\"Compatibility test: platform is Linux and uses same package manager.\n \"\"\"\n runs, packages, other_files = config\n\n orig_distribution = runs[0]['distribution'][0].lower()\n if not THIS_DISTRIBUTION:\n return COMPAT_NO, \"This machine is not running Linux\"\n elif THIS_DISTRIBUTION == orig_distribution:\n return COMPAT_OK\n else:\n return COMPAT_NO, \"Different distributions. Then: %s, now: %s\" % (\n orig_distribution, THIS_DISTRIBUTION)\n\n\ndef test_linux_same_arch(pack, config, **kwargs):\n \"\"\"Compatibility test: this platform is Linux and arch is compatible.\n \"\"\"\n runs, packages, other_files = config\n\n orig_architecture = runs[0]['architecture']\n current_architecture = platform.machine().lower()\n if not sys.platform.startswith('linux'):\n return COMPAT_NO, \"This machine is not running Linux\"\n elif (orig_architecture == current_architecture or\n (orig_architecture == 'i386' and current_architecture == 'amd64')):\n return COMPAT_OK\n else:\n return COMPAT_NO, \"Different architectures. Then: %s, now: %s\" % (\n orig_architecture, current_architecture)\n\n\ndef setup_installpkgs(parser):\n \"\"\"Installs the required packages on this system\n \"\"\"\n parser.add_argument('pack', nargs=1, help=\"Pack to process\")\n parser.add_argument(\n '-y', '--assume-yes', action='store_true', default=False,\n help=\"Assumes yes for package manager's questions (if supported)\")\n parser.add_argument(\n '--missing', action='store_true',\n help=\"Only install packages that weren't packed\")\n parser.add_argument(\n '--summary', action='store_true',\n help=\"Don't install, print which packages are installed or not\")\n parser.set_defaults(func=installpkgs)\n\n return {'test_compatibility': test_same_pkgmngr}\n\n\ndef setup_directory(parser, **kwargs):\n \"\"\"Unpacks the files in a directory and runs with PATH and LD_LIBRARY_PATH\n\n setup creates the directory (needs the pack filename)\n upload replaces input files in the directory\n (without arguments, lists input files)\n run runs the experiment\n download gets output files\n (without arguments, lists output files)\n destroy removes the unpacked directory\n\n Upload specifications are either:\n :input_id restores the original input file from the pack\n filename:input_id replaces the input file with the specified local\n file\n\n Download specifications are either:\n output_id: print the output file to stdout\n output_id:filename extracts the output file to the corresponding local\n path\n \"\"\"\n subparsers = parser.add_subparsers(title=\"actions\",\n metavar='', help=argparse.SUPPRESS)\n\n def add_opt_general(opts):\n opts.add_argument('target', nargs=1, help=\"Experiment directory\")\n\n # setup\n parser_setup = subparsers.add_parser('setup')\n parser_setup.add_argument('pack', nargs=1, help=\"Pack to extract\")\n # Note: add_opt_general is called later so that 'pack' is before 'target'\n add_opt_general(parser_setup)\n parser_setup.set_defaults(func=directory_create)\n\n # upload\n parser_upload = subparsers.add_parser('upload')\n add_opt_general(parser_upload)\n parser_upload.add_argument('file', nargs=argparse.ZERO_OR_MORE,\n help=\":\")\n parser_upload.set_defaults(func=upload, type='directory')\n\n # run\n parser_run = subparsers.add_parser('run')\n add_opt_general(parser_run)\n parser_run.add_argument('run', default=None, nargs=argparse.OPTIONAL)\n parser_run.add_argument('--cmdline', nargs=argparse.REMAINDER,\n help=\"Command line to run\")\n parser_run.add_argument('--enable-x11', action='store_true', default=False,\n dest='x11',\n help=\"Enable X11 support (needs an X server)\")\n add_environment_options(parser_run)\n parser_run.set_defaults(func=directory_run)\n\n # download\n parser_download = subparsers.add_parser('download')\n add_opt_general(parser_download)\n parser_download.add_argument('file', nargs=argparse.ZERO_OR_MORE,\n help=\"[:]\")\n parser_download.add_argument('--all', action='store_true',\n help=\"Download all output files to the \"\n \"current directory\")\n parser_download.set_defaults(func=download, type='directory')\n\n # destroy\n parser_destroy = subparsers.add_parser('destroy')\n add_opt_general(parser_destroy)\n parser_destroy.set_defaults(func=directory_destroy)\n\n return {'test_compatibility': test_linux_same_arch}\n\n\ndef chroot_setup(args):\n \"\"\"Does both create and mount depending on --bind-magic-dirs.\n \"\"\"\n do_mount = should_mount_magic_dirs(args.bind_magic_dirs)\n chroot_create(args)\n if do_mount:\n chroot_mount(args)\n\n\ndef setup_chroot(parser, **kwargs):\n \"\"\"Unpacks the files and run with chroot\n\n setup/create creates the directory (needs the pack filename)\n setup/mount mounts --bind /dev and /proc inside the chroot\n (do NOT rm -Rf the directory after that!)\n upload replaces input files in the directory\n (without arguments, lists input files)\n run runs the experiment\n download gets output files\n (without arguments, lists output files)\n destroy/unmount unmounts /dev and /proc from the directory\n destroy/dir removes the unpacked directory\n\n Upload specifications are either:\n :input_id restores the original input file from the pack\n filename:input_id replaces the input file with the specified local\n file\n\n Download specifications are either:\n output_id: print the output file to stdout\n output_id:filename extracts the output file to the corresponding local\n path\n \"\"\"\n subparsers = parser.add_subparsers(title=\"actions\",\n metavar='', help=argparse.SUPPRESS)\n\n def add_opt_general(opts):\n opts.add_argument('target', nargs=1, help=\"Experiment directory\")\n\n # setup/create\n def add_opt_setup(opts):\n opts.add_argument('pack', nargs=1, help=\"Pack to extract\")\n\n def add_opt_owner(opts):\n opts.add_argument('--preserve-owner', action='store_true',\n dest='restore_owner', default=None,\n help=\"Restore files' owner/group when extracting\")\n opts.add_argument('--dont-preserve-owner', action='store_false',\n dest='restore_owner', default=None,\n help=\"Don't restore files' owner/group when \"\n \"extracting, use current users\")\n\n parser_setup_create = subparsers.add_parser('setup/create')\n add_opt_setup(parser_setup_create)\n add_opt_general(parser_setup_create)\n add_opt_owner(parser_setup_create)\n parser_setup_create.set_defaults(func=chroot_create)\n\n # setup/mount\n parser_setup_mount = subparsers.add_parser('setup/mount')\n add_opt_general(parser_setup_mount)\n parser_setup_mount.set_defaults(func=chroot_mount)\n\n # setup\n parser_setup = subparsers.add_parser('setup')\n add_opt_setup(parser_setup)\n add_opt_general(parser_setup)\n add_opt_owner(parser_setup)\n parser_setup.add_argument(\n '--bind-magic-dirs', action='store_true',\n dest='bind_magic_dirs', default=None,\n help=\"Mount /dev and /proc inside the chroot\")\n parser_setup.add_argument(\n '--dont-bind-magic-dirs', action='store_false',\n dest='bind_magic_dirs', default=None,\n help=\"Don't mount /dev and /proc inside the chroot\")\n parser_setup.set_defaults(func=chroot_setup)\n\n # upload\n parser_upload = subparsers.add_parser('upload')\n add_opt_general(parser_upload)\n add_opt_owner(parser_upload)\n parser_upload.add_argument('file', nargs=argparse.ZERO_OR_MORE,\n help=\":\")\n parser_upload.set_defaults(func=upload, type='chroot')\n\n # run\n parser_run = subparsers.add_parser('run')\n add_opt_general(parser_run)\n parser_run.add_argument('run', default=None, nargs=argparse.OPTIONAL)\n parser_run.add_argument('--cmdline', nargs=argparse.REMAINDER,\n help=\"Command line to run\")\n parser_run.add_argument('--enable-x11', action='store_true', default=False,\n dest='x11',\n help=\"Enable X11 support (needs an X server on \"\n \"the host)\")\n parser_run.add_argument('--x11-display', dest='x11_display',\n help=\"Display number to use on the experiment \"\n \"side (change the host display with the \"\n \"DISPLAY environment variable)\")\n add_environment_options(parser_run)\n parser_run.set_defaults(func=chroot_run)\n\n # download\n parser_download = subparsers.add_parser('download')\n add_opt_general(parser_download)\n parser_download.add_argument('file', nargs=argparse.ZERO_OR_MORE,\n help=\"[:]\")\n parser_download.add_argument('--all', action='store_true',\n help=\"Download all output files to the \"\n \"current directory\")\n parser_download.set_defaults(func=download, type='chroot')\n\n # destroy/unmount\n parser_destroy_unmount = subparsers.add_parser('destroy/unmount')\n add_opt_general(parser_destroy_unmount)\n parser_destroy_unmount.set_defaults(func=chroot_destroy_unmount)\n\n # destroy/dir\n parser_destroy_dir = subparsers.add_parser('destroy/dir')\n add_opt_general(parser_destroy_dir)\n parser_destroy_dir.set_defaults(func=chroot_destroy_dir)\n\n # destroy\n parser_destroy = subparsers.add_parser('destroy')\n add_opt_general(parser_destroy)\n parser_destroy.set_defaults(func=chroot_destroy)\n\n return {'test_compatibility': test_linux_same_arch}\n" }, { "path": "reprounzip/reprounzip/unpackers/graph.py", "content": "# Copyright (C) 2014 New York University\n# This file is part of ReproZip which is released under the Revised BSD License\n# See file LICENSE for full license details.\n\n\"\"\"Graph plugin for reprounzip.\n\nThis is not actually an unpacker, it just creates a graph from the metadata\ncollected by the reprozip tracer (either from a pack file or the initial .rpz\ndirectory).\n\nIt creates a file in GraphViz DOT format, which can be turned into an image by\nusing the dot utility.\n\nSee http://www.graphviz.org/\n\"\"\"\n\nfrom __future__ import division, print_function, unicode_literals\n\nimport argparse\nfrom distutils.version import LooseVersion\nimport heapq\nimport json\nimport logging\nimport re\nfrom rpaths import PosixPath, Path\nimport sqlite3\nimport sys\n\nfrom reprounzip.common import FILE_READ, FILE_WRITE, FILE_WDIR, RPZPack, \\\n load_config\nfrom reprounzip.orderedset import OrderedSet\nfrom reprounzip.unpackers.common import COMPAT_OK, COMPAT_NO\nfrom reprounzip.utils import PY3, izip, iteritems, itervalues, stderr, \\\n unicode_, escape, normalize_path\n\n\nlogger = logging.getLogger('reprounzip.graph')\n\n\nC_INITIAL = 0 # First process or don't know\nC_FORK = 1 # Might actually be any one of fork, vfork or clone\nC_EXEC = 2 # Replaced image with execve\nC_FORKEXEC = 3 # A fork then an exec, folded as one because all_forks==False\n\n\nFORMAT_DOT = 0\nFORMAT_JSON = 1\n\n\nLVL_PKG_FILE = 0 # Show individual files in packages\nLVL_PKG_PACKAGE = 1 # Aggregate by package\nLVL_PKG_IGNORE = 2 # Ignore packages, treat them like any file\nLVL_PKG_DROP = 3 # Drop every file that comes from a package\n\nLVL_PROC_THREAD = 0 # Show every process and thread\nLVL_PROC_PROCESS = 1 # Only show processes, not threads\nLVL_PROC_RUN = 2 # Don't show individual processes, aggregate by run\n\nLVL_OTHER_ALL = 0 # Show every file, aggregate through directory list\nLVL_OTHER_IO = 1 # Only show input & output files\nLVL_OTHER_NO = 3 # Don't show other files\n\n\nclass Run(object):\n \"\"\"Structure representing a whole run.\n \"\"\"\n def __init__(self, nb):\n self.nb = nb\n self.name = \"run %d\" % nb\n self.processes = []\n\n def dot(self, fp, level_processes):\n assert self.processes\n if level_processes == LVL_PROC_RUN:\n fp.write(' run%d [label=\"%d: %s\"];\\n' % (\n self.nb, self.nb, self.processes[0].binary or \"-\"))\n else:\n fp.write(' subgraph cluster_run%d {\\n label=\"%s\";\\n' % (\n self.nb, escape(self.name)))\n for process in self.processes:\n if level_processes == LVL_PROC_THREAD or not process.thread:\n process.dot(fp, level_processes, indent=2)\n fp.write(' }\\n')\n\n def dot_endpoint(self, level_processes):\n return 'run%d' % self.nb\n\n def json(self, prog_map, level_processes):\n assert self.processes\n if level_processes == LVL_PROC_RUN:\n json_process = self.processes[0].json()\n for process in self.processes:\n prog_map[process] = json_process\n processes = [json_process]\n else:\n processes = []\n process_idx_map = {}\n for process in self.processes:\n if level_processes == LVL_PROC_THREAD or not process.thread:\n process_idx_map[process] = len(processes)\n json_process = process.json(process_idx_map)\n prog_map[process] = json_process\n processes.append(json_process)\n else:\n p_process = process\n while p_process.thread:\n p_process = p_process.parent\n prog_map[process] = prog_map[p_process]\n return {'name': self.name, 'processes': processes}\n\n\nclass Process(object):\n \"\"\"Structure representing a process in the experiment.\n \"\"\"\n _id_gen = 0\n\n def __init__(self, pid, run, parent, timestamp, thread, acted, binary,\n argv, created):\n self.id = Process._id_gen\n Process._id_gen += 1\n self.pid = pid\n self.run = run\n self.parent = parent\n self.timestamp = timestamp\n self.thread = bool(thread)\n # Whether that process has done something yet. If it execve()s and\n # hasn't done anything since it forked, no need for it to appear\n self.acted = acted\n # Executable file\n self.binary = binary\n # Command-line if this was created by an exec\n self.argv = argv\n # How was this process created, one of the C_* constants\n self.created = created\n\n def dot(self, fp, level_processes, indent=1):\n thread_style = ',fillcolor=\"#666666\"' if self.thread else ''\n fp.write(' ' * indent + 'prog%d [label=\"%s (%d)\"%s];\\n' % (\n self.id, escape(unicode_(self.binary) or \"-\"),\n self.pid, thread_style))\n if self.parent is not None:\n reason = ''\n if self.created == C_FORK:\n if self.thread:\n reason = \"thread\"\n else:\n reason = \"fork\"\n elif self.created == C_EXEC:\n reason = \"exec\"\n elif self.created == C_FORKEXEC:\n reason = \"fork+exec\"\n fp.write(' ' * indent + 'prog%d -> prog%d [label=\"%s\"];\\n' % (\n self.parent.id, self.id, reason))\n\n def dot_endpoint(self, level_processes):\n if level_processes == LVL_PROC_RUN:\n return self.run.dot_endpoint(level_processes)\n else:\n prog = self\n if level_processes == LVL_PROC_PROCESS:\n while prog.thread:\n prog = prog.parent\n return 'prog%d' % prog.id\n\n def json(self, process_map):\n name = \"%d\" % self.pid\n long_name = \"%s (%d)\" % (PosixPath(self.binary).components[-1]\n if self.binary else \"-\",\n self.pid)\n description = \"%s\\n%d\" % (self.binary, self.pid)\n if self.parent is not None:\n if self.created == C_FORK:\n reason = \"fork\"\n elif self.created == C_EXEC:\n reason = \"exec\"\n elif self.created == C_FORKEXEC:\n reason = \"fork+exec\"\n else:\n assert False\n parent = [process_map[self.parent], reason]\n else:\n parent = None\n return {'name': name, 'parent': parent, 'reads': [], 'writes': [],\n 'long_name': long_name, 'description': description,\n 'argv': self.argv, 'is_thread': self.thread,\n 'start_time': self.timestamp}\n\n\nclass Package(object):\n \"\"\"Structure representing a system package.\n \"\"\"\n def __init__(self, name, version=None):\n self.id = None\n self.name = name\n self.version = version\n self.files = set()\n\n def dot(self, fp, level_pkgs):\n assert self.id is not None\n if not self.files:\n return\n\n if level_pkgs == LVL_PKG_PACKAGE:\n fp.write(' \"pkg %s\" [shape=box,label=' % escape(self.name))\n if self.version:\n fp.write('\"%s %s\"];\\n' % (\n escape(self.name), escape(self.version)))\n else:\n fp.write('\"%s\"];\\n' % escape(self.name))\n elif level_pkgs == LVL_PKG_FILE:\n fp.write(' subgraph cluster_pkg%d {\\n label=' % self.id)\n if self.version:\n fp.write('\"%s %s\";\\n' % (\n escape(self.name), escape(self.version)))\n else:\n fp.write('\"%s\";\\n' % escape(self.name))\n for f in sorted(unicode_(f) for f in self.files):\n fp.write(' \"%s\";\\n' % escape(f))\n fp.write(' }\\n')\n\n def dot_endpoint(self, f, level_pkgs):\n if level_pkgs == LVL_PKG_PACKAGE:\n return '\"pkg %s\"' % escape(self.name)\n else:\n return '\"%s\"' % escape(unicode_(f))\n\n def json_endpoint(self, f, level_pkgs):\n if level_pkgs == LVL_PKG_PACKAGE:\n return self.name\n else:\n return unicode_(f)\n\n def json(self, level_pkgs):\n if level_pkgs == LVL_PKG_PACKAGE:\n logger.critical(\"JSON output doesn't support --packages package\")\n sys.exit(1)\n elif level_pkgs == LVL_PKG_FILE:\n files = sorted(unicode_(f) for f in self.files)\n else:\n assert False\n return {'name': self.name, 'version': self.version or None,\n 'files': files}\n\n\ndef parse_levels(level_pkgs, level_processes, level_other_files):\n try:\n level_pkgs = {'file': LVL_PKG_FILE,\n 'files': LVL_PKG_FILE,\n 'package': LVL_PKG_PACKAGE,\n 'packages': LVL_PKG_PACKAGE,\n 'ignore': LVL_PKG_IGNORE,\n 'drop': LVL_PKG_DROP}[level_pkgs]\n except KeyError:\n logger.critical(\"Unknown level of detail for packages: '%s'\",\n level_pkgs)\n sys.exit(1)\n try:\n level_processes = {'thread': LVL_PROC_THREAD,\n 'threads': LVL_PROC_THREAD,\n 'process': LVL_PROC_PROCESS,\n 'processes': LVL_PROC_PROCESS,\n 'run': LVL_PROC_RUN,\n 'runs': LVL_PROC_RUN}[level_processes]\n except KeyError:\n logger.critical(\"Unknown level of detail for processes: '%s'\",\n level_processes)\n sys.exit(1)\n if level_other_files.startswith('depth:'):\n file_depth = int(level_other_files[6:])\n level_other_files = 'all'\n else:\n file_depth = None\n try:\n level_other_files = {'all': LVL_OTHER_ALL,\n 'io': LVL_OTHER_IO,\n 'inputoutput': LVL_OTHER_IO,\n 'no': LVL_OTHER_NO,\n 'none': LVL_OTHER_NO,\n 'drop': LVL_OTHER_NO}[level_other_files]\n except KeyError:\n logger.critical(\"Unknown level of detail for other files: '%s'\",\n level_other_files)\n sys.exit(1)\n\n return level_pkgs, level_processes, level_other_files, file_depth\n\n\ndef read_events(database, all_forks, has_thread_flag):\n # In here, a file is any file on the filesystem. A binary is a file, that\n # gets executed. A process is a system-level task, identified by its pid\n # (pids don't get reused in the database).\n # What I call program is the couple (process, binary), so forking creates a\n # new program (with the same binary) and exec'ing creates a new program as\n # well (with the same process)\n # Because of this, fork+exec will create an intermediate program that\n # doesn't do anything (new process but still old binary). If that program\n # doesn't do anything worth showing on the graph, it will be erased, unless\n # all_forks is True (--all-forks).\n\n assert database.is_file()\n if PY3:\n # On PY3, connect() only accepts unicode\n conn = sqlite3.connect(str(database))\n else:\n conn = sqlite3.connect(database.path)\n conn.row_factory = sqlite3.Row\n\n # This is a bit weird. We need to iterate on all types of events at the\n # same time, ordering by timestamp, so we decorate-sort-undecorate\n # Decoration adds timestamp (for sorting) and tags by event type, one of\n # 'process', 'open' or 'exec'\n\n # Reads processes from the database\n process_cursor = conn.cursor()\n if has_thread_flag:\n sql = '''\n SELECT id, parent, timestamp, is_thread\n FROM processes\n ORDER BY id\n '''\n else:\n sql = '''\n SELECT id, parent, timestamp, 0 as is_thread\n FROM processes\n ORDER BY id\n '''\n process_rows = process_cursor.execute(sql)\n processes = {}\n all_programs = []\n\n # ... and opened files...\n file_cursor = conn.cursor()\n file_rows = file_cursor.execute(\n '''\n SELECT name, timestamp, mode, process, is_directory\n FROM opened_files\n ORDER BY id\n ''')\n binaries = set()\n files = set()\n edges = OrderedSet()\n\n # ... as well as executed files.\n exec_cursor = conn.cursor()\n exec_rows = exec_cursor.execute(\n '''\n SELECT name, timestamp, process, argv\n FROM executed_files\n ORDER BY id\n ''')\n\n # Loop on all event lists\n logger.info(\"Getting all events from database...\")\n rows = heapq.merge(((r[2], 'process', r) for r in process_rows),\n ((r[1], 'open', r) for r in file_rows),\n ((r[1], 'exec', r) for r in exec_rows))\n runs = []\n run = None\n for ts, event_type, data in rows:\n if event_type == 'process':\n r_id, r_parent, r_timestamp, r_thread = data\n logger.debug(\"Process %d created (parent %r)\", r_id, r_parent)\n if r_parent is not None:\n parent = processes[r_parent]\n binary = parent.binary\n else:\n run = Run(len(runs))\n runs.append(run)\n parent = None\n binary = None\n if r_parent is not None:\n argv = processes[r_parent].argv\n else:\n argv = None\n process = Process(r_id,\n run,\n parent,\n r_timestamp,\n r_thread,\n False,\n binary,\n argv,\n C_INITIAL if r_parent is None else C_FORK)\n processes[r_id] = process\n all_programs.append(process)\n run.processes.append(process)\n\n elif event_type == 'open':\n r_name, r_timestamp, r_mode, r_process, r_directory = data\n r_name = normalize_path(r_name)\n logger.debug(\"File open: %s, process %d\", r_name, r_process)\n if not (r_mode & FILE_WDIR or r_directory):\n process = processes[r_process]\n files.add(r_name)\n edges.add((process, r_name, r_mode, None))\n\n elif event_type == 'exec':\n r_name, r_timestamp, r_process, r_argv = data\n r_name = normalize_path(r_name)\n argv = tuple(r_argv.split('\\0'))\n if not argv[-1]:\n argv = argv[:-1]\n logger.debug(\"File exec: %s, process %d\", r_name, r_process)\n process = processes[r_process]\n binaries.add(r_name)\n # Here we split this process in two \"programs\", unless the previous\n # one hasn't done anything since it was created via fork()\n if not all_forks and not process.acted:\n process.binary = r_name\n process.created = C_FORKEXEC\n process.acted = True\n process.argv = argv\n else:\n process = Process(process.pid,\n run,\n process,\n r_timestamp,\n False,\n True, # Hides exec only once\n r_name,\n argv,\n C_EXEC)\n all_programs.append(process)\n processes[r_process] = process\n run.processes.append(process)\n files.add(r_name)\n edges.add((process, r_name, None, argv))\n\n process_cursor.close()\n file_cursor.close()\n exec_cursor.close()\n conn.close()\n\n return runs, files, edges\n\n\ndef format_argv(argv):\n joined = ' '.join(argv)\n if len(joined) < 50:\n return joined\n else:\n return \"%s ...\" % argv[0]\n\n\ndef generate(target, configfile, database, all_forks=False, graph_format='dot',\n level_pkgs='file', level_processes='thread',\n level_other_files='all',\n regex_filters=None, regex_includes=None,\n regex_replaces=None, aggregates=None):\n \"\"\"Main function for the graph subcommand.\n \"\"\"\n try:\n graph_format = {'dot': FORMAT_DOT, 'DOT': FORMAT_DOT,\n 'json': FORMAT_JSON, 'JSON': FORMAT_JSON}[graph_format]\n except KeyError:\n logger.critical(\"Unknown output format %r\", graph_format)\n sys.exit(1)\n\n level_pkgs, level_processes, level_other_files, file_depth = \\\n parse_levels(level_pkgs, level_processes, level_other_files)\n\n if target.exists():\n logger.critical(\"Output file %s exists\", target)\n sys.exit(1)\n\n # Reads package ownership from the configuration\n if not configfile.is_file():\n logger.critical(\"Configuration file does not exist!\\n\"\n \"Did you forget to run 'reprozip trace'?\\n\"\n \"If not, you might want to use --dir to specify an \"\n \"alternate location.\")\n sys.exit(1)\n config = load_config(configfile, canonical=False)\n inputs_outputs = config.inputs_outputs\n inputs_outputs_map = dict((f.path, n)\n for n, f in iteritems(config.inputs_outputs))\n has_thread_flag = config.format_version >= LooseVersion('0.7')\n\n runs, files, edges = read_events(database, all_forks,\n has_thread_flag)\n\n # Label the runs\n if len(runs) != len(config.runs):\n logger.warning(\"Configuration file doesn't list the same number of \"\n \"runs we found in the database!\")\n else:\n for config_run, run in izip(config.runs, runs):\n run.name = config_run['id']\n\n # Apply regexes\n ignore = [lambda path, r=re.compile(p): r.search(path) is not None\n for p in regex_filters or []]\n include = [lambda path, r=re.compile(p): r.search(path) is not None\n for p in regex_includes or []]\n replace = [lambda path, r=re.compile(p): r.sub(repl, path)\n for p, repl in regex_replaces or []]\n\n def filefilter(path):\n pathuni = unicode_(path)\n if include and not any(f(pathuni) for f in include):\n logger.debug(\"IGN(include) %s\", pathuni)\n return None\n if any(f(pathuni) for f in ignore):\n logger.debug(\"IGN %s\", pathuni)\n return None\n if not (replace or aggregates):\n return path\n for fi in replace:\n pathuni_ = fi(pathuni)\n if pathuni_ != pathuni:\n logger.debug(\"SUB %s -> %s\", pathuni, pathuni_)\n pathuni = pathuni_\n for prefix in aggregates or []:\n if pathuni.startswith(prefix):\n logger.debug(\"AGG %s -> %s\", pathuni, prefix)\n pathuni = prefix\n break\n return PosixPath(pathuni)\n\n files_new = set()\n for fi in files:\n fi = filefilter(fi)\n if fi is not None:\n files_new.add(fi)\n files = files_new\n\n edges_new = OrderedSet()\n for prog, fi, mode, argv in edges:\n fi = filefilter(fi)\n if fi is not None:\n edges_new.add((prog, fi, mode, argv))\n edges = edges_new\n\n # Puts files in packages\n package_map = {}\n if level_pkgs == LVL_PKG_IGNORE:\n packages = []\n other_files = files\n else:\n logger.info(\"Organizes packages...\")\n file2package = dict((f.path, pkg)\n for pkg in config.packages for f in pkg.files)\n packages = {}\n other_files = []\n for fi in files:\n pkg = file2package.get(fi)\n if pkg is not None:\n package = packages.get(pkg.name)\n if package is None:\n package = Package(pkg.name, pkg.version)\n packages[pkg.name] = package\n package.files.add(fi)\n package_map[fi] = package\n else:\n other_files.append(fi)\n packages = sorted(itervalues(packages), key=lambda pkg: pkg.name)\n for i, pkg in enumerate(packages):\n pkg.id = i\n\n # Filter other files\n if level_other_files == LVL_OTHER_ALL and file_depth is not None:\n other_files = set(PosixPath(*f.components[:file_depth + 1])\n for f in other_files)\n edges = OrderedSet((prog,\n f if f in package_map\n else PosixPath(*f.components[:file_depth + 1]),\n mode,\n argv)\n for prog, f, mode, argv in edges)\n else:\n if level_other_files == LVL_OTHER_IO:\n other_files = set(f\n for f in other_files if f in inputs_outputs_map)\n edges = [(prog, f, mode, argv)\n for prog, f, mode, argv in edges\n if f in package_map or f in other_files]\n elif level_other_files == LVL_OTHER_NO:\n other_files = set()\n edges = [(prog, f, mode, argv)\n for prog, f, mode, argv in edges\n if f in package_map]\n\n args = (target, runs, packages, other_files, package_map, edges,\n inputs_outputs, inputs_outputs_map,\n level_pkgs, level_processes, level_other_files)\n if graph_format == FORMAT_DOT:\n graph_dot(*args)\n elif graph_format == FORMAT_JSON:\n graph_json(*args)\n else:\n assert False\n\n\ndef graph_dot(target, runs, packages, other_files, package_map, edges,\n inputs_outputs, inputs_outputs_map,\n level_pkgs, level_processes, level_other_files):\n \"\"\"Writes a GraphViz DOT file from the collected information.\n \"\"\"\n with target.open('w', encoding='utf-8', newline='\\n') as fp:\n fp.write('digraph G {\\n rankdir=LR;\\n\\n /* programs */\\n'\n ' node [shape=box fontcolor=white '\n 'fillcolor=black style=\"filled,rounded\"];\\n')\n\n # Programs\n logger.info(\"Writing programs...\")\n for run in runs:\n run.dot(fp, level_processes)\n\n fp.write('\\n'\n ' node [shape=ellipse fontcolor=\"#131C39\" '\n 'fillcolor=\"#C9D2ED\"];\\n')\n\n # Packages\n if level_pkgs not in (LVL_PKG_IGNORE, LVL_PKG_DROP):\n logger.info(\"Writing packages...\")\n fp.write('\\n /* system packages */\\n')\n for package in sorted(packages, key=lambda pkg: pkg.name):\n package.dot(fp, level_pkgs)\n\n fp.write('\\n /* other files */\\n')\n\n # Other files\n logger.info(\"Writing other files...\")\n for fi in sorted(other_files):\n if fi in inputs_outputs_map:\n fp.write(' \"%(path)s\" [fillcolor=\"#A3B4E0\", '\n 'label=\"%(name)s\\\\n%(path)s\"];\\n' %\n {'path': escape(unicode_(fi)),\n 'name': inputs_outputs_map[fi]})\n else:\n fp.write(' \"%s\";\\n' % escape(unicode_(fi)))\n\n fp.write('\\n')\n\n # Edges\n logger.info(\"Connecting edges...\")\n done_edges = set()\n for prog, fi, mode, argv in edges:\n endp_prog = prog.dot_endpoint(level_processes)\n if fi in package_map:\n if level_pkgs == LVL_PKG_DROP:\n continue\n endp_file = package_map[fi].dot_endpoint(fi, level_pkgs)\n e = endp_prog, endp_file, mode\n if e in done_edges:\n continue\n else:\n done_edges.add(e)\n else:\n endp_file = '\"%s\"' % escape(unicode_(fi))\n\n if mode is None:\n fp.write(' %s -> %s [style=bold, label=\"%s\"];\\n' % (\n endp_file,\n endp_prog,\n escape(format_argv(argv))))\n elif mode & FILE_WRITE:\n fp.write(' %s -> %s [color=\"#000088\"];\\n' % (\n endp_prog, endp_file))\n elif mode & FILE_READ:\n fp.write(' %s -> %s [color=\"#8888CC\"];\\n' % (\n endp_file, endp_prog))\n\n fp.write('}\\n')\n\n\ndef graph_json(target, runs, packages, other_files, package_map, edges,\n inputs_outputs, inputs_outputs_map,\n level_pkgs, level_processes, level_other_files):\n \"\"\"Writes a JSON file suitable for further processing.\n \"\"\"\n # Packages\n if level_pkgs in (LVL_PKG_IGNORE, LVL_PKG_DROP):\n json_packages = []\n else:\n json_packages = [pkg.json(level_pkgs) for pkg in packages]\n\n # Other files\n json_other_files = [unicode_(fi) for fi in sorted(other_files)]\n\n # Programs\n prog_map = {}\n json_runs = [run.json(prog_map, level_processes) for run in runs]\n\n # Connect edges\n done_edges = set()\n for prog, fi, mode, argv in edges:\n endp_prog = prog_map[prog]\n if fi in package_map:\n if level_pkgs == LVL_PKG_DROP:\n continue\n endp_file = package_map[fi].json_endpoint(fi, level_pkgs)\n e = endp_prog['name'], endp_file, mode\n if e in done_edges:\n continue\n else:\n done_edges.add(e)\n else:\n endp_file = unicode_(fi)\n if mode is None:\n endp_prog['reads'].append(endp_file)\n # TODO: argv?\n elif mode & FILE_WRITE:\n endp_prog['writes'].append(endp_file)\n elif mode & FILE_READ:\n endp_prog['reads'].append(endp_file)\n\n json_other_files.sort()\n\n if PY3:\n fp = target.open('w', encoding='utf-8', newline='\\n')\n else:\n fp = target.open('wb')\n try:\n json.dump({'packages': sorted(json_packages,\n key=lambda p: p['name']),\n 'other_files': json_other_files,\n 'runs': json_runs,\n 'inputs_outputs': [\n {'name': k, 'path': unicode_(v.path),\n 'read_by_runs': v.read_runs,\n 'written_by_runs': v.write_runs}\n for k, v in sorted(iteritems(inputs_outputs))]},\n fp,\n ensure_ascii=False,\n indent=2,\n sort_keys=True)\n finally:\n fp.close()\n\n\ndef graph(args):\n \"\"\"graph subcommand.\n\n Reads in the trace sqlite3 database and writes out a graph in GraphViz DOT\n format or JSON.\n \"\"\"\n def call_generate(args, config, trace):\n generate(Path(args.target[0]), config, trace, args.all_forks,\n args.format, args.packages, args.processes, args.otherfiles,\n args.regex_filter, args.regex_include,\n args.regex_replace, args.aggregate)\n\n if args.pack is not None:\n rpz_pack = RPZPack(args.pack)\n with rpz_pack.with_config() as config:\n with rpz_pack.with_trace() as trace:\n call_generate(args, config, trace)\n else:\n call_generate(args,\n Path(args.dir) / 'config.yml',\n Path(args.dir) / 'trace.sqlite3')\n\n\ndef disabled_bug13676(args):\n stderr.write(\"Error: your version of Python, %s, is not supported\\n\"\n \"Versions before 2.7.3 are affected by bug 13676 and will \"\n \"not be able to read\\nthe trace \"\n \"database\\n\" % sys.version.split(' ', 1)[0])\n sys.exit(1)\n\n\ndef setup(parser, **kwargs):\n \"\"\"Generates a provenance graph from the trace data\n \"\"\"\n\n # http://bugs.python.org/issue13676\n # This prevents repro(un)zip from reading argv and envp arrays from trace\n if sys.version_info < (2, 7, 3):\n parser.add_argument('rest_of_cmdline', nargs=argparse.REMAINDER,\n help=argparse.SUPPRESS)\n parser.set_defaults(func=disabled_bug13676)\n return {'test_compatibility': (COMPAT_NO, \"Python >2.7.3 required\")}\n\n parser.add_argument('target', nargs=1, help=\"Destination DOT file\")\n parser.add_argument('-F', '--all-forks', action='store_true',\n help=\"Show forked processes before they exec\")\n parser.add_argument('--packages', default='file',\n help=\"Level of detail for packages; 'file', \"\n \"'package', 'drop' or 'ignore' (default: 'file')\")\n parser.add_argument('--processes', default='thread',\n help=\"Level of detail for processes; 'thread', \"\n \"'process' or 'run' (default: 'thread')\")\n parser.add_argument('--otherfiles', default='all',\n help=\"Level of detail for non-package files; 'all', \"\n \"'io' or 'no' (default: 'all')\")\n parser.add_argument('--aggregate', action='append',\n help=\"Aggregate all files under this path\")\n parser.add_argument('--regex-include', action='append',\n help=\"Glob patterns of files to include (checked \"\n \"before --regex-filter)\")\n parser.add_argument('--regex-filter', action='append',\n help=\"Glob patterns of files to ignore (checked \"\n \"after --regex-include)\")\n parser.add_argument('--regex-replace', action='append', nargs=2,\n help=\"Apply regular expression replacement to files\")\n parser.add_argument('--dot', action='store_const', dest='format',\n const='dot', default='dot',\n help=\"Set the output format to DOT (this is the \"\n \"default)\")\n parser.add_argument('--json', action='store_const', dest='format',\n const='json', help=\"Set the output format to JSON\")\n parser.add_argument(\n '-d', '--dir', default='.reprozip-trace',\n help=\"where the database and configuration file are stored (default: \"\n \"./.reprozip-trace)\")\n parser.add_argument(\n 'pack', nargs=argparse.OPTIONAL,\n help=\"Pack to extract (defaults to reading from --dir)\")\n parser.set_defaults(func=graph)\n\n return {'test_compatibility': COMPAT_OK}\n" }, { "path": "reprounzip/reprounzip/unpackers/provviewer.py", "content": "# Copyright (C) 2014 New York University\n# This file is part of ReproZip which is released under the Revised BSD License\n# See file LICENSE for full license details.\n\n\"\"\"Prov Viewer exporter.\n\nThis exports the trace data into a format suitable for the Prov Viewer tool\n(https://github.com/gems-uff/prov-viewer).\n\nSee schema: https://git.io/provviewer-xsd\n\"\"\"\n\nfrom __future__ import division, print_function, unicode_literals\n\nimport argparse\nimport logging\nfrom distutils.version import LooseVersion\nfrom rpaths import Path\nimport sqlite3\nimport sys\n\nfrom reprounzip.common import FILE_WRITE, RPZPack, load_config\nfrom reprounzip.unpackers.common import COMPAT_OK, COMPAT_NO, shell_escape\nfrom reprounzip.utils import PY3, iteritems, stderr\n\n\nlogger = logging.getLogger('reprounzip.provviewer')\n\n\ndef xml_escape(s):\n \"\"\"Escapes for XML.\n \"\"\"\n return ((\"%s\" % s).replace('&', '&').replace('\"', '"')\n .replace('<', '≶').replace('>', '>'))\n\n\ndef generate(target, configfile, database):\n \"\"\"Go over the trace and generate the graph file.\n \"\"\"\n # Reads package ownership from the configuration\n if not configfile.is_file():\n logger.critical(\"Configuration file does not exist!\\n\"\n \"Did you forget to run 'reprozip trace'?\\n\"\n \"If not, you might want to use --dir to specify an \"\n \"alternate location.\")\n sys.exit(1)\n\n config = load_config(configfile, canonical=False)\n\n has_thread_flag = config.format_version >= LooseVersion('0.7')\n\n assert database.is_file()\n if PY3:\n # On PY3, connect() only accepts unicode\n conn = sqlite3.connect(str(database))\n else:\n conn = sqlite3.connect(database.path)\n conn.row_factory = sqlite3.Row\n\n vertices = []\n edges = []\n\n # Create user entity, that initiates the runs\n vertices.append({'ID': 'user',\n 'type': 'Agent',\n 'subtype': 'User',\n 'label': 'User'})\n\n run = -1\n\n # Read processes\n cur = conn.cursor()\n rows = cur.execute(\n '''\n SELECT id, parent, timestamp, is_thread, exitcode\n FROM processes;\n ''' if has_thread_flag else '''\n SELECT id, parent, timestamp, 0 as is_thread, exitcode\n FROM processes;\n ''')\n for r_id, r_parent, r_timestamp, r_isthread, r_exitcode in rows:\n if r_parent is None:\n # Create run entity\n run += 1\n vertices.append({'ID': 'run%d' % run,\n 'type': 'Activity',\n 'subtype': 'Run',\n 'label': \"Run #%d\" % run,\n 'date': r_timestamp})\n # User -> run\n edges.append({'ID': 'user_run%d' % run,\n 'type': 'UserRuns',\n 'label': \"User runs command\",\n 'sourceID': 'user',\n 'targetID': 'run%d' % run})\n # Run -> process\n edges.append({'ID': 'run_start%d' % run,\n 'type': 'RunStarts',\n 'label': \"Run #%d command\",\n 'sourceID': 'run%d' % run,\n 'targetID': 'process%d' % r_id})\n\n # Create process entity\n vertices.append({'ID': 'process%d' % r_id,\n 'type': 'Agent',\n 'subtype': 'Thread' if r_isthread else 'Process',\n 'label': 'Process #%d' % r_id,\n 'date': r_timestamp})\n # TODO: add process end time (use master branch?)\n\n # Add process creation activity\n if r_parent is not None:\n # Process creation activity\n vertex = {'ID': 'fork%d' % r_id,\n 'type': 'Activity',\n 'subtype': 'Fork',\n 'label': \"#%d creates %s #%d\" % (\n r_parent,\n \"thread\" if r_isthread else \"process\",\n r_id),\n 'date': r_timestamp}\n if has_thread_flag:\n vertex['thread'] = 'true' if r_isthread else 'false'\n vertices.append(vertex)\n\n # Parent -> creation\n edges.append({'ID': 'fork_p_%d' % r_id,\n 'type': 'PerformsFork',\n 'label': \"Performs fork\",\n 'sourceID': 'process%d' % r_parent,\n 'targetID': 'fork%d' % r_id})\n # Creation -> child\n edges.append({'ID': 'fork_c_%d' % r_id,\n 'type': 'ForkCreates',\n 'label': \"Fork creates\",\n 'sourceID': 'fork%d' % r_id,\n 'targetID': 'process%d' % r_id})\n cur.close()\n\n file2package = dict((f.path.path, pkg)\n for pkg in config.packages\n for f in pkg.files)\n inputs_outputs = dict((f.path.path, (bool(f.write_runs),\n bool(f.read_runs)))\n for n, f in iteritems(config.inputs_outputs))\n\n # Read opened files\n cur = conn.cursor()\n rows = cur.execute(\n '''\n SELECT name, is_directory\n FROM opened_files\n GROUP BY name;\n ''')\n for r_name, r_directory in rows:\n # Create file entity\n vertex = {'ID': r_name,\n 'type': 'Entity',\n 'subtype': 'Directory' if r_directory else 'File',\n 'label': r_name}\n if r_name in file2package:\n vertex['package'] = file2package[r_name].name\n if r_name in inputs_outputs:\n out_, in_ = inputs_outputs[r_name]\n if in_:\n vertex['input'] = True\n if out_:\n vertex['output'] = True\n vertices.append(vertex)\n cur.close()\n\n # Read file opens\n cur = conn.cursor()\n rows = cur.execute(\n '''\n SELECT id, name, timestamp, mode, process\n FROM opened_files;\n ''')\n for r_id, r_name, r_timestamp, r_mode, r_process in rows:\n # Create file access activity\n vertices.append({'ID': 'access%d' % r_id,\n 'type': 'Activity',\n 'subtype': ('FileWrites' if r_mode & FILE_WRITE\n else 'FileReads'),\n 'label': (\"File write: %s\" if r_mode & FILE_WRITE\n else \"File read: %s\") % r_name,\n 'date': r_timestamp,\n 'mode': r_mode})\n # Process -> access\n edges.append({'ID': 'proc_access%d' % r_id,\n 'type': 'PerformsFileAccess',\n 'label': \"Process does file access\",\n 'sourceID': 'process%d' % r_process,\n 'targetID': 'access%d' % r_id})\n # Access -> file\n edges.append({'ID': 'access_file%d' % r_id,\n 'type': 'AccessFile',\n 'label': \"File access touches\",\n 'sourceID': 'access%d' % r_id,\n 'targetID': r_name})\n cur.close()\n\n # Read executions\n cur = conn.cursor()\n rows = cur.execute(\n '''\n SELECT id, name, timestamp, process, argv\n FROM executed_files;\n ''')\n for r_id, r_name, r_timestamp, r_process, r_argv in rows:\n argv = r_argv.split('\\0')\n if not argv[-1]:\n argv = argv[:-1]\n cmdline = ' '.join(shell_escape(a) for a in argv)\n\n # Create execution activity\n vertices.append({'ID': 'exec%d' % r_id,\n 'type': 'Activity',\n 'subtype': 'ProcessExecutes',\n 'label': \"Process #%d executes file %s\" % (r_process,\n r_name),\n 'date': r_timestamp,\n 'cmdline': cmdline,\n 'process': r_process,\n 'file': r_name})\n # Process -> execution\n edges.append({'ID': 'proc_exec%d' % r_id,\n 'type': 'ProcessExecution',\n 'label': \"Process does exec()\",\n 'sourceID': 'process%d' % r_process,\n 'targetID': 'exec%d' % r_id})\n # Execution -> file\n edges.append({'ID': 'exec_file%d' % r_id,\n 'type': 'ExecutionFile',\n 'label': \"Execute file\",\n 'sourceID': 'exec%d' % r_id,\n 'targetID': r_name})\n cur.close()\n\n # Write the file from the created lists\n with target.open('w', encoding='utf-8', newline='\\n') as out:\n out.write('\\n\\n'\n '\\n'\n ' \\n')\n\n for vertex in vertices:\n if 'date' not in vertex:\n vertex['date'] = '-1'\n tags = {}\n for k in ('ID', 'type', 'label', 'date'):\n if k not in vertex:\n vertex.update(tags)\n raise ValueError(\"Vertex is missing tag '%s': %r\" % (\n k, vertex))\n tags[k] = vertex.pop(k)\n out.write(' \\n ' +\n '\\n '.join('<{k}>{v}'.format(k=k,\n v=xml_escape(v))\n for k, v in iteritems(tags)))\n if vertex:\n out.write('\\n \\n')\n for k, v in iteritems(vertex):\n out.write(' \\n'\n ' {k}\\n'\n ' {v}\\n'\n ' \\n'\n .format(k=xml_escape(k),\n v=xml_escape(v)))\n out.write(' ')\n out.write('\\n \\n')\n out.write(' \\n'\n ' \\n')\n for edge in edges:\n for k in ('ID', 'type', 'label', 'sourceID', 'targetID'):\n if k not in edge:\n raise ValueError(\"Edge is missing tag '%s': %r\" % (\n k, edge))\n if 'value' not in edge:\n edge['value'] = ''\n out.write(' \\n ' +\n '\\n '.join('<{k}>{v}'.format(k=k,\n v=xml_escape(v))\n for k, v in iteritems(edge)) +\n '\\n \\n')\n out.write(' \\n'\n '\\n')\n\n conn.close()\n\n\ndef provgraph(args):\n \"\"\"provgraph subcommand.\n\n Reads in the trace sqlite3 database and writes out a graph in Provenance\n Viewer graph format.\"\"\"\n def call_generate(args, config, trace):\n generate(Path(args.target[0]), config, trace)\n\n if args.pack is not None:\n rpz_pack = RPZPack(args.pack)\n with rpz_pack.with_config() as config:\n with rpz_pack.with_trace() as trace:\n call_generate(args, config, trace)\n else:\n call_generate(args,\n Path(args.dir) / 'config.yml',\n Path(args.dir) / 'trace.sqlite3')\n\n\ndef disabled_bug13676(args):\n stderr.write(\"Error: your version of Python, %s, is not supported\\n\"\n \"Versions before 2.7.3 are affected by bug 13676 and will \"\n \"not be able to read\\nthe trace \"\n \"database\\n\" % sys.version.split(' ', 1)[0])\n sys.exit(1)\n\n\ndef setup(parser, **kwargs):\n \"\"\"Generates a Prov Viewer graph from the trace data\n \"\"\"\n\n # http://bugs.python.org/issue13676\n # This prevents repro(un)zip from reading argv and envp arrays from trace\n if sys.version_info < (2, 7, 3):\n parser.add_argument('rest_of_cmdline', nargs=argparse.REMAINDER,\n help=argparse.SUPPRESS)\n parser.set_defaults(func=disabled_bug13676)\n return {'test_compatibility': (COMPAT_NO, \"Python >2.7.3 required\")}\n\n parser.add_argument('target', nargs=1, help=\"Destination DOT file\")\n parser.add_argument(\n '-d', '--dir', default='.reprozip-trace',\n help=\"where the database and configuration file are stored (default: \"\n \"./.reprozip-trace)\")\n parser.add_argument(\n 'pack', nargs=argparse.OPTIONAL,\n help=\"Pack to extract (defaults to reading from --dir)\")\n parser.set_defaults(func=provgraph)\n\n return {'test_compatibility': COMPAT_OK}\n" }, { "path": "reprounzip/reprounzip/utils.py", "content": "# Copyright (C) 2014 New York University\n# This file is part of ReproZip which is released under the Revised BSD License\n# See file LICENSE for full license details.\n\n# This file is shared:\n# reprozip/reprozip/utils.py\n# reprounzip/reprounzip/utils.py\n\n\"\"\"Utility functions.\n\nThese functions are shared between reprozip and reprounzip but are not specific\nto this software (more utilities).\n\n\"\"\"\n\nfrom __future__ import division, print_function, unicode_literals\n\nimport codecs\nimport contextlib\nfrom datetime import datetime\nimport email.utils\nimport itertools\nimport locale\nimport logging\nimport operator\nimport os\nimport requests\nfrom rpaths import Path, PosixPath\nimport stat\nimport subprocess\nimport sys\nimport time\n\n\nlogger = logging.getLogger(__name__.split('.', 1)[0])\n\n\nclass StreamWriter(object):\n def __init__(self, stream):\n writer = codecs.getwriter(locale.getpreferredencoding())\n self._writer = writer(stream, 'replace')\n self.buffer = stream\n\n def writelines(self, lines):\n self.write(str('').join(lines))\n\n def write(self, obj):\n if isinstance(obj, bytes):\n self.buffer.write(obj)\n else:\n self._writer.write(obj)\n\n def __getattr__(self, name,\n getattr=getattr):\n\n \"\"\" Inherit all other methods from the underlying stream.\n \"\"\"\n return getattr(self._writer, name)\n\n\nPY3 = sys.version_info[0] == 3\n\n\nif PY3:\n izip = zip\n irange = range\n iteritems = lambda d: d.items()\n itervalues = lambda d: d.values()\n listvalues = lambda d: list(d.values())\n\n stdout_bytes = sys.stdout.buffer if sys.stdout is not None else None\n stderr_bytes = sys.stderr.buffer if sys.stderr is not None else None\n stdin_bytes = sys.stdin.buffer if sys.stdin is not None else None\n stdout, stderr = sys.stdout, sys.stderr\nelse:\n izip = itertools.izip\n irange = xrange # noqa: F821\n iteritems = lambda d: d.iteritems()\n itervalues = lambda d: d.itervalues()\n listvalues = lambda d: d.values()\n\n _writer = codecs.getwriter(locale.getpreferredencoding())\n stdout_bytes, stderr_bytes = sys.stdout, sys.stderr\n stdin_bytes = sys.stdin\n stdout, stderr = StreamWriter(sys.stdout), StreamWriter(sys.stderr)\n\n\nif PY3:\n int_types = int,\n unicode_ = str\nelse:\n int_types = int, long # noqa: F821\n unicode_ = unicode # noqa: F821\n\n\ndef flatten(n, iterable):\n \"\"\"Flattens an iterable by repeatedly calling chain.from_iterable() on it.\n\n >>> a = [[1, 2, 3], [4, 5, 6]]\n >>> b = [[7, 8], [9, 10, 11, 12, 13, 14, 15, 16]]\n >>> l = [a, b]\n >>> list(flatten(0, a))\n [[1, 2, 3], [4, 5, 6]]\n >>> list(flatten(1, a))\n [1, 2, 3, 4, 5, 6]\n >>> list(flatten(1, l))\n [[1, 2, 3], [4, 5, 6], [7, 8], [9, 10, 11, 12, 13, 14, 15, 16]]\n >>> list(flatten(2, l))\n [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]\n \"\"\"\n for _ in irange(n):\n iterable = itertools.chain.from_iterable(iterable)\n return iterable\n\n\nclass UniqueNames(object):\n \"\"\"Makes names unique amongst the ones it's already seen.\n \"\"\"\n def __init__(self):\n self.names = set()\n\n def insert(self, name):\n assert name not in self.names\n self.names.add(name)\n\n def __call__(self, name):\n nb = 1\n attempt = name\n while attempt in self.names:\n nb += 1\n attempt = '%s_%d' % (name, nb)\n self.names.add(attempt)\n return attempt\n\n\ndef escape(s):\n \"\"\"Escapes backslashes and double quotes in strings.\n\n This does NOT add quotes around the string.\n \"\"\"\n return s.replace('\\\\', '\\\\\\\\').replace('\"', '\\\\\"')\n\n\ndef optional_return_type(req_args, other_args):\n \"\"\"Sort of namedtuple but with name-only fields.\n\n When deconstructing a namedtuple, you have to get all the fields:\n\n >>> o = namedtuple('T', ['a', 'b', 'c'])(1, 2, 3)\n >>> a, b = o\n ValueError: too many values to unpack\n\n You thus cannot easily add new return values. This class allows it:\n\n >>> o2 = optional_return_type(['a', 'b'], ['c'])(1, 2, 3)\n >>> a, b = o2\n >>> c = o2.c\n \"\"\"\n if len(set(req_args) | set(other_args)) != len(req_args) + len(other_args):\n raise ValueError\n\n # Maps argument name to position in each list\n req_args_pos = dict((n, i) for i, n in enumerate(req_args))\n other_args_pos = dict((n, i) for i, n in enumerate(other_args))\n\n def cstr(cls, *args, **kwargs):\n if len(args) > len(req_args) + len(other_args):\n raise TypeError(\n \"Too many arguments (expected at least %d and no more than \"\n \"%d)\" % (len(req_args),\n len(req_args) + len(other_args)))\n\n args1, args2 = args[:len(req_args)], args[len(req_args):]\n req = dict((i, v) for i, v in enumerate(args1))\n other = dict(izip(other_args, args2))\n\n for k, v in iteritems(kwargs):\n if k in req_args_pos:\n pos = req_args_pos[k]\n if pos in req:\n raise TypeError(\"Multiple values for field %s\" % k)\n req[pos] = v\n elif k in other_args_pos:\n if k in other:\n raise TypeError(\"Multiple values for field %s\" % k)\n other[k] = v\n else:\n raise TypeError(\"Unknown field name %s\" % k)\n\n args = []\n for i, k in enumerate(req_args):\n if i not in req:\n raise TypeError(\"Missing value for field %s\" % k)\n args.append(req[i])\n\n inst = tuple.__new__(cls, args)\n inst.__dict__.update(other)\n return inst\n\n dct = {'__new__': cstr}\n for i, n in enumerate(req_args):\n dct[n] = property(operator.itemgetter(i))\n return type(str('OptionalReturnType'), (tuple,), dct)\n\n\ndef tz_offset():\n offset = time.timezone if time.localtime().tm_isdst == 0 else time.altzone\n return -offset\n\n\ndef isodatetime():\n offset = tz_offset()\n sign = '+'\n if offset < 0:\n sign = '-'\n offset = -offset\n if offset % 60 == 0:\n offset = '%02d:%02d' % (offset // 3600, (offset // 60) % 60)\n else:\n offset = '%02d:%02d:%02d' % (offset // 3600, (offset // 60) % 60,\n offset % 60)\n # Remove microsecond\n now = datetime.now()\n now = datetime(year=now.year, month=now.month, day=now.day,\n hour=now.hour, minute=now.minute, second=now.second)\n return '%s%s%s' % (now.isoformat(),\n sign,\n offset)\n\n\ndef hsize(nbytes):\n \"\"\"Readable size.\n \"\"\"\n if nbytes is None:\n return \"unknown\"\n\n KB = 1 << 10\n MB = 1 << 20\n GB = 1 << 30\n TB = 1 << 40\n PB = 1 << 50\n\n nbytes = float(nbytes)\n\n if nbytes < KB:\n return \"{0} bytes\".format(nbytes)\n elif nbytes < MB:\n return \"{0:.2f} KB\".format(nbytes / KB)\n elif nbytes < GB:\n return \"{0:.2f} MB\".format(nbytes / MB)\n elif nbytes < TB:\n return \"{0:.2f} GB\".format(nbytes / GB)\n elif nbytes < PB:\n return \"{0:.2f} TB\".format(nbytes / TB)\n else:\n return \"{0:.2f} PB\".format(nbytes / PB)\n\n\ndef normalize_path(path):\n \"\"\"Normalize a path obtained from the database.\n \"\"\"\n # For some reason, os.path.normpath() keeps multiple leading slashes\n # We don't want this since it has no meaning on Linux\n path = PosixPath(path)\n if path.path.startswith(path._sep + path._sep):\n path = PosixPath(path.path[1:])\n return path\n\n\ndef find_all_links_recursive(filename, files):\n path = Path('/')\n for c in filename.components[1:]:\n # At this point, path is a canonical path, and all links in it have\n # been resolved\n\n # We add the next path component\n path = path / c\n\n # That component is possibly a link\n if path.is_link():\n # Adds the link itself\n files.add(path)\n\n target = path.read_link(absolute=True)\n # Here, target might contain a number of symlinks\n if target not in files:\n # Recurse on this new path\n find_all_links_recursive(target, files)\n # Restores the invariant; realpath might resolve several links here\n path = path.resolve()\n return path\n\n\ndef find_all_links(filename, include_target=False):\n \"\"\"Dereferences symlinks from a path.\n\n If include_target is True, this also returns the real path of the final\n target.\n\n Example:\n /\n a -> b\n b\n g -> c\n c -> ../a/d\n d\n e -> /f\n f\n >>> find_all_links('/a/g/e', True)\n ['/a', '/b/c', '/b/g', '/b/d/e', '/f']\n \"\"\"\n files = set()\n filename = Path(filename)\n assert filename.absolute()\n path = find_all_links_recursive(filename, files)\n files = list(files)\n if include_target:\n files.append(path)\n return files\n\n\ndef join_root(root, path):\n \"\"\"Prepends `root` to the absolute path `path`.\n \"\"\"\n p_root, p_loc = path.split_root()\n assert p_root == b'/'\n return root / p_loc\n\n\n@contextlib.contextmanager\ndef make_dir_writable(directory):\n \"\"\"Context-manager that sets write permission on a directory.\n\n This assumes that the directory belongs to you. If the u+w permission\n wasn't set, it gets set in the context, and restored to what it was when\n leaving the context. u+x also gets set on all the directories leading to\n that path.\n \"\"\"\n uid = os.getuid()\n\n try:\n sb = directory.stat()\n except OSError:\n pass\n else:\n if sb.st_uid != uid or sb.st_mode & 0o700 == 0o700:\n yield\n return\n\n # These are the permissions to be restored, in reverse order\n restore_perms = []\n try:\n # Add u+x to all directories up to the target\n path = Path('/')\n for c in directory.components[1:-1]:\n path = path / c\n sb = path.stat()\n if sb.st_uid == uid and not sb.st_mode & 0o100:\n logger.debug(\"Temporarily setting u+x on %s\", path)\n restore_perms.append((path, sb.st_mode))\n path.chmod(sb.st_mode | 0o700)\n\n # Add u+wx to the target\n sb = directory.stat()\n if sb.st_uid == uid and sb.st_mode & 0o700 != 0o700:\n logger.debug(\"Temporarily setting u+wx on %s\", directory)\n restore_perms.append((directory, sb.st_mode))\n directory.chmod(sb.st_mode | 0o700)\n\n yield\n finally:\n for path, mod in reversed(restore_perms):\n path.chmod(mod)\n\n\ndef rmtree_fixed(path):\n \"\"\"Like :func:`shutil.rmtree` but doesn't choke on annoying permissions.\n\n If a directory with -w or -x is encountered, it gets fixed and deletion\n continues.\n \"\"\"\n if path.is_link():\n raise OSError(\"Cannot call rmtree on a symbolic link\")\n\n uid = os.getuid()\n st = path.lstat()\n\n if st.st_uid == uid and st.st_mode & 0o700 != 0o700:\n path.chmod(st.st_mode | 0o700)\n\n for entry in path.listdir():\n if stat.S_ISDIR(entry.lstat().st_mode):\n rmtree_fixed(entry)\n else:\n entry.remove()\n\n path.rmdir()\n\n\n# Compatibility with ReproZip <= 1.0.3\ncheck_output = subprocess.check_output\n\n\ndef copyfile(source, destination, CHUNK_SIZE=4096):\n \"\"\"Copies from one file object to another.\n \"\"\"\n while True:\n chunk = source.read(CHUNK_SIZE)\n if chunk:\n destination.write(chunk)\n if len(chunk) != CHUNK_SIZE:\n break\n\n\ndef download_file(url, dest, cachename=None, ssl_verify=None):\n \"\"\"Downloads a file using a local cache.\n\n If the file cannot be downloaded or if it wasn't modified, the cached\n version will be used instead.\n\n The cache lives in ``~/.cache/reprozip/``.\n \"\"\"\n if cachename is None:\n if dest is None:\n raise ValueError(\"One of 'dest' or 'cachename' must be specified\")\n cachename = dest.components[-1]\n\n headers = {}\n\n if 'XDG_CACHE_HOME' in os.environ:\n cache = Path(os.environ['XDG_CACHE_HOME'])\n else:\n cache = Path('~/.cache').expand_user()\n cache = cache / 'reprozip' / cachename\n if cache.exists():\n mtime = email.utils.formatdate(cache.mtime(), usegmt=True)\n headers['If-Modified-Since'] = mtime\n\n cache.parent.mkdir(parents=True)\n\n try:\n response = requests.get(url, headers=headers,\n timeout=2 if cache.exists() else 10,\n stream=True, verify=ssl_verify)\n response.raise_for_status()\n if response.status_code == 304:\n raise requests.HTTPError(\n '304 File is up to date, no data returned',\n response=response)\n except requests.RequestException as e:\n if cache.exists():\n if e.response and e.response.status_code == 304:\n logger.info(\"Download %s: cache is up to date\", cachename)\n else:\n logger.warning(\"Download %s: error downloading %s: %s\",\n cachename, url, e)\n if dest is not None:\n cache.copy(dest)\n return dest\n else:\n return cache\n else:\n raise\n\n logger.info(\"Download %s: downloading %s\", cachename, url)\n try:\n with cache.open('wb') as f:\n for chunk in response.iter_content(4096):\n f.write(chunk)\n response.close()\n except Exception as e: # pragma: no cover\n try:\n cache.remove()\n except OSError:\n pass\n raise e\n logger.info(\"Downloaded %s successfully\", cachename)\n\n if dest is not None:\n cache.copy(dest)\n return dest\n else:\n return cache\n" }, { "path": "reprounzip/setup.cfg", "content": "[bdist_wheel]\nuniversal = 1\n" }, { "path": "reprounzip/setup.py", "content": "import io\nimport os\nfrom setuptools import setup\n\n\n# pip workaround\nos.chdir(os.path.abspath(os.path.dirname(__file__)))\n\n\n# Need to specify encoding for PY3, which has the worst unicode handling ever\nwith io.open('README.rst', encoding='utf-8') as fp:\n description = fp.read()\nreq = [\n 'importlib-metadata',\n 'PyYAML',\n 'rpaths>=0.8',\n 'usagestats>=0.3',\n 'requests',\n 'distro',\n 'pyelftools']\nsetup(name='reprounzip',\n version='1.3.2',\n packages=['reprounzip', 'reprounzip.unpackers',\n 'reprounzip.unpackers.common', 'reprounzip.plugins'],\n entry_points={\n 'console_scripts': [\n 'reprounzip = reprounzip.main:main'],\n 'reprounzip.unpackers': [\n 'info = reprounzip.pack_info:setup_info',\n 'showfiles = reprounzip.pack_info:setup_showfiles',\n 'graph = reprounzip.unpackers.graph:setup',\n 'provviewer = reprounzip.unpackers.provviewer:setup',\n 'installpkgs = reprounzip.unpackers.default:setup_installpkgs',\n 'directory = reprounzip.unpackers.default:setup_directory',\n 'chroot = reprounzip.unpackers.default:setup_chroot']},\n namespace_packages=['reprounzip', 'reprounzip.unpackers'],\n install_requires=req,\n extras_require={\n 'all': ['reprounzip-vagrant>=1.0', 'reprounzip-docker>=1.0',\n 'reprounzip-vistrails>=1.0']},\n description=\"Linux tool enabling reproducible experiments (unpacker)\",\n author=\"Remi Rampin, Fernando Chirigati, Dennis Shasha, Juliana Freire\",\n author_email='reprozip@nyu.edu',\n maintainer=\"Remi Rampin\",\n maintainer_email='remi@rampin.org',\n url='https://www.reprozip.org/',\n project_urls={\n 'Documentation': 'https://docs.reprozip.org/',\n 'Examples': 'https://examples.reprozip.org/',\n 'Source': 'https://github.com/VIDA-NYU/reprozip',\n 'Bug Tracker': 'https://github.com/VIDA-NYU/reprozip/issues',\n 'Chat': 'https://riot.im/app/#/room/#reprozip:matrix.org',\n 'Changelog':\n 'https://github.com/VIDA-NYU/reprozip/blob/1.x/CHANGELOG.md',\n },\n long_description=description,\n license='BSD-3-Clause',\n keywords=['reprozip', 'reprounzip', 'reproducibility', 'provenance',\n 'vida', 'nyu'],\n classifiers=[\n 'Development Status :: 5 - Production/Stable',\n 'Intended Audience :: Science/Research',\n 'License :: OSI Approved :: BSD License',\n 'Programming Language :: Python :: 2.7',\n 'Programming Language :: Python :: 3',\n 'Topic :: Scientific/Engineering',\n 'Topic :: System :: Archiving'])\n" }, { "path": "reprounzip-docker/LICENSE.txt", "content": "Copyright (C) 2014, New York University\nAll rights reserved.\n\nRedistribution and use in source and binary forms, with or without\nmodification, are permitted provided that the following conditions are met:\n\n1. Redistributions of source code must retain the above copyright notice, this\n list of conditions and the following disclaimer.\n\n2. Redistributions in binary form must reproduce the above copyright notice,\n this list of conditions and the following disclaimer in the documentation\n and/or other materials provided with the distribution.\n\n3. Neither the name of the copyright holder nor the names of its contributors\n may be used to endorse or promote products derived from this software\n without specific prior written permission.\n\nTHIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \"AS IS\" AND\nANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED\nWARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE\nDISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE\nFOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL\nDAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR\nSERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER\nCAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,\nOR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE\nOF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n" }, { "path": "reprounzip-docker/MANIFEST.in", "content": "include README.rst\ninclude LICENSE.txt\n" }, { "path": "reprounzip-docker/README.rst", "content": "ReproZip project\n================\n\n`ReproZip `__ is a tool aimed at simplifying the process of creating reproducible experiments from command-line executions, a frequently-used common denominator in computational science. It tracks operating system calls and creates a bundle that contains all the binaries, files and dependencies required to run a given command on the author's computational environment (packing step). A reviewer can then extract the experiment in his environment to reproduce the results (unpacking step).\n\nreprounzip-docker\n-----------------\n\nThis is the component responsible for the unpacking step on different environments (Linux, Windows, and Mac OS X) by using a `Docker `_ container. Please refer to `reprozip `__, `reprounzip `_, and `reprounzip-vagrant `_ for other components and plugins.\n\nAdditional Information\n----------------------\n\nFor more detailed information, please refer to our `website `_, as well as to our `documentation `_.\n\nReproZip is currently being developed at `NYU `_. The team includes:\n\n* `Fernando Chirigati `_\n* `Juliana Freire `_\n* `Remi Rampin `_\n* `Dennis Shasha `_\n* `Vicky Rampin `_\n" }, { "path": "reprounzip-docker/reprounzip/__init__.py", "content": "try: # pragma: no cover\n __import__('pkg_resources').declare_namespace(__name__)\nexcept ImportError: # pragma: no cover\n from pkgutil import extend_path\n __path__ = extend_path(__path__, __name__)\n" }, { "path": "reprounzip-docker/reprounzip/unpackers/__init__.py", "content": "try: # pragma: no cover\n __import__('pkg_resources').declare_namespace(__name__)\nexcept ImportError: # pragma: no cover\n from pkgutil import extend_path\n __path__ = extend_path(__path__, __name__)\n" }, { "path": "reprounzip-docker/reprounzip/unpackers/docker.py", "content": "# Copyright (C) 2014 New York University\n# This file is part of ReproZip which is released under the Revised BSD License\n# See file LICENSE for full license details.\n\n\"\"\"Docker plugin for reprounzip.\n\nThis files contains the 'docker' unpacker, which builds a Dockerfile from a\nreprozip pack. You can then build a container and run it with Docker.\n\nSee http://www.docker.io/\n\"\"\"\n\nfrom __future__ import division, print_function, unicode_literals\n\nimport argparse\nfrom itertools import chain\nimport json\nimport logging\nimport os\nimport re\nfrom rpaths import Path, PosixPath\nimport socket\nimport subprocess\nimport sys\nimport tarfile\n\nfrom reprounzip.common import load_config, record_usage, RPZPack\nfrom reprounzip import signals\nfrom reprounzip.parameters import get_parameter\nfrom reprounzip.unpackers.common import COMPAT_OK, COMPAT_MAYBE, \\\n UsageError, CantFindInstaller, composite_action, target_must_exist, \\\n make_unique_name, shell_escape, select_installer, busybox_url, sudo_url, \\\n rpztar_url, \\\n FileUploader, FileDownloader, get_runs, add_environment_options, \\\n fixup_environment, interruptible_call, metadata_read, metadata_write, \\\n metadata_initial_iofiles, metadata_update_run, parse_ports\nfrom reprounzip.unpackers.common.x11 import X11Handler, LocalForwarder\nfrom reprounzip.utils import unicode_, iteritems, stderr, join_root, \\\n download_file\n\n\nlogger = logging.getLogger('reprounzip.docker')\n\n\n# How this all works:\n# - setup/create just copies file to the target directory and writes the\n# Dockerfile\n# - setup/build creates the image and stores it in the unpacker info as\n# 'initial_image' and 'current_image'\n# - run runs a container from 'current_image', the commits it into the new\n# 'current_image'\n# - upload creates a Dockerfile in a temporary directory, copies all the files\n# to upload there, and builds it. This creates a new 'current_image' with\n# the files replaced\n# - download creates a temporary container from 'current_image' and uses\n# docker cp from it\n# - reset destroys 'current_image' and resets it to 'initial_image'\n# This means that a lot of images will get layered on top of each other,\n# unfortunately this is necessary so that successive runs carry over the global\n# state as expected.\n\n\ndef select_image(runs):\n \"\"\"Selects a base image for the experiment, with the correct distribution.\n \"\"\"\n distribution, version = runs[0]['distribution']\n distribution = distribution.lower()\n architecture = runs[0]['architecture']\n\n record_usage(docker_select_box='%s;%s;%s' % (distribution, version,\n architecture))\n\n if architecture == 'i686':\n logger.info(\"Wanted architecture was i686, but we'll use x86_64 with \"\n \"Docker\")\n elif architecture != 'x86_64':\n logger.error(\"Error: unsupported architecture %s\", architecture)\n sys.exit(1)\n\n def find_distribution(parameter, distribution, version):\n images = parameter['images']\n default = parameter['default']\n\n for distrib_name, distrib in iteritems(images):\n if distribution == distrib_name:\n result = find_version(distrib, version)\n if result is not None:\n return result\n distrib = images[default]\n logger.warning(\"Unsupported distribution '%s', using %s\",\n distribution, default)\n return find_version(distrib, None)\n\n def find_version(distrib, version):\n if version is not None:\n for image in distrib['versions']:\n if re.match(image['version'], version) is not None:\n return image['distribution'], image['image']\n image = distrib['default']\n if version is not None:\n logger.warning(\"Using %s instead of '%s'\",\n image['name'], version)\n return image['distribution'], image['image']\n\n return find_distribution(get_parameter('docker_images'),\n distribution, version)\n\n\ndef write_dict(path, dct):\n metadata_write(path, dct, 'docker')\n\n\ndef read_dict(path):\n return metadata_read(path, 'docker')\n\n\ndef docker_setup(args):\n \"\"\"Does both create and build.\n\n Removes the directory if building fails.\n \"\"\"\n docker_setup_create(args)\n try:\n docker_setup_build(args)\n except BaseException:\n Path(args.target[0]).rmtree(ignore_errors=True)\n raise\n\n\ndef docker_setup_create(args):\n \"\"\"Sets up the experiment to be run in a Docker-built container.\n \"\"\"\n pack = Path(args.pack[0])\n target = Path(args.target[0])\n if target.exists():\n logger.critical(\"Target directory exists\")\n sys.exit(1)\n\n signals.pre_setup(target=target, pack=pack)\n\n target.mkdir()\n\n try:\n # Unpacks configuration file\n rpz_pack = RPZPack(pack)\n rpz_pack.extract_config(target / 'config.yml')\n\n # Loads config\n runs, packages, other_files = config = load_config(\n target / 'config.yml', True)\n\n if args.base_image:\n record_usage(docker_explicit_base=True)\n base_image = args.base_image[0]\n if args.distribution:\n target_distribution = args.distribution[0]\n else:\n target_distribution = None\n else:\n target_distribution, base_image = select_image(runs)\n logger.info(\"Using base image %s\", base_image)\n logger.debug(\"Distribution: %s\", target_distribution or \"unknown\")\n\n rpz_pack.copy_data_tar(target / 'data.tgz')\n\n arch = runs[0]['architecture']\n\n # Writes Dockerfile\n logger.info(\"Writing %s...\", target / 'Dockerfile')\n with (target / 'Dockerfile').open('w', encoding='utf-8',\n newline='\\n') as dockerfile:\n dockerfile.write('FROM %s\\n\\n' % base_image)\n\n # Installs busybox\n download_file(busybox_url(arch),\n target / 'busybox',\n 'busybox-%s' % arch)\n dockerfile.write('COPY busybox /busybox\\n')\n\n # Installs rpzsudo\n download_file(sudo_url(arch),\n target / 'rpzsudo',\n 'rpzsudo-%s' % arch)\n dockerfile.write('COPY rpzsudo /rpzsudo\\n\\n')\n\n # Installs rpztar\n download_file(rpztar_url(arch),\n target / 'rpztar',\n 'rpztar-%s' % arch)\n dockerfile.write('COPY rpztar /rpztar\\n\\n')\n\n dockerfile.write('COPY data.tgz /reprozip_data.tgz\\n\\n')\n dockerfile.write('COPY rpz-files.list /rpz-files.list\\n')\n dockerfile.write('RUN \\\\\\n'\n ' chmod +x /busybox /rpzsudo /rpztar && \\\\\\n')\n\n if args.install_pkgs:\n # Install every package through package manager\n missing_packages = []\n else:\n # Only install packages that were not packed\n missing_packages = [pkg for pkg in packages if pkg.packfiles]\n packages = [pkg for pkg in packages if not pkg.packfiles]\n if packages:\n record_usage(docker_install_pkgs=True)\n try:\n installer = select_installer(pack, runs,\n target_distribution)\n except CantFindInstaller as e:\n logger.error(\"Need to install %d packages but couldn't \"\n \"select a package installer: %s\",\n len(packages), e)\n sys.exit(1)\n # Updates package sources\n update_script = installer.update_script()\n if update_script:\n dockerfile.write(' %s && \\\\\\n' % update_script)\n # Installs necessary packages\n dockerfile.write(\n ' %s && \\\\\\n' % installer.install_script(packages),\n )\n logger.info(\"Dockerfile will install the %d software \"\n \"packages that were not packed\", len(packages))\n else:\n record_usage(docker_install_pkgs=False)\n\n # Untar\n paths = set()\n pathlist = []\n # Add intermediate directories, and check for existence in the tar\n logger.info(\"Generating file list...\")\n missing_files = chain.from_iterable(pkg.files\n for pkg in missing_packages)\n data_files = rpz_pack.data_filenames()\n listoffiles = list(chain(other_files, missing_files))\n for f in listoffiles:\n if f.path.unicodename in ('resolv.conf', 'hosts') and (\n f.path.lies_under('/etc') or\n f.path.lies_under('/run') or\n f.path.lies_under('/var')):\n continue\n path = PosixPath('/')\n for c in rpz_pack.remove_data_prefix(f.path).components:\n path = path / c\n if path in paths:\n continue\n paths.add(path)\n if path in data_files:\n if path != '/etc':\n pathlist.append(path)\n else:\n logger.info(\"Missing file %s\", path)\n rpz_pack.close()\n with (target / 'rpz-files.list').open('wb') as filelist:\n for p in pathlist:\n filelist.write(join_root(PosixPath(''), p).path)\n filelist.write(b'\\0')\n dockerfile.write(\n ' cd / && '\n + '/rpztar /reprozip_data.tgz /rpz-files.list\\n',\n )\n\n # Meta-data for reprounzip\n write_dict(target, metadata_initial_iofiles(config))\n\n signals.post_setup(target=target, pack=pack)\n except Exception:\n target.rmtree(ignore_errors=True)\n raise\n\n\n@target_must_exist\ndef docker_setup_build(args):\n \"\"\"Builds the container from the Dockerfile\n \"\"\"\n target = Path(args.target[0])\n unpacked_info = read_dict(target)\n if 'initial_image' in unpacked_info:\n logger.critical(\"Image already built\")\n sys.exit(1)\n\n if args.image_name:\n image = args.image_name[0]\n if not isinstance(image, bytes):\n image = image.encode('ascii')\n else:\n image = make_unique_name(b'reprounzip_image_')\n\n logger.info(\"Calling 'docker build'...\")\n try:\n retcode = subprocess.call(args.docker_cmd.split() + ['build', '-t'] +\n args.docker_option + [image, '.'],\n cwd=target.path)\n except OSError:\n logger.critical(\"docker executable not found\")\n sys.exit(1)\n else:\n if retcode != 0:\n logger.critical(\"docker build failed with code %d\", retcode)\n sys.exit(1)\n logger.info(\"Initial image created: %s\", image.decode('ascii'))\n\n unpacked_info['initial_image'] = image\n unpacked_info['current_image'] = image\n if 'DOCKER_MACHINE_NAME' in os.environ:\n unpacked_info['docker_host'] = {\n 'type': 'docker-machine',\n 'name': os.environ['DOCKER_MACHINE_NAME']}\n elif 'DOCKER_HOST' in os.environ:\n unpacked_info['docker_host'] = {\n 'type': 'custom',\n 'env': dict((k, v)\n for k, v in iteritems(os.environ)\n if k.startswith('DOCKER_'))}\n\n write_dict(target, unpacked_info)\n\n\n@target_must_exist\ndef docker_reset(args):\n \"\"\"Reset the image to the initial one.\n\n This will quickly undo the effects of all the 'upload' and 'run' commands\n on the environment.\n \"\"\"\n target = Path(args.target[0])\n unpacked_info = read_dict(target)\n if 'initial_image' not in unpacked_info:\n logger.critical(\"Image doesn't exist yet, have you run setup/build?\")\n sys.exit(1)\n image = unpacked_info['current_image']\n initial = unpacked_info['initial_image']\n\n if image == initial:\n logger.warning(\"Image is already in the initial state, nothing to \"\n \"reset\")\n else:\n logger.info(\"Removing image %s\", image.decode('ascii'))\n retcode = subprocess.call(args.docker_cmd.split() + ['rmi', image])\n if retcode != 0:\n logger.warning(\"Can't remove previous image, docker returned %d\",\n retcode)\n unpacked_info['current_image'] = initial\n write_dict(target, unpacked_info)\n\n\n_addr_re = re.compile(r'^(?:[a-z]+://)?([0-9a-zA-Z_.-]+)(?::[0-9]+)?$')\n\n\ndef get_local_addr():\n \"\"\"Gets the local IP address of the local machine.\n\n This finds the address used to connect to the Docker host by establishing a\n network connection to it and reading the local address of the socket.\n\n Returns an IP address as a unicode object, in digits-and-dots format.\n\n >>> get_local_addr()\n '172.17.42.1'\n \"\"\"\n # This function works by creating a socket and connecting to a remote IP.\n # The local address of this socket is assumed to be the address of this\n # machine, that the Docker container can reach.\n target = None\n\n # Find hostname or IP address in DOCKER_HOST\n if 'DOCKER_HOST' in os.environ:\n m = _addr_re.match(os.environ['DOCKER_HOST'])\n if m is not None:\n target = m.group(1)\n if target.startswith('127.'):\n target = None\n\n # Else, use whatever local interface lets you connect to google.com\n if target is None:\n target = 'google.com'\n\n try:\n addresses = socket.getaddrinfo(target, 9, socket.AF_UNSPEC,\n socket.SOCK_STREAM)\n except socket.gaierror:\n pass\n else:\n for address in addresses:\n try:\n af, socktype, proto, canonname, sa = address\n sock = socket.socket(af, socktype, proto)\n sock.settimeout(1)\n sock.connect(sa)\n except socket.error:\n pass\n else:\n addr = sock.getsockname()[0]\n if isinstance(addr, bytes):\n addr = addr.decode('ascii')\n sock.close()\n return addr\n\n return '127.0.0.1'\n\n\n_dockerhost_re = re.compile(r'^tcp://([0-9.]+):[0-9]+$')\n\n\n@target_must_exist\ndef docker_run(args):\n \"\"\"Runs the experiment in the container.\n \"\"\"\n target = Path(args.target[0])\n unpacked_info = read_dict(target)\n cmdline = args.cmdline\n\n # Sanity check\n if args.detach and args.x11:\n logger.critical(\"Error: Can't use X11 forwarding if you're detaching\")\n raise UsageError\n\n # Loads config\n config = load_config(target / 'config.yml', True)\n runs = config.runs\n\n selected_runs = get_runs(runs, args.run, cmdline)\n\n # Get current image name\n if 'current_image' in unpacked_info:\n image = unpacked_info['current_image']\n logger.debug(\"Running from image %s\", image.decode('ascii'))\n else:\n logger.critical(\"Image doesn't exist yet, have you run setup/build?\")\n sys.exit(1)\n\n # Name of new container\n if args.detach:\n container = make_unique_name(b'reprounzip_detached_')\n else:\n container = make_unique_name(b'reprounzip_run_')\n\n hostname = runs[selected_runs[0]].get('hostname', 'reprounzip')\n\n # Port forwarding\n port_options = []\n for port_host, port_container, proto in parse_ports(args.expose_port):\n port_options.extend(['-p',\n '%s:%s/%s' % (port_host, port_container, proto)])\n\n # X11 handler\n if args.x11:\n local_ip = get_local_addr()\n\n docker_host = local_ip\n if os.environ.get('DOCKER_HOST'):\n m = _dockerhost_re.match(os.environ['DOCKER_HOST'])\n if m is not None:\n docker_host = m.group(1)\n\n if args.tunneled_x11:\n x11 = X11Handler(True, ('internet', docker_host), args.x11_display)\n else:\n x11 = X11Handler(True, ('internet', local_ip), args.x11_display)\n\n if (docker_host != local_ip and docker_host != 'localhost' and\n not docker_host.startswith('127.') and\n not docker_host.startswith('192.168.99.')):\n ssh_cmdline = ' '.join(\n '-R*:%(p)d:127.0.0.1:%(p)d' % {'p': port}\n for port, connector in x11.port_forward)\n logger.warning(\n \"You requested X11 forwarding but the Docker container \"\n \"appears to be running remotely. It is probable that it \"\n \"won't be able to connect to the local display. Creating \"\n \"a remote SSH tunnel and running with --tunneled-x11 \"\n \"might help (%s).\",\n ssh_cmdline)\n else:\n x11 = X11Handler(False, ('local', hostname), args.x11_display)\n\n cmds = []\n for run_number in selected_runs:\n run = runs[run_number]\n cmd = 'cd %s && ' % shell_escape(run['workingdir'])\n cmd += '/busybox env -i '\n environ = x11.fix_env(run['environ'])\n environ = fixup_environment(environ, args)\n cmd += ' '.join('%s=%s' % (shell_escape(k), shell_escape(v))\n for k, v in iteritems(environ))\n cmd += ' '\n # FIXME : Use exec -a or something if binary != argv[0]\n if cmdline is None:\n argv = [run['binary']] + run['argv'][1:]\n else:\n argv = cmdline\n cmd += ' '.join(shell_escape(a) for a in argv)\n uid = run.get('uid', 1000)\n gid = run.get('gid', 1000)\n cmd = '/rpzsudo \\'#%d\\' \\'#%d\\' /busybox sh -c %s' % (\n uid, gid,\n shell_escape(cmd))\n cmds.append(cmd)\n cmds = x11.init_cmds + cmds\n cmds = ' && '.join(cmds)\n\n signals.pre_run(target=target)\n\n # Creates forwarders\n forwarders = []\n for port, connector in x11.port_forward:\n forwarders.append(LocalForwarder(connector, port))\n\n if args.detach:\n logger.info(\"Start container %s (detached)\",\n container.decode('ascii'))\n retcode = interruptible_call(args.docker_cmd.split() +\n ['run', b'--name=' + container,\n '-h', hostname,\n '-d', '-t'] +\n port_options +\n args.docker_option +\n [image, '/busybox', 'sh', '-c', cmds])\n if retcode != 0:\n logger.critical(\"docker run failed with code %d\", retcode)\n subprocess.call(['docker', 'rm', '-f', container])\n sys.exit(1)\n return\n\n # Run command in container\n logger.info(\"Starting container %s\", container.decode('ascii'))\n retcode = interruptible_call(args.docker_cmd.split() +\n ['run', b'--name=' + container,\n '-h', hostname,\n '-i', '-t'] +\n port_options +\n args.docker_option +\n [image, '/busybox', 'sh', '-c', cmds],\n request_tty=True)\n\n # Get exit status from \"docker inspect\"\n try:\n out = subprocess.check_output(args.docker_cmd.split() +\n ['inspect', container])\n except subprocess.CalledProcessError:\n logger.critical(\"docker run failed with code %d\", retcode)\n subprocess.call(['docker', 'rm', '-f', container])\n sys.exit(1)\n outjson = json.loads(out.decode('utf-8'))\n if (outjson[0][\"State\"][\"Running\"] is not False or\n outjson[0][\"State\"][\"Paused\"] is not False):\n logger.error(\"Invalid container state after execution:\\n%s\",\n json.dumps(outjson[0][\"State\"]))\n retcode = outjson[0][\"State\"][\"ExitCode\"]\n stderr.write(\"\\n*** Command finished, status: %d\\n\" % retcode)\n\n # Commit to create new image\n new_image = make_unique_name(b'reprounzip_image_')\n logger.info(\"Committing container %s to image %s\",\n container.decode('ascii'), new_image.decode('ascii'))\n subprocess.check_call(args.docker_cmd.split() +\n ['commit', container, new_image])\n\n # Update image name\n unpacked_info['current_image'] = new_image\n write_dict(target, unpacked_info)\n\n # Remove the container\n logger.info(\"Destroying container %s\", container.decode('ascii'))\n retcode = subprocess.call(args.docker_cmd.split() + ['rm', container])\n if retcode != 0:\n logger.error(\"Error deleting container %s\", container.decode('ascii'))\n\n # Untag previous image, unless it is the initial_image\n if image != unpacked_info['initial_image']:\n logger.info(\"Untagging previous image %s\", image.decode('ascii'))\n subprocess.check_call(args.docker_cmd.split() + ['rmi', image])\n\n # Update input file status\n metadata_update_run(config, unpacked_info, selected_runs)\n write_dict(target, unpacked_info)\n\n signals.post_run(target=target, retcode=retcode)\n\n\nclass ContainerUploader(FileUploader):\n def __init__(self, target, input_files, files, unpacked_info,\n docker_cmd=['docker']):\n self.unpacked_info = unpacked_info\n self.docker_cmd = docker_cmd\n config = load_config(target / 'config.yml', True)\n if config.runs:\n first_run = config.runs[0]\n self.default_ownership = (\n first_run.get('uid'),\n first_run.get('gid'),\n )\n else:\n self.default_ownership = None, None\n\n FileUploader.__init__(self, target, input_files, files)\n\n def prepare_upload(self, files):\n if 'current_image' not in self.unpacked_info:\n stderr.write(\"Image doesn't exist yet, have you run \"\n \"setup/build?\\n\")\n sys.exit(1)\n\n self.build_directory = Path.tempdir(prefix='reprozip_build_')\n self.docker_copy = []\n\n def upload_file(self, local_path, input_path):\n stem, ext = local_path.stem, local_path.ext\n name = local_path.name\n nb = 0\n while (self.build_directory / name).exists():\n nb += 1\n name = stem + ('_%d' % nb).encode('ascii') + ext\n name = Path(name)\n local_path.copyfile(self.build_directory / name)\n logger.info(\"Copied file %s to %s\", local_path, name)\n self.docker_copy.append((name, input_path))\n\n def finalize(self):\n if not self.docker_copy:\n self.build_directory.rmtree()\n return\n\n from_image = self.unpacked_info['current_image']\n\n with self.build_directory.open('w', 'Dockerfile',\n encoding='utf-8',\n newline='\\n') as dockerfile:\n dockerfile.write('FROM %s\\n\\n' % from_image.decode('ascii'))\n for src, target in self.docker_copy:\n # FIXME : spaces in filenames will probably break Docker\n dockerfile.write(\n 'COPY \\\\\\n %s \\\\\\n %s\\n' % (\n shell_escape(unicode_(src)),\n shell_escape(unicode_(target))))\n\n for src, target in self.docker_copy:\n uid = gid = None\n\n # Keep permissions if the file is already in there\n tar = tarfile.open(str(self.target / 'data.tgz'), 'r:*')\n try:\n info = tar.getmember(str(\n join_root(PosixPath(b'DATA'), target)\n ))\n uid, gid = info.uid, info.gid\n except KeyError:\n pass\n\n # Otherwise default on the first run's UID/GID\n if uid is None:\n uid, gid = self.default_ownership\n\n # Lastly, use 1000\n if uid is None:\n uid = gid = 1000\n\n dockerfile.write(\n 'RUN [\"/busybox\", \"chown\", \"%d:%d\", %s]\\n' % (\n uid, gid, json.dumps(unicode_(target)),\n )\n )\n\n image = make_unique_name(b'reprounzip_image_')\n retcode = subprocess.call(self.docker_cmd +\n ['build', '-t', image, '.'],\n cwd=self.build_directory.path)\n if retcode != 0:\n logger.critical(\"docker build failed with code %d\", retcode)\n sys.exit(1)\n else:\n logger.info(\"New image created: %s\", image.decode('ascii'))\n if from_image != self.unpacked_info['initial_image']:\n logger.info(\"Untagging previous image %s\",\n from_image.decode('ascii'))\n retcode = subprocess.call(self.docker_cmd +\n ['rmi', from_image])\n if retcode != 0:\n logger.warning(\"Can't remove previous image, docker \"\n \"returned %d\", retcode)\n self.unpacked_info['current_image'] = image\n write_dict(self.target, self.unpacked_info)\n\n self.build_directory.rmtree()\n\n\n@target_must_exist\ndef docker_upload(args):\n \"\"\"Replaces an input file in the container.\n \"\"\"\n target = Path(args.target[0])\n files = args.file\n unpacked_info = read_dict(target)\n input_files = unpacked_info.setdefault('input_files', {})\n\n try:\n ContainerUploader(target, input_files, files, unpacked_info,\n docker_cmd=args.docker_cmd.split())\n finally:\n write_dict(target, unpacked_info)\n\n\nclass ContainerDownloader(FileDownloader):\n def __init__(self, target, files, image, all_=False,\n docker_cmd=['docker']):\n self.image = image\n self.docker_cmd = docker_cmd\n FileDownloader.__init__(self, target, files, all_=all_)\n\n def prepare_download(self, files):\n # Create a container from the image\n self.container = make_unique_name(b'reprounzip_dl_')\n logger.info(\"Creating container %s\", self.container.decode('ascii'))\n subprocess.check_call(self.docker_cmd +\n ['create',\n b'--name=' + self.container,\n self.image])\n\n def download(self, remote_path, local_path):\n # Docker copies to a file in the specified directory, cannot just take\n # a file name (#4272)\n tmpdir = Path.tempdir(prefix='reprozip_docker_output_')\n try:\n ret = subprocess.call(self.docker_cmd +\n ['cp',\n self.container + b':' + remote_path.path,\n tmpdir.path])\n if ret != 0:\n logger.critical(\"Can't get output file: %s\", remote_path)\n return False\n (tmpdir / remote_path.name).copyfile(local_path)\n finally:\n tmpdir.rmtree()\n return True\n\n def finalize(self):\n logger.info(\"Removing container %s\", self.container.decode('ascii'))\n retcode = subprocess.call(self.docker_cmd + ['rm', self.container])\n if retcode != 0:\n logger.warning(\"Can't remove temporary container, docker \"\n \"returned %d\", retcode)\n\n\n@target_must_exist\ndef docker_download(args):\n \"\"\"Gets an output file out of the container.\n \"\"\"\n target = Path(args.target[0])\n files = args.file\n unpacked_info = read_dict(target)\n\n if 'current_image' not in unpacked_info:\n logger.critical(\"Image doesn't exist yet, have you run setup/build?\")\n sys.exit(1)\n image = unpacked_info['current_image']\n logger.debug(\"Downloading from image %s\", image.decode('ascii'))\n\n ContainerDownloader(target, files, image,\n all_=args.all, docker_cmd=args.docker_cmd.split())\n\n\n@target_must_exist\ndef docker_destroy_docker(args):\n \"\"\"Destroys the container and images.\n \"\"\"\n target = Path(args.target[0])\n unpacked_info = read_dict(target)\n if 'initial_image' not in unpacked_info:\n logger.critical(\"Image not created\")\n sys.exit(1)\n\n initial_image = unpacked_info.pop('initial_image')\n\n if 'current_image' in unpacked_info:\n image = unpacked_info.pop('current_image')\n if image != initial_image:\n logger.info(\"Destroying image %s...\", image.decode('ascii'))\n retcode = subprocess.call(args.docker_cmd.split() + ['rmi', image])\n if retcode != 0:\n logger.error(\"Error deleting image %s\", image.decode('ascii'))\n\n logger.info(\"Destroying image %s...\", initial_image.decode('ascii'))\n retcode = subprocess.call(args.docker_cmd.split() + ['rmi', initial_image])\n if retcode != 0:\n logger.error(\"Error deleting image %s\", initial_image.decode('ascii'))\n\n\n@target_must_exist\ndef docker_destroy_dir(args):\n \"\"\"Destroys the directory.\n \"\"\"\n target = Path(args.target[0])\n read_dict(target)\n\n logger.info(\"Removing directory %s...\", target)\n signals.pre_destroy(target=target)\n target.rmtree()\n signals.post_destroy(target=target)\n\n\ndef test_has_docker(pack, **kwargs):\n \"\"\"Compatibility test: has docker (ok) or not (maybe).\n \"\"\"\n pathlist = os.environ['PATH'].split(os.pathsep) + ['.']\n pathexts = os.environ.get('PATHEXT', '').split(os.pathsep)\n for path in pathlist:\n for ext in pathexts:\n fullpath = os.path.join(path, 'docker') + ext\n if os.path.isfile(fullpath):\n return COMPAT_OK\n return COMPAT_MAYBE, \"docker not found in PATH\"\n\n\ndef setup(parser, **kwargs):\n \"\"\"Runs the experiment in a Docker container\n\n You will need Docker to be installed on your machine if you want to run the\n experiment.\n\n setup setup/create creates Dockerfile (needs the pack filename)\n setup/build builds the container from the Dockerfile\n reset resets the Docker image to the initial state (just\n after setup)\n upload replaces input files in the container\n (without arguments, lists input files)\n run runs the experiment in the container\n download gets output files from the container\n (without arguments, lists output files)\n destroy destroy/docker destroys the container and associated images\n destroy/dir removes the unpacked directory\n\n For example:\n\n $ reprounzip docker setup mypack.rpz experiment; cd experiment\n $ reprounzip docker run .\n $ reprounzip docker download . results:/home/user/theresults.txt\n $ cd ..; reprounzip docker destroy experiment\n\n Upload specifications are either:\n :input_id restores the original input file from the pack\n filename:input_id replaces the input file with the specified local\n file\n\n Download specifications are either:\n output_id: print the output file to stdout\n output_id:filename extracts the output file to the corresponding local\n path\n \"\"\"\n parser.add_argument('--docker-cmd', action='store', default='docker',\n help=\"Change the Docker command; split on spaces\")\n\n subparsers = parser.add_subparsers(title=\"actions\",\n metavar='', help=argparse.SUPPRESS)\n\n def add_opt_general(opts):\n opts.add_argument('target', nargs=1, help=\"Experiment directory\")\n\n # Common between setup and setup/create\n def add_opt_setup_create(opts):\n opts.add_argument('pack', nargs=1, help=\"Pack to extract\")\n opts.add_argument('--base-image', nargs=1, help=\"Base image to use\")\n opts.add_argument('--distribution', nargs=1,\n help=\"Distribution used in the base image (for \"\n \"package installer selection)\")\n opts.add_argument('--install-pkgs', action='store_true',\n default=False,\n help=\"Install packages rather than extracting \"\n \"them from RPZ file\")\n opts.add_argument('--unpack-pkgs', action='store_false',\n default=False, dest='install_pkgs',\n help=argparse.SUPPRESS)\n\n # Common between setup and setup/build\n def add_opt_setup_build(opts):\n opts.add_argument('--image-name', nargs=1,\n help=\"Name of the image to create (by default, a \"\n \"random name beginning with \"\n \"'reprounzip_image_' is generated)\")\n add_raw_docker_option(opts)\n\n # --docker-option\n def add_raw_docker_option(opts):\n opts.add_argument('--docker-option', action='append',\n default=[],\n help=\"Argument passed to Docker directly; may be \"\n \"specified multiple times\")\n\n parser_setup_create = subparsers.add_parser('setup/create')\n add_opt_setup_create(parser_setup_create)\n add_opt_general(parser_setup_create)\n parser_setup_create.set_defaults(func=docker_setup_create)\n\n # setup/build\n parser_setup_build = subparsers.add_parser('setup/build')\n add_opt_general(parser_setup_build)\n add_opt_setup_build(parser_setup_build)\n parser_setup_build.set_defaults(func=docker_setup_build)\n\n # setup\n parser_setup = subparsers.add_parser('setup')\n add_opt_setup_create(parser_setup)\n add_opt_setup_build(parser_setup)\n add_opt_general(parser_setup)\n parser_setup.set_defaults(func=docker_setup)\n\n # reset\n parser_reset = subparsers.add_parser('reset')\n add_opt_general(parser_reset)\n parser_reset.set_defaults(func=docker_reset)\n\n # upload\n parser_upload = subparsers.add_parser('upload')\n add_opt_general(parser_upload)\n parser_upload.add_argument('file', nargs=argparse.ZERO_OR_MORE,\n help=\":\")\n parser_upload.set_defaults(func=docker_upload)\n\n # run\n parser_run = subparsers.add_parser('run')\n add_opt_general(parser_run)\n parser_run.add_argument('run', default=None, nargs=argparse.OPTIONAL)\n parser_run.add_argument('--cmdline', nargs=argparse.REMAINDER,\n help=\"Command line to run\")\n parser_run.add_argument('--expose-port', '-p', action='append', default=[],\n help=\"Expose a network port, \"\n \"host[:experiment[/proto]]. Example: 8000:80\")\n parser_run.add_argument('--enable-x11', action='store_true', default=False,\n dest='x11',\n help=\"Enable X11 support (needs an X server on \"\n \"the host)\")\n parser_run.add_argument('--x11-display', dest='x11_display',\n help=\"Display number to use on the experiment \"\n \"side (change the host display with the \"\n \"DISPLAY environment variable)\")\n parser_run.add_argument(\n '--tunneled-x11', dest='tunneled_x11',\n action='store_true', default=False,\n help=\"Connect X11 to local machine from Docker container instead of \"\n \"trying to connect to this one (useful if the Docker machine has \"\n \"an X server or if a tunnel is used to access this one)\")\n parser_run.add_argument('-d', '--detach', action='store_true',\n help=\"Don't attach or commit the created \"\n \"container, just start it and leave it be\")\n add_raw_docker_option(parser_run)\n add_environment_options(parser_run)\n parser_run.set_defaults(func=docker_run)\n\n # download\n parser_download = subparsers.add_parser('download')\n add_opt_general(parser_download)\n parser_download.add_argument('file', nargs=argparse.ZERO_OR_MORE,\n help=\"[:]\")\n parser_download.add_argument('--all', action='store_true',\n help=\"Download all output files to the \"\n \"current directory\")\n parser_download.set_defaults(func=docker_download)\n\n # destroy/docker\n parser_destroy_docker = subparsers.add_parser('destroy/docker')\n add_opt_general(parser_destroy_docker)\n parser_destroy_docker.set_defaults(func=docker_destroy_docker)\n\n # destroy/dir\n parser_destroy_dir = subparsers.add_parser('destroy/dir')\n add_opt_general(parser_destroy_dir)\n parser_destroy_dir.set_defaults(func=docker_destroy_dir)\n\n # destroy\n parser_destroy = subparsers.add_parser('destroy')\n add_opt_general(parser_destroy)\n parser_destroy.set_defaults(func=composite_action(docker_destroy_docker,\n docker_destroy_dir))\n\n return {'test_compatibility': test_has_docker}\n" }, { "path": "reprounzip-docker/setup.cfg", "content": "[bdist_wheel]\nuniversal = 1\n" }, { "path": "reprounzip-docker/setup.py", "content": "import io\nimport os\nfrom setuptools import setup\n\n\n# pip workaround\nos.chdir(os.path.abspath(os.path.dirname(__file__)))\n\n\n# Need to specify encoding for PY3, which has the worst unicode handling ever\nwith io.open('README.rst', encoding='utf-8') as fp:\n description = fp.read()\nsetup(name='reprounzip-docker',\n version='1.2',\n packages=['reprounzip', 'reprounzip.unpackers'],\n entry_points={\n 'reprounzip.unpackers': [\n 'docker = reprounzip.unpackers.docker:setup']},\n namespace_packages=['reprounzip', 'reprounzip.unpackers'],\n install_requires=[\n 'reprounzip>=1.1',\n 'rpaths>=0.8'],\n description=\"Allows the ReproZip unpacker to create Docker containers\",\n author=\"Remi Rampin, Fernando Chirigati, Dennis Shasha, Juliana Freire\",\n author_email='reprozip@nyu.edu',\n maintainer=\"Remi Rampin\",\n maintainer_email='remi@rampin.org',\n url='https://www.reprozip.org/',\n project_urls={\n 'Documentation': 'https://docs.reprozip.org/',\n 'Examples': 'https://examples.reprozip.org/',\n 'Source': 'https://github.com/VIDA-NYU/reprozip',\n 'Bug Tracker': 'https://github.com/VIDA-NYU/reprozip/issues',\n 'Chat': 'https://riot.im/app/#/room/#reprozip:matrix.org',\n 'Changelog':\n 'https://github.com/VIDA-NYU/reprozip/blob/1.x/CHANGELOG.md',\n },\n long_description=description,\n license='BSD-3-Clause',\n keywords=['reprozip', 'reprounzip', 'reproducibility', 'provenance',\n 'vida', 'nyu', 'docker'],\n classifiers=[\n 'Development Status :: 5 - Production/Stable',\n 'Intended Audience :: Science/Research',\n 'License :: OSI Approved :: BSD License',\n 'Programming Language :: Python :: 2.7',\n 'Programming Language :: Python :: 3',\n 'Topic :: Scientific/Engineering',\n 'Topic :: System :: Archiving'])\n" }, { "path": "reprounzip-qt/LICENSE.txt", "content": "Copyright (C) 2014, New York University\nAll rights reserved.\n\nRedistribution and use in source and binary forms, with or without\nmodification, are permitted provided that the following conditions are met:\n\n1. Redistributions of source code must retain the above copyright notice, this\n list of conditions and the following disclaimer.\n\n2. Redistributions in binary form must reproduce the above copyright notice,\n this list of conditions and the following disclaimer in the documentation\n and/or other materials provided with the distribution.\n\n3. Neither the name of the copyright holder nor the names of its contributors\n may be used to endorse or promote products derived from this software\n without specific prior written permission.\n\nTHIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \"AS IS\" AND\nANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED\nWARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE\nDISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE\nFOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL\nDAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR\nSERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER\nCAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,\nOR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE\nOF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n" }, { "path": "reprounzip-qt/MANIFEST.in", "content": "include README.rst\ninclude LICENSE.txt\n" }, { "path": "reprounzip-qt/README.rst", "content": "ReproZip project\n================\n\n`ReproZip `__ is a tool aimed at simplifying the process of creating reproducible experiments from command-line executions, a frequently-used common denominator in computational science. It tracks operating system calls and creates a bundle that contains all the binaries, files and dependencies required to run a given command on the author's computational environment (packing step). A reviewer can then extract the experiment in his environment to reproduce the results (unpacking step).\n\nreprounzip-qt\n-------------\n\nThis is a graphical user interface for the reprounzip package, allowing to unpack and run packed applications without having to use the command-line interface.\n\nPlease refer to `reprozip `__ and `reprounzip `_ for more information.\n\nAdditional Information\n----------------------\n\nFor more detailed information, please refer to our `website `_, as well as to our `documentation `_.\n\nReproZip is currently being developed at `NYU `_. The team includes:\n\n* `Fernando Chirigati `_\n* `Juliana Freire `_\n* `Remi Rampin `_\n* `Dennis Shasha `_\n* `Vicky Rampin `_\n" }, { "path": "reprounzip-qt/reprounzip_qt/__init__.py", "content": "__version__ = '1.2.1'\n" }, { "path": "reprounzip-qt/reprounzip_qt/gui/__init__.py", "content": "# Copyright (C) 2014 New York University\n# This file is part of ReproZip which is released under the Revised BSD License\n# See file LICENSE for full license details.\n\nfrom __future__ import division, print_function, unicode_literals\n\nimport logging\nimport os\nfrom qtpy import QtCore, QtWidgets\nimport shutil\nimport subprocess\nimport sys\nimport tempfile\nimport traceback\nimport usagestats\n\nimport reprounzip_qt\nfrom reprounzip_qt.gui.common import error_msg\nfrom reprounzip_qt.gui.unpack import UnpackTab\nfrom reprounzip_qt.gui.run import RunTab\nfrom reprounzip_qt.usage import record_usage, _usage_report as usage_report\n\n\nlogger = logging.getLogger('reprounzip_qt')\n\n\nclass Application(QtWidgets.QApplication):\n def __init__(self, argv):\n QtWidgets.QApplication.__init__(self, argv)\n self.first_window = None\n self.windows = set()\n\n def linux_try_register_default(self, window):\n rcpath = os.path.expanduser('~/.reprozip')\n rcname = 'rpuzqt-nodefault'\n if os.path.exists(os.path.join(rcpath, rcname)):\n logger.info(\"Registering application disabled\")\n return\n try:\n proc = subprocess.Popen(['xdg-mime', 'query', 'default',\n 'application/x-reprozip'],\n stdout=subprocess.PIPE,\n stderr=subprocess.PIPE)\n out, err = proc.communicate()\n registered = bool(out.strip())\n except OSError:\n record_usage(appregister='fail xdg-mime')\n logger.info(\"xdg-mime call failed, not registering application\")\n else:\n if not registered:\n r = QtWidgets.QMessageBox.question(\n window, \"Default application\",\n \"Do you want to set ReproUnzip as the default to open \"\n \".rpz files?\",\n QtWidgets.QMessageBox.Yes | QtWidgets.QMessageBox.No)\n if r == QtWidgets.QMessageBox.Yes:\n self.linux_register_default(window)\n elif r == QtWidgets.QMessageBox.No:\n record_usage(appregister='no')\n if not os.path.exists(rcpath):\n os.mkdir(rcpath)\n with open(os.path.join(rcpath, rcname), 'w') as fp:\n fp.write('1\\n')\n\n def linux_register_default(self, window):\n record_usage(appregister='yes')\n command = os.path.abspath(sys.argv[0])\n if not os.path.isfile(command):\n logger.error(\"Couldn't find argv[0] location!\")\n return\n dirname = tempfile.mkdtemp(prefix='reprozip_mime_')\n try:\n # Install x-reprozip mimetype\n logger.info(\"Installing application/x-reprozip mimetype for .rpz\")\n filename = os.path.join(dirname, 'nyuvida-reprozip.xml')\n with open(filename, 'w') as fp:\n fp.write('''\\\n\n\n \n ReproZip Package\n \n \n\n''')\n subprocess.check_call(['xdg-mime', 'install', filename])\n subprocess.check_call(['update-mime-database',\n os.path.join(os.environ['HOME'],\n '.local/share/mime')])\n\n # Install icon\n logger.info(\"Copying icon\")\n icon_dest_root = os.path.join(os.environ['HOME'],\n '.local/share/icons/hicolor')\n icon_dest_subdir = os.path.join(icon_dest_root, '48x48/mimetypes')\n icon_dest_file = os.path.join(icon_dest_subdir,\n 'application-x-reprozip.png')\n icon_src = os.path.join(os.path.dirname(reprounzip_qt.__file__),\n 'icon.png')\n if not os.path.exists(icon_dest_subdir):\n os.makedirs(icon_dest_subdir)\n shutil.copyfile(icon_src, icon_dest_file)\n subprocess.check_call(['update-icon-caches', icon_dest_root])\n\n # Install desktop file\n logger.info(\"Installing reprounzip.desktop file\")\n app_dir = os.path.join(os.environ['HOME'],\n '.local/share/applications')\n if not os.path.exists(app_dir):\n os.makedirs(app_dir)\n with open(os.path.join(app_dir, 'reprounzip.desktop'), 'w') as fp:\n fp.write('''\\\n[Desktop Entry]\nName=ReproUnzip\nExec={0} %f\nType=Application\nMimeType=application/x-reprozip\nIcon={1}\n'''.format(command, icon_dest_file))\n subprocess.check_call(['update-desktop-database', app_dir])\n\n logger.info(\"Application registered!\")\n except (OSError, subprocess.CalledProcessError):\n error_msg(window, \"Error setting default application\",\n 'error', traceback.format_exc())\n finally:\n shutil.rmtree(dirname)\n\n def ask_enable_usage_report(self):\n dialog = QtWidgets.QDialog()\n dialog.setWindowTitle(\"Anonymous usage statistics\")\n layout = QtWidgets.QVBoxLayout()\n layout.addWidget(QtWidgets.QLabel(\"Send anonymous usage reports to \"\n \"the developers?\"))\n dont_ask = QtWidgets.QCheckBox(\"Don't ask again\")\n layout.addWidget(dont_ask)\n buttons = QtWidgets.QDialogButtonBox(\n QtWidgets.QDialogButtonBox.Yes | QtWidgets.QDialogButtonBox.No)\n layout.addWidget(buttons)\n buttons.accepted.connect(dialog.accept)\n buttons.rejected.connect(dialog.reject)\n dialog.setLayout(layout)\n\n res = dialog.exec_()\n if res == QtWidgets.QDialog.Accepted:\n usage_report.enable_reporting()\n elif dont_ask.isChecked():\n usage_report.disable_reporting()\n\n def event(self, event):\n if event.type() == QtCore.QEvent.FileOpen:\n record_usage(fileopenevent=True)\n # Create new window for this RPZ\n window = ReprounzipUi(unpack=dict(package=event.file()))\n window.setVisible(True)\n self.windows.add(window)\n # Close first window if it exists\n if self.first_window and self.first_window.replaceable():\n self.first_window.close()\n self.first_window.deleteLater()\n self.first_window = None\n return True\n return QtWidgets.QApplication.event(self, event)\n\n def set_first_window(self, window):\n self.first_window = window\n self.windows.add(window)\n if sys.platform.startswith('linux'):\n self.linux_try_register_default(window)\n if usage_report.status is usagestats.Stats.UNSET:\n self.ask_enable_usage_report()\n\n\nclass ReprounzipUi(QtWidgets.QMainWindow):\n def __init__(self, unpack={}, run={}, tab=None, **kwargs):\n super(ReprounzipUi, self).__init__(**kwargs)\n\n self.setWindowTitle(\"ReproUnzip\")\n\n self.tabs = QtWidgets.QTabWidget()\n self.tabs.addTab(UnpackTab(**unpack), \"Open package\")\n self.tabs.addTab(RunTab(**run), \"Run unpacked experiment\")\n self.tabs.widget(0).unpacked.connect(self._unpacked)\n if tab is not None:\n self.tabs.setCurrentIndex(tab)\n self.setCentralWidget(self.tabs)\n\n def _unpacked(self, directory, root):\n self.tabs.widget(1).set_directory(directory, root=root)\n self.tabs.setCurrentIndex(1)\n\n def closeEvent(self, event):\n if self.tabs.widget(1).should_exit():\n Application.instance().windows.discard(self)\n event.accept()\n else:\n event.ignore()\n\n def replaceable(self):\n return (self.tabs.widget(0).replaceable() and\n self.tabs.widget(1).replaceable())\n" }, { "path": "reprounzip-qt/reprounzip_qt/gui/common.py", "content": "# Copyright (C) 2014 New York University\n# This file is part of ReproZip which is released under the Revised BSD License\n# See file LICENSE for full license details.\n\nfrom __future__ import division, print_function, unicode_literals\n\nfrom qtpy import QtCore, QtWidgets\nimport re\n\n\ndef error_msg(parent, message, severity, details=None):\n if severity == 'information':\n icon = QtWidgets.QMessageBox.Information\n elif severity == 'warning':\n icon = QtWidgets.QMessageBox.Warning\n else:\n icon = QtWidgets.QMessageBox.Critical\n\n msgbox = QtWidgets.QMessageBox(icon, \"Error\", message,\n QtWidgets.QMessageBox.Ok, parent,\n detailedText=details,\n textFormat=QtCore.Qt.PlainText)\n msgbox.exec_()\n\n\ndef handle_error(parent, result):\n if result in (True, False):\n return result\n else:\n error_msg(parent, *result)\n return False\n\n\nclass ResizableStack(QtWidgets.QStackedWidget):\n # See http://stackoverflow.com/a/14485901/711380\n def __init__(self, **kwargs):\n super(ResizableStack, self).__init__(**kwargs)\n\n self.currentChanged[int].connect(self._current_changed)\n\n def addWidget(self, widget):\n widget.setSizePolicy(QtWidgets.QSizePolicy.Ignored,\n QtWidgets.QSizePolicy.Ignored)\n super(ResizableStack, self).addWidget(widget)\n\n def _current_changed(self, idx):\n widget = self.widget(idx)\n widget.setSizePolicy(QtWidgets.QSizePolicy.Expanding,\n QtWidgets.QSizePolicy.Expanding)\n widget.adjustSize()\n self.adjustSize()\n\n\nclass ROOT(object):\n OPTION_TO_INDEX = {None: 0, 'sudo': 1, 'su': 2}\n INDEX_TO_OPTION = {0: None, 1: 'sudo', 2: 'su'}\n TEXT = [\"no\", \"with sudo\", \"with su\"]\n\n\n_port_re = re.compile('^(?:([0-9]+):)?([0-9]+)(?:/([a-z]+))?$')\n\n\ndef parse_ports(string, widget):\n ports = []\n\n for port in string.split():\n port = port.strip()\n if not port:\n continue\n\n m = _port_re.match(port)\n if m is None:\n error_msg(widget, \"Invalid port specification: '%s'\" % port,\n 'warning')\n return None\n else:\n host, experiment, proto = m.groups()\n if not host:\n host = experiment\n if not proto:\n proto = 'tcp'\n ports.append((int(host), int(experiment), proto))\n\n return ports\n" }, { "path": "reprounzip-qt/reprounzip_qt/gui/run.py", "content": "# Copyright (C) 2014 New York University\n# This file is part of ReproZip which is released under the Revised BSD License\n# See file LICENSE for full license details.\n\nfrom __future__ import division, print_function, unicode_literals\n\nfrom qtpy import QtCore, QtWidgets\nimport yaml\n\nimport reprounzip_qt.reprounzip_interface as reprounzip\nfrom reprounzip_qt.gui.common import ROOT, ResizableStack, handle_error, \\\n error_msg, parse_ports\nfrom reprounzip_qt.usage import record_usage\n\n\nclass RunOptions(QtWidgets.QWidget):\n x11 = None\n\n def __init__(self):\n super(RunOptions, self).__init__()\n self.setLayout(QtWidgets.QGridLayout())\n\n def add_row(self, label, widget):\n layout = self.layout()\n row = layout.rowCount()\n layout.addWidget(QtWidgets.QLabel(label), row, 0)\n layout.addWidget(widget, row, 1)\n\n def add_row_layout(self, label, rowlayout):\n layout = self.layout()\n row = layout.rowCount()\n layout.addWidget(QtWidgets.QLabel(label), row, 0)\n layout.addLayout(rowlayout, row, 1)\n\n def add_x11(self):\n self.x11 = QtWidgets.QCheckBox(\"enabled\", checked=False)\n self.add_row(\"X11 display:\", self.x11)\n\n def options(self):\n options = {'args': []}\n\n if self.x11 is not None and self.x11.isChecked():\n options['args'].append('--enable-x11')\n\n return options\n\n\nclass DirectoryOptions(RunOptions):\n def __init__(self):\n super(DirectoryOptions, self).__init__()\n self.add_x11()\n\n\nclass ChrootOptions(RunOptions):\n def __init__(self):\n super(ChrootOptions, self).__init__()\n self.add_x11()\n\n\nclass DockerOptions(RunOptions):\n def __init__(self):\n super(DockerOptions, self).__init__()\n\n self.x11 = QtWidgets.QCheckBox(\"enabled\", checked=False)\n self.tunneled_x11 = QtWidgets.QCheckBox(\"use tunnel\", checked=False)\n row = QtWidgets.QHBoxLayout()\n row.addWidget(self.x11)\n row.addWidget(self.tunneled_x11)\n row.addStretch(1)\n self.add_row_layout(\"X11 display:\", row)\n\n self.detach = QtWidgets.QCheckBox(\"start background container and \"\n \"leave it running\",\n checked=False)\n self.add_row(\"Detach:\", self.detach)\n\n self.raw_options = QtWidgets.QLineEdit('')\n self.add_row(\"Raw Docker options:\", self.raw_options)\n\n self.ports = QtWidgets.QLineEdit(\n '',\n toolTip=\"Space-separated host:guest port mappings\")\n self.add_row(\"Expose ports:\", self.ports)\n\n def options(self):\n options = super(DockerOptions, self).options()\n\n if self.tunneled_x11.isChecked():\n options['args'].append('--tunneled-x11')\n record_usage(docker_tunneled_x11=True)\n\n if self.detach.isChecked():\n options['args'].append('--detach')\n record_usage(docker_detach=True)\n\n nb_raw = 0\n for opt in self.raw_options.text().split():\n opt = opt.strip()\n if opt:\n nb_raw += 1\n options['args'].append('--docker-option=%s' % opt)\n if nb_raw:\n record_usage(docker_raw_options=nb_raw)\n\n ports = parse_ports(self.ports.text(), self)\n if ports is None:\n return None\n for host, container, proto in ports:\n options['args'].extend(\n ['--docker-option=-p',\n '--docker-option=%s:%s/%s' % (host, container, proto)])\n record_usage(docker_run_port_fwd=bool(ports))\n\n return options\n\n\nclass VagrantOptions(RunOptions):\n def __init__(self):\n super(VagrantOptions, self).__init__()\n self.add_x11()\n\n self.ports = QtWidgets.QLineEdit(\n '',\n toolTip=\"Space-separated host:guest port mappings\")\n self.add_row(\"Expose ports:\", self.ports)\n\n def options(self):\n options = super(VagrantOptions, self).options()\n\n ports = parse_ports(self.ports.text(), self)\n if ports is None:\n return None\n for host, container, proto in parse_ports(self.ports.text(), self):\n options['args'].append('--expose-port=%s:%s/%s' %\n (host, container, proto))\n record_usage(vagrant_run_port_fwd=bool(ports))\n\n return options\n\n\nclass FilesManager(QtWidgets.QDialog):\n def __init__(self, directory, unpacker=None, root=None, **kwargs):\n super(FilesManager, self).__init__(**kwargs)\n self.directory = directory\n self.unpacker = unpacker\n self.root = root\n\n layout = QtWidgets.QHBoxLayout()\n\n self.files_widget = QtWidgets.QListWidget(\n selectionMode=QtWidgets.QListWidget.SingleSelection)\n self.files_widget.itemSelectionChanged.connect(self._file_changed)\n layout.addWidget(self.files_widget)\n\n right_layout = QtWidgets.QGridLayout()\n right_layout.addWidget(QtWidgets.QLabel(\"name:\"), 0, 0)\n self.f_name = QtWidgets.QLineEdit('', readOnly=True)\n right_layout.addWidget(self.f_name, 0, 1)\n right_layout.addWidget(QtWidgets.QLabel(\"Path:\"), 1, 0)\n self.f_path = QtWidgets.QLineEdit('', readOnly=True)\n right_layout.addWidget(self.f_path, 1, 1)\n right_layout.addWidget(QtWidgets.QLabel(\"Current:\"), 2, 0)\n self.f_status = QtWidgets.QLineEdit('', readOnly=True)\n right_layout.addWidget(self.f_status, 2, 1)\n self.b_upload = QtWidgets.QPushButton(\"Upload a replacement\",\n enabled=False)\n self.b_upload.clicked.connect(self._upload)\n right_layout.addWidget(self.b_upload, 3, 0, 1, 2)\n self.b_download = QtWidgets.QPushButton(\"Download to disk\",\n enabled=False)\n self.b_download.clicked.connect(self._download)\n right_layout.addWidget(self.b_download, 4, 0, 1, 2)\n self.b_reset = QtWidgets.QPushButton(\"Reset file\", enabled=False)\n self.b_reset.clicked.connect(self._reset)\n right_layout.addWidget(self.b_reset, 5, 0, 1, 2)\n right_layout.setRowStretch(6, 1)\n layout.addLayout(right_layout)\n\n self.setLayout(layout)\n\n self.files_status = reprounzip.FilesStatus(directory)\n\n for file_status in self.files_status:\n text = \"[%s%s] %s\" % ((\"I\" if file_status.is_input else ''),\n (\"O\" if file_status.is_output else ''),\n file_status.name)\n self.files_widget.addItem(text)\n record_usage(iofiles=self.files_widget.count())\n\n def _file_changed(self):\n selected = [i.row() for i in self.files_widget.selectedIndexes()]\n if not selected:\n self.f_name.setText('')\n self.f_path.setText('')\n self.f_status.setText('')\n self.b_upload.setEnabled(False)\n self.b_download.setEnabled(False)\n self.b_reset.setEnabled(False)\n else:\n file_status = self.files_status[selected[0]]\n self.b_upload.setEnabled(True)\n self.b_download.setEnabled(True)\n self.b_reset.setEnabled(False)\n self.f_name.setText(file_status.name)\n self.f_path.setText(file_status.path)\n self.f_status.setEnabled(False)\n if file_status.assigned is None:\n self.f_status.setText(\"(original)\")\n self.b_reset.setEnabled(False)\n elif file_status.assigned is False:\n self.f_status.setText(\"(not created)\")\n elif file_status.assigned is True:\n self.f_status.setText(\"(generated)\")\n else:\n caption = file_status.assigned\n if isinstance(caption, bytes):\n caption = caption.decode('utf-8', 'replace')\n self.f_status.setText(caption)\n self.f_status.setEnabled(True)\n\n def _upload(self):\n selected = self.files_widget.selectedIndexes()[0].row()\n file_status = self.files_status[selected]\n picked, _ = QtWidgets.QFileDialog.getOpenFileName(\n self, \"Pick file to upload\",\n QtCore.QDir.currentPath())\n if picked:\n record_usage(file_upload=True)\n handle_error(self, reprounzip.upload(\n self.directory, file_status.name, picked,\n unpacker=self.unpacker, root=self.root))\n self._file_changed()\n\n def _download(self):\n selected = self.files_widget.selectedIndexes()[0].row()\n file_status = self.files_status[selected]\n picked, _ = QtWidgets.QFileDialog.getSaveFileName(\n self, \"Pick destination\",\n QtCore.QDir.currentPath() + '/' + file_status.name)\n if picked:\n record_usage(file_download=True)\n handle_error(self, reprounzip.download(\n self.directory, file_status.name, picked,\n unpacker=self.unpacker, root=self.root))\n self._file_changed()\n\n def _reset(self):\n selected = self.files_widget.selectedIndexes()[0].row()\n file_status = self.files_status[selected]\n record_usage(file_reset=True)\n handle_error(self, reprounzip.upload(\n self.directory, file_status.name, None,\n unpacker=self.unpacker, root=self.root))\n self._file_changed()\n\n\nclass RunTab(QtWidgets.QWidget):\n \"\"\"The main window, that allows you to run/change an unpacked experiment.\n \"\"\"\n UNPACKERS = [\n ('directory', DirectoryOptions),\n ('chroot', ChrootOptions),\n ('docker', DockerOptions),\n ('vagrant', VagrantOptions),\n ]\n\n directory = None\n unpacker = None\n\n def __init__(self, unpacked_directory='', **kwargs):\n super(RunTab, self).__init__(**kwargs)\n\n layout = QtWidgets.QGridLayout()\n layout.addWidget(QtWidgets.QLabel(\"Experiment directory:\"), 0, 0)\n self.directory_widget = QtWidgets.QLineEdit(unpacked_directory)\n self.directory_widget.editingFinished.connect(self._directory_changed)\n layout.addWidget(self.directory_widget, 0, 1)\n browse = QtWidgets.QPushButton(\"Browse\")\n browse.clicked.connect(self._browse)\n layout.addWidget(browse, 0, 2)\n\n layout.addWidget(QtWidgets.QLabel(\"Unpacker:\"), 1, 0,\n QtCore.Qt.AlignTop)\n self.unpacker_widget = QtWidgets.QLabel(\"-\")\n layout.addWidget(self.unpacker_widget, 1, 1, 1, 2)\n\n layout.addWidget(QtWidgets.QLabel(\"Input/output files:\"), 2, 0,\n QtCore.Qt.AlignTop)\n self.files_button = QtWidgets.QPushButton(\"Manage files\",\n enabled=False)\n self.files_button.clicked.connect(self._open_files_manager)\n layout.addWidget(self.files_button, 2, 1, 1, 2)\n\n layout.addWidget(QtWidgets.QLabel(\"Runs:\"), 3, 0,\n QtCore.Qt.AlignTop)\n self.runs_widget = QtWidgets.QListWidget(\n selectionMode=QtWidgets.QListWidget.MultiSelection)\n layout.addWidget(self.runs_widget, 3, 1, 3, 1)\n select_all = QtWidgets.QPushButton(\"Select All\")\n select_all.clicked.connect(self.runs_widget.selectAll)\n layout.addWidget(select_all, 3, 2)\n deselect_all = QtWidgets.QPushButton(\"Deselect All\")\n deselect_all.clicked.connect(self.runs_widget.clearSelection)\n layout.addWidget(deselect_all, 4, 2)\n\n layout.addWidget(QtWidgets.QLabel(\"Elevate privileges:\"), 6, 0)\n self.root = QtWidgets.QComboBox(editable=False)\n self.root.addItems(ROOT.TEXT)\n layout.addWidget(self.root, 6, 1, 1, 2)\n\n layout.addWidget(QtWidgets.QLabel(\"Jupyter integration:\"),\n 7, 0)\n self.run_jupyter_notebook = QtWidgets.QCheckBox(\"Run notebook server\",\n checked=False,\n enabled=False)\n layout.addWidget(self.run_jupyter_notebook, 7, 1, 1, 2)\n\n group = QtWidgets.QGroupBox(title=\"Unpacker options\")\n group_layout = QtWidgets.QVBoxLayout()\n self.unpacker_options = ResizableStack()\n scroll = QtWidgets.QScrollArea(widgetResizable=True)\n scroll.setWidget(self.unpacker_options)\n group_layout.addWidget(scroll)\n group.setLayout(group_layout)\n layout.addWidget(group, 8, 0, 1, 3)\n layout.setRowStretch(8, 1)\n\n for i, (name, WidgetClass) in enumerate(self.UNPACKERS):\n widget = WidgetClass()\n self.unpacker_options.addWidget(widget)\n\n self.unpacker_options.addWidget(\n QtWidgets.QLabel(\"Select a directory to display options...\"))\n self.unpacker_options.setCurrentIndex(len(self.UNPACKERS))\n\n buttons = QtWidgets.QHBoxLayout()\n buttons.addStretch(1)\n self.run_widget = QtWidgets.QPushButton(\"Run experiment\")\n self.run_widget.clicked.connect(self._run)\n buttons.addWidget(self.run_widget)\n self.destroy_widget = QtWidgets.QPushButton(\"Destroy unpacked \"\n \"experiment\")\n self.destroy_widget.clicked.connect(self._destroy)\n buttons.addWidget(self.destroy_widget)\n layout.addLayout(buttons, 9, 0, 1, 3)\n\n self.setLayout(layout)\n\n self._directory_changed()\n\n def _browse(self):\n picked = QtWidgets.QFileDialog.getExistingDirectory(\n self, \"Pick directory\",\n QtCore.QDir.currentPath())\n if picked:\n record_usage(browse_unpacked=True)\n self.directory_widget.setText(picked)\n self._directory_changed()\n\n def _directory_changed(self, new_dir=None, force=False):\n if not force and self.directory_widget.text() == self.directory:\n return\n self.directory = self.directory_widget.text()\n\n unpacker = reprounzip.check_directory(self.directory)\n\n self.run_jupyter_notebook.setChecked(False)\n self.run_jupyter_notebook.setEnabled(False)\n\n self.runs_widget.clear()\n if unpacker is not None:\n with open(self.directory + '/config.yml') as fp:\n self.config = yaml.safe_load(fp)\n self.run_widget.setEnabled(True)\n self.destroy_widget.setEnabled(True)\n self.files_button.setEnabled(True)\n self.unpacker = unpacker\n self.unpacker_widget.setText(unpacker)\n for run in self.config['runs']:\n self.runs_widget.addItem(' '.join(reprounzip.shell_escape(arg)\n for arg in run['argv']))\n self.runs_widget.selectAll()\n self.unpacker_options.setCurrentIndex(\n dict((n, i) for i, (n, w) in enumerate(self.UNPACKERS))\n .get(unpacker, 4))\n\n if (unpacker == 'docker' and\n reprounzip.find_command('reprozip-jupyter') is not None and\n reprounzip.is_jupyter(self.directory)):\n self.run_jupyter_notebook.setEnabled(True)\n self.run_jupyter_notebook.setChecked(True)\n else:\n self.run_widget.setEnabled(False)\n self.destroy_widget.setEnabled(False)\n self.files_button.setEnabled(False)\n self.unpacker = None\n self.unpacker_widget.setText(\"-\")\n self.unpacker_options.setCurrentIndex(len(self.UNPACKERS))\n\n def _run(self):\n options = self.unpacker_options.currentWidget().options()\n if options is None:\n return\n runs = sorted(i.row() for i in self.runs_widget.selectedIndexes())\n if not runs:\n error_msg(self, \"No run selected\", 'warning')\n return\n record_usage(run='%d/%d' % (len(runs), self.runs_widget.count()))\n handle_error(self, reprounzip.run(\n self.directory, runs=runs,\n unpacker=self.unpacker,\n root=ROOT.INDEX_TO_OPTION[self.root.currentIndex()],\n jupyter=self.run_jupyter_notebook.isChecked(),\n **options))\n\n def _destroy(self):\n handle_error(self, reprounzip.destroy(\n self.directory, unpacker=self.unpacker,\n root=ROOT.INDEX_TO_OPTION[self.root.currentIndex()]))\n self._directory_changed(force=True)\n\n def _open_files_manager(self):\n manager = FilesManager(\n parent=self,\n directory=self.directory_widget.text(),\n unpacker=self.unpacker,\n root=ROOT.INDEX_TO_OPTION[self.root.currentIndex()])\n manager.exec_()\n\n def set_directory(self, directory, root=None):\n self.root.setCurrentIndex(ROOT.OPTION_TO_INDEX[root])\n self.directory_widget.setText(directory)\n self._directory_changed(force=True)\n\n def should_exit(self):\n if self.unpacker:\n r = QtWidgets.QMessageBox.question(\n self, \"Close Confirmation\",\n \"The experiment is still unpacked with '%s'. Are you sure you \"\n \"want to exit without removing it?\" % self.unpacker,\n QtWidgets.QMessageBox.Yes | QtWidgets.QMessageBox.No)\n if r == QtWidgets.QMessageBox.Yes:\n record_usage(leave_unpacked=True)\n return True\n else:\n return False\n else:\n return True\n\n def replaceable(self):\n return not self.unpacker\n" }, { "path": "reprounzip-qt/reprounzip_qt/gui/unpack.py", "content": "# Copyright (C) 2014 New York University\n# This file is part of ReproZip which is released under the Revised BSD License\n# See file LICENSE for full license details.\n\nfrom __future__ import division, print_function, unicode_literals\n\nimport os\nfrom qtpy import QtCore, QtWidgets\nimport subprocess\nimport sys\n\nimport reprounzip_qt.reprounzip_interface as reprounzip\nfrom reprounzip_qt.gui.common import ROOT, ResizableStack, \\\n handle_error, error_msg, parse_ports\nfrom reprounzip_qt.usage import record_usage\n\n\nclass UnpackerOptions(QtWidgets.QWidget):\n def __init__(self):\n super(UnpackerOptions, self).__init__()\n self.setLayout(QtWidgets.QGridLayout())\n\n def add_row(self, label, widget):\n layout = self.layout()\n row = layout.rowCount()\n layout.addWidget(QtWidgets.QLabel(label), row, 0)\n layout.addWidget(widget, row, 1)\n\n def options(self):\n return {'args': []}\n\n @staticmethod\n def available():\n return True\n\n\nclass DirectoryOptions(UnpackerOptions):\n def __init__(self):\n super(DirectoryOptions, self).__init__()\n self.layout().addWidget(\n QtWidgets.QLabel(\"(directory unpacker has no option)\"),\n 0, 0, 1, 2)\n\n @staticmethod\n def available():\n return sys.platform.startswith('linux')\n\n\nclass ChrootOptions(UnpackerOptions):\n def __init__(self):\n super(ChrootOptions, self).__init__()\n\n self.root = QtWidgets.QComboBox(editable=False)\n self.root.addItems(ROOT.TEXT)\n self.add_row(\"Elevate privileges:\", self.root)\n\n self.preserve_owner = QtWidgets.QCheckBox(\"enabled\", tristate=True)\n self.preserve_owner.setCheckState(QtCore.Qt.PartiallyChecked)\n self.add_row(\"Preserve file ownership:\", self.preserve_owner)\n\n self.magic_dirs = QtWidgets.QCheckBox(\n \"mount /dev and /proc inside the chroot\", tristate=True)\n self.magic_dirs.setCheckState(QtCore.Qt.PartiallyChecked)\n self.add_row(\"Mount magic dirs:\", self.magic_dirs)\n\n def options(self):\n options = super(ChrootOptions, self).options()\n\n options['root'] = ROOT.INDEX_TO_OPTION[self.root.currentIndex()]\n\n if self.preserve_owner.checkState() == QtCore.Qt.Unchecked:\n options['args'].append('--dont-preserve-owner')\n elif self.preserve_owner.checkState() == QtCore.Qt.Checked:\n options['args'].append('--preserve-owner')\n\n if self.magic_dirs.checkState() == QtCore.Qt.Unchecked:\n options['args'].append('--dont-bind-magic-dirs')\n elif self.magic_dirs.checkState() == QtCore.Qt.Checked:\n options['args'].append('--bind-magic-dirs')\n\n record_usage(\n chroot_preserve_owner=self.preserve_owner.checkState(),\n chroot_magic_dirs=self.magic_dirs.checkState())\n\n return options\n\n @staticmethod\n def available():\n return sys.platform.startswith('linux')\n\n\nclass DockerOptions(UnpackerOptions):\n def __init__(self):\n super(DockerOptions, self).__init__()\n\n self.root = QtWidgets.QComboBox(editable=False)\n self.root.addItems(ROOT.TEXT)\n self.add_row(\"Elevate privileges:\", self.root)\n\n try:\n cmd = ['docker-machine', 'ls', '-q']\n query = subprocess.Popen(cmd, stdout=subprocess.PIPE)\n out, _ = query.communicate()\n if query.returncode != 0:\n raise subprocess.CalledProcessError(query.returncode, cmd)\n self.machine = QtWidgets.QComboBox(editable=False)\n if 'DOCKER_HOST' in os.environ:\n self.machine.addItem(\"Custom config from environment\", None)\n else:\n self.machine.addItem(\"Default (no machine)\", None)\n nb_machines = 0\n for machine in out.splitlines():\n machine = machine.strip()\n if machine:\n self.machine.addItem(machine.decode('utf-8', 'replace'),\n machine)\n nb_machines += 1\n record_usage(docker_machines=nb_machines)\n except (OSError, subprocess.CalledProcessError):\n self.machine = QtWidgets.QComboBox(editable=False, enabled=False)\n self.machine.addItem(\"docker-machine unavailable\", None)\n record_usage(docker_machines=False)\n self.add_row(\"docker-machine:\", self.machine)\n\n self.image = QtWidgets.QLineEdit(placeholderText='detect')\n self.add_row(\"Base image:\", self.image)\n\n self.distribution = QtWidgets.QLineEdit(placeholderText='detect')\n self.add_row(\"Distribution:\", self.distribution)\n\n self.install_pkgs = QtWidgets.QCheckBox(\"install packages rather than \"\n \"extracting them from RPZ\")\n self.add_row(\"Install packages:\", self.install_pkgs)\n\n def options(self):\n options = super(DockerOptions, self).options()\n\n if self.machine.currentIndex() != -1:\n options['docker-machine'] = self.machine.itemData(\n self.machine.currentIndex())\n record_usage(\n use_docker_machine=options['docker-machine'] is not None)\n\n options['root'] = ROOT.INDEX_TO_OPTION[self.root.currentIndex()]\n\n if self.image.text():\n options['args'].extend(['--base-image', self.image.text()])\n record_usage(docker_base_image=True)\n\n if self.distribution.text():\n options['args'].extend(['--distribution',\n self.distribution.text()])\n record_usage(docker_distribution=True)\n\n if self.install_pkgs.isChecked():\n options['args'].append('--install-pkgs')\n\n record_usage(root=options['root'],\n docker_install_pkgs=self.install_pkgs.isChecked())\n\n return options\n\n\nclass VagrantOptions(UnpackerOptions):\n def __init__(self):\n super(VagrantOptions, self).__init__()\n\n self.image = QtWidgets.QLineEdit(placeholderText='detect')\n self.add_row(\"Base box:\", self.image)\n\n self.distribution = QtWidgets.QLineEdit(placeholderText='detect')\n self.add_row(\"Distribution:\", self.distribution)\n\n self.memory = QtWidgets.QSpinBox(suffix=\"MB\",\n minimum=99, maximum=64000, value=99,\n specialValueText=\"(default)\")\n self.add_row(\"Memory:\", self.memory)\n\n self.gui = QtWidgets.QCheckBox(\"Enable local GUI\")\n self.add_row(\"GUI:\", self.gui)\n\n self.ports = QtWidgets.QLineEdit(\n '',\n toolTip=\"Space-separated host:guest port mappings\")\n self.add_row(\"Expose ports:\", self.ports)\n\n self.use_chroot = QtWidgets.QCheckBox(\n \"use chroot and prefer packed files over the virtual machines' \"\n \"files\",\n checked=True)\n self.add_row(\"Chroot:\", self.use_chroot)\n\n self.magic_dirs = QtWidgets.QCheckBox(\"mount /dev and /proc inside \"\n \"the chroot\", checked=True)\n self.add_row(\"Mount magic dirs:\", self.magic_dirs)\n\n def options(self):\n options = super(VagrantOptions, self).options()\n\n if self.image.text():\n options['args'].extend(['--base-image', self.image.text()])\n record_usage(vagrant_base_image=True)\n\n if self.distribution.text():\n options['args'].extend(['--distribution',\n self.distribution.text()])\n record_usage(vagrant_distribution=True)\n\n if self.memory.value() != 99:\n options['args'].extend(['--memory', '%d' % self.memory.value()])\n record_usage(vagrant_memory=self.memory.value())\n\n if self.gui.isChecked():\n options['args'].append('--use-gui')\n record_usage(vagrant_gui=True)\n\n ports = parse_ports(self.ports.text(), self)\n if ports is None:\n return None\n record_usage(vagrant_unpack_port_fwd=bool(ports))\n for host, container, proto in ports:\n options['args'].append('--expose-port=%s:%s/%s' % (\n host, container, proto))\n\n if not self.use_chroot.isChecked():\n options['args'].append('--dont-use-chroot')\n record_usage(vagrant_no_chroot=True)\n\n if not self.magic_dirs.isChecked():\n options['args'].append('--dont-bind-magic-dirs')\n record_usage(vagrant_magic_dirs=False)\n\n return options\n\n\nclass UnpackTab(QtWidgets.QWidget):\n \"\"\"The unpack window, that sets up a .RPZ file in a directory.\n \"\"\"\n UNPACKERS = [\n ('directory', DirectoryOptions),\n ('chroot', ChrootOptions),\n ('docker', DockerOptions),\n ('vagrant', VagrantOptions),\n ]\n\n unpacked = QtCore.Signal(str, object)\n\n def __init__(self, package='', **kwargs):\n super(UnpackTab, self).__init__(**kwargs)\n\n layout = QtWidgets.QGridLayout()\n layout.addWidget(QtWidgets.QLabel(\"RPZ package:\"), 0, 0)\n self.package_widget = QtWidgets.QLineEdit(package, enabled=False)\n layout.addWidget(self.package_widget, 0, 1)\n browse_pkg = QtWidgets.QPushButton(\"Browse\")\n browse_pkg.clicked.connect(self._browse_pkg)\n layout.addWidget(browse_pkg, 0, 2)\n\n layout.addWidget(QtWidgets.QLabel(\"Unpacker:\"), 1, 0,\n QtCore.Qt.AlignTop)\n ulayout = QtWidgets.QVBoxLayout()\n self.unpackers = QtWidgets.QButtonGroup()\n for i, (name, opts) in enumerate(self.UNPACKERS):\n radio = QtWidgets.QRadioButton(name)\n radio.unpacker = name\n self.unpackers.addButton(radio, i)\n ulayout.addWidget(radio)\n if not opts.available():\n radio.setDisabled(True)\n layout.addLayout(ulayout, 1, 1, 1, 2)\n\n group = QtWidgets.QGroupBox(title=\"Unpacker options\")\n group_layout = QtWidgets.QVBoxLayout()\n self.unpacker_options = ResizableStack()\n self.unpackers.buttonClicked[int].connect(\n self.unpacker_options.setCurrentIndex)\n scroll = QtWidgets.QScrollArea(widgetResizable=True)\n scroll.setWidget(self.unpacker_options)\n group_layout.addWidget(scroll)\n group.setLayout(group_layout)\n layout.addWidget(group, 2, 0, 1, 3)\n layout.setRowStretch(2, 1)\n\n for i, (name, WidgetClass) in enumerate(self.UNPACKERS):\n widget = WidgetClass()\n self.unpacker_options.addWidget(widget)\n\n self.unpacker_options.addWidget(\n QtWidgets.QLabel(\"Select an unpacker to display options...\"))\n self.unpacker_options.setCurrentIndex(len(self.UNPACKERS))\n\n layout.addWidget(QtWidgets.QLabel(\"Destination directory:\"), 3, 0)\n self.directory_widget = QtWidgets.QLineEdit()\n self.directory_widget.editingFinished.connect(self._directory_changed)\n layout.addWidget(self.directory_widget, 3, 1)\n browse_dir = QtWidgets.QPushButton(\"Browse\")\n browse_dir.clicked.connect(self._browse_dir)\n layout.addWidget(browse_dir, 3, 2)\n\n buttons = QtWidgets.QHBoxLayout()\n buttons.addStretch(1)\n self.unpack_widget = QtWidgets.QPushButton(\"Unpack experiment\",\n enabled=False)\n self.unpack_widget.clicked.connect(self._unpack)\n buttons.addWidget(self.unpack_widget)\n layout.addLayout(buttons, 4, 0, 1, 3)\n\n self.setLayout(layout)\n\n self._package_changed()\n\n def replaceable(self):\n return not self.package_widget.text()\n\n def _browse_pkg(self):\n picked, _ = QtWidgets.QFileDialog.getOpenFileName(\n self, \"Pick package file\",\n QtCore.QDir.currentPath(), \"ReproZip Packages (*.rpz)\")\n if picked:\n record_usage(browse_pkg=True)\n self.package_widget.setText(picked)\n self._package_changed()\n\n def _package_changed(self, new_pkg=None):\n package = self.package_widget.text()\n if package.lower().endswith('.rpz'):\n self.directory_widget.setText(package[:-4])\n self._directory_changed()\n\n def _browse_dir(self):\n picked, _ = QtWidgets.QFileDialog.getSaveFileName(\n self, \"Pick directory\",\n QtCore.QDir.currentPath())\n if picked:\n if os.path.exists(picked):\n error_msg(self, \"This directory already exists\", 'warning')\n else:\n self.directory_widget.setText(picked)\n self._directory_changed()\n\n def _directory_changed(self, new_dir=None):\n self.unpack_widget.setEnabled(bool(self.directory_widget.text()))\n\n def _unpack(self):\n directory = self.directory_widget.text()\n if not directory:\n return\n unpacker = self.unpackers.checkedButton()\n if unpacker:\n record_usage(unpacker=unpacker.text())\n options = self.unpacker_options.currentWidget().options()\n if options is None:\n return\n if handle_error(self, reprounzip.unpack(\n self.package_widget.text(),\n unpacker.unpacker,\n directory,\n options)):\n self.unpacked.emit(os.path.abspath(directory),\n options.get('root'))\n else:\n error_msg(self, \"No unpacker selected\", 'warning')\n" }, { "path": "reprounzip-qt/reprounzip_qt/main.py", "content": "# Copyright (C) 2014 New York University\n# This file is part of ReproZip which is released under the Revised BSD License\n# See file LICENSE for full license details.\n\nfrom __future__ import division, print_function, unicode_literals\n\nimport argparse\nimport locale\nimport logging\nimport sys\n\nfrom reprounzip.common import setup_logging\nfrom reprounzip_qt import __version__\nfrom reprounzip_qt.usage import record_usage, submit_usage_report\nfrom reprounzip.utils import stderr\n\n\nlogger = logging.getLogger('reprounzip_qt')\n\n\ndef main():\n \"\"\"Entry point when called on the command-line.\n \"\"\"\n # Locale\n try:\n locale.setlocale(locale.LC_ALL, '')\n except locale.Error as e:\n stderr.write(\"Couldn't set locale: %s\\n\" % e)\n\n parser = argparse.ArgumentParser(\n description=\"Graphical user interface for reprounzip\",\n epilog=\"Please report issues to reprozip@nyu.edu\")\n parser.add_argument('--version', action='version',\n version=\"reprounzip-qt version %s\" % __version__)\n parser.add_argument('-v', '--verbose', action='count', default=1,\n dest='verbosity', help=\"augments verbosity level\")\n parser.add_argument('package', nargs=argparse.OPTIONAL)\n parser.add_argument('--unpacked', action='append', default=[])\n\n argv = sys.argv[1:]\n i = 0\n while i < len(argv):\n if argv[i].startswith('-psn'):\n del argv[i]\n else:\n i += 1\n args = parser.parse_args(argv)\n\n setup_logging('REPROUNZIP-QT', args.verbosity)\n\n from reprounzip_qt.gui import Application, ReprounzipUi\n\n app = Application(sys.argv)\n\n window_args = {}\n if args.package and args.unpacked:\n sys.stderr.write(\"You can't pass both a package and a unpacked \"\n \"directory\\n\")\n sys.exit(2)\n elif args.package:\n logger.info(\"Got package on the command-line: %s\", args.package)\n record_usage(cmdline='package')\n window_args = dict(unpack=dict(package=args.package))\n elif len(args.unpacked) == 1:\n logger.info(\"Got unpacked directory on the command-line: %s\",\n args.unpacked)\n record_usage(cmdline='directory')\n window_args = dict(run=dict(unpacked_directory=args.unpacked[0]),\n tab=1)\n elif args.unpacked:\n sys.stderr.write(\"You may only use --unpacked once\\n\")\n sys.exit(2)\n else:\n record_usage(cmdline='empty')\n\n window = ReprounzipUi(**window_args)\n app.set_first_window(window)\n window.setVisible(True)\n\n app.exec_()\n submit_usage_report()\n sys.exit(0)\n\n\nif __name__ == '__main__':\n main()\n" }, { "path": "reprounzip-qt/reprounzip_qt/qt_terminal.py", "content": "# Copyright (C) 2014 New York University\n# This file is part of ReproZip which is released under the Revised BSD License\n# See file LICENSE for full license details.\n\nfrom __future__ import division, print_function, unicode_literals\n\nimport locale\nimport logging\nfrom qtpy import QtCore, QtWidgets\nimport sys\n\n\nif sys.version_info >= (3,):\n from html import escape\nelse:\n from cgi import escape\n\n\nlogger = logging.getLogger('reprounzip_qt')\n\n\nclass Terminal(QtWidgets.QWidget):\n finished = QtCore.Signal(int)\n\n def __init__(self, cmdline, env, input_enabled=False,\n success_msg=None, fail_msg=None,\n **kwargs):\n super(Terminal, self).__init__(**kwargs)\n\n self.success_msg = success_msg or \"Command finished\"\n self.fail_msg = fail_msg or \"Command failed\"\n\n layout = QtWidgets.QVBoxLayout()\n self.text = QtWidgets.QTextEdit(readOnly=True)\n layout.addWidget(self.text)\n if input_enabled:\n self.input = QtWidgets.QLineEdit()\n self.input.returnPressed.connect(self._enter)\n layout.addWidget(self.input)\n else:\n self.input = None\n self.setLayout(layout)\n\n self.process = QtCore.QProcess(self)\n\n # Dodge py2app issues\n environ = QtCore.QProcessEnvironment.systemEnvironment()\n if environ.contains('PYTHONHOME'):\n environ.remove('PYTHONPATH')\n environ.remove('PYTHONHOME')\n if sys.platform == 'darwin':\n environ.insert(\n 'PATH',\n (environ.value('PATH', '/usr/bin:/bin:/usr/sbin:/sbin') +\n ':/usr/local/bin:/opt/reprounzip'))\n\n # Unset TERM to avoid ansi escapes\n environ.remove('TERM')\n\n # Add additional environment variables\n for k, v in env.items():\n environ.insert(k, v)\n\n logger.info(\"Running in builtin Qt terminal: %r\", cmdline)\n\n self.process.setProcessEnvironment(environ)\n self.process.setProcessChannelMode(QtCore.QProcess.SeparateChannels)\n if input_enabled:\n mode = QtCore.QIODevice.ReadWrite\n else:\n mode = QtCore.QIODevice.ReadOnly\n self.process.start(cmdline[0], cmdline[1:], mode)\n if not input_enabled:\n self.process.closeWriteChannel()\n self.process.readyReadStandardOutput.connect(self._read_stdout)\n self.process.readyReadStandardError.connect(self._read_stderr)\n self.process.finished.connect(self._finished)\n self.text.setHtml('''\\\n\n''')\n self.text.append('%s' %\n escape(' '.join(cmdline)))\n\n def _enter(self):\n cmd = self.input.text()\n self.input.setText('')\n self.process.write(cmd + '\\n')\n\n def _read_stdout(self):\n out = self.process.readAllStandardOutput()\n out = bytes(out).decode(locale.getpreferredencoding() or 'UTF-8',\n 'replace')\n self.text.append('%s' % escape(out))\n\n def _read_stderr(self):\n out = self.process.readAllStandardError()\n out = bytes(out).decode(locale.getpreferredencoding() or 'UTF-8',\n 'replace')\n self.text.append('%s' % escape(out))\n\n def _finished(self, code, status):\n good = False\n if status == QtCore.QProcess.NormalExit:\n msg = \"returned %d\" % code\n self.finished.emit(code)\n good = code == 0\n else:\n msg = \"crashed\"\n self.finished.emit(-1)\n if good:\n msg = self.success_msg\n else:\n msg = \"%s (%s)\" % (self.fail_msg, msg)\n self.text.append('
%s' % msg)\n\n\ndef run_in_builtin_terminal(cmd, env,\n text=None, success_msg=None, fail_msg=None):\n result = [False]\n\n def store_result(code):\n result[:] = [code]\n\n dialog = QtWidgets.QDialog()\n layout = QtWidgets.QVBoxLayout()\n if text is not None:\n layout.addWidget(QtWidgets.QLabel(text))\n terminal = Terminal(cmd, env, input_enabled=False,\n success_msg=success_msg, fail_msg=fail_msg)\n terminal.finished.connect(store_result)\n layout.addWidget(terminal)\n buttons = QtWidgets.QHBoxLayout()\n buttons.addStretch(1)\n accept = QtWidgets.QPushButton(\"Close\", enabled=False)\n accept.clicked.connect(dialog.accept)\n terminal.finished.connect(lambda _: accept.setEnabled(True))\n buttons.addWidget(accept)\n layout.addLayout(buttons)\n dialog.setLayout(layout)\n dialog.exec_()\n\n return result[0]\n" }, { "path": "reprounzip-qt/reprounzip_qt/reprounzip_interface.py", "content": "# Copyright (C) 2014 New York University\n# This file is part of ReproZip which is released under the Revised BSD License\n# See file LICENSE for full license details.\n\nfrom __future__ import division, print_function, unicode_literals\n\nimport itertools\nimport logging\nimport os\nfrom rpaths import Path\nimport pickle\nimport subprocess\nimport sys\nimport time\nimport yaml\n\nfrom reprounzip.common import load_config\nfrom reprounzip_qt.qt_terminal import run_in_builtin_terminal\n\n\nlogger = logging.getLogger('reprounzip_qt')\n\n\nsafe_shell_chars = set(\"ABCDEFGHIJKLMNOPQRSTUVWXYZ\"\n \"abcdefghijklmnopqrstuvwxyz\"\n \"0123456789\"\n \"-+=/:.,%_\")\n\n\ndef shell_escape(s):\n r\"\"\"Given bl\"a, returns \"bl\\\\\"a\".\n \"\"\"\n if isinstance(s, bytes):\n s = s.decode('utf-8')\n if not s or any(c not in safe_shell_chars for c in s):\n return '\"%s\"' % (s.replace('\\\\', '\\\\\\\\')\n .replace('\"', '\\\\\"')\n .replace('`', '\\\\`')\n .replace('$', '\\\\$'))\n else:\n return s\n\n\nsafe_win_chars = set(\"ABCDEFGHIJKLMNOPQRSTUVWXYZ\"\n \"abcdefghijklmnopqrstuvwxyz\"\n \"0123456789\"\n \"-+=/:.,_\\\\$\")\n\n\ndef win_escape(s):\n r\"\"\"Given bl\"a, returns \"bl^\"a\".\n \"\"\"\n if isinstance(s, bytes):\n s = s.decode('utf-8')\n if any(c not in safe_win_chars for c in s):\n return '\"%s\"' % (s.replace('^', '^^')\n .replace('\"', '^\"')\n .replace('%', '^%'))\n else:\n return s\n\n\nif sys.platform.startswith('win'):\n native_escape = win_escape\nelse:\n native_escape = shell_escape\n\n\ndef check_directory(directory):\n if os.path.isdir(directory):\n filename = os.path.join(directory, '.reprounzip')\n if os.path.isfile(filename):\n with open(filename, 'rb') as fp:\n unpacked_info = pickle.load(fp)\n logger.debug(\"Directory was created by unpacker '%s': %s\",\n unpacked_info['unpacker'], directory)\n return unpacked_info['unpacker']\n logger.debug(\"Not an unpacked directory: %s\", directory)\n return None\n\n\ndef is_jupyter(directory):\n with open(os.path.join(directory, 'config.yml')) as fp:\n config = yaml.safe_load(fp)\n iofiles = config.get('inputs_outputs', None)\n detected = iofiles and any(iofile['name'] == 'jupyter_connection_file'\n for iofile in config.get('inputs_outputs'))\n if detected:\n logger.debug(\"Jupyter kernel detected\")\n return detected\n\n\nclass FileStatus(object):\n def __init__(self, name, path, is_input, is_output):\n self.name = name\n self.path = path\n self.assigned = None\n self.is_input = is_input\n self.is_output = is_output\n\n def __repr__(self):\n return '' % self.name\n\n\nclass FilesStatus(object):\n def __init__(self, directory):\n self.directory = directory\n config = load_config(Path(directory) / 'config.yml', True)\n\n self.files = [\n FileStatus(name, f.path, f.read_runs, f.write_runs)\n for name, f in config.inputs_outputs.items()\n ]\n logger.info(\"Loaded %d files from the configuration\", len(self.files))\n self._refresh()\n\n def _refresh(self):\n with open(os.path.join(self.directory, '.reprounzip'), 'rb') as fp:\n unpacked_info = pickle.load(fp)\n assigned_input_files = unpacked_info.get('input_files', {})\n for f in self.files:\n f.assigned = assigned_input_files.get(f.name)\n\n def __getitem__(self, item):\n self._refresh()\n return self.files[item]\n\n def __iter__(self):\n self._refresh()\n return iter(self.files)\n\n\ndef find_command(cmd):\n if sys.platform.startswith('win'):\n for path in os.environ.get('PATH', '').split(os.pathsep):\n for ext in ('.bat', '.exe', '.cmd'):\n filename = os.path.join(path, cmd + ext)\n if os.path.exists(filename):\n logger.info(\"Using %s\", filename)\n return filename\n else:\n for path in itertools.chain(\n os.environ.get('PATH', '').split(os.pathsep),\n ['/usr/local/bin', '/opt/reprounzip']):\n filename = os.path.join(path, cmd)\n if os.path.exists(filename):\n logger.info(\"Using %s\", filename)\n return filename\n logger.warning(\"Command not found: %s\", cmd)\n return None\n\n\ndef run(directory, unpacker=None, runs=None,\n root=None, jupyter=False, args=[]):\n if unpacker is None:\n unpacker = check_directory(directory)\n\n if jupyter:\n reprounzip_jupyter = find_command('reprozip-jupyter')\n if reprounzip_jupyter is None:\n return (\"Couldn't find reprozip-jupyter command -- is it \"\n \"installed?\", 'critical')\n cmd = [reprounzip_jupyter, 'run', os.path.abspath(directory)]\n run_in_system_terminal(\n ['sh', '-c',\n 'cd %s && %s' % (shell_escape(os.getcwd()),\n ' '.join(shell_escape(a) for a in cmd))],\n root=root)\n return True\n\n reprounzip = find_command('reprounzip')\n if reprounzip is None:\n return (\"Couldn't find reprounzip command -- is reprounzip installed?\",\n 'critical')\n\n env = {}\n\n with open(os.path.join(directory, '.reprounzip'), 'rb') as fp:\n docker_host = pickle.load(fp).get('docker_host')\n if docker_host and docker_host['type']:\n if docker_host['type'] == 'docker-machine':\n env.update(docker_machine_env(docker_host['name']))\n elif docker_host['type'] == 'custom':\n env.update(docker_host['env'])\n else:\n raise ValueError(\"Unrecognized docker host type %r\" %\n docker_host['type'])\n\n run_in_system_terminal(\n [reprounzip, unpacker, 'run'] +\n args +\n [os.path.abspath(directory)] +\n ([','.join('%d' % r for r in runs)] if runs is not None else []),\n env=env,\n root=root)\n return True\n\n\ndef docker_machine_env(machine):\n cmd = ['docker-machine', 'env', machine]\n getconf = subprocess.Popen(cmd, stdout=subprocess.PIPE)\n out, _ = getconf.communicate()\n if getconf.returncode != 0:\n raise subprocess.CalledProcessError(getconf.returncode, cmd)\n env = {}\n for line in out.splitlines():\n line = line.strip()\n if not line or line[0] == b'#':\n continue\n if line[0:7] == b'export ':\n line = line[7:]\n sep = line.index(b'=')\n key = line[:sep]\n if line[sep + 1] != b'\"' or line[-1] != b'\"':\n raise ValueError(\"docker-machine env format not recognized\")\n value = line[sep + 2:-1]\n env[key] = value\n logger.info(\"Got environment from docker-machine: %r\", env)\n return env\n\n\ndef unpack(package, unpacker, directory, options=None):\n if options is None:\n options = {}\n\n reprounzip = find_command('reprounzip')\n if reprounzip is None:\n return (\"Couldn't find reprounzip command -- is reprounzip installed?\",\n 'critical')\n\n env = {}\n\n docker_machine = options.get('docker-machine', None)\n if docker_machine:\n env.update(docker_machine_env(docker_machine))\n\n cmd = ([reprounzip, unpacker, 'setup'] +\n options.get('args', []) +\n [os.path.abspath(package), os.path.abspath(directory)])\n\n code = run_in_builtin_terminal_maybe(\n cmd, options.get('root', None),\n env=env,\n text=\"Unpacking experiment...\",\n success_msg=\"Successfully setup experiment\",\n fail_msg=\"Error setting up experiment\")\n if code is None:\n return os.path.exists(directory)\n else:\n return code == 0\n\n\ndef destroy(directory, unpacker=None, root=None):\n if unpacker is None:\n unpacker = check_directory(directory)\n\n reprounzip = find_command('reprounzip')\n if reprounzip is None:\n return (\"Couldn't find reprounzip command -- is reprounzip installed?\",\n 'critical')\n\n code = run_in_builtin_terminal_maybe(\n [reprounzip, unpacker, 'destroy', os.path.abspath(directory)], root,\n text=\"Destroying experiment directory...\",\n success_msg=\"Successfully destroyed experiment directory\",\n fail_msg=\"Error destroying experiment\")\n if code is None:\n return not os.path.exists(directory)\n else:\n return code == 0\n\n\ndef upload(directory, name, path, unpacker=None, root=None):\n if unpacker is None:\n unpacker = check_directory(directory)\n\n reprounzip = find_command('reprounzip')\n if reprounzip is None:\n return (\"Couldn't find reprounzip command -- is reprounzip installed?\",\n 'critical')\n\n if path is None:\n spec = ':%s' % name\n else:\n spec = '%s:%s' % (path, name)\n\n code = run_in_builtin_terminal_maybe(\n [reprounzip, unpacker, 'upload', os.path.abspath(directory), spec],\n root,\n text=\"Uploading file...\",\n success_msg=\"Successfully replaced file\",\n fail_msg=\"Error uploading file\")\n if code is None:\n return True\n else:\n return code == 0\n\n\ndef download(directory, name, path, unpacker=None, root=None):\n if unpacker is None:\n unpacker = check_directory(directory)\n\n reprounzip = find_command('reprounzip')\n if reprounzip is None:\n return (\"Couldn't find reprounzip command -- is reprounzip installed?\",\n 'critical')\n\n spec = '%s:%s' % (name, path)\n\n code = run_in_builtin_terminal_maybe(\n [reprounzip, unpacker, 'download', os.path.abspath(directory), spec],\n root,\n text=\"Downloading file...\",\n success_msg=\"Successfully downloaded file\",\n fail_msg=\"Error downloading file\")\n if code is None:\n return True\n else:\n return code == 0\n\n\ndef run_in_builtin_terminal_maybe(cmd, root, env={}, **kwargs):\n if root is None:\n code = run_in_builtin_terminal(cmd, env, **kwargs)\n return code\n else:\n run_in_system_terminal(cmd, env, root=root)\n return None\n\n\ndef run_in_system_terminal(cmd, env={},\n wait=True, close_on_success=False, root=None):\n if root is None:\n pass\n elif root == 'sudo':\n cmd = ['sudo'] + cmd\n elif root == 'su':\n cmd = ['su', '-c', ' '.join(native_escape(a) for a in cmd)]\n else:\n assert False\n\n cmd = ' '.join(native_escape(c) for c in cmd)\n\n logger.info(\"Running in system terminal: %s\", cmd)\n\n environ = dict(os.environ)\n environ.update(env)\n\n if sys.platform == 'darwin':\n # Dodge py2app issues\n environ.pop('PYTHONPATH', None)\n environ.pop('PYTHONHOME', None)\n proc = subprocess.Popen(['/usr/bin/osascript', '-'],\n stdin=subprocess.PIPE, env=env)\n run_script = \"\"\"\\\ntell application \"Terminal\"\n activate\n set w to do script %s\n%s\n%s\nend tell\n\"\"\"\n wait_script = \"\"\"\\\n repeat\n delay 1\n if not busy of w then exit repeat\n end repeat\n\"\"\"\n close_script = \"\"\"\\\n activate\n tell (first window whose tabs contain w)\n set selected tab to w\n tell application \"System Events\"\n tell process \"Terminal\"\n keystroke \"w\" using {command down}\n end tell\n end tell\n end tell\n\"\"\"\n\n if not wait:\n wait_script = ''\n if close_on_success:\n cmd = cmd + ' && exit'\n else:\n cmd = cmd + '; exit'\n close_script = ''\n run_script = run_script % (shell_escape(cmd),\n wait_script,\n close_script)\n\n proc.communicate(run_script)\n proc.wait()\n if wait:\n time.sleep(0.5)\n return None\n elif sys.platform.startswith('win'):\n if not close_on_success:\n cmd = cmd + ' ^& pause'\n subprocess.check_call(\n 'start%s cmd /c %s' % (\n ' /wait' if wait else '',\n cmd,\n ),\n shell=True)\n return None\n elif sys.platform.startswith('linux'):\n if not close_on_success:\n cmd = '/bin/sh -c %s' % \\\n shell_escape(cmd + ' ; echo \"Press enter...\"; read r')\n for term, arg_factory in [('konsole', lambda a: ['--nofork', '-e', a]),\n ('gnome-terminal', lambda a: [\n '--disable-factory-', '--',\n '/bin/sh', '-c', a]),\n ('lxterminal', lambda a: ['--command=' + a]),\n ('rxvt', lambda a: ['-e', a]),\n ('xterm', lambda a: ['-e', a])]:\n if find_command(term) is not None:\n args = arg_factory(cmd)\n proc = subprocess.Popen([term] + args,\n stdin=subprocess.PIPE)\n proc.stdin.close()\n if wait:\n retcode = proc.wait()\n if retcode != 0:\n raise subprocess.CalledProcessError(retcode,\n [term] + args)\n\n return None\n return \"Couldn't start a terminal\", 'critical'\n" }, { "path": "reprounzip-qt/reprounzip_qt/usage.py", "content": "# Copyright (C) 2014 New York University\n# This file is part of ReproZip which is released under the Revised BSD License\n# See file LICENSE for full license details.\n\nfrom __future__ import division, print_function, unicode_literals\n\nimport os\nfrom reprounzip.common import get_reprozip_ca_certificate\nimport usagestats\n\nfrom reprounzip_qt import __version__ as version\n\n\n_certificate_file = get_reprozip_ca_certificate()\n\n_usage_report = usagestats.Stats(\n '~/.reprozip/usage_stats',\n usagestats.Prompt(''),\n os.environ.get('REPROZIP_USAGE_URL', 'https://stats.reprozip.org/'),\n version='%s %s' % ('reprounzip-qt', version),\n unique_user_id=True,\n env_var='REPROZIP_USAGE_STATS',\n ssl_verify=_certificate_file.path\n)\n\n\ndef record_usage(**kwargs):\n \"\"\"Records some info in the current usage report.\n \"\"\"\n if _usage_report is not None:\n _usage_report.note(kwargs)\n\n\ndef submit_usage_report(**kwargs):\n \"\"\"Submits the current usage report to the usagestats server.\n \"\"\"\n _usage_report.submit(kwargs,\n usagestats.OPERATING_SYSTEM,\n usagestats.SESSION_TIME,\n usagestats.PYTHON_VERSION)\n" }, { "path": "reprounzip-qt/setup.cfg", "content": "[bdist_wheel]\nuniversal = 1\n" }, { "path": "reprounzip-qt/setup.py", "content": "import io\nimport os\nfrom setuptools import setup\n\n\n# pip workaround\nos.chdir(os.path.abspath(os.path.dirname(__file__)))\n\n\n# Need to specify encoding for PY3, which has the worst unicode handling ever\nwith io.open('README.rst', encoding='utf-8') as fp:\n description = fp.read()\nsetup(name='reprounzip-qt',\n version='1.2.1',\n packages=['reprounzip_qt', 'reprounzip_qt.gui'],\n package_data={'reprounzip_qt': ['icon.png']},\n entry_points={\n 'gui_scripts': [\n 'reprounzip-qt = reprounzip_qt.main:main']},\n install_requires=['PyYAML', 'qtpy', 'reprounzip>=1.0'],\n description=\"Graphical user interface for reprounzip, using Qt\",\n author=\"Remi Rampin, Fernando Chirigati, Dennis Shasha, Juliana Freire\",\n author_email='reprozip@nyu.edu',\n maintainer=\"Remi Rampin\",\n maintainer_email='remi@rampin.org',\n url='https://www.reprozip.org/',\n project_urls={\n 'Documentation': 'https://docs.reprozip.org/',\n 'Examples': 'https://examples.reprozip.org/',\n 'Source': 'https://github.com/VIDA-NYU/reprozip',\n 'Bug Tracker': 'https://github.com/VIDA-NYU/reprozip/issues',\n 'Chat': 'https://riot.im/app/#/room/#reprozip:matrix.org',\n 'Changelog':\n 'https://github.com/VIDA-NYU/reprozip/blob/1.x/CHANGELOG.md',\n },\n long_description=description,\n license='BSD-3-Clause',\n keywords=['reprozip', 'reprounzip', 'reproducibility', 'provenance',\n 'vida', 'nyu', 'gui'],\n classifiers=[\n 'Development Status :: 4 - Beta',\n 'Environment :: X11 Applications :: Qt',\n 'Intended Audience :: Science/Research',\n 'License :: OSI Approved :: BSD License',\n 'Programming Language :: Python :: 2.7',\n 'Programming Language :: Python :: 3',\n 'Topic :: Scientific/Engineering',\n 'Topic :: System :: Archiving'])\n" }, { "path": "reprounzip-vagrant/LICENSE.txt", "content": "Copyright (C) 2014, New York University\nAll rights reserved.\n\nRedistribution and use in source and binary forms, with or without\nmodification, are permitted provided that the following conditions are met:\n\n1. Redistributions of source code must retain the above copyright notice, this\n list of conditions and the following disclaimer.\n\n2. Redistributions in binary form must reproduce the above copyright notice,\n this list of conditions and the following disclaimer in the documentation\n and/or other materials provided with the distribution.\n\n3. Neither the name of the copyright holder nor the names of its contributors\n may be used to endorse or promote products derived from this software\n without specific prior written permission.\n\nTHIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \"AS IS\" AND\nANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED\nWARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE\nDISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE\nFOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL\nDAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR\nSERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER\nCAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,\nOR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE\nOF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n" }, { "path": "reprounzip-vagrant/MANIFEST.in", "content": "include README.rst\ninclude LICENSE.txt\n" }, { "path": "reprounzip-vagrant/README.rst", "content": "ReproZip project\n================\n\n`ReproZip `__ is a tool aimed at simplifying the process of creating reproducible experiments from command-line executions, a frequently-used common denominator in computational science. It tracks operating system calls and creates a bundle that contains all the binaries, files and dependencies required to run a given command on the author's computational environment (packing step). A reviewer can then extract the experiment in his environment to reproduce the results (unpacking step).\n\nreprounzip-vagrant\n------------------\n\nThis is the component responsible for the unpacking step on different environments (Linux, Windows, and Mac OS X) by using a `Vagrant `_ virtual machine.\n\nPlease refer to `reprozip `__, `reprounzip `_, and `reprounzip-docker `_ for other components and plugins.\n\nAdditional Information\n----------------------\n\nFor more detailed information, please refer to our `website `_, as well as to our `documentation `_.\n\nReproZip is currently being developed at `NYU `_. The team includes:\n\n* `Fernando Chirigati `_\n* `Juliana Freire `_\n* `Remi Rampin `_\n* `Dennis Shasha `_\n* `Vicky Rampin `_\n" }, { "path": "reprounzip-vagrant/reprounzip/__init__.py", "content": "try: # pragma: no cover\n __import__('pkg_resources').declare_namespace(__name__)\nexcept ImportError: # pragma: no cover\n from pkgutil import extend_path\n __path__ = extend_path(__path__, __name__)\n" }, { "path": "reprounzip-vagrant/reprounzip/unpackers/__init__.py", "content": "try: # pragma: no cover\n __import__('pkg_resources').declare_namespace(__name__)\nexcept ImportError: # pragma: no cover\n from pkgutil import extend_path\n __path__ = extend_path(__path__, __name__)\n" }, { "path": "reprounzip-vagrant/reprounzip/unpackers/vagrant/__init__.py", "content": "# Copyright (C) 2014 New York University\n# This file is part of ReproZip which is released under the Revised BSD License\n# See file LICENSE for full license details.\n\n\"\"\"Vagrant plugin for reprounzip.\n\nThis files contains the 'vagrant' unpacker, which builds a Vagrant template\nfrom a reprozip pack. That template can then be run as a virtual machine via\nVagrant (``vagrant up``).\n\nSee http://www.vagrantup.com/\n\"\"\"\n\nfrom __future__ import division, print_function, unicode_literals\n\nimport argparse\nfrom distutils.version import LooseVersion\nimport logging\nimport os\nimport paramiko\nimport re\nfrom rpaths import PosixPath, Path\nimport subprocess\nimport sys\n\nfrom reprounzip.common import load_config, record_usage, RPZPack\nfrom reprounzip import signals\nfrom reprounzip.parameters import get_parameter\nfrom reprounzip.unpackers.common import COMPAT_OK, COMPAT_MAYBE, COMPAT_NO, \\\n CantFindInstaller, composite_action, target_must_exist, \\\n make_unique_name, shell_escape, select_installer, busybox_url, join_root, \\\n rpztar_url, \\\n FileUploader, FileDownloader, get_runs, add_environment_options, \\\n fixup_environment, metadata_read, metadata_write, \\\n metadata_initial_iofiles, metadata_update_run, parse_ports\nfrom reprounzip.unpackers.common.x11 import BaseX11Handler, X11Handler\nfrom reprounzip.unpackers.vagrant.run_command import IgnoreMissingKey, \\\n run_interactive\nfrom reprounzip.utils import unicode_, iteritems, stderr, download_file\n\n\nlogger = logging.getLogger('reprounzip.vagrant')\n\n\ndef _find_version(distrib, version, architecture):\n if version is not None:\n for box in distrib['versions']:\n if re.match(box['version'], version) is not None:\n result = box['architectures'].get(architecture)\n if result is not None:\n return box['distribution'], result\n box = distrib['default']\n if version is not None:\n logger.warning(\"Using %s instead of '%s'\",\n box['name'], version)\n result = box['architectures'].get(architecture)\n if result is not None:\n return box['distribution'], result\n\n\ndef _find_distribution(parameter, distribution, version, architecture):\n boxes = parameter['boxes']\n default = parameter['default']\n\n for distrib_name, distrib in iteritems(boxes):\n if distribution == distrib_name:\n result = _find_version(distrib, version, architecture)\n if result is not None:\n return result\n distrib = boxes[default]\n logger.warning(\"Unsupported distribution '%s', using %s\",\n distribution, default)\n return _find_version(distrib, None, architecture)\n\n\ndef select_box(runs, gui=False):\n \"\"\"Selects a box for the experiment, with the correct distribution.\n \"\"\"\n distribution, version = runs[0]['distribution']\n distribution = distribution.lower()\n architecture = runs[0]['architecture']\n\n record_usage(vagrant_select_box='%s;%s;%s;gui=%s' % (distribution, version,\n architecture, gui))\n\n if architecture not in ('i686', 'x86_64'):\n logger.critical(\"Error: unsupported architecture %s\", architecture)\n sys.exit(1)\n\n if gui:\n vagrant_param = get_parameter('vagrant_boxes_x')\n if vagrant_param is None: # Compatibility with old parameters\n return 'debian', 'remram/debian-8-amd64-x'\n else:\n vagrant_param = get_parameter('vagrant_boxes')\n\n result = _find_distribution(vagrant_param,\n distribution, version, architecture)\n if result is None:\n logger.critical(\"Error: couldn't find a base box for required \"\n \"architecture\")\n sys.exit(1)\n return result\n\n\ndef write_dict(path, dct):\n metadata_write(path, dct, 'vagrant')\n\n\ndef read_dict(path):\n return metadata_read(path, 'vagrant')\n\n\ndef machine_setup(target):\n \"\"\"Prepare the machine and get SSH parameters from ``vagrant ssh``.\n \"\"\"\n try:\n out = subprocess.check_output(['vagrant', 'ssh-config'],\n cwd=target.path,\n stderr=subprocess.PIPE)\n except subprocess.CalledProcessError:\n # Makes sure the VM is running\n logger.info(\"Calling 'vagrant up'...\")\n try:\n retcode = subprocess.check_call(['vagrant', 'up'], cwd=target.path)\n except OSError:\n logger.critical(\"vagrant executable not found\")\n sys.exit(1)\n else:\n if retcode != 0:\n logger.critical(\"vagrant up failed with code %d\", retcode)\n sys.exit(1)\n # Try again\n out = subprocess.check_output(['vagrant', 'ssh-config'],\n cwd=target.path)\n\n vagrant_info = {}\n for line in out.split(b'\\n'):\n line = line.strip().split(b' ', 1)\n if len(line) != 2:\n continue\n value = line[1].decode('utf-8')\n if len(value) >= 2 and value[0] == '\"' and value[-1] == '\"':\n # Vagrant should really be escaping special characters here, but\n # it's not -- https://github.com/mitchellh/vagrant/issues/6428\n value = value[1:-1]\n vagrant_info[line[0].decode('utf-8').lower()] = value\n\n if 'identityfile' in vagrant_info:\n key_file = vagrant_info['identityfile']\n else:\n key_file = Path('~/.vagrant.d/insecure_private_key').expand_user()\n info = dict(hostname=vagrant_info.get('hostname', '127.0.0.1'),\n port=int(vagrant_info.get('port', 2222)),\n username=vagrant_info.get('user', 'vagrant'),\n key_filename=key_file)\n logger.debug(\"SSH parameters from Vagrant: %s@%s:%s, key=%s\",\n info['username'], info['hostname'], info['port'],\n info['key_filename'])\n\n unpacked_info = read_dict(target)\n use_chroot = unpacked_info['use_chroot']\n gui = unpacked_info['gui']\n\n if use_chroot:\n # Mount directories\n logger.debug(\"Mounting directories\")\n ssh = paramiko.SSHClient()\n ssh.set_missing_host_key_policy(IgnoreMissingKey())\n ssh.connect(**info)\n chan = ssh.get_transport().open_session()\n chan.exec_command(\n '/usr/bin/sudo /bin/sh -c %s' % shell_escape(\n 'if ! grep -q \"/experimentroot \" /etc/mtab; then '\n 'mount -o bind /.experimentdata /experimentroot; '\n 'fi; '\n 'if ! grep -q \"/experimentroot/dev \" /etc/mtab; then '\n 'mount -o rbind /dev /experimentroot/dev; '\n 'fi; '\n 'if ! grep -q \"/experimentroot/proc \" /etc/mtab; then '\n 'mount -t proc none /experimentroot/proc; '\n 'fi'))\n if chan.recv_exit_status() != 0:\n logger.critical(\"Couldn't mount directories in chroot\")\n sys.exit(1)\n if gui:\n # Mount X11 socket\n logger.debug(\"Mounting X11 socket\")\n chan = ssh.get_transport().open_session()\n chan.exec_command(\n '/usr/bin/sudo /bin/sh -c %s' % shell_escape(\n 'if [ -d /tmp/.X11-unix ]; then '\n '[ -d /experimentroot/tmp/.X11-unix ] || '\n 'mkdir /experimentroot/tmp/.X11-unix; '\n 'mount -o bind '\n '/tmp/.X11-unix /experimentroot/tmp/.X11-unix; '\n 'fi; exit 0'))\n if chan.recv_exit_status() != 0:\n logger.critical(\"Couldn't mount X11 sockets in chroot\")\n sys.exit(1)\n ssh.close()\n else:\n logger.debug(\"NOT mounting directories\")\n\n return info\n\n\ndef vagrant_setup_create(args):\n \"\"\"Sets up the experiment to be run in a Vagrant-built virtual machine.\n\n This can either build a chroot or not.\n\n If building a chroot, we do just like without Vagrant: we copy all the\n files and only get what's missing from the host. But we do install\n automatically the packages whose files are required.\n\n If not building a chroot, we install all the packages, and only unpack\n files that don't come from packages.\n\n In short: files from packages with packfiles=True will only be used if\n building a chroot.\n \"\"\"\n if not args.pack:\n logger.critical(\"setup/create needs the pack filename\")\n sys.exit(1)\n\n pack = Path(args.pack[0])\n target = Path(args.target[0])\n if target.exists():\n logger.critical(\"Target directory exists\")\n sys.exit(1)\n use_chroot = args.use_chroot\n mount_bind = args.bind_magic_dirs\n record_usage(use_chroot=use_chroot,\n mount_bind=mount_bind)\n\n signals.pre_setup(target=target, pack=pack)\n\n # Unpacks configuration file\n rpz_pack = RPZPack(pack)\n rpz_pack.extract_config(target / 'config.yml')\n\n # Loads config\n runs, packages, other_files = config = load_config(target / 'config.yml',\n True)\n\n if not args.memory:\n memory = None\n else:\n try:\n memory = int(args.memory[-1])\n except ValueError:\n logger.critical(\"Invalid value for memory size: %r\", args.memory)\n sys.exit(1)\n\n ports = parse_ports(args.expose_port)\n\n if args.base_image and args.base_image[0]:\n record_usage(vagrant_explicit_image=True)\n box = args.base_image[0]\n if args.distribution:\n target_distribution = args.distribution[0]\n else:\n target_distribution = None\n else:\n target_distribution, box = select_box(runs, gui=args.gui)\n logger.info(\"Using box %s\", box)\n logger.debug(\"Distribution: %s\", target_distribution or \"unknown\")\n\n # If using chroot, we might still need to install packages to get missing\n # (not packed) files\n if use_chroot:\n packages = [pkg for pkg in packages if not pkg.packfiles]\n if packages:\n record_usage(vagrant_install_pkgs=True)\n logger.info(\"Some packages were not packed, so we'll install and \"\n \"copy their files\\n\"\n \"Packages that are missing:\\n%s\",\n ' '.join(pkg.name for pkg in packages))\n\n if packages:\n try:\n installer = select_installer(pack, runs, target_distribution)\n except CantFindInstaller as e:\n logger.error(\"Need to install %d packages but couldn't select a \"\n \"package installer: %s\",\n len(packages), e)\n sys.exit(1)\n\n target.mkdir(parents=True)\n\n try:\n # Writes setup script\n logger.info(\"Writing setup script %s...\", target / 'setup.sh')\n with (target / 'setup.sh').open('w', encoding='utf-8',\n newline='\\n') as script:\n script.write('#!/bin/sh\\n\\nset -e\\n\\n')\n if packages:\n # Updates package sources\n update_script = installer.update_script()\n if update_script:\n script.write(update_script)\n script.write('\\n')\n # Installs necessary packages\n script.write(installer.install_script(packages))\n script.write('\\n')\n # TODO : Compare package versions (painful because of sh)\n\n # Copies rpztar\n if not use_chroot:\n arch = runs[0]['architecture']\n download_file(rpztar_url(arch),\n target / 'rpztar',\n 'rpztar-%s' % arch)\n script.write(r'''\ncp /vagrant/rpztar /usr/local/bin/rpztar\nchmod +x /usr/local/bin/rpztar\n''')\n\n # Untar\n if use_chroot:\n script.write('\\n'\n 'mkdir /experimentroot\\n'\n 'mkdir /.experimentdata; cd /.experimentdata\\n')\n script.write('tar zpxf /vagrant/data.tgz --numeric-owner '\n '--strip=1 %s\\n' % rpz_pack.data_prefix)\n if mount_bind:\n script.write('\\n'\n 'mkdir -p /.experimentdata/dev\\n'\n 'mkdir -p /.experimentdata/proc\\n')\n\n for pkg in packages:\n script.write('\\n# Copies files from package %s\\n'\n % pkg.name)\n for f in pkg.files:\n f = f.path\n dest = join_root(PosixPath('/.experimentdata'), f)\n script.write('mkdir -p %s\\n' %\n shell_escape(unicode_(f.parent)))\n script.write('cp -L %s %s\\n' % (\n shell_escape(unicode_(f)),\n shell_escape(unicode_(dest))))\n script.write(\n '\\n'\n 'cp /etc/resolv.conf /.experimentdata/etc/resolv.conf\\n')\n else:\n script.write('\\ncd /\\n')\n paths = set()\n pathlist = []\n # Adds intermediate directories, and checks for existence in\n # the tar\n logger.info(\"Generating file list...\")\n data_files = rpz_pack.data_filenames()\n for f in other_files:\n if f.path.name == 'resolv.conf' and (\n f.path.lies_under('/etc') or\n f.path.lies_under('/run') or\n f.path.lies_under('/var')):\n continue\n path = PosixPath('/')\n for c in rpz_pack.remove_data_prefix(f.path).components:\n path = path / c\n if path in paths:\n continue\n paths.add(path)\n if path in data_files:\n pathlist.append(path)\n else:\n logger.info(\"Missing file %s\", path)\n with (target / 'rpz-files.list').open('wb') as filelist:\n for p in pathlist:\n filelist.write(join_root(PosixPath(''), p).path)\n filelist.write(b'\\0')\n script.write('/usr/local/bin/rpztar '\n '/vagrant/data.tgz '\n '/vagrant/rpz-files.list\\n')\n\n # Copies busybox\n if use_chroot:\n arch = runs[0]['architecture']\n download_file(busybox_url(arch),\n target / 'busybox',\n 'busybox-%s' % arch)\n script.write(r'''\ncp /vagrant/busybox /.experimentdata/busybox\nchmod +x /.experimentdata/busybox\nmkdir -p /.experimentdata/bin\n[ -e /.experimentdata/bin/sh ] || \\\n ln -s /busybox /.experimentdata/bin/sh\n''')\n\n # Copies pack\n logger.info(\"Copying pack file...\")\n rpz_pack.copy_data_tar(target / 'data.tgz')\n\n rpz_pack.close()\n\n # Meta-data for reprounzip\n metadata = metadata_initial_iofiles(config,\n {'use_chroot': use_chroot,\n 'gui': args.gui,\n 'ports': ports,\n 'box': box,\n 'memory': memory})\n write_dict(target, metadata)\n\n # Writes Vagrant file\n write_vagrantfile(target, metadata)\n\n signals.post_setup(target=target, pack=pack)\n except Exception:\n target.rmtree(ignore_errors=True)\n raise\n\n\ndef write_vagrantfile(target, unpacked_info):\n box = unpacked_info['box']\n gui = unpacked_info.get('gui', False)\n ports = unpacked_info.get('ports', [])\n memory = unpacked_info.get('memory', None)\n\n logger.info(\"Writing %s...\", target / 'Vagrantfile')\n with (target / 'Vagrantfile').open('w', encoding='utf-8',\n newline='\\n') as vgfile:\n # Vagrant header and version\n vgfile.write(\n '# -*- mode: ruby -*-\\n'\n '# vi: set ft=ruby\\n\\n'\n 'VAGRANTFILE_API_VERSION = \"2\"\\n\\n'\n 'Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|\\n')\n # Selects which box to install\n vgfile.write(' config.vm.box = \"%s\"\\n' % box)\n # Run the setup script on the virtual machine\n vgfile.write(' config.vm.provision \"shell\", path: \"setup.sh\"\\n')\n\n # Memory size\n if memory is not None or gui:\n vgfile.write(' config.vm.provider \"virtualbox\" do |v|\\n')\n if memory is not None:\n vgfile.write(' v.memory = %d\\n' % memory)\n if gui:\n vgfile.write(' v.gui = true\\n')\n vgfile.write(' end\\n')\n\n # Port forwarding\n for port in ports:\n vgfile.write(' config.vm.network \"forwarded_port\", host: '\n '%s, guest: %s, protocol: \"%s\"\\n' % port)\n\n vgfile.write('end\\n')\n\n\n@target_must_exist\ndef vagrant_setup_start(args):\n \"\"\"Starts the vagrant-built virtual machine.\n \"\"\"\n target = Path(args.target[0])\n\n check_vagrant_version()\n\n machine_setup(target)\n\n\nclass LocalX11Handler(BaseX11Handler):\n port_forward = []\n init_cmds = []\n\n @staticmethod\n def fix_env(env):\n \"\"\"Sets ``$XAUTHORITY`` and ``$DISPLAY`` in the environment.\n \"\"\"\n new_env = dict(env)\n new_env.pop('XAUTHORITY', None)\n new_env['DISPLAY'] = ':0'\n return new_env\n\n\n@target_must_exist\ndef vagrant_run(args):\n \"\"\"Runs the experiment in the virtual machine.\n \"\"\"\n target = Path(args.target[0])\n unpacked_info = read_dict(target)\n use_chroot = unpacked_info['use_chroot']\n cmdline = args.cmdline\n\n check_vagrant_version()\n\n # Loads config\n config = load_config(target / 'config.yml', True)\n runs = config.runs\n\n selected_runs = get_runs(runs, args.run, cmdline)\n\n hostname = runs[selected_runs[0]].get('hostname', 'reprounzip')\n\n # Port forwarding\n ports = parse_ports(args.expose_port)\n\n # If the requested ports are not a subset of the ones already set on the\n # VM, we have to update the Vagrantfile and issue `vagrant reload`, which\n # will reboot the machine\n req_ports = set(ports)\n set_ports = set(unpacked_info.get('ports', []))\n if not req_ports.issubset(set_ports):\n # Build new set of forwarded ports: the ones already set + the one just\n # requested\n # The ones we request now override the previous config\n all_ports = dict((host, (guest, proto))\n for host, guest, proto in set_ports)\n for host, guest, proto in req_ports:\n all_ports[host] = guest, proto\n unpacked_info['ports'] = sorted(\n (host, guest, proto)\n for host, (guest, proto) in iteritems(all_ports))\n\n write_vagrantfile(target, unpacked_info)\n logger.info(\"Some requested ports are not yet forwarded, running \"\n \"'vagrant reload'\")\n retcode = subprocess.call(['vagrant', 'reload', '--no-provision'],\n cwd=target.path)\n if retcode != 0:\n logger.critical(\"vagrant reload failed with code %d, aborting\",\n retcode)\n sys.exit(1)\n write_dict(target, unpacked_info)\n\n # X11 handler\n if unpacked_info['gui']:\n x11 = LocalX11Handler()\n else:\n x11 = X11Handler(args.x11, ('local', hostname), args.x11_display)\n\n cmds = []\n for run_number in selected_runs:\n run = runs[run_number]\n cmd = 'cd %s && ' % shell_escape(run['workingdir'])\n if use_chroot:\n cmd += '/busybox env -i '\n else:\n cmd += '/usr/bin/env -i '\n environ = x11.fix_env(run['environ'])\n environ = fixup_environment(environ, args)\n cmd += ' '.join('%s=%s' % (shell_escape(k), shell_escape(v))\n for k, v in iteritems(environ))\n cmd += ' '\n # FIXME : Use exec -a or something if binary != argv[0]\n if cmdline is None:\n argv = [run['binary']] + run['argv'][1:]\n else:\n argv = cmdline\n cmd += ' '.join(shell_escape(a) for a in argv)\n uid = run.get('uid', 1000)\n gid = run.get('gid', 1000)\n if use_chroot:\n userspec = '%s:%s' % (uid, gid)\n cmd = ('chroot --userspec=%s /experimentroot '\n '/bin/sh -c %s' % (\n userspec,\n shell_escape(cmd)))\n else:\n cmd = 'sudo -u \\'#%d\\' sh -c %s' % (uid, shell_escape(cmd))\n cmds.append(cmd)\n if use_chroot:\n cmds = ['chroot /experimentroot /bin/sh -c %s' % shell_escape(c)\n for c in x11.init_cmds] + cmds\n else:\n cmds = x11.init_cmds + cmds\n cmds = ' && '.join(cmds)\n # Sets the hostname to the original experiment's machine's\n # FIXME: not reentrant: this restores the Vagrant machine's hostname after\n # the run, which might cause issues if several \"reprounzip vagrant run\" are\n # running at once\n cmds = ('OLD_HOSTNAME=$(/bin/hostname); /bin/hostname %s; ' % hostname +\n cmds +\n '; RES=$?; /bin/hostname \"$OLD_HOSTNAME\"; exit $RES')\n cmds = '/usr/bin/sudo /bin/sh -c %s' % shell_escape(cmds)\n\n # Gets vagrant SSH parameters\n info = machine_setup(target)\n\n signals.pre_run(target=target)\n\n interactive = not (args.no_stdin or\n os.environ.get('REPROUNZIP_NON_INTERACTIVE'))\n retcode = run_interactive(info, interactive,\n cmds,\n not args.no_pty,\n x11.port_forward)\n stderr.write(\"\\r\\n*** Command finished, status: %d\\r\\n\" % retcode)\n\n # Update input file status\n metadata_update_run(config, unpacked_info, selected_runs)\n write_dict(target, unpacked_info)\n\n signals.post_run(target=target, retcode=retcode)\n\n\nclass SSHUploader(FileUploader):\n def __init__(self, target, input_files, files, use_chroot):\n self.use_chroot = use_chroot\n FileUploader.__init__(self, target, input_files, files)\n\n def prepare_upload(self, files):\n # Checks whether the VM is running\n try:\n ssh_info = machine_setup(self.target)\n except subprocess.CalledProcessError:\n logger.critical(\"Failed to get the status of the machine -- is \"\n \"it running?\")\n sys.exit(1)\n\n # Connect with SSH\n self.ssh = paramiko.SSHClient()\n self.ssh.set_missing_host_key_policy(IgnoreMissingKey())\n self.ssh.connect(**ssh_info)\n\n def upload_file(self, local_path, input_path):\n if self.use_chroot:\n remote_path = join_root(PosixPath('/experimentroot'),\n input_path)\n else:\n remote_path = input_path\n\n temp = make_unique_name(b'reprozip_input_')\n ltemp = self.target / temp\n rtemp = PosixPath('/vagrant') / temp\n\n # Copy file to shared folder\n logger.info(\"Copying file to shared folder...\")\n local_path.copyfile(ltemp)\n\n # Move it\n logger.info(\"Moving file into place...\")\n chan = self.ssh.get_transport().open_session()\n chown_cmd = '/bin/chown --reference=%s %s' % (\n shell_escape(remote_path.path),\n shell_escape(rtemp.path))\n chmod_cmd = '/bin/chmod --reference=%s %s' % (\n shell_escape(remote_path.path),\n shell_escape(rtemp.path))\n mv_cmd = '/bin/mv %s %s' % (\n shell_escape(rtemp.path),\n shell_escape(remote_path.path))\n chan.exec_command('/usr/bin/sudo /bin/sh -c %s' % shell_escape(\n ' && '.join((chown_cmd, chmod_cmd, mv_cmd))))\n if chan.recv_exit_status() != 0:\n logger.critical(\"Couldn't move file in virtual machine\")\n try:\n ltemp.remove()\n except OSError:\n pass\n sys.exit(1)\n chan.close()\n\n def finalize(self):\n self.ssh.close()\n\n\n@target_must_exist\ndef vagrant_upload(args):\n \"\"\"Replaces an input file in the VM.\n \"\"\"\n target = Path(args.target[0])\n files = args.file\n unpacked_info = read_dict(target)\n input_files = unpacked_info.setdefault('input_files', {})\n use_chroot = unpacked_info['use_chroot']\n\n try:\n SSHUploader(target, input_files, files, use_chroot)\n finally:\n write_dict(target, unpacked_info)\n\n\nclass SSHDownloader(FileDownloader):\n def __init__(self, target, files, use_chroot, all_=False):\n self.use_chroot = use_chroot\n FileDownloader.__init__(self, target, files, all_=all_)\n\n def prepare_download(self, files):\n # Checks whether the VM is running\n try:\n info = machine_setup(self.target)\n except subprocess.CalledProcessError:\n logger.critical(\"Failed to get the status of the machine -- is \"\n \"it running?\")\n sys.exit(1)\n\n # Connect with SSH\n self.ssh = paramiko.SSHClient()\n self.ssh.set_missing_host_key_policy(IgnoreMissingKey())\n self.ssh.connect(**info)\n\n def download(self, remote_path, local_path):\n if self.use_chroot:\n remote_path = join_root(PosixPath('/experimentroot'), remote_path)\n\n temp = make_unique_name(b'reprozip_output_')\n rtemp = PosixPath('/vagrant') / temp\n ltemp = self.target / temp\n\n # Copy file to shared folder\n logger.info(\"Copying file to shared folder...\")\n chan = self.ssh.get_transport().open_session()\n cp_cmd = '/bin/cp %s %s' % (\n shell_escape(remote_path.path),\n shell_escape(rtemp.path))\n chown_cmd = '/bin/chown vagrant %s' % shell_escape(rtemp.path)\n chmod_cmd = '/bin/chmod 644 %s' % shell_escape(rtemp.path)\n chan.exec_command('/usr/bin/sudo /bin/sh -c %s' % shell_escape(\n ' && '.join((cp_cmd, chown_cmd, chmod_cmd))))\n if chan.recv_exit_status() != 0:\n logger.critical(\"Couldn't copy file in virtual machine\")\n try:\n ltemp.remove()\n except OSError:\n pass\n return False\n\n # Move file to final destination\n try:\n ltemp.move(local_path)\n except OSError as e:\n logger.critical(\"Couldn't download output file: %s\\n%s\",\n remote_path, str(e))\n ltemp.remove()\n return False\n return True\n\n def finalize(self):\n self.ssh.close()\n\n\n@target_must_exist\ndef vagrant_download(args):\n \"\"\"Gets an output file out of the VM.\n \"\"\"\n target = Path(args.target[0])\n files = args.file\n use_chroot = read_dict(target)['use_chroot']\n\n SSHDownloader(target, files, use_chroot, all_=args.all)\n\n\n@target_must_exist\ndef vagrant_suspend(args):\n \"\"\"Suspends the VM through Vagrant, without destroying it.\n \"\"\"\n target = Path(args.target[0])\n\n retcode = subprocess.call(['vagrant', 'suspend'], cwd=target.path)\n if retcode != 0:\n logger.critical(\"vagrant suspend failed with code %d, ignoring...\",\n retcode)\n\n\n@target_must_exist\ndef vagrant_destroy_vm(args):\n \"\"\"Destroys the VM through Vagrant.\n \"\"\"\n target = Path(args.target[0])\n read_dict(target)\n\n retcode = subprocess.call(['vagrant', 'destroy', '-f'], cwd=target.path)\n if retcode != 0:\n logger.critical(\"vagrant destroy failed with code %d, ignoring...\",\n retcode)\n\n\n@target_must_exist\ndef vagrant_destroy_dir(args):\n \"\"\"Destroys the directory.\n \"\"\"\n target = Path(args.target[0])\n read_dict(target)\n\n signals.pre_destroy(target=target)\n target.rmtree()\n signals.post_destroy(target=target)\n\n\ndef _executable_in_path(executable):\n pathlist = os.environ['PATH'].split(os.pathsep) + ['.']\n pathexts = os.environ.get('PATHEXT', '').split(os.pathsep)\n for path in pathlist:\n for ext in pathexts:\n fullpath = os.path.join(path, executable) + ext\n if os.path.isfile(fullpath):\n return True\n return False\n\n\ndef check_vagrant_version():\n try:\n out = subprocess.check_output(['vagrant', '--version'])\n except (subprocess.CalledProcessError, OSError):\n logger.error(\"Couldn't run vagrant\")\n sys.exit(1)\n out = out.decode('ascii').strip().lower().split()\n if out[0] == 'vagrant':\n if LooseVersion(out[1]) < LooseVersion('1.1'):\n logger.error(\"Vagrant >=1.1 is required; detected version: %s\",\n out[1])\n sys.exit(1)\n else:\n logger.error(\"Vagrant >=1.1 is required\")\n sys.exit(1)\n\n\ndef test_has_vagrant(pack, **kwargs):\n \"\"\"Compatibility test: has vagrant (ok) or not (maybe).\n \"\"\"\n if not _executable_in_path('vagrant'):\n return COMPAT_MAYBE, \"vagrant not found in PATH\"\n\n try:\n out = subprocess.check_output(['vagrant', '--version'])\n except subprocess.CalledProcessError:\n return COMPAT_NO, (\"vagrant was found in PATH but doesn't seem to \"\n \"work properly\")\n out = out.decode('ascii').strip().lower().split()\n if out[0] == 'vagrant':\n if LooseVersion(out[1]) >= LooseVersion('1.1'):\n return COMPAT_OK\n else:\n return COMPAT_NO, (\"Vagrant >=1.1 is required; detected version: \"\n \"%s\" % out[1])\n else:\n return COMPAT_NO, \"Vagrant >=1.1 is required\"\n\n\ndef setup(parser, **kwargs):\n \"\"\"Runs the experiment in a virtual machine created through Vagrant\n\n You will need Vagrant to be installed on your machine if you want to run\n the experiment.\n\n setup setup/create creates Vagrantfile (needs the pack filename)\n setup/start starts or resume the virtual machine\n upload replaces input files in the machine\n (without arguments, lists input files)\n run runs the experiment in the virtual machine\n suspend suspend the virtual machine without destroying it\n download gets output files from the machine\n (without arguments, lists output files)\n destroy destroy/vm destroys the virtual machine\n destroy/dir removes the unpacked directory\n\n For example:\n\n $ reprounzip vagrant setup mypack.rpz experiment; cd experiment\n $ reprounzip vagrant run .\n $ reprounzip vagrant download . results:/home/user/theresults.txt\n $ cd ..; reprounzip vagrant destroy experiment\n\n Upload specifications are either:\n :input_id restores the original input file from the pack\n filename:input_id replaces the input file with the specified local\n file\n\n Download specifications are either:\n output_id: print the output file to stdout\n output_id:filename extracts the output file to the corresponding local\n path\n \"\"\"\n subparsers = parser.add_subparsers(title=\"actions\",\n metavar='', help=argparse.SUPPRESS)\n\n def add_opt_general(opts):\n opts.add_argument('target', nargs=1, help=\"Experiment directory\")\n\n # setup/create\n def add_opt_setup(opts):\n opts.add_argument('pack', nargs=1, help=\"Pack to extract\")\n opts.add_argument(\n '--use-chroot', action='store_true',\n default=True,\n help=argparse.SUPPRESS)\n opts.add_argument(\n '--dont-use-chroot', action='store_false', dest='use_chroot',\n default=True,\n help=\"Don't prefer original files nor use chroot in the virtual \"\n \"machine\")\n opts.add_argument(\n '--no-use-chroot', action='store_false', dest='use_chroot',\n default=True, help=argparse.SUPPRESS)\n opts.add_argument(\n '--dont-bind-magic-dirs', action='store_false', default=True,\n dest='bind_magic_dirs',\n help=\"Don't mount /dev and /proc inside the chroot (no effect if \"\n \"--dont-use-chroot is set)\")\n opts.add_argument('--base-image', nargs=1, help=\"Vagrant box to use\")\n opts.add_argument('--distribution', nargs=1,\n help=\"Distribution used in the Vagrant box (for \"\n \"package installer selection)\")\n opts.add_argument('--memory', nargs=1,\n help=\"Amount of RAM to allocate to VM (megabytes, \"\n \"default: box default)\")\n opts.add_argument('--use-gui', action='store_true', default=False,\n dest='gui', help=\"Use the VM's X server\")\n opts.add_argument('--expose-port', '-p', action='append', default=[],\n help=\"Expose a network port, \"\n \"host[:experiment[/proto]]. Example: 8000:80\")\n\n parser_setup_create = subparsers.add_parser('setup/create')\n add_opt_setup(parser_setup_create)\n add_opt_general(parser_setup_create)\n parser_setup_create.set_defaults(func=vagrant_setup_create)\n\n # setup/start\n parser_setup_start = subparsers.add_parser('setup/start')\n add_opt_general(parser_setup_start)\n parser_setup_start.set_defaults(func=vagrant_setup_start)\n\n # setup\n parser_setup = subparsers.add_parser('setup')\n add_opt_setup(parser_setup)\n add_opt_general(parser_setup)\n parser_setup.set_defaults(func=composite_action(vagrant_setup_create,\n vagrant_setup_start))\n\n # upload\n parser_upload = subparsers.add_parser('upload')\n add_opt_general(parser_upload)\n parser_upload.add_argument('file', nargs=argparse.ZERO_OR_MORE,\n help=\":\")\n parser_upload.set_defaults(func=vagrant_upload)\n\n # run\n parser_run = subparsers.add_parser('run')\n add_opt_general(parser_run)\n parser_run.add_argument('run', default=None, nargs=argparse.OPTIONAL)\n parser_run.add_argument('--no-stdin', action='store_true', default=False,\n help=\"Don't connect program's input stream to \"\n \"this terminal\")\n parser_run.add_argument('--no-pty', action='store_true', default=False,\n help=\"Don't request a PTY from the SSH server\")\n parser_run.add_argument('--cmdline', nargs=argparse.REMAINDER,\n help=\"Command line to run\")\n parser_run.add_argument('--expose-port', '-p', action='append', default=[],\n help=\"Expose a network port, \"\n \"host[:experiment[/proto]]. Example: 8000:80\")\n parser_run.add_argument('--enable-x11', action='store_true', default=False,\n dest='x11',\n help=\"Enable X11 support (needs an X server on \"\n \"the host)\")\n parser_run.add_argument('--x11-display', dest='x11_display',\n help=\"Display number to use on the experiment \"\n \"side (change the host display with the \"\n \"DISPLAY environment variable)\")\n add_environment_options(parser_run)\n parser_run.set_defaults(func=vagrant_run)\n\n # download\n parser_download = subparsers.add_parser('download')\n add_opt_general(parser_download)\n parser_download.add_argument('file', nargs=argparse.ZERO_OR_MORE,\n help=\"[:]\")\n parser_download.add_argument('--all', action='store_true',\n help=\"Download all output files to the \"\n \"current directory\")\n parser_download.set_defaults(func=vagrant_download)\n\n parser_suspend = subparsers.add_parser('suspend')\n add_opt_general(parser_suspend)\n parser_suspend.set_defaults(func=vagrant_suspend)\n\n # destroy/vm\n parser_destroy_vm = subparsers.add_parser('destroy/vm')\n add_opt_general(parser_destroy_vm)\n parser_destroy_vm.set_defaults(func=vagrant_destroy_vm)\n\n # destroy/dir\n parser_destroy_dir = subparsers.add_parser('destroy/dir')\n add_opt_general(parser_destroy_dir)\n parser_destroy_dir.set_defaults(func=vagrant_destroy_dir)\n\n # destroy\n parser_destroy = subparsers.add_parser('destroy')\n add_opt_general(parser_destroy)\n parser_destroy.set_defaults(func=composite_action(vagrant_destroy_vm,\n vagrant_destroy_dir))\n\n return {'test_compatibility': test_has_vagrant}\n" }, { "path": "reprounzip-vagrant/reprounzip/unpackers/vagrant/interaction.py", "content": "# This is paramiko/demos/interactive.py\n# Part of the Paramiko project; https://github.com/paramiko/paramiko/\n# Adapted by Remi Rampin, New York University for ReproZip, 2015\n\n# Copyright (C) 2003-2007 Robey Pointer \n#\n# This file is part of paramiko.\n#\n# Paramiko is free software; you can redistribute it and/or modify it under the\n# terms of the GNU Lesser General Public License as published by the Free\n# Software Foundation; either version 2.1 of the License, or (at your option)\n# any later version.\n#\n# Paramiko is distributed in the hope that it will be useful, but WITHOUT ANY\n# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR\n# A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more\n# details.\n#\n# You should have received a copy of the GNU Lesser General Public License\n# along with Paramiko; if not, write to the Free Software Foundation, Inc.,\n# 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA.\n\n\nfrom __future__ import division, print_function, unicode_literals\n\nimport socket\n\nfrom reprounzip.utils import stdout, stdout_bytes, stderr_bytes, stdin_bytes\n\n# windows does not have termios...\ntry:\n import termios\n import tty\n has_termios = True\nexcept ImportError:\n has_termios = False\n\n\ndef interactive_shell(chan, raw=True):\n if has_termios:\n posix_shell(chan, raw)\n else:\n windows_shell(chan)\n\n\ndef posix_shell(chan, raw):\n # set signal somehow\n import select\n\n oldtty = termios.tcgetattr(stdin_bytes)\n try:\n if raw:\n tty.setraw(stdin_bytes.fileno())\n tty.setcbreak(stdin_bytes.fileno())\n chan.settimeout(0.0)\n\n while True:\n r, w, e = select.select([chan, stdin_bytes], [], [])\n if chan in r:\n try:\n if chan.recv_stderr_ready():\n x = chan.recv_stderr(1024)\n if len(x) > 0:\n stderr_bytes.write(x)\n stderr_bytes.flush()\n else:\n x = chan.recv(1024)\n if len(x) == 0:\n break\n stdout_bytes.write(x)\n stdout_bytes.flush()\n except socket.timeout:\n pass\n if stdin_bytes in r:\n x = stdin_bytes.read(1)\n if len(x) == 0:\n break\n chan.send(x)\n\n finally:\n if raw:\n termios.tcsetattr(stdin_bytes, termios.TCSADRAIN, oldtty)\n\n\n# thanks to Mike Looijmans for this code\ndef windows_shell(chan):\n # set signal somehow\n import threading\n\n stdout.write(\"*** Emulating terminal on Windows; press F6 or Ctrl+Z then \"\n \"enter to send EOF,\\r\\nor at the end of the execution.\\r\\n\")\n stdout.flush()\n\n out_lock = threading.RLock()\n\n def write(recv, std):\n while True:\n data = recv(256)\n if not data:\n if std:\n with out_lock:\n stdout.write(\n \"\\r\\n*** EOF reached; (press F6 or ^Z then enter \"\n \"to end)\\r\\n\")\n stdout.flush()\n break\n stream = [stderr_bytes, stdout_bytes][std]\n with out_lock:\n stream.write(data)\n stream.flush()\n\n threading.Thread(target=write, args=(chan.recv, True)).start()\n threading.Thread(target=write, args=(chan.recv_stderr, False,)).start()\n\n try:\n while True:\n d = stdin_bytes.read(1)\n if not d:\n chan.shutdown_write()\n break\n try:\n chan.send(d)\n except socket.error:\n break\n except EOFError:\n # user hit ^Z or F6\n pass\n" }, { "path": "reprounzip-vagrant/reprounzip/unpackers/vagrant/run_command.py", "content": "# Copyright (C) 2014 New York University\n# This file is part of ReproZip which is released under the Revised BSD License\n# See file LICENSE for full license details.\n\n\"\"\"SSH command runner.\n\nThis contains `run_interactive()`, used to run a command via SSH.\n\"\"\"\n\nfrom __future__ import division, print_function, unicode_literals\n\nimport logging\nimport os\nimport paramiko\nfrom paramiko.client import MissingHostKeyPolicy\nimport sys\nimport threading\n\nfrom reprounzip.common import record_usage\nfrom reprounzip.unpackers.common import interruptible_call\nfrom reprounzip.unpackers.common.x11 import BaseForwarder, LocalForwarder\nfrom reprounzip.unpackers.vagrant.interaction import interactive_shell\nfrom reprounzip.utils import irange, stdout_bytes\n\n\nlogger = logging.getLogger('reprounzip.vagrant')\n\n\nclass IgnoreMissingKey(MissingHostKeyPolicy):\n \"\"\"Policy that just ignores missing SSH host keys.\n\n We are connecting to vagrant, checking the host doesn't make sense, and\n accepting keys permanently is a security risk.\n \"\"\"\n def missing_host_key(self, client, hostname, key):\n pass\n\n\ndef find_ssh_executable(name='ssh'):\n exts = os.environ.get('PATHEXT', '').split(os.pathsep)\n dirs = list(os.environ.get('PATH', '').split(os.pathsep))\n par, join = os.path.dirname, os.path.join\n # executable might be bin/python or ReproUnzip\\python\n # or ReproUnzip\\Python27\\python or ReproUnzip\\Python27\\Scripts\\something\n loc = par(sys.executable)\n local_dirs = []\n for i in irange(3):\n local_dirs.extend([loc, join(loc, 'ssh')])\n loc = par(loc)\n for pathdir in local_dirs + dirs:\n for ext in exts:\n fullpath = os.path.join(pathdir, name + ext)\n if os.path.isfile(fullpath):\n return fullpath\n return None\n\n\nclass SSHForwarder(BaseForwarder):\n \"\"\"Gets a remote port from paramiko and forwards to the given connector.\n\n The `connector` is a function which takes the address of remote process\n connecting on the port on the SSH server, and gives out a socket object\n that is the second endpoint of the tunnel. The socket object must provide\n ``recv()``, ``sendall()`` and ``close()``.\n \"\"\"\n def __init__(self, ssh_transport, remote_port, connector):\n BaseForwarder.__init__(self, connector)\n ssh_transport.request_port_forward('', remote_port,\n self._new_connection)\n\n class _ChannelWrapper(object):\n def __init__(self, channel):\n self.channel = channel\n\n def sendall(self, data):\n return self.channel.send(data)\n\n def recv(self, data):\n return self.channel.recv(data)\n\n def close(self):\n self.channel.close()\n\n def _new_connection(self, channel, src_addr, dest_addr):\n # Wraps the channel as a socket-like object that _forward() can use\n socklike = self._ChannelWrapper(channel)\n t = threading.Thread(target=self._forward,\n args=(socklike, src_addr))\n t.setDaemon(True)\n t.start()\n\n\ndef run_interactive(ssh_info, interactive, cmd, request_pty, forwarded_ports):\n \"\"\"Runs a command on an SSH server.\n\n If `interactive` is True, we'll try to find an ``ssh`` executable, falling\n back to paramiko if it's not found. The terminal handling code is a bit\n wonky, so using ``ssh`` is definitely a good idea, especially on Windows.\n Non-interactive commands should run fine.\n\n :param ssh_info: dict with `hostname`, `port`, `username`, `key_filename`,\n passed directly to paramiko\n :type ssh_info: dict\n :param interactive: whether to connect local input to the remote process\n :type interactive: bool\n :param cmd: command-line to run on the server\n :type cmd: str\n :param request_pty: whether to request a PTY from the SSH server\n :type request_pty: bool\n :param forwarded_ports: ports to forward back to us; iterable of pairs\n ``(port_number, connector)`` where `port_number` is the remote port number\n and `connector` is the connector object used to build the connected socket\n to forward to on this side\n :type forwarded_ports: collections.Iterable[(int, object)]\n \"\"\"\n if interactive:\n ssh_exe = find_ssh_executable()\n else:\n ssh_exe = None\n\n if interactive and ssh_exe:\n record_usage(vagrant_ssh='ssh')\n args = [ssh_exe,\n '-t' if request_pty else '-T', # Force allocation of PTY\n '-o', 'StrictHostKeyChecking=no', # Silently accept host keys\n '-o', 'UserKnownHostsFile=/dev/null', # Don't store host keys\n '-o', 'LogLevel=FATAL', # Hide messages\n '-o', 'PasswordAuthentication=no', # Login using key\n '-i', ssh_info['key_filename'],\n '-p', '%d' % ssh_info['port']]\n for remote_port, connector in forwarded_ports:\n # Remote port will connect to a local port\n fwd = LocalForwarder(connector)\n args.append('-R%d:127.0.0.1:%d' % (remote_port, fwd.local_port))\n args.append('%s@%s' % (ssh_info['username'],\n ssh_info['hostname']))\n args.append(cmd)\n return interruptible_call(args)\n\n else:\n record_usage(vagrant_ssh='interactive' if interactive else 'simple')\n # Connects to the machine\n ssh = paramiko.SSHClient()\n ssh.set_missing_host_key_policy(IgnoreMissingKey())\n ssh.connect(**ssh_info)\n\n # Starts forwarding\n forwarders = []\n for remote_port, connector in forwarded_ports:\n forwarders.append(\n SSHForwarder(ssh.get_transport(), remote_port, connector))\n\n chan = ssh.get_transport().open_session()\n if request_pty:\n chan.get_pty()\n\n # Execute command\n logger.info(\"Connected via SSH, running command...\")\n chan.exec_command(cmd)\n\n # Get output\n if interactive:\n interactive_shell(chan)\n else:\n chan.shutdown_write()\n while True:\n data = chan.recv(1024)\n if len(data) == 0:\n break\n stdout_bytes.write(data)\n stdout_bytes.flush()\n retcode = chan.recv_exit_status()\n ssh.close()\n return retcode\n" }, { "path": "reprounzip-vagrant/setup.cfg", "content": "[bdist_wheel]\nuniversal = 1\n" }, { "path": "reprounzip-vagrant/setup.py", "content": "import io\nimport os\nfrom setuptools import setup\n\n\n# pip workaround\nos.chdir(os.path.abspath(os.path.dirname(__file__)))\n\n\n# Need to specify encoding for PY3, which has the worst unicode handling ever\nwith io.open('README.rst', encoding='utf-8') as fp:\n description = fp.read()\nsetup(name='reprounzip-vagrant',\n version='1.2',\n packages=['reprounzip', 'reprounzip.unpackers',\n 'reprounzip.unpackers.vagrant'],\n entry_points={\n 'reprounzip.unpackers': [\n 'vagrant = reprounzip.unpackers.vagrant:setup']},\n namespace_packages=['reprounzip', 'reprounzip.unpackers'],\n install_requires=[\n 'reprounzip>=1.1',\n 'rpaths>=0.8',\n 'paramiko'],\n description=\"Allows the ReproZip unpacker to create virtual machines\",\n author=\"Remi Rampin, Fernando Chirigati, Dennis Shasha, Juliana Freire\",\n author_email='reprozip@nyu.edu',\n maintainer=\"Remi Rampin\",\n maintainer_email='remi@rampin.org',\n url='https://www.reprozip.org/',\n project_urls={\n 'Documentation': 'https://docs.reprozip.org/',\n 'Examples': 'https://examples.reprozip.org/',\n 'Source': 'https://github.com/VIDA-NYU/reprozip',\n 'Bug Tracker': 'https://github.com/VIDA-NYU/reprozip/issues',\n 'Chat': 'https://riot.im/app/#/room/#reprozip:matrix.org',\n 'Changelog':\n 'https://github.com/VIDA-NYU/reprozip/blob/1.x/CHANGELOG.md',\n },\n long_description=description,\n license='BSD-3-Clause',\n keywords=['reprozip', 'reprounzip', 'reproducibility', 'provenance',\n 'vida', 'nyu', 'vagrant'],\n classifiers=[\n 'Development Status :: 5 - Production/Stable',\n 'Intended Audience :: Science/Research',\n 'License :: OSI Approved :: BSD License',\n 'Programming Language :: Python :: 2.7',\n 'Programming Language :: Python :: 3',\n 'Topic :: Scientific/Engineering',\n 'Topic :: System :: Archiving'])\n" }, { "path": "reprounzip-vistrails/LICENSE.txt", "content": "Copyright (C) 2014, New York University\nAll rights reserved.\n\nRedistribution and use in source and binary forms, with or without\nmodification, are permitted provided that the following conditions are met:\n\n1. Redistributions of source code must retain the above copyright notice, this\n list of conditions and the following disclaimer.\n\n2. Redistributions in binary form must reproduce the above copyright notice,\n this list of conditions and the following disclaimer in the documentation\n and/or other materials provided with the distribution.\n\n3. Neither the name of the copyright holder nor the names of its contributors\n may be used to endorse or promote products derived from this software\n without specific prior written permission.\n\nTHIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \"AS IS\" AND\nANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED\nWARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE\nDISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE\nFOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL\nDAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR\nSERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER\nCAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,\nOR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE\nOF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n" }, { "path": "reprounzip-vistrails/MANIFEST.in", "content": "include README.rst\ninclude LICENSE.txt\n" }, { "path": "reprounzip-vistrails/README.rst", "content": "ReproZip project\n================\n\n`ReproZip `__ is a tool aimed at simplifying the process of creating reproducible experiments from command-line executions, a frequently-used common denominator in computational science. It tracks operating system calls and creates a bundle that contains all the binaries, files and dependencies required to run a given command on the author's computational environment (packing step). A reviewer can then extract the experiment in his environment to reproduce the results (unpacking step).\n\nreprounzip-vistrails\n--------------------\n\nThis is a plugin that creates VisTrails workflows from unpacked experiments. See the `VisTrails website `_ for more information about VisTrails.\n\nPlease refer to `reprozip `__ and `reprounzip `_ for other components.\n\nAdditional Information\n----------------------\n\nFor more detailed information, please refer to our `website `_, as well as to our `documentation `_.\n\nReproZip is currently being developed at `NYU `_. The team includes:\n\n* `Fernando Chirigati `_\n* `Juliana Freire `_\n* `Remi Rampin `_\n* `Dennis Shasha `_\n* `Vicky Rampin `_\n" }, { "path": "reprounzip-vistrails/reprounzip/__init__.py", "content": "try: # pragma: no cover\n __import__('pkg_resources').declare_namespace(__name__)\nexcept ImportError: # pragma: no cover\n from pkgutil import extend_path\n __path__ = extend_path(__path__, __name__)\n" }, { "path": "reprounzip-vistrails/reprounzip/plugins/__init__.py", "content": "try: # pragma: no cover\n __import__('pkg_resources').declare_namespace(__name__)\nexcept ImportError: # pragma: no cover\n from pkgutil import extend_path\n __path__ = extend_path(__path__, __name__)\n" }, { "path": "reprounzip-vistrails/reprounzip/plugins/vistrails.py", "content": "# Copyright (C) 2014 New York University\n# This file is part of ReproZip which is released under the Revised BSD License\n# See file LICENSE for full license details.\n\n\"\"\"VisTrails runner for reprounzip.\n\nThis file provides the reprounzip plugin that builds a VisTrails pipeline\nalongside an unpacked experiment. Although you don't need VisTrails to generate\nthe .vt file, you will need it if you want to run it.\n\nSee http://www.vistrails.org/\n\"\"\"\n\nfrom __future__ import division, print_function, unicode_literals\n\nif __name__ == '__main__': # noqa\n from reprounzip.plugins.vistrails import run_from_vistrails\n run_from_vistrails()\n\nimport argparse\nfrom datetime import datetime\nimport itertools\nimport logging\nimport os\nfrom rpaths import Path\nimport subprocess\nimport sys\nimport zipfile\n\nfrom reprounzip.common import load_config, setup_logging, record_usage\nfrom reprounzip import signals\nfrom reprounzip.unpackers.common import shell_escape\nfrom reprounzip.utils import iteritems\n\n\n__version__ = '1.0.16'\n\n\nlogger = logging.getLogger('reprounzip.vistrails')\n\n\ndef escape_xml(s):\n \"\"\"Escapes for XML.\n \"\"\"\n return (\"%s\" % s).replace('&', '&').replace('\"', '"')\n\n\nclass IdScope(object):\n def __init__(self):\n self._ids = {'add': 0,\n 'module': 0,\n 'location': 0,\n 'annotation': 0,\n 'function': 0,\n 'parameter': 0,\n 'connection': 0,\n 'port': 0,\n 'portspec': 0,\n 'portspecitem': 0}\n\n def _add(type_):\n def getter(self):\n i = self._ids[type_]\n self._ids[type_] += 1\n return i\n return getter\n\n add = _add('add')\n module = _add('module')\n location = _add('location')\n annotation = _add('annotation')\n function = _add('function')\n parameter = _add('parameter')\n connection = _add('connection')\n port = _add('port')\n portspec = _add('portspec')\n portspecitem = _add('portspecitem')\n\n del _add\n\n\ndef split_sig(sig):\n pkg, name = sig.rsplit(':', 1)\n return name, pkg\n\n\nclass Workflow(object):\n def __init__(self, file_, ids):\n self._file = file_\n self._ids = ids\n self._mod_y = 0\n\n file_.write('\\n'\n '\\n'\n ' \\n'.format(\n date=datetime.now().strftime('%Y-%m-%d %H:%M:%S'),\n version=__version__))\n\n def close(self):\n self._file.write(\n ' \\n'\n '\\n')\n\n def add_module(self, sig, version, desc=None):\n mod_id = self._ids.module()\n name, pkg = split_sig(sig)\n self._file.write(\n ' \\n'\n ' \\n'\n ' \\n'.format(\n add_id=self._ids.add(), mod_id=mod_id,\n mod_name=name, mod_pkg=pkg, version=version))\n self._file.write(\n ' \\n'\n ' \\n'\n ' \\n'.format(\n add_id=self._ids.add(), mod_id=mod_id,\n loc_id=self._ids.location(), y=self._mod_y))\n if desc is not None:\n self._file.write(\n ' \\n'\n ' \\n'\n ' \\n'.format(\n add_id=self._ids.add(), mod_id=mod_id,\n ann_id=self._ids.annotation(), text=escape_xml(desc)))\n self._mod_y -= 100\n return mod_id\n\n def add_function(self, mod_id, name, param_values):\n func_id = self._ids.function()\n self._file.write(\n ' \\n'\n ' \\n'\n ' \\n'.format(\n add_id=self._ids.add(), mod_id=mod_id, func_id=func_id,\n name=name))\n\n for i, (sig, val) in enumerate(param_values):\n self._file.write(\n ' \\n'\n ' \\n'\n ' \\n'.format(\n add_id=self._ids.add(), param_id=self._ids.parameter(),\n func_id=func_id, pos=i, type=sig, val=escape_xml(val)))\n\n def connect(self, from_id, from_sig, from_port, to_id, to_sig, to_port):\n self._file.write(\n ' \\n'\n ' \\n'\n ' \\n'\n ' \\n'\n ' \\n'\n ' \\n'\n ' \\n'\n ' \\n'\n ' \\n'.format(\n add1_id=self._ids.add(), add2_id=self._ids.add(),\n add3_id=self._ids.add(), conn_id=self._ids.connection(),\n port1_id=self._ids.port(), from_id=from_id, from_sig=from_sig,\n from_mod=split_sig(from_sig)[0], from_port=from_port,\n port2_id=self._ids.port(), to_id=to_id, to_sig=to_sig,\n to_mod=split_sig(to_sig)[0], to_port=to_port\n ))\n\n def add_port_spec(self, mod_id, name, type_, sigs, optional=True):\n self._file.write(\n ' \\n'\n ' \\n'.format(\n add_id=self._ids.add(), ps_id=self._ids.portspec(),\n mod_id=mod_id, name=escape_xml(name), type=type_,\n opt='1' if optional else '0'))\n for i, (pkg, mod) in enumerate(sigs):\n self._file.write(\n ' \\n'.format(\n psi_id=self._ids.portspecitem(), mod=mod, pkg=pkg,\n pos=i))\n self._file.write(\n ' \\n'\n ' \\n')\n\n\ndirectory_sig = 'org.vistrails.vistrails.basic:Directory'\nfile_pkg_mod = 'org.vistrails.vistrails.basic', 'File'\ninteger_sig = 'org.vistrails.vistrails.basic:Integer'\nstring_sig = 'org.vistrails.vistrails.basic:String'\nrpz_id = 'io.github.vida-nyu.reprozip.reprounzip'\nrpz_version = '0.1'\nexperiment_sig = '%s:Directory' % rpz_id\n\n\ndef do_vistrails(target, pack=None, **kwargs):\n \"\"\"Create a VisTrails workflow that runs the experiment.\n\n This is called from signals after an experiment has been setup by any\n unpacker.\n \"\"\"\n record_usage(do_vistrails=True)\n\n config = load_config(target / 'config.yml', canonical=True)\n\n # Writes VisTrails workflow\n bundle = target / 'vistrails.vt'\n logger.info(\"Writing VisTrails workflow %s...\", bundle)\n vtdir = Path.tempdir(prefix='reprounzip_vistrails_')\n ids = IdScope()\n try:\n with vtdir.open('w', 'vistrail',\n encoding='utf-8', newline='\\n') as fp:\n wf = Workflow(fp, ids)\n\n # Directory module, refering to this directory\n d = wf.add_module('%s:Directory' % rpz_id, rpz_version)\n wf.add_function(d, 'directory',\n [(directory_sig, str(target.resolve()))])\n\n connect_from = d\n\n for i, run in enumerate(config.runs):\n inputs = sorted(n for n, f in iteritems(config.inputs_outputs)\n if i in f.read_runs)\n outputs = sorted(n for n, f in iteritems(config.inputs_outputs)\n if i in f.write_runs)\n ports = itertools.chain((('input', p) for p in inputs),\n (('output', p) for p in outputs))\n\n # Run module\n r = wf.add_module('%s:Run' % rpz_id, rpz_version,\n desc=run.get('id', 'run%d' % i))\n wf.add_function(r, 'cmdline', [\n (string_sig,\n ' '.join(shell_escape(arg)\n for arg in run['argv']))])\n wf.add_function(r, 'run_number', [(integer_sig, i)])\n\n # Port specs for input/output files\n for type_, name in ports:\n wf.add_port_spec(r, name, type_, [file_pkg_mod])\n\n # Draw connection\n wf.connect(connect_from, experiment_sig, 'experiment',\n r, experiment_sig, 'experiment')\n connect_from = r\n\n wf.close()\n\n with bundle.open('wb') as fp:\n z = zipfile.ZipFile(fp, 'w')\n with vtdir.in_dir():\n for path in Path('.').recursedir():\n z.write(str(path))\n z.close()\n finally:\n vtdir.rmtree()\n\n\ndef setup_vistrails():\n \"\"\"Setup the plugin.\n \"\"\"\n signals.post_setup.subscribe(do_vistrails)\n\n\ndef run_from_vistrails():\n setup_logging('REPROUNZIP-VISTRAILS', logging.INFO)\n\n cli_version = 1\n if len(sys.argv) > 1:\n try:\n cli_version = int(sys.argv[1])\n except ValueError:\n logger.info(\"Compatibility mode: reprounzip-vistrails didn't get \"\n \"a version number\")\n if cli_version != 1:\n logger.critical(\"Unknown interface version %d; you are probably \"\n \"using a version of reprounzip-vistrails too old for \"\n \"your VisTrails package. Consider upgrading.\",\n cli_version)\n sys.exit(1)\n\n parser = argparse.ArgumentParser()\n parser.add_argument('unpacker')\n parser.add_argument('directory')\n parser.add_argument('run')\n parser.add_argument('--input-file', action='append', default=[])\n parser.add_argument('--output-file', action='append', default=[])\n parser.add_argument('--cmdline', action='store')\n\n args = parser.parse_args(sys.argv[2:])\n\n config = load_config(Path(args.directory) / 'config.yml', canonical=True)\n\n python = sys.executable\n rpuz = [python, '-c', 'from reprounzip.main import main; main()',\n args.unpacker]\n\n os.environ['REPROUNZIP_NON_INTERACTIVE'] = 'y'\n\n def cmd(lst, add=None):\n if add:\n logger.info(\"cmd: %s %s\", ' '.join(rpuz + lst), add)\n string = ' '.join(shell_escape(a) for a in (rpuz + lst))\n string += ' ' + add\n subprocess.check_call(string, shell=True,\n cwd=args.directory)\n else:\n logger.info(\"cmd: %s\", ' '.join(rpuz + lst))\n subprocess.check_call(rpuz + lst,\n cwd=args.directory)\n\n logger.info(\"reprounzip-vistrails calling reprounzip; dir=%s\",\n args.directory)\n\n # Parses input files from the command-line\n upload_command = []\n seen_input_names = set()\n for input_file in args.input_file:\n input_name, filename = input_file.split(':', 1)\n upload_command.append('%s:%s' % (filename, input_name))\n seen_input_names.add(input_name)\n\n # Resets the input files that are used by this run and were not given\n for name, f in iteritems(config.inputs_outputs):\n if name not in seen_input_names and int(args.run) in f.read_runs:\n upload_command.append(':%s' % name)\n\n # Runs the command\n cmd(['upload', '.'] + upload_command)\n\n # Runs the experiment\n if args.cmdline:\n cmd(['run', '.', args.run, '--cmdline'], add=args.cmdline)\n else:\n cmd(['run', '.', args.run])\n\n # Gets output files\n for output_file in args.output_file:\n output_name, filename = output_file.split(':', 1)\n cmd(['download', '.',\n '%s:%s' % (output_name, filename)])\n\n sys.exit(0)\n" }, { "path": "reprounzip-vistrails/setup.cfg", "content": "[bdist_wheel]\nuniversal = 1\n" }, { "path": "reprounzip-vistrails/setup.py", "content": "import io\nimport os\nfrom setuptools import setup\nimport sys\n\n\n# pip workaround\nos.chdir(os.path.abspath(os.path.dirname(__file__)))\n\n\n# Need to specify encoding for PY3, which has the worst unicode handling ever\nwith io.open('README.rst', encoding='utf-8') as fp:\n description = fp.read()\nreq = [\n 'reprounzip>=1.0.0',\n 'rpaths>=0.8']\nif sys.version_info < (2, 7):\n req.append('argparse')\nsetup(name='reprounzip-vistrails',\n version='1.2',\n packages=['reprounzip', 'reprounzip.plugins'],\n entry_points={\n 'reprounzip.plugins': [\n 'vistrails = reprounzip.plugins.vistrails:setup_vistrails']},\n namespace_packages=['reprounzip', 'reprounzip.plugins'],\n install_requires=req,\n description=\"Integrates the ReproZip unpacker with the VisTrails \"\n \"workflow management system\",\n author=\"Remi Rampin, Fernando Chirigati, Dennis Shasha, Juliana Freire\",\n author_email='reprozip@nyu.edu',\n maintainer=\"Remi Rampin\",\n maintainer_email='remi@rampin.org',\n url='https://www.reprozip.org/',\n project_urls={\n 'Documentation': 'https://docs.reprozip.org/',\n 'Examples': 'https://examples.reprozip.org/',\n 'Source': 'https://github.com/VIDA-NYU/reprozip',\n 'Bug Tracker': 'https://github.com/VIDA-NYU/reprozip/issues',\n 'Chat': 'https://riot.im/app/#/room/#reprozip:matrix.org',\n 'Changelog':\n 'https://github.com/VIDA-NYU/reprozip/blob/1.x/CHANGELOG.md',\n },\n long_description=description,\n license='BSD-3-Clause',\n keywords=['reprozip', 'reprounzip', 'reproducibility', 'provenance',\n 'vida', 'nyu', 'vistrails'],\n classifiers=[\n 'Development Status :: 4 - Beta',\n 'Intended Audience :: Science/Research',\n 'License :: OSI Approved :: BSD License',\n 'Programming Language :: Python :: 2.7',\n 'Programming Language :: Python :: 3',\n 'Topic :: Scientific/Engineering',\n 'Topic :: System :: Archiving'])\n" }, { "path": "reprozip/LICENSE.txt", "content": "Copyright (C) 2014, New York University\nAll rights reserved.\n\nRedistribution and use in source and binary forms, with or without\nmodification, are permitted provided that the following conditions are met:\n\n1. Redistributions of source code must retain the above copyright notice, this\n list of conditions and the following disclaimer.\n\n2. Redistributions in binary form must reproduce the above copyright notice,\n this list of conditions and the following disclaimer in the documentation\n and/or other materials provided with the distribution.\n\n3. Neither the name of the copyright holder nor the names of its contributors\n may be used to endorse or promote products derived from this software\n without specific prior written permission.\n\nTHIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \"AS IS\" AND\nANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED\nWARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE\nDISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE\nFOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL\nDAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR\nSERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER\nCAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,\nOR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE\nOF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n" }, { "path": "reprozip/MANIFEST.in", "content": "include README.rst\ninclude LICENSE.txt\ngraft native\n" }, { "path": "reprozip/README.rst", "content": "ReproZip project\n================\n\n`ReproZip `__ is a tool aimed at simplifying the process of creating reproducible experiments from command-line executions, a frequently-used common denominator in computational science. It tracks operating system calls and creates a bundle that contains all the binaries, files and dependencies required to run a given command on the author's computational environment (packing step). A reviewer can then extract the experiment in his environment to reproduce the results (unpacking step).\n\nreprozip\n--------\n\nThis is the component responsible for the packing step on Linux distributions.\n\nPlease refer to `reprounzip `_, `reprounzip-vagrant `_, and `reprounzip-docker `_ for other components and plugins.\n\nAdditional Information\n----------------------\n\nFor more detailed information, please refer to our `website `_, as well as to our `documentation `_.\n\nReproZip is currently being developed at `NYU `_. The team includes:\n\n* `Fernando Chirigati `_\n* `Juliana Freire `_\n* `Remi Rampin `_\n* `Dennis Shasha `_\n* `Vicky Rampin `_\n" }, { "path": "reprozip/native/config.h", "content": "#ifndef CONFIG_H\n#define CONFIG_H\n\n#define WORD_SIZE sizeof(int)\n\n#if !defined(X86) && !defined(X86_64)\n# if defined(__x86_64__) || defined(__x86_64)\n# define X86_64\n# elif defined(__i386__) || defined(__i386) || defined(_M_I86) || defined(_M_IX86)\n# define I386\n# else\n# error Unrecognized architecture!\n# endif\n#endif\n\n/* Static assertion trick */\n#define STATIC_ASSERT(name, condition) \\\n enum { name = 1/(!!( \\\n condition \\\n )) }\n\nSTATIC_ASSERT(ASSERT_POINTER_FITS_IN_LONG_INT,\n sizeof(long int) >= sizeof(void*));\n\n#endif\n" }, { "path": "reprozip/native/database.c", "content": "#include \n#include \n#include \n#include \n#include \n#include \n\n#include \n\n#include \"database.h\"\n#include \"log.h\"\n\n#define count(x) (sizeof((x))/sizeof(*(x)))\n#define check(r) if((r) != SQLITE_OK) { goto sqlerror; }\n\nstatic sqlite3_uint64 gettime(void)\n{\n sqlite3_uint64 timestamp;\n struct timespec now;\n if(clock_gettime(CLOCK_MONOTONIC, &now) == -1)\n {\n /* LCOV_EXCL_START : clock_gettime() is unlikely to fail */\n log_critical(0, \"getting time failed (clock_gettime): %s\",\n strerror(errno));\n exit(125);\n /* LCOV_EXCL_STOP */\n }\n timestamp = now.tv_sec;\n timestamp *= 1000000000;\n timestamp += now.tv_nsec;\n return timestamp;\n}\n\nstatic sqlite3 *db;\nstatic sqlite3_stmt *stmt_last_rowid;\nstatic sqlite3_stmt *stmt_insert_process;\nstatic sqlite3_stmt *stmt_set_exitcode;\nstatic sqlite3_stmt *stmt_insert_file;\nstatic sqlite3_stmt *stmt_insert_exec;\n\nstatic int run_id = -1;\n\nint db_init(const char *filename)\n{\n int tables_exist;\n\n check(sqlite3_open(filename, &db));\n log_debug(0, \"database file opened: %s\", filename);\n\n check(sqlite3_exec(db, \"BEGIN IMMEDIATE;\", NULL, NULL, NULL))\n\n {\n int ret;\n const char *sql = \"\"\n \"SELECT name FROM SQLITE_MASTER \"\n \"WHERE type='table';\";\n sqlite3_stmt *stmt_get_tables;\n unsigned int found = 0x00;\n check(sqlite3_prepare_v2(db, sql, -1, &stmt_get_tables, NULL));\n while((ret = sqlite3_step(stmt_get_tables)) == SQLITE_ROW)\n {\n const char *colname = (const char*)sqlite3_column_text(\n stmt_get_tables, 0);\n if(strcmp(\"processes\", colname) == 0)\n found |= 0x01;\n else if(strcmp(\"opened_files\", colname) == 0)\n found |= 0x02;\n else if(strcmp(\"executed_files\", colname) == 0)\n found |= 0x04;\n else\n goto wrongschema;\n }\n if(found == 0x00)\n tables_exist = 0;\n else if(found == 0x07)\n tables_exist = 1;\n else\n {\n wrongschema:\n log_critical(0, \"database schema is wrong\");\n return -1;\n }\n sqlite3_finalize(stmt_get_tables);\n if(ret != SQLITE_DONE)\n goto sqlerror; /* LCOV_EXCL_LINE */\n }\n\n if(!tables_exist)\n {\n const char *sql[] = {\n \"CREATE TABLE processes(\"\n \" id INTEGER NOT NULL PRIMARY KEY,\"\n \" run_id INTEGER NOT NULL,\"\n \" parent INTEGER,\"\n \" timestamp INTEGER NOT NULL,\"\n \" is_thread BOOLEAN NOT NULL,\"\n \" exitcode INTEGER\"\n \" );\",\n \"CREATE INDEX proc_parent_idx ON processes(parent);\",\n \"CREATE TABLE opened_files(\"\n \" id INTEGER NOT NULL PRIMARY KEY,\"\n \" run_id INTEGER NOT NULL,\"\n \" name TEXT NOT NULL,\"\n \" timestamp INTEGER NOT NULL,\"\n \" mode INTEGER NOT NULL,\"\n \" is_directory BOOLEAN NOT NULL,\"\n \" process INTEGER NOT NULL\"\n \" );\",\n \"CREATE INDEX open_proc_idx ON opened_files(process);\",\n \"CREATE TABLE executed_files(\"\n \" id INTEGER NOT NULL PRIMARY KEY,\"\n \" name TEXT NOT NULL,\"\n \" run_id INTEGER NOT NULL,\"\n \" timestamp INTEGER NOT NULL,\"\n \" process INTEGER NOT NULL,\"\n \" argv TEXT NOT NULL,\"\n \" envp TEXT NOT NULL,\"\n \" workingdir TEXT NOT NULL\"\n \" );\",\n \"CREATE INDEX exec_proc_idx ON executed_files(process);\",\n };\n size_t i;\n for(i = 0; i < count(sql); ++i)\n check(sqlite3_exec(db, sql[i], NULL, NULL, NULL));\n }\n\n /* Get the first unused run_id */\n {\n sqlite3_stmt *stmt_get_run_id;\n const char *sql = \"SELECT max(run_id) + 1 FROM processes;\";\n check(sqlite3_prepare_v2(db, sql, -1, &stmt_get_run_id, NULL));\n if(sqlite3_step(stmt_get_run_id) != SQLITE_ROW)\n {\n /* LCOV_EXCL_START */\n sqlite3_finalize(stmt_get_run_id);\n goto sqlerror;\n /* LCOV_EXCL_STOP */\n }\n run_id = sqlite3_column_int(stmt_get_run_id, 0);\n if(sqlite3_step(stmt_get_run_id) != SQLITE_DONE)\n {\n /* LCOV_EXCL_START */\n sqlite3_finalize(stmt_get_run_id);\n goto sqlerror;\n /* LCOV_EXCL_STOP */\n }\n sqlite3_finalize(stmt_get_run_id);\n }\n log_debug(0, \"This is run %d\", run_id);\n\n {\n const char *sql = \"\"\n \"SELECT last_insert_rowid()\";\n check(sqlite3_prepare_v2(db, sql, -1, &stmt_last_rowid, NULL));\n }\n\n {\n const char *sql = \"\"\n \"INSERT INTO processes(run_id, parent, timestamp, is_thread)\"\n \"VALUES(?, ?, ?, ?)\";\n check(sqlite3_prepare_v2(db, sql, -1, &stmt_insert_process, NULL));\n }\n\n {\n const char *sql = \"\"\n \"UPDATE processes SET exitcode=?\"\n \"WHERE id=?\";\n check(sqlite3_prepare_v2(db, sql, -1, &stmt_set_exitcode, NULL));\n }\n\n {\n const char *sql = \"\"\n \"INSERT INTO opened_files(run_id, name, timestamp, \"\n \" mode, is_directory, process)\"\n \"VALUES(?, ?, ?, ?, ?, ?)\";\n check(sqlite3_prepare_v2(db, sql, -1, &stmt_insert_file, NULL));\n }\n\n {\n const char *sql = \"\"\n \"INSERT INTO executed_files(run_id, name, timestamp, process, \"\n \" argv, envp, workingdir)\"\n \"VALUES(?, ?, ?, ?, ?, ?, ?)\";\n check(sqlite3_prepare_v2(db, sql, -1, &stmt_insert_exec, NULL));\n }\n\n return 0;\n\n /* LCOV_EXCL_START */\nsqlerror:\n log_critical(0, \"sqlite3 error creating database: %s\", sqlite3_errmsg(db));\n return -1;\n /* LCOV_EXCL_STOP */\n}\n\nint db_close(int rollback)\n{\n if(rollback)\n {\n check(sqlite3_exec(db, \"ROLLBACK;\", NULL, NULL, NULL));\n }\n else\n {\n check(sqlite3_exec(db, \"COMMIT;\", NULL, NULL, NULL));\n }\n log_debug(0, \"database file closed%s\", rollback?\" (rolled back)\":\"\");\n check(sqlite3_finalize(stmt_last_rowid));\n check(sqlite3_finalize(stmt_insert_process));\n check(sqlite3_finalize(stmt_set_exitcode));\n check(sqlite3_finalize(stmt_insert_file));\n check(sqlite3_finalize(stmt_insert_exec));\n check(sqlite3_close(db));\n run_id = -1;\n return 0;\n\n /* LCOV_EXCL_START */\nsqlerror:\n log_critical(0, \"sqlite3 error on exit: %s\", sqlite3_errmsg(db));\n return -1;\n /* LCOV_EXCL_STOP */\n}\n\n#define DB_NO_PARENT ((unsigned int)-2)\n\nint db_add_process(unsigned int *id, unsigned int parent_id,\n const char *working_dir, int is_thread)\n{\n check(sqlite3_bind_int(stmt_insert_process, 1, run_id));\n if(parent_id == DB_NO_PARENT)\n {\n check(sqlite3_bind_null(stmt_insert_process, 2));\n }\n else\n {\n check(sqlite3_bind_int(stmt_insert_process, 2, parent_id));\n }\n /* This assumes that we won't go over 2^32 seconds (~135 years) */\n check(sqlite3_bind_int64(stmt_insert_process, 3, gettime()));\n check(sqlite3_bind_int(stmt_insert_process, 4, is_thread?1:0));\n\n if(sqlite3_step(stmt_insert_process) != SQLITE_DONE)\n goto sqlerror; /* LCOV_EXCL_LINE */\n sqlite3_reset(stmt_insert_process);\n\n /* Get id */\n if(sqlite3_step(stmt_last_rowid) != SQLITE_ROW)\n goto sqlerror; /* LCOV_EXCL_LINE */\n *id = sqlite3_column_int(stmt_last_rowid, 0);\n if(sqlite3_step(stmt_last_rowid) != SQLITE_DONE)\n goto sqlerror; /* LCOV_EXCL_LINE */\n sqlite3_reset(stmt_last_rowid);\n\n return db_add_file_open(*id, working_dir, FILE_WDIR, 1);\n\n /* LCOV_EXCL_START : Insertions shouldn't fail */\nsqlerror:\n log_critical(0, \"sqlite3 error inserting process: %s\", sqlite3_errmsg(db));\n return -1;\n /* LCOV_EXCL_STOP */\n}\n\nint db_add_first_process(unsigned int *id, const char *working_dir)\n{\n return db_add_process(id, DB_NO_PARENT, working_dir, 0);\n}\n\nint db_add_exit(unsigned int id, int exitcode)\n{\n check(sqlite3_bind_int(stmt_set_exitcode, 1, exitcode));\n check(sqlite3_bind_int(stmt_set_exitcode, 2, id));\n\n if(sqlite3_step(stmt_set_exitcode) != SQLITE_DONE)\n goto sqlerror; /* LCOV_EXCL_LINE */\n sqlite3_reset(stmt_set_exitcode);\n\n return 0;\n\n /* LCOV_EXCL_START : Insertions shouldn't fail */\nsqlerror:\n log_critical(0, \"sqlite3 error setting exitcode: %s\", sqlite3_errmsg(db));\n return -1;\n /* LCOV_EXCL_STOP */\n}\n\nint db_add_file_open(unsigned int process, const char *name,\n unsigned int mode, int is_dir)\n{\n check(sqlite3_bind_int(stmt_insert_file, 1, run_id));\n check(sqlite3_bind_text(stmt_insert_file, 2, name, -1, SQLITE_TRANSIENT));\n /* This assumes that we won't go over 2^32 seconds (~135 years) */\n check(sqlite3_bind_int64(stmt_insert_file, 3, gettime()));\n check(sqlite3_bind_int(stmt_insert_file, 4, mode));\n check(sqlite3_bind_int(stmt_insert_file, 5, is_dir));\n check(sqlite3_bind_int(stmt_insert_file, 6, process));\n\n if(sqlite3_step(stmt_insert_file) != SQLITE_DONE)\n goto sqlerror; /* LCOV_EXCL_LINE */\n sqlite3_reset(stmt_insert_file);\n return 0;\n\n /* LCOV_EXCL_START : Insertions shouldn't fail */\nsqlerror:\n log_critical(0, \"sqlite3 error inserting file: %s\", sqlite3_errmsg(db));\n return -1;\n /* LCOV_EXCL_STOP */\n}\n\nstatic char *strarray2nulsep(const char *const *array, size_t *plen)\n{\n char *list;\n size_t len = 0;\n {\n const char *const *a = array;\n while(*a)\n {\n len += strlen(*a) + 1;\n ++a;\n }\n }\n {\n const char *const *a = array;\n char *p;\n p = list = malloc(len);\n while(*a)\n {\n const char *s = *a;\n while(*s)\n *p++ = *s++;\n *p++ = '\\0';\n ++a;\n }\n }\n *plen = len;\n return list;\n}\n\nint db_add_exec(unsigned int process, const char *binary,\n const char *const *argv, const char *const *envp,\n const char *workingdir)\n{\n check(sqlite3_bind_int(stmt_insert_exec, 1, run_id));\n check(sqlite3_bind_text(stmt_insert_exec, 2, binary,\n -1, SQLITE_TRANSIENT));\n /* This assumes that we won't go over 2^32 seconds (~135 years) */\n check(sqlite3_bind_int64(stmt_insert_exec, 3, gettime()));\n check(sqlite3_bind_int(stmt_insert_exec, 4, process));\n {\n size_t len;\n char *arglist = strarray2nulsep(argv, &len);\n check(sqlite3_bind_text(stmt_insert_exec, 5, arglist, len,\n SQLITE_TRANSIENT));\n free(arglist);\n }\n {\n size_t len;\n char *envlist = strarray2nulsep(envp, &len);\n check(sqlite3_bind_text(stmt_insert_exec, 6, envlist, len,\n SQLITE_TRANSIENT));\n free(envlist);\n }\n check(sqlite3_bind_text(stmt_insert_exec, 7, workingdir,\n -1, SQLITE_TRANSIENT));\n\n if(sqlite3_step(stmt_insert_exec) != SQLITE_DONE)\n goto sqlerror; /* LCOV_EXCL_LINE */\n sqlite3_reset(stmt_insert_exec);\n return 0;\n\n /* LCOV_EXCL_START : Insertions shouldn't fail */\nsqlerror:\n log_critical(0, \"sqlite3 error inserting exec: %s\", sqlite3_errmsg(db));\n return -1;\n /* LCOV_EXCL_STOP */\n}\n" }, { "path": "reprozip/native/database.h", "content": "#ifndef DATABASE_H\n#define DATABASE_H\n\n#define FILE_READ 0x01\n#define FILE_WRITE 0x02\n#define FILE_WDIR 0x04 /* File is used as a process's working dir */\n#define FILE_STAT 0x08 /* File is stat()d (only metadata is read) */\n#define FILE_LINK 0x10 /* The link itself is accessed, no dereference */\n#define FILE_SOCKET 0x20 /* The file is a UNIX domain socket */\n\nint db_init(const char *filename);\nint db_close(int rollback);\nint db_add_process(unsigned int *id, unsigned int parent_id,\n const char *working_dir, int is_thread);\nint db_add_exit(unsigned int id, int exitcode);\nint db_add_first_process(unsigned int *id, const char *working_dir);\nint db_add_file_open(unsigned int process,\n const char *name, unsigned int mode,\n int is_dir);\nint db_add_exec(unsigned int process, const char *binary,\n const char *const *argv, const char *const *envp,\n const char *workingdir);\n\n#endif\n" }, { "path": "reprozip/native/log.h", "content": "#ifndef LOG_H\n#define LOG_H\n\n#include \n#include \n\n#include \n#include \n\n\nextern int logging_level;\n\nint log_setup(void);\nvoid log_real_(pid_t tid, int lvl, const char *format, ...);\n\n\n#ifdef __GNUC__\n\n#define log_critical(i, s, ...) log_real_(i, 50, s, ## __VA_ARGS__)\n#define log_error(i, s, ...) log_real_(i, 40, s, ## __VA_ARGS__)\n#define log_warn(i, s, ...) log_real_(i, 30, s, ## __VA_ARGS__)\n#define log_info(i, s, ...) log_real_(i, 20, s, ## __VA_ARGS__)\n#define log_debug(i, s, ...) log_real_(i, 10, s, ## __VA_ARGS__)\n\n#else\n\n#define log_critical(i, s, ...) log_real_(i, 50, s, __VA_ARGS__)\n#define log_error(i, s, ...) log_real_(i, 40, s, __VA_ARGS__)\n#define log_warn(i, s, ...) log_real_(i, 30, s, __VA_ARGS__)\n#define log_info(i, s, ...) log_real_(i, 20, s, __VA_ARGS__)\n#define log_debug(i, s, ...) log_real_(i, 10, s, __VA_ARGS__)\n#endif\n\n#endif\n" }, { "path": "reprozip/native/ptrace_utils.c", "content": "#include \n#include \n#include \n#include \n#include \n#include \n#include \n\n#include \"config.h\"\n#include \"log.h\"\n#include \"ptrace_utils.h\"\n#include \"tracer.h\"\n\n\nstatic long tracee_getword(pid_t tid, const void *addr)\n{\n long res;\n errno = 0;\n res = ptrace(PTRACE_PEEKDATA, tid, addr, NULL);\n if(errno)\n {\n /* LCOV_EXCL_START : We only do that on things that went through the\n * kernel successfully, and so should be valid. The exception is\n * execve(), which will dup arguments when entering the syscall */\n log_error(tid, \"tracee_getword() failed: %s\", strerror(errno));\n return 0;\n /* LCOV_EXCL_STOP */\n }\n return res;\n}\n\nvoid *tracee_getptr(int mode, pid_t tid, const void *addr)\n{\n if(mode == MODE_I386)\n {\n /* Pointers are 32 bits */\n uint32_t ptr;\n tracee_read(tid, (void*)&ptr, addr, sizeof(ptr));\n return (void*)(uint64_t)ptr;\n }\n else /* mode == MODE_X86_64 */\n {\n /* Pointers are 64 bits */\n uint64_t ptr;\n tracee_read(tid, (void*)&ptr, addr, sizeof(ptr));\n return (void*)ptr;\n }\n}\n\nuint64_t tracee_getlong(int mode, pid_t tid, const void *addr)\n{\n if(mode == MODE_I386)\n {\n /* Longs are 32 bits */\n uint32_t val;\n tracee_read(tid, (void*)&val, addr, sizeof(val));\n return (uint64_t)val;\n }\n else /* mode == MODE_X86_64 */\n {\n /* Longs are 64 bits */\n uint64_t val;\n tracee_read(tid, (void*)&val, addr, sizeof(val));\n return val;\n }\n}\n\nuint64_t tracee_getu64(pid_t tid, const void *addr)\n{\n /* Longs are 64 bits */\n uint64_t val;\n tracee_read(tid, (void*)&val, addr, sizeof(val));\n return val;\n}\n\nsize_t tracee_getwordsize(int mode)\n{\n if(mode == MODE_I386)\n /* Pointers are 32 bits */\n return 4;\n else /* mode == MODE_X86_64 */\n /* Pointers are 64 bits */\n return 8;\n}\n\nsize_t tracee_strlen(pid_t tid, const char *str)\n{\n uintptr_t ptr = (uintptr_t)str;\n size_t j = ptr % WORD_SIZE;\n uintptr_t i = ptr - j;\n size_t size = 0;\n int done = 0;\n for(; !done; i += WORD_SIZE)\n {\n unsigned long data = tracee_getword(tid, (const void*)i);\n for(; !done && j < WORD_SIZE; ++j)\n {\n unsigned char byte = data >> (8 * j);\n if(byte == 0)\n done = 1;\n else\n ++size;\n }\n j = 0;\n }\n return size;\n}\n\nvoid tracee_read(pid_t tid, char *dst, const char *src, size_t size)\n{\n uintptr_t ptr = (uintptr_t)src;\n size_t j = ptr % WORD_SIZE;\n uintptr_t i = ptr - j;\n uintptr_t end = ptr + size;\n for(; i < end; i += WORD_SIZE)\n {\n unsigned long data = tracee_getword(tid, (const void*)i);\n for(; j < WORD_SIZE && i + j < end; ++j)\n *dst++ = data >> (8 * j);\n j = 0;\n }\n}\n\nchar *tracee_strdup(pid_t tid, const char *str)\n{\n size_t length = tracee_strlen(tid, str);\n char *res = malloc(length + 1);\n tracee_read(tid, res, str, length);\n res[length] = '\\0';\n return res;\n}\n\nchar **tracee_strarraydup(int mode, pid_t tid, const char *const *argv)\n{\n /* FIXME : This is probably broken on x32 */\n char **array;\n /* Reads number of pointers in pointer array */\n size_t nb_args = 0;\n {\n const char *const *a = argv;\n /* xargv = *a */\n const char *xargv = tracee_getptr(mode, tid, a);\n while(xargv != NULL)\n {\n ++nb_args;\n ++a;\n xargv = tracee_getptr(mode, tid, a);\n }\n }\n /* Allocs pointer array */\n array = malloc((nb_args + 1) * sizeof(char*));\n /* Dups array elements */\n {\n size_t i = 0;\n /* xargv = argv[0] */\n const char *xargv = tracee_getptr(mode, tid, argv);\n while(xargv != NULL)\n {\n array[i] = tracee_strdup(tid, xargv);\n ++i;\n /* xargv = argv[i] */\n xargv = tracee_getptr(mode, tid, argv + i);\n }\n array[i] = NULL;\n }\n return array;\n}\n\nvoid free_strarray(char **array)\n{\n char **ptr = array;\n while(*ptr)\n {\n free(*ptr);\n ++ptr;\n }\n free(array);\n}\n" }, { "path": "reprozip/native/ptrace_utils.h", "content": "#ifndef PTRACE_UTILS_H\n#define PTRACE_UTILS_H\n\nvoid *tracee_getptr(int mode, pid_t tid, const void *addr);\nuint64_t tracee_getlong(int mode, pid_t tid, const void *addr);\nuint64_t tracee_getu64(pid_t tid, const void *addr);\nsize_t tracee_getwordsize(int mode);\n\nsize_t tracee_strlen(pid_t tid, const char *str);\n\nvoid tracee_read(pid_t tid, char *dst, const char *src, size_t size);\n\nchar *tracee_strdup(pid_t tid, const char *str);\n\nchar **tracee_strarraydup(int mode, pid_t tid, const char *const *argv);\nvoid free_strarray(char **array);\n\n#endif\n" }, { "path": "reprozip/native/pylog.c", "content": "#include \n#include \n#include \n#include \n#include \n#include \n\n#include \n\n#include \"log.h\"\n\n\nstatic PyObject *py_logger = NULL;\nstatic PyObject *py_logger_log = NULL;\nint logging_level = 0;\n\n\nint log_setup()\n{\n if(py_logger == NULL)\n {\n // Import Python's logging module\n PyObject *logging = PyImport_ImportModuleEx(\"logging\",\n NULL, NULL, NULL);\n if(logging == NULL)\n return -1;\n\n // Get the logger\n {\n PyObject *func = PyObject_GetAttrString(logging, \"getLogger\");\n py_logger = PyObject_CallFunction(func, \"(s)\", \"reprozip\");\n Py_DECREF(logging);\n Py_DECREF(func);\n if(py_logger == NULL)\n return -1;\n }\n\n // Get the log function\n py_logger_log = PyObject_GetAttrString(py_logger, \"log\");\n if(py_logger_log == NULL)\n {\n /* LCOV_EXCL_START : Logger objects always have a 'log' method */\n Py_DECREF(py_logger);\n py_logger = NULL;\n return -1;\n /* LCOV_EXCL_STOP */\n }\n }\n\n // Get the effective logging level\n {\n PyObject *meth = PyObject_GetAttrString(py_logger,\n \"getEffectiveLevel\");\n PyObject *level = PyObject_CallFunctionObjArgs(meth, NULL);\n Py_DECREF(meth);\n if(level == NULL)\n return -1;\n logging_level = PyLong_AsLong(level);\n if(PyErr_Occurred())\n {\n /* LCOV_EXCL_START : Logger objects are reliable */\n Py_DECREF(level);\n return -1;\n /* LCOV_EXCL_STOP */\n }\n Py_DECREF(level);\n }\n\n return 0;\n}\n\nvoid log_real_(pid_t tid, int lvl, const char *format, ...)\n{\n va_list args;\n char datestr[13]; /* HH:MM:SS.mmm */\n static char *buffer = NULL;\n static size_t bufsize = 4096;\n size_t length;\n\n /* Fast filter: don't call Python if level is not enough */\n if(lvl < logging_level)\n return;\n\n if(buffer == NULL)\n buffer = malloc(bufsize);\n {\n struct timeval tv;\n gettimeofday(&tv, NULL);\n strftime(datestr, 13, \"%H:%M:%S\", localtime(&tv.tv_sec));\n sprintf(datestr+8, \".%03u\", (unsigned int)(tv.tv_usec / 1000));\n }\n va_start(args, format);\n length = (size_t)vsnprintf(buffer, bufsize, format, args);\n va_end(args);\n if(length + 1 >= bufsize)\n {\n while(length + 1 >= bufsize)\n bufsize *= 2;\n free(buffer);\n buffer = malloc(bufsize);\n va_start(args, format);\n length = vsnprintf(buffer, bufsize, format, args);\n va_end(args);\n }\n\n if(tid > 0)\n PyObject_CallFunction(py_logger_log, \"(l, s, l, s)\",\n lvl, \"[%d] %s\", tid, buffer);\n else\n PyObject_CallFunction(py_logger_log, \"(l, s, s)\",\n lvl, \"%s\", buffer);\n}\n" }, { "path": "reprozip/native/pytracer.c", "content": "#include \n\n#include \"database.h\"\n#include \"log.h\"\n#include \"tracer.h\"\n\n\nPyObject *Err_Base;\n\n\n/**\n * Makes a C string from a Python unicode or bytes object.\n *\n * If successful, the result is a string that the caller must free().\n * Else, returns NULL.\n */\nstatic char *get_string(PyObject *obj)\n{\n if(PyUnicode_Check(obj))\n {\n const char *str;\n PyObject *pyutf8 = PyUnicode_AsUTF8String(obj);\n if(pyutf8 == NULL)\n return NULL;\n#if PY_MAJOR_VERSION >= 3\n str = PyBytes_AsString(pyutf8);\n#else\n str = PyString_AsString(pyutf8);\n#endif\n if(str == NULL)\n return NULL;\n {\n char *ret = strdup(str);\n Py_DECREF(pyutf8);\n return ret;\n }\n }\n else if(\n#if PY_MAJOR_VERSION >= 3\n PyBytes_Check(obj)\n#else\n PyString_Check(obj)\n#endif\n )\n {\n const char *str;\n#if PY_MAJOR_VERSION >= 3\n str = PyBytes_AsString(obj);\n#else\n str = PyString_AsString(obj);\n#endif\n if(str == NULL)\n return NULL;\n return strdup(str);\n }\n else\n return NULL;\n}\n\n\nstatic PyObject *pytracer_execute(PyObject *self, PyObject *args)\n{\n PyObject *ret = NULL;\n int exit_status;\n\n char *binary = NULL, *databasepath = NULL;\n char **argv = NULL;\n size_t argv_len;\n PyObject *py_binary, *py_argv, *py_databasepath;\n\n if(log_setup() != 0)\n {\n /* LCOV_EXCL_START : Can't fail unless Python is in a broken state */\n PyErr_SetString(Err_Base, \"Error occurred\");\n return NULL;\n /* LCOV_EXCL_STOP */\n }\n\n /* Reads arguments */\n if(!PyArg_ParseTuple(args, \"OO!O\",\n &py_binary,\n &PyList_Type, &py_argv,\n &py_databasepath))\n return NULL;\n\n binary = get_string(py_binary);\n if(binary == NULL)\n goto done;\n databasepath = get_string(py_databasepath);\n if(databasepath == NULL)\n goto done;\n\n /* Converts argv from Python list to char[][] */\n {\n size_t i;\n int bad = 0;\n argv_len = PyList_Size(py_argv);\n argv = malloc((argv_len + 1) * sizeof(char*));\n for(i = 0; i < argv_len; ++i)\n {\n PyObject *arg = PyList_GetItem(py_argv, i);\n char *str = get_string(arg);\n if(str == NULL)\n {\n bad = 1;\n break;\n }\n argv[i] = str;\n }\n if(bad)\n {\n size_t j;\n for(j = 0; j < i; ++j)\n free(argv[j]);\n free(argv);\n argv = NULL;\n goto done;\n }\n argv[argv_len] = NULL;\n }\n\n if(fork_and_trace(binary, argv_len, argv, databasepath, &exit_status) == 0)\n {\n ret = PyLong_FromLong(exit_status);\n }\n else\n {\n PyErr_SetString(Err_Base, \"Error occurred\");\n ret = NULL;\n }\n\ndone:\n free(binary);\n free(databasepath);\n\n /* Deallocs argv */\n if(argv)\n {\n size_t i;\n for(i = 0; i < argv_len; ++i)\n free(argv[i]);\n free(argv);\n }\n\n return ret;\n}\n\n\nstatic PyMethodDef methods[] = {\n {\"execute\", pytracer_execute, METH_VARARGS,\n \"execute(binary, argv, databasepath)\\n\"\n \"\\n\"\n \"Runs the specified binary with the argument list argv under trace and \"\n \"writes\\nthe captured events to SQLite3 database databasepath.\"},\n { NULL, NULL, 0, NULL }\n};\n\n#if PY_MAJOR_VERSION >= 3\nstatic struct PyModuleDef moduledef = {\n PyModuleDef_HEAD_INIT,\n \"reprozip._pytracer\", /* m_name */\n \"C interface to tracer\", /* m_doc */\n -1, /* m_size */\n methods, /* m_methods */\n NULL, /* m_reload */\n NULL, /* m_traverse */\n NULL, /* m_clear */\n NULL, /* m_free */\n};\n#endif\n\n#if PY_MAJOR_VERSION >= 3\nPyMODINIT_FUNC PyInit__pytracer(void)\n#else\nPyMODINIT_FUNC init_pytracer(void)\n#endif\n{\n PyObject *mod;\n\n#if PY_MAJOR_VERSION >= 3\n mod = PyModule_Create(&moduledef);\n#else\n mod = Py_InitModule(\"reprozip._pytracer\", methods);\n#endif\n if(mod == NULL)\n {\n#if PY_MAJOR_VERSION >= 3\n return NULL;\n#else\n return;\n#endif\n }\n\n Err_Base = PyErr_NewException(\"_pytracer.Error\", NULL, NULL);\n Py_INCREF(Err_Base);\n PyModule_AddObject(mod, \"Error\", Err_Base);\n\n#if PY_MAJOR_VERSION >= 3\n return mod;\n#endif\n}\n" }, { "path": "reprozip/native/syscalls.c", "content": "#include \n#include \n#include \n#include \n\n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n\n#include \"config.h\"\n#include \"database.h\"\n#include \"log.h\"\n#include \"ptrace_utils.h\"\n#include \"syscalls.h\"\n#include \"tracer.h\"\n#include \"utils.h\"\n\n\n#ifndef __X32_SYSCALL_BIT\n#define __X32_SYSCALL_BIT 0x40000000\n#endif\n#ifndef SYS_CONNECT\n#define SYS_CONNECT 3\n#endif\n#ifndef SYS_ACCEPT\n#define SYS_ACCEPT 5\n#endif\n\n\n#define SYSCALL_I386 0\n#define SYSCALL_X86_64 1\n#define SYSCALL_X86_64_x32 2\n\n\nstruct syscall_table_entry {\n const char *name;\n int (*proc_entry)(const char*, struct Process *, unsigned int);\n int (*proc_exit)(const char*, struct Process *, unsigned int);\n unsigned int udata;\n};\n\nstruct syscall_table {\n size_t length;\n struct syscall_table_entry *entries;\n};\n\nstruct syscall_table *syscall_tables = NULL;\n\n\nstatic char *abs_path_arg(const struct Process *process, size_t arg)\n{\n char *pathname = tracee_strdup(process->tid, process->params[arg].p);\n if(pathname[0] != '/')\n {\n char *oldpath = pathname;\n pathname = abspath(process->threadgroup->wd, oldpath);\n free(oldpath);\n }\n return pathname;\n}\n\n\n/* ********************\n * Other syscalls that might be of interest but that we don't handle yet\n */\n\nstatic int syscall_unhandled_path1(const char *name, struct Process *process,\n unsigned int udata)\n{\n if(logging_level <= 30 && process->in_syscall && process->retvalue.i >= 0\n && name != NULL)\n {\n char *pathname = abs_path_arg(process, 0);\n log_info(process->tid, \"process used unhandled system call %s(\\\"%s\\\")\",\n name, pathname);\n free(pathname);\n }\n return 0;\n}\n\nstatic int syscall_unhandled_other(const char *name, struct Process *process,\n unsigned int udata)\n{\n if(process->in_syscall && process->retvalue.i >= 0 && name != NULL)\n log_info(process->tid, \"process used unhandled system call %s\", name);\n return 0;\n}\n\n\n/* ********************\n * open(), creat(), access()\n */\n\n#define SYSCALL_OPENING_OPEN 1\n#define SYSCALL_OPENING_ACCESS 2\n#define SYSCALL_OPENING_CREAT 3\n\nstatic int syscall_fileopening_in(const char *name, struct Process *process,\n unsigned int udata)\n{\n unsigned int mode = flags2mode(process->params[1].u);\n if( (mode & FILE_READ) && (mode & FILE_WRITE) )\n {\n char *pathname = abs_path_arg(process, 0);\n if(access(pathname, F_OK) != 0 && errno == ENOENT)\n {\n log_debug(process->tid, \"Doing RW open, file exists: no\");\n process->flags &= ~PROCFLAG_OPEN_EXIST;\n }\n else\n {\n log_debug(process->tid, \"Doing RW open, file exists: yes\");\n process->flags |= PROCFLAG_OPEN_EXIST;\n }\n free(pathname);\n }\n return 0;\n}\n\nstatic const char *mode_to_s(char *buf, unsigned int mode)\n{\n if(mode & FILE_READ)\n strcat(buf, \"|FILE_READ\");\n if(mode & FILE_WRITE)\n strcat(buf, \"|FILE_WRITE\");\n if(mode & FILE_WDIR)\n strcat(buf, \"|FILE_WDIR\");\n if(mode & FILE_STAT)\n strcat(buf, \"|FILE_STAT\");\n return buf[0]?buf + 1:\"0\";\n}\n\nstatic int syscall_fileopening_out(const char *name, struct Process *process,\n unsigned int syscall)\n{\n unsigned int mode;\n char *pathname = abs_path_arg(process, 0);\n\n if(syscall == SYSCALL_OPENING_ACCESS)\n mode = FILE_STAT;\n else if(syscall == SYSCALL_OPENING_CREAT)\n mode = flags2mode(process->params[1].u |\n O_CREAT | O_WRONLY | O_TRUNC);\n else /* syscall == SYSCALL_OPENING_OPEN */\n {\n mode = flags2mode(process->params[1].u);\n if( (process->retvalue.i >= 0) /* Open succeeded */\n && (mode & FILE_READ) && (mode & FILE_WRITE) ) /* In readwrite mode */\n {\n /* But the file doesn't exist */\n if(!(process->flags & PROCFLAG_OPEN_EXIST))\n /* Consider this a simple write */\n mode &= ~FILE_READ;\n }\n }\n\n if(logging_level <= 10)\n {\n /* Converts mode to string s_mode */\n char mode_buf[42] = \"\";\n const char *s_mode = mode_to_s(mode_buf, mode);\n\n if(syscall == SYSCALL_OPENING_OPEN)\n log_debug(process->tid,\n \"open(\\\"%s\\\", mode=%s) = %d (%s)\",\n pathname,\n s_mode,\n (int)process->retvalue.i,\n (process->retvalue.i >= 0)?\"success\":\"failure\");\n else /* creat or access */\n log_debug(process->tid,\n \"%s(\\\"%s\\\") (mode=%s) = %d (%s)\",\n (syscall == SYSCALL_OPENING_OPEN)?\"open\":\n (syscall == SYSCALL_OPENING_CREAT)?\"creat\":\"access\",\n pathname,\n s_mode,\n (int)process->retvalue.i,\n (process->retvalue.i >= 0)?\"success\":\"failure\");\n }\n\n if(process->retvalue.i >= 0)\n {\n if(db_add_file_open(process->identifier,\n pathname,\n mode,\n path_is_dir(pathname)) != 0)\n return -1; /* LCOV_EXCL_LINE */\n }\n\n free(pathname);\n return 0;\n}\n\nstatic int syscall_openat2_in(const char *name, struct Process *process,\n unsigned int udata)\n{\n if((int32_t)(process->params[0].u & 0xFFFFFFFF) != AT_FDCWD)\n {\n char *pathname = tracee_strdup(process->tid, process->params[1].p);\n log_info(process->tid,\n \"process used unhandled system call %s(%d, \\\"%s\\\")\",\n name, process->params[0].i, pathname);\n free(pathname);\n }\n else\n {\n void *flags_ptr = process->params[2].p; /* struct open_how* */\n unsigned int mode = flags2mode(tracee_getu64(process->tid, flags_ptr));\n if( (mode & FILE_READ) && (mode & FILE_WRITE) )\n {\n char *pathname = abs_path_arg(process, 1);\n if(access(pathname, F_OK) != 0 && errno == ENOENT)\n {\n log_debug(process->tid, \"Doing RW open, file exists: no\");\n process->flags &= ~PROCFLAG_OPEN_EXIST;\n }\n else\n {\n log_debug(process->tid, \"Doing RW open, file exists: yes\");\n process->flags |= PROCFLAG_OPEN_EXIST;\n }\n free(pathname);\n }\n }\n return 0;\n}\n\nstatic int syscall_openat2_out(const char *name, struct Process *process,\n unsigned int udata)\n{\n if((int32_t)(process->params[0].u & 0xFFFFFFFF) == AT_FDCWD)\n {\n char *pathname = abs_path_arg(process, 1);\n void *flags_ptr = process->params[2].p; /* struct open_how* */\n unsigned int mode = flags2mode(tracee_getu64(process->tid, flags_ptr));\n if( (process->retvalue.i >= 0) /* Open succeeded */\n && (mode & FILE_READ) && (mode & FILE_WRITE) ) /* In readwrite mode */\n {\n /* But the file doesn't exist */\n if(!(process->flags & PROCFLAG_OPEN_EXIST))\n /* Consider this a simple write */\n mode &= ~FILE_READ;\n }\n\n if(logging_level <= 10)\n {\n /* Converts mode to string s_mode */\n char mode_buf[42] = \"\";\n const char *s_mode = mode_to_s(mode_buf, mode);\n\n log_debug(process->tid,\n \"openat2(AT_FDCWD, \\\"%s\\\", mode=%s) = %d (%s)\",\n pathname,\n s_mode,\n (int)process->retvalue.i,\n (process->retvalue.i >= 0)?\"success\":\"failure\");\n }\n\n if(process->retvalue.i >= 0)\n {\n if(db_add_file_open(process->identifier,\n pathname,\n mode,\n path_is_dir(pathname)) != 0)\n return -1; /* LCOV_EXCL_LINE */\n }\n\n free(pathname);\n }\n return 0;\n}\n\n\n/* ********************\n * rename(), link(), symlink()\n */\n\nstatic int syscall_filecreating(const char *name, struct Process *process,\n unsigned int is_symlink)\n{\n if(process->retvalue.i >= 0)\n {\n char *written_path = abs_path_arg(process, 1);\n int is_dir = path_is_dir(written_path);\n /* symlink doesn't actually read the source */\n if(!is_symlink)\n {\n char *read_path = abs_path_arg(process, 0);\n if(db_add_file_open(process->identifier,\n read_path,\n FILE_READ | FILE_LINK,\n is_dir) != 0)\n return -1; /* LCOV_EXCL_LINE */\n free(read_path);\n }\n if(db_add_file_open(process->identifier,\n written_path,\n FILE_WRITE | FILE_LINK,\n is_dir) != 0)\n return -1; /* LCOV_EXCL_LINE */\n free(written_path);\n }\n return 0;\n}\n\nstatic int syscall_filecreating_at(const char *name, struct Process *process,\n unsigned int is_symlink)\n{\n if(process->retvalue.i >= 0)\n {\n if( (process->params[0].i == AT_FDCWD)\n && (process->params[2].i == AT_FDCWD) )\n {\n char *written_path = abs_path_arg(process, 3);\n int is_dir = path_is_dir(written_path);\n /* symlink doesn't actually read the source */\n if(!is_symlink)\n {\n char *read_path = abs_path_arg(process, 1);\n if(db_add_file_open(process->identifier,\n read_path,\n FILE_READ | FILE_LINK,\n is_dir) != 0)\n return -1; /* LCOV_EXCL_LINE */\n free(read_path);\n }\n if(db_add_file_open(process->identifier,\n written_path,\n FILE_WRITE | FILE_LINK,\n is_dir) != 0)\n return -1; /* LCOV_EXCL_LINE */\n free(written_path);\n }\n else\n return syscall_unhandled_other(name, process, 0);\n }\n return 0;\n}\n\n\n/* ********************\n * stat(), lstat()\n */\n\nstatic int syscall_filestat(const char *name, struct Process *process,\n unsigned int no_deref)\n{\n if(process->retvalue.i >= 0)\n {\n char *pathname = abs_path_arg(process, 0);\n if(db_add_file_open(process->identifier,\n pathname,\n FILE_STAT | (no_deref?FILE_LINK:0),\n path_is_dir(pathname)) != 0)\n return -1; /* LCOV_EXCL_LINE */\n free(pathname);\n }\n return 0;\n}\n\n\n/* ********************\n * readlink()\n */\n\nstatic int syscall_readlink(const char *name, struct Process *process,\n unsigned int udata)\n{\n if(process->retvalue.i >= 0)\n {\n char *pathname = abs_path_arg(process, 0);\n if(db_add_file_open(process->identifier,\n pathname,\n FILE_STAT | FILE_LINK,\n 0) != 0)\n return -1; /* LCOV_EXCL_LINE */\n free(pathname);\n }\n return 0;\n}\n\n\n/* ********************\n * mkdir()\n */\n\nstatic int syscall_mkdir(const char *name, struct Process *process,\n unsigned int udata)\n{\n if(process->retvalue.i >= 0)\n {\n char *pathname = abs_path_arg(process, 0);\n log_debug(process->tid, \"mkdir(\\\"%s\\\")\", pathname);\n if(db_add_file_open(process->identifier,\n pathname,\n FILE_WRITE,\n 1) != 0)\n return -1; /* LCOV_EXCL_LINE */\n free(pathname);\n }\n return 0;\n}\n\n\n/* ********************\n * chdir()\n */\n\nstatic int syscall_chdir(const char *name, struct Process *process,\n unsigned int udata)\n{\n if(process->retvalue.i >= 0)\n {\n char *pathname = abs_path_arg(process, 0);\n free(process->threadgroup->wd);\n process->threadgroup->wd = pathname;\n if(db_add_file_open(process->identifier,\n pathname,\n FILE_WDIR,\n 1) != 0)\n return -1; /* LCOV_EXCL_LINE */\n }\n return 0;\n}\n\n\n/* ********************\n * execve()\n *\n * See also special handling in syscall_handle() and PTRACE_EVENT_EXEC case\n * in trace().\n */\n\n#define SHEBANG_MAX_LEN 128 /* = Linux's BINPRM_BUF_SIZE */\n\nstatic int record_shebangs(struct Process *process, const char *exec_target)\n{\n const char *wd = process->threadgroup->wd;\n char buffer[SHEBANG_MAX_LEN];\n char target_buffer[SHEBANG_MAX_LEN];\n int step;\n for(step = 0; step < 4; ++step)\n {\n FILE *execd = fopen(exec_target, \"rb\");\n size_t ret = 0;\n if(execd != NULL)\n {\n ret = fread(buffer, 1, SHEBANG_MAX_LEN - 1, execd);\n fclose(execd);\n }\n if(ret == 0)\n {\n log_error(process->tid, \"couldn't open executed file %s\", exec_target);\n return 0;\n }\n if(buffer[0] != '#' || buffer[1] != '!')\n {\n // Check if executable is set-uid or set-gid\n struct stat statbuf;\n if(stat(exec_target, &statbuf) != 0)\n {\n /* LCOV_EXCL_START : stat() shouldn't fail if fopen() above worked */\n log_error(process->tid, \"couldn't stat executed file %s\", exec_target);\n /* LCOV_EXCL_STOP */\n }\n else\n {\n if((statbuf.st_mode & 04000) == 04000)\n {\n if(statbuf.st_uid != getuid())\n {\n log_warn(process->tid,\n \"executing set-uid binary! For security, \"\n \"Linux will not give the process any \"\n \"privileges from set-uid while it is being \"\n \"traced. This will probably break whatever \"\n \"you are tracing. Executable: %s\",\n exec_target);\n }\n else\n {\n log_info(process->tid,\n \"binary has set-uid bit set, not a problem \"\n \"because it is owned by our user\");\n }\n }\n if((statbuf.st_mode & 02000) == 02000)\n {\n int is_our_group = 0;\n size_t i, size;\n // Get the list of groups\n gid_t *groups = NULL;\n int ret = getgroups(0, NULL);\n if(ret >= 0)\n {\n size = (size_t)ret;\n groups = malloc(sizeof(gid_t) * size);\n ret = getgroups(ret, groups);\n }\n if(ret < 0)\n {\n /* LCOV_EXCL_START : Shouldn't ever fail */\n free(groups);\n log_critical(process->tid, \"getgroups() failed: %s\",\n strerror(errno));\n return -1;\n /* LCOV_EXCL_STOP */\n }\n\n // Check if the gid is one of our groups\n for(i = 0; i < size; ++i)\n {\n if(groups[i] == statbuf.st_gid)\n {\n is_our_group = 1;\n break;\n }\n }\n free(groups);\n\n if(!is_our_group)\n {\n log_warn(process->tid,\n \"executing set-gid binary! For security, \"\n \"Linux will not give the process any \"\n \"privileges from set-gid while it is being \"\n \"traced. This will probably break whatever \"\n \"you are tracing. Executable: %s\",\n exec_target);\n }\n else\n {\n log_info(process->tid,\n \"binary has set-gid bit set, not a problem \"\n \"because it is in one of our groups\");\n }\n }\n }\n return 0;\n }\n else\n {\n char *start = buffer + 2;\n buffer[ret] = '\\0';\n while(*start == '\\t' || *start == ' ')\n ++start;\n if(*start == '\\n' || *start == '\\0')\n {\n log_info(process->tid, \"empty shebang in %s\", exec_target);\n return 0;\n }\n {\n char *end = start;\n while(*end != '\\t' && *end != ' ' &&\n *end != '\\n' && *end != '\\0')\n ++end;\n *end = '\\0';\n }\n log_info(process->tid, \"read shebang: %s -> %s\", exec_target, start);\n if(*start != '/')\n {\n char *pathname = abspath(wd, start);\n if(db_add_file_open(process->identifier,\n pathname,\n FILE_READ,\n 0) != 0)\n return -1; /* LCOV_EXCL_LINE */\n free(pathname);\n }\n else\n if(db_add_file_open(process->identifier,\n start,\n FILE_READ,\n 0) != 0)\n return -1; /* LCOV_EXCL_LINE */\n exec_target = strcpy(target_buffer, start);\n }\n }\n log_error(process->tid, \"reached maximum shebang depth\");\n return 0;\n}\n\nstatic int syscall_execve_in(const char *name, struct Process *process,\n unsigned int udata)\n{\n /* int execve(const char *filename,\n * char *const argv[],\n * char *const envp[]); */\n struct ExecveInfo *execi = malloc(sizeof(struct ExecveInfo));\n execi->binary = abs_path_arg(process, 0);\n execi->argv = tracee_strarraydup(process->mode, process->tid,\n process->params[1].p);\n execi->envp = tracee_strarraydup(process->mode, process->tid,\n process->params[2].p);\n if(logging_level <= 10)\n {\n log_debug(process->tid, \"execve called:\\n binary=%s\\n argv:\",\n execi->binary);\n {\n /* Note: this conversion is correct and shouldn't need a cast */\n const char *const *v = (const char* const*)execi->argv;\n while(*v)\n {\n log_debug(process->tid, \" %s\", *v);\n ++v;\n }\n }\n {\n size_t nb = 0;\n while(execi->envp[nb] != NULL)\n ++nb;\n log_debug(process->tid, \" envp: (%u entries)\", (unsigned int)nb);\n }\n }\n process->execve_info = execi;\n return 0;\n}\n\nint syscall_execve_event(struct Process *process)\n{\n struct Process *exec_process = process;\n struct ExecveInfo *execi = exec_process->execve_info;\n if(execi == NULL)\n {\n /* On Linux, execve changes tid to the thread leader's tid, no\n * matter which thread made the call. This means that the process\n * that just returned from execve might not be the one which\n * called.\n * So we start by finding the one which called execve.\n * No possible confusion here since all other threads will have been\n * terminated by the kernel. */\n size_t i;\n for(i = 0; i < processes_size; ++i)\n {\n if(processes[i]->status == PROCSTAT_ATTACHED\n && processes[i]->threadgroup == process->threadgroup\n && processes[i]->in_syscall\n && processes[i]->execve_info != NULL)\n {\n exec_process = processes[i];\n break;\n }\n }\n if(exec_process == NULL)\n {\n /* LCOV_EXCL_START : internal error */\n log_critical(process->tid,\n \"execve() completed but call wasn't recorded\");\n return -1;\n /* LCOV_EXCL_STOP */\n }\n execi = exec_process->execve_info;\n\n /* The process that called execve() disappears without any trace */\n if(db_add_exit(exec_process->identifier, 0) != 0)\n return -1; /* LCOV_EXCL_LINE */\n log_debug(exec_process->tid,\n \"original exec'ing thread removed, tgid: %d\",\n process->tid);\n exec_process->execve_info = NULL;\n trace_free_process(exec_process);\n }\n else\n exec_process->execve_info = NULL;\n\n process->flags = PROCFLAG_EXECD;\n\n /* Note: execi->argv needs a cast to suppress a bogus warning\n * While conversion from char** to const char** is invalid, conversion from\n * char** to const char*const* is, in fact, safe.\n * G++ accepts it, GCC issues a warning. */\n if(db_add_exec(process->identifier, execi->binary,\n (const char *const*)execi->argv,\n (const char *const*)execi->envp,\n process->threadgroup->wd) != 0)\n return -1; /* LCOV_EXCL_LINE */\n /* Note that here, the database records that the thread leader called\n * execve, instead of thread exec_process->tid. */\n log_info(process->tid, \"successfully exec'd %s\", execi->binary);\n\n /* Follow shebangs */\n if(record_shebangs(process, execi->binary) != 0)\n return -1; /* LCOV_EXCL_LINE */\n\n if(trace_add_files_from_proc(process->identifier, process->tid,\n execi->binary) != 0)\n return -1; /* LCOV_EXCL_LINE */\n\n free_execve_info(execi);\n return 0;\n}\n\nstatic int syscall_execve_out(const char *name, struct Process *process,\n unsigned int udata)\n{\n log_debug(process->tid, \"execve() failed\");\n if(process->execve_info != NULL)\n {\n free_execve_info(process->execve_info);\n process->execve_info = NULL;\n }\n return 0;\n}\n\n\n/* ********************\n * fork(), clone(), ...\n */\n\nstatic int syscall_fork_in(const char *name, struct Process *process,\n unsigned int flags)\n{\n process->flags |= PROCFLAG_FORKING | flags;\n return 0;\n}\n\nstatic int syscall_fork_out(const char *name, struct Process *process,\n unsigned int udata)\n{\n process->flags &= ~(PROCFLAG_FORKING | PROCFLAG_CLONE3);\n return 0;\n}\n\nint syscall_fork_event(struct Process *process, unsigned int event)\n{\n#ifndef CLONE_THREAD\n#define CLONE_THREAD 0x00010000\n#endif\n\n int is_thread = 0;\n struct Process *new_process;\n unsigned long new_tid;\n\n ptrace(PTRACE_GETEVENTMSG, process->tid, NULL, &new_tid);\n\n if( (process->flags & PROCFLAG_FORKING) == 0)\n {\n /* LCOV_EXCL_START : internal error */\n log_critical(process->tid,\n \"process created new process %d but we didn't see syscall \"\n \"entry\", new_tid);\n return -1;\n /* LCOV_EXCL_STOP */\n }\n else if(event == PTRACE_EVENT_CLONE)\n {\n if( (process->flags & PROCFLAG_CLONE3) == 0) /* clone */\n is_thread = process->params[0].u & CLONE_THREAD;\n else /* clone3 */\n {\n void *flag_ptr = process->params[0].p; /* struct clone_args* */\n uint64_t flags = tracee_getu64(process->tid, flag_ptr);\n is_thread = flags & CLONE_THREAD;\n }\n }\n process->flags &= ~(PROCFLAG_FORKING | PROCFLAG_CLONE3);\n\n if(logging_level <= 20)\n log_info(new_tid, \"process created by %d via %s\\n\"\n \" (thread: %s) (working directory: %s)\",\n process->tid,\n (event == PTRACE_EVENT_FORK)?\"fork()\":\n (event == PTRACE_EVENT_VFORK)?\"vfork()\":\n \"clone()\",\n is_thread?\"yes\":\"no\",\n process->threadgroup->wd);\n\n /* At this point, the process might have been seen by waitpid in trace() or\n * not */\n new_process = trace_find_process(new_tid);\n if(new_process != NULL)\n {\n /* Process has been seen before and options were set */\n if(new_process->status != PROCSTAT_UNKNOWN)\n {\n /* LCOV_EXCL_START: : internal error */\n log_critical(new_tid,\n \"just created process that is already running \"\n \"(status=%d)\", new_process->status);\n return -1;\n /* LCOV_EXCL_STOP */\n }\n new_process->status = PROCSTAT_ATTACHED;\n ptrace(PTRACE_SYSCALL, new_process->tid, NULL, NULL);\n if(logging_level <= 20)\n {\n unsigned int nproc, unknown;\n trace_count_processes(&nproc, &unknown);\n log_info(0, \"%d processes (inc. %d unattached)\",\n nproc, unknown);\n }\n }\n else\n {\n /* Process hasn't been seen before (event happened first) */\n new_process = trace_get_empty_process();\n new_process->status = PROCSTAT_ALLOCATED;\n new_process->flags = 0;\n /* New process gets a SIGSTOP, but we resume on attach */\n new_process->tid = new_tid;\n new_process->in_syscall = 0;\n }\n\n if(is_thread)\n {\n new_process->threadgroup = process->threadgroup;\n process->threadgroup->refs++;\n log_debug(process->threadgroup->tgid, \"threadgroup refs=%d\",\n process->threadgroup->refs);\n }\n else\n new_process->threadgroup = trace_new_threadgroup(\n new_process->tid,\n strdup(process->threadgroup->wd));\n\n /* Parent will also get a SIGTRAP with PTRACE_EVENT_FORK */\n\n if(db_add_process(&new_process->identifier,\n process->identifier,\n process->threadgroup->wd, is_thread) != 0)\n return -1; /* LCOV_EXCL_LINE */\n\n return 0;\n}\n\n\n/* ********************\n * Network connections\n */\n\nstatic int handle_socket(struct Process *process, const char *msg,\n void *address, socklen_t addrlen)\n{\n const short family = ((struct sockaddr*)address)->sa_family;\n if(family == AF_INET && addrlen >= sizeof(struct sockaddr_in))\n {\n struct sockaddr_in *address_ = address;\n log_info(process->tid, \"%s %s:%d\", msg,\n inet_ntoa(address_->sin_addr),\n ntohs(address_->sin_port));\n }\n else if(family == AF_INET6\n && addrlen >= sizeof(struct sockaddr_in6))\n {\n struct sockaddr_in6 *address_ = address;\n char buf[50];\n inet_ntop(AF_INET6, &address_->sin6_addr, buf, sizeof(buf));\n log_info(process->tid, \"%s [%s]:%d\", msg,\n buf, ntohs(address_->sin6_port));\n }\n else if(family == AF_UNIX)\n {\n struct sockaddr_un *address_ = address;\n char buf[109];\n strncpy(buf, address_->sun_path, 108);\n buf[108] = 0;\n log_info(process->tid, \"%s unix:%s\", msg, buf);\n\n if(db_add_file_open(process->identifier,\n buf,\n FILE_SOCKET | FILE_WRITE,\n 0) != 0)\n return -1; /* LCOV_EXCL_LINE */\n }\n else\n log_info(process->tid, \"%s \",\n msg, family);\n return 0;\n}\n\nstatic int handle_accept(struct Process *process,\n void *arg1, void *arg2)\n{\n socklen_t addrlen;\n tracee_read(process->tid, (void*)&addrlen, arg2, sizeof(addrlen));\n if(addrlen >= sizeof(short))\n {\n void *address = malloc(addrlen);\n tracee_read(process->tid, address, arg1, addrlen);\n if(handle_socket(process, \"process accepted a connection from\",\n address, addrlen) != 0)\n return -1; /* LCOV_EXCL_LINE */\n free(address);\n }\n return 0;\n}\n\nstatic int handle_connect(struct Process *process,\n void *arg1, socklen_t addrlen)\n{\n if(addrlen >= sizeof(short))\n {\n void *address = malloc(addrlen);\n tracee_read(process->tid, address, arg1, addrlen);\n if(handle_socket(process, \"process connected to\",\n address, addrlen) != 0)\n return -1; /* LCOV_EXCL_LINE */\n free(address);\n }\n return 0;\n}\n\nstatic int syscall_socketcall(const char *name, struct Process *process,\n unsigned int udata)\n{\n if(process->retvalue.i >= 0)\n {\n /* Argument 1 is an array of longs, which are either numbers of pointers */\n uint64_t args = process->params[1].u;\n /* Size of each element in the array */\n const size_t wordsize = tracee_getwordsize(process->mode);\n /* Note that void* pointer arithmetic is illegal, hence the uint */\n if(process->params[0].u == SYS_ACCEPT)\n return handle_accept(process,\n tracee_getptr(process->mode, process->tid,\n (void*)(args + 1*wordsize)),\n tracee_getptr(process->mode, process->tid,\n (void*)(args + 2*wordsize)));\n else if(process->params[0].u == SYS_CONNECT)\n return handle_connect(process,\n tracee_getptr(process->mode, process->tid,\n (void*)(args + 1*wordsize)),\n tracee_getlong(process->mode, process->tid,\n (void*)(args + 2*wordsize)));\n }\n return 0;\n}\n\nstatic int syscall_accept(const char *name, struct Process *process,\n unsigned int udata)\n{\n if(process->retvalue.i >= 0)\n return handle_accept(process,\n process->params[1].p, process->params[2].p);\n else\n return 0;\n}\n\nstatic int syscall_connect(const char *name, struct Process *process,\n unsigned int udata)\n{\n if(process->retvalue.i >= 0)\n return handle_connect(process,\n process->params[1].p, process->params[2].u);\n else\n return 0;\n}\n\n\n/* ********************\n * *at variants, handled if dirfd is AT_FDCWD\n */\nstatic int syscall_xxx_at(const char *name, struct Process *process,\n unsigned int real_syscall)\n{\n /* Argument 0 is a file descriptor, we assume that the rest of them match\n * the non-at variant of the syscall */\n /* It seems that Linux accepts both AT_FDCWD=-100 sign-extended to 32 bit\n * and 64 bit. This is weird, but a process is unlikely to have 2**32 open\n * file descriptors anyway, so we only check bottom 32 bits. See #293 */\n if((int32_t)(process->params[0].u & 0xFFFFFFFF) == AT_FDCWD)\n {\n struct syscall_table_entry *entry = NULL;\n struct syscall_table *tbl;\n size_t syscall_type;\n if(process->mode == MODE_I386)\n syscall_type = SYSCALL_I386;\n else if(process->current_syscall & __X32_SYSCALL_BIT)\n syscall_type = SYSCALL_X86_64_x32;\n else\n syscall_type = SYSCALL_X86_64;\n tbl = &syscall_tables[syscall_type];\n if(real_syscall < tbl->length)\n entry = &tbl->entries[real_syscall];\n if(entry == NULL || entry->name == NULL)\n {\n /* LCOV_EXCL_START : Internal error, our syscall table is broken */\n log_critical(process->tid, \"INVALID SYSCALL in *at dispatch: %d\",\n real_syscall);\n return 0;\n /* LCOV_EXCL_STOP */\n }\n else\n {\n int ret = 0;\n /* Shifts arguments */\n size_t i;\n register_type arg0 = process->params[0];\n for(i = 0; i < PROCESS_ARGS - 1; ++i)\n process->params[i] = process->params[i + 1];\n if(!process->in_syscall && entry->proc_entry)\n ret = entry->proc_entry(name, process, entry->udata);\n else if(process->in_syscall && entry->proc_exit)\n ret = entry->proc_exit(name, process, entry->udata);\n for(i = PROCESS_ARGS; i > 1; --i)\n process->params[i - 1] = process->params[i - 2];\n process->params[0] = arg0;\n return ret;\n }\n }\n else if(!process->in_syscall)\n {\n char *pathname = tracee_strdup(process->tid, process->params[1].p);\n log_info(process->tid,\n \"process used unhandled system call %s(%d, \\\"%s\\\")\",\n name, process->params[0].i, pathname);\n free(pathname);\n }\n return 0;\n}\n\n\n/* ********************\n * Building the syscall table\n */\n\nstruct unprocessed_table_entry {\n unsigned int n;\n const char *name;\n int (*proc_entry)(const char*, struct Process *, unsigned int);\n int (*proc_exit)(const char*, struct Process *, unsigned int);\n unsigned int udata;\n\n};\n\nstruct syscall_table *process_table(struct syscall_table *table,\n const struct unprocessed_table_entry *orig)\n{\n size_t i, length = 0;\n const struct unprocessed_table_entry *pos;\n\n /* Measure required table */\n pos = orig;\n while(pos->proc_entry || pos->proc_exit)\n {\n if(pos->n + 1 > length)\n length = pos->n + 1;\n ++pos;\n }\n\n /* Allocate table */\n table->length = length;\n table->entries = malloc(sizeof(struct syscall_table_entry) * length);\n\n /* Initialize to NULL */\n for(i = 0; i < length; ++i)\n {\n table->entries[i].name = NULL;\n table->entries[i].proc_entry = NULL;\n table->entries[i].proc_exit = NULL;\n }\n\n /* Copy from unordered list */\n {\n pos = orig;\n while(pos->proc_entry || pos->proc_exit)\n {\n table->entries[pos->n].name = pos->name;\n table->entries[pos->n].proc_entry = pos->proc_entry;\n table->entries[pos->n].proc_exit = pos->proc_exit;\n table->entries[pos->n].udata = pos->udata;\n ++pos;\n }\n }\n\n return table;\n}\n\nvoid syscall_build_table(void)\n{\n if(syscall_tables != NULL)\n return ;\n\n#if defined(I386)\n syscall_tables = malloc(1 * sizeof(struct syscall_table));\n#elif defined(X86_64)\n syscall_tables = malloc(3 * sizeof(struct syscall_table));\n#else\n# error Unrecognized architecture!\n#endif\n /* https://github.com/torvalds/linux/blob/master/arch/x86/entry/syscalls/syscall_32.tbl */\n\n /* i386 */\n {\n struct unprocessed_table_entry list[] = {\n { 5, \"open\", syscall_fileopening_in, syscall_fileopening_out,\n SYSCALL_OPENING_OPEN},\n { 8, \"creat\", NULL, syscall_fileopening_out, SYSCALL_OPENING_CREAT},\n { 33, \"access\", NULL, syscall_fileopening_out, SYSCALL_OPENING_ACCESS},\n\n {106, \"stat\", NULL, syscall_filestat, 0},\n {107, \"lstat\", NULL, syscall_filestat, 1},\n {195, \"stat64\", NULL, syscall_filestat, 0},\n { 18, \"oldstat\", NULL, syscall_filestat, 0},\n {196, \"lstat64\", NULL, syscall_filestat, 1},\n { 84, \"oldlstat\", NULL, syscall_filestat, 1},\n\n { 85, \"readlink\", NULL, syscall_readlink, 0},\n\n { 39, \"mkdir\", NULL, syscall_mkdir, 0},\n\n { 12, \"chdir\", NULL, syscall_chdir, 0},\n\n { 11, \"execve\", syscall_execve_in, syscall_execve_out, 0},\n\n { 2, \"fork\", syscall_fork_in, syscall_fork_out, 0},\n {190, \"vfork\", syscall_fork_in, syscall_fork_out, 0},\n {120, \"clone\", syscall_fork_in, syscall_fork_out, 0},\n {435, \"clone3\", syscall_fork_in, syscall_fork_out, PROCFLAG_CLONE3},\n\n {364, \"accept4\", NULL, syscall_accept, 0},\n {362, \"connect\", NULL, syscall_connect, 0},\n {102, \"socketcall\", NULL, syscall_socketcall, 0},\n /*{369, \"sendto\", NULL, syscall_sendto, 0},*/\n\n /* File-creating syscalls: created path is second argument */\n { 38, \"rename\", NULL, syscall_filecreating, 0},\n { 9, \"link\", NULL, syscall_filecreating, 0},\n { 83, \"symlink\", NULL, syscall_filecreating, 1},\n\n /* File-creating syscalls, at variants: unhandled if first or third\n * argument is not AT_FDCWD, second is read, fourth is created */\n {302, \"renameat\", NULL, syscall_filecreating_at, 0},\n {353, \"renameat2\", NULL, syscall_filecreating_at, 0},\n {303, \"linkat\", NULL, syscall_filecreating_at, 0},\n {304, \"symlinkat\", NULL, syscall_filecreating_at, 1},\n\n /* Half-implemented: *at() variants, when dirfd is AT_FDCWD */\n {437, \"openat2\", syscall_openat2_in, syscall_openat2_out, 0},\n {296, \"mkdirat\", NULL, syscall_xxx_at, 39},\n {295, \"openat\", syscall_xxx_at, syscall_xxx_at, 5},\n {307, \"faccessat\", NULL, syscall_xxx_at, 33},\n {439, \"faccessat2\", NULL, syscall_xxx_at, 33},\n {305, \"readlinkat\", NULL, syscall_xxx_at, 85},\n {300, \"fstatat64\", NULL, syscall_xxx_at, 195},\n {383, \"statx\", NULL, syscall_xxx_at, 107},\n {358, \"execveat\", syscall_xxx_at, syscall_xxx_at, 11},\n {298, \"fchownat\", NULL, syscall_xxx_at, 182},\n {306, \"fchmodat\", NULL, syscall_xxx_at, 15},\n\n /* Unhandled with path as first argument */\n { 40, \"rmdir\", NULL, syscall_unhandled_path1, 0},\n { 92, \"truncate\", NULL, syscall_unhandled_path1, 0},\n {193, \"truncate64\", NULL, syscall_unhandled_path1, 0},\n { 10, \"unlink\", NULL, syscall_unhandled_path1, 0},\n { 15, \"chmod\", NULL, syscall_unhandled_path1, 0},\n {182, \"chown\", NULL, syscall_unhandled_path1, 0},\n {212, \"chown32\", NULL, syscall_unhandled_path1, 0},\n { 16, \"lchown\", NULL, syscall_unhandled_path1, 0},\n {198, \"lchown32\", NULL, syscall_unhandled_path1, 0},\n { 30, \"utime\", NULL, syscall_unhandled_path1, 0},\n {271, \"utimes\", NULL, syscall_unhandled_path1, 0},\n {277, \"mq_open\", NULL, syscall_unhandled_path1, 0},\n {278, \"mq_unlink\", NULL, syscall_unhandled_path1, 0},\n {188, \"setxattr\", NULL, syscall_unhandled_path1, 0},\n {189, \"lsetxattr\", NULL, syscall_unhandled_path1, 0},\n {197, \"removexattr\", NULL, syscall_unhandled_path1, 0},\n {198, \"lremovexattr\", NULL, syscall_unhandled_path1, 0},\n {191, \"getxattr\", NULL, syscall_unhandled_path1, 0},\n {192, \"lgetxattr\", NULL, syscall_unhandled_path1, 0},\n {194, \"listxattr\", NULL, syscall_unhandled_path1, 0},\n {195, \"llistxattr\", NULL, syscall_unhandled_path1, 0},\n\n /* Unhandled which use open descriptors */\n {301, \"unlinkat\", NULL, syscall_unhandled_other, 0},\n {133, \"fchdir\", NULL, syscall_unhandled_other, 0},\n\n /* Other unhandled */\n { 26, \"ptrace\", NULL, syscall_unhandled_other, 0},\n {341, \"name_to_handle_at\", NULL, syscall_unhandled_other, 0},\n\n /* Sentinel */\n {0, NULL, NULL, NULL, 0}\n };\n process_table(&syscall_tables[SYSCALL_I386], list);\n }\n\n#ifdef X86_64\n /* https://github.com/torvalds/linux/blob/master/arch/x86/entry/syscalls/syscall_64.tbl */\n\n /* x64 */\n {\n struct unprocessed_table_entry list[] = {\n { 2, \"open\", syscall_fileopening_in, syscall_fileopening_out,\n SYSCALL_OPENING_OPEN},\n { 85, \"creat\", NULL, syscall_fileopening_out, SYSCALL_OPENING_CREAT},\n { 21, \"access\", NULL, syscall_fileopening_out, SYSCALL_OPENING_ACCESS},\n\n { 4, \"stat\", NULL, syscall_filestat, 0},\n { 6, \"lstat\", NULL, syscall_filestat, 1},\n\n { 89, \"readlink\", NULL, syscall_readlink, 0},\n\n { 83, \"mkdir\", NULL, syscall_mkdir, 0},\n\n { 80, \"chdir\", NULL, syscall_chdir, 0},\n\n { 59, \"execve\", syscall_execve_in, syscall_execve_out, 0},\n\n { 57, \"fork\", syscall_fork_in, syscall_fork_out, 0},\n { 58, \"vfork\", syscall_fork_in, syscall_fork_out, 0},\n { 56, \"clone\", syscall_fork_in, syscall_fork_out, 0},\n {435, \"clone3\", syscall_fork_in, syscall_fork_out, PROCFLAG_CLONE3},\n\n { 43, \"accept\", NULL, syscall_accept, 0},\n {288, \"accept4\", NULL, syscall_accept, 0},\n { 42, \"connect\", NULL, syscall_connect, 0},\n /*{ 44, \"sendto\", NULL, syscall_sendto, 0},*/\n\n /* File-creating syscalls: created path is second argument */\n { 82, \"rename\", NULL, syscall_filecreating, 0},\n { 86, \"link\", NULL, syscall_filecreating, 0},\n { 88, \"symlink\", NULL, syscall_filecreating, 1},\n\n /* File-creating syscalls, at variants: unhandled if first or third\n * argument is not AT_FDCWD, second is read, fourth is created */\n {264, \"renameat\", NULL, syscall_filecreating_at, 0},\n {265, \"linkat\", NULL, syscall_filecreating_at, 0},\n {266, \"symlinkat\", NULL, syscall_filecreating_at, 1},\n\n /* Half-implemented: *at() variants, when dirfd is AT_FDCWD */\n {437, \"openat2\", syscall_openat2_in, syscall_openat2_out, 0},\n {258, \"mkdirat\", NULL, syscall_xxx_at, 83},\n {257, \"openat\", syscall_xxx_at, syscall_xxx_at, 2},\n {269, \"faccessat\", NULL, syscall_xxx_at, 21},\n {439, \"faccessat2\", NULL, syscall_xxx_at, 21},\n {267, \"readlinkat\", NULL, syscall_xxx_at, 89},\n {262, \"newfstatat\", NULL, syscall_xxx_at, 4},\n {332, \"statx\", NULL, syscall_xxx_at, 6},\n {322, \"execveat\", syscall_xxx_at, syscall_xxx_at, 59},\n {260, \"fchownat\", NULL, syscall_xxx_at, 92},\n {268, \"fchmodat\", NULL, syscall_xxx_at, 90},\n\n /* Unhandled with path as first argument */\n { 84, \"rmdir\", NULL, syscall_unhandled_path1, 0},\n { 76, \"truncate\", NULL, syscall_unhandled_path1, 0},\n { 87, \"unlink\", NULL, syscall_unhandled_path1, 0},\n { 90, \"chmod\", NULL, syscall_unhandled_path1, 0},\n { 92, \"chown\", NULL, syscall_unhandled_path1, 0},\n { 94, \"lchown\", NULL, syscall_unhandled_path1, 0},\n {132, \"utime\", NULL, syscall_unhandled_path1, 0},\n {235, \"utimes\", NULL, syscall_unhandled_path1, 0},\n {240, \"mq_open\", NULL, syscall_unhandled_path1, 0},\n {241, \"mq_unlink\", NULL, syscall_unhandled_path1, 0},\n {188, \"setxattr\", NULL, syscall_unhandled_path1, 0},\n {189, \"lsetxattr\", NULL, syscall_unhandled_path1, 0},\n {197, \"removexattr\", NULL, syscall_unhandled_path1, 0},\n {198, \"lremovexattr\", NULL, syscall_unhandled_path1, 0},\n {191, \"getxattr\", NULL, syscall_unhandled_path1, 0},\n {192, \"lgetxattr\", NULL, syscall_unhandled_path1, 0},\n {194, \"listxattr\", NULL, syscall_unhandled_path1, 0},\n {195, \"llistxattr\", NULL, syscall_unhandled_path1, 0},\n\n /* Unhandled which use open descriptors */\n {263, \"unlinkat\", NULL, syscall_unhandled_other, 0},\n { 81, \"fchdir\", NULL, syscall_unhandled_other, 0},\n\n /* Other unhandled */\n {101, \"ptrace\", NULL, syscall_unhandled_other, 0},\n {303, \"name_to_handle_at\", NULL, syscall_unhandled_other, 0},\n\n /* Sentinel */\n {0, NULL, NULL, NULL, 0}\n };\n process_table(&syscall_tables[SYSCALL_X86_64], list);\n }\n\n /* x32 */\n {\n struct unprocessed_table_entry list[] = {\n { 2, \"open\", syscall_fileopening_in, syscall_fileopening_out,\n SYSCALL_OPENING_OPEN},\n { 85, \"creat\", NULL, syscall_fileopening_out, SYSCALL_OPENING_CREAT},\n { 21, \"access\", NULL, syscall_fileopening_out, SYSCALL_OPENING_ACCESS},\n\n { 4, \"stat\", NULL, syscall_filestat, 0},\n { 6, \"lstat\", NULL, syscall_filestat, 1},\n\n { 89, \"readlink\", NULL, syscall_readlink, 0},\n\n { 83, \"mkdir\", NULL, syscall_mkdir, 0},\n\n { 80, \"chdir\", NULL, syscall_chdir, 0},\n\n {520, \"execve\", syscall_execve_in, syscall_execve_out},\n\n { 57, \"fork\", syscall_fork_in, syscall_fork_out, 0},\n { 58, \"vfork\", syscall_fork_in, syscall_fork_out, 0},\n { 56, \"clone\", syscall_fork_in, syscall_fork_out, 0},\n {435, \"clone3\", syscall_fork_in, syscall_fork_out, PROCFLAG_CLONE3},\n\n { 43, \"accept\", NULL, syscall_accept, 0},\n {288, \"accept4\", NULL, syscall_accept, 0},\n { 42, \"connect\", NULL, syscall_connect, 0},\n /*{ 44, \"sendto\", NULL, syscall_sendto, 0},*/\n\n /* File-creating syscalls: created path is second argument */\n { 82, \"rename\", NULL, syscall_filecreating, 0},\n { 86, \"link\", NULL, syscall_filecreating, 0},\n { 88, \"symlink\", NULL, syscall_filecreating, 1},\n\n /* File-creating syscalls, at variants: unhandled if first or third\n * argument is not AT_FDCWD, second is read, fourth is created */\n {264, \"renameat\", NULL, syscall_filecreating_at, 0},\n {265, \"linkat\", NULL, syscall_filecreating_at, 0},\n {266, \"symlinkat\", NULL, syscall_filecreating_at, 1},\n\n /* Half-implemented: *at() variants, when dirfd is AT_FDCWD */\n {437, \"openat2\", syscall_openat2_in, syscall_openat2_out, 0},\n {258, \"mkdirat\", NULL, syscall_xxx_at, 83},\n {257, \"openat\", syscall_xxx_at, syscall_xxx_at, 2},\n {269, \"faccessat\", NULL, syscall_xxx_at, 21},\n {439, \"faccessat2\", NULL, syscall_xxx_at, 21},\n {267, \"readlinkat\", NULL, syscall_xxx_at, 89},\n {262, \"newfstatat\", NULL, syscall_xxx_at, 4},\n {332, \"statx\", NULL, syscall_xxx_at, 6},\n {545, \"execveat\", syscall_xxx_at, syscall_xxx_at, 59},\n {260, \"fchownat\", NULL, syscall_xxx_at, 92},\n {268, \"fchmodat\", NULL, syscall_xxx_at, 90},\n\n /* Unhandled with path as first argument */\n { 84, \"rmdir\", NULL, syscall_unhandled_path1, 0},\n { 76, \"truncate\", NULL, syscall_unhandled_path1, 0},\n { 87, \"unlink\", NULL, syscall_unhandled_path1, 0},\n { 90, \"chmod\", NULL, syscall_unhandled_path1, 0},\n { 92, \"chown\", NULL, syscall_unhandled_path1, 0},\n { 94, \"lchown\", NULL, syscall_unhandled_path1, 0},\n {132, \"utime\", NULL, syscall_unhandled_path1, 0},\n {235, \"utimes\", NULL, syscall_unhandled_path1, 0},\n {240, \"mq_open\", NULL, syscall_unhandled_path1, 0},\n {241, \"mq_unlink\", NULL, syscall_unhandled_path1, 0},\n {188, \"setxattr\", NULL, syscall_unhandled_path1, 0},\n {189, \"lsetxattr\", NULL, syscall_unhandled_path1, 0},\n {197, \"removexattr\", NULL, syscall_unhandled_path1, 0},\n {198, \"lremovexattr\", NULL, syscall_unhandled_path1, 0},\n {191, \"getxattr\", NULL, syscall_unhandled_path1, 0},\n {192, \"lgetxattr\", NULL, syscall_unhandled_path1, 0},\n {194, \"listxattr\", NULL, syscall_unhandled_path1, 0},\n {195, \"llistxattr\", NULL, syscall_unhandled_path1, 0},\n\n /* Unhandled which use open descriptors */\n {263, \"unlinkat\", NULL, syscall_unhandled_other, 0},\n { 81, \"fchdir\", NULL, syscall_unhandled_other, 0},\n\n /* Other unhandled */\n {521, \"ptrace\", NULL, syscall_unhandled_other, 0},\n {303, \"name_to_handle_at\", NULL, syscall_unhandled_other, 0},\n\n /* Sentinel */\n {0, NULL, NULL, NULL, 0}\n };\n process_table(&syscall_tables[SYSCALL_X86_64_x32], list);\n }\n#endif\n}\n\n\n/* ********************\n * Handle a syscall via the table\n */\n\nint syscall_handle(struct Process *process)\n{\n pid_t tid = process->tid;\n const int syscall = process->current_syscall & ~__X32_SYSCALL_BIT;\n size_t syscall_type;\n const char *inout = process->in_syscall?\"out\":\"in\";\n if(process->mode == MODE_I386)\n {\n syscall_type = SYSCALL_I386;\n if(logging_level <= 5)\n log_debug(process->tid, \"syscall %d (i386) (%s)\", syscall, inout);\n }\n else if(process->current_syscall & __X32_SYSCALL_BIT)\n {\n /* LCOV_EXCL_START : x32 is not supported right now */\n syscall_type = SYSCALL_X86_64_x32;\n if(logging_level <= 5)\n log_debug(process->tid, \"syscall %d (x32) (%s)\", syscall, inout);\n /* LCOV_EXCL_STOP */\n }\n else\n {\n syscall_type = SYSCALL_X86_64;\n if(logging_level <= 5)\n log_debug(process->tid, \"syscall %d (x64) (%s)\", syscall, inout);\n }\n\n if(process->flags & PROCFLAG_EXECD)\n {\n if(logging_level <= 5)\n log_debug(process->tid,\n \"ignoring, EXEC'D is set -- just post-exec syscall-\"\n \"return stop\");\n process->flags &= ~PROCFLAG_EXECD;\n if(process->execve_info != NULL)\n {\n free_execve_info(process->execve_info);\n process->execve_info = NULL;\n }\n process->in_syscall = 1; /* set to 0 before function returns */\n }\n else\n {\n struct syscall_table_entry *entry = NULL;\n struct syscall_table *tbl = &syscall_tables[syscall_type];\n if(syscall < 0 || syscall >= 2000)\n /* LCOV_EXCL_START : internal error */\n log_error(process->tid, \"INVALID SYSCALL %d\", syscall);\n /* LCOV_EXCL_STOP */\n if(syscall >= 0 && (size_t)syscall < tbl->length)\n entry = &tbl->entries[syscall];\n if(entry != NULL)\n {\n int ret = 0;\n if(entry->name)\n log_debug(process->tid, \"%s()\", entry->name);\n if(!process->in_syscall && entry->proc_entry)\n ret = entry->proc_entry(entry->name, process, entry->udata);\n else if(process->in_syscall && entry->proc_exit)\n ret = entry->proc_exit(entry->name, process, entry->udata);\n if(ret != 0)\n return -1;\n }\n }\n\n /* Run to next syscall */\n if(process->in_syscall)\n {\n process->in_syscall = 0;\n if(process->execve_info != NULL)\n {\n /* LCOV_EXCL_START : internal error */\n log_error(process->tid, \"out of syscall with execve_info != NULL\");\n return -1;\n /* LCOV_EXCL_STOP */\n\n }\n process->current_syscall = -1;\n }\n else\n process->in_syscall = 1;\n ptrace(PTRACE_SYSCALL, tid, NULL, NULL);\n\n return 0;\n}\n" }, { "path": "reprozip/native/syscalls.h", "content": "#ifndef SYSCALL_H\n#define SYSCALL_H\n\n#include \"tracer.h\"\n\nvoid syscall_build_table(void);\n\nint syscall_handle(struct Process *process);\n\nint syscall_execve_event(struct Process *process);\nint syscall_fork_event(struct Process *process, unsigned int event);\n\n#endif\n" }, { "path": "reprozip/native/tracer.c", "content": "#include \n#include \n#include \n#include \n#include \n#include \n\n#include \n#include \n#include \n#include \n#include \n#include \n#include \n\n#include \"config.h\"\n#include \"database.h\"\n#include \"log.h\"\n#include \"ptrace_utils.h\"\n#include \"syscalls.h\"\n#include \"tracer.h\"\n#include \"utils.h\"\n\n\n#ifndef NT_PRSTATUS\n#define NT_PRSTATUS 1\n#endif\n\n\nstruct i386_regs {\n int32_t ebx;\n int32_t ecx;\n int32_t edx;\n int32_t esi;\n int32_t edi;\n int32_t ebp;\n int32_t eax;\n int32_t xds;\n int32_t xes;\n int32_t xfs;\n int32_t xgs;\n int32_t orig_eax;\n int32_t eip;\n int32_t xcs;\n int32_t eflags;\n int32_t esp;\n int32_t xss;\n};\n\n\nstruct x86_64_regs {\n int64_t r15;\n int64_t r14;\n int64_t r13;\n int64_t r12;\n int64_t rbp;\n int64_t rbx;\n int64_t r11;\n int64_t r10;\n int64_t r9;\n int64_t r8;\n int64_t rax;\n int64_t rcx;\n int64_t rdx;\n int64_t rsi;\n int64_t rdi;\n int64_t orig_rax;\n int64_t rip;\n int64_t cs;\n int64_t eflags;\n int64_t rsp;\n int64_t ss;\n int64_t fs_base;\n int64_t gs_base;\n int64_t ds;\n int64_t es;\n int64_t fs;\n int64_t gs;\n};\n\n\nstatic void get_i386_reg(register_type *reg, uint32_t value)\n{\n reg->i = (int32_t)value;\n reg->u = value;\n reg->p = (void*)(uint64_t)value;\n}\n\nstatic void get_x86_64_reg(register_type *reg, uint64_t value)\n{\n reg->i = (int64_t)value;\n reg->u = value;\n reg->p = (void*)value;\n}\n\n\nvoid free_execve_info(struct ExecveInfo *execi)\n{\n free_strarray(execi->argv);\n free_strarray(execi->envp);\n free(execi->binary);\n free(execi);\n}\n\n\nstruct Process **processes = NULL;\nsize_t processes_size;\n\nstruct Process *trace_find_process(pid_t tid)\n{\n size_t i;\n for(i = 0; i < processes_size; ++i)\n {\n if(processes[i]->status != PROCSTAT_FREE && processes[i]->tid == tid)\n return processes[i];\n }\n return NULL;\n}\n\nstruct Process *trace_get_empty_process(void)\n{\n size_t i;\n for(i = 0; i < processes_size; ++i)\n {\n if(processes[i]->status == PROCSTAT_FREE)\n return processes[i];\n }\n\n /* Count unknown processes */\n if(logging_level <= 10)\n {\n size_t unknown = 0;\n for(i = 0; i < processes_size; ++i)\n if(processes[i]->status == PROCSTAT_UNKNOWN)\n ++unknown;\n log_debug(0, \"there are %u/%u UNKNOWN processes\",\n (unsigned int)unknown, (unsigned int)processes_size);\n }\n\n /* Allocate more! */\n log_debug(0, \"process table full (%d), reallocating\",\n (int)processes_size);\n {\n struct Process *pool;\n size_t prev_size = processes_size;\n processes_size *= 2;\n pool = malloc((processes_size - prev_size) * sizeof(*pool));\n processes = realloc(processes, processes_size * sizeof(*processes));\n for(; i < processes_size; ++i)\n {\n processes[i] = pool++;\n processes[i]->status = PROCSTAT_FREE;\n processes[i]->threadgroup = NULL;\n processes[i]->execve_info = NULL;\n }\n return processes[prev_size];\n }\n}\n\nstruct ThreadGroup *trace_new_threadgroup(pid_t tgid, char *wd)\n{\n struct ThreadGroup *threadgroup = malloc(sizeof(struct ThreadGroup));\n threadgroup->tgid = tgid;\n threadgroup->wd = wd;\n threadgroup->refs = 1;\n log_debug(tgid, \"threadgroup (= process) created\");\n return threadgroup;\n}\n\nvoid trace_free_process(struct Process *process)\n{\n process->status = PROCSTAT_FREE;\n if(process->threadgroup != NULL)\n {\n process->threadgroup->refs--;\n log_debug(process->tid,\n \"process died, threadgroup tgid=%d refs=%d\",\n process->threadgroup->tgid, process->threadgroup->refs);\n if(process->threadgroup->refs == 0)\n {\n log_debug(process->threadgroup->tgid,\n \"deallocating threadgroup\");\n if(process->threadgroup->wd != NULL)\n free(process->threadgroup->wd);\n free(process->threadgroup);\n }\n process->threadgroup = NULL;\n }\n else\n log_debug(process->tid, \"threadgroup==NULL\"); /* LCOV_EXCL_LINE */\n if(process->execve_info != NULL)\n {\n free_execve_info(process->execve_info);\n process->execve_info = NULL;\n }\n}\n\nvoid trace_count_processes(unsigned int *p_nproc, unsigned int *p_unknown)\n{\n unsigned int nproc = 0, unknown = 0;\n size_t i;\n for(i = 0; i < processes_size; ++i)\n {\n switch(processes[i]->status)\n {\n case PROCSTAT_FREE:\n break;\n case PROCSTAT_UNKNOWN:\n /* Exists but no corresponding syscall has returned yet */\n ++unknown;\n ++nproc;\n break;\n case PROCSTAT_ALLOCATED:\n /* Not yet attached but it will show up eventually */\n case PROCSTAT_ATTACHED:\n /* Running */\n ++nproc;\n break;\n }\n }\n if(p_nproc != NULL)\n *p_nproc = nproc;\n if(p_unknown != NULL)\n *p_unknown = unknown;\n}\n\nint trace_add_files_from_proc(unsigned int process, pid_t tid,\n const char *binary)\n{\n FILE *fp;\n char dummy;\n char *line = NULL;\n size_t length = 0;\n size_t previous_path_size = 4096;\n char *previous_path = malloc(previous_path_size);\n previous_path[0] = 0;\n\n const char *const fmt = \"/proc/%d/maps\";\n int len = snprintf(&dummy, 1, fmt, tid);\n char *procfile = malloc(len + 1);\n snprintf(procfile, len + 1, fmt, tid);\n\n /* Loops on lines\n * Format:\n * 08134000-0813a000 rw-p 000eb000 fe:00 868355 /bin/bash\n * 0813a000-0813f000 rw-p 00000000 00:00 0\n * b7721000-b7740000 r-xp 00000000 fe:00 901950 /lib/ld-2.18.so\n * bfe44000-bfe65000 rw-p 00000000 00:00 0 [stack]\n */\n\n#ifdef DEBUG_PROC_PARSER\n log_info(tid, \"parsing %s\", procfile);\n#endif\n fp = fopen(procfile, \"r\");\n free(procfile);\n\n while((line = read_line(line, &length, fp)) != NULL)\n {\n unsigned long int addr_start, addr_end;\n char perms[5];\n unsigned long int offset;\n unsigned int dev_major, dev_minor;\n unsigned long int inode;\n int path_offset;\n int ret = sscanf(line,\n \"%lx-%lx %4s %lx %x:%x %lu %n\",\n &addr_start, &addr_end,\n perms,\n &offset,\n &dev_major, &dev_minor,\n &inode,\n &path_offset);\n char *pathname = line + path_offset;\n if(ret != 7)\n {\n /* LCOV_EXCL_START : Broken or unexpected proc file format*/\n log_error(tid, \"Invalid format in /proc/%d/maps (%d):\\n %s\", tid,\n ret, line);\n free(line);\n fclose(fp);\n return -1;\n /* LCOV_EXCL_STOP */\n }\n\n#ifdef DEBUG_PROC_PARSER\n log_info(tid,\n \"proc line:\\n\"\n \" addr_start: %lx\\n\"\n \" addr_end: %lx\\n\"\n \" perms: %s\\n\"\n \" offset: %lx\\n\"\n \" dev_major: %x\\n\"\n \" dev_minor: %x\\n\"\n \" inode: %lu\\n\"\n \" pathname: %s\",\n addr_start, addr_end,\n perms,\n offset,\n dev_major, dev_minor,\n inode,\n pathname);\n#endif\n if(inode > 0)\n {\n if(strcmp(pathname, binary) != 0\n && strcmp(pathname, previous_path) != 0)\n {\n#ifdef DEBUG_PROC_PARSER\n log_info(tid, \" adding to database\");\n#endif\n if(db_add_file_open(process, pathname,\n FILE_READ, path_is_dir(pathname)) != 0)\n return -1;\n {\n size_t needed_size = strlen(pathname) + 1;\n if(needed_size > previous_path_size) {\n while(needed_size > previous_path_size) {\n previous_path_size *= 2;\n }\n free(previous_path);\n previous_path = malloc(previous_path_size);\n }\n }\n strcpy(previous_path, pathname);\n }\n }\n }\n fclose(fp);\n free(previous_path);\n return 0;\n}\n\nstatic void trace_set_options(pid_t tid)\n{\n ptrace(PTRACE_SETOPTIONS, tid, 0,\n PTRACE_O_TRACESYSGOOD | /* Adds 0x80 bit to SIGTRAP signals\n * if paused because of syscall */\n#ifdef PTRACE_O_EXITKILL\n PTRACE_O_EXITKILL |\n#endif\n PTRACE_O_TRACECLONE |\n PTRACE_O_TRACEFORK |\n PTRACE_O_TRACEVFORK |\n PTRACE_O_TRACEEXEC);\n}\n\nstatic int trace(pid_t first_proc, int *first_exit_code)\n{\n for(;;)\n {\n int status;\n pid_t tid;\n struct Process *process;\n\n /* Wait for a process */\n tid = waitpid(-1, &status, __WALL);\n if(tid == -1)\n {\n /* LCOV_EXCL_START : internal error: waitpid() won't fail unless we\n * mistakingly call it while there is no child to wait for */\n log_critical(0, \"waitpid failed: %s\", strerror(errno));\n return -1;\n /* LCOV_EXCL_STOP */\n }\n if(WIFEXITED(status) || WIFSIGNALED(status))\n {\n unsigned int nprocs, unknown;\n int exitcode;\n if(WIFSIGNALED(status))\n /* exit codes are 8 bits */\n exitcode = 0x0100 | WTERMSIG(status);\n else\n exitcode = WEXITSTATUS(status);\n\n if(tid == first_proc && first_exit_code != NULL)\n *first_exit_code = exitcode;\n process = trace_find_process(tid);\n if(process != NULL)\n {\n if(db_add_exit(process->identifier, exitcode) != 0)\n return -1; /* LCOV_EXCL_LINE */\n trace_free_process(process);\n }\n trace_count_processes(&nprocs, &unknown);\n log_info(tid, \"process exited (%s %d), %d processes remain\",\n (exitcode & 0x0100)?\"signal\":\"code\", exitcode & 0xFF,\n (unsigned int)nprocs);\n if(nprocs <= 0)\n break;\n if(unknown >= nprocs)\n {\n /* LCOV_EXCL_START : This can't happen because UNKNOWN\n * processes are the forked processes whose creator has not\n * returned yet. Therefore, if there is an UNKNOWN process, its\n * creator has to exist as well (and it is not UNKNOWN). */\n log_critical(0, \"only UNKNOWN processes remaining (%d)\",\n (unsigned int)nprocs);\n return -1;\n /* LCOV_EXCL_STOP */\n }\n continue;\n }\n\n process = trace_find_process(tid);\n if(process == NULL)\n {\n log_debug(tid, \"process appeared\");\n process = trace_get_empty_process();\n process->status = PROCSTAT_UNKNOWN;\n process->flags = 0;\n process->tid = tid;\n process->threadgroup = NULL;\n process->in_syscall = 0;\n trace_set_options(tid);\n /* Don't resume, it will be set to ATTACHED and resumed when fork()\n * returns */\n continue;\n }\n else if(process->status == PROCSTAT_ALLOCATED)\n {\n process->status = PROCSTAT_ATTACHED;\n\n log_debug(tid, \"process attached\");\n trace_set_options(tid);\n ptrace(PTRACE_SYSCALL, tid, NULL, NULL);\n if(logging_level <= 20)\n {\n unsigned int nproc, unknown;\n trace_count_processes(&nproc, &unknown);\n log_info(0, \"%d processes (inc. %d unattached)\",\n nproc, unknown);\n }\n continue;\n }\n\n if(WIFSTOPPED(status) && WSTOPSIG(status) & 0x80)\n {\n size_t len = 0;\n#ifdef I386\n struct i386_regs regs;\n#else /* def X86_64 */\n struct x86_64_regs regs;\n#endif\n /* Try to use GETREGSET first, since iov_len allows us to know if\n * 32bit or 64bit mode was used */\n#ifdef PTRACE_GETREGSET\n#ifndef NT_PRSTATUS\n#define NT_PRSTATUS 1\n#endif\n {\n struct iovec iov;\n iov.iov_base = ®s;\n iov.iov_len = sizeof(regs);\n if(ptrace(PTRACE_GETREGSET, tid, NT_PRSTATUS, &iov) == 0)\n len = iov.iov_len;\n }\n if(len == 0)\n#endif\n /* GETREGSET undefined or call failed, fallback on GETREGS */\n {\n /* LCOV_EXCL_START : GETREGSET was added by Linux 2.6.34 in\n * May 2010 (2225a122) */\n ptrace(PTRACE_GETREGS, tid, NULL, ®s);\n /* LCOV_EXCL_STOP */\n }\n#if defined(I386)\n if(!process->in_syscall)\n process->current_syscall = regs.orig_eax;\n if(process->in_syscall)\n get_i386_reg(&process->retvalue, regs.eax);\n else\n {\n get_i386_reg(&process->params[0], regs.ebx);\n get_i386_reg(&process->params[1], regs.ecx);\n get_i386_reg(&process->params[2], regs.edx);\n get_i386_reg(&process->params[3], regs.esi);\n get_i386_reg(&process->params[4], regs.edi);\n get_i386_reg(&process->params[5], regs.ebp);\n }\n process->mode = MODE_I386;\n#elif defined(X86_64)\n /* On x86_64, process might be 32 or 64 bits */\n /* If len is known (not 0) and not that of x86_64 registers,\n * or if len is not known (0) and CS is 0x23 (not as reliable) */\n if( (len != 0 && len != sizeof(regs))\n || (len == 0 && regs.cs == 0x23) )\n {\n /* 32 bit mode */\n struct i386_regs *x86regs = (struct i386_regs*)®s;\n if(!process->in_syscall)\n process->current_syscall = x86regs->orig_eax;\n if(process->in_syscall)\n get_i386_reg(&process->retvalue, x86regs->eax);\n else\n {\n get_i386_reg(&process->params[0], x86regs->ebx);\n get_i386_reg(&process->params[1], x86regs->ecx);\n get_i386_reg(&process->params[2], x86regs->edx);\n get_i386_reg(&process->params[3], x86regs->esi);\n get_i386_reg(&process->params[4], x86regs->edi);\n get_i386_reg(&process->params[5], x86regs->ebp);\n }\n process->mode = MODE_I386;\n }\n else\n {\n /* 64 bit mode */\n if(!process->in_syscall)\n process->current_syscall = regs.orig_rax;\n if(process->in_syscall)\n get_x86_64_reg(&process->retvalue, regs.rax);\n else\n {\n get_x86_64_reg(&process->params[0], regs.rdi);\n get_x86_64_reg(&process->params[1], regs.rsi);\n get_x86_64_reg(&process->params[2], regs.rdx);\n get_x86_64_reg(&process->params[3], regs.r10);\n get_x86_64_reg(&process->params[4], regs.r8);\n get_x86_64_reg(&process->params[5], regs.r9);\n }\n /* Might still be either native x64 or Linux's x32 layer */\n process->mode = MODE_X86_64;\n }\n#endif\n if(syscall_handle(process) != 0)\n return -1; /* LCOV_EXCL_LINE */\n }\n /* Handle signals */\n else if(WIFSTOPPED(status))\n {\n int signum = WSTOPSIG(status) & 0x7F;\n\n /* Synthetic signal for ptrace event: resume */\n if(signum == SIGTRAP && status & 0xFF0000)\n {\n int event = status >> 16;\n if(event == PTRACE_EVENT_EXEC)\n {\n log_debug(tid,\n \"got EVENT_EXEC, an execve() was successful and \"\n \"will return soon\");\n if(syscall_execve_event(process) != 0)\n return -1;\n }\n else if( (event == PTRACE_EVENT_FORK)\n || (event == PTRACE_EVENT_VFORK)\n || (event == PTRACE_EVENT_CLONE))\n {\n if(syscall_fork_event(process, event) != 0)\n return -1;\n }\n ptrace(PTRACE_SYSCALL, tid, NULL, NULL);\n }\n else if(signum == SIGTRAP)\n {\n /* LCOV_EXCL_START : Processes shouldn't be getting SIGTRAPs */\n log_error(0,\n \"NOT delivering SIGTRAP to %d\\n\"\n \" waitstatus=0x%X\", tid, status);\n ptrace(PTRACE_SYSCALL, tid, NULL, NULL);\n /* LCOV_EXCL_STOP */\n }\n /* Other signal, let the process handle it */\n else\n {\n siginfo_t si;\n log_info(tid, \"caught signal %d\", signum);\n if(ptrace(PTRACE_GETSIGINFO, tid, 0, (long)&si) >= 0)\n ptrace(PTRACE_SYSCALL, tid, NULL, signum);\n else\n {\n /* LCOV_EXCL_START : Not sure what this is for... doesn't\n * seem to happen in practice */\n log_error(tid, \" NOT delivering: %s\", strerror(errno));\n if(signum != SIGSTOP)\n ptrace(PTRACE_SYSCALL, tid, NULL, NULL);\n /* LCOV_EXCL_STOP */\n }\n }\n }\n }\n\n return 0;\n}\n\nstatic void (*python_sigchld_handler)(int) = NULL;\nstatic void (*python_sigint_handler)(int) = NULL;\n\nstatic void restore_signals(void)\n{\n if(python_sigchld_handler != NULL)\n {\n signal(SIGCHLD, python_sigchld_handler);\n python_sigchld_handler = NULL;\n }\n if(python_sigint_handler != NULL)\n {\n signal(SIGINT, python_sigint_handler);\n python_sigint_handler = NULL;\n }\n}\n\nstatic void cleanup(void)\n{\n size_t i;\n {\n size_t nb = 0;\n for(i = 0; i < processes_size; ++i)\n if(processes[i]->status != PROCSTAT_FREE)\n ++nb;\n /* size_t size is implementation dependent; %u for size_t can trigger\n * a warning */\n log_error(0, \"cleaning up, %u processes to kill...\", (unsigned int)nb);\n }\n for(i = 0; i < processes_size; ++i)\n {\n if(processes[i]->status != PROCSTAT_FREE)\n {\n kill(processes[i]->tid, SIGKILL);\n trace_free_process(processes[i]);\n }\n }\n}\n\nstatic time_t last_int = 0;\n\nstatic void sigint_handler(int signo)\n{\n time_t now = time(NULL);\n (void)signo;\n if(now - last_int < 2)\n {\n log_error(0, \"Got 2 SIGINT, aborting ReproZip\");\n cleanup();\n restore_signals();\n exit(128 + 2);\n }\n else\n log_error(0, \"Got SIGINT...\");\n log_error(0, \"If you want to abort ReproZip, press Ctrl-C twice (no trace will be generated)\");\n last_int = now;\n}\n\nstatic void trace_init(void)\n{\n /* Store Python's handlers for restore_signals() */\n python_sigchld_handler = signal(SIGCHLD, SIG_DFL);\n python_sigint_handler = signal(SIGINT, sigint_handler);\n\n if(processes == NULL)\n {\n size_t i;\n struct Process *pool;\n processes_size = 16;\n processes = malloc(processes_size * sizeof(*processes));\n pool = malloc(processes_size * sizeof(*pool));\n for(i = 0; i < processes_size; ++i)\n {\n processes[i] = pool++;\n processes[i]->status = PROCSTAT_FREE;\n processes[i]->threadgroup = NULL;\n processes[i]->execve_info = NULL;\n }\n }\n\n syscall_build_table();\n}\n\nint fork_and_trace(const char *binary, int argc, char **argv,\n const char *database_path, int *exit_status)\n{\n pid_t child;\n\n trace_init();\n\n child = fork();\n\n if(child != 0)\n log_info(0, \"child created, pid=%d\", child);\n\n if(child == 0)\n {\n char **args = malloc((argc + 1) * sizeof(char*));\n memcpy(args, argv, argc * sizeof(char*));\n args[argc] = NULL;\n /* Trace this process */\n if(ptrace(PTRACE_TRACEME, 0, NULL, NULL) != 0)\n {\n log_critical(\n 0,\n \"couldn't use ptrace: %s\\n\"\n \"This could be caused by a security policy or isolation \"\n \"mechanism (such as Docker), see http://bit.ly/2bZd8Fa\",\n strerror(errno));\n exit(125);\n }\n /* Stop this once so tracer can set options */\n kill(getpid(), SIGSTOP);\n /* Execute the target */\n execvp(binary, args);\n log_critical(0, \"couldn't execute the target command (execvp \"\n \"returned): %s\", strerror(errno));\n exit(127);\n }\n\n if(db_init(database_path) != 0)\n {\n kill(child, SIGKILL);\n restore_signals();\n return 1;\n }\n\n /* Creates entry for first process */\n {\n struct Process *process = trace_get_empty_process();\n process->status = PROCSTAT_ALLOCATED; /* Not yet attached... */\n process->flags = 0;\n /* We sent a SIGSTOP, but we resume on attach */\n process->tid = child;\n process->threadgroup = trace_new_threadgroup(child, get_wd());\n process->in_syscall = 0;\n\n log_info(0, \"process %d created by initial fork()\", child);\n if( (db_add_first_process(&process->identifier,\n process->threadgroup->wd) != 0)\n || (db_add_file_open(process->identifier, process->threadgroup->wd,\n FILE_WDIR, 1) != 0) )\n {\n /* LCOV_EXCL_START : Database insertion shouldn't fail */\n db_close(1);\n cleanup();\n restore_signals();\n return 1;\n /* LCOV_EXCL_STOP */\n }\n }\n\n if(trace(child, exit_status) != 0)\n {\n cleanup();\n db_close(1);\n restore_signals();\n return 1;\n }\n\n if(db_close(0) != 0)\n {\n /* LCOV_EXCL_START : Closing database shouldn't fail */\n restore_signals();\n return 1;\n /* LCOV_EXCL_STOP */\n }\n\n restore_signals();\n return 0;\n}\n" }, { "path": "reprozip/native/tracer.h", "content": "#ifndef TRACER_H\n#define TRACER_H\n\n#include \"config.h\"\n\n\nint fork_and_trace(const char *binary, int argc, char **argv,\n const char *database_path, int *exit_status);\n\n\n/* This is NOT a union because sign-extension rules depend on actual register\n * sizes. */\ntypedef struct S_register_type {\n signed long int i;\n unsigned long int u;\n void *p;\n} register_type;\n\n\n#define PROCESS_ARGS 6\n\nstruct ExecveInfo {\n char *binary;\n char **argv;\n char **envp;\n};\n\nvoid free_execve_info(struct ExecveInfo *execi);\n\nstruct ThreadGroup {\n pid_t tgid;\n char *wd;\n unsigned int refs;\n};\n\nstruct Process {\n unsigned int identifier;\n unsigned int mode;\n struct ThreadGroup *threadgroup;\n pid_t tid;\n int status;\n unsigned int flags;\n int in_syscall;\n int current_syscall;\n register_type retvalue;\n register_type params[PROCESS_ARGS];\n struct ExecveInfo *execve_info;\n};\n\n#define PROCSTAT_FREE 0 /* unallocated entry in table */\n#define PROCSTAT_ALLOCATED 1 /* fork() done but not yet attached */\n#define PROCSTAT_ATTACHED 2 /* running process */\n#define PROCSTAT_UNKNOWN 3 /* attached but no corresponding fork() call\n * has finished yet */\n\n#define MODE_I386 1\n#define MODE_X86_64 2 /* In x86_64 mode, syscalls might be native x64\n * or x32 */\n\n#define PROCFLAG_EXECD 1 /* Process is coming out of execve */\n#define PROCFLAG_FORKING 2 /* Process is spawning another with\n * fork/vfork/clone */\n#define PROCFLAG_CLONE3 4 /* Process is executing clone3 (not clone) */\n#define PROCFLAG_OPEN_EXIST 8 /* Process is opening a file that exists */\n\n/* FIXME : This is only exposed because of execve() workaround */\nextern struct Process **processes;\nextern size_t processes_size;\n\n\nstruct Process *trace_find_process(pid_t tid);\n\nstruct Process *trace_get_empty_process(void);\n\nstruct ThreadGroup *trace_new_threadgroup(pid_t tgid, char *wd);\n\nvoid trace_free_process(struct Process *process);\n\nvoid trace_count_processes(unsigned int *p_nproc, unsigned int *p_unknown);\n\nint trace_add_files_from_proc(unsigned int process, pid_t tid,\n const char *binary);\n\n#endif\n" }, { "path": "reprozip/native/utils.c", "content": "#include \n#include \n#include \n#include \n\n#include \n#include \n#include \n#include \n\n#include \"config.h\"\n#include \"database.h\"\n#include \"log.h\"\n\n\nunsigned int flags2mode(int flags)\n{\n unsigned int mode = 0;\n if(!O_RDONLY)\n {\n if(flags & O_WRONLY)\n mode |= FILE_WRITE;\n else if(flags & O_RDWR)\n mode |= FILE_READ | FILE_WRITE;\n else\n mode |= FILE_READ;\n }\n else if(!O_WRONLY)\n {\n if(flags & O_RDONLY)\n mode |= FILE_READ;\n else if(flags & O_RDWR)\n mode |= FILE_READ | FILE_WRITE;\n else\n mode |= FILE_WRITE;\n }\n else\n {\n if( (flags & (O_RDONLY | O_WRONLY)) == (O_RDONLY | O_WRONLY) )\n log_error(0, \"encountered bogus open() flags O_RDONLY|O_WRONLY\");\n /* Carry on anyway */\n if(flags & O_RDONLY)\n mode |= FILE_READ;\n if(flags & O_WRONLY)\n mode |= FILE_WRITE;\n if(flags & O_RDWR)\n mode |= FILE_READ | FILE_WRITE;\n if( (mode & FILE_READ) && (mode & FILE_WRITE) && (flags & O_TRUNC) )\n /* If O_TRUNC is set, consider this a write */\n mode &= ~FILE_READ;\n }\n return mode;\n}\n\nchar *abspath(const char *wd, const char *path)\n{\n size_t len_wd = strlen(wd);\n if(wd[len_wd-1] == '/')\n {\n /* LCOV_EXCL_START : We usually get canonical path names, so we don't\n * run into this one */\n char *result = malloc(len_wd + strlen(path) + 1);\n memcpy(result, wd, len_wd);\n strcpy(result + len_wd, path);\n return result;\n /* LCOV_EXCL_STOP */\n }\n else\n {\n char *result = malloc(len_wd + 1 + strlen(path) + 1);\n memcpy(result, wd, len_wd);\n result[len_wd] = '/';\n strcpy(result + len_wd + 1, path);\n return result;\n }\n}\n\nchar *get_wd(void)\n{\n /* PATH_MAX has issues, don't use it */\n size_t size = 1024;\n char *path;\n for(;;)\n {\n path = malloc(size);\n if(getcwd(path, size) != NULL)\n return path;\n else\n {\n if(errno != ERANGE)\n {\n /* LCOV_EXCL_START : getcwd() really shouldn't fail */\n free(path);\n log_error(0, \"getcwd failed: %s\", strerror(errno));\n return strdup(\"/UNKNOWN\");\n /* LCOV_EXCL_STOP */\n }\n free(path);\n size <<= 1;\n }\n }\n}\n\nchar *read_line(char *buffer, size_t *size, FILE *fp)\n{\n size_t pos = 0;\n if(buffer == NULL)\n {\n *size = 4096;\n buffer = malloc(*size);\n }\n for(;;)\n {\n char c;\n {\n int t = getc(fp);\n if(t == EOF)\n {\n free(buffer);\n return NULL;\n }\n c = t;\n }\n if(c == '\\n')\n {\n buffer[pos] = '\\0';\n return buffer;\n }\n else\n {\n if(pos + 1 >= *size)\n {\n *size <<= 2;\n buffer = realloc(buffer, *size);\n }\n buffer[pos++] = c;\n }\n }\n}\n\nint path_is_dir(const char *pathname)\n{\n struct stat buf;\n if(lstat(pathname, &buf) != 0)\n {\n /* LCOV_EXCL_START : shouldn't happen because a tracer process just\n * accessed it */\n log_error(0, \"error stat()ing %s: %s\", pathname, strerror(errno));\n return 0;\n /* LCOV_EXCL_STOP */\n }\n return S_ISDIR(buf.st_mode)?1:0;\n}\n" }, { "path": "reprozip/native/utils.h", "content": "#ifndef UTILS_H\n#define UTILS_H\n\n#include \n#include \n#include \n\n#include \n#include \n\n\nunsigned int flags2mode(int flags);\n\nchar *abspath(const char *wd, const char *path);\n\nchar *get_wd(void);\n\nchar *read_line(char *buffer, size_t *size, FILE *fp);\n\nint path_is_dir(const char *pathname);\n\n#endif\n" }, { "path": "reprozip/reprozip/__init__.py", "content": "# Copyright (C) 2014 New York University\n# This file is part of ReproZip which is released under the Revised BSD License\n# See file LICENSE for full license details.\n\n__version__ = '1.3.2'\n" }, { "path": "reprozip/reprozip/common.py", "content": "# Copyright (C) 2014 New York University\n# This file is part of ReproZip which is released under the Revised BSD License\n# See file LICENSE for full license details.\n\n# This file is shared:\n# reprozip/reprozip/common.py\n# reprounzip/reprounzip/common.py\n\n\"\"\"Common functions between reprozip and reprounzip.\n\nThis module contains functions that are specific to the reprozip software and\nits data formats, but that are shared between the reprozip and reprounzip\npackages. Because the packages can be installed separately, these functions are\nin a separate module which is duplicated between the packages.\n\nAs long as these are small in number, they are not worth putting in a separate\npackage that reprozip and reprounzip would both depend on.\n\n\"\"\"\n\nfrom __future__ import division, print_function, unicode_literals\n\nimport atexit\nimport contextlib\nimport copy\nfrom datetime import datetime\nfrom distutils.version import LooseVersion\nimport functools\nimport gzip\nimport logging\nimport logging.handlers\nimport os\nfrom rpaths import PosixPath, Path\nimport sys\nimport tarfile\nimport usagestats\nimport yaml\nimport zipfile\n\nfrom .utils import iteritems, itervalues, unicode_, stderr, UniqueNames, \\\n escape, optional_return_type, isodatetime, hsize, join_root, copyfile\n\n\nlogger = logging.getLogger(__name__.split('.', 1)[0])\n\n\nFILE_READ = 0x01\nFILE_WRITE = 0x02\nFILE_WDIR = 0x04\nFILE_STAT = 0x08\nFILE_LINK = 0x10\nFILE_SOCKET = 0x20\n\n\nclass File(object):\n \"\"\"A file, used at some point during the experiment.\n \"\"\"\n comment = None\n\n def __init__(self, path, size=None):\n self.path = path\n self.size = size\n\n def __eq__(self, other):\n return (isinstance(other, File) and\n self.path == other.path)\n\n def __ne__(self, other):\n return not self.__eq__(other)\n\n def __hash__(self):\n return hash(self.path)\n\n\nclass Package(object):\n \"\"\"A distribution package, containing a set of files.\n \"\"\"\n def __init__(self, name, version, files=None, packfiles=True, size=None):\n self.name = name\n self.version = version\n self.files = list(files) if files is not None else []\n self.packfiles = packfiles\n self.size = size\n\n def __eq__(self, other):\n return (isinstance(other, Package) and\n self.name == other.name and\n self.version == other.version)\n\n def __ne__(self, other):\n return not self.__eq__(other)\n\n def add_file(self, file_):\n self.files.append(file_)\n\n def __unicode__(self):\n return '%s (%s)' % (self.name, self.version)\n __str__ = __unicode__\n\n\n# Pack format history:\n# 1: used by reprozip 0.2 through 0.7. Single tar.gz file, metadata under\n# METADATA/, data under DATA/\n# 2: pack is usually not compressed, metadata under METADATA/, data in another\n# DATA.tar.gz (files inside it still have the DATA/ prefix for ease-of-use\n# in unpackers)\n#\n# Pack metadata history:\n# 0.2: used by reprozip 0.2\n# 0.2.1:\n# config: comments directories as such in config\n# trace database: adds executed_files.workingdir, adds processes.exitcode\n# data: packs dynamic linkers\n# 0.3:\n# config: don't list missing (unpacked) files in config\n# trace database: adds opened_files.is_directory\n# 0.3.1: no change\n# 0.3.2: no change\n# 0.4:\n# config: adds input_files, output_files, lists parent directories\n# 0.4.1: no change\n# 0.5: no change\n# 0.6: no change\n# 0.7:\n# moves input_files and output_files from run to global scope\n# adds processes.is_thread column to trace database\n# 0.8: adds 'id' field to run\n\n\nclass RPZPack(object):\n \"\"\"Encapsulates operations on the RPZ pack format.\n \"\"\"\n data = zip = tar = None\n\n def __init__(self, pack):\n self.pack = Path(pack)\n\n if self._open_tar():\n pass\n elif self._open_zip():\n pass\n else:\n raise ValueError(\"File doesn't appear to be an RPZ pack\")\n\n def _open_tar(self):\n try:\n self.tar = tarfile.open(str(self.pack), 'r:*')\n except tarfile.TarError:\n return False\n try:\n f = self.tar.extractfile('METADATA/version')\n except KeyError:\n raise ValueError(\"Invalid ReproZip file\")\n version = f.read()\n f.close()\n if version.startswith(b'REPROZIP VERSION '):\n try:\n version = int(version[17:].rstrip())\n except ValueError:\n version = None\n if version in (1, 2):\n self.version = version\n self.data_prefix = PosixPath(b'DATA')\n else:\n raise ValueError(\n \"Unknown format version %r (maybe you should upgrade \"\n \"reprounzip? I only know versions 1 and 2\" % version)\n else:\n raise ValueError(\"File doesn't appear to be an RPZ pack\")\n\n if self.version == 1:\n self.data = self.tar\n elif version == 2:\n self.data = tarfile.open(\n fileobj=self.tar.extractfile('DATA.tar.gz'),\n mode='r:*')\n else:\n assert False\n return True\n\n def _open_zip(self):\n try:\n self.zip = zipfile.ZipFile(str(self.pack))\n except zipfile.BadZipfile:\n return False\n try:\n f = self.zip.open('METADATA/version')\n except KeyError:\n raise ValueError(\"Invalid ReproZip file\")\n version = f.read()\n f.close()\n if version.startswith(b'REPROZIP VERSION '):\n try:\n version = int(version[17:].rstrip())\n except ValueError:\n version = None\n if version == 1:\n raise ValueError(\"Format version 1 is not accepted for ZIP\")\n elif version == 2:\n self.version = 2\n self.data_prefix = PosixPath(b'DATA')\n else:\n raise ValueError(\n \"Unknown format version %r (maybe you should upgrade \"\n \"reprounzip? I only know versions 1 and 2\" % version)\n else:\n raise ValueError(\"File doesn't appear to be an RPZ pack\")\n\n if sys.version_info < (3, 7):\n # zip.open() doesn't return a seekable file object before 3.6\n # Extract to a temporary file instead\n fd, temporary_data = Path.tempfile(\n prefix='reprounzip_data_',\n suffix='.zip',\n )\n os.close(fd)\n self._extract_file('DATA.tar.gz', temporary_data)\n self.data = tarfile.open(str(temporary_data), mode='r:*')\n atexit.register(os.remove, temporary_data.path)\n else:\n self.data = tarfile.open(fileobj=self.zip.open('DATA.tar.gz'),\n mode='r:*')\n return True\n\n def remove_data_prefix(self, path):\n if not isinstance(path, PosixPath):\n path = PosixPath(path)\n components = path.components[1:]\n if not components:\n return path.__class__('')\n return path.__class__(*components)\n\n def open_config(self):\n \"\"\"Gets the configuration file.\n \"\"\"\n if self.tar is not None:\n return self.tar.extractfile('METADATA/config.yml')\n else:\n return self.zip.open('METADATA/config.yml')\n\n def extract_config(self, target):\n \"\"\"Extracts the config to the specified path.\n\n It is up to the caller to remove that file once done.\n \"\"\"\n self._extract_file('METADATA/config.yml', target)\n\n def _extract_file(self, name, target):\n if self.tar is not None:\n member = copy.copy(self.tar.getmember(name))\n member.name = str(target.components[-1])\n self.tar.extract(member, path=str(Path.cwd() / target.parent))\n else:\n member = copy.copy(self.zip.getinfo(name))\n member.filename = str(target.components[-1])\n self.zip.extract(member, path=str(Path.cwd() / target.parent))\n target.chmod(0o644)\n assert target.is_file()\n\n def _extract_file_gz(self, name, target):\n if self.tar is not None:\n f_in = self.tar.extractfile(name)\n else:\n f_in = self.zip.open(name)\n f_in_gz = gzip.open(f_in)\n f_out = target.open('wb')\n try:\n chunk = f_in_gz.read(4096)\n while len(chunk) == 4096:\n f_out.write(chunk)\n chunk = f_in_gz.read(4096)\n if chunk:\n f_out.write(chunk)\n finally:\n f_out.close()\n f_in_gz.close()\n f_in.close()\n target.chmod(0o644)\n\n @contextlib.contextmanager\n def with_config(self):\n \"\"\"Context manager that extracts the config to a temporary file.\n \"\"\"\n fd, tmp = Path.tempfile(prefix='reprounzip_')\n os.close(fd)\n self.extract_config(tmp)\n yield tmp\n tmp.remove()\n\n def extract_trace(self, target):\n \"\"\"Extracts the trace database to the specified path.\n\n It is up to the caller to remove that file once done.\n \"\"\"\n target = Path(target)\n if self.version == 2:\n try:\n if self.tar is not None:\n self.tar.getmember('METADATA/trace.sqlite3.gz')\n else:\n self.zip.getinfo('METADATA/trace.sqlite3.gz')\n except KeyError:\n pass\n else:\n self._extract_file_gz('METADATA/trace.sqlite3.gz', target)\n return\n elif self.version != 2:\n assert False\n self._extract_file('METADATA/trace.sqlite3', target)\n\n @contextlib.contextmanager\n def with_trace(self):\n \"\"\"Context manager extracting the trace database to a temporary file.\n \"\"\"\n fd, tmp = Path.tempfile(prefix='reprounzip_')\n os.close(fd)\n self.extract_trace(tmp)\n yield tmp\n tmp.remove()\n\n def list_data(self):\n \"\"\"Returns tarfile.TarInfo objects for all the data paths.\n \"\"\"\n return [copy.copy(m)\n for m in self.data.getmembers()\n if m.name.startswith('DATA/')]\n\n def data_filenames(self):\n \"\"\"Returns a set of filenames for all the data paths.\n\n Those paths begin with a slash / and the 'DATA' prefix has been\n removed.\n \"\"\"\n return set(PosixPath(m.name[4:])\n for m in self.data.getmembers()\n if m.name.startswith('DATA/'))\n\n def get_data(self, path):\n \"\"\"Returns a tarfile.TarInfo object for the data path.\n\n Raises KeyError if no such path exists.\n \"\"\"\n path = PosixPath(path)\n path = join_root(PosixPath(b'DATA'), path)\n return copy.copy(self.data.getmember(path))\n\n def extract_data(self, root, members):\n \"\"\"Extracts the given members from the data tarball.\n\n The members must come from get_data().\n \"\"\"\n # Check for CVE-2007-4559\n abs_root = root.absolute()\n for member in members:\n member_path = (root / member.name).absolute()\n if not member_path.lies_under(abs_root):\n raise ValueError(\"Invalid path in data tar\")\n\n self.data.extractall(str(root), members)\n\n def copy_data_tar(self, target):\n \"\"\"Copies the file in which the data lies to the specified destination.\n \"\"\"\n if self.tar is not None:\n if self.version == 1:\n self.pack.copyfile(target)\n elif self.version == 2:\n with target.open('wb') as fp:\n data = self.tar.extractfile('DATA.tar.gz')\n copyfile(data, fp)\n data.close()\n else:\n with target.open('wb') as fp:\n data = self.zip.open('DATA.tar.gz')\n copyfile(data, fp)\n data.close()\n\n def extensions(self):\n \"\"\"Get a list of extensions present in this pack.\n \"\"\"\n extensions = set()\n if self.tar is not None:\n for m in self.tar.getmembers():\n if m.name.startswith('EXTENSIONS/'):\n name = m.name[11:]\n if '/' in name:\n name = name[:name.index('/')]\n if name:\n extensions.add(name)\n else:\n for m in self.zip.infolist():\n if m.filename.startswith('EXTENSIONS/'):\n name = m.filename[11:]\n if '/' in name:\n name = name[:name.index('/')]\n if name:\n extensions.add(name)\n return extensions\n\n def close(self):\n if self.data is not self.tar:\n self.data.close()\n if self.tar is not None:\n self.tar.close()\n elif self.zip is not None:\n self.zip.close()\n self.data = self.zip = self.tar = None\n\n\nclass InvalidConfig(ValueError):\n \"\"\"Configuration file is invalid.\n \"\"\"\n\n\ndef read_files(files, File=File):\n if files is None:\n return []\n return [File(PosixPath(f)) for f in files]\n\n\ndef read_packages(packages, File=File, Package=Package):\n if packages is None:\n return []\n new_pkgs = []\n for pkg in packages:\n pkg['files'] = read_files(pkg['files'], File)\n new_pkgs.append(Package(**pkg))\n return new_pkgs\n\n\nConfig = optional_return_type(['runs', 'packages', 'other_files'],\n ['inputs_outputs', 'additional_patterns',\n 'format_version'])\n\n\n@functools.total_ordering\nclass InputOutputFile(object):\n def __init__(self, path, read_runs, write_runs):\n self.path = path\n self.read_runs = read_runs\n self.write_runs = write_runs\n\n def __eq__(self, other):\n return ((self.path, self.read_runs, self.write_runs) ==\n (other.path, other.read_runs, other.write_runs))\n\n def __lt__(self, other):\n return self.path < other.path\n\n def __repr__(self):\n return \"\" % (\n self.path, self.read_runs, self.write_runs)\n\n\ndef load_iofiles(config, runs):\n \"\"\"Loads the inputs_outputs part of the configuration.\n\n This tests for duplicates, merge the lists of executions, and optionally\n loads from the runs for reprozip < 0.7 compatibility.\n \"\"\"\n files_list = config.get('inputs_outputs') or []\n\n # reprozip < 0.7 compatibility: read input_files and output_files from runs\n if 'inputs_outputs' not in config:\n for i, run in enumerate(runs):\n for rkey, wkey in (('input_files', 'read_by_runs'),\n ('output_files', 'written_by_runs')):\n for k, p in iteritems(run.pop(rkey, {})):\n files_list.append({'name': k,\n 'path': p,\n wkey: [i]})\n\n files = {} # name:str: InputOutputFile\n paths = {} # path:PosixPath: name:str\n required_keys = set(['name', 'path'])\n optional_keys = set(['read_by_runs', 'written_by_runs'])\n uniquenames = UniqueNames()\n for i, f in enumerate(files_list):\n keys = set(f)\n if (not keys.issubset(required_keys | optional_keys) or\n not keys.issuperset(required_keys)):\n raise InvalidConfig(\"File #%d has invalid keys\")\n name = f['name']\n path = PosixPath(f['path'])\n readers = sorted(f.get('read_by_runs', []))\n writers = sorted(f.get('written_by_runs', []))\n if (\n not isinstance(readers, (tuple, list))\n or not all(isinstance(e, int) for e in readers)\n ):\n raise InvalidConfig(\"read_by_runs should be a list of integers\")\n if (\n not isinstance(writers, (tuple, list))\n or not all(isinstance(e, int) for e in writers)\n ):\n raise InvalidConfig(\"written_by_runs should be a list of integers\")\n if name in files:\n if files[name].path != path:\n old_name, name = name, uniquenames(name)\n logger.warning(\"File name appears multiple times: %s\\n\"\n \"Using name %s instead\",\n old_name, name)\n else:\n uniquenames.insert(name)\n if path in paths:\n if paths[path] == name:\n logger.warning(\"File appears multiple times: %s\", name)\n else:\n logger.warning(\"Two files have the same path (but different \"\n \"names): %s, %s\\nUsing name %s\",\n name, paths[path], paths[path])\n name = paths[path]\n files[name].read_runs.update(readers)\n files[name].write_runs.update(writers)\n else:\n paths[path] = name\n files[name] = InputOutputFile(path, readers, writers)\n\n return files\n\n\ndef load_config(filename, canonical, File=File, Package=Package):\n \"\"\"Loads a YAML configuration file.\n\n `File` and `Package` parameters can be used to override the classes that\n will be used to hold files and distribution packages; useful during the\n packing step.\n\n `canonical` indicates whether a canonical configuration file is expected\n (in which case the ``additional_patterns`` section is not accepted). Note\n that this changes the number of returned values of this function.\n \"\"\"\n with filename.open(encoding='utf-8') as fp:\n config = yaml.safe_load(fp)\n\n ver = LooseVersion(config['version'])\n\n keys_ = set(config)\n if 'version' not in keys_:\n raise InvalidConfig(\"Missing version\")\n # Accepts versions from 0.2 to 0.8 inclusive\n elif not LooseVersion('0.2') <= ver < LooseVersion('0.9'):\n pkgname = (__package__ or __name__).split('.', 1)[0]\n raise InvalidConfig(\"Loading configuration file in unknown format %s; \"\n \"this probably means that you should upgrade \"\n \"%s\" % (ver, pkgname))\n unknown_keys = keys_ - set(['pack_id', 'version', 'runs',\n 'inputs_outputs',\n 'packages', 'other_files',\n 'additional_patterns',\n # Deprecated\n 'input_files', 'output_files'])\n if unknown_keys:\n logger.warning(\"Unrecognized sections in configuration: %s\",\n ', '.join(unknown_keys))\n\n runs = config.get('runs') or []\n packages = read_packages(config.get('packages'), File, Package)\n other_files = read_files(config.get('other_files'), File)\n\n inputs_outputs = load_iofiles(config, runs)\n\n # reprozip < 0.7 compatibility: set inputs/outputs on runs (for plugins)\n for i, run in enumerate(runs):\n run['input_files'] = dict((n, f.path)\n for n, f in iteritems(inputs_outputs)\n if i in f.read_runs)\n run['output_files'] = dict((n, f.path)\n for n, f in iteritems(inputs_outputs)\n if i in f.write_runs)\n\n # reprozip < 0.8 compatibility: assign IDs to runs\n for i, run in enumerate(runs):\n if run.get('id') is None:\n run['id'] = \"run%d\" % i\n\n record_usage_package(runs, packages, other_files,\n inputs_outputs,\n pack_id=config.get('pack_id'))\n\n kwargs = {'format_version': ver,\n 'inputs_outputs': inputs_outputs}\n\n if canonical:\n if 'additional_patterns' in config:\n raise InvalidConfig(\"Canonical configuration file shouldn't have \"\n \"additional_patterns key anymore\")\n else:\n kwargs['additional_patterns'] = config.get('additional_patterns') or []\n\n return Config(runs, packages, other_files,\n **kwargs)\n\n\ndef write_file(fp, fi, indent=0):\n fp.write(\"%s - \\\"%s\\\"%s\\n\" % (\n \" \" * indent,\n escape(unicode_(fi.path)),\n ' # %s' % fi.comment if fi.comment is not None else ''))\n\n\ndef write_package(fp, pkg, indent=0):\n indent_str = \" \" * indent\n fp.write(\"%s - name: \\\"%s\\\"\\n\" % (indent_str, escape(pkg.name)))\n fp.write(\"%s version: \\\"%s\\\"\\n\" % (indent_str, escape(pkg.version)))\n if pkg.size is not None:\n fp.write(\"%s size: %d\\n\" % (indent_str, pkg.size))\n fp.write(\"%s packfiles: %s\\n\" % (indent_str, 'true' if pkg.packfiles\n else 'false'))\n fp.write(\"%s files:\\n\"\n \"%s # Total files used: %s\\n\" % (\n indent_str, indent_str,\n hsize(sum(fi.size\n for fi in pkg.files\n if fi.size is not None))))\n if pkg.size is not None:\n fp.write(\"%s # Installed package size: %s\\n\" % (\n indent_str, hsize(pkg.size)))\n for fi in sorted(pkg.files, key=lambda fi_: fi_.path):\n write_file(fp, fi, indent + 1)\n\n\ndef save_config(filename, runs, packages, other_files, reprozip_version,\n inputs_outputs=None,\n canonical=False, pack_id=None):\n \"\"\"Saves the configuration to a YAML file.\n\n `canonical` indicates whether this is a canonical configuration file\n (no ``additional_patterns`` section).\n \"\"\"\n dump = lambda x: yaml.safe_dump(x, encoding='utf-8', allow_unicode=True)\n with filename.open('w', encoding='utf-8', newline='\\n') as fp:\n # Writes preamble\n fp.write(\"\"\"\\\n# ReproZip configuration file\n# This file was generated by reprozip {version} at {date}\n\n{what}\n\n# Run info{pack_id}\nversion: \"{format!s}\"\n\"\"\".format(pack_id=(('\\npack_id: \"%s\"' % pack_id) if pack_id is not None\n else ''),\n version=escape(reprozip_version),\n format='0.8',\n date=isodatetime(),\n what=(\"# It was generated by the packer and you shouldn't need to \"\n \"edit it\" if canonical\n else \"# You might want to edit this file before running the \"\n \"packer\\n# See 'reprozip pack -h' for help\")))\n\n fp.write(\"runs:\\n\")\n for i, run in enumerate(runs):\n # Remove reprozip < 0.7 compatibility fields\n run = dict((k, v) for k, v in iteritems(run)\n if k not in ('input_files', 'output_files'))\n fp.write(\"# Run %d\\n\" % i)\n fp.write(dump([run]).decode('utf-8'))\n fp.write(\"\\n\")\n\n fp.write(\"\"\"\\\n# Input and output files\n\n# Inputs are files that are only read by a run; reprounzip can replace these\n# files on demand to run the experiment with custom data.\n# Outputs are files that are generated by a run; reprounzip can extract these\n# files from the experiment on demand, for the user to examine.\n# The name field is the identifier the user will use to access these files.\ninputs_outputs:\"\"\")\n for n, f in iteritems(inputs_outputs):\n fp.write(\"\"\"\\\n\n- name: {name}\n path: {path}\n written_by_runs: {writers}\n read_by_runs: {readers}\"\"\".format(name=n, path=unicode_(f.path),\n readers=repr(f.read_runs),\n writers=repr(f.write_runs)))\n\n fp.write(\"\"\"\\\n\n\n# Files to pack\n# All the files below were used by the program; they will be included in the\n# generated package\n\n# These files come from packages; we can thus choose not to include them, as it\n# will simply be possible to install that package on the destination system\n# They are included anyway by default\npackages:\n\"\"\")\n\n # Writes files\n for pkg in sorted(packages, key=lambda p: p.name):\n write_package(fp, pkg)\n\n fp.write(\"\"\"\\\n\n# These files do not appear to come with an installed package -- you probably\n# want them packed\nother_files:\n\"\"\")\n for f in sorted(other_files, key=lambda fi: fi.path):\n write_file(fp, f)\n\n if not canonical:\n fp.write(\"\"\"\\\n\n# If you want to include additional files in the pack, you can list additional\n# patterns of files that will be included\nadditional_patterns:\n# Examples:\n# - /etc/apache2/** # Everything under apache2/\n# - /var/log/apache2/*.log # Log files directly under apache2/\n# - /var/lib/lxc/*/rootfs/home/**/*.py # All Python files of all users in\n# # that container\n\"\"\")\n\n\nclass LoggingDateFormatter(logging.Formatter):\n \"\"\"Formatter that puts milliseconds in the timestamp.\n \"\"\"\n converter = datetime.fromtimestamp\n\n def formatTime(self, record, datefmt=None):\n ct = self.converter(record.created)\n t = ct.strftime(\"%H:%M:%S\")\n s = \"%s.%03d\" % (t, record.msecs)\n return s\n\n\ndef setup_logging(tag, verbosity):\n \"\"\"Sets up the logging module.\n \"\"\"\n levels = [logging.CRITICAL, logging.WARNING, logging.INFO, logging.DEBUG]\n console_level = levels[min(verbosity, 3)]\n file_level = logging.INFO\n min_level = min(console_level, file_level)\n\n # Create formatter\n fmt = \"[%s] %%(asctime)s %%(levelname)s: %%(message)s\" % tag\n formatter = LoggingDateFormatter(fmt)\n\n # Console logger\n handler = logging.StreamHandler()\n handler.setLevel(console_level)\n handler.setFormatter(formatter)\n\n # Set up logger\n rootlogger = logging.root\n rootlogger.setLevel(min_level)\n rootlogger.addHandler(handler)\n\n # File logger\n if os.environ.get('REPROZIP_NO_LOGFILE', '').lower() in ('', 'false',\n '0', 'off'):\n dotrpz = Path('~/.reprozip').expand_user()\n try:\n if not dotrpz.is_dir():\n dotrpz.mkdir()\n filehandler = logging.handlers.RotatingFileHandler(\n str(dotrpz / 'log'), mode='a',\n delay=False, maxBytes=400000, backupCount=5)\n except (IOError, OSError):\n logger.warning(\"Couldn't create log file %s\", dotrpz / 'log')\n else:\n filehandler.setFormatter(formatter)\n filehandler.setLevel(file_level)\n rootlogger.addHandler(filehandler)\n\n filehandler.emit(logging.root.makeRecord(\n __name__.split('.', 1)[0],\n logging.INFO,\n \"(log start)\", 0,\n \"Log opened %s %s\",\n (datetime.now().strftime(\"%Y-%m-%d\"), sys.argv),\n None))\n\n logging.getLogger('urllib3').setLevel(logging.INFO)\n\n\n_usage_report = None\n\n\ndef setup_usage_report(name, version):\n \"\"\"Sets up the usagestats module.\n \"\"\"\n global _usage_report\n\n certificate_file = get_reprozip_ca_certificate()\n\n _usage_report = usagestats.Stats(\n '~/.reprozip/usage_stats',\n usagestats.Prompt(enable='%s usage_report --enable' % name,\n disable='%s usage_report --disable' % name),\n os.environ.get('REPROZIP_USAGE_URL',\n 'https://stats.reprozip.org/'),\n version='%s %s' % (name, version),\n unique_user_id=True,\n env_var='REPROZIP_USAGE_STATS',\n ssl_verify=certificate_file.path)\n try:\n os.getcwd().encode('ascii')\n except (UnicodeEncodeError, UnicodeDecodeError):\n record_usage(cwd_ascii=False)\n else:\n record_usage(cwd_ascii=True)\n\n\ndef enable_usage_report(enable):\n \"\"\"Enables or disables usage reporting.\n \"\"\"\n if enable:\n _usage_report.enable_reporting()\n stderr.write(\"Thank you, usage reports will be sent automatically \"\n \"from now on.\\n\")\n else:\n _usage_report.disable_reporting()\n stderr.write(\"Usage reports will not be collected nor sent.\\n\")\n\n\ndef record_usage(**kwargs):\n \"\"\"Records some info in the current usage report.\n \"\"\"\n if _usage_report is not None:\n _usage_report.note(kwargs)\n\n\ndef record_usage_package(runs, packages, other_files,\n inputs_outputs,\n pack_id=None):\n \"\"\"Records the info on some pack file into the current usage report.\n \"\"\"\n if _usage_report is None:\n return\n for run in runs:\n record_usage(argv0=run['argv'][0])\n record_usage(pack_id=pack_id or '',\n nb_packages=len(packages),\n nb_package_files=sum(len(pkg.files)\n for pkg in packages),\n packed_packages=sum(1 for pkg in packages\n if pkg.packfiles),\n nb_other_files=len(other_files),\n nb_input_outputs_files=len(inputs_outputs),\n nb_input_files=sum(1 for f in itervalues(inputs_outputs)\n if f.read_runs),\n nb_output_files=sum(1 for f in itervalues(inputs_outputs)\n if f.write_runs))\n\n\ndef submit_usage_report(**kwargs):\n \"\"\"Submits the current usage report to the usagestats server.\n \"\"\"\n _usage_report.submit(kwargs,\n usagestats.OPERATING_SYSTEM,\n usagestats.SESSION_TIME,\n usagestats.PYTHON_VERSION)\n\n\ndef get_reprozip_ca_certificate():\n \"\"\"Gets the ReproZip CA certificate filename.\n \"\"\"\n fd, certificate_file = Path.tempfile(prefix='rpz_stats_ca_', suffix='.pem')\n with certificate_file.open('wb') as fp:\n fp.write(usage_report_ca)\n os.close(fd)\n atexit.register(os.remove, certificate_file.path)\n return certificate_file\n\n\nusage_report_ca = b'''\\\n-----BEGIN CERTIFICATE-----\nMIIDzzCCAregAwIBAgIJAMmlcDnTidBEMA0GCSqGSIb3DQEBCwUAMH4xCzAJBgNV\nBAYTAlVTMREwDwYDVQQIDAhOZXcgWW9yazERMA8GA1UEBwwITmV3IFlvcmsxDDAK\nBgNVBAoMA05ZVTERMA8GA1UEAwwIUmVwcm9aaXAxKDAmBgkqhkiG9w0BCQEWGXJl\ncHJvemlwLWRldkB2Z2MucG9seS5lZHUwHhcNMTQxMTA3MDUxOTA5WhcNMjQxMTA0\nMDUxOTA5WjB+MQswCQYDVQQGEwJVUzERMA8GA1UECAwITmV3IFlvcmsxETAPBgNV\nBAcMCE5ldyBZb3JrMQwwCgYDVQQKDANOWVUxETAPBgNVBAMMCFJlcHJvWmlwMSgw\nJgYJKoZIhvcNAQkBFhlyZXByb3ppcC1kZXZAdmdjLnBvbHkuZWR1MIIBIjANBgkq\nhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA1fuTW2snrVji51vGVl9hXAAZbNJ+dxG+\n/LOOxZrF2f1RRNy8YWpeCfGbsZqiIEjorBv8lvdd9P+tD3M5sh9L0zQPU9dFvDb+\nOOrV0jx59hbK3QcCQju3YFuAtD1lu8TBIPgGEab0eJhLVIX+XU5cYXrfoBmwCpN/\n1wXWkUhN91ZVMA0ylATAxTpnoNuMKzfTxT8pyOWajiTskYkKmVBAxgYJQe1YDFA8\nfglBNkQuHqP8jgYAniEBCAPZRMMq8WpOtyFx+L9LX9/WcHtAQyDPPb9M81KKgPQq\nurtCqtuDKxuqcX9zg4/O8l4nZ50pwaJjbH4kMW/wnLzTPvzZCPtJYQIDAQABo1Aw\nTjAdBgNVHQ4EFgQUJjhDDOup4P0cdrAVq1F9ap3yTj8wHwYDVR0jBBgwFoAUJjhD\nDOup4P0cdrAVq1F9ap3yTj8wDAYDVR0TBAUwAwEB/zANBgkqhkiG9w0BAQsFAAOC\nAQEAeKpTiy2WYPqevHseTCJDIL44zghDJ9w5JmECOhFgPXR9Hl5Nh9S1j4qHBs4G\ncn8d1p2+8tgcJpNAysjuSl4/MM6hQNecW0QVqvJDQGPn33bruMB4DYRT5du1Zpz1\nYIKRjGU7Of3CycOCbaT50VZHhEd5GS2Lvg41ngxtsE8JKnvPuim92dnCutD0beV+\n4TEvoleIi/K4AZWIaekIyqazd0c7eQjgSclNGgePcdbaxIo0u6tmdTYk3RNzo99t\nDCfXxuMMg3wo5pbqG+MvTdECaLwt14zWU259z8JX0BoeVG32kHlt2eUpm5PCfxqc\ndYuwZmAXksp0T0cWo0DnjJKRGQ==\n-----END CERTIFICATE-----\n-----BEGIN CERTIFICATE-----\nMIIDRDCCAiygAwIBAgIUXaa8P7qR4c0P51hCDIqj4GUbG/owDQYJKoZIhvcNAQEL\nBQAwEzERMA8GA1UEAwwIUmVwcm9aaXAwIBcNMjEwNDI5MjEwNTUzWhgPMjEyMTA0\nMjkyMTA1NTNaMBMxETAPBgNVBAMMCFJlcHJvWmlwMIIBIjANBgkqhkiG9w0BAQEF\nAAOCAQ8AMIIBCgKCAQEA3udPriZ8kziQE+OyLVozJFDSZTe8RLlpFsu/ZQjSnIh1\nTsENMMu1lwv0GVEpT/EbtD5ORtZzwYQ7Vuh+IO4TQDhA5KvyJD2gZW8hE4txkkQd\nyI5vSj0iiViA80tKB7FSDLsvz9iiDxShYHJI947gswbaLmampHIXD/Rjjs7+hmL5\nRRS5NL8vCp2/2QVj5wnJupa5O2l2T1M6S/SyFcAgBMM/FhDsaA/yf4NPcOG6gFuO\nb5mYz2ERSf4v9mRL+G3r6YULYGZWS5ThY0QoZ0lYt2nlthzwfftazrJ9+yfYBkoJ\nK6Ug8UGtyOb5m3mK00c4wS7/wzuGgLMszkE0nE9SfwIDAQABo4GNMIGKMB0GA1Ud\nDgQWBBSqrIPVnO5vkHj9ImGvOr38r4rcNjBOBgNVHSMERzBFgBSqrIPVnO5vkHj9\nImGvOr38r4rcNqEXpBUwEzERMA8GA1UEAwwIUmVwcm9aaXCCFF2mvD+6keHND+dY\nQgyKo+BlGxv6MAwGA1UdEwQFMAMBAf8wCwYDVR0PBAQDAgEGMA0GCSqGSIb3DQEB\nCwUAA4IBAQC2g8yX1c5JutH/qAKUVvqSwP2KBm3IyOjdbN7cvnwn0gMkwEj88j7p\ndKhfO0Kfp/N4iKj1YK7PBXfrdlYhxINSbfPSVS3A9XWi8pJwiwgBfjrrACRMhBsv\nHAQPnkXnaEJrQm//k8s4etT25JYaPgXS8t+dgVS0iqonYlpCB1XkE0gw1kLNCW5F\n1SimUehJ5bZrJYGgo6kp44F12kPMzNHvk50Sf2p3nm2d9/BNbbJQxUBKt9n+i6Ir\nxGGDWfg5F+BLKURWkoGBnnLPqkRxlkaGvk6QpIAfD1h99fTyuWUno3+NoQ7hS952\nyWbmqwbavTIyG+D+kfhbGFEdRxLF5BeK\n-----END CERTIFICATE-----\n'''\n" }, { "path": "reprozip/reprozip/filters.py", "content": "# Copyright (C) 2014 New York University\n# This file is part of ReproZip which is released under the Revised BSD License\n# See file LICENSE for full license details.\n\nfrom __future__ import division, print_function, unicode_literals\n\nimport logging\nimport re\nfrom rpaths import Path\n\nfrom reprozip.tracer.trace import TracedFile\nfrom reprozip.utils import irange, iteritems\n\n\nlogger = logging.getLogger('reprozip')\n\n\n_so_file = re.compile(br'\\.so(\\.[0-9]+)*$')\n\n\ndef builtin(input_files, **kwargs):\n \"\"\"Default heuristics for input files.\n \"\"\"\n for i in irange(len(input_files)):\n lst = []\n for path in input_files[i]:\n if (\n # Hidden files\n path.unicodename[0] == '.' or\n # Shared libraries\n _so_file.search(path.name)\n ):\n logger.info(\"Removing input %s\", path)\n else:\n lst.append(path)\n\n input_files[i] = lst\n\n\ndef python(files, input_files, **kwargs):\n add = []\n for path, fi in iteritems(files):\n # Include .py files instead of .pyc\n if path.ext == b'.pyc':\n if path.parent.name == b'__pycache__':\n # Python 3: /dir/__pycache__/mod.cpython-38.pyc -> /dir/mod.py\n basename = path.unicodename.split('.', 1)[0]\n pyfile = path.parent.parent / basename + '.py'\n else:\n # Python2: /dir/mod.pyc -> /dir/moc.py\n pyfile = path.parent / path.stem + '.py'\n if pyfile.is_file():\n if pyfile not in files:\n logger.info(\"Adding %s\", pyfile)\n add.append(TracedFile(pyfile))\n\n for fi in add:\n files[fi.path] = fi\n\n for i in irange(len(input_files)):\n lst = []\n for path in input_files[i]:\n if (\n path.ext in (b'.py', b'.pyc')\n or (\n b'site-packages' in path.components\n and path.ext == b'.pth'\n )\n ):\n logger.info(\"Removing input %s\", path)\n else:\n lst.append(path)\n\n input_files[i] = lst\n\n\ndef ruby(files, input_files, **kwargs):\n extensions = set(ext.encode('utf-8') for ext in [\n '.rb', '.haml', '.slim', '.erb', '.js', '.html',\n ])\n ignored_dirs = set(name.encode('utf-8') for name in [\n 'spec', 'test', 'tests', 'guides', 'doc-api', 'rdoc', 'doc',\n ])\n\n gemy_path = re.compile(r'^.*/ruby[-/]\\d+\\.\\d+\\.\\d+/gems')\n appdir_paths = re.compile(r'^.*/app/(views|controllers|models|helpers)')\n\n directories = set()\n\n for path, fi in iteritems(files):\n m1 = gemy_path.match(str(path))\n if m1:\n directories.add(Path(m1.group(0)))\n\n m2 = appdir_paths.match(str(path))\n if m2:\n app_root = Path(m2.group(0)).parent.parent\n if (app_root / 'config/application.rb').is_file():\n directories.add(app_root)\n\n def add_recursive(dir_or_file):\n if (\n dir_or_file.is_file()\n and dir_or_file.ext in extensions\n ):\n logger.info(\"Adding %s\", dir_or_file)\n files[dir_or_file] = TracedFile(dir_or_file)\n elif (\n dir_or_file.is_dir()\n and dir_or_file.name not in ignored_dirs\n ):\n for child in dir_or_file.listdir():\n add_recursive(child)\n\n for directory in directories:\n add_recursive(directory)\n\n for i in irange(len(input_files)):\n lst = []\n for path in input_files[i]:\n if gemy_path.match(str(path)) or appdir_paths.match(str(path)):\n logger.info(\"Removing input %s\", path)\n else:\n lst.append(path)\n\n input_files[i] = lst\n" }, { "path": "reprozip/reprozip/main.py", "content": "# Copyright (C) 2014 New York University\n# This file is part of ReproZip which is released under the Revised BSD License\n# See file LICENSE for full license details.\n\n\"\"\"Entry point for the reprozip utility.\n\nThis contains :func:`~reprozip.main.main`, which is the entry point declared to\nsetuptools. It is also callable directly.\n\nIt dispatchs to other routines, or handles the testrun command.\n\"\"\"\n\nfrom __future__ import division, print_function, unicode_literals\n\nif __name__ == '__main__': # noqa\n from reprozip.main import main\n main()\n\nimport argparse\nimport locale\nimport logging\nimport os\nfrom rpaths import Path\nimport sqlite3\nimport sys\nimport traceback\n\nfrom reprozip import __version__ as reprozip_version\nfrom reprozip import _pytracer\nfrom reprozip.common import setup_logging, \\\n setup_usage_report, enable_usage_report, \\\n submit_usage_report, record_usage\nimport reprozip.pack\nimport reprozip.tracer.trace\nimport reprozip.traceutils\nfrom reprozip.utils import PY3, unicode_, stderr\n\n\nlogger = logging.getLogger('reprozip')\n\n\nsafe_shell_chars = set(\"ABCDEFGHIJKLMNOPQRSTUVWXYZ\"\n \"abcdefghijklmnopqrstuvwxyz\"\n \"0123456789\"\n \"-+=/:.,%_\")\n\n\ndef shell_escape(s):\n r\"\"\"Given bl\"a, returns \"bl\\\\\"a\".\n \"\"\"\n if isinstance(s, bytes):\n s = s.decode('utf-8')\n if not s or any(c not in safe_shell_chars for c in s):\n return '\"%s\"' % (s.replace('\\\\', '\\\\\\\\')\n .replace('\"', '\\\\\"')\n .replace('`', '\\\\`')\n .replace('$', '\\\\$'))\n else:\n return s\n\n\ndef print_db(database):\n \"\"\"Prints out database content.\n \"\"\"\n assert database.is_file()\n if PY3:\n # On PY3, connect() only accepts unicode\n conn = sqlite3.connect(str(database))\n else:\n conn = sqlite3.connect(database.path)\n conn.row_factory = sqlite3.Row\n conn.text_factory = lambda x: unicode_(x, 'utf-8', 'replace')\n\n cur = conn.cursor()\n rows = cur.execute(\n '''\n SELECT id, parent, timestamp, exitcode\n FROM processes;\n ''')\n print(\"\\nProcesses:\")\n header = \"+------+--------+-------+------------------+\"\n print(header)\n print(\"| id | parent | exit | timestamp |\")\n print(header)\n for r_id, r_parent, r_timestamp, r_exit in rows:\n f_id = \"{0: 5d} \".format(r_id)\n if r_parent is not None:\n f_parent = \"{0: 7d} \".format(r_parent)\n else:\n f_parent = \" \"\n if r_exit & 0x0100:\n f_exit = \" sig{0: <2d} \".format(r_exit)\n else:\n f_exit = \" {0: <2d} \".format(r_exit)\n f_timestamp = \"{0: 17d} \".format(r_timestamp)\n print('|'.join(('', f_id, f_parent, f_exit, f_timestamp, '')))\n print(header)\n cur.close()\n\n cur = conn.cursor()\n rows = cur.execute(\n '''\n SELECT id, name, timestamp, process, argv\n FROM executed_files;\n ''')\n print(\"\\nExecuted files:\")\n header = (\"+--------+------------------+---------+------------------------\"\n \"---------------+\")\n print(header)\n print(\"| id | timestamp | process | name and argv \"\n \" |\")\n print(header)\n for r_id, r_name, r_timestamp, r_process, r_argv in rows:\n f_id = \"{0: 7d} \".format(r_id)\n f_timestamp = \"{0: 17d} \".format(r_timestamp)\n f_proc = \"{0: 8d} \".format(r_process)\n argv = r_argv.split('\\0')\n if not argv[-1]:\n argv = argv[:-1]\n cmdline = ' '.join(shell_escape(a) for a in argv)\n if argv[0] != r_name:\n cmdline = \"(%s) %s\" % (shell_escape(r_name), cmdline)\n f_cmdline = \" {0: <37s} \".format(cmdline)\n print('|'.join(('', f_id, f_timestamp, f_proc, f_cmdline, '')))\n print(header)\n cur.close()\n\n cur = conn.cursor()\n rows = cur.execute(\n '''\n SELECT id, name, timestamp, mode, process\n FROM opened_files;\n ''')\n print(\"\\nFiles:\")\n header = (\"+--------+------------------+---------+------+-----------------\"\n \"---------------+\")\n print(header)\n print(\"| id | timestamp | process | mode | name \"\n \" |\")\n print(header)\n for r_id, r_name, r_timestamp, r_mode, r_process in rows:\n f_id = \"{0: 7d} \".format(r_id)\n f_timestamp = \"{0: 17d} \".format(r_timestamp)\n f_proc = \"{0: 8d} \".format(r_process)\n f_mode = \"{0: 5d} \".format(r_mode)\n f_name = \" {0: <30s} \".format(r_name)\n print('|'.join(('', f_id, f_timestamp, f_proc, f_mode, f_name, '')))\n print(header)\n cur.close()\n\n conn.close()\n\n\ndef testrun(args):\n \"\"\"testrun subcommand.\n\n Runs the command with the tracer using a temporary sqlite3 database, then\n reads it and dumps it out.\n\n Not really useful, except for debugging.\n \"\"\"\n fd, database = Path.tempfile(prefix='reprozip_', suffix='.sqlite3')\n os.close(fd)\n try:\n if args.arg0 is not None:\n argv = [args.arg0] + args.cmdline[1:]\n else:\n argv = args.cmdline\n logger.debug(\"Starting tracer, binary=%r, argv=%r\",\n args.cmdline[0], argv)\n c = _pytracer.execute(args.cmdline[0], argv, database.path)\n print(\"\\n\\n-----------------------------------------------------------\"\n \"--------------------\")\n print_db(database)\n if c != 0:\n if c & 0x0100:\n print(\"\\nWarning: program appears to have been terminated by \"\n \"signal %d\" % (c & 0xFF))\n else:\n print(\"\\nWarning: program exited with non-zero code %d\" % c)\n\n return c\n finally:\n database.remove()\n\n\ndef trace(args):\n \"\"\"trace subcommand.\n\n Simply calls reprozip.tracer.trace() with the arguments from argparse.\n \"\"\"\n if args.arg0 is not None:\n argv = [args.arg0] + args.cmdline[1:]\n else:\n argv = args.cmdline\n if args.append and args.overwrite:\n logger.critical(\"You can't use both --continue and --overwrite\")\n sys.exit(2)\n elif args.append:\n append = True\n elif args.overwrite:\n append = False\n else:\n append = None\n status = reprozip.tracer.trace.trace(args.cmdline[0],\n argv,\n Path(args.dir),\n append,\n args.verbosity)\n reprozip.tracer.trace.write_configuration(Path(args.dir),\n args.identify_packages,\n args.find_inputs_outputs,\n overwrite=False)\n return status\n\n\ndef reset(args):\n \"\"\"reset subcommand.\n\n Just regenerates the configuration (config.yml) from the trace\n (trace.sqlite3).\n \"\"\"\n reprozip.tracer.trace.write_configuration(Path(args.dir),\n args.identify_packages,\n args.find_inputs_outputs,\n overwrite=True)\n\n\ndef pack(args):\n \"\"\"pack subcommand.\n\n Reads in the configuration file and writes out a tarball.\n \"\"\"\n target = Path(args.target)\n if not target.unicodename.lower().endswith('.rpz'):\n target = Path(target.path + b'.rpz')\n logger.warning(\"Changing output filename to %s\", target.unicodename)\n reprozip.pack.pack(target, Path(args.dir), args.identify_packages)\n\n\ndef combine(args):\n \"\"\"combine subcommand.\n\n Reads in multiple trace databases and combines them into one.\n\n The runs from the original traces are appended ('run_id' field gets\n translated to avoid conflicts).\n \"\"\"\n traces = []\n for tracepath in args.traces:\n if tracepath == '-':\n tracepath = Path(args.dir) / 'trace.sqlite3'\n else:\n tracepath = Path(tracepath)\n if tracepath.is_dir():\n tracepath = tracepath / 'trace.sqlite3'\n traces.append(tracepath)\n\n reprozip.traceutils.combine_traces(traces, Path(args.dir))\n reprozip.tracer.trace.write_configuration(Path(args.dir),\n args.identify_packages,\n args.find_inputs_outputs,\n overwrite=True)\n\n\ndef usage_report(args):\n if bool(args.enable) == bool(args.disable):\n logger.critical(\"What do you want to do?\")\n sys.exit(2)\n enable_usage_report(args.enable)\n sys.exit(0)\n\n\ndef main():\n \"\"\"Entry point when called on the command-line.\n \"\"\"\n # Locale\n try:\n locale.setlocale(locale.LC_ALL, '')\n except locale.Error as e:\n stderr.write(\"Couldn't set locale: %s\\n\" % e)\n\n # http://bugs.python.org/issue13676\n # This prevents reprozip from reading argv and envp arrays from trace\n if sys.version_info < (2, 7, 3):\n stderr.write(\"Error: your version of Python, %s, is not supported\\n\"\n \"Versions before 2.7.3 are affected by bug 13676 and \"\n \"will not work with ReproZip\\n\" %\n sys.version.split(' ', 1)[0])\n sys.exit(125)\n\n # Parses command-line\n\n # General options\n def add_options(opt):\n opt.add_argument('--version', action='version',\n version=\"reprozip version %s\" % reprozip_version)\n opt.add_argument('-d', '--dir', default='.reprozip-trace',\n help=\"where to store database and configuration file \"\n \"(default: ./.reprozip-trace)\")\n opt.add_argument(\n '--dont-identify-packages', action='store_false', default=True,\n dest='identify_packages',\n help=\"do not try identify which package each file comes from\")\n opt.add_argument(\n '--dont-find-inputs-outputs', action='store_false',\n default=True, dest='find_inputs_outputs',\n help=\"do not try to identify input and output files\")\n\n parser = argparse.ArgumentParser(\n description=\"reprozip is the ReproZip component responsible for \"\n \"tracing and packing the execution of an experiment\",\n epilog=\"Please report issues to reprozip@nyu.edu\")\n add_options(parser)\n parser.add_argument('-v', '--verbose', action='count', default=1,\n dest='verbosity',\n help=\"augments verbosity level\")\n subparsers = parser.add_subparsers(title=\"commands\", metavar='',\n dest='selected_command')\n\n # usage_report subcommand\n parser_stats = subparsers.add_parser(\n 'usage_report',\n help=\"Enables or disables anonymous usage reports\")\n add_options(parser_stats)\n parser_stats.add_argument('--enable', action='store_true')\n parser_stats.add_argument('--disable', action='store_true')\n parser_stats.set_defaults(func=usage_report)\n\n # trace command\n parser_trace = subparsers.add_parser(\n 'trace',\n help=\"Runs the program and writes out database and configuration file\")\n add_options(parser_trace)\n parser_trace.add_argument(\n '-a',\n dest='arg0',\n help=\"argument 0 to program, if different from program path\")\n parser_trace.add_argument(\n '-c', '--continue', action='store_true', dest='append',\n help=\"add to the previous trace, don't replace it\")\n parser_trace.add_argument(\n '-w', '--overwrite', action='store_true', dest='overwrite',\n help=\"overwrite the previous trace, don't add to it\")\n parser_trace.add_argument('cmdline', nargs=argparse.REMAINDER,\n help=\"command-line to run under trace\")\n parser_trace.set_defaults(func=trace)\n\n # testrun command\n parser_testrun = subparsers.add_parser(\n 'testrun',\n help=\"Runs the program and writes out the database contents\")\n add_options(parser_testrun)\n parser_testrun.add_argument(\n '-a',\n dest='arg0',\n help=\"argument 0 to program, if different from program path\")\n parser_testrun.add_argument('cmdline', nargs=argparse.REMAINDER)\n parser_testrun.set_defaults(func=testrun)\n\n # reset command\n parser_reset = subparsers.add_parser(\n 'reset',\n help=\"Resets the configuration file\")\n add_options(parser_reset)\n parser_reset.set_defaults(func=reset)\n\n # pack command\n parser_pack = subparsers.add_parser(\n 'pack',\n help=\"Packs the experiment according to the current configuration\")\n add_options(parser_pack)\n parser_pack.add_argument('target', nargs=argparse.OPTIONAL,\n default='experiment.rpz',\n help=\"Destination file\")\n parser_pack.set_defaults(func=pack)\n\n # combine command\n parser_combine = subparsers.add_parser(\n 'combine',\n help=\"Combine multiple traces into one (possibly as subsequent runs)\")\n add_options(parser_combine)\n parser_combine.add_argument('traces', nargs=argparse.ONE_OR_MORE)\n parser_combine.set_defaults(func=combine)\n\n args = parser.parse_args()\n setup_logging('REPROZIP', args.verbosity)\n if getattr(args, 'func', None) is None:\n parser.print_help(sys.stderr)\n sys.exit(2)\n setup_usage_report('reprozip', reprozip_version)\n if 'cmdline' in args and not args.cmdline:\n parser.error(\"missing command-line\")\n record_usage(command=args.selected_command)\n try:\n status = args.func(args)\n except Exception as e:\n traceback.print_exc()\n submit_usage_report(result=type(e).__name__)\n sys.exit(125)\n else:\n submit_usage_report(result='success')\n if status is None:\n sys.exit(0)\n else:\n sys.exit(int(status))\n" }, { "path": "reprozip/reprozip/pack.py", "content": "# Copyright (C) 2014 New York University\n# This file is part of ReproZip which is released under the Revised BSD License\n# See file LICENSE for full license details.\n\n\"\"\"Packing logic for reprozip.\n\nThis module contains the :func:`~reprozip.pack.pack` function and associated\nutilities that are used to build the .rpz pack file from the trace SQLite file\nand config YAML.\n\"\"\"\n\nfrom __future__ import division, print_function, unicode_literals\n\nimport itertools\nimport logging\nimport os\nfrom rpaths import Path\nimport string\nimport sys\nimport tarfile\nimport uuid\n\nfrom reprozip import __version__ as reprozip_version\nfrom reprozip.common import File, load_config, save_config, \\\n record_usage_package\nfrom reprozip.tracer.linux_pkgs import identify_packages\nfrom reprozip.traceutils import combine_files\nfrom reprozip.utils import iteritems\n\n\nlogger = logging.getLogger('reprozip')\n\n\ndef expand_patterns(patterns):\n files = set()\n dirs = set()\n\n # Finds all matching paths\n for pattern in patterns:\n if logger.isEnabledFor(logging.DEBUG):\n logger.debug(\"Expanding pattern %r into %d paths\",\n pattern,\n len(list(Path('/').recursedir(pattern))))\n for path in Path('/').recursedir(pattern):\n if path.is_dir():\n dirs.add(path)\n else:\n files.add(path)\n\n # Don't include directories whose files are included\n non_empty_dirs = set([Path('/')])\n for p in files | dirs:\n path = Path('/')\n for c in p.components[1:]:\n path = path / c\n non_empty_dirs.add(path)\n\n # Builds the final list\n return [File(p) for p in itertools.chain(dirs - non_empty_dirs, files)]\n\n\ndef canonicalize_config(packages, other_files, additional_patterns,\n sort_packages):\n \"\"\"Expands ``additional_patterns`` from the configuration file.\n \"\"\"\n if additional_patterns:\n add_files = expand_patterns(additional_patterns)\n logger.info(\"Found %d files from expanding additional_patterns...\",\n len(add_files))\n if add_files:\n if sort_packages:\n add_files, add_packages = identify_packages(add_files)\n else:\n add_packages = []\n other_files, packages = combine_files(add_files, add_packages,\n other_files, packages)\n return packages, other_files\n\n\ndef data_path(filename, prefix=Path('DATA')):\n \"\"\"Computes the filename to store in the archive.\n\n Turns an absolute path containing '..' into a filename without '..', and\n prefixes with DATA/.\n\n Example:\n\n >>> data_path(PosixPath('/var/lib/../../../../tmp/test'))\n PosixPath(b'DATA/tmp/test')\n >>> data_path(PosixPath('/var/lib/../www/index.html'))\n PosixPath(b'DATA/var/www/index.html')\n \"\"\"\n return prefix / filename.split_root()[1]\n\n\nclass PackBuilder(object):\n \"\"\"Higher layer on tarfile that adds intermediate directories.\n \"\"\"\n def __init__(self, filename):\n self.tar = tarfile.open(str(filename), 'w:gz')\n self.seen = set()\n\n def add_data(self, filename):\n if filename in self.seen:\n return\n path = Path('/')\n for c in filename.components[1:]:\n path = path / c\n if path in self.seen:\n continue\n logger.debug(\"%s -> %s\", path, data_path(path))\n self.tar.add(str(path), str(data_path(path)), recursive=False)\n self.seen.add(path)\n\n def close(self):\n self.tar.close()\n self.seen = None\n\n\ndef pack(target, directory, sort_packages):\n \"\"\"Main function for the pack subcommand.\n \"\"\"\n if target.exists():\n # Don't overwrite packs...\n logger.critical(\"Target file exists!\")\n sys.exit(1)\n\n # Reads configuration\n configfile = directory / 'config.yml'\n if not configfile.is_file():\n logger.critical(\"Configuration file does not exist!\\n\"\n \"Did you forget to run 'reprozip trace'?\\n\"\n \"If not, you might want to use --dir to specify an \"\n \"alternate location.\")\n sys.exit(1)\n runs, packages, other_files = config = load_config(\n configfile,\n canonical=False)\n additional_patterns = config.additional_patterns\n inputs_outputs = config.inputs_outputs\n\n # Validate run ids\n run_chars = ('0123456789_-@() .:%'\n 'abcdefghijklmnopqrstuvwxyz'\n 'ABCDEFGHIJKLMNOPQRSTUVWXYZ')\n for i, run in enumerate(runs):\n if (any(c not in run_chars for c in run['id']) or\n all(c in string.digits for c in run['id'])):\n logger.critical(\"Illegal run id: %r (run number %d)\",\n run['id'], i)\n sys.exit(1)\n\n # Canonicalize config (re-sort, expand 'additional_files' patterns)\n packages, other_files = canonicalize_config(\n packages, other_files, additional_patterns, sort_packages)\n\n logger.info(\"Creating pack %s...\", target)\n tar = tarfile.open(str(target), 'w:')\n\n fd, tmp = Path.tempfile()\n os.close(fd)\n try:\n datatar = PackBuilder(tmp)\n # Add the files from the packages\n for pkg in packages:\n if pkg.packfiles:\n logger.info(\"Adding files from package %s...\", pkg.name)\n files = []\n for f in pkg.files:\n if not Path(f.path).exists():\n logger.warning(\"Missing file %s from package %s\",\n f.path, pkg.name)\n else:\n datatar.add_data(f.path)\n files.append(f)\n pkg.files = files\n else:\n logger.info(\"NOT adding files from package %s\", pkg.name)\n\n # Add the rest of the files\n logger.info(\"Adding other files...\")\n files = set()\n for f in other_files:\n if not Path(f.path).exists():\n logger.warning(\"Missing file %s\", f.path)\n else:\n datatar.add_data(f.path)\n files.add(f)\n other_files = files\n datatar.close()\n\n tar.add(str(tmp), 'DATA.tar.gz')\n finally:\n tmp.remove()\n\n logger.info(\"Adding metadata...\")\n # Stores pack version\n fd, manifest = Path.tempfile(prefix='reprozip_', suffix='.txt')\n os.close(fd)\n try:\n with manifest.open('wb') as fp:\n fp.write(b'REPROZIP VERSION 2\\n')\n tar.add(str(manifest), 'METADATA/version')\n finally:\n manifest.remove()\n\n # Stores the original trace\n trace = directory / 'trace.sqlite3'\n if not trace.is_file():\n logger.critical(\"trace.sqlite3 is gone! Aborting\")\n sys.exit(1)\n tar.add(str(trace), 'METADATA/trace.sqlite3')\n\n # Checks that input files are packed\n for name, f in iteritems(inputs_outputs):\n if f.read_runs and not Path(f.path).exists():\n logger.warning(\"File is designated as input (name %s) but is not \"\n \"to be packed: %s\", name, f.path)\n\n # Generates a unique identifier for the pack (for usage reports purposes)\n pack_id = str(uuid.uuid4())\n\n # Stores canonical config\n fd, can_configfile = Path.tempfile(suffix='.yml', prefix='rpz_config_')\n os.close(fd)\n try:\n save_config(can_configfile, runs, packages, other_files,\n reprozip_version,\n inputs_outputs, canonical=True,\n pack_id=pack_id)\n\n tar.add(str(can_configfile), 'METADATA/config.yml')\n finally:\n can_configfile.remove()\n\n tar.close()\n\n # Record some info to the usage report\n record_usage_package(runs, packages, other_files,\n inputs_outputs,\n pack_id)\n" }, { "path": "reprozip/reprozip/tracer/__init__.py", "content": "" }, { "path": "reprozip/reprozip/tracer/linux_pkgs.py", "content": "# Copyright (C) 2014 New York University\n# This file is part of ReproZip which is released under the Revised BSD License\n# See file LICENSE for full license details.\n\n\"\"\"Package identification routines.\n\nThis module contains the :func:`~reprozip.tracer.linux_pkgs.identify_packages`\nfunction that sorts a list of files between their distribution packages,\ndepending on what Linux distribution we are running on.\n\nCurrently supported package managers:\n- dpkg (Debian, Ubuntu)\n- rpm (CentOS, Fedora)\n\"\"\"\n\nfrom __future__ import division, print_function, unicode_literals\n\nimport distro\nimport itertools\nimport logging\nfrom rpaths import Path\nimport subprocess\nimport time\n\nfrom reprozip.common import Package\nfrom reprozip.utils import iteritems, listvalues\n\n\nlogger = logging.getLogger('reprozip')\n\n\nmagic_dirs = ('/dev', '/proc', '/sys')\nsystem_dirs = ('/bin', '/etc', '/lib', '/sbin', '/usr', '/var', '/run')\n\n\nclass PkgManager(object):\n \"\"\"Base class for package identifiers.\n\n Subclasses should provide either `search_for_files` or `search_for_file`\n which actually identifies the package for a file.\n \"\"\"\n def __init__(self):\n # Files that were not part of a package\n self.unknown_files = set()\n # All the packages identified, with their `files` attribute set\n self.packages = {}\n\n def filter_files(self, files):\n seen_files = set()\n for f in files:\n if f.path not in seen_files:\n if not self._filter(f):\n yield f\n seen_files.add(f.path)\n\n def search_for_files(self, files):\n nb_pkg_files = 0\n\n for f in self.filter_files(files):\n pkgnames = self._get_packages_for_file(f.path)\n\n # Stores the file\n if not pkgnames:\n self.unknown_files.add(f)\n else:\n pkgs = []\n for pkgname in pkgnames:\n if pkgname in self.packages:\n pkgs.append(self.packages[pkgname])\n else:\n pkg = self._create_package(pkgname)\n if pkg is not None:\n self.packages[pkgname] = pkg\n pkgs.append(self.packages[pkgname])\n if len(pkgs) == 1:\n pkgs[0].add_file(f)\n nb_pkg_files += 1\n else:\n self.unknown_files.add(f)\n\n # Filter out packages with no files\n self.packages = {pkgname: pkg\n for pkgname, pkg in iteritems(self.packages)\n if pkg.files}\n\n logger.info(\"%d packages with %d files, and %d other files\",\n len(self.packages),\n nb_pkg_files,\n len(self.unknown_files))\n\n def _filter(self, f):\n # Special files\n if any(f.path.lies_under(c) for c in magic_dirs):\n return True\n\n # If it's not in a system directory, no need to look for it\n if (f.path.lies_under('/usr/local') or\n not any(f.path.lies_under(c) for c in system_dirs)):\n self.unknown_files.add(f)\n return True\n\n return False\n\n def _get_packages_for_file(self, filename):\n raise NotImplementedError\n\n def _create_package(self, pkgname):\n raise NotImplementedError\n\n\n# Before Linux 2.6.23, maximum argv is 128kB\nMAX_ARGV = 800\n\n\nclass DpkgManager(PkgManager):\n \"\"\"Package identifier for deb-based systems (Debian, Ubuntu).\n \"\"\"\n def search_for_files(self, files):\n # Make a set of all the requested files\n requested = dict((f.path, f) for f in self.filter_files(files))\n found = {} # {path: pkgname}\n\n # Request a few files at a time so we don't hit the command-line size\n # limit\n iter_batch = iter(requested)\n while True:\n batch = list(itertools.islice(iter_batch, MAX_ARGV))\n if not batch:\n break\n\n proc = subprocess.Popen(['dpkg-query', '-S'] +\n [path.path for path in batch],\n stdout=subprocess.PIPE,\n stderr=subprocess.PIPE)\n out, err = proc.communicate()\n for line in out.splitlines():\n pkgname, path = line.split(b': ', 1)\n path = Path(path.strip())\n # 8-bit safe encoding, because this might be a localized error\n # message (that we don't care about)\n pkgname = pkgname.decode('iso-8859-1')\n if ', ' in pkgname: # Multiple packages\n found[path] = None\n continue\n pkgname = pkgname.split(':', 1)[0] # Remove :arch\n if path in requested:\n if ' ' not in pkgname:\n # If we had assigned it to a package already, undo\n if path in found:\n found[path] = None\n # Else assign to the package\n else:\n found[path] = pkgname\n\n # Remaining files are not from packages\n self.unknown_files.update(\n f for f in files\n if f.path in requested and found.get(f.path) is None)\n\n nb_pkg_files = 0\n\n for path, pkgname in iteritems(found):\n if pkgname is None:\n continue\n if pkgname in self.packages:\n package = self.packages[pkgname]\n else:\n package = self._create_package(pkgname)\n self.packages[pkgname] = package\n package.add_file(requested.pop(path))\n nb_pkg_files += 1\n\n logger.info(\"%d packages with %d files, and %d other files\",\n len(self.packages),\n nb_pkg_files,\n len(self.unknown_files))\n\n def _get_packages_for_file(self, filename):\n # This method is not used for dpkg: instead, we query multiple files at\n # once since it is faster\n assert False\n\n def _create_package(self, pkgname):\n p = subprocess.Popen(['dpkg-query',\n '--showformat=${Package}\\t'\n '${Version}\\t'\n '${Installed-Size}\\n',\n '-W',\n pkgname],\n stdout=subprocess.PIPE)\n try:\n size = version = None\n for line in p.stdout:\n fields = line.split()\n # Removes :arch\n name = fields[0].decode('ascii').split(':', 1)[0]\n if name == pkgname:\n version = fields[1].decode('ascii')\n size = int(fields[2].decode('ascii')) * 1024 # kbytes\n break\n for line in p.stdout: # finish draining stdout\n pass\n finally:\n p.wait()\n if p.returncode == 0:\n pkg = Package(pkgname, version, size=size)\n logger.debug(\"Found package %s\", pkg)\n return pkg\n else:\n return None\n\n\nclass RpmManager(PkgManager):\n \"\"\"Package identifier for rpm-based systems (Fedora, CentOS).\n \"\"\"\n def _get_packages_for_file(self, filename):\n p = subprocess.Popen(['rpm', '-qf', filename.path,\n '--qf', '%{NAME}'],\n stdout=subprocess.PIPE,\n stderr=subprocess.PIPE)\n out, err = p.communicate()\n if p.returncode != 0:\n return None\n return [line.strip().decode('iso-8859-1')\n for line in out.splitlines()\n if line]\n\n def _create_package(self, pkgname):\n p = subprocess.Popen(['rpm', '-q', pkgname,\n '--qf', '%{VERSION}-%{RELEASE} %{SIZE}'],\n stdout=subprocess.PIPE,\n stderr=subprocess.PIPE)\n out, err = p.communicate()\n if p.returncode == 0:\n version, size = out.strip().decode('iso-8859-1').rsplit(' ', 1)\n size = int(size)\n pkg = Package(pkgname, version, size=size)\n logger.debug(\"Found package %s\", pkg)\n return pkg\n else:\n return None\n\n\ndef identify_packages(files):\n \"\"\"Organizes the files, using the distribution's package manager.\n \"\"\"\n distribution = distro.id()\n if distribution in ('debian', 'ubuntu'):\n logger.info(\"Identifying Debian packages for %d files...\", len(files))\n manager = DpkgManager()\n elif (distribution in ('centos', 'centos linux',\n 'fedora', 'scientific linux') or\n distribution.startswith('red hat')):\n logger.info(\"Identifying RPM packages for %d files...\", len(files))\n manager = RpmManager()\n else:\n logger.info(\"Unknown distribution, can't identify packages\")\n return files, []\n\n begin = time.time()\n manager.search_for_files(files)\n logger.debug(\"Assigning files to packages took %f seconds\",\n (time.time() - begin))\n\n return manager.unknown_files, listvalues(manager.packages)\n" }, { "path": "reprozip/reprozip/tracer/trace.py", "content": "# Copyright (C) 2014 New York University\n# This file is part of ReproZip which is released under the Revised BSD License\n# See file LICENSE for full license details.\n\n\"\"\"Tracing logic for reprozip.\n\nThis module contains the :func:`~reprozip.tracer.tracer.tracer` function that\ninvokes the C tracer (_pytracer) to build the SQLite trace file, and the\ngeneration logic for the config YAML file.\n\"\"\"\n\nfrom __future__ import division, print_function, unicode_literals\n\nimport contextlib\nimport distro\nfrom collections import defaultdict\nfrom importlib_metadata import entry_points\nfrom itertools import count\nimport logging\nimport os\nimport platform\nfrom rpaths import Path\nimport sqlite3\nimport sys\nimport warnings\n\nfrom reprozip import __version__ as reprozip_version\nfrom reprozip import _pytracer\nfrom reprozip.common import File, InputOutputFile, load_config, save_config, \\\n FILE_READ, FILE_WRITE, FILE_LINK, FILE_SOCKET\nfrom reprozip.tracer.linux_pkgs import magic_dirs, system_dirs, \\\n identify_packages\nfrom reprozip.utils import PY3, izip, iteritems, itervalues, \\\n unicode_, flatten, UniqueNames, hsize, normalize_path, find_all_links\n\n\nlogger = logging.getLogger('reprozip')\n\n\nsystemd_sockets = ('/run/systemd/private', '/run/dbus/system_bus_socket')\n\n\n@contextlib.contextmanager\ndef stderr_in_red():\n if os.isatty(sys.stderr.fileno()):\n try:\n print('\\x1b[31;20m', file=sys.stderr, end='', flush=True)\n yield\n finally:\n print('\\x1b[0m', file=sys.stderr, end='', flush=True)\n else:\n yield\n\n\nclass TracedFile(File):\n \"\"\"Override of `~reprozip.common.File` that reads stats from filesystem.\n\n It also memorizes how files are used, to select files that are only read,\n and accurately guess input and output files.\n \"\"\"\n # read\n # +------+\n # | |\n # read v + write\n # (init) +------------------> ONLY_READ +-------> READ_THEN_WRITTEN\n # | ^ +\n # | | |\n # +-------> WRITTEN +--+ +---------+\n # write ^ | read, write\n # | |\n # +---------+\n # read, write\n READ_THEN_WRITTEN = 0\n ONLY_READ = 1\n WRITTEN = 2\n\n what = None\n\n def __init__(self, path):\n path = Path(path)\n size = None\n if path.exists():\n if path.is_link():\n self.comment = \"Link to %s\" % path.read_link(absolute=True)\n elif path.is_dir():\n self.comment = \"Directory\"\n else:\n size = path.size()\n self.comment = hsize(size)\n self.what = None\n self.runs = defaultdict(lambda: None)\n File.__init__(self, path, size)\n\n def read(self, run):\n if self.what is None:\n self.what = TracedFile.ONLY_READ\n\n if run is not None:\n if self.runs[run] is None:\n self.runs[run] = TracedFile.ONLY_READ\n\n def write(self, run):\n if self.what is None:\n self.what = TracedFile.WRITTEN\n elif self.what == TracedFile.ONLY_READ:\n self.what = TracedFile.READ_THEN_WRITTEN\n\n if run is not None:\n if self.runs[run] is None:\n self.runs[run] = TracedFile.WRITTEN\n elif self.runs[run] == TracedFile.ONLY_READ:\n self.runs[run] = TracedFile.READ_THEN_WRITTEN\n\n\ndef run_filter_plugins(files, input_files):\n for entry_point in entry_points().select(group='reprozip.filters'):\n func = entry_point.load()\n name = entry_point.name\n\n logger.info(\"Running filter plugin %s\", name)\n func(files=files, input_files=input_files)\n\n\ndef get_files(conn):\n \"\"\"Find all the files used by the experiment by reading the trace.\n \"\"\"\n files = {}\n access_files = [set()]\n\n # Finds run timestamps, so we can sort input/output files by run\n proc_cursor = conn.cursor()\n executions = proc_cursor.execute(\n '''\n SELECT timestamp\n FROM processes\n WHERE parent ISNULL\n ORDER BY id;\n ''')\n run_timestamps = [r_timestamp for r_timestamp, in executions][1:]\n proc_cursor.close()\n\n # Adds dynamic linkers\n for libdir in (Path('/lib'), Path('/lib64')):\n if libdir.exists():\n for linker in libdir.listdir('*ld-linux*'):\n for filename in find_all_links(linker, True):\n if filename not in files:\n f = TracedFile(filename)\n f.read(None)\n files[f.path] = f\n\n # Loops on executed files, and opened files, at the same time\n cur = conn.cursor()\n rows = cur.execute(\n '''\n SELECT 'exec' AS event_type, name, NULL AS mode, timestamp\n FROM executed_files\n UNION ALL\n SELECT 'open' AS event_type, name, mode, timestamp\n FROM opened_files\n ORDER BY timestamp;\n ''')\n executed = set()\n systemd_accessed = False\n run = 0\n for event_type, r_name, r_mode, r_timestamp in rows:\n if event_type == 'exec':\n r_mode = FILE_READ\n r_name = Path(normalize_path(r_name))\n\n # Stays on the current run\n while run_timestamps and r_timestamp > run_timestamps[0]:\n del run_timestamps[0]\n access_files.append(set())\n run += 1\n\n # Adds symbolic links as read files\n for filename in find_all_links(r_name.parent if r_mode & FILE_LINK\n else r_name, False):\n try:\n f = files[filename]\n except KeyError:\n f = TracedFile(filename)\n files[f.path] = f\n f.read(run)\n # Go to final target\n if not r_mode & FILE_LINK:\n r_name = r_name.resolve()\n if event_type == 'exec':\n executed.add(r_name)\n if r_name not in files:\n f = TracedFile(r_name)\n files[f.path] = f\n else:\n f = files[r_name]\n if r_mode & FILE_READ:\n f.read(run)\n if r_mode & FILE_SOCKET:\n if r_name in systemd_sockets:\n systemd_accessed = True\n if r_mode & FILE_WRITE:\n f.write(run)\n # Mark the parent directory as read\n if r_name.parent not in files:\n fp = TracedFile(r_name.parent)\n fp.read(run)\n files[fp.path] = fp\n\n # Identifies input files\n if (\n not r_mode & FILE_LINK\n and r_name.is_file()\n and r_name not in executed\n ):\n access_files[-1].add(f)\n cur.close()\n\n # Further filters input files\n inputs = [[fi.path\n for fi in lst\n # Input files are regular files,\n if fi.path.is_file() and\n # ONLY_READ,\n fi.runs[r] == TracedFile.ONLY_READ and\n # not executable,\n # FIXME : currently disabled; only remove executed files\n # not fi.path.stat().st_mode & 0b111 and\n fi.path not in executed and\n # not in a system directory\n not any(fi.path.lies_under(m)\n for m in magic_dirs + system_dirs)]\n for r, lst in enumerate(access_files)]\n\n # Identify output files\n outputs = [[fi.path\n for fi in lst\n # Output files are regular files,\n if fi.path.is_file() and\n # WRITTEN\n fi.runs[r] == TracedFile.WRITTEN and\n # not in a system directory\n not any(fi.path.lies_under(m)\n for m in magic_dirs + system_dirs)]\n for r, lst in enumerate(access_files)]\n\n # Run the list of files through the filter plugins\n run_filter_plugins(files, inputs)\n\n # Files removed from plugins should be removed from inputs as well\n inputs = [[path for path in lst if path in files]\n for lst in inputs]\n\n # Display a warning for READ_THEN_WRITTEN files\n read_then_written_files = [\n fi\n for fi in itervalues(files)\n if fi.what == TracedFile.READ_THEN_WRITTEN and\n not any(fi.path.lies_under(m) for m in magic_dirs)]\n if read_then_written_files:\n with stderr_in_red():\n logger.warning(\n \"Some files were read and then written. We will only pack the \"\n \"final version of the file; reproducible experiments \"\n \"shouldn't change their input files\")\n logger.info(\"Paths:\\n%s\",\n \", \".join(unicode_(fi.path)\n for fi in read_then_written_files))\n\n # Display a warning for systemd\n if systemd_accessed:\n with stderr_in_red():\n logger.warning(\n \"A connection to systemd was detected. If systemd was asked \"\n \"to start a process, it won't be captured by reprozip, \"\n \"because it is an independent server. Please see \"\n \"https://docs.reprozip.org/s/systemd for more information\")\n\n files = set(\n fi\n for fi in itervalues(files)\n if fi.what != TracedFile.WRITTEN and not any(fi.path.lies_under(m)\n for m in magic_dirs))\n return files, inputs, outputs\n\n\ndef tty_prompt(prompt, chars):\n \"\"\"Get input from the terminal.\n\n On Linux, this will find the controlling terminal and ask there.\n\n :param prompt: String to be displayed on the terminal before reading the\n input.\n :param chars: Accepted character responses.\n \"\"\"\n try:\n import termios\n\n # Can't use O_RDWR/\"w+\" for a single fd/stream because of PY3 bug 20074\n ofd = os.open('/dev/tty', os.O_WRONLY | os.O_NOCTTY)\n ostream = os.fdopen(ofd, 'w', 1)\n ifd = os.open('/dev/tty', os.O_RDONLY | os.O_NOCTTY)\n istream = os.fdopen(ifd, 'r', 1)\n old = termios.tcgetattr(ifd)\n except (ImportError, AttributeError, IOError, OSError):\n ostream = sys.stdout\n istream = sys.stdin\n if not os.isatty(sys.stdin.fileno()):\n return None\n\n while True:\n ostream.write(prompt)\n ostream.flush()\n line = istream.readline()\n if not line:\n return None\n elif line[0] in chars:\n return line[0]\n else:\n new = old[:]\n new[3] &= ~termios.ICANON # 3 == 'lflags'\n tcsetattr_flags = termios.TCSAFLUSH | getattr(termios, 'TCSASOFT', 0)\n try:\n termios.tcsetattr(ifd, tcsetattr_flags, new)\n ostream.write(prompt)\n ostream.flush()\n while True:\n char = istream.read(1)\n if char in chars:\n ostream.write(\"\\n\")\n return char\n finally:\n termios.tcsetattr(ifd, tcsetattr_flags, old)\n ostream.flush()\n\n\ndef trace(binary, argv, directory, append, verbosity='unset'):\n \"\"\"Main function for the trace subcommand.\n \"\"\"\n if verbosity != 'unset':\n warnings.warn(\"The 'verbosity' parameter for trace() is deprecated. \"\n \"Please set a level on the 'reprozip' logger instead.\",\n DeprecationWarning)\n if not isinstance(directory, Path):\n directory = Path(directory)\n if isinstance(binary, Path):\n binary = binary.path\n\n cwd = Path.cwd()\n if (any(cwd.lies_under(c) for c in magic_dirs + system_dirs) and\n not cwd.lies_under('/usr/local')):\n logger.warning(\n \"You are running this experiment from a system directory! \"\n \"Autodetection of non-system files will probably not work as \"\n \"intended\")\n\n # Trace directory\n if directory.exists():\n if append is None:\n r = tty_prompt(\n \"Trace directory %s exists\\n\"\n \"(a)ppend run to the trace, (o)verwrite it, \"\n \"or (s)top? [a/o/s] \" %\n directory,\n 'aAdDoOsS')\n if r is None:\n logger.critical(\n \"Trace directory %s exists\\n\"\n \"Please use either --continue or --overwrite\\n\",\n directory)\n sys.exit(125)\n elif r in 'sS':\n sys.exit(125)\n elif r in 'oOdD': # keep accepting 'd' for delete\n directory.rmtree()\n directory.mkdir()\n logger.warning(\n \"You can use --overwrite to replace the existing trace \"\n \"(or --continue to append\\nwithout prompt)\")\n elif append is False:\n logger.info(\"Removing existing trace directory %s\", directory)\n directory.rmtree()\n directory.mkdir(parents=True)\n else:\n if append is True:\n logger.warning(\"--continue was set but trace doesn't exist yet\")\n directory.mkdir()\n\n # Runs the trace\n database = directory / 'trace.sqlite3'\n logger.info(\"Running program\")\n # Might raise _pytracer.Error\n c = _pytracer.execute(binary, argv, database.path)\n if c != 0:\n if c & 0x0100:\n logger.warning(\"Program appears to have been terminated by \"\n \"signal %d\", c & 0xFF)\n else:\n logger.warning(\"Program exited with non-zero code %d\", c)\n logger.info(\"Program completed\")\n\n return c\n\n\ndef write_configuration(directory, sort_packages, find_inputs_outputs,\n overwrite=False):\n \"\"\"Writes the canonical YAML configuration file.\n \"\"\"\n database = directory / 'trace.sqlite3'\n\n assert database.is_file()\n if PY3:\n # On PY3, connect() only accepts unicode\n conn = sqlite3.connect(str(database))\n else:\n conn = sqlite3.connect(database.path)\n conn.row_factory = sqlite3.Row\n\n # Reads info from database\n files, inputs, outputs = get_files(conn)\n\n # Identifies which file comes from which package\n if sort_packages:\n files, packages = identify_packages(files)\n else:\n packages = []\n\n # Writes configuration file\n config = directory / 'config.yml'\n distribution = [distro.id(), distro.version()]\n cur = conn.cursor()\n if overwrite or not config.exists():\n runs = []\n # This gets all the top-level processes (p.parent ISNULL) and the first\n # executed file for that process (sorting by ids, which are\n # chronological)\n executions = cur.execute(\n '''\n SELECT e.name, e.argv, e.envp, e.workingdir, p.exitcode\n FROM processes p\n JOIN executed_files e ON e.id=(\n SELECT id FROM executed_files e2\n WHERE e2.process=p.id\n ORDER BY e2.id\n LIMIT 1\n )\n WHERE p.parent ISNULL;\n ''')\n else:\n # Loads in previous config\n runs, oldpkgs, oldfiles = load_config(config,\n canonical=False,\n File=TracedFile)\n\n # Same query as previous block but only gets last process\n executions = cur.execute(\n '''\n SELECT e.name, e.argv, e.envp, e.workingdir, p.exitcode\n FROM processes p\n JOIN executed_files e ON e.id=(\n SELECT id FROM executed_files e2\n WHERE e2.process=p.id\n ORDER BY e2.id\n LIMIT 1\n )\n WHERE p.parent ISNULL\n ORDER BY p.id\n LIMIT 2147483647 OFFSET ?;\n ''',\n (len(runs),))\n for r_name, r_argv, r_envp, r_workingdir, r_exitcode in executions:\n # Decodes command-line\n argv = r_argv.split('\\0')\n if not argv[-1]:\n argv = argv[:-1]\n\n # Decodes environment\n envp = r_envp.split('\\0')\n if not envp[-1]:\n envp = envp[:-1]\n environ = dict(v.split('=', 1) for v in envp)\n\n runs.append({'id': \"run%d\" % len(runs),\n 'binary': r_name, 'argv': argv,\n 'workingdir': unicode_(Path(r_workingdir)),\n 'architecture': platform.machine().lower(),\n 'distribution': distribution,\n 'hostname': platform.node(),\n 'system': [platform.system(), platform.release()],\n 'environ': environ,\n 'uid': os.getuid(),\n 'gid': os.getgid(),\n 'signal' if r_exitcode & 0x0100 else 'exitcode':\n r_exitcode & 0xFF})\n\n cur.close()\n conn.close()\n\n if find_inputs_outputs:\n inputs_outputs = compile_inputs_outputs(runs, inputs, outputs)\n else:\n inputs_outputs = {}\n\n save_config(config, runs, packages, files, reprozip_version,\n inputs_outputs)\n\n print(\"Configuration file written in {0!s}\".format(config))\n print(\"Edit that file then run the packer -- \"\n \"use 'reprozip pack -h' for help\")\n\n\ndef compile_inputs_outputs(runs, inputs, outputs):\n \"\"\"Gives names to input/output files and creates InputOutputFile objects.\n \"\"\"\n # {path: (run_nb, arg_nb) or None}\n runs_with_file = {}\n # run_nb: number_of_file_arguments\n nb_file_args = []\n # {path: [runs]}\n readers = {}\n writers = {}\n\n for run_nb, run, in_files, out_files in izip(count(), runs,\n inputs, outputs):\n # List which runs read or write each file\n for p in in_files:\n readers.setdefault(p, []).append(run_nb)\n for p in out_files:\n writers.setdefault(p, []).append(run_nb)\n\n # Locate files that appear on a run's command line\n files_set = set(in_files) | set(out_files)\n nb_files = 0\n for arg_nb, arg in enumerate(run['argv']):\n p = Path(run['workingdir'], arg).resolve()\n if p in files_set:\n nb_files += 1\n if p not in runs_with_file:\n runs_with_file[p] = run_nb, arg_nb\n elif runs_with_file[p] is not None:\n runs_with_file[p] = None\n nb_file_args.append(nb_files)\n\n file_names = {}\n make_unique = UniqueNames()\n\n for fi in flatten(2, (inputs, outputs)):\n if fi in file_names:\n continue\n\n # If it appears in at least one of the command-lines\n if fi in runs_with_file:\n # If it only appears once in the command-lines\n if runs_with_file[fi] is not None:\n run_nb, arg_nb = runs_with_file[fi]\n parts = []\n # Run number, if there are more than one runs\n if len(runs) > 1:\n parts.append(run_nb)\n # Argument number, if there are more than one file arguments\n if nb_file_args[run_nb] > 1:\n parts.append(arg_nb)\n file_names[fi] = make_unique(\n 'arg%s' % '_'.join('%s' % s for s in parts))\n else:\n file_names[fi] = make_unique('arg_%s' % fi.unicodename)\n else:\n file_names[fi] = make_unique(fi.unicodename)\n\n return dict((n, InputOutputFile(p, readers.get(p, []), writers.get(p, [])))\n for p, n in iteritems(file_names))\n" }, { "path": "reprozip/reprozip/traceutils.py", "content": "# Copyright (C) 2014 New York University\n# This file is part of ReproZip which is released under the Revised BSD License\n# See file LICENSE for full license details.\n\n\"\"\"Additional manipulations for traces.\n\nThese are operations on traces that are not directly related to the tracing\nprocess itself.\n\"\"\"\n\nfrom __future__ import division, print_function, unicode_literals\n\nimport logging\nimport os\nfrom rpaths import Path\nimport sqlite3\n\nfrom reprozip.tracer.trace import TracedFile\nfrom reprozip.utils import PY3, listvalues\n\n\nlogger = logging.getLogger('reprozip')\n\n\ndef create_schema(conn):\n \"\"\"Create the trace database schema on a given SQLite3 connection.\n \"\"\"\n sql = [\n '''\n CREATE TABLE processes(\n id INTEGER NOT NULL PRIMARY KEY,\n run_id INTEGER NOT NULL,\n parent INTEGER,\n timestamp INTEGER NOT NULL,\n is_thread BOOLEAN NOT NULL,\n exitcode INTEGER\n );\n ''',\n '''\n CREATE INDEX proc_parent_idx ON processes(parent);\n ''',\n '''\n CREATE TABLE opened_files(\n id INTEGER NOT NULL PRIMARY KEY,\n run_id INTEGER NOT NULL,\n name TEXT NOT NULL,\n timestamp INTEGER NOT NULL,\n mode INTEGER NOT NULL,\n is_directory BOOLEAN NOT NULL,\n process INTEGER NOT NULL\n );\n ''',\n '''\n CREATE INDEX open_proc_idx ON opened_files(process);\n ''',\n '''\n CREATE TABLE executed_files(\n id INTEGER NOT NULL PRIMARY KEY,\n name TEXT NOT NULL,\n run_id INTEGER NOT NULL,\n timestamp INTEGER NOT NULL,\n process INTEGER NOT NULL,\n argv TEXT NOT NULL,\n envp TEXT NOT NULL,\n workingdir TEXT NOT NULL\n );\n ''',\n '''\n CREATE INDEX exec_proc_idx ON executed_files(process);\n ''',\n ]\n for stmt in sql:\n conn.execute(stmt)\n\n\ndef combine_files(newfiles, newpackages, oldfiles, oldpackages):\n \"\"\"Merges two sets of packages and files.\n \"\"\"\n files = set(oldfiles)\n files.update(newfiles)\n\n packages = dict((pkg.name, pkg) for pkg in newpackages)\n for oldpkg in oldpackages:\n if oldpkg.name in packages:\n pkg = packages[oldpkg.name]\n # Here we build TracedFiles from the Files so that the comment\n # (size, etc) gets set\n s = set(TracedFile(fi.path) for fi in oldpkg.files)\n s.update(pkg.files)\n oldpkg.files = list(s)\n packages[oldpkg.name] = oldpkg\n else:\n oldpkg.files = [TracedFile(fi.path) for fi in oldpkg.files]\n packages[oldpkg.name] = oldpkg\n packages = listvalues(packages)\n\n return files, packages\n\n\ndef combine_traces(traces, target):\n \"\"\"Combines multiple trace databases into one.\n\n The runs from the original traces are appended ('run_id' field gets\n translated to avoid conflicts).\n\n :param traces: List of trace database filenames.\n :type traces: [Path]\n :param target: Directory where to write the new database and associated\n configuration file.\n :type target: Path\n \"\"\"\n # We are probably overwriting one of the traces we're reading, so write to\n # a temporary file first then move it\n fd, output = Path.tempfile('.sqlite3', 'reprozip_combined_')\n if PY3:\n # On PY3, connect() only accepts unicode\n conn = sqlite3.connect(str(output))\n else:\n conn = sqlite3.connect(output.path)\n os.close(fd)\n conn.row_factory = sqlite3.Row\n\n # Create the schema\n create_schema(conn)\n\n # Temporary database with lookup tables\n conn.execute(\n '''\n ATTACH DATABASE '' AS maps;\n ''')\n conn.execute(\n '''\n CREATE TABLE maps.map_runs(\n old INTEGER NOT NULL,\n new INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT\n );\n ''')\n conn.execute(\n '''\n CREATE TABLE maps.map_processes(\n old INTEGER NOT NULL,\n new INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT\n );\n ''')\n\n # Do the merge\n for other in traces:\n logger.info(\"Attaching database %s\", other)\n\n # Attach the other trace\n conn.execute(\n '''\n ATTACH DATABASE ? AS trace;\n ''',\n (str(other),))\n\n # Add runs to lookup table\n conn.execute(\n '''\n INSERT INTO maps.map_runs(old)\n SELECT DISTINCT run_id AS old\n FROM trace.processes\n ORDER BY run_id;\n ''')\n\n logger.info(\n \"%d rows in maps.map_runs\",\n list(conn.execute('SELECT COUNT(*) FROM maps.map_runs;'))[0][0])\n\n # Add processes to lookup table\n conn.execute(\n '''\n INSERT INTO maps.map_processes(old)\n SELECT id AS old\n FROM trace.processes\n ORDER BY id;\n ''')\n\n logger.info(\n \"%d rows in maps.map_processes\",\n list(conn.execute('SELECT COUNT(*) FROM maps.map_processes;'))\n [0][0])\n\n # processes\n logger.info(\"Insert processes...\")\n conn.execute(\n '''\n INSERT INTO processes(id, run_id, parent,\n timestamp, is_thread, exitcode)\n SELECT p.new AS id, r.new AS run_id, parent,\n timestamp, is_thread, exitcode\n FROM trace.processes t\n INNER JOIN maps.map_runs r ON t.run_id = r.old\n INNER JOIN maps.map_processes p ON t.id = p.old\n ORDER BY t.id;\n ''')\n\n # opened_files\n logger.info(\"Insert opened_files...\")\n conn.execute(\n '''\n INSERT INTO opened_files(run_id, name, timestamp,\n mode, is_directory, process)\n SELECT r.new AS run_id, name, timestamp,\n mode, is_directory, p.new AS process\n FROM trace.opened_files t\n INNER JOIN maps.map_runs r ON t.run_id = r.old\n INNER JOIN maps.map_processes p ON t.process = p.old\n ORDER BY t.id;\n ''')\n\n # executed_files\n logger.info(\"Insert executed_files...\")\n conn.execute(\n '''\n INSERT INTO executed_files(name, run_id, timestamp, process,\n argv, envp, workingdir)\n SELECT name, r.new AS run_id, timestamp, p.new AS process,\n argv, envp, workingdir\n FROM trace.executed_files t\n INNER JOIN maps.map_runs r ON t.run_id = r.old\n INNER JOIN maps.map_processes p ON t.process = p.old\n ORDER BY t.id;\n ''')\n\n # Flush maps\n conn.execute(\n '''\n DELETE FROM maps.map_runs;\n ''')\n conn.execute(\n '''\n DELETE FROM maps.map_processes;\n ''')\n\n # An implicit transaction gets created. Python used to implicitly\n # commit it, but no longer does as of 3.6, so we have to explicitly\n # commit before detaching.\n conn.commit()\n\n # Detach\n conn.execute(\n '''\n DETACH DATABASE trace;\n ''')\n\n # See above.\n conn.commit()\n\n conn.execute(\n '''\n DETACH DATABASE maps;\n ''')\n\n conn.commit()\n conn.close()\n\n # Move database to final destination\n if not target.exists():\n target.mkdir()\n output.move(target / 'trace.sqlite3')\n" }, { "path": "reprozip/reprozip/utils.py", "content": "# Copyright (C) 2014 New York University\n# This file is part of ReproZip which is released under the Revised BSD License\n# See file LICENSE for full license details.\n\n# This file is shared:\n# reprozip/reprozip/utils.py\n# reprounzip/reprounzip/utils.py\n\n\"\"\"Utility functions.\n\nThese functions are shared between reprozip and reprounzip but are not specific\nto this software (more utilities).\n\n\"\"\"\n\nfrom __future__ import division, print_function, unicode_literals\n\nimport codecs\nimport contextlib\nfrom datetime import datetime\nimport email.utils\nimport itertools\nimport locale\nimport logging\nimport operator\nimport os\nimport requests\nfrom rpaths import Path, PosixPath\nimport stat\nimport subprocess\nimport sys\nimport time\n\n\nlogger = logging.getLogger(__name__.split('.', 1)[0])\n\n\nclass StreamWriter(object):\n def __init__(self, stream):\n writer = codecs.getwriter(locale.getpreferredencoding())\n self._writer = writer(stream, 'replace')\n self.buffer = stream\n\n def writelines(self, lines):\n self.write(str('').join(lines))\n\n def write(self, obj):\n if isinstance(obj, bytes):\n self.buffer.write(obj)\n else:\n self._writer.write(obj)\n\n def __getattr__(self, name,\n getattr=getattr):\n\n \"\"\" Inherit all other methods from the underlying stream.\n \"\"\"\n return getattr(self._writer, name)\n\n\nPY3 = sys.version_info[0] == 3\n\n\nif PY3:\n izip = zip\n irange = range\n iteritems = lambda d: d.items()\n itervalues = lambda d: d.values()\n listvalues = lambda d: list(d.values())\n\n stdout_bytes = sys.stdout.buffer if sys.stdout is not None else None\n stderr_bytes = sys.stderr.buffer if sys.stderr is not None else None\n stdin_bytes = sys.stdin.buffer if sys.stdin is not None else None\n stdout, stderr = sys.stdout, sys.stderr\nelse:\n izip = itertools.izip\n irange = xrange # noqa: F821\n iteritems = lambda d: d.iteritems()\n itervalues = lambda d: d.itervalues()\n listvalues = lambda d: d.values()\n\n _writer = codecs.getwriter(locale.getpreferredencoding())\n stdout_bytes, stderr_bytes = sys.stdout, sys.stderr\n stdin_bytes = sys.stdin\n stdout, stderr = StreamWriter(sys.stdout), StreamWriter(sys.stderr)\n\n\nif PY3:\n int_types = int,\n unicode_ = str\nelse:\n int_types = int, long # noqa: F821\n unicode_ = unicode # noqa: F821\n\n\ndef flatten(n, iterable):\n \"\"\"Flattens an iterable by repeatedly calling chain.from_iterable() on it.\n\n >>> a = [[1, 2, 3], [4, 5, 6]]\n >>> b = [[7, 8], [9, 10, 11, 12, 13, 14, 15, 16]]\n >>> l = [a, b]\n >>> list(flatten(0, a))\n [[1, 2, 3], [4, 5, 6]]\n >>> list(flatten(1, a))\n [1, 2, 3, 4, 5, 6]\n >>> list(flatten(1, l))\n [[1, 2, 3], [4, 5, 6], [7, 8], [9, 10, 11, 12, 13, 14, 15, 16]]\n >>> list(flatten(2, l))\n [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]\n \"\"\"\n for _ in irange(n):\n iterable = itertools.chain.from_iterable(iterable)\n return iterable\n\n\nclass UniqueNames(object):\n \"\"\"Makes names unique amongst the ones it's already seen.\n \"\"\"\n def __init__(self):\n self.names = set()\n\n def insert(self, name):\n assert name not in self.names\n self.names.add(name)\n\n def __call__(self, name):\n nb = 1\n attempt = name\n while attempt in self.names:\n nb += 1\n attempt = '%s_%d' % (name, nb)\n self.names.add(attempt)\n return attempt\n\n\ndef escape(s):\n \"\"\"Escapes backslashes and double quotes in strings.\n\n This does NOT add quotes around the string.\n \"\"\"\n return s.replace('\\\\', '\\\\\\\\').replace('\"', '\\\\\"')\n\n\ndef optional_return_type(req_args, other_args):\n \"\"\"Sort of namedtuple but with name-only fields.\n\n When deconstructing a namedtuple, you have to get all the fields:\n\n >>> o = namedtuple('T', ['a', 'b', 'c'])(1, 2, 3)\n >>> a, b = o\n ValueError: too many values to unpack\n\n You thus cannot easily add new return values. This class allows it:\n\n >>> o2 = optional_return_type(['a', 'b'], ['c'])(1, 2, 3)\n >>> a, b = o2\n >>> c = o2.c\n \"\"\"\n if len(set(req_args) | set(other_args)) != len(req_args) + len(other_args):\n raise ValueError\n\n # Maps argument name to position in each list\n req_args_pos = dict((n, i) for i, n in enumerate(req_args))\n other_args_pos = dict((n, i) for i, n in enumerate(other_args))\n\n def cstr(cls, *args, **kwargs):\n if len(args) > len(req_args) + len(other_args):\n raise TypeError(\n \"Too many arguments (expected at least %d and no more than \"\n \"%d)\" % (len(req_args),\n len(req_args) + len(other_args)))\n\n args1, args2 = args[:len(req_args)], args[len(req_args):]\n req = dict((i, v) for i, v in enumerate(args1))\n other = dict(izip(other_args, args2))\n\n for k, v in iteritems(kwargs):\n if k in req_args_pos:\n pos = req_args_pos[k]\n if pos in req:\n raise TypeError(\"Multiple values for field %s\" % k)\n req[pos] = v\n elif k in other_args_pos:\n if k in other:\n raise TypeError(\"Multiple values for field %s\" % k)\n other[k] = v\n else:\n raise TypeError(\"Unknown field name %s\" % k)\n\n args = []\n for i, k in enumerate(req_args):\n if i not in req:\n raise TypeError(\"Missing value for field %s\" % k)\n args.append(req[i])\n\n inst = tuple.__new__(cls, args)\n inst.__dict__.update(other)\n return inst\n\n dct = {'__new__': cstr}\n for i, n in enumerate(req_args):\n dct[n] = property(operator.itemgetter(i))\n return type(str('OptionalReturnType'), (tuple,), dct)\n\n\ndef tz_offset():\n offset = time.timezone if time.localtime().tm_isdst == 0 else time.altzone\n return -offset\n\n\ndef isodatetime():\n offset = tz_offset()\n sign = '+'\n if offset < 0:\n sign = '-'\n offset = -offset\n if offset % 60 == 0:\n offset = '%02d:%02d' % (offset // 3600, (offset // 60) % 60)\n else:\n offset = '%02d:%02d:%02d' % (offset // 3600, (offset // 60) % 60,\n offset % 60)\n # Remove microsecond\n now = datetime.now()\n now = datetime(year=now.year, month=now.month, day=now.day,\n hour=now.hour, minute=now.minute, second=now.second)\n return '%s%s%s' % (now.isoformat(),\n sign,\n offset)\n\n\ndef hsize(nbytes):\n \"\"\"Readable size.\n \"\"\"\n if nbytes is None:\n return \"unknown\"\n\n KB = 1 << 10\n MB = 1 << 20\n GB = 1 << 30\n TB = 1 << 40\n PB = 1 << 50\n\n nbytes = float(nbytes)\n\n if nbytes < KB:\n return \"{0} bytes\".format(nbytes)\n elif nbytes < MB:\n return \"{0:.2f} KB\".format(nbytes / KB)\n elif nbytes < GB:\n return \"{0:.2f} MB\".format(nbytes / MB)\n elif nbytes < TB:\n return \"{0:.2f} GB\".format(nbytes / GB)\n elif nbytes < PB:\n return \"{0:.2f} TB\".format(nbytes / TB)\n else:\n return \"{0:.2f} PB\".format(nbytes / PB)\n\n\ndef normalize_path(path):\n \"\"\"Normalize a path obtained from the database.\n \"\"\"\n # For some reason, os.path.normpath() keeps multiple leading slashes\n # We don't want this since it has no meaning on Linux\n path = PosixPath(path)\n if path.path.startswith(path._sep + path._sep):\n path = PosixPath(path.path[1:])\n return path\n\n\ndef find_all_links_recursive(filename, files):\n path = Path('/')\n for c in filename.components[1:]:\n # At this point, path is a canonical path, and all links in it have\n # been resolved\n\n # We add the next path component\n path = path / c\n\n # That component is possibly a link\n if path.is_link():\n # Adds the link itself\n files.add(path)\n\n target = path.read_link(absolute=True)\n # Here, target might contain a number of symlinks\n if target not in files:\n # Recurse on this new path\n find_all_links_recursive(target, files)\n # Restores the invariant; realpath might resolve several links here\n path = path.resolve()\n return path\n\n\ndef find_all_links(filename, include_target=False):\n \"\"\"Dereferences symlinks from a path.\n\n If include_target is True, this also returns the real path of the final\n target.\n\n Example:\n /\n a -> b\n b\n g -> c\n c -> ../a/d\n d\n e -> /f\n f\n >>> find_all_links('/a/g/e', True)\n ['/a', '/b/c', '/b/g', '/b/d/e', '/f']\n \"\"\"\n files = set()\n filename = Path(filename)\n assert filename.absolute()\n path = find_all_links_recursive(filename, files)\n files = list(files)\n if include_target:\n files.append(path)\n return files\n\n\ndef join_root(root, path):\n \"\"\"Prepends `root` to the absolute path `path`.\n \"\"\"\n p_root, p_loc = path.split_root()\n assert p_root == b'/'\n return root / p_loc\n\n\n@contextlib.contextmanager\ndef make_dir_writable(directory):\n \"\"\"Context-manager that sets write permission on a directory.\n\n This assumes that the directory belongs to you. If the u+w permission\n wasn't set, it gets set in the context, and restored to what it was when\n leaving the context. u+x also gets set on all the directories leading to\n that path.\n \"\"\"\n uid = os.getuid()\n\n try:\n sb = directory.stat()\n except OSError:\n pass\n else:\n if sb.st_uid != uid or sb.st_mode & 0o700 == 0o700:\n yield\n return\n\n # These are the permissions to be restored, in reverse order\n restore_perms = []\n try:\n # Add u+x to all directories up to the target\n path = Path('/')\n for c in directory.components[1:-1]:\n path = path / c\n sb = path.stat()\n if sb.st_uid == uid and not sb.st_mode & 0o100:\n logger.debug(\"Temporarily setting u+x on %s\", path)\n restore_perms.append((path, sb.st_mode))\n path.chmod(sb.st_mode | 0o700)\n\n # Add u+wx to the target\n sb = directory.stat()\n if sb.st_uid == uid and sb.st_mode & 0o700 != 0o700:\n logger.debug(\"Temporarily setting u+wx on %s\", directory)\n restore_perms.append((directory, sb.st_mode))\n directory.chmod(sb.st_mode | 0o700)\n\n yield\n finally:\n for path, mod in reversed(restore_perms):\n path.chmod(mod)\n\n\ndef rmtree_fixed(path):\n \"\"\"Like :func:`shutil.rmtree` but doesn't choke on annoying permissions.\n\n If a directory with -w or -x is encountered, it gets fixed and deletion\n continues.\n \"\"\"\n if path.is_link():\n raise OSError(\"Cannot call rmtree on a symbolic link\")\n\n uid = os.getuid()\n st = path.lstat()\n\n if st.st_uid == uid and st.st_mode & 0o700 != 0o700:\n path.chmod(st.st_mode | 0o700)\n\n for entry in path.listdir():\n if stat.S_ISDIR(entry.lstat().st_mode):\n rmtree_fixed(entry)\n else:\n entry.remove()\n\n path.rmdir()\n\n\n# Compatibility with ReproZip <= 1.0.3\ncheck_output = subprocess.check_output\n\n\ndef copyfile(source, destination, CHUNK_SIZE=4096):\n \"\"\"Copies from one file object to another.\n \"\"\"\n while True:\n chunk = source.read(CHUNK_SIZE)\n if chunk:\n destination.write(chunk)\n if len(chunk) != CHUNK_SIZE:\n break\n\n\ndef download_file(url, dest, cachename=None, ssl_verify=None):\n \"\"\"Downloads a file using a local cache.\n\n If the file cannot be downloaded or if it wasn't modified, the cached\n version will be used instead.\n\n The cache lives in ``~/.cache/reprozip/``.\n \"\"\"\n if cachename is None:\n if dest is None:\n raise ValueError(\"One of 'dest' or 'cachename' must be specified\")\n cachename = dest.components[-1]\n\n headers = {}\n\n if 'XDG_CACHE_HOME' in os.environ:\n cache = Path(os.environ['XDG_CACHE_HOME'])\n else:\n cache = Path('~/.cache').expand_user()\n cache = cache / 'reprozip' / cachename\n if cache.exists():\n mtime = email.utils.formatdate(cache.mtime(), usegmt=True)\n headers['If-Modified-Since'] = mtime\n\n cache.parent.mkdir(parents=True)\n\n try:\n response = requests.get(url, headers=headers,\n timeout=2 if cache.exists() else 10,\n stream=True, verify=ssl_verify)\n response.raise_for_status()\n if response.status_code == 304:\n raise requests.HTTPError(\n '304 File is up to date, no data returned',\n response=response)\n except requests.RequestException as e:\n if cache.exists():\n if e.response and e.response.status_code == 304:\n logger.info(\"Download %s: cache is up to date\", cachename)\n else:\n logger.warning(\"Download %s: error downloading %s: %s\",\n cachename, url, e)\n if dest is not None:\n cache.copy(dest)\n return dest\n else:\n return cache\n else:\n raise\n\n logger.info(\"Download %s: downloading %s\", cachename, url)\n try:\n with cache.open('wb') as f:\n for chunk in response.iter_content(4096):\n f.write(chunk)\n response.close()\n except Exception as e: # pragma: no cover\n try:\n cache.remove()\n except OSError:\n pass\n raise e\n logger.info(\"Downloaded %s successfully\", cachename)\n\n if dest is not None:\n cache.copy(dest)\n return dest\n else:\n return cache\n" }, { "path": "reprozip/setup.py", "content": "import io\nimport os\nfrom setuptools import setup, Extension\nimport sys\n\n\n# pip workaround\nos.chdir(os.path.abspath(os.path.dirname(__file__)))\n\n\n# This won't build on non-Linux -- don't even try\nif not sys.platform.startswith('linux'):\n sys.stderr.write(\"reprozip uses ptrace and thus only works on Linux\\n\"\n \"You can however install reprounzip and plugins on other \"\n \"platforms\\n\")\n sys.exit(1)\n\n\n# List the source files\nsources = ['pytracer.c', 'tracer.c', 'syscalls.c', 'database.c',\n 'ptrace_utils.c', 'utils.c', 'pylog.c']\n# They can be found under native/\nsources = [os.path.join('native', n) for n in sources]\n\n\n# Setup the libraries\nlibraries = ['sqlite3', 'rt']\n\n\n# Build the C module\npytracer = Extension('reprozip._pytracer',\n sources=sources,\n libraries=libraries)\n\n# Need to specify encoding for PY3, which has the worst unicode handling ever\nwith io.open('README.rst', encoding='utf-8') as fp:\n description = fp.read()\nreq = [\n 'importlib-metadata',\n 'PyYAML',\n 'rpaths>=0.8',\n 'usagestats>=0.3',\n 'requests',\n 'distro']\nsetup(name='reprozip',\n version='1.3.2',\n ext_modules=[pytracer],\n packages=['reprozip', 'reprozip.tracer'],\n entry_points={\n 'console_scripts': [\n 'reprozip = reprozip.main:main'],\n 'reprozip.filters': [\n 'python = reprozip.filters:python',\n 'builtin = reprozip.filters:builtin',\n 'ruby = reprozip.filters:ruby']},\n install_requires=req,\n description=\"Linux tool enabling reproducible experiments (packer)\",\n author=\"Remi Rampin, Fernando Chirigati, Dennis Shasha, Juliana Freire\",\n author_email='reprozip@nyu.edu',\n maintainer=\"Remi Rampin\",\n maintainer_email='remi@rampin.org',\n url='https://www.reprozip.org/',\n project_urls={\n 'Documentation': 'https://docs.reprozip.org/',\n 'Examples': 'https://examples.reprozip.org/',\n 'Source': 'https://github.com/VIDA-NYU/reprozip',\n 'Bug Tracker': 'https://github.com/VIDA-NYU/reprozip/issues',\n 'Chat': 'https://riot.im/app/#/room/#reprozip:matrix.org',\n 'Changelog':\n 'https://github.com/VIDA-NYU/reprozip/blob/1.x/CHANGELOG.md',\n },\n long_description=description,\n license='BSD-3-Clause',\n keywords=['reprozip', 'reprounzip', 'reproducibility', 'provenance',\n 'vida', 'nyu'],\n classifiers=[\n 'Development Status :: 5 - Production/Stable',\n 'Intended Audience :: Science/Research',\n 'License :: OSI Approved :: BSD License',\n 'Programming Language :: Python :: 2.7',\n 'Programming Language :: Python :: 3',\n 'Operating System :: POSIX :: Linux',\n 'Programming Language :: C',\n 'Topic :: Scientific/Engineering',\n 'Topic :: System :: Archiving'])\n" }, { "path": "reprozip-jupyter/LICENSE.txt", "content": "Copyright (C) 2014, New York University\nAll rights reserved.\n\nRedistribution and use in source and binary forms, with or without\nmodification, are permitted provided that the following conditions are met:\n\n1. Redistributions of source code must retain the above copyright notice, this\n list of conditions and the following disclaimer.\n\n2. Redistributions in binary form must reproduce the above copyright notice,\n this list of conditions and the following disclaimer in the documentation\n and/or other materials provided with the distribution.\n\n3. Neither the name of the copyright holder nor the names of its contributors\n may be used to endorse or promote products derived from this software\n without specific prior written permission.\n\nTHIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \"AS IS\" AND\nANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED\nWARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE\nDISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE\nFOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL\nDAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR\nSERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER\nCAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,\nOR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE\nOF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n" }, { "path": "reprozip-jupyter/MANIFEST.in", "content": "include README.rst\ninclude LICENSE.txt\n" }, { "path": "reprozip-jupyter/README.rst", "content": "ReproZip project\n================\n\n`ReproZip `__ is a tool aimed at simplifying the process of creating reproducible experiments from command-line executions, a frequently-used common denominator in computational science. It tracks operating system calls and creates a bundle that contains all the binaries, files and dependencies required to run a given command on the author's computational environment (packing step). A reviewer can then extract the experiment in his environment to reproduce the results (unpacking step).\n\nreprozip-jupyter\n----------------\n\nThis package provides tracing and reproduction of Jupyter notebooks, allowing one to pack all the libraries and data used in their notebook to allow anyone to re-run it easily.\n\nYou can use it from the command-line::\n\n # Trace & pack\n $ reprozip-jupyter trace mynotebook.ipynb\n $ reprozip pack notebook_environment.rpz\n\n # Unpack and reproduce\n $ reprounzip docker setup notebook_environment.rpz /tmp/notebook\n $ reprozip-jupyter run /tmp/notebook\n\nOr you can pack directly from the Jupyter notebook interface, if you enable the extension::\n\n $ jupyter nbextension install --py reprozip_jupyter --user\n $ jupyter nbextension enable --py reprozip_jupyter --user\n $ jupyter serverextension enable --py reprozip_jupyter --user\n\nPlease refer to `reprozip `__ and `reprounzip `_ for more information.\n\nAdditional Information\n----------------------\n\nFor more detailed information, please refer to our `website `_, as well as to our `documentation `_.\n\nReproZip is currently being developed at `NYU `_. The team includes:\n\n* `Fernando Chirigati `_\n* `Juliana Freire `_\n* `Remi Rampin `_\n* `Dennis Shasha `_\n* `Vicky Rampin `_\n" }, { "path": "reprozip-jupyter/reprozip_jupyter/__init__.py", "content": "\"\"\"Traces and packs notebook environments with ReproZip.\n\"\"\"\n\n__version__ = '1.2'\n\n\ndef _jupyter_nbextension_paths():\n return [\n dict(\n section='notebook',\n src='notebook-extension.js',\n dest='reprozip-jupyter.js',\n require='reprozip-jupyter',\n ),\n ]\n\n\ndef _jupyter_server_extension_paths():\n return [\n dict(module='reprozip_jupyter.server_extension'),\n ]\n" }, { "path": "reprozip-jupyter/reprozip_jupyter/main.py", "content": "# Copyright (C) 2014 New York University\n# This file is part of ReproZip which is released under the Revised BSD License\n# See file LICENSE for full license details.\n\nimport argparse\nimport sys\n\nfrom . import __version__\nfrom . import run\nfrom . import trace\n\n\ndef main():\n def add_options(opts):\n opts.add_argument('-v', '--verbose', action='count', default=0,\n dest='verbosity', help=\"augments verbosity level\")\n opts.add_argument('--version', action='version',\n version=\"reprounzip-jupyter version %s\" %\n __version__)\n\n parser = argparse.ArgumentParser(\n description=\"Jupyter Notebook tracing/reproduction for ReproZip\",\n epilog=\"Please report issues to reprozip@nyu.edu\")\n add_options(parser)\n subparser = parser.add_subparsers(title=\"subcommands\", metavar='',\n dest='cmd')\n\n parser_trace = subparser.add_parser(\n 'trace',\n help=\"Runs a Jupyter notebook under ReproZip trace to generate the \"\n \"accompanying environment package\")\n add_options(parser_trace)\n trace.setup(parser_trace)\n\n parser_run = subparser.add_parser(\n 'run',\n help=\"Runs a Jupyter notebook server that will spawn notebooks in \"\n \"Docker containers running in the given unpacked environment\")\n add_options(parser_run)\n run.setup(parser_run)\n\n args = parser.parse_args()\n if getattr(args, 'func', None) is None:\n parser.print_help(sys.stderr)\n sys.exit(2)\n args.func(args)\n" }, { "path": "reprozip-jupyter/reprozip_jupyter/notebook-extension.js", "content": "define([\"jquery\",\n \"base/js/namespace\",\n \"base/js/utils\",\n \"base/js/dialog\"],\nfunction notebook_extension($, IPython, utils, dialog) {\n var ajax = utils.ajax || $.ajax;\n\n // Called when the user clicks the \"trace\" action\n function fn_trace(env) {\n // TODO: Cancel trace?\n var cancel = function cancel() {};\n\n var modal = dialog.modal({\n body: \"Executing notebook under trace...\",\n title: \"Packing notebook with ReproZip\",\n buttons: {\n \"Cancel\": {\n class: \"btn-warning\",\n click: cancel\n }\n },\n notebook: env.notebook,\n keyboard_manager: env.notebook.keyboard_manager\n });\n\n // Save the notebook first, get called back when it is done\n // (there is a round-trip to the server)\n var saved = function saved() {\n env.notebook.events.off(\"notebook_saved.Notebook\", saved);\n console.log(\"reprozip: notebook saved, triggering trace\");\n\n ajax(utils.url_path_join(IPython.notebook.base_url,\n \"reprozip/trace\"), {\n method: \"POST\",\n data: {file: IPython.notebook.notebook_path}\n }).then(function packed(response) {\n if(response.error) {\n console.log(\"reprozip: got error response\");\n modal.modal(\"hide\");\n var m = dialog.modal({\n body: response.error,\n title: \"Packing notebook with ReproZip\",\n buttons: {\n \"Close\": {\n class: \"btn-danger\"\n }\n }\n });\n } else {\n console.log(\"reprozip: packing successful\");\n modal.modal(\"hide\");\n dialog.modal({\n body: \"Created package \" + response.bundle,\n title: \"Packing notebook with ReproZip\",\n buttons: {\n \"Close\": {\n class: \"btn-primary\"\n }\n }\n });\n }\n }, function failed() {\n console.error(\"reprozip: packing failed\");\n modal.modal(\"hide\");\n var m = dialog.modal({\n body: \"Packing failed!\",\n title: \"Packing notebook with ReproZip\",\n show: false,\n buttons: {\n \"Close\": {\n class: \"btn-danger\"\n }\n }\n });\n m.find(\".modal-body\").html(\"
Packing failed! Check the server console for details.
\");\n m.modal(\"show\");\n });\n };\n console.log(\"reprozip: saving notebook\")\n env.notebook.events.on(\"notebook_saved.Notebook\", saved);\n env.notebook.save_checkpoint();\n }\n\n function _on_load() {\n console.info(\"reprozip: extension reprozip-jupyter loaded\");\n\n var trace_action = {\n help: \"Trace this notebook using ReproZip\",\n icon: \"fa-file-archive-o\",\n help_index: \"\",\n handler: fn_trace\n };\n var trace_action_name = IPython.keyboard_manager.actions.register(\n trace_action,\n \"trace-with-reprozip\",\n \"reprozip\");\n console.log(\"reprozip: created action\", trace_action_name);\n\n // Add icon to toolbar\n IPython.toolbar.add_buttons_group([trace_action_name]);\n\n // Add icon to \"kernel\" menu\n var kernel_menu = $(\"#kernel_menu\");\n if(!kernel_menu.length) {\n console.log(\"reprozip: no kernel menu?\");\n } else {\n var after_entry = $(\"#restart_run_all\");\n if(!after_entry.length) {\n console.log(\"reprozip: Couldn't find restart_run_all, using end of menu\");\n var children = kernel_menu.children();\n after_entry = children[children.length - 1];\n }\n var entry = $(\"Trace with ReproZip\");\n entry.on(\"click\", fn_trace.bind(null, IPython));\n after_entry.after($(\"
  • \").attr(\"title\", \"Trace this notebook using ReproZip\").append(entry));\n }\n }\n\n return {load_ipython_extension: _on_load};\n});\n" }, { "path": "reprozip-jupyter/reprozip_jupyter/run.py", "content": "# Copyright (C) 2014 New York University\n# This file is part of ReproZip which is released under the Revised BSD License\n# See file LICENSE for full license details.\n\n\"\"\"Run notebooks in a packed environment.\n\"\"\"\n\nfrom __future__ import division, print_function, unicode_literals\n\nimport argparse\nimport contextlib\nimport json\nfrom jupyter_client.ioloop import IOLoopKernelManager\nimport logging\nfrom notebook.notebookapp import NotebookApp\nfrom notebook.services.kernels.kernelmanager import MappingKernelManager\nimport os\nfrom rpaths import Path\nimport subprocess\nimport sys\n\nfrom reprounzip.common import setup_logging\n\n\nlogger = logging.getLogger('reprozip_jupyter')\n\n\n@contextlib.contextmanager\ndef process_connection_file(original):\n with original.open('r') as fp:\n data = json.load(fp)\n\n data['ip'] = '0.0.0.0' # Kernel should listen on all interfaces\n\n ports = [value for key, value in data.items() if key.endswith('_port')]\n\n fd, fixed_file = Path.tempfile(suffix='.json')\n with fixed_file.open('w') as fp:\n json.dump(data, fp)\n os.close(fd)\n\n yield fixed_file, ports\n\n fixed_file.remove()\n\n\nclass RPZKernelManager(IOLoopKernelManager):\n rpz_target = None\n rpz_verbosity = 1\n\n def _launch_kernel(self, kernel_cmd, **kw):\n # Need to parse kernel command-line to find the connection file\n logger.info(\"Kernel command-line: %s\", ' '.join(kernel_cmd))\n connection_file = None\n for i, arg in enumerate(kernel_cmd):\n if arg == '-f':\n connection_file = Path(kernel_cmd[i + 1])\n break\n\n if connection_file is None:\n logger.critical(\"The notebook didn't pass a connection file to \"\n \"the kernel\")\n sys.exit(1)\n\n with process_connection_file(connection_file) as (fixed_file, ports):\n # Upload connection file to environment\n subprocess.check_call(\n ['reprounzip'] + (['-v'] * (self.rpz_verbosity - 1)) +\n ['docker', 'upload', self.rpz_target,\n '%s:jupyter_connection_file' % fixed_file])\n\n docker_options = []\n for port in ports:\n docker_options.extend(['-p', '%d:%d' % (port, port)])\n\n return subprocess.Popen(\n ['reprounzip'] + (['-v'] * (self.rpz_verbosity - 1)) +\n ['docker', 'run'] +\n ['--docker-option=%s' % opt for opt in docker_options] +\n [self.rpz_target])\n\n\nclass RPZMappingKernelManager(MappingKernelManager):\n def __init__(self, **kwargs):\n kwargs['kernel_manager_class'] = \\\n 'reprozip_jupyter.run.RPZKernelManager'\n super(RPZMappingKernelManager, self).__init__(**kwargs)\n\n\ndef run_server(target, jupyter_args=None, verbosity=1):\n RPZKernelManager.rpz_target = target\n RPZKernelManager.rpz_verbosity = verbosity\n\n if not jupyter_args:\n jupyter_args = []\n\n logger.info(\"Starting Jupyter notebook\")\n NotebookApp.launch_instance(argv=jupyter_args,\n kernel_manager_class=RPZMappingKernelManager)\n\n\ndef cmd_run_server(args):\n setup_logging('REPROZIP-JUPYTER-SERVER', args.verbosity)\n if not args.target:\n sys.stderr.write(\"Missing experiment directory\\n\")\n sys.exit(2)\n\n run_server(args.target, args.jupyter_args, verbosity=args.verbosity)\n\n\ndef setup(parser):\n parser.add_argument('target', help=\"Experiment directory\")\n parser.add_argument('jupyter_args', nargs=argparse.REMAINDER,\n help=\"Arguments to pass to the notebook server\")\n parser.set_defaults(func=cmd_run_server)\n" }, { "path": "reprozip-jupyter/reprozip_jupyter/server_extension.py", "content": "# Copyright (C) 2014 New York University\n# This file is part of ReproZip which is released under the Revised BSD License\n# See file LICENSE for full license details.\n\nfrom datetime import datetime\nfrom notebook.utils import url_path_join as ujoin\nfrom rpaths import Path\nimport subprocess\nimport sys\nfrom tornado.concurrent import Future\nfrom tornado.process import Subprocess\nfrom tornado.web import RequestHandler\n\n\ntry:\n unicode_ = unicode\nexcept NameError:\n unicode_ = str\n\n\nclass TraceHandler(RequestHandler):\n def initialize(self, nbapp=None):\n self.nbapp = nbapp\n self._tempdir = None\n self._future = None\n\n def post(self):\n self._notebook_file = Path(self.get_body_argument('file'))\n name = self._notebook_file.unicodename\n if name.endswith('.ipynb'):\n name = name[:-6]\n name = u'%s_%s.rpz' % (name, datetime.now().strftime('%Y%m%d-%H%M%S'))\n self._pack_file = self._notebook_file.parent / name\n self.nbapp.log.info(\"reprozip: tracing request from client: file=%r\",\n self._notebook_file)\n self._tempdir = Path.tempdir()\n self.nbapp.log.info(\"reprozip: created temp directory %r\",\n self._tempdir)\n self._future = Future()\n proc = Subprocess(\n [sys.executable, '-c',\n 'from reprozip_jupyter.main import main; main()',\n 'trace',\n '--dont-save-notebook',\n '-d', self._tempdir.path,\n self._notebook_file.path],\n stdin=subprocess.PIPE)\n proc.stdin.close()\n proc.set_exit_callback(self._trace_done)\n self.nbapp.log.info(\"reprozip: started tracing...\")\n return self._future\n\n def _trace_done(self, returncode):\n self.nbapp.log.info(\"reprozip: tracing done, returned %d\", returncode)\n if returncode == 0:\n # Pack\n if self._pack_file.exists():\n self._pack_file.remove()\n proc = Subprocess(\n ['reprozip', 'pack', '-d',\n self._tempdir.path,\n self._pack_file.path],\n stdin=subprocess.PIPE)\n proc.stdin.close()\n proc.set_exit_callback(self._packing_done)\n self.nbapp.log.info(\"reprozip: started packing...\")\n else:\n self._tempdir.rmtree()\n if returncode == 3:\n self.set_header('Content-Type', 'application/json')\n self.finish(\n {'error': \"There was an error running the notebook. \"\n \"Please make sure that it can run from top to \"\n \"bottom without error before packing.\"})\n else:\n self.send_error(500)\n self._future.set_result(None)\n\n def _packing_done(self, returncode):\n self.nbapp.log.info(\"reprozip: packing done, returned %d\", returncode)\n if returncode == 0:\n # Send the response\n self.set_header('Content-Type', 'application/json')\n self.finish({'bundle': unicode_(self._pack_file)})\n self.nbapp.log.info(\"reprozip: response sent!\")\n else:\n self.send_error(500)\n self._tempdir.rmtree()\n self._future.set_result(None)\n\n\ndef load_jupyter_server_extension(nbapp):\n nbapp.log.info('reprozip: notebook extension loaded')\n\n webapp = nbapp.web_app\n base_url = webapp.settings['base_url']\n webapp.add_handlers(\".*$\", [\n (ujoin(base_url, r\"/reprozip/trace\"), TraceHandler, {'nbapp': nbapp}),\n ])\n" }, { "path": "reprozip-jupyter/reprozip_jupyter/trace.py", "content": "# Copyright (C) 2014 New York University\n# This file is part of ReproZip which is released under the Revised BSD License\n# See file LICENSE for full license details.\n\n\"\"\"Trace a notebook to generate accompanying RPZ pack.\n\"\"\"\n\nfrom __future__ import division, print_function, unicode_literals\n\nimport argparse\nfrom jupyter_client.launcher import launch_kernel\nfrom jupyter_client.manager import KernelManager\nfrom jupyter_client.managerabc import KernelManagerABC\nimport logging\nfrom nbconvert.preprocessors import ExecutePreprocessor\nfrom nbconvert.preprocessors.execute import CellExecutionError\nimport nbformat\nimport os\nfrom rpaths import Path\nimport sys\n\nfrom reprounzip.common import setup_logging\n\n\nlogger = logging.getLogger('reprozip_jupyter')\n\n\nclass RPZOptions(object):\n def __init__(self, verbosity=1, dir=None, identify_packages=True,\n find_inputs_outputs=True, append=None):\n self.verbosity = verbosity\n self.dir = dir\n self.identify_packages = identify_packages\n self.find_inputs_outputs = find_inputs_outputs\n self.append = append\n\n def trace_command_line(self, kernel_cmd):\n cmd = ['reprozip']\n cmd.extend(['-v'] * (self.verbosity - 1))\n cmd.append('trace')\n if self.dir:\n cmd.extend(['--dir', self.dir])\n if not self.identify_packages:\n cmd.append('--dont-identify-packages')\n if not self.find_inputs_outputs:\n cmd.append('--dont-find-inputs-outputs')\n if self.append:\n cmd.append('--continue')\n else:\n cmd.append('--overwrite')\n cmd.extend(kernel_cmd)\n return cmd\n\n def config_file(self):\n if self.dir:\n return Path(self.dir) / 'config.yml'\n else:\n return Path('.reprozip-trace/config.yml')\n\n\nclass RPZKernelManager(KernelManager):\n rpz_options = None\n\n def _launch_kernel(self, kernel_cmd, **kw):\n cmd = self.rpz_options.trace_command_line(kernel_cmd)\n\n logger.info(\"Kernel requested, connection file: %s\",\n self.connection_file)\n logger.info(\"Executing: %r\", cmd)\n return launch_kernel(cmd, **kw)\n\n def finish_shutdown(self, *args, **kwargs):\n kwargs['waittime'] = 600\n super(RPZKernelManager, self).finish_shutdown(*args, **kwargs)\n\n # Add the input file to the configuration\n config = self.rpz_options.config_file()\n\n with config.rewrite(encoding='utf-8') as (read, write):\n for line in read:\n write.write(line)\n if line == 'inputs_outputs:\\n':\n write.write(' - name: jupyter_connection_file'\n ' # Needed for reprozip-jupyter operations\\n'\n ' read_by_runs: [0]\\n'\n ' path: %s\\n' % self.connection_file)\n\n\nKernelManagerABC.register(RPZKernelManager)\n\n\nclass RPZExecutePreprocessor(ExecutePreprocessor):\n def __init__(self, options):\n self.rpz_options = options\n super(RPZExecutePreprocessor, self).__init__()\n\n def preprocess(self, nb, resources):\n # no change {\n path = resources.get('metadata', {}).get('path', '')\n if path == '':\n path = None\n\n kernel_name = nb.metadata.get('kernelspec', {}).get('name', 'python')\n if self.kernel_name:\n kernel_name = self.kernel_name\n self.log.info(\"Executing notebook with kernel: %s\" % kernel_name)\n # } no change\n\n logger.info(\"Starting kernel...\")\n\n # copied from start_new_kernel(), but using our KernelManager class {\n km = RPZKernelManager(kernel_name=kernel_name)\n km.rpz_options = self.rpz_options\n km.start_kernel(extra_arguments=self.extra_arguments,\n cwd=path) # changed not to hide stderr\n kc = km.client()\n kc.start_channels()\n try:\n kc.wait_for_ready(timeout=60)\n except RuntimeError:\n kc.stop_channels()\n km.shutdown_kernel()\n raise\n # } start_new_kernel()\n\n self.km, self.kc = km, kc\n logger.info(\"Kernel started\")\n\n # no change {\n self.kc.allow_stdin = False\n\n try:\n nb, resources = super(ExecutePreprocessor, self).preprocess(\n nb, resources)\n except CellExecutionError:\n sys.exit(3)\n finally:\n self.kc.stop_channels()\n self.km.shutdown_kernel(now=False) # changed from now=False\n\n return nb, resources\n # } no change\n\n\ndef trace_notebook(filename, save_notebook=True, **kwargs):\n with open(filename) as fp:\n notebook = nbformat.read(fp, as_version=4)\n preprocessor = RPZExecutePreprocessor(RPZOptions(**kwargs))\n preprocessor.preprocess(\n notebook,\n {'metadata': {'path': os.path.dirname(filename)}})\n if save_notebook:\n with open(filename, 'wt') as fp:\n nbformat.write(notebook, fp)\n return notebook\n\n\ndef cmd_trace_notebook(args):\n setup_logging('REPROZIP-JUPYTER-TRACE', args.verbosity)\n if not args.notebook:\n sys.stderr.write(\"missing notebook\\n\")\n sys.exit(2)\n return trace_notebook(args.notebook,\n save_notebook=args.save_notebook,\n verbosity=args.verbosity, dir=args.dir,\n identify_packages=args.identify_packages,\n find_inputs_outputs=args.find_inputs_outputs,\n append=args.append)\n\n\ndef setup(parser):\n parser.add_argument('-d', '--dir',\n help=\"where to store database and configuration file \"\n \"(default: ./.reprozip-trace)\")\n parser.add_argument(\n '--dont-save-notebook', action='store_false', default=True,\n dest='save_notebook',\n help=\"do not update the notebook file when executing\")\n parser.add_argument(\n '--dont-identify-packages', action='store_false', default=True,\n dest='identify_packages',\n help=\"do not try identify which package each file comes from\")\n parser.add_argument(\n '--find-inputs-outputs', action='store_true',\n default=False, dest='find_inputs_outputs',\n help=\"try to identify input and output files\")\n parser.add_argument(\n '--dont-find-inputs-outputs', action='store_false',\n default=False, dest='find_inputs_outputs',\n help=argparse.SUPPRESS)\n parser.add_argument(\n '-c', '--continue', action='store_true', dest='append',\n help=\"add to the previous trace, don't replace it\")\n parser.add_argument(\n '-w', '--overwrite', action='store_true', dest='overwrite',\n help=\"overwrite the previous trace, don't add to it\")\n parser.add_argument('notebook', help=\"command-line to run under trace\")\n parser.set_defaults(func=cmd_trace_notebook)\n" }, { "path": "reprozip-jupyter/setup.cfg", "content": "[bdist_wheel]\nuniversal = 1\n" }, { "path": "reprozip-jupyter/setup.py", "content": "import io\nimport os\nfrom setuptools import setup\n\n\n# pip workaround\nos.chdir(os.path.abspath(os.path.dirname(__file__)))\n\n\n# Need to specify encoding for PY3, which has the worst unicode handling ever\nwith io.open('README.rst', encoding='utf-8') as fp:\n description = fp.read()\nsetup(name='reprozip-jupyter',\n version='1.2',\n packages=['reprozip_jupyter'],\n package_data={'reprozip_jupyter': ['notebook-extension.js']},\n entry_points={\n 'console_scripts': [\n 'reprozip-jupyter = reprozip_jupyter.main:main']},\n install_requires=['rpaths',\n 'notebook', 'jupyter_client', 'nbformat', 'nbconvert',\n 'reprounzip>=1.0'],\n description=\"Jupyter Notebook tracing/reproduction using ReproZip\",\n author=\"Remi Rampin, Fernando Chirigati, Dennis Shasha, Juliana Freire\",\n author_email='reprozip@nyu.edu',\n maintainer=\"Remi Rampin\",\n maintainer_email='remi@rampin.org',\n url='https://www.reprozip.org/',\n project_urls={\n 'Documentation': 'https://docs.reprozip.org/',\n 'Examples': 'https://examples.reprozip.org/',\n 'Source': 'https://github.com/VIDA-NYU/reprozip',\n 'Bug Tracker': 'https://github.com/VIDA-NYU/reprozip/issues',\n 'Chat': 'https://riot.im/app/#/room/#reprozip:matrix.org',\n 'Changelog':\n 'https://github.com/VIDA-NYU/reprozip/blob/1.x/CHANGELOG.md',\n },\n long_description=description,\n license='BSD',\n keywords=['reprozip', 'reprounzip', 'reproducibility', 'provenance',\n 'vida', 'nyu', 'jupyter', 'notebook'],\n classifiers=[\n 'Development Status :: 4 - Beta',\n 'Intended Audience :: Science/Research',\n 'License :: OSI Approved :: BSD License',\n 'Programming Language :: Python :: 2.7',\n 'Programming Language :: Python :: 3',\n 'Topic :: Scientific/Engineering',\n 'Topic :: System :: Archiving'])\n" }, { "path": "scripts/RELEASE", "content": "Update the CHANGELOG.md file\n\nUpload parameters to stats.reprozip.org\n\nBump versions in setup.py files, __init__.py files, docs/conf.py, installers\n\nSet correct metadata on Zenodo, update badge in README\n\nCreate a signed tag for the version\n git tag -s 1.0.11\n\nMake sdists, wheels, upload to PyPI using twine\n\nMake binary Linux wheels\n\nBuild Windows installer using native Python installer & InnoDB\n\nBuild MacOS installer using Python built from source, App shim, Packages.app\n\nCreate a release on GitHub, add the CHANGELOG to it, link to the attached files\n Mention which packages didn't change\n Don't forget to update the links in the description\n Attach installers\n\nUpdate conda-forge feedstocks\n\nUpdate S3 bucket so website links redirect to newest binaries\n" }, { "path": "scripts/linux-wheels.sh", "content": "#!/bin/sh\n\ncd \"$(dirname \"$0\")/..\"\nif [ ! -d dist ]; then mkdir dist; fi\nfor image in manylinux2010_x86_64 manylinux2014_x86_64; do\ndocker run -i --rm -v \"${PWD}:/src\" quay.io/pypa/${image} <<'END'\nyum install -y sqlite-devel\ncd /src/reprozip\nfor PYBIN in /opt/python/*/bin; do \"${PYBIN}/pip\" wheel . -w ../dist; done\nfor WHEEL in ../dist/*.whl; do auditwheel repair \"${WHEEL}\" -w ../dist; done\nexit 0\nEND\ndone\n" }, { "path": "scripts/macos/README.txt", "content": "This will install Python, ReproUnzip and all plugins to /opt/reprounzip.\n\nOnce this setup is done, you will be able to run reprounzip from the terminal like so:\n\n /opt/reprounzip/reprounzip --help\n\nYou can also add reprounzip to your PATH by adding the line:\n\n export PATH=/opt/reprounzip:$PATH\n\nto .profile in your home directory.\n\nReproUnzip now comes with a graphical application that you can use to unpack and reproduce experiments. If you choose to install it, it will be placed in /Applications for you automatically.\n" }, { "path": "scripts/macos/ReproUnzip.pkgproj", "content": "\n\n\n\n\tPACKAGES\n\t\n\t\t\n\t\t\tMUST-CLOSE-APPLICATION-ITEMS\n\t\t\t\n\t\t\tMUST-CLOSE-APPLICATIONS\n\t\t\t\n\t\t\tPACKAGE_FILES\n\t\t\t\n\t\t\t\tDEFAULT_INSTALL_LOCATION\n\t\t\t\t/\n\t\t\t\tHIERARCHY\n\t\t\t\t\n\t\t\t\t\tCHILDREN\n\t\t\t\t\t\n\t\t\t\t\t\t\n\t\t\t\t\t\t\tCHILDREN\n\t\t\t\t\t\t\t\n\t\t\t\t\t\t\t\t\n\t\t\t\t\t\t\t\t\tCHILDREN\n\t\t\t\t\t\t\t\t\t\n\t\t\t\t\t\t\t\t\tGID\n\t\t\t\t\t\t\t\t\t80\n\t\t\t\t\t\t\t\t\tPATH\n\t\t\t\t\t\t\t\t\tUtilities\n\t\t\t\t\t\t\t\t\tPATH_TYPE\n\t\t\t\t\t\t\t\t\t0\n\t\t\t\t\t\t\t\t\tPERMISSIONS\n\t\t\t\t\t\t\t\t\t493\n\t\t\t\t\t\t\t\t\tTYPE\n\t\t\t\t\t\t\t\t\t1\n\t\t\t\t\t\t\t\t\tUID\n\t\t\t\t\t\t\t\t\t0\n\t\t\t\t\t\t\t\t\n\t\t\t\t\t\t\t\n\t\t\t\t\t\t\tGID\n\t\t\t\t\t\t\t80\n\t\t\t\t\t\t\tPATH\n\t\t\t\t\t\t\tApplications\n\t\t\t\t\t\t\tPATH_TYPE\n\t\t\t\t\t\t\t0\n\t\t\t\t\t\t\tPERMISSIONS\n\t\t\t\t\t\t\t509\n\t\t\t\t\t\t\tTYPE\n\t\t\t\t\t\t\t1\n\t\t\t\t\t\t\tUID\n\t\t\t\t\t\t\t0\n\t\t\t\t\t\t\n\t\t\t\t\t\t\n\t\t\t\t\t\t\tCHILDREN\n\t\t\t\t\t\t\t\n\t\t\t\t\t\t\t\t\n\t\t\t\t\t\t\t\t\tCHILDREN\n\t\t\t\t\t\t\t\t\t\n\t\t\t\t\t\t\t\t\tGID\n\t\t\t\t\t\t\t\t\t80\n\t\t\t\t\t\t\t\t\tPATH\n\t\t\t\t\t\t\t\t\tApplication Support\n\t\t\t\t\t\t\t\t\tPATH_TYPE\n\t\t\t\t\t\t\t\t\t0\n\t\t\t\t\t\t\t\t\tPERMISSIONS\n\t\t\t\t\t\t\t\t\t493\n\t\t\t\t\t\t\t\t\tTYPE\n\t\t\t\t\t\t\t\t\t1\n\t\t\t\t\t\t\t\t\tUID\n\t\t\t\t\t\t\t\t\t0\n\t\t\t\t\t\t\t\t\n\t\t\t\t\t\t\t\t\n\t\t\t\t\t\t\t\t\tCHILDREN\n\t\t\t\t\t\t\t\t\t\n\t\t\t\t\t\t\t\t\tGID\n\t\t\t\t\t\t\t\t\t0\n\t\t\t\t\t\t\t\t\tPATH\n\t\t\t\t\t\t\t\t\tAutomator\n\t\t\t\t\t\t\t\t\tPATH_TYPE\n\t\t\t\t\t\t\t\t\t0\n\t\t\t\t\t\t\t\t\tPERMISSIONS\n\t\t\t\t\t\t\t\t\t493\n\t\t\t\t\t\t\t\t\tTYPE\n\t\t\t\t\t\t\t\t\t1\n\t\t\t\t\t\t\t\t\tUID\n\t\t\t\t\t\t\t\t\t0\n\t\t\t\t\t\t\t\t\n\t\t\t\t\t\t\t\t\n\t\t\t\t\t\t\t\t\tCHILDREN\n\t\t\t\t\t\t\t\t\t\n\t\t\t\t\t\t\t\t\tGID\n\t\t\t\t\t\t\t\t\t0\n\t\t\t\t\t\t\t\t\tPATH\n\t\t\t\t\t\t\t\t\tDocumentation\n\t\t\t\t\t\t\t\t\tPATH_TYPE\n\t\t\t\t\t\t\t\t\t0\n\t\t\t\t\t\t\t\t\tPERMISSIONS\n\t\t\t\t\t\t\t\t\t493\n\t\t\t\t\t\t\t\t\tTYPE\n\t\t\t\t\t\t\t\t\t1\n\t\t\t\t\t\t\t\t\tUID\n\t\t\t\t\t\t\t\t\t0\n\t\t\t\t\t\t\t\t\n\t\t\t\t\t\t\t\t\n\t\t\t\t\t\t\t\t\tCHILDREN\n\t\t\t\t\t\t\t\t\t\n\t\t\t\t\t\t\t\t\tGID\n\t\t\t\t\t\t\t\t\t0\n\t\t\t\t\t\t\t\t\tPATH\n\t\t\t\t\t\t\t\t\tFilesystems\n\t\t\t\t\t\t\t\t\tPATH_TYPE\n\t\t\t\t\t\t\t\t\t0\n\t\t\t\t\t\t\t\t\tPERMISSIONS\n\t\t\t\t\t\t\t\t\t493\n\t\t\t\t\t\t\t\t\tTYPE\n\t\t\t\t\t\t\t\t\t1\n\t\t\t\t\t\t\t\t\tUID\n\t\t\t\t\t\t\t\t\t0\n\t\t\t\t\t\t\t\t\n\t\t\t\t\t\t\t\t\n\t\t\t\t\t\t\t\t\tCHILDREN\n\t\t\t\t\t\t\t\t\t\n\t\t\t\t\t\t\t\t\tGID\n\t\t\t\t\t\t\t\t\t0\n\t\t\t\t\t\t\t\t\tPATH\n\t\t\t\t\t\t\t\t\tFrameworks\n\t\t\t\t\t\t\t\t\tPATH_TYPE\n\t\t\t\t\t\t\t\t\t0\n\t\t\t\t\t\t\t\t\tPERMISSIONS\n\t\t\t\t\t\t\t\t\t493\n\t\t\t\t\t\t\t\t\tTYPE\n\t\t\t\t\t\t\t\t\t1\n\t\t\t\t\t\t\t\t\tUID\n\t\t\t\t\t\t\t\t\t0\n\t\t\t\t\t\t\t\t\n\t\t\t\t\t\t\t\t\n\t\t\t\t\t\t\t\t\tCHILDREN\n\t\t\t\t\t\t\t\t\t\n\t\t\t\t\t\t\t\t\tGID\n\t\t\t\t\t\t\t\t\t0\n\t\t\t\t\t\t\t\t\tPATH\n\t\t\t\t\t\t\t\t\tInput Methods\n\t\t\t\t\t\t\t\t\tPATH_TYPE\n\t\t\t\t\t\t\t\t\t0\n\t\t\t\t\t\t\t\t\tPERMISSIONS\n\t\t\t\t\t\t\t\t\t493\n\t\t\t\t\t\t\t\t\tTYPE\n\t\t\t\t\t\t\t\t\t1\n\t\t\t\t\t\t\t\t\tUID\n\t\t\t\t\t\t\t\t\t0\n\t\t\t\t\t\t\t\t\n\t\t\t\t\t\t\t\t\n\t\t\t\t\t\t\t\t\tCHILDREN\n\t\t\t\t\t\t\t\t\t\n\t\t\t\t\t\t\t\t\tGID\n\t\t\t\t\t\t\t\t\t0\n\t\t\t\t\t\t\t\t\tPATH\n\t\t\t\t\t\t\t\t\tInternet Plug-Ins\n\t\t\t\t\t\t\t\t\tPATH_TYPE\n\t\t\t\t\t\t\t\t\t0\n\t\t\t\t\t\t\t\t\tPERMISSIONS\n\t\t\t\t\t\t\t\t\t493\n\t\t\t\t\t\t\t\t\tTYPE\n\t\t\t\t\t\t\t\t\t1\n\t\t\t\t\t\t\t\t\tUID\n\t\t\t\t\t\t\t\t\t0\n\t\t\t\t\t\t\t\t\n\t\t\t\t\t\t\t\t\n\t\t\t\t\t\t\t\t\tCHILDREN\n\t\t\t\t\t\t\t\t\t\n\t\t\t\t\t\t\t\t\tGID\n\t\t\t\t\t\t\t\t\t0\n\t\t\t\t\t\t\t\t\tPATH\n\t\t\t\t\t\t\t\t\tLaunchAgents\n\t\t\t\t\t\t\t\t\tPATH_TYPE\n\t\t\t\t\t\t\t\t\t0\n\t\t\t\t\t\t\t\t\tPERMISSIONS\n\t\t\t\t\t\t\t\t\t493\n\t\t\t\t\t\t\t\t\tTYPE\n\t\t\t\t\t\t\t\t\t1\n\t\t\t\t\t\t\t\t\tUID\n\t\t\t\t\t\t\t\t\t0\n\t\t\t\t\t\t\t\t\n\t\t\t\t\t\t\t\t\n\t\t\t\t\t\t\t\t\tCHILDREN\n\t\t\t\t\t\t\t\t\t\n\t\t\t\t\t\t\t\t\tGID\n\t\t\t\t\t\t\t\t\t0\n\t\t\t\t\t\t\t\t\tPATH\n\t\t\t\t\t\t\t\t\tLaunchDaemons\n\t\t\t\t\t\t\t\t\tPATH_TYPE\n\t\t\t\t\t\t\t\t\t0\n\t\t\t\t\t\t\t\t\tPERMISSIONS\n\t\t\t\t\t\t\t\t\t493\n\t\t\t\t\t\t\t\t\tTYPE\n\t\t\t\t\t\t\t\t\t1\n\t\t\t\t\t\t\t\t\tUID\n\t\t\t\t\t\t\t\t\t0\n\t\t\t\t\t\t\t\t\n\t\t\t\t\t\t\t\t\n\t\t\t\t\t\t\t\t\tCHILDREN\n\t\t\t\t\t\t\t\t\t\n\t\t\t\t\t\t\t\t\tGID\n\t\t\t\t\t\t\t\t\t0\n\t\t\t\t\t\t\t\t\tPATH\n\t\t\t\t\t\t\t\t\tPreferencePanes\n\t\t\t\t\t\t\t\t\tPATH_TYPE\n\t\t\t\t\t\t\t\t\t0\n\t\t\t\t\t\t\t\t\tPERMISSIONS\n\t\t\t\t\t\t\t\t\t493\n\t\t\t\t\t\t\t\t\tTYPE\n\t\t\t\t\t\t\t\t\t1\n\t\t\t\t\t\t\t\t\tUID\n\t\t\t\t\t\t\t\t\t0\n\t\t\t\t\t\t\t\t\n\t\t\t\t\t\t\t\t\n\t\t\t\t\t\t\t\t\tCHILDREN\n\t\t\t\t\t\t\t\t\t\n\t\t\t\t\t\t\t\t\tGID\n\t\t\t\t\t\t\t\t\t0\n\t\t\t\t\t\t\t\t\tPATH\n\t\t\t\t\t\t\t\t\tPreferences\n\t\t\t\t\t\t\t\t\tPATH_TYPE\n\t\t\t\t\t\t\t\t\t0\n\t\t\t\t\t\t\t\t\tPERMISSIONS\n\t\t\t\t\t\t\t\t\t493\n\t\t\t\t\t\t\t\t\tTYPE\n\t\t\t\t\t\t\t\t\t1\n\t\t\t\t\t\t\t\t\tUID\n\t\t\t\t\t\t\t\t\t0\n\t\t\t\t\t\t\t\t\n\t\t\t\t\t\t\t\t\n\t\t\t\t\t\t\t\t\tCHILDREN\n\t\t\t\t\t\t\t\t\t\n\t\t\t\t\t\t\t\t\tGID\n\t\t\t\t\t\t\t\t\t80\n\t\t\t\t\t\t\t\t\tPATH\n\t\t\t\t\t\t\t\t\tPrinters\n\t\t\t\t\t\t\t\t\tPATH_TYPE\n\t\t\t\t\t\t\t\t\t0\n\t\t\t\t\t\t\t\t\tPERMISSIONS\n\t\t\t\t\t\t\t\t\t493\n\t\t\t\t\t\t\t\t\tTYPE\n\t\t\t\t\t\t\t\t\t1\n\t\t\t\t\t\t\t\t\tUID\n\t\t\t\t\t\t\t\t\t0\n\t\t\t\t\t\t\t\t\n\t\t\t\t\t\t\t\t\n\t\t\t\t\t\t\t\t\tCHILDREN\n\t\t\t\t\t\t\t\t\t\n\t\t\t\t\t\t\t\t\tGID\n\t\t\t\t\t\t\t\t\t0\n\t\t\t\t\t\t\t\t\tPATH\n\t\t\t\t\t\t\t\t\tPrivilegedHelperTools\n\t\t\t\t\t\t\t\t\tPATH_TYPE\n\t\t\t\t\t\t\t\t\t0\n\t\t\t\t\t\t\t\t\tPERMISSIONS\n\t\t\t\t\t\t\t\t\t1005\n\t\t\t\t\t\t\t\t\tTYPE\n\t\t\t\t\t\t\t\t\t1\n\t\t\t\t\t\t\t\t\tUID\n\t\t\t\t\t\t\t\t\t0\n\t\t\t\t\t\t\t\t\n\t\t\t\t\t\t\t\t\n\t\t\t\t\t\t\t\t\tCHILDREN\n\t\t\t\t\t\t\t\t\t\n\t\t\t\t\t\t\t\t\tGID\n\t\t\t\t\t\t\t\t\t0\n\t\t\t\t\t\t\t\t\tPATH\n\t\t\t\t\t\t\t\t\tQuickLook\n\t\t\t\t\t\t\t\t\tPATH_TYPE\n\t\t\t\t\t\t\t\t\t0\n\t\t\t\t\t\t\t\t\tPERMISSIONS\n\t\t\t\t\t\t\t\t\t493\n\t\t\t\t\t\t\t\t\tTYPE\n\t\t\t\t\t\t\t\t\t1\n\t\t\t\t\t\t\t\t\tUID\n\t\t\t\t\t\t\t\t\t0\n\t\t\t\t\t\t\t\t\n\t\t\t\t\t\t\t\t\n\t\t\t\t\t\t\t\t\tCHILDREN\n\t\t\t\t\t\t\t\t\t\n\t\t\t\t\t\t\t\t\tGID\n\t\t\t\t\t\t\t\t\t0\n\t\t\t\t\t\t\t\t\tPATH\n\t\t\t\t\t\t\t\t\tQuickTime\n\t\t\t\t\t\t\t\t\tPATH_TYPE\n\t\t\t\t\t\t\t\t\t0\n\t\t\t\t\t\t\t\t\tPERMISSIONS\n\t\t\t\t\t\t\t\t\t493\n\t\t\t\t\t\t\t\t\tTYPE\n\t\t\t\t\t\t\t\t\t1\n\t\t\t\t\t\t\t\t\tUID\n\t\t\t\t\t\t\t\t\t0\n\t\t\t\t\t\t\t\t\n\t\t\t\t\t\t\t\t\n\t\t\t\t\t\t\t\t\tCHILDREN\n\t\t\t\t\t\t\t\t\t\n\t\t\t\t\t\t\t\t\tGID\n\t\t\t\t\t\t\t\t\t0\n\t\t\t\t\t\t\t\t\tPATH\n\t\t\t\t\t\t\t\t\tScreen Savers\n\t\t\t\t\t\t\t\t\tPATH_TYPE\n\t\t\t\t\t\t\t\t\t0\n\t\t\t\t\t\t\t\t\tPERMISSIONS\n\t\t\t\t\t\t\t\t\t493\n\t\t\t\t\t\t\t\t\tTYPE\n\t\t\t\t\t\t\t\t\t1\n\t\t\t\t\t\t\t\t\tUID\n\t\t\t\t\t\t\t\t\t0\n\t\t\t\t\t\t\t\t\n\t\t\t\t\t\t\t\t\n\t\t\t\t\t\t\t\t\tCHILDREN\n\t\t\t\t\t\t\t\t\t\n\t\t\t\t\t\t\t\t\tGID\n\t\t\t\t\t\t\t\t\t0\n\t\t\t\t\t\t\t\t\tPATH\n\t\t\t\t\t\t\t\t\tScripts\n\t\t\t\t\t\t\t\t\tPATH_TYPE\n\t\t\t\t\t\t\t\t\t0\n\t\t\t\t\t\t\t\t\tPERMISSIONS\n\t\t\t\t\t\t\t\t\t493\n\t\t\t\t\t\t\t\t\tTYPE\n\t\t\t\t\t\t\t\t\t1\n\t\t\t\t\t\t\t\t\tUID\n\t\t\t\t\t\t\t\t\t0\n\t\t\t\t\t\t\t\t\n\t\t\t\t\t\t\t\t\n\t\t\t\t\t\t\t\t\tCHILDREN\n\t\t\t\t\t\t\t\t\t\n\t\t\t\t\t\t\t\t\tGID\n\t\t\t\t\t\t\t\t\t0\n\t\t\t\t\t\t\t\t\tPATH\n\t\t\t\t\t\t\t\t\tServices\n\t\t\t\t\t\t\t\t\tPATH_TYPE\n\t\t\t\t\t\t\t\t\t0\n\t\t\t\t\t\t\t\t\tPERMISSIONS\n\t\t\t\t\t\t\t\t\t493\n\t\t\t\t\t\t\t\t\tTYPE\n\t\t\t\t\t\t\t\t\t1\n\t\t\t\t\t\t\t\t\tUID\n\t\t\t\t\t\t\t\t\t0\n\t\t\t\t\t\t\t\t\n\t\t\t\t\t\t\t\t\n\t\t\t\t\t\t\t\t\tCHILDREN\n\t\t\t\t\t\t\t\t\t\n\t\t\t\t\t\t\t\t\tGID\n\t\t\t\t\t\t\t\t\t0\n\t\t\t\t\t\t\t\t\tPATH\n\t\t\t\t\t\t\t\t\tWidgets\n\t\t\t\t\t\t\t\t\tPATH_TYPE\n\t\t\t\t\t\t\t\t\t0\n\t\t\t\t\t\t\t\t\tPERMISSIONS\n\t\t\t\t\t\t\t\t\t493\n\t\t\t\t\t\t\t\t\tTYPE\n\t\t\t\t\t\t\t\t\t1\n\t\t\t\t\t\t\t\t\tUID\n\t\t\t\t\t\t\t\t\t0\n\t\t\t\t\t\t\t\t\n\t\t\t\t\t\t\t\t\n\t\t\t\t\t\t\t\t\tCHILDREN\n\t\t\t\t\t\t\t\t\t\n\t\t\t\t\t\t\t\t\tGID\n\t\t\t\t\t\t\t\t\t0\n\t\t\t\t\t\t\t\t\tPATH\n\t\t\t\t\t\t\t\t\tExtensions\n\t\t\t\t\t\t\t\t\tPATH_TYPE\n\t\t\t\t\t\t\t\t\t0\n\t\t\t\t\t\t\t\t\tPERMISSIONS\n\t\t\t\t\t\t\t\t\t493\n\t\t\t\t\t\t\t\t\tTYPE\n\t\t\t\t\t\t\t\t\t1\n\t\t\t\t\t\t\t\t\tUID\n\t\t\t\t\t\t\t\t\t0\n\t\t\t\t\t\t\t\t\n\t\t\t\t\t\t\t\n\t\t\t\t\t\t\tGID\n\t\t\t\t\t\t\t0\n\t\t\t\t\t\t\tPATH\n\t\t\t\t\t\t\tLibrary\n\t\t\t\t\t\t\tPATH_TYPE\n\t\t\t\t\t\t\t0\n\t\t\t\t\t\t\tPERMISSIONS\n\t\t\t\t\t\t\t493\n\t\t\t\t\t\t\tTYPE\n\t\t\t\t\t\t\t1\n\t\t\t\t\t\t\tUID\n\t\t\t\t\t\t\t0\n\t\t\t\t\t\t\n\t\t\t\t\t\t\n\t\t\t\t\t\t\tCHILDREN\n\t\t\t\t\t\t\t\n\t\t\t\t\t\t\t\t\n\t\t\t\t\t\t\t\t\tCHILDREN\n\t\t\t\t\t\t\t\t\t\n\t\t\t\t\t\t\t\t\tGID\n\t\t\t\t\t\t\t\t\t0\n\t\t\t\t\t\t\t\t\tPATH\n\t\t\t\t\t\t\t\t\t/opt/reprounzip\n\t\t\t\t\t\t\t\t\tPATH_TYPE\n\t\t\t\t\t\t\t\t\t0\n\t\t\t\t\t\t\t\t\tPERMISSIONS\n\t\t\t\t\t\t\t\t\t493\n\t\t\t\t\t\t\t\t\tTYPE\n\t\t\t\t\t\t\t\t\t3\n\t\t\t\t\t\t\t\t\tUID\n\t\t\t\t\t\t\t\t\t0\n\t\t\t\t\t\t\t\t\n\t\t\t\t\t\t\t\n\t\t\t\t\t\t\tGID\n\t\t\t\t\t\t\t0\n\t\t\t\t\t\t\tPATH\n\t\t\t\t\t\t\topt\n\t\t\t\t\t\t\tPATH_TYPE\n\t\t\t\t\t\t\t0\n\t\t\t\t\t\t\tPERMISSIONS\n\t\t\t\t\t\t\t493\n\t\t\t\t\t\t\tTYPE\n\t\t\t\t\t\t\t2\n\t\t\t\t\t\t\tUID\n\t\t\t\t\t\t\t0\n\t\t\t\t\t\t\n\t\t\t\t\t\t\n\t\t\t\t\t\t\tCHILDREN\n\t\t\t\t\t\t\t\n\t\t\t\t\t\t\t\t\n\t\t\t\t\t\t\t\t\tCHILDREN\n\t\t\t\t\t\t\t\t\t\n\t\t\t\t\t\t\t\t\t\t\n\t\t\t\t\t\t\t\t\t\t\tCHILDREN\n\t\t\t\t\t\t\t\t\t\t\t\n\t\t\t\t\t\t\t\t\t\t\tGID\n\t\t\t\t\t\t\t\t\t\t\t0\n\t\t\t\t\t\t\t\t\t\t\tPATH\n\t\t\t\t\t\t\t\t\t\t\tExtensions\n\t\t\t\t\t\t\t\t\t\t\tPATH_TYPE\n\t\t\t\t\t\t\t\t\t\t\t0\n\t\t\t\t\t\t\t\t\t\t\tPERMISSIONS\n\t\t\t\t\t\t\t\t\t\t\t493\n\t\t\t\t\t\t\t\t\t\t\tTYPE\n\t\t\t\t\t\t\t\t\t\t\t1\n\t\t\t\t\t\t\t\t\t\t\tUID\n\t\t\t\t\t\t\t\t\t\t\t0\n\t\t\t\t\t\t\t\t\t\t\n\t\t\t\t\t\t\t\t\t\n\t\t\t\t\t\t\t\t\tGID\n\t\t\t\t\t\t\t\t\t0\n\t\t\t\t\t\t\t\t\tPATH\n\t\t\t\t\t\t\t\t\tLibrary\n\t\t\t\t\t\t\t\t\tPATH_TYPE\n\t\t\t\t\t\t\t\t\t0\n\t\t\t\t\t\t\t\t\tPERMISSIONS\n\t\t\t\t\t\t\t\t\t493\n\t\t\t\t\t\t\t\t\tTYPE\n\t\t\t\t\t\t\t\t\t1\n\t\t\t\t\t\t\t\t\tUID\n\t\t\t\t\t\t\t\t\t0\n\t\t\t\t\t\t\t\t\n\t\t\t\t\t\t\t\n\t\t\t\t\t\t\tGID\n\t\t\t\t\t\t\t0\n\t\t\t\t\t\t\tPATH\n\t\t\t\t\t\t\tSystem\n\t\t\t\t\t\t\tPATH_TYPE\n\t\t\t\t\t\t\t0\n\t\t\t\t\t\t\tPERMISSIONS\n\t\t\t\t\t\t\t493\n\t\t\t\t\t\t\tTYPE\n\t\t\t\t\t\t\t1\n\t\t\t\t\t\t\tUID\n\t\t\t\t\t\t\t0\n\t\t\t\t\t\t\n\t\t\t\t\t\t\n\t\t\t\t\t\t\tCHILDREN\n\t\t\t\t\t\t\t\n\t\t\t\t\t\t\t\t\n\t\t\t\t\t\t\t\t\tCHILDREN\n\t\t\t\t\t\t\t\t\t\n\t\t\t\t\t\t\t\t\tGID\n\t\t\t\t\t\t\t\t\t0\n\t\t\t\t\t\t\t\t\tPATH\n\t\t\t\t\t\t\t\t\tShared\n\t\t\t\t\t\t\t\t\tPATH_TYPE\n\t\t\t\t\t\t\t\t\t0\n\t\t\t\t\t\t\t\t\tPERMISSIONS\n\t\t\t\t\t\t\t\t\t1023\n\t\t\t\t\t\t\t\t\tTYPE\n\t\t\t\t\t\t\t\t\t1\n\t\t\t\t\t\t\t\t\tUID\n\t\t\t\t\t\t\t\t\t0\n\t\t\t\t\t\t\t\t\n\t\t\t\t\t\t\t\n\t\t\t\t\t\t\tGID\n\t\t\t\t\t\t\t80\n\t\t\t\t\t\t\tPATH\n\t\t\t\t\t\t\tUsers\n\t\t\t\t\t\t\tPATH_TYPE\n\t\t\t\t\t\t\t0\n\t\t\t\t\t\t\tPERMISSIONS\n\t\t\t\t\t\t\t493\n\t\t\t\t\t\t\tTYPE\n\t\t\t\t\t\t\t1\n\t\t\t\t\t\t\tUID\n\t\t\t\t\t\t\t0\n\t\t\t\t\t\t\n\t\t\t\t\t\n\t\t\t\t\tGID\n\t\t\t\t\t0\n\t\t\t\t\tPATH\n\t\t\t\t\t/\n\t\t\t\t\tPATH_TYPE\n\t\t\t\t\t0\n\t\t\t\t\tPERMISSIONS\n\t\t\t\t\t493\n\t\t\t\t\tTYPE\n\t\t\t\t\t1\n\t\t\t\t\tUID\n\t\t\t\t\t0\n\t\t\t\t\n\t\t\t\tPAYLOAD_TYPE\n\t\t\t\t0\n\t\t\t\tSHOW_INVISIBLE\n\t\t\t\t\n\t\t\t\tSPLIT_FORKS\n\t\t\t\t\n\t\t\t\tTREAT_MISSING_FILES_AS_WARNING\n\t\t\t\t\n\t\t\t\tVERSION\n\t\t\t\t5\n\t\t\t\n\t\t\tPACKAGE_SCRIPTS\n\t\t\t\n\t\t\t\tRESOURCES\n\t\t\t\t\n\t\t\t\n\t\t\tPACKAGE_SETTINGS\n\t\t\t\n\t\t\t\tAUTHENTICATION\n\t\t\t\t1\n\t\t\t\tCONCLUSION_ACTION\n\t\t\t\t0\n\t\t\t\tFOLLOW_SYMBOLIC_LINKS\n\t\t\t\t\n\t\t\t\tIDENTIFIER\n\t\t\t\tedu.nyu.vida.reprounzip\n\t\t\t\tLOCATION\n\t\t\t\t0\n\t\t\t\tNAME\n\t\t\t\tReproUnzip\n\t\t\t\tOVERWRITE_PERMISSIONS\n\t\t\t\t\n\t\t\t\tPAYLOAD_SIZE\n\t\t\t\t-1\n\t\t\t\tREFERENCE_PATH\n\t\t\t\t\n\t\t\t\tRELOCATABLE\n\t\t\t\t\n\t\t\t\tUSE_HFS+_COMPRESSION\n\t\t\t\t\n\t\t\t\tVERSION\n\t\t\t\t1.2\n\t\t\t\n\t\t\tTYPE\n\t\t\t0\n\t\t\tUUID\n\t\t\tD5E9C540-1489-4E0E-A914-B81FB6DCED9C\n\t\t\n\t\t\n\t\t\tMUST-CLOSE-APPLICATION-ITEMS\n\t\t\t\n\t\t\tMUST-CLOSE-APPLICATIONS\n\t\t\t\n\t\t\tPACKAGE_FILES\n\t\t\t\n\t\t\t\tDEFAULT_INSTALL_LOCATION\n\t\t\t\t/\n\t\t\t\tHIERARCHY\n\t\t\t\t\n\t\t\t\t\tCHILDREN\n\t\t\t\t\t\n\t\t\t\t\t\t\n\t\t\t\t\t\t\tCHILDREN\n\t\t\t\t\t\t\t\n\t\t\t\t\t\t\t\t\n\t\t\t\t\t\t\t\t\tBUNDLE_CAN_DOWNGRADE\n\t\t\t\t\t\t\t\t\t\n\t\t\t\t\t\t\t\t\tBUNDLE_POSTINSTALL_PATH\n\t\t\t\t\t\t\t\t\t\n\t\t\t\t\t\t\t\t\t\tPATH_TYPE\n\t\t\t\t\t\t\t\t\t\t0\n\t\t\t\t\t\t\t\t\t\n\t\t\t\t\t\t\t\t\tBUNDLE_PREINSTALL_PATH\n\t\t\t\t\t\t\t\t\t\n\t\t\t\t\t\t\t\t\t\tPATH_TYPE\n\t\t\t\t\t\t\t\t\t\t0\n\t\t\t\t\t\t\t\t\t\n\t\t\t\t\t\t\t\t\tCHILDREN\n\t\t\t\t\t\t\t\t\t\n\t\t\t\t\t\t\t\t\tGID\n\t\t\t\t\t\t\t\t\t0\n\t\t\t\t\t\t\t\t\tPATH\n\t\t\t\t\t\t\t\t\tapp-shim/ReproUnzip.app\n\t\t\t\t\t\t\t\t\tPATH_TYPE\n\t\t\t\t\t\t\t\t\t1\n\t\t\t\t\t\t\t\t\tPERMISSIONS\n\t\t\t\t\t\t\t\t\t493\n\t\t\t\t\t\t\t\t\tTYPE\n\t\t\t\t\t\t\t\t\t3\n\t\t\t\t\t\t\t\t\tUID\n\t\t\t\t\t\t\t\t\t0\n\t\t\t\t\t\t\t\t\n\t\t\t\t\t\t\t\t\n\t\t\t\t\t\t\t\t\tCHILDREN\n\t\t\t\t\t\t\t\t\t\n\t\t\t\t\t\t\t\t\tGID\n\t\t\t\t\t\t\t\t\t80\n\t\t\t\t\t\t\t\t\tPATH\n\t\t\t\t\t\t\t\t\tUtilities\n\t\t\t\t\t\t\t\t\tPATH_TYPE\n\t\t\t\t\t\t\t\t\t0\n\t\t\t\t\t\t\t\t\tPERMISSIONS\n\t\t\t\t\t\t\t\t\t493\n\t\t\t\t\t\t\t\t\tTYPE\n\t\t\t\t\t\t\t\t\t1\n\t\t\t\t\t\t\t\t\tUID\n\t\t\t\t\t\t\t\t\t0\n\t\t\t\t\t\t\t\t\n\t\t\t\t\t\t\t\n\t\t\t\t\t\t\tGID\n\t\t\t\t\t\t\t80\n\t\t\t\t\t\t\tPATH\n\t\t\t\t\t\t\tApplications\n\t\t\t\t\t\t\tPATH_TYPE\n\t\t\t\t\t\t\t0\n\t\t\t\t\t\t\tPERMISSIONS\n\t\t\t\t\t\t\t509\n\t\t\t\t\t\t\tTYPE\n\t\t\t\t\t\t\t1\n\t\t\t\t\t\t\tUID\n\t\t\t\t\t\t\t0\n\t\t\t\t\t\t\n\t\t\t\t\t\t\n\t\t\t\t\t\t\tCHILDREN\n\t\t\t\t\t\t\t\n\t\t\t\t\t\t\t\t\n\t\t\t\t\t\t\t\t\tCHILDREN\n\t\t\t\t\t\t\t\t\t\n\t\t\t\t\t\t\t\t\tGID\n\t\t\t\t\t\t\t\t\t80\n\t\t\t\t\t\t\t\t\tPATH\n\t\t\t\t\t\t\t\t\tApplication Support\n\t\t\t\t\t\t\t\t\tPATH_TYPE\n\t\t\t\t\t\t\t\t\t0\n\t\t\t\t\t\t\t\t\tPERMISSIONS\n\t\t\t\t\t\t\t\t\t493\n\t\t\t\t\t\t\t\t\tTYPE\n\t\t\t\t\t\t\t\t\t1\n\t\t\t\t\t\t\t\t\tUID\n\t\t\t\t\t\t\t\t\t0\n\t\t\t\t\t\t\t\t\n\t\t\t\t\t\t\t\t\n\t\t\t\t\t\t\t\t\tCHILDREN\n\t\t\t\t\t\t\t\t\t\n\t\t\t\t\t\t\t\t\tGID\n\t\t\t\t\t\t\t\t\t0\n\t\t\t\t\t\t\t\t\tPATH\n\t\t\t\t\t\t\t\t\tAutomator\n\t\t\t\t\t\t\t\t\tPATH_TYPE\n\t\t\t\t\t\t\t\t\t0\n\t\t\t\t\t\t\t\t\tPERMISSIONS\n\t\t\t\t\t\t\t\t\t493\n\t\t\t\t\t\t\t\t\tTYPE\n\t\t\t\t\t\t\t\t\t1\n\t\t\t\t\t\t\t\t\tUID\n\t\t\t\t\t\t\t\t\t0\n\t\t\t\t\t\t\t\t\n\t\t\t\t\t\t\t\t\n\t\t\t\t\t\t\t\t\tCHILDREN\n\t\t\t\t\t\t\t\t\t\n\t\t\t\t\t\t\t\t\tGID\n\t\t\t\t\t\t\t\t\t0\n\t\t\t\t\t\t\t\t\tPATH\n\t\t\t\t\t\t\t\t\tDocumentation\n\t\t\t\t\t\t\t\t\tPATH_TYPE\n\t\t\t\t\t\t\t\t\t0\n\t\t\t\t\t\t\t\t\tPERMISSIONS\n\t\t\t\t\t\t\t\t\t493\n\t\t\t\t\t\t\t\t\tTYPE\n\t\t\t\t\t\t\t\t\t1\n\t\t\t\t\t\t\t\t\tUID\n\t\t\t\t\t\t\t\t\t0\n\t\t\t\t\t\t\t\t\n\t\t\t\t\t\t\t\t\n\t\t\t\t\t\t\t\t\tCHILDREN\n\t\t\t\t\t\t\t\t\t\n\t\t\t\t\t\t\t\t\tGID\n\t\t\t\t\t\t\t\t\t0\n\t\t\t\t\t\t\t\t\tPATH\n\t\t\t\t\t\t\t\t\tFilesystems\n\t\t\t\t\t\t\t\t\tPATH_TYPE\n\t\t\t\t\t\t\t\t\t0\n\t\t\t\t\t\t\t\t\tPERMISSIONS\n\t\t\t\t\t\t\t\t\t493\n\t\t\t\t\t\t\t\t\tTYPE\n\t\t\t\t\t\t\t\t\t1\n\t\t\t\t\t\t\t\t\tUID\n\t\t\t\t\t\t\t\t\t0\n\t\t\t\t\t\t\t\t\n\t\t\t\t\t\t\t\t\n\t\t\t\t\t\t\t\t\tCHILDREN\n\t\t\t\t\t\t\t\t\t\n\t\t\t\t\t\t\t\t\tGID\n\t\t\t\t\t\t\t\t\t0\n\t\t\t\t\t\t\t\t\tPATH\n\t\t\t\t\t\t\t\t\tFrameworks\n\t\t\t\t\t\t\t\t\tPATH_TYPE\n\t\t\t\t\t\t\t\t\t0\n\t\t\t\t\t\t\t\t\tPERMISSIONS\n\t\t\t\t\t\t\t\t\t493\n\t\t\t\t\t\t\t\t\tTYPE\n\t\t\t\t\t\t\t\t\t1\n\t\t\t\t\t\t\t\t\tUID\n\t\t\t\t\t\t\t\t\t0\n\t\t\t\t\t\t\t\t\n\t\t\t\t\t\t\t\t\n\t\t\t\t\t\t\t\t\tCHILDREN\n\t\t\t\t\t\t\t\t\t\n\t\t\t\t\t\t\t\t\tGID\n\t\t\t\t\t\t\t\t\t0\n\t\t\t\t\t\t\t\t\tPATH\n\t\t\t\t\t\t\t\t\tInput Methods\n\t\t\t\t\t\t\t\t\tPATH_TYPE\n\t\t\t\t\t\t\t\t\t0\n\t\t\t\t\t\t\t\t\tPERMISSIONS\n\t\t\t\t\t\t\t\t\t493\n\t\t\t\t\t\t\t\t\tTYPE\n\t\t\t\t\t\t\t\t\t1\n\t\t\t\t\t\t\t\t\tUID\n\t\t\t\t\t\t\t\t\t0\n\t\t\t\t\t\t\t\t\n\t\t\t\t\t\t\t\t\n\t\t\t\t\t\t\t\t\tCHILDREN\n\t\t\t\t\t\t\t\t\t\n\t\t\t\t\t\t\t\t\tGID\n\t\t\t\t\t\t\t\t\t0\n\t\t\t\t\t\t\t\t\tPATH\n\t\t\t\t\t\t\t\t\tInternet Plug-Ins\n\t\t\t\t\t\t\t\t\tPATH_TYPE\n\t\t\t\t\t\t\t\t\t0\n\t\t\t\t\t\t\t\t\tPERMISSIONS\n\t\t\t\t\t\t\t\t\t493\n\t\t\t\t\t\t\t\t\tTYPE\n\t\t\t\t\t\t\t\t\t1\n\t\t\t\t\t\t\t\t\tUID\n\t\t\t\t\t\t\t\t\t0\n\t\t\t\t\t\t\t\t\n\t\t\t\t\t\t\t\t\n\t\t\t\t\t\t\t\t\tCHILDREN\n\t\t\t\t\t\t\t\t\t\n\t\t\t\t\t\t\t\t\tGID\n\t\t\t\t\t\t\t\t\t0\n\t\t\t\t\t\t\t\t\tPATH\n\t\t\t\t\t\t\t\t\tLaunchAgents\n\t\t\t\t\t\t\t\t\tPATH_TYPE\n\t\t\t\t\t\t\t\t\t0\n\t\t\t\t\t\t\t\t\tPERMISSIONS\n\t\t\t\t\t\t\t\t\t493\n\t\t\t\t\t\t\t\t\tTYPE\n\t\t\t\t\t\t\t\t\t1\n\t\t\t\t\t\t\t\t\tUID\n\t\t\t\t\t\t\t\t\t0\n\t\t\t\t\t\t\t\t\n\t\t\t\t\t\t\t\t\n\t\t\t\t\t\t\t\t\tCHILDREN\n\t\t\t\t\t\t\t\t\t\n\t\t\t\t\t\t\t\t\tGID\n\t\t\t\t\t\t\t\t\t0\n\t\t\t\t\t\t\t\t\tPATH\n\t\t\t\t\t\t\t\t\tLaunchDaemons\n\t\t\t\t\t\t\t\t\tPATH_TYPE\n\t\t\t\t\t\t\t\t\t0\n\t\t\t\t\t\t\t\t\tPERMISSIONS\n\t\t\t\t\t\t\t\t\t493\n\t\t\t\t\t\t\t\t\tTYPE\n\t\t\t\t\t\t\t\t\t1\n\t\t\t\t\t\t\t\t\tUID\n\t\t\t\t\t\t\t\t\t0\n\t\t\t\t\t\t\t\t\n\t\t\t\t\t\t\t\t\n\t\t\t\t\t\t\t\t\tCHILDREN\n\t\t\t\t\t\t\t\t\t\n\t\t\t\t\t\t\t\t\tGID\n\t\t\t\t\t\t\t\t\t0\n\t\t\t\t\t\t\t\t\tPATH\n\t\t\t\t\t\t\t\t\tPreferencePanes\n\t\t\t\t\t\t\t\t\tPATH_TYPE\n\t\t\t\t\t\t\t\t\t0\n\t\t\t\t\t\t\t\t\tPERMISSIONS\n\t\t\t\t\t\t\t\t\t493\n\t\t\t\t\t\t\t\t\tTYPE\n\t\t\t\t\t\t\t\t\t1\n\t\t\t\t\t\t\t\t\tUID\n\t\t\t\t\t\t\t\t\t0\n\t\t\t\t\t\t\t\t\n\t\t\t\t\t\t\t\t\n\t\t\t\t\t\t\t\t\tCHILDREN\n\t\t\t\t\t\t\t\t\t\n\t\t\t\t\t\t\t\t\tGID\n\t\t\t\t\t\t\t\t\t0\n\t\t\t\t\t\t\t\t\tPATH\n\t\t\t\t\t\t\t\t\tPreferences\n\t\t\t\t\t\t\t\t\tPATH_TYPE\n\t\t\t\t\t\t\t\t\t0\n\t\t\t\t\t\t\t\t\tPERMISSIONS\n\t\t\t\t\t\t\t\t\t493\n\t\t\t\t\t\t\t\t\tTYPE\n\t\t\t\t\t\t\t\t\t1\n\t\t\t\t\t\t\t\t\tUID\n\t\t\t\t\t\t\t\t\t0\n\t\t\t\t\t\t\t\t\n\t\t\t\t\t\t\t\t\n\t\t\t\t\t\t\t\t\tCHILDREN\n\t\t\t\t\t\t\t\t\t\n\t\t\t\t\t\t\t\t\tGID\n\t\t\t\t\t\t\t\t\t80\n\t\t\t\t\t\t\t\t\tPATH\n\t\t\t\t\t\t\t\t\tPrinters\n\t\t\t\t\t\t\t\t\tPATH_TYPE\n\t\t\t\t\t\t\t\t\t0\n\t\t\t\t\t\t\t\t\tPERMISSIONS\n\t\t\t\t\t\t\t\t\t493\n\t\t\t\t\t\t\t\t\tTYPE\n\t\t\t\t\t\t\t\t\t1\n\t\t\t\t\t\t\t\t\tUID\n\t\t\t\t\t\t\t\t\t0\n\t\t\t\t\t\t\t\t\n\t\t\t\t\t\t\t\t\n\t\t\t\t\t\t\t\t\tCHILDREN\n\t\t\t\t\t\t\t\t\t\n\t\t\t\t\t\t\t\t\tGID\n\t\t\t\t\t\t\t\t\t0\n\t\t\t\t\t\t\t\t\tPATH\n\t\t\t\t\t\t\t\t\tPrivilegedHelperTools\n\t\t\t\t\t\t\t\t\tPATH_TYPE\n\t\t\t\t\t\t\t\t\t0\n\t\t\t\t\t\t\t\t\tPERMISSIONS\n\t\t\t\t\t\t\t\t\t1005\n\t\t\t\t\t\t\t\t\tTYPE\n\t\t\t\t\t\t\t\t\t1\n\t\t\t\t\t\t\t\t\tUID\n\t\t\t\t\t\t\t\t\t0\n\t\t\t\t\t\t\t\t\n\t\t\t\t\t\t\t\t\n\t\t\t\t\t\t\t\t\tCHILDREN\n\t\t\t\t\t\t\t\t\t\n\t\t\t\t\t\t\t\t\tGID\n\t\t\t\t\t\t\t\t\t0\n\t\t\t\t\t\t\t\t\tPATH\n\t\t\t\t\t\t\t\t\tQuickLook\n\t\t\t\t\t\t\t\t\tPATH_TYPE\n\t\t\t\t\t\t\t\t\t0\n\t\t\t\t\t\t\t\t\tPERMISSIONS\n\t\t\t\t\t\t\t\t\t493\n\t\t\t\t\t\t\t\t\tTYPE\n\t\t\t\t\t\t\t\t\t1\n\t\t\t\t\t\t\t\t\tUID\n\t\t\t\t\t\t\t\t\t0\n\t\t\t\t\t\t\t\t\n\t\t\t\t\t\t\t\t\n\t\t\t\t\t\t\t\t\tCHILDREN\n\t\t\t\t\t\t\t\t\t\n\t\t\t\t\t\t\t\t\tGID\n\t\t\t\t\t\t\t\t\t0\n\t\t\t\t\t\t\t\t\tPATH\n\t\t\t\t\t\t\t\t\tQuickTime\n\t\t\t\t\t\t\t\t\tPATH_TYPE\n\t\t\t\t\t\t\t\t\t0\n\t\t\t\t\t\t\t\t\tPERMISSIONS\n\t\t\t\t\t\t\t\t\t493\n\t\t\t\t\t\t\t\t\tTYPE\n\t\t\t\t\t\t\t\t\t1\n\t\t\t\t\t\t\t\t\tUID\n\t\t\t\t\t\t\t\t\t0\n\t\t\t\t\t\t\t\t\n\t\t\t\t\t\t\t\t\n\t\t\t\t\t\t\t\t\tCHILDREN\n\t\t\t\t\t\t\t\t\t\n\t\t\t\t\t\t\t\t\tGID\n\t\t\t\t\t\t\t\t\t0\n\t\t\t\t\t\t\t\t\tPATH\n\t\t\t\t\t\t\t\t\tScreen Savers\n\t\t\t\t\t\t\t\t\tPATH_TYPE\n\t\t\t\t\t\t\t\t\t0\n\t\t\t\t\t\t\t\t\tPERMISSIONS\n\t\t\t\t\t\t\t\t\t493\n\t\t\t\t\t\t\t\t\tTYPE\n\t\t\t\t\t\t\t\t\t1\n\t\t\t\t\t\t\t\t\tUID\n\t\t\t\t\t\t\t\t\t0\n\t\t\t\t\t\t\t\t\n\t\t\t\t\t\t\t\t\n\t\t\t\t\t\t\t\t\tCHILDREN\n\t\t\t\t\t\t\t\t\t\n\t\t\t\t\t\t\t\t\tGID\n\t\t\t\t\t\t\t\t\t0\n\t\t\t\t\t\t\t\t\tPATH\n\t\t\t\t\t\t\t\t\tScripts\n\t\t\t\t\t\t\t\t\tPATH_TYPE\n\t\t\t\t\t\t\t\t\t0\n\t\t\t\t\t\t\t\t\tPERMISSIONS\n\t\t\t\t\t\t\t\t\t493\n\t\t\t\t\t\t\t\t\tTYPE\n\t\t\t\t\t\t\t\t\t1\n\t\t\t\t\t\t\t\t\tUID\n\t\t\t\t\t\t\t\t\t0\n\t\t\t\t\t\t\t\t\n\t\t\t\t\t\t\t\t\n\t\t\t\t\t\t\t\t\tCHILDREN\n\t\t\t\t\t\t\t\t\t\n\t\t\t\t\t\t\t\t\tGID\n\t\t\t\t\t\t\t\t\t0\n\t\t\t\t\t\t\t\t\tPATH\n\t\t\t\t\t\t\t\t\tServices\n\t\t\t\t\t\t\t\t\tPATH_TYPE\n\t\t\t\t\t\t\t\t\t0\n\t\t\t\t\t\t\t\t\tPERMISSIONS\n\t\t\t\t\t\t\t\t\t493\n\t\t\t\t\t\t\t\t\tTYPE\n\t\t\t\t\t\t\t\t\t1\n\t\t\t\t\t\t\t\t\tUID\n\t\t\t\t\t\t\t\t\t0\n\t\t\t\t\t\t\t\t\n\t\t\t\t\t\t\t\t\n\t\t\t\t\t\t\t\t\tCHILDREN\n\t\t\t\t\t\t\t\t\t\n\t\t\t\t\t\t\t\t\tGID\n\t\t\t\t\t\t\t\t\t0\n\t\t\t\t\t\t\t\t\tPATH\n\t\t\t\t\t\t\t\t\tWidgets\n\t\t\t\t\t\t\t\t\tPATH_TYPE\n\t\t\t\t\t\t\t\t\t0\n\t\t\t\t\t\t\t\t\tPERMISSIONS\n\t\t\t\t\t\t\t\t\t493\n\t\t\t\t\t\t\t\t\tTYPE\n\t\t\t\t\t\t\t\t\t1\n\t\t\t\t\t\t\t\t\tUID\n\t\t\t\t\t\t\t\t\t0\n\t\t\t\t\t\t\t\t\n\t\t\t\t\t\t\t\t\n\t\t\t\t\t\t\t\t\tCHILDREN\n\t\t\t\t\t\t\t\t\t\n\t\t\t\t\t\t\t\t\tGID\n\t\t\t\t\t\t\t\t\t0\n\t\t\t\t\t\t\t\t\tPATH\n\t\t\t\t\t\t\t\t\tExtensions\n\t\t\t\t\t\t\t\t\tPATH_TYPE\n\t\t\t\t\t\t\t\t\t0\n\t\t\t\t\t\t\t\t\tPERMISSIONS\n\t\t\t\t\t\t\t\t\t493\n\t\t\t\t\t\t\t\t\tTYPE\n\t\t\t\t\t\t\t\t\t1\n\t\t\t\t\t\t\t\t\tUID\n\t\t\t\t\t\t\t\t\t0\n\t\t\t\t\t\t\t\t\n\t\t\t\t\t\t\t\n\t\t\t\t\t\t\tGID\n\t\t\t\t\t\t\t0\n\t\t\t\t\t\t\tPATH\n\t\t\t\t\t\t\tLibrary\n\t\t\t\t\t\t\tPATH_TYPE\n\t\t\t\t\t\t\t0\n\t\t\t\t\t\t\tPERMISSIONS\n\t\t\t\t\t\t\t493\n\t\t\t\t\t\t\tTYPE\n\t\t\t\t\t\t\t1\n\t\t\t\t\t\t\tUID\n\t\t\t\t\t\t\t0\n\t\t\t\t\t\t\n\t\t\t\t\t\t\n\t\t\t\t\t\t\tCHILDREN\n\t\t\t\t\t\t\t\n\t\t\t\t\t\t\t\t\n\t\t\t\t\t\t\t\t\tCHILDREN\n\t\t\t\t\t\t\t\t\t\n\t\t\t\t\t\t\t\t\t\t\n\t\t\t\t\t\t\t\t\t\t\tCHILDREN\n\t\t\t\t\t\t\t\t\t\t\t\n\t\t\t\t\t\t\t\t\t\t\tGID\n\t\t\t\t\t\t\t\t\t\t\t0\n\t\t\t\t\t\t\t\t\t\t\tPATH\n\t\t\t\t\t\t\t\t\t\t\tExtensions\n\t\t\t\t\t\t\t\t\t\t\tPATH_TYPE\n\t\t\t\t\t\t\t\t\t\t\t0\n\t\t\t\t\t\t\t\t\t\t\tPERMISSIONS\n\t\t\t\t\t\t\t\t\t\t\t493\n\t\t\t\t\t\t\t\t\t\t\tTYPE\n\t\t\t\t\t\t\t\t\t\t\t1\n\t\t\t\t\t\t\t\t\t\t\tUID\n\t\t\t\t\t\t\t\t\t\t\t0\n\t\t\t\t\t\t\t\t\t\t\n\t\t\t\t\t\t\t\t\t\n\t\t\t\t\t\t\t\t\tGID\n\t\t\t\t\t\t\t\t\t0\n\t\t\t\t\t\t\t\t\tPATH\n\t\t\t\t\t\t\t\t\tLibrary\n\t\t\t\t\t\t\t\t\tPATH_TYPE\n\t\t\t\t\t\t\t\t\t0\n\t\t\t\t\t\t\t\t\tPERMISSIONS\n\t\t\t\t\t\t\t\t\t493\n\t\t\t\t\t\t\t\t\tTYPE\n\t\t\t\t\t\t\t\t\t1\n\t\t\t\t\t\t\t\t\tUID\n\t\t\t\t\t\t\t\t\t0\n\t\t\t\t\t\t\t\t\n\t\t\t\t\t\t\t\n\t\t\t\t\t\t\tGID\n\t\t\t\t\t\t\t0\n\t\t\t\t\t\t\tPATH\n\t\t\t\t\t\t\tSystem\n\t\t\t\t\t\t\tPATH_TYPE\n\t\t\t\t\t\t\t0\n\t\t\t\t\t\t\tPERMISSIONS\n\t\t\t\t\t\t\t493\n\t\t\t\t\t\t\tTYPE\n\t\t\t\t\t\t\t1\n\t\t\t\t\t\t\tUID\n\t\t\t\t\t\t\t0\n\t\t\t\t\t\t\n\t\t\t\t\t\t\n\t\t\t\t\t\t\tCHILDREN\n\t\t\t\t\t\t\t\n\t\t\t\t\t\t\t\t\n\t\t\t\t\t\t\t\t\tCHILDREN\n\t\t\t\t\t\t\t\t\t\n\t\t\t\t\t\t\t\t\tGID\n\t\t\t\t\t\t\t\t\t0\n\t\t\t\t\t\t\t\t\tPATH\n\t\t\t\t\t\t\t\t\tShared\n\t\t\t\t\t\t\t\t\tPATH_TYPE\n\t\t\t\t\t\t\t\t\t0\n\t\t\t\t\t\t\t\t\tPERMISSIONS\n\t\t\t\t\t\t\t\t\t1023\n\t\t\t\t\t\t\t\t\tTYPE\n\t\t\t\t\t\t\t\t\t1\n\t\t\t\t\t\t\t\t\tUID\n\t\t\t\t\t\t\t\t\t0\n\t\t\t\t\t\t\t\t\n\t\t\t\t\t\t\t\n\t\t\t\t\t\t\tGID\n\t\t\t\t\t\t\t80\n\t\t\t\t\t\t\tPATH\n\t\t\t\t\t\t\tUsers\n\t\t\t\t\t\t\tPATH_TYPE\n\t\t\t\t\t\t\t0\n\t\t\t\t\t\t\tPERMISSIONS\n\t\t\t\t\t\t\t493\n\t\t\t\t\t\t\tTYPE\n\t\t\t\t\t\t\t1\n\t\t\t\t\t\t\tUID\n\t\t\t\t\t\t\t0\n\t\t\t\t\t\t\n\t\t\t\t\t\n\t\t\t\t\tGID\n\t\t\t\t\t0\n\t\t\t\t\tPATH\n\t\t\t\t\t/\n\t\t\t\t\tPATH_TYPE\n\t\t\t\t\t0\n\t\t\t\t\tPERMISSIONS\n\t\t\t\t\t493\n\t\t\t\t\tTYPE\n\t\t\t\t\t1\n\t\t\t\t\tUID\n\t\t\t\t\t0\n\t\t\t\t\n\t\t\t\tPAYLOAD_TYPE\n\t\t\t\t0\n\t\t\t\tSHOW_INVISIBLE\n\t\t\t\t\n\t\t\t\tSPLIT_FORKS\n\t\t\t\t\n\t\t\t\tTREAT_MISSING_FILES_AS_WARNING\n\t\t\t\t\n\t\t\t\tVERSION\n\t\t\t\t5\n\t\t\t\n\t\t\tPACKAGE_SCRIPTS\n\t\t\t\n\t\t\t\tRESOURCES\n\t\t\t\t\n\t\t\t\n\t\t\tPACKAGE_SETTINGS\n\t\t\t\n\t\t\t\tAUTHENTICATION\n\t\t\t\t1\n\t\t\t\tCONCLUSION_ACTION\n\t\t\t\t0\n\t\t\t\tFOLLOW_SYMBOLIC_LINKS\n\t\t\t\t\n\t\t\t\tIDENTIFIER\n\t\t\t\tedu.nyu.vida.reprounzip-qt\n\t\t\t\tLOCATION\n\t\t\t\t0\n\t\t\t\tNAME\n\t\t\t\tQt Interface\n\t\t\t\tOVERWRITE_PERMISSIONS\n\t\t\t\t\n\t\t\t\tPAYLOAD_SIZE\n\t\t\t\t-1\n\t\t\t\tREFERENCE_PATH\n\t\t\t\t\n\t\t\t\tRELOCATABLE\n\t\t\t\t\n\t\t\t\tUSE_HFS+_COMPRESSION\n\t\t\t\t\n\t\t\t\tVERSION\n\t\t\t\t1.2\n\t\t\t\n\t\t\tTYPE\n\t\t\t0\n\t\t\tUUID\n\t\t\t5650AD76-E68A-44EF-86A4-F8A1511A5ED4\n\t\t\n\t\n\tPROJECT\n\t\n\t\tPROJECT_COMMENTS\n\t\t\n\t\t\tNOTES\n\t\t\t\n\t\t\tPCFET0NUWVBFIGh0bWwgUFVCTElDICItLy9XM0MvL0RURCBIVE1M\n\t\t\tIDQuMDEvL0VOIiAiaHR0cDovL3d3dy53My5vcmcvVFIvaHRtbDQv\n\t\t\tc3RyaWN0LmR0ZCI+CjxodG1sPgo8aGVhZD4KPG1ldGEgaHR0cC1l\n\t\t\tcXVpdj0iQ29udGVudC1UeXBlIiBjb250ZW50PSJ0ZXh0L2h0bWw7\n\t\t\tIGNoYXJzZXQ9VVRGLTgiPgo8bWV0YSBodHRwLWVxdWl2PSJDb250\n\t\t\tZW50LVN0eWxlLVR5cGUiIGNvbnRlbnQ9InRleHQvY3NzIj4KPHRp\n\t\t\tdGxlPjwvdGl0bGU+CjxtZXRhIG5hbWU9IkdlbmVyYXRvciIgY29u\n\t\t\tdGVudD0iQ29jb2EgSFRNTCBXcml0ZXIiPgo8bWV0YSBuYW1lPSJD\n\t\t\tb2NvYVZlcnNpb24iIGNvbnRlbnQ9IjEzNDguMTciPgo8c3R5bGUg\n\t\t\tdHlwZT0idGV4dC9jc3MiPgo8L3N0eWxlPgo8L2hlYWQ+Cjxib2R5\n\t\t\tPgo8L2JvZHk+CjwvaHRtbD4K\n\t\t\t\n\t\t\n\t\tPROJECT_PRESENTATION\n\t\t\n\t\t\tBACKGROUND\n\t\t\t\n\t\t\tINSTALLATION TYPE\n\t\t\t\n\t\t\t\tHIERARCHIES\n\t\t\t\t\n\t\t\t\t\tINSTALLER\n\t\t\t\t\t\n\t\t\t\t\t\tLIST\n\t\t\t\t\t\t\n\t\t\t\t\t\t\t\n\t\t\t\t\t\t\t\tDESCRIPTION\n\t\t\t\t\t\t\t\t\n\t\t\t\t\t\t\t\t\t\n\t\t\t\t\t\t\t\t\t\tLANGUAGE\n\t\t\t\t\t\t\t\t\t\tEnglish\n\t\t\t\t\t\t\t\t\t\tVALUE\n\t\t\t\t\t\t\t\t\t\tPython 3.9, ReproUnzip and plugins.\n\t\t\t\t\t\t\t\t\t\n\t\t\t\t\t\t\t\t\n\t\t\t\t\t\t\t\tOPTIONS\n\t\t\t\t\t\t\t\t\n\t\t\t\t\t\t\t\t\tHIDDEN\n\t\t\t\t\t\t\t\t\t\n\t\t\t\t\t\t\t\t\tSTATE\n\t\t\t\t\t\t\t\t\t1\n\t\t\t\t\t\t\t\t\n\t\t\t\t\t\t\t\tPACKAGE_UUID\n\t\t\t\t\t\t\t\tD5E9C540-1489-4E0E-A914-B81FB6DCED9C\n\t\t\t\t\t\t\t\tREQUIREMENTS\n\t\t\t\t\t\t\t\t\n\t\t\t\t\t\t\t\t\t\n\t\t\t\t\t\t\t\t\t\tBEHAVIOR\n\t\t\t\t\t\t\t\t\t\t1\n\t\t\t\t\t\t\t\t\t\tDICTIONARY\n\t\t\t\t\t\t\t\t\t\t\n\t\t\t\t\t\t\t\t\t\t\tIC_REQUIREMENT_CPU_ARCHITECTURE_FAMILY\n\t\t\t\t\t\t\t\t\t\t\t2\n\t\t\t\t\t\t\t\t\t\t\tIC_REQUIREMENT_CPU_INTEL_ARCHITECTURE_TYPE\n\t\t\t\t\t\t\t\t\t\t\t2\n\t\t\t\t\t\t\t\t\t\t\tIC_REQUIREMENT_CPU_MINIMUM_CPU_CORES_COUNT\n\t\t\t\t\t\t\t\t\t\t\t1\n\t\t\t\t\t\t\t\t\t\t\tIC_REQUIREMENT_CPU_MINIMUM_FREQUENCY\n\t\t\t\t\t\t\t\t\t\t\t866666\n\t\t\t\t\t\t\t\t\t\t\tIC_REQUIREMENT_CPU_POWERPC_ARCHITECTURE_TYPE\n\t\t\t\t\t\t\t\t\t\t\t0\n\t\t\t\t\t\t\t\t\t\t\n\t\t\t\t\t\t\t\t\t\tIDENTIFIER\n\t\t\t\t\t\t\t\t\t\tfr.whitebox.Packages.requirement.cpu\n\t\t\t\t\t\t\t\t\t\tMESSAGE\n\t\t\t\t\t\t\t\t\t\t\n\t\t\t\t\t\t\t\t\t\tNAME\n\t\t\t\t\t\t\t\t\t\tProcessor\n\t\t\t\t\t\t\t\t\t\tSTATE\n\t\t\t\t\t\t\t\t\t\t\n\t\t\t\t\t\t\t\t\t\n\t\t\t\t\t\t\t\t\n\t\t\t\t\t\t\t\tTITLE\n\t\t\t\t\t\t\t\t\n\t\t\t\t\t\t\t\tTOOLTIP\n\t\t\t\t\t\t\t\t\n\t\t\t\t\t\t\t\tTYPE\n\t\t\t\t\t\t\t\t0\n\t\t\t\t\t\t\t\tUUID\n\t\t\t\t\t\t\t\tE02412A5-D04B-498D-A459-8F90160AF7BB\n\t\t\t\t\t\t\t\n\t\t\t\t\t\t\t\n\t\t\t\t\t\t\t\tDESCRIPTION\n\t\t\t\t\t\t\t\t\n\t\t\t\t\t\t\t\t\t\n\t\t\t\t\t\t\t\t\t\tLANGUAGE\n\t\t\t\t\t\t\t\t\t\tEnglish\n\t\t\t\t\t\t\t\t\t\tVALUE\n\t\t\t\t\t\t\t\t\t\tReproUnzip.app, graphical interface for reprounzip with Qt.\n\t\t\t\t\t\t\t\t\t\n\t\t\t\t\t\t\t\t\n\t\t\t\t\t\t\t\tOPTIONS\n\t\t\t\t\t\t\t\t\n\t\t\t\t\t\t\t\t\tHIDDEN\n\t\t\t\t\t\t\t\t\t\n\t\t\t\t\t\t\t\t\tSTATE\n\t\t\t\t\t\t\t\t\t1\n\t\t\t\t\t\t\t\t\n\t\t\t\t\t\t\t\tPACKAGE_UUID\n\t\t\t\t\t\t\t\t5650AD76-E68A-44EF-86A4-F8A1511A5ED4\n\t\t\t\t\t\t\t\tREQUIREMENTS\n\t\t\t\t\t\t\t\t\n\t\t\t\t\t\t\t\t\t\n\t\t\t\t\t\t\t\t\t\tBEHAVIOR\n\t\t\t\t\t\t\t\t\t\t1\n\t\t\t\t\t\t\t\t\t\tDICTIONARY\n\t\t\t\t\t\t\t\t\t\t\n\t\t\t\t\t\t\t\t\t\t\tIC_REQUIREMENT_CPU_ARCHITECTURE_FAMILY\n\t\t\t\t\t\t\t\t\t\t\t2\n\t\t\t\t\t\t\t\t\t\t\tIC_REQUIREMENT_CPU_INTEL_ARCHITECTURE_TYPE\n\t\t\t\t\t\t\t\t\t\t\t2\n\t\t\t\t\t\t\t\t\t\t\tIC_REQUIREMENT_CPU_MINIMUM_CPU_CORES_COUNT\n\t\t\t\t\t\t\t\t\t\t\t1\n\t\t\t\t\t\t\t\t\t\t\tIC_REQUIREMENT_CPU_MINIMUM_FREQUENCY\n\t\t\t\t\t\t\t\t\t\t\t866666\n\t\t\t\t\t\t\t\t\t\t\tIC_REQUIREMENT_CPU_POWERPC_ARCHITECTURE_TYPE\n\t\t\t\t\t\t\t\t\t\t\t0\n\t\t\t\t\t\t\t\t\t\t\n\t\t\t\t\t\t\t\t\t\tIDENTIFIER\n\t\t\t\t\t\t\t\t\t\tfr.whitebox.Packages.requirement.cpu\n\t\t\t\t\t\t\t\t\t\tMESSAGE\n\t\t\t\t\t\t\t\t\t\t\n\t\t\t\t\t\t\t\t\t\tNAME\n\t\t\t\t\t\t\t\t\t\tProcessor\n\t\t\t\t\t\t\t\t\t\tSTATE\n\t\t\t\t\t\t\t\t\t\t\n\t\t\t\t\t\t\t\t\t\n\t\t\t\t\t\t\t\t\n\t\t\t\t\t\t\t\tTITLE\n\t\t\t\t\t\t\t\t\n\t\t\t\t\t\t\t\tTOOLTIP\n\t\t\t\t\t\t\t\t\n\t\t\t\t\t\t\t\tTYPE\n\t\t\t\t\t\t\t\t0\n\t\t\t\t\t\t\t\tUUID\n\t\t\t\t\t\t\t\tAE811B09-E193-439B-B49C-8E63D3E026FD\n\t\t\t\t\t\t\t\n\t\t\t\t\t\t\n\t\t\t\t\t\tREMOVED\n\t\t\t\t\t\t\n\t\t\t\t\t\n\t\t\t\t\n\t\t\t\tMODE\n\t\t\t\t1\n\t\t\t\n\t\t\tINSTALLATION_STEPS\n\t\t\t\n\t\t\t\t\n\t\t\t\t\tICPRESENTATION_CHAPTER_VIEW_CONTROLLER_CLASS\n\t\t\t\t\tICPresentationViewIntroductionController\n\t\t\t\t\tINSTALLER_PLUGIN\n\t\t\t\t\tIntroduction\n\t\t\t\t\tLIST_TITLE_KEY\n\t\t\t\t\tInstallerSectionTitle\n\t\t\t\t\n\t\t\t\t\n\t\t\t\t\tICPRESENTATION_CHAPTER_VIEW_CONTROLLER_CLASS\n\t\t\t\t\tICPresentationViewReadMeController\n\t\t\t\t\tINSTALLER_PLUGIN\n\t\t\t\t\tReadMe\n\t\t\t\t\tLIST_TITLE_KEY\n\t\t\t\t\tInstallerSectionTitle\n\t\t\t\t\n\t\t\t\t\n\t\t\t\t\tICPRESENTATION_CHAPTER_VIEW_CONTROLLER_CLASS\n\t\t\t\t\tICPresentationViewLicenseController\n\t\t\t\t\tINSTALLER_PLUGIN\n\t\t\t\t\tLicense\n\t\t\t\t\tLIST_TITLE_KEY\n\t\t\t\t\tInstallerSectionTitle\n\t\t\t\t\n\t\t\t\t\n\t\t\t\t\tICPRESENTATION_CHAPTER_VIEW_CONTROLLER_CLASS\n\t\t\t\t\tICPresentationViewDestinationSelectController\n\t\t\t\t\tINSTALLER_PLUGIN\n\t\t\t\t\tTargetSelect\n\t\t\t\t\tLIST_TITLE_KEY\n\t\t\t\t\tInstallerSectionTitle\n\t\t\t\t\n\t\t\t\t\n\t\t\t\t\tICPRESENTATION_CHAPTER_VIEW_CONTROLLER_CLASS\n\t\t\t\t\tICPresentationViewInstallationTypeController\n\t\t\t\t\tINSTALLER_PLUGIN\n\t\t\t\t\tPackageSelection\n\t\t\t\t\tLIST_TITLE_KEY\n\t\t\t\t\tInstallerSectionTitle\n\t\t\t\t\n\t\t\t\t\n\t\t\t\t\tICPRESENTATION_CHAPTER_VIEW_CONTROLLER_CLASS\n\t\t\t\t\tICPresentationViewInstallationController\n\t\t\t\t\tINSTALLER_PLUGIN\n\t\t\t\t\tInstall\n\t\t\t\t\tLIST_TITLE_KEY\n\t\t\t\t\tInstallerSectionTitle\n\t\t\t\t\n\t\t\t\t\n\t\t\t\t\tICPRESENTATION_CHAPTER_VIEW_CONTROLLER_CLASS\n\t\t\t\t\tICPresentationViewSummaryController\n\t\t\t\t\tINSTALLER_PLUGIN\n\t\t\t\t\tSummary\n\t\t\t\t\tLIST_TITLE_KEY\n\t\t\t\t\tInstallerSectionTitle\n\t\t\t\t\n\t\t\t\n\t\t\tINTRODUCTION\n\t\t\t\n\t\t\t\tLOCALIZATIONS\n\t\t\t\t\n\t\t\t\n\t\t\tLICENSE\n\t\t\t\n\t\t\t\tLOCALIZATIONS\n\t\t\t\t\n\t\t\t\t\t\n\t\t\t\t\t\tLANGUAGE\n\t\t\t\t\t\tEnglish\n\t\t\t\t\t\tVALUE\n\t\t\t\t\t\t\n\t\t\t\t\t\t\tPATH\n\t\t\t\t\t\t\t../../LICENSE.txt\n\t\t\t\t\t\t\tPATH_TYPE\n\t\t\t\t\t\t\t1\n\t\t\t\t\t\t\n\t\t\t\t\t\n\t\t\t\t\n\t\t\t\tMODE\n\t\t\t\t0\n\t\t\t\n\t\t\tREADME\n\t\t\t\n\t\t\t\tLOCALIZATIONS\n\t\t\t\t\n\t\t\t\t\t\n\t\t\t\t\t\tLANGUAGE\n\t\t\t\t\t\tEnglish\n\t\t\t\t\t\tVALUE\n\t\t\t\t\t\t\n\t\t\t\t\t\t\tPATH\n\t\t\t\t\t\t\tREADME.txt\n\t\t\t\t\t\t\tPATH_TYPE\n\t\t\t\t\t\t\t1\n\t\t\t\t\t\t\n\t\t\t\t\t\n\t\t\t\t\n\t\t\t\n\t\t\tSUMMARY\n\t\t\t\n\t\t\t\tLOCALIZATIONS\n\t\t\t\t\n\t\t\t\n\t\t\tTITLE\n\t\t\t\n\t\t\t\tLOCALIZATIONS\n\t\t\t\t\n\t\t\t\t\t\n\t\t\t\t\t\tLANGUAGE\n\t\t\t\t\t\tEnglish\n\t\t\t\t\t\tVALUE\n\t\t\t\t\t\tReproUnzip\n\t\t\t\t\t\n\t\t\t\t\n\t\t\t\n\t\t\n\t\tPROJECT_REQUIREMENTS\n\t\t\n\t\t\tLIST\n\t\t\t\n\t\t\tRESOURCES\n\t\t\t\n\t\t\tROOT_VOLUME_ONLY\n\t\t\t\n\t\t\n\t\tPROJECT_SETTINGS\n\t\t\n\t\t\tBUILD_FORMAT\n\t\t\t0\n\t\t\tBUILD_PATH\n\t\t\t\n\t\t\t\tPATH\n\t\t\t\tbuild\n\t\t\t\tPATH_TYPE\n\t\t\t\t1\n\t\t\t\n\t\t\tEXCLUDED_FILES\n\t\t\t\n\t\t\t\t\n\t\t\t\t\tPATTERNS_ARRAY\n\t\t\t\t\t\n\t\t\t\t\t\t\n\t\t\t\t\t\t\tREGULAR_EXPRESSION\n\t\t\t\t\t\t\t\n\t\t\t\t\t\t\tSTRING\n\t\t\t\t\t\t\t.DS_Store\n\t\t\t\t\t\t\tTYPE\n\t\t\t\t\t\t\t0\n\t\t\t\t\t\t\n\t\t\t\t\t\n\t\t\t\t\tPROTECTED\n\t\t\t\t\t\n\t\t\t\t\tPROXY_NAME\n\t\t\t\t\tRemove .DS_Store files\n\t\t\t\t\tPROXY_TOOLTIP\n\t\t\t\t\tRemove \".DS_Store\" files created by the Finder.\n\t\t\t\t\tSTATE\n\t\t\t\t\t\n\t\t\t\t\n\t\t\t\t\n\t\t\t\t\tPATTERNS_ARRAY\n\t\t\t\t\t\n\t\t\t\t\t\t\n\t\t\t\t\t\t\tREGULAR_EXPRESSION\n\t\t\t\t\t\t\t\n\t\t\t\t\t\t\tSTRING\n\t\t\t\t\t\t\t.pbdevelopment\n\t\t\t\t\t\t\tTYPE\n\t\t\t\t\t\t\t0\n\t\t\t\t\t\t\n\t\t\t\t\t\n\t\t\t\t\tPROTECTED\n\t\t\t\t\t\n\t\t\t\t\tPROXY_NAME\n\t\t\t\t\tRemove .pbdevelopment files\n\t\t\t\t\tPROXY_TOOLTIP\n\t\t\t\t\tRemove \".pbdevelopment\" files created by ProjectBuilder or Xcode.\n\t\t\t\t\tSTATE\n\t\t\t\t\t\n\t\t\t\t\n\t\t\t\t\n\t\t\t\t\tPATTERNS_ARRAY\n\t\t\t\t\t\n\t\t\t\t\t\t\n\t\t\t\t\t\t\tREGULAR_EXPRESSION\n\t\t\t\t\t\t\t\n\t\t\t\t\t\t\tSTRING\n\t\t\t\t\t\t\tCVS\n\t\t\t\t\t\t\tTYPE\n\t\t\t\t\t\t\t1\n\t\t\t\t\t\t\n\t\t\t\t\t\t\n\t\t\t\t\t\t\tREGULAR_EXPRESSION\n\t\t\t\t\t\t\t\n\t\t\t\t\t\t\tSTRING\n\t\t\t\t\t\t\t.cvsignore\n\t\t\t\t\t\t\tTYPE\n\t\t\t\t\t\t\t0\n\t\t\t\t\t\t\n\t\t\t\t\t\t\n\t\t\t\t\t\t\tREGULAR_EXPRESSION\n\t\t\t\t\t\t\t\n\t\t\t\t\t\t\tSTRING\n\t\t\t\t\t\t\t.cvspass\n\t\t\t\t\t\t\tTYPE\n\t\t\t\t\t\t\t0\n\t\t\t\t\t\t\n\t\t\t\t\t\t\n\t\t\t\t\t\t\tREGULAR_EXPRESSION\n\t\t\t\t\t\t\t\n\t\t\t\t\t\t\tSTRING\n\t\t\t\t\t\t\t.svn\n\t\t\t\t\t\t\tTYPE\n\t\t\t\t\t\t\t1\n\t\t\t\t\t\t\n\t\t\t\t\t\t\n\t\t\t\t\t\t\tREGULAR_EXPRESSION\n\t\t\t\t\t\t\t\n\t\t\t\t\t\t\tSTRING\n\t\t\t\t\t\t\t.git\n\t\t\t\t\t\t\tTYPE\n\t\t\t\t\t\t\t1\n\t\t\t\t\t\t\n\t\t\t\t\t\t\n\t\t\t\t\t\t\tREGULAR_EXPRESSION\n\t\t\t\t\t\t\t\n\t\t\t\t\t\t\tSTRING\n\t\t\t\t\t\t\t.gitignore\n\t\t\t\t\t\t\tTYPE\n\t\t\t\t\t\t\t0\n\t\t\t\t\t\t\n\t\t\t\t\t\n\t\t\t\t\tPROTECTED\n\t\t\t\t\t\n\t\t\t\t\tPROXY_NAME\n\t\t\t\t\tRemove SCM metadata\n\t\t\t\t\tPROXY_TOOLTIP\n\t\t\t\t\tRemove helper files and folders used by the CVS, SVN or Git Source Code Management systems.\n\t\t\t\t\tSTATE\n\t\t\t\t\t\n\t\t\t\t\n\t\t\t\t\n\t\t\t\t\tPATTERNS_ARRAY\n\t\t\t\t\t\n\t\t\t\t\t\t\n\t\t\t\t\t\t\tREGULAR_EXPRESSION\n\t\t\t\t\t\t\t\n\t\t\t\t\t\t\tSTRING\n\t\t\t\t\t\t\tclasses.nib\n\t\t\t\t\t\t\tTYPE\n\t\t\t\t\t\t\t0\n\t\t\t\t\t\t\n\t\t\t\t\t\t\n\t\t\t\t\t\t\tREGULAR_EXPRESSION\n\t\t\t\t\t\t\t\n\t\t\t\t\t\t\tSTRING\n\t\t\t\t\t\t\tdesignable.db\n\t\t\t\t\t\t\tTYPE\n\t\t\t\t\t\t\t0\n\t\t\t\t\t\t\n\t\t\t\t\t\t\n\t\t\t\t\t\t\tREGULAR_EXPRESSION\n\t\t\t\t\t\t\t\n\t\t\t\t\t\t\tSTRING\n\t\t\t\t\t\t\tinfo.nib\n\t\t\t\t\t\t\tTYPE\n\t\t\t\t\t\t\t0\n\t\t\t\t\t\t\n\t\t\t\t\t\n\t\t\t\t\tPROTECTED\n\t\t\t\t\t\n\t\t\t\t\tPROXY_NAME\n\t\t\t\t\tOptimize nib files\n\t\t\t\t\tPROXY_TOOLTIP\n\t\t\t\t\tRemove \"classes.nib\", \"info.nib\" and \"designable.nib\" files within .nib bundles.\n\t\t\t\t\tSTATE\n\t\t\t\t\t\n\t\t\t\t\n\t\t\t\t\n\t\t\t\t\tPATTERNS_ARRAY\n\t\t\t\t\t\n\t\t\t\t\t\t\n\t\t\t\t\t\t\tREGULAR_EXPRESSION\n\t\t\t\t\t\t\t\n\t\t\t\t\t\t\tSTRING\n\t\t\t\t\t\t\tResources Disabled\n\t\t\t\t\t\t\tTYPE\n\t\t\t\t\t\t\t1\n\t\t\t\t\t\t\n\t\t\t\t\t\n\t\t\t\t\tPROTECTED\n\t\t\t\t\t\n\t\t\t\t\tPROXY_NAME\n\t\t\t\t\tRemove Resources Disabled folders\n\t\t\t\t\tPROXY_TOOLTIP\n\t\t\t\t\tRemove \"Resources Disabled\" folders.\n\t\t\t\t\tSTATE\n\t\t\t\t\t\n\t\t\t\t\n\t\t\t\t\n\t\t\t\t\tSEPARATOR\n\t\t\t\t\t\n\t\t\t\t\n\t\t\t\n\t\t\tNAME\n\t\t\tReproUnzip\n\t\t\tPAYLOAD_ONLY\n\t\t\t\n\t\t\tTREAT_MISSING_PRESENTATION_DOCUMENTS_AS_WARNING\n\t\t\t\n\t\t\n\t\n\tTYPE\n\t0\n\tVERSION\n\t2\n\n\n" }, { "path": "scripts/macos/app-shim/ReproUnzip_app/Contents/Info.plist", "content": "\n\n\n\n\tCFBundleDevelopmentRegion\n\tEnglish\n\tCFBundleDisplayName\n\tReproUnzip\n\tCFBundleDocumentTypes\n\t\n\t\t\n\t\t\tCFBundleTypeExtensions\n\t\t\t\n\t\t\t\trpz\n\t\t\t\n\t\t\tCFBundleTypeName\n\t\t\tReproZip Package\n\t\t\tCFBundleTypeIconFile\n\t\t\ticon\n\t\t\tCFBundleTypeOSTypes\n\t\t\t\n\t\t\t\t????\n\t\t\t\n\t\t\tCFBundleTypeRole\n\t\t\tViewer\n\t\t\n\t\n\tCFBundleExecutable\n\treprounzip-qt\n\tCFBundleIconFile\n\ticon\n\tCFBundleIdentifier\n\tedu.nyu.engineering.vida.reprounzip\n\tCFBundleInfoDictionaryVersion\n\t6.0\n\tCFBundleName\n\treprounzip-qt\n\tCFBundlePackageType\n\tAPPL\n\tCFBundleShortVersionString\n\t1.0\n\tCFBundleVersion\n\t1.0\n\tLSHasLocalizedDisplayName\n\t\n\n\n" }, { "path": "scripts/macos/app-shim/reprounzip-qt.c", "content": "#include \n#include \n#include \n\nint main(int argc, char **argv) {\n char **args = malloc(sizeof(char*) * (argc + 1));\n size_t i;\n for(i = 1; i < argc; ++i) {\n args[i] = argv[i];\n }\n args[argc] = 0;\n execv(\"/opt/reprounzip/reprounzip-qt\", args);\n perror(\"execv failed\");\n return 1;\n}\n" }, { "path": "scripts/macos/app-shim/reprounzip-qt.debug.c", "content": "#include \n#include \n#include \n\n/* This version of reprounzip-qt runs from a virtualenv, useful for local\n * development on reprounzip-qt while still starting it as a Mac app bundle */\n\nint main(int argc, char **argv) {\n char **args = malloc(sizeof(char*) * (argc + 4));\n size_t i;\n args[0] = \"/bin/sh\";\n args[1] = \"-c\";\n args[2] = \". /Users/remram/Documents/programming/_venvs/reprozip/bin/activate && reprounzip-qt \\\"$@\\\"\";\n args[3] = \"-\";\n for(i = 1; i < argc; ++i) {\n args[i + 3] = argv[i];\n }\n args[argc + 3] = 0;\n execv(\"/bin/sh\", args);\n perror(\"execv failed\");\n return 1;\n}\n" }, { "path": "scripts/macos/instructions.txt", "content": "Reset PATH to not include macports, etc\nexport DYLD_FRAMEWORK_PATH=/opt/reprounzip/Frameworks\nexport CFLAGS=\"-mmacosx-version-min=10.8 -isysroot /Developer/SDKs/MacOSX10.8.sdk\"\nexport LDFLAGS=\"-mmacosx-version-min=10.8 -isysroot /Developer/SDKs/MacOSX10.8.sdk\"\nexport MACOS_DEPLOYMENT_TARGET=10.8\nBuild Python with --prefix=/opt/reprounzip/python3\nCopy reprounzip, reprounzip-qt, and reprozip-jupyter to /opt/reprounzip\nInstall pip with get-pip.py\nInstall reprounzip, reprounzip-docker, reprounzip-vagrant, reprounzip-vistrails\nInstall sip and PyQt5\nInstall reprounzip-qt\nBuild shim application:\n gcc -mmacosx-version-min=10.8 -isysroot /Developer/SDKs/MacOSX10.8.sdk \\\n reprounzip-qt.c -o ReproUnzip.app/Contents/MacOS/reprounzip-qt\nPut QtCore.framework and QtGui.framework in /opt/reprounzip/Frameworks\nBuild installer using ReproUnzip.pkgproj\n Packages application: http://s.sudre.free.fr/Software/Packages/about.html\n" }, { "path": "scripts/macos/reprounzip", "content": "#!/bin/sh\n\nexec \"$(dirname \"$0\")/python3/bin/reprounzip\" \"$@\"\n" }, { "path": "scripts/macos/reprounzip-qt", "content": "#!/bin/sh\n\nDYLD_FRAMEWORK_PATH=/opt/reprounzip/Frameworks; export DYLD_FRAMEWORK_PATH\nPATH=\"/opt/reprounzip:$PATH\"\nexec \"$(dirname \"$0\")/python3/bin/reprounzip-qt\" \"$@\"\n" }, { "path": "scripts/macos/reprozip-jupyter", "content": "#!/bin/sh\n\nexec \"$(dirname \"$0\")/python3/bin/reprozip-jupyter\" \"$@\"\n" }, { "path": "scripts/register-linux.sh", "content": "#!/bin/sh\n\nif ! which reprounzip-qt &>/dev/null; then\n echo \"reprounzip-qt is not in PATH\" >&2\n exit 1\nfi\n\n# Install x-reprozip mimetype\ncat >/tmp/reprozip-mime.xml <<'END'\n\n\n \n ReproZip Package\n \n \n\nEND\nxdg-mime install /tmp/reprozip-mime.xml\nupdate-mime-database \"$HOME/.local/share/mime\"\n\n# Install desktop file\nmkdir -p \"$HOME/.local/share/applications\"\ncat >\"$HOME/.local/share/applications/reprounzip.desktop\" <\n#include \n\n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n\n\nint main(void)\n{\n int sockfd;\n struct sockaddr_in addr;\n\n sockfd = socket(AF_INET, SOCK_STREAM, 0);\n\n memset(&addr, 0, sizeof(addr));\n addr.sin_family = AF_INET;\n {\n struct hostent *h = gethostbyname(\"www.google.com\");\n if(h == NULL)\n return 1;\n addr.sin_addr = *((struct in_addr*)h->h_addr);\n }\n addr.sin_port = htons(80);\n\n if(connect(sockfd, (struct sockaddr*)&addr, sizeof(addr)) == -1)\n return 2;\n\n close(sockfd);\n\n return 0;\n}\n" }, { "path": "tests/exec_echo.c", "content": "/* exec_echo.c\n *\n * This is a very simple program that executes /bin/echo to display a message.\n * Used to try an i386 -> x64 transition.\n *\n * usage: ./exec_echo somestr\n */\n\n#include \n\n\nint main(int argc, char **argv, char **envp)\n{\n if(argc != 2)\n return 2;\n\n {\n char *args[] = {\"echo\", argv[1], NULL};\n execve(\"/bin/echo\", args, envp);\n }\n\n return 1;\n}\n" }, { "path": "tests/functional.py", "content": "#!/usr/bin/env python\n\n# Copyright (C) 2014 New York University\n# This file is part of ReproZip which is released under the Revised BSD License\n# See file LICENSE for full license details.\n\nfrom __future__ import unicode_literals\n\nimport functools\nimport logging\nimport os\nimport re\nimport requests\nfrom rpaths import Path, unicode\nimport sqlite3\nimport subprocess\nimport sys\nimport yaml\n\nfrom reprounzip.common import FILE_READ, FILE_WRITE\nfrom reprounzip.unpackers.common import join_root\nfrom reprounzip.utils import PY3, stderr_bytes, stderr\n\ntests = Path(__file__).parent.absolute()\n\n\nlogger = logging.getLogger('reprozip-tests')\n\n\ndef download_file_retry(url, dest):\n attempt = 0\n while True:\n attempt += 1\n try:\n logger.info(\"Download %s, attempt %d\", url, attempt)\n response = requests.get(url, stream=True)\n response.raise_for_status()\n try:\n with dest.open('wb') as f:\n for chunk in response.iter_content(4096):\n f.write(chunk)\n response.close()\n except Exception as e:\n try:\n dest.remove()\n except OSError:\n pass\n raise e\n return\n except requests.RequestException as e:\n if attempt == 2:\n raise e\n logger.warning(\"Download %s: retrying after error: %s\", url, e)\n\n\ndef in_temp_dir(f):\n @functools.wraps(f)\n def wrapper(*args, **kwargs):\n tmp = Path.tempdir(prefix='reprozip_tests_')\n try:\n with tmp.in_dir():\n return f(*args, **kwargs)\n finally:\n tmp.rmtree(ignore_errors=True)\n return wrapper\n\n\ndef print_arg_list(f):\n \"\"\"Decorator printing the sole argument (list of strings) first.\n \"\"\"\n @functools.wraps(f)\n def wrapper(arglist, *args, **kwargs):\n print(\"\\nreprozip-tests$ \" +\n \" \".join(a if isinstance(a, unicode)\n else a.decode('utf-8', 'replace')\n for a in arglist))\n return f(arglist, *args, **kwargs)\n return wrapper\n\n\n@print_arg_list\ndef call(args):\n r = subprocess.call(args)\n print(\"---> %d\" % r)\n return r\n\n\n@print_arg_list\ndef check_call(args):\n return subprocess.check_call(args)\n\n\n@print_arg_list\ndef call_output(args, stream='out'):\n if stream == 'out':\n kw = {'stdout': subprocess.PIPE}\n elif stream == 'err':\n kw = {'stderr': subprocess.PIPE}\n else:\n raise ValueError\n p = subprocess.Popen(args, **kw)\n if stream == 'out':\n fp = p.stdout\n else:\n fp = p.stderr\n output = []\n while True:\n line = fp.readline()\n if not line:\n break\n stderr_bytes.write(line)\n output.append(line)\n retcode = p.wait()\n return retcode, b''.join(output)\n\n\ndef check_output(args, stream='out'):\n retcode, output = call_output(args, stream)\n if retcode != 0:\n raise subprocess.CalledProcessError(retcode, args)\n return output\n\n\ndef build(target, sources, args=[]):\n check_call(['/usr/bin/env', 'CFLAGS=', 'cc', '-o', target] +\n [(tests / s).path\n for s in sources] +\n args)\n\n\n@in_temp_dir\ndef functional_tests(raise_warnings, interactive, run_vagrant, run_docker):\n python_exe = os.environ.get('REPROZIP_TEST_PYTHON', sys.executable)\n python_exe = python_exe.split(' ')\n\n # Can't match on the SignalWarning category here because of a Python bug\n # http://bugs.python.org/issue22543\n if raise_warnings:\n python_exe.extend(['-W', 'error:signal'])\n\n if 'COVER' in os.environ:\n python_exe.extend(['-m'] + os.environ['COVER'].split(' '))\n\n reprozip_main = tests.parent / 'reprozip/reprozip/main.py'\n reprounzip_main = tests.parent / 'reprounzip/reprounzip/main.py'\n\n verbose = ['-v'] * 3\n rpz = python_exe + [reprozip_main.absolute().path] + verbose\n rpuz = python_exe + [reprounzip_main.absolute().path] + verbose\n\n print(\"Command lines are:\\n%r\\n%r\" % (rpz, rpuz))\n\n # ########################################\n # testrun /bin/echo\n #\n\n output = check_output(rpz + ['testrun', '/bin/echo', 'outputhere'])\n assert any(b' 1 | /bin/echo outputhere ' in line\n for line in output.splitlines())\n\n output = check_output(rpz + ['testrun', '-a', '/fake/path/echo',\n '/bin/echo', 'outputhere'])\n assert any(b' 1 | (/bin/echo) /fake/path/echo outputhere ' in line\n for line in output.splitlines())\n\n # ########################################\n # testrun multiple commands\n #\n\n check_call(rpz + ['testrun', 'bash', '-c',\n 'cat ../../../../../etc/passwd;'\n 'cd /var/lib;'\n 'cat ../../etc/group'])\n check_call(rpz + ['trace', '--overwrite',\n 'bash', '-c', 'cat /etc/passwd;echo'])\n check_call(rpz + ['trace', '--continue',\n 'sh', '-c', 'cat /etc/group;/usr/bin/id'])\n check_call(rpz + ['pack'])\n check_call(rpuz + ['graph', 'graph.dot'])\n check_call(rpuz + ['graph', 'graph2.dot', 'experiment.rpz'])\n\n if os.getuid() == 0:\n sudo = []\n else:\n sudo = ['sudo', '-E'] # -E to keep REPROZIP_USAGE_STATS\n\n # ########################################\n # 'simple' program: trace, pack, info, unpack\n #\n\n def check_simple(args, stream, infile=1):\n output = check_output(args, stream).splitlines()\n try:\n first = output.index(b\"Read 6 bytes\")\n except ValueError:\n stderr.write(\"output = %r\\n\" % output)\n raise\n if infile == 1:\n assert output[first + 1] == b\"a = 29, b = 13\"\n assert output[first + 2] == b\"result = 42\"\n else: # infile == 2\n assert output[first + 1] == b\"a = 25, b = 11\"\n assert output[first + 2] == b\"result = 36\"\n\n # Build\n build('simple', ['simple.c'])\n # Trace\n check_call(rpz + ['trace', '--overwrite', '-d', 'rpz-simple',\n './simple',\n (tests / 'simple_input.txt').path,\n 'simple_output.txt'])\n orig_output_location = Path('simple_output.txt').absolute()\n assert orig_output_location.is_file()\n with orig_output_location.open(encoding='utf-8') as fp:\n assert fp.read().strip() == '42'\n orig_output_location.remove()\n # Read config\n with Path('rpz-simple/config.yml').open(encoding='utf-8') as fp:\n conf = yaml.safe_load(fp)\n other_files = set(Path(f).absolute() for f in conf['other_files'])\n expected = [Path('simple'), (tests / 'simple_input.txt')]\n assert other_files.issuperset([f.resolve() for f in expected])\n # Check input and output files\n inputs_outputs = conf['inputs_outputs']\n # Exactly one input: \"arg1\", \"...simple_input.txt\"\n # Output: 'arg2', \"...simple_output.txt\"\n # There might be more output files: the C coverage files\n found = 0\n for fdict in inputs_outputs:\n if Path(fdict['path']).name == b'simple_input.txt':\n assert fdict['name'] == 'arg1'\n assert fdict['read_by_runs'] == [0]\n assert not fdict.get('written_by_runs')\n found |= 0x01\n elif Path(fdict['path']).name == b'simple_output.txt':\n assert fdict['name'] == 'arg2'\n assert not fdict.get('read_by_runs')\n assert fdict['written_by_runs'] == [0]\n found |= 0x02\n else:\n # No other inputs\n assert not fdict.get('read_by_runs')\n assert found == 0x03\n # Pack\n check_call(rpz + ['pack', '-d', 'rpz-simple', 'simple.rpz'])\n Path('simple').remove()\n # Info\n check_call(rpuz + ['info', 'simple.rpz'])\n # Show files\n check_call(rpuz + ['showfiles', 'simple.rpz'])\n # Lists packages\n check_call(rpuz + ['installpkgs', '--summary', 'simple.rpz'])\n # Unpack directory\n check_call(rpuz + ['directory', 'setup', 'simple.rpz', 'simpledir'])\n # Run directory\n check_simple(rpuz + ['directory', 'run', 'simpledir'], 'err')\n output_in_dir = join_root(Path('simpledir/root'), orig_output_location)\n with output_in_dir.open(encoding='utf-8') as fp:\n assert fp.read().strip() == '42'\n # Delete with wrong command (should fail)\n p = subprocess.Popen(rpuz + ['chroot', 'destroy', 'simpledir'],\n stderr=subprocess.PIPE)\n out, err = p.communicate()\n assert p.poll() != 0\n err = err.splitlines()\n while b\"DeprecationWarning\" in err[0] or b\"ImportWarning\" in err[0]:\n err = err[2:]\n assert b\"Wrong unpacker used\" in err[0]\n assert err[1].startswith(b\"usage: \")\n # Delete directory\n check_call(rpuz + ['directory', 'destroy', 'simpledir'])\n # Unpack chroot\n check_call(sudo + rpuz + ['chroot', 'setup', '--bind-magic-dirs',\n 'simple.rpz', 'simplechroot'])\n try:\n output_in_chroot = join_root(Path('simplechroot/root'),\n orig_output_location)\n # Run chroot\n check_simple(sudo + rpuz + ['chroot', 'run', 'simplechroot'], 'err')\n with output_in_chroot.open(encoding='utf-8') as fp:\n assert fp.read().strip() == '42'\n # Get output file\n check_call(sudo + rpuz + ['chroot', 'download', 'simplechroot',\n 'arg2:output1.txt'])\n with Path('output1.txt').open(encoding='utf-8') as fp:\n assert fp.read().strip() == '42'\n # Replace input file\n check_call(sudo + rpuz + ['chroot', 'upload', 'simplechroot',\n '%s:arg1' % (tests / 'simple_input2.txt')])\n check_call(sudo + rpuz + ['chroot', 'upload', 'simplechroot'])\n # Run again\n check_simple(sudo + rpuz + ['chroot', 'run', 'simplechroot'], 'err', 2)\n with output_in_chroot.open(encoding='utf-8') as fp:\n assert fp.read().strip() == '36'\n # Reset input file\n check_call(sudo + rpuz + ['chroot', 'upload', 'simplechroot', ':arg1'])\n # Run again\n check_simple(sudo + rpuz + ['chroot', 'run', 'simplechroot'], 'err')\n with output_in_chroot.open(encoding='utf-8') as fp:\n assert fp.read().strip() == '42'\n # Delete with wrong command (should fail)\n p = subprocess.Popen(rpuz + ['directory', 'destroy', 'simplechroot'],\n stderr=subprocess.PIPE)\n out, err = p.communicate()\n assert p.poll() != 0\n err = err.splitlines()\n while b\"DeprecationWarning\" in err[0] or b\"ImportWarning\" in err[0]:\n err = err[2:]\n assert b\"Wrong unpacker used\" in err[0]\n assert err[1].startswith(b\"usage:\")\n finally:\n # Delete chroot\n check_call(sudo + rpuz + ['chroot', 'destroy', 'simplechroot'])\n\n # Use reprounzip-vistrails with chroot\n check_call(sudo + rpuz + ['chroot', 'setup', '--bind-magic-dirs',\n 'simple.rpz', 'simplechroot_vt'])\n try:\n output_in_chroot = join_root(Path('simplechroot_vt/root'),\n orig_output_location)\n # Run using reprounzip-vistrails\n check_simple(\n sudo + python_exe +\n ['-m', 'reprounzip.plugins.vistrails', '1',\n 'chroot', 'simplechroot_vt', '0',\n '--input-file', 'arg1:%s' % (tests / 'simple_input2.txt'),\n '--output-file', 'arg2:output_vt.txt'],\n 'err', 2)\n with output_in_chroot.open(encoding='utf-8') as fp:\n assert fp.read().strip() == '36'\n finally:\n # Delete chroot\n check_call(sudo + rpuz + ['chroot', 'destroy', 'simplechroot_vt'])\n\n if not (tests / 'vagrant').exists():\n check_call(\n sudo + [\n 'sh', '-c',\n 'mkdir %(d)s; chmod 777 %(d)s' % {'d': tests / 'vagrant'},\n ]\n )\n\n # Unpack Vagrant-chroot\n check_call(rpuz + ['vagrant', 'setup/create', '--memory', '512',\n '--use-chroot', 'simple.rpz',\n (tests / 'vagrant/simplevagrantchroot').path])\n print(\"\\nVagrant project set up in simplevagrantchroot\")\n try:\n if run_vagrant:\n check_simple(rpuz + ['vagrant', 'run', '--no-stdin',\n (tests / 'vagrant/simplevagrantchroot').path],\n 'out')\n # Get output file\n check_call(rpuz + ['vagrant', 'download',\n (tests / 'vagrant/simplevagrantchroot').path,\n 'arg2:voutput1.txt'])\n with Path('voutput1.txt').open(encoding='utf-8') as fp:\n assert fp.read().strip() == '42'\n # Replace input file\n check_call(rpuz + ['vagrant', 'upload',\n (tests / 'vagrant/simplevagrantchroot').path,\n '%s:arg1' % (tests / 'simple_input2.txt')])\n check_call(rpuz + ['vagrant', 'upload',\n (tests / 'vagrant/simplevagrantchroot').path])\n # Run again\n check_simple(rpuz + ['vagrant', 'run', '--no-stdin',\n (tests / 'vagrant/simplevagrantchroot').path],\n 'out', 2)\n # Get output file\n check_call(rpuz + ['vagrant', 'download',\n (tests / 'vagrant/simplevagrantchroot').path,\n 'arg2:voutput2.txt'])\n with Path('voutput2.txt').open(encoding='utf-8') as fp:\n assert fp.read().strip() == '36'\n # Reset input file\n check_call(rpuz + ['vagrant', 'upload',\n (tests / 'vagrant/simplevagrantchroot').path,\n ':arg1'])\n # Run again\n check_simple(rpuz + ['vagrant', 'run', '--no-stdin',\n (tests / 'vagrant/simplevagrantchroot').path],\n 'out')\n # Get output file\n check_call(rpuz + ['vagrant', 'download',\n (tests / 'vagrant/simplevagrantchroot').path,\n 'arg2:voutput3.txt'])\n with Path('voutput3.txt').open(encoding='utf-8') as fp:\n assert fp.read().strip() == '42'\n # Destroy\n check_call(rpuz + ['vagrant', 'destroy',\n (tests / 'vagrant/simplevagrantchroot').path])\n elif interactive:\n print(\"Test and press enter\")\n sys.stdin.readline()\n finally:\n if (tests / 'vagrant/simplevagrantchroot').exists():\n (tests / 'vagrant/simplevagrantchroot').rmtree()\n # Unpack Vagrant without chroot\n check_call(rpuz + ['vagrant', 'setup/create', '--dont-use-chroot',\n 'simple.rpz',\n (tests / 'vagrant/simplevagrant').path])\n print(\"\\nVagrant project set up in simplevagrant\")\n try:\n if run_vagrant:\n check_simple(rpuz + ['vagrant', 'run', '--no-stdin',\n (tests / 'vagrant/simplevagrant').path],\n 'out')\n # Get output file\n check_call(rpuz + ['vagrant', 'download',\n (tests / 'vagrant/simplevagrant').path,\n 'arg2:woutput1.txt'])\n with Path('woutput1.txt').open(encoding='utf-8') as fp:\n assert fp.read().strip() == '42'\n # Replace input file\n check_call(rpuz + ['vagrant', 'upload',\n (tests / 'vagrant/simplevagrant').path,\n '%s:arg1' % (tests / 'simple_input2.txt')])\n check_call(rpuz + ['vagrant', 'upload',\n (tests / 'vagrant/simplevagrant').path])\n # Run again\n check_simple(rpuz + ['vagrant', 'run', '--no-stdin',\n (tests / 'vagrant/simplevagrant').path],\n 'out', 2)\n # Get output file\n check_call(rpuz + ['vagrant', 'download',\n (tests / 'vagrant/simplevagrant').path,\n 'arg2:woutput2.txt'])\n with Path('woutput2.txt').open(encoding='utf-8') as fp:\n assert fp.read().strip() == '36'\n # Reset input file\n check_call(rpuz + ['vagrant', 'upload',\n (tests / 'vagrant/simplevagrant').path,\n ':arg1'])\n # Run again\n check_simple(rpuz + ['vagrant', 'run', '--no-stdin',\n (tests / 'vagrant/simplevagrant').path],\n 'out')\n # Get output file\n check_call(rpuz + ['vagrant', 'download',\n (tests / 'vagrant/simplevagrant').path,\n 'arg2:woutput3.txt'])\n with Path('woutput3.txt').open(encoding='utf-8') as fp:\n assert fp.read().strip() == '42'\n # Destroy\n check_call(rpuz + ['vagrant', 'destroy',\n (tests / 'vagrant/simplevagrant').path])\n elif interactive:\n print(\"Test and press enter\")\n sys.stdin.readline()\n finally:\n if (tests / 'vagrant/simplevagrant').exists():\n (tests / 'vagrant/simplevagrant').rmtree()\n\n # Unpack Docker\n check_call(rpuz + ['docker', 'setup/create', 'simple.rpz', 'simpledocker'])\n print(\"\\nDocker project set up in simpledocker\")\n try:\n if run_docker:\n check_call(rpuz + ['docker', 'setup/build', 'simpledocker'])\n check_simple(rpuz + ['docker', 'run', 'simpledocker'], 'out')\n # Get output file\n check_call(rpuz + ['docker', 'download', 'simpledocker',\n 'arg2:doutput1.txt'])\n with Path('doutput1.txt').open(encoding='utf-8') as fp:\n assert fp.read().strip() == '42'\n # Replace input file\n check_call(rpuz + ['docker', 'upload', 'simpledocker',\n '%s:arg1' % (tests / 'simple_input2.txt')])\n check_call(rpuz + ['docker', 'upload', 'simpledocker'])\n check_call(rpuz + ['showfiles', 'simpledocker'])\n # Run again\n check_simple(rpuz + ['docker', 'run', 'simpledocker'], 'out', 2)\n # Get output file\n check_call(rpuz + ['docker', 'download', 'simpledocker',\n 'arg2:doutput2.txt'])\n with Path('doutput2.txt').open(encoding='utf-8') as fp:\n assert fp.read().strip() == '36'\n # Reset input file\n check_call(rpuz + ['docker', 'upload', 'simpledocker',\n ':arg1'])\n # Run again\n check_simple(rpuz + ['docker', 'run', 'simpledocker'], 'out')\n # Get output file\n check_call(rpuz + ['docker', 'download', 'simpledocker',\n 'arg2:doutput1.txt'])\n with Path('doutput1.txt').open(encoding='utf-8') as fp:\n assert fp.read().strip() == '42'\n # Destroy\n check_call(rpuz + ['docker', 'destroy', 'simpledocker'])\n elif interactive:\n print(\"Test and press enter\")\n sys.stdin.readline()\n finally:\n if Path('simpledocker').exists():\n Path('simpledocker').rmtree()\n\n # ########################################\n # 'threads' program: testrun\n #\n\n # Build\n build('threads', ['threads.c'], ['-lpthread'])\n # Trace\n output = check_output(rpz + ['testrun', './threads'], 'err')\n assert any(b'successfully exec\\'d /bin/./echo' in line\n for line in output.splitlines())\n\n # ########################################\n # 'threads2' program: testrun\n #\n\n # Build\n build('threads2', ['threads2.c'], ['-lpthread'])\n # Trace\n output = check_output(rpz + ['testrun', './threads2'], 'err')\n assert any(b'successfully exec\\'d /bin/echo' in line\n for line in output.splitlines())\n\n # ########################################\n # 'segv' program: testrun\n #\n\n # Build\n build('segv', ['segv.c'])\n # Trace\n check_call(rpz + ['testrun', './segv'])\n\n # ########################################\n # 'exec_echo' program: trace, pack, run --cmdline\n #\n\n # Build\n build('exec_echo', ['exec_echo.c'])\n # Trace\n check_call(rpz + ['trace', '--overwrite',\n './exec_echo', 'originalexecechooutput'])\n # Pack\n check_call(rpz + ['pack', 'exec_echo.rpz'])\n # Unpack chroot\n check_call(sudo + rpuz + ['chroot', 'setup',\n 'exec_echo.rpz', 'echochroot'])\n try:\n # Run original command-line\n output = check_output(sudo + rpuz + ['chroot', 'run',\n 'echochroot'])\n assert output == b'originalexecechooutput\\n'\n # Prints out command-line\n output = check_output(sudo + rpuz + ['chroot', 'run',\n 'echochroot', '--cmdline'])\n assert any(b'./exec_echo originalexecechooutput' == s.strip()\n for s in output.split(b'\\n'))\n # Run with different command-line\n output = check_output(sudo + rpuz + [\n 'chroot', 'run', 'echochroot',\n '--cmdline', './exec_echo', 'changedexecechooutput'])\n assert output == b'changedexecechooutput\\n'\n finally:\n check_call(sudo + rpuz + ['chroot', 'destroy', 'echochroot'])\n\n # ########################################\n # 'exec_echo' program: testrun\n # This is built with -m32 so that we transition:\n # python (x64) -> exec_echo (i386) -> echo (x64)\n #\n\n if sys.maxsize > 2 ** 32:\n # Build\n build('exec_echo32', ['exec_echo.c'], ['-m32'])\n # Trace\n check_call(rpz + ['testrun', './exec_echo32', '42'])\n else:\n print(\"Can't try exec_echo transitions: not running on 64bits\")\n\n # ########################################\n # Tracing non-existing program\n #\n\n ret = call(rpz + ['testrun', './doesntexist'])\n assert ret == 127\n\n # ########################################\n # 'connect' program: testrun\n #\n\n # Build\n build('connect', ['connect.c'])\n # Trace\n err = check_output(rpz + ['testrun', './connect'], 'err')\n err = err.split(b'\\n')\n assert not any(b'program exited with non-zero code' in line\n for line in err)\n assert any(re.search(br'process connected to [0-9.]+:80', line)\n for line in err)\n\n # ########################################\n # 'vfork' program: testrun\n #\n\n # Build\n build('vfork', ['vfork.c'])\n # Trace\n err = check_output(rpz + ['testrun', './vfork'], 'err')\n err = err.split(b'\\n')\n assert not any(b'program exited with non-zero code' in line\n for line in err)\n\n # ########################################\n # 'rename' program: trace\n #\n\n # Build\n build('rename', ['rename.c'])\n # Trace\n check_call(rpz + ['trace', '--overwrite', '-d', 'rename-trace',\n './rename'])\n with Path('rename-trace/config.yml').open(encoding='utf-8') as fp:\n config = yaml.safe_load(fp)\n # Check that written files were logged\n database = Path.cwd() / 'rename-trace/trace.sqlite3'\n if PY3:\n # On PY3, connect() only accepts unicode\n conn = sqlite3.connect(str(database))\n else:\n conn = sqlite3.connect(database.path)\n conn.row_factory = sqlite3.Row\n rows = conn.execute(\n '''\n SELECT name FROM opened_files\n ''')\n files = set(Path(r[0]) for r in rows)\n for n in ('dir1/file', 'dir2/file', 'dir2/brokensymlink', 'dir2/symlink'):\n if (Path.cwd() / n) not in files:\n raise AssertionError(\"Missing file: %s\" % (Path.cwd() / n))\n conn.close()\n # Check that created files won't be packed\n for f in config.get('other_files'):\n if 'dir2' in Path(f).parent.components:\n raise AssertionError(\"Created file shouldn't be packed: %s\" %\n Path(f))\n\n # ########################################\n # 'readwrite' program: trace\n\n # Build\n build('readwrite', ['readwrite.c'])\n # Create test folder\n Path('readwrite_test').mkdir()\n with Path('readwrite_test/existing').open('w'):\n pass\n\n # Trace existing one\n check_call(rpz + ['trace', '--overwrite', '-d', 'readwrite-E-trace',\n './readwrite', 'readwrite_test/existing'])\n # Check that file was logged as read and written\n database = Path.cwd() / 'readwrite-E-trace/trace.sqlite3'\n if PY3:\n # On PY3, connect() only accepts unicode\n conn = sqlite3.connect(str(database))\n else:\n conn = sqlite3.connect(database.path)\n conn.row_factory = sqlite3.Row\n rows = list(conn.execute(\n '''\n SELECT mode FROM opened_files\n WHERE name = ?\n ''',\n (str(Path('readwrite_test/existing').absolute()),)))\n conn.close()\n assert rows\n assert rows[0][0] == FILE_READ | FILE_WRITE\n\n # Trace non-existing one\n check_call(rpz + ['trace', '--overwrite', '-d', 'readwrite-N-trace',\n './readwrite', 'readwrite_test/nonexisting'])\n # Check that file was logged as written only\n database = Path.cwd() / 'readwrite-N-trace/trace.sqlite3'\n if PY3:\n # On PY3, connect() only accepts unicode\n conn = sqlite3.connect(str(database))\n else:\n conn = sqlite3.connect(database.path)\n conn.row_factory = sqlite3.Row\n rows = list(conn.execute(\n '''\n SELECT mode FROM opened_files\n WHERE name = ?\n ''',\n (str(Path('readwrite_test/nonexisting').absolute()),)))\n conn.close()\n assert rows\n assert rows[0][0] == FILE_WRITE\n\n # Trace a failure: inaccessible file\n ret = call(rpz + ['trace', '--overwrite', '-d', 'readwrite-F-trace',\n './readwrite', 'readwrite_test/non/existing/file'])\n assert ret == 1\n\n # ########################################\n # Test shebang corner-cases\n #\n\n Path('a').symlink('b')\n with Path('b').open('w') as fp:\n fp.write('#!%s 0\\nsome content\\n' % (Path.cwd() / 'c'))\n Path('b').chmod(0o744)\n Path('c').symlink('d')\n with Path('d').open('w') as fp:\n fp.write('#!e')\n Path('d').chmod(0o744)\n with Path('e').open('w') as fp:\n fp.write('#!/bin/echo')\n Path('e').chmod(0o744)\n\n # Trace\n out = check_output(rpz + ['trace', '--overwrite', '-d', 'shebang-trace',\n '--dont-identify-packages', './a', '1', '2'])\n out = out.splitlines()[0]\n assert out == ('e %s 0 ./a 1 2' % (Path.cwd() / 'c')).encode('ascii')\n\n # Check config\n with Path('shebang-trace/config.yml').open(encoding='utf-8') as fp:\n config = yaml.safe_load(fp)\n other_files = set(Path(f) for f in config['other_files']\n if f.startswith('%s/' % Path.cwd()))\n\n # Check database\n database = Path.cwd() / 'shebang-trace/trace.sqlite3'\n if PY3:\n # On PY3, connect() only accepts unicode\n conn = sqlite3.connect(str(database))\n else:\n conn = sqlite3.connect(database.path)\n conn.row_factory = sqlite3.Row\n rows = conn.execute(\n '''\n SELECT name FROM opened_files\n ''')\n opened = [Path(r[0]) for r in rows\n if r[0].startswith('%s/' % Path.cwd())]\n rows = conn.execute(\n '''\n SELECT name, argv FROM executed_files\n ''')\n executed = [(Path(r[0]), r[1]) for r in rows\n if Path(r[0]).lies_under(Path.cwd())]\n\n print(\"other_files: %r\" % sorted(other_files))\n print(\"opened: %r\" % opened)\n print(\"executed: %r\" % executed)\n\n assert other_files == set(Path.cwd() / p\n for p in ('a', 'b', 'c', 'd', 'e'))\n assert opened == [Path.cwd() / 'c', Path.cwd() / 'e']\n assert executed == [(Path.cwd() / 'a', './a\\x001\\x002\\x00')]\n\n # ########################################\n # Test set-uid warning\n #\n\n if os.getuid() != 0:\n # Find a set-uid program\n executable = '/bin/su'\n if not os.path.exists(executable) and os.path.exists('/usr/bin/su'):\n executable = '/usr/bin/su'\n assert os.stat(executable).st_mode & 0o4000 == 0o4000\n # Trace\n # Pass a wrong username to su to make it exit without reading a\n # password\n _, err = call_output(\n rpz + [\n 'testrun', executable,\n '94627ebfafbf81cd77a17d4ed646a80c94bf4202',\n ],\n 'err',\n )\n err = err.split(b'\\n')\n assert any(b'executing set-uid binary!' in line for line in err)\n\n # ########################################\n # Test set-gid warning\n #\n\n if os.getuid() != 0:\n # Find a set-gid program\n executable = '/usr/bin/crontab'\n if not os.path.exists(executable) and os.path.exists('/bin/crontab'):\n executable = '/bin/crontab'\n assert os.stat(executable).st_mode & 0o2000 == 0o2000\n # Trace\n # Pass a wrong username to su to make it exit without reading a\n # password\n _, err = call_output(\n rpz + ['testrun', executable, '-l'],\n 'err',\n )\n err = err.split(b'\\n')\n assert any(b'executing set-gid binary!' in line for line in err)\n\n # ########################################\n # Test old packages\n #\n\n old_packages = [\n ('simple-0.4.0.rpz',\n 'https://drive.google.com/uc?export=download&id=0B3ucPz7GSthBVG4xZW1V'\n 'eDhXNTQ',\n 'arg_2'),\n ('simple-0.6.0.rpz',\n 'https://drive.google.com/uc?export=download&id=0B3ucPz7GSthBbl9SUjhr'\n 'cUdtbGs',\n 'arg_2'),\n ('simple-0.7.1.rpz',\n 'https://drive.google.com/uc?export=download&id=0B3ucPz7GSthBRGp2Vm5V'\n 'QVpWOGs',\n 'arg2'),\n ]\n for name, url, outputname in old_packages:\n print(\"Testing old package %s\" % name)\n f = Path(name)\n if not f.exists():\n download_file_retry(url, f)\n # Info\n check_call(rpuz + ['info', name])\n # Show files\n check_call(rpuz + ['showfiles', name])\n # Lists packages\n check_call(rpuz + ['installpkgs', '--summary', name])\n if not run_docker:\n # Unpack directory\n check_call(rpuz + ['directory', 'setup', name, 'simpledir'])\n # Run directory\n check_simple(rpuz + ['directory', 'run', 'simpledir'], 'err')\n output_in_dir = Path('simpledir/root/tmp')\n output_in_dir = output_in_dir.listdir('reprozip_*')[0]\n output_in_dir = output_in_dir / 'simple_output.txt'\n with output_in_dir.open(encoding='utf-8') as fp:\n assert fp.read().strip() == '42'\n # Delete directory\n check_call(rpuz + ['directory', 'destroy', 'simpledir'])\n else:\n # Unpack docker\n check_call(rpuz + ['docker', 'setup', name, 'simpledocker'])\n # Run docker\n check_simple(rpuz + ['docker', 'run', 'simpledocker'], 'out')\n # Get output file\n check_call(rpuz + ['docker', 'download', 'simpledocker',\n outputname + ':doutput1.txt'])\n with Path('doutput1.txt').open(encoding='utf-8') as fp:\n assert fp.read().strip() == '42'\n # Delete docker\n check_call(rpuz + ['docker', 'destroy', 'simpledocker'])\n\n # ########################################\n # Copies back coverage report\n #\n\n coverage = Path('.coverage')\n if coverage.exists():\n coverage.copyfile(tests.parent / '.coverage.runpy')\n" }, { "path": "tests/readwrite.c", "content": "/* readwrite.c\n *\n * This tests the readwrite access mode.\n *\n * usage: ./readwrite file\n */\n\n#include \n\n\nint main(int argc, char **argv)\n{\n FILE *fp;\n\n if(argc != 2)\n return 1;\n\n fp = fopen(argv[1], \"a+\");\n if(fp)\n {\n fclose(fp);\n return 0;\n }\n else\n return 1;\n}\n" }, { "path": "tests/rename.c", "content": "/* rename.c\n *\n * This is a very simple program that creates a file, renames it, and opens the\n * renamed files. It tests the handler for rename(2) & co: the renamed file\n * shouldn't be packed, but it would be if the handler didn't behave correctly.\n *\n * usage: ./rename\n */\n\n#include \n\n\nint main(int argc, char **argv)\n{\n if(mkdir(\"dir1\", 0755) == -1)\n return 1;\n if(mkdir(\"dir2\", 0755) == -1)\n return 1;\n\n /* rename */\n {\n FILE *orig = fopen(\"dir1/file\", \"w\");\n if(orig == NULL)\n return 1;\n fclose(orig);\n if(rename(\"dir1/file\", \"dir2/file\") == -1)\n return 1;\n }\n\n /* broken symlink */\n if(symlink(\"dirN/something\", \"dir2/brokensymlink\") == -1)\n return 1;\n\n /* working symlink */\n if(symlink(\"dir1\", \"dir2/symlink\") == -1)\n return 1;\n\n return 0;\n}\n" }, { "path": "tests/segv.c", "content": "/* segv.c\n *\n * This program will spawn a process that will seg fault before the main\n * process finishes.\n *\n * usage: ./segv\n */\n\n#include \n\n\nint main(int argc, char **argv)\n{\n pid_t child = fork();\n if(child < 0)\n return 1;\n else if(child == 0)\n {\n char *ptr = NULL;\n *ptr = 42; /* segv */\n return 0;\n }\n else\n {\n usleep(100000);\n return 0;\n }\n}\n" }, { "path": "tests/simple.c", "content": "/* simple.c\n *\n * This is a very simple program that loads data from a file and creates\n * another file with a result.\n *\n * usage: ./simple input.txt output.txt\n */\n\n#include \n\n\nint main(int argc, char **argv)\n{\n int a, b;\n if(argc != 3)\n return 1;\n {\n char data[256];\n size_t len;\n FILE *inputfile = fopen(argv[1], \"r\");\n if(inputfile == NULL)\n {\n fprintf(stderr, \"Opening file failed\\n\");\n return 1;\n }\n len = fread(data, 1, 256, inputfile);\n fprintf(stderr, \"Read %d bytes\\n\", (int)len);\n fclose(inputfile);\n data[len] = '\\0';\n sscanf(data, \"%d %d\", &a, &b);\n fprintf(stderr, \"a = %d, b = %d\\n\", a, b);\n }\n\n {\n int result = a + b;\n fprintf(stderr, \"result = %d\\n\", result);\n FILE *outputfile = fopen(argv[2], \"w\");\n if(outputfile == NULL)\n {\n fprintf(stderr, \"Opening result failed\\n\");\n return 1;\n }\n fprintf(outputfile, \"%d\\n\", result);\n fclose(outputfile);\n }\n\n return 0;\n}\n" }, { "path": "tests/simple_input.txt", "content": "29 13\n" }, { "path": "tests/simple_input2.txt", "content": "25 11\n" }, { "path": "tests/test_graph.py", "content": "# Copyright (C) 2014 New York University\n# This file is part of ReproZip which is released under the Revised BSD License\n# See file LICENSE for full license details.\n\nfrom __future__ import print_function, unicode_literals\n\nimport json\nfrom rpaths import Path\nimport sys\nimport unittest\n\nfrom reprounzip.common import FILE_READ, FILE_WRITE, FILE_WDIR, FILE_STAT\nfrom reprounzip.unpackers import graph\nfrom reprounzip.unpackers.common import UsageError\n\nfrom tests.common import make_database\n\n\nclass TestGraph(unittest.TestCase):\n \"\"\"Generates graphs from a fabricated trace database.\"\"\"\n maxDiff = None\n\n @classmethod\n def setUpClass(cls):\n if sys.version_info < (2, 7, 3):\n raise unittest.SkipTest(\"Python version not supported by reprozip\")\n\n cls._trace = Path.tempdir(prefix='rpz_testdb_')\n conn = make_database([\n ('proc', 0, None, False),\n ('open', 0, \"/some/dir\", True, FILE_WDIR),\n ('exec', 0, \"/bin/sh\", \"/some/dir\", \"sh\\0script_1\\0\"),\n ('open', 0, \"/usr/share/1_one.pyc\", False, FILE_READ),\n ('open', 0, \"/some/dir/one\", False, FILE_WRITE),\n ('exec', 0, \"/usr/bin/python\", \"/some/dir\", \"python\\0drive.py\\0\"),\n ('open', 0, \"/some/dir/drive.py\", False, FILE_READ),\n ('open', 0, \"/some/dir/one\", False, FILE_READ),\n ('open', 0, \"/etc/2_two.cfg\", False, FILE_READ),\n ('proc', 1, 0, False),\n ('open', 1, \"/some/dir\", True, FILE_WDIR),\n ('exec', 1, \"/some/dir/experiment\", \"/some/dir\", \"experiment\\0\"),\n ('open', 1, \"/some/dir/one\", False, FILE_STAT),\n ('open', 1, \"/usr/lib/2_one.so\", False, FILE_READ),\n ('open', 1, \"/some/dir/two\", False, FILE_WRITE),\n ('exec', 0, \"/usr/bin/wc\", \"/some/dir\", \"wc\\0out.txt\\0\"),\n ('open', 0, \"/some/dir/two\", False, FILE_READ),\n\n ('proc', 2, None, False),\n ('open', 2, \"/some/dir\", True, FILE_WDIR),\n ('exec', 2, \"/bin/sh\", \"/some/dir\", \"sh\\0script_2\\0\"),\n ('proc', 3, 2, True),\n ('proc', 4, 2, False),\n ('open', 4, \"/some/dir\", True, FILE_WDIR),\n ('exec', 4, \"/usr/bin/python\", \"/some/dir\", \"python\\0-\\0\"),\n ('open', 4, \"/some/dir/one\", False, FILE_READ),\n ('open', 4, \"/some/dir/thing\", False, FILE_WRITE),\n ('exec', 2, \"/some/dir/report\", \"/some/dir\", \"./report\\0-v\\0\"),\n ('open', 2, \"/some/dir/thing\", False, FILE_READ),\n ('open', 2, \"/some/dir/result\", False, FILE_WRITE),\n ], cls._trace / 'trace.sqlite3')\n conn.close()\n with (cls._trace / 'config.yml').open('w', encoding='utf-8') as fp:\n fp.write(\"\"\"\\\nversion: \"0.7\"\nruns:\n- id: first run\n architecture: x86_64\n argv: [sh, \"script_1\"]\n binary: /some/dir/one\n distribution: [debian, '8.0']\n environ: {USER: remram}\n exitcode: 0\n uid: 1000\n gid: 1000\n hostname: test\n workingdir: /user/dir\n- architecture: x86_64\n argv: [\"sh\", \"script_2\"]\n binary: /some/dir/one\n distribution: [debian, '8.0']\n environ: {USER: remram}\n exitcode: 0\n uid: 1000\n gid: 1000\n hostname: test\n workingdir: /user/dir\n\ninputs_outputs:\n- name: important\n path: \"/some/dir/one\"\n written_by_runs: [0]\n read_by_runs: [1]\n\npackages:\n- name: pkg1\n version: \"1.0\"\n size: 10000\n packfiles: true\n files:\n - \"/usr/share/1_one.py\"\n - \"/usr/share/1_two.py\"\n - \"/usr/bin/wc\"\n- name: pkg2\n version: \"1.0\"\n size: 10000\n packfiles: true\n files:\n - \"/usr/lib/2_one.so\"\n - \"/etc/2_two.cfg\"\n- name: python\n version: \"2.7\"\n size: 5000000\n packfiles: true\n files:\n - \"/usr/bin/python\"\n- name: unused\n version: \"0.1\"\n size: 100\n packfiles: true\n files:\n - \"/an/unused/file\"\n\nother_files:\n- \"/bin/sh\"\n- \"/usr/share/1_one.pyc\"\n- \"/some/dir/drive.py\"\n- \"/some/dir/experiment\"\n- \"/some/dir/report\"\n\"\"\")\n\n @classmethod\n def tearDownClass(cls):\n cls._trace.rmtree()\n\n def do_dot_test(self, expected, **kwargs):\n graph.Process._id_gen = 0\n tmpdir = Path.tempdir(prefix='rpz_testgraph_')\n target = tmpdir / 'graph.dot'\n try:\n graph.generate(target,\n self._trace / 'config.yml',\n self._trace / 'trace.sqlite3',\n **kwargs)\n if expected is False:\n self.fail(\"DOT generation didn't fail as expected\")\n with target.open('r') as fp:\n self.assertEqual(expected, fp.read())\n except UsageError:\n if expected is not False:\n raise\n finally:\n tmpdir.rmtree()\n\n def do_json_test(self, expected, **kwargs):\n graph.Process._id_gen = 0\n tmpdir = Path.tempdir(prefix='rpz_testgraph_')\n target = tmpdir / 'graph.json'\n try:\n graph.generate(target,\n self._trace / 'config.yml',\n self._trace / 'trace.sqlite3',\n graph_format='json', **kwargs)\n if expected is False:\n self.fail(\"JSON generation didn't fail as expected\")\n with target.open('r', encoding='utf-8') as fp:\n obj = json.load(fp)\n self.assertEqual(expected, obj)\n except SystemExit:\n if expected is not False:\n raise\n finally:\n tmpdir.rmtree()\n\n def do_tests(self, expected_dot, expected_json, **kwargs):\n self.do_dot_test(expected_dot, **kwargs)\n self.do_json_test(expected_json, **kwargs)\n\n def test_simple(self):\n self.do_tests(\n \"\"\"\\\ndigraph G {\n rankdir=LR;\n\n /* programs */\n node [shape=box fontcolor=white fillcolor=black style=\"filled,rounded\"];\n subgraph cluster_run0 {\n label=\"first run\";\n prog0 [label=\"/bin/sh (0)\"];\n prog1 [label=\"/usr/bin/python (0)\"];\n prog0 -> prog1 [label=\"exec\"];\n prog2 [label=\"/some/dir/experiment (1)\"];\n prog1 -> prog2 [label=\"fork+exec\"];\n prog3 [label=\"/usr/bin/wc (0)\"];\n prog1 -> prog3 [label=\"exec\"];\n }\n subgraph cluster_run1 {\n label=\"run1\";\n prog4 [label=\"/bin/sh (2)\"];\n prog5 [label=\"/bin/sh (3)\",fillcolor=\"#666666\"];\n prog4 -> prog5 [label=\"thread\"];\n prog6 [label=\"/usr/bin/python (4)\"];\n prog4 -> prog6 [label=\"fork+exec\"];\n prog7 [label=\"/some/dir/report (2)\"];\n prog4 -> prog7 [label=\"exec\"];\n }\n\n node [shape=ellipse fontcolor=\"#131C39\" fillcolor=\"#C9D2ED\"];\n\n /* system packages */\n subgraph cluster_pkg0 {\n label=\"pkg1 1.0\";\n \"/usr/bin/wc\";\n }\n subgraph cluster_pkg1 {\n label=\"pkg2 1.0\";\n \"/etc/2_two.cfg\";\n \"/usr/lib/2_one.so\";\n }\n subgraph cluster_pkg2 {\n label=\"python 2.7\";\n \"/usr/bin/python\";\n }\n\n /* other files */\n \"/bin/sh\";\n \"/some/dir/drive.py\";\n \"/some/dir/experiment\";\n \"/some/dir/one\" [fillcolor=\"#A3B4E0\", label=\"important\\\\n/some/dir/one\"];\n \"/some/dir/report\";\n \"/some/dir/result\";\n \"/some/dir/thing\";\n \"/some/dir/two\";\n \"/usr/share/1_one.pyc\";\n\n \"/bin/sh\" -> prog0 [style=bold, label=\"sh script_1\"];\n \"/usr/share/1_one.pyc\" -> prog0 [color=\"#8888CC\"];\n prog0 -> \"/some/dir/one\" [color=\"#000088\"];\n \"/usr/bin/python\" -> prog1 [style=bold, label=\"python drive.py\"];\n \"/some/dir/drive.py\" -> prog1 [color=\"#8888CC\"];\n \"/some/dir/one\" -> prog1 [color=\"#8888CC\"];\n \"/etc/2_two.cfg\" -> prog1 [color=\"#8888CC\"];\n \"/some/dir/experiment\" -> prog2 [style=bold, label=\"experiment\"];\n \"/usr/lib/2_one.so\" -> prog2 [color=\"#8888CC\"];\n prog2 -> \"/some/dir/two\" [color=\"#000088\"];\n \"/usr/bin/wc\" -> prog3 [style=bold, label=\"wc out.txt\"];\n \"/some/dir/two\" -> prog3 [color=\"#8888CC\"];\n \"/bin/sh\" -> prog4 [style=bold, label=\"sh script_2\"];\n \"/usr/bin/python\" -> prog6 [style=bold, label=\"python -\"];\n \"/some/dir/one\" -> prog6 [color=\"#8888CC\"];\n prog6 -> \"/some/dir/thing\" [color=\"#000088\"];\n \"/some/dir/report\" -> prog7 [style=bold, label=\"./report -v\"];\n \"/some/dir/thing\" -> prog7 [color=\"#8888CC\"];\n prog7 -> \"/some/dir/result\" [color=\"#000088\"];\n}\n\"\"\",\n {'packages': [{'name': 'pkg1', 'version': '1.0',\n 'files': ['/usr/bin/wc']},\n {'name': 'pkg2', 'version': '1.0',\n 'files': ['/etc/2_two.cfg',\n '/usr/lib/2_one.so']},\n {'name': 'python', 'version': '2.7',\n 'files': ['/usr/bin/python']}],\n 'other_files': ['/bin/sh',\n '/some/dir/drive.py',\n '/some/dir/experiment',\n '/some/dir/one',\n '/some/dir/report',\n '/some/dir/result',\n '/some/dir/thing',\n '/some/dir/two',\n '/usr/share/1_one.pyc'],\n 'inputs_outputs': [{'name': \"important\", 'path': \"/some/dir/one\",\n 'written_by_runs': [0], 'read_by_runs': [1]}],\n 'runs': [{'name': \"first run\",\n 'processes': [\n {'name': '0',\n 'long_name': 'sh (0)',\n 'description': '/bin/sh\\n0',\n 'argv': ['sh', 'script_1'],\n 'start_time': 0,\n 'is_thread': False,\n 'parent': None,\n 'reads': ['/bin/sh', '/usr/share/1_one.pyc'],\n 'writes': ['/some/dir/one']},\n {'name': '0',\n 'long_name': 'python (0)',\n 'description': '/usr/bin/python\\n0',\n 'argv': ['python', 'drive.py'],\n 'start_time': 5,\n 'is_thread': False,\n 'parent': [0, 'exec'],\n 'reads': ['/usr/bin/python',\n '/some/dir/drive.py',\n '/some/dir/one',\n '/etc/2_two.cfg'],\n 'writes': []},\n {'name': '1',\n 'long_name': 'experiment (1)',\n 'description': '/some/dir/experiment\\n1',\n 'argv': ['experiment'],\n 'start_time': 9,\n 'is_thread': False,\n 'parent': [1, 'fork+exec'],\n 'reads': ['/some/dir/experiment',\n '/usr/lib/2_one.so'],\n 'writes': ['/some/dir/two']},\n {'name': '0',\n 'long_name': 'wc (0)',\n 'description': '/usr/bin/wc\\n0',\n 'argv': ['wc', 'out.txt'],\n 'start_time': 15,\n 'is_thread': False,\n 'parent': [1, 'exec'],\n 'reads': ['/usr/bin/wc', '/some/dir/two'],\n 'writes': []}]},\n\n {'name': \"run1\",\n 'processes': [\n {'name': '2',\n 'long_name': 'sh (2)',\n 'description': '/bin/sh\\n2',\n 'argv': ['sh', 'script_2'],\n 'start_time': 17,\n 'is_thread': False,\n 'parent': None,\n 'reads': ['/bin/sh'],\n 'writes': []},\n {'name': '3',\n 'long_name': 'sh (3)',\n 'description': '/bin/sh\\n3',\n 'argv': ['sh', 'script_2'],\n 'start_time': 20,\n 'is_thread': True,\n 'parent': [0, 'fork'],\n 'reads': [],\n 'writes': []},\n {'name': '4',\n 'long_name': 'python (4)',\n 'description': '/usr/bin/python\\n4',\n 'argv': ['python', '-'],\n 'start_time': 21,\n 'is_thread': False,\n 'parent': [0, 'fork+exec'],\n 'reads': ['/usr/bin/python', '/some/dir/one'],\n 'writes': ['/some/dir/thing']},\n {'name': '2',\n 'long_name': 'report (2)',\n 'description': '/some/dir/report\\n2',\n 'argv': ['./report', '-v'],\n 'start_time': 26,\n 'is_thread': False,\n 'parent': [0, 'exec'],\n 'reads': ['/some/dir/report', '/some/dir/thing'],\n 'writes': ['/some/dir/result']}]}]})\n\n def test_collapsed_packages(self):\n self.do_tests(\n \"\"\"\\\ndigraph G {\n rankdir=LR;\n\n /* programs */\n node [shape=box fontcolor=white fillcolor=black style=\"filled,rounded\"];\n subgraph cluster_run0 {\n label=\"first run\";\n prog0 [label=\"/bin/sh (0)\"];\n prog1 [label=\"/usr/bin/python (0)\"];\n prog0 -> prog1 [label=\"exec\"];\n prog2 [label=\"/some/dir/experiment (1)\"];\n prog1 -> prog2 [label=\"fork+exec\"];\n prog3 [label=\"/usr/bin/wc (0)\"];\n prog1 -> prog3 [label=\"exec\"];\n }\n subgraph cluster_run1 {\n label=\"run1\";\n prog4 [label=\"/bin/sh (2)\"];\n prog5 [label=\"/bin/sh (3)\",fillcolor=\"#666666\"];\n prog4 -> prog5 [label=\"thread\"];\n prog6 [label=\"/usr/bin/python (4)\"];\n prog4 -> prog6 [label=\"fork+exec\"];\n prog7 [label=\"/some/dir/report (2)\"];\n prog4 -> prog7 [label=\"exec\"];\n }\n\n node [shape=ellipse fontcolor=\"#131C39\" fillcolor=\"#C9D2ED\"];\n\n /* system packages */\n \"pkg pkg1\" [shape=box,label=\"pkg1 1.0\"];\n \"pkg pkg2\" [shape=box,label=\"pkg2 1.0\"];\n \"pkg python\" [shape=box,label=\"python 2.7\"];\n\n /* other files */\n \"/bin/sh\";\n \"/some/dir/drive.py\";\n \"/some/dir/experiment\";\n \"/some/dir/one\" [fillcolor=\"#A3B4E0\", label=\"important\\\\n/some/dir/one\"];\n \"/some/dir/report\";\n \"/some/dir/result\";\n \"/some/dir/thing\";\n \"/some/dir/two\";\n\n \"/bin/sh\" -> prog0 [style=bold, label=\"sh script_1\"];\n \"pkg pkg1\" -> prog0 [color=\"#8888CC\"];\n prog0 -> \"/some/dir/one\" [color=\"#000088\"];\n \"pkg python\" -> prog1 [style=bold, label=\"python drive.py\"];\n \"/some/dir/drive.py\" -> prog1 [color=\"#8888CC\"];\n \"/some/dir/one\" -> prog1 [color=\"#8888CC\"];\n \"pkg pkg2\" -> prog1 [color=\"#8888CC\"];\n \"/some/dir/experiment\" -> prog2 [style=bold, label=\"experiment\"];\n \"pkg pkg2\" -> prog2 [color=\"#8888CC\"];\n prog2 -> \"/some/dir/two\" [color=\"#000088\"];\n \"pkg pkg1\" -> prog3 [style=bold, label=\"wc out.txt\"];\n \"/some/dir/two\" -> prog3 [color=\"#8888CC\"];\n \"/bin/sh\" -> prog4 [style=bold, label=\"sh script_2\"];\n \"pkg python\" -> prog6 [style=bold, label=\"python -\"];\n \"/some/dir/one\" -> prog6 [color=\"#8888CC\"];\n prog6 -> \"/some/dir/thing\" [color=\"#000088\"];\n \"/some/dir/report\" -> prog7 [style=bold, label=\"./report -v\"];\n \"/some/dir/thing\" -> prog7 [color=\"#8888CC\"];\n prog7 -> \"/some/dir/result\" [color=\"#000088\"];\n}\n\"\"\",\n False,\n level_pkgs='package',\n regex_replaces=[('.pyc$', '.py')])\n" }, { "path": "tests/test_parameters.py", "content": "# Copyright (C) 2014 New York University\n# This file is part of ReproZip which is released under the Revised BSD License\n# See file LICENSE for full license details.\n\nfrom __future__ import print_function, unicode_literals\n\nimport unittest\n\nfrom reprounzip import parameters\nfrom reprounzip.unpackers.docker import select_image\nfrom reprounzip.unpackers.vagrant import select_box\n\n\nclass TestSelection(unittest.TestCase):\n @classmethod\n def setUpClass(cls):\n # Make sure parameters are loaded\n parameters.update_parameters()\n\n # Reset them to bundled parameters temporarily\n cls._old_parameters = parameters.parameters\n parameters.parameters = parameters._bundled_parameters\n\n @classmethod\n def tearDownClass(cls):\n parameters.parameters = cls._old_parameters\n\n def test_docker(self):\n def get(architecture, distribution, version):\n return select_image([{'architecture': architecture,\n 'distribution': (distribution, version)}])\n\n self.assertEqual(get('i686', 'Ubuntu', '14.10'),\n ('ubuntu', 'ubuntu:14.10'))\n self.assertEqual(get('x86_64', 'Ubuntu', '14.10'),\n ('ubuntu', 'ubuntu:14.10'))\n self.assertEqual(get('x86_64', 'Ubuntu', '1.1'),\n ('ubuntu', 'ubuntu:19.04'))\n self.assertRaises(SystemExit, get, 'armv7', 'Debian', '8.2')\n self.assertEqual(get('x86_64', 'Arch', '2015.06.01'),\n ('debian', 'debian:stretch'))\n self.assertEqual(get('x86_64', 'Debian', '1'),\n ('debian', 'debian:stretch'))\n self.assertEqual(get('x86_64', 'CentOS', '1'),\n ('centos', 'centos:centos7'))\n\n def test_vagrant(self):\n def get(architecture, distribution, version, gui=False):\n return select_box([{'architecture': architecture,\n 'distribution': (distribution, version)}],\n gui=gui)\n\n self.assertEqual(get('i686', 'Ubuntu', '14.04'),\n ('ubuntu', 'ubuntu/trusty32'))\n self.assertEqual(get('x86_64', 'Ubuntu', '12.04'),\n ('ubuntu', 'hashicorp/precise64'))\n self.assertEqual(get('i686', 'Ubuntu', '1.1'),\n ('ubuntu', 'bento/ubuntu-17.04-i386'))\n self.assertRaises(SystemExit, get, 'armv7', 'Debian', '8.2')\n self.assertEqual(get('x86_64', 'Arch', '2015.06.01'),\n ('debian', 'bento/debian-10'))\n self.assertEqual(get('x86_64', 'Debian', '1'),\n ('debian', 'bento/debian-10'))\n self.assertEqual(get('x86_64', 'CentOS', '1'),\n ('centos', 'bento/centos-8'))\n self.assertEqual(get('x86_64', 'Fedora', '22'),\n ('fedora', 'remram/fedora-22-amd64'))\n self.assertEqual(get('x86_64', 'Fedora', '22', gui=True),\n ('debian', 'remram/debian-8-amd64-x'))\n self.assertEqual(get('x86_64', 'Ubuntu', '14.04', gui=True),\n ('ubuntu', 'remram/ubuntu-1604-amd64-x'))\n" }, { "path": "tests/test_rails_filter.py", "content": "# Copyright (C) 2014 New York University\n# This file is part of ReproZip which is released under the Revised BSD License\n# See file LICENSE for full license details.\n\nfrom __future__ import division, print_function, unicode_literals\n\nfrom reprozip.common import File\nfrom reprozip.filters import ruby\nfrom rpaths import Path\nimport unittest\n\n\nclass MockTracedFile(File):\n def __init__(self, path):\n File.__init__(self, path, None)\n\n\nclass RailsFilterTest(unittest.TestCase):\n def setUp(self):\n self.tmp = Path.tempdir(prefix='reprozip_tests_')\n\n def tearDown(self):\n self.tmp.rmtree()\n\n @classmethod\n def touch(cls, test_files):\n for fi in test_files:\n if not fi.parent.is_dir():\n fi.parent.mkdir(parents=True)\n with fi.open('a'):\n pass\n\n def test_consuming_entire_gem(self):\n gemdir = self.tmp / 'gems/ruby-2.2.3/gems/kaminari-0.16.3'\n gemfiles = [\n 'app/views/kaminari/_first_page.html.erb',\n 'app/views/kaminari/_first_page.html.haml',\n 'app/views/kaminari/_first_page.html.slim',\n 'app/views/kaminari/_gap.html.erb',\n 'app/views/kaminari/_gap.html.haml',\n 'app/views/kaminari/_gap.html.slim',\n 'app/views/kaminari/_last_page.html.erb',\n 'app/views/kaminari/_last_page.html.haml',\n 'app/views/kaminari/_last_page.html.slim',\n ]\n\n self.touch(gemdir / f for f in gemfiles)\n\n input_files = [[]]\n files = {}\n\n for path in gemdir.recursedir():\n if b'_first' in path.name:\n f = MockTracedFile(path)\n files[f.path] = f\n input_files[0].append(path)\n\n ruby(files=files, input_files=input_files)\n\n self.assertEqual(set(files.keys()), set(gemdir / f for f in gemfiles))\n\n def test_consuming_rails_files(self):\n # Should be recognized: has a config file\n railsfiles = [\n 'yes/config/application.rb',\n 'yes/app/views/application.html.erb',\n 'yes/app/views/discussion-sidebar.html.erb',\n 'yes/app/views/payments_listing.html.erb',\n 'yes/app/views/print-friendly.html.erb',\n 'yes/app/views/w-sidebar.html.erb',\n 'yes/app/views/widget.html.erb',\n ]\n # Should NOT be: no config file\n notrailsfiles = [\n # 'no/config/application.rb',\n 'no/app/views/application.html.erb',\n 'no/app/views/discussion-sidebar.html.erb',\n 'no/app/views/payments_listing.html.erb',\n 'no/app/views/print-friendly.html.erb',\n 'no/app/views/w-sidebar.html.erb',\n 'no/app/views/widget.html.erb',\n ]\n\n self.touch(self.tmp / f for f in railsfiles)\n self.touch(self.tmp / f for f in notrailsfiles)\n\n input_files = [[]]\n files = {}\n\n viewsdir = MockTracedFile(self.tmp / railsfiles[-1])\n files[viewsdir.path] = viewsdir\n viewsdir = MockTracedFile(self.tmp / notrailsfiles[-1])\n files[viewsdir.path] = viewsdir\n\n ruby(files, input_files)\n\n self.assertEqual(\n set(files.keys()),\n set(self.tmp / f for f in railsfiles + [notrailsfiles[-1]]),\n )\n" }, { "path": "tests/test_reprounzip.py", "content": "# Copyright (C) 2014 New York University\n# This file is part of ReproZip which is released under the Revised BSD License\n# See file LICENSE for full license details.\n\nfrom __future__ import print_function, unicode_literals\n\nimport io\nimport os\nimport sys\nimport shutil\nimport tarfile\nimport tempfile\nimport unittest\nimport warnings\nimport zipfile\n\nfrom reprounzip.common import RPZPack\nfrom reprounzip.signals import Signal\n\n\nclass TestSignals(unittest.TestCase):\n def test_make_signal(self):\n \"\"\"Tests signal creation.\"\"\"\n Signal(['p1', 'p2'])\n\n sig = Signal(['p1'], new_args=['p3'], old_args=['p4'])\n self.assertEqual(sig._args,\n {'p1': Signal.REQUIRED,\n 'p3': Signal.OPTIONAL,\n 'p4': Signal.DEPRECATED})\n\n with self.assertRaises(ValueError):\n Signal(['p1', 'p2'], new_args=['p2', 'p3'])\n\n def test_subscription(self):\n \"\"\"Tests subscribe() and unsubscribe().\"\"\"\n def foo(info):\n pass\n\n sig = Signal()\n sig.subscribe(foo)\n with self.assertRaises(TypeError):\n sig.subscribe(object())\n with self.assertRaises(TypeError):\n sig.subscribe(2)\n sig.unsubscribe(4)\n self.assertEqual(sig._listeners, set([foo]))\n sig.unsubscribe(foo)\n self.assertFalse(sig._listeners)\n\n def test_calls(self):\n \"\"\"Tests actually emitting signals.\"\"\"\n def called(**kwargs):\n called.last = kwargs\n\n def callsig(res_type, **kwargs):\n if res_type not in ('succ', 'warn', 'fail'):\n raise TypeError\n called.last = None\n with warnings.catch_warnings(record=True) as w:\n warnings.resetwarnings()\n sig(**kwargs)\n if res_type == 'fail' or res_type == 'warn':\n self.assertEqual(len(w), 1)\n if res_type == 'succ' or res_type == 'warn':\n self.assertEqual(called.last, kwargs)\n\n sig = Signal(['a', 'b'], new_args=['c'], old_args=['d'])\n sig.subscribe(called)\n\n callsig('succ', a=1, b=2)\n callsig('succ', a=1, b=2, c=3)\n callsig('fail', a=1)\n callsig('warn', a=1, b=2, d=3)\n\n\nclass TestArgs(unittest.TestCase):\n def test_argparse(self):\n \"\"\"Tests argument parsing\"\"\"\n calls = []\n\n def chroot_run(args):\n calls.append(('c', args.verbosity))\n\n def setup_logging(tag, verbosity):\n calls.append(('l', verbosity))\n\n import reprounzip.main\n import reprounzip.unpackers.default\n\n old_funcs = (reprounzip.unpackers.default.chroot_run,\n reprounzip.main.setup_logging)\n reprounzip.unpackers.default.chroot_run = chroot_run\n reprounzip.main.setup_logging = setup_logging\n old_argv = sys.argv\n print(\"<<<<< argparse tests for reprounzip (disregard usage warnings)\",\n file=sys.stderr)\n try:\n for a, c, v in [('reprounzip', 2, -1),\n ('reprounzip -v', 2, -1),\n ('reprounzip chroot run a', 0, 1),\n ('reprounzip chroot -v run a', 2, -1),\n ('reprounzip -v chroot run a', 0, 2),\n ('reprounzip -v -v chroot run a', 0, 3),\n ('reprounzip chroot run -v a', 2, -1)]:\n sys.argv = a.split()\n with self.assertRaises(SystemExit) as cm:\n reprounzip.main.main()\n self.assertEqual(cm.exception.code, c)\n if c == 0:\n self.assertEqual(calls, [('l', v), ('c', v)])\n calls = []\n finally:\n print(\">>>>> argparse tests\", file=sys.stderr)\n sys.argv = old_argv\n (reprounzip.unpackers.default.chroot_run,\n reprounzip.main.setup_logging) = old_funcs\n\n\nclass TarBuilder(object):\n def __init__(self, filename, mode):\n self.tar = tarfile.open(filename, mode)\n\n def write_data(self, path, data, mode=None):\n info = tarfile.TarInfo(path)\n info.size = len(data)\n if mode is not None:\n info.mode = mode\n self.tar.addfile(info, io.BytesIO(data))\n\n def add_file(self, name, arcname):\n self.tar.add(name, arcname)\n\n def close(self):\n self.tar.close()\n self.tar = None\n\n\nclass ZipBuilder(object):\n def __init__(self, filename):\n self.zip = zipfile.ZipFile(filename, 'w')\n\n def write_data(self, path, data, mode=None):\n self.zip.writestr(zipfile.ZipInfo(filename=path), data)\n\n def add_file(self, name, arcname):\n self.zip.write(name, arcname)\n\n def close(self):\n self.zip.close()\n self.zip = None\n\n\nclass TestCommon(unittest.TestCase):\n def test_rpzpack_v1(self):\n tmp = tempfile.mkdtemp()\n try:\n # Create rpz\n rpz = os.path.join(tmp, 'test.rpz')\n arc = TarBuilder(rpz, 'w:gz')\n arc.write_data(\n 'METADATA/version',\n b'REPROZIP VERSION 1\\n',\n )\n arc.write_data(\n 'METADATA/trace.sqlite3',\n b'',\n )\n arc.write_data(\n 'METADATA/config.yml',\n b'{}',\n )\n\n # Add data\n arc.write_data(\n 'DATA/bin/init',\n b'#!/bin/sh\\necho \"Success.\"\\n',\n mode=0o755,\n )\n\n # Add directory extension\n arc.write_data(\n 'EXTENSIONS/foo/one.txt',\n b'',\n )\n arc.write_data(\n 'EXTENSIONS/foo/two.txt',\n b'',\n )\n\n # Add single file extension\n arc.write_data(\n 'EXTENSIONS/bar',\n b'',\n )\n\n arc.close()\n\n rpz_obj = RPZPack(rpz)\n self.assertEqual(rpz_obj.open_config().read(), b'{}')\n self.assertEqual(rpz_obj.extensions(), {'foo', 'bar'})\n finally:\n shutil.rmtree(tmp)\n\n def test_rpzpack_v2_tar(self):\n tmp = tempfile.mkdtemp()\n try:\n # Create data tar.gz\n data = os.path.join(tmp, 'DATA.tar.gz')\n arc = TarBuilder(data, 'w:gz')\n arc.write_data(\n 'DATA/bin/init',\n b'#!/bin/sh\\necho \"Success.\"\\n',\n mode=0o755,\n )\n arc.close()\n\n # Create rpz\n rpz = os.path.join(tmp, 'test.rpz')\n arc = TarBuilder(rpz, 'w:')\n arc.write_data(\n 'METADATA/version',\n b'REPROZIP VERSION 2\\n',\n )\n arc.write_data(\n 'METADATA/trace.sqlite3',\n b'',\n )\n arc.write_data(\n 'METADATA/config.yml',\n b'{}',\n )\n arc.add_file(\n data,\n 'DATA.tar.gz',\n )\n\n # Add directory extension\n arc.write_data(\n 'EXTENSIONS/foo/one.txt',\n b'',\n )\n arc.write_data(\n 'EXTENSIONS/foo/two.txt',\n b'',\n )\n\n # Add single file extension\n arc.write_data(\n 'EXTENSIONS/bar',\n b'',\n )\n\n arc.close()\n\n rpz_obj = RPZPack(rpz)\n self.assertEqual(rpz_obj.open_config().read(), b'{}')\n self.assertEqual(rpz_obj.extensions(), {'foo', 'bar'})\n finally:\n shutil.rmtree(tmp)\n\n def test_rpzpack_v2_zip(self):\n tmp = tempfile.mkdtemp()\n try:\n # Create data tar.gz\n data = os.path.join(tmp, 'DATA.tar.gz')\n arc = TarBuilder(data, 'w:gz')\n arc.write_data(\n 'DATA/bin/init',\n b'#!/bin/sh\\necho \"Success.\"\\n',\n mode=0o755,\n )\n arc.close()\n\n # Create rpz\n rpz = os.path.join(tmp, 'test.rpz')\n arc = ZipBuilder(rpz)\n arc.write_data(\n 'METADATA/version',\n b'REPROZIP VERSION 2\\n',\n )\n arc.write_data(\n 'METADATA/trace.sqlite3',\n b'',\n )\n arc.write_data(\n 'METADATA/config.yml',\n b'{}',\n )\n arc.add_file(\n data,\n 'DATA.tar.gz',\n )\n\n # Add directory extension\n arc.write_data(\n 'EXTENSIONS/foo/one.txt',\n b'',\n )\n arc.write_data(\n 'EXTENSIONS/foo/two.txt',\n b'',\n )\n\n # Add single file extension\n arc.write_data(\n 'EXTENSIONS/bar',\n b'',\n )\n\n arc.close()\n\n rpz_obj = RPZPack(rpz)\n self.assertEqual(rpz_obj.open_config().read(), b'{}')\n self.assertEqual(rpz_obj.extensions(), {'foo', 'bar'})\n finally:\n shutil.rmtree(tmp)\n" }, { "path": "tests/test_reprozip.py", "content": "# Copyright (C) 2014 New York University\n# This file is part of ReproZip which is released under the Revised BSD License\n# See file LICENSE for full license details.\n\nfrom __future__ import print_function, unicode_literals\n\nimport os\n\nimport sqlite3\nfrom rpaths import AbstractPath, Path\nimport sys\nimport unittest\n\nfrom reprozip.common import FILE_READ, FILE_WRITE, FILE_WDIR, InputOutputFile\nfrom reprozip.tracer.trace import get_files, compile_inputs_outputs\nfrom reprozip import traceutils\nfrom reprozip.utils import PY3, unicode_, UniqueNames, make_dir_writable\n\nfrom tests.common import make_database\n\n\nclass TestReprozip(unittest.TestCase):\n @unittest.skipUnless(hasattr(os, 'chown'), \"No POSIX file permissions\")\n def test_make_dir_writable(self):\n \"\"\"Tests make_dir_writable with read-only dir.\"\"\"\n def check_mode(mod, path):\n self.assertEqual(oct((path.stat().st_mode & 0o0700) >> 6),\n oct(mod))\n\n tmp = Path.tempdir()\n try:\n (tmp / 'some' / 'path').mkdir(parents=True)\n (tmp / 'some' / 'path').chmod(0o555)\n with make_dir_writable(tmp / 'some' / 'path'):\n check_mode(7, tmp / 'some')\n check_mode(7, tmp / 'some' / 'path')\n check_mode(7, tmp / 'some')\n check_mode(5, tmp / 'some' / 'path')\n finally:\n (tmp / 'some').chmod(0o755)\n (tmp / 'some' / 'path').chmod(0o755)\n tmp.rmtree()\n\n @unittest.skipUnless(hasattr(os, 'chown'), \"No POSIX file permissions\")\n def test_make_dir_writable2(self):\n \"\"\"Tests make_dir_writable with read-only and no-executable dirs.\"\"\"\n def check_mode(mod, path):\n self.assertEqual(oct((path.stat().st_mode & 0o0700) >> 6),\n oct(mod))\n\n tmp = Path.tempdir()\n try:\n (tmp / 'some' / 'complete' / 'path').mkdir(parents=True)\n (tmp / 'some' / 'complete' / 'path').chmod(0o555)\n (tmp / 'some' / 'complete').chmod(0o444)\n with make_dir_writable(tmp / 'some' / 'complete' / 'path'):\n check_mode(7, tmp / 'some')\n check_mode(7, tmp / 'some' / 'complete')\n check_mode(7, tmp / 'some' / 'complete' / 'path')\n check_mode(7, tmp / 'some')\n check_mode(4, tmp / 'some' / 'complete')\n (tmp / 'some' / 'complete').chmod(0o755)\n check_mode(5, tmp / 'some' / 'complete' / 'path')\n finally:\n (tmp / 'some').chmod(0o755)\n (tmp / 'some' / 'complete').chmod(0o755)\n (tmp / 'some' / 'complete' / 'path').chmod(0o755)\n tmp.rmtree()\n\n @unittest.skipIf(sys.version_info < (2, 7, 3),\n \"Python version not supported by reprozip\")\n def test_argparse(self):\n \"\"\"Tests argument parsing\"\"\"\n calls = []\n\n def testrun(args):\n calls.append(('t', args.verbosity))\n\n def setup_logging(tag, verbosity):\n calls.append(('l', verbosity))\n\n import reprozip.main\n\n old_funcs = reprozip.main.testrun, reprozip.main.setup_logging\n reprozip.main.testrun = testrun\n reprozip.main.setup_logging = setup_logging\n old_argv = sys.argv\n print(\"<<<<< argparse tests for reprozip (disregard usage warnings)\",\n file=sys.stderr)\n try:\n for a, c, v in [('reprozip', 2, -1),\n ('reprozip -v', 2, -1),\n ('reprozip testrun true', 0, 1),\n ('reprozip testrun -v true', 2, -1),\n ('reprozip -v testrun true', 0, 2),\n ('reprozip -v -v testrun true', 0, 3)]:\n sys.argv = a.split()\n with self.assertRaises(SystemExit) as cm:\n reprozip.main.main()\n self.assertEqual(cm.exception.code, c)\n if c == 0:\n self.assertEqual(calls, [('l', v), ('t', v)])\n calls = []\n finally:\n print(\">>>>> argparse tests\", file=sys.stderr)\n sys.argv = old_argv\n reprozip.main.testrun, reprozip.main.setup_logging = old_funcs\n\n\nclass TestNames(unittest.TestCase):\n def test_uniquenames(self):\n \"\"\"Tests UniqueNames.\"\"\"\n u = UniqueNames()\n self.assertEqual(u('test'), 'test')\n self.assertEqual(u('test'), 'test_2')\n self.assertEqual(u('test'), 'test_3')\n self.assertEqual(u('test_2'), 'test_2_2')\n self.assertEqual(u('test_'), 'test_')\n self.assertEqual(u('test_'), 'test__2')\n\n def test_label_files(self):\n \"\"\"Tests input/output file labelling.\"\"\"\n wd = Path('/fakeworkingdir')\n self.assertEqual(\n compile_inputs_outputs(\n [{'argv': ['aa', 'bb.txt'], 'workingdir': wd}],\n [[wd / 'aa', Path('/other/cc.bin'), wd / 'bb.txt']],\n [[]]),\n {'arg0': InputOutputFile(wd / 'aa', [0], []),\n 'cc.bin': InputOutputFile(Path('/other/cc.bin'), [0], []),\n 'arg1': InputOutputFile(wd / 'bb.txt', [0], [])})\n\n\nclass TestFiles(unittest.TestCase):\n def do_test(self, insert):\n conn = make_database(insert)\n\n try:\n files, inputs, outputs = get_files(conn)\n files = set(fi for fi in files\n if not fi.path.path.startswith((b'/lib', b'/usr/lib')))\n return files, inputs, outputs\n finally:\n conn.close()\n\n @classmethod\n def make_paths(cls, obj):\n if isinstance(obj, set):\n return set(cls.make_paths(e) for e in obj)\n elif isinstance(obj, list):\n return [cls.make_paths(e) for e in obj]\n elif isinstance(obj, AbstractPath):\n return obj\n elif isinstance(obj, (bytes, unicode_)):\n return Path(obj)\n else:\n assert False\n\n def assertEqualPaths(self, objs, second):\n self.assertEqual(self.make_paths(objs), second)\n\n def test_get_files(self):\n files, inputs, outputs = self.do_test([\n ('proc', 0, None, False),\n ('open', 0, \"/some/dir\", True, FILE_WDIR),\n ('exec', 0, \"/some/dir/ls\", \"/some/dir\", \"ls\\0\"),\n ('open', 0, \"/some/otherdir/in\", False, FILE_READ),\n ('open', 0, \"/some/thing/created\", True, FILE_WRITE),\n ('proc', 1, 0, False),\n ('open', 1, \"/some/thing/created/file\", False, FILE_WRITE),\n ('open', 1, \"/some/thing/created/file\", False, FILE_READ),\n ('open', 1, \"/some/thing/created\", True, FILE_WDIR),\n ('exec', 0, \"/some/thing/created/file\", \"/some/thing/created\",\n \"created\\0\"),\n ])\n expected = set([\n '/some/dir',\n '/some/dir/ls',\n '/some/otherdir/in',\n '/some/thing',\n ])\n self.assertEqualPaths(expected,\n set(fi.path for fi in files))\n\n def test_multiple_runs(self):\n def fail(s):\n assert False, \"Shouldn't be called?\"\n old = Path.is_file, Path.stat\n Path.is_file = lambda s: True\n Path.stat = fail\n try:\n files, inputs, outputs = self.do_test([\n ('proc', 0, None, False),\n ('open', 0, \"/some/dir\", True, FILE_WDIR),\n ('exec', 0, \"/some/dir/ls\", \"/some/dir\", b'ls\\0/some/cli\\0'),\n ('open', 0, \"/some/cli\", False, FILE_WRITE),\n ('open', 0, \"/some/r\", False, FILE_READ),\n ('open', 0, \"/some/rw\", False, FILE_READ),\n ('proc', 1, None, False),\n ('open', 1, \"/some/dir\", True, FILE_WDIR),\n ('exec', 1, \"/some/dir/ls\", \"/some/dir\", b'ls\\0'),\n ('open', 1, \"/some/cli\", False, FILE_READ),\n ('proc', 2, 1, True),\n ('open', 2, \"/some/r\", False, FILE_READ),\n ('open', 1, \"/some/rw\", False, FILE_WRITE),\n ])\n expected = set([\n '/some',\n '/some/dir',\n '/some/dir/ls',\n '/some/r',\n '/some/rw',\n ])\n self.assertEqualPaths(expected,\n set(fi.path for fi in files))\n self.assertEqualPaths([set([\"/some/r\", \"/some/rw\"]),\n set([\"/some/cli\", \"/some/r\"])],\n [set(run) for run in inputs])\n self.assertEqualPaths([set([\"/some/cli\"]), set([\"/some/rw\"])],\n [set(run) for run in outputs])\n finally:\n Path.is_file, Path.stat = old\n\n\nclass TestCombine(unittest.TestCase):\n def setUp(self):\n self.tmpdir = Path.tempdir()\n\n def tearDown(self):\n self.tmpdir.rmtree()\n\n def test_combine(self):\n traces = []\n schema = '''\nPRAGMA foreign_keys=OFF;\nBEGIN TRANSACTION;\nCREATE TABLE processes(\n id INTEGER NOT NULL PRIMARY KEY,\n run_id INTEGER NOT NULL,\n parent INTEGER,\n timestamp INTEGER NOT NULL,\n is_thread BOOLEAN NOT NULL,\n exitcode INTEGER\n );\nCREATE TABLE opened_files(\n id INTEGER NOT NULL PRIMARY KEY,\n run_id INTEGER NOT NULL,\n name TEXT NOT NULL,\n timestamp INTEGER NOT NULL,\n mode INTEGER NOT NULL,\n is_directory BOOLEAN NOT NULL,\n process INTEGER NOT NULL\n );\nCREATE TABLE executed_files(\n id INTEGER NOT NULL PRIMARY KEY,\n name TEXT NOT NULL,\n run_id INTEGER NOT NULL,\n timestamp INTEGER NOT NULL,\n process INTEGER NOT NULL,\n argv TEXT NOT NULL,\n envp TEXT NOT NULL,\n workingdir TEXT NOT NULL\n );\nCREATE INDEX proc_parent_idx ON processes(parent);\nCREATE INDEX open_proc_idx ON opened_files(process);\nCREATE INDEX exec_proc_idx ON executed_files(process);\n '''\n sql_data = [\n schema + '''\nINSERT INTO \"processes\" VALUES(1,0,NULL,12345678901001,0,0);\nINSERT INTO \"opened_files\" VALUES(1,0,'/home/vagrant',12345678901001,4,1,1);\nINSERT INTO \"opened_files\" VALUES(2,0,'/lib/ld.so',12345678901003,1,0,1);\nINSERT INTO \"executed_files\" VALUES(1,'/usr/bin/id',0,12345678901002,1,'id',\n 'RUN=first','/home/vagrant');\n ''',\n schema + '''\nINSERT INTO \"processes\" VALUES(1,0,NULL,12345678902001,0,0);\nINSERT INTO \"processes\" VALUES(2,0,1,12345678902002,1,0);\nINSERT INTO \"processes\" VALUES(3,1,NULL,12345678902004,0,0);\nINSERT INTO \"processes\" VALUES(4,1,3,12345678902005,0,1);\nINSERT INTO \"opened_files\" VALUES(1,0,'/usr',12345678902001,4,1,1);\nINSERT INTO \"opened_files\" VALUES(2,0,'/lib/ld.so',12345678902003,1,0,2);\nINSERT INTO \"opened_files\" VALUES(3,1,'/usr/bin',12345678902004,4,1,3);\nINSERT INTO \"executed_files\" VALUES(1,'/usr/bin/id',1,12345678902006,4,'id',\n 'RUN=third','/home/vagrant');\n ''',\n schema + '''\nINSERT INTO \"processes\" VALUES(0,0,NULL,12345678903001,0,1);\nINSERT INTO \"opened_files\" VALUES(0,0,'/home',12345678903001,4,1,0);\nINSERT INTO \"executed_files\" VALUES(1,'/bin/false',0,12345678903002,0,'false',\n 'RUN=fourth','/home');\n ''']\n\n for i, dat in enumerate(sql_data):\n trace = self.tmpdir / ('trace%d.sqlite3' % i)\n if PY3:\n conn = sqlite3.connect(str(trace))\n else:\n conn = sqlite3.connect(trace.path)\n conn.row_factory = sqlite3.Row\n conn.executescript(dat + 'COMMIT;')\n conn.commit()\n conn.close()\n\n traces.append(trace)\n\n target = self.tmpdir / 'target'\n traceutils.combine_traces(traces, target)\n target = target / 'trace.sqlite3'\n\n if PY3:\n conn = sqlite3.connect(str(target))\n else:\n conn = sqlite3.connect(target.path)\n conn.row_factory = None\n processes = list(conn.execute(\n '''\n SELECT * FROM processes;\n '''))\n opened_files = list(conn.execute(\n '''\n SELECT * FROM opened_files;\n '''))\n executed_files = list(conn.execute(\n '''\n SELECT * FROM executed_files;\n '''))\n\n self.assertEqual([processes, opened_files, executed_files], [\n [(1, 1, None, 12345678901001, 0, 0),\n (2, 2, None, 12345678902001, 0, 0),\n (3, 2, 1, 12345678902002, 1, 0),\n (4, 3, None, 12345678902004, 0, 0),\n (5, 3, 3, 12345678902005, 0, 1),\n (6, 4, None, 12345678903001, 0, 1)],\n\n [(1, 1, '/home/vagrant', 12345678901001, 4, 1, 1),\n (2, 1, '/lib/ld.so', 12345678901003, 1, 0, 1),\n (3, 2, '/usr', 12345678902001, 4, 1, 2),\n (4, 2, '/lib/ld.so', 12345678902003, 1, 0, 3),\n (5, 3, '/usr/bin', 12345678902004, 4, 1, 4),\n (6, 4, '/home', 12345678903001, 4, 1, 6)],\n\n [(1, '/usr/bin/id', 1, 12345678901002, 1, 'id',\n 'RUN=first', '/home/vagrant'),\n (2, '/usr/bin/id', 3, 12345678902006, 5, 'id',\n 'RUN=third', '/home/vagrant'),\n (3, '/bin/false', 4, 12345678903002, 6, 'false',\n 'RUN=fourth', '/home')],\n ])\n" }, { "path": "tests/test_unpackers_common.py", "content": "# Copyright (C) 2014 New York University\n# This file is part of ReproZip which is released under the Revised BSD License\n# See file LICENSE for full license details.\n\nfrom __future__ import print_function, unicode_literals\n\nimport os\nimport sys\nimport unittest\n\nfrom reprounzip.unpackers.common import UsageError, \\\n unique_names, make_unique_name, get_runs, fixup_environment\nfrom reprounzip.utils import irange\n\n\nclass TestCommon(unittest.TestCase):\n def test_unique_names(self):\n \"\"\"Tests the unique_names generator.\"\"\"\n names = [next(unique_names) for i in irange(3)]\n for n in names:\n self.assertTrue(n and isinstance(n, bytes))\n self.assertEqual(len(set(names)), len(names))\n\n def test_make_unique_name(self):\n \"\"\"Tests the make_unique_name() function.\"\"\"\n names = [make_unique_name(b'/some/prefix_') for i in irange(3)]\n for n in names:\n self.assertTrue(n and isinstance(n, bytes) and\n n[:13] == b'/some/prefix_')\n self.assertEqual(len(set(names)), len(names))\n\n def test_env(self):\n \"\"\"Tests fixing environment variables\"\"\"\n outer_env = {\n 'OUTONLY': 'outvalue',\n 'COMMON': 'commonvalueout',\n 'SHARED': 'sharedvalue',\n 'EMPTY': '',\n }\n inner_env = {\n 'INONLY': 'invalue',\n 'COMMON': 'commonvaluein',\n 'SHARED': 'sharedvalue',\n }\n\n class FakeArgs(object):\n def __init__(self, pass_env, set_env):\n self.pass_env = pass_env\n self.set_env = set_env\n\n old_environ, os.environ = os.environ, outer_env\n try:\n self.assertEqual(\n fixup_environment(\n inner_env,\n FakeArgs([], [])),\n {\n 'INONLY': 'invalue',\n 'COMMON': 'commonvaluein',\n 'SHARED': 'sharedvalue',\n })\n\n self.assertEqual(\n fixup_environment(\n inner_env,\n FakeArgs(['COMMON', 'INONLY', 'OUTONLY', 'EMPTY'], [])),\n {\n 'INONLY': 'invalue',\n 'OUTONLY': 'outvalue',\n 'COMMON': 'commonvalueout',\n 'SHARED': 'sharedvalue',\n 'EMPTY': '',\n })\n\n self.assertEqual(\n fixup_environment(\n inner_env,\n FakeArgs(['OUTONLY'],\n ['SHARED=surprise', 'COMMON=', 'INONLY'])),\n {\n 'OUTONLY': 'outvalue',\n 'COMMON': '',\n 'SHARED': 'surprise',\n })\n\n self.assertEqual(\n fixup_environment(\n inner_env,\n FakeArgs(['.*Y$'], [])),\n {\n 'INONLY': 'invalue',\n 'OUTONLY': 'outvalue',\n 'COMMON': 'commonvaluein',\n 'SHARED': 'sharedvalue',\n 'EMPTY': '',\n })\n finally:\n os.environ = old_environ\n\n\nclass TestMisc(unittest.TestCase):\n def do_ok(self, arg, expected, nruns=4):\n try:\n config_runs = [{'id': 'one'}, {'id': 'two-heh'},\n {'id': 'three'}, {}]\n runs = get_runs(\n config_runs[:nruns],\n arg, None)\n except (SystemExit, UsageError):\n self.fail(\"get_runs(<4 runs>, %r) raised\" % arg)\n self.assertEqual(list(runs), expected)\n\n def do_fail(self, arg):\n self.assertRaises((SystemExit, UsageError),\n get_runs, [{}, {}, {}, {}], arg, None)\n\n def test_get_runs(self):\n \"\"\"Tests get_runs(), parsing runs from the command-line.\"\"\"\n print(\"<<<<< get_runs tests for reprounzip (disregard parse errors)\",\n file=sys.stderr)\n try:\n self.do_fail('')\n self.do_fail('a-')\n self.do_fail('1-k')\n self.do_ok(None, [0, 1, 2, 3])\n self.do_ok('-', [0, 1, 2, 3])\n self.do_ok(None, [0], nruns=1)\n self.do_ok('-', [0], nruns=1)\n self.do_ok('1-', [1, 2, 3])\n self.do_ok('-2', [0, 1, 2])\n self.do_ok('1-2', [1, 2])\n self.do_ok('0-2', [0, 1, 2])\n self.do_ok('1-3', [1, 2, 3])\n self.do_ok('1-1', [1])\n self.do_fail('2-1')\n self.do_fail('0-8')\n self.do_fail('0-4')\n self.do_ok('0-3', [0, 1, 2, 3])\n self.do_ok('one', [0])\n self.do_ok('two-heh', [1]),\n self.do_ok('one,three', [0, 2])\n self.do_ok('1,three', [1, 2])\n self.do_ok('2-3,two-heh', [2, 3, 1])\n finally:\n print(\">>>>> get_runs tests\", file=sys.stderr)\n" }, { "path": "tests/test_utils.py", "content": "# Copyright (C) 2014 New York University\n# This file is part of ReproZip which is released under the Revised BSD License\n# See file LICENSE for full license details.\n\nimport unittest\n\nfrom reprounzip.utils import optional_return_type\n\n\nclass TestOptionalReturnType(unittest.TestCase):\n def test_namedtuple(self):\n T = optional_return_type(['a', 'b', 'c'], [])\n for o in [T(1, 2, 3), T(1, b=2, c=3), T(a=1, b=2, c=3)]:\n self.assertEqual(o, (1, 2, 3))\n self.assertEqual(o[0], 1)\n self.assertEqual(o[2], 3)\n self.assertRaises(IndexError, lambda: o[3])\n self.assertEqual(o.a, 1)\n self.assertEqual(o.c, 3)\n self.assertRaises(TypeError, lambda: T(1, 2))\n self.assertRaises(TypeError, lambda: T(a=1, b=2))\n self.assertRaises(TypeError, lambda: T(a=1, b=2))\n\n def test_with_opt(self):\n T = optional_return_type(['a', 'b'], ['c'])\n for o in [T(1, 2, 3), T(1, b=2, c=3), T(a=1, b=2, c=3)]:\n self.assertEqual(o, (1, 2))\n self.assertEqual(o[0], 1)\n self.assertEqual(o[1], 2)\n self.assertRaises(IndexError, lambda: o[2])\n self.assertEqual(o.a, 1)\n self.assertEqual(o.b, 2)\n self.assertEqual(o.c, 3)\n for o in [T(1, 2), T(1, b=2), T(a=1, b=2)]:\n self.assertEqual(o, (1, 2))\n self.assertEqual(o[0], 1)\n self.assertEqual(o[1], 2)\n self.assertRaises(IndexError, lambda: o[2])\n self.assertEqual(o.a, 1)\n self.assertEqual(o.b, 2)\n self.assertRaises(AttributeError, lambda: o.c)\n self.assertRaises(TypeError, lambda: T(1))\n self.assertRaises(TypeError, lambda: T(b=1, c=2))\n self.assertRaises(TypeError, lambda: T(c=1))\n" }, { "path": "tests/threads.c", "content": "/* threads.c\n *\n * This is a very simple threaded program.\n *\n * usage: ./threads\n */\n\n#include \n#include \n\n\nvoid *func1(void *param)\n{\n static retvalue = 42;\n chdir(\"/bin\");\n usleep(300000);\n return &retvalue;\n}\n\nvoid *func23(void *param)\n{\n usleep(1000000);\n return NULL;\n}\n\nvoid *func4(void *param)\n{\n char *argv[3] = {\"echo\", \"42\", NULL};\n usleep(600000);\n execvp(\"./echo\", argv);\n perror(\"execvp\");\n return NULL;\n}\n\nint main(void)\n{\n pthread_t th1, th2, th3, th4;\n pthread_create(&th1, NULL, func1, NULL);\n pthread_create(&th2, NULL, func23, NULL);\n pthread_create(&th3, NULL, func23, NULL);\n pthread_create(&th4, NULL, func4, NULL);\n\n {\n void *retval;\n pthread_join(th1, &retval);\n if(*(int*)retval != 42)\n {\n fprintf(stderr, \"Invalid return from thread 1\\n\");\n return 1;\n }\n }\n\n pthread_join(th4, NULL);\n /* Won't be reached */\n return 2;\n}\n" }, { "path": "tests/threads2.c", "content": "/* threads2.c\n *\n * A second multithreaded program.\n *\n * usage: ./threads2\n */\n\n#include \n#include \n\n\nint search(void)\n{\n int i;\n int sum = 0;\n for(i = 0; sum != 10; ++i)\n {\n int a = 1, b = 1;\n int j;\n for(j = 1; j < i; ++j)\n {\n int c = a + b;\n a = b;\n b = c;\n }\n sum += b;\n }\n return i;\n}\n\nvoid *funcT(void *param)\n{\n char *argv[3] = {\"echo\", \"42\", NULL};\n usleep(200000);\n execvp(\"/bin/echo\", argv);\n perror(\"execvp\");\n return NULL;\n}\n\nint main(void)\n{\n int r;\n pthread_t th;\n pthread_create(&th, NULL, funcT, NULL);\n\n r = search();\n /* Won't be reached */\n printf(\"%d\\n\", r);\n pthread_join(th, NULL);\n return 2;\n}\n" }, { "path": "tests/vfork.c", "content": "/* vfork.c\n *\n * This just calls vfork() then execve().\n *\n * usage: ./vfork\n */\n\n#include \n#include \n#include \n\n\nint main(void)\n{\n pid_t res = vfork();\n if(res == 0)\n {\n char *args[] = {\"echo\", \"hello\", NULL};\n execv(\"/bin/echo\", args);\n }\n else if(res > 0)\n {\n int status;\n waitpid(res, &status, 0);\n if(WIFEXITED(status))\n return 0;\n else\n return 1;\n }\n else\n {\n perror(\"vfork\");\n return 2;\n }\n}\n" } ]