[
  {
    "path": ".github/ISSUE_TEMPLATE/bug_report.md",
    "content": "---\nname: Bug report\nabout: Create a report to help us improve\ntitle: ''\nlabels: ''\nassignees: ''\n\n---\n\n**Describe the bug**\nA clear and concise description of what the bug is.\n\n**To Reproduce**\nSteps to reproduce the behavior:\n1. Go to '...'\n2. Click on '....'\n3. Scroll down to '....'\n4. See error\n\n**Expected behavior**\nA clear and concise description of what you expected to happen.\n\n**Screenshots**\nIf applicable, add screenshots to help explain your problem.\n\n**Desktop (please complete the following information):**\n - OS: [e.g. iOS]\n - Version [e.g. v0.57, can be obtained by running `mashtree --version`.]\n - which method did you install with?  E.g., `cpanm`, `git clone`, ...\n\n\n**Additional context**\nAdd any other context about the problem here.\n"
  },
  {
    "path": ".github/ISSUE_TEMPLATE/feature_request.md",
    "content": "---\nname: Feature request\nabout: Suggest an idea for this project\ntitle: ''\nlabels: ''\nassignees: ''\n\n---\n\n**Is your feature request related to a problem? Please describe.**\nA clear and concise description of what the problem is. Ex. I'm always frustrated when [...]\n\n**Describe the solution you'd like**\nA clear and concise description of what you want to happen.\n\n**Describe alternatives you've considered**\nA clear and concise description of any alternative solutions or features you've considered.\n\n**Additional context**\nAdd any other context or screenshots about the feature request here.\n"
  },
  {
    "path": ".github/workflows/cache.yml",
    "content": "name: force-cache\non: [push]\n\njobs:\n  build:\n    runs-on: ${{ matrix.os }}\n    strategy:\n      matrix:\n        os: ['ubuntu-22.04']\n        perl: [ '5.34' ]\n    name: Perl ${{ matrix.perl }} on ${{ matrix.os }}\n    steps:\n      - name: apt-get to prepare for cached perl packages\n        run:  |\n          sudo apt-get update -y\n          sudo apt-get install -y libdb-dev sqlite3 libgd-dev libsqlite3-dev libberkeleydb-perl libcdb-file-perl\n      - uses: actions/checkout@v3\n      - uses: shogo82148/actions-setup-perl@v1\n        with:\n          perl-version: '5.34'\n          install-modules-with: cpanm\n          install-modules-args: --with-develop --with-configure\n          multi-thread: true\n          install-modules: |\n            JSON\n            Bio::Perl\n            Bio::Kmer\n            Bio::Tree::Statistics\n            Bio::Matrix::IO\n            Bio::Tree::DistanceFactory\n      - run: |\n          echo \"hello world!\"\n          # prove -lv t\n"
  },
  {
    "path": ".github/workflows/perl_ci.yml",
    "content": "# https://docs.github.com/en/actions/reference/workflow-syntax-for-github-actions\n# https://docs.github.com/en/actions/guides/publishing-docker-images\n# https://github.com/docker/build-push-action/blob/master/docs/advanced/share-image-jobs.md\nname: unit-testing\non: [push]\n\njobs:\n  build:\n    runs-on: ${{ matrix.os }}\n    strategy:\n      matrix:\n        os: ['ubuntu-22.04']\n        perl: [ '5.34' ]\n        #perl: [ '5.34', '5.32', '5.16.3' ]\n    name: Perl ${{ matrix.perl }} on ${{ matrix.os }}\n    steps:\n      - uses: actions/checkout@v3\n      - name: Set up perl\n        uses: shogo82148/actions-setup-perl@v1\n        with:\n          perl-version: ${{ matrix.perl }}\n          install-modules-with: cpanm\n          install-modules-args: --with-develop --with-configure --force --notest\n          enable-modules-cache: true\n          multi-thread: true\n          install-modules: |\n            JSON\n            Bio::Perl\n            Bio::TreeIO\n            Bio::Kmer\n            Bio::Tree::Statistics\n            Bio::Matrix::IO\n            Bio::Tree::DistanceFactory\n            Bio::Sketch::Mash\n      - name: apt-get to prepare for cached perl packages\n        run:  |\n          sudo apt-get update -y\n          sudo apt-get install -y libdb-dev sqlite3 libgd-dev libsqlite3-dev libberkeleydb-perl libcdb-file-perl quicktree mash\n      #- name: cpanm installdeps --notest --force\n      #  run: cpanm --installdeps . --notest --force --verbose\n      - name: versions\n        run:  |\n          mash --version\n          quicktree -v\n          sqlite3 --version\n      - run: perl -V\n      - name: perl modules checks\n        run: |\n          for i in Bio::Tree::Statistics threads Bio::Matrix::IO Bio::Tree::DistanceFactory; do perl -M$i -e 'print \"checking module $i\";'; done\n      - name: make test\n        run: perl Makefile.PL && make && make test\n"
  },
  {
    "path": ".gitignore",
    "content": "!Build/\n.last_cover_stats\n/META.yml\n/META.json\n/MYMETA.*\n*.o\n*.pm.tdy\n*.bs\n\n# Devel::Cover\ncover_db/\n\n# Devel::NYTProf\nnytprof.out\n\n# Dist::Zilla\n/.build/\n\n# Module::Build\n_build/\nBuild\nBuild.bat\n\n# Module::Install\ninc/\n\n# ExtUtils::MakeMaker\n/blib/\n/_eumm/\n/*.gz\n/Makefile\n/Makefile.old\n/MANIFEST.bak\n/pm_to_blib\n/*.zip\n\n# temp testing files\nt/lambda/lambda.fofn\nt/lambda/lambda/\nt/lambda/list.lst\nt/lambda/mashtree2.tmp/\nt/lambda/sketches/\nt/lambda/tmp.dnd\nt/lambda/tmp/\nbin/ROSS-0.3*\n"
  },
  {
    "path": ".travis.yml.bak",
    "content": "language: perl\nperl:\n  - \"5.30-shrplib\"\nenv: PERL_CPANM_OPT=\"--notest --force --skip-satisfied\"\naddons:\n  apt:\n    update: true\n    packages:\n      - bioperl\n      - libgd-dev\n      - sqlite3\n      - libsqlite3-dev\n      - libberkeleydb-perl\n      - libcdb-file-perl\nbefore_install:\n  - cpanm --local-lib=~/perl5 local::lib && eval $(perl -I ~/perl5/lib/perl5/ -Mlocal::lib)\n  # bootstrap modules needed to manipulate dist\n  - \"yes no | cpanm --force --notest Bio::Perl\"\n  #- cpanm -v -L . DBD::SQLite\n  #- \"yes no | cpanm --force --notest DBI\"\n  #- wget https://github.com/marbl/Mash/releases/download/v1.1.1/mash-Linux64-v1.1.1.tar.gz\n  #- tar zxvf mash-Linux64-v1.1.1.tar.gz\n  - wget https://github.com/marbl/Mash/releases/download/v2.2/mash-Linux64-v2.2.tar\n  - tar xvf mash-Linux64-v2.2.tar\n  - git clone https://github.com/khowe/quicktree.git\n  - make -C quicktree\ninstall:\n  - export PATH=$PATH:$HOME/bin\n  - export PERL5LIB=$PERL5LIB:$(pwd)/lib/perl5\n  - export PERL5LIB=$PERL5LIB:$HOME/perl5:$HOME/perl5/lib/perl5\nbefore_script:\n  - ls -F # if this test fails, just be sure what the directory is\n  - pwd; pwd -P\n  - PATH=$PATH:./mash-Linux64-v2.2\n  - export PATH=$PATH:./quicktree\nscript:\n  - cpanm -l . --notest --force -v Bio::Tree::Statistics Bio::Matrix::IO Bio::Tree::DistanceFactory\n  - find . -type f -name Statistics.pm\n  #- find . -type f -name DistanceFactory.pm || true\n  #- cpanm --notest --force --verbose -l . GD\n  #- cpanm --notest --force -l . CDRAUG/BioPerl-1.7.4.tar.gz\n  - cpanm --installdeps --notest .\n  - perl Makefile.PL \n  - perl bin/mashtree -h 2>&1 | grep .\n  - perl bin/mashtree --help 2>&1 | grep .\n  - make test\n\n"
  },
  {
    "path": "CHANGES",
    "content": "v1.4 removed SQL dependency\n\nv0.55 bootstrapping\n\nv0.41\n* Fixed some unit test bugs from v0.40. Addresses issue #37 on GitHub.\n\nv0.40\n* jack knifing enabled with hashes\n\nv0.37 \n* mashtree reps more stablized; reps unit test\n\nv0.36\n* Fixed a bug with multithreaded accurate mode (--min-depth 0) and bumped the version up so that cpan could catch it.\n"
  },
  {
    "path": "CONTRIBUTING.md",
    "content": "# How to Contribute\n\n## Getting Started\n\n* Make sure you have a GitHub account\n* Make sure you have a travis-ci account\n* **Add an issue to the github site**. Discuss the problem you are fixing with the maintainer.\n  * Has the issue already been discussed on the GitHub issues page?\n  * Be detailed and focused\n\n## Making changes\n\n* Fork the Mashtree repo\n* Create your own new branch and name it with something descriptive and your username, e.g., `lskatz-fix-dendrogram-formatting`.\n* Fix the issue on your new branch\n  * Make commits of logical and atomic units\n  * Add any new unit tests to help with longetivity. Unit tests have a file extension `.t` and are written in perl.  More details on unit testing can be found here: https://perldoc.perl.org/Test/More.html\n  * If the unit test(s) show any errors on your local computer or Travis-CI, then fix these errors.\n* Make a pull request when all unit tests show no errors.\n\n"
  },
  {
    "path": "LICENSE",
    "content": "                    GNU GENERAL PUBLIC LICENSE\n                       Version 3, 29 June 2007\n\n Copyright (C) 2007 Free Software Foundation, Inc. <http://fsf.org/>\n Everyone is permitted to copy and distribute verbatim copies\n of this license document, but changing it is not allowed.\n\n                            Preamble\n\n  The GNU General Public License is a free, copyleft license for\nsoftware and other kinds of works.\n\n  The licenses for most software and other practical works are designed\nto take away your freedom to share and change the works.  By contrast,\nthe GNU General Public License is intended to guarantee your freedom to\nshare and change all versions of a program--to make sure it remains free\nsoftware for all its users.  We, the Free Software Foundation, use the\nGNU General Public License for most of our software; it applies also to\nany other work released this way by its authors.  You can apply it to\nyour programs, too.\n\n  When we speak of free software, we are referring to freedom, not\nprice.  Our General Public Licenses are designed to make sure that you\nhave the freedom to distribute copies of free software (and charge for\nthem if you wish), that you receive source code or can get it if you\nwant it, that you can change the software or use pieces of it in new\nfree programs, and that you know you can do these things.\n\n  To protect your rights, we need to prevent others from denying you\nthese rights or asking you to surrender the rights.  Therefore, you have\ncertain responsibilities if you distribute copies of the software, or if\nyou modify it: responsibilities to respect the freedom of others.\n\n  For example, if you distribute copies of such a program, whether\ngratis or for a fee, you must pass on to the recipients the same\nfreedoms that you received.  You must make sure that they, too, receive\nor can get the source code.  And you must show them these terms so they\nknow their rights.\n\n  Developers that use the GNU GPL protect your rights with two steps:\n(1) assert copyright on the software, and (2) offer you this License\ngiving you legal permission to copy, distribute and/or modify it.\n\n  For the developers' and authors' protection, the GPL clearly explains\nthat there is no warranty for this free software.  For both users' and\nauthors' sake, the GPL requires that modified versions be marked as\nchanged, so that their problems will not be attributed erroneously to\nauthors of previous versions.\n\n  Some devices are designed to deny users access to install or run\nmodified versions of the software inside them, although the manufacturer\ncan do so.  This is fundamentally incompatible with the aim of\nprotecting users' freedom to change the software.  The systematic\npattern of such abuse occurs in the area of products for individuals to\nuse, which is precisely where it is most unacceptable.  Therefore, we\nhave designed this version of the GPL to prohibit the practice for those\nproducts.  If such problems arise substantially in other domains, we\nstand ready to extend this provision to those domains in future versions\nof the GPL, as needed to protect the freedom of users.\n\n  Finally, every program is threatened constantly by software patents.\nStates should not allow patents to restrict development and use of\nsoftware on general-purpose computers, but in those that do, we wish to\navoid the special danger that patents applied to a free program could\nmake it effectively proprietary.  To prevent this, the GPL assures that\npatents cannot be used to render the program non-free.\n\n  The precise terms and conditions for copying, distribution and\nmodification follow.\n\n                       TERMS AND CONDITIONS\n\n  0. Definitions.\n\n  \"This License\" refers to version 3 of the GNU General Public License.\n\n  \"Copyright\" also means copyright-like laws that apply to other kinds of\nworks, such as semiconductor masks.\n\n  \"The Program\" refers to any copyrightable work licensed under this\nLicense.  Each licensee is addressed as \"you\".  \"Licensees\" and\n\"recipients\" may be individuals or organizations.\n\n  To \"modify\" a work means to copy from or adapt all or part of the work\nin a fashion requiring copyright permission, other than the making of an\nexact copy.  The resulting work is called a \"modified version\" of the\nearlier work or a work \"based on\" the earlier work.\n\n  A \"covered work\" means either the unmodified Program or a work based\non the Program.\n\n  To \"propagate\" a work means to do anything with it that, without\npermission, would make you directly or secondarily liable for\ninfringement under applicable copyright law, except executing it on a\ncomputer or modifying a private copy.  Propagation includes copying,\ndistribution (with or without modification), making available to the\npublic, and in some countries other activities as well.\n\n  To \"convey\" a work means any kind of propagation that enables other\nparties to make or receive copies.  Mere interaction with a user through\na computer network, with no transfer of a copy, is not conveying.\n\n  An interactive user interface displays \"Appropriate Legal Notices\"\nto the extent that it includes a convenient and prominently visible\nfeature that (1) displays an appropriate copyright notice, and (2)\ntells the user that there is no warranty for the work (except to the\nextent that warranties are provided), that licensees may convey the\nwork under this License, and how to view a copy of this License.  If\nthe interface presents a list of user commands or options, such as a\nmenu, a prominent item in the list meets this criterion.\n\n  1. Source Code.\n\n  The \"source code\" for a work means the preferred form of the work\nfor making modifications to it.  \"Object code\" means any non-source\nform of a work.\n\n  A \"Standard Interface\" means an interface that either is an official\nstandard defined by a recognized standards body, or, in the case of\ninterfaces specified for a particular programming language, one that\nis widely used among developers working in that language.\n\n  The \"System Libraries\" of an executable work include anything, other\nthan the work as a whole, that (a) is included in the normal form of\npackaging a Major Component, but which is not part of that Major\nComponent, and (b) serves only to enable use of the work with that\nMajor Component, or to implement a Standard Interface for which an\nimplementation is available to the public in source code form.  A\n\"Major Component\", in this context, means a major essential component\n(kernel, window system, and so on) of the specific operating system\n(if any) on which the executable work runs, or a compiler used to\nproduce the work, or an object code interpreter used to run it.\n\n  The \"Corresponding Source\" for a work in object code form means all\nthe source code needed to generate, install, and (for an executable\nwork) run the object code and to modify the work, including scripts to\ncontrol those activities.  However, it does not include the work's\nSystem Libraries, or general-purpose tools or generally available free\nprograms which are used unmodified in performing those activities but\nwhich are not part of the work.  For example, Corresponding Source\nincludes interface definition files associated with source files for\nthe work, and the source code for shared libraries and dynamically\nlinked subprograms that the work is specifically designed to require,\nsuch as by intimate data communication or control flow between those\nsubprograms and other parts of the work.\n\n  The Corresponding Source need not include anything that users\ncan regenerate automatically from other parts of the Corresponding\nSource.\n\n  The Corresponding Source for a work in source code form is that\nsame work.\n\n  2. Basic Permissions.\n\n  All rights granted under this License are granted for the term of\ncopyright on the Program, and are irrevocable provided the stated\nconditions are met.  This License explicitly affirms your unlimited\npermission to run the unmodified Program.  The output from running a\ncovered work is covered by this License only if the output, given its\ncontent, constitutes a covered work.  This License acknowledges your\nrights of fair use or other equivalent, as provided by copyright law.\n\n  You may make, run and propagate covered works that you do not\nconvey, without conditions so long as your license otherwise remains\nin force.  You may convey covered works to others for the sole purpose\nof having them make modifications exclusively for you, or provide you\nwith facilities for running those works, provided that you comply with\nthe terms of this License in conveying all material for which you do\nnot control copyright.  Those thus making or running the covered works\nfor you must do so exclusively on your behalf, under your direction\nand control, on terms that prohibit them from making any copies of\nyour copyrighted material outside their relationship with you.\n\n  Conveying under any other circumstances is permitted solely under\nthe conditions stated below.  Sublicensing is not allowed; section 10\nmakes it unnecessary.\n\n  3. Protecting Users' Legal Rights From Anti-Circumvention Law.\n\n  No covered work shall be deemed part of an effective technological\nmeasure under any applicable law fulfilling obligations under article\n11 of the WIPO copyright treaty adopted on 20 December 1996, or\nsimilar laws prohibiting or restricting circumvention of such\nmeasures.\n\n  When you convey a covered work, you waive any legal power to forbid\ncircumvention of technological measures to the extent such circumvention\nis effected by exercising rights under this License with respect to\nthe covered work, and you disclaim any intention to limit operation or\nmodification of the work as a means of enforcing, against the work's\nusers, your or third parties' legal rights to forbid circumvention of\ntechnological measures.\n\n  4. Conveying Verbatim Copies.\n\n  You may convey verbatim copies of the Program's source code as you\nreceive it, in any medium, provided that you conspicuously and\nappropriately publish on each copy an appropriate copyright notice;\nkeep intact all notices stating that this License and any\nnon-permissive terms added in accord with section 7 apply to the code;\nkeep intact all notices of the absence of any warranty; and give all\nrecipients a copy of this License along with the Program.\n\n  You may charge any price or no price for each copy that you convey,\nand you may offer support or warranty protection for a fee.\n\n  5. Conveying Modified Source Versions.\n\n  You may convey a work based on the Program, or the modifications to\nproduce it from the Program, in the form of source code under the\nterms of section 4, provided that you also meet all of these conditions:\n\n    a) The work must carry prominent notices stating that you modified\n    it, and giving a relevant date.\n\n    b) The work must carry prominent notices stating that it is\n    released under this License and any conditions added under section\n    7.  This requirement modifies the requirement in section 4 to\n    \"keep intact all notices\".\n\n    c) You must license the entire work, as a whole, under this\n    License to anyone who comes into possession of a copy.  This\n    License will therefore apply, along with any applicable section 7\n    additional terms, to the whole of the work, and all its parts,\n    regardless of how they are packaged.  This License gives no\n    permission to license the work in any other way, but it does not\n    invalidate such permission if you have separately received it.\n\n    d) If the work has interactive user interfaces, each must display\n    Appropriate Legal Notices; however, if the Program has interactive\n    interfaces that do not display Appropriate Legal Notices, your\n    work need not make them do so.\n\n  A compilation of a covered work with other separate and independent\nworks, which are not by their nature extensions of the covered work,\nand which are not combined with it such as to form a larger program,\nin or on a volume of a storage or distribution medium, is called an\n\"aggregate\" if the compilation and its resulting copyright are not\nused to limit the access or legal rights of the compilation's users\nbeyond what the individual works permit.  Inclusion of a covered work\nin an aggregate does not cause this License to apply to the other\nparts of the aggregate.\n\n  6. Conveying Non-Source Forms.\n\n  You may convey a covered work in object code form under the terms\nof sections 4 and 5, provided that you also convey the\nmachine-readable Corresponding Source under the terms of this License,\nin one of these ways:\n\n    a) Convey the object code in, or embodied in, a physical product\n    (including a physical distribution medium), accompanied by the\n    Corresponding Source fixed on a durable physical medium\n    customarily used for software interchange.\n\n    b) Convey the object code in, or embodied in, a physical product\n    (including a physical distribution medium), accompanied by a\n    written offer, valid for at least three years and valid for as\n    long as you offer spare parts or customer support for that product\n    model, to give anyone who possesses the object code either (1) a\n    copy of the Corresponding Source for all the software in the\n    product that is covered by this License, on a durable physical\n    medium customarily used for software interchange, for a price no\n    more than your reasonable cost of physically performing this\n    conveying of source, or (2) access to copy the\n    Corresponding Source from a network server at no charge.\n\n    c) Convey individual copies of the object code with a copy of the\n    written offer to provide the Corresponding Source.  This\n    alternative is allowed only occasionally and noncommercially, and\n    only if you received the object code with such an offer, in accord\n    with subsection 6b.\n\n    d) Convey the object code by offering access from a designated\n    place (gratis or for a charge), and offer equivalent access to the\n    Corresponding Source in the same way through the same place at no\n    further charge.  You need not require recipients to copy the\n    Corresponding Source along with the object code.  If the place to\n    copy the object code is a network server, the Corresponding Source\n    may be on a different server (operated by you or a third party)\n    that supports equivalent copying facilities, provided you maintain\n    clear directions next to the object code saying where to find the\n    Corresponding Source.  Regardless of what server hosts the\n    Corresponding Source, you remain obligated to ensure that it is\n    available for as long as needed to satisfy these requirements.\n\n    e) Convey the object code using peer-to-peer transmission, provided\n    you inform other peers where the object code and Corresponding\n    Source of the work are being offered to the general public at no\n    charge under subsection 6d.\n\n  A separable portion of the object code, whose source code is excluded\nfrom the Corresponding Source as a System Library, need not be\nincluded in conveying the object code work.\n\n  A \"User Product\" is either (1) a \"consumer product\", which means any\ntangible personal property which is normally used for personal, family,\nor household purposes, or (2) anything designed or sold for incorporation\ninto a dwelling.  In determining whether a product is a consumer product,\ndoubtful cases shall be resolved in favor of coverage.  For a particular\nproduct received by a particular user, \"normally used\" refers to a\ntypical or common use of that class of product, regardless of the status\nof the particular user or of the way in which the particular user\nactually uses, or expects or is expected to use, the product.  A product\nis a consumer product regardless of whether the product has substantial\ncommercial, industrial or non-consumer uses, unless such uses represent\nthe only significant mode of use of the product.\n\n  \"Installation Information\" for a User Product means any methods,\nprocedures, authorization keys, or other information required to install\nand execute modified versions of a covered work in that User Product from\na modified version of its Corresponding Source.  The information must\nsuffice to ensure that the continued functioning of the modified object\ncode is in no case prevented or interfered with solely because\nmodification has been made.\n\n  If you convey an object code work under this section in, or with, or\nspecifically for use in, a User Product, and the conveying occurs as\npart of a transaction in which the right of possession and use of the\nUser Product is transferred to the recipient in perpetuity or for a\nfixed term (regardless of how the transaction is characterized), the\nCorresponding Source conveyed under this section must be accompanied\nby the Installation Information.  But this requirement does not apply\nif neither you nor any third party retains the ability to install\nmodified object code on the User Product (for example, the work has\nbeen installed in ROM).\n\n  The requirement to provide Installation Information does not include a\nrequirement to continue to provide support service, warranty, or updates\nfor a work that has been modified or installed by the recipient, or for\nthe User Product in which it has been modified or installed.  Access to a\nnetwork may be denied when the modification itself materially and\nadversely affects the operation of the network or violates the rules and\nprotocols for communication across the network.\n\n  Corresponding Source conveyed, and Installation Information provided,\nin accord with this section must be in a format that is publicly\ndocumented (and with an implementation available to the public in\nsource code form), and must require no special password or key for\nunpacking, reading or copying.\n\n  7. Additional Terms.\n\n  \"Additional permissions\" are terms that supplement the terms of this\nLicense by making exceptions from one or more of its conditions.\nAdditional permissions that are applicable to the entire Program shall\nbe treated as though they were included in this License, to the extent\nthat they are valid under applicable law.  If additional permissions\napply only to part of the Program, that part may be used separately\nunder those permissions, but the entire Program remains governed by\nthis License without regard to the additional permissions.\n\n  When you convey a copy of a covered work, you may at your option\nremove any additional permissions from that copy, or from any part of\nit.  (Additional permissions may be written to require their own\nremoval in certain cases when you modify the work.)  You may place\nadditional permissions on material, added by you to a covered work,\nfor which you have or can give appropriate copyright permission.\n\n  Notwithstanding any other provision of this License, for material you\nadd to a covered work, you may (if authorized by the copyright holders of\nthat material) supplement the terms of this License with terms:\n\n    a) Disclaiming warranty or limiting liability differently from the\n    terms of sections 15 and 16 of this License; or\n\n    b) Requiring preservation of specified reasonable legal notices or\n    author attributions in that material or in the Appropriate Legal\n    Notices displayed by works containing it; or\n\n    c) Prohibiting misrepresentation of the origin of that material, or\n    requiring that modified versions of such material be marked in\n    reasonable ways as different from the original version; or\n\n    d) Limiting the use for publicity purposes of names of licensors or\n    authors of the material; or\n\n    e) Declining to grant rights under trademark law for use of some\n    trade names, trademarks, or service marks; or\n\n    f) Requiring indemnification of licensors and authors of that\n    material by anyone who conveys the material (or modified versions of\n    it) with contractual assumptions of liability to the recipient, for\n    any liability that these contractual assumptions directly impose on\n    those licensors and authors.\n\n  All other non-permissive additional terms are considered \"further\nrestrictions\" within the meaning of section 10.  If the Program as you\nreceived it, or any part of it, contains a notice stating that it is\ngoverned by this License along with a term that is a further\nrestriction, you may remove that term.  If a license document contains\na further restriction but permits relicensing or conveying under this\nLicense, you may add to a covered work material governed by the terms\nof that license document, provided that the further restriction does\nnot survive such relicensing or conveying.\n\n  If you add terms to a covered work in accord with this section, you\nmust place, in the relevant source files, a statement of the\nadditional terms that apply to those files, or a notice indicating\nwhere to find the applicable terms.\n\n  Additional terms, permissive or non-permissive, may be stated in the\nform of a separately written license, or stated as exceptions;\nthe above requirements apply either way.\n\n  8. Termination.\n\n  You may not propagate or modify a covered work except as expressly\nprovided under this License.  Any attempt otherwise to propagate or\nmodify it is void, and will automatically terminate your rights under\nthis License (including any patent licenses granted under the third\nparagraph of section 11).\n\n  However, if you cease all violation of this License, then your\nlicense from a particular copyright holder is reinstated (a)\nprovisionally, unless and until the copyright holder explicitly and\nfinally terminates your license, and (b) permanently, if the copyright\nholder fails to notify you of the violation by some reasonable means\nprior to 60 days after the cessation.\n\n  Moreover, your license from a particular copyright holder is\nreinstated permanently if the copyright holder notifies you of the\nviolation by some reasonable means, this is the first time you have\nreceived notice of violation of this License (for any work) from that\ncopyright holder, and you cure the violation prior to 30 days after\nyour receipt of the notice.\n\n  Termination of your rights under this section does not terminate the\nlicenses of parties who have received copies or rights from you under\nthis License.  If your rights have been terminated and not permanently\nreinstated, you do not qualify to receive new licenses for the same\nmaterial under section 10.\n\n  9. Acceptance Not Required for Having Copies.\n\n  You are not required to accept this License in order to receive or\nrun a copy of the Program.  Ancillary propagation of a covered work\noccurring solely as a consequence of using peer-to-peer transmission\nto receive a copy likewise does not require acceptance.  However,\nnothing other than this License grants you permission to propagate or\nmodify any covered work.  These actions infringe copyright if you do\nnot accept this License.  Therefore, by modifying or propagating a\ncovered work, you indicate your acceptance of this License to do so.\n\n  10. Automatic Licensing of Downstream Recipients.\n\n  Each time you convey a covered work, the recipient automatically\nreceives a license from the original licensors, to run, modify and\npropagate that work, subject to this License.  You are not responsible\nfor enforcing compliance by third parties with this License.\n\n  An \"entity transaction\" is a transaction transferring control of an\norganization, or substantially all assets of one, or subdividing an\norganization, or merging organizations.  If propagation of a covered\nwork results from an entity transaction, each party to that\ntransaction who receives a copy of the work also receives whatever\nlicenses to the work the party's predecessor in interest had or could\ngive under the previous paragraph, plus a right to possession of the\nCorresponding Source of the work from the predecessor in interest, if\nthe predecessor has it or can get it with reasonable efforts.\n\n  You may not impose any further restrictions on the exercise of the\nrights granted or affirmed under this License.  For example, you may\nnot impose a license fee, royalty, or other charge for exercise of\nrights granted under this License, and you may not initiate litigation\n(including a cross-claim or counterclaim in a lawsuit) alleging that\nany patent claim is infringed by making, using, selling, offering for\nsale, or importing the Program or any portion of it.\n\n  11. Patents.\n\n  A \"contributor\" is a copyright holder who authorizes use under this\nLicense of the Program or a work on which the Program is based.  The\nwork thus licensed is called the contributor's \"contributor version\".\n\n  A contributor's \"essential patent claims\" are all patent claims\nowned or controlled by the contributor, whether already acquired or\nhereafter acquired, that would be infringed by some manner, permitted\nby this License, of making, using, or selling its contributor version,\nbut do not include claims that would be infringed only as a\nconsequence of further modification of the contributor version.  For\npurposes of this definition, \"control\" includes the right to grant\npatent sublicenses in a manner consistent with the requirements of\nthis License.\n\n  Each contributor grants you a non-exclusive, worldwide, royalty-free\npatent license under the contributor's essential patent claims, to\nmake, use, sell, offer for sale, import and otherwise run, modify and\npropagate the contents of its contributor version.\n\n  In the following three paragraphs, a \"patent license\" is any express\nagreement or commitment, however denominated, not to enforce a patent\n(such as an express permission to practice a patent or covenant not to\nsue for patent infringement).  To \"grant\" such a patent license to a\nparty means to make such an agreement or commitment not to enforce a\npatent against the party.\n\n  If you convey a covered work, knowingly relying on a patent license,\nand the Corresponding Source of the work is not available for anyone\nto copy, free of charge and under the terms of this License, through a\npublicly available network server or other readily accessible means,\nthen you must either (1) cause the Corresponding Source to be so\navailable, or (2) arrange to deprive yourself of the benefit of the\npatent license for this particular work, or (3) arrange, in a manner\nconsistent with the requirements of this License, to extend the patent\nlicense to downstream recipients.  \"Knowingly relying\" means you have\nactual knowledge that, but for the patent license, your conveying the\ncovered work in a country, or your recipient's use of the covered work\nin a country, would infringe one or more identifiable patents in that\ncountry that you have reason to believe are valid.\n\n  If, pursuant to or in connection with a single transaction or\narrangement, you convey, or propagate by procuring conveyance of, a\ncovered work, and grant a patent license to some of the parties\nreceiving the covered work authorizing them to use, propagate, modify\nor convey a specific copy of the covered work, then the patent license\nyou grant is automatically extended to all recipients of the covered\nwork and works based on it.\n\n  A patent license is \"discriminatory\" if it does not include within\nthe scope of its coverage, prohibits the exercise of, or is\nconditioned on the non-exercise of one or more of the rights that are\nspecifically granted under this License.  You may not convey a covered\nwork if you are a party to an arrangement with a third party that is\nin the business of distributing software, under which you make payment\nto the third party based on the extent of your activity of conveying\nthe work, and under which the third party grants, to any of the\nparties who would receive the covered work from you, a discriminatory\npatent license (a) in connection with copies of the covered work\nconveyed by you (or copies made from those copies), or (b) primarily\nfor and in connection with specific products or compilations that\ncontain the covered work, unless you entered into that arrangement,\nor that patent license was granted, prior to 28 March 2007.\n\n  Nothing in this License shall be construed as excluding or limiting\nany implied license or other defenses to infringement that may\notherwise be available to you under applicable patent law.\n\n  12. No Surrender of Others' Freedom.\n\n  If conditions are imposed on you (whether by court order, agreement or\notherwise) that contradict the conditions of this License, they do not\nexcuse you from the conditions of this License.  If you cannot convey a\ncovered work so as to satisfy simultaneously your obligations under this\nLicense and any other pertinent obligations, then as a consequence you may\nnot convey it at all.  For example, if you agree to terms that obligate you\nto collect a royalty for further conveying from those to whom you convey\nthe Program, the only way you could satisfy both those terms and this\nLicense would be to refrain entirely from conveying the Program.\n\n  13. Use with the GNU Affero General Public License.\n\n  Notwithstanding any other provision of this License, you have\npermission to link or combine any covered work with a work licensed\nunder version 3 of the GNU Affero General Public License into a single\ncombined work, and to convey the resulting work.  The terms of this\nLicense will continue to apply to the part which is the covered work,\nbut the special requirements of the GNU Affero General Public License,\nsection 13, concerning interaction through a network will apply to the\ncombination as such.\n\n  14. Revised Versions of this License.\n\n  The Free Software Foundation may publish revised and/or new versions of\nthe GNU General Public License from time to time.  Such new versions will\nbe similar in spirit to the present version, but may differ in detail to\naddress new problems or concerns.\n\n  Each version is given a distinguishing version number.  If the\nProgram specifies that a certain numbered version of the GNU General\nPublic License \"or any later version\" applies to it, you have the\noption of following the terms and conditions either of that numbered\nversion or of any later version published by the Free Software\nFoundation.  If the Program does not specify a version number of the\nGNU General Public License, you may choose any version ever published\nby the Free Software Foundation.\n\n  If the Program specifies that a proxy can decide which future\nversions of the GNU General Public License can be used, that proxy's\npublic statement of acceptance of a version permanently authorizes you\nto choose that version for the Program.\n\n  Later license versions may give you additional or different\npermissions.  However, no additional obligations are imposed on any\nauthor or copyright holder as a result of your choosing to follow a\nlater version.\n\n  15. Disclaimer of Warranty.\n\n  THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY\nAPPLICABLE LAW.  EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT\nHOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM \"AS IS\" WITHOUT WARRANTY\nOF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO,\nTHE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR\nPURPOSE.  THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM\nIS WITH YOU.  SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF\nALL NECESSARY SERVICING, REPAIR OR CORRECTION.\n\n  16. Limitation of Liability.\n\n  IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING\nWILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS\nTHE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY\nGENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE\nUSE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF\nDATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD\nPARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS),\nEVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF\nSUCH DAMAGES.\n\n  17. Interpretation of Sections 15 and 16.\n\n  If the disclaimer of warranty and limitation of liability provided\nabove cannot be given local legal effect according to their terms,\nreviewing courts shall apply local law that most closely approximates\nan absolute waiver of all civil liability in connection with the\nProgram, unless a warranty or assumption of liability accompanies a\ncopy of the Program in return for a fee.\n\n                     END OF TERMS AND CONDITIONS\n\n            How to Apply These Terms to Your New Programs\n\n  If you develop a new program, and you want it to be of the greatest\npossible use to the public, the best way to achieve this is to make it\nfree software which everyone can redistribute and change under these terms.\n\n  To do so, attach the following notices to the program.  It is safest\nto attach them to the start of each source file to most effectively\nstate the exclusion of warranty; and each file should have at least\nthe \"copyright\" line and a pointer to where the full notice is found.\n\n    {one line to give the program's name and a brief idea of what it does.}\n    Copyright (C) {year}  {name of author}\n\n    This program is free software: you can redistribute it and/or modify\n    it under the terms of the GNU General Public License as published by\n    the Free Software Foundation, either version 3 of the License, or\n    (at your option) any later version.\n\n    This program is distributed in the hope that it will be useful,\n    but WITHOUT ANY WARRANTY; without even the implied warranty of\n    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the\n    GNU General Public License for more details.\n\n    You should have received a copy of the GNU General Public License\n    along with this program.  If not, see <http://www.gnu.org/licenses/>.\n\nAlso add information on how to contact you by electronic and paper mail.\n\n  If the program does terminal interaction, make it output a short\nnotice like this when it starts in an interactive mode:\n\n    {project}  Copyright (C) {year}  {fullname}\n    This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'.\n    This is free software, and you are welcome to redistribute it\n    under certain conditions; type `show c' for details.\n\nThe hypothetical commands `show w' and `show c' should show the appropriate\nparts of the General Public License.  Of course, your program's commands\nmight be different; for a GUI interface, you would use an \"about box\".\n\n  You should also get your employer (if you work as a programmer) or school,\nif any, to sign a \"copyright disclaimer\" for the program, if necessary.\nFor more information on this, and how to apply and follow the GNU GPL, see\n<http://www.gnu.org/licenses/>.\n\n  The GNU General Public License does not permit incorporating your program\ninto proprietary programs.  If your program is a subroutine library, you\nmay consider it more useful to permit linking proprietary applications with\nthe library.  If this is what you want to do, use the GNU Lesser General\nPublic License instead of this License.  But first, please read\n<http://www.gnu.org/philosophy/why-not-lgpl.html>.\n"
  },
  {
    "path": "MANIFEST",
    "content": "bin/mashtree\nbin/mashtree_bootstrap.pl\nbin/mashtree_cluster.pl\nbin/mashtree_init.pl\nbin/mashtree_jackknife.pl\nbin/min_abundance_finder.pl\nCHANGES\nCONTRIBUTING.md\ndocs/ALGORITHM.md\ndocs/INSTALL.md\ndocs/TIPS.md\ndocs/TROUBLESHOOTING.md\nlib/Mashtree.pm\nlib/Mashtree/Db.pm\nlib/Mashtree/Mash.pm\nplugins/mashtree_optimize.pl\nplugins/README.md\nREADME.md\nt/file-of-files.txt\nt/filetypes/CFSAN000189.gbk.gz\nt/filetypes/CFSAN000189.ref.fasta.gz\nt/filetypes/CFSAN000191.ref.fasta.gz\nt/filetypes/CFSAN000211.gbk.gz\nt/filetypes/CFSAN000961.gbk.gz\nt/filetypes/CFSAN000968.ref.fasta.gz\nt/filetypes/CFSAN001112.ref.fasta.gz\nt/filetypes/CFSAN001115.ref.fasta.bz2\nt/filetypes/CFSAN001140_1.fastq.gz\nt/lambda/sample1.fastq.gz\nt/lambda/sample2.fastq.gz\nt/lambda/sample3.fastq.gz\nt/lambda/sample4.fastq.gz\nt/00_env.t\nt/01_filetypes.t\nt/02_lambda.t\nt/03_subsample.t\nt/04_fileoffiles.t\nt/06_jackknifingHashes.t\nt/08_bootstrap.t\nt/11_tsv-db.t\nMANIFEST\nMakefile.PL\n\n"
  },
  {
    "path": "MANIFEST.SKIP",
    "content": ".travis.yml*\n.git/*\nbin/mashtree_wrapper_deprecated.pl\nLICENSE\nMakefile.PL\nman/*\nmisc/*\nt/00_env.t\nt/01_filetypes.t\nt/02_lambda.t\nt/03_subsample.t\nt/04_fileoffiles.t\nt/06_jackknifingHashes.t\nt/10_sqlite.t\nt/11_tsv-db.t\nt/lambda/mashtree.dnd\ndist/*\nMakefile.old\nMANIFEST.*\n./*.msh\nlib/perl5\n\n"
  },
  {
    "path": "Makefile.PL",
    "content": "#!/usr/bin/env perl\nuse strict;\nuse warnings;\n \nuse 5.010;\n \nuse ExtUtils::MakeMaker;\n\nuse FindBin qw/$RealBin/;\nuse lib \"$RealBin/lib\";\nuse lib \"$RealBin/lib/perl5\";\n\n# Some help from\n# http://blogs.perl.org/users/michal_wojciechowski/2011/11/github-friendly-readme-files-with-extutils-makemaker-and-module-build.html\nmy $preop = 'true;';\n  #'sed -i.bak \"s/.5f/.10f/g\" bin/quicktree-2.3/src/tree.c bin/quicktree-2.3/src/distancemat.c;' .\n  #'make -C bin/quicktree-2.3;' .\n  #'perldoc -uT $(VERSION_FROM) | tee $(DISTVNAME)/README.pod > README.pod;' .\n  #'pod2text README.pod | tee $(DISTVNAME)/README > README';\n\n# Execute something before `make`\nmy $preBuild= 'true';\n#'rm -f bin/ROSS-v0.3.tar.gz;' .\n#  'if [ ! -e ./bin/ROSS-0.3/LICENSE ]; then ' .\n#  'wget https://github.com/lskatz/ROSS/archive/v0.3.tar.gz -O bin/ROSS-v0.3.tar.gz;'.\n#  'tar -zxvf ./bin/ROSS-v0.3.tar.gz --directory ./bin;'.\n#  'fi;' .\n#  '';\n\nWriteMakefile1(\n  NAME         => 'Mashtree',\n  VERSION_FROM => 'lib/Mashtree.pm',\n  #ABSTRACT_FROM => 'lib/Mashtree.pm',\n  #ABSTRACT_FROM => 'README.md',\n  AUTHOR => q{Lee S. Katz <gzu2@cdc.gov>},\n  EXE_FILES    => [\n    \"bin/mashtree\",\n    \"bin/mashtree_jackknife.pl\",\n    \"bin/min_abundance_finder.pl\",\n    \"bin/mashtree_bootstrap.pl\",\n  ],\n  BEFORE_BUILD=> {\n    exec => $preBuild,\n  },\n  PREREQ_PM    => {\n    # Core modules\n    'File::Basename' => 0,\n    'Data::Dumper'   => 0,\n    'List::Util'     => 0,\n    'List::MoreUtils'=> 0,\n    'Exporter'       => 0,\n    # Threads modules\n    'threads'        => 0,\n    'threads::shared'=> 0,\n    'Thread::Queue'  => 0,\n    # Not core (I think?)\n    'File::Which'    => 0,\n    'Bio::SeqIO'     => 0,\n    'Bio::TreeIO'    => 0,\n    'Bio::Matrix::IO'=> 0,\n    'Bio::Tree::Statistics'=> 0,\n    'Bio::Tree::DistanceFactory'=> 0,\n    'Bio::Sketch::Mash'    => 0, # ensures that Mash gets loaded\n    'Bio::Kmer'      => 0.26,\n    #'Graph::Dijkstra'=> 0,\n    #'Readonly'       => 0,\n    'JSON'           => '2.90',\n  },\n  BUILD_REQUIRES => {\n    'Test::More'     => 0.47,\n  },\n  MIN_PERL_VERSION => '5.10.0',\n  LICENSE          => \"gpl_3\",\n  dist=>{\n    COMPRESS => \"gzip -9f\",\n    SUFFIX   => \"gz\",\n    PREOP    => $preop,\n  },\n  META_MERGE => {\n    'meta-spec' => { version => 2 },\n    resources => {\n      repository => {\n        type => 'git',\n        url => 'git://github.com/lskatz/mashtree.git',\n        web => 'https://github.com/lskatz/mashtree',\n      },\n    },\n  },\n);\n\nsub WriteMakefile1 {  #Compatibility code for old versions of EU::MM. Written by Alexandr Ciornii, version 2. Added by eumm-upgrade.\n  my %params=@_;\n  my $eumm_version=$ExtUtils::MakeMaker::VERSION;\n  $eumm_version=eval $eumm_version;\n  die \"EXTRA_META is deprecated\" if exists $params{EXTRA_META};\n  die \"License not specified\" if not exists $params{LICENSE};\n  if ($params{AUTHOR} and ref($params{AUTHOR}) eq 'ARRAY' and $eumm_version < 6.5705) {\n    $params{META_ADD}->{author}=$params{AUTHOR};\n    $params{AUTHOR}=join(', ',@{$params{AUTHOR}});\n  }\n  if ($params{TEST_REQUIRES} and $eumm_version < 6.64) {\n    $params{BUILD_REQUIRES}={ %{$params{BUILD_REQUIRES} || {}} , %{$params{TEST_REQUIRES}} };\n    delete $params{TEST_REQUIRES};\n  }\n  if ($params{BUILD_REQUIRES} and $eumm_version < 6.5503) {\n    #EUMM 6.5502 has problems with BUILD_REQUIRES\n    $params{PREREQ_PM}={ %{$params{PREREQ_PM} || {}} , %{$params{BUILD_REQUIRES}} };\n    delete $params{BUILD_REQUIRES};\n  }\n  delete $params{CONFIGURE_REQUIRES} if $eumm_version < 6.52;\n  delete $params{MIN_PERL_VERSION} if $eumm_version < 6.48;\n  delete $params{META_MERGE} if $eumm_version < 6.46;\n  delete $params{META_ADD} if $eumm_version < 6.46;\n  delete $params{LICENSE} if $eumm_version < 6.31;\n\n  # LK\n  if(defined($params{BEFORE_BUILD})){\n    if(defined($params{BEFORE_BUILD}{exec})){\n      system($params{BEFORE_BUILD}{exec});\n      die if $?;\n    }\n    delete($params{BEFORE_BUILD});\n  }\n\n  WriteMakefile(%params);\n}\n"
  },
  {
    "path": "README.md",
    "content": "# mashtree\n\n[![DOI](https://joss.theoj.org/papers/10.21105/joss.01762/status.svg)](https://doi.org/10.21105/joss.01762)\n\nCreate a tree using Mash distances.\n\nFor simple usage, see `mashtree --help`. This is an example command:\n\n    mashtree *.fastq.gz > tree.dnd\n    \nFor confidence values, run either with `--help`: `mashtree_bootstrap.pl` or `mashtree_jackknife.pl`.\n\n## Two modes: fast or accurate\n\n**Input files**: fastq files are interpreted as raw read files. Fasta,\nGenBank, and EMBL files are interpreted as genome\nassemblies. Compressed files are also accepted of any of the\nabove file types.  You can compress with gz, bz2, or zip.\n\n**Output files**: Newick (.dnd).  If `--outmatrix` is supplied, then\na distance matrix too.\n\nSee the documentation on [the algorithms](docs/ALGORITHM.md) for more information.\n\n### Faster\n\n    mashtree --numcpus 12 *.fastq.gz [*.fasta] > mashtree.dnd\n\n### More accurate\n\nYou can get a more accurate tree with the minimum abundance finder. Simply\ngive `--mindepth 0`.  This step helps ignore very unique kmers that are \nmore likely read errors.\n\n    mashtree --mindepth 0 --numcpus 12 *.fastq.gz [*.fasta] > mashtree.dnd\n\n### Adding confidence values\n\nMashtree can add confidence values using jack knifing. For each\njack knife tree, 50% of hashes are used. Confidence values are calculated from\nthe jack knife trees using BioPerl. When using this method, you can pass\nflags to `mashtree` using the double-dash like in the example below.\n\nAdded in version 0.40.\n\n    mashtree_jackknife.pl --reps 100 --numcpus 12 *.fastq.gz -- --min-depth 0 > mashtree.jackknife.dnd\n    mashtree_jackknife.pl --help # additional usage help\n\nBootsrapping was added in version 0.55.  This runs mashtree itself multiple times, each\nwith a random seed.\n\n    mashtree_bootstrap.pl --reps 100 --numcpus 12 *.fastq.gz -- --min-depth 0 > mashtree.bootstrap.dnd\n\n## Usage\n\n    Usage: mashtree [options] *.fastq *.fasta *.gbk *.msh > tree.dnd\n    NOTE: fastq files are read as raw reads;\n          fasta, gbk, and embl files are read as assemblies;\n          Input files can be gzipped.\n    --tempdir            ''   If specified, this directory will not be\n                              removed at the end of the script and can\n                              be used to cache results for future\n                              analyses.\n                              If not specified, a dir will be made for you\n                              and then deleted at the end of this script.\n    --numcpus            1    This script uses Perl threads.\n    --outmatrix          ''   If specified, will write a distance matrix\n                              in tab-delimited format\n    --file-of-files           If specified, mashtree will try to read\n                              filenames from each input file. The file of\n                              files format is one filename per line. This\n                              file of files cannot be compressed.\n    --outtree                 If specified, the tree will be written to\n                              this file and not to stdout. Log messages\n                              will still go to stderr.\n    --version                 Display the version and exit\n\n    TREE OPTIONS\n    --truncLength        250  How many characters to keep in a filename\n    --sort-order         ABC  For neighbor-joining, the sort order can\n                              make a difference. Options include:\n                              ABC (alphabetical), random, input-order\n\n    MASH SKETCH OPTIONS\n    --genomesize         5000000\n    --mindepth           5    If mindepth is zero, then it will be\n                              chosen in a smart but slower method,\n                              to discard lower-abundance kmers.\n    --kmerlength         21\n    --sketch-size        10000\n\n\n## Installation\n\n**Please see [INSTALL.md](docs/INSTALL.md)**\n\n## Further documentation\n\nFor perl library help, run `perldoc` on a `.pm` file, e.g., `perldoc lib/Mashtree/Db.pm`.\n\nFor executable help run `--help`, e.g., `mashtree_bootstrap.pl --help`.\n\nFor more information and help please see the [docs folder](docs/)\n\nFor more information on plugins, see the [plugins folder](plugins). (in development)\n\nFor more information on contributions, please see [CONTRIBUTING.md](CONTRIBUTING.md).\n\n## References\n\n*  Mash: http://mash.readthedocs.io\n*  BioPerl: http://bioperl.org\n\n## Citation\n\n### JOSS\n\nKatz, L. S., Griswold, T., Morrison, S., Caravas, J., Zhang, S., den Bakker, H.C., Deng, X., and Carleton, H. A., (2019). Mashtree: a rapid comparison of whole genome sequence files. Journal of Open Source Software, 4(44), 1762, https://doi.org/10.21105/joss.01762\n\n### Poster\n\nKatz, L. S., Griswold, T., & Carleton, H. A. (2017, October 8-11). [_Generating WGS Trees with Mashtree_](misc/mashtree%20ASM%20NGS.pptx). Poster presented at the American Society for Microbiology Conference on Rapid Applied Microbial Next-Generation Sequencing and Bioinformatic Pipelines, Washington, DC. Poster number 27.\n\n\n"
  },
  {
    "path": "bin/mashtree",
    "content": "#!/usr/bin/env perl\n# Author: Lee Katz <lkatz@cdc.gov>\n# Uses Mash and BioPerl to create a NJ tree based on distances.\n# Run this script with -h for help and usage.\n\nuse strict;\nuse warnings;\nuse Data::Dumper;\nuse Getopt::Long;\nuse File::Temp qw/tempdir tempfile/;\nuse File::Basename qw/basename dirname fileparse/;\nuse File::Copy qw/copy/;\nuse POSIX qw/floor/;\nuse List::Util qw/min max/;\nuse Scalar::Util qw/looks_like_number/;\n\nuse threads;\nuse threads::shared;\n\nuse FindBin;\nuse lib \"$FindBin::RealBin/../lib\";\nuse lib \"$FindBin::RealBin/../lib/perl5\";\nuse File::Which qw/which/;\nuse Mashtree qw/logmsg @fastqExt @fastaExt @mshExt @richseqExt _truncateFilename createTreeFromPhylip $MASHTREE_VERSION/;\nuse Mashtree::Db;\nuse Bio::Tree::DistanceFactory;\nuse Bio::Matrix::IO;\nuse Bio::Tree::Statistics;\nuse Bio::SeqIO;\n\n#my %delta :shared=(); # change in amplitude for peak detection, for each fastq\nmy $scriptDir=dirname $0;\nmy $dbhLock :shared;  # Use this as a lock so that only one thread writes to the db at a time\nmy $abundanceFinderLock :shared; # a lock to limit min abundance finder instances\nlocal $0=basename $0;\n\nexit main();\n\nsub main{\n  my $settings={};\n  GetOptions($settings,qw(help sigfigs|significant-digits=i file-of-files outtree=s outmatrix=s tempdir=s numcpus=i genomesize=i mindepth|min-depth=i truncLength=i kmerlength=i sort-order=s sketch-size=i citation version save-sketches:s seed=i)) or die $!;\n  $$settings{numcpus}||=1;\n  $$settings{truncLength}||=250;  # how long a genome name is\n  $$settings{sigfigs} ||= 10;\n  $$settings{tempdir}||=tempdir(\"MASHTREE.XXXXXX\",CLEANUP=>1,TMPDIR=>1);\n  $$settings{'sort-order'}||=\"ABC\";\n\n  # Mash-specific options\n  $$settings{genomesize}||=5000000;\n  $$settings{mindepth}//=5;\n  $$settings{kmerlength}||=21;\n  $$settings{'sketch-size'}||=10000;\n  $$settings{seed}||=42;\n\n  # Make some settings lowercase\n  for(qw(sort-order)){\n    $$settings{$_}=lc($$settings{$_});\n  }\n\n  if($$settings{help}){\n    print usage();\n    return 0;\n  }\n  if($$settings{version}){\n    print \"Mashtree $MASHTREE_VERSION\\n\";\n    return 0;\n  }\n  if($$settings{citation}){\n    print \"Katz, L. S., Griswold, T., Morrison, S., Caravas, J., Zhang, S., den Bakker, H.C., Deng, X., and Carleton, H. A., (2019). Mashtree: a rapid comparison of whole genome sequence files. Journal of Open Source Software, 4(44), 1762, https://doi.org/10.21105/joss.01762\\n\";\n    return 0;\n  }\n\n  # \"reads\" are either fasta assemblies or fastq reads\n  my @reads=@ARGV;\n  die \"need more arguments\\n\".usage() if(@reads < 1);\n\n  # Check for prereq executables.\n  for my $exe(qw(mash quicktree)){\n    if(!which($exe)){\n      die \"ERROR: could not find $exe in your PATH\";\n    }\n  }\n\n  # Check mash version\n  my $mashVersion = `mash --version`;\n  die \"ERROR running mash: $!\" if $?;\n  chomp($mashVersion);\n  $mashVersion=~s/\\..*//;\n  die \"ERROR: need mash version > 2\" if($mashVersion < 2);\n  logmsg \"Found mash version $mashVersion - \".which('mash');\n\n  # Distributed cpus if we have few genomes but high numcpus\n  $$settings{cpus_per_mash}=floor($$settings{numcpus}/@reads);\n  $$settings{cpus_per_mash}=1 if($$settings{cpus_per_mash} < 1);\n  $$settings{numthreads}=min(scalar(@reads), $$settings{numcpus});\n\n  #die Dumper [$$settings{cpus_per_mash},$$settings{numthreads},\\@reads];\n  #$$settings{cpus_per_mash}=1;\n  #$$settings{numthreads}=$$settings{numcpus};\n\n  logmsg \"Temporary directory will be $$settings{tempdir}\";\n  logmsg \"$0 on \".scalar(@reads).\" files\";\n\n  my %seen;\n  my @tmp;\n  for my $reads(@ARGV){\n    if(!-e $reads){\n      die \"ERROR: I could not find reads path at $reads\";\n    }\n    my $basename=basename($reads);\n    if($seen{$basename}++){\n      logmsg \"Skipping $reads: already seen $basename\";\n      next;\n    }\n\n    # If the user wants to specify files of files then check\n    # on whether it is a file of files.  Otherwise do not check\n    # because of disk IO speeds.\n    if($$settings{'file-of-files'}){\n      # If we have file-of-files in play, don't try to optimize\n      # too early.\n      $$settings{cpus_per_mash}=1;\n      $$settings{numthreads}=$$settings{numcpus};\n      open(my $possibleFileOfFiles, \"<\", $reads) or die \"ERROR: could not read $reads: $!\";\n      my $firstfile = <$possibleFileOfFiles>;\n      $firstfile=~s/^\\s+|\\s+$//g; #whitespace trim\n      chomp($firstfile);\n      if(-e $firstfile){\n        push(@tmp, $firstfile);\n        while(<$possibleFileOfFiles>){\n          s/^\\s+|\\s+$//g; #whitespace trim\n          push(@tmp, $_);\n        }\n      } else {\n        logmsg \"WARNING: user specified --file-of-files but $reads does not seem to be a file of files (ignore this warning if this was not supposed to be a file of files\";\n        push(@tmp, $reads);\n      }\n      close $possibleFileOfFiles;\n    } else {\n      push(@tmp,$reads);\n    }\n  }\n  @reads=@tmp;\n\n  my $sketches=sketchAll(\\@reads,\"$$settings{tempdir}\",$settings);\n\n  my $phylip = mashDistance($sketches,\\@reads,$$settings{tempdir},$settings);\n\n  my $treeObj = createTreeFromPhylip($phylip,$$settings{tempdir},$settings);\n\n  # Write the tree\n  if($$settings{outtree}){\n    open(my $treeFh, \">\", $$settings{outtree}) or die \"ERROR writing tree to $$settings{outtree}: $!\";\n    print $treeFh $treeObj->as_text('newick');\n    close $treeFh\n  } else {\n    print $treeObj->as_text('newick');\n    print \"\\n\";\n  }\n\n  return 0;\n}\n\n# Run mash sketch on everything, multithreaded.\nsub sketchAll{\n  my($reads,$sketchDir,$settings)=@_;\n\n  mkdir $sketchDir if(!-d $sketchDir);\n\n  # Make an array of genomes that would distribute well\n  # across threads.  For example, don't put all raw-read\n  # genomes into a single thread and all the assemblies\n  # into another.\n  my %filesize=();\n  for(@$reads){\n    $filesize{$_} = -s $_;\n  }\n  my @sortedReads=sort {$filesize{$a} <=> $filesize{$b}} @$reads;\n  my @threadArr=();\n  for(my $i=0; $i<@sortedReads; $i++){\n    # Since each genome is sorted smallest to leargest,\n    # they can be sent round-robin to each thread to \n    # ensure balance.\n    my $threadIndex = $i % $$settings{numcpus};\n    push(@{ $threadArr[$threadIndex] }, $sortedReads[$i]);\n  }\n\n  # Initiate the threads\n  my @thr;\n  for(0..$$settings{numthreads}-1){\n    # If there are more threads than samples, then we need to set the number in this thread to empty array\n    $threadArr[$_] //= [];\n\n    $thr[$_]=threads->new(\\&mashSketch,$sketchDir,$threadArr[$_],$settings);\n  }\n\n  my @mshList;\n  for(@thr){\n    my $mashfiles=$_->join;\n    for my $file(@$mashfiles){\n      push(@mshList,$file);\n    }\n  }\n\n  return \\@mshList;\n}\n\n# Individual mash sketch worker\nsub mashSketch{\n  my($sketchDir,$genomeArr,$settings)=@_;\n\n  # If any file needs to be converted, it will end up in\n  # this directory.\n  my $tempdir=tempdir(\"$$settings{tempdir}/convertSeq.XXXXXX\", CLEANUP=>1);\n\n  my $numFiles = scalar(@$genomeArr);\n  logmsg \"This thread will work on $numFiles sketches\";\n\n  my @msh;\n  my $fileCounter=0;\n  # $fastq is a misnomer: it could be any kind of accepted sequence file\n  for my $fastq(@$genomeArr){\n    logmsg \"Working on file \".++$fileCounter.\" out of $numFiles\";\n    my($fileName,$filePath,$fileExt)=fileparse($fastq,@fastqExt,@fastaExt,@richseqExt,@mshExt);\n\n    # Unzip the file. This temporary file will\n    # only exist if the correct extensions are detected.\n    my $unzipped=\"$tempdir/\".basename($fastq);\n    $unzipped=~s/\\.(gz|bz2?|zip)$//i;\n    my $was_unzipped=0;\n    # Don't bother unzipping if it's a fastq or fasta file b/c Mash can read those\n    if(!grep {$_ eq $fileExt} (@fastqExt,@fastaExt)){\n      if($fastq=~/\\.gz$/i){\n        # Don't actually decompress fastq.gz files because they are\n        # read natively by mash.\n        if($fastq =~ /\\.fastq.gz$|\\.fq.gz$/){\n          $was_unzipped=0;\n        } else {\n          system(\"gzip -cd $fastq > $unzipped\");\n          die \"ERROR with gzip -cd $fastq\" if $?;\n          $was_unzipped=1;\n        }\n      } elsif($fastq=~/\\.bz2?$/i){\n        system(\"bzip2 -cd $fastq > $unzipped\");\n        die \"ERROR with bzip2 -cd $fastq\" if $?;\n        $was_unzipped=1;\n      } elsif($fastq=~/\\.zip$/i){\n        system(\"unzip -p  $fastq > $unzipped\");\n        die \"ERROR with unzip -p  $fastq\" if $?;\n        $was_unzipped=1;\n      }\n    }\n\n    # If the file was uncompressed, parse the filename again.\n    if($was_unzipped){\n      $fastq=$unzipped;\n      ($fileName,$filePath,$fileExt)=fileparse($fastq,@fastqExt,@fastaExt,@richseqExt,@mshExt);\n    }\n\n    # If we see a richseq (e.g., gbk or embl), then convert it to fasta\n    # TODO If Mash itself accepts richseq, then consider\n    # doing away with this section.\n    if(grep {$_ eq $fileExt} @richseqExt){\n      # Make a temporary fasta file, but it needs to have a\n      # consistent name in case Mashtree is being run with\n      # the wrapper for bootstrap values.\n      # I can't exactly make a consistent filename in case\n      # different mashtree invocations collide, so\n      # I need to make a new temporary directory with a \n      # consistent filename.\n      my $tmpfasta=\"$tempdir/$fileName$fileExt.fasta\";\n      my $in=Bio::SeqIO->new(-file=>$fastq);\n      my $out=Bio::SeqIO->new(-file=>\">$tmpfasta\", -format=>\"fasta\");\n      while(my $seq=$in->next_seq){\n        $out->write_seq($seq);\n      }\n      logmsg \"Wrote $tmpfasta\";\n\n      # Update our filename for downstream\n      $fastq=$tmpfasta;\n      ($fileName,$filePath,$fileExt)=fileparse($tmpfasta, @fastaExt);\n    }\n\n    my $outPrefix=\"$sketchDir/\".basename($fastq, @mshExt);\n    if(-e \"$outPrefix.msh\"){\n      logmsg \"already mashed: $fastq. Skipping.\";\n      push(@msh,\"$outPrefix.msh\");\n      next;\n    }\n\n    # Do different things depending on fastq vs fasta\n    my $sketchXopts=\"\";\n    if(grep {$_ eq $fileExt} @fastqExt){\n      my $minDepth=determineMinimumDepth($fastq,$$settings{mindepth},$$settings{kmerlength},$settings);\n      $sketchXopts.=\"-m $minDepth -g $$settings{genomesize} \";\n    } elsif(grep {$_ eq $fileExt} @fastaExt) {\n      $sketchXopts.=\" \";\n    } elsif(grep {$_ eq $fileExt} @mshExt){\n      $sketchXopts.=\" \";\n    } else {\n      logmsg \"WARNING: I could not understand what kind of file this is by its extension ($fileExt): $fastq\";\n    }\n      \n    # See if the user already mashed this file locally\n    if(-e \"$fastq.msh\"){\n      logmsg \"Found locally mashed file $fastq.msh. I will use it.\";\n      copy(\"$fastq.msh\",\"$outPrefix.msh\");\n    }\n    if(grep {$_ eq $fileExt} @mshExt){\n      logmsg \"Input file is a sketch file itself and will be used as such: $fastq\";\n      copy($fastq, \"$outPrefix.msh\");\n    }\n\n    if(-e \"$outPrefix.msh\"){\n      logmsg \"WARNING: \".basename($fastq).\" was already mashed.\";\n    } elsif(-s $fastq < 1){\n      logmsg \"WARNING: $fastq is a zero byte file. Skipping.\";\n      next;\n    } else {\n      my $sketchCommand=\"mash sketch -S $$settings{seed} -k $$settings{kmerlength} -s $$settings{'sketch-size'} $sketchXopts -o \\Q$outPrefix\\E \\Q$fastq\\E 2>&1\";\n      my $stdout = `$sketchCommand`;\n      if ($?){\n        logmsg \"ERROR running $sketchCommand!\\n  $stdout\";\n        die;\n      }\n    }\n\n    push(@msh,\"$outPrefix.msh\");\n\n    # Save sketches into a local directory if requested\n    if($$settings{'save-sketches'}){\n      my $dir=$$settings{'save-sketches'};\n      my $target = \"$fastq.msh\";\n      if(defined($dir)){\n        mkdir($dir) if(!-e $dir);\n        $target = \"$dir/\".basename($fastq).\".msh\";\n      }\n      #try to hard link because this is basically a read-only file,\n      #but if not, copying is fine.\n      link(\"$outPrefix.msh\", $target) || copy(\"$outPrefix.msh\", $target)\n        || die \"ERROR hard-linking or copying $outPrefix.msh to $target: $!\";\n    }\n  }\n\n  system(\"rm -rf $tempdir\");\n\n  return \\@msh;\n}\n\n# Parallelized mash distance\nsub mashDistance{\n  my($mshList,$reads,$outdir,$settings)=@_;\n\n  # Make a list of names that will appear in the database\n  # in exactly the right format.\n  my @genomeName;\n\n  # Make a temporary file with one line per mash file.\n  # Helps with not running into the max number of command line args.\n  my $mshListFilename=\"$outdir/mshList.txt\";\n  open(my $mshListFh,\">\",$mshListFilename) or die \"ERROR: could not write to $mshListFilename: $!\";\n  for(@$mshList){\n    print $mshListFh $_.\"\\n\";\n    push(@genomeName,_truncateFilename($_,$settings));\n  }\n  close $mshListFh;\n\n  # Instatiate the database and create the table before the threads get to it\n  my $mashtreeDbFilename=\"$outdir/distances.db.tsv\";\n  my $mashtreeDb=Mashtree::Db->new($mashtreeDbFilename,{significant_figures=>$$settings{sigfigs}});\n\n  # Make an array of distance files for each thread.\n  # Because distance files take about the same amount\n  # of time to analyze, there is no need to sort.\n  my @threadArr=();\n  for(my $i=0; $i<@$mshList; $i++){\n    my $threadIndex = $i % $$settings{numcpus};\n    push(@{ $threadArr[$threadIndex] }, $$mshList[$i]);\n  }\n\n  # Initialize the threads\n  my @thr;\n  for(0..$$settings{numthreads}-1){\n    $thr[$_]=threads->new(\\&mashDist,$outdir,$threadArr[$_],$mshListFilename,$settings);\n  }\n\n  my $numThreads=@thr;\n  for(my $t=0; $t<@thr; $t++){\n    my $T=$t+1;\n    my $formattedThreadName = \"$T/$numThreads, TID\".$thr[$t]->tid;\n    logmsg \"Waiting to join thread ($formattedThreadName)\";\n    my $distHash = $thr[$t]->join; # returns ref to scalar b/c the str could be very large\n\n    logmsg \"Databasing distances ($formattedThreadName)\";\n    $mashtreeDb->addDistancesFromHash($distHash);\n  }\n\n  my $phylip = \"$outdir/distances.phylip\";\n  logmsg \"Converting to phylip format into $phylip\";\n  open(my $phylipFh, \">\", $phylip) or die \"ERROR: could not write to $phylip: $!\";\n  print $phylipFh $mashtreeDb->toString(\\@genomeName,\"phylip\");\n  close $phylipFh;\n\n  if($$settings{outmatrix}){\n    logmsg \"Writing a distance matrix to $$settings{outmatrix}\";\n    open(my $matrixFh, \">\", $$settings{outmatrix}) or die \"ERROR: could not write to $$settings{outmatrix}: $!\";\n    print $matrixFh $mashtreeDb->toString(\\@genomeName,\"matrix\");\n    close $matrixFh;\n  }\n  \n\n  return $phylip;\n}\n\n# Individual mash distance\nsub mashDist{\n  my($outdir,$mshArr,$mshList,$settings)=@_;\n\n  my $numQueries=0;\n  my %dist;\n  for my $msh(@$mshArr){\n    #my $outfile=\"$outdir/\".basename($msh).\".tsv\";\n    logmsg \"Distances for $msh\";\n    my $cmd = \"mash dist -t \\Q$msh\\E -l $mshList\";\n    my @distRes = `$cmd`;\n    die \"ERROR with $cmd\" if $?;\n    chomp(@distRes);\n\n    my $query=\"\";\n    my $queryLine = shift(@distRes);\n    if($queryLine=~/#\\s*query\\s+(.+)/){\n      $query = _truncateFilename($1);\n    } else {\n      die \"ERROR parsing for the query in $queryLine\";\n    }\n    for(@distRes){\n      my($hit, $dist)=split /\\t/;\n      $hit = _truncateFilename($hit);\n      $dist{$query}{$hit} = $dist;\n    }\n\n    $numQueries++;\n  }\n  return \\%dist;\n  #return \\$distStr; # return the reference because the str could be quite large\n}\n\nsub determineMinimumDepth{\n  my($fastq,$mindepth,$kmerlength,$settings)=@_;\n\n  my $defaultDepth=5; # if no valley is detected\n\n  return $mindepth if($mindepth > 0);\n\n  my $basename=basename($fastq,@fastqExt);\n  \n  # Run the min abundance finder to find the valleys\n  my $minAbundanceTempdir=\"$$settings{tempdir}/$basename.minAbundance.tmp\";\n  mkdir $minAbundanceTempdir;\n  my $minAbundanceCommand=\"min_abundance_finder.pl --numcpus $$settings{cpus_per_mash} $fastq --kmer $kmerlength --tempdir $minAbundanceTempdir\";\n  #lock($abundanceFinderLock); logmsg \"DEBUG: running single mode for $fastq with $$settings{numcpus} cpus in the single thread\";\n  my $minKmerCount = `$minAbundanceCommand`;\n  # If there is an error, just try running one at a time.\n  # I am not sure why there is a seg fault sometimes when\n  # more than one are running at the same time though.\n  #if($?){\n  #  lock($abundanceFinderLock);\n  #  @valleyLines=`$minAbundanceCommand`;\n  #}\n  die \"ERROR with min_abundance_finder.pl on $fastq: $!\" if($?);\n  chomp($minKmerCount);\n  # Some cleanup of large files\n  # Avoid some weird segfault error associated with the unlink()\n  #unlink $_ for(glob(\"$minAbundanceTempdir/*\"));\n  #rmdir $minAbundanceTempdir;\n  system(\"rm -rf $minAbundanceTempdir\");\n\n  # If there is no valley, return a default value\n  if(!$minKmerCount){\n    return $defaultDepth;\n  }\n  \n  # Set a minimum just in case\n  $minKmerCount=1 if($minKmerCount < 1);\n\n  logmsg \"Setting the min depth as $minKmerCount for $fastq\";\n\n  return $minKmerCount;\n}\n\nsub usage{\n  \"$0: use distances from Mash (min-hash algorithm) to make a NJ tree\n  Usage: $0 [options] *.fastq *.fasta *.gbk *.msh > tree.dnd\n  NOTE: fastq files are read as raw reads;\n        fasta, gbk, and embl files are read as assemblies;\n        Input files can be gzipped.\n  --tempdir            ''   If specified, this directory will not be\n                            removed at the end of the script and can\n                            be used to cache results for future\n                            analyses.\n                            If not specified, a dir will be made for you\n                            and then deleted at the end of this script.\n  --numcpus            1    This script uses Perl threads.\n  --outmatrix          ''   If specified, will write a distance matrix\n                            in tab-delimited format\n  --file-of-files           If specified, mashtree will try to read \n                            filenames from each input file. The file of\n                            files format is one filename per line. This\n                            file of files cannot be compressed.\n  --outtree                 If specified, the tree will be written to \n                            this file and not to stdout. Log messages\n                            will still go to stderr.\n  --version                 Display the version and exit\n  --citation                Display the preferred citation and exit\n\n  TREE OPTIONS\n  --truncLength        250  How many characters to keep in a filename\n  --sigfigs            10   How many decimal places to use in mash distances \n  --sort-order         ABC  For neighbor-joining, the sort order can\n                            make a difference. Options include:\n                            ABC (alphabetical), random, input-order\n\n  MASH SKETCH OPTIONS\n  --genomesize         5000000\n  --mindepth           5    If mindepth is zero, then it will be\n                            chosen in a smart but slower method,\n                            to discard lower-abundance kmers.\n  --kmerlength         21\n  --sketch-size        10000\n  --seed               42   Seed for mash sketch\n  --save-sketches      ''   If a directory is supplied, then sketches\n                            will be saved in it.\n                            If no directory is supplied, then sketches\n                            will be saved alongside source files.\n  \"\n}\n\n"
  },
  {
    "path": "bin/mashtree_bootstrap.pl",
    "content": "#!/usr/bin/env perl\n# Author: Lee Katz <lkatz@cdc.gov>\n# Uses Mash and BioPerl to create a NJ tree based on distances.\n# Run this script with -h for help and usage.\n\nuse strict;\nuse warnings;\nuse Data::Dumper;\nuse Getopt::Long;\nuse File::Temp qw/tempdir tempfile/;\nuse File::Basename qw/basename dirname fileparse/;\nuse File::Copy qw/cp mv/;\nuse List::Util qw/shuffle/;\nuse List::MoreUtils qw/part/;\nuse POSIX qw/floor/;\nuse IO::Handle; # allows me to autoflush file handles\n\n#use Fcntl qw/:flock LOCK_EX LOCK_UN/;\n\nuse threads;\nuse Thread::Queue;\nuse threads::shared;\n\nuse FindBin;\nuse lib \"$FindBin::RealBin/../lib\";\nuse Mashtree qw/logmsg @fastqExt @fastaExt createTreeFromPhylip mashDist/;\nuse Mashtree::Db;\nuse Bio::SeqIO;\nuse Bio::TreeIO;\nuse Bio::Tree::DistanceFactory;\nuse Bio::Tree::Statistics;\n\nmy $writeStick :shared;\nmy $readStick  :shared; # limit disk IO by reading some files one at a time\n\nlocal $0=basename $0;\n\nexit main();\n\nsub main{\n  my $settings={};\n  my @wrapperOptions=qw(help file-of-files outmatrix=s tempdir=s reps=i numcpus=i);\n  GetOptions($settings,@wrapperOptions) or die $!;\n  $$settings{reps}//=1;\n  $$settings{numcpus}||=1;\n  die usage() if($$settings{help});\n  die usage() if(@ARGV < 1);\n\n  $$settings{tempdir}||=tempdir(\"MASHTREE_BOOTSTRAP.XXXXXX\",CLEANUP=>1,TMPDIR=>1);\n  mkdir($$settings{tempdir}) if(!-d $$settings{tempdir});\n  logmsg \"Temporary directory will be $$settings{tempdir}\";\n\n  if($$settings{reps} < 1){\n    die \"ERROR: no reps were given!\";\n  }\n  if($$settings{reps} < 100){\n    logmsg \"WARNING: You have very few reps planned on this mashtree run. Recommended reps are at least 100.\";\n  }\n  \n  ## Catch some options that are not allowed to be passed\n  # Tempdir: All mashtree temporary directories will be under the\n  # wrapper's tempdir.\n  if(grep(/^\\-+tempdir$/,@ARGV) || grep(/^\\-+t$/,@ARGV)){\n    die \"ERROR: tempdir was specified for mashtree but should be an option for $0\";\n  }\n  # Numcpus: this needs to be specified in the wrapper and will\n  # appropriately be transferred to the mashtree script\n  if(grep(/^\\-+numcpus$/,@ARGV) || grep(/^\\-+n$/,@ARGV)){\n    die \"ERROR: numcpus was specified for mashtree but should be an option for $0\";\n  }\n  # Outmatrix: the wrapper script needs to control where\n  # the matrix goes because it can only have the outmatrix\n  # for the observed run and not the replicates for speed's\n  # sake.\n  if(grep(/^\\-+outmatrix$/,@ARGV) || grep(/^\\-+o$/,@ARGV)){\n    die \"ERROR: outmatrix was specified for mashtree but should be an option for $0\";\n  }\n  # --file-of-files: should be something to give to mashtree directly\n  if(grep(/^\\-+file-of-files$/,@ARGV) || grep(/^\\-+f$/,@ARGV)){\n    die \"ERROR: file-of-files was specified for mashtree but should be an option for $0\";\n  }\n  \n  # Separate flagged options from reads in the mashtree options\n  my @reads = ();\n  my @mashOptions = ();\n  for(my $i=0;$i<@ARGV;$i++){\n    if(-e $ARGV[$i]){\n      push(@reads, $ARGV[$i]);\n    } else {\n      push(@mashOptions, $ARGV[$i]);\n    }\n  }\n  if($$settings{'file-of-files'}){\n    push(@mashOptions, '--file-of-files');\n  }\n  my $mashOptions=join(\" \",@mashOptions);\n  my $reads = join(\" \", @reads);\n  \n  # Some filenames we'll expect\n  my $observeddir=\"$$settings{tempdir}/observed\";\n  my $obsDistances=\"$observeddir/distances.phylip\";\n  my $observedTree=\"$$settings{tempdir}/observed.dnd\";\n  my $outmatrix=\"$$settings{tempdir}/observeddistances.tsv\";\n  my $mshList = \"$$settings{tempdir}/mash.list\";\n  my $mergedMash = \"$$settings{tempdir}/merged.msh\";\n\n  # Make the observed directory and run Mash\n  logmsg \"Running mashtree on full data (\".scalar(@reads).\" targets)\";\n  mkdir($observeddir);\n  system(\"$FindBin::RealBin/mashtree --outmatrix $outmatrix.tmp --tempdir $observeddir --numcpus $$settings{numcpus} $mashOptions $reads > $observedTree.tmp\");\n  die if $?;\n  mv(\"$observedTree.tmp\",$observedTree) or die $?;\n  mv(\"$outmatrix.tmp\",$outmatrix) or die $?;\n\n  # Round-robin sample the replicate number, so that it's\n  # well-balanced.\n  my @repNumber = (1..$$settings{reps});\n  my @reps;\n  for(my $i=0;$i<$$settings{reps};$i++){\n    my $thrIndex = $i % $$settings{numcpus};\n    push(@{$reps[$thrIndex]}, $i);\n  }\n\n  # Sample the mash sketches with replacement for rapid bootstrapping\n  logmsg \"Running mashtree single-threaded among $$settings{reps} replicates using $$settings{numcpus} total threads.\";\n  my @bsThread;\n  for my $i(0..$$settings{numcpus}-1){\n    my %settingsCopy = %$settings;\n    $bsThread[$i] = threads->new(\\&mashtreeRepWorker, $reps[$i], $reads, $mashOptions, \\%settingsCopy);\n  }\n\n  my @bsTree;\n  for my $t(@bsThread){\n    my $treeArr = $t->join;\n    for my $file(@$treeArr){\n      my $treein = Bio::TreeIO->new(-file=>$file);\n      while(my $tree=$treein->next_tree){\n        push(@bsTree, $tree);\n      }\n    }\n  }\n  \n  # Combine trees into a bootstrapped tree and write it \n  # to an output file. Then print it to stdout.\n  logmsg \"Adding bootstraps to tree\";\n  my $guideTree=Bio::TreeIO->new(-file=>\"$observeddir/tree.dnd\")->next_tree;\n  my $bsTree=assess_bootstrap($guideTree,\\@bsTree,$guideTree);\n  for my $node($bsTree->get_nodes){\n    next if($node->is_Leaf);\n    my $id=$node->bootstrap||$node->id||0;\n    $node->id($id);\n  }\n  open(my $treeFh,\">\",\"$$settings{tempdir}/bstree.dnd\") or die \"ERROR: could not write to $$settings{tempdir}/bstree.dnd: $!\";\n  print $treeFh $bsTree->as_text('newick');\n  print $treeFh \"\\n\";\n  close $treeFh;\n\n  system(\"cat $$settings{tempdir}/bstree.dnd\"); die if $?;\n\n  if($$settings{'outmatrix'}){\n    cp($outmatrix,$$settings{'outmatrix'});\n  }\n  \n  return 0;\n}\n\nsub mashtreeRepWorker{\n  my($reps, $reads, $mashtreeOptions, $settings) = @_;\n\n  my @tree;\n  my $largeInt = ~0;\n\n  for my $rep(@$reps){\n    logmsg \"Rep $rep\";\n    my $tempdir = $$settings{tempdir}.\"/$rep\";\n    my $seed = int(rand(4.29497e+09)); # apparently 4.29497e+09 is the largest int for mash sketch\n    my $repMashtreeOptions = $mashtreeOptions . \" --seed $seed\";\n    my $stdout = `$FindBin::RealBin/mashtree --tempdir $tempdir --numcpus 1 $repMashtreeOptions $reads 2>&1`;\n    die \"ERROR on rep $rep\\n$stdout\" if $?;\n\n    push(@tree, \"$$settings{tempdir}/$rep/tree.dnd\");\n  }\n\n  return \\@tree;\n}\n\n\n# Fixed bootstrapping function from bioperl\n# https://github.com/bioperl/bioperl-live/pull/304\nsub assess_bootstrap{\n   my ($self,$bs_trees,$guide_tree) = @_;\n   my @consensus;\n\n   if(!defined($bs_trees) || ref($bs_trees) ne 'ARRAY'){\n     die \"ERROR: second parameter in assess_bootstrap() must be a list\";\n   }\n   my $num_bs_trees = scalar(@$bs_trees);\n   if($num_bs_trees < 1){\n     die \"ERROR: no bootstrap trees were passed to assess_bootstrap()\";\n   }\n\n   # internal nodes are defined by their children\n\n   my (%lookup,%internal);\n   my $i = 0;\n   for my $tree ( $guide_tree, @$bs_trees ) {\n       # Do this as a top down approach, can probably be\n       # improved by caching internal node states, but not going\n       # to worry about it right now.\n\n       my @allnodes = $tree->get_nodes;\n       my @internalnodes = grep { ! $_->is_Leaf } @allnodes;\n       for my $node ( @internalnodes ) {\n           my @tips = sort map { $_->id } \n                      grep { $_->is_Leaf() } $node->get_all_Descendents;\n           my $id = \"(\".join(\",\", @tips).\")\";\n           if( $i == 0 ) {\n               $internal{$id} = $node->internal_id;\n           } else { \n               $lookup{$id}++;\n           }\n       }\n       $i++;\n   }\n   #my @save; # unsure why this variable is needed\n   for my $l ( keys %lookup ) {\n       if( defined $internal{$l} ) {#&& $lookup{$l} > $min_seen ) {\n           my $intnode = $guide_tree->find_node(-internal_id => $internal{$l});\n           $intnode->bootstrap(sprintf(\"%d\",100 * $lookup{$l} / $num_bs_trees));\n       }\n   }\n   return $guide_tree;\n}\n\n#######\n# Utils\n#######\n\nsub usage{\n  my $usage=\"$0: a wrapper around mashtree for bootstrapping.\n  Usage: $0 [options] [-- mashtree options] *.fastq.gz *.fasta > tree.dnd\n  --outmatrix          ''   Output file for distance matrix\n  --reps               0    How many bootstrap repetitions to run;\n                            If zero, no bootstrapping.\n  --numcpus            1    This will be passed to mashtree and will\n                            be used to multithread reps.\n  --tempdir                 A directory path that will be created to\n                            store temporary files\n  \n  --file-of-files           If specified, mashtree will try to read\n                            filenames from each input file. The file of\n                            files format is one filename per line. This\n                            file of files cannot be compressed.\n\n  --                        Used to separate options for $0 and mashtree\n  MASHTREE OPTIONS:\\n\".\n  # Print the mashtree options starting with numcpus,\n  # skipping the tempdir option.\n  `mashtree --help 2>&1 | grep -A 999 \"TREE OPTIONS\" | grep -v ^Stopped`;\n\n  return $usage;\n}\n\n"
  },
  {
    "path": "bin/mashtree_cluster.pl",
    "content": "#!/usr/bin/env perl\n# Author: Lee Katz <lkatz@cdc.gov>\n# Read a mashtree database to cluster genomes\n\nuse strict;\nuse warnings;\nuse Data::Dumper;\nuse Getopt::Long;\nuse File::Basename qw/basename dirname fileparse/;\nuse File::Copy qw/copy/;\nuse POSIX qw/floor/;\nuse List::Util qw/min max/;\nuse Scalar::Util qw/looks_like_number/;\n\nuse DBI;\n\nuse threads;\nuse threads::shared;\n\nuse FindBin;\nuse lib \"$FindBin::RealBin/../lib\";\nuse Mashtree qw/logmsg/;\n#use Mashtree qw/logmsg @fastqExt @fastaExt @mshExt @richseqExt _truncateFilename createTreeFromPhylip $MASHTREE_VERSION/;\n#use Mashtree::Db;\n#use Graph::Dijkstra;\n\nlocal $0=basename $0;\n\nexit main();\n\nsub main{\n  my $settings={};\n  GetOptions($settings,qw(help threshold|cutoff=f numcpus=i nonzero=f)) or die $!;\n  $$settings{numcpus}||=1;\n  $$settings{nonzero}||=1e-99;\n  $$settings{threshold}||=0.1;\n\n  my($dbFile) = @ARGV;\n\n  die usage() if($$settings{help} || !$dbFile);\n\n  if(!-e $dbFile){\n    die \"ERROR: could not find database file $dbFile\";\n  }\n\n  my $clusters = makeClusters($dbFile,$settings);\n  for my $clusterArr(values(%$clusters)){\n    print join(\"\\t\",@$clusterArr).\"\\n\";\n  }\n\n  return 0;\n}\n\nsub makeClusters{\n  my($db,$settings)=@_;\n\n  my %C; #clustering hash, where the key is the seed\n\n  #my $graph = Graph::Dijkstra->new(edgedefault=>\"undirected\");\n  my %graph=();# hash of pairwise nodes\n\n  my $dbh = DBI->connect(\"dbi:SQLite:dbname=$db\",\"\",\"\",{\n      RaiseError => 1\n  });\n\n  my $sth = $dbh->prepare(qq(SELECT GENOME1,GENOME2,DISTANCE\n    FROM DISTANCE\n    WHERE GENOME1 != GENOME2\n    ORDER BY DISTANCE ASC, \n             GENOME1  ASC, \n             GENOME2  ASC\n    ));\n  my $rv = $sth->execute() or die $DBI::errstr;\n  if($rv < 0){\n    die \"ERROR: no distances were found in the database $db\";\n  }\n\n  my $rowCounter=0;\n  my %genomeIndex;\n  while(my @row=$sth->fetchrow_array()){\n    my($g1,$g2,$dist)=@row;\n    $genomeIndex{$g1}=1;\n    $genomeIndex{$g2}=1;\n\n    #$dist=$$settings{nonzero} if(!$dist);\n    my($G1,$G2)=sort{$a cmp $b} ($g1,$g2);\n    $graph{$G1}{$G2}=$dist;\n    $graph{$G2}{$G1}=$dist;\n    $rowCounter++;\n\n    if($rowCounter % 1000000 == 0){\n      logmsg \"Read the distances for $rowCounter pairs\";\n    }\n  }\n  logmsg \"Done reading the distances for $rowCounter pairs\";\n  # TODO? close the database fh\n\n  logmsg \"Calculating clusters\";\n  #my @genome=sort {$a cmp $b} map {$$_{id}} $graph->nodeList;\n  my @genome = sort{$a cmp $b} keys(%genomeIndex);\n  my $numGenomes=@genome;\n  for(my $i=0;$i<$numGenomes;$i++){\n    # Is this genome $i close to anything?\n    my $query = $genome[$i];\n    logmsg \"Querying with $query\";\n    my $hit = undef;\n\n    # See if the query is close to any of the current seeds first\n    my @seed           = sort{$a cmp $b} keys(%C);\n    my $numSeeds       = scalar(@seed);\n    my $possibleHit;\n    my $closestSeedDist = ~0;\n    for(my $j=0; $j<$numSeeds; $j++){\n      next if(!defined($graph{$seed[$j]}{$query}));\n      my $dist = $graph{$seed[$j]}{$query};\n      next if($dist > $$settings{threshold});\n\n      if($dist < $closestSeedDist){\n        $possibleHit = $seed[$j];\n        $closestSeedDist = $dist;\n      }\n    }\n\n    # If it's close to a seed, then record it.\n    if($closestSeedDist <= $$settings{threshold}){\n      logmsg \"  => $possibleHit\";\n      push(@{ $C{$possibleHit} }, $query);\n    }\n    # If it's not close to a seed, start a new seeded cluster.\n    else {\n      logmsg \"  => new cluster seeded with $query\";\n      $C{$query} = [$query];\n    }\n    \n  }\n  return \\%C;\n}\n\nsub usage{\n  \"Output clusters of genomes, based on a mashtree sqlite database.\n  The stdout will be a tab-delimited list of genomes, one\n  line per cluster.\n  Usage: $0 file.sql > clusters.tsv\n\n  --threshold  0.1       Maximum any two genomes can be\n                         from the seed in a given cluster\n  --nonzero    1e-99     Zero distance is not tolerated\n                         in this script. Give a nonzero\n                         value in case a zero distance\n                         is found.\n  --numcpus    1         Not currently used.\n  \"\n}\n\n"
  },
  {
    "path": "bin/mashtree_init.pl",
    "content": "#!/usr/bin/env perl\n# Author: Lee Katz <lkatz@cdc.gov>\n# Uses Mash to create a database of distances\n# Run this script with -h for help and usage.\n\nuse strict;\nuse warnings;\nuse Data::Dumper;\nuse Getopt::Long;\nuse File::Temp qw/tempdir tempfile/;\nuse File::Basename qw/basename dirname fileparse/;\nuse File::Copy qw/copy move/;\nuse POSIX qw/floor/;\nuse List::Util qw/min max/;\nuse Scalar::Util qw/looks_like_number/;\n\nuse threads;\nuse threads::shared;\n\nuse FindBin;\nuse lib \"$FindBin::RealBin/../lib\";\nuse Mashtree qw/logmsg @fastqExt @fastaExt @mshExt @richseqExt _truncateFilename $MASHTREE_VERSION/;\nuse Mashtree::Db;\nuse Bio::Tree::DistanceFactory;\nuse Bio::Matrix::IO;\nuse Bio::Tree::Statistics;\nuse Bio::SeqIO;\n\nmy %delta :shared=(); # change in amplitude for peak detection, for each fastq\nmy $scriptDir=dirname $0;\nmy $dbhLock :shared;  # Use this as a lock so that only one thread writes to the db at a time\nmy $abundanceFinderLock :shared; # a lock to limit min abundance finder instances\nlocal $0=basename $0;\n\nexit main();\n\nsub main{\n  my $settings={};\n  GetOptions($settings,qw(help outfile|output=s tempdir=s numcpus=i genomesize=i mindepth|min-depth=i truncLength=i kmerlength=i sort-order=s sketch-size=i version)) or die $!;\n  $$settings{numcpus}||=1;\n  $$settings{truncLength}||=250;  # how long a genome name is\n  $$settings{tempdir}||=tempdir(\"MASHTREE.XXXXXX\",CLEANUP=>1,TMPDIR=>1);\n  $$settings{'sort-order'}||=\"ABC\";\n\n  # Mash-specific options\n  $$settings{genomesize}||=5000000;\n  $$settings{mindepth}//=5;\n  $$settings{kmerlength}||=21;\n  $$settings{'sketch-size'}||=10000;\n\n  # Make some settings lowercase\n  for(qw(sort-order)){\n    $$settings{$_}=lc($$settings{$_});\n  }\n\n  die usage() if($$settings{help});\n  if($$settings{version}){\n    print \"Mashtree $MASHTREE_VERSION\\n\";\n    return 0;\n  }\n\n  # \"reads\" are either fasta assemblies or fastq reads\n  my @reads=@ARGV;\n  die usage() if(@reads < 2);\n  die \"ERROR: need --outfile\" if(!$$settings{outfile});\n\n  # Check for prereq executables.\n  for my $exe(qw(mash)){\n    system(\"$exe -h > /dev/null 2>&1\");\n    die \"ERROR: could not find $exe in your PATH\" if $?;\n  }\n\n  # Distributed cpus if we have few genomes but high numcpus\n  $$settings{cpus_per_mash}=floor($$settings{numcpus}/@reads);\n  $$settings{cpus_per_mash}=1 if($$settings{cpus_per_mash} < 1);\n  $$settings{numthreads}=min(scalar(@reads), $$settings{numcpus});\n\n  #die Dumper [$$settings{cpus_per_mash},$$settings{numthreads},\\@reads];\n  #$$settings{cpus_per_mash}=1;\n  #$$settings{numthreads}=$$settings{numcpus};\n\n  logmsg \"Temporary directory will be $$settings{tempdir}\";\n  logmsg \"$0 on \".scalar(@reads).\" files\";\n\n  my %seen;\n  my @tmp;\n  for my $reads(@ARGV){\n    if(!-e $reads){\n      die \"ERROR: I could not find reads path at $reads\";\n    }\n    my $basename=basename($reads);\n    if($seen{$basename}++){\n      logmsg \"Skipping $reads: already seen $basename\";\n      next;\n    }\n    push(@tmp,$reads);\n  }\n  @reads=@tmp;\n\n  my $sketches=sketchAll(\\@reads,\"$$settings{tempdir}\",$settings);\n\n  my $db = mashDistance($sketches,\\@reads,$$settings{tempdir},$settings);\n\n  logmsg \"New file in $$settings{outfile}\";\n  move($db, $$settings{outfile});\n\n  return 0;\n}\n\n# Run mash sketch on everything, multithreaded.\nsub sketchAll{\n  my($reads,$sketchDir,$settings)=@_;\n\n  mkdir $sketchDir if(!-d $sketchDir);\n\n  # Make an array of genomes that would distribute well\n  # across threads.  For example, don't put all raw-read\n  # genomes into a single thread and all the assemblies\n  # into another.\n  my %filesize=();\n  for(@$reads){\n    $filesize{$_} = -s $_;\n  }\n  my @sortedReads=sort {$filesize{$a} <=> $filesize{$b}} @$reads;\n  my @threadArr=();\n  for(my $i=0; $i<@sortedReads; $i++){\n    # Since each genome is sorted smallest to leargest,\n    # they can be sent round-robin to each thread to \n    # ensure balance.\n    my $threadIndex = $i % $$settings{numcpus};\n    push(@{ $threadArr[$threadIndex] }, $sortedReads[$i]);\n  }\n\n  # Initiate the threads\n  my @thr;\n  for(0..$$settings{numthreads}-1){\n    $thr[$_]=threads->new(\\&mashSketch,$sketchDir,$threadArr[$_],$settings);\n  }\n\n  my @mshList;\n  for(@thr){\n    my $mashfiles=$_->join;\n    for my $file(@$mashfiles){\n      push(@mshList,$file);\n    }\n  }\n\n  return \\@mshList;\n}\n\n# Individual mash sketch\nsub mashSketch{\n  my($sketchDir,$genomeArr,$settings)=@_;\n\n  # If any file needs to be converted, it will end up in\n  # this directory.\n  my $tempdir=tempdir(\"$$settings{tempdir}/convertSeq.XXXXXX\", CLEANUP=>1);\n\n  my @msh;\n  # $fastq is a misnomer: it could be any kind of accepted sequence file\n  for my $fastq(@$genomeArr){\n    my($fileName,$filePath,$fileExt)=fileparse($fastq,@fastqExt,@fastaExt,@richseqExt,@mshExt);\n\n    # Unzip the file. This temporary file will\n    # only exist if the correct extensions are detected.\n    my $unzipped=\"$tempdir/\".basename($fastq);\n    $unzipped=~s/\\.(gz|bz2?|zip)$//i;\n    my $was_unzipped=0;\n    # Don't bother unzipping if it's a fastq or fasta file b/c Mash can read those\n    if(!grep {$_ eq $fileExt} (@fastqExt,@fastaExt)){\n      if($fastq=~/\\.gz$/i){\n        system(\"gzip  -cd $fastq > $unzipped\");\n        die \"ERROR with gzip  -cd $fastq\" if $?;\n        $was_unzipped=1;\n      } elsif($fastq=~/\\.bz2?$/i){\n        system(\"bzip2 -cd $fastq > $unzipped\");\n        die \"ERROR with bzip2 -cd $fastq\" if $?;\n        $was_unzipped=1;\n      } elsif($fastq=~/\\.zip$/i){\n        system(\"unzip -p  $fastq > $unzipped\");\n        die \"ERROR with unzip -p  $fastq\" if $?;\n        $was_unzipped=1;\n      }\n    }\n\n    # If the file was uncompressed, parse the filename again.\n    if($was_unzipped){\n      $fastq=$unzipped;\n      ($fileName,$filePath,$fileExt)=fileparse($fastq,@fastqExt,@fastaExt,@richseqExt,@mshExt);\n    }\n\n    # If we see a richseq (e.g., gbk or embl), then convert it to fasta\n    # TODO If Mash itself accepts richseq, then consider\n    # doing away with this section.\n    if(grep {$_ eq $fileExt} @richseqExt){\n      # Make a temporary fasta file, but it needs to have a\n      # consistent name in case Mashtree is being run with\n      # the wrapper for bootstrap values.\n      # I can't exactly make a consistent filename in case\n      # different mashtree invocations collide, so\n      # I need to make a new temporary directory with a \n      # consistent filename.\n      my $tmpfasta=\"$tempdir/$fileName$fileExt.fasta\";\n      my $in=Bio::SeqIO->new(-file=>$fastq);\n      my $out=Bio::SeqIO->new(-file=>\">$tmpfasta\", -format=>\"fasta\");\n      while(my $seq=$in->next_seq){\n        $out->write_seq($seq);\n      }\n      logmsg \"Wrote $tmpfasta\";\n\n      # Update our filename for downstream\n      $fastq=$tmpfasta;\n      ($fileName,$filePath,$fileExt)=fileparse($tmpfasta, @fastaExt);\n    }\n\n    # Do different things depending on fastq vs fasta\n    my $sketchXopts=\"\";\n    if(grep {$_ eq $fileExt} @fastqExt){\n      my $minDepth=determineMinimumDepth($fastq,$$settings{mindepth},$$settings{kmerlength},$settings);\n      $sketchXopts.=\"-m $minDepth -g $$settings{genomesize} \";\n    } elsif(grep {$_ eq $fileExt} @fastaExt) {\n      $sketchXopts.=\" \";\n    } elsif(grep {$_ eq $fileExt} @mshExt){\n      $sketchXopts.=\" \";\n    } else {\n      logmsg \"WARNING: I could not understand what kind of file this is by its extension ($fileExt): $fastq\";\n    }\n      \n    my $outPrefix=\"$sketchDir/\".basename($fastq, @mshExt);\n\n    # See if the user already mashed this file locally\n    if(-e \"$fastq.msh\"){\n      logmsg \"Found locally mashed file $fastq.msh. I will use it.\";\n      copy(\"$fastq.msh\",\"$outPrefix.msh\");\n    }\n    if(grep {$_ eq $fileExt} @mshExt){\n      logmsg \"Input file is a sketch file itself and will be used as such: $fastq\";\n      copy($fastq, \"$outPrefix.msh\");\n    }\n\n    if(-e \"$outPrefix.msh\"){\n      logmsg \"WARNING: \".basename($fastq).\" was already mashed.\";\n    } elsif(-s $fastq < 1){\n      logmsg \"WARNING: $fastq is a zero byte file. Skipping.\";\n      next;\n    } else {\n      logmsg \"Sketching $fastq\";\n      my $sketchCommand=\"mash sketch -k $$settings{kmerlength} -s $$settings{'sketch-size'} $sketchXopts -o $outPrefix $fastq  1>&2\";\n      system($sketchCommand);\n      die if $?;\n    }\n\n    push(@msh,\"$outPrefix.msh\");\n  }\n\n  system(\"rm -rf $tempdir\");\n\n  return \\@msh;\n}\n\n# Parallelized mash distance\nsub mashDistance{\n  my($mshList,$reads,$outdir,$settings)=@_;\n\n  # Make a list of names that will appear in the database\n  # in exactly the right format.\n  my @genomeName;\n\n  # Make a temporary file with one line per mash file.\n  # Helps with not running into the max number of command line args.\n  my $mshListFilename=\"$outdir/mshList.txt\";\n  open(my $mshListFh,\">\",$mshListFilename) or die \"ERROR: could not write to $mshListFilename: $!\";\n  for(@$mshList){\n    print $mshListFh $_.\"\\n\";\n    push(@genomeName,_truncateFilename($_,$settings));\n  }\n  close $mshListFh;\n\n  # Instatiate the database and create the table before the threads get to it\n  my $mashtreeDbFilename=\"$outdir/distances.sqlite\";\n  my $mashtreeDb=Mashtree::Db->new($mashtreeDbFilename);\n\n  # Make an array of distance files for each thread.\n  # Because distance files take about the same amount\n  # of time to analyze, there is no need to sort.\n  my @threadArr=();\n  for(my $i=0; $i<@$mshList; $i++){\n    my $threadIndex = $i % $$settings{numcpus};\n    push(@{ $threadArr[$threadIndex] }, $$mshList[$i]);\n  }\n\n  # Initialize the threads\n  my @thr;\n  for(0..$$settings{numthreads}-1){\n    $thr[$_]=threads->new(\\&mashDist,$outdir,$threadArr[$_],$mshListFilename,$mashtreeDbFilename,$settings);\n  }\n\n  for(@thr){\n    logmsg \"Waiting to join thread TID\".$_->tid;\n    my $distfiles=$_->join;\n    logmsg \"Joined TID\".$_->tid;\n  }\n\n  return $mashtreeDbFilename;\n}\n\n# Individual mash distance\nsub mashDist{\n  my($outdir,$mshArr,$mshList,$mashtreeDbFilename,$settings)=@_;\n\n  # One distance file for all queries in this thread\n  my($distFileFh,$distFile)=tempfile(\"mashdistXXXXXX\", SUFFIX=>\".tsv\", DIR=>$outdir);\n\n  my $numQueries=0;\n  my $mashtreeDb=Mashtree::Db->new($mashtreeDbFilename);\n  for my $msh(@$mshArr){\n    #my $outfile=\"$outdir/\".basename($msh).\".tsv\";\n    logmsg \"Distances for $msh\";\n    system(\"mash dist -t $msh -l $mshList >> $distFile\");\n    die \"ERROR with 'mash dist -t $msh -l $mshList'\" if $?;\n    $numQueries++;\n  }\n\n  # If there is anything to add to the database, lock\n  # the database.  Only lock it once per thread, optimally.\n  if($numQueries > 0){\n    lock($dbhLock);\n    $mashtreeDb->addDistances($distFile);\n  }\n\n  # I think that the thread disconnects the db when\n  # this sub ends but I wanted to do it directly and\n  # in a readable fashion.\n  $mashtreeDb->disconnect();\n  close($distFileFh);\n  unlink($distFile);\n}\n\nsub determineMinimumDepth{\n  my($fastq,$mindepth,$kmerlength,$settings)=@_;\n\n  $delta{$fastq}//=10;\n  my $defaultDepth=2; # if no valley is detected # TODO should it be five?\n\n  return $mindepth if($mindepth > 0);\n\n  my $basename=basename($fastq,@fastqExt);\n  \n  # Run the min abundance finder to find the valleys\n  my $minAbundanceTempdir=\"$$settings{tempdir}/$basename.minAbundance.tmp\";\n  mkdir $minAbundanceTempdir;\n  my $minAbundanceCommand=\"min_abundance_finder.pl --numcpus $$settings{cpus_per_mash} $fastq --kmer $kmerlength --tempdir $minAbundanceTempdir --delta $delta{$fastq}\";\n  lock($abundanceFinderLock); logmsg \"DEBUG: running single mode for $fastq\";\n  my @valleyLines=`$minAbundanceCommand`;\n  # If there is an error, just try running one at a time.\n  # I am not sure why there is a seg fault sometimes when\n  # more than one are running at the same time though.\n  #if($?){\n  #  lock($abundanceFinderLock);\n  #  @valleyLines=`$minAbundanceCommand`;\n  #}\n  die \"ERROR with min_abundance_finder.pl on $fastq: $!\" if($?);\n  chomp(@valleyLines);\n  # Some cleanup of large files\n  unlink $_ for(glob(\"$minAbundanceTempdir/*\"));\n  rmdir $minAbundanceTempdir;\n\n  # If there is no valley, return a default value\n  #if(!defined $valleyLines[1] || !looks_like_number($valley[1]) || @valley < 1){\n  if(!defined $valleyLines[1] || @valleyLines < 1){\n    $delta{$fastq}=int($delta{$fastq}/2);\n    if($delta{$fastq} > 10){\n      logmsg \"Trying again to determine a min depth with delta==$delta{$fastq} on $fastq\";\n      return determineMinimumDepth($fastq,$mindepth,$kmerlength,$settings);\n    }\n    logmsg \"WARNING: no valleys were found! Reporting minimum kmer coverage as $defaultDepth.\";\n    return $defaultDepth;\n  }\n  \n  # Discard the header but keep the first line\n  my($minKmerCount, $countOfCounts)=split(/\\t/,$valleyLines[1]);\n  # force an \"empty\" value to zero\n  if(!defined($minKmerCount) || !looks_like_number($minKmerCount)){\n    $minKmerCount=0;\n  }\n  # However, the minimum count can't be zero, and so it is one.\n  $minKmerCount=1 if($minKmerCount < 1);\n\n  logmsg \"Setting the min depth as $minKmerCount for $fastq (delta==$delta{$fastq})\";\n\n  return $minKmerCount;\n}\n\nsub usage{\n  \"$0: use distances from Mash (min-hash algorithm) to make a database of distances\n  Usage: $0 [options] -o mash.sqlite *.fastq *.fasta *.gbk *.msh\n  NOTE: fastq files are read as raw reads;\n        fasta, gbk, and embl files are read as assemblies;\n        Input files can be gzipped.\n  --outfile            ''   Required output sqlite file\n  --tempdir            ''   If specified, this directory will not be\n                            removed at the end of the script and can\n                            be used to cache results for future\n                            analyses.\n                            If not specified, a dir will be made for you\n                            and then deleted at the end of this script.\n  --numcpus            1    This script uses Perl threads.\n  --version                 Display the version and exit\n\n  TREE OPTIONS\n  --truncLength        250  How many characters to keep in a filename\n  --sort-order         ABC  For neighbor-joining, the sort order can\n                            make a difference. Options include:\n                            ABC (alphabetical), random, input-order\n\n  MASH SKETCH OPTIONS\n  --genomesize         5000000\n  --mindepth           5    If mindepth is zero, then it will be\n                            chosen in a smart but slower method,\n                            to discard lower-abundance kmers.\n  --kmerlength         21\n  --sketch-size        10000\n  \"\n}\n\n"
  },
  {
    "path": "bin/mashtree_jackknife.pl",
    "content": "#!/usr/bin/env perl\n# Author: Lee Katz <lkatz@cdc.gov>\n# Uses Mash and BioPerl to create a NJ tree based on distances.\n# Run this script with -h for help and usage.\n\nuse strict;\nuse warnings;\nuse Data::Dumper;\nuse Getopt::Long;\nuse File::Temp qw/tempdir tempfile/;\nuse File::Basename qw/basename dirname fileparse/;\nuse File::Copy qw/cp mv/;\nuse List::Util qw/shuffle/;\nuse List::MoreUtils qw/part/;\nuse POSIX qw/floor/;\n# if JSON::XS is installed, it will be automatically used.\n# Specify () so that no functions are exported. We will use OO.\nuse JSON (); \nuse IO::Handle; # allows me to autoflush file handles\n\n#use Fcntl qw/:flock LOCK_EX LOCK_UN/;\n\nuse threads;\nuse Thread::Queue;\nuse threads::shared;\n\nuse FindBin;\nuse lib \"$FindBin::RealBin/../lib\";\nuse Mashtree qw/logmsg @fastqExt @fastaExt createTreeFromPhylip mashDist/;\nuse Mashtree::Db;\nuse Bio::SeqIO;\nuse Bio::TreeIO;\nuse Bio::Tree::DistanceFactory;\nuse Bio::Tree::Statistics;\n\nmy $writeStick :shared;\nmy $readStick  :shared; # limit disk IO by reading some files one at a time\n\nlocal $0=basename $0;\n\nexit main();\n\nsub main{\n  my $settings={};\n  my @wrapperOptions=qw(help file-of-files outmatrix=s tempdir=s reps=i numcpus=i);\n  GetOptions($settings,@wrapperOptions) or die $!;\n  $$settings{reps}//=1;\n  $$settings{numcpus}||=1;\n  die usage() if($$settings{help});\n  die usage() if(@ARGV < 1);\n\n  $$settings{tempdir}||=tempdir(\"MASHTREE_WRAPPER.XXXXXX\",CLEANUP=>1,TMPDIR=>1);\n  mkdir($$settings{tempdir}) if(!-d $$settings{tempdir});\n  logmsg \"Temporary directory will be $$settings{tempdir}\";\n\n  if($$settings{reps} < 1){\n    die \"ERROR: no reps were given!\";\n  }\n  if($$settings{reps} < 100){\n    logmsg \"WARNING: You have very few reps planned on this mashtree run. Recommended reps are at least 100.\";\n  }\n  # Give a warning if JSON is going to be slow later\n  my $jsonTmp = JSON->new();\n  if(! $jsonTmp->is_xs){\n    logmsg \"WARNING: the currently installed JSON module will make this script very slow when jack knifing. To avoid this error, install the JSON::XS module like so: `cpanm ~l ~ JSON::XS`.\";\n  }\n  \n  ## Catch some options that are not allowed to be passed\n  # Tempdir: All mashtree temporary directories will be under the\n  # wrapper's tempdir.\n  if(grep(/^\\-+tempdir$/,@ARGV) || grep(/^\\-+t$/,@ARGV)){\n    die \"ERROR: tempdir was specified for mashtree but should be an option for $0\";\n  }\n  # Numcpus: this needs to be specified in the wrapper and will\n  # appropriately be transferred to the mashtree script\n  if(grep(/^\\-+numcpus$/,@ARGV) || grep(/^\\-+n$/,@ARGV)){\n    die \"ERROR: numcpus was specified for mashtree but should be an option for $0\";\n  }\n  # Outmatrix: the wrapper script needs to control where\n  # the matrix goes because it can only have the outmatrix\n  # for the observed run and not the replicates for speed's\n  # sake.\n  if(grep(/^\\-+outmatrix$/,@ARGV) || grep(/^\\-+o$/,@ARGV)){\n    die \"ERROR: outmatrix was specified for mashtree but should be an option for $0\";\n  }\n  # --file-of-files: should be something to give to mashtree directly\n  if(grep(/^\\-+file-of-files$/,@ARGV) || grep(/^\\-+f$/,@ARGV)){\n    die \"ERROR: file-of-files was specified for mashtree but should be an option for $0\";\n  }\n  \n  # Separate flagged options from reads in the mashtree options\n  my @reads = ();\n  my @mashOptions = ();\n  for(my $i=0;$i<@ARGV;$i++){\n    if(-e $ARGV[$i]){\n      push(@reads, $ARGV[$i]);\n    } else {\n      push(@mashOptions, $ARGV[$i]);\n    }\n  }\n  if($$settings{'file-of-files'}){\n    push(@mashOptions, '--file-of-files');\n  }\n  my $mashOptions=join(\" \",@mashOptions);\n  my $reads = join(\" \", @reads);\n  \n  # Some filenames we'll expect\n  my $observeddir=\"$$settings{tempdir}/observed\";\n  my $obsDistances=\"$observeddir/distances.phylip\";\n  my $observedTree=\"$$settings{tempdir}/observed.dnd\";\n  my $outmatrix=\"$$settings{tempdir}/observeddistances.tsv\";\n  my $mshList = \"$$settings{tempdir}/mash.list\";\n  my $mergedMash = \"$$settings{tempdir}/merged.msh\";\n  my $mergedJSON = \"$$settings{tempdir}/merged.msh.json.gz\";\n\n  # Make the observed directory and run Mash\n  logmsg \"Running mashtree on full data (\".scalar(@reads).\" targets)\";\n  mkdir($observeddir);\n  system(\"$FindBin::RealBin/mashtree --outmatrix $outmatrix.tmp --tempdir $observeddir --numcpus $$settings{numcpus} $mashOptions $reads > $observedTree.tmp\");\n  die if $?;\n  mv(\"$observedTree.tmp\",$observedTree) or die $?;\n  mv(\"$outmatrix.tmp\",$outmatrix) or die $?;\n\n  # Merge the mash files to make the threads go faster later\n  my @msh = glob(\"$observeddir/*.msh\");\n  open(my $mshListFh, \">\", $mshList) or die \"ERROR writing to mash list $mshList: $!\";\n  for(@msh){\n    print $mshListFh $_.\"\\n\";\n  }\n  close $mshListFh;\n  unlink($mergedMash) if(-e $mergedMash); # mash complains about overwriting an existing file\n  system(\"mash paste -l $mergedMash $mshList >&2\"); die \"ERROR merging mash files\" if $?; \n  # max compression on the json file so that it can be a\n  # smaller footprint and so that it can be read faster\n  # in each thread.\n  system(\"mash info -d $mergedMash | gzip -c9 > $mergedJSON\");\n  die \"ERROR with mash info | gzip -c9\" if $?;\n  unlink($_) for(@msh); # remove redundant files to the merged msh\n  \n  # Round-robin sample the replicate number, so that it's\n  # well-balanced.\n  my @repNumber = (1..$$settings{reps});\n  my @reps;\n  for(my $i=0;$i<$$settings{reps};$i++){\n    my $thrIndex = $i % $$settings{numcpus};\n    push(@{$reps[$thrIndex]}, $i);\n  }\n\n  # Sample the mash sketches with replacement for rapid bootstrapping\n  my @bsThread;\n  for my $i(0..$$settings{numcpus}-1){\n    my %settingsCopy = %$settings;\n    $bsThread[$i] = threads->new(\\&subsampleMashSketchesWorker, $mergedJSON, $reps[$i], \\%settingsCopy);\n  }\n\n  my @bsTree;\n  for my $thr(@bsThread){\n    my $fileArr = $thr->join;\n    if(ref($fileArr) ne 'ARRAY'){\n      die \"ERROR: one or more threads did not return an array of jack knife tree files as expected.\";\n    }\n    for my $file(@$fileArr){\n      my $treein = Bio::TreeIO->new(-file=>$file);\n      while(my $tree=$treein->next_tree){\n        push(@bsTree, $tree);\n      }\n    }\n  }\n  \n  # Combine trees into a bootstrapped tree and write it \n  # to an output file. Then print it to stdout.\n  logmsg \"Adding bootstraps to tree\";\n  my $guideTree=Bio::TreeIO->new(-file=>\"$observeddir/tree.dnd\")->next_tree;\n  my $bsTree=assess_bootstrap($guideTree,\\@bsTree,$guideTree);\n  for my $node($bsTree->get_nodes){\n    next if($node->is_Leaf);\n    my $id=$node->bootstrap||$node->id||0;\n    $node->id($id);\n  }\n  open(my $treeFh,\">\",\"$$settings{tempdir}/bstree.dnd\") or die \"ERROR: could not write to $$settings{tempdir}/bstree.dnd: $!\";\n  print $treeFh $bsTree->as_text('newick');\n  print $treeFh \"\\n\";\n  close $treeFh;\n\n  system(\"cat $$settings{tempdir}/bstree.dnd\"); die if $?;\n\n  if($$settings{'outmatrix'}){\n    cp($outmatrix,$$settings{'outmatrix'});\n  }\n  \n  return 0;\n}\n\nsub subsampleMashSketchesWorker{\n  my($mergedJSON, $reps, $settings) = @_;\n  $reps //= [];\n\n  my @treeFile;\n  return \\@treeFile if(@$reps <1);\n\n  # Initialize our JSON reader\n  my $json = JSON->new;\n  $json->utf8;           # If we only expect characters 0..255. Makes it fast.\n  $json->allow_nonref;   # can convert a non-reference into its corresponding string\n  $json->allow_blessed;  # encode method will not barf when it encounters a blessed reference \n  $json->pretty;         # enables indent, space_before and space_after \n  # Only let one thread at a time read the large JSON file\n  my $mashInfoStr=\"\"; # must be declared outside of the block because we are parsing it after the block\n  {\n    lock($readStick);\n    logmsg \"Reading huge JSON file describing all mash distances, $mergedJSON\";\n    $mashInfoStr = `gzip -cd $mergedJSON`; \n    die \"ERROR running gzip -cd $mergedJSON: $!\" if $?;\n  }\n  my $mashInfoHash = $json->decode($mashInfoStr);\n  $mashInfoStr=\"\"; # clear some ram\n  my $kmerlength = $$mashInfoHash{kmer}; # find the kmer length right away\n\n  my $numReps = @$reps;\n  my $numSketches = scalar(@{ $$mashInfoHash{sketches} });\n  my $msg=\"Initializing thread with $numReps replicates: \";\n  for my $i(0..2){\n    my $repID = $$reps[$i] // \"\";\n    $msg.=\"$repID, \";\n  }\n  $msg=~s/[ ,]+$//; # right trim\n  if($numReps > 3){\n    $msg.=\"...\";\n  }\n  logmsg $msg;\n  for(my $repI=0;$repI<@$reps;$repI++){\n    my $rep = $$reps[$repI];\n\n    my $subsampleDir = \"$$settings{tempdir}/rep$rep\";\n    mkdir $subsampleDir;\n    my $log = \"$subsampleDir/jackknife.log\";\n    open(my $logFh, \">\", $log) or die \"ERROR: cannot write to log $log: $!\";\n    $logFh->autoflush();\n    logmsg \"rep$rep: $subsampleDir (\".($repI+1).\" out of $numReps in this thread)\";\n\n    my @name;\n    # Start off a distances string for printing to file later\n    my %dist = ();\n    for(my $sketchCounter=0; $sketchCounter<$numSketches; $sketchCounter++){\n      my $nameI = basename($$mashInfoHash{sketches}[$sketchCounter]{name},(@fastqExt,@fastaExt));\n      print $logFh \"Subsampling sketches from entry $sketchCounter, $nameI\\n\";\n\n      # subsample the hashes of one genome at a time as compared\n      # to the other genomes, to get a jack knife distance.\n      my $numHashes = scalar(@{ $$mashInfoHash{sketches}[$sketchCounter]{hashes} });\n      my $keepHashes = int($numHashes / 2); # based on half the hashes\n      my @subsampleHash = @{ $$mashInfoHash{sketches}[$sketchCounter]{hashes} };\n      @subsampleHash = (shuffle(@subsampleHash))[0..$keepHashes-1];\n      @subsampleHash = sort{$a<=>$b} @subsampleHash;\n\n      print $logFh \"Distances between $nameI and other genomes\";\n      # Initialize the distances string with the query name.\n      # Use this string as a buffer for the distances file\n      # to help avoid too much disk IO.\n      push(@name, $nameI);\n      for(my $j=0; $j<$numSketches; $j++){\n        my $nameJ = basename($$mashInfoHash{sketches}[$j]{name},(@fastqExt,@fastaExt));\n        my $distance = mashDist(\\@subsampleHash, $$mashInfoHash{sketches}[$j]{hashes}, $kmerlength);\n        $dist{$nameI}{$nameJ} = $distance;\n        print $logFh \".\";\n      }\n      print $logFh \"\\n\";\n    }\n\n    # Add distances to database\n    print $logFh \"Creating database, $subsampleDir/distances.db.tsv\\n\";\n    my $mashtreeDb = Mashtree::Db->new(\"$subsampleDir/distances.db.tsv\");\n    $mashtreeDb->addDistancesFromHash(\\%dist);\n    # Convert to Phylip\n    my $phylipFile = \"$subsampleDir/distances.phylip\";\n    print $logFh \"Creating phylip file, $phylipFile\\n\";\n    open(my $phylipFh, \">\", $phylipFile) or die \"ERROR: could not write to $phylipFile: $!\";\n    print $phylipFh $mashtreeDb->toString(\\@name, \"phylip\");\n    close $phylipFh;\n\n    print $logFh \"Creating tree file, $subsampleDir/tree.dnd\\n\";\n    my $treeObj = createTreeFromPhylip($phylipFile, $subsampleDir, $settings);\n    push(@treeFile, \"$subsampleDir/tree.dnd\");\n\n    print $logFh \"DONE!\\n\";\n    close $logFh;\n  }\n\n  return \\@treeFile;\n}\n\n# Fixed bootstrapping function from bioperl\n# https://github.com/bioperl/bioperl-live/pull/304\nsub assess_bootstrap{\n   my ($self,$bs_trees,$guide_tree) = @_;\n   my @consensus;\n\n   if(!defined($bs_trees) || ref($bs_trees) ne 'ARRAY'){\n     die \"ERROR: second parameter in assess_bootstrap() must be a list\";\n   }\n   my $num_bs_trees = scalar(@$bs_trees);\n   if($num_bs_trees < 1){\n     die \"ERROR: no bootstrap trees were passed to assess_bootstrap()\";\n   }\n\n   # internal nodes are defined by their children\n\n   my (%lookup,%internal);\n   my $i = 0;\n   for my $tree ( $guide_tree, @$bs_trees ) {\n       # Do this as a top down approach, can probably be\n       # improved by caching internal node states, but not going\n       # to worry about it right now.\n\n       my @allnodes = $tree->get_nodes;\n       my @internalnodes = grep { ! $_->is_Leaf } @allnodes;\n       for my $node ( @internalnodes ) {\n           my @tips = sort map { $_->id } \n                      grep { $_->is_Leaf() } $node->get_all_Descendents;\n           my $id = \"(\".join(\",\", @tips).\")\";\n           if( $i == 0 ) {\n               $internal{$id} = $node->internal_id;\n           } else { \n               $lookup{$id}++;\n           }\n       }\n       $i++;\n   }\n   #my @save; # unsure why this variable is needed\n   for my $l ( keys %lookup ) {\n       if( defined $internal{$l} ) {#&& $lookup{$l} > $min_seen ) {\n           my $intnode = $guide_tree->find_node(-internal_id => $internal{$l});\n           $intnode->bootstrap(sprintf(\"%d\",100 * $lookup{$l} / $num_bs_trees));\n       }\n   }\n   return $guide_tree;\n}\n\n#######\n# Utils\n#######\n\nsub usage{\n  my $usage=\"$0: a wrapper around mashtree.\n  Usage: $0 [options] [-- mashtree options] *.fastq.gz *.fasta > tree.dnd\n  --outmatrix          ''   Output file for distance matrix\n  --reps               0    How many bootstrap repetitions to run;\n                            If zero, no bootstrapping.\n  --numcpus            1    This will be passed to mashtree and will\n                            be used to multithread reps.\n  --tempdir                 A directory path that will be created to\n                            store temporary files\n  --file-of-files           If specified, mashtree will try to read\n                            filenames from each input file. The file of\n                            files format is one filename per line. This\n                            file of files cannot be compressed.\n  \n  --                        Used to separate options for $0 and mashtree\n  MASHTREE OPTIONS:\\n\".\n  # Print the mashtree options starting with numcpus,\n  # skipping the tempdir option.\n  `mashtree --help 2>&1 | grep -A 999 \"TREE OPTIONS\" | grep -v ^Stopped`;\n\n  return $usage;\n}\n\n"
  },
  {
    "path": "bin/mashtree_wrapper_deprecated.pl",
    "content": "#!/usr/bin/env perl\n# Author: Lee Katz <lkatz@cdc.gov>\n# Uses Mash and BioPerl to create a NJ tree based on distances.\n# Run this script with -h for help and usage.\n\nuse strict;\nuse warnings;\nuse Data::Dumper;\nuse Getopt::Long;\nuse File::Temp qw/tempdir tempfile/;\nuse File::Basename qw/basename dirname fileparse/;\nuse File::Copy qw/cp mv/;\nuse List::Util qw/shuffle/;\nuse List::MoreUtils qw/part/;\nuse POSIX qw/floor/;\n\nuse Fcntl qw/:flock LOCK_EX/;\n\nuse threads;\nuse Thread::Queue;\nuse threads::shared;\n\nuse FindBin;\nuse lib \"$FindBin::RealBin/../lib\";\nuse Mashtree qw/logmsg @fastqExt @fastaExt createTreeFromPhylip/;\nuse Mashtree::Db;\nuse Bio::SeqIO;\nuse Bio::TreeIO;\nuse Bio::Tree::DistanceFactory;\nuse Bio::Tree::Statistics;\nuse Bio::Matrix::IO;\n\nmy $writeStick :shared;\n\nlocal $0=basename $0;\n\nexit main();\n\nsub main{\n  my $settings={};\n  my @wrapperOptions=qw(help outmatrix=s tempdir=s reps=i numcpus=i);\n  GetOptions($settings,@wrapperOptions) or die $!;\n  $$settings{reps}||=0;\n  $$settings{numcpus}||=1;\n  die usage() if($$settings{help});\n  die usage() if(@ARGV < 1);\n\n  $$settings{tempdir}||=tempdir(\"MASHTREE_WRAPPER.XXXXXX\",CLEANUP=>1,TMPDIR=>1);\n  mkdir($$settings{tempdir}) if(!-d $$settings{tempdir});\n  logmsg \"Temporary directory will be $$settings{tempdir}\";\n\n  if($$settings{reps} < 10){\n    logmsg \"WARNING: You have very few reps planned on this mashtree run. Recommended reps are at least 10 or 100.\";\n  }\n  \n  ## Catch some options that are not allowed to be passed\n  # Tempdir: All mashtree temporary directories will be under the\n  # wrapper's tempdir.\n  if(grep(/^\\-+tempdir$/,@ARGV) || grep(/^\\-+t$/,@ARGV)){\n    die \"ERROR: tempdir was specified for mashtree but should be an option for $0\";\n  }\n  # Numcpus: this needs to be specified in the wrapper and will\n  # appropriately be transferred to the mashtree script\n  if(grep(/^\\-+numcpus$/,@ARGV) || grep(/^\\-+n$/,@ARGV)){\n    die \"ERROR: numcpus was specified for mashtree but should be an option for $0\";\n  }\n  # Outmatrix: the wrapper script needs to control where\n  # the matrix goes because it can only have the outmatrix\n  # for the observed run and not the replicates for speed's\n  # sake.\n  if(grep(/^\\-+outmatrix$/,@ARGV) || grep(/^\\-+o$/,@ARGV)){\n    die \"ERROR: outmatrix was specified for mashtree but should be an option for $0\";\n  }\n  \n  # Separate flagged options from reads in the mashtree options\n  my @reads = ();\n  my @mashOptions = ();\n  for(my $i=0;$i<@ARGV;$i++){\n    if(-e $ARGV[$i]){\n      push(@reads, $ARGV[$i]);\n    } else {\n      push(@mashOptions, $ARGV[$i]);\n    }\n  }\n\n  # Copy reads over to the temp storage where I assume it is faster\n  # and where we can write .lock files.\n  my $inputdir = \"$$settings{tempdir}/input\";\n  mkdir $inputdir;\n  my $tmp_i= 0;\n  my @reads_per_thread = ([@reads]);\n  if($$settings{numcpus} > 1){\n    @reads_per_thread = part { $tmp_i++ % ($$settings{numcpus}-1) } @reads;\n  }\n  my @cpThread;\n  for(0..$$settings{numcpus}-1){\n    $cpThread[$_] = threads->new(sub{\n      my($fileArr) = @_;\n      my @copiedReads;\n      for my $file(@$fileArr){\n        logmsg \"Copying $file to temp space - $inputdir\";\n        my $copiedFile = \"$inputdir/\".basename($file);\n        cp($file, $copiedFile);\n        push(@copiedReads, $copiedFile);\n      }\n      return \\@copiedReads;\n    }, $reads_per_thread[$_]);\n  }\n  @reads = ();\n  for(@cpThread){\n    my $tmp = $_->join;\n    push(@reads, @$tmp);\n  }\n  logmsg \"Finished copying input data to $inputdir\";\n\n  my $mashOptions=join(\" \",@mashOptions);\n  my $reads = join(\" \", @reads);\n  \n  # Some filenames we'll expect\n  my $observeddir=\"$$settings{tempdir}/observed\";\n  my $obsDistances=\"$observeddir/distances.phylip\";\n  my $observedTree=\"$$settings{tempdir}/observed.dnd\";\n  my $outmatrix=\"$$settings{tempdir}/observeddistances.tsv\";\n\n  # Multithreaded reps\n  my @rep_id = (1..$$settings{reps});\n  my $repsPerThread = int($$settings{reps} / $$settings{numcpus}) + 1;\n  my @thr;\n  for(0..$$settings{numcpus}-1){\n    my @theseReps = splice(@rep_id, 0, $repsPerThread);\n    $thr[$_]=threads->new(\\&repWorker, \\@mashOptions, \\@reads, \\@theseReps, $settings);\n  }\n\n  my @bsTree;\n  for(@thr){\n    my $treeArr=$_->join;\n    for(@$treeArr){\n      push(@bsTree,Bio::TreeIO->new(-file=>$_)->next_tree);\n    }\n  }\n  \n  # Make the observed directory and run Mash\n  logmsg \"Running mashtree on full data\";\n  mkdir($observeddir);\n  system(\"$FindBin::RealBin/mashtree --outmatrix $outmatrix.tmp --tempdir $observeddir --numcpus $$settings{numcpus} $mashOptions $reads > $observedTree.tmp\");\n  die if $?;\n  mv(\"$observedTree.tmp\",$observedTree) or die $?;\n  mv(\"$outmatrix.tmp\",$outmatrix) or die $?;\n\n  # Combine trees into a bootstrapped tree and write it \n  # to an output file. Then print it to stdout.\n  logmsg \"Adding bootstraps to tree\";\n  my $biostat=Bio::Tree::Statistics->new;\n  my $guideTree=Bio::TreeIO->new(-file=>\"$observeddir/tree.dnd\")->next_tree;\n  my $bsTree=$biostat->assess_bootstrap(\\@bsTree,$guideTree);\n  for my $node($bsTree->get_nodes){\n    next if($node->is_Leaf);\n    my $id=$node->bootstrap||$node->id||0;\n    $node->id($id);\n  }\n  open(my $treeFh,\">\",\"$$settings{tempdir}/bstree.dnd\") or die \"ERROR: could not write to $$settings{tempdir}/bstree.dnd: $!\";\n  print $treeFh $bsTree->as_text('newick');\n  print $treeFh \"\\n\";\n  close $treeFh;\n\n  system(\"cat $$settings{tempdir}/bstree.dnd\"); die if $?;\n\n  if($$settings{'outmatrix'}){\n    cp($outmatrix,$$settings{'outmatrix'});\n  }\n  \n  return 0;\n}\n\nsub repWorker{\n  my($mashOptions, $reads, $reps,$settings)=@_;\n  my @bsTree;\n  if(!defined($reps) || ref($reps) ne 'ARRAY' || !@$reps){\n    return \\@bsTree;\n  }\n\n  my @mashOptions = @$mashOptions;\n  my @reads       = @$reads;\n\n  my $numcpus = floor($$settings{numcpus}/$$settings{reps});\n  $numcpus = 1 if($numcpus < 1);\n\n  for my $rep(@$reps){\n    my $repTempdir=\"$$settings{tempdir}/rep$rep\";\n    mkdir $repTempdir;\n    logmsg \"Starting mashtree replicate $rep - $repTempdir\";\n    \n    #logmsg \"Downsampling reads (replicate $rep).\";\n    # Downsample the reads\n    my @downsampledReads=();\n    for my $r(@reads){\n      my $newReads = \"$repTempdir/\".basename($r);\n      #logmsg \"DEBUG\";push(@downsampledReads, $newReads);next;\n\n      my @buffer = ();\n      open(my $lockFh, \">\", \"$r.lock\") or die \"ERROR: could not make lockfile $r.lock: $!\";\n      flock($lockFh, LOCK_EX) or die \"ERROR locking file $r.lock: $!\";\n\n      open(my $inFh, \"zcat $r | \") or die \"ERROR reading $r for downsampling: $!\";\n      open(my $outFh,\" | gzip -c > $newReads\") or die \"ERROR gzipping to $newReads: $!\";\n      close $outFh;\n      while(my $id=<$inFh>){\n        my $seq =<$inFh>;\n        my $plus=<$inFh>;\n        my $qual=<$inFh>;\n        if(rand(1) < 0.5){\n          push(@buffer, $id.$seq.$plus.$qual);\n          # The buffer size is something like 600 bytes per entry\n          # times 100,000 entries times 12 threads = 720Mb.\n          if(scalar(@buffer) > 100000){\n            {\n              # Only let one thread at a time write to the file but\n              # this makes us open the file in append mode.\n              lock($writeStick);\n              open($outFh, \" | gzip -c >> $newReads\") or die \"ERROR gzipping to $newReads: $!\";\n              print $outFh join(\"\",@buffer);\n              close $outFh;\n            }\n            @buffer = (); # flush the buffer\n          }\n        }\n      } \n      # Finish the remaining entries in the buffer\n      {\n        lock($writeStick);\n        open($outFh, \" | gzip -c >> $newReads\") or die \"ERROR gzipping to $newReads: $!\";\n        print $outFh join(\"\",@buffer);\n        close $outFh;\n      }\n      close $inFh;\n      close $lockFh;\n\n      push(@downsampledReads, $newReads);\n    }\n\n    logmsg \"Done downsampling for replicate $rep. Running mashtree on files in $repTempdir\";\n    my $log = `mashtree --numcpus $numcpus @mashOptions @downsampledReads 2>&1 > $repTempdir/tree.dnd`;\n    if($?){\n      die \"ERROR with mashtree on rep $rep (exit code $?):\\n$log\";\n    }\n\n    logmsg \"Finished with rep $rep\";\n    push(@bsTree,\"$repTempdir/tree.dnd\");\n  }\n\n  return \\@bsTree;\n}\n\n#######\n# Utils\n#######\n\nsub usage{\n  my $usage=\"$0: a wrapper around mashtree.\n  Usage: $0 [options] [-- mashtree options] *.fastq.gz *.fasta > tree.dnd\n  --outmatrix          ''   Output file for distance matrix\n  --reps               0    How many bootstrap repetitions to run;\n                            If zero, no bootstrapping.\n                            Bootstrapping will only work on compressed fastq\n                            files.\n  --numcpus            1    This will be passed to mashtree and will\n                            be used to multithread reps.\n  \n  --                        Used to separate options for $0 and mashtree\n  MASHTREE OPTIONS:\\n\".\n  # Print the mashtree options starting with numcpus,\n  # skipping the tempdir option.\n  `mashtree --help 2>&1 | grep -A 999 \"TREE OPTIONS\" | grep -v ^Stopped`;\n\n  return $usage;\n}\n\n"
  },
  {
    "path": "bin/min_abundance_finder.pl",
    "content": "#!/usr/bin/env perl\n\n# Find the minimum abundance of kmers\n# Original script was in Python at \n#   https://gist.github.com/alexjironkin/4ed43412878723491240814a0d5a6ed6/223dea45d70c9136703a4afaab0178cdbfbd2042\n# Original author of python script: @alexjironkin, Public Health Englad\n# I wanted to increase capatibility with Perl and have a more\n# standalone script instead of relying on the khmer package.\n# Author: Lee Katz <lkatz@cdc.gov>\n\nuse strict;\nuse warnings;\nuse Getopt::Long qw/GetOptions/;\nuse Data::Dumper qw/Dumper/;\nuse File::Basename qw/basename fileparse/;\nuse File::Temp qw/tempdir/;\nuse List::Util qw/max/;\n\nuse threads;\nuse Thread::Queue;\n\n# http://perldoc.perl.org/perlop.html#Symbolic-Unary-Operators\n# +Inf and -Inf will be a binary complement of all zeros\nuse constant MAXINT =>  ~0;\nuse constant MININT => -~0;\n\nlocal $0=basename $0;\nsub logmsg{print STDERR \"$0: @_\\n\";}\n\nexit main();\n\nsub main{\n  my $settings={};\n  GetOptions($settings,qw(help kmerlength|kmer=i kmerCounter=s delta=i gt|greaterthan=i tempdir=s numcpus=i)) or die $!;\n  $$settings{kmerlength} ||=21;\n  $$settings{kmerCounter}||=\"\";\n  $$settings{gt}         ||=1;\n  $$settings{tempdir}    ||=tempdir(TEMPLATE=>\"$0.XXXXXX\",CLEANUP=>1,TMPDIR=>1);\n  $$settings{numcpus}    ||=1;\n\n  if($$settings{delta}){\n    logmsg \"WARNING: --delta has been deprecated\";\n  }\n\n  my($fastq)=@ARGV;\n  die usage() if(!$fastq || $$settings{help});\n  die \"ERROR: I could not find fastq at $fastq\" if(!-e $fastq);\n\n  ## Find valleys with multithreading\n  # List all kmer sizes per thread.\n  my @kmerlength;\n  my $i=0;\n  for(my $k=7; $k<=32; $k+=3){\n    my $whichThread = $i % $$settings{numcpus};\n    push(@{$kmerlength[$whichThread]}, $k);\n    $i++;\n  }\n      \n  # Launch the threads with the various kmer length sub-arrays.\n  my @thr;\n  for my $kArray(@kmerlength){\n    # Ensure that each thing getting sent to the thread is independent\n    # of this main subroutine memory space.\n    my %settingsCopy = %$settings;\n    my @kArrayCopy   = @$kArray;\n    # Launch the thread.\n    push(@thr,\n      threads->new(\\&valleyWorker, $fastq, \\@kArrayCopy,\\%settingsCopy)\n    );\n  }\n\n  my %firstValleyVote;\n  for(@thr){\n    my $firstValley = $_->join;\n    for my $minimum(@$firstValley){\n      next if(!$minimum);\n      $firstValleyVote{$minimum}++;\n    }\n  }\n\n  # If no valleys were found, simply set the stage so that\n  # the minimum depth will be set to 0.\n  if(keys(%firstValleyVote) < 1){\n    logmsg \"NOTE: no valleys were found and so I am inserting an imaginary vote for a valley at cov=1\";\n    $firstValleyVote{0} = 1;\n  }\n\n  logmsg \"For various values of k, valleys were found:\";\n  my $firstValley=0;\n  # Average out the votes\n  my $totalFirstValley = 0;\n  my $totalVotes = 0;\n  my @vote;\n  # Sort bins by their votes, highest to lowest\n  for my $bin(sort{$firstValleyVote{$b}<=>$firstValleyVote{$a} || $a<=>$b} keys(%firstValleyVote)){\n    my $value=$firstValleyVote{$bin};\n    $firstValley||=$bin; # set the valley to the first bin we come to\n    for(1..$value){\n      push(@vote, $bin);\n    }\n\n    $totalFirstValley += $bin * $value;\n    $totalVotes += $value;\n  }\n  @vote = sort {$a<=>$b} @vote;\n  logmsg @vote;\n  my $medianFirstValley = $vote[ int(scalar(@vote)/2) ];\n\n  # Get the average first valley across many kmers\n  my $avgFirstValley = $totalFirstValley/$totalVotes;\n  logmsg \"    Average first valley is $avgFirstValley\";\n  logmsg \"    However, I will use the median valley: $medianFirstValley\";\n  printf(\"%0.0f\\n\", $medianFirstValley);\n\n  #print join(\"\\t\",qw(kmer count)).\"\\n\";\n  #print join(\"\\t\", $firstValley, 1).\"\\n\";\n  return 0;\n\n}\n\n# Poor man's way of subsampling\n# Thanks to Nick Greenfield for pointing this out.\nsub mashHistogram{\n  my($fastq,$k,$settings)=@_;\n  my $sketch=\"$$settings{tempdir}/sketch.$k.msh\";\n  system(\"mash sketch -k $k -m 1 -o $sketch $fastq > /dev/null 2>&1\");\n  die if $?;\n  \n  my @histogram;\n  open(my $fh, \"mash info -c $sketch | \") or die \"ERROR: could not get mash info on sketch $sketch\";\n  while(my $line=<$fh>){\n    $line=~s/^\\s+|\\s+$//g;  # whitespace trim\n    next if($line=~/^#/);\n    my($filename, $bin, $frequency)=split(/\\t/, $line);\n    $histogram[$bin]=$frequency;\n  }\n  close $fh;\n  $histogram[$_]||=0 for(0..@histogram);\n  return \\@histogram;\n}\n\nsub readHistogram{\n  my($infile,$settings)=@_;\n  my @hist=(0);\n  open(HIST,$infile) or die \"ERROR: could not read $infile: $!\";\n  logmsg \"Reading histogram from $infile\";\n  while(<HIST>){\n    chomp;\n    my($count,$countOfCounts)=split /\\t/;\n    $hist[$count]=$countOfCounts;\n  }\n  close HIST;\n\n  # ensure defined values\n  for(my $i=0;$i<@hist;$i++){\n    $hist[$i] //= 0;\n  }\n\n  return \\@hist;\n}\n\nsub valleyWorker{\n  my($fastq, $kArray, $settings)=@_;\n  my @firstValley;\n\n  for my $kmerlength(@$kArray){\n    logmsg \"Counting $kmerlength-kmers in $fastq\";\n    my $histogram   = mashHistogram($fastq,$kmerlength,$settings);\n    my $firstValley = findFirstValley($histogram, $settings);\n    push(@firstValley, $firstValley);\n  }\n  return \\@firstValley;\n}\n\n# https://www.perlmonks.org/?node_id=629742\nsub localMinimaMaxima{\n  my($array, $settings)=@_;\n\n  my @arr = @$array;\n\n  my @minima;\n  my @maxima;\n  my $prev_cmp = 0;\n\n  my $num = @arr - 2;\n  for my $i (0 .. $num){\n    my $cmp = $arr[$i] <=> $arr[$i+1];\n    if ($cmp != $prev_cmp) {\n      # have a minimum only after there has been a maximum\n      if($cmp < 0 && @maxima > 0){\n        push @minima, $i;\n      }\n      elsif($cmp > 0){\n        push @maxima, $i;\n      }\n      # when this and next elements are ==, defer checking for\n      # minima/maxima till next loop iteration\n      $prev_cmp = $cmp if $cmp;\n    }\n  }\n\n  # Uncomment the following if we want to look at the very\n  # last number in the array.\n  #if (@$array) {\n  #  push @minima, $num if $prev_cmp >= 0;\n  #  push @maxima, $num if $prev_cmp <= 0;\n  #}\n\n  ## debugging\n  #splice(@$array, 30);\n  #print join(\".\",@minima).\"\\n\".join(\",\", @$array).\"\\n\\n\";\n\n  return(\\@minima, \\@maxima);\n}\n\nsub findFirstValley{\n  my($array, $settings)=@_;\n  my($minima, $maxima) = localMinimaMaxima($array, $settings);\n\n  # Return the first minimum if it's not the first element\n  # or if there are no other minima.\n  if($$minima[0] > 0 || !$$minima[1]){\n    return $$minima[0];\n  } else {\n    return $$minima[1];\n  }\n}\n\n# http://www.perlmonks.org/?node_id=761662\nsub which{\n  my($exe,$settings)=@_;\n  \n  my $tool_path=\"\";\n  for my $path ( split /:/, $ENV{PATH} ) {\n      if ( -f \"$path/$exe\" && -x \"$path/$exe\" ) {\n          $tool_path = \"$path/$exe\";\n          last;\n      }\n  }\n  \n  return $tool_path;\n}\n\nsub usage{\n  \"\n  $0: \n       Find the valley between two peaks on a set of kmers\n       such that you can discard the kmers that are \n       likely representative of contamination.\n       This script does not require any dependencies.\n\n  Usage: $0 file.fastq[.gz]\n  --gt     1   Look for the first peak at this kmer count\n               and then the next valley.\n  --kmer   21  kmer length\n  --numcpus 1  This script will apply one thread per kmer\n               length. Multiple values of k are tested to\n               get a consensus value.\n\n  MISC\n  --kmerCounter ''  The kmer counting program to use.\n                    Default: (empty string) auto-choose\n                    Options: perl, jellyfish\n  \"\n}\n\n"
  },
  {
    "path": "docs/ALGORITHM.md",
    "content": "# Algorithms\n\n## Fast or accurate mode\n\nWhile Mashtree has been developed for speed, some might want higher accuracy.\nIn response, we have implemented an optional minimum abundance filter for raw read input files.\nThis filter creates a histogram of kmers using Mash and detects the valleys between peaks.\nThe first (left-most) peak of a kmer histogram represents kmers that occur only once or rarely.\nThese \"rare\" kmers usually represent sequencing errors.\nTherefore, the minimum abundance filter can optionally be used to see where the first valley is and automatically set that value for the minimum kmer coverage parameter.\nUsing this filter helps avoid noise in the input data.\nRunning in accurate mode is shorthand for running Mashtree with the minimum abundance filter turned on.\nOtherwise by default, a user is running in fast mode.\n\nTo invoke accurate mode, run with `--mindepth 0`, e.g.,\n\n```bash\nmashtree --mindepth 0 --numcpus 12 *.fastq.gz [*.fasta] > mashtree.dnd\n```\n\n## Confidence values\n\nMost phylogenetic trees have confidence scores associated with each hypothetical ancestor node.\nThese support scores are typically generated by resampling the input data many times, analyzing each replicate in context of the original phylogeny.\nStrong  support for a node in the input data will result in consistent recovery of that node in the majority of resampled replicate trees.  Conversely, nodes which have low support in the input data will be sensitive to stochastic changes in the resampled sketches which can result in alternate topologies of involved isolates.  Therefore, the support value correlates with the robustness and reproducibility of a recovered relationship.  As the NJ tree recovered from a single tree reconstruction will always be a fully resolved binary tree, the addition of these resampled support values provides a valuable tool for assessing the confidence and reliability of a node.  This can be especially helpful when trying to assess the exact topology of clades containing  three or more very closely related genomes.\n\nAlthough Mashtree does not infer phylogeny, we have borrowed the ideas behind phylogenetic confidence values to yield confidence values for each parent node in the tree. There are two resampling methods implemented in Mashtree to assign support values to infernal nodes, bootstrapping and jackknifing. Initially, both methods create a tree as depicted in [Figure 1](../paper/Mashtree_workflow.png).\n\nThe bootstrapping method infers confidence on the tree by creating one new tree per replication [Figure 2](../paper/Bootstrap_workflow.png). However, instead of accepting the default seed value for Mash, a new one is generated.\nChanging this seed value will affect the value of hashed kmers and how they sort, therefore providing a different sampling of kmers in the sketch.\nDue to changes in sketch sampling, distances between genomes can differ and the resulting tree in the replication can differ. Then, nodes in the original tree are assigned support values based on how frequently the same bipartition was recovered in the replicate trees.\n\nIn the jackknife method, for each replication, each genome is picked one at a time as the query genome [Figure 3](../paper/Jackknife_workflow.png).  Its hashes are sampled without replacement for half of the original hashes.  Then, a Mash distance is recomputed between the query genome and all other genomes in the analysis.\nWhen the distances are recorded in the replicate's pairwise distance matrix, the average distance between each comparison is used of each query-reference genome pair, and vice versa. This new distance matrix is used for computing a NJ tree. Nodes in the original tree are assigned support values based on how frequently the same bipartition was recovered in replicate trees.\n\n# Figures\n\n![Figure 1](../paper/Mashtree_workflow.png)  \n\n**Figure 1.** The Mashtree workflow.  Step 1) Sketch genomes with Mash. In this schematic, there is a green circle representing each genome in the analysis.  Filled-in brown circles indicate the presence of a kmer.  Missing circles represent true absence.  After hashing with a sketch size of six (after the arrow), some kmers are not represented in the Mash sketch either because they are not present in the original genome or because only a finite number of kmers are sketched (e.g., six in this example).  Henceforth, truly missing hashes or hashes not included in the Mash sketch are represented by empty circles.  Step 2) Calculate distances with `Mash dist`.  Distances in the figure are represented by Jaccard distances, which are calculated as the intersection divided by the union.  In this example, the genomes are separated by Jaccard distances of 5/9, 4/9, and 3/9.  These jaccard distances are internally transformed into Mash distances [@Ondov:2016].  Step 3) Create dendrogram with Quicktree using the Mash distance matrix.\n\n![Figure 2](../paper/Bootstrap_workflow.png)  \n\n**Figure 2.** The Mashtree bootstrap workflow.  Step 1) Generate a tree with the normal workflow as in Figure 1. This is the main tree.  Step 2) Run the normal workflow once per replicate but with a different random seed. In this example, the top right replicate differs from the main tree.  All ten of these trees are the bootstrap tree replicates.  Step 3) For each parent node in the main tree, quantify how many bootstrap tree replicates have the same node with the same children. Record that percentage next to each parent node. This percentage quantifies how confident the Mashtree cluster is, controlling for the random seed in the Mash program.\n\n![Figure 3](../paper/Jackknife_workflow.png)  \n\n**FIgure 3.** The Mashtree jackknife workflow.  Step 1) Generate a tree with the normal workflow as in Figure 1. This is the main tree.  Step 2) For each replicate, sample the half hashes without replacement for each query genome. Recalculate the Mash distance between the query genome and all other genomes, reducing the denominator to one half, rounding up, to reflect the smaller pool of hashes.  After all genomes have been selected for query genomes, average the distances to create a new distance matrix.  Create the dendrogram from the new distance matrix.  For brevity, only one detailed replicate is shown.  Step 3) For each replication, calculate the new tree from the new distance matrix. In this example, the top right replication differs from the main tree. All ten of these trees are the jack knife tree replicates.  Step 4) For each parent node in the main tree, quantify how many jack knife tree replicates have the same node with the same children. Record that percentage next to each parent node.  This percentage quantifies how confident Mashtree is at clustering, controlling for stochasticity in hashes.\n\n\n"
  },
  {
    "path": "docs/INSTALL.md",
    "content": "# Installation\n\n## Prerequisite software\n\n* Mash >= v2.0\n* ~~SQLite3~~ Removed in version 1.4\n* Perl\n  * multithreading\n  * BioPerl library\n  * ~~`DBD::SQLite`~~ Removed in version 1.4\n* Quicktree\n\n### Environment\n\n#### Environment variables\n\nFor most Linux OSs, you will need to set up your environment like this:\n\n    export PATH=$HOME/bin:$PATH\n    export PERL5LIB=$PERL5LIB:$HOME/lib/perl5\n\n#### System packages\n\nSome system packages are needed.  On Ubuntu, this is how you might install these packages.\n\n    apt update\n    apt install -y build-essential cpanminus libexpat1-dev wget \n\n#### Perl packages\n\nThere are some perl packages too and so this is how you would install those on most Linux OSs:\n\n    cpanm -l ~ --notest BioPerl Bio::Sketch::Mash\n\n#### Quicktree\n\n    mkdir -pv $HOME/bin/build\n    cd $HOME/bin/build\n    wget https://github.com/khowe/quicktree/archive/v2.5.tar.gz\n    tar xvf v2.5.tar.gz \n    cd quicktree-2.5\n    make\n    mv quicktree $HOME/bin/\n\n#### Mash\n\n    mkdir -pv $HOME/bin/build\n    cd $HOME/bin/build\n    wget https://github.com/marbl/Mash/releases/download/v2.2/mash-Linux64-v2.2.tar\n    tar xvf mash-Linux64-v2.2.tar\n    mv -v mash-Linux64-v2.2/mash $HOME/bin/\n\n## Mashtree Installation from CPAN\n\nInstalling from CPAN installs the latest stable version of Mashtree.  This method adds the Mashtree perl modules to the correct place in your home directory and adds the executables to your home bin directory.\n\n    export PERL5LIB=$PERL5LIB:$HOME/lib/perl5\n    cpanm -l ~ Mashtree\n    mashtree --help # verify it shows usage and not an error\n\n## Uninstallation from CPAN\n\n    cpanm --uninstall Mashtree --local-lib=$HOME\n\n## Other sources for Mashtree\n\nOther instances of Mashtree can be found in the wild. Although I did not create these,\nothers have found them useful. I cannot provide support for these outside instances.\n\n### Docker\n\n* https://hub.docker.com/r/bioedge/mashtree\n* https://hub.docker.com/r/staphb/mashtree\n\n### Conda\n\n* https://anaconda.org/bioconda/mashtree\n\n\n"
  },
  {
    "path": "docs/TIPS.md",
    "content": "# Tips and tricks\n\n## Leveraging a high performance computer for bootstrapping\n\nBootstrapping is simply running mashtree several times and then quantifying how often a parent node\noccurs for each group of genomes.  This frequency is usually displayed as a percentage\nat each node.\n\n### Bootstrap trees\n\nHere is a bash loop to make 100 bootstrap trees\n\n    mkdir bootstrapTrees\n    export FASTA=$(ls *.fasta)\n    # Run the first tree without using --seed, to form your actual tree\n    # without bootstraps, using many threads so that it can be done before\n    # any of the bootstrap trees.\n    echo 'mashtree --numcpus $NSLOTS --mindepth 0 $FASTA' | \\\n      qsub -V -cwd -pe smp 1-36 -N mashtree -o mashtree.dnd -e mashtree.log\n    \n    # Make 100 bootstrap trees\n    for i in `seq 1 100`; do \n      # The echo statement makes a short bash script, which\n      # is piped to qsub\n      echo '\n        mashtree --numcpus $NSLOTS --mindepth 0 --seed $RANDOM $FASTA' \n      |\\\n        qsub -V -cwd -pe smp 1-12 -N mashtree$i -o bootstrapTrees/$i.dnd -e bootstrapTrees/$i.log; \n    done;\n    \n### Combine bootstrap trees\n\n#### Bootstrapping with RAxML\n\nRAxML is not a prerequisite for Mashtree, and so it might not be installed on your computer even if Mashtree is working.\n\n    cat bootstrapTrees/*.dnd > bootstrapTrees.dnd\n    raxmlHPC -f b -t mashtree.dnd -z bootstrapTrees.dnd -m GTRCAT -n TEST\n    # output file: RAxML_bipartitions.TEST\n    ln -sv RAxML_bipartitions.TEST bsTree.dnd # in case you want to have a sane file extension\n\n#### Bootstrapping with BioPerl\n\nThis script requires bioperl, which is a prerequisite for Mashtree.\n\n    perl -MBio::TreeIO -MBio::Tree::Statistics -e '\n      # Gather all the trees into an array\n      for my $file(glob(\"bootstrapTrees/*.dnd\")){\n        next if(! -s $file);\n        my $in=Bio::TreeIO->new(-file=>$file);\n        while($tree = $in->next_tree){\n          push(@tree, $tree);\n        }\n      }\n      \n      # Combine trees\n      my $baseTree = Bio::TreeIO->new(-file=>\"mashtree.dnd\")->next_tree;\n      $stats=Bio::Tree::Statistics->new; \n      my $bsTree = $stats->assess_bootstrap(\\@tree,$baseTree);\n      # BioPerl is very respectful about which values are IDs vs which are\n      # bootstraps. However, most tree drawing programs look at IDs, so\n      # we have to alter the ID\n      for ($bsTree->get_nodes){\n        next if($_->is_Leaf); # Do not alter leaves\n        $_->id($_->bootstrap);\n      }\n      \n      print $bsTree->as_text(\"newick\") .\"\\n\";\n    ' > bsTree.dnd\n    \n"
  },
  {
    "path": "docs/TROUBLESHOOTING.md",
    "content": "# Troubleshooting\n\n## General\n\nHere are some general tips for when things go wrong.\n\n### Read integrity\n\nYour reads might be badly formatted or the number of reads might be too low.\nYou can optionally download [Fasten](https://github.com/lskatz/fasten) and install it separately.\nThen, run `fasten_validate --help` or `fasten_inspect --help`\nand follow instructions on how to check each read set.  \n\n"
  },
  {
    "path": "lib/Mashtree/Db.pm",
    "content": "#!/usr/bin/env perl\npackage Mashtree::Db;\nuse strict;\nuse warnings;\nuse Exporter qw(import);\nuse File::Basename qw/fileparse basename dirname/;\nuse List::Util qw/shuffle/;\nuse Data::Dumper;\n\nuse lib dirname($INC{\"Mashtree/Db.pm\"});\nuse lib dirname($INC{\"Mashtree/Db.pm\"}).\"/..\";\n\nuse Mashtree qw/_truncateFilename logmsg sortNames/;\n\n\nour @EXPORT_OK = qw(\n         );\n\nlocal $0=basename $0;\n\n=pod\n\n=head1 NAME\n\nMashtree::Db - functions for Mashtree databasing \n\n=head1 SYNOPSIS\n\n  use strict;\n  use warnings\n  use Mashtree::Db;\n\n  my $dbFile = \"mashtree.tsv\";\n  my $db=Mashtree::Db->new($dbFile);\n\n  # Add 10 distances from genome \"test\" to other genomes\n  my %distHash;\n  for(my $dist=0;$dist<10;$dist++){\n    my $otherGenome = \"genome\" . $dist;\n    $distHash{\"test\"}{$otherGenome} = $dist;\n  }\n  $db->addDistancesFromHash(\\$distHash);\n\n  my $firstDistance = $db->findDistance(\"test\", \"genome0\");\n  # => 0\n\n=head1 DESCRIPTION\n\nThis is a helper module, usually not used directly.\nThis is how Mashtree reads and writes to the internal database.\n\n=cut\n\n=pod\n\n=head1 METHODS\n\n=over\n\n=item Mashtree::Db->new($dbFile, \\%settings)\n\nCreate a new Mashtree::Db object.\n\nThe database file is a tab-separated file and will be created if it doesn't exist.\nIf it does exist, then it will be read into memory.\n\nArguments:\n\n  * $dbFile - a file path\n  * $settings - a hash of key/values (currently unused)\n\n=back\n\n=cut\n\n# Properties of this object:\n#   dbFile\n#   cache\n#   settings, a hashref with keys:\n#     significant_figures, how many sigfigs in mash distances default:10\nsub new{\n  my($class,$dbFile,$settings)=@_;\n\n  # How many significant digits to go into the mash dists \n  $$settings{significant_figures} ||= 10;\n\n  my $self={\n    _significant_figures => $$settings{significant_figures},\n    cache                => {},\n  };\n  bless($self,$class);\n\n  $self->selectDb($dbFile);\n  $self->readDatabase();\n\n  return $self;\n}\n\n=pod\n\n=over \n\n=item $db->selectDb\n\nSelects a database. If it doesn't exist, then it will be created.\nThen, it sets the object property `dbFile` to the file path.\n\n=back\n\n=cut\n\n# Create a database from a TSV\n# Returns 1 if created a new database\nsub selectDb{\n  my($self, $dbFile)=@_;\n\n  $self->{dbFile}=$dbFile;\n\n  if(!-e $dbFile){\n    open(my $fh, '>', $dbFile) or die \"ERROR: could not write to $dbFile: $!\";\n    print $fh join(\"\\t\", qw(genome1 genome2 distance)).\"\\n\";\n    close $fh;\n  }\n\n  return 1;\n}\n\n=pod\n\n=over\n\n=item $db->readDatabase\n\nReads the database from the dbFile set by `selectDb`.\nReturns a hash of distances, e.g., genome1 => {genome2=>dist}\n\nThen, this hash of distances is set in the object property `cache`.\n\n=back\n\n=cut\n\nsub readDatabase{\n  my($self) = @_;\n\n  my %dist;\n\n  open(my $fh, \"<\", $self->{dbFile}) or die \"ERROR: could not read from \".$self->{dbFile}.\": $!\";\n  my $header = <$fh>;\n  chomp($header);\n  my @header = split(/\\t/, $header);\n  my $numHeaders = @header;\n  while(my $line = <$fh>){\n    chomp($line);\n    my @F = split(/\\t/, $line);\n    my %F;\n    for(my $i=0;$i<$numHeaders;$i++){\n      $F{$header[$i]} = $F[$i];\n    }\n    \n    $dist{$F{genome1}}{$F{genome2}} = $F{distance};\n  }\n\n  $self->{cache} = \\%dist;\n  return \\%dist;\n}\n\n=pod\n\n=over\n\n=item addDistancesFromHash\n\nAdd distances from a perl hash, $distHash\n$distHash is { genome1 => {$genome2 => $dist} }\n\n=back\n\n=cut\n\nsub addDistancesFromHash{\n  my($self,$distHash)=@_;\n\n  my $numInserted=0;   # how many are going to be inserted?\n\n  open(my $fh, \">>\", $self->{dbFile}) or die \"ERROR: could not append to \".$self->{dbFile}.\": $!\";\n  for my $genome1(sort keys(%$distHash)){\n    for my $genome2(sort keys(%{ $$distHash{$genome1} })){\n      my $dist = $$distHash{$genome1}{$genome2};\n      if(!defined($dist)){\n        logmsg \"WARNING: distance was not defined for $genome1 <=> $genome2\";\n        logmsg \"  Setting distance to 1.\";\n        $dist = 1;\n      }\n      print $fh join(\"\\t\", $genome1, $genome2, $dist).\"\\n\";\n      $numInserted++;\n    }\n  }\n  close $fh;\n\n  return $numInserted;\n}\n\n=pod \n\n=over\n\n=item $db->addDistances\n\nAdd distances from a TSV file.\nTSV file should be a mash distances tsv file and is in the format of, e.g.,\n   # query t/lambda/sample1.fastq.gz\n   t/lambda/sample2.fastq.gz  0.059\n   t/lambda/sample3.fastq.gz  0.061\n\n=back\n\n=cut\n\nsub addDistances{\n  my($self,$distancesFile)=@_;\n\n  # update the cache just in case\n  # it helps us skip some redundancies.\n  $self->readDatabase();\n\n  my $numInserted=0;   # how many are going to be inserted?\n\n  open(my $inFh, \"<\", $distancesFile) or die \"ERROR: could not read $distancesFile: $!\";\n  open(my $outFh, \">>\", $self->{dbFile}) or die \"ERROR: could not append to \".$self->{dbFile}.\": $!\";\n  my $query = \"\";\n  while(<$inFh>){\n    chomp;\n    if(/^#\\s*query\\s+(.+)/){\n      $query=$1;\n      $query=~s/^\\s+|\\s+$//g;  # whitespace trim before right-padding is added\n      $query=_truncateFilename($query);\n      next;\n    }\n    die \"ERROR: query was not stated in the mash dist -t output: $distancesFile\" if(!$query);\n    my($subject,$distance)=split(/\\t/,$_);\n    $subject=~s/^\\s+|\\s+$//g;  # whitespace trim before right-padding is added\n    $subject=_truncateFilename($subject);\n\n    next if(defined($self->findDistance($query,$subject)));\n    print $outFh join(\"\\t\", $query, $subject, $distance).\"\\n\";\n    $numInserted++;\n  }\n  close $outFh;\n  close $inFh;\n\n  return $numInserted;\n}\n\n=pod\n\n=over\n\n=item $db->findDistance\n\nFind the distance between any two genomes.\nReturn undef if not found.\n\n=back\n\n=cut\n\nsub findDistance{\n  my($self, $query, $subject) = @_;\n  \n  my $dists = $self->{cache};\n  if(defined($$dists{$query}{$subject})){\n    return $$dists{$query}{$subject};\n  }\n  if(defined($$dists{$subject}{$query})){\n    return $$dists{$subject}{$query};\n  }\n  \n  return undef;\n}\n\n=pod\n\n=over\n\n=item $db->findDistances\n\nFind all distances from one genome to all others\nReturn undef if not found.\n\n=back\n\n=cut\n\nsub findDistances{\n  my($self, $query) = @_;\n\n  my $dists = $self->{cache};\n  return $$dists{$query};\n}\n\n=pod\n\n=over\n\n=item $db->toString\n\nTurn the database into a string representation.\n\nArguments:\n\n  * genomeArray - list of genomes to include, or undef for all genomes\n  * format - can be a string of one of these values:\n    * tsv    3-column format (default)\n    * matrix all-vs all tsv format\n    * phylip Phylip matrix format\n  * sortBy - can be:\n    * abc (default)\n    * rand\n\n=back\n\n=cut\n\nsub toString{\n  my($self,$genome,$format,$sortBy)=@_;\n  $format//=\"tsv\";\n  $format=lc($format);\n  $sortBy//=\"abc\";\n  $sortBy=lc($sortBy);\n  \n  if($format eq \"tsv\"){\n    return $self->toString_tsv($genome,$sortBy);\n  } elsif($format eq \"matrix\"){\n    return $self->toString_matrix($genome,$sortBy);\n  } elsif($format eq \"phylip\"){\n    return $self->toString_phylip($genome,$sortBy);\n  }\n\n  die \"ERROR: could not format \".ref($self).\" as $format.\";\n}\n\nsub toString_matrix{\n  my($self,$genome,$sortBy)=@_;\n\n  my %distance=$self->toString_tsv($genome,$sortBy);\n  my @name=keys(%distance);\n  my $numNames=@name;\n\n  my $str=join(\"\\t\",\".\",@name).\"\\n\";\n  for(my $i=0;$i<$numNames;$i++){\n    $str.=$name[$i];\n    for(my $j=0;$j<$numNames;$j++){\n      my $dist = $distance{$name[$i]}{$name[$j]} || $distance{$name[$j]}{$name[$i]};\n      $str.=\"\\t$dist\";\n    }\n    $str.=\"\\n\";\n  }\n\n  return $str;\n}\n\n# Return the database in a TSV formatted file along with a header,\n# or if you ask for an array/hash, then it will return a hash of genome1=>{genome2=>dist}\nsub toString_tsv{\n  my($self,$genome,$sortBy)=@_;\n  $sortBy||=\"abc\";\n  $genome||=[];\n\n  my $dbh=$self->{dbh};\n\n  # Index the genome array\n  my %genome;\n  $genome{_truncateFilename($_)}=1 for(@$genome);\n\n  my %dist;\n\n  open(my $fh, \"<\", $self->{dbFile}) or die \"ERROR: could not read \".$self->{dbFile}.\": $!\";\n  my $str = <$fh>;\n  my @header = split(/\\t/, $str);\n  chomp(@header);\n  while(my $line = <$fh>){\n    $str .= $line;\n\n    my %d;\n    chomp($line);\n    my @F = split(/\\t/, $line);\n    @d{@header} = @F;\n\n    $dist{$d{genome1}}{$d{genome2}}=$d{distance};\n  }\n  return %dist if(wantarray);\n  return $str;\n}\n\nsub toString_phylip{\n  my($self,$genome,$sortBy)=@_;\n\n  my %dist = $self->toString_tsv();\n  $sortBy||=\"abc\";\n  $genome||=[];\n\n  my $sigfigs = $$self{_significant_figures} || die \"INTERNAL ERROR: could not set sigfigs\";\n\n  # Index the genome array\n  my %genome;\n  $genome{_truncateFilename($_)}=1 for(@$genome);\n\n  my $str=\"\";\n\n  # The way phylip is, I need to know the genome names\n  # a priori. Get the genome names from the db.\n  my @rawName = sort{$a cmp $b} keys(%dist);\n\n  my $maxGenomeLength=0;\n  my @name;\n  for my $name(@rawName){\n    # If the parameter was given to filter genome names,\n    # do it here.\n    my $truncatedName = _truncateFilename($name);\n    next if(@$genome && !$genome{$truncatedName});\n\n    push(@name,$name);\n    $maxGenomeLength=length($name) if(length($name) > $maxGenomeLength);\n  }\n\n  # We are already sorted alphabetically, so just worry\n  # about whether or not we sort by random\n  if($sortBy eq 'rand'){\n    @name = shuffle(@name);\n  }\n\n  my $numGenomes=@name;\n\n  $str.=(\" \" x 4) . \"$numGenomes\\n\";\n  for(my $i=0;$i<$numGenomes;$i++){\n\n    # sanitize sample names by removing spaces\n    my $sanitizedName = $name[$i];\n    $sanitizedName =~ s/\\s/_/g;\n\n    $str.=$sanitizedName;\n    $str.=\" \" x ($maxGenomeLength - length($name[$i]) + 2);\n    my $distanceHash = $dist{$name[$i]};\n\n    for(my $j=0;$j<$numGenomes;$j++){\n      if(!defined($$distanceHash{$name[$j]})){\n        logmsg \"WARNING: could not find distance for $name[$i] and $name[$j]\";\n      }\n      $str.=sprintf(\"%0.${sigfigs}f  \",$$distanceHash{$name[$j]});\n    }\n    $str=~s/ +$/\\n/; # replace that trailing whitespace with a newline\n  }\n  return $str;\n}\n\n1; # gotta love how we we return 1 in modules. TRUTH!!!\n\n"
  },
  {
    "path": "lib/Mashtree/Mash.pm",
    "content": "#!/usr/bin/env perl\npackage Mashtree::Mash;\nuse strict;\nuse warnings;\nuse Exporter qw(import);\nuse File::Basename qw/fileparse basename dirname/;\nuse Data::Dumper;\n\nuse lib dirname($INC{\"Mashtree/Mash.pm\"});\nuse lib dirname($INC{\"Mashtree/Mash.pm\"}).\"/..\";\n\nuse Mashtree qw/_truncateFilename logmsg/;\nuse JSON qw/from_json/;\nuse Bio::Seq;\nuse Bio::SimpleAlign;\nuse Bio::Align::DNAStatistics;\nuse Bio::Tree::DistanceFactory;\n\nour @EXPORT_OK = qw(\n         );\n\nlocal $0=basename $0;\n\n\n# Properties of this object:\nsub new{\n  my($class,$file,$settings)=@_;\n\n  if(ref($file) ne 'ARRAY'){\n    die \"ERROR: parameter \\$file must be a list of file(s)\";\n  }\n\n  my $self={\n    file=>$file,\n    info=>{},\n    hashes=>{},\n    distance=>{},\n  };\n  bless($self,$class);\n\n  # Gather info from each file. $self->{info} and\n  # $self->{hashes} gets updated.\n  for my $f(@$file){\n    $self->info($f);\n  }\n\n  # TODO are all mash metadata compatible?  Else: ERROR\n\n  return $self;\n}\n\nsub info{\n  my($self,$msh)=@_;\n  \n  my %info = %{ $self->{info} };\n\n  my $mashInfo=from_json(`mash info -d $msh`);\n\n  for my $sketch(@{ $$mashInfo{sketches} }){\n    #delete($$sketch{hashes});\n    $info{$$sketch{name}}=$sketch;\n\n    my %sketchHash;\n    for my $pos(@{ $$sketch{hashes} }){\n      $$self{hashes}{$pos}++;\n      $sketchHash{$pos}=1;\n    }\n    $info{$$sketch{name}}{hashes}=\\%sketchHash;\n  }\n\n  $self->{info}=\\%info;\n\n  return \\%info;\n}\n\nsub mashDistances{\n  my($self)=@_;\n  my @file=@{$self->{file}};\n  my $numFiles=@file;\n  \n  my %distance;\n  for(my $i=0;$i<$numFiles;$i++){\n    for(my $j=0;$j<$numFiles;$j++){\n      open(my $fh, \"mash dist '$file[$i]' '$file[$j]' | \") or die \"ERROR: running mash dist on $file[$i] and $file[$j]: $!\";\n      while(<$fh>){\n        chomp;\n        my($genome1,$genome2,$dist,$p,$sharedFraction)=split(/\\t/,$_);\n        $distance{$genome1}{$genome2}=$dist;\n      }\n      close $fh;\n    }\n  }\n  $self->{distance}=\\%distance;\n  return \\%distance;\n}\n\n# TODO make sets of alignments and also report the \n# mash distance between them all.\nsub alignment{\n  my($self)=@_;\n\n  # print an A for present, a T for not present\n  my %nt=(present=>\"A\",absent=>\"T\");\n\n  my $aln=Bio::SimpleAlign->new();\n  for my $name(keys(%{ $self->{info} })){\n    my $sketch=$$self{info}{$name};\n    my $sequence=\"\";\n    for my $hash(sort {$a<=>$b} keys(%{ $self->{hashes} })){\n      if($$sketch{hashes}{$hash}){\n        $sequence.= $nt{present};\n      } else {\n        $sequence.= $nt{absent};\n      }\n    } \n    my $seq=Bio::LocatableSeq->new(-id=>$name, -seq=>$sequence);\n    $aln->add_seq($seq);\n  }\n  return $aln;\n}\n\nsub refinedTree{\n  my($self,$aln)=@_;\n  my $dfactory = Bio::Tree::DistanceFactory->new(-method=>\"NJ\");\n  my $stats    = Bio::Align::DNAStatistics->new;\n  my $matrix   = $stats->distance(-method=>'Kimura', -align => $aln);\n  my $tree     = $dfactory->make_tree($matrix);\n  #print $tree->as_text().\"\\n\";\n  return $tree;\n}\n\n1; # gotta love how we we return 1 in modules. TRUTH!!!\n\n"
  },
  {
    "path": "lib/Mashtree.pm",
    "content": "#!/usr/bin/env perl\npackage Mashtree;\nuse strict;\nuse warnings;\nuse Exporter qw(import);\nuse File::Basename qw/fileparse basename dirname/;\nuse Data::Dumper;\nuse List::Util qw/shuffle/;\nuse Scalar::Util qw/looks_like_number/;\n\nuse threads;\nuse threads::shared;\n\nuse lib dirname($INC{\"Mashtree.pm\"});\nuse Bio::Matrix::IO;\nuse Bio::TreeIO;\n\nour @EXPORT_OK = qw(\n           logmsg openFastq _truncateFilename distancesToPhylip createTreeFromPhylip sortNames treeDist mashDist mashHashes raw_mash_distance raw_mash_distance_unequal_sizes\n           @fastqExt @fastaExt @bamExt @vcfExt @richseqExt @mshExt\n           $MASHTREE_VERSION\n         );\n\nlocal $0=basename $0;\n\n=pod\n\n=head1 NAME Mashtree\n\n=head1 SYNOPSIS\n\nHelps run a mashtree analysis to make rapid trees for genomes.\nPlease see github.com/lskatz/Mashtree for more information.\n\n=over\n\n=item mashtree executables\n\nThis document covers the Mashtree library, but the highlight\nthe mashtree package is the executable `mashtree`.\nSee github.com/lskatz/Mashtree for more information.\n\nFast method:\n\n    mashtree --numcpus 12 *.fastq.gz [*.fasta] > mashtree.dnd\n\nMore accurate method:\n\n    mashtree --mindepth 0 --numcpus 12 *.fastq.gz [*.fasta] > mashtree.dnd\n\nBootstrapping and jackknifing\n\n    mashtree_bootstrap.pl --reps 100 --numcpus 12 *.fastq.gz -- --min-depth 0 > mashtree.jackknife.dnd\n    mashtree_jackknife.pl --reps 100 --numcpus 12 *.fastq.gz -- --min-depth 0 > mashtree.jackknife.dnd\n\n=back\n\n=head1 VARIABLES\n\n=over\n\n=item $VERSION\n\n=item $MASHTREE_VERSION (same value as $VERSION)\n\n=item @fastqExt = qw(.fastq.gz .fastq .fq .fq.gz)\n\n=item @fastaExt = qw(.fasta .fna .faa .mfa .fas .fsa .fa)\n\n=item @bamExt = qw(.sorted.bam .bam)\n\n=item @vcfExt = qw(.vcf.gz .vcf)\n\n=item @mshExt = qw(.msh)\n\n=item @richseqExt = qw(.gb .gbank .genbank .gbk .gbs .gbf .embl .ebl .emb .dat .swiss .sp)\n\n=item $fhStick :shared \n\nUsed to mark whether a file is being read, so that Mashtree limits disk I/O\n\n=back\n\n=cut\n\n######\n# CONSTANTS\n\nour $VERSION = \"1.4.6\";\nour $MASHTREE_VERSION=$VERSION;\nour @fastqExt=qw(.fastq.gz .fastq .fq .fq.gz);\nour @fastaExt=qw(.fasta .fna .faa .mfa .fas .fsa .fa);\nour @bamExt=qw(.sorted.bam .bam);\nour @vcfExt=qw(.vcf.gz .vcf);\nour @mshExt=qw(.msh);\n# Richseq extensions were obtained mostly from bioperl under\n# the genbank, embl, and swissprot entries, under\n# the source for Bio::SeqIO\nour @richseqExt=qw(.gb .gbank .genbank .gbk .gbs .gbf .embl .ebl .emb .dat .swiss .sp);\n\n# Helpful things\nmy $fhStick :shared;  # A thread can only open a fastq file if it has the talking stick.\n\n=head1 METHODS\n\n=over\n\n=item $SIG{'__DIE__'}\n\nRemakes how `die` works, so that it references the caller\n\n=cut\n\n#################################################\n### COMMON SUBS/TOOLS (not object subroutines) ##\n#################################################\n# Redefine how scripts die\n$SIG{'__DIE__'} = sub {\n  local $0=basename($0);\n  my $e = $_[0] || \"\"; \n  my $callerSub=(caller(1))[3] || (caller(0))[3] || \"UnknownSub\";\n\n  $e =~ s/(at [^\\s]+? line \\d+\\.$)/\\nStopped $1/; \n  die(\"$0: $callerSub: $e\"); \n};\n\n=pod\n\n=item logmsg\n\nPrints a message to STDERR with the thread number and the program name, with a trailing newline.\n\n=cut\n\n# Centralized logmsg\n#sub logmsg {print STDERR \"$0: \".(caller(1))[3].\": @_\\n\";}\nsub logmsg {\n  local $0=basename $0;\n  my $parentSub=(caller(1))[3] || (caller(0))[3];\n  $parentSub=~s/^main:://;\n\n  # Find the thread ID and stringify it\n  my $tid=threads->tid;\n  $tid=($tid) ? \"(TID$tid)\" : \"\";\n\n  my $msg=\"$0: $parentSub$tid: @_\\n\";\n\n  print STDERR $msg;\n}\n\n=pod\n\n=item openFastq\n\n Opens a fastq file in a thread-safe way.\n\n=cut\n\nsub openFastq{\n  my($fastq,$settings)=@_;\n\n  my $fh;\n\n  lock($fhStick);\n\n  my @fastqExt=qw(.fastq.gz .fastq .fq.gz .fq);\n  my($name,$dir,$ext)=fileparse($fastq,@fastqExt);\n  if($ext =~/\\.gz$/){\n    open($fh,\"zcat $fastq | \") or die \"ERROR: could not open $fastq for reading!: $!\";\n  } else {\n    open($fh,\"<\",$fastq) or die \"ERROR: could not open $fastq for reading!: $!\";\n  }\n  return $fh;\n}\n\n=pod\n\n=item _truncateFilename\n\n Removes fastq extension, removes directory name,\n\n=cut\n\nsub _truncateFilename{\n  my($file,$settings)=@_;\n  # strip off msh and any other known extentions\n  my $name=$file;\n  my $oldName=\"\";\n  # Strip until we get convergence\n  while($name ne $oldName){\n    $oldName = $name;\n    $name = basename($name,@mshExt,@fastqExt,@richseqExt,@fastaExt);\n  }\n  return $name;\n}\n\n=pod\n\n=item distancesToPhylip\n\n1. Read the mash distances\n2. Create a phylip file\n\nArguments: hash of distances, output directory, settings hash\n\n=cut\n\nsub distancesToPhylip{\n  my($distances,$outdir,$settings)=@_;\n\n  my $phylip = \"$outdir/distances.phylip\"; \n  # NOTE: need to regenerate the combined distances each time\n  # because I need to allow variation in the input samples.\n  #return $phylip if(-e $phylip);\n\n  # The way phylip is, I need to know the genome names\n  # a priori\n  my %name;\n  open(MASHDIST,\"<\",$distances) or die \"ERROR: could not open $distances for reading: $!\";\n  while(<MASHDIST>){\n    next if(/^#/);\n    my($name)=split(/\\t/,$_);\n    $name=~s/^\\s+|\\s+$//g;  # whitespace trim before right-padding is added\n    $name{_truncateFilename($name,$settings)}=1;\n  }\n  close MASHDIST;\n  my @name=sortNames([keys(%name)],$settings);\n  # Index the array\n  my $columnIndex=0;\n  for(@name){\n    $name{$_}=$columnIndex++;\n  }\n\n  # Load up the matrix object\n  logmsg \"Reading the distances file at $distances\";\n  open(MASHDIST,\"<\",$distances) or die \"ERROR: could not open $distances for reading: $!\";\n  my $query=\"UNKNOWN\"; # Default ID in case anything goes wrong\n  my @m;\n  while(<MASHDIST>){\n    chomp;\n    if(/^#query\\s+(.+)/){\n      $query=_truncateFilename($1,$settings);\n    } else {\n      my ($reference,$distance)=split(/\\t/,$_);\n      $reference=_truncateFilename($reference,$settings);\n      $distance=sprintf(\"%0.10f\",$distance);\n      $m[$name{$query}][$name{$reference}]=$distance;\n      $m[$name{$reference}][$name{$query}]=$distance;\n    }\n  }\n  close MASHDIST;\n  #my $matrixObj=Bio::Matrix::Generic->new(-rownames=>\\@name,-colnames=>\\@name,-values=>\\@m);\n\n  # taking this method from write_matrix in http://cpansearch.perl.org/src/CJFIELDS/BioPerl-1.6.924/Bio/Matrix/IO/phylip.pm\n  my $str;\n  $str.=(\" \" x 4) . scalar(@name).\"\\n\";\n  for(my $i=0;$i<@name;$i++){\n    $str.=$name[$i];\n    my $count=0;\n    for(my $j=0;$j<@name;$j++){\n      if($count < $#name){\n        $str.=$m[$i][$j]. \"  \";\n      } else {\n        $str.=$m[$i][$j];\n      }\n      $count++;\n    }\n    $str.=\"\\n\";\n  }\n  open(PHYLIP,\">\",$phylip) or die \"ERROR: could not write to $phylip: $!\";\n  print PHYLIP $str;\n  close PHYLIP;\n  return $phylip;\n}\n\n=pod\n\n=item sortNames\n\nSorts names.\n\nArguments: \n\n1. $name - array of names\n2. $settings - options\n  * $$settings{'sort-order'} is either \"abc\", \"random\", \"input-order\"\n\n=cut\n\nsub sortNames{\n  my($name,$settings)=@_;\n  my @sorted;\n  if($$settings{'sort-order'} =~ /^(abc|alphabet)$/){\n    @sorted=sort { $a cmp $b } @$name;\n  } elsif($$settings{'sort-order'}=~/^rand(om)?/){\n    @sorted=shuffle(@$name);\n  } elsif($$settings{'sort-order'} eq 'input-order'){\n    @sorted=@$name;\n  } else {\n    die \"ERROR: I don't understand sort-order $$settings{'sort-order'}\";\n  }\n  return @sorted;\n}\n\n=pod\n\n=item createTreeFromPhylip($phylip, $outdir, $settings)\n\n Create tree file with Quicktree but bioperl \n as a backup.\n\n=cut\n\nsub createTreeFromPhylip{\n  my($phylip,$outdir,$settings)=@_;\n\n  my $treeObj;\n\n  my $quicktreePath=`which quicktree 2>/dev/null`;\n  # bioperl if there was an error with which quicktree\n  if($?){\n    logmsg \"DEPRECATION WARNING: CANNOT FIND QUICKTREE IN YOUR PATH. I will use BioPerl to make the tree this time, but it will be removed in the next version.\";\n    #logmsg \"Creating tree with BioPerl\";\n    my $dfactory = Bio::Tree::DistanceFactory->new(-method=>\"NJ\");\n    my $matrix   = Bio::Matrix::IO->new(-format=>\"phylip\", -file=>$phylip)->next_matrix;\n    $treeObj = $dfactory->make_tree($matrix);\n    open(TREE,\">\",\"$outdir/tree.dnd\") or die \"ERROR: could not open $outdir/tree.dnd: $!\";\n    print TREE $treeObj->as_text(\"newick\");\n    print TREE \"\\n\";\n    close TREE;\n  }\n  # quicktree\n  else {\n    #logmsg \"Creating tree with QuickTree\";\n    system(\"quicktree -in m $phylip > $outdir/tree.dnd.tmp\");\n    die \"ERROR with quicktree\" if $?;\n    $treeObj=Bio::TreeIO->new(-file=>\"$outdir/tree.dnd.tmp\")->next_tree;\n    open(my $treeFh, \">\", \"$outdir/tree.dnd\") or die \"ERROR: could not write to $outdir/tree.dnd: $!\";\n    print $treeFh $treeObj->as_text(\"newick\").\"\\n\";\n    #my $outtree=Bio::TreeIO->new(-file=>\">$outdir/tree.dnd\", -format=>\"newick\");\n    #$outtree->write_tree($treeObj);\n\n    unlink(\"$outdir/tree.dnd.tmp\");\n  }\n\n  return $treeObj;\n\n}\n\n=pod\n\n=item treeDist($treeObj1, $treeObj2)\n\n Lee's implementation of a tree distance. The objective\n is to return zero if two trees are the same.\n\n=cut\n\nsub treeDist{\n  my($treeObj1,$treeObj2)=@_;\n\n  # If the tree objects are really strings, then make Bio::Tree::Tree objects\n  if(!ref($treeObj1)){\n    if(-e $treeObj1){ # if this is a file, get the contents\n      $treeObj1=`cat $treeObj1`;\n    }\n    $treeObj1=Bio::TreeIO->new(-string=>$treeObj1)->next_tree;\n  }\n  if(!ref($treeObj2)){\n    if(-e $treeObj2){ # if this is a file, get the contents\n      $treeObj2=`cat $treeObj2`;\n    }\n    $treeObj2=Bio::TreeIO->new(-string=>$treeObj2)->next_tree;\n  }\n  for($treeObj1,$treeObj2){\n    #$_->force_binary;\n  }\n  \n  # Get all leaf nodes so that they can be compared\n  my @nodes1=sort {$a->id cmp $b->id} grep{$_->is_Leaf} $treeObj1->get_nodes;\n  my @nodes2=sort {$a->id cmp $b->id} grep{$_->is_Leaf} $treeObj2->get_nodes;\n  my $numNodes=@nodes1;\n\n  # Test 1: are these the same nodes?\n  my $nodeString1=join(\" \",map{$_->id} @nodes1);\n  my $nodeString2=join(\" \",map{$_->id} @nodes2);\n  if($nodeString1 ne $nodeString2){\n    # TODO print out the differing nodes?\n    logmsg \"ERROR: nodes are not the same in both trees!\\n  $nodeString1\\n  $nodeString2\";\n    return ~0; #largest int\n  }\n\n  # Find the number of branches it takes to get to each node.\n  # Turn it into a Euclidean distance\n  my $euclideanDistance=0;\n  for(my $i=0;$i<$numNodes;$i++){\n    for(my $j=$i+1;$j<$numNodes;$j++){\n      my ($numBranches1,$numBranches2);\n\n      my $lca1=$treeObj1->get_lca($nodes1[$i],$nodes1[$j]);\n      my $lca2=$treeObj2->get_lca($nodes2[$i],$nodes2[$j]);\n      \n      # Distance in tree1\n      my $distance1=0;\n      my @ancestory1=reverse $treeObj1->get_lineage_nodes($nodes1[$i]);\n      my @ancestory2=reverse $treeObj1->get_lineage_nodes($nodes1[$j]);\n      for my $currentNode(@ancestory1){\n        $distance1++;\n        last if($currentNode eq $lca1);\n      }\n      for my $currentNode(@ancestory2){\n        $distance1++;\n        last if($currentNode eq $lca1);\n      }\n      \n      # Distance in tree2\n      my $distance2=0;\n      my @ancestory3=reverse $treeObj2->get_lineage_nodes($nodes2[$i]);\n      my @ancestory4=reverse $treeObj2->get_lineage_nodes($nodes2[$j]);\n      for my $currentNode(@ancestory3){\n        $distance2++;\n        last if($currentNode eq $lca2);\n      }\n      for my $currentNode(@ancestory4){\n        $distance2++;\n        last if($currentNode eq $lca2);\n      }\n\n      if($distance1 != $distance2){\n        logmsg \"These two nodes do not have the same distance between trees: \".$nodes1[$i]->id.\" and \".$nodes1[$j]->id;\n      }\n\n      # Add up the Euclidean distance\n      $euclideanDistance+=($distance1 - $distance2) ** 2;\n    }\n  }\n  $euclideanDistance=sqrt($euclideanDistance);\n  return $euclideanDistance;\n}\n\n=pod\n\n=item mashDist($file1, $file2, $k, $settings)\n\nFind the distance between two mash sketch files\nAlternatively: two hash lists.\n\n=cut\n\nsub mashDist{\n  my($file1, $file2, $k, $settings)=@_;\n\n  my($hashes1, $hashes2, $kmer1, $kmer2);\n  if(ref($file1) eq 'ARRAY'){\n    $hashes1 = $file1;\n    $kmer1 = -1;\n  } else {\n    ($hashes1, $kmer1) = mashHashes($file1);\n  }\n  if(ref($file2) eq 'ARRAY'){\n    $hashes2 = $file2;\n    $kmer2 = -1;\n  } else {\n    ($hashes2, $kmer2) = mashHashes($file2);\n  }\n\n  if($kmer1 ne $kmer2){\n    die \"ERROR: kmer lengths do not match($kmer1 vs $kmer2)\";\n  }\n\n  # Set the default kmer length and perform sanity check\n  $k ||= $kmer1;\n  if(!looks_like_number($k)){\n    die \"ERROR: k was not set to an integer\";\n  }\n  if($k < 1){\n    die \"ERROR: k was undefined or set to less than 1\";\n  }\n\n  my($common, $total) = (0,0);\n  if(scalar(@$hashes1) != scalar(@$hashes2)){\n    ($common, $total) = raw_mash_distance_unequal_sizes($hashes1, $hashes2);\n  } else {\n    ($common, $total) = raw_mash_distance($hashes1, $hashes2);\n  }\n  my $jaccard = $common/$total;\n  my $mash_distance = -1/$k * log(2*$jaccard / (1+$jaccard));\n  #logmsg \"========== $mash_distance = -1/$k * log(2*$jaccard / (1+$jaccard)) ==============\";\n\n  return $mash_distance;\n}\n\n=pod\n\n=item mashHashes($sketch)\n\nReturn an array of hashes, the kmer length, and the genome estimated length\n\n=cut\n\nsub mashHashes{\n  my($sketch)=@_;\n  my @hash;\n  my $length = 0;\n  my $kmer   = 0;\n\n  if(!-e $sketch){\n    die \"ERROR: file not found: $sketch\";\n  }\n\n  my $fh;\n  if($sketch =~ /\\.msh$/){\n    open($fh, \"mash info -d $sketch | \") or die \"ERROR: could not run mash info -d on $sketch: $!\";\n  } elsif($sketch =~ /\\.json$/){\n    open($fh, $sketch) or die \"ERROR: could not read $sketch: $!\";\n  }\n  while(<$fh>){\n    if(/kmer\\D+(\\d+)/){\n      $kmer = $1;\n    }\n    elsif(/length\\D+(\\d+)/){\n      $length = $1;\n    }\n    elsif(/hashes/){\n      while(<$fh>){\n        last if(/\\]/);\n        next if(!/\\d/);\n        s/\\D+//g;\n        s/^\\s+|\\s+$//g;\n        push(@hash, $_);\n      }\n    }\n  }\n  if(!@hash){\n    die \"ERROR: no hashes found in $sketch\";\n  }\n  return (\\@hash, $kmer, $length);\n}\n\n=pod\n\n=item raw_mash_distance_unequal_sizes($hashes1, $hashes2)\n\nCompare unequal sized hashes. Treat the first\nset of hashes as the reference (denominator)\nset.\n\n=cut\n\nsub raw_mash_distance_unequal_sizes{\n  my($hashes1, $hashes2) = @_;\n\n  my (%sketch1,%sketch2);\n  @sketch1{@$hashes1} = (1) x scalar(@$hashes1);\n  @sketch2{@$hashes2} = (1) x scalar(@$hashes2);\n\n  my %union;\n  for my $h(@$hashes1){\n    if($sketch2{$h}){\n      $union{$h}++;\n    }\n  }\n\n  my $common = scalar(keys(%union));\n  my $total  = scalar(@$hashes1);\n\n  return($common,$total);\n}\n\n=pod\n\n=item raw_mash_distance($hashes1, $hashes2)\n\nReturn the number of kmers in common and the number compared total. inspiration from\nhttps://github.com/onecodex/finch-rs/blob/master/src/distance.rs#L34\n\n=cut\n\nsub raw_mash_distance{\n  my($hashes1, $hashes2) = @_;\n\n  my @sketch1 = sort {$a <=> $b} @$hashes1;\n  my @sketch2 = sort {$a <=> $b} @$hashes2;\n\n  my $i      = 0;\n  my $j      = 0;\n  my $common = 0;\n  my $total  = 0;\n\n  my $sketch_size = @sketch1;\n  while($total < $sketch_size && $i < @sketch1 && $j < @sketch2){\n    my $ltgt = ($sketch1[$i] <=> $sketch2[$j]); # -1 if sketch1 is less than, +1 if sketch1 is greater than\n\n    if($ltgt == -1){\n      $i += 1;\n    } elsif($ltgt == 1){\n      $j += 1;\n    } elsif($ltgt==0) {\n      $i += 1;\n      $j += 1;\n      $common += 1;\n    } else {\n      die \"Internal error\";\n    }\n\n    $total += 1;\n  }\n\n  if($total < $sketch_size){\n    if($i < @sketch1){\n      $total += @sketch1 - 1;\n    }\n\n    if($j < @sketch2){\n      $total += @sketch2 - 1;\n    }\n\n    if($total > $sketch_size){\n      $total = $sketch_size;\n    }\n  }\n\n  return ($common, $total);\n}\n\n# Calculates the Transfer Bootstrap Expectation (TBE) for internal nodes based on\n# the methods outlined in Lemoine et al, Nature, 2018.\n# Currently experimental.\n# I entered this sub into bioperl > v1.7.5 but it is worth\n# having locally here to help maintain compatibility.\n# The only difference is that it isn't an object method\n# and that it is called without an OO implementation.\n\n=item transfer_bootstrap_expectation\n\n Title   : transfer_bootstrap_expectation\n Usage   : my $tree_with_bs = transfer_bootstrap_expectation(\\@bs_trees,$guide_tree);\n Function: Calculates the Transfer Bootstrap Expectation (TBE) for internal nodes based on \n           the methods outlined in Lemoine et al, Nature, 2018.\n           Currently experimental.\n Returns : L<Bio::Tree::TreeI>\n Args    : Arrayref of L<Bio::Tree::TreeI>s\n           Guide tree, L<Bio::Tree::TreeI>s\n\n=back\n\n=cut\n\nsub transfer_bootstrap_expectation{\n  my ($bs_trees,$guide_tree) = @_;\n\n  if(!defined($bs_trees) || ref($bs_trees) ne 'ARRAY'){\n    die \"ERROR: second parameter in assess_bootstrap() must be a list\";\n  }\n  my $num_bs_trees = scalar(@$bs_trees);\n  if($num_bs_trees < 1){\n    die \"ERROR: no bootstrap trees were passed to \".(caller(0))[3];\n  }\n\n  # internal nodes are defined by their children\n  my %internal = ();\n  my %leafNameId = ();\n  my @idLookup = ();\n  my @internalLookup = ();\n  my @tree = ($guide_tree, @$bs_trees);\n  my $numTrees = scalar(@tree);\n  for(my $i = 0; $i < $numTrees; $i++){ # guide tree's index is $i==0\n    # Do this as a top down approach, can probably be\n    # improved by caching internal node states, but not going\n    # to worry about it right now.\n\n    my @allnodes = $tree[$i]->get_nodes;\n    my @internalnodes = grep { ! $_->is_Leaf } @allnodes;\n    for my $node ( @internalnodes ) {\n      my @tips = sort map { $_->id } \n                      grep { $_->is_Leaf() } $node->get_all_Descendents;\n      my $id = join(\",\", @tips);\n      # Map the concatenated-leaf ID to the internal ID on the guide tree\n      if( $i == 0 ) {\n        $internal{$id} = $node->internal_id;\n        $leafNameId{$node->internal_id} = $id;\n      }\n\n      # Record the tips for each tree's internal node\n      # ID lookup (concatenated string of leaf names)\n      $idLookup[$i]{$id} = \\@tips;\n      # Internal ID lookup\n      $internalLookup[$i]{$internal{$id}} = \\@tips;\n    }\n  }\n\n  # Find the average distance from branch b to all\n  # bootstrap trees' branches b*\n  my @id = sort keys %internal;\n  my $numIds = @id;\n  # Loop through all internal nodes of the guide tree\n  for(my $j=0; $j<$numIds; $j++){\n    my $refNode = $guide_tree->find_node(-internal_id => $internal{$id[$j]});\n    my $refNodeId = $refNode->internal_id;\n    my $refJoinId = $leafNameId{$refNodeId};\n    my $refLeaves = $idLookup[0]{$refJoinId};\n    my %refLeafIndex = map{$_=>1} @$refLeaves;\n    #next if(!defined($refLeaves));\n\n    # For each internal node, start calculating for\n    # an average TBE distance.\n    my $nodeTbeTotal = 0;\n  \n    # Loop through all bootstrap trees, skipping the 0th\n    # tree which is the guide tree.\n    for(my $i=1;$i<$numTrees;$i++){\n\n      # Find the right branch to bootstrap with. The right\n      # branch will be the one that has the smallest\n      # TBE distance.\n      my @bsNode   = grep {!$_->is_Leaf} $tree[$i]->get_nodes;\n      my $numBsIds = scalar(@bsNode);\n      my $minDistance = ~0; # large int\n      for(my $k=0;$k<$numBsIds;$k++){\n        my @queryLeaves = sort map { $_->id }\n                      grep { $_->is_Leaf() } $bsNode[$k]->get_all_Descendents;\n\n        my %queryLeafIndex = map{$_=>1} @queryLeaves;\n\n        # How many moves does it take to go from query to ref?\n        my $dist=0;\n        for my $queryLeaf(@queryLeaves){\n          if(!$refLeafIndex{$queryLeaf}){\n            $dist++;\n          }\n        }\n        for my $refLeaf(@$refLeaves){\n          if(!$queryLeafIndex{$refLeaf}){\n            $dist++;\n          }\n        }\n\n        if($dist < $minDistance){\n          $minDistance = $dist;\n        }\n      }\n      $nodeTbeTotal += $minDistance;\n    }\n    my $avgTbe = $nodeTbeTotal / $numTrees;\n    \n    # Calculate the average of all b to b* distances\n    # But it is also 1 - average.\n    my $numRefLeaves = scalar(@$refLeaves);\n    my $nodeTbe = 1 - $avgTbe/$numRefLeaves;\n    # Round to an integer\n    $refNode->bootstrap(sprintf(\"%0.0f\",100 * $nodeTbe));\n  }\n   \n  return $guide_tree;\n}\n\n1;\n\n\n__END__\n\n"
  },
  {
    "path": "misc/.dummy",
    "content": "\n"
  },
  {
    "path": "paper/paper.bib",
    "content": "@article{Ondov:2016,\r\n  title={Mash: fast genome and metagenome distance estimation using MinHash},\r\n  author={Ondov, Brian D and Treangen, Todd J and Melsted, P{\\'a}ll and Mallonee, Adam B and Bergman, Nicholas H and Koren, Sergey and Phillippy, Adam M},\r\n  journal={Genome Biology},\r\n  volume={17},\r\n  number={1},\r\n  pages={132},\r\n  year={2016},\r\n  publisher={BioMed Central},\r\n  doi={10.1186/s13059-016-0997-x}\r\n}\r\n\r\n@article{Saitou:1987,\r\n  title={The neighbor-joining method: a new method for reconstructing phylogenetic trees.},\r\n  author={Saitou, Naruya and Nei, Masatoshi},\r\n  journal={Molecular Biology and Evolution},\r\n  volume={4},\r\n  number={4},\r\n  pages={406--425},\r\n  year={1987},\r\n  doi={10.1093/oxfordjournals.molbev.a040454}\r\n}\r\n\r\n@article{Howe:2002,\r\n  title={QuickTree: building huge Neighbour-Joining trees of protein sequences},\r\n  author={Howe, Kevin and Bateman, Alex and Durbin, Richard},\r\n  journal={Bioinformatics},\r\n  volume={18},\r\n  number={11},\r\n  pages={1546--1547},\r\n  year={2002},\r\n  publisher={Oxford University Press},\r\n  doi={10.1093/bioinformatics/18.11.1546}\r\n}\r\n\r\n@article{Harris:2018,\r\n  title={SKA: Split Kmer Analysis Toolkit for Bacterial Genomic Epidemiology},\r\n  author={Harris, Simon R},\r\n  journal={BioRxiv},\r\n  pages={453142},\r\n  year={2018},\r\n  publisher={Cold Spring Harbor Laboratory},\r\n  doi={10.1101/453142}\r\n}\r\n\r\n@article{Gardner:2015,\r\n  title={kSNP3. 0: SNP detection and phylogenetic analysis of genomes without genome alignment or reference genome},\r\n  author={Gardner, Shea N and Slezak, Tom and Hall, Barry G},\r\n  journal={Bioinformatics},\r\n  volume={31},\r\n  number={17},\r\n  pages={2877--2878},\r\n  year={2015},\r\n  publisher={Oxford University Press},\r\n  doi={10.1093/bioinformatics/btv271}\r\n}\r\n\r\n@article{Timme:2017,\r\n  title={Benchmark datasets for phylogenomic pipeline validation, applications for foodborne pathogen surveillance},\r\n  author={Timme, Ruth E and Rand, Hugh and Shumway, Martin and Trees, Eija K and Simmons, Mustafa and Agarwala, Richa and Davis, Steven and Tillman, Glenn E and Defibaugh-Chavez, Stephanie and Carleton, Heather A and others},\r\n  journal={PeerJ},\r\n  volume={5},\r\n  pages={e3893},\r\n  year={2017},\r\n  publisher={PeerJ Inc.},\r\n  doi={10.7717/peerj.3893}\r\n}\r\n\r\n@article {Baker:2019,\r\n\tauthor = {Baker, Daniel N and Langmead, Ben},\r\n\ttitle = {Dashing: Fast and Accurate Genomic Distances with HyperLogLog},\r\n\telocation-id = {501726},\r\n\tyear = {2019},\r\n\tdoi = {10.1101/501726},\r\n\tpublisher = {Cold Spring Harbor Laboratory},\r\n\tabstract = {Dashing is a fast and accurate software tool for estimating similarities of genomes or sequencing datasets. It uses the HyperLogLog sketch together with cardinality estimation methods that are specialized for set unions and intersections. Dashing summarizes genomes more rapidly than previous MinHash-based methods while providing greater accuracy across a wide range of input sizes and sketch sizes. It can sketch and calculate pairwise distances for over 87K genomes in 6 minutes. Dashing is open source and available at https://github.com/dnbaker/dashing.},\r\n\tURL = {https://www.biorxiv.org/content/early/2019/02/04/501726},\r\n\teprint = {https://www.biorxiv.org/content/early/2019/02/04/501726.full.pdf},\r\n\tjournal = {bioRxiv}\r\n}\r\n\r\n@article{Zhao:2018,\r\n    author = {Zhao, XiaoFei},\r\n    title = \"{BinDash, software for fast genome distance estimation on a typical personal laptop}\",\r\n    journal = {Bioinformatics},\r\n    volume = {35},\r\n    number = {4},\r\n    pages = {671-673},\r\n    year = {2018},\r\n    month = {07},\r\n    abstract = \"{The number of genomes (including meta-genomes) is increasing at an accelerating pace. In the near future, we may need to estimate pairwise distances between millions of genomes. Even with the use of cloud computing, very few softwares can perform such estimation.The multi-threaded software BinDash can perform such estimation using only a typical personal laptop. BinDash implemented b-bit one-permutation rolling MinHash with optimal densification, an existing data-mining technique. BinDash empirically outperforms the state-of-the-art software in terms of precision, compression ratio, memory usage and runtime according to our evaluation. Our evaluation is performed with a Dell Inspiron 157 559 Notebook on all bacterial genomes in RefSeq.BinDash is released under the Apache 2.0 license at https://github.com/zhaoxiaofei/BinDash.Supplementary data are available at Bioinformatics online.}\",\r\n    issn = {1367-4803},\r\n    doi = {10.1093/bioinformatics/bty651},\r\n    url = {https://doi.org/10.1093/bioinformatics/bty651},\r\n    eprint = {http://oup.prod.sis.lan/bioinformatics/article-pdf/35/4/671/27838595/bty651.pdf},\r\n}\r\n\r\n@article{Brown:2019,\r\n  title={Use of whole-genome sequencing for food safety and public health in the United States},\r\n  author={Brown, Eric and Dessai, Uday and McGarry, Sherri and Gerner-Smidt, Peter},\r\n  journal={Foodborne Pathogens and Disease},\r\n  volume={16},\r\n  number={7},\r\n  pages={441--450},\r\n  year={2019},\r\n  publisher={Mary Ann Liebert, Inc., publishers 140 Huguenot Street, 3rd Floor New~.},\r\n  doi={10.1089/fpd.2019.2662}\r\n}\r\n\r\n@article{bloom:1970,\r\n  title={Space/time trade-offs in hash coding with allowable errors},\r\n  author={Bloom, Burton H},\r\n  journal={Communications of the ACM},\r\n  volume={13},\r\n  number={7},\r\n  pages={422--426},\r\n  year={1970},\r\n  publisher={ACM},\r\n  doi={10.1145/362686.362692}\r\n}\r\n"
  },
  {
    "path": "paper/paper.md",
    "content": "---\ntitle: 'Mashtree: a rapid comparison of whole genome sequence files'\nauthors:\n- affiliation: \"1, 2\"\n  name: Lee S. Katz\n  orcid: 0000-0002-2533-9161\n- affiliation: 1\n  name: Taylor Griswold\n- affiliation: 3\n  name: Shatavia S. Morrison\n  orcid: 0000-0002-4658-5951\n- affiliation: 3\n  name: Jason A. Caravas\n  orcid: 0000-0001-9111-406X\n- affiliation: 2\n  name: Shaokang Zhang\n  orcid: 0000-0003-0874-2212\n- affiliation: 2\n  name: Henk C. den Bakker\n  orcid: 0000-0002-4086-1580\n- affiliation: 2\n  name: Xiangyu Deng\n- affiliation: 1\n  name: Heather A. Carleton\ndate: \"11 September 2019\"\noutput:\n  html_document:\n    df_print: paged\n  pdf_document: default\n  word_document: default\nbibliography: paper.bib\ntags:\n- dendrogram\n- mash\n- sketch\n- tree\n- rapid\naffiliations:\n- index: 1\n  name: Enteric Diseases Laboratory Branch, Centers for Disease Control and Prevention,\n    Atlanta, GA, USA\n- index: 2\n  name: Center for Food Safety, University of Georgia, Griffin, GA, USA\n- index: 3\n  name: Respiratory Diseases Laboratory Branch, Centers for Disease Control and Prevention,\n    Atlanta, GA, USA\n---\n\n# Summary\n\nIn the past decade, the number of publicly available bacterial genomes has increased dramatically.\nThese genomes have been generated for impactful initiatives, especially in the field of genomic epidemiology [@Timme:2017; @Brown:2019].\nGenomes are sequenced, shared publicly, and subsequently analyzed for phylogenetic relatedness.\nIf two genomes of epidemiological interest are found to be related, further investigation might be prompted.\nHowever, comparing the multitudes of genomes for phylogenetic relatedness is computationally expensive and, with large numbers, laborious.\nConsequently, there are many strategies to reduce the complexity of the data for downstream analysis,\nespecially using nucleotide stretches of length _k_ (kmers).\n\nOne major kmer strategy is to reduce each genome to split kmers. With split kmer analysis, kmers on both sides of a variable site are recorded, and the variable nucleotide is identified.\nWhen comparing two or more genomes, the variable sites are compared. Split kmers have been implemented in software packages such as KSNP and SKA [@Gardner:2015; @Harris:2018].\n\nAnother major kmer strategy is to convert genomic data into manageable datasets, usually called sketches [@Ondov:2016; @Baker:2019; @Zhao:2018].\nMost notably, an algorithm called min-hash was implemented in the Mash package [@Ondov:2016].  In the min-hash algorithm, all kmers are recorded and transformed into integers using hashing and a Bloom filter [@bloom:1970].  These hashed kmers are sorted and only the first several kmers are retained.  The kmers that appear at the top of the sorted list are collectively called the sketch.\nAny two sketches can be compared by counting how many hashed kmers they have in common. \n\nBecause min-hash creates distances between any two genomes,\nmin-hash values can be used to rapidly cluster genomes into trees using the neighbor-joining algorithm [@Saitou:1987].\nWe implemented this idea in software called Mashtree, which quickly and efficiently generates large trees that would be too computationally intensive using other methods.\n\n# Implementation\n\n## Workflow\n\nMashtree builds on two major algorithms that are already implemented in other software packages. The first is the min-hash algorithm, which is implemented in the software Mash [@Ondov:2016].  Mashtree uses Mash to create sketches of the genomes with the function `mash sketch`.  We elected to keep most default Mash parameters but increased the sketch size (number of hashed kmers) from 1,000 to 10,000 to increase discriminatory power.  Then, Mash is used to calculate the distances between genomes with `mash dist`.  Mashtree records these distances into a pairwise distance matrix. Next, Mashtree calls the neighbor-joining (NJ) algorithm which is implemented in the software QuickTree [@Howe:2002].  The Mash distance matrix is used with QuickTree with default options to generate a dendrogram.  The workflow is depicted in Figure 1.\n\n## Confidence values\n\nAlthough Mashtree does not infer phylogeny, we have borrowed the ideas behind phylogenetic confidence values to yield confidence values for each parent node in the tree.\nThere are two resampling methods implemented in Mashtree to assign support values to internal nodes: bootstrapping and jackknifing.\nInitially, both methods create a tree as depicted in Figure 1.\nThen, confidence values can be calculated for the tree using either the bootstrapping approach or the jackknifing approach (Figures 2 and 3).\n\n## Other features\n\nMashtree has several other useful features.\nFirst, Mashtree can read any common sequence file type and can read gzip-compressed files (e.g., fastq, fastq.gz, fasta).\nThis is a major advantage in being compatible with a wide variety of databases and with space-saving file compression.\nSecond, Mashtree takes advantage of multithreading.\nThe number of requested threads is used to determine how many genomes are sketched at the same time and how many sketches can be compared at the same time.\nWhen the number of threads requested outnumbers the number of operations that it can parallelize, Mashtree uses the multithreading already encoded in Mash sketches and distances.\nThird, Mashtree uses an SQLite database which can be used to cache results between runs.\n\n# Installation\n\nThe Mashtree package is programmed in Perl, and is available in the CPAN repository.\nDocumentation can be found at https://github.com/lskatz/mashtree.\n\n# Figures\n\n![The Mashtree workflow.  Step 1) Sketch genomes with Mash. In this schematic, there is a green circle representing each genome in the analysis.  Filled-in brown circles indicate the presence of a kmer.  Missing circles represent true absence.  After hashing with a sketch size of six (after the arrow), some kmers are not represented in the Mash sketch either because they are not present in the original genome or because only a finite number of kmers are sketched (e.g., six in this example).  Henceforth, truly missing hashes or hashes not included in the Mash sketch are represented by empty circles.  Step 2) Calculate distances with `Mash dist`.  Distances in the figure are represented by Jaccard distances, which are calculated as the intersection divided by the union.  In this example, the genomes are separated by Jaccard distances of 5/9, 4/9, and 3/9.  These Jaccard distances are internally transformed into Mash distances [@Ondov:2016].  Step 3) Create dendrogram with Quicktree using the Mash distance matrix.  ](Mashtree_workflow.png)\n\n\n![The Mashtree bootstrap workflow.  Step 1) Generate a tree with the normal workflow as in Figure 1. This is the main tree.  Step 2) Run the normal workflow once per replicate but with a different random seed. In this example, the top right replicate differs from the main tree.  All ten of these trees are the bootstrap tree replicates.  Step 3) For each parent node in the main tree, quantify how many bootstrap tree replicates have the same node with the same children. Record that percentage next to each parent node. This percentage quantifies how confident the Mashtree cluster is, controlling for the random seed in the Mash program.](Bootstrap_workflow.png)\n\n![The Mashtree jackknife workflow.  Step 1) Generate a tree with the normal workflow as in Figure 1. This is the main tree.  Step 2) For each replicate, sample the half hashes without replacement for each query genome. Recalculate the Mash distance between the query genome and all other genomes, reducing the denominator to one half, rounding up, to reflect the smaller pool of hashes.  After all genomes have been selected for query genomes, average the distances to create a new distance matrix.  Create the dendrogram from the new distance matrix.  For brevity, only one detailed replicate is shown.  Step 3) For each replication, calculate the new tree from the new distance matrix. In this example, the top right replication differs from the main tree. All ten of these trees are the jackknife tree replicates.  Step 4) For each parent node in the main tree, quantify how many jackknife tree replicates have the same node with the same children. Record that percentage next to each parent node.  This percentage quantifies how confident Mashtree is at clustering, controlling for stochasticity in hashes.  ](Jackknife_workflow.png)\n\n# Acknowledgements\n\nThis work was made possible through support from the Advanced Molecular Detection (AMD) Initiative at the Centers for Disease Control and Prevention.\nThank you Sam Minot, Andrew Page, Brian Raphael, and Torsten Seemann for helpful discussions.\nThe findings and conclusions in this report are those of the authors and do not necessarily represent the official position of the Centers for Disease Control and Prevention.\n\n# References\n"
  },
  {
    "path": "plugins/README.md",
    "content": "# Mashtree plugins\n\nMashtree plugins are executed such that a mashtree database\nis the first positional parameter. Flag options are allowed.\nAdditional tables within the database might get created, but\noverwriting the DISTANCE table is not allowed.\n\nFor more information on the standard database, please see\n[the documentation](../docs/SQL.md).\n\n## Synopses for individual plugins\n\n### mashtree_optimize\n\nThis plugin optimizes distances between all genomes. Currently\nonly uses Dijkstra's algorithm but potentially could be expanded.\n\n    Usage: mashtree_optimize.pl [options] mashtree.sqlite\n\n"
  },
  {
    "path": "plugins/mashtree_optimize.pl",
    "content": "#!/usr/bin/env perl\n# Author: Lee Katz <lkatz@cdc.gov>\n# Read a mashtree database to cluster genomes\n\nuse strict;\nuse warnings;\nuse Data::Dumper;\nuse Getopt::Long;\nuse File::Temp qw/tempdir tempfile/;\nuse File::Basename qw/basename dirname fileparse/;\nuse File::Copy qw/mv/;\nuse POSIX qw/floor/;\nuse List::Util qw/min max/;\nuse Scalar::Util qw/looks_like_number/;\n\nuse DBI;\n\nuse threads;\nuse threads::shared;\n\nuse FindBin;\nuse lib \"$FindBin::RealBin/../lib\";\nuse Mashtree qw/logmsg/;\nuse Graph::Dijkstra;\n\nlocal $0=basename $0;\n\nexit main();\n\nsub main{\n  my $settings={};\n  GetOptions($settings,qw(help tempdir=s method=s threshold|cutoff=f numcpus=i nonzero=f)) or die $!;\n  $$settings{numcpus}||=1;\n  $$settings{nonzero}||=1e-99;\n  $$settings{threshold}||=0.1;\n\n  my($dbFile) = @ARGV;\n  die usage() if($$settings{help} || !$dbFile);\n\n  if(!$$settings{method}){\n    $$settings{method}=\"dijkstra\";\n    logmsg \"Setting method to dijkstra.\";\n  }\n  $$settings{method}=lc($$settings{method});\n  $$settings{tempdir}||=tempdir(\"MASHTREE_OPTIMIZE_$$settings{method}.XXXXXX\",CLEANUP=>1,TMPDIR=>1);\n\n  if(!-e $dbFile){\n    die \"ERROR: could not find database file $dbFile\";\n  }\n\n  optimizeDb($dbFile,$$settings{method},$settings);\n\n  return 0;\n}\n\n# Optimize the database using the method supplied. This\n# subroutine will redirect the workload to whichever\n# specialized subroutine.\nsub optimizeDb{\n  my($dbFile,$method,$settings)=@_;\n\n  no strict 'refs';\n  my $subroutine=\"optimizeDb_$method\";\n  \n  if(!defined(&$subroutine)){\n    die \"ERROR: optimization method $method is not defined in this script\";\n  }\n\n  return &$subroutine($dbFile,$settings);\n}\n\nsub optimizeDb_dijkstra{\n  my($db,$settings)=@_;\n\n  my $graph = Graph::Dijkstra->new(edgedefault=>\"undirected\");\n\n  my $dbh = DBI->connect(\"dbi:SQLite:dbname=$db\",\"\",\"\",{\n      RaiseError => 1,\n      AutoCommit => 0,\n  });\n\n  logmsg \"Reading the database\";\n  my $sth = $dbh->prepare(qq(SELECT GENOME1,GENOME2,DISTANCE\n    FROM DISTANCE\n    WHERE GENOME1 != GENOME2\n    ));\n  my $rv = $sth->execute() or die $DBI::errstr;\n  if($rv < 0){\n    die \"ERROR: no distances were found in the database $db\";\n  }\n\n  my $rowCounter=0;\n  my %seenNode; # a fast hash to track whether we need to define a node\n  while(my @row=$sth->fetchrow_array()){\n    my($g1,$g2,$dist)=@row;\n    # next if($g1 eq $g2); # take care of this with SQL\n    $dist=$$settings{nonzero} if(!$dist);\n\n    if(!$seenNode{$g1}){\n      $graph->node({id=>$g1});\n      $seenNode{$g1}=1;\n    }\n    if(!$seenNode{$g2}){\n      $graph->node({id=>$g2});\n      $seenNode{$g2}=1;\n    }\n    $graph->edge({sourceID=>$g1,targetID=>$g2,weight=>$dist});\n    $rowCounter++;\n\n    if($rowCounter % 1000000 == 0){\n      logmsg \"$rowCounter distances read\";\n    }\n  }\n  undef(%seenNode);\n\n  # Create the new table\n  $sth = $dbh->prepare(qq(\n    DROP TABLE IF EXISTS OPTIMIZED_DISTANCE\n    ));\n  $sth->execute();\n  $sth = $dbh->prepare(qq(\n    CREATE TABLE OPTIMIZED_DISTANCE(\n      GENOME1     CHAR(255)    NOT NULL,\n      GENOME2     CHAR(255)    NOT NULL,\n      DISTANCE    REAL         NOT NULL,\n      PRIMARY KEY(GENOME1,GENOME2)\n    )) \n  );\n  $sth->execute();\n\n  my $insertSth = $dbh->prepare(\"INSERT INTO OPTIMIZED_DISTANCE(GENOME1,GENOME2,DISTANCE)\n    VALUES(?,?,?);\");\n  $dbh->{AutoCommit}=0;\n\n  my %distBuffer;       # buffer for db transactions\n  my $bufferSize=10000; # how many transactions at a time for the db\n  my @genome=sort {$a cmp $b} map {$$_{id}} $graph->nodeList;\n  my $numGenomes=@genome;\n  my $distancesCounter=0;\n  logmsg \"Optimizing $numGenomes genomes\";\n  for(my $i=0;$i<$numGenomes;$i++){\n    logmsg \"Optimizing $genome[$i] (\".($i+1).\"/$numGenomes)\";\n    for(my $j=$i+1;$j<$numGenomes;$j++){\n      # Sort the genome names so that we only store the pair once\n      # in a predictable fashion.\n      my($G1,$G2)=sort{$a cmp $b} ($genome[$i],$genome[$j]);\n      my %solution=(originID=>$G1, destinationID=>$G2);\n      $graph->shortestPath(\\%solution);\n      if(!$solution{edges}){\n        logmsg \"WARNING: could not find distance between $G1 and $G2. Setting to large INT.\";\n        $solution{weight}= ~0; # largest int\n        $solution{edges}=[];\n      }\n\n      $distBuffer{$G1}{$G2}=$solution{weight};\n      $distancesCounter++;\n\n      # Insert the distances if the buffer is full\n      if($distancesCounter % $bufferSize == 0){\n        my $insertValues=\"\";\n        while(my($G1,$distances)=each(%distBuffer)){\n          while(my($G2,$dist) = each(%$distances)){\n            $insertSth->execute($G1,$G2,$dist);\n          }\n        }\n        $dbh->commit();\n        %distBuffer=(); # purge the buffer\n      }\n    }\n  }\n\n  # One last set of inserts\n  while(my($G1,$distances)=each(%distBuffer)){\n    while(my($G2,$dist) = each(%$distances)){\n      $insertSth->execute($G1,$G2,$dist);\n    }\n  }\n  $dbh->commit();\n   \n}\n\nsub usage{\n  \"Optimizes a mashtree database using different methods\n  Usage: $0 [options] mashtree.sqlite\n\n  Note: only one optimization is available at this time\n\n  --method   Dijkstra   Optimize distances using Dijkstra's algorithm\n  \"\n}\n\n"
  },
  {
    "path": "t/00_env.t",
    "content": "#!/usr/bin/env perl\n\nuse strict;\nuse warnings;\nuse Test::More tests => 5;\nuse FindBin qw/$RealBin/;\n\nuse lib './lib';\nuse_ok 'Mashtree';\n$ENV{PATH}=\"$RealBin/../bin:\".$ENV{PATH};\n\n# Is Mash installed?\nmy $mash_is_missing = system(\"mash > mash.log 2>&1\");\nis $mash_is_missing, 0, \"Found Mash in PATH\" or\n  diag(\"The executable Mash was not found in PATH: \".`cat mash.log`);\n\nEND{unlink(\"mash.log\");}\n\n# Is Quicktree installed?\nmy $quicktree_is_missing = system(\"quicktree 2>&1 | grep -C 50 -i quicktree > mash.log 2>&1\"); # overwrite the mash log\nmy $quicktree_is_missing_mod = $quicktree_is_missing >> 8; # shift the exit code down 8 bits\nis $quicktree_is_missing_mod, 0, \"Found quicktree in PATH\" or \n  diag(\"The executable quicktree was not found in PATH: \".`cat mash.log`);\n\n# Test out mashtree exe\nmy $version = `mashtree --version`;\nmy $exit_code = $? >> 8;\nis $exit_code, 0, \"Mashtree --version exit code: $exit_code\";\n\n# If mash, quicktree, or mashtree gave an exit code, bail out of the whole test\nif($mash_is_missing || $quicktree_is_missing || $exit_code){\n  BAIL_OUT(\"Prerequisite software was not found\");\n}\n\n$version =~ s/Mashtree\\s*//; # Mashtree trim\n$version =~ s/^\\s+|\\s+$//;   # whitespace trim\nmy $found_nonversion = !! ($version=~/([^\\.\\d])/) + 0;\nis($found_nonversion, 0, \"Looking for version numbers in the format of 1.2.3 (Version returned was $version)\");\n\n"
  },
  {
    "path": "t/01_filetypes.t",
    "content": "#!/usr/bin/env perl\n\nuse strict;\nuse warnings;\nuse lib './lib';\nuse File::Basename qw/dirname/;\nuse Getopt::Long qw/GetOptions/;\n\nuse Test::More tests => 2;\n\nuse_ok 'Mashtree';\nuse Mashtree qw/treeDist/;\n\n$ENV{PATH}=\"./bin:$ENV{PATH}\";\n\nmy $settings={};\nGetOptions($settings,qw(numcpus=i)) or die $!;\n$$settings{numcpus}||=1;\n\n#my $correctMashtree=\"(CFSAN000189.gbk:0.00000,(CFSAN001112.ref:0.00001,CFSAN001140_1:0.00019):0.00001,((CFSAN000968.ref:0.00000,CFSAN001115.ref:0.00000):0.00001,(CFSAN000189.ref:0.00000,(CFSAN000191.ref:0.00002,(CFSAN000211.gbk:0.00045,CFSAN000961.gbk:0.00005):0.00003):0.00000):0.00001):0.00000);\";\n#my $correctMashtree=\"(CFSAN001112.ref:0.0000197713,(CFSAN001115.ref:0.0000107516,((CFSAN000968.ref:0.0000186460,(CFSAN000961.gbk:0.0000252712,CFSAN000211.gbk:0.0004753628):0.0000146923):0.0000077003,(CFSAN000191.ref:0.0000276279,(CFSAN000189.ref:0.0000023811,CFSAN000189.gbk:0.0000000000):0.0000033570):0.0000037416):0.0000038797):0.0000011147,CFSAN001140_1:0.0002225407);\";\nmy $correctMashtree=\"(CFSAN001112.ref:0.0000197713,(CFSAN001115.ref:0.0000107516,((CFSAN000968.ref:0.0000186460,(CFSAN000961:0.0000252712,CFSAN000211:0.0004753628):0.0000146923):0.0000077003,(CFSAN000191.ref:0.0000276279,(CFSAN000189.ref:0.0000023811,CFSAN000189:0.0000000000):0.0000033570):0.0000037416):0.0000038797):0.0000011147,CFSAN001140_1:0.0002225407);\";\n$correctMashtree=~s/(\\d+\\.)(\\d+)/$1 . substr($2,0,4)/ge; # global and expression\n\n# Test to see if the correct tree is made\nmy $mashtree=`mashtree --numcpus $$settings{numcpus} t/filetypes/*`;\nchomp($mashtree);\n$mashtree=~s/(\\d+\\.)(\\d+)/$1 . substr($2,0,4)/ge; # global and expression\n\nok treeDist($mashtree,$correctMashtree) < 11, \"Produce correct tree from mixed file types\";\n\n"
  },
  {
    "path": "t/02_lambda.t",
    "content": "#!/usr/bin/env perl\n\nuse strict;\nuse warnings;\nuse Data::Dumper;\nuse FindBin qw/$RealBin/;\n\nuse lib \"$RealBin/../lib\";\nuse File::Basename qw/dirname basename/;\nuse File::Temp qw/tempdir/;\nuse File::Copy qw/cp/;\nuse Digest::MD5 qw/md5_hex/;\n\nuse Test::More tests => 7;\n\nuse_ok 'Mashtree';\nuse Mashtree qw/treeDist mashDist raw_mash_distance mashHashes/;\n\n$ENV{PATH}=\"./bin:$ENV{PATH}\";\n\nmy $tempdir = tempdir(basename($0).\".XXXXXX\", TMP=>1, CLEANUP=>1);\n\nsubtest 'space in filename' => sub {\n  my $wd = \"$tempdir/spaces\";\n  mkdir $wd;\n  my @target;\n  for my $filename(glob(\"$RealBin/lambda/*.fastq.gz\")){\n    my $target = \"$wd/\".basename($filename);\n    $target =~ s/sample(\\d)/sample $1/; # add in a space for funsies\n    cp($filename, $target) or die \"ERROR: could not copy $filename to $target: $!\";\n\n    push(@target, $target);\n  }\n\n  # e.g., '02_lambda.tmp.tiF_bn/spaces/sample 1.fastq.gz' '02_lambda.tmp.tiF_bn/spaces/sample 2.fastq.gz' '02_lambda.tmp.tiF_bn/spaces/sample 3.fastq.gz' '02_lambda.tmp.tiF_bn/spaces/sample 4.fastq.gz'\n  my $targets = \"'\" . join(\"' '\", @target) . \"'\";\n  my $cmd = \"mashtree --outmatrix lambdadist.tsv --genomesize 40000 --numcpus 1 $targets 2>$0.log\";\n  note $cmd;\n  system($cmd);\n  my $exit_code = $? >> 8;\n  if($exit_code){\n    note `cat $0.log`;\n  }\n  is($exit_code, 0, \"Ran mashtree with exit code $exit_code\");\n};\n\nmy $correctMashtree=\"(sample3:0.00195,sample4:0.00205,(sample1:0.00205,sample2:0.00205):0.00010);\";\n$correctMashtree=~s/(\\d+\\.)(\\d+)/$1 . substr($2,0,4)/ge; # global and expression\n\n# Test to see if the correct tree is made\nEND{unlink \"lambdadist.tsv\"; unlink \"$0.log\";}\nmy $mashtree=`mashtree --outmatrix lambdadist.tsv --genomesize 40000 --save-sketches $RealBin/lambda/sketches --numcpus 1 $RealBin/lambda/*.fastq.gz 2>$0.log`;\nif($?){\n  BAIL_OUT(\"ERROR running mashtree: $!\\n \".`cat $0.log`);\n}\nchomp($mashtree);\n$mashtree=~s/(\\d+\\.)(\\d+)/$1 . substr($2,0,4)/ge; # global and expression\nmy $dist=treeDist($mashtree,$correctMashtree);\nis $dist , 0, \"Lambda test set tree, distance should be zero between trees\";\nif($dist!=0){\n  note \"Correct tree: $correctMashtree\";\n  note \"This tree:    $mashtree\";\n  BAIL_OUT(\"Incorrect tree found. Will not continue.\");\n}\n\n# Test for the correct distance matrix\nmy %matrix=(\n          'sample4' => {\n                         'sample4' => 0,\n                         'sample2' => '0.00417555',\n                         'sample1' => '0.0042153'\n                       },\n          'sample2' => {\n                         'sample2' => 0,\n                         'sample1' => '0.00414809'\n                       },\n          'sample3' => {\n                         'sample4' => '0.00402957',\n                         'sample2' => '0.00405078',\n                         'sample3' => 0,\n                         'sample1' => '0.0041298'\n                       },\n          'sample1' => {\n                         'sample1' => 0\n                       }\n        );\n# mirror the matrix\nwhile(my($ref,$queryHash)=each(%matrix)){\n  while(my($query,$dist)=each(%$queryHash)){\n    $matrix{$query}{$ref}=$dist;\n  }\n}\n\nsubtest \"Test matrix\" => sub {\n  plan tests => 16;\n  open(MATRIX, \"lambdadist.tsv\") or die \"ERROR: could not read lambdadist.tsv: $!\";\n  my $header=<MATRIX>;\n  chomp($header);\n  my (undef,@header)=split(/\\t/,$header);\n  while(my $distances=<MATRIX>){\n    chomp($distances);\n    my($label,@dist)=split /\\t/,$distances;\n    for(my $i=0;$i<@header;$i++){\n      is $dist[$i], $matrix{$label}{$header[$i]}, \"Distance between $label and $header[$i]\"\n        or note \"Should have been $dist[$i]\";\n    }\n  }\n  close MATRIX;\n};\n\n# Did we get exactly the right sketches?\nmy %sketches = (\n  \"$RealBin/lambda/sketches/sample1.fastq.gz.msh\" => \"3b11eed05ee26156758e3df04816e742\",\n  \"$RealBin/lambda/sketches/sample2.fastq.gz.msh\" => \"3b11eed05ee26156758e3df04816e742\",\n  \"$RealBin/lambda/sketches/sample3.fastq.gz.msh\" => \"3b11eed05ee26156758e3df04816e742\",\n  \"$RealBin/lambda/sketches/sample4.fastq.gz.msh\" => \"3b11eed05ee26156758e3df04816e742\",\n);\nsubtest \"Saving sketches\" => sub {\n  plan tests => 4;\n  for my $file(sort keys(%sketches)){\n    my $md5sum = $sketches{$file};\n    open(my $fh, \"mash info $file | \") or die \"ERROR running mash info on $file: $!\";\n    my @content = grep {!/sample/} <$fh>;\n    close $fh;\n    my $content = join(\"\", @content);\n    pass(\"TODO: check md5sum, host computer agnostic\"); next;\n    is(md5_hex($content), $md5sum, \"MD5 of \".basename($file))\n      or note \"Should have been $sketches{$file}. Check on file size and/or `mash info` to follow up.\";\n  }\n};\n\n# test Mash module functions\n# # 8460/10000\nmy ($hashes1, $k1, $length1) = mashHashes(\"$RealBin/lambda/sketches/sample1.fastq.gz.msh\");\nmy ($hashes2, $k2, $length2) = mashHashes(\"$RealBin/lambda/sketches/sample2.fastq.gz.msh\");\nmy ($common, $total) = raw_mash_distance($hashes1, $hashes2);\nis($common/$total, 8460/10000, \"Raw mash distance\")\n  or note \"Got: $common / $total but it should be 8460/10000\";\n\nmy $mashDist = mashDist(\"$RealBin/lambda/sketches/sample1.fastq.gz.msh\",\"$RealBin/lambda/sketches/sample2.fastq.gz.msh\");\nis(sprintf(\"%0.8f\",$mashDist), 0.00414809, \"Mash distance function\");\n\n"
  },
  {
    "path": "t/03_subsample.t",
    "content": "#!/usr/bin/env perl\n\nuse strict;\nuse warnings;\nuse lib './lib';\nuse File::Basename qw/dirname/;\nuse Getopt::Long qw/GetOptions/;\n\nuse Test::More tests => 2;\n\nuse_ok 'Mashtree';\nuse Mashtree qw/treeDist/;\n\n$ENV{PATH}=\"./bin:$ENV{PATH}\";\n\nmy $settings={};\nGetOptions($settings,qw(numcpus=i)) or die $!;\n$$settings{numcpus}||=1;\n\nmy $correctMashtree='((CFSAN000191.ref:0.0000277081,(CFSAN000189.ref:0.0000023811,CFSAN000189:0.0000000000):0.0000032768):0.0000050668,(CFSAN001115.ref:0.0000099241,CFSAN001112.ref:0.0000210585):0.0000032901,(((CFSAN001140_1:0.0112883298,CFSAN000211:0.0004680698):0.0000119053,CFSAN000968.ref:0.0000164555):0.0000085410,CFSAN000961:0.0000418070):0.0000036369);';\n$correctMashtree=~s/(\\d+\\.)(\\d+)/$1 . substr($2,0,4)/ge; # global and expression\n\n# Test to see if the correct tree is made\nmy $mashtree=`mashtree --min-depth 0 --numcpus $$settings{numcpus} t/filetypes/*`;\nchomp($mashtree);\n$mashtree=~s/(\\d+\\.)(\\d+)/$1 . substr($2,0,4)/ge; # global and expression\n\n#print STDERR \"$mashtree\\n\";\n\nmy $treedist=treeDist($correctMashtree, $mashtree);\nok $treedist < 2, \"Correct min_abundance_filter tree (dist: $treedist)\";\n\n"
  },
  {
    "path": "t/04_fileoffiles.t",
    "content": "#!/usr/bin/env perl\n\nuse strict;\nuse warnings;\nuse Data::Dumper;\nuse lib './lib';\nuse File::Basename qw/dirname/;\n\nuse Test::More tests => 2;\n\nuse_ok 'Mashtree';\nuse Mashtree qw/treeDist/;\n\n$ENV{PATH}=\"./bin:$ENV{PATH}\";\n\nmy $correctMashtree=\"((sample2:0.0020443525,sample1:0.0021037373):0.0000540274,sample3:0.0019622177,sample4:0.0020673526);\";\n\n# Pattern match for numbers and round them to 5 decimal places\n$correctMashtree=~s/(\\d+\\.)(\\d+)/$1 . substr($2,0,4)/ge; # global and expression\n\n# Test to see if the correct tree is made\nmy $mashtree=`mashtree --file-of-files --numcpus 1 t/file-of-files.txt`;\nchomp($mashtree);\n$mashtree=~s/(\\d+\\.)(\\d+)/$1 . substr($2,0,4)/ge; # global and expression\n\nis($mashtree, $correctMashtree, \"File of files test on lambda\");\n\n"
  },
  {
    "path": "t/06_jackknifingHashes.t",
    "content": "#!/usr/bin/env perl\n\nuse strict;\nuse warnings;\nuse Data::Dumper;\nuse FindBin qw/$RealBin/;\nuse lib \"$RealBin/../lib\";\nuse lib \"$RealBin/../lib/perl5\";\n\n# Need to actually update env because\n# of the executables called later.\n$ENV{PERL5LIB}=$ENV{PERL5LIB}.\":$RealBin/../lib/perl5\";\n\nuse File::Basename qw/dirname/;\nuse File::Path qw/rmtree/;\nuse Bio::TreeIO;\nuse IO::String;\nuse Scalar::Util qw/looks_like_number/;\n\nuse Test::More tests => 6;\n\nuse_ok 'Mashtree';\nuse Mashtree;\nuse Mashtree::Db;\n\n$ENV{PATH}=\"./bin:$ENV{PATH}\";\n\nmy $correctMashtree=\"((sample2:0.0020443525,sample1:0.0021037373)66:0.0000540274,sample3:0.0019622177,sample4:0.0020673526)83;\";\n$correctMashtree=~s/(\\d+\\.)(\\d+)/$1 . substr($2,0,4)/ge; # global and expression\n\n# Cleanup\nEND{\n  rmtree(\"$RealBin/lambda/jackknife.tmp\");\n  unlink(\"$RealBin/lambda/jackknife.log\");\n}\n\n# Test to see if the correct tree is made\nmy $mashtree=`mashtree_jackknife.pl --tempdir $RealBin/lambda/jackknife.tmp --reps 100 --numcpus 2 $RealBin/lambda/*.fastq.gz 2>$RealBin/lambda/jackknife.log`;\nif($?){\n  my $log = `cat $RealBin/lambda/jackknife.log`;\n  diag $log;\n  BAIL_OUT(\"mashtree_jackknife.pl exited with an error code $?\");\n}\nmy $passed = ok(defined($mashtree),\"Mashtree_jackknife.pl ran and produced a string\");\n$mashtree=~s/(\\d+\\.)(\\d+)/$1 . substr($2,0,4)/ge; # global and expression\n\nmy $fh = IO::String->new($mashtree);\nmy $tree = Bio::TreeIO->new(-fh=>$fh, -format=>\"newick\")->next_tree;\n$passed = is(ref($tree),\"Bio::Tree::Tree\",\"Produced a BioPerl tree object\");\nif(!$passed){\n  diag \"Tree string produced was $mashtree\";\n  BAIL_OUT(\"Tree object was not produced out of the tree string\");\n}\n\nsubtest \"Parts of the tree file intact\" => sub{\n  plan tests => 3;\n  my @nodes = $tree->get_nodes;\n  my @expectedBootstrap = (100, 11);\n  my $nodeCounter=0;\n  for my $node(grep {!$_->is_Leaf} @nodes){\n    ok(looks_like_number($node->id), \"Bootstrap is a number: \".$node->id);\n    note(\"Usually this bootstrap is around $expectedBootstrap[$nodeCounter], give or take 5%\");\n    $nodeCounter++;\n  }\n\n  my $correctNodeString = \"sample1 sample2 sample3 sample4\";\n  my $nodeString = join(\" \", sort map{$_->id} grep { $_->is_Leaf} @nodes);\n  is $correctNodeString, $nodeString, \"Taxon names in the tree: $nodeString\";\n};\n  \n# Test to validate distances on the first rep\nsubtest \"Database of distances for rep1\" => sub{\n  plan tests=>20;\n  my $db = Mashtree::Db->new(\"$RealBin/lambda/jackknife.tmp/rep1/distances.db.tsv\");\n  my @dist = split(/\\n/, $db->toString('','phylip'));\n  chomp(@dist);\n  shift(@dist);\n  for(my $i=0;$i<@dist;$i++){\n    my ($name, @dist) = split(/\\s+/, $dist[$i]);\n    for(my $j=0; $j<@dist; $j++){\n      ok(looks_like_number($dist[$j]), \"Distance is a number\")\n        or note \"  Found $dist[$j]\";\n      if($j==$i){\n        ok($dist[$j] == 0,\"Distance against self is 0\");\n      }\n    }\n  }\n};\n\n# Address the bug fix https://github.com/lskatz/mashtree/issues/51#issuecomment-604495598\nsubtest \"file-of-filenames\" => sub{\n  plan tests=>2;\n\n  # Make the file of filenames\n  my $fofn = \"$RealBin/lambda/lambda.fofn\";\n  my @file = glob(\"$RealBin/lambda/*.fastq.gz\");\n  open(my $fh, \">\", $fofn) or die \"ERROR: could not write to $fofn: $!\";\n  for my $f(@file){\n    print $fh \"$f\\n\";\n  }\n  close $fh;\n\n  my $mashtreeFofn = `mashtree_jackknife.pl --tempdir $RealBin/lambda/jackknife.tmp --reps 100 --numcpus 2 --file-of-files $fofn 2>$RealBin/lambda/jackknife.log`;\n  note $mashtreeFofn;\n  \n  my $treeFh = IO::String->new($mashtree);\n  my $tree = Bio::TreeIO->new(-fh=>$treeFh, -format=>\"newick\")->next_tree;\n  $passed = is(ref($tree),\"Bio::Tree::Tree\",\"Produced a BioPerl tree object\");\n  if(!$passed){\n    diag \"Tree string produced was $mashtree\";\n    BAIL_OUT(\"Tree object was not produced out of the tree string\");\n  }\n\n  my @leaf = sort map{$_->id} grep{$_->is_Leaf} $tree->get_nodes;\n  is(scalar(@leaf), 4, \"correct number of nodes\");\n  note \"leaves found: @leaf\";\n  note \"tree: \".$tree->as_text(\"newick\");\n};\n\n"
  },
  {
    "path": "t/08_bootstrap.t",
    "content": "#!/usr/bin/env perl\n\nuse strict;\nuse warnings;\nuse Data::Dumper;\nuse FindBin qw/$RealBin/;\nuse lib \"$RealBin/../lib\";\nuse File::Basename qw/dirname/;\nuse File::Path qw/rmtree/;\nuse Bio::TreeIO;\nuse IO::String;\nuse Scalar::Util qw/looks_like_number/;\n\nuse Test::More tests => 3;\n\nuse_ok 'Mashtree';\nuse Mashtree;\nuse Mashtree::Db;\n\n$ENV{PATH}=\"$RealBin/../bin:$ENV{PATH}\";\n\nmy $correctMashtree=\"((sample2:0.0020443525,sample1:0.0021037373)66:0.0000540274,sample3:0.0019622177,sample4:0.0020673526)83;\";\n$correctMashtree=~s/(\\d+\\.)(\\d+)/$1 . substr($2,0,4)/ge; # global and expression\n\n# Cleanup\nEND{\n  rmtree(\"$RealBin/lambda/bootstrap.tmp\");\n  unlink(\"$RealBin/lambda/bootstrap.log\");\n}\n\nsubtest \"run mashtree\" => sub{\n  # Test to see if the correct tree is made\n  my $mashtree=`mashtree_bootstrap.pl --tempdir $RealBin/lambda/bootstrap.tmp --reps 10 --numcpus 2 $RealBin/lambda/*.fastq.gz 2>$RealBin/lambda/bootstrap.log`;\n  if($?){\n    my $log = `cat $RealBin/lambda/bootstrap.log`;\n    diag $log;\n    BAIL_OUT(\"mashtree_bootstrap.pl exited with an error code $?\");\n  }\n  my $passed = ok(defined($mashtree),\"Mashtree_bootstrap.pl ran and produced a string\");\n  $mashtree=~s/(\\d+\\.)(\\d+)/$1 . substr($2,0,4)/ge; # global and expression\n\n  my $fh = IO::String->new($mashtree);\n  my $tree = Bio::TreeIO->new(-fh=>$fh, -format=>\"newick\")->next_tree;\n  $passed = is(ref($tree),\"Bio::Tree::Tree\",\"Produced a BioPerl tree object\");\n  if(!$passed){\n    diag \"Tree string produced was $mashtree\";\n    BAIL_OUT(\"Tree object was not produced out of the tree string\");\n  }\n\n  subtest \"Parts of the tree file intact\" => sub{\n    plan tests => 3;\n    my @nodes = $tree->get_nodes;\n    my @expectedBootstrap = (100, 11);\n    my $nodeCounter=0;\n    for my $node(grep {!$_->is_Leaf} @nodes){\n      ok(looks_like_number($node->id), \"Bootstrap is a number: \".$node->id);\n      note(\"Usually this bootstrap is around $expectedBootstrap[$nodeCounter], give or take 5%\");\n      $nodeCounter++;\n    }\n\n    my $correctNodeString = \"sample1 sample2 sample3 sample4\";\n    my $nodeString = join(\" \", sort map{$_->id} grep { $_->is_Leaf} @nodes);\n    is $correctNodeString, $nodeString, \"Taxon names in the tree: $nodeString\";\n  };\n\n};\n\n# Address the bug fix https://github.com/lskatz/mashtree/issues/51#issuecomment-604495598\nsubtest \"file-of-filenames\" => sub{\n  plan tests=>2;\n\n  # Make the file of filenames\n  my $fofn = \"$RealBin/lambda/lambda.fofn\";\n  my @file = glob(\"$RealBin/lambda/*.fastq.gz\");\n  open(my $fh, \">\", $fofn) or die \"ERROR: could not write to $fofn: $!\";\n  for my $f(@file){\n    print $fh \"$f\\n\";\n  }\n  close $fh;\n\n  my $mashtreeFofn = `mashtree_bootstrap.pl --tempdir $RealBin/lambda/bootstrap.tmp --reps 100 --numcpus 2 --file-of-files $fofn 2>$RealBin/lambda/bootstrap.log`;\n  note \"mashtree text is: $mashtreeFofn\";\n  \n  my $treeFh = IO::String->new($mashtreeFofn);\n  my $tree = Bio::TreeIO->new(-fh=>$treeFh, -format=>\"newick\")->next_tree;\n  my $passed = is(ref($tree),\"Bio::Tree::Tree\",\"Produced a BioPerl tree object\");\n  if(!$passed){\n    diag \"Tree string produced was $tree\";\n    BAIL_OUT(\"Tree object was not produced out of the tree string\");\n  }\n\n  my @leaf = sort map{$_->id} grep{$_->is_Leaf} $tree->get_nodes;\n  is(scalar(@leaf), 4, \"correct number of nodes\");\n  note \"@leaf\";\n};\n\n"
  },
  {
    "path": "t/11_tsv-db.t",
    "content": "#!/usr/bin/env perl\n\nuse strict;\nuse warnings;\nuse lib './lib';\nuse File::Basename qw/dirname/;\nuse File::Temp qw/tempdir/;\nuse Data::Dumper;\n\nuse FindBin qw/$RealBin/;\nuse lib \"$RealBin/../lib/perl5\";\nuse lib \"$RealBin/../lib/perl5/x86_64-linux-thread-multi/\";\n\nuse Test::More tests => 4;\n\nuse_ok 'Mashtree';\nuse_ok 'Mashtree::Db';\nuse Mashtree qw/_truncateFilename/;\n\n$ENV{PATH}=\"./bin:$ENV{PATH}\";\n$ENV{PATH}=~s/quicktree//gi; # remove quicktree for now because it produces a diff ordering of trees\n\nmy $tempdir=tempdir(\"testMashDb.XXXXXX\",CLEANUP=>1,TMPDIR=>1);\n\nmy $mashDistancesFile=\"$tempdir/testdistances.txt\";\nmy $dbFile=\"$tempdir/testdb.db.tsv\";\n\n# Create a large distances file to see if it can handle large inserts\nopen(my $fh, \">\", $mashDistancesFile) or die \"ERROR: could not write to $mashDistancesFile: $!\";\n# Each genome is labeled as genomeX and is X distance from genome 'test'\nprint $fh '#query test'.\"\\n\";\nfor(my $i=0;$i<20000;$i++){\n  print $fh join(\"\\t\",\"genome$i\",$i).\"\\n\";\n}\nclose $fh;\n\n# Make the large inserts and test the hashsum of the db\n\nmy $db=Mashtree::Db->new($dbFile);\nmy $numInserted=$db->addDistances($mashDistancesFile);\nok($numInserted == 20000, \"Added 20k distances to the database (actually added: $numInserted)\");\n\nsubtest 'Testing for specific distances' => sub {\n  plan tests=>16;\n  # Reload the database internally\n  $db->readDatabase;\n\n  # Test for these expected distances one at a time\n  for my $expectedDistance(0,10,100,256,987,1234,1432){\n    my $dist = $db->findDistance(\"genome$expectedDistance\",\"test\");\n    is $expectedDistance,$dist, \"Expected distance: $expectedDistance\";\n\n    my $distRev = $db->findDistance(\"test\",\"genome$expectedDistance\",);\n    is $expectedDistance,$distRev, \"Expected distance in reverse: $expectedDistance\";\n  }\n\n  my $distances = $db->findDistances(\"test\");\n  my $expectedNumDistances=20000;\n  is $expectedNumDistances, scalar(keys(%$distances)), \"Expected number of distances with Mashtree::Db::findDistances(): $expectedNumDistances\";\n\n  is $$distances{\"genome654\"}, 654, \"Specific distance from Mashtree::Db::findDistances()==654\";\n};\n\n"
  },
  {
    "path": "t/51_benchmark_mashtree.t",
    "content": "#!/usr/bin/env perl\n\nuse strict;\nuse warnings;\nuse Benchmark ':all';\nuse Test::More tests=>1;\nuse File::Temp qw/tempdir/;\nuse File::Which qw/which/;\nuse Data::Dumper;\n\nuse FindBin qw/$RealBin/;\nuse lib \"$RealBin/../lib/perl5\";\nuse lib \"$RealBin/../lib\";\n\n$ENV{PATH}=\"$RealBin/../bin:$ENV{PATH}\";\n\nmy $exe = which('mashtree');\nok($exe ne \"\", \"found a path to mashtree: $exe\");\n\ntimethese(20, {\n  'vanilla mashtree' => sub{\n    my $dnd = `mashtree --numcpus 1 --genomesize 40000 $RealBin/lambda/*.fastq.gz 2>/dev/null`;\n    die if $?;\n  },\n});\ntimethese(5, {\n  'bootstrap mashtree' => sub{\n    my $dnd = `mashtree_bootstrap.pl --reps 10 --numcpus 1 $RealBin/lambda/*.fastq.gz -- --genomesize 40000 2>/dev/null`;\n    die if $?;\n  },\n  'jackknife mashtree' => sub{\n    my $dnd = `mashtree_jackknife.pl --reps 10 --numcpus 1 $RealBin/lambda/*.fastq.gz -- --genomesize 40000 2>/dev/null`;\n    die if $?;\n  },\n});\n\n"
  },
  {
    "path": "t/55_benchmark_db.t",
    "content": "#!/usr/bin/env perl\n\nuse strict;\nuse warnings;\nuse Benchmark ':all';\nuse Test::More tests=>4;\nuse File::Temp qw/tempdir/;\nuse Data::Dumper;\n\nuse FindBin qw/$RealBin/;\nuse lib \"$RealBin/../lib/perl5\";\nuse lib \"$RealBin/../lib\";\nuse Mashtree::Db;\n\nmy $tempdir=tempdir(\"testMashDb.XXXXXX\",CLEANUP=>1,TMPDIR=>1);\n\nmy $mashDistancesFile=\"$tempdir/testdistances.txt\";\nmy $dbFile=\"$tempdir/testdb.db.tsv\";\n\n# Create a large distances file to see if it can handle large inserts\nopen(my $fh, \">\", $mashDistancesFile) or die \"ERROR: could not write to $mashDistancesFile: $!\";\n# Each genome is labeled as genomeX and is X distance from genome 'test'\nprint $fh '#query test'.\"\\n\";\nfor(my $i=0;$i<20000;$i++){\n  print $fh join(\"\\t\",\"genome$i\",$i).\"\\n\";\n}\nclose $fh;\n\nmy $db = Mashtree::Db->new($dbFile);\nmy $numInserted=$db->addDistances($mashDistancesFile);\nok($numInserted == 20000, \"Added 20k distances to the database (actually added: $numInserted)\");\n\nmy $distsHash = $db->readDatabase;\nis(keys(%$distsHash), 1, \"Keys of dists file total 1\");\nis(keys(%{ $$distsHash{test} }), 20000, \"Number of other genomes compared against 'test' genome is 20k.\");\n\nunlink($dbFile);\n$db = Mashtree::Db->new($dbFile);\n$numInserted=$db->addDistancesFromHash($distsHash);\nok($numInserted == 20000, \"Added 20k distances to the database (actually added: $numInserted)\");\n\ntimethese(30, {\n  'addDistancesFromMashFile' => sub{\n    unlink($dbFile);\n    my $db = Mashtree::Db->new($dbFile);\n    $db->addDistances($mashDistancesFile);\n  },\n  'addDistancesFromHash'     => sub{\n    unlink($dbFile);\n    my $db = Mashtree::Db->new($dbFile);\n    $db->addDistancesFromHash($distsHash);\n  },\n  'readDatabase'             => sub{\n    $db->readDatabase;\n  },\n});\n\n"
  },
  {
    "path": "t/archive/10_sqlite.t",
    "content": "#!/usr/bin/env perl\n\nuse strict;\nuse warnings;\nuse lib './lib';\nuse File::Basename qw/dirname/;\nuse File::Temp qw/tempdir/;\nuse Data::Dumper;\n\nuse FindBin qw/$RealBin/;\nuse lib \"$RealBin/../lib/perl5\";\nuse lib \"$RealBin/../lib/perl5/x86_64-linux-thread-multi/\";\n\nuse Test::More tests => 5;\n\nuse_ok 'Mashtree';\nuse_ok 'Mashtree::Db';\nuse Mashtree qw/_truncateFilename/;\n\n$ENV{PATH}=\"./bin:$ENV{PATH}\";\n$ENV{PATH}=~s/quicktree//gi; # remove quicktree for now because it produces a diff ordering of trees\n\n# Test for the sqlite3 executable\nsystem(\"which sqlite3\");\nok($?==0, \"SQLite executable\") or\n  diag(\"Could not find sqlite. Make sure that the executable sqlite3 is set up properly\");\n\nmy $tempdir=tempdir(\"testMash.XXXXXX\",CLEANUP=>1,TMPDIR=>1);\n\nmy $mashDistancesFile=\"$tempdir/testdistances.txt\";\nmy $sqliteFile=\"$tempdir/testdb.sqlite\";\n\n# Create a large distances file to see if it can handle large inserts\nopen(my $fh, \">\", $mashDistancesFile) or die \"ERROR: could not write to $mashDistancesFile: $!\";\nprint $fh '#query test'.\"\\n\";\nfor(my $i=0;$i<20000;$i++){\n  print $fh join(\"\\t\",\"genome$i\",$i).\"\\n\";\n}\nclose $fh;\n\n# Make the large inserts and test the hashsum of the db\n\nmy $db=Mashtree::Db->new($sqliteFile);\nmy $numInserted=$db->addDistances($mashDistancesFile);\nok($numInserted == 20000, \"Added 20k distances to the database\");\n\nsubtest 'Testing for specific distances' => sub {\n  plan tests=>9;\n  for my $expectedDistance(0,10,100,256,987,1234,1432){\n    my $dist = $db->findDistance(\"genome$expectedDistance\",\"test\");\n    is $expectedDistance,$dist, \"Expected distance: $expectedDistance\";\n  }\n\n  my $distances = $db->findDistances(\"test\");\n  my $expectedNumDistances=20000;\n  is $expectedNumDistances, scalar(keys(%$distances)), \"Expected number of distances with Mashtree::Db::findDistances(): $expectedNumDistances\";\n\n  is $$distances{\"genome654\"}, 654, \"Specific distance from Mashtree::Db::findDistances()==654\";\n};\n\n"
  },
  {
    "path": "t/file-of-files.txt",
    "content": "t/lambda/sample1.fastq.gz\nt/lambda/sample2.fastq.gz\nt/lambda/sample3.fastq.gz\nt/lambda/sample4.fastq.gz\n"
  },
  {
    "path": "t/lambda/mashtree.dnd",
    "content": "(sample3:0.00195,sample4:0.00205,(sample1:0.00205,sample2:0.00205):0.00010);\n"
  }
]