Repository: lskatz/mashtree Branch: master Commit: c0853a86cf52 Files: 47 Total size: 212.5 KB Directory structure: gitextract_qa42dgor/ ├── .github/ │ ├── ISSUE_TEMPLATE/ │ │ ├── bug_report.md │ │ └── feature_request.md │ └── workflows/ │ ├── cache.yml │ └── perl_ci.yml ├── .gitignore ├── .travis.yml.bak ├── CHANGES ├── CONTRIBUTING.md ├── LICENSE ├── MANIFEST ├── MANIFEST.SKIP ├── Makefile.PL ├── README.md ├── bin/ │ ├── mashtree │ ├── mashtree_bootstrap.pl │ ├── mashtree_cluster.pl │ ├── mashtree_init.pl │ ├── mashtree_jackknife.pl │ ├── mashtree_wrapper_deprecated.pl │ └── min_abundance_finder.pl ├── docs/ │ ├── ALGORITHM.md │ ├── INSTALL.md │ ├── TIPS.md │ └── TROUBLESHOOTING.md ├── lib/ │ ├── Mashtree/ │ │ ├── Db.pm │ │ └── Mash.pm │ └── Mashtree.pm ├── misc/ │ ├── .dummy │ └── mashtree ASM NGS.pptx ├── paper/ │ ├── paper.bib │ └── paper.md ├── plugins/ │ ├── README.md │ └── mashtree_optimize.pl └── t/ ├── 00_env.t ├── 01_filetypes.t ├── 02_lambda.t ├── 03_subsample.t ├── 04_fileoffiles.t ├── 06_jackknifingHashes.t ├── 08_bootstrap.t ├── 11_tsv-db.t ├── 51_benchmark_mashtree.t ├── 55_benchmark_db.t ├── archive/ │ └── 10_sqlite.t ├── file-of-files.txt ├── filetypes/ │ └── CFSAN001115.ref.fasta.bz2 └── lambda/ └── mashtree.dnd ================================================ FILE CONTENTS ================================================ ================================================ FILE: .github/ISSUE_TEMPLATE/bug_report.md ================================================ --- name: Bug report about: Create a report to help us improve title: '' labels: '' assignees: '' --- **Describe the bug** A clear and concise description of what the bug is. **To Reproduce** Steps to reproduce the behavior: 1. Go to '...' 2. Click on '....' 3. Scroll down to '....' 4. See error **Expected behavior** A clear and concise description of what you expected to happen. **Screenshots** If applicable, add screenshots to help explain your problem. **Desktop (please complete the following information):** - OS: [e.g. iOS] - Version [e.g. v0.57, can be obtained by running `mashtree --version`.] - which method did you install with? E.g., `cpanm`, `git clone`, ... **Additional context** Add any other context about the problem here. ================================================ FILE: .github/ISSUE_TEMPLATE/feature_request.md ================================================ --- name: Feature request about: Suggest an idea for this project title: '' labels: '' assignees: '' --- **Is your feature request related to a problem? Please describe.** A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] **Describe the solution you'd like** A clear and concise description of what you want to happen. **Describe alternatives you've considered** A clear and concise description of any alternative solutions or features you've considered. **Additional context** Add any other context or screenshots about the feature request here. ================================================ FILE: .github/workflows/cache.yml ================================================ name: force-cache on: [push] jobs: build: runs-on: ${{ matrix.os }} strategy: matrix: os: ['ubuntu-22.04'] perl: [ '5.34' ] name: Perl ${{ matrix.perl }} on ${{ matrix.os }} steps: - name: apt-get to prepare for cached perl packages run: | sudo apt-get update -y sudo apt-get install -y libdb-dev sqlite3 libgd-dev libsqlite3-dev libberkeleydb-perl libcdb-file-perl - uses: actions/checkout@v3 - uses: shogo82148/actions-setup-perl@v1 with: perl-version: '5.34' install-modules-with: cpanm install-modules-args: --with-develop --with-configure multi-thread: true install-modules: | JSON Bio::Perl Bio::Kmer Bio::Tree::Statistics Bio::Matrix::IO Bio::Tree::DistanceFactory - run: | echo "hello world!" # prove -lv t ================================================ FILE: .github/workflows/perl_ci.yml ================================================ # https://docs.github.com/en/actions/reference/workflow-syntax-for-github-actions # https://docs.github.com/en/actions/guides/publishing-docker-images # https://github.com/docker/build-push-action/blob/master/docs/advanced/share-image-jobs.md name: unit-testing on: [push] jobs: build: runs-on: ${{ matrix.os }} strategy: matrix: os: ['ubuntu-22.04'] perl: [ '5.34' ] #perl: [ '5.34', '5.32', '5.16.3' ] name: Perl ${{ matrix.perl }} on ${{ matrix.os }} steps: - uses: actions/checkout@v3 - name: Set up perl uses: shogo82148/actions-setup-perl@v1 with: perl-version: ${{ matrix.perl }} install-modules-with: cpanm install-modules-args: --with-develop --with-configure --force --notest enable-modules-cache: true multi-thread: true install-modules: | JSON Bio::Perl Bio::TreeIO Bio::Kmer Bio::Tree::Statistics Bio::Matrix::IO Bio::Tree::DistanceFactory Bio::Sketch::Mash - name: apt-get to prepare for cached perl packages run: | sudo apt-get update -y sudo apt-get install -y libdb-dev sqlite3 libgd-dev libsqlite3-dev libberkeleydb-perl libcdb-file-perl quicktree mash #- name: cpanm installdeps --notest --force # run: cpanm --installdeps . --notest --force --verbose - name: versions run: | mash --version quicktree -v sqlite3 --version - run: perl -V - name: perl modules checks run: | for i in Bio::Tree::Statistics threads Bio::Matrix::IO Bio::Tree::DistanceFactory; do perl -M$i -e 'print "checking module $i";'; done - name: make test run: perl Makefile.PL && make && make test ================================================ FILE: .gitignore ================================================ !Build/ .last_cover_stats /META.yml /META.json /MYMETA.* *.o *.pm.tdy *.bs # Devel::Cover cover_db/ # Devel::NYTProf nytprof.out # Dist::Zilla /.build/ # Module::Build _build/ Build Build.bat # Module::Install inc/ # ExtUtils::MakeMaker /blib/ /_eumm/ /*.gz /Makefile /Makefile.old /MANIFEST.bak /pm_to_blib /*.zip # temp testing files t/lambda/lambda.fofn t/lambda/lambda/ t/lambda/list.lst t/lambda/mashtree2.tmp/ t/lambda/sketches/ t/lambda/tmp.dnd t/lambda/tmp/ bin/ROSS-0.3* ================================================ FILE: .travis.yml.bak ================================================ language: perl perl: - "5.30-shrplib" env: PERL_CPANM_OPT="--notest --force --skip-satisfied" addons: apt: update: true packages: - bioperl - libgd-dev - sqlite3 - libsqlite3-dev - libberkeleydb-perl - libcdb-file-perl before_install: - cpanm --local-lib=~/perl5 local::lib && eval $(perl -I ~/perl5/lib/perl5/ -Mlocal::lib) # bootstrap modules needed to manipulate dist - "yes no | cpanm --force --notest Bio::Perl" #- cpanm -v -L . DBD::SQLite #- "yes no | cpanm --force --notest DBI" #- wget https://github.com/marbl/Mash/releases/download/v1.1.1/mash-Linux64-v1.1.1.tar.gz #- tar zxvf mash-Linux64-v1.1.1.tar.gz - wget https://github.com/marbl/Mash/releases/download/v2.2/mash-Linux64-v2.2.tar - tar xvf mash-Linux64-v2.2.tar - git clone https://github.com/khowe/quicktree.git - make -C quicktree install: - export PATH=$PATH:$HOME/bin - export PERL5LIB=$PERL5LIB:$(pwd)/lib/perl5 - export PERL5LIB=$PERL5LIB:$HOME/perl5:$HOME/perl5/lib/perl5 before_script: - ls -F # if this test fails, just be sure what the directory is - pwd; pwd -P - PATH=$PATH:./mash-Linux64-v2.2 - export PATH=$PATH:./quicktree script: - cpanm -l . --notest --force -v Bio::Tree::Statistics Bio::Matrix::IO Bio::Tree::DistanceFactory - find . -type f -name Statistics.pm #- find . -type f -name DistanceFactory.pm || true #- cpanm --notest --force --verbose -l . GD #- cpanm --notest --force -l . CDRAUG/BioPerl-1.7.4.tar.gz - cpanm --installdeps --notest . - perl Makefile.PL - perl bin/mashtree -h 2>&1 | grep . - perl bin/mashtree --help 2>&1 | grep . - make test ================================================ FILE: CHANGES ================================================ v1.4 removed SQL dependency v0.55 bootstrapping v0.41 * Fixed some unit test bugs from v0.40. Addresses issue #37 on GitHub. v0.40 * jack knifing enabled with hashes v0.37 * mashtree reps more stablized; reps unit test v0.36 * Fixed a bug with multithreaded accurate mode (--min-depth 0) and bumped the version up so that cpan could catch it. ================================================ FILE: CONTRIBUTING.md ================================================ # How to Contribute ## Getting Started * Make sure you have a GitHub account * Make sure you have a travis-ci account * **Add an issue to the github site**. Discuss the problem you are fixing with the maintainer. * Has the issue already been discussed on the GitHub issues page? * Be detailed and focused ## Making changes * Fork the Mashtree repo * Create your own new branch and name it with something descriptive and your username, e.g., `lskatz-fix-dendrogram-formatting`. * Fix the issue on your new branch * Make commits of logical and atomic units * Add any new unit tests to help with longetivity. Unit tests have a file extension `.t` and are written in perl. More details on unit testing can be found here: https://perldoc.perl.org/Test/More.html * If the unit test(s) show any errors on your local computer or Travis-CI, then fix these errors. * Make a pull request when all unit tests show no errors. ================================================ FILE: LICENSE ================================================ GNU GENERAL PUBLIC LICENSE Version 3, 29 June 2007 Copyright (C) 2007 Free Software Foundation, Inc. Everyone is permitted to copy and distribute verbatim copies of this license document, but changing it is not allowed. Preamble The GNU General Public License is a free, copyleft license for software and other kinds of works. The licenses for most software and other practical works are designed to take away your freedom to share and change the works. By contrast, the GNU General Public License is intended to guarantee your freedom to share and change all versions of a program--to make sure it remains free software for all its users. We, the Free Software Foundation, use the GNU General Public License for most of our software; it applies also to any other work released this way by its authors. You can apply it to your programs, too. When we speak of free software, we are referring to freedom, not price. Our General Public Licenses are designed to make sure that you have the freedom to distribute copies of free software (and charge for them if you wish), that you receive source code or can get it if you want it, that you can change the software or use pieces of it in new free programs, and that you know you can do these things. To protect your rights, we need to prevent others from denying you these rights or asking you to surrender the rights. Therefore, you have certain responsibilities if you distribute copies of the software, or if you modify it: responsibilities to respect the freedom of others. For example, if you distribute copies of such a program, whether gratis or for a fee, you must pass on to the recipients the same freedoms that you received. You must make sure that they, too, receive or can get the source code. And you must show them these terms so they know their rights. Developers that use the GNU GPL protect your rights with two steps: (1) assert copyright on the software, and (2) offer you this License giving you legal permission to copy, distribute and/or modify it. For the developers' and authors' protection, the GPL clearly explains that there is no warranty for this free software. For both users' and authors' sake, the GPL requires that modified versions be marked as changed, so that their problems will not be attributed erroneously to authors of previous versions. Some devices are designed to deny users access to install or run modified versions of the software inside them, although the manufacturer can do so. This is fundamentally incompatible with the aim of protecting users' freedom to change the software. The systematic pattern of such abuse occurs in the area of products for individuals to use, which is precisely where it is most unacceptable. Therefore, we have designed this version of the GPL to prohibit the practice for those products. If such problems arise substantially in other domains, we stand ready to extend this provision to those domains in future versions of the GPL, as needed to protect the freedom of users. Finally, every program is threatened constantly by software patents. States should not allow patents to restrict development and use of software on general-purpose computers, but in those that do, we wish to avoid the special danger that patents applied to a free program could make it effectively proprietary. To prevent this, the GPL assures that patents cannot be used to render the program non-free. The precise terms and conditions for copying, distribution and modification follow. TERMS AND CONDITIONS 0. Definitions. "This License" refers to version 3 of the GNU General Public License. "Copyright" also means copyright-like laws that apply to other kinds of works, such as semiconductor masks. "The Program" refers to any copyrightable work licensed under this License. Each licensee is addressed as "you". "Licensees" and "recipients" may be individuals or organizations. To "modify" a work means to copy from or adapt all or part of the work in a fashion requiring copyright permission, other than the making of an exact copy. The resulting work is called a "modified version" of the earlier work or a work "based on" the earlier work. A "covered work" means either the unmodified Program or a work based on the Program. To "propagate" a work means to do anything with it that, without permission, would make you directly or secondarily liable for infringement under applicable copyright law, except executing it on a computer or modifying a private copy. Propagation includes copying, distribution (with or without modification), making available to the public, and in some countries other activities as well. To "convey" a work means any kind of propagation that enables other parties to make or receive copies. Mere interaction with a user through a computer network, with no transfer of a copy, is not conveying. An interactive user interface displays "Appropriate Legal Notices" to the extent that it includes a convenient and prominently visible feature that (1) displays an appropriate copyright notice, and (2) tells the user that there is no warranty for the work (except to the extent that warranties are provided), that licensees may convey the work under this License, and how to view a copy of this License. If the interface presents a list of user commands or options, such as a menu, a prominent item in the list meets this criterion. 1. Source Code. The "source code" for a work means the preferred form of the work for making modifications to it. "Object code" means any non-source form of a work. A "Standard Interface" means an interface that either is an official standard defined by a recognized standards body, or, in the case of interfaces specified for a particular programming language, one that is widely used among developers working in that language. The "System Libraries" of an executable work include anything, other than the work as a whole, that (a) is included in the normal form of packaging a Major Component, but which is not part of that Major Component, and (b) serves only to enable use of the work with that Major Component, or to implement a Standard Interface for which an implementation is available to the public in source code form. A "Major Component", in this context, means a major essential component (kernel, window system, and so on) of the specific operating system (if any) on which the executable work runs, or a compiler used to produce the work, or an object code interpreter used to run it. The "Corresponding Source" for a work in object code form means all the source code needed to generate, install, and (for an executable work) run the object code and to modify the work, including scripts to control those activities. However, it does not include the work's System Libraries, or general-purpose tools or generally available free programs which are used unmodified in performing those activities but which are not part of the work. For example, Corresponding Source includes interface definition files associated with source files for the work, and the source code for shared libraries and dynamically linked subprograms that the work is specifically designed to require, such as by intimate data communication or control flow between those subprograms and other parts of the work. The Corresponding Source need not include anything that users can regenerate automatically from other parts of the Corresponding Source. The Corresponding Source for a work in source code form is that same work. 2. Basic Permissions. All rights granted under this License are granted for the term of copyright on the Program, and are irrevocable provided the stated conditions are met. This License explicitly affirms your unlimited permission to run the unmodified Program. The output from running a covered work is covered by this License only if the output, given its content, constitutes a covered work. This License acknowledges your rights of fair use or other equivalent, as provided by copyright law. You may make, run and propagate covered works that you do not convey, without conditions so long as your license otherwise remains in force. You may convey covered works to others for the sole purpose of having them make modifications exclusively for you, or provide you with facilities for running those works, provided that you comply with the terms of this License in conveying all material for which you do not control copyright. Those thus making or running the covered works for you must do so exclusively on your behalf, under your direction and control, on terms that prohibit them from making any copies of your copyrighted material outside their relationship with you. Conveying under any other circumstances is permitted solely under the conditions stated below. Sublicensing is not allowed; section 10 makes it unnecessary. 3. Protecting Users' Legal Rights From Anti-Circumvention Law. No covered work shall be deemed part of an effective technological measure under any applicable law fulfilling obligations under article 11 of the WIPO copyright treaty adopted on 20 December 1996, or similar laws prohibiting or restricting circumvention of such measures. When you convey a covered work, you waive any legal power to forbid circumvention of technological measures to the extent such circumvention is effected by exercising rights under this License with respect to the covered work, and you disclaim any intention to limit operation or modification of the work as a means of enforcing, against the work's users, your or third parties' legal rights to forbid circumvention of technological measures. 4. Conveying Verbatim Copies. You may convey verbatim copies of the Program's source code as you receive it, in any medium, provided that you conspicuously and appropriately publish on each copy an appropriate copyright notice; keep intact all notices stating that this License and any non-permissive terms added in accord with section 7 apply to the code; keep intact all notices of the absence of any warranty; and give all recipients a copy of this License along with the Program. You may charge any price or no price for each copy that you convey, and you may offer support or warranty protection for a fee. 5. Conveying Modified Source Versions. You may convey a work based on the Program, or the modifications to produce it from the Program, in the form of source code under the terms of section 4, provided that you also meet all of these conditions: a) The work must carry prominent notices stating that you modified it, and giving a relevant date. b) The work must carry prominent notices stating that it is released under this License and any conditions added under section 7. This requirement modifies the requirement in section 4 to "keep intact all notices". c) You must license the entire work, as a whole, under this License to anyone who comes into possession of a copy. This License will therefore apply, along with any applicable section 7 additional terms, to the whole of the work, and all its parts, regardless of how they are packaged. This License gives no permission to license the work in any other way, but it does not invalidate such permission if you have separately received it. d) If the work has interactive user interfaces, each must display Appropriate Legal Notices; however, if the Program has interactive interfaces that do not display Appropriate Legal Notices, your work need not make them do so. A compilation of a covered work with other separate and independent works, which are not by their nature extensions of the covered work, and which are not combined with it such as to form a larger program, in or on a volume of a storage or distribution medium, is called an "aggregate" if the compilation and its resulting copyright are not used to limit the access or legal rights of the compilation's users beyond what the individual works permit. Inclusion of a covered work in an aggregate does not cause this License to apply to the other parts of the aggregate. 6. Conveying Non-Source Forms. You may convey a covered work in object code form under the terms of sections 4 and 5, provided that you also convey the machine-readable Corresponding Source under the terms of this License, in one of these ways: a) Convey the object code in, or embodied in, a physical product (including a physical distribution medium), accompanied by the Corresponding Source fixed on a durable physical medium customarily used for software interchange. b) Convey the object code in, or embodied in, a physical product (including a physical distribution medium), accompanied by a written offer, valid for at least three years and valid for as long as you offer spare parts or customer support for that product model, to give anyone who possesses the object code either (1) a copy of the Corresponding Source for all the software in the product that is covered by this License, on a durable physical medium customarily used for software interchange, for a price no more than your reasonable cost of physically performing this conveying of source, or (2) access to copy the Corresponding Source from a network server at no charge. c) Convey individual copies of the object code with a copy of the written offer to provide the Corresponding Source. This alternative is allowed only occasionally and noncommercially, and only if you received the object code with such an offer, in accord with subsection 6b. d) Convey the object code by offering access from a designated place (gratis or for a charge), and offer equivalent access to the Corresponding Source in the same way through the same place at no further charge. You need not require recipients to copy the Corresponding Source along with the object code. If the place to copy the object code is a network server, the Corresponding Source may be on a different server (operated by you or a third party) that supports equivalent copying facilities, provided you maintain clear directions next to the object code saying where to find the Corresponding Source. Regardless of what server hosts the Corresponding Source, you remain obligated to ensure that it is available for as long as needed to satisfy these requirements. e) Convey the object code using peer-to-peer transmission, provided you inform other peers where the object code and Corresponding Source of the work are being offered to the general public at no charge under subsection 6d. A separable portion of the object code, whose source code is excluded from the Corresponding Source as a System Library, need not be included in conveying the object code work. A "User Product" is either (1) a "consumer product", which means any tangible personal property which is normally used for personal, family, or household purposes, or (2) anything designed or sold for incorporation into a dwelling. In determining whether a product is a consumer product, doubtful cases shall be resolved in favor of coverage. For a particular product received by a particular user, "normally used" refers to a typical or common use of that class of product, regardless of the status of the particular user or of the way in which the particular user actually uses, or expects or is expected to use, the product. A product is a consumer product regardless of whether the product has substantial commercial, industrial or non-consumer uses, unless such uses represent the only significant mode of use of the product. "Installation Information" for a User Product means any methods, procedures, authorization keys, or other information required to install and execute modified versions of a covered work in that User Product from a modified version of its Corresponding Source. The information must suffice to ensure that the continued functioning of the modified object code is in no case prevented or interfered with solely because modification has been made. If you convey an object code work under this section in, or with, or specifically for use in, a User Product, and the conveying occurs as part of a transaction in which the right of possession and use of the User Product is transferred to the recipient in perpetuity or for a fixed term (regardless of how the transaction is characterized), the Corresponding Source conveyed under this section must be accompanied by the Installation Information. But this requirement does not apply if neither you nor any third party retains the ability to install modified object code on the User Product (for example, the work has been installed in ROM). The requirement to provide Installation Information does not include a requirement to continue to provide support service, warranty, or updates for a work that has been modified or installed by the recipient, or for the User Product in which it has been modified or installed. Access to a network may be denied when the modification itself materially and adversely affects the operation of the network or violates the rules and protocols for communication across the network. Corresponding Source conveyed, and Installation Information provided, in accord with this section must be in a format that is publicly documented (and with an implementation available to the public in source code form), and must require no special password or key for unpacking, reading or copying. 7. Additional Terms. "Additional permissions" are terms that supplement the terms of this License by making exceptions from one or more of its conditions. Additional permissions that are applicable to the entire Program shall be treated as though they were included in this License, to the extent that they are valid under applicable law. If additional permissions apply only to part of the Program, that part may be used separately under those permissions, but the entire Program remains governed by this License without regard to the additional permissions. When you convey a copy of a covered work, you may at your option remove any additional permissions from that copy, or from any part of it. (Additional permissions may be written to require their own removal in certain cases when you modify the work.) You may place additional permissions on material, added by you to a covered work, for which you have or can give appropriate copyright permission. Notwithstanding any other provision of this License, for material you add to a covered work, you may (if authorized by the copyright holders of that material) supplement the terms of this License with terms: a) Disclaiming warranty or limiting liability differently from the terms of sections 15 and 16 of this License; or b) Requiring preservation of specified reasonable legal notices or author attributions in that material or in the Appropriate Legal Notices displayed by works containing it; or c) Prohibiting misrepresentation of the origin of that material, or requiring that modified versions of such material be marked in reasonable ways as different from the original version; or d) Limiting the use for publicity purposes of names of licensors or authors of the material; or e) Declining to grant rights under trademark law for use of some trade names, trademarks, or service marks; or f) Requiring indemnification of licensors and authors of that material by anyone who conveys the material (or modified versions of it) with contractual assumptions of liability to the recipient, for any liability that these contractual assumptions directly impose on those licensors and authors. All other non-permissive additional terms are considered "further restrictions" within the meaning of section 10. If the Program as you received it, or any part of it, contains a notice stating that it is governed by this License along with a term that is a further restriction, you may remove that term. If a license document contains a further restriction but permits relicensing or conveying under this License, you may add to a covered work material governed by the terms of that license document, provided that the further restriction does not survive such relicensing or conveying. If you add terms to a covered work in accord with this section, you must place, in the relevant source files, a statement of the additional terms that apply to those files, or a notice indicating where to find the applicable terms. Additional terms, permissive or non-permissive, may be stated in the form of a separately written license, or stated as exceptions; the above requirements apply either way. 8. Termination. You may not propagate or modify a covered work except as expressly provided under this License. Any attempt otherwise to propagate or modify it is void, and will automatically terminate your rights under this License (including any patent licenses granted under the third paragraph of section 11). However, if you cease all violation of this License, then your license from a particular copyright holder is reinstated (a) provisionally, unless and until the copyright holder explicitly and finally terminates your license, and (b) permanently, if the copyright holder fails to notify you of the violation by some reasonable means prior to 60 days after the cessation. Moreover, your license from a particular copyright holder is reinstated permanently if the copyright holder notifies you of the violation by some reasonable means, this is the first time you have received notice of violation of this License (for any work) from that copyright holder, and you cure the violation prior to 30 days after your receipt of the notice. Termination of your rights under this section does not terminate the licenses of parties who have received copies or rights from you under this License. If your rights have been terminated and not permanently reinstated, you do not qualify to receive new licenses for the same material under section 10. 9. Acceptance Not Required for Having Copies. You are not required to accept this License in order to receive or run a copy of the Program. Ancillary propagation of a covered work occurring solely as a consequence of using peer-to-peer transmission to receive a copy likewise does not require acceptance. However, nothing other than this License grants you permission to propagate or modify any covered work. These actions infringe copyright if you do not accept this License. Therefore, by modifying or propagating a covered work, you indicate your acceptance of this License to do so. 10. Automatic Licensing of Downstream Recipients. Each time you convey a covered work, the recipient automatically receives a license from the original licensors, to run, modify and propagate that work, subject to this License. You are not responsible for enforcing compliance by third parties with this License. An "entity transaction" is a transaction transferring control of an organization, or substantially all assets of one, or subdividing an organization, or merging organizations. If propagation of a covered work results from an entity transaction, each party to that transaction who receives a copy of the work also receives whatever licenses to the work the party's predecessor in interest had or could give under the previous paragraph, plus a right to possession of the Corresponding Source of the work from the predecessor in interest, if the predecessor has it or can get it with reasonable efforts. You may not impose any further restrictions on the exercise of the rights granted or affirmed under this License. For example, you may not impose a license fee, royalty, or other charge for exercise of rights granted under this License, and you may not initiate litigation (including a cross-claim or counterclaim in a lawsuit) alleging that any patent claim is infringed by making, using, selling, offering for sale, or importing the Program or any portion of it. 11. Patents. A "contributor" is a copyright holder who authorizes use under this License of the Program or a work on which the Program is based. The work thus licensed is called the contributor's "contributor version". A contributor's "essential patent claims" are all patent claims owned or controlled by the contributor, whether already acquired or hereafter acquired, that would be infringed by some manner, permitted by this License, of making, using, or selling its contributor version, but do not include claims that would be infringed only as a consequence of further modification of the contributor version. For purposes of this definition, "control" includes the right to grant patent sublicenses in a manner consistent with the requirements of this License. Each contributor grants you a non-exclusive, worldwide, royalty-free patent license under the contributor's essential patent claims, to make, use, sell, offer for sale, import and otherwise run, modify and propagate the contents of its contributor version. In the following three paragraphs, a "patent license" is any express agreement or commitment, however denominated, not to enforce a patent (such as an express permission to practice a patent or covenant not to sue for patent infringement). To "grant" such a patent license to a party means to make such an agreement or commitment not to enforce a patent against the party. If you convey a covered work, knowingly relying on a patent license, and the Corresponding Source of the work is not available for anyone to copy, free of charge and under the terms of this License, through a publicly available network server or other readily accessible means, then you must either (1) cause the Corresponding Source to be so available, or (2) arrange to deprive yourself of the benefit of the patent license for this particular work, or (3) arrange, in a manner consistent with the requirements of this License, to extend the patent license to downstream recipients. "Knowingly relying" means you have actual knowledge that, but for the patent license, your conveying the covered work in a country, or your recipient's use of the covered work in a country, would infringe one or more identifiable patents in that country that you have reason to believe are valid. If, pursuant to or in connection with a single transaction or arrangement, you convey, or propagate by procuring conveyance of, a covered work, and grant a patent license to some of the parties receiving the covered work authorizing them to use, propagate, modify or convey a specific copy of the covered work, then the patent license you grant is automatically extended to all recipients of the covered work and works based on it. A patent license is "discriminatory" if it does not include within the scope of its coverage, prohibits the exercise of, or is conditioned on the non-exercise of one or more of the rights that are specifically granted under this License. You may not convey a covered work if you are a party to an arrangement with a third party that is in the business of distributing software, under which you make payment to the third party based on the extent of your activity of conveying the work, and under which the third party grants, to any of the parties who would receive the covered work from you, a discriminatory patent license (a) in connection with copies of the covered work conveyed by you (or copies made from those copies), or (b) primarily for and in connection with specific products or compilations that contain the covered work, unless you entered into that arrangement, or that patent license was granted, prior to 28 March 2007. Nothing in this License shall be construed as excluding or limiting any implied license or other defenses to infringement that may otherwise be available to you under applicable patent law. 12. No Surrender of Others' Freedom. If conditions are imposed on you (whether by court order, agreement or otherwise) that contradict the conditions of this License, they do not excuse you from the conditions of this License. If you cannot convey a covered work so as to satisfy simultaneously your obligations under this License and any other pertinent obligations, then as a consequence you may not convey it at all. For example, if you agree to terms that obligate you to collect a royalty for further conveying from those to whom you convey the Program, the only way you could satisfy both those terms and this License would be to refrain entirely from conveying the Program. 13. Use with the GNU Affero General Public License. Notwithstanding any other provision of this License, you have permission to link or combine any covered work with a work licensed under version 3 of the GNU Affero General Public License into a single combined work, and to convey the resulting work. The terms of this License will continue to apply to the part which is the covered work, but the special requirements of the GNU Affero General Public License, section 13, concerning interaction through a network will apply to the combination as such. 14. Revised Versions of this License. The Free Software Foundation may publish revised and/or new versions of the GNU General Public License from time to time. Such new versions will be similar in spirit to the present version, but may differ in detail to address new problems or concerns. Each version is given a distinguishing version number. If the Program specifies that a certain numbered version of the GNU General Public License "or any later version" applies to it, you have the option of following the terms and conditions either of that numbered version or of any later version published by the Free Software Foundation. If the Program does not specify a version number of the GNU General Public License, you may choose any version ever published by the Free Software Foundation. If the Program specifies that a proxy can decide which future versions of the GNU General Public License can be used, that proxy's public statement of acceptance of a version permanently authorizes you to choose that version for the Program. Later license versions may give you additional or different permissions. However, no additional obligations are imposed on any author or copyright holder as a result of your choosing to follow a later version. 15. Disclaimer of Warranty. THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, REPAIR OR CORRECTION. 16. Limitation of Liability. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGES. 17. Interpretation of Sections 15 and 16. If the disclaimer of warranty and limitation of liability provided above cannot be given local legal effect according to their terms, reviewing courts shall apply local law that most closely approximates an absolute waiver of all civil liability in connection with the Program, unless a warranty or assumption of liability accompanies a copy of the Program in return for a fee. END OF TERMS AND CONDITIONS How to Apply These Terms to Your New Programs If you develop a new program, and you want it to be of the greatest possible use to the public, the best way to achieve this is to make it free software which everyone can redistribute and change under these terms. To do so, attach the following notices to the program. It is safest to attach them to the start of each source file to most effectively state the exclusion of warranty; and each file should have at least the "copyright" line and a pointer to where the full notice is found. {one line to give the program's name and a brief idea of what it does.} Copyright (C) {year} {name of author} This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. You should have received a copy of the GNU General Public License along with this program. If not, see . Also add information on how to contact you by electronic and paper mail. If the program does terminal interaction, make it output a short notice like this when it starts in an interactive mode: {project} Copyright (C) {year} {fullname} This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'. This is free software, and you are welcome to redistribute it under certain conditions; type `show c' for details. The hypothetical commands `show w' and `show c' should show the appropriate parts of the General Public License. Of course, your program's commands might be different; for a GUI interface, you would use an "about box". You should also get your employer (if you work as a programmer) or school, if any, to sign a "copyright disclaimer" for the program, if necessary. For more information on this, and how to apply and follow the GNU GPL, see . The GNU General Public License does not permit incorporating your program into proprietary programs. If your program is a subroutine library, you may consider it more useful to permit linking proprietary applications with the library. If this is what you want to do, use the GNU Lesser General Public License instead of this License. But first, please read . ================================================ FILE: MANIFEST ================================================ bin/mashtree bin/mashtree_bootstrap.pl bin/mashtree_cluster.pl bin/mashtree_init.pl bin/mashtree_jackknife.pl bin/min_abundance_finder.pl CHANGES CONTRIBUTING.md docs/ALGORITHM.md docs/INSTALL.md docs/TIPS.md docs/TROUBLESHOOTING.md lib/Mashtree.pm lib/Mashtree/Db.pm lib/Mashtree/Mash.pm plugins/mashtree_optimize.pl plugins/README.md README.md t/file-of-files.txt t/filetypes/CFSAN000189.gbk.gz t/filetypes/CFSAN000189.ref.fasta.gz t/filetypes/CFSAN000191.ref.fasta.gz t/filetypes/CFSAN000211.gbk.gz t/filetypes/CFSAN000961.gbk.gz t/filetypes/CFSAN000968.ref.fasta.gz t/filetypes/CFSAN001112.ref.fasta.gz t/filetypes/CFSAN001115.ref.fasta.bz2 t/filetypes/CFSAN001140_1.fastq.gz t/lambda/sample1.fastq.gz t/lambda/sample2.fastq.gz t/lambda/sample3.fastq.gz t/lambda/sample4.fastq.gz t/00_env.t t/01_filetypes.t t/02_lambda.t t/03_subsample.t t/04_fileoffiles.t t/06_jackknifingHashes.t t/08_bootstrap.t t/11_tsv-db.t MANIFEST Makefile.PL ================================================ FILE: MANIFEST.SKIP ================================================ .travis.yml* .git/* bin/mashtree_wrapper_deprecated.pl LICENSE Makefile.PL man/* misc/* t/00_env.t t/01_filetypes.t t/02_lambda.t t/03_subsample.t t/04_fileoffiles.t t/06_jackknifingHashes.t t/10_sqlite.t t/11_tsv-db.t t/lambda/mashtree.dnd dist/* Makefile.old MANIFEST.* ./*.msh lib/perl5 ================================================ FILE: Makefile.PL ================================================ #!/usr/bin/env perl use strict; use warnings; use 5.010; use ExtUtils::MakeMaker; use FindBin qw/$RealBin/; use lib "$RealBin/lib"; use lib "$RealBin/lib/perl5"; # Some help from # http://blogs.perl.org/users/michal_wojciechowski/2011/11/github-friendly-readme-files-with-extutils-makemaker-and-module-build.html my $preop = 'true;'; #'sed -i.bak "s/.5f/.10f/g" bin/quicktree-2.3/src/tree.c bin/quicktree-2.3/src/distancemat.c;' . #'make -C bin/quicktree-2.3;' . #'perldoc -uT $(VERSION_FROM) | tee $(DISTVNAME)/README.pod > README.pod;' . #'pod2text README.pod | tee $(DISTVNAME)/README > README'; # Execute something before `make` my $preBuild= 'true'; #'rm -f bin/ROSS-v0.3.tar.gz;' . # 'if [ ! -e ./bin/ROSS-0.3/LICENSE ]; then ' . # 'wget https://github.com/lskatz/ROSS/archive/v0.3.tar.gz -O bin/ROSS-v0.3.tar.gz;'. # 'tar -zxvf ./bin/ROSS-v0.3.tar.gz --directory ./bin;'. # 'fi;' . # ''; WriteMakefile1( NAME => 'Mashtree', VERSION_FROM => 'lib/Mashtree.pm', #ABSTRACT_FROM => 'lib/Mashtree.pm', #ABSTRACT_FROM => 'README.md', AUTHOR => q{Lee S. Katz }, EXE_FILES => [ "bin/mashtree", "bin/mashtree_jackknife.pl", "bin/min_abundance_finder.pl", "bin/mashtree_bootstrap.pl", ], BEFORE_BUILD=> { exec => $preBuild, }, PREREQ_PM => { # Core modules 'File::Basename' => 0, 'Data::Dumper' => 0, 'List::Util' => 0, 'List::MoreUtils'=> 0, 'Exporter' => 0, # Threads modules 'threads' => 0, 'threads::shared'=> 0, 'Thread::Queue' => 0, # Not core (I think?) 'File::Which' => 0, 'Bio::SeqIO' => 0, 'Bio::TreeIO' => 0, 'Bio::Matrix::IO'=> 0, 'Bio::Tree::Statistics'=> 0, 'Bio::Tree::DistanceFactory'=> 0, 'Bio::Sketch::Mash' => 0, # ensures that Mash gets loaded 'Bio::Kmer' => 0.26, #'Graph::Dijkstra'=> 0, #'Readonly' => 0, 'JSON' => '2.90', }, BUILD_REQUIRES => { 'Test::More' => 0.47, }, MIN_PERL_VERSION => '5.10.0', LICENSE => "gpl_3", dist=>{ COMPRESS => "gzip -9f", SUFFIX => "gz", PREOP => $preop, }, META_MERGE => { 'meta-spec' => { version => 2 }, resources => { repository => { type => 'git', url => 'git://github.com/lskatz/mashtree.git', web => 'https://github.com/lskatz/mashtree', }, }, }, ); sub WriteMakefile1 { #Compatibility code for old versions of EU::MM. Written by Alexandr Ciornii, version 2. Added by eumm-upgrade. my %params=@_; my $eumm_version=$ExtUtils::MakeMaker::VERSION; $eumm_version=eval $eumm_version; die "EXTRA_META is deprecated" if exists $params{EXTRA_META}; die "License not specified" if not exists $params{LICENSE}; if ($params{AUTHOR} and ref($params{AUTHOR}) eq 'ARRAY' and $eumm_version < 6.5705) { $params{META_ADD}->{author}=$params{AUTHOR}; $params{AUTHOR}=join(', ',@{$params{AUTHOR}}); } if ($params{TEST_REQUIRES} and $eumm_version < 6.64) { $params{BUILD_REQUIRES}={ %{$params{BUILD_REQUIRES} || {}} , %{$params{TEST_REQUIRES}} }; delete $params{TEST_REQUIRES}; } if ($params{BUILD_REQUIRES} and $eumm_version < 6.5503) { #EUMM 6.5502 has problems with BUILD_REQUIRES $params{PREREQ_PM}={ %{$params{PREREQ_PM} || {}} , %{$params{BUILD_REQUIRES}} }; delete $params{BUILD_REQUIRES}; } delete $params{CONFIGURE_REQUIRES} if $eumm_version < 6.52; delete $params{MIN_PERL_VERSION} if $eumm_version < 6.48; delete $params{META_MERGE} if $eumm_version < 6.46; delete $params{META_ADD} if $eumm_version < 6.46; delete $params{LICENSE} if $eumm_version < 6.31; # LK if(defined($params{BEFORE_BUILD})){ if(defined($params{BEFORE_BUILD}{exec})){ system($params{BEFORE_BUILD}{exec}); die if $?; } delete($params{BEFORE_BUILD}); } WriteMakefile(%params); } ================================================ FILE: README.md ================================================ # mashtree [![DOI](https://joss.theoj.org/papers/10.21105/joss.01762/status.svg)](https://doi.org/10.21105/joss.01762) Create a tree using Mash distances. For simple usage, see `mashtree --help`. This is an example command: mashtree *.fastq.gz > tree.dnd For confidence values, run either with `--help`: `mashtree_bootstrap.pl` or `mashtree_jackknife.pl`. ## Two modes: fast or accurate **Input files**: fastq files are interpreted as raw read files. Fasta, GenBank, and EMBL files are interpreted as genome assemblies. Compressed files are also accepted of any of the above file types. You can compress with gz, bz2, or zip. **Output files**: Newick (.dnd). If `--outmatrix` is supplied, then a distance matrix too. See the documentation on [the algorithms](docs/ALGORITHM.md) for more information. ### Faster mashtree --numcpus 12 *.fastq.gz [*.fasta] > mashtree.dnd ### More accurate You can get a more accurate tree with the minimum abundance finder. Simply give `--mindepth 0`. This step helps ignore very unique kmers that are more likely read errors. mashtree --mindepth 0 --numcpus 12 *.fastq.gz [*.fasta] > mashtree.dnd ### Adding confidence values Mashtree can add confidence values using jack knifing. For each jack knife tree, 50% of hashes are used. Confidence values are calculated from the jack knife trees using BioPerl. When using this method, you can pass flags to `mashtree` using the double-dash like in the example below. Added in version 0.40. mashtree_jackknife.pl --reps 100 --numcpus 12 *.fastq.gz -- --min-depth 0 > mashtree.jackknife.dnd mashtree_jackknife.pl --help # additional usage help Bootsrapping was added in version 0.55. This runs mashtree itself multiple times, each with a random seed. mashtree_bootstrap.pl --reps 100 --numcpus 12 *.fastq.gz -- --min-depth 0 > mashtree.bootstrap.dnd ## Usage Usage: mashtree [options] *.fastq *.fasta *.gbk *.msh > tree.dnd NOTE: fastq files are read as raw reads; fasta, gbk, and embl files are read as assemblies; Input files can be gzipped. --tempdir '' If specified, this directory will not be removed at the end of the script and can be used to cache results for future analyses. If not specified, a dir will be made for you and then deleted at the end of this script. --numcpus 1 This script uses Perl threads. --outmatrix '' If specified, will write a distance matrix in tab-delimited format --file-of-files If specified, mashtree will try to read filenames from each input file. The file of files format is one filename per line. This file of files cannot be compressed. --outtree If specified, the tree will be written to this file and not to stdout. Log messages will still go to stderr. --version Display the version and exit TREE OPTIONS --truncLength 250 How many characters to keep in a filename --sort-order ABC For neighbor-joining, the sort order can make a difference. Options include: ABC (alphabetical), random, input-order MASH SKETCH OPTIONS --genomesize 5000000 --mindepth 5 If mindepth is zero, then it will be chosen in a smart but slower method, to discard lower-abundance kmers. --kmerlength 21 --sketch-size 10000 ## Installation **Please see [INSTALL.md](docs/INSTALL.md)** ## Further documentation For perl library help, run `perldoc` on a `.pm` file, e.g., `perldoc lib/Mashtree/Db.pm`. For executable help run `--help`, e.g., `mashtree_bootstrap.pl --help`. For more information and help please see the [docs folder](docs/) For more information on plugins, see the [plugins folder](plugins). (in development) For more information on contributions, please see [CONTRIBUTING.md](CONTRIBUTING.md). ## References * Mash: http://mash.readthedocs.io * BioPerl: http://bioperl.org ## Citation ### JOSS Katz, L. S., Griswold, T., Morrison, S., Caravas, J., Zhang, S., den Bakker, H.C., Deng, X., and Carleton, H. A., (2019). Mashtree: a rapid comparison of whole genome sequence files. Journal of Open Source Software, 4(44), 1762, https://doi.org/10.21105/joss.01762 ### Poster Katz, L. S., Griswold, T., & Carleton, H. A. (2017, October 8-11). [_Generating WGS Trees with Mashtree_](misc/mashtree%20ASM%20NGS.pptx). Poster presented at the American Society for Microbiology Conference on Rapid Applied Microbial Next-Generation Sequencing and Bioinformatic Pipelines, Washington, DC. Poster number 27. ================================================ FILE: bin/mashtree ================================================ #!/usr/bin/env perl # Author: Lee Katz # Uses Mash and BioPerl to create a NJ tree based on distances. # Run this script with -h for help and usage. use strict; use warnings; use Data::Dumper; use Getopt::Long; use File::Temp qw/tempdir tempfile/; use File::Basename qw/basename dirname fileparse/; use File::Copy qw/copy/; use POSIX qw/floor/; use List::Util qw/min max/; use Scalar::Util qw/looks_like_number/; use threads; use threads::shared; use FindBin; use lib "$FindBin::RealBin/../lib"; use lib "$FindBin::RealBin/../lib/perl5"; use File::Which qw/which/; use Mashtree qw/logmsg @fastqExt @fastaExt @mshExt @richseqExt _truncateFilename createTreeFromPhylip $MASHTREE_VERSION/; use Mashtree::Db; use Bio::Tree::DistanceFactory; use Bio::Matrix::IO; use Bio::Tree::Statistics; use Bio::SeqIO; #my %delta :shared=(); # change in amplitude for peak detection, for each fastq my $scriptDir=dirname $0; my $dbhLock :shared; # Use this as a lock so that only one thread writes to the db at a time my $abundanceFinderLock :shared; # a lock to limit min abundance finder instances local $0=basename $0; exit main(); sub main{ my $settings={}; GetOptions($settings,qw(help sigfigs|significant-digits=i file-of-files outtree=s outmatrix=s tempdir=s numcpus=i genomesize=i mindepth|min-depth=i truncLength=i kmerlength=i sort-order=s sketch-size=i citation version save-sketches:s seed=i)) or die $!; $$settings{numcpus}||=1; $$settings{truncLength}||=250; # how long a genome name is $$settings{sigfigs} ||= 10; $$settings{tempdir}||=tempdir("MASHTREE.XXXXXX",CLEANUP=>1,TMPDIR=>1); $$settings{'sort-order'}||="ABC"; # Mash-specific options $$settings{genomesize}||=5000000; $$settings{mindepth}//=5; $$settings{kmerlength}||=21; $$settings{'sketch-size'}||=10000; $$settings{seed}||=42; # Make some settings lowercase for(qw(sort-order)){ $$settings{$_}=lc($$settings{$_}); } if($$settings{help}){ print usage(); return 0; } if($$settings{version}){ print "Mashtree $MASHTREE_VERSION\n"; return 0; } if($$settings{citation}){ print "Katz, L. S., Griswold, T., Morrison, S., Caravas, J., Zhang, S., den Bakker, H.C., Deng, X., and Carleton, H. A., (2019). Mashtree: a rapid comparison of whole genome sequence files. Journal of Open Source Software, 4(44), 1762, https://doi.org/10.21105/joss.01762\n"; return 0; } # "reads" are either fasta assemblies or fastq reads my @reads=@ARGV; die "need more arguments\n".usage() if(@reads < 1); # Check for prereq executables. for my $exe(qw(mash quicktree)){ if(!which($exe)){ die "ERROR: could not find $exe in your PATH"; } } # Check mash version my $mashVersion = `mash --version`; die "ERROR running mash: $!" if $?; chomp($mashVersion); $mashVersion=~s/\..*//; die "ERROR: need mash version > 2" if($mashVersion < 2); logmsg "Found mash version $mashVersion - ".which('mash'); # Distributed cpus if we have few genomes but high numcpus $$settings{cpus_per_mash}=floor($$settings{numcpus}/@reads); $$settings{cpus_per_mash}=1 if($$settings{cpus_per_mash} < 1); $$settings{numthreads}=min(scalar(@reads), $$settings{numcpus}); #die Dumper [$$settings{cpus_per_mash},$$settings{numthreads},\@reads]; #$$settings{cpus_per_mash}=1; #$$settings{numthreads}=$$settings{numcpus}; logmsg "Temporary directory will be $$settings{tempdir}"; logmsg "$0 on ".scalar(@reads)." files"; my %seen; my @tmp; for my $reads(@ARGV){ if(!-e $reads){ die "ERROR: I could not find reads path at $reads"; } my $basename=basename($reads); if($seen{$basename}++){ logmsg "Skipping $reads: already seen $basename"; next; } # If the user wants to specify files of files then check # on whether it is a file of files. Otherwise do not check # because of disk IO speeds. if($$settings{'file-of-files'}){ # If we have file-of-files in play, don't try to optimize # too early. $$settings{cpus_per_mash}=1; $$settings{numthreads}=$$settings{numcpus}; open(my $possibleFileOfFiles, "<", $reads) or die "ERROR: could not read $reads: $!"; my $firstfile = <$possibleFileOfFiles>; $firstfile=~s/^\s+|\s+$//g; #whitespace trim chomp($firstfile); if(-e $firstfile){ push(@tmp, $firstfile); while(<$possibleFileOfFiles>){ s/^\s+|\s+$//g; #whitespace trim push(@tmp, $_); } } else { logmsg "WARNING: user specified --file-of-files but $reads does not seem to be a file of files (ignore this warning if this was not supposed to be a file of files"; push(@tmp, $reads); } close $possibleFileOfFiles; } else { push(@tmp,$reads); } } @reads=@tmp; my $sketches=sketchAll(\@reads,"$$settings{tempdir}",$settings); my $phylip = mashDistance($sketches,\@reads,$$settings{tempdir},$settings); my $treeObj = createTreeFromPhylip($phylip,$$settings{tempdir},$settings); # Write the tree if($$settings{outtree}){ open(my $treeFh, ">", $$settings{outtree}) or die "ERROR writing tree to $$settings{outtree}: $!"; print $treeFh $treeObj->as_text('newick'); close $treeFh } else { print $treeObj->as_text('newick'); print "\n"; } return 0; } # Run mash sketch on everything, multithreaded. sub sketchAll{ my($reads,$sketchDir,$settings)=@_; mkdir $sketchDir if(!-d $sketchDir); # Make an array of genomes that would distribute well # across threads. For example, don't put all raw-read # genomes into a single thread and all the assemblies # into another. my %filesize=(); for(@$reads){ $filesize{$_} = -s $_; } my @sortedReads=sort {$filesize{$a} <=> $filesize{$b}} @$reads; my @threadArr=(); for(my $i=0; $i<@sortedReads; $i++){ # Since each genome is sorted smallest to leargest, # they can be sent round-robin to each thread to # ensure balance. my $threadIndex = $i % $$settings{numcpus}; push(@{ $threadArr[$threadIndex] }, $sortedReads[$i]); } # Initiate the threads my @thr; for(0..$$settings{numthreads}-1){ # If there are more threads than samples, then we need to set the number in this thread to empty array $threadArr[$_] //= []; $thr[$_]=threads->new(\&mashSketch,$sketchDir,$threadArr[$_],$settings); } my @mshList; for(@thr){ my $mashfiles=$_->join; for my $file(@$mashfiles){ push(@mshList,$file); } } return \@mshList; } # Individual mash sketch worker sub mashSketch{ my($sketchDir,$genomeArr,$settings)=@_; # If any file needs to be converted, it will end up in # this directory. my $tempdir=tempdir("$$settings{tempdir}/convertSeq.XXXXXX", CLEANUP=>1); my $numFiles = scalar(@$genomeArr); logmsg "This thread will work on $numFiles sketches"; my @msh; my $fileCounter=0; # $fastq is a misnomer: it could be any kind of accepted sequence file for my $fastq(@$genomeArr){ logmsg "Working on file ".++$fileCounter." out of $numFiles"; my($fileName,$filePath,$fileExt)=fileparse($fastq,@fastqExt,@fastaExt,@richseqExt,@mshExt); # Unzip the file. This temporary file will # only exist if the correct extensions are detected. my $unzipped="$tempdir/".basename($fastq); $unzipped=~s/\.(gz|bz2?|zip)$//i; my $was_unzipped=0; # Don't bother unzipping if it's a fastq or fasta file b/c Mash can read those if(!grep {$_ eq $fileExt} (@fastqExt,@fastaExt)){ if($fastq=~/\.gz$/i){ # Don't actually decompress fastq.gz files because they are # read natively by mash. if($fastq =~ /\.fastq.gz$|\.fq.gz$/){ $was_unzipped=0; } else { system("gzip -cd $fastq > $unzipped"); die "ERROR with gzip -cd $fastq" if $?; $was_unzipped=1; } } elsif($fastq=~/\.bz2?$/i){ system("bzip2 -cd $fastq > $unzipped"); die "ERROR with bzip2 -cd $fastq" if $?; $was_unzipped=1; } elsif($fastq=~/\.zip$/i){ system("unzip -p $fastq > $unzipped"); die "ERROR with unzip -p $fastq" if $?; $was_unzipped=1; } } # If the file was uncompressed, parse the filename again. if($was_unzipped){ $fastq=$unzipped; ($fileName,$filePath,$fileExt)=fileparse($fastq,@fastqExt,@fastaExt,@richseqExt,@mshExt); } # If we see a richseq (e.g., gbk or embl), then convert it to fasta # TODO If Mash itself accepts richseq, then consider # doing away with this section. if(grep {$_ eq $fileExt} @richseqExt){ # Make a temporary fasta file, but it needs to have a # consistent name in case Mashtree is being run with # the wrapper for bootstrap values. # I can't exactly make a consistent filename in case # different mashtree invocations collide, so # I need to make a new temporary directory with a # consistent filename. my $tmpfasta="$tempdir/$fileName$fileExt.fasta"; my $in=Bio::SeqIO->new(-file=>$fastq); my $out=Bio::SeqIO->new(-file=>">$tmpfasta", -format=>"fasta"); while(my $seq=$in->next_seq){ $out->write_seq($seq); } logmsg "Wrote $tmpfasta"; # Update our filename for downstream $fastq=$tmpfasta; ($fileName,$filePath,$fileExt)=fileparse($tmpfasta, @fastaExt); } my $outPrefix="$sketchDir/".basename($fastq, @mshExt); if(-e "$outPrefix.msh"){ logmsg "already mashed: $fastq. Skipping."; push(@msh,"$outPrefix.msh"); next; } # Do different things depending on fastq vs fasta my $sketchXopts=""; if(grep {$_ eq $fileExt} @fastqExt){ my $minDepth=determineMinimumDepth($fastq,$$settings{mindepth},$$settings{kmerlength},$settings); $sketchXopts.="-m $minDepth -g $$settings{genomesize} "; } elsif(grep {$_ eq $fileExt} @fastaExt) { $sketchXopts.=" "; } elsif(grep {$_ eq $fileExt} @mshExt){ $sketchXopts.=" "; } else { logmsg "WARNING: I could not understand what kind of file this is by its extension ($fileExt): $fastq"; } # See if the user already mashed this file locally if(-e "$fastq.msh"){ logmsg "Found locally mashed file $fastq.msh. I will use it."; copy("$fastq.msh","$outPrefix.msh"); } if(grep {$_ eq $fileExt} @mshExt){ logmsg "Input file is a sketch file itself and will be used as such: $fastq"; copy($fastq, "$outPrefix.msh"); } if(-e "$outPrefix.msh"){ logmsg "WARNING: ".basename($fastq)." was already mashed."; } elsif(-s $fastq < 1){ logmsg "WARNING: $fastq is a zero byte file. Skipping."; next; } else { my $sketchCommand="mash sketch -S $$settings{seed} -k $$settings{kmerlength} -s $$settings{'sketch-size'} $sketchXopts -o \Q$outPrefix\E \Q$fastq\E 2>&1"; my $stdout = `$sketchCommand`; if ($?){ logmsg "ERROR running $sketchCommand!\n $stdout"; die; } } push(@msh,"$outPrefix.msh"); # Save sketches into a local directory if requested if($$settings{'save-sketches'}){ my $dir=$$settings{'save-sketches'}; my $target = "$fastq.msh"; if(defined($dir)){ mkdir($dir) if(!-e $dir); $target = "$dir/".basename($fastq).".msh"; } #try to hard link because this is basically a read-only file, #but if not, copying is fine. link("$outPrefix.msh", $target) || copy("$outPrefix.msh", $target) || die "ERROR hard-linking or copying $outPrefix.msh to $target: $!"; } } system("rm -rf $tempdir"); return \@msh; } # Parallelized mash distance sub mashDistance{ my($mshList,$reads,$outdir,$settings)=@_; # Make a list of names that will appear in the database # in exactly the right format. my @genomeName; # Make a temporary file with one line per mash file. # Helps with not running into the max number of command line args. my $mshListFilename="$outdir/mshList.txt"; open(my $mshListFh,">",$mshListFilename) or die "ERROR: could not write to $mshListFilename: $!"; for(@$mshList){ print $mshListFh $_."\n"; push(@genomeName,_truncateFilename($_,$settings)); } close $mshListFh; # Instatiate the database and create the table before the threads get to it my $mashtreeDbFilename="$outdir/distances.db.tsv"; my $mashtreeDb=Mashtree::Db->new($mashtreeDbFilename,{significant_figures=>$$settings{sigfigs}}); # Make an array of distance files for each thread. # Because distance files take about the same amount # of time to analyze, there is no need to sort. my @threadArr=(); for(my $i=0; $i<@$mshList; $i++){ my $threadIndex = $i % $$settings{numcpus}; push(@{ $threadArr[$threadIndex] }, $$mshList[$i]); } # Initialize the threads my @thr; for(0..$$settings{numthreads}-1){ $thr[$_]=threads->new(\&mashDist,$outdir,$threadArr[$_],$mshListFilename,$settings); } my $numThreads=@thr; for(my $t=0; $t<@thr; $t++){ my $T=$t+1; my $formattedThreadName = "$T/$numThreads, TID".$thr[$t]->tid; logmsg "Waiting to join thread ($formattedThreadName)"; my $distHash = $thr[$t]->join; # returns ref to scalar b/c the str could be very large logmsg "Databasing distances ($formattedThreadName)"; $mashtreeDb->addDistancesFromHash($distHash); } my $phylip = "$outdir/distances.phylip"; logmsg "Converting to phylip format into $phylip"; open(my $phylipFh, ">", $phylip) or die "ERROR: could not write to $phylip: $!"; print $phylipFh $mashtreeDb->toString(\@genomeName,"phylip"); close $phylipFh; if($$settings{outmatrix}){ logmsg "Writing a distance matrix to $$settings{outmatrix}"; open(my $matrixFh, ">", $$settings{outmatrix}) or die "ERROR: could not write to $$settings{outmatrix}: $!"; print $matrixFh $mashtreeDb->toString(\@genomeName,"matrix"); close $matrixFh; } return $phylip; } # Individual mash distance sub mashDist{ my($outdir,$mshArr,$mshList,$settings)=@_; my $numQueries=0; my %dist; for my $msh(@$mshArr){ #my $outfile="$outdir/".basename($msh).".tsv"; logmsg "Distances for $msh"; my $cmd = "mash dist -t \Q$msh\E -l $mshList"; my @distRes = `$cmd`; die "ERROR with $cmd" if $?; chomp(@distRes); my $query=""; my $queryLine = shift(@distRes); if($queryLine=~/#\s*query\s+(.+)/){ $query = _truncateFilename($1); } else { die "ERROR parsing for the query in $queryLine"; } for(@distRes){ my($hit, $dist)=split /\t/; $hit = _truncateFilename($hit); $dist{$query}{$hit} = $dist; } $numQueries++; } return \%dist; #return \$distStr; # return the reference because the str could be quite large } sub determineMinimumDepth{ my($fastq,$mindepth,$kmerlength,$settings)=@_; my $defaultDepth=5; # if no valley is detected return $mindepth if($mindepth > 0); my $basename=basename($fastq,@fastqExt); # Run the min abundance finder to find the valleys my $minAbundanceTempdir="$$settings{tempdir}/$basename.minAbundance.tmp"; mkdir $minAbundanceTempdir; my $minAbundanceCommand="min_abundance_finder.pl --numcpus $$settings{cpus_per_mash} $fastq --kmer $kmerlength --tempdir $minAbundanceTempdir"; #lock($abundanceFinderLock); logmsg "DEBUG: running single mode for $fastq with $$settings{numcpus} cpus in the single thread"; my $minKmerCount = `$minAbundanceCommand`; # If there is an error, just try running one at a time. # I am not sure why there is a seg fault sometimes when # more than one are running at the same time though. #if($?){ # lock($abundanceFinderLock); # @valleyLines=`$minAbundanceCommand`; #} die "ERROR with min_abundance_finder.pl on $fastq: $!" if($?); chomp($minKmerCount); # Some cleanup of large files # Avoid some weird segfault error associated with the unlink() #unlink $_ for(glob("$minAbundanceTempdir/*")); #rmdir $minAbundanceTempdir; system("rm -rf $minAbundanceTempdir"); # If there is no valley, return a default value if(!$minKmerCount){ return $defaultDepth; } # Set a minimum just in case $minKmerCount=1 if($minKmerCount < 1); logmsg "Setting the min depth as $minKmerCount for $fastq"; return $minKmerCount; } sub usage{ "$0: use distances from Mash (min-hash algorithm) to make a NJ tree Usage: $0 [options] *.fastq *.fasta *.gbk *.msh > tree.dnd NOTE: fastq files are read as raw reads; fasta, gbk, and embl files are read as assemblies; Input files can be gzipped. --tempdir '' If specified, this directory will not be removed at the end of the script and can be used to cache results for future analyses. If not specified, a dir will be made for you and then deleted at the end of this script. --numcpus 1 This script uses Perl threads. --outmatrix '' If specified, will write a distance matrix in tab-delimited format --file-of-files If specified, mashtree will try to read filenames from each input file. The file of files format is one filename per line. This file of files cannot be compressed. --outtree If specified, the tree will be written to this file and not to stdout. Log messages will still go to stderr. --version Display the version and exit --citation Display the preferred citation and exit TREE OPTIONS --truncLength 250 How many characters to keep in a filename --sigfigs 10 How many decimal places to use in mash distances --sort-order ABC For neighbor-joining, the sort order can make a difference. Options include: ABC (alphabetical), random, input-order MASH SKETCH OPTIONS --genomesize 5000000 --mindepth 5 If mindepth is zero, then it will be chosen in a smart but slower method, to discard lower-abundance kmers. --kmerlength 21 --sketch-size 10000 --seed 42 Seed for mash sketch --save-sketches '' If a directory is supplied, then sketches will be saved in it. If no directory is supplied, then sketches will be saved alongside source files. " } ================================================ FILE: bin/mashtree_bootstrap.pl ================================================ #!/usr/bin/env perl # Author: Lee Katz # Uses Mash and BioPerl to create a NJ tree based on distances. # Run this script with -h for help and usage. use strict; use warnings; use Data::Dumper; use Getopt::Long; use File::Temp qw/tempdir tempfile/; use File::Basename qw/basename dirname fileparse/; use File::Copy qw/cp mv/; use List::Util qw/shuffle/; use List::MoreUtils qw/part/; use POSIX qw/floor/; use IO::Handle; # allows me to autoflush file handles #use Fcntl qw/:flock LOCK_EX LOCK_UN/; use threads; use Thread::Queue; use threads::shared; use FindBin; use lib "$FindBin::RealBin/../lib"; use Mashtree qw/logmsg @fastqExt @fastaExt createTreeFromPhylip mashDist/; use Mashtree::Db; use Bio::SeqIO; use Bio::TreeIO; use Bio::Tree::DistanceFactory; use Bio::Tree::Statistics; my $writeStick :shared; my $readStick :shared; # limit disk IO by reading some files one at a time local $0=basename $0; exit main(); sub main{ my $settings={}; my @wrapperOptions=qw(help file-of-files outmatrix=s tempdir=s reps=i numcpus=i); GetOptions($settings,@wrapperOptions) or die $!; $$settings{reps}//=1; $$settings{numcpus}||=1; die usage() if($$settings{help}); die usage() if(@ARGV < 1); $$settings{tempdir}||=tempdir("MASHTREE_BOOTSTRAP.XXXXXX",CLEANUP=>1,TMPDIR=>1); mkdir($$settings{tempdir}) if(!-d $$settings{tempdir}); logmsg "Temporary directory will be $$settings{tempdir}"; if($$settings{reps} < 1){ die "ERROR: no reps were given!"; } if($$settings{reps} < 100){ logmsg "WARNING: You have very few reps planned on this mashtree run. Recommended reps are at least 100."; } ## Catch some options that are not allowed to be passed # Tempdir: All mashtree temporary directories will be under the # wrapper's tempdir. if(grep(/^\-+tempdir$/,@ARGV) || grep(/^\-+t$/,@ARGV)){ die "ERROR: tempdir was specified for mashtree but should be an option for $0"; } # Numcpus: this needs to be specified in the wrapper and will # appropriately be transferred to the mashtree script if(grep(/^\-+numcpus$/,@ARGV) || grep(/^\-+n$/,@ARGV)){ die "ERROR: numcpus was specified for mashtree but should be an option for $0"; } # Outmatrix: the wrapper script needs to control where # the matrix goes because it can only have the outmatrix # for the observed run and not the replicates for speed's # sake. if(grep(/^\-+outmatrix$/,@ARGV) || grep(/^\-+o$/,@ARGV)){ die "ERROR: outmatrix was specified for mashtree but should be an option for $0"; } # --file-of-files: should be something to give to mashtree directly if(grep(/^\-+file-of-files$/,@ARGV) || grep(/^\-+f$/,@ARGV)){ die "ERROR: file-of-files was specified for mashtree but should be an option for $0"; } # Separate flagged options from reads in the mashtree options my @reads = (); my @mashOptions = (); for(my $i=0;$i<@ARGV;$i++){ if(-e $ARGV[$i]){ push(@reads, $ARGV[$i]); } else { push(@mashOptions, $ARGV[$i]); } } if($$settings{'file-of-files'}){ push(@mashOptions, '--file-of-files'); } my $mashOptions=join(" ",@mashOptions); my $reads = join(" ", @reads); # Some filenames we'll expect my $observeddir="$$settings{tempdir}/observed"; my $obsDistances="$observeddir/distances.phylip"; my $observedTree="$$settings{tempdir}/observed.dnd"; my $outmatrix="$$settings{tempdir}/observeddistances.tsv"; my $mshList = "$$settings{tempdir}/mash.list"; my $mergedMash = "$$settings{tempdir}/merged.msh"; # Make the observed directory and run Mash logmsg "Running mashtree on full data (".scalar(@reads)." targets)"; mkdir($observeddir); system("$FindBin::RealBin/mashtree --outmatrix $outmatrix.tmp --tempdir $observeddir --numcpus $$settings{numcpus} $mashOptions $reads > $observedTree.tmp"); die if $?; mv("$observedTree.tmp",$observedTree) or die $?; mv("$outmatrix.tmp",$outmatrix) or die $?; # Round-robin sample the replicate number, so that it's # well-balanced. my @repNumber = (1..$$settings{reps}); my @reps; for(my $i=0;$i<$$settings{reps};$i++){ my $thrIndex = $i % $$settings{numcpus}; push(@{$reps[$thrIndex]}, $i); } # Sample the mash sketches with replacement for rapid bootstrapping logmsg "Running mashtree single-threaded among $$settings{reps} replicates using $$settings{numcpus} total threads."; my @bsThread; for my $i(0..$$settings{numcpus}-1){ my %settingsCopy = %$settings; $bsThread[$i] = threads->new(\&mashtreeRepWorker, $reps[$i], $reads, $mashOptions, \%settingsCopy); } my @bsTree; for my $t(@bsThread){ my $treeArr = $t->join; for my $file(@$treeArr){ my $treein = Bio::TreeIO->new(-file=>$file); while(my $tree=$treein->next_tree){ push(@bsTree, $tree); } } } # Combine trees into a bootstrapped tree and write it # to an output file. Then print it to stdout. logmsg "Adding bootstraps to tree"; my $guideTree=Bio::TreeIO->new(-file=>"$observeddir/tree.dnd")->next_tree; my $bsTree=assess_bootstrap($guideTree,\@bsTree,$guideTree); for my $node($bsTree->get_nodes){ next if($node->is_Leaf); my $id=$node->bootstrap||$node->id||0; $node->id($id); } open(my $treeFh,">","$$settings{tempdir}/bstree.dnd") or die "ERROR: could not write to $$settings{tempdir}/bstree.dnd: $!"; print $treeFh $bsTree->as_text('newick'); print $treeFh "\n"; close $treeFh; system("cat $$settings{tempdir}/bstree.dnd"); die if $?; if($$settings{'outmatrix'}){ cp($outmatrix,$$settings{'outmatrix'}); } return 0; } sub mashtreeRepWorker{ my($reps, $reads, $mashtreeOptions, $settings) = @_; my @tree; my $largeInt = ~0; for my $rep(@$reps){ logmsg "Rep $rep"; my $tempdir = $$settings{tempdir}."/$rep"; my $seed = int(rand(4.29497e+09)); # apparently 4.29497e+09 is the largest int for mash sketch my $repMashtreeOptions = $mashtreeOptions . " --seed $seed"; my $stdout = `$FindBin::RealBin/mashtree --tempdir $tempdir --numcpus 1 $repMashtreeOptions $reads 2>&1`; die "ERROR on rep $rep\n$stdout" if $?; push(@tree, "$$settings{tempdir}/$rep/tree.dnd"); } return \@tree; } # Fixed bootstrapping function from bioperl # https://github.com/bioperl/bioperl-live/pull/304 sub assess_bootstrap{ my ($self,$bs_trees,$guide_tree) = @_; my @consensus; if(!defined($bs_trees) || ref($bs_trees) ne 'ARRAY'){ die "ERROR: second parameter in assess_bootstrap() must be a list"; } my $num_bs_trees = scalar(@$bs_trees); if($num_bs_trees < 1){ die "ERROR: no bootstrap trees were passed to assess_bootstrap()"; } # internal nodes are defined by their children my (%lookup,%internal); my $i = 0; for my $tree ( $guide_tree, @$bs_trees ) { # Do this as a top down approach, can probably be # improved by caching internal node states, but not going # to worry about it right now. my @allnodes = $tree->get_nodes; my @internalnodes = grep { ! $_->is_Leaf } @allnodes; for my $node ( @internalnodes ) { my @tips = sort map { $_->id } grep { $_->is_Leaf() } $node->get_all_Descendents; my $id = "(".join(",", @tips).")"; if( $i == 0 ) { $internal{$id} = $node->internal_id; } else { $lookup{$id}++; } } $i++; } #my @save; # unsure why this variable is needed for my $l ( keys %lookup ) { if( defined $internal{$l} ) {#&& $lookup{$l} > $min_seen ) { my $intnode = $guide_tree->find_node(-internal_id => $internal{$l}); $intnode->bootstrap(sprintf("%d",100 * $lookup{$l} / $num_bs_trees)); } } return $guide_tree; } ####### # Utils ####### sub usage{ my $usage="$0: a wrapper around mashtree for bootstrapping. Usage: $0 [options] [-- mashtree options] *.fastq.gz *.fasta > tree.dnd --outmatrix '' Output file for distance matrix --reps 0 How many bootstrap repetitions to run; If zero, no bootstrapping. --numcpus 1 This will be passed to mashtree and will be used to multithread reps. --tempdir A directory path that will be created to store temporary files --file-of-files If specified, mashtree will try to read filenames from each input file. The file of files format is one filename per line. This file of files cannot be compressed. -- Used to separate options for $0 and mashtree MASHTREE OPTIONS:\n". # Print the mashtree options starting with numcpus, # skipping the tempdir option. `mashtree --help 2>&1 | grep -A 999 "TREE OPTIONS" | grep -v ^Stopped`; return $usage; } ================================================ FILE: bin/mashtree_cluster.pl ================================================ #!/usr/bin/env perl # Author: Lee Katz # Read a mashtree database to cluster genomes use strict; use warnings; use Data::Dumper; use Getopt::Long; use File::Basename qw/basename dirname fileparse/; use File::Copy qw/copy/; use POSIX qw/floor/; use List::Util qw/min max/; use Scalar::Util qw/looks_like_number/; use DBI; use threads; use threads::shared; use FindBin; use lib "$FindBin::RealBin/../lib"; use Mashtree qw/logmsg/; #use Mashtree qw/logmsg @fastqExt @fastaExt @mshExt @richseqExt _truncateFilename createTreeFromPhylip $MASHTREE_VERSION/; #use Mashtree::Db; #use Graph::Dijkstra; local $0=basename $0; exit main(); sub main{ my $settings={}; GetOptions($settings,qw(help threshold|cutoff=f numcpus=i nonzero=f)) or die $!; $$settings{numcpus}||=1; $$settings{nonzero}||=1e-99; $$settings{threshold}||=0.1; my($dbFile) = @ARGV; die usage() if($$settings{help} || !$dbFile); if(!-e $dbFile){ die "ERROR: could not find database file $dbFile"; } my $clusters = makeClusters($dbFile,$settings); for my $clusterArr(values(%$clusters)){ print join("\t",@$clusterArr)."\n"; } return 0; } sub makeClusters{ my($db,$settings)=@_; my %C; #clustering hash, where the key is the seed #my $graph = Graph::Dijkstra->new(edgedefault=>"undirected"); my %graph=();# hash of pairwise nodes my $dbh = DBI->connect("dbi:SQLite:dbname=$db","","",{ RaiseError => 1 }); my $sth = $dbh->prepare(qq(SELECT GENOME1,GENOME2,DISTANCE FROM DISTANCE WHERE GENOME1 != GENOME2 ORDER BY DISTANCE ASC, GENOME1 ASC, GENOME2 ASC )); my $rv = $sth->execute() or die $DBI::errstr; if($rv < 0){ die "ERROR: no distances were found in the database $db"; } my $rowCounter=0; my %genomeIndex; while(my @row=$sth->fetchrow_array()){ my($g1,$g2,$dist)=@row; $genomeIndex{$g1}=1; $genomeIndex{$g2}=1; #$dist=$$settings{nonzero} if(!$dist); my($G1,$G2)=sort{$a cmp $b} ($g1,$g2); $graph{$G1}{$G2}=$dist; $graph{$G2}{$G1}=$dist; $rowCounter++; if($rowCounter % 1000000 == 0){ logmsg "Read the distances for $rowCounter pairs"; } } logmsg "Done reading the distances for $rowCounter pairs"; # TODO? close the database fh logmsg "Calculating clusters"; #my @genome=sort {$a cmp $b} map {$$_{id}} $graph->nodeList; my @genome = sort{$a cmp $b} keys(%genomeIndex); my $numGenomes=@genome; for(my $i=0;$i<$numGenomes;$i++){ # Is this genome $i close to anything? my $query = $genome[$i]; logmsg "Querying with $query"; my $hit = undef; # See if the query is close to any of the current seeds first my @seed = sort{$a cmp $b} keys(%C); my $numSeeds = scalar(@seed); my $possibleHit; my $closestSeedDist = ~0; for(my $j=0; $j<$numSeeds; $j++){ next if(!defined($graph{$seed[$j]}{$query})); my $dist = $graph{$seed[$j]}{$query}; next if($dist > $$settings{threshold}); if($dist < $closestSeedDist){ $possibleHit = $seed[$j]; $closestSeedDist = $dist; } } # If it's close to a seed, then record it. if($closestSeedDist <= $$settings{threshold}){ logmsg " => $possibleHit"; push(@{ $C{$possibleHit} }, $query); } # If it's not close to a seed, start a new seeded cluster. else { logmsg " => new cluster seeded with $query"; $C{$query} = [$query]; } } return \%C; } sub usage{ "Output clusters of genomes, based on a mashtree sqlite database. The stdout will be a tab-delimited list of genomes, one line per cluster. Usage: $0 file.sql > clusters.tsv --threshold 0.1 Maximum any two genomes can be from the seed in a given cluster --nonzero 1e-99 Zero distance is not tolerated in this script. Give a nonzero value in case a zero distance is found. --numcpus 1 Not currently used. " } ================================================ FILE: bin/mashtree_init.pl ================================================ #!/usr/bin/env perl # Author: Lee Katz # Uses Mash to create a database of distances # Run this script with -h for help and usage. use strict; use warnings; use Data::Dumper; use Getopt::Long; use File::Temp qw/tempdir tempfile/; use File::Basename qw/basename dirname fileparse/; use File::Copy qw/copy move/; use POSIX qw/floor/; use List::Util qw/min max/; use Scalar::Util qw/looks_like_number/; use threads; use threads::shared; use FindBin; use lib "$FindBin::RealBin/../lib"; use Mashtree qw/logmsg @fastqExt @fastaExt @mshExt @richseqExt _truncateFilename $MASHTREE_VERSION/; use Mashtree::Db; use Bio::Tree::DistanceFactory; use Bio::Matrix::IO; use Bio::Tree::Statistics; use Bio::SeqIO; my %delta :shared=(); # change in amplitude for peak detection, for each fastq my $scriptDir=dirname $0; my $dbhLock :shared; # Use this as a lock so that only one thread writes to the db at a time my $abundanceFinderLock :shared; # a lock to limit min abundance finder instances local $0=basename $0; exit main(); sub main{ my $settings={}; GetOptions($settings,qw(help outfile|output=s tempdir=s numcpus=i genomesize=i mindepth|min-depth=i truncLength=i kmerlength=i sort-order=s sketch-size=i version)) or die $!; $$settings{numcpus}||=1; $$settings{truncLength}||=250; # how long a genome name is $$settings{tempdir}||=tempdir("MASHTREE.XXXXXX",CLEANUP=>1,TMPDIR=>1); $$settings{'sort-order'}||="ABC"; # Mash-specific options $$settings{genomesize}||=5000000; $$settings{mindepth}//=5; $$settings{kmerlength}||=21; $$settings{'sketch-size'}||=10000; # Make some settings lowercase for(qw(sort-order)){ $$settings{$_}=lc($$settings{$_}); } die usage() if($$settings{help}); if($$settings{version}){ print "Mashtree $MASHTREE_VERSION\n"; return 0; } # "reads" are either fasta assemblies or fastq reads my @reads=@ARGV; die usage() if(@reads < 2); die "ERROR: need --outfile" if(!$$settings{outfile}); # Check for prereq executables. for my $exe(qw(mash)){ system("$exe -h > /dev/null 2>&1"); die "ERROR: could not find $exe in your PATH" if $?; } # Distributed cpus if we have few genomes but high numcpus $$settings{cpus_per_mash}=floor($$settings{numcpus}/@reads); $$settings{cpus_per_mash}=1 if($$settings{cpus_per_mash} < 1); $$settings{numthreads}=min(scalar(@reads), $$settings{numcpus}); #die Dumper [$$settings{cpus_per_mash},$$settings{numthreads},\@reads]; #$$settings{cpus_per_mash}=1; #$$settings{numthreads}=$$settings{numcpus}; logmsg "Temporary directory will be $$settings{tempdir}"; logmsg "$0 on ".scalar(@reads)." files"; my %seen; my @tmp; for my $reads(@ARGV){ if(!-e $reads){ die "ERROR: I could not find reads path at $reads"; } my $basename=basename($reads); if($seen{$basename}++){ logmsg "Skipping $reads: already seen $basename"; next; } push(@tmp,$reads); } @reads=@tmp; my $sketches=sketchAll(\@reads,"$$settings{tempdir}",$settings); my $db = mashDistance($sketches,\@reads,$$settings{tempdir},$settings); logmsg "New file in $$settings{outfile}"; move($db, $$settings{outfile}); return 0; } # Run mash sketch on everything, multithreaded. sub sketchAll{ my($reads,$sketchDir,$settings)=@_; mkdir $sketchDir if(!-d $sketchDir); # Make an array of genomes that would distribute well # across threads. For example, don't put all raw-read # genomes into a single thread and all the assemblies # into another. my %filesize=(); for(@$reads){ $filesize{$_} = -s $_; } my @sortedReads=sort {$filesize{$a} <=> $filesize{$b}} @$reads; my @threadArr=(); for(my $i=0; $i<@sortedReads; $i++){ # Since each genome is sorted smallest to leargest, # they can be sent round-robin to each thread to # ensure balance. my $threadIndex = $i % $$settings{numcpus}; push(@{ $threadArr[$threadIndex] }, $sortedReads[$i]); } # Initiate the threads my @thr; for(0..$$settings{numthreads}-1){ $thr[$_]=threads->new(\&mashSketch,$sketchDir,$threadArr[$_],$settings); } my @mshList; for(@thr){ my $mashfiles=$_->join; for my $file(@$mashfiles){ push(@mshList,$file); } } return \@mshList; } # Individual mash sketch sub mashSketch{ my($sketchDir,$genomeArr,$settings)=@_; # If any file needs to be converted, it will end up in # this directory. my $tempdir=tempdir("$$settings{tempdir}/convertSeq.XXXXXX", CLEANUP=>1); my @msh; # $fastq is a misnomer: it could be any kind of accepted sequence file for my $fastq(@$genomeArr){ my($fileName,$filePath,$fileExt)=fileparse($fastq,@fastqExt,@fastaExt,@richseqExt,@mshExt); # Unzip the file. This temporary file will # only exist if the correct extensions are detected. my $unzipped="$tempdir/".basename($fastq); $unzipped=~s/\.(gz|bz2?|zip)$//i; my $was_unzipped=0; # Don't bother unzipping if it's a fastq or fasta file b/c Mash can read those if(!grep {$_ eq $fileExt} (@fastqExt,@fastaExt)){ if($fastq=~/\.gz$/i){ system("gzip -cd $fastq > $unzipped"); die "ERROR with gzip -cd $fastq" if $?; $was_unzipped=1; } elsif($fastq=~/\.bz2?$/i){ system("bzip2 -cd $fastq > $unzipped"); die "ERROR with bzip2 -cd $fastq" if $?; $was_unzipped=1; } elsif($fastq=~/\.zip$/i){ system("unzip -p $fastq > $unzipped"); die "ERROR with unzip -p $fastq" if $?; $was_unzipped=1; } } # If the file was uncompressed, parse the filename again. if($was_unzipped){ $fastq=$unzipped; ($fileName,$filePath,$fileExt)=fileparse($fastq,@fastqExt,@fastaExt,@richseqExt,@mshExt); } # If we see a richseq (e.g., gbk or embl), then convert it to fasta # TODO If Mash itself accepts richseq, then consider # doing away with this section. if(grep {$_ eq $fileExt} @richseqExt){ # Make a temporary fasta file, but it needs to have a # consistent name in case Mashtree is being run with # the wrapper for bootstrap values. # I can't exactly make a consistent filename in case # different mashtree invocations collide, so # I need to make a new temporary directory with a # consistent filename. my $tmpfasta="$tempdir/$fileName$fileExt.fasta"; my $in=Bio::SeqIO->new(-file=>$fastq); my $out=Bio::SeqIO->new(-file=>">$tmpfasta", -format=>"fasta"); while(my $seq=$in->next_seq){ $out->write_seq($seq); } logmsg "Wrote $tmpfasta"; # Update our filename for downstream $fastq=$tmpfasta; ($fileName,$filePath,$fileExt)=fileparse($tmpfasta, @fastaExt); } # Do different things depending on fastq vs fasta my $sketchXopts=""; if(grep {$_ eq $fileExt} @fastqExt){ my $minDepth=determineMinimumDepth($fastq,$$settings{mindepth},$$settings{kmerlength},$settings); $sketchXopts.="-m $minDepth -g $$settings{genomesize} "; } elsif(grep {$_ eq $fileExt} @fastaExt) { $sketchXopts.=" "; } elsif(grep {$_ eq $fileExt} @mshExt){ $sketchXopts.=" "; } else { logmsg "WARNING: I could not understand what kind of file this is by its extension ($fileExt): $fastq"; } my $outPrefix="$sketchDir/".basename($fastq, @mshExt); # See if the user already mashed this file locally if(-e "$fastq.msh"){ logmsg "Found locally mashed file $fastq.msh. I will use it."; copy("$fastq.msh","$outPrefix.msh"); } if(grep {$_ eq $fileExt} @mshExt){ logmsg "Input file is a sketch file itself and will be used as such: $fastq"; copy($fastq, "$outPrefix.msh"); } if(-e "$outPrefix.msh"){ logmsg "WARNING: ".basename($fastq)." was already mashed."; } elsif(-s $fastq < 1){ logmsg "WARNING: $fastq is a zero byte file. Skipping."; next; } else { logmsg "Sketching $fastq"; my $sketchCommand="mash sketch -k $$settings{kmerlength} -s $$settings{'sketch-size'} $sketchXopts -o $outPrefix $fastq 1>&2"; system($sketchCommand); die if $?; } push(@msh,"$outPrefix.msh"); } system("rm -rf $tempdir"); return \@msh; } # Parallelized mash distance sub mashDistance{ my($mshList,$reads,$outdir,$settings)=@_; # Make a list of names that will appear in the database # in exactly the right format. my @genomeName; # Make a temporary file with one line per mash file. # Helps with not running into the max number of command line args. my $mshListFilename="$outdir/mshList.txt"; open(my $mshListFh,">",$mshListFilename) or die "ERROR: could not write to $mshListFilename: $!"; for(@$mshList){ print $mshListFh $_."\n"; push(@genomeName,_truncateFilename($_,$settings)); } close $mshListFh; # Instatiate the database and create the table before the threads get to it my $mashtreeDbFilename="$outdir/distances.sqlite"; my $mashtreeDb=Mashtree::Db->new($mashtreeDbFilename); # Make an array of distance files for each thread. # Because distance files take about the same amount # of time to analyze, there is no need to sort. my @threadArr=(); for(my $i=0; $i<@$mshList; $i++){ my $threadIndex = $i % $$settings{numcpus}; push(@{ $threadArr[$threadIndex] }, $$mshList[$i]); } # Initialize the threads my @thr; for(0..$$settings{numthreads}-1){ $thr[$_]=threads->new(\&mashDist,$outdir,$threadArr[$_],$mshListFilename,$mashtreeDbFilename,$settings); } for(@thr){ logmsg "Waiting to join thread TID".$_->tid; my $distfiles=$_->join; logmsg "Joined TID".$_->tid; } return $mashtreeDbFilename; } # Individual mash distance sub mashDist{ my($outdir,$mshArr,$mshList,$mashtreeDbFilename,$settings)=@_; # One distance file for all queries in this thread my($distFileFh,$distFile)=tempfile("mashdistXXXXXX", SUFFIX=>".tsv", DIR=>$outdir); my $numQueries=0; my $mashtreeDb=Mashtree::Db->new($mashtreeDbFilename); for my $msh(@$mshArr){ #my $outfile="$outdir/".basename($msh).".tsv"; logmsg "Distances for $msh"; system("mash dist -t $msh -l $mshList >> $distFile"); die "ERROR with 'mash dist -t $msh -l $mshList'" if $?; $numQueries++; } # If there is anything to add to the database, lock # the database. Only lock it once per thread, optimally. if($numQueries > 0){ lock($dbhLock); $mashtreeDb->addDistances($distFile); } # I think that the thread disconnects the db when # this sub ends but I wanted to do it directly and # in a readable fashion. $mashtreeDb->disconnect(); close($distFileFh); unlink($distFile); } sub determineMinimumDepth{ my($fastq,$mindepth,$kmerlength,$settings)=@_; $delta{$fastq}//=10; my $defaultDepth=2; # if no valley is detected # TODO should it be five? return $mindepth if($mindepth > 0); my $basename=basename($fastq,@fastqExt); # Run the min abundance finder to find the valleys my $minAbundanceTempdir="$$settings{tempdir}/$basename.minAbundance.tmp"; mkdir $minAbundanceTempdir; my $minAbundanceCommand="min_abundance_finder.pl --numcpus $$settings{cpus_per_mash} $fastq --kmer $kmerlength --tempdir $minAbundanceTempdir --delta $delta{$fastq}"; lock($abundanceFinderLock); logmsg "DEBUG: running single mode for $fastq"; my @valleyLines=`$minAbundanceCommand`; # If there is an error, just try running one at a time. # I am not sure why there is a seg fault sometimes when # more than one are running at the same time though. #if($?){ # lock($abundanceFinderLock); # @valleyLines=`$minAbundanceCommand`; #} die "ERROR with min_abundance_finder.pl on $fastq: $!" if($?); chomp(@valleyLines); # Some cleanup of large files unlink $_ for(glob("$minAbundanceTempdir/*")); rmdir $minAbundanceTempdir; # If there is no valley, return a default value #if(!defined $valleyLines[1] || !looks_like_number($valley[1]) || @valley < 1){ if(!defined $valleyLines[1] || @valleyLines < 1){ $delta{$fastq}=int($delta{$fastq}/2); if($delta{$fastq} > 10){ logmsg "Trying again to determine a min depth with delta==$delta{$fastq} on $fastq"; return determineMinimumDepth($fastq,$mindepth,$kmerlength,$settings); } logmsg "WARNING: no valleys were found! Reporting minimum kmer coverage as $defaultDepth."; return $defaultDepth; } # Discard the header but keep the first line my($minKmerCount, $countOfCounts)=split(/\t/,$valleyLines[1]); # force an "empty" value to zero if(!defined($minKmerCount) || !looks_like_number($minKmerCount)){ $minKmerCount=0; } # However, the minimum count can't be zero, and so it is one. $minKmerCount=1 if($minKmerCount < 1); logmsg "Setting the min depth as $minKmerCount for $fastq (delta==$delta{$fastq})"; return $minKmerCount; } sub usage{ "$0: use distances from Mash (min-hash algorithm) to make a database of distances Usage: $0 [options] -o mash.sqlite *.fastq *.fasta *.gbk *.msh NOTE: fastq files are read as raw reads; fasta, gbk, and embl files are read as assemblies; Input files can be gzipped. --outfile '' Required output sqlite file --tempdir '' If specified, this directory will not be removed at the end of the script and can be used to cache results for future analyses. If not specified, a dir will be made for you and then deleted at the end of this script. --numcpus 1 This script uses Perl threads. --version Display the version and exit TREE OPTIONS --truncLength 250 How many characters to keep in a filename --sort-order ABC For neighbor-joining, the sort order can make a difference. Options include: ABC (alphabetical), random, input-order MASH SKETCH OPTIONS --genomesize 5000000 --mindepth 5 If mindepth is zero, then it will be chosen in a smart but slower method, to discard lower-abundance kmers. --kmerlength 21 --sketch-size 10000 " } ================================================ FILE: bin/mashtree_jackknife.pl ================================================ #!/usr/bin/env perl # Author: Lee Katz # Uses Mash and BioPerl to create a NJ tree based on distances. # Run this script with -h for help and usage. use strict; use warnings; use Data::Dumper; use Getopt::Long; use File::Temp qw/tempdir tempfile/; use File::Basename qw/basename dirname fileparse/; use File::Copy qw/cp mv/; use List::Util qw/shuffle/; use List::MoreUtils qw/part/; use POSIX qw/floor/; # if JSON::XS is installed, it will be automatically used. # Specify () so that no functions are exported. We will use OO. use JSON (); use IO::Handle; # allows me to autoflush file handles #use Fcntl qw/:flock LOCK_EX LOCK_UN/; use threads; use Thread::Queue; use threads::shared; use FindBin; use lib "$FindBin::RealBin/../lib"; use Mashtree qw/logmsg @fastqExt @fastaExt createTreeFromPhylip mashDist/; use Mashtree::Db; use Bio::SeqIO; use Bio::TreeIO; use Bio::Tree::DistanceFactory; use Bio::Tree::Statistics; my $writeStick :shared; my $readStick :shared; # limit disk IO by reading some files one at a time local $0=basename $0; exit main(); sub main{ my $settings={}; my @wrapperOptions=qw(help file-of-files outmatrix=s tempdir=s reps=i numcpus=i); GetOptions($settings,@wrapperOptions) or die $!; $$settings{reps}//=1; $$settings{numcpus}||=1; die usage() if($$settings{help}); die usage() if(@ARGV < 1); $$settings{tempdir}||=tempdir("MASHTREE_WRAPPER.XXXXXX",CLEANUP=>1,TMPDIR=>1); mkdir($$settings{tempdir}) if(!-d $$settings{tempdir}); logmsg "Temporary directory will be $$settings{tempdir}"; if($$settings{reps} < 1){ die "ERROR: no reps were given!"; } if($$settings{reps} < 100){ logmsg "WARNING: You have very few reps planned on this mashtree run. Recommended reps are at least 100."; } # Give a warning if JSON is going to be slow later my $jsonTmp = JSON->new(); if(! $jsonTmp->is_xs){ logmsg "WARNING: the currently installed JSON module will make this script very slow when jack knifing. To avoid this error, install the JSON::XS module like so: `cpanm ~l ~ JSON::XS`."; } ## Catch some options that are not allowed to be passed # Tempdir: All mashtree temporary directories will be under the # wrapper's tempdir. if(grep(/^\-+tempdir$/,@ARGV) || grep(/^\-+t$/,@ARGV)){ die "ERROR: tempdir was specified for mashtree but should be an option for $0"; } # Numcpus: this needs to be specified in the wrapper and will # appropriately be transferred to the mashtree script if(grep(/^\-+numcpus$/,@ARGV) || grep(/^\-+n$/,@ARGV)){ die "ERROR: numcpus was specified for mashtree but should be an option for $0"; } # Outmatrix: the wrapper script needs to control where # the matrix goes because it can only have the outmatrix # for the observed run and not the replicates for speed's # sake. if(grep(/^\-+outmatrix$/,@ARGV) || grep(/^\-+o$/,@ARGV)){ die "ERROR: outmatrix was specified for mashtree but should be an option for $0"; } # --file-of-files: should be something to give to mashtree directly if(grep(/^\-+file-of-files$/,@ARGV) || grep(/^\-+f$/,@ARGV)){ die "ERROR: file-of-files was specified for mashtree but should be an option for $0"; } # Separate flagged options from reads in the mashtree options my @reads = (); my @mashOptions = (); for(my $i=0;$i<@ARGV;$i++){ if(-e $ARGV[$i]){ push(@reads, $ARGV[$i]); } else { push(@mashOptions, $ARGV[$i]); } } if($$settings{'file-of-files'}){ push(@mashOptions, '--file-of-files'); } my $mashOptions=join(" ",@mashOptions); my $reads = join(" ", @reads); # Some filenames we'll expect my $observeddir="$$settings{tempdir}/observed"; my $obsDistances="$observeddir/distances.phylip"; my $observedTree="$$settings{tempdir}/observed.dnd"; my $outmatrix="$$settings{tempdir}/observeddistances.tsv"; my $mshList = "$$settings{tempdir}/mash.list"; my $mergedMash = "$$settings{tempdir}/merged.msh"; my $mergedJSON = "$$settings{tempdir}/merged.msh.json.gz"; # Make the observed directory and run Mash logmsg "Running mashtree on full data (".scalar(@reads)." targets)"; mkdir($observeddir); system("$FindBin::RealBin/mashtree --outmatrix $outmatrix.tmp --tempdir $observeddir --numcpus $$settings{numcpus} $mashOptions $reads > $observedTree.tmp"); die if $?; mv("$observedTree.tmp",$observedTree) or die $?; mv("$outmatrix.tmp",$outmatrix) or die $?; # Merge the mash files to make the threads go faster later my @msh = glob("$observeddir/*.msh"); open(my $mshListFh, ">", $mshList) or die "ERROR writing to mash list $mshList: $!"; for(@msh){ print $mshListFh $_."\n"; } close $mshListFh; unlink($mergedMash) if(-e $mergedMash); # mash complains about overwriting an existing file system("mash paste -l $mergedMash $mshList >&2"); die "ERROR merging mash files" if $?; # max compression on the json file so that it can be a # smaller footprint and so that it can be read faster # in each thread. system("mash info -d $mergedMash | gzip -c9 > $mergedJSON"); die "ERROR with mash info | gzip -c9" if $?; unlink($_) for(@msh); # remove redundant files to the merged msh # Round-robin sample the replicate number, so that it's # well-balanced. my @repNumber = (1..$$settings{reps}); my @reps; for(my $i=0;$i<$$settings{reps};$i++){ my $thrIndex = $i % $$settings{numcpus}; push(@{$reps[$thrIndex]}, $i); } # Sample the mash sketches with replacement for rapid bootstrapping my @bsThread; for my $i(0..$$settings{numcpus}-1){ my %settingsCopy = %$settings; $bsThread[$i] = threads->new(\&subsampleMashSketchesWorker, $mergedJSON, $reps[$i], \%settingsCopy); } my @bsTree; for my $thr(@bsThread){ my $fileArr = $thr->join; if(ref($fileArr) ne 'ARRAY'){ die "ERROR: one or more threads did not return an array of jack knife tree files as expected."; } for my $file(@$fileArr){ my $treein = Bio::TreeIO->new(-file=>$file); while(my $tree=$treein->next_tree){ push(@bsTree, $tree); } } } # Combine trees into a bootstrapped tree and write it # to an output file. Then print it to stdout. logmsg "Adding bootstraps to tree"; my $guideTree=Bio::TreeIO->new(-file=>"$observeddir/tree.dnd")->next_tree; my $bsTree=assess_bootstrap($guideTree,\@bsTree,$guideTree); for my $node($bsTree->get_nodes){ next if($node->is_Leaf); my $id=$node->bootstrap||$node->id||0; $node->id($id); } open(my $treeFh,">","$$settings{tempdir}/bstree.dnd") or die "ERROR: could not write to $$settings{tempdir}/bstree.dnd: $!"; print $treeFh $bsTree->as_text('newick'); print $treeFh "\n"; close $treeFh; system("cat $$settings{tempdir}/bstree.dnd"); die if $?; if($$settings{'outmatrix'}){ cp($outmatrix,$$settings{'outmatrix'}); } return 0; } sub subsampleMashSketchesWorker{ my($mergedJSON, $reps, $settings) = @_; $reps //= []; my @treeFile; return \@treeFile if(@$reps <1); # Initialize our JSON reader my $json = JSON->new; $json->utf8; # If we only expect characters 0..255. Makes it fast. $json->allow_nonref; # can convert a non-reference into its corresponding string $json->allow_blessed; # encode method will not barf when it encounters a blessed reference $json->pretty; # enables indent, space_before and space_after # Only let one thread at a time read the large JSON file my $mashInfoStr=""; # must be declared outside of the block because we are parsing it after the block { lock($readStick); logmsg "Reading huge JSON file describing all mash distances, $mergedJSON"; $mashInfoStr = `gzip -cd $mergedJSON`; die "ERROR running gzip -cd $mergedJSON: $!" if $?; } my $mashInfoHash = $json->decode($mashInfoStr); $mashInfoStr=""; # clear some ram my $kmerlength = $$mashInfoHash{kmer}; # find the kmer length right away my $numReps = @$reps; my $numSketches = scalar(@{ $$mashInfoHash{sketches} }); my $msg="Initializing thread with $numReps replicates: "; for my $i(0..2){ my $repID = $$reps[$i] // ""; $msg.="$repID, "; } $msg=~s/[ ,]+$//; # right trim if($numReps > 3){ $msg.="..."; } logmsg $msg; for(my $repI=0;$repI<@$reps;$repI++){ my $rep = $$reps[$repI]; my $subsampleDir = "$$settings{tempdir}/rep$rep"; mkdir $subsampleDir; my $log = "$subsampleDir/jackknife.log"; open(my $logFh, ">", $log) or die "ERROR: cannot write to log $log: $!"; $logFh->autoflush(); logmsg "rep$rep: $subsampleDir (".($repI+1)." out of $numReps in this thread)"; my @name; # Start off a distances string for printing to file later my %dist = (); for(my $sketchCounter=0; $sketchCounter<$numSketches; $sketchCounter++){ my $nameI = basename($$mashInfoHash{sketches}[$sketchCounter]{name},(@fastqExt,@fastaExt)); print $logFh "Subsampling sketches from entry $sketchCounter, $nameI\n"; # subsample the hashes of one genome at a time as compared # to the other genomes, to get a jack knife distance. my $numHashes = scalar(@{ $$mashInfoHash{sketches}[$sketchCounter]{hashes} }); my $keepHashes = int($numHashes / 2); # based on half the hashes my @subsampleHash = @{ $$mashInfoHash{sketches}[$sketchCounter]{hashes} }; @subsampleHash = (shuffle(@subsampleHash))[0..$keepHashes-1]; @subsampleHash = sort{$a<=>$b} @subsampleHash; print $logFh "Distances between $nameI and other genomes"; # Initialize the distances string with the query name. # Use this string as a buffer for the distances file # to help avoid too much disk IO. push(@name, $nameI); for(my $j=0; $j<$numSketches; $j++){ my $nameJ = basename($$mashInfoHash{sketches}[$j]{name},(@fastqExt,@fastaExt)); my $distance = mashDist(\@subsampleHash, $$mashInfoHash{sketches}[$j]{hashes}, $kmerlength); $dist{$nameI}{$nameJ} = $distance; print $logFh "."; } print $logFh "\n"; } # Add distances to database print $logFh "Creating database, $subsampleDir/distances.db.tsv\n"; my $mashtreeDb = Mashtree::Db->new("$subsampleDir/distances.db.tsv"); $mashtreeDb->addDistancesFromHash(\%dist); # Convert to Phylip my $phylipFile = "$subsampleDir/distances.phylip"; print $logFh "Creating phylip file, $phylipFile\n"; open(my $phylipFh, ">", $phylipFile) or die "ERROR: could not write to $phylipFile: $!"; print $phylipFh $mashtreeDb->toString(\@name, "phylip"); close $phylipFh; print $logFh "Creating tree file, $subsampleDir/tree.dnd\n"; my $treeObj = createTreeFromPhylip($phylipFile, $subsampleDir, $settings); push(@treeFile, "$subsampleDir/tree.dnd"); print $logFh "DONE!\n"; close $logFh; } return \@treeFile; } # Fixed bootstrapping function from bioperl # https://github.com/bioperl/bioperl-live/pull/304 sub assess_bootstrap{ my ($self,$bs_trees,$guide_tree) = @_; my @consensus; if(!defined($bs_trees) || ref($bs_trees) ne 'ARRAY'){ die "ERROR: second parameter in assess_bootstrap() must be a list"; } my $num_bs_trees = scalar(@$bs_trees); if($num_bs_trees < 1){ die "ERROR: no bootstrap trees were passed to assess_bootstrap()"; } # internal nodes are defined by their children my (%lookup,%internal); my $i = 0; for my $tree ( $guide_tree, @$bs_trees ) { # Do this as a top down approach, can probably be # improved by caching internal node states, but not going # to worry about it right now. my @allnodes = $tree->get_nodes; my @internalnodes = grep { ! $_->is_Leaf } @allnodes; for my $node ( @internalnodes ) { my @tips = sort map { $_->id } grep { $_->is_Leaf() } $node->get_all_Descendents; my $id = "(".join(",", @tips).")"; if( $i == 0 ) { $internal{$id} = $node->internal_id; } else { $lookup{$id}++; } } $i++; } #my @save; # unsure why this variable is needed for my $l ( keys %lookup ) { if( defined $internal{$l} ) {#&& $lookup{$l} > $min_seen ) { my $intnode = $guide_tree->find_node(-internal_id => $internal{$l}); $intnode->bootstrap(sprintf("%d",100 * $lookup{$l} / $num_bs_trees)); } } return $guide_tree; } ####### # Utils ####### sub usage{ my $usage="$0: a wrapper around mashtree. Usage: $0 [options] [-- mashtree options] *.fastq.gz *.fasta > tree.dnd --outmatrix '' Output file for distance matrix --reps 0 How many bootstrap repetitions to run; If zero, no bootstrapping. --numcpus 1 This will be passed to mashtree and will be used to multithread reps. --tempdir A directory path that will be created to store temporary files --file-of-files If specified, mashtree will try to read filenames from each input file. The file of files format is one filename per line. This file of files cannot be compressed. -- Used to separate options for $0 and mashtree MASHTREE OPTIONS:\n". # Print the mashtree options starting with numcpus, # skipping the tempdir option. `mashtree --help 2>&1 | grep -A 999 "TREE OPTIONS" | grep -v ^Stopped`; return $usage; } ================================================ FILE: bin/mashtree_wrapper_deprecated.pl ================================================ #!/usr/bin/env perl # Author: Lee Katz # Uses Mash and BioPerl to create a NJ tree based on distances. # Run this script with -h for help and usage. use strict; use warnings; use Data::Dumper; use Getopt::Long; use File::Temp qw/tempdir tempfile/; use File::Basename qw/basename dirname fileparse/; use File::Copy qw/cp mv/; use List::Util qw/shuffle/; use List::MoreUtils qw/part/; use POSIX qw/floor/; use Fcntl qw/:flock LOCK_EX/; use threads; use Thread::Queue; use threads::shared; use FindBin; use lib "$FindBin::RealBin/../lib"; use Mashtree qw/logmsg @fastqExt @fastaExt createTreeFromPhylip/; use Mashtree::Db; use Bio::SeqIO; use Bio::TreeIO; use Bio::Tree::DistanceFactory; use Bio::Tree::Statistics; use Bio::Matrix::IO; my $writeStick :shared; local $0=basename $0; exit main(); sub main{ my $settings={}; my @wrapperOptions=qw(help outmatrix=s tempdir=s reps=i numcpus=i); GetOptions($settings,@wrapperOptions) or die $!; $$settings{reps}||=0; $$settings{numcpus}||=1; die usage() if($$settings{help}); die usage() if(@ARGV < 1); $$settings{tempdir}||=tempdir("MASHTREE_WRAPPER.XXXXXX",CLEANUP=>1,TMPDIR=>1); mkdir($$settings{tempdir}) if(!-d $$settings{tempdir}); logmsg "Temporary directory will be $$settings{tempdir}"; if($$settings{reps} < 10){ logmsg "WARNING: You have very few reps planned on this mashtree run. Recommended reps are at least 10 or 100."; } ## Catch some options that are not allowed to be passed # Tempdir: All mashtree temporary directories will be under the # wrapper's tempdir. if(grep(/^\-+tempdir$/,@ARGV) || grep(/^\-+t$/,@ARGV)){ die "ERROR: tempdir was specified for mashtree but should be an option for $0"; } # Numcpus: this needs to be specified in the wrapper and will # appropriately be transferred to the mashtree script if(grep(/^\-+numcpus$/,@ARGV) || grep(/^\-+n$/,@ARGV)){ die "ERROR: numcpus was specified for mashtree but should be an option for $0"; } # Outmatrix: the wrapper script needs to control where # the matrix goes because it can only have the outmatrix # for the observed run and not the replicates for speed's # sake. if(grep(/^\-+outmatrix$/,@ARGV) || grep(/^\-+o$/,@ARGV)){ die "ERROR: outmatrix was specified for mashtree but should be an option for $0"; } # Separate flagged options from reads in the mashtree options my @reads = (); my @mashOptions = (); for(my $i=0;$i<@ARGV;$i++){ if(-e $ARGV[$i]){ push(@reads, $ARGV[$i]); } else { push(@mashOptions, $ARGV[$i]); } } # Copy reads over to the temp storage where I assume it is faster # and where we can write .lock files. my $inputdir = "$$settings{tempdir}/input"; mkdir $inputdir; my $tmp_i= 0; my @reads_per_thread = ([@reads]); if($$settings{numcpus} > 1){ @reads_per_thread = part { $tmp_i++ % ($$settings{numcpus}-1) } @reads; } my @cpThread; for(0..$$settings{numcpus}-1){ $cpThread[$_] = threads->new(sub{ my($fileArr) = @_; my @copiedReads; for my $file(@$fileArr){ logmsg "Copying $file to temp space - $inputdir"; my $copiedFile = "$inputdir/".basename($file); cp($file, $copiedFile); push(@copiedReads, $copiedFile); } return \@copiedReads; }, $reads_per_thread[$_]); } @reads = (); for(@cpThread){ my $tmp = $_->join; push(@reads, @$tmp); } logmsg "Finished copying input data to $inputdir"; my $mashOptions=join(" ",@mashOptions); my $reads = join(" ", @reads); # Some filenames we'll expect my $observeddir="$$settings{tempdir}/observed"; my $obsDistances="$observeddir/distances.phylip"; my $observedTree="$$settings{tempdir}/observed.dnd"; my $outmatrix="$$settings{tempdir}/observeddistances.tsv"; # Multithreaded reps my @rep_id = (1..$$settings{reps}); my $repsPerThread = int($$settings{reps} / $$settings{numcpus}) + 1; my @thr; for(0..$$settings{numcpus}-1){ my @theseReps = splice(@rep_id, 0, $repsPerThread); $thr[$_]=threads->new(\&repWorker, \@mashOptions, \@reads, \@theseReps, $settings); } my @bsTree; for(@thr){ my $treeArr=$_->join; for(@$treeArr){ push(@bsTree,Bio::TreeIO->new(-file=>$_)->next_tree); } } # Make the observed directory and run Mash logmsg "Running mashtree on full data"; mkdir($observeddir); system("$FindBin::RealBin/mashtree --outmatrix $outmatrix.tmp --tempdir $observeddir --numcpus $$settings{numcpus} $mashOptions $reads > $observedTree.tmp"); die if $?; mv("$observedTree.tmp",$observedTree) or die $?; mv("$outmatrix.tmp",$outmatrix) or die $?; # Combine trees into a bootstrapped tree and write it # to an output file. Then print it to stdout. logmsg "Adding bootstraps to tree"; my $biostat=Bio::Tree::Statistics->new; my $guideTree=Bio::TreeIO->new(-file=>"$observeddir/tree.dnd")->next_tree; my $bsTree=$biostat->assess_bootstrap(\@bsTree,$guideTree); for my $node($bsTree->get_nodes){ next if($node->is_Leaf); my $id=$node->bootstrap||$node->id||0; $node->id($id); } open(my $treeFh,">","$$settings{tempdir}/bstree.dnd") or die "ERROR: could not write to $$settings{tempdir}/bstree.dnd: $!"; print $treeFh $bsTree->as_text('newick'); print $treeFh "\n"; close $treeFh; system("cat $$settings{tempdir}/bstree.dnd"); die if $?; if($$settings{'outmatrix'}){ cp($outmatrix,$$settings{'outmatrix'}); } return 0; } sub repWorker{ my($mashOptions, $reads, $reps,$settings)=@_; my @bsTree; if(!defined($reps) || ref($reps) ne 'ARRAY' || !@$reps){ return \@bsTree; } my @mashOptions = @$mashOptions; my @reads = @$reads; my $numcpus = floor($$settings{numcpus}/$$settings{reps}); $numcpus = 1 if($numcpus < 1); for my $rep(@$reps){ my $repTempdir="$$settings{tempdir}/rep$rep"; mkdir $repTempdir; logmsg "Starting mashtree replicate $rep - $repTempdir"; #logmsg "Downsampling reads (replicate $rep)."; # Downsample the reads my @downsampledReads=(); for my $r(@reads){ my $newReads = "$repTempdir/".basename($r); #logmsg "DEBUG";push(@downsampledReads, $newReads);next; my @buffer = (); open(my $lockFh, ">", "$r.lock") or die "ERROR: could not make lockfile $r.lock: $!"; flock($lockFh, LOCK_EX) or die "ERROR locking file $r.lock: $!"; open(my $inFh, "zcat $r | ") or die "ERROR reading $r for downsampling: $!"; open(my $outFh," | gzip -c > $newReads") or die "ERROR gzipping to $newReads: $!"; close $outFh; while(my $id=<$inFh>){ my $seq =<$inFh>; my $plus=<$inFh>; my $qual=<$inFh>; if(rand(1) < 0.5){ push(@buffer, $id.$seq.$plus.$qual); # The buffer size is something like 600 bytes per entry # times 100,000 entries times 12 threads = 720Mb. if(scalar(@buffer) > 100000){ { # Only let one thread at a time write to the file but # this makes us open the file in append mode. lock($writeStick); open($outFh, " | gzip -c >> $newReads") or die "ERROR gzipping to $newReads: $!"; print $outFh join("",@buffer); close $outFh; } @buffer = (); # flush the buffer } } } # Finish the remaining entries in the buffer { lock($writeStick); open($outFh, " | gzip -c >> $newReads") or die "ERROR gzipping to $newReads: $!"; print $outFh join("",@buffer); close $outFh; } close $inFh; close $lockFh; push(@downsampledReads, $newReads); } logmsg "Done downsampling for replicate $rep. Running mashtree on files in $repTempdir"; my $log = `mashtree --numcpus $numcpus @mashOptions @downsampledReads 2>&1 > $repTempdir/tree.dnd`; if($?){ die "ERROR with mashtree on rep $rep (exit code $?):\n$log"; } logmsg "Finished with rep $rep"; push(@bsTree,"$repTempdir/tree.dnd"); } return \@bsTree; } ####### # Utils ####### sub usage{ my $usage="$0: a wrapper around mashtree. Usage: $0 [options] [-- mashtree options] *.fastq.gz *.fasta > tree.dnd --outmatrix '' Output file for distance matrix --reps 0 How many bootstrap repetitions to run; If zero, no bootstrapping. Bootstrapping will only work on compressed fastq files. --numcpus 1 This will be passed to mashtree and will be used to multithread reps. -- Used to separate options for $0 and mashtree MASHTREE OPTIONS:\n". # Print the mashtree options starting with numcpus, # skipping the tempdir option. `mashtree --help 2>&1 | grep -A 999 "TREE OPTIONS" | grep -v ^Stopped`; return $usage; } ================================================ FILE: bin/min_abundance_finder.pl ================================================ #!/usr/bin/env perl # Find the minimum abundance of kmers # Original script was in Python at # https://gist.github.com/alexjironkin/4ed43412878723491240814a0d5a6ed6/223dea45d70c9136703a4afaab0178cdbfbd2042 # Original author of python script: @alexjironkin, Public Health Englad # I wanted to increase capatibility with Perl and have a more # standalone script instead of relying on the khmer package. # Author: Lee Katz use strict; use warnings; use Getopt::Long qw/GetOptions/; use Data::Dumper qw/Dumper/; use File::Basename qw/basename fileparse/; use File::Temp qw/tempdir/; use List::Util qw/max/; use threads; use Thread::Queue; # http://perldoc.perl.org/perlop.html#Symbolic-Unary-Operators # +Inf and -Inf will be a binary complement of all zeros use constant MAXINT => ~0; use constant MININT => -~0; local $0=basename $0; sub logmsg{print STDERR "$0: @_\n";} exit main(); sub main{ my $settings={}; GetOptions($settings,qw(help kmerlength|kmer=i kmerCounter=s delta=i gt|greaterthan=i tempdir=s numcpus=i)) or die $!; $$settings{kmerlength} ||=21; $$settings{kmerCounter}||=""; $$settings{gt} ||=1; $$settings{tempdir} ||=tempdir(TEMPLATE=>"$0.XXXXXX",CLEANUP=>1,TMPDIR=>1); $$settings{numcpus} ||=1; if($$settings{delta}){ logmsg "WARNING: --delta has been deprecated"; } my($fastq)=@ARGV; die usage() if(!$fastq || $$settings{help}); die "ERROR: I could not find fastq at $fastq" if(!-e $fastq); ## Find valleys with multithreading # List all kmer sizes per thread. my @kmerlength; my $i=0; for(my $k=7; $k<=32; $k+=3){ my $whichThread = $i % $$settings{numcpus}; push(@{$kmerlength[$whichThread]}, $k); $i++; } # Launch the threads with the various kmer length sub-arrays. my @thr; for my $kArray(@kmerlength){ # Ensure that each thing getting sent to the thread is independent # of this main subroutine memory space. my %settingsCopy = %$settings; my @kArrayCopy = @$kArray; # Launch the thread. push(@thr, threads->new(\&valleyWorker, $fastq, \@kArrayCopy,\%settingsCopy) ); } my %firstValleyVote; for(@thr){ my $firstValley = $_->join; for my $minimum(@$firstValley){ next if(!$minimum); $firstValleyVote{$minimum}++; } } # If no valleys were found, simply set the stage so that # the minimum depth will be set to 0. if(keys(%firstValleyVote) < 1){ logmsg "NOTE: no valleys were found and so I am inserting an imaginary vote for a valley at cov=1"; $firstValleyVote{0} = 1; } logmsg "For various values of k, valleys were found:"; my $firstValley=0; # Average out the votes my $totalFirstValley = 0; my $totalVotes = 0; my @vote; # Sort bins by their votes, highest to lowest for my $bin(sort{$firstValleyVote{$b}<=>$firstValleyVote{$a} || $a<=>$b} keys(%firstValleyVote)){ my $value=$firstValleyVote{$bin}; $firstValley||=$bin; # set the valley to the first bin we come to for(1..$value){ push(@vote, $bin); } $totalFirstValley += $bin * $value; $totalVotes += $value; } @vote = sort {$a<=>$b} @vote; logmsg @vote; my $medianFirstValley = $vote[ int(scalar(@vote)/2) ]; # Get the average first valley across many kmers my $avgFirstValley = $totalFirstValley/$totalVotes; logmsg " Average first valley is $avgFirstValley"; logmsg " However, I will use the median valley: $medianFirstValley"; printf("%0.0f\n", $medianFirstValley); #print join("\t",qw(kmer count))."\n"; #print join("\t", $firstValley, 1)."\n"; return 0; } # Poor man's way of subsampling # Thanks to Nick Greenfield for pointing this out. sub mashHistogram{ my($fastq,$k,$settings)=@_; my $sketch="$$settings{tempdir}/sketch.$k.msh"; system("mash sketch -k $k -m 1 -o $sketch $fastq > /dev/null 2>&1"); die if $?; my @histogram; open(my $fh, "mash info -c $sketch | ") or die "ERROR: could not get mash info on sketch $sketch"; while(my $line=<$fh>){ $line=~s/^\s+|\s+$//g; # whitespace trim next if($line=~/^#/); my($filename, $bin, $frequency)=split(/\t/, $line); $histogram[$bin]=$frequency; } close $fh; $histogram[$_]||=0 for(0..@histogram); return \@histogram; } sub readHistogram{ my($infile,$settings)=@_; my @hist=(0); open(HIST,$infile) or die "ERROR: could not read $infile: $!"; logmsg "Reading histogram from $infile"; while(){ chomp; my($count,$countOfCounts)=split /\t/; $hist[$count]=$countOfCounts; } close HIST; # ensure defined values for(my $i=0;$i<@hist;$i++){ $hist[$i] //= 0; } return \@hist; } sub valleyWorker{ my($fastq, $kArray, $settings)=@_; my @firstValley; for my $kmerlength(@$kArray){ logmsg "Counting $kmerlength-kmers in $fastq"; my $histogram = mashHistogram($fastq,$kmerlength,$settings); my $firstValley = findFirstValley($histogram, $settings); push(@firstValley, $firstValley); } return \@firstValley; } # https://www.perlmonks.org/?node_id=629742 sub localMinimaMaxima{ my($array, $settings)=@_; my @arr = @$array; my @minima; my @maxima; my $prev_cmp = 0; my $num = @arr - 2; for my $i (0 .. $num){ my $cmp = $arr[$i] <=> $arr[$i+1]; if ($cmp != $prev_cmp) { # have a minimum only after there has been a maximum if($cmp < 0 && @maxima > 0){ push @minima, $i; } elsif($cmp > 0){ push @maxima, $i; } # when this and next elements are ==, defer checking for # minima/maxima till next loop iteration $prev_cmp = $cmp if $cmp; } } # Uncomment the following if we want to look at the very # last number in the array. #if (@$array) { # push @minima, $num if $prev_cmp >= 0; # push @maxima, $num if $prev_cmp <= 0; #} ## debugging #splice(@$array, 30); #print join(".",@minima)."\n".join(",", @$array)."\n\n"; return(\@minima, \@maxima); } sub findFirstValley{ my($array, $settings)=@_; my($minima, $maxima) = localMinimaMaxima($array, $settings); # Return the first minimum if it's not the first element # or if there are no other minima. if($$minima[0] > 0 || !$$minima[1]){ return $$minima[0]; } else { return $$minima[1]; } } # http://www.perlmonks.org/?node_id=761662 sub which{ my($exe,$settings)=@_; my $tool_path=""; for my $path ( split /:/, $ENV{PATH} ) { if ( -f "$path/$exe" && -x "$path/$exe" ) { $tool_path = "$path/$exe"; last; } } return $tool_path; } sub usage{ " $0: Find the valley between two peaks on a set of kmers such that you can discard the kmers that are likely representative of contamination. This script does not require any dependencies. Usage: $0 file.fastq[.gz] --gt 1 Look for the first peak at this kmer count and then the next valley. --kmer 21 kmer length --numcpus 1 This script will apply one thread per kmer length. Multiple values of k are tested to get a consensus value. MISC --kmerCounter '' The kmer counting program to use. Default: (empty string) auto-choose Options: perl, jellyfish " } ================================================ FILE: docs/ALGORITHM.md ================================================ # Algorithms ## Fast or accurate mode While Mashtree has been developed for speed, some might want higher accuracy. In response, we have implemented an optional minimum abundance filter for raw read input files. This filter creates a histogram of kmers using Mash and detects the valleys between peaks. The first (left-most) peak of a kmer histogram represents kmers that occur only once or rarely. These "rare" kmers usually represent sequencing errors. Therefore, the minimum abundance filter can optionally be used to see where the first valley is and automatically set that value for the minimum kmer coverage parameter. Using this filter helps avoid noise in the input data. Running in accurate mode is shorthand for running Mashtree with the minimum abundance filter turned on. Otherwise by default, a user is running in fast mode. To invoke accurate mode, run with `--mindepth 0`, e.g., ```bash mashtree --mindepth 0 --numcpus 12 *.fastq.gz [*.fasta] > mashtree.dnd ``` ## Confidence values Most phylogenetic trees have confidence scores associated with each hypothetical ancestor node. These support scores are typically generated by resampling the input data many times, analyzing each replicate in context of the original phylogeny. Strong support for a node in the input data will result in consistent recovery of that node in the majority of resampled replicate trees. Conversely, nodes which have low support in the input data will be sensitive to stochastic changes in the resampled sketches which can result in alternate topologies of involved isolates. Therefore, the support value correlates with the robustness and reproducibility of a recovered relationship. As the NJ tree recovered from a single tree reconstruction will always be a fully resolved binary tree, the addition of these resampled support values provides a valuable tool for assessing the confidence and reliability of a node. This can be especially helpful when trying to assess the exact topology of clades containing three or more very closely related genomes. Although Mashtree does not infer phylogeny, we have borrowed the ideas behind phylogenetic confidence values to yield confidence values for each parent node in the tree. There are two resampling methods implemented in Mashtree to assign support values to infernal nodes, bootstrapping and jackknifing. Initially, both methods create a tree as depicted in [Figure 1](../paper/Mashtree_workflow.png). The bootstrapping method infers confidence on the tree by creating one new tree per replication [Figure 2](../paper/Bootstrap_workflow.png). However, instead of accepting the default seed value for Mash, a new one is generated. Changing this seed value will affect the value of hashed kmers and how they sort, therefore providing a different sampling of kmers in the sketch. Due to changes in sketch sampling, distances between genomes can differ and the resulting tree in the replication can differ. Then, nodes in the original tree are assigned support values based on how frequently the same bipartition was recovered in the replicate trees. In the jackknife method, for each replication, each genome is picked one at a time as the query genome [Figure 3](../paper/Jackknife_workflow.png). Its hashes are sampled without replacement for half of the original hashes. Then, a Mash distance is recomputed between the query genome and all other genomes in the analysis. When the distances are recorded in the replicate's pairwise distance matrix, the average distance between each comparison is used of each query-reference genome pair, and vice versa. This new distance matrix is used for computing a NJ tree. Nodes in the original tree are assigned support values based on how frequently the same bipartition was recovered in replicate trees. # Figures ![Figure 1](../paper/Mashtree_workflow.png) **Figure 1.** The Mashtree workflow. Step 1) Sketch genomes with Mash. In this schematic, there is a green circle representing each genome in the analysis. Filled-in brown circles indicate the presence of a kmer. Missing circles represent true absence. After hashing with a sketch size of six (after the arrow), some kmers are not represented in the Mash sketch either because they are not present in the original genome or because only a finite number of kmers are sketched (e.g., six in this example). Henceforth, truly missing hashes or hashes not included in the Mash sketch are represented by empty circles. Step 2) Calculate distances with `Mash dist`. Distances in the figure are represented by Jaccard distances, which are calculated as the intersection divided by the union. In this example, the genomes are separated by Jaccard distances of 5/9, 4/9, and 3/9. These jaccard distances are internally transformed into Mash distances [@Ondov:2016]. Step 3) Create dendrogram with Quicktree using the Mash distance matrix. ![Figure 2](../paper/Bootstrap_workflow.png) **Figure 2.** The Mashtree bootstrap workflow. Step 1) Generate a tree with the normal workflow as in Figure 1. This is the main tree. Step 2) Run the normal workflow once per replicate but with a different random seed. In this example, the top right replicate differs from the main tree. All ten of these trees are the bootstrap tree replicates. Step 3) For each parent node in the main tree, quantify how many bootstrap tree replicates have the same node with the same children. Record that percentage next to each parent node. This percentage quantifies how confident the Mashtree cluster is, controlling for the random seed in the Mash program. ![Figure 3](../paper/Jackknife_workflow.png) **FIgure 3.** The Mashtree jackknife workflow. Step 1) Generate a tree with the normal workflow as in Figure 1. This is the main tree. Step 2) For each replicate, sample the half hashes without replacement for each query genome. Recalculate the Mash distance between the query genome and all other genomes, reducing the denominator to one half, rounding up, to reflect the smaller pool of hashes. After all genomes have been selected for query genomes, average the distances to create a new distance matrix. Create the dendrogram from the new distance matrix. For brevity, only one detailed replicate is shown. Step 3) For each replication, calculate the new tree from the new distance matrix. In this example, the top right replication differs from the main tree. All ten of these trees are the jack knife tree replicates. Step 4) For each parent node in the main tree, quantify how many jack knife tree replicates have the same node with the same children. Record that percentage next to each parent node. This percentage quantifies how confident Mashtree is at clustering, controlling for stochasticity in hashes. ================================================ FILE: docs/INSTALL.md ================================================ # Installation ## Prerequisite software * Mash >= v2.0 * ~~SQLite3~~ Removed in version 1.4 * Perl * multithreading * BioPerl library * ~~`DBD::SQLite`~~ Removed in version 1.4 * Quicktree ### Environment #### Environment variables For most Linux OSs, you will need to set up your environment like this: export PATH=$HOME/bin:$PATH export PERL5LIB=$PERL5LIB:$HOME/lib/perl5 #### System packages Some system packages are needed. On Ubuntu, this is how you might install these packages. apt update apt install -y build-essential cpanminus libexpat1-dev wget #### Perl packages There are some perl packages too and so this is how you would install those on most Linux OSs: cpanm -l ~ --notest BioPerl Bio::Sketch::Mash #### Quicktree mkdir -pv $HOME/bin/build cd $HOME/bin/build wget https://github.com/khowe/quicktree/archive/v2.5.tar.gz tar xvf v2.5.tar.gz cd quicktree-2.5 make mv quicktree $HOME/bin/ #### Mash mkdir -pv $HOME/bin/build cd $HOME/bin/build wget https://github.com/marbl/Mash/releases/download/v2.2/mash-Linux64-v2.2.tar tar xvf mash-Linux64-v2.2.tar mv -v mash-Linux64-v2.2/mash $HOME/bin/ ## Mashtree Installation from CPAN Installing from CPAN installs the latest stable version of Mashtree. This method adds the Mashtree perl modules to the correct place in your home directory and adds the executables to your home bin directory. export PERL5LIB=$PERL5LIB:$HOME/lib/perl5 cpanm -l ~ Mashtree mashtree --help # verify it shows usage and not an error ## Uninstallation from CPAN cpanm --uninstall Mashtree --local-lib=$HOME ## Other sources for Mashtree Other instances of Mashtree can be found in the wild. Although I did not create these, others have found them useful. I cannot provide support for these outside instances. ### Docker * https://hub.docker.com/r/bioedge/mashtree * https://hub.docker.com/r/staphb/mashtree ### Conda * https://anaconda.org/bioconda/mashtree ================================================ FILE: docs/TIPS.md ================================================ # Tips and tricks ## Leveraging a high performance computer for bootstrapping Bootstrapping is simply running mashtree several times and then quantifying how often a parent node occurs for each group of genomes. This frequency is usually displayed as a percentage at each node. ### Bootstrap trees Here is a bash loop to make 100 bootstrap trees mkdir bootstrapTrees export FASTA=$(ls *.fasta) # Run the first tree without using --seed, to form your actual tree # without bootstraps, using many threads so that it can be done before # any of the bootstrap trees. echo 'mashtree --numcpus $NSLOTS --mindepth 0 $FASTA' | \ qsub -V -cwd -pe smp 1-36 -N mashtree -o mashtree.dnd -e mashtree.log # Make 100 bootstrap trees for i in `seq 1 100`; do # The echo statement makes a short bash script, which # is piped to qsub echo ' mashtree --numcpus $NSLOTS --mindepth 0 --seed $RANDOM $FASTA' |\ qsub -V -cwd -pe smp 1-12 -N mashtree$i -o bootstrapTrees/$i.dnd -e bootstrapTrees/$i.log; done; ### Combine bootstrap trees #### Bootstrapping with RAxML RAxML is not a prerequisite for Mashtree, and so it might not be installed on your computer even if Mashtree is working. cat bootstrapTrees/*.dnd > bootstrapTrees.dnd raxmlHPC -f b -t mashtree.dnd -z bootstrapTrees.dnd -m GTRCAT -n TEST # output file: RAxML_bipartitions.TEST ln -sv RAxML_bipartitions.TEST bsTree.dnd # in case you want to have a sane file extension #### Bootstrapping with BioPerl This script requires bioperl, which is a prerequisite for Mashtree. perl -MBio::TreeIO -MBio::Tree::Statistics -e ' # Gather all the trees into an array for my $file(glob("bootstrapTrees/*.dnd")){ next if(! -s $file); my $in=Bio::TreeIO->new(-file=>$file); while($tree = $in->next_tree){ push(@tree, $tree); } } # Combine trees my $baseTree = Bio::TreeIO->new(-file=>"mashtree.dnd")->next_tree; $stats=Bio::Tree::Statistics->new; my $bsTree = $stats->assess_bootstrap(\@tree,$baseTree); # BioPerl is very respectful about which values are IDs vs which are # bootstraps. However, most tree drawing programs look at IDs, so # we have to alter the ID for ($bsTree->get_nodes){ next if($_->is_Leaf); # Do not alter leaves $_->id($_->bootstrap); } print $bsTree->as_text("newick") ."\n"; ' > bsTree.dnd ================================================ FILE: docs/TROUBLESHOOTING.md ================================================ # Troubleshooting ## General Here are some general tips for when things go wrong. ### Read integrity Your reads might be badly formatted or the number of reads might be too low. You can optionally download [Fasten](https://github.com/lskatz/fasten) and install it separately. Then, run `fasten_validate --help` or `fasten_inspect --help` and follow instructions on how to check each read set. ================================================ FILE: lib/Mashtree/Db.pm ================================================ #!/usr/bin/env perl package Mashtree::Db; use strict; use warnings; use Exporter qw(import); use File::Basename qw/fileparse basename dirname/; use List::Util qw/shuffle/; use Data::Dumper; use lib dirname($INC{"Mashtree/Db.pm"}); use lib dirname($INC{"Mashtree/Db.pm"})."/.."; use Mashtree qw/_truncateFilename logmsg sortNames/; our @EXPORT_OK = qw( ); local $0=basename $0; =pod =head1 NAME Mashtree::Db - functions for Mashtree databasing =head1 SYNOPSIS use strict; use warnings use Mashtree::Db; my $dbFile = "mashtree.tsv"; my $db=Mashtree::Db->new($dbFile); # Add 10 distances from genome "test" to other genomes my %distHash; for(my $dist=0;$dist<10;$dist++){ my $otherGenome = "genome" . $dist; $distHash{"test"}{$otherGenome} = $dist; } $db->addDistancesFromHash(\$distHash); my $firstDistance = $db->findDistance("test", "genome0"); # => 0 =head1 DESCRIPTION This is a helper module, usually not used directly. This is how Mashtree reads and writes to the internal database. =cut =pod =head1 METHODS =over =item Mashtree::Db->new($dbFile, \%settings) Create a new Mashtree::Db object. The database file is a tab-separated file and will be created if it doesn't exist. If it does exist, then it will be read into memory. Arguments: * $dbFile - a file path * $settings - a hash of key/values (currently unused) =back =cut # Properties of this object: # dbFile # cache # settings, a hashref with keys: # significant_figures, how many sigfigs in mash distances default:10 sub new{ my($class,$dbFile,$settings)=@_; # How many significant digits to go into the mash dists $$settings{significant_figures} ||= 10; my $self={ _significant_figures => $$settings{significant_figures}, cache => {}, }; bless($self,$class); $self->selectDb($dbFile); $self->readDatabase(); return $self; } =pod =over =item $db->selectDb Selects a database. If it doesn't exist, then it will be created. Then, it sets the object property `dbFile` to the file path. =back =cut # Create a database from a TSV # Returns 1 if created a new database sub selectDb{ my($self, $dbFile)=@_; $self->{dbFile}=$dbFile; if(!-e $dbFile){ open(my $fh, '>', $dbFile) or die "ERROR: could not write to $dbFile: $!"; print $fh join("\t", qw(genome1 genome2 distance))."\n"; close $fh; } return 1; } =pod =over =item $db->readDatabase Reads the database from the dbFile set by `selectDb`. Returns a hash of distances, e.g., genome1 => {genome2=>dist} Then, this hash of distances is set in the object property `cache`. =back =cut sub readDatabase{ my($self) = @_; my %dist; open(my $fh, "<", $self->{dbFile}) or die "ERROR: could not read from ".$self->{dbFile}.": $!"; my $header = <$fh>; chomp($header); my @header = split(/\t/, $header); my $numHeaders = @header; while(my $line = <$fh>){ chomp($line); my @F = split(/\t/, $line); my %F; for(my $i=0;$i<$numHeaders;$i++){ $F{$header[$i]} = $F[$i]; } $dist{$F{genome1}}{$F{genome2}} = $F{distance}; } $self->{cache} = \%dist; return \%dist; } =pod =over =item addDistancesFromHash Add distances from a perl hash, $distHash $distHash is { genome1 => {$genome2 => $dist} } =back =cut sub addDistancesFromHash{ my($self,$distHash)=@_; my $numInserted=0; # how many are going to be inserted? open(my $fh, ">>", $self->{dbFile}) or die "ERROR: could not append to ".$self->{dbFile}.": $!"; for my $genome1(sort keys(%$distHash)){ for my $genome2(sort keys(%{ $$distHash{$genome1} })){ my $dist = $$distHash{$genome1}{$genome2}; if(!defined($dist)){ logmsg "WARNING: distance was not defined for $genome1 <=> $genome2"; logmsg " Setting distance to 1."; $dist = 1; } print $fh join("\t", $genome1, $genome2, $dist)."\n"; $numInserted++; } } close $fh; return $numInserted; } =pod =over =item $db->addDistances Add distances from a TSV file. TSV file should be a mash distances tsv file and is in the format of, e.g., # query t/lambda/sample1.fastq.gz t/lambda/sample2.fastq.gz 0.059 t/lambda/sample3.fastq.gz 0.061 =back =cut sub addDistances{ my($self,$distancesFile)=@_; # update the cache just in case # it helps us skip some redundancies. $self->readDatabase(); my $numInserted=0; # how many are going to be inserted? open(my $inFh, "<", $distancesFile) or die "ERROR: could not read $distancesFile: $!"; open(my $outFh, ">>", $self->{dbFile}) or die "ERROR: could not append to ".$self->{dbFile}.": $!"; my $query = ""; while(<$inFh>){ chomp; if(/^#\s*query\s+(.+)/){ $query=$1; $query=~s/^\s+|\s+$//g; # whitespace trim before right-padding is added $query=_truncateFilename($query); next; } die "ERROR: query was not stated in the mash dist -t output: $distancesFile" if(!$query); my($subject,$distance)=split(/\t/,$_); $subject=~s/^\s+|\s+$//g; # whitespace trim before right-padding is added $subject=_truncateFilename($subject); next if(defined($self->findDistance($query,$subject))); print $outFh join("\t", $query, $subject, $distance)."\n"; $numInserted++; } close $outFh; close $inFh; return $numInserted; } =pod =over =item $db->findDistance Find the distance between any two genomes. Return undef if not found. =back =cut sub findDistance{ my($self, $query, $subject) = @_; my $dists = $self->{cache}; if(defined($$dists{$query}{$subject})){ return $$dists{$query}{$subject}; } if(defined($$dists{$subject}{$query})){ return $$dists{$subject}{$query}; } return undef; } =pod =over =item $db->findDistances Find all distances from one genome to all others Return undef if not found. =back =cut sub findDistances{ my($self, $query) = @_; my $dists = $self->{cache}; return $$dists{$query}; } =pod =over =item $db->toString Turn the database into a string representation. Arguments: * genomeArray - list of genomes to include, or undef for all genomes * format - can be a string of one of these values: * tsv 3-column format (default) * matrix all-vs all tsv format * phylip Phylip matrix format * sortBy - can be: * abc (default) * rand =back =cut sub toString{ my($self,$genome,$format,$sortBy)=@_; $format//="tsv"; $format=lc($format); $sortBy//="abc"; $sortBy=lc($sortBy); if($format eq "tsv"){ return $self->toString_tsv($genome,$sortBy); } elsif($format eq "matrix"){ return $self->toString_matrix($genome,$sortBy); } elsif($format eq "phylip"){ return $self->toString_phylip($genome,$sortBy); } die "ERROR: could not format ".ref($self)." as $format."; } sub toString_matrix{ my($self,$genome,$sortBy)=@_; my %distance=$self->toString_tsv($genome,$sortBy); my @name=keys(%distance); my $numNames=@name; my $str=join("\t",".",@name)."\n"; for(my $i=0;$i<$numNames;$i++){ $str.=$name[$i]; for(my $j=0;$j<$numNames;$j++){ my $dist = $distance{$name[$i]}{$name[$j]} || $distance{$name[$j]}{$name[$i]}; $str.="\t$dist"; } $str.="\n"; } return $str; } # Return the database in a TSV formatted file along with a header, # or if you ask for an array/hash, then it will return a hash of genome1=>{genome2=>dist} sub toString_tsv{ my($self,$genome,$sortBy)=@_; $sortBy||="abc"; $genome||=[]; my $dbh=$self->{dbh}; # Index the genome array my %genome; $genome{_truncateFilename($_)}=1 for(@$genome); my %dist; open(my $fh, "<", $self->{dbFile}) or die "ERROR: could not read ".$self->{dbFile}.": $!"; my $str = <$fh>; my @header = split(/\t/, $str); chomp(@header); while(my $line = <$fh>){ $str .= $line; my %d; chomp($line); my @F = split(/\t/, $line); @d{@header} = @F; $dist{$d{genome1}}{$d{genome2}}=$d{distance}; } return %dist if(wantarray); return $str; } sub toString_phylip{ my($self,$genome,$sortBy)=@_; my %dist = $self->toString_tsv(); $sortBy||="abc"; $genome||=[]; my $sigfigs = $$self{_significant_figures} || die "INTERNAL ERROR: could not set sigfigs"; # Index the genome array my %genome; $genome{_truncateFilename($_)}=1 for(@$genome); my $str=""; # The way phylip is, I need to know the genome names # a priori. Get the genome names from the db. my @rawName = sort{$a cmp $b} keys(%dist); my $maxGenomeLength=0; my @name; for my $name(@rawName){ # If the parameter was given to filter genome names, # do it here. my $truncatedName = _truncateFilename($name); next if(@$genome && !$genome{$truncatedName}); push(@name,$name); $maxGenomeLength=length($name) if(length($name) > $maxGenomeLength); } # We are already sorted alphabetically, so just worry # about whether or not we sort by random if($sortBy eq 'rand'){ @name = shuffle(@name); } my $numGenomes=@name; $str.=(" " x 4) . "$numGenomes\n"; for(my $i=0;$i<$numGenomes;$i++){ # sanitize sample names by removing spaces my $sanitizedName = $name[$i]; $sanitizedName =~ s/\s/_/g; $str.=$sanitizedName; $str.=" " x ($maxGenomeLength - length($name[$i]) + 2); my $distanceHash = $dist{$name[$i]}; for(my $j=0;$j<$numGenomes;$j++){ if(!defined($$distanceHash{$name[$j]})){ logmsg "WARNING: could not find distance for $name[$i] and $name[$j]"; } $str.=sprintf("%0.${sigfigs}f ",$$distanceHash{$name[$j]}); } $str=~s/ +$/\n/; # replace that trailing whitespace with a newline } return $str; } 1; # gotta love how we we return 1 in modules. TRUTH!!! ================================================ FILE: lib/Mashtree/Mash.pm ================================================ #!/usr/bin/env perl package Mashtree::Mash; use strict; use warnings; use Exporter qw(import); use File::Basename qw/fileparse basename dirname/; use Data::Dumper; use lib dirname($INC{"Mashtree/Mash.pm"}); use lib dirname($INC{"Mashtree/Mash.pm"})."/.."; use Mashtree qw/_truncateFilename logmsg/; use JSON qw/from_json/; use Bio::Seq; use Bio::SimpleAlign; use Bio::Align::DNAStatistics; use Bio::Tree::DistanceFactory; our @EXPORT_OK = qw( ); local $0=basename $0; # Properties of this object: sub new{ my($class,$file,$settings)=@_; if(ref($file) ne 'ARRAY'){ die "ERROR: parameter \$file must be a list of file(s)"; } my $self={ file=>$file, info=>{}, hashes=>{}, distance=>{}, }; bless($self,$class); # Gather info from each file. $self->{info} and # $self->{hashes} gets updated. for my $f(@$file){ $self->info($f); } # TODO are all mash metadata compatible? Else: ERROR return $self; } sub info{ my($self,$msh)=@_; my %info = %{ $self->{info} }; my $mashInfo=from_json(`mash info -d $msh`); for my $sketch(@{ $$mashInfo{sketches} }){ #delete($$sketch{hashes}); $info{$$sketch{name}}=$sketch; my %sketchHash; for my $pos(@{ $$sketch{hashes} }){ $$self{hashes}{$pos}++; $sketchHash{$pos}=1; } $info{$$sketch{name}}{hashes}=\%sketchHash; } $self->{info}=\%info; return \%info; } sub mashDistances{ my($self)=@_; my @file=@{$self->{file}}; my $numFiles=@file; my %distance; for(my $i=0;$i<$numFiles;$i++){ for(my $j=0;$j<$numFiles;$j++){ open(my $fh, "mash dist '$file[$i]' '$file[$j]' | ") or die "ERROR: running mash dist on $file[$i] and $file[$j]: $!"; while(<$fh>){ chomp; my($genome1,$genome2,$dist,$p,$sharedFraction)=split(/\t/,$_); $distance{$genome1}{$genome2}=$dist; } close $fh; } } $self->{distance}=\%distance; return \%distance; } # TODO make sets of alignments and also report the # mash distance between them all. sub alignment{ my($self)=@_; # print an A for present, a T for not present my %nt=(present=>"A",absent=>"T"); my $aln=Bio::SimpleAlign->new(); for my $name(keys(%{ $self->{info} })){ my $sketch=$$self{info}{$name}; my $sequence=""; for my $hash(sort {$a<=>$b} keys(%{ $self->{hashes} })){ if($$sketch{hashes}{$hash}){ $sequence.= $nt{present}; } else { $sequence.= $nt{absent}; } } my $seq=Bio::LocatableSeq->new(-id=>$name, -seq=>$sequence); $aln->add_seq($seq); } return $aln; } sub refinedTree{ my($self,$aln)=@_; my $dfactory = Bio::Tree::DistanceFactory->new(-method=>"NJ"); my $stats = Bio::Align::DNAStatistics->new; my $matrix = $stats->distance(-method=>'Kimura', -align => $aln); my $tree = $dfactory->make_tree($matrix); #print $tree->as_text()."\n"; return $tree; } 1; # gotta love how we we return 1 in modules. TRUTH!!! ================================================ FILE: lib/Mashtree.pm ================================================ #!/usr/bin/env perl package Mashtree; use strict; use warnings; use Exporter qw(import); use File::Basename qw/fileparse basename dirname/; use Data::Dumper; use List::Util qw/shuffle/; use Scalar::Util qw/looks_like_number/; use threads; use threads::shared; use lib dirname($INC{"Mashtree.pm"}); use Bio::Matrix::IO; use Bio::TreeIO; our @EXPORT_OK = qw( logmsg openFastq _truncateFilename distancesToPhylip createTreeFromPhylip sortNames treeDist mashDist mashHashes raw_mash_distance raw_mash_distance_unequal_sizes @fastqExt @fastaExt @bamExt @vcfExt @richseqExt @mshExt $MASHTREE_VERSION ); local $0=basename $0; =pod =head1 NAME Mashtree =head1 SYNOPSIS Helps run a mashtree analysis to make rapid trees for genomes. Please see github.com/lskatz/Mashtree for more information. =over =item mashtree executables This document covers the Mashtree library, but the highlight the mashtree package is the executable `mashtree`. See github.com/lskatz/Mashtree for more information. Fast method: mashtree --numcpus 12 *.fastq.gz [*.fasta] > mashtree.dnd More accurate method: mashtree --mindepth 0 --numcpus 12 *.fastq.gz [*.fasta] > mashtree.dnd Bootstrapping and jackknifing mashtree_bootstrap.pl --reps 100 --numcpus 12 *.fastq.gz -- --min-depth 0 > mashtree.jackknife.dnd mashtree_jackknife.pl --reps 100 --numcpus 12 *.fastq.gz -- --min-depth 0 > mashtree.jackknife.dnd =back =head1 VARIABLES =over =item $VERSION =item $MASHTREE_VERSION (same value as $VERSION) =item @fastqExt = qw(.fastq.gz .fastq .fq .fq.gz) =item @fastaExt = qw(.fasta .fna .faa .mfa .fas .fsa .fa) =item @bamExt = qw(.sorted.bam .bam) =item @vcfExt = qw(.vcf.gz .vcf) =item @mshExt = qw(.msh) =item @richseqExt = qw(.gb .gbank .genbank .gbk .gbs .gbf .embl .ebl .emb .dat .swiss .sp) =item $fhStick :shared Used to mark whether a file is being read, so that Mashtree limits disk I/O =back =cut ###### # CONSTANTS our $VERSION = "1.4.6"; our $MASHTREE_VERSION=$VERSION; our @fastqExt=qw(.fastq.gz .fastq .fq .fq.gz); our @fastaExt=qw(.fasta .fna .faa .mfa .fas .fsa .fa); our @bamExt=qw(.sorted.bam .bam); our @vcfExt=qw(.vcf.gz .vcf); our @mshExt=qw(.msh); # Richseq extensions were obtained mostly from bioperl under # the genbank, embl, and swissprot entries, under # the source for Bio::SeqIO our @richseqExt=qw(.gb .gbank .genbank .gbk .gbs .gbf .embl .ebl .emb .dat .swiss .sp); # Helpful things my $fhStick :shared; # A thread can only open a fastq file if it has the talking stick. =head1 METHODS =over =item $SIG{'__DIE__'} Remakes how `die` works, so that it references the caller =cut ################################################# ### COMMON SUBS/TOOLS (not object subroutines) ## ################################################# # Redefine how scripts die $SIG{'__DIE__'} = sub { local $0=basename($0); my $e = $_[0] || ""; my $callerSub=(caller(1))[3] || (caller(0))[3] || "UnknownSub"; $e =~ s/(at [^\s]+? line \d+\.$)/\nStopped $1/; die("$0: $callerSub: $e"); }; =pod =item logmsg Prints a message to STDERR with the thread number and the program name, with a trailing newline. =cut # Centralized logmsg #sub logmsg {print STDERR "$0: ".(caller(1))[3].": @_\n";} sub logmsg { local $0=basename $0; my $parentSub=(caller(1))[3] || (caller(0))[3]; $parentSub=~s/^main:://; # Find the thread ID and stringify it my $tid=threads->tid; $tid=($tid) ? "(TID$tid)" : ""; my $msg="$0: $parentSub$tid: @_\n"; print STDERR $msg; } =pod =item openFastq Opens a fastq file in a thread-safe way. =cut sub openFastq{ my($fastq,$settings)=@_; my $fh; lock($fhStick); my @fastqExt=qw(.fastq.gz .fastq .fq.gz .fq); my($name,$dir,$ext)=fileparse($fastq,@fastqExt); if($ext =~/\.gz$/){ open($fh,"zcat $fastq | ") or die "ERROR: could not open $fastq for reading!: $!"; } else { open($fh,"<",$fastq) or die "ERROR: could not open $fastq for reading!: $!"; } return $fh; } =pod =item _truncateFilename Removes fastq extension, removes directory name, =cut sub _truncateFilename{ my($file,$settings)=@_; # strip off msh and any other known extentions my $name=$file; my $oldName=""; # Strip until we get convergence while($name ne $oldName){ $oldName = $name; $name = basename($name,@mshExt,@fastqExt,@richseqExt,@fastaExt); } return $name; } =pod =item distancesToPhylip 1. Read the mash distances 2. Create a phylip file Arguments: hash of distances, output directory, settings hash =cut sub distancesToPhylip{ my($distances,$outdir,$settings)=@_; my $phylip = "$outdir/distances.phylip"; # NOTE: need to regenerate the combined distances each time # because I need to allow variation in the input samples. #return $phylip if(-e $phylip); # The way phylip is, I need to know the genome names # a priori my %name; open(MASHDIST,"<",$distances) or die "ERROR: could not open $distances for reading: $!"; while(){ next if(/^#/); my($name)=split(/\t/,$_); $name=~s/^\s+|\s+$//g; # whitespace trim before right-padding is added $name{_truncateFilename($name,$settings)}=1; } close MASHDIST; my @name=sortNames([keys(%name)],$settings); # Index the array my $columnIndex=0; for(@name){ $name{$_}=$columnIndex++; } # Load up the matrix object logmsg "Reading the distances file at $distances"; open(MASHDIST,"<",$distances) or die "ERROR: could not open $distances for reading: $!"; my $query="UNKNOWN"; # Default ID in case anything goes wrong my @m; while(){ chomp; if(/^#query\s+(.+)/){ $query=_truncateFilename($1,$settings); } else { my ($reference,$distance)=split(/\t/,$_); $reference=_truncateFilename($reference,$settings); $distance=sprintf("%0.10f",$distance); $m[$name{$query}][$name{$reference}]=$distance; $m[$name{$reference}][$name{$query}]=$distance; } } close MASHDIST; #my $matrixObj=Bio::Matrix::Generic->new(-rownames=>\@name,-colnames=>\@name,-values=>\@m); # taking this method from write_matrix in http://cpansearch.perl.org/src/CJFIELDS/BioPerl-1.6.924/Bio/Matrix/IO/phylip.pm my $str; $str.=(" " x 4) . scalar(@name)."\n"; for(my $i=0;$i<@name;$i++){ $str.=$name[$i]; my $count=0; for(my $j=0;$j<@name;$j++){ if($count < $#name){ $str.=$m[$i][$j]. " "; } else { $str.=$m[$i][$j]; } $count++; } $str.="\n"; } open(PHYLIP,">",$phylip) or die "ERROR: could not write to $phylip: $!"; print PHYLIP $str; close PHYLIP; return $phylip; } =pod =item sortNames Sorts names. Arguments: 1. $name - array of names 2. $settings - options * $$settings{'sort-order'} is either "abc", "random", "input-order" =cut sub sortNames{ my($name,$settings)=@_; my @sorted; if($$settings{'sort-order'} =~ /^(abc|alphabet)$/){ @sorted=sort { $a cmp $b } @$name; } elsif($$settings{'sort-order'}=~/^rand(om)?/){ @sorted=shuffle(@$name); } elsif($$settings{'sort-order'} eq 'input-order'){ @sorted=@$name; } else { die "ERROR: I don't understand sort-order $$settings{'sort-order'}"; } return @sorted; } =pod =item createTreeFromPhylip($phylip, $outdir, $settings) Create tree file with Quicktree but bioperl as a backup. =cut sub createTreeFromPhylip{ my($phylip,$outdir,$settings)=@_; my $treeObj; my $quicktreePath=`which quicktree 2>/dev/null`; # bioperl if there was an error with which quicktree if($?){ logmsg "DEPRECATION WARNING: CANNOT FIND QUICKTREE IN YOUR PATH. I will use BioPerl to make the tree this time, but it will be removed in the next version."; #logmsg "Creating tree with BioPerl"; my $dfactory = Bio::Tree::DistanceFactory->new(-method=>"NJ"); my $matrix = Bio::Matrix::IO->new(-format=>"phylip", -file=>$phylip)->next_matrix; $treeObj = $dfactory->make_tree($matrix); open(TREE,">","$outdir/tree.dnd") or die "ERROR: could not open $outdir/tree.dnd: $!"; print TREE $treeObj->as_text("newick"); print TREE "\n"; close TREE; } # quicktree else { #logmsg "Creating tree with QuickTree"; system("quicktree -in m $phylip > $outdir/tree.dnd.tmp"); die "ERROR with quicktree" if $?; $treeObj=Bio::TreeIO->new(-file=>"$outdir/tree.dnd.tmp")->next_tree; open(my $treeFh, ">", "$outdir/tree.dnd") or die "ERROR: could not write to $outdir/tree.dnd: $!"; print $treeFh $treeObj->as_text("newick")."\n"; #my $outtree=Bio::TreeIO->new(-file=>">$outdir/tree.dnd", -format=>"newick"); #$outtree->write_tree($treeObj); unlink("$outdir/tree.dnd.tmp"); } return $treeObj; } =pod =item treeDist($treeObj1, $treeObj2) Lee's implementation of a tree distance. The objective is to return zero if two trees are the same. =cut sub treeDist{ my($treeObj1,$treeObj2)=@_; # If the tree objects are really strings, then make Bio::Tree::Tree objects if(!ref($treeObj1)){ if(-e $treeObj1){ # if this is a file, get the contents $treeObj1=`cat $treeObj1`; } $treeObj1=Bio::TreeIO->new(-string=>$treeObj1)->next_tree; } if(!ref($treeObj2)){ if(-e $treeObj2){ # if this is a file, get the contents $treeObj2=`cat $treeObj2`; } $treeObj2=Bio::TreeIO->new(-string=>$treeObj2)->next_tree; } for($treeObj1,$treeObj2){ #$_->force_binary; } # Get all leaf nodes so that they can be compared my @nodes1=sort {$a->id cmp $b->id} grep{$_->is_Leaf} $treeObj1->get_nodes; my @nodes2=sort {$a->id cmp $b->id} grep{$_->is_Leaf} $treeObj2->get_nodes; my $numNodes=@nodes1; # Test 1: are these the same nodes? my $nodeString1=join(" ",map{$_->id} @nodes1); my $nodeString2=join(" ",map{$_->id} @nodes2); if($nodeString1 ne $nodeString2){ # TODO print out the differing nodes? logmsg "ERROR: nodes are not the same in both trees!\n $nodeString1\n $nodeString2"; return ~0; #largest int } # Find the number of branches it takes to get to each node. # Turn it into a Euclidean distance my $euclideanDistance=0; for(my $i=0;$i<$numNodes;$i++){ for(my $j=$i+1;$j<$numNodes;$j++){ my ($numBranches1,$numBranches2); my $lca1=$treeObj1->get_lca($nodes1[$i],$nodes1[$j]); my $lca2=$treeObj2->get_lca($nodes2[$i],$nodes2[$j]); # Distance in tree1 my $distance1=0; my @ancestory1=reverse $treeObj1->get_lineage_nodes($nodes1[$i]); my @ancestory2=reverse $treeObj1->get_lineage_nodes($nodes1[$j]); for my $currentNode(@ancestory1){ $distance1++; last if($currentNode eq $lca1); } for my $currentNode(@ancestory2){ $distance1++; last if($currentNode eq $lca1); } # Distance in tree2 my $distance2=0; my @ancestory3=reverse $treeObj2->get_lineage_nodes($nodes2[$i]); my @ancestory4=reverse $treeObj2->get_lineage_nodes($nodes2[$j]); for my $currentNode(@ancestory3){ $distance2++; last if($currentNode eq $lca2); } for my $currentNode(@ancestory4){ $distance2++; last if($currentNode eq $lca2); } if($distance1 != $distance2){ logmsg "These two nodes do not have the same distance between trees: ".$nodes1[$i]->id." and ".$nodes1[$j]->id; } # Add up the Euclidean distance $euclideanDistance+=($distance1 - $distance2) ** 2; } } $euclideanDistance=sqrt($euclideanDistance); return $euclideanDistance; } =pod =item mashDist($file1, $file2, $k, $settings) Find the distance between two mash sketch files Alternatively: two hash lists. =cut sub mashDist{ my($file1, $file2, $k, $settings)=@_; my($hashes1, $hashes2, $kmer1, $kmer2); if(ref($file1) eq 'ARRAY'){ $hashes1 = $file1; $kmer1 = -1; } else { ($hashes1, $kmer1) = mashHashes($file1); } if(ref($file2) eq 'ARRAY'){ $hashes2 = $file2; $kmer2 = -1; } else { ($hashes2, $kmer2) = mashHashes($file2); } if($kmer1 ne $kmer2){ die "ERROR: kmer lengths do not match($kmer1 vs $kmer2)"; } # Set the default kmer length and perform sanity check $k ||= $kmer1; if(!looks_like_number($k)){ die "ERROR: k was not set to an integer"; } if($k < 1){ die "ERROR: k was undefined or set to less than 1"; } my($common, $total) = (0,0); if(scalar(@$hashes1) != scalar(@$hashes2)){ ($common, $total) = raw_mash_distance_unequal_sizes($hashes1, $hashes2); } else { ($common, $total) = raw_mash_distance($hashes1, $hashes2); } my $jaccard = $common/$total; my $mash_distance = -1/$k * log(2*$jaccard / (1+$jaccard)); #logmsg "========== $mash_distance = -1/$k * log(2*$jaccard / (1+$jaccard)) =============="; return $mash_distance; } =pod =item mashHashes($sketch) Return an array of hashes, the kmer length, and the genome estimated length =cut sub mashHashes{ my($sketch)=@_; my @hash; my $length = 0; my $kmer = 0; if(!-e $sketch){ die "ERROR: file not found: $sketch"; } my $fh; if($sketch =~ /\.msh$/){ open($fh, "mash info -d $sketch | ") or die "ERROR: could not run mash info -d on $sketch: $!"; } elsif($sketch =~ /\.json$/){ open($fh, $sketch) or die "ERROR: could not read $sketch: $!"; } while(<$fh>){ if(/kmer\D+(\d+)/){ $kmer = $1; } elsif(/length\D+(\d+)/){ $length = $1; } elsif(/hashes/){ while(<$fh>){ last if(/\]/); next if(!/\d/); s/\D+//g; s/^\s+|\s+$//g; push(@hash, $_); } } } if(!@hash){ die "ERROR: no hashes found in $sketch"; } return (\@hash, $kmer, $length); } =pod =item raw_mash_distance_unequal_sizes($hashes1, $hashes2) Compare unequal sized hashes. Treat the first set of hashes as the reference (denominator) set. =cut sub raw_mash_distance_unequal_sizes{ my($hashes1, $hashes2) = @_; my (%sketch1,%sketch2); @sketch1{@$hashes1} = (1) x scalar(@$hashes1); @sketch2{@$hashes2} = (1) x scalar(@$hashes2); my %union; for my $h(@$hashes1){ if($sketch2{$h}){ $union{$h}++; } } my $common = scalar(keys(%union)); my $total = scalar(@$hashes1); return($common,$total); } =pod =item raw_mash_distance($hashes1, $hashes2) Return the number of kmers in common and the number compared total. inspiration from https://github.com/onecodex/finch-rs/blob/master/src/distance.rs#L34 =cut sub raw_mash_distance{ my($hashes1, $hashes2) = @_; my @sketch1 = sort {$a <=> $b} @$hashes1; my @sketch2 = sort {$a <=> $b} @$hashes2; my $i = 0; my $j = 0; my $common = 0; my $total = 0; my $sketch_size = @sketch1; while($total < $sketch_size && $i < @sketch1 && $j < @sketch2){ my $ltgt = ($sketch1[$i] <=> $sketch2[$j]); # -1 if sketch1 is less than, +1 if sketch1 is greater than if($ltgt == -1){ $i += 1; } elsif($ltgt == 1){ $j += 1; } elsif($ltgt==0) { $i += 1; $j += 1; $common += 1; } else { die "Internal error"; } $total += 1; } if($total < $sketch_size){ if($i < @sketch1){ $total += @sketch1 - 1; } if($j < @sketch2){ $total += @sketch2 - 1; } if($total > $sketch_size){ $total = $sketch_size; } } return ($common, $total); } # Calculates the Transfer Bootstrap Expectation (TBE) for internal nodes based on # the methods outlined in Lemoine et al, Nature, 2018. # Currently experimental. # I entered this sub into bioperl > v1.7.5 but it is worth # having locally here to help maintain compatibility. # The only difference is that it isn't an object method # and that it is called without an OO implementation. =item transfer_bootstrap_expectation Title : transfer_bootstrap_expectation Usage : my $tree_with_bs = transfer_bootstrap_expectation(\@bs_trees,$guide_tree); Function: Calculates the Transfer Bootstrap Expectation (TBE) for internal nodes based on the methods outlined in Lemoine et al, Nature, 2018. Currently experimental. Returns : L Args : Arrayref of Ls Guide tree, Ls =back =cut sub transfer_bootstrap_expectation{ my ($bs_trees,$guide_tree) = @_; if(!defined($bs_trees) || ref($bs_trees) ne 'ARRAY'){ die "ERROR: second parameter in assess_bootstrap() must be a list"; } my $num_bs_trees = scalar(@$bs_trees); if($num_bs_trees < 1){ die "ERROR: no bootstrap trees were passed to ".(caller(0))[3]; } # internal nodes are defined by their children my %internal = (); my %leafNameId = (); my @idLookup = (); my @internalLookup = (); my @tree = ($guide_tree, @$bs_trees); my $numTrees = scalar(@tree); for(my $i = 0; $i < $numTrees; $i++){ # guide tree's index is $i==0 # Do this as a top down approach, can probably be # improved by caching internal node states, but not going # to worry about it right now. my @allnodes = $tree[$i]->get_nodes; my @internalnodes = grep { ! $_->is_Leaf } @allnodes; for my $node ( @internalnodes ) { my @tips = sort map { $_->id } grep { $_->is_Leaf() } $node->get_all_Descendents; my $id = join(",", @tips); # Map the concatenated-leaf ID to the internal ID on the guide tree if( $i == 0 ) { $internal{$id} = $node->internal_id; $leafNameId{$node->internal_id} = $id; } # Record the tips for each tree's internal node # ID lookup (concatenated string of leaf names) $idLookup[$i]{$id} = \@tips; # Internal ID lookup $internalLookup[$i]{$internal{$id}} = \@tips; } } # Find the average distance from branch b to all # bootstrap trees' branches b* my @id = sort keys %internal; my $numIds = @id; # Loop through all internal nodes of the guide tree for(my $j=0; $j<$numIds; $j++){ my $refNode = $guide_tree->find_node(-internal_id => $internal{$id[$j]}); my $refNodeId = $refNode->internal_id; my $refJoinId = $leafNameId{$refNodeId}; my $refLeaves = $idLookup[0]{$refJoinId}; my %refLeafIndex = map{$_=>1} @$refLeaves; #next if(!defined($refLeaves)); # For each internal node, start calculating for # an average TBE distance. my $nodeTbeTotal = 0; # Loop through all bootstrap trees, skipping the 0th # tree which is the guide tree. for(my $i=1;$i<$numTrees;$i++){ # Find the right branch to bootstrap with. The right # branch will be the one that has the smallest # TBE distance. my @bsNode = grep {!$_->is_Leaf} $tree[$i]->get_nodes; my $numBsIds = scalar(@bsNode); my $minDistance = ~0; # large int for(my $k=0;$k<$numBsIds;$k++){ my @queryLeaves = sort map { $_->id } grep { $_->is_Leaf() } $bsNode[$k]->get_all_Descendents; my %queryLeafIndex = map{$_=>1} @queryLeaves; # How many moves does it take to go from query to ref? my $dist=0; for my $queryLeaf(@queryLeaves){ if(!$refLeafIndex{$queryLeaf}){ $dist++; } } for my $refLeaf(@$refLeaves){ if(!$queryLeafIndex{$refLeaf}){ $dist++; } } if($dist < $minDistance){ $minDistance = $dist; } } $nodeTbeTotal += $minDistance; } my $avgTbe = $nodeTbeTotal / $numTrees; # Calculate the average of all b to b* distances # But it is also 1 - average. my $numRefLeaves = scalar(@$refLeaves); my $nodeTbe = 1 - $avgTbe/$numRefLeaves; # Round to an integer $refNode->bootstrap(sprintf("%0.0f",100 * $nodeTbe)); } return $guide_tree; } 1; __END__ ================================================ FILE: misc/.dummy ================================================ ================================================ FILE: paper/paper.bib ================================================ @article{Ondov:2016, title={Mash: fast genome and metagenome distance estimation using MinHash}, author={Ondov, Brian D and Treangen, Todd J and Melsted, P{\'a}ll and Mallonee, Adam B and Bergman, Nicholas H and Koren, Sergey and Phillippy, Adam M}, journal={Genome Biology}, volume={17}, number={1}, pages={132}, year={2016}, publisher={BioMed Central}, doi={10.1186/s13059-016-0997-x} } @article{Saitou:1987, title={The neighbor-joining method: a new method for reconstructing phylogenetic trees.}, author={Saitou, Naruya and Nei, Masatoshi}, journal={Molecular Biology and Evolution}, volume={4}, number={4}, pages={406--425}, year={1987}, doi={10.1093/oxfordjournals.molbev.a040454} } @article{Howe:2002, title={QuickTree: building huge Neighbour-Joining trees of protein sequences}, author={Howe, Kevin and Bateman, Alex and Durbin, Richard}, journal={Bioinformatics}, volume={18}, number={11}, pages={1546--1547}, year={2002}, publisher={Oxford University Press}, doi={10.1093/bioinformatics/18.11.1546} } @article{Harris:2018, title={SKA: Split Kmer Analysis Toolkit for Bacterial Genomic Epidemiology}, author={Harris, Simon R}, journal={BioRxiv}, pages={453142}, year={2018}, publisher={Cold Spring Harbor Laboratory}, doi={10.1101/453142} } @article{Gardner:2015, title={kSNP3. 0: SNP detection and phylogenetic analysis of genomes without genome alignment or reference genome}, author={Gardner, Shea N and Slezak, Tom and Hall, Barry G}, journal={Bioinformatics}, volume={31}, number={17}, pages={2877--2878}, year={2015}, publisher={Oxford University Press}, doi={10.1093/bioinformatics/btv271} } @article{Timme:2017, title={Benchmark datasets for phylogenomic pipeline validation, applications for foodborne pathogen surveillance}, author={Timme, Ruth E and Rand, Hugh and Shumway, Martin and Trees, Eija K and Simmons, Mustafa and Agarwala, Richa and Davis, Steven and Tillman, Glenn E and Defibaugh-Chavez, Stephanie and Carleton, Heather A and others}, journal={PeerJ}, volume={5}, pages={e3893}, year={2017}, publisher={PeerJ Inc.}, doi={10.7717/peerj.3893} } @article {Baker:2019, author = {Baker, Daniel N and Langmead, Ben}, title = {Dashing: Fast and Accurate Genomic Distances with HyperLogLog}, elocation-id = {501726}, year = {2019}, doi = {10.1101/501726}, publisher = {Cold Spring Harbor Laboratory}, abstract = {Dashing is a fast and accurate software tool for estimating similarities of genomes or sequencing datasets. It uses the HyperLogLog sketch together with cardinality estimation methods that are specialized for set unions and intersections. Dashing summarizes genomes more rapidly than previous MinHash-based methods while providing greater accuracy across a wide range of input sizes and sketch sizes. It can sketch and calculate pairwise distances for over 87K genomes in 6 minutes. Dashing is open source and available at https://github.com/dnbaker/dashing.}, URL = {https://www.biorxiv.org/content/early/2019/02/04/501726}, eprint = {https://www.biorxiv.org/content/early/2019/02/04/501726.full.pdf}, journal = {bioRxiv} } @article{Zhao:2018, author = {Zhao, XiaoFei}, title = "{BinDash, software for fast genome distance estimation on a typical personal laptop}", journal = {Bioinformatics}, volume = {35}, number = {4}, pages = {671-673}, year = {2018}, month = {07}, abstract = "{The number of genomes (including meta-genomes) is increasing at an accelerating pace. In the near future, we may need to estimate pairwise distances between millions of genomes. Even with the use of cloud computing, very few softwares can perform such estimation.The multi-threaded software BinDash can perform such estimation using only a typical personal laptop. BinDash implemented b-bit one-permutation rolling MinHash with optimal densification, an existing data-mining technique. BinDash empirically outperforms the state-of-the-art software in terms of precision, compression ratio, memory usage and runtime according to our evaluation. Our evaluation is performed with a Dell Inspiron 157 559 Notebook on all bacterial genomes in RefSeq.BinDash is released under the Apache 2.0 license at https://github.com/zhaoxiaofei/BinDash.Supplementary data are available at Bioinformatics online.}", issn = {1367-4803}, doi = {10.1093/bioinformatics/bty651}, url = {https://doi.org/10.1093/bioinformatics/bty651}, eprint = {http://oup.prod.sis.lan/bioinformatics/article-pdf/35/4/671/27838595/bty651.pdf}, } @article{Brown:2019, title={Use of whole-genome sequencing for food safety and public health in the United States}, author={Brown, Eric and Dessai, Uday and McGarry, Sherri and Gerner-Smidt, Peter}, journal={Foodborne Pathogens and Disease}, volume={16}, number={7}, pages={441--450}, year={2019}, publisher={Mary Ann Liebert, Inc., publishers 140 Huguenot Street, 3rd Floor New~.}, doi={10.1089/fpd.2019.2662} } @article{bloom:1970, title={Space/time trade-offs in hash coding with allowable errors}, author={Bloom, Burton H}, journal={Communications of the ACM}, volume={13}, number={7}, pages={422--426}, year={1970}, publisher={ACM}, doi={10.1145/362686.362692} } ================================================ FILE: paper/paper.md ================================================ --- title: 'Mashtree: a rapid comparison of whole genome sequence files' authors: - affiliation: "1, 2" name: Lee S. Katz orcid: 0000-0002-2533-9161 - affiliation: 1 name: Taylor Griswold - affiliation: 3 name: Shatavia S. Morrison orcid: 0000-0002-4658-5951 - affiliation: 3 name: Jason A. Caravas orcid: 0000-0001-9111-406X - affiliation: 2 name: Shaokang Zhang orcid: 0000-0003-0874-2212 - affiliation: 2 name: Henk C. den Bakker orcid: 0000-0002-4086-1580 - affiliation: 2 name: Xiangyu Deng - affiliation: 1 name: Heather A. Carleton date: "11 September 2019" output: html_document: df_print: paged pdf_document: default word_document: default bibliography: paper.bib tags: - dendrogram - mash - sketch - tree - rapid affiliations: - index: 1 name: Enteric Diseases Laboratory Branch, Centers for Disease Control and Prevention, Atlanta, GA, USA - index: 2 name: Center for Food Safety, University of Georgia, Griffin, GA, USA - index: 3 name: Respiratory Diseases Laboratory Branch, Centers for Disease Control and Prevention, Atlanta, GA, USA --- # Summary In the past decade, the number of publicly available bacterial genomes has increased dramatically. These genomes have been generated for impactful initiatives, especially in the field of genomic epidemiology [@Timme:2017; @Brown:2019]. Genomes are sequenced, shared publicly, and subsequently analyzed for phylogenetic relatedness. If two genomes of epidemiological interest are found to be related, further investigation might be prompted. However, comparing the multitudes of genomes for phylogenetic relatedness is computationally expensive and, with large numbers, laborious. Consequently, there are many strategies to reduce the complexity of the data for downstream analysis, especially using nucleotide stretches of length _k_ (kmers). One major kmer strategy is to reduce each genome to split kmers. With split kmer analysis, kmers on both sides of a variable site are recorded, and the variable nucleotide is identified. When comparing two or more genomes, the variable sites are compared. Split kmers have been implemented in software packages such as KSNP and SKA [@Gardner:2015; @Harris:2018]. Another major kmer strategy is to convert genomic data into manageable datasets, usually called sketches [@Ondov:2016; @Baker:2019; @Zhao:2018]. Most notably, an algorithm called min-hash was implemented in the Mash package [@Ondov:2016]. In the min-hash algorithm, all kmers are recorded and transformed into integers using hashing and a Bloom filter [@bloom:1970]. These hashed kmers are sorted and only the first several kmers are retained. The kmers that appear at the top of the sorted list are collectively called the sketch. Any two sketches can be compared by counting how many hashed kmers they have in common. Because min-hash creates distances between any two genomes, min-hash values can be used to rapidly cluster genomes into trees using the neighbor-joining algorithm [@Saitou:1987]. We implemented this idea in software called Mashtree, which quickly and efficiently generates large trees that would be too computationally intensive using other methods. # Implementation ## Workflow Mashtree builds on two major algorithms that are already implemented in other software packages. The first is the min-hash algorithm, which is implemented in the software Mash [@Ondov:2016]. Mashtree uses Mash to create sketches of the genomes with the function `mash sketch`. We elected to keep most default Mash parameters but increased the sketch size (number of hashed kmers) from 1,000 to 10,000 to increase discriminatory power. Then, Mash is used to calculate the distances between genomes with `mash dist`. Mashtree records these distances into a pairwise distance matrix. Next, Mashtree calls the neighbor-joining (NJ) algorithm which is implemented in the software QuickTree [@Howe:2002]. The Mash distance matrix is used with QuickTree with default options to generate a dendrogram. The workflow is depicted in Figure 1. ## Confidence values Although Mashtree does not infer phylogeny, we have borrowed the ideas behind phylogenetic confidence values to yield confidence values for each parent node in the tree. There are two resampling methods implemented in Mashtree to assign support values to internal nodes: bootstrapping and jackknifing. Initially, both methods create a tree as depicted in Figure 1. Then, confidence values can be calculated for the tree using either the bootstrapping approach or the jackknifing approach (Figures 2 and 3). ## Other features Mashtree has several other useful features. First, Mashtree can read any common sequence file type and can read gzip-compressed files (e.g., fastq, fastq.gz, fasta). This is a major advantage in being compatible with a wide variety of databases and with space-saving file compression. Second, Mashtree takes advantage of multithreading. The number of requested threads is used to determine how many genomes are sketched at the same time and how many sketches can be compared at the same time. When the number of threads requested outnumbers the number of operations that it can parallelize, Mashtree uses the multithreading already encoded in Mash sketches and distances. Third, Mashtree uses an SQLite database which can be used to cache results between runs. # Installation The Mashtree package is programmed in Perl, and is available in the CPAN repository. Documentation can be found at https://github.com/lskatz/mashtree. # Figures ![The Mashtree workflow. Step 1) Sketch genomes with Mash. In this schematic, there is a green circle representing each genome in the analysis. Filled-in brown circles indicate the presence of a kmer. Missing circles represent true absence. After hashing with a sketch size of six (after the arrow), some kmers are not represented in the Mash sketch either because they are not present in the original genome or because only a finite number of kmers are sketched (e.g., six in this example). Henceforth, truly missing hashes or hashes not included in the Mash sketch are represented by empty circles. Step 2) Calculate distances with `Mash dist`. Distances in the figure are represented by Jaccard distances, which are calculated as the intersection divided by the union. In this example, the genomes are separated by Jaccard distances of 5/9, 4/9, and 3/9. These Jaccard distances are internally transformed into Mash distances [@Ondov:2016]. Step 3) Create dendrogram with Quicktree using the Mash distance matrix. ](Mashtree_workflow.png) ![The Mashtree bootstrap workflow. Step 1) Generate a tree with the normal workflow as in Figure 1. This is the main tree. Step 2) Run the normal workflow once per replicate but with a different random seed. In this example, the top right replicate differs from the main tree. All ten of these trees are the bootstrap tree replicates. Step 3) For each parent node in the main tree, quantify how many bootstrap tree replicates have the same node with the same children. Record that percentage next to each parent node. This percentage quantifies how confident the Mashtree cluster is, controlling for the random seed in the Mash program.](Bootstrap_workflow.png) ![The Mashtree jackknife workflow. Step 1) Generate a tree with the normal workflow as in Figure 1. This is the main tree. Step 2) For each replicate, sample the half hashes without replacement for each query genome. Recalculate the Mash distance between the query genome and all other genomes, reducing the denominator to one half, rounding up, to reflect the smaller pool of hashes. After all genomes have been selected for query genomes, average the distances to create a new distance matrix. Create the dendrogram from the new distance matrix. For brevity, only one detailed replicate is shown. Step 3) For each replication, calculate the new tree from the new distance matrix. In this example, the top right replication differs from the main tree. All ten of these trees are the jackknife tree replicates. Step 4) For each parent node in the main tree, quantify how many jackknife tree replicates have the same node with the same children. Record that percentage next to each parent node. This percentage quantifies how confident Mashtree is at clustering, controlling for stochasticity in hashes. ](Jackknife_workflow.png) # Acknowledgements This work was made possible through support from the Advanced Molecular Detection (AMD) Initiative at the Centers for Disease Control and Prevention. Thank you Sam Minot, Andrew Page, Brian Raphael, and Torsten Seemann for helpful discussions. The findings and conclusions in this report are those of the authors and do not necessarily represent the official position of the Centers for Disease Control and Prevention. # References ================================================ FILE: plugins/README.md ================================================ # Mashtree plugins Mashtree plugins are executed such that a mashtree database is the first positional parameter. Flag options are allowed. Additional tables within the database might get created, but overwriting the DISTANCE table is not allowed. For more information on the standard database, please see [the documentation](../docs/SQL.md). ## Synopses for individual plugins ### mashtree_optimize This plugin optimizes distances between all genomes. Currently only uses Dijkstra's algorithm but potentially could be expanded. Usage: mashtree_optimize.pl [options] mashtree.sqlite ================================================ FILE: plugins/mashtree_optimize.pl ================================================ #!/usr/bin/env perl # Author: Lee Katz # Read a mashtree database to cluster genomes use strict; use warnings; use Data::Dumper; use Getopt::Long; use File::Temp qw/tempdir tempfile/; use File::Basename qw/basename dirname fileparse/; use File::Copy qw/mv/; use POSIX qw/floor/; use List::Util qw/min max/; use Scalar::Util qw/looks_like_number/; use DBI; use threads; use threads::shared; use FindBin; use lib "$FindBin::RealBin/../lib"; use Mashtree qw/logmsg/; use Graph::Dijkstra; local $0=basename $0; exit main(); sub main{ my $settings={}; GetOptions($settings,qw(help tempdir=s method=s threshold|cutoff=f numcpus=i nonzero=f)) or die $!; $$settings{numcpus}||=1; $$settings{nonzero}||=1e-99; $$settings{threshold}||=0.1; my($dbFile) = @ARGV; die usage() if($$settings{help} || !$dbFile); if(!$$settings{method}){ $$settings{method}="dijkstra"; logmsg "Setting method to dijkstra."; } $$settings{method}=lc($$settings{method}); $$settings{tempdir}||=tempdir("MASHTREE_OPTIMIZE_$$settings{method}.XXXXXX",CLEANUP=>1,TMPDIR=>1); if(!-e $dbFile){ die "ERROR: could not find database file $dbFile"; } optimizeDb($dbFile,$$settings{method},$settings); return 0; } # Optimize the database using the method supplied. This # subroutine will redirect the workload to whichever # specialized subroutine. sub optimizeDb{ my($dbFile,$method,$settings)=@_; no strict 'refs'; my $subroutine="optimizeDb_$method"; if(!defined(&$subroutine)){ die "ERROR: optimization method $method is not defined in this script"; } return &$subroutine($dbFile,$settings); } sub optimizeDb_dijkstra{ my($db,$settings)=@_; my $graph = Graph::Dijkstra->new(edgedefault=>"undirected"); my $dbh = DBI->connect("dbi:SQLite:dbname=$db","","",{ RaiseError => 1, AutoCommit => 0, }); logmsg "Reading the database"; my $sth = $dbh->prepare(qq(SELECT GENOME1,GENOME2,DISTANCE FROM DISTANCE WHERE GENOME1 != GENOME2 )); my $rv = $sth->execute() or die $DBI::errstr; if($rv < 0){ die "ERROR: no distances were found in the database $db"; } my $rowCounter=0; my %seenNode; # a fast hash to track whether we need to define a node while(my @row=$sth->fetchrow_array()){ my($g1,$g2,$dist)=@row; # next if($g1 eq $g2); # take care of this with SQL $dist=$$settings{nonzero} if(!$dist); if(!$seenNode{$g1}){ $graph->node({id=>$g1}); $seenNode{$g1}=1; } if(!$seenNode{$g2}){ $graph->node({id=>$g2}); $seenNode{$g2}=1; } $graph->edge({sourceID=>$g1,targetID=>$g2,weight=>$dist}); $rowCounter++; if($rowCounter % 1000000 == 0){ logmsg "$rowCounter distances read"; } } undef(%seenNode); # Create the new table $sth = $dbh->prepare(qq( DROP TABLE IF EXISTS OPTIMIZED_DISTANCE )); $sth->execute(); $sth = $dbh->prepare(qq( CREATE TABLE OPTIMIZED_DISTANCE( GENOME1 CHAR(255) NOT NULL, GENOME2 CHAR(255) NOT NULL, DISTANCE REAL NOT NULL, PRIMARY KEY(GENOME1,GENOME2) )) ); $sth->execute(); my $insertSth = $dbh->prepare("INSERT INTO OPTIMIZED_DISTANCE(GENOME1,GENOME2,DISTANCE) VALUES(?,?,?);"); $dbh->{AutoCommit}=0; my %distBuffer; # buffer for db transactions my $bufferSize=10000; # how many transactions at a time for the db my @genome=sort {$a cmp $b} map {$$_{id}} $graph->nodeList; my $numGenomes=@genome; my $distancesCounter=0; logmsg "Optimizing $numGenomes genomes"; for(my $i=0;$i<$numGenomes;$i++){ logmsg "Optimizing $genome[$i] (".($i+1)."/$numGenomes)"; for(my $j=$i+1;$j<$numGenomes;$j++){ # Sort the genome names so that we only store the pair once # in a predictable fashion. my($G1,$G2)=sort{$a cmp $b} ($genome[$i],$genome[$j]); my %solution=(originID=>$G1, destinationID=>$G2); $graph->shortestPath(\%solution); if(!$solution{edges}){ logmsg "WARNING: could not find distance between $G1 and $G2. Setting to large INT."; $solution{weight}= ~0; # largest int $solution{edges}=[]; } $distBuffer{$G1}{$G2}=$solution{weight}; $distancesCounter++; # Insert the distances if the buffer is full if($distancesCounter % $bufferSize == 0){ my $insertValues=""; while(my($G1,$distances)=each(%distBuffer)){ while(my($G2,$dist) = each(%$distances)){ $insertSth->execute($G1,$G2,$dist); } } $dbh->commit(); %distBuffer=(); # purge the buffer } } } # One last set of inserts while(my($G1,$distances)=each(%distBuffer)){ while(my($G2,$dist) = each(%$distances)){ $insertSth->execute($G1,$G2,$dist); } } $dbh->commit(); } sub usage{ "Optimizes a mashtree database using different methods Usage: $0 [options] mashtree.sqlite Note: only one optimization is available at this time --method Dijkstra Optimize distances using Dijkstra's algorithm " } ================================================ FILE: t/00_env.t ================================================ #!/usr/bin/env perl use strict; use warnings; use Test::More tests => 5; use FindBin qw/$RealBin/; use lib './lib'; use_ok 'Mashtree'; $ENV{PATH}="$RealBin/../bin:".$ENV{PATH}; # Is Mash installed? my $mash_is_missing = system("mash > mash.log 2>&1"); is $mash_is_missing, 0, "Found Mash in PATH" or diag("The executable Mash was not found in PATH: ".`cat mash.log`); END{unlink("mash.log");} # Is Quicktree installed? my $quicktree_is_missing = system("quicktree 2>&1 | grep -C 50 -i quicktree > mash.log 2>&1"); # overwrite the mash log my $quicktree_is_missing_mod = $quicktree_is_missing >> 8; # shift the exit code down 8 bits is $quicktree_is_missing_mod, 0, "Found quicktree in PATH" or diag("The executable quicktree was not found in PATH: ".`cat mash.log`); # Test out mashtree exe my $version = `mashtree --version`; my $exit_code = $? >> 8; is $exit_code, 0, "Mashtree --version exit code: $exit_code"; # If mash, quicktree, or mashtree gave an exit code, bail out of the whole test if($mash_is_missing || $quicktree_is_missing || $exit_code){ BAIL_OUT("Prerequisite software was not found"); } $version =~ s/Mashtree\s*//; # Mashtree trim $version =~ s/^\s+|\s+$//; # whitespace trim my $found_nonversion = !! ($version=~/([^\.\d])/) + 0; is($found_nonversion, 0, "Looking for version numbers in the format of 1.2.3 (Version returned was $version)"); ================================================ FILE: t/01_filetypes.t ================================================ #!/usr/bin/env perl use strict; use warnings; use lib './lib'; use File::Basename qw/dirname/; use Getopt::Long qw/GetOptions/; use Test::More tests => 2; use_ok 'Mashtree'; use Mashtree qw/treeDist/; $ENV{PATH}="./bin:$ENV{PATH}"; my $settings={}; GetOptions($settings,qw(numcpus=i)) or die $!; $$settings{numcpus}||=1; #my $correctMashtree="(CFSAN000189.gbk:0.00000,(CFSAN001112.ref:0.00001,CFSAN001140_1:0.00019):0.00001,((CFSAN000968.ref:0.00000,CFSAN001115.ref:0.00000):0.00001,(CFSAN000189.ref:0.00000,(CFSAN000191.ref:0.00002,(CFSAN000211.gbk:0.00045,CFSAN000961.gbk:0.00005):0.00003):0.00000):0.00001):0.00000);"; #my $correctMashtree="(CFSAN001112.ref:0.0000197713,(CFSAN001115.ref:0.0000107516,((CFSAN000968.ref:0.0000186460,(CFSAN000961.gbk:0.0000252712,CFSAN000211.gbk:0.0004753628):0.0000146923):0.0000077003,(CFSAN000191.ref:0.0000276279,(CFSAN000189.ref:0.0000023811,CFSAN000189.gbk:0.0000000000):0.0000033570):0.0000037416):0.0000038797):0.0000011147,CFSAN001140_1:0.0002225407);"; my $correctMashtree="(CFSAN001112.ref:0.0000197713,(CFSAN001115.ref:0.0000107516,((CFSAN000968.ref:0.0000186460,(CFSAN000961:0.0000252712,CFSAN000211:0.0004753628):0.0000146923):0.0000077003,(CFSAN000191.ref:0.0000276279,(CFSAN000189.ref:0.0000023811,CFSAN000189:0.0000000000):0.0000033570):0.0000037416):0.0000038797):0.0000011147,CFSAN001140_1:0.0002225407);"; $correctMashtree=~s/(\d+\.)(\d+)/$1 . substr($2,0,4)/ge; # global and expression # Test to see if the correct tree is made my $mashtree=`mashtree --numcpus $$settings{numcpus} t/filetypes/*`; chomp($mashtree); $mashtree=~s/(\d+\.)(\d+)/$1 . substr($2,0,4)/ge; # global and expression ok treeDist($mashtree,$correctMashtree) < 11, "Produce correct tree from mixed file types"; ================================================ FILE: t/02_lambda.t ================================================ #!/usr/bin/env perl use strict; use warnings; use Data::Dumper; use FindBin qw/$RealBin/; use lib "$RealBin/../lib"; use File::Basename qw/dirname basename/; use File::Temp qw/tempdir/; use File::Copy qw/cp/; use Digest::MD5 qw/md5_hex/; use Test::More tests => 7; use_ok 'Mashtree'; use Mashtree qw/treeDist mashDist raw_mash_distance mashHashes/; $ENV{PATH}="./bin:$ENV{PATH}"; my $tempdir = tempdir(basename($0).".XXXXXX", TMP=>1, CLEANUP=>1); subtest 'space in filename' => sub { my $wd = "$tempdir/spaces"; mkdir $wd; my @target; for my $filename(glob("$RealBin/lambda/*.fastq.gz")){ my $target = "$wd/".basename($filename); $target =~ s/sample(\d)/sample $1/; # add in a space for funsies cp($filename, $target) or die "ERROR: could not copy $filename to $target: $!"; push(@target, $target); } # e.g., '02_lambda.tmp.tiF_bn/spaces/sample 1.fastq.gz' '02_lambda.tmp.tiF_bn/spaces/sample 2.fastq.gz' '02_lambda.tmp.tiF_bn/spaces/sample 3.fastq.gz' '02_lambda.tmp.tiF_bn/spaces/sample 4.fastq.gz' my $targets = "'" . join("' '", @target) . "'"; my $cmd = "mashtree --outmatrix lambdadist.tsv --genomesize 40000 --numcpus 1 $targets 2>$0.log"; note $cmd; system($cmd); my $exit_code = $? >> 8; if($exit_code){ note `cat $0.log`; } is($exit_code, 0, "Ran mashtree with exit code $exit_code"); }; my $correctMashtree="(sample3:0.00195,sample4:0.00205,(sample1:0.00205,sample2:0.00205):0.00010);"; $correctMashtree=~s/(\d+\.)(\d+)/$1 . substr($2,0,4)/ge; # global and expression # Test to see if the correct tree is made END{unlink "lambdadist.tsv"; unlink "$0.log";} my $mashtree=`mashtree --outmatrix lambdadist.tsv --genomesize 40000 --save-sketches $RealBin/lambda/sketches --numcpus 1 $RealBin/lambda/*.fastq.gz 2>$0.log`; if($?){ BAIL_OUT("ERROR running mashtree: $!\n ".`cat $0.log`); } chomp($mashtree); $mashtree=~s/(\d+\.)(\d+)/$1 . substr($2,0,4)/ge; # global and expression my $dist=treeDist($mashtree,$correctMashtree); is $dist , 0, "Lambda test set tree, distance should be zero between trees"; if($dist!=0){ note "Correct tree: $correctMashtree"; note "This tree: $mashtree"; BAIL_OUT("Incorrect tree found. Will not continue."); } # Test for the correct distance matrix my %matrix=( 'sample4' => { 'sample4' => 0, 'sample2' => '0.00417555', 'sample1' => '0.0042153' }, 'sample2' => { 'sample2' => 0, 'sample1' => '0.00414809' }, 'sample3' => { 'sample4' => '0.00402957', 'sample2' => '0.00405078', 'sample3' => 0, 'sample1' => '0.0041298' }, 'sample1' => { 'sample1' => 0 } ); # mirror the matrix while(my($ref,$queryHash)=each(%matrix)){ while(my($query,$dist)=each(%$queryHash)){ $matrix{$query}{$ref}=$dist; } } subtest "Test matrix" => sub { plan tests => 16; open(MATRIX, "lambdadist.tsv") or die "ERROR: could not read lambdadist.tsv: $!"; my $header=; chomp($header); my (undef,@header)=split(/\t/,$header); while(my $distances=){ chomp($distances); my($label,@dist)=split /\t/,$distances; for(my $i=0;$i<@header;$i++){ is $dist[$i], $matrix{$label}{$header[$i]}, "Distance between $label and $header[$i]" or note "Should have been $dist[$i]"; } } close MATRIX; }; # Did we get exactly the right sketches? my %sketches = ( "$RealBin/lambda/sketches/sample1.fastq.gz.msh" => "3b11eed05ee26156758e3df04816e742", "$RealBin/lambda/sketches/sample2.fastq.gz.msh" => "3b11eed05ee26156758e3df04816e742", "$RealBin/lambda/sketches/sample3.fastq.gz.msh" => "3b11eed05ee26156758e3df04816e742", "$RealBin/lambda/sketches/sample4.fastq.gz.msh" => "3b11eed05ee26156758e3df04816e742", ); subtest "Saving sketches" => sub { plan tests => 4; for my $file(sort keys(%sketches)){ my $md5sum = $sketches{$file}; open(my $fh, "mash info $file | ") or die "ERROR running mash info on $file: $!"; my @content = grep {!/sample/} <$fh>; close $fh; my $content = join("", @content); pass("TODO: check md5sum, host computer agnostic"); next; is(md5_hex($content), $md5sum, "MD5 of ".basename($file)) or note "Should have been $sketches{$file}. Check on file size and/or `mash info` to follow up."; } }; # test Mash module functions # # 8460/10000 my ($hashes1, $k1, $length1) = mashHashes("$RealBin/lambda/sketches/sample1.fastq.gz.msh"); my ($hashes2, $k2, $length2) = mashHashes("$RealBin/lambda/sketches/sample2.fastq.gz.msh"); my ($common, $total) = raw_mash_distance($hashes1, $hashes2); is($common/$total, 8460/10000, "Raw mash distance") or note "Got: $common / $total but it should be 8460/10000"; my $mashDist = mashDist("$RealBin/lambda/sketches/sample1.fastq.gz.msh","$RealBin/lambda/sketches/sample2.fastq.gz.msh"); is(sprintf("%0.8f",$mashDist), 0.00414809, "Mash distance function"); ================================================ FILE: t/03_subsample.t ================================================ #!/usr/bin/env perl use strict; use warnings; use lib './lib'; use File::Basename qw/dirname/; use Getopt::Long qw/GetOptions/; use Test::More tests => 2; use_ok 'Mashtree'; use Mashtree qw/treeDist/; $ENV{PATH}="./bin:$ENV{PATH}"; my $settings={}; GetOptions($settings,qw(numcpus=i)) or die $!; $$settings{numcpus}||=1; my $correctMashtree='((CFSAN000191.ref:0.0000277081,(CFSAN000189.ref:0.0000023811,CFSAN000189:0.0000000000):0.0000032768):0.0000050668,(CFSAN001115.ref:0.0000099241,CFSAN001112.ref:0.0000210585):0.0000032901,(((CFSAN001140_1:0.0112883298,CFSAN000211:0.0004680698):0.0000119053,CFSAN000968.ref:0.0000164555):0.0000085410,CFSAN000961:0.0000418070):0.0000036369);'; $correctMashtree=~s/(\d+\.)(\d+)/$1 . substr($2,0,4)/ge; # global and expression # Test to see if the correct tree is made my $mashtree=`mashtree --min-depth 0 --numcpus $$settings{numcpus} t/filetypes/*`; chomp($mashtree); $mashtree=~s/(\d+\.)(\d+)/$1 . substr($2,0,4)/ge; # global and expression #print STDERR "$mashtree\n"; my $treedist=treeDist($correctMashtree, $mashtree); ok $treedist < 2, "Correct min_abundance_filter tree (dist: $treedist)"; ================================================ FILE: t/04_fileoffiles.t ================================================ #!/usr/bin/env perl use strict; use warnings; use Data::Dumper; use lib './lib'; use File::Basename qw/dirname/; use Test::More tests => 2; use_ok 'Mashtree'; use Mashtree qw/treeDist/; $ENV{PATH}="./bin:$ENV{PATH}"; my $correctMashtree="((sample2:0.0020443525,sample1:0.0021037373):0.0000540274,sample3:0.0019622177,sample4:0.0020673526);"; # Pattern match for numbers and round them to 5 decimal places $correctMashtree=~s/(\d+\.)(\d+)/$1 . substr($2,0,4)/ge; # global and expression # Test to see if the correct tree is made my $mashtree=`mashtree --file-of-files --numcpus 1 t/file-of-files.txt`; chomp($mashtree); $mashtree=~s/(\d+\.)(\d+)/$1 . substr($2,0,4)/ge; # global and expression is($mashtree, $correctMashtree, "File of files test on lambda"); ================================================ FILE: t/06_jackknifingHashes.t ================================================ #!/usr/bin/env perl use strict; use warnings; use Data::Dumper; use FindBin qw/$RealBin/; use lib "$RealBin/../lib"; use lib "$RealBin/../lib/perl5"; # Need to actually update env because # of the executables called later. $ENV{PERL5LIB}=$ENV{PERL5LIB}.":$RealBin/../lib/perl5"; use File::Basename qw/dirname/; use File::Path qw/rmtree/; use Bio::TreeIO; use IO::String; use Scalar::Util qw/looks_like_number/; use Test::More tests => 6; use_ok 'Mashtree'; use Mashtree; use Mashtree::Db; $ENV{PATH}="./bin:$ENV{PATH}"; my $correctMashtree="((sample2:0.0020443525,sample1:0.0021037373)66:0.0000540274,sample3:0.0019622177,sample4:0.0020673526)83;"; $correctMashtree=~s/(\d+\.)(\d+)/$1 . substr($2,0,4)/ge; # global and expression # Cleanup END{ rmtree("$RealBin/lambda/jackknife.tmp"); unlink("$RealBin/lambda/jackknife.log"); } # Test to see if the correct tree is made my $mashtree=`mashtree_jackknife.pl --tempdir $RealBin/lambda/jackknife.tmp --reps 100 --numcpus 2 $RealBin/lambda/*.fastq.gz 2>$RealBin/lambda/jackknife.log`; if($?){ my $log = `cat $RealBin/lambda/jackknife.log`; diag $log; BAIL_OUT("mashtree_jackknife.pl exited with an error code $?"); } my $passed = ok(defined($mashtree),"Mashtree_jackknife.pl ran and produced a string"); $mashtree=~s/(\d+\.)(\d+)/$1 . substr($2,0,4)/ge; # global and expression my $fh = IO::String->new($mashtree); my $tree = Bio::TreeIO->new(-fh=>$fh, -format=>"newick")->next_tree; $passed = is(ref($tree),"Bio::Tree::Tree","Produced a BioPerl tree object"); if(!$passed){ diag "Tree string produced was $mashtree"; BAIL_OUT("Tree object was not produced out of the tree string"); } subtest "Parts of the tree file intact" => sub{ plan tests => 3; my @nodes = $tree->get_nodes; my @expectedBootstrap = (100, 11); my $nodeCounter=0; for my $node(grep {!$_->is_Leaf} @nodes){ ok(looks_like_number($node->id), "Bootstrap is a number: ".$node->id); note("Usually this bootstrap is around $expectedBootstrap[$nodeCounter], give or take 5%"); $nodeCounter++; } my $correctNodeString = "sample1 sample2 sample3 sample4"; my $nodeString = join(" ", sort map{$_->id} grep { $_->is_Leaf} @nodes); is $correctNodeString, $nodeString, "Taxon names in the tree: $nodeString"; }; # Test to validate distances on the first rep subtest "Database of distances for rep1" => sub{ plan tests=>20; my $db = Mashtree::Db->new("$RealBin/lambda/jackknife.tmp/rep1/distances.db.tsv"); my @dist = split(/\n/, $db->toString('','phylip')); chomp(@dist); shift(@dist); for(my $i=0;$i<@dist;$i++){ my ($name, @dist) = split(/\s+/, $dist[$i]); for(my $j=0; $j<@dist; $j++){ ok(looks_like_number($dist[$j]), "Distance is a number") or note " Found $dist[$j]"; if($j==$i){ ok($dist[$j] == 0,"Distance against self is 0"); } } } }; # Address the bug fix https://github.com/lskatz/mashtree/issues/51#issuecomment-604495598 subtest "file-of-filenames" => sub{ plan tests=>2; # Make the file of filenames my $fofn = "$RealBin/lambda/lambda.fofn"; my @file = glob("$RealBin/lambda/*.fastq.gz"); open(my $fh, ">", $fofn) or die "ERROR: could not write to $fofn: $!"; for my $f(@file){ print $fh "$f\n"; } close $fh; my $mashtreeFofn = `mashtree_jackknife.pl --tempdir $RealBin/lambda/jackknife.tmp --reps 100 --numcpus 2 --file-of-files $fofn 2>$RealBin/lambda/jackknife.log`; note $mashtreeFofn; my $treeFh = IO::String->new($mashtree); my $tree = Bio::TreeIO->new(-fh=>$treeFh, -format=>"newick")->next_tree; $passed = is(ref($tree),"Bio::Tree::Tree","Produced a BioPerl tree object"); if(!$passed){ diag "Tree string produced was $mashtree"; BAIL_OUT("Tree object was not produced out of the tree string"); } my @leaf = sort map{$_->id} grep{$_->is_Leaf} $tree->get_nodes; is(scalar(@leaf), 4, "correct number of nodes"); note "leaves found: @leaf"; note "tree: ".$tree->as_text("newick"); }; ================================================ FILE: t/08_bootstrap.t ================================================ #!/usr/bin/env perl use strict; use warnings; use Data::Dumper; use FindBin qw/$RealBin/; use lib "$RealBin/../lib"; use File::Basename qw/dirname/; use File::Path qw/rmtree/; use Bio::TreeIO; use IO::String; use Scalar::Util qw/looks_like_number/; use Test::More tests => 3; use_ok 'Mashtree'; use Mashtree; use Mashtree::Db; $ENV{PATH}="$RealBin/../bin:$ENV{PATH}"; my $correctMashtree="((sample2:0.0020443525,sample1:0.0021037373)66:0.0000540274,sample3:0.0019622177,sample4:0.0020673526)83;"; $correctMashtree=~s/(\d+\.)(\d+)/$1 . substr($2,0,4)/ge; # global and expression # Cleanup END{ rmtree("$RealBin/lambda/bootstrap.tmp"); unlink("$RealBin/lambda/bootstrap.log"); } subtest "run mashtree" => sub{ # Test to see if the correct tree is made my $mashtree=`mashtree_bootstrap.pl --tempdir $RealBin/lambda/bootstrap.tmp --reps 10 --numcpus 2 $RealBin/lambda/*.fastq.gz 2>$RealBin/lambda/bootstrap.log`; if($?){ my $log = `cat $RealBin/lambda/bootstrap.log`; diag $log; BAIL_OUT("mashtree_bootstrap.pl exited with an error code $?"); } my $passed = ok(defined($mashtree),"Mashtree_bootstrap.pl ran and produced a string"); $mashtree=~s/(\d+\.)(\d+)/$1 . substr($2,0,4)/ge; # global and expression my $fh = IO::String->new($mashtree); my $tree = Bio::TreeIO->new(-fh=>$fh, -format=>"newick")->next_tree; $passed = is(ref($tree),"Bio::Tree::Tree","Produced a BioPerl tree object"); if(!$passed){ diag "Tree string produced was $mashtree"; BAIL_OUT("Tree object was not produced out of the tree string"); } subtest "Parts of the tree file intact" => sub{ plan tests => 3; my @nodes = $tree->get_nodes; my @expectedBootstrap = (100, 11); my $nodeCounter=0; for my $node(grep {!$_->is_Leaf} @nodes){ ok(looks_like_number($node->id), "Bootstrap is a number: ".$node->id); note("Usually this bootstrap is around $expectedBootstrap[$nodeCounter], give or take 5%"); $nodeCounter++; } my $correctNodeString = "sample1 sample2 sample3 sample4"; my $nodeString = join(" ", sort map{$_->id} grep { $_->is_Leaf} @nodes); is $correctNodeString, $nodeString, "Taxon names in the tree: $nodeString"; }; }; # Address the bug fix https://github.com/lskatz/mashtree/issues/51#issuecomment-604495598 subtest "file-of-filenames" => sub{ plan tests=>2; # Make the file of filenames my $fofn = "$RealBin/lambda/lambda.fofn"; my @file = glob("$RealBin/lambda/*.fastq.gz"); open(my $fh, ">", $fofn) or die "ERROR: could not write to $fofn: $!"; for my $f(@file){ print $fh "$f\n"; } close $fh; my $mashtreeFofn = `mashtree_bootstrap.pl --tempdir $RealBin/lambda/bootstrap.tmp --reps 100 --numcpus 2 --file-of-files $fofn 2>$RealBin/lambda/bootstrap.log`; note "mashtree text is: $mashtreeFofn"; my $treeFh = IO::String->new($mashtreeFofn); my $tree = Bio::TreeIO->new(-fh=>$treeFh, -format=>"newick")->next_tree; my $passed = is(ref($tree),"Bio::Tree::Tree","Produced a BioPerl tree object"); if(!$passed){ diag "Tree string produced was $tree"; BAIL_OUT("Tree object was not produced out of the tree string"); } my @leaf = sort map{$_->id} grep{$_->is_Leaf} $tree->get_nodes; is(scalar(@leaf), 4, "correct number of nodes"); note "@leaf"; }; ================================================ FILE: t/11_tsv-db.t ================================================ #!/usr/bin/env perl use strict; use warnings; use lib './lib'; use File::Basename qw/dirname/; use File::Temp qw/tempdir/; use Data::Dumper; use FindBin qw/$RealBin/; use lib "$RealBin/../lib/perl5"; use lib "$RealBin/../lib/perl5/x86_64-linux-thread-multi/"; use Test::More tests => 4; use_ok 'Mashtree'; use_ok 'Mashtree::Db'; use Mashtree qw/_truncateFilename/; $ENV{PATH}="./bin:$ENV{PATH}"; $ENV{PATH}=~s/quicktree//gi; # remove quicktree for now because it produces a diff ordering of trees my $tempdir=tempdir("testMashDb.XXXXXX",CLEANUP=>1,TMPDIR=>1); my $mashDistancesFile="$tempdir/testdistances.txt"; my $dbFile="$tempdir/testdb.db.tsv"; # Create a large distances file to see if it can handle large inserts open(my $fh, ">", $mashDistancesFile) or die "ERROR: could not write to $mashDistancesFile: $!"; # Each genome is labeled as genomeX and is X distance from genome 'test' print $fh '#query test'."\n"; for(my $i=0;$i<20000;$i++){ print $fh join("\t","genome$i",$i)."\n"; } close $fh; # Make the large inserts and test the hashsum of the db my $db=Mashtree::Db->new($dbFile); my $numInserted=$db->addDistances($mashDistancesFile); ok($numInserted == 20000, "Added 20k distances to the database (actually added: $numInserted)"); subtest 'Testing for specific distances' => sub { plan tests=>16; # Reload the database internally $db->readDatabase; # Test for these expected distances one at a time for my $expectedDistance(0,10,100,256,987,1234,1432){ my $dist = $db->findDistance("genome$expectedDistance","test"); is $expectedDistance,$dist, "Expected distance: $expectedDistance"; my $distRev = $db->findDistance("test","genome$expectedDistance",); is $expectedDistance,$distRev, "Expected distance in reverse: $expectedDistance"; } my $distances = $db->findDistances("test"); my $expectedNumDistances=20000; is $expectedNumDistances, scalar(keys(%$distances)), "Expected number of distances with Mashtree::Db::findDistances(): $expectedNumDistances"; is $$distances{"genome654"}, 654, "Specific distance from Mashtree::Db::findDistances()==654"; }; ================================================ FILE: t/51_benchmark_mashtree.t ================================================ #!/usr/bin/env perl use strict; use warnings; use Benchmark ':all'; use Test::More tests=>1; use File::Temp qw/tempdir/; use File::Which qw/which/; use Data::Dumper; use FindBin qw/$RealBin/; use lib "$RealBin/../lib/perl5"; use lib "$RealBin/../lib"; $ENV{PATH}="$RealBin/../bin:$ENV{PATH}"; my $exe = which('mashtree'); ok($exe ne "", "found a path to mashtree: $exe"); timethese(20, { 'vanilla mashtree' => sub{ my $dnd = `mashtree --numcpus 1 --genomesize 40000 $RealBin/lambda/*.fastq.gz 2>/dev/null`; die if $?; }, }); timethese(5, { 'bootstrap mashtree' => sub{ my $dnd = `mashtree_bootstrap.pl --reps 10 --numcpus 1 $RealBin/lambda/*.fastq.gz -- --genomesize 40000 2>/dev/null`; die if $?; }, 'jackknife mashtree' => sub{ my $dnd = `mashtree_jackknife.pl --reps 10 --numcpus 1 $RealBin/lambda/*.fastq.gz -- --genomesize 40000 2>/dev/null`; die if $?; }, }); ================================================ FILE: t/55_benchmark_db.t ================================================ #!/usr/bin/env perl use strict; use warnings; use Benchmark ':all'; use Test::More tests=>4; use File::Temp qw/tempdir/; use Data::Dumper; use FindBin qw/$RealBin/; use lib "$RealBin/../lib/perl5"; use lib "$RealBin/../lib"; use Mashtree::Db; my $tempdir=tempdir("testMashDb.XXXXXX",CLEANUP=>1,TMPDIR=>1); my $mashDistancesFile="$tempdir/testdistances.txt"; my $dbFile="$tempdir/testdb.db.tsv"; # Create a large distances file to see if it can handle large inserts open(my $fh, ">", $mashDistancesFile) or die "ERROR: could not write to $mashDistancesFile: $!"; # Each genome is labeled as genomeX and is X distance from genome 'test' print $fh '#query test'."\n"; for(my $i=0;$i<20000;$i++){ print $fh join("\t","genome$i",$i)."\n"; } close $fh; my $db = Mashtree::Db->new($dbFile); my $numInserted=$db->addDistances($mashDistancesFile); ok($numInserted == 20000, "Added 20k distances to the database (actually added: $numInserted)"); my $distsHash = $db->readDatabase; is(keys(%$distsHash), 1, "Keys of dists file total 1"); is(keys(%{ $$distsHash{test} }), 20000, "Number of other genomes compared against 'test' genome is 20k."); unlink($dbFile); $db = Mashtree::Db->new($dbFile); $numInserted=$db->addDistancesFromHash($distsHash); ok($numInserted == 20000, "Added 20k distances to the database (actually added: $numInserted)"); timethese(30, { 'addDistancesFromMashFile' => sub{ unlink($dbFile); my $db = Mashtree::Db->new($dbFile); $db->addDistances($mashDistancesFile); }, 'addDistancesFromHash' => sub{ unlink($dbFile); my $db = Mashtree::Db->new($dbFile); $db->addDistancesFromHash($distsHash); }, 'readDatabase' => sub{ $db->readDatabase; }, }); ================================================ FILE: t/archive/10_sqlite.t ================================================ #!/usr/bin/env perl use strict; use warnings; use lib './lib'; use File::Basename qw/dirname/; use File::Temp qw/tempdir/; use Data::Dumper; use FindBin qw/$RealBin/; use lib "$RealBin/../lib/perl5"; use lib "$RealBin/../lib/perl5/x86_64-linux-thread-multi/"; use Test::More tests => 5; use_ok 'Mashtree'; use_ok 'Mashtree::Db'; use Mashtree qw/_truncateFilename/; $ENV{PATH}="./bin:$ENV{PATH}"; $ENV{PATH}=~s/quicktree//gi; # remove quicktree for now because it produces a diff ordering of trees # Test for the sqlite3 executable system("which sqlite3"); ok($?==0, "SQLite executable") or diag("Could not find sqlite. Make sure that the executable sqlite3 is set up properly"); my $tempdir=tempdir("testMash.XXXXXX",CLEANUP=>1,TMPDIR=>1); my $mashDistancesFile="$tempdir/testdistances.txt"; my $sqliteFile="$tempdir/testdb.sqlite"; # Create a large distances file to see if it can handle large inserts open(my $fh, ">", $mashDistancesFile) or die "ERROR: could not write to $mashDistancesFile: $!"; print $fh '#query test'."\n"; for(my $i=0;$i<20000;$i++){ print $fh join("\t","genome$i",$i)."\n"; } close $fh; # Make the large inserts and test the hashsum of the db my $db=Mashtree::Db->new($sqliteFile); my $numInserted=$db->addDistances($mashDistancesFile); ok($numInserted == 20000, "Added 20k distances to the database"); subtest 'Testing for specific distances' => sub { plan tests=>9; for my $expectedDistance(0,10,100,256,987,1234,1432){ my $dist = $db->findDistance("genome$expectedDistance","test"); is $expectedDistance,$dist, "Expected distance: $expectedDistance"; } my $distances = $db->findDistances("test"); my $expectedNumDistances=20000; is $expectedNumDistances, scalar(keys(%$distances)), "Expected number of distances with Mashtree::Db::findDistances(): $expectedNumDistances"; is $$distances{"genome654"}, 654, "Specific distance from Mashtree::Db::findDistances()==654"; }; ================================================ FILE: t/file-of-files.txt ================================================ t/lambda/sample1.fastq.gz t/lambda/sample2.fastq.gz t/lambda/sample3.fastq.gz t/lambda/sample4.fastq.gz ================================================ FILE: t/lambda/mashtree.dnd ================================================ (sample3:0.00195,sample4:0.00205,(sample1:0.00205,sample2:0.00205):0.00010);