Repository: chapmanb/cloudbiolinux Branch: master Commit: cd1b727402e2 Files: 458 Total size: 1.7 MB Directory structure: gitextract_z4wsjush/ ├── .gitignore ├── .gitmodules ├── LICENSE.txt ├── MANIFEST.in ├── README.rst ├── cloudbio/ │ ├── __init__.py │ ├── biodata/ │ │ ├── __init__.py │ │ ├── galaxy.py │ │ ├── genomes.py │ │ ├── ggd.py │ │ └── rnaseq.py │ ├── cloudbiolinux.py │ ├── cloudman.py │ ├── config_management/ │ │ ├── __init__.py │ │ ├── chef.py │ │ ├── puppet.py │ │ └── utils.py │ ├── custom/ │ │ ├── __init__.py │ │ ├── bio_general.py │ │ ├── bio_nextgen.py │ │ ├── bio_proteomics.py │ │ ├── bio_proteomics_wine.py │ │ ├── cloudman.py │ │ ├── distributed.py │ │ ├── galaxy.py │ │ ├── galaxy_tools.py │ │ ├── galaxyp.py │ │ ├── java.py │ │ ├── millstone.py │ │ ├── phylogeny.py │ │ ├── python.py │ │ ├── shared.py │ │ ├── system.py │ │ ├── vcr.py │ │ └── versioncheck.py │ ├── deploy/ │ │ ├── __init__.py │ │ ├── config.py │ │ ├── main.py │ │ ├── plugins/ │ │ │ ├── __init__.py │ │ │ ├── cloudman.py │ │ │ ├── galaxy.py │ │ │ └── gvl.py │ │ ├── util.py │ │ ├── vmlauncher/ │ │ │ ├── __init__.py │ │ │ ├── config.md │ │ │ └── transfer.py │ │ └── volume.py │ ├── distribution.py │ ├── fabutils.py │ ├── flavor/ │ │ ├── __init__.py │ │ └── config.py │ ├── galaxy/ │ │ ├── __init__.py │ │ ├── applications.py │ │ ├── r.py │ │ ├── tools.py │ │ └── utils.py │ ├── libraries.py │ ├── manifest.py │ ├── package/ │ │ ├── __init__.py │ │ ├── brew.py │ │ ├── conda.py │ │ ├── cpan.py │ │ ├── deb.py │ │ ├── nix.py │ │ ├── rpm.py │ │ └── shared.py │ └── utils.py ├── config/ │ ├── README.md │ ├── biodata.yaml │ ├── chef/ │ │ └── cookbooks/ │ │ └── .gitkeep │ ├── chef_recipes.yaml │ ├── custom.yaml │ ├── fabricrc.txt │ ├── haskell-libs.yaml │ ├── main.yaml │ ├── node_extra.json │ ├── packages-debian.yaml │ ├── packages-homebrew.yaml │ ├── packages-nix.yaml │ ├── packages-scientificlinux.yaml │ ├── packages-yum.yaml │ ├── packages.yaml │ ├── perl-libs.yaml │ ├── puppet/ │ │ └── modules/ │ │ └── .gitkeep │ ├── puppet_classes.yaml │ ├── python-libs.yaml │ ├── r-libs.yaml │ └── ruby-libs.yaml ├── contrib/ │ ├── __init__.py │ └── flavor/ │ ├── __init__.py │ ├── biocloudcentral/ │ │ └── main.yaml │ ├── biopython/ │ │ ├── custom.yaml │ │ ├── fabricrc.txt │ │ ├── main.yaml │ │ ├── packages-homebrew.yaml │ │ ├── packages.yaml │ │ └── python-libs.yaml │ ├── boinc/ │ │ ├── __init__.py │ │ ├── boincflavor.py │ │ ├── fabricrc_debian.txt │ │ └── main.yaml │ ├── cloudman/ │ │ ├── README.md │ │ ├── cloudman/ │ │ │ └── main.yaml │ │ ├── cloudman_and_galaxy/ │ │ │ └── main.yaml │ │ ├── cloudman_and_galaxyp/ │ │ │ └── main.yaml │ │ ├── cloudman_desktop_and_galaxyp/ │ │ │ ├── main.yaml │ │ │ └── ruby-libs.yaml │ │ ├── migration_checklist.md │ │ └── tools.yaml │ ├── cwl_dockers/ │ │ └── packages-bcbio-alignment.yaml │ ├── demo/ │ │ ├── README.md │ │ ├── custom.yaml │ │ ├── fabricrc.txt │ │ ├── main.yaml │ │ └── packages-homebrew.yaml │ ├── edx_course/ │ │ ├── custom.yaml │ │ ├── edx_setup.sh │ │ ├── fabricrc.txt │ │ ├── main.yaml │ │ ├── packages-homebrew.yaml │ │ └── python-libs.yaml │ ├── globus/ │ │ └── main.yaml │ ├── millstone/ │ │ ├── README.md │ │ ├── __init__.py │ │ ├── custom.yaml │ │ ├── installer.py │ │ ├── main.yaml │ │ ├── millstoneflavor.py │ │ └── python-libs.yaml │ ├── minimal/ │ │ ├── fabricrc_debian.txt │ │ └── main.yaml │ ├── neuro/ │ │ ├── __init__.py │ │ ├── custom.yaml │ │ ├── fabricrc.txt │ │ ├── main.yaml │ │ ├── neuro.py │ │ └── packages.yaml │ ├── ngs_pipeline_minimal/ │ │ ├── custom.yaml │ │ ├── main.yaml │ │ ├── packages-conda.yaml │ │ ├── packages-homebrew.yaml │ │ ├── perl-libs.yaml │ │ └── r-libs.yaml │ ├── phylogeny/ │ │ ├── __init__.py │ │ ├── fabricrc_debian.txt │ │ ├── fabricrc_ubuntu.txt │ │ ├── install_debian.sh │ │ ├── install_ubuntu.sh │ │ ├── main.yaml │ │ ├── phylogenyflavor.py │ │ └── virtualbox.md │ ├── pjotrp/ │ │ ├── __init__.py │ │ └── biotest/ │ │ ├── __init__.py │ │ ├── biotestflavor.py │ │ ├── fabricrc_debian.txt │ │ └── main.yaml │ ├── proteomics/ │ │ ├── galaxyp/ │ │ │ ├── README.md │ │ │ ├── main.yaml │ │ │ ├── settings-sample-galaxyp.yaml │ │ │ └── tools.yaml │ │ └── swift/ │ │ ├── custom.yaml │ │ ├── main.yaml │ │ └── r-libs.yaml │ ├── seal/ │ │ ├── __init__.py │ │ ├── fabricrc_sl.txt │ │ ├── main.yaml │ │ └── sealflavor.py │ └── variantviz/ │ ├── custom.yaml │ ├── fabricrc.txt │ ├── main.yaml │ └── packages-yum.yaml ├── contributors.mkd ├── data_fabfile.py ├── deploy/ │ ├── README.md │ ├── TODO │ ├── Vagrantfile │ ├── cloudman.html │ ├── cloudman.md │ ├── config/ │ │ └── tool_data_table_conf.xml │ ├── deploy.sh │ ├── deploy_bourne.sh │ ├── deploy_no_deps.sh │ ├── requirements.txt │ ├── settings-sample-cm.yaml │ ├── settings-sample-minimal.yaml │ ├── settings-sample-oldgalaxyvmlauncher.yaml │ ├── test_install_galaxy_tool.py │ └── update_dependencies.sh ├── doc/ │ ├── Makefile │ ├── hacking.md │ ├── intro/ │ │ ├── FAQ.tex │ │ ├── README │ │ ├── basicTerminology.aux │ │ ├── basicTerminology.tex │ │ ├── cloudbl_desktopIntro.aux │ │ ├── cloudbl_desktopIntro.tex │ │ ├── getReady.aux │ │ ├── getReady.tex │ │ ├── gettingStarted_CloudBioLinux.aux │ │ ├── gettingStarted_CloudBioLinux.out │ │ ├── gettingStarted_CloudBioLinux.tex │ │ ├── gettingStarted_CloudBioLinux.toc │ │ ├── images/ │ │ │ ├── createAndMountVol-1.odg │ │ │ ├── nutshell.odg │ │ │ └── unmountDetach-1.odg │ │ ├── tips.tex │ │ ├── usefulLinks.tex │ │ ├── workingOnCloudBL.aux │ │ ├── workingOnCloudBL.tex │ │ ├── workingWithData.aux │ │ └── workingWithData.tex │ ├── linux_kvm.md │ ├── private_cloud.md │ ├── remote_gui.md │ ├── source/ │ │ ├── conf.py │ │ ├── framework.rst │ │ └── index.rst │ └── virtualbox.md ├── fabfile.py ├── ggd-recipes/ │ ├── BDGP6/ │ │ ├── gtf.yaml │ │ ├── mirbase.yaml │ │ ├── seq.yaml │ │ └── transcripts.yaml │ ├── GRCh37/ │ │ ├── 1000g.yaml │ │ ├── 1000g_omni_snps.yaml │ │ ├── 1000g_snps.yaml │ │ ├── ACMG56_genes.yaml │ │ ├── GA4GH_problem_regions.yaml │ │ ├── GRCh37_NCBI2ensembl.txt │ │ ├── MIG.yaml │ │ ├── RADAR.yaml │ │ ├── af_only_gnomad.yaml │ │ ├── ancestral.yaml │ │ ├── battenberg.yaml │ │ ├── capture_regions.yaml │ │ ├── clinvar.yaml │ │ ├── cosmic.yaml │ │ ├── dbnsfp.yaml │ │ ├── dbscsnv.yaml │ │ ├── dbsnp.yaml │ │ ├── dream-syn3.yaml │ │ ├── dream-syn4.yaml │ │ ├── ericscript.yaml │ │ ├── esp.yaml │ │ ├── exac.yaml │ │ ├── fusion-blacklist.yaml │ │ ├── genesplicer.yaml │ │ ├── giab-NA12878-NA24385-somatic.yaml │ │ ├── giab-NA12878.yaml │ │ ├── giab-NA24143.yaml │ │ ├── giab-NA24149.yaml │ │ ├── giab-NA24385.yaml │ │ ├── giab-NA24631.yaml │ │ ├── giab-NA24694.yaml │ │ ├── giab-NA24695.yaml │ │ ├── gnomad.yaml │ │ ├── gnomad_exome.yaml │ │ ├── gnomad_sv.yaml │ │ ├── hapmap.yaml │ │ ├── mills_indels.yaml │ │ ├── prioritize.yaml │ │ ├── qsignature.yaml │ │ ├── seq.yaml │ │ ├── topmed.yaml │ │ ├── transcripts.yaml │ │ ├── twobit.yaml │ │ ├── varpon.yaml │ │ ├── vcfanno.yaml │ │ └── viral.yaml │ ├── GRCz11/ │ │ ├── seq.yaml │ │ ├── transcripts.yaml │ │ └── twobit.yaml │ ├── README.md │ ├── Sscrofa11.1/ │ │ ├── seq.yaml │ │ ├── transcripts.yaml │ │ └── twobit.yaml │ ├── TAIR10/ │ │ └── mirbase.yaml │ ├── canFam3/ │ │ ├── dbsnp.yaml │ │ ├── mirbase.yaml │ │ ├── transcripts.yaml │ │ └── twobit.yaml │ ├── hg19/ │ │ ├── 1000g.yaml │ │ ├── 1000g_omni_snps.yaml │ │ ├── 1000g_snps.yaml │ │ ├── ACMG56_genes.yaml │ │ ├── GA4GH_problem_regions.yaml │ │ ├── MIG.yaml │ │ ├── RADAR.yaml │ │ ├── af_only_gnomad.yaml │ │ ├── battenberg.yaml │ │ ├── capture_regions.yaml │ │ ├── clinvar.yaml │ │ ├── cosmic.yaml │ │ ├── dbsnp.yaml │ │ ├── effects_transcripts.yaml │ │ ├── esp.yaml │ │ ├── exac.yaml │ │ ├── fusion-blacklist.yaml │ │ ├── giab-NA12878.yaml │ │ ├── giab-NA24143.yaml │ │ ├── giab-NA24149.yaml │ │ ├── giab-NA24385.yaml │ │ ├── giab-NA24631.yaml │ │ ├── gnomad.yaml │ │ ├── gnomad_exome.yaml │ │ ├── gnomad_genome.grch37_to_hg19.sh │ │ ├── gtf.yaml │ │ ├── hapmap.yaml │ │ ├── mills_indels.yaml │ │ ├── mirbase.yaml │ │ ├── platinum-genome-NA12878.yaml │ │ ├── prioritize.yaml │ │ ├── purecn_mappability.yaml │ │ ├── rmsk.yaml │ │ ├── seq.yaml │ │ ├── simple_repeat.yaml │ │ ├── topmed.yaml │ │ ├── transcripts.yaml │ │ ├── twobit.yaml │ │ ├── varpon.yaml │ │ └── viral.yaml │ ├── hg38/ │ │ ├── 1000g_indels.yaml │ │ ├── 1000g_omni_snps.yaml │ │ ├── 1000g_snps.yaml │ │ ├── ACMG56_genes.yaml │ │ ├── RADAR.yaml │ │ ├── README.md │ │ ├── af_only_gnomad.yaml │ │ ├── bwa.yaml │ │ ├── canonical_cancer_99.txt │ │ ├── capture_regions.yaml │ │ ├── ccds.yaml │ │ ├── clinvar.yaml │ │ ├── coverage.yaml │ │ ├── dbnsfp.yaml │ │ ├── dbscsnv.yaml │ │ ├── dbsnp.yaml │ │ ├── dream-syn3-crossmap.yaml │ │ ├── dream-syn4-crossmap.yaml │ │ ├── effects_transcripts.yaml │ │ ├── ericscript.yaml │ │ ├── esp.yaml │ │ ├── exac.yaml │ │ ├── fusion-blacklist.yaml │ │ ├── genesplicer.yaml │ │ ├── genotype2phenotype.yaml │ │ ├── giab-NA12878-NA24385-somatic.yaml │ │ ├── giab-NA12878-crossmap.yaml │ │ ├── giab-NA12878-remap.yaml │ │ ├── giab-NA12878.yaml │ │ ├── giab-NA24143.yaml │ │ ├── giab-NA24149.yaml │ │ ├── giab-NA24385.yaml │ │ ├── giab-NA24631.yaml │ │ ├── giab-NA24694.yaml │ │ ├── giab-NA24695.yaml │ │ ├── gnomad.yaml │ │ ├── gnomad_exome.yaml │ │ ├── gnomad_fields_to_keep.txt │ │ ├── gtf.yaml │ │ ├── hapmap_snps.yaml │ │ ├── hisat2.yaml │ │ ├── mills_indels.yaml │ │ ├── mirbase.yaml │ │ ├── platinum-genome-NA12878.yaml │ │ ├── prioritize.yaml │ │ ├── purecn_mappability.yaml │ │ ├── qsignature.yaml │ │ ├── rmsk.yaml │ │ ├── salmon-decoys.yaml │ │ ├── seq.yaml │ │ ├── simple_repeat.yaml │ │ ├── topmed.yaml │ │ ├── transcripts.yaml │ │ ├── twobit.yaml │ │ ├── varpon.yaml │ │ ├── vcfanno.yaml │ │ └── viral.yaml │ ├── hg38-noalt/ │ │ ├── RADAR.yaml │ │ ├── README.md │ │ ├── bowtie2.yaml │ │ ├── bwa.yaml │ │ ├── gtf.yaml │ │ ├── mirbase.yaml │ │ ├── seq.yaml │ │ └── transcripts.yaml │ ├── mm10/ │ │ ├── dbsnp.yaml │ │ ├── mirbase.yaml │ │ ├── prioritize.yaml │ │ ├── problem_regions.yaml │ │ ├── rmsk.yaml │ │ ├── seq.yaml │ │ ├── transcripts.yaml │ │ ├── twobit.yaml │ │ └── vcfanno.yaml │ ├── rn6/ │ │ ├── mirbase.yaml │ │ ├── seq.yaml │ │ ├── transcripts.yaml │ │ └── twobit.yaml │ └── sacCer3/ │ ├── seq.yaml │ └── transcripts.yaml ├── installed_files/ │ ├── bash_history │ ├── bash_login │ ├── ec2autorun.py │ ├── galaxy_default.template │ ├── galaxy_init │ ├── galaxyp_nginx.conf.template │ ├── image_user_data │ ├── ipython_config.py │ ├── jwmrc.xml │ ├── nginx.conf.template │ ├── nginx_init │ ├── novnc_default.template │ ├── novnc_init │ ├── pg_ctl │ ├── proftpd.conf.template │ ├── protvis_default.template │ ├── protvis_init │ ├── psql │ ├── setupnx.sh │ ├── tool_data_table_conf.xml │ ├── vncserver_default.template │ ├── vncserver_init │ ├── xstartup │ ├── xvfb_default │ └── xvfb_init ├── manifest/ │ ├── custom-packages.yaml │ ├── debian-packages.yaml │ ├── python-packages.yaml │ └── r-packages.yaml ├── setup.py ├── test/ │ ├── README │ ├── test_biolinux │ ├── test_vagrant │ └── testlib/ │ ├── test_biolinux.rb │ └── test_support.rb └── utils/ ├── bootstrap.sh ├── cbl_exome_setup.py ├── cbl_installed_software.py ├── convert_to_xz.py ├── cwl2yaml_packages.py ├── get_biolinux_packages.py ├── get_yum_packages.py ├── images_and_snapshots.py ├── prep_esp_hg38.py ├── prepare_cosmic.py ├── prepare_dbsnp.py ├── prepare_tx_gff.py ├── prioritize/ │ ├── AZ300.txt │ ├── AZ300_with_known.txt │ ├── az-cancer-panel.txt │ ├── az300_to_bed.py │ ├── prep_ccds_genes.py │ └── prep_prioritize_downloads.sh ├── query_conda_deps.py ├── s3_multipart_upload.py └── sv/ ├── NA24385_crowd_dels.py └── NA24385_giab_dels.py ================================================ FILE CONTENTS ================================================ ================================================ FILE: .gitignore ================================================ *.pyc tags cloudbiolinux.egg-info/ build/ doc/build dist/ *.pem dpkg.lst *.log venv/ venv_cbl/ deploy/build deploy/keys deploy/settings.yaml deploy/.venv-deploy deploy/.vagrant .ropeproject .idea ================================================ FILE: .gitmodules ================================================ [submodule "config/puppet/modules/apache"] path = config/puppet/modules/apache url = git://github.com/puppetlabs/puppetlabs-apache.git [submodule "config/puppet/modules/concat"] path = config/puppet/modules/concat url = git://github.com/puppetlabs/puppetlabs-concat.git [submodule "config/puppet/modules/firewall"] path = config/puppet/modules/firewall url = git://github.com/puppetlabs/puppetlabs-firewall.git [submodule "config/puppet/modules/stdlib"] path = config/puppet/modules/stdlib url = git://github.com/puppetlabs/puppetlabs-stdlib.git [submodule "config/puppet/modules/vcsrepo"] path = config/puppet/modules/vcsrepo url = git://github.com/puppetlabs/puppetlabs-vcsrepo.git [submodule "config/puppet/modules/biocloudcentral"] path = config/puppet/modules/biocloudcentral url = git://github.com/bioconfig/puppet-biocloudcentral.git [submodule "config/puppet/modules/python"] path = config/puppet/modules/python url = git://github.com/bioconfig/puppet-python.git [submodule "config/chef/cookbooks/globus"] path = config/chef/cookbooks/globus url = git://github.com/bioconfig/chef-globus.git [submodule "config/puppet/modules/lwr"] path = config/puppet/modules/lwr url = git://github.com/bioconfig/puppet-lwr.git ================================================ FILE: LICENSE.txt ================================================ Copyright (c) 2013 CloudBioLinux contributors Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ================================================ FILE: MANIFEST.in ================================================ include *fabfile.py include *.md include *.mkd include config/*.yaml include config/fabricrc.txt include config/README.md include doc/*.md include installed_files/* include utils/* ================================================ FILE: README.rst ================================================ CloudBioLinux is a build and deployment system which installs an easily customizable selection of bioinformatics and machine learning libraries on a linux container, bare virtual machine (VM) image, freshly installed PC, or in the cloud. CloudBioLinux is a curated and community developed set of instructions for tools provided by operating system packages (debs and RPMs), external packaging efforts (`bioconda `_ and `homebrew-science `_) and language specific library installers (Python, R, Perl and Ruby). CloudBioLinux included software packages are fully customizable. In addition to the default configuration, we support custom configuration builds through flavors. Flavors support overriding default package installations, making it simple to create derived installs for specific purposes. CloudBioLinux is a single install route for `Docker containers `_ ,desktop VMs such as `VirtualBox `_, cloud providers such as `Amazon EC2 `_ or desktop machines. This works equally well for other virtual machines and private cloud environments, including `XEN `_, Linux `KVM `_, `Eucalyptus `_ and `Openstack `_. Quick start =========== `bcbio `_ uses CloudBioLinux as the basis for tool installation and provides a large set of supported and tested tools. If you're looking to bootstrap a system with tools and data for high throughput sequencing analysis, we suggest using the `bcbio installer `_ which fully wraps CloudBioLinux and provides an easy path to customize install directories, organisms and biological data installed. To modify/add data/package recipies to bcbio, edit the below configs: - bcbio data recipes: https://github.com/chapmanb/cloudbiolinux/tree/master/ggd-recipes - bcbio data index1: https://github.com/chapmanb/cloudbiolinux/blob/master/config/biodata.yaml - bcbio data index2: https://github.com/bcbio/bcbio-nextgen/tree/master/config/genomes - bcbio conda packages index: https://github.com/chapmanb/cloudbiolinux/blob/master/contrib/flavor/ngs_pipeline_minimal/packages-conda.yaml We recommend using, or developing, a custom flavor to choose tools of interest to install. The amount of bioinformatics software continues to increase -- there are `over 1000 recipes in bioconda `_ -- and it's difficult to come up with a default installation that includes everything for everyone. The ``ngs_pipeline_minimal`` flavor has the set of NGS analysis tools installed with bcbio and is a good starting point for understanding the CloudBioLinux install process. To install inside an isolated conda environment on a bare machine do:: git clone https://github.com/chapmanb/cloudbiolinux.git cd cloudbiolinux wget https://repo.continuum.io/miniconda/Miniconda2-latest-Linux-x86_64.sh bash Miniconda2-latest-Linux-x86_64.sh -b -p ~/cblenv ~/cblenv/bin/conda install -y -c bioconda -c conda-forge pip fabric pyyaml ~/cblenv/bin/fab -f fabfile.py -H localhost install_biolinux:flavor=ngs_pipeline_minimal --set keep_isolated=true Installing CloudBioLinux on a local machine =========================================== The install process for CloudBioLinux is fully automated through a `Fabric build file `_ written in Python. Everything is fully configurable through plain text YAML configuration files, and custom build targets allow installation of a subset of the total available packages. Setup ----- Retrieve the CloudBioLinux code base and install fabric:: pip install fabric git clone git://github.com/chapmanb/cloudbiolinux.git cd cloudbiolinux Usage ----- The basic usage specifies the hostname of a machine accessible via ssh or the local machine:: fab -f fabfile.py -H localhost install_biolinux Fabric contains some other useful commandline arguments for customizing this to your environments: - ``-c your_fabricrc.txt`` -- Specify the path to a fabricrc configuration files. This allows customization of install directories and other server specific details. See the default ``config/fabricrc.txt`` for a full list of options. - ``-u username`` -- The username on a remote machine, overriding the default of your current username. Customization with flavors -------------------------- In most cases you want to customize a specific set of packages, or install into an isolated directory without root access, using flavors:: fab -f fabfile.py -H localhost install_biolinux:flavor=my_flavor ``my_flavor`` can be the name of an existing flavor in ``contrib/flavor`` or the path to a directory with customization information. The files in your flavor directory replace those in the standard ``config`` directory, allowing replacement of any of the configuration files like ``main.yaml`` with customized copies. If you desire even more control, flavors allow custom python hooks. See ``doc/hacking.md`` for more details. The best place to get started is the `demo flavor `_ included with CloudBioLinux. This installs a small number of common packages into an isolated directory (``~/tmp/cbl_demo`` by default), without root access. Run the example with:: fab -f fabfile.py -H localhost install_biolinux:flavor=demo Specific install targets ------------------------ You can substitute ``install_biolinux`` with more specific targets to only build portions of CloudBioLinux: - ``install_biolinux:packages`` -- Install all of the defined system packages. - ``install_biolinux:libraries`` -- Install all libraries for various programming languages. - ``install_biolinux:brew`` -- Install homebrew packages only. - ``install_libraries:language`` -- Install libraries for a specific language. - ``install_biolinux:custom`` -- Install all custom programs. - ``install_brew:a_package_name`` -- Install a specific brew package. - ``install_custom:a_package_name`` -- Install a specific custom program. Homebrew package installation ----------------------------- `Homebrew `_ and `Linuxbrew `_ provide a Ruby-based environment for installing packages on MacOSX and Linux. The active `homebrew-science `_ packaging community maintains a number of common scientific tools. We also maintain a `homebrew-cbl `_ repository with tools not yet integrated into homebrew-science. CloudBioLinux manages installation of the Linuxbrew or Homebrew framework and pulls in the ``homebrew/science`` and ``chapmanb/cbl`` taps, as well as injecting your current compilers into the homebrew build scripts. To install a `supported package `_ using CloudBioLinux:: fab -f fabfile.py -H localhost install_custom:bedtools Specific package installation ----------------------------- The custom directory contains installation instructions for programs that are not available from standard package repositories, written in Python using the `Fabric `_ remote deployment tool. To install individual `custom packages `_:: fab -f fabfile.py -H localhost install_custom:your_package_name We prefer using the Homebrew framework for new packages over writing custom packages. Biological data --------------- We manage a repository of useful public biological data on an `Amazon S3 bucket `_. Currently this includes whole genomes pre-indexed for a number of popular aligners. Downloading and installing these saves a ton of time over running the indexing steps yourself, and eases running next-generation analyses on cloud machines. A Fabric build script is provided to install this data on your local machine. A `biodata configuration file in YAML format `_, ``config/biodata.yaml``, specifies the genomes of interest and the aligner indexes to use. The ``config/fabricrc.txt`` file specifies details about the system and where to install the data. The basic commandline is:: fab -f data_fabfile.py -H your_machine install_data_s3 and you can pass in custom biodata and fabricrc files with:: fab -f data_fabfile.py -H your_machine -c your_fabricrc.txt install_data_s3:your_biodata.yaml In addition to downloading and preparing the data, the script will integrate these files with a Galaxy instance by updating appropriate Galaxy configuration files. This makes it useful for installing data to a local or `cloud-based `_ Galaxy server. Not all of the genomes are hosted on the S3 bucket, but are still supported. If your genome fails to install with install_data_s3, you might be able to download the genome from Ensembl, etc and prepare it:: fab -f data_fabfile.py -H your_machine -c your_fabricrc.txt install_data:your_biodata.yaml Using pre-built cloud images ============================ Amazon ------ See the 'Getting Started with CloudBioLinux' guide on the `CloudBioLinux website `_ for a detailed description. The short version for users familiar with Amazon is: - Login to the `Amazon EC2 console `_. - Click Launch Instance, and choose the latest CloudBioLinux AMI from the `website `_ in the community AMI section (search for 'CloudBioLinux'). - After launching the instance, find the host details of your running instance from the Instances section. - Connect to your machine via ssh or VNC (using the Amazon PEM keys) Supported environments ====================== Docker ------ `Docker `_ provides lightweight local containers for Linux machines, allowing isolation without the associated overhead of full virtual machines. Include any of the standard CloudBioLinux commands inside a `Dockerfile `_ to use CloudBioLinux to build up the set of tools on your instance. See the `Dockerfile examples `_ for information how to write Dockerfiles. To use a pre-built Docker image made with CloudBioLinux infrastructure, using this `bcbio-nextgen Dockerfile `_, you can import the `bcbio-nextgen `_ container into your local docker environment:: docker import https://s3.amazonaws.com/bcbio_nextgen/bcbio-nextgen-docker-image.gz chapmanb/bcbio-nextgen-cbl Amazon ------ A bare Linux image launched in Amazon EC2 is configured from another machine, i.e. your local desktop, using ssh and cloudbiolinux. See the Installation section for installing CloudBioLinux with fabric. Any cloudbiolinux distribution can be used, including Ubuntu, Debian Linux and CentOS. We recommend using m1.medium or better instance for building a CloudBioLinux image from scratch, due to resource usage while compiling software. 1. Go to the cloudbiolinux source and edit the ``config/fabricrc.txt``, to match the system you plan to install on. Specifically, ``distribution`` and ``dist_name`` parameters specify details about the type of target. 2. Start an Amazon EC2 base instance and retrieve it's DNS hostname: - `Alestic Ubuntu images `_ - `Camptocamp Debian images `_ 3. From your local machine, have CloudBioLinux install your Amazon instance: :: fab -f fabfile.py -H hostname -u username -i private_key_file install_biolinux 4. When finished, use the `Amazon console `_ to create an AMI. Thereafter make it public so it can be used by others. Vagrant and VirtualBox ---------------------- Vagrant allows easy deploying and connecting to VirtualBox images. The setup is ideal for running CloudBioLinux on a desktop computer. Install `VirtualBox `_ and `vagrant `_. See `the VirtualBox and Vagrant documentation `_ for details on creating a local virtual machine from scratch with CloudBioLinux. Through Vagrant additional facilities are available, such as a shared network drive. It is also possible to tweak the image (e.g. RAM/CPU settings, and getting the all important guest additions) by firing up virtualbox itself. For more information, see the documentation on the `Vagrant website `_. OpenStack/XEN/KVM/Eucalyptus private Cloud ------------------------------------------ As long as there is an 'ssh' entry to an running VM, CloudBioLinux can install itself. For more on private Cloud and CloudBioLinux see ./doc/private\_cloud.md. EC2 quickstart ============== This provides a quick cheat sheet of commands for getting up and running on EC2 using Amazon's command line tools. Initial set up -------------- The first time using EC2, you'll need to install the toolkit and credentials for connecting on your local machine, following the `getting started guide `_. Login to your `Amazon EC2 account `_ and go to Security Credentials/X.509. Create a new certificate and download the public ``cert-*.pem`` and ``private pk-*.pem`` files. Put these in ``~.ec2``. Install the `ec2 api tools `_, which require java. Set up .zshrc/.bashrc: :: export EC2_PRIVATE_KEY=~/.ec2/pk-UBH43XTAWVNQMIZRAV3RP5IIBAPBIFVP.pem export EC2_CERT=~/.ec2/cert-UBH43XTAWVNQMIZRAV3RP5IIBAPBIFVP.pem export AWS_ACCESS_KEY_ID= export AWS_SECRET_ACCESS_KEY= To test, you should be able to run the command: :: % ec2-describe-regions Now generate a privatekey for logging in: :: % ec2-add-keypair yourmachine-keypair This will produce an RSA private key. You should copy and paste this to your .ec2 directory for future use: :: % vim ~/.ec2/id-yourmachine.keypair % chmod 600 ~/.ec2/id-yourmachine.keypair Allow ssh and web access to your instances: :: % ec2-authorize default -p 22 % ec2-authorize default -p 80 Starting an instance -------------------- Each time you'd like to use EC2, you need to create a remote instance to work with; the `AWS console `_ is useful for managing this process. When building from scratch with Alestic images, you will need to increase the size of the root filesystem to fit all of the CloudBioLinux data and libraries. This is done by starting the instance from the commandline with: :: % ec2-run-instances ami-1aad5273 -k kunkel-keypair -t m1.large -b /dev/sda1=:20 % ec2-describe-instances i-0ca39764 On Ubuntu 10.04, you then need to ssh into the instance and resize the filesystem with: :: % sudo resize2fs /dev/sda1 On 11.04 the resize happens automatically and this is not required. Testing ======= BioLinux comes with an integration testing frame work - currently based on Vagrant. Try: :: cd test ./testing_vagrant --help Target VMs can be listed with :: ./testing_vagrant --list Build a minimal VM :: ./testing_vagrant Minimal Documentation ============= Additional documentation can be found in the `./doc directory `_ in the BioLinux source tree. LICENSE ======= The code is freely available under the `MIT license `_. ================================================ FILE: cloudbio/__init__.py ================================================ """Module level code supporting CloudBioLinux installations. This provides a reusable architecture allowing definitions and helper code to be used in other projects. """ __import__('pkg_resources').declare_namespace(__name__) ================================================ FILE: cloudbio/biodata/__init__.py ================================================ """Download, installation and configuration of biological data. """ ================================================ FILE: cloudbio/biodata/galaxy.py ================================================ """Retrieve indexed genomes using Galaxy's rsync server resources. http://wiki.galaxyproject.org/Admin/Data%20Integration """ from __future__ import print_function import os import shutil import subprocess from xml.etree import ElementTree from cloudbio.custom import shared # ## Compatibility definitions server = "rsync://datacache.g2.bx.psu.edu" index_map = {"bismark": "bismark_index", "bowtie": "bowtie_index", "bowtie2": "bowtie2_index", "bwa": "bwa_index", "novoalign": "novoalign_index", "ucsc": "seq", "seq": "sam_index"} org_remap = {"phix": "phiX", "GRCh37": "hg_g1k_v37", "araTha_tair9": "Arabidopsis_thaliana_TAIR9", "araTha_tair10": "Arabidopsis_thaliana_TAIR10", "WS210": "ce10", "WS220": "ce10"} galaxy_subdirs = ["", "/microbes"] # ## Galaxy location files class LocCols(object): # Hold all possible .loc file column fields making sure the local # variable names match column names in Galaxy's tool_data_table_conf.xml def __init__(self, config, dbkey, file_path): self.dbkey = dbkey self.path = file_path self.value = config.get("value", dbkey) self.name = config.get("name", dbkey) self.species = config.get('species', '') self.index = config.get('index', 'index') self.formats = config.get('index', 'fastqsanger') self.dbkey1 = config.get('index', dbkey) self.dbkey2 = config.get('index', dbkey) def _get_tool_conf(env, tool_name): """ Parse the tool_data_table_conf.xml from installed_files subfolder and extract values for the 'columns' tag and 'path' parameter for the 'file' tag, returning those as a dict. """ tool_conf = {} tdtc = ElementTree.parse(env.tool_data_table_conf_file) tables = tdtc.getiterator('table') for t in tables: if tool_name in t.attrib.get('name', ''): tool_conf['columns'] = t.find('columns').text.replace(' ', '').split(',') tool_conf['file'] = t.find('file').attrib.get('path', '') return tool_conf def _build_galaxy_loc_line(env, dbkey, file_path, config, prefix, tool_name): """Prepare genome information to write to a Galaxy *.loc config file. """ if tool_name: str_parts = [] tool_conf = _get_tool_conf(env, tool_name) loc_cols = LocCols(config, dbkey, file_path) # Compose the .loc file line as str_parts list by looking for column values # from the retrieved tool_conf (as defined in tool_data_table_conf.xml). # Any column values required but missing in the tool_conf are # supplemented by the defaults defined in LocCols class for col in tool_conf.get('columns', []): str_parts.append(config.get(col, getattr(loc_cols, col))) else: str_parts = [dbkey, file_path] if prefix: str_parts.insert(0, prefix) return str_parts def update_loc_file(env, ref_file, line_parts): """Add a reference to the given genome to the base index file. """ if getattr(env, "galaxy_home", None) is not None: tools_dir = os.path.join(env.galaxy_home, "tool-data") if not os.path.exists(tools_dir): subprocess.check_call("mkdir -p %s" % tools_dir, shell=True) dt_file = os.path.join(env.galaxy_home, "tool_data_table_conf.xml") if not os.path.exists(dt_file): shutil.copy(env.tool_data_table_conf_file, dt_file) add_str = "\t".join(line_parts) with shared.chdir(tools_dir): if not os.path.exists(ref_file): subprocess.check_call("touch %s" % ref_file, shell=True) has_line = False with open(ref_file) as in_handle: for line in in_handle: if line.strip() == add_str.strip(): has_line = True if not has_line: with open(ref_file, "a") as out_handle: out_handle.write(add_str + "\n") def prep_locs(env, gid, indexes, config): """Prepare Galaxy location files for all available indexes. """ for ref_index_file, cur_index, prefix, tool_name in [ ("alignseq.loc", indexes.get("ucsc", None), "seq", None), ("bismark_indices.loc", indexes.get("bismark", None), "", 'bismark_indexes'), ("bowtie2_indices.loc", indexes.get("bowtie2", None), "", 'bowtie2_indexes'), ("bowtie_indices.loc", indexes.get("bowtie", None), "", 'bowtie_indexes'), ("bwa_index.loc", indexes.get("bwa", None), "", 'bwa_indexes'), ("gatk_sorted_picard_index.loc", indexes.get("seq", None), "", "gatk_picard_indexes"), ("mosaik_index.loc", indexes.get("mosaik", None), "", "mosaik_indexes"), ("novoalign_indices.loc", indexes.get("novoalign", None), "", "novoalign_indexes"), ("picard_index.loc", indexes.get("seq", None), "", "picard_indexes"), ("sam_fa_indices.loc", indexes.get("seq", None), "", 'sam_fa_indexes'), ("twobit.loc", indexes.get("ucsc", None), "", None)]: if cur_index: str_parts = _build_galaxy_loc_line(env, gid, cur_index, config, prefix, tool_name) update_loc_file(env, ref_index_file, str_parts) # ## Finalize downloads def index_picard(ref_file): """Provide a Picard style dict index file for a reference genome. """ index_file = "%s.dict" % os.path.splitext(ref_file)[0] if not os.path.exists(index_file): subprocess.check_call("picard -Xms500m -Xmx3500m CreateSequenceDictionary REFERENCE={ref} OUTPUT={out}" .format(ref=ref_file, out=index_file), shell=True) return index_file def _finalize_index_seq(fname): """Convert UCSC 2bit file into fasta file. """ out_fasta = fname + ".fa" if not os.path.exists(out_fasta): subprocess.check_call("twoBitToFa {base}.2bit {out}".format( base=fname, out=out_fasta), shell=True) finalize_fns = {"ucsc": _finalize_index_seq, "seq": index_picard} def _finalize_index(idx, fname): """Perform final processing on an rsync'ed index file if necessary. """ finalize_fn = finalize_fns.get(idx) if finalize_fn: finalize_fn(fname) # ## Retrieve data from Galaxy def rsync_genomes(genome_dir, genomes, genome_indexes): """Top level entry point to retrieve rsync'ed indexes from Galaxy. """ for gid in (x[1] for x in genomes): galaxy_gid = org_remap.get(gid, gid) indexes = _get_galaxy_genomes(galaxy_gid, genome_dir, genomes, genome_indexes) _finalize_index("ucsc", indexes["ucsc"]) for idx, fname in indexes.iteritems(): _finalize_index(idx, fname) prep_locs(galaxy_gid, indexes, {}) def _get_galaxy_genomes(gid, genome_dir, genomes, genome_indexes): """Retrieve the provided genomes and indexes from Galaxy rsync. """ out = {} org_dir = os.path.join(genome_dir, gid) if not os.path.exists(org_dir): subprocess.check_call('mkdir -p %s' % org_dir, shell=True) for idx in genome_indexes: galaxy_index_name = index_map.get(idx) index_file = None if galaxy_index_name: index_file = _rsync_genome_index(gid, galaxy_index_name, org_dir) if index_file: out[idx] = index_file else: print("Galaxy does not support {0} for {1}".format(idx, gid)) return out def _rsync_genome_index(gid, idx, org_dir): """Retrieve index for a genome from rsync server, returning path to files. """ idx_dir = os.path.join(org_dir, idx) if not os.path.exists(idx_dir): org_rsync = None for subdir in galaxy_subdirs: test_rsync = "{server}/indexes{subdir}/{gid}/{idx}/".format( server=server, subdir=subdir, gid=gid, idx=idx) try: subprocess.check_output("rsync --list-only {server}".format(server=test_rsync)) org_rsync = test_rsync except subprocess.CalledProcessError: pass if org_rsync is None: raise ValueError("Could not find genome %s on Galaxy rsync" % gid) try: subprocess.check_call("rsync --list-only {server}".format(server=org_rsync), shell=True) if not os.path.exists(idx_dir): subprocess.check_call('mkdir -p %s' % idx_dir, shell=True) with cd(idx_dir): subprocess.check_call("rsync -avzP {server} {idx_dir}".format(server=org_rsync, idx_dir=idx_dir), shell=True) except subprocess.CalledProcessError: pass if os.path.exists(idx_dir): try: subprocess.check_call("ls {idx_dir}/{gid}.fa*".format(idx_dir=idx_dir, gid=gid), shell=True) ext = ".fa" if (has_fa_ext.succeeded and idx not in ["seq"]) else "" except subprocess.CalledProcessError: pass return os.path.join(idx_dir, gid + ext) ================================================ FILE: cloudbio/biodata/genomes.py ================================================ """Download and install structured genome data and aligner index files. Downloads prepared FASTA, indexes for aligners like BWA, Bowtie and novoalign and other genome data in automated pipelines. Specify the genomes and aligners to use in an input biodata.yaml configuration file. The main targets are fabric functions: - install_data -- Install biological data from scratch, including indexing genomes. - install_data_s3 -- Install biological data, downloading pre-computed indexes from S3. - upload_s3 -- Upload created indexes to biodata S3 bucket. """ from __future__ import print_function import collections import os import operator import socket import subprocess import sys import traceback from math import log try: import yaml except ImportError: yaml = None try: import boto except ImportError: boto = None from cloudbio.biodata import galaxy, ggd, rnaseq from cloudbio.custom import shared # -- Configuration for genomes to download and prepare class _DownloadHelper: def __init__(self): self.config = {} def ucsc_name(self): return None def _exists(self, fname, seq_dir): """Check if a file exists in either download or final destination. """ return os.path.exists(fname) or os.path.exists(os.path.join(seq_dir, fname)) class UCSCGenome(_DownloadHelper): def __init__(self, genome_name, dl_name=None): _DownloadHelper.__init__(self) self.data_source = "UCSC" self._name = genome_name self.dl_name = dl_name if dl_name is not None else genome_name self._url = "ftp://hgdownload.cse.ucsc.edu/goldenPath/%s/bigZips" % \ genome_name def ucsc_name(self): return self._name def _karyotype_sort(self, xs): """Sort reads in karyotypic order to work with GATK's defaults. """ def karyotype_keyfn(x): base = os.path.splitext(os.path.basename(x))[0] if base.startswith("chr"): base = base[3:] parts = base.split("_") try: base = int(parts[0]) except ValueError: base = sys.maxsize # unplaced at the very end if parts[0] == "Un": parts.insert(0, "z") # mitochondrial special case -- after X/Y elif parts[0] in ["M", "MT"]: parts.insert(0, "x") # sort random and extra chromosomes after M elif len(parts) > 1: parts.insert(0, "y") # standard integers, sort first else: parts.insert(0, "a") return [base] + parts return sorted(xs, key=karyotype_keyfn) def _split_multifasta(self, fasta_file): chrom = "" file_handle = None file_names = [] out_dir = os.path.dirname(fasta_file) with open(fasta_file) as in_handle: for line in in_handle: if line.startswith(">"): chrom = line.split(">")[1].strip() file_handle.close() if file_handle else None file_names.append(chrom + ".fa") file_handle = open(os.path.join(out_dir, chrom + ".fa"), "w") file_handle.write(line) else: file_handle.write(line) file_handle.close() return file_names def download(self, seq_dir): zipped_file = None genome_file = "%s.fa" % self._name if not self._exists(genome_file, seq_dir): prep_dir = "seq_prep" subprocess.check_call("mkdir -p %s" % prep_dir, shell=True) with shared.chdir(prep_dir): zipped_file = self._download_zip(seq_dir) if zipped_file.endswith(".tar.gz"): subprocess.check_call("tar -xzpf %s" % zipped_file, shell=True) elif zipped_file.endswith(".zip"): subprocess.check_call("unzip %s" % zipped_file, shell=True) elif zipped_file.endswith(".gz"): if not os.path.exists("out.fa"): subprocess.check_call("gunzip -c %s > out.fa" % zipped_file, shell=True) else: raise ValueError("Do not know how to handle: %s" % zipped_file) tmp_file = genome_file.replace(".fa", ".txt") result = subprocess.check_output("find `pwd` -name '*.fa'", shell=True).decode() result = [x.strip() for x in result.split("\n")] if len(result) == 1: orig_result = result[0] result = self._split_multifasta(result[0]) subprocess.check_call("rm %s" % orig_result, shell=True) result = self._karyotype_sort(result) subprocess.check_call("rm -f inputs.txt", shell=True) for fname in result: subprocess.check_output("echo '%s' >> inputs.txt" % fname, shell=True).decode() subprocess.check_call("cat `cat inputs.txt` > %s" % (tmp_file), shell=True) for fname in result: subprocess.check_output("rm -f %s" % fname, shell=True).decode() subprocess.check_call("mv %s %s" % (tmp_file, genome_file), shell=True) zipped_file = os.path.join(prep_dir, zipped_file) genome_file = os.path.join(prep_dir, genome_file) return genome_file, [zipped_file] def _download_zip(self, seq_dir): for zipped_file in ["chromFa.tar.gz", "%s.fa.gz" % self._name, "chromFa.zip"]: if not self._exists(zipped_file, seq_dir): result = shared._remote_fetch(None, "%s/%s" % (self._url, zipped_file), allow_fail=True) if result: break else: break return zipped_file class NCBIRest(_DownloadHelper): """Retrieve files using the TogoWS REST server pointed at NCBI. """ def __init__(self, name, refs, dl_name=None): _DownloadHelper.__init__(self) self.data_source = "NCBI" self._name = name self._refs = refs self.dl_name = dl_name if dl_name is not None else name self._base_url = "http://togows.dbcls.jp/entry/ncbi-nucleotide/%s.fasta" def download(self, seq_dir): genome_file = "%s.fa" % self._name if not self._exists(genome_file, seq_dir): for ref in self._refs: shared._remote_fetch(None, self._base_url % ref) subprocess.check_call("ls -l", shell=True) subprocess.check_call(r"sed -i 's/^>.*$/>%s/' %s.fasta" % (ref, ref), shell=True) tmp_file = genome_file.replace(".fa", ".txt") subprocess.check_call("cat *.fasta > %s" % tmp_file, shell=True) subprocess.check_call("rm -f *.fasta", shell=True) subprocess.check_call("rm -f *.bak", shell=True) subprocess.check_call("mv %s %s" % (tmp_file, genome_file), shell=True) return genome_file, [] class VectorBase(_DownloadHelper): """Retrieve genomes from VectorBase) """ def __init__(self, name, genus, species, strain, release, assembly_types): _DownloadHelper.__init__(self) self._name = name self.data_source = "VectorBase" self._base_url = ("http://www.vectorbase.org/sites/default/files/ftp/" "downloads/") _base_file = ("{genus}-{species}-{strain}_{assembly}" "_{release}.fa.gz") self._to_get = [] for assembly in assembly_types: self._to_get.append(_base_file.format(**locals())) def download(self, seq_dir): genome_file = "%s.fa" % self._name for fn in self._to_get: url = self._base_url + fn if not self._exists(fn, seq_dir): shared._remote_fetch(None, url) subprocess.check_call("gunzip -c %s >> %s" % (fn, genome_file), shell=True) return genome_file, [] class EnsemblGenome(_DownloadHelper): """Retrieve genome FASTA files from Ensembl. ftp://ftp.ensemblgenomes.org/pub/plants/release-22/fasta/ arabidopsis_thaliana/dna/Arabidopsis_thaliana.TAIR10.22.dna.toplevel.fa.gz ftp://ftp.ensembl.org/pub/release-75/fasta/ caenorhabditis_elegans/dna/Caenorhabditis_elegans.WBcel235.75.dna.toplevel.fa.gz ftp://ftp.ensemblgenomes.org/pub/bacteria/release-23/bacteria/fasta/ bacteria_17_collection/pseudomonas_aeruginosa_ucbpp_pa14/dna/ Pseudomonas_aeruginosa_ucbpp_pa14.GCA_000014625.1.23.dna.toplevel.fa.gz """ def __init__(self, ensembl_section, release, organism, name, subsection=None): _DownloadHelper.__init__(self) self.data_source = "Ensembl" if ensembl_section == "standard": url = "ftp://ftp.ensembl.org/pub/" else: url = "ftp://ftp.ensemblgenomes.org/pub/%s/" % ensembl_section url += "release-%s/fasta/" % release if subsection: url += "%s/" % subsection url += "%s/dna/" % organism.lower() self._url = url if ensembl_section == "standard": self._get_file = "%s.%s.dna.toplevel.fa.gz" % (organism, name) else: self._get_file = "%s.%s.%s.dna.toplevel.fa.gz" % (organism, name, release) self._name = name self.dl_name = name def download(self, seq_dir): genome_file = "%s.fa" % self._name if not self._exists(self._get_file, seq_dir): shared._remote_fetch(None, "%s%s" % (self._url, self._get_file)) if not self._exists(genome_file, seq_dir): subprocess.check_call("gunzip -c %s > %s" % (self._get_file, genome_file), shell=True) return genome_file, [self._get_file] class BroadGenome(_DownloadHelper): """Retrieve genomes organized and sorted by Broad for use with GATK. Uses the UCSC-name compatible versions of the GATK bundles. """ def __init__(self, name, target_fasta, dl_name=None): _DownloadHelper.__init__(self) self.data_source = "UCSC" self._name = name self.dl_name = dl_name if dl_name is not None else name self._target = target_fasta self._ftp_url = "ftp://gsapubftp-anonymous:@ftp.broadinstitute.org/bundle/" + \ "{org}/".format(org=self.dl_name) def download(self, seq_dir): org_file = "%s.fa" % self._name if not self._exists(org_file, seq_dir): shared._remote_fetch(None, "%s%s.gz" % (self._ftp_url, self._target)) subprocess.check_call("gunzip %s.gz" % self._target, shell=True) subprocess.check_call("mv %s %s" % (self._target, org_file), shell=True) return org_file, [] class GGDGenome: """Genome with download specified via a GGD recipe. """ def __init__(self, name): self._name = name GENOMES_SUPPORTED = [ ("phiX174", "phix", NCBIRest("phix", ["NC_001422.1"])), ("Scerevisiae", "sacCer3", UCSCGenome("sacCer3")), ("Mmusculus", "mm10", UCSCGenome("mm10")), ("Mmusculus", "mm9", UCSCGenome("mm9")), ("Mmusculus", "mm8", UCSCGenome("mm8")), ("Hsapiens", "hg18", BroadGenome("hg18", "Homo_sapiens_assembly18.fasta")), ("Hsapiens", "hg19", BroadGenome("hg19", "ucsc.hg19.fasta")), ("Hsapiens", "GRCh37", BroadGenome("GRCh37", "human_g1k_v37.fasta", "b37")), ("Hsapiens", "hg38", GGDGenome("hg38")), ("Hsapiens", "hg38-noalt", GGDGenome("hg38-noalt")), ("Rnorvegicus", "rn6", GGDGenome("rn6")), ("Rnorvegicus", "rn5", UCSCGenome("rn5")), ("Rnorvegicus", "rn4", UCSCGenome("rn4")), ("Xtropicalis", "xenTro3", UCSCGenome("xenTro3")), ("Athaliana", "TAIR10", EnsemblGenome("plants", "26", "Arabidopsis_thaliana", "TAIR10")), ("Dmelanogaster", "dm3", UCSCGenome("dm3")), ("Dmelanogaster", "BDGP6", GGDGenome("BDGP6")), ("Celegans", "WBcel235", EnsemblGenome("standard", "80", "Caenorhabditis_elegans", "WBcel235")), ("Mtuberculosis_H37Rv", "mycoTube_H37RV", NCBIRest("mycoTube_H37RV", ["NC_000962"])), ("Msmegmatis", "92", NCBIRest("92", ["NC_008596.1"])), ("Paeruginosa_UCBPP-PA14", "pseudomonas_aeruginosa_ucbpp_pa14", EnsemblGenome("bacteria", "26", "Pseudomonas_aeruginosa_ucbpp_pa14", "GCA_000014625.1", "bacteria_17_collection")), ("Ecoli", "eschColi_K12", NCBIRest("eschColi_K12", ["U00096.2"])), ("Amellifera_Honeybee", "apiMel3", UCSCGenome("apiMel3")), ("Cfamiliaris_Dog", "canFam3", UCSCGenome("canFam3")), ("Cfamiliaris_Dog", "canFam2", UCSCGenome("canFam2")), ("Drerio_Zebrafish", "Zv9", EnsemblGenome("standard", "80", "Danio_rerio", "Zv9")), ("Drerio_Zebrafish", "GRCz10", EnsemblGenome("standard", "81", "Danio_rerio", "GRCz10")), ("Drerio_Zebrafish", "GRCz11", EnsemblGenome("standard", "92", "Danio_rerio", "GRCz11")), ("Sscrofa", "Sscrofa11.1", EnsemblGenome("standard", "92", "Sus_scrofa", "Sscrofa11.1")), ("Ecaballus_Horse", "equCab2", UCSCGenome("equCab2")), ("Fcatus_Cat", "felCat3", UCSCGenome("felCat3")), ("Ggallus_Chicken", "galGal4", UCSCGenome("galGal4")), ("Tguttata_Zebra_finch", "taeGut1", UCSCGenome("taeGut1")), ("Aalbimanus", "AalbS1", VectorBase("AalbS1", "Anopheles", "albimanus", "STECLA", "AalbS1", ["SCAFFOLDS"])), ("Agambiae", "AgamP3", VectorBase("AgamP3", "Anopheles", "gambiae", "PEST", "AgamP3", ["CHROMOSOMES"]))] GENOME_INDEXES_SUPPORTED = ["bowtie", "bowtie2", "bwa", "maq", "minimap2", "novoalign", "novoalign-cs", "ucsc", "mosaik", "snap", "star", "rtg", "hisat2", "bbmap", "bismark"] DEFAULT_GENOME_INDEXES = ["seq"] # -- Fabric instructions def _check_version(env): version = env.version if int(version.split(".")[0]) < 1: raise NotImplementedError("Please install fabric version 1 or better") def install_data(config_source, approaches=None): """Main entry point for installing useful biological data, back compatible. """ from fabric.api import env _check_version(env) install_data_local(config_source, env.system_install, env.data_files, env.galaxy_home, env.tool_data_table_conf_file, env.cores, approaches) def install_data_local(config_source, system_installdir, data_filedir, galaxy_home=None, tool_data_table_conf_file=None, cores=None, approaches=None): """Local installation of biological data, avoiding fabric usage. """ if not cores: cores = 1 PREP_FNS = {"s3": _download_s3_index, "ggd": _install_with_ggd, "raw": _prep_raw_index} if approaches is None: approaches = ["ggd", "s3", "raw"] ready_approaches = [] Env = collections.namedtuple("Env", "system_install, galaxy_home, tool_data_table_conf_file, cores") env = Env(system_installdir, galaxy_home, tool_data_table_conf_file, cores) for approach in approaches: ready_approaches.append((approach, PREP_FNS[approach])) # Append a potentially custom system install path to PATH so tools are found os.environ["PATH"] = "%s/bin:%s" % (os.path.join(system_installdir), os.environ["PATH"]) genomes, genome_indexes, config = _get_genomes(config_source) genome_indexes = [x for x in DEFAULT_GENOME_INDEXES if x not in genome_indexes] + genome_indexes _make_genome_directories(genomes, data_filedir) rnaseq.cleanup(genomes, data_filedir) _prep_genomes(env, genomes, genome_indexes, ready_approaches, data_filedir) rnaseq.finalize(genomes, data_filedir) def install_data_s3(config_source): """Install data using pre-existing genomes present on Amazon s3. """ from fabric.api import env _check_version(env) genomes, genome_indexes, config = _get_genomes(config_source) genome_indexes += [x for x in DEFAULT_GENOME_INDEXES if x not in genome_indexes] _make_genome_directories(genomes, env.data_files) rnaseq.cleanup(genomes, env.data_files) _download_genomes(env, genomes, genome_indexes) rnaseq.finalize(genomes, env.data_files) _install_additional_data(env, genomes, genome_indexes, config) def install_data_rsync(config_source): """Install data using pre-existing genomes from Galaxy rsync servers. """ from fabric.api import env _check_version(env) genomes, genome_indexes, config = _get_genomes(config_source) genome_indexes += [x for x in DEFAULT_GENOME_INDEXES if x not in genome_indexes] # Galaxy stores FASTAs in ucsc format and generates on the fly if "ucsc" not in genome_indexes: genome_indexes.append("ucsc") genome_dir = _make_genome_dir(env.data_files) galaxy.rsync_genomes(genome_dir, genomes, genome_indexes) def upload_s3(config_source): """Upload prepared genome files by identifier to Amazon s3 buckets. """ from fabric.api import env if boto is None: raise ImportError("install boto to upload to Amazon s3") if env.host != "localhost" and not env.host.startswith(socket.gethostname()): raise ValueError("Need to run S3 upload on a local machine") _check_version(env) genomes, genome_indexes, config = _get_genomes(config_source) genome_indexes += [x for x in DEFAULT_GENOME_INDEXES if x not in genome_indexes] _data_ngs_genomes(env, genomes, genome_indexes) _upload_genomes(env, genomes, genome_indexes) def _install_additional_data(env, genomes, genome_indexes, config): for custom in (config.get("custom") or []): _prep_custom_genome(custom, genomes, genome_indexes, env) if config.get("install_liftover", False): lift_over_genomes = [g.ucsc_name() for (_, _, g) in genomes if g.ucsc_name()] _data_liftover(env, lift_over_genomes) if config.get("install_uniref", False): _data_uniref(env) def _get_genomes(config_source): if isinstance(config_source, dict): config = config_source else: if yaml is None: raise ImportError("install yaml to read configuration from %s" % config_source) with open(config_source) as in_handle: config = yaml.load(in_handle) genomes = [] genomes_config = config["genomes"] or [] print("List of genomes to get (from the config file at '{0}'): {1}" .format(config_source, ', '.join(g.get('name', g["dbkey"]) for g in genomes_config))) for g in genomes_config: ginfo = None for info in GENOMES_SUPPORTED: if info[1] == g["dbkey"]: ginfo = info break assert ginfo is not None, "Did not find download info for %s" % g["dbkey"] name, gid, manager = ginfo manager.config = g genomes.append((name, gid, manager)) indexes = config["genome_indexes"] or [] if "seq" in indexes: indexes.remove("seq") indexes.insert(0, "seq") return genomes, indexes, config # ## Decorators and context managers def _if_installed(pname): """Run if the given program name is installed. """ def argcatcher(func): def decorator(*args, **kwargs): envs = [x for x in args if hasattr(x, "system_install")] env = envs[0] if envs else None if shared.which(pname, env): return func(*args, **kwargs) return decorator return argcatcher # ## Generic preparation functions def _make_genome_dir(data_filedir): genome_dir = os.path.join(data_filedir, "genomes") subprocess.check_output("mkdir -p %s" % genome_dir, shell=True).decode() return genome_dir def _make_genome_directories(genomes, data_filedir): genome_dir = _make_genome_dir(data_filedir) for (orgname, gid, manager) in genomes: org_dir = os.path.join(genome_dir, orgname, gid) if not os.path.exists(org_dir): subprocess.check_call('mkdir -p %s' % org_dir, shell=True) def _prep_genomes(env, genomes, genome_indexes, retrieve_fns, data_filedir): """Prepare genomes with the given indexes, supporting multiple retrieval methods. """ genome_dir = _make_genome_dir(data_filedir) for (orgname, gid, manager) in genomes: org_dir = os.path.join(genome_dir, orgname, gid) if not os.path.exists(org_dir): subprocess.check_call('mkdir -p %s' % org_dir, shell=True) ggd_recipes = manager.config.get("annotations", []) + manager.config.get("validation", []) ggd_recipes += [x for x in manager.config.get("indexes", []) if x in genome_indexes] for idx in genome_indexes + ggd_recipes: with shared.chdir(org_dir): if idx in ggd_recipes or not os.path.exists(idx): finished = False last_exc = None for method, retrieve_fn in retrieve_fns: try: retrieve_fn(env, manager, gid, idx) finished = True break except KeyboardInterrupt: raise except BaseException as e: # Fail on incorrect GGD recipes if idx in ggd_recipes and method == "ggd": raise else: last_exc = traceback.format_exc() print("Moving on to next genome prep method after trying {0}\n{1}".format( method, str(e))) if not finished: raise IOError("Could not prepare index {0} for {1} by any method\n{2}" .format(idx, gid, last_exc)) ref_file = os.path.join(org_dir, "seq", "%s.fa" % gid) if not os.path.exists(ref_file): ref_file = os.path.join(org_dir, "seq", "%s.fa" % manager._name) assert os.path.exists(ref_file), ref_file _index_to_galaxy(env, org_dir, ref_file, gid, genome_indexes, manager.config) # ## Genomes index for next-gen sequencing tools def _get_ref_seq(manager): """Check for or retrieve the reference sequence. """ seq_dir = os.path.join(os.getcwd(), "seq") ref_file = os.path.join(seq_dir, "%s.fa" % manager._name) if not os.path.exists(ref_file): ref_file, base_zips = manager.download(seq_dir) ref_file = _move_seq_files(ref_file, base_zips, seq_dir) return ref_file def _prep_raw_index(env, manager, gid, idx): """Prepare genome from raw downloads and indexes. """ print("Preparing genome {0} with index {1}".format(gid, idx)) ref_file = _get_ref_seq(manager) get_index_fn(idx)(env, ref_file) def _data_ngs_genomes(env, genomes, genome_indexes): """Download and create index files for next generation genomes. """ genome_dir = _make_genome_dir(env.data_files) for organism, genome, manager in genomes: cur_dir = os.path.join(genome_dir, organism, genome) print("Processing genome {0} and putting it to {1}".format(organism, cur_dir)) if not os.path.exists(cur_dir): subprocess.check_call('mkdir -p %s' % cur_dir, shell=True) with shared.chdir(cur_dir): if hasattr(env, "remove_old_genomes") and env.remove_old_genomes: _clean_genome_directory() seq_dir = 'seq' ref_file, base_zips = manager.download(seq_dir) ref_file = _move_seq_files(ref_file, base_zips, seq_dir) cur_indexes = manager.config.get("indexes", genome_indexes) _index_to_galaxy(env, cur_dir, ref_file, genome, cur_indexes, manager.config) def _index_to_galaxy(env, work_dir, ref_file, gid, genome_indexes, config): """Index sequence files and update associated Galaxy loc files. """ indexes = {} with shared.chdir(work_dir): for idx in genome_indexes: index_file = get_index_fn(idx)(env, ref_file) if index_file: indexes[idx] = os.path.join(work_dir, index_file) galaxy.prep_locs(env, gid, indexes, config) class CustomMaskManager: """Create a custom genome based on masking an existing genome. """ def __init__(self, custom, config): assert "mask" in custom self._custom = custom self.config = config def download(self, seq_dir): base_seq = os.path.join(os.pardir, self._custom["base"], "seq", "{0}.fa".format(self._custom["base"])) assert os.path.exists(base_seq) mask_file = os.path.basename(self._custom["mask"]) ready_mask = apply("{0}-complement{1}".format, os.path.splitext(mask_file)) out_fasta = "{0}.fa".format(self._custom["dbkey"]) if not os.path.exists(os.path.join(seq_dir, out_fasta)): if not os.path.exists(mask_file): shared._remote_fetch(None, self._custom["mask"]) if not os.path.exists(ready_mask): subprocess.check_call("bedtools complement -i {i} -g {g}.fai > {o}".format( i=mask_file, g=base_seq, o=ready_mask), shell=True) if not os.path.exists(out_fasta): subprocess.check_call("bedtools maskfasta -fi {fi} -bed {bed} -fo {fo}".format( fi=base_seq, bed=ready_mask, fo=out_fasta), shell=True) return out_fasta, [mask_file, ready_mask] def _prep_custom_genome(custom, genomes, genome_indexes, env): """Prepare a custom genome derived from existing genome. Allows creation of masked genomes for specific purposes. """ cur_org = None cur_manager = None for org, gid, manager in genomes: if gid == custom["base"]: cur_org = org cur_manager = manager break assert cur_org is not None _data_ngs_genomes(env, [[cur_org, custom["dbkey"], CustomMaskManager(custom, cur_manager.config)]], genome_indexes) def _clean_genome_directory(): """Remove any existing sequence information in the current directory. """ for dirname in GENOME_INDEXES_SUPPORTED + DEFAULT_GENOME_INDEXES: if os.path.exists(dirname): subprocess.check_call("rm -rf %s" % dirname, shell=True) def _move_seq_files(ref_file, base_zips, seq_dir): if not os.path.exists(seq_dir): subprocess.check_call('mkdir %s' % seq_dir, shell=True) for move_file in [ref_file] + base_zips: if os.path.exists(move_file): subprocess.check_call("mv %s %s" % (move_file, seq_dir), shell=True) path, fname = os.path.split(ref_file) moved_ref = os.path.join(path, seq_dir, fname) assert os.path.exists(moved_ref), moved_ref return moved_ref # ## Indexing for specific aligners def _index_w_command(env, dir_name, command, ref_file, pre=None, post=None, ext=None): """Low level function to do the indexing and paths with an index command. """ path_export = _get_path_export(env) index_name = os.path.splitext(os.path.basename(ref_file))[0] if ext is not None: index_name += ext full_ref_path = os.path.join(os.pardir, ref_file) if not os.path.exists(dir_name): subprocess.check_call("mkdir %s" % dir_name, shell=True) with shared.chdir(dir_name): if pre: full_ref_path = pre(full_ref_path) subprocess.check_call(path_export + command.format(ref_file=full_ref_path, index_name=index_name), shell=True) if post: post(full_ref_path) return os.path.join(dir_name, index_name) @_if_installed("faToTwoBit") def _index_twobit(env, ref_file): """Index reference files using 2bit for random access. """ dir_name = "ucsc" cmd = "faToTwoBit {ref_file} {index_name}" return _index_w_command(env, dir_name, cmd, ref_file) def _index_bowtie(env, ref_file): dir_name = "bowtie" cmd = "bowtie-build -f {ref_file} {index_name}" return _index_w_command(env, dir_name, cmd, ref_file) def _index_bowtie2(env, ref_file): dir_name = "bowtie2" cmd = "bowtie2-build {ref_file} {index_name}" out_suffix = _index_w_command(env, dir_name, cmd, ref_file) bowtie_link = os.path.normpath(os.path.join(os.path.dirname(ref_file), os.path.pardir, out_suffix + ".fa")) relative_ref_file = os.path.relpath(ref_file, os.path.dirname(bowtie_link)) if not os.path.exists(bowtie_link): subprocess.check_call("ln -sf %s %s" % (relative_ref_file, bowtie_link), shell=True) return out_suffix def _index_bwa(env, ref_file): dir_name = "bwa" local_ref = os.path.split(ref_file)[-1] if not os.path.exists(os.path.join(dir_name, "%s.bwt" % local_ref)): subprocess.check_call("mkdir -p %s" % dir_name, shell=True) with shared.chdir(dir_name): subprocess.check_call("ln -sf %s" % os.path.join(os.pardir, ref_file), shell=True) try: subprocess.check_call("bwa index -a bwtsw %s" % local_ref, shell=True) except subprocess.CalledProcessError: # work around a bug in bwa indexing for small files subprocess.check_call("bwa index %s" % local_ref, shell=True) subprocess.check_call("rm -f %s" % local_ref, shell=True) return os.path.join(dir_name, local_ref) def _index_bbmap(env, ref_file): dir_name = "bbmap" try: cores = env.cores except: cores = 1 if not os.path.exists(os.path.join(dir_name, "ref", "genome", "1", "summary.txt")): subprocess.check_call("mkdir -p %s" % dir_name, shell=True) subprocess.check_call("bbmap.sh -Xms%sg -Xmx24g path=%s ref=%s" % (cores, dir_name, ref_file), shell=True) return dir_name def _index_bismark(env, ref_file): dir_name = "bismark" subprocess.check_call("mkdir -p %s" % dir_name, shell=True) out_dir = os.path.join(dir_name, "Bisulfite_Genome") if os.path.exists(out_dir): return out_dir with shared.chdir(dir_name): local = os.path.basename(ref_file) subprocess.check_call("ln -sf {0} {1}".format(ref_file, local), shell=True) cmd= f"bismark_genome_preparation ." subprocess.check_call(cmd, shell=True) return out_dir def _index_maq(env, ref_file): dir_name = "maq" cmd = "maq fasta2bfa {ref_file} {index_name}" def link_local(ref_file): local = os.path.basename(ref_file) subprocess.check_call("ln -sf {0} {1}".format(ref_file, local), shell=True) return local def rm_local(local_file): subprocess.check_call("rm -f {0}".format(local_file), shell=True) return _index_w_command(env, dir_name, cmd, ref_file, pre=link_local, post=rm_local) def _index_minimap2(env, ref_file): dir_name = "minimap2" indexes = [] for preset in ["sr"]: index_name = "%s-%s.mmi" % (os.path.splitext(os.path.basename(ref_file))[0], preset) cmd = "minimap2 -x %s -d %s {ref_file}" % (preset, index_name) out_basename = _index_w_command(env, dir_name, cmd, ref_file) indexes.append(os.path.join(os.path.dirname(out_basename), index_name)) return indexes[0] @_if_installed("novoindex") def _index_novoalign(env, ref_file): dir_name = "novoalign" cmd = "novoindex {index_name} {ref_file}" return _index_w_command(env, dir_name, cmd, ref_file) @_if_installed("novoalignCS") def _index_novoalign_cs(env, ref_file): dir_name = "novoalign_cs" cmd = "novoindex -c {index_name} {ref_file}" return _index_w_command(env, dir_name, cmd, ref_file) def _index_sam(env, ref_file): (ref_dir, local_file) = os.path.split(ref_file) with shared.chdir(ref_dir): if not os.path.exists("%s.fai" % local_file): subprocess.check_call("samtools faidx %s" % local_file, shell=True) galaxy.index_picard(ref_file) return ref_file @_if_installed("STAR") def _index_star(env, ref_file): (ref_dir, local_file) = os.path.split(ref_file) build = os.path.basename(os.path.splitext(ref_file)[0]) dir_name = os.path.normpath(os.path.join(ref_dir, os.pardir, "star")) sentinel_file = os.path.join(dir_name, "SA") if os.path.exists(sentinel_file): return dir_name if build == "hg38": simple_file = os.path.splitext(ref_file)[0] + "-simple.fa" print(f"hg38 detected, building a simple reference with no alts, decoys or HLA from {ref_file} to {simple_file}.") ref_file = prepare_simple_reference(ref_file, simple_file) GenomeLength = os.path.getsize(ref_file) Nbases = int(round(min(14, log(GenomeLength, 2) / 2 - 2), 0)) # if there is a large number of contigs, scale nbits down # https://github.com/alexdobin/STAR/issues/103#issuecomment-173009628 # if there is a small genome, scale nbits down # https://groups.google.com/forum/#!topic/rna-star/9g8Uoe1Igho cmd = 'grep ">" {ref_file} | wc -l'.format(ref_file=ref_file) nrefs = float(subprocess.check_output(cmd, shell=True).decode()) nbits = int(round(min(14, log(GenomeLength / nrefs, 2), log(GenomeLength, 2) / 2 - 1))) # first we estimate the number of bits we need to hold the genome and allocate # double that plus some padding to build the index mem = ((GenomeLength + 1) / nbits + 1) * nbits mem = (mem + 10000) * 2 mem = mem + mem / 3 mem = max(mem, 30000000000) try: cpu = env.cores except: cpu = 1 print(f"Preparing STAR index from {ref_file}.") cmd = ("STAR --genomeDir %s --genomeFastaFiles {ref_file} " "--runThreadN %s " "--limitGenomeGenerateRAM %s " "--genomeChrBinNbits %s " "--runMode genomeGenerate " "--genomeSAindexNbases %s" % (dir_name, str(cpu), str(mem), Nbases, nbits)) if not os.path.exists(os.path.join(dir_name, "SA")): _index_w_command(env, dir_name, cmd, ref_file) if build == "hg38": print(f"Removing {ref_file}.") os.remove(ref_file) return dir_name @_if_installed("hisat2-build") def _index_hisat2(env, ref_file): path_export = _get_path_export(env) build = os.path.splitext(os.path.basename(ref_file))[0] (ref_dir, local_file) = os.path.split(ref_file) gtf_file = os.path.join(ref_dir, os.pardir, "rnaseq", "ref-transcripts.gtf") dir_name = os.path.normpath(os.path.join(ref_dir, os.pardir, "hisat2")) index_prefix = os.path.join(dir_name, build) if os.path.exists(os.path.join(index_prefix + ".1.ht2")): return dir_name if not os.path.exists(dir_name): subprocess.check_call('mkdir -p %s' % dir_name, shell=True) try: cpu = env.cores except: cpu = 1 cmd = "{path_export}hisat2-build -p {cpu} " exons_file = index_prefix + ".exons" splicesites_file = index_prefix + ".splicesites" if os.path.exists(gtf_file): if not os.path.exists(exons_file): with open(exons_file, "w") as out_handle: exons_cmd = ["hisat2_extract_exons.py", gtf_file] subprocess.check_call(path_export + " ".join(exons_cmd), stdout=out_handle, shell=True) if not os.path.exists(splicesites_file): with open(splicesites_file, "w") as out_handle: splicesites_cmd = ["hisat2_extract_splice_sites.py", gtf_file] subprocess.check_call(path_export + " ".join(splicesites_cmd), stdout=out_handle, shell=True) if os.stat(exons_file).st_size > 0 and os.stat(splicesites_file).st_size > 0: cmd += "--exon {exons_file} --ss {splicesites_file} " cmd += "{ref_file} {index_prefix} " if not os.path.exists(os.path.join(index_prefix + ".1.ht2")): subprocess.check_call(cmd.format(**locals()), shell=True) return dir_name def _index_snap(env, ref_file): """Snap indexing is computationally expensive. Requests all cores and 64Gb of memory. """ dir_name = "snap" index_name = os.path.splitext(os.path.basename(ref_file))[0] org_arg = "-hg19" if index_name in ["hg19", "GRCh37"] else "" cmd = "snap-aligner index {ref_file} {dir_name} -bSpace {org_arg}" if not os.path.exists(os.path.join(dir_name, "GenomeIndex")): subprocess.check_call(cmd.format(**locals()), shell=True) return dir_name def _get_path_export(env): """Ensure PATH points to local install directory. """ path_export = "" if hasattr(env, "system_install") and env.system_install: local_bin = os.path.join(env.system_install, 'bin') if os.path.exists(local_bin): path_export = "export PATH=%s:$PATH && " % local_bin return path_export def _index_rtg(env, ref_file): """Perform indexing for use with Real Time Genomics tools. https://github.com/RealTimeGenomics/rtg-tools """ path_export = _get_path_export(env) dir_name = "rtg" index_name = "%s.sdf" % os.path.splitext(os.path.basename(ref_file))[0] if not os.path.exists(os.path.join(dir_name, index_name, "done")): cmd = ("{path_export}export RTG_JAVA_OPTS='-Xms1g' && export RTG_MEM=2g && " "rtg format -o {dir_name}/{index_name} {ref_file}") subprocess.check_call(cmd.format(**locals()), shell=True) return dir_name @_if_installed("MosaikJump") def _index_mosaik(env, ref_file): hash_size = 15 dir_name = "mosaik" cmd = "MosaikBuild -fr {ref_file} -oa {index_name}" def create_jumpdb(ref_file): jmp_base = os.path.splitext(os.path.basename(ref_file))[0] dat_file = "{0}.dat".format(jmp_base) if not os.path.exists("{0}_keys.jmp".format(jmp_base)): cmd = "export MOSAIK_TMP=`pwd` && MosaikJump -hs {hash_size} -ia {ref_file} -out {index_name}".format( hash_size=hash_size, ref_file=dat_file, index_name=jmp_base) subprocess.check_call(cmd, shell=True) return _index_w_command(env, dir_name, cmd, ref_file, post=create_jumpdb, ext=".dat") # -- Retrieve using GGD recipes def _install_with_ggd(env, manager, gid, recipe): recipe_dir = os.path.normpath(os.path.join(os.path.dirname(__file__), os.pardir, os.pardir, "ggd-recipes")) recipe_file = os.path.join(recipe_dir, gid, "%s.yaml" % recipe) if os.path.exists(recipe_file): ggd.install_recipe(os.getcwd(), env.system_install, recipe_file, gid) else: raise NotImplementedError("GGD recipe not available for %s %s" % (gid, recipe)) # -- Genome upload and download to Amazon s3 buckets def _download_s3_index(env, manager, gid, idx): print("Downloading genome from s3: {0} {1}".format(gid, idx)) url = "https://s3.amazonaws.com/biodata/genomes/%s-%s.tar.xz" % (gid, idx) if gid in ["GRCh37", "hg19", "mm10"] and idx in ["bowtie2", "bwa", "novoalign"]: out_file = shared._remote_fetch(env, url, samedir=True) subprocess.check_call("xz -dc %s | tar -xvpf -" % out_file, shell=True) subprocess.check_call("rm -f %s" % out_file, shell=True) else: raise NotImplementedError("No pre-computed indices for %s %s" % (gid, idx)) def _download_genomes(env, genomes, genome_indexes): """Download a group of genomes from Amazon s3 bucket. """ genome_dir = _make_genome_dir(env.data_files) for (orgname, gid, manager) in genomes: org_dir = os.path.join(genome_dir, orgname, gid) if not os.path.exists(org_dir): subprocess.check_call('mkdir -p %s' % org_dir, shell=True) for idx in genome_indexes: with shared.chdir(org_dir): if not os.path.exists(idx): _download_s3_index(env, manager, gid, idx) ref_file = os.path.join(org_dir, "seq", "%s.fa" % gid) if not os.path.exists(ref_file): ref_file = os.path.join(org_dir, "seq", "%s.fa" % manager._name) assert os.path.exists(ref_file), ref_file cur_indexes = manager.config.get("indexes", genome_indexes) _index_to_galaxy(env, org_dir, ref_file, gid, cur_indexes, manager.config) def _upload_genomes(env, genomes, genome_indexes): """Upload our configured genomes to Amazon s3 bucket. """ conn = boto.connect_s3() bucket = conn.create_bucket("biodata") genome_dir = os.path.join(env.data_files, "genomes") for (orgname, gid, _) in genomes: cur_dir = os.path.join(genome_dir, orgname, gid) _clean_directory(cur_dir, gid) for idx in genome_indexes: idx_dir = os.path.join(cur_dir, idx) tarball = _tar_directory(idx_dir, "%s-%s" % (gid, idx)) _upload_to_s3(tarball, bucket) bucket.make_public() def _upload_to_s3(tarball, bucket): """Upload the genome tarball to s3. """ upload_script = os.path.join(os.path.dirname(__file__), os.pardir, os.pardir, "utils", "s3_multipart_upload.py") s3_key_name = os.path.join("genomes", os.path.basename(tarball)) if not bucket.get_key(s3_key_name): gb_size = int(subprocess.check_output("du -sm %s" % tarball, shell=True).decode().split()[0]) / 1000.0 print("Uploading %s %.1fGb" % (s3_key_name, gb_size)) cl = ["python", upload_script, tarball, bucket.name, s3_key_name, "--public"] subprocess.check_call(cl) def _tar_directory(dir, tar_name): """Create a tarball of the directory. """ base_dir, tar_dir = os.path.split(dir) tarball = os.path.join(base_dir, "%s.tar.xz" % tar_name) if not os.path.exists(tarball): with shared.chdir(base_dir): subprocess.check_call("tar -cvpf - %s | xz -zc - > %s" % (tar_dir, os.path.basename(tarball)), shell=True) return tarball def _clean_directory(dir, gid): """Clean duplicate files from directories before tar and upload. """ # get rid of softlinks bowtie_ln = os.path.join(dir, "bowtie", "%s.fa" % gid) maq_ln = os.path.join(dir, "maq", "%s.fa" % gid) for to_remove in [bowtie_ln, maq_ln]: if os.path.exists(to_remove): subprocess.check_call("rm -f %s" % to_remove, shell=True) # remove any downloaded original sequence files remove_exts = ["*.gz", "*.zip"] with shared.chdir(os.path.join(dir, "seq")): for rext in remove_exts: fnames = subprocess.check_output("find . -name '%s'" % rext, shell=True).decode() for fname in (f.strip() for f in fnames.split("\n") if f.strip()): subprocess.check_call("rm -f %s" % fname, shell=True) # == Liftover files def _data_liftover(env, lift_over_genomes): """Download chain files for running liftOver. Does not install liftOver binaries automatically. """ lo_dir = os.path.join(env.data_files, "liftOver") if not os.path.exists(lo_dir): subprocess.check_call("mkdir %s" % lo_dir, shell=True) lo_base_url = "ftp://hgdownload.cse.ucsc.edu/goldenPath/%s/liftOver/%s" lo_base_file = "%sTo%s.over.chain.gz" for g1 in lift_over_genomes: for g2 in [g for g in lift_over_genomes if g != g1]: g2u = g2[0].upper() + g2[1:] cur_file = lo_base_file % (g1, g2u) non_zip = os.path.splitext(cur_file)[0] worked = False with shared.chdir(lo_dir): if not os.path.exists(non_zip): result = shared._remote_fetch(env, "%s" % (lo_base_url % (g1, cur_file)), allow_fail=True) # Lift over back and forths don't always exist # Only move forward if we found the file if result: worked = True subprocess.check_call("gunzip %s" % result, shell=True) if worked: ref_parts = [g1, g2, os.path.join(lo_dir, non_zip)] galaxy.update_loc_file(env, "liftOver.loc", ref_parts) # == UniRef def _data_uniref(env): """Retrieve and index UniRef databases for protein searches. http://www.ebi.ac.uk/uniref/ These are currently indexed for FASTA searches. Are other indexes desired? Should this be separated out and organized by program like genome data? This should also check the release note and automatically download and replace older versions. """ site = "ftp://ftp.uniprot.org" base_url = site + "/pub/databases/uniprot/" \ "current_release/uniref/%s/%s" for uniref_db in ["uniref50", "uniref90", "uniref100"]: work_dir = os.path.join(env.data_files, "uniref", uniref_db) if not os.path.exists(work_dir): subprocess.check_call("mkdir -p %s" % work_dir, shell=True) base_work_url = base_url % (uniref_db, uniref_db) fasta_url = base_work_url + ".fasta.gz" base_file = os.path.splitext(os.path.basename(fasta_url))[0] with shared.chdir(work_dir): if not os.path.exists(base_file): out_file = shared._remote_fetch(env, fasta_url) subprocess.check_call("gunzip %s" % out_file, shell=True) shared._remote_fetch(env, base_work_url + ".release_note") _index_blast_db(work_dir, base_file, "prot") def _index_blast_db(work_dir, base_file, db_type): """Index a database using blast+ for similary searching. """ type_to_ext = dict(prot=("phr", "pal"), nucl=("nhr", "nal")) db_name = os.path.splitext(base_file)[0] with shared.chdir(work_dir): if not reduce(operator.or_, (os.path.exists("%s.%s" % (db_name, ext)) for ext in type_to_ext[db_type])): subprocess.check_call("makeblastdb -in %s -dbtype %s -out %s" % (base_file, db_type, db_name), shell=True) def get_index_fn(index): """ return the index function for an index, if it is missing return a function that is a no-op """ def noop(env, ref_file): pass return INDEX_FNS.get(index, noop) def prepare_simple_reference(ref_file, out_file): """ given an hg38 FASTA file, create a FASTA file with no alts, HLA or decoys """ if os.path.exists(out_file): return out_file with open(ref_file + ".fai") as in_handle: chroms = [x.split()[0].strip() for x in in_handle] chroms = [x for x in chroms if not (is_alt(x) or is_decoy(x) or is_HLA(x))] cmd = ["samtools", "faidx", ref_file] + chroms with open(out_file, "w") as out_handle: subprocess.check_call(cmd, stdout=out_handle) return out_file def is_alt(chrom): return chrom.endswith("_alt") def is_decoy(chrom): return chrom.endswith("_decoy") def is_HLA(chrom): return chrom.startswith("HLA") INDEX_FNS = { "seq": _index_sam, "bbmap": _index_bbmap, "bismark": _index_bismark, "bwa": _index_bwa, "bowtie": _index_bowtie, "bowtie2": _index_bowtie2, "maq": _index_maq, "mosaik": _index_mosaik, "minimap2": _index_minimap2, "novoalign": _index_novoalign, "novoalign_cs": _index_novoalign_cs, "ucsc": _index_twobit, "twobit": _index_twobit, "star": _index_star, "snap": _index_snap, "rtg": _index_rtg, "hisat2": _index_hisat2 } ================================================ FILE: cloudbio/biodata/ggd.py ================================================ """Process GGD (Get Genomics Data) configurations for installation in biodata directories. Builds off work done by Aaron Quinlan to define and install genomic data: https://github.com/arq5x/ggd """ from __future__ import print_function import collections import contextlib from distutils.version import LooseVersion import os import shutil import subprocess import yaml def install_recipe(base_dir, system_install, recipe_file, genome_build): """Install data in a biodata directory given instructions from GGD YAML recipe. """ if not os.path.exists(base_dir): os.makedirs(base_dir) recipe = _read_recipe(recipe_file) if not version_uptodate(base_dir, recipe): if _has_required_programs(recipe["recipe"]["full"].get("required", [])): with tx_tmpdir(base_dir) as tmpdir: with chdir(tmpdir): print("Running GGD recipe: %s %s %s" % (genome_build, recipe["attributes"]["name"], recipe["attributes"]["version"])) _run_recipe(tmpdir, recipe["recipe"]["full"]["recipe_cmds"], recipe["recipe"]["full"]["recipe_type"], system_install) _move_files(tmpdir, base_dir, recipe["recipe"]["full"]["recipe_outfiles"]) add_version(base_dir, recipe) def _has_required_programs(programs): """Ensure the provided programs exist somewhere in the current PATH. http://stackoverflow.com/questions/377017/test-if-executable-exists-in-python """ def is_exe(fpath): return os.path.isfile(fpath) and os.access(fpath, os.X_OK) for p in programs: found = False for path in os.environ["PATH"].split(os.pathsep): path = path.strip('"') exe_file = os.path.join(path, p) if is_exe(exe_file): found = True break if not found: return False return True def _run_recipe(work_dir, recipe_cmds, recipe_type, system_install): """Create a bash script and run the recipe to download data. """ assert recipe_type == "bash", "Can only currently run bash recipes" run_file = os.path.join(work_dir, "ggd-run.sh") with open(run_file, "w") as out_handle: out_handle.write("#!/bin/bash\nset -eu -o pipefail\nexport PATH=%s/bin:$PATH\n" % system_install) out_handle.write("\n".join(recipe_cmds)) subprocess.check_output(["bash", run_file]) def _move_files(tmp_dir, final_dir, targets): for target in targets: if os.path.isdir(os.path.join(tmp_dir, target)): out_files = [os.path.join(target, f) for f in os.listdir(os.path.join(tmp_dir, target))] else: out_files = [target] for out_file in out_files: orig = os.path.join(tmp_dir, out_file) final = os.path.join(final_dir, out_file) assert os.path.exists(orig), ("Did not find expected output file %s in %s" % (out_file, tmp_dir)) cur_dir = os.path.dirname(final) if not os.path.exists(cur_dir): os.makedirs(cur_dir) if os.path.exists(final): os.remove(final) shutil.move(orig, final) def _read_recipe(in_file): in_file = os.path.abspath(os.path.expanduser(in_file)) with open(in_file) as in_handle: recipe = yaml.safe_load(in_handle) return recipe # ## Versioning def version_uptodate(base_dir, recipe): """Check if we have an up to date GGD installation in this directory. """ versions = _get_versions(base_dir) return (recipe["attributes"]["name"] in versions and LooseVersion(versions[recipe["attributes"]["name"]]) >= LooseVersion(str(recipe["attributes"]["version"]))) def add_version(base_dir, recipe): versions = _get_versions(base_dir) versions[recipe["attributes"]["name"]] = recipe["attributes"]["version"] with open(_get_version_file(base_dir), "w") as out_handle: for n, v in versions.items(): out_handle.write("%s,%s\n" % (n, v)) def _get_versions(base_dir): version_file = _get_version_file(base_dir) versions = collections.OrderedDict() if os.path.exists(version_file): with open(version_file) as in_handle: for line in in_handle: name, version = line.strip().split(",") versions[name] = version return versions def _get_version_file(base_dir): return os.path.join(base_dir, "versions.csv") # ## Transactional utilities @contextlib.contextmanager def tx_tmpdir(base_dir): """Context manager to create and remove a transactional temporary directory. """ tmp_dir = os.path.join(base_dir, "txtmp") if not os.path.exists(tmp_dir): os.makedirs(tmp_dir) yield tmp_dir shutil.rmtree(tmp_dir, ignore_errors=True) @contextlib.contextmanager def chdir(new_dir): """Context manager to temporarily change to a new directory. http://lucentbeing.com/blog/context-managers-and-the-with-statement-in-python/ """ cur_dir = os.getcwd() os.chdir(new_dir) try: yield finally: os.chdir(cur_dir) ================================================ FILE: cloudbio/biodata/rnaseq.py ================================================ """Infrastructure for RNA-seq supporting files. """ import os import subprocess from cloudbio.custom import shared def finalize(genomes, data_filedir): """Provide symlinks back to reference genomes so tophat avoids generating FASTA genomes. """ genome_dir = os.path.join(data_filedir, "genomes") for (orgname, gid, manager) in genomes: org_dir = os.path.join(genome_dir, orgname) for aligner in ["bowtie", "bowtie2"]: aligner_dir = os.path.join(org_dir, gid, aligner) if os.path.exists(aligner_dir): with shared.chdir(aligner_dir): for ext in ["", ".fai"]: orig_seq = os.path.join(os.pardir, "seq", "%s.fa%s" % (gid, ext)) if os.path.exists(orig_seq) and not os.path.exists(os.path.basename(orig_seq)): subprocess.check_call("ln -sf %s" % orig_seq, shell=True) def cleanup(genomes, data_filedir): """Cleanup for GGD recipe installation, removing old rnaseq symlinks. """ folder_name = "rnaseq" genome_dir = os.path.join(data_filedir, "genomes") for (orgname, gid, manager) in genomes: org_dir = os.path.join(genome_dir, orgname) target_dir = os.path.join(org_dir, gid, folder_name) if os.path.lexists(target_dir) and os.path.islink(target_dir): os.remove(target_dir) ================================================ FILE: cloudbio/cloudbiolinux.py ================================================ """CloudBioLinux specific scripts """ import os from fabric.api import * from fabric.contrib.files import * from cloudbio.custom import shared def _freenx_scripts(env): """Provide graphical access to clients via FreeNX. """ home_dir = env.safe_run_output("echo $HOME") setup_script = "setupnx.sh" bin_dir = shared._get_bin_dir(env) install_file_dir = os.path.join(env.config_dir, os.pardir, "installed_files") if not env.safe_exists(os.path.join(bin_dir, setup_script)): env.safe_put(os.path.join(install_file_dir, setup_script), os.path.join(home_dir, setup_script)) env.safe_run("chmod 0777 %s" % os.path.join(home_dir, setup_script)) env.safe_sudo("mv %s %s" % (os.path.join(home_dir, setup_script), bin_dir)) remote_login = "configure_freenx.sh" if not env.safe_exists(os.path.join(home_dir, remote_login)): env.safe_put(os.path.join(install_file_dir, 'bash_login'), os.path.join(home_dir, remote_login)) env.safe_run("chmod 0777 %s" % os.path.join(home_dir, remote_login)) _configure_gnome(env) def _cleanup_space(env): """Cleanup to recover space from builds and packages. """ env.logger.info("Cleaning up space from package builds") with settings(warn_only=True): env.safe_sudo("rm -rf .cpanm") env.safe_sudo("rm -f /var/crash/*") env.safe_run("rm -f ~/*.dot") env.safe_run("rm -f ~/*.log") def _configure_gnome(env): """Configure NX server to use classic GNOME. http://askubuntu.com/questions/50503/why-do-i-get-unity-instead-of-classic-when-using-nx http://notepad2.blogspot.com/2012/04/install-freenx-server-on-ubuntu-1110.html """ add = 'COMMAND_START_GNOME="gnome-session --session gnome-fallback"' fname = "/etc/nxserver/node.conf" if env.safe_exists("/etc/nxserver/"): env.safe_append(fname, add, use_sudo=True) ================================================ FILE: cloudbio/cloudman.py ================================================ """Build instructions associated with CloudMan. http://wiki.g2.bx.psu.edu/Admin/Cloud Adapted from Enis Afgan's code: https://bitbucket.org/afgane/mi-deployment """ cm_upstart = """ description "Start CloudMan contextualization script" start on runlevel [2345] task exec python %s 2> %s.log """ import os from fabric.api import sudo, cd, run, put from fabric.contrib.files import exists, settings from cloudbio.galaxy import _setup_users from cloudbio.flavor.config import get_config_file from cloudbio.package.shared import _yaml_to_packages from cloudbio.custom.shared import (_make_tmp_dir, _write_to_file, _get_install, _configure_make, _if_not_installed, _setup_conf_file, _add_to_profiles, _create_python_virtualenv, _setup_simple_service, _read_boolean) from cloudbio.package.deb import (_apt_packages, _setup_apt_automation) MI_REPO_ROOT_URL = "https://bitbucket.org/afgane/mi-deployment/raw/tip" CM_REPO_ROOT_URL = "https://bitbucket.org/galaxy/cloudman/raw/tip" def _configure_cloudman(env, use_repo_autorun=False): """ Configure the machine to be capable of running CloudMan. ..Also see: ``custom/cloudman.py`` """ env.logger.debug("Configuring CloudMan") _setup_users(env) _setup_env(env) _configure_logrotate(env) _configure_ec2_autorun(env, use_repo_autorun) _configure_sge(env) _configure_hadoop(env) _configure_nfs(env) _configure_novnc(env) _configure_desktop(env) install_s3fs(env) def _configure_desktop(env): """ Configure a desktop manager to work with VNC. Note that `xfce4` (or `jwm`) and `vnc4server` packages need to be installed for this to have effect. """ if not _read_boolean(env, "configure_desktop", False): return # Set nginx PAM module to allow logins for any system user if env.safe_exists("/etc/pam.d"): env.safe_sudo('echo "@include common-auth" > /etc/pam.d/nginx') env.safe_sudo('usermod -a -G shadow galaxy') # Create a start script for X _setup_conf_file(env, "/home/ubuntu/.vnc/xstartup", "xstartup", default_source="xstartup") # Create jwmrc config file (uncomment this if using jwm window manager) # _setup_conf_file(env, "/home/ubuntu/.jwmrc", "jwmrc.xml", # default_source="jwmrc.xml", mode="0644") env.logger.info("----- Done configuring desktop -----") def _configure_novnc(env): if not _read_boolean(env, "configure_novnc", False): # Longer term would like this enabled by default. -John return if not "novnc_install_dir" in env: env.novnc_install_dir = "/opt/novnc" if not "vnc_password" in env: env.vnc_password = "cl0udbi0l1nux" if not "vnc_user" in env: env.vnc_user = env.user if not "vnc_display" in env: env.vnc_display = "1" if not "vnc_depth" in env: env.vnc_depth = "16" if not "vnc_geometry" in env: env.vnc_geometry = "1024x768" _configure_vncpasswd(env) novnc_dir = env.novnc_install_dir env.safe_sudo("mkdir -p '%s'" % novnc_dir) env.safe_sudo("chown %s '%s'" % (env.user, novnc_dir)) clone_cmd = "NOVNC_DIR='%s'; rm -rf $NOVNC_DIR; git clone https://github.com/kanaka/noVNC.git $NOVNC_DIR" % novnc_dir run(clone_cmd) ## Move vnc_auto.html which takes vnc_password as query argument ## to index.html and rewrite it so that password is autoset, no ## need to specify via query parameter. run("sed s/password\\ =\\ /password\\ =\\ \\\'%s\\\'\\;\\\\\\\\/\\\\\\\\// '%s/vnc_auto.html' > '%s/index.html'" % (env.vnc_password, novnc_dir, novnc_dir)) _setup_conf_file(env, "/etc/init.d/novnc", "novnc_init", default_source="novnc_init") _setup_conf_file(env, "/etc/default/novnc", "novnc_default", default_source="novnc_default.template") _setup_conf_file(env, "/etc/init.d/vncserver", "vncserver_init", default_source="vncserver_init") _setup_conf_file(env, "/etc/default/vncserver", "vncserver_default", default_source="vncserver_default.template") _setup_simple_service("novnc") _setup_simple_service("vncserver") def _configure_vncpasswd(env): with cd("~"): run("mkdir -p ~/.vnc") run("rm -rf vncpasswd") run("git clone https://github.com/trinitronx/vncpasswd.py vncpasswd") run("python vncpasswd/vncpasswd.py '%s' -f ~/.vnc/passwd" % env.vnc_password) run("chmod 600 ~/.vnc/passwd") run("rm -rf vncpasswd") def _setup_env(env): """ Setup the system environment required to run CloudMan. This means installing required system-level packages (as defined in CBL's ``packages.yaml``, or a flavor thereof) and Python dependencies (i.e., libraries) as defined in CloudMan's ``requirements.txt`` file. """ # Get and install required system packages if env.distribution in ["debian", "ubuntu"]: config_file = get_config_file(env, "packages.yaml") (packages, _) = _yaml_to_packages(config_file.base, 'cloudman') # Allow flavors to modify the package list packages = env.flavor.rewrite_config_items("packages", packages) _setup_apt_automation() _apt_packages(pkg_list=packages) elif env.distribution in ["centos", "scientificlinux"]: env.logger.warn("No CloudMan system package dependencies for CentOS") pass # Get and install required Python libraries with _make_tmp_dir() as work_dir: with cd(work_dir): url = os.path.join(CM_REPO_ROOT_URL, 'requirements.txt') _create_python_virtualenv(env, 'CM', reqs_url=url) # Add a custom vimrc vimrc_url = os.path.join(MI_REPO_ROOT_URL, 'conf_files', 'vimrc') remote_file = '/etc/vim/vimrc' if env.safe_exists("/etc/vim"): env.safe_sudo("wget --output-document=%s %s" % (remote_file, vimrc_url)) env.logger.debug("Added a custom vimrc to {0}".format(remote_file)) # Setup profile aliases = ['alias lt="ls -ltr"', 'alias ll="ls -l"'] for alias in aliases: _add_to_profiles(alias, ['/etc/bash.bashrc']) env.logger.info("Done setting up CloudMan's environment") def _configure_logrotate(env): """ Add logrotate config file, which will automatically rotate CloudMan's log """ conf_file = "cloudman.logrotate" remote = '/etc/logrotate.d/cloudman' url = os.path.join(MI_REPO_ROOT_URL, 'conf_files', conf_file) env.safe_sudo("wget --output-document=%s %s" % (remote, url)) env.logger.info("----- Added logrotate file to {0} -----".format(remote)) def _configure_ec2_autorun(env, use_repo_autorun=False): """ ec2autorun.py is a script that launches CloudMan on instance boot and is thus required on an instance. See the script itself for the details of what it does. This script also adds a cloudman service to ``/etc/init``, which actually runs ec2autorun.py as a system-level service at system boot. """ script = "ec2autorun.py" remote = os.path.join(env.install_dir, "bin", script) if not env.safe_exists(os.path.dirname(remote)): env.safe_sudo('mkdir -p {0}'.format(os.path.dirname(remote))) if use_repo_autorun: # Is this used, can we eliminate use_repo_autorun? url = os.path.join(MI_REPO_ROOT_URL, script) env.safe_sudo("wget --output-document=%s %s" % (remote, url)) else: install_file_dir = os.path.join(env.config_dir, os.pardir, "installed_files") tmp_remote = os.path.join("/tmp", os.path.basename(remote)) env.safe_put(os.path.join(install_file_dir, script), tmp_remote) env.safe_sudo("mv %s %s" % (tmp_remote, remote)) env.safe_sudo("chmod 0777 %s" % remote) # Create upstart configuration file for boot-time script cloudman_boot_file = 'cloudman.conf' remote_file = '/etc/init/%s' % cloudman_boot_file _write_to_file(cm_upstart % (remote, os.path.splitext(remote)[0]), remote_file, mode="0644") # Setup default image user data (if configured by image_user_data_path or # image_user_data_template_path). This specifies defaults for CloudMan when # used with resulting image, normal userdata supplied by user will override # these defaults. image_user_data_path = os.path.join(env.install_dir, "bin", "IMAGE_USER_DATA") if "image_user_data_dict" in env: # Explicit YAML contents defined in env, just dump them as is. import yaml _write_to_file(yaml.dump(env.get("image_user_data_dict")), image_user_data_path, mode="0644") else: # Else use file or template file. _setup_conf_file(env, image_user_data_path, "image_user_data", default_source="image_user_data") env.logger.info("Done configuring CloudMan's ec2_autorun") def _configure_sge(env): """ This method sets up the environment for SGE w/o actually setting up SGE; it basically makes sure system paths expected by CloudMan exist on the system. TODO: Merge this with ``install_sge`` method in ``custom/cloudman.py``. """ sge_root = '/opt/sge' if not env.safe_exists(sge_root): env.safe_sudo("mkdir -p %s" % sge_root) env.safe_sudo("chown sgeadmin:sgeadmin %s" % sge_root) # Link our installed SGE to CloudMan's expected directory sge_package_dir = "/opt/galaxy/pkg" sge_dir = "ge6.2u5" if not env.safe_exists(os.path.join(sge_package_dir, sge_dir)): env.safe_sudo("mkdir -p %s" % sge_package_dir) if not env.safe_exists(os.path.join(sge_package_dir, sge_dir)): env.safe_sudo("ln --force -s %s/%s %s/%s" % (env.install_dir, sge_dir, sge_package_dir, sge_dir)) env.logger.info("Done configuring SGE for CloudMan") def _configure_hadoop(env): """ Grab files required by CloudMan to setup a Hadoop cluster atop SGE. """ hadoop_root = '/opt/hadoop' url_root = 'https://s3.amazonaws.com/cloudman' hcm_file = 'hadoop.1.0.4__1.0.tar.gz' si_file = 'sge_integration.1.0.tar.gz' # Make sure we're working with a clean hadoop_home dir to avoid any version conflicts env.safe_sudo("rm -rf {0}".format(hadoop_root)) env.safe_sudo("mkdir -p %s" % hadoop_root) with cd(hadoop_root): env.safe_sudo("wget --output-document={0} {1}/{0}".format(hcm_file, url_root)) env.safe_sudo("wget --output-document={0} {1}/{0}".format(si_file, url_root)) env.safe_sudo("chown -R {0} {1}".format(env.user, hadoop_root)) env.logger.info("Done configuring Hadoop for CloudMan") def _configure_nfs(env): """ Edit ``/etc/exports`` to append paths that are shared over NFS by CloudMan. In addition to the hard coded paths listed here, additional paths can be included by setting ``extra_nfs_exports`` in ``fabricrc.txt`` as a comma-separated list of directories. """ nfs_dir = "/export/data" cloudman_dir = "/mnt/galaxy/export" if not env.safe_exists(nfs_dir): # For the case of rerunning this script, ensure the nfs_dir does # not exist (exists() method does not recognize it as a file because # by default it points to a non-existing dir/file). with settings(warn_only=True): env.safe_sudo('rm -rf {0}'.format(nfs_dir)) env.safe_sudo("mkdir -p %s" % os.path.dirname(nfs_dir)) env.safe_sudo("ln -s %s %s" % (cloudman_dir, nfs_dir)) env.safe_sudo("chown -R %s %s" % (env.user, os.path.dirname(nfs_dir))) # Setup /etc/exports paths, to be used as NFS mount points # galaxy_data_mount = env.get("galaxy_data_mount", "/mnt/galaxyData") # galaxy_indices_mount = env.get("galaxy_indices_mount", "/mnt/galaxyIndices") # galaxy_tools_mount = env.get("galaxy_tools_mount", "/mnt/galaxyTools") exports = ['/opt/sge *(rw,sync,no_root_squash,no_subtree_check)', '/opt/hadoop *(rw,sync,no_root_squash,no_subtree_check)', # '%s *(rw,sync,no_root_squash,subtree_check,no_wdelay)' % galaxy_data_mount, # '%s *(rw,sync,no_root_squash,no_subtree_check)' % galaxy_indices_mount, # '%s *(rw,sync,no_root_squash,no_subtree_check)' % galaxy_tools_mount, # '%s *(rw,sync,no_root_squash,no_subtree_check)' % nfs_dir, # '%s/openmpi *(rw,sync,no_root_squash,no_subtree_check)' % env.install_dir ] extra_nfs_exports = env.get("extra_nfs_exports", "") if extra_nfs_exports: for extra_nfs_export in extra_nfs_exports.split(","): exports.append('%s *(rw,sync,no_root_squash,no_subtree_check)' % extra_nfs_export) env.safe_append('/etc/exports', exports, use_sudo=True) # Create a symlink for backward compatibility where all of CloudMan's # stuff is expected to be in /opt/galaxy old_dir = '/opt/galaxy' # Because stow is used, the equivalent to CloudMan's expected path # is actually the parent of the install_dir so use it for the symlink new_dir = os.path.dirname(env.install_dir) if not env.safe_exists(old_dir) and exists(new_dir): env.safe_sudo('ln -s {0} {1}'.format(new_dir, old_dir)) env.logger.info("Done configuring NFS for CloudMan") @_if_not_installed("s3fs") def install_s3fs(env): """ Install s3fs, allowing S3 buckets to be mounted as ~POSIX file systems """ default_version = "1.61" version = env.get("tool_version", default_version) url = "http://s3fs.googlecode.com/files/s3fs-%s.tar.gz" % version _get_install(url, env, _configure_make) def _cleanup_ec2(env): """ Clean up any extra files after building. This method must be called on an instance after being built and before creating a new machine image. *Note* that after this method has run, key-based ssh access to the machine is no longer possible. """ env.logger.info("Cleaning up for EC2 AMI creation") # Clean up log files and such fnames = [".bash_history", "/var/log/firstboot.done", ".nx_setup_done", "/var/crash/*", "%s/ec2autorun.py.log" % env.install_dir, "%s/ec2autorun.err" % env.install_dir, "%s/ec2autorun.log" % env.install_dir, "%s/bin/ec2autorun.log" % env.install_dir] for fname in fnames: sudo("rm -f %s" % fname) rmdirs = ["/mnt/galaxyData", "/mnt/cm", "/tmp/cm"] for rmdir in rmdirs: sudo("rm -rf %s" % rmdir) # Seed the history with frequently used commands env.logger.debug("Setting bash history") local = os.path.join(env.config_dir, os.pardir, "installed_files", "bash_history") remote = os.path.join('/home', 'ubuntu', '.bash_history') put(local, remote, mode="0660", use_sudo=True) # Make sure the default config dir is owned by ubuntu sudo("chown ubuntu:ubuntu ~/.config") # Stop Apache from starting automatically at boot (it conflicts with Galaxy's nginx) sudo('/usr/sbin/update-rc.d -f apache2 remove') with settings(warn_only=True): # RabbitMQ fails to start if its database is embedded into the image # because it saves the current IP address or host name so delete it now. # When starting up, RabbitMQ will recreate that directory. sudo('/etc/init.d/rabbitmq-server stop') sudo('service rabbitmq-server stop') # Clean up packages that are causing issues or are unnecessary pkgs_to_remove = ['tntnet', 'tntnet-runtime', 'libtntnet9', 'vsftpd'] for ptr in pkgs_to_remove: sudo('apt-get -y --force-yes remove --purge {0}'.format(ptr)) sudo('initctl reload-configuration') for db_location in ['/var/lib/rabbitmq/mnesia', '/mnesia']: if exists(db_location): sudo('rm -rf %s' % db_location) # remove existing ssh host key pairs # http://docs.amazonwebservices.com/AWSEC2/latest/UserGuide/AESDG-chapter-sharingamis.html sudo("rm -f /etc/ssh/ssh_host_*") sudo("rm -f ~/.ssh/authorized_keys*") sudo("rm -f /root/.ssh/authorized_keys*") ================================================ FILE: cloudbio/config_management/__init__.py ================================================ """ This module contains code related to integrating the configuration management tools `chef` and `puppet` into CloudBioLinux. """ ================================================ FILE: cloudbio/config_management/chef.py ================================================ import os import json from fabric.api import cd from fabric.contrib import files from fabric.state import _AttributeDict from cloudbio.flavor.config import get_config_file from utils import build_properties, upload_config, config_dir # Code based heavily on fabric-provision. https://github.com/caffeinehit/fabric-provision DEFAULTS = dict( path='/var/chef', data_bags=config_dir(os.path.join('chef', 'data_bags')), roles=config_dir(os.path.join('chef', 'roles')), cookbooks=config_dir(os.path.join('chef', 'cookbooks')), log_level='info', recipes=[], run_list=[], json={}, ) SOLO_RB = """ log_level :%(log_level)s log_location STDOUT file_cache_path "%(path)s" data_bag_path "%(path)s/data_bags" role_path [ "%(path)s/roles" ] cookbook_path [ "%(path)s/cookbooks" ] Chef::Log::Formatter.show_time = true """ class ChefDict(_AttributeDict): def add_recipe(self, recipe): self.run_list.append('recipe[{0}]'.format(recipe)) def add_role(self, role): self.run_list.append('role[{0}]'.format(role)) def _get_json(self): the_json = self['json'].copy() the_json['run_list'] = self['run_list'] return the_json json = property(fget=_get_json) chef = ChefDict(DEFAULTS) def omnibus(env): """ Install Chef from Opscode's Omnibus installer """ ctx = { 'filename': '%(path)s/install-chef.sh' % chef, 'url': 'http://opscode.com/chef/install.sh', } if not files.exists(ctx['filename']): env.safe_sudo('wget -O %(filename)s %(url)s' % ctx) with cd(chef.path): env.safe_sudo('bash install-chef.sh') def _chef_provision(env, _omnibus=True): env.safe_sudo('mkdir -p %(path)s' % chef) omnibus(env) config_files = {'node.json': json.dumps(chef.json), 'solo.rb': SOLO_RB % chef} upload_config(chef, config_folder_names=['cookbooks', 'data_bags', 'roles'], config_files=config_files) with cd(chef.path): env.safe_sudo('chef-solo -c solo.rb -j node.json') def _configure_chef(env, chef): # Set node json properties node_json_path = get_config_file(env, "node_extra.json").base chef.json = _build_chef_properties(env, node_json_path) # Set whether to use the Opscode Omnibus Installer to load Chef. use_omnibus_installer_str = env.get("use_chef_omnibus_installer", "false") chef.use_omnibus_installer = use_omnibus_installer_str.upper() in ["TRUE", "YES"] def _build_chef_properties(env, config_file): """ Build python object representation of the Chef-solo node.json file from node_extra.json in config dir and the fabric environment. """ json_properties = _parse_json(config_file) return build_properties(env, "chef", json_properties) def _parse_json(filename): """ Parse a JSON file First remove comments and then use the json module package Comments look like : // ... """ with open(filename) as f: lines = f.readlines() content = ''.join([line for line in lines if not line.startswith('//')]) return json.loads(content) ================================================ FILE: cloudbio/config_management/puppet.py ================================================ from fabric.state import _AttributeDict from fabric.api import cd from utils import upload_config, config_dir, build_properties from cloudbio.package.deb import _apt_packages import os DEFAULTS = dict( path='/var/puppet', log_level='info', modules=config_dir(os.path.join('puppet', 'modules')) ) puppet = _AttributeDict(DEFAULTS) def _puppet_provision(env, classes): env.safe_sudo('mkdir -p %(path)s' % puppet) manifest_body = "node default {\n%s\n}\n" % _build_node_def_body(env, classes) config_files = {"manifest.pp": manifest_body} upload_config(puppet, config_folder_names=["modules"], config_files=config_files) # TODO: Allow yum based install _apt_packages(pkg_list=["puppet"]) with cd(puppet.path): env.safe_sudo("sudo puppet apply --modulepath=modules manifest.pp") def _build_node_def_body(env, classes): contents = "" properties = build_properties(env, "puppet") contents += "\n".join(["$%s = '%s'" % (key, value.replace("'", r"\'")) for key, value in properties.iteritems()]) contents += "\n" contents += "\n".join([_build_class_include(env, class_name) for class_name in classes]) return contents def _build_class_include(env, class_name): """ If parentns::classname is included and fabric properties such as puppet_parentns__classname_prop = val1 are set, the class included in puppet will be something like class { 'parentns::classname': prop => 'val1', } """ include_def = "class { '%s': \n" % class_name property_prefix = _property_prefix(class_name) for name, value in env.iteritems(): if name.startswith(property_prefix): property_name = name[len(property_prefix):] if not property_name.startswith("_"): # else subclass property include_def += " %s => '%s',\n" % (property_name, value) include_def += "\n}" return include_def def _property_prefix(class_name): return "puppet_%s_" % class_name.replace("::", "__") ================================================ FILE: cloudbio/config_management/utils.py ================================================ from tempfile import mkdtemp import os from fabric.api import settings, local, put, sudo, cd from fabric.contrib import files def config_dir(relative_path): cloudbiolinux_dir = os.path.join(os.path.dirname(__file__), os.pardir, os.pardir) return os.path.join(cloudbiolinux_dir, "config", relative_path) def build_properties(env, prefix, overrides={}): # Prefix will be either chef or puppet prefix = "%s_" % prefix # Clone fresh dictonary to modify overrides = dict(overrides) # Load fabric environment properties into properties. for key, value in env.iteritems(): # Skip invalid properties. if key in overrides or not isinstance(value, str): continue if key.startswith(prefix): # If a property starts with chef_ assume it is meant for chef and # add without this prefix. So chef_apache_dir would be available # as apache_dir. overrides[key[len(prefix):]] = value else: # Otherwise, allow chef to access property anyway but prefix with # cloudbiolinux_ so it doesn't clash with anything explicitly # configured for chef. overrides["cloudbiolinux_%s" % key] = value return overrides def upload_config(config, config_folder_names=[], config_files={}): """ Common code to upload puppet and chef config files to remote server. Heavily based on upload procedure from fabric-provision: https://github.com/caffeinehit/fabric-provision/blob/master/provision/__init__.py """ names = config_folder_names + config_files.keys() ctx = dict(map(lambda name: (name, '%s/%s' % (config.path, name)), names)) tmpfolder = mkdtemp() listify = lambda what: what if isinstance(what, list) else [what] for folder_name in config_folder_names: setattr(config, folder_name, listify(getattr(config, folder_name))) for folder_name in config_folder_names: local('mkdir %s/%s' % (tmpfolder, folder_name)) def copyfolder(folder, what): if not os.path.exists(folder): os.makedirs(folder) with settings(warn_only=True): local('cp -r %(folder)s/* %(tmpfolder)s/%(what)s' % dict( folder=folder, tmpfolder=tmpfolder, what=what)) for what in config_folder_names: map(lambda f: copyfolder(f, what), getattr(config, what)) folder_paths = " ".join(map(lambda folder_name: "./%s" % folder_name, config_folder_names)) local('cd %s && tar -f config_dir.tgz -cz %s' % (tmpfolder, folder_paths)) # Get rid of old files with settings(warn_only=True): map(lambda what: sudo("rm -rf '%s'" % ctx[what]), ctx.keys()) # Upload put('%s/config_dir.tgz' % tmpfolder, config.path, use_sudo=True) with cd(config.path): sudo('tar -xf config_dir.tgz') for file, contents in config_files.iteritems(): files.append(ctx[file], contents, use_sudo=True) ================================================ FILE: cloudbio/custom/__init__.py ================================================ """Fabric sub-modules providing custom installation for non-packaged programs. """ ================================================ FILE: cloudbio/custom/bio_general.py ================================================ """Custom installs for biological packages. """ import os from fabric.api import * from fabric.contrib.files import * from cloudbio.custom import shared from shared import (_if_not_installed, _get_install, _configure_make, _java_install, _make_tmp_dir) def install_anaconda(env): """Pre-packaged Anaconda Python installed from Continuum. http://docs.continuum.io/anaconda/index.html """ version = "2.0.0" outdir = os.path.join(env.system_install, "anaconda") if env.distribution in ["ubuntu", "centos", "scientificlinux", "debian", "arch", "suse"]: platform = "Linux" elif env.distribution in ["macosx"]: platform = "MacOSX" else: raise ValueError("Unexpected distribution: %s" % env.distribution) url = "http://09c8d0b2229f813c1b93-c95ac804525aac4b6dba79b00b39d1d3.r79.cf1.rackcdn.com/" \ "Anaconda-%s-%s-x86_64.sh" % (version, platform) if not env.safe_exists(outdir): with _make_tmp_dir() as work_dir: with cd(work_dir): installer = shared._remote_fetch(env, url) env.safe_sed(os.path.basename(url), "more < %s """ % (share_dir, executable)) env.safe_sudo("chmod +x '%s'" % executable) _unzip_install("mayu", version, url, env, install_fn) def install_pride_inspector(env): default_version = "1.3.0" version = env.get("tool_version", default_version) url = "http://pride-toolsuite.googlecode.com/files/pride-inspector-%s.zip" % version def install_fn(env, install_dir): _get_gist_script(env, "https://gist.github.com/jmchilton/5474788/raw/6bcffd8680ec0e0301af44961184529a1f76dd3b/pride-inspector") # Hack for multi-user environment. env.safe_sudo("chmod -R o+w log config") env.safe_sudo("mv * '%s'" % install_dir) bin_dir = os.path.join(env.get("system_install"), "bin") env.safe_sudo("mkdir -p '%s'" % bin_dir) env.safe_sudo("ln -s '%s' %s" % (os.path.join(install_dir, "pride-inspector"), os.path.join(bin_dir, "pride-inspector"))) _unzip_install("pride_inspector", version, url, env, install_fn, "PRIDE_Inspector") def install_pride_converter2(env): default_version = "2.0.17" version = env.get("tool_version", default_version) url = "http://pride-converter-2.googlecode.com/files/pride-converter-%s-bin.zip" % version def install_fn(env, install_dir): _get_gist_script(env, "https://gist.github.com/jmchilton/5475119/raw/4e9135ada5114ba149f3ebc8965aee242bfc776f/pride-converter") # Hack for multi-user environment. env.safe_sudo("mkdir log; chmod o+w log") env.safe_sudo("mv * '%s'" % install_dir) bin_dir = os.path.join(env.get("system_install"), "bin") env.safe_sudo("mkdir -p '%s'" % bin_dir) env.safe_sudo("ln -s '%s' %s" % (os.path.join(install_dir, "pride-converter"), os.path.join(bin_dir, "pride-converter"))) _unzip_install("pride_converter2", version, url, env, install_fn, ".") def _unzip_install(pname, version, url, env, install_fn, dir_name="."): install_dir = _symlinked_java_version_dir(pname, version, env) if install_dir: with _make_tmp_dir() as work_dir: with cd(work_dir): _fetch_and_unpack(url, need_dir=False) with cd(dir_name): install_fn(env, install_dir) @_if_not_installed("SuperHirnv03") def install_superhirn(env): default_version = "0.03" version = env.get("tool_version", default_version) url = "https://github.com/jmchilton/SuperHirn/zipball/%s/SuperHirn.zip" % version def _chdir(work_cmd): def do_work(env): with cd("SuperHirnv03/make"): work_cmd(env) return do_work _get_install(url, env, _chdir(_make_copy(find_cmd="find -perm -100 -name 'SuperHirn*'"))) @_if_not_installed("percolator") def install_percolator(env): default_version = "2_04" version = env.get("tool_version", default_version) url = "https://github.com/downloads/percolator/percolator/percolator_%s_full_src.tar.gz" % version def make(env): with cd(".."): env.safe_run("env") env.safe_run("cmake -DCMAKE_INSTALL_PREFIX='%s' . " % env.system_install) env.safe_run("make -j8") env.safe_sudo("make install") _get_install(url, env, make) @_if_not_installed("PepNovo") def install_pepnovo(env): default_version = "20120423" version = env.get("tool_version", default_version) url = "http://proteomics.ucsd.edu/Downloads/PepNovo.%s.zip" % version def install_fn(env, install_dir): with cd("src"): env.safe_run("make") env.safe_sudo("mkdir -p '%s/bin'" % env.system_install) env.safe_sudo("mkdir -p '%s/share/pepnovo'" % env.system_install) env.safe_sudo("mv PepNovo_bin '%s/bin/PepNovo'" % env.system_install) env.safe_sudo("cp -r '../Models' '%s/share/pepnovo'" % env.system_install) _unzip_install("pepnovo", version, url, env, install_fn) @_if_not_installed("crux") def install_crux(env): default_version = "1.39" version = env.get("tool_version", default_version) url = "http://noble.gs.washington.edu/proj/crux/download/crux_%s-x86_64-Linux.zip" % version def _move(env): bin_dir = _get_bin_dir(env) env.safe_sudo("mv bin/* '%s'" % (bin_dir)) _get_install(url, env, _move) @_if_not_installed("Fido") def install_fido(env): version = "2011" url = 'http://noble.gs.washington.edu/proj/fido/fido.tar.gz' # Adapted from Jorrit Boekel's mi-deployment fork # https://bitbucket.org/glormph/mi-deployment-protoeimcs def _chdir_src(work_cmd): def do_work(env): with cd("src/cpp"): env.safe_append('tmpmake', 'SHELL=/bin/bash') env.safe_append('tmpmake', 'prefix=%s' % env.get("system_install")) env.safe_append('tmpmake', 'CPPFLAGS=-Wall -ffast-math -march=x86-64 -pipe -O4 -g') env.safe_run('cat makefile |grep BINPATH -A 9999 >> tmpmake') env.safe_run('cp tmpmake makefile') work_cmd(env) return do_work _get_install(url, env, _chdir_src(_make_copy(find_cmd="find ../../bin -perm -100 -name 'Fido*'"))) def install_ipig(env): """ This tool is installed in Galaxy's jars dir """ # This galaxy specific download probable doesn't belong in this file. default_version = "r5" version = env.get("tool_version", default_version) url = 'http://downloads.sourceforge.net/project/ipig/ipig_%s.zip' % version pkg_name = 'ipig' install_dir = os.path.join(env.galaxy_jars_dir, pkg_name) install_cmd = env.safe_sudo if env.use_sudo else env.safe_run install_cmd("mkdir -p %s" % install_dir) with cd(install_dir): install_cmd("wget %s -O %s" % (url, os.path.split(url)[-1])) install_cmd("unzip -u %s" % (os.path.split(url)[-1])) install_cmd("rm %s" % (os.path.split(url)[-1])) install_cmd('chown --recursive %s:%s %s' % (env.galaxy_user, env.galaxy_user, install_dir)) def install_peptide_to_gff(env): default_version = "master" version = env.get("tool_version", default_version) repository = "hg clone https://jmchilton@bitbucket.org/galaxyp/peptide_to_gff" def install_fn(env, install_dir): env.safe_sudo("cp -r peptide_to_gff/* '%s'" % install_dir) _create_python_virtualenv(env, "peptide_to_gff", "%s/requirements.txt" % install_dir) bin_dir = os.path.join(env.get("system_install"), "bin") env.safe_sudo("mkdir -p '%s'" % bin_dir) env.safe_sudo("ln -s '%s' '%s'" % (os.path.join(install_dir, "peptide_to_gff"), os.path.join(bin_dir, "peptide_to_gff"))) _unzip_install("peptide_to_gff", version, repository, env, install_fn) def install_galaxy_protk(env): """This method installs Ira Cooke's ProtK framework. Very galaxy specific, can only be installed in context of custom Galaxy tool. By default this will install ProtK from rubygems server, but if env.protk_version is set to @ (e.g. 1.1.5@https://bitbucket.org/iracooke/protk-working) the gem will be cloned with hg and installed from source. """ if not env.get('galaxy_tool_install', False): from cloudbio.custom.galaxy import _prep_galaxy _prep_galaxy(env) default_version = "1.2.2" version = env.get("tool_version", default_version) version_and_revision = version install_from_source = version_and_revision.find("@") > 0 # e.g. protk_version = 1.1.5@https://bitbucket.org/iracooke/protk-working if install_from_source: (version, revision) = version_and_revision.split("@") url = _get_bitbucket_download_url(revision, "https://bitbucket.org/iracooke/protk") else: version = version_and_revision ruby_version = "1.9.3" force_rvm_install = False with prefix("HOME=~%s" % env.galaxy_user): def rvm_exec(env, cmd="", rvm_cmd="use", with_gemset=False): target = ruby_version if not with_gemset else "%s@%s" % (ruby_version, "protk-%s" % version) prefix = ". $HOME/.rvm/scripts/rvm; rvm %s %s; " % (rvm_cmd, target) env.safe_sudo("%s %s" % (prefix, cmd), user=env.galaxy_user) if not env.safe_exists("$HOME/.rvm") or force_rvm_install: env.safe_sudo("curl -L get.rvm.io | bash -s stable; source ~%s/.rvm/scripts/rvm" % (env.galaxy_user), user=env.galaxy_user) rvm_exec(env, rvm_cmd="install") rvm_exec(env, cmd="rvm gemset create protk-%s" % version) if not install_from_source: # Typical rubygem install rvm_exec(env, "gem install --no-ri --no-rdoc protk -v %s" % version, with_gemset=True) else: with cd("~%s" % env.galaxy_user): env.safe_sudo("rm -rf protk_source; hg clone '%s' protk_source" % url, user=env.galaxy_user) rvm_exec(env, "cd protk_source; gem build protk.gemspec; gem install protk", with_gemset=True) protk_properties = {} ## ProtK can set these up itself, should make that an option. protk_properties["tpp_root"] = os.path.join(env.galaxy_tools_dir, "transproteomic_pipeline", "default") protk_properties['openms_root'] = "/usr" # os.path.join(env.galaxy_tools_dir, "openms", "default", "bin") ### Assumes omssa, blast, and transproteomic_pipeline CBL galaxy installs. protk_properties['omssa_root'] = os.path.join(env.galaxy_tools_dir, "omssa", "default", "bin") protk_properties['blast_root'] = os.path.join(env.galaxy_tools_dir, "blast+", "default") protk_properties['pwiz_root'] = os.path.join(env.galaxy_tools_dir, "transproteomic_pipeline", "default", "bin") # Other properties: log_file, blast_root env.safe_sudo("mkdir -p \"$HOME/.protk\"", user=env.galaxy_user) env.safe_sudo("mkdir -p \"$HOME/.protk/Databases\"", user=env.galaxy_user) import yaml _write_to_file(yaml.dump(protk_properties), "/home/%s/.protk/config.yml" % env.galaxy_user, "0755") rvm_exec(env, "protk_setup.rb galaxyenv", with_gemset=True) install_dir = os.path.join(env.galaxy_tools_dir, "galaxy_protk", version) env.safe_sudo("mkdir -p '%s'" % install_dir) _chown_galaxy(env, install_dir) env.safe_sudo('ln -s -f "$HOME/.protk/galaxy/env.sh" "%s/env.sh"' % install_dir, user=env.galaxy_user) with cd(install_dir): with cd(".."): env.safe_sudo("ln -s -f '%s' default" % version) @_if_not_installed("myrimatch") def install_myrimatch(env): default_version = "2.1.131" _install_tabb_tool(env, default_version, "myrimatch-bin-linux-x86_64-gcc41-release", ["myrimatch"]) @_if_not_installed("pepitome") def install_pepitome(env): default_version = "1.0.45" _install_tabb_tool(env, default_version, "pepitome-bin-linux-x86_64-gcc41-release", ["pepitome"]) @_if_not_installed("directag") def install_directag(env): default_version = "1.3.62" _install_tabb_tool(env, default_version, "directag-bin-linux-x86_64-gcc41-release", ["adjustScanRankerScoreByGroup", "directag"]) @_if_not_installed("tagrecon") def install_tagrecon(env): default_version = "1.4.63" # TODO: Should consider a better way to handle the unimod xml and blosum matrix. _install_tabb_tool(env, default_version, "tagrecon-bin-linux-x86_64-gcc41-release", ["tagrecon", "unimod.xml", "blosum62.fas"]) @_if_not_installed("idpQonvert") def install_idpqonvert(env): default_version = "3.0.475" version = env.get("tool_version", default_version) url = "%s/idpQonvert_%s" % (PROTEOMICS_APP_ARCHIVE_URL, version) env.safe_run("wget --no-check-certificate -O %s '%s'" % ("idpQonvert", url)) env.safe_run("chmod 755 idpQonvert") env.safe_sudo("mkdir -p '%s/bin'" % env["system_install"]) env.safe_sudo("mv %s '%s/bin'" % ("idpQonvert", env["system_install"])) env.safe_sudo("chmod +x '%s/bin/idpQonvert'" % env["system_install"]) def _install_tabb_tool(env, default_version, download_name, exec_names): version = env.get("tool_version", default_version) url = "%s/%s-%s.tar.bz2" \ % (PROTEOMICS_APP_ARCHIVE_URL, download_name, version.replace(".", "_")) _fetch_and_unpack(url, False) env.safe_sudo("mkdir -p '%s/bin'" % env["system_install"]) for exec_name in exec_names: env.safe_sudo("mv %s '%s/bin'" % (exec_name, env["system_install"])) ================================================ FILE: cloudbio/custom/bio_proteomics_wine.py ================================================ from fabric.api import cd from shared import (_make_tmp_dir, _fetch_and_unpack, _write_to_file, _get_bin_dir) import os def install_proteomics_wine_env(env): script_src = env.get("setup_proteomics_wine_env_script") script_dest = "%s/bin/setup_proteomics_wine_env.sh" % env.get("system_install") if not env.safe_exists(script_dest): env.safe_put(script_src, script_dest, mode="0755", use_sudo=True) def install_multiplierz(env): """ Assumes your wine environment contains an install Python 2.6 in C:\Python26. """ wine_user = _get_wine_user(env) install_proteomics_wine_env(env) env.safe_sudo("setup_proteomics_wine_env.sh", user=wine_user) with _make_tmp_dir() as work_dir: with cd(work_dir): _fetch_and_unpack("hg clone http://multiplierz.hg.sourceforge.net:8000/hgroot/multiplierz/multiplierz") with cd("multiplierz"): wine_prefix = _get_wine_prefix(env) env.safe_sudo("%s; wine %s/drive_c/Python26/python.exe setup.py install" % (_conf_wine(env), wine_prefix), user=wine_user) def install_proteowizard(env): build_id = "85131" version = "3_0_4624" url = "http://teamcity.labkey.org:8080/repository/download/bt36/%s:id/pwiz-bin-windows-x86-vc100-release-%s.tar.bz2?guest=1" % (build_id, version) install_dir = env.get("install_dir") share_dir = "%s/share/proteowizard" % install_dir with _make_tmp_dir() as work_dir: with cd(work_dir): _fetch_and_unpack(url, need_dir=False) env.safe_sudo("cp -r . '%s'" % share_dir) proteowizard_apps = ["msconvert", "msaccess", "chainsaw", "msdiff", "mspicture", "mscat", "txt2mzml", "MSConvertGUI", "Skyline", "Topograph", "SeeMS"] for app in proteowizard_apps: setup_wine_wrapper(env, "%s/%s" % (share_dir, app)) def install_morpheus(env): url = "http://www.chem.wisc.edu/~coon/Downloads/Morpheus/latest/Morpheus.zip" # TODO: install_dir = env.get("install_dir") share_dir = "%s/share/morpheus" % install_dir with _make_tmp_dir() as work_dir: with cd(work_dir): _fetch_and_unpack(url, need_dir=False) env.safe_sudo("cp -r Morpheus '%s'" % share_dir) morpheus_exes = ["morpheus_cl.exe", "Morpheus.exe"] for app in morpheus_exes: setup_wine_wrapper(env, "%s/%s" % (share_dir, app)) def setup_wine_wrapper(env, to): basename = os.path.basename(to) contents = """#!/bin/bash setup_proteomics_wine_env.sh export WINEPREFIX=$HOME/.wine-proteomics wine %s "$@" """ % to bin_dir = _get_bin_dir(env) dest = "%s/%s" % (bin_dir, basename) _write_to_file(contents, dest, '0755') def _conf_wine(env): return "export WINEPREFIX=%s" % _get_wine_prefix(env) def _get_wine_prefix(env): wine_user = _get_wine_user(env) return "~%s/.wine-proteomics" % wine_user def _get_wine_user(env): return env.get("wine_user", env.get("user")) ================================================ FILE: cloudbio/custom/cloudman.py ================================================ """Custom install scripts for CloudMan environment. From Enis Afgan: https://bitbucket.org/afgane/mi-deployment """ import os import contextlib from fabric.api import cd from fabric.contrib.files import settings, hide from cloudbio.custom.shared import (_make_tmp_dir, _setup_conf_file) from cloudbio.cloudman import (_configure_cloudman, _configure_novnc, _configure_desktop, _configure_ec2_autorun) from cloudbio.galaxy import _install_nginx CDN_ROOT_URL = "http://linuxcourse.rutgers.edu/rate/Clusters/download" REPO_ROOT_URL = "https://bitbucket.org/afgane/mi-deployment/raw/tip" def install_cloudman(env): """ A meta method for installing all of CloudMan components. Allows CloudMan and all of its dependencies to be installed via: fab -f fabfile.py -i -H ubuntu@ install_custom:cloudman """ env.logger.debug("Installing CloudMan") _configure_cloudman(env, use_repo_autorun=False) install_nginx(env) install_proftpd(env) install_sge(env) install_novnc(env) def install_ec2_autorun(env): _configure_ec2_autorun(env) def install_novnc(env): _configure_novnc(env) _configure_desktop(env) def install_nginx(env): _install_nginx(env) def install_proftpd(env): """Highly configurable GPL-licensed FTP server software. http://proftpd.org/ """ version = "1.3.4c" postgres_ver = "9.1" url = "ftp://ftp.tpnet.pl/pub/linux/proftpd/distrib/source/proftpd-%s.tar.gz" % version modules = "mod_sql:mod_sql_postgres:mod_sql_passwd" extra_modules = env.get("extra_proftp_modules", "") # Comma separated list of extra modules if extra_modules: modules = "%s:%s" % (modules, extra_modules.replace(",", ":")) install_dir = os.path.join(env.install_dir, 'proftpd') remote_conf_dir = os.path.join(install_dir, "etc") # Skip install if already available if env.safe_exists(remote_conf_dir): env.logger.debug("ProFTPd seems to already be installed in {0}".format(install_dir)) return with _make_tmp_dir() as work_dir: with cd(work_dir): env.safe_run("wget %s" % url) with settings(hide('stdout')): env.safe_run("tar xvzf %s" % os.path.split(url)[1]) with cd("proftpd-%s" % version): env.safe_run("CFLAGS='-I/usr/include/postgresql' ./configure --prefix=%s " "--disable-auth-file --disable-ncurses --disable-ident --disable-shadow " "--enable-openssl --with-modules=%s " "--with-libraries=/usr/lib/postgresql/%s/lib" % (install_dir, modules, postgres_ver)) env.safe_sudo("make") env.safe_sudo("make install") env.safe_sudo("make clean") # Get the init.d startup script initd_script = 'proftpd.initd' initd_url = os.path.join(REPO_ROOT_URL, 'conf_files', initd_script) remote_file = "/etc/init.d/proftpd" env.safe_sudo("wget --output-document=%s %s" % (remote_file, initd_url)) env.safe_sed(remote_file, 'REPLACE_THIS_WITH_CUSTOM_INSTALL_DIR', install_dir, use_sudo=True) env.safe_sudo("chmod 755 %s" % remote_file) # Set the configuration file conf_file = 'proftpd.conf' remote_file = os.path.join(remote_conf_dir, conf_file) if "postgres_port" not in env: env.postgres_port = '5910' if "galaxy_ftp_user_password" not in env: env.galaxy_ftp_user_password = 'fu5yOj2sn' proftpd_conf = {'galaxy_uid': env.safe_run('id -u galaxy'), 'galaxy_fs': '/mnt/galaxy', # Should be a var but uncertain how to get it 'install_dir': install_dir} _setup_conf_file(env, remote_file, conf_file, overrides=proftpd_conf, default_source="proftpd.conf.template") # Get the custom welcome msg file welcome_msg_file = 'welcome_msg.txt' welcome_url = os.path.join(REPO_ROOT_URL, 'conf_files', welcome_msg_file) env.safe_sudo("wget --output-document=%s %s" % (os.path.join(remote_conf_dir, welcome_msg_file), welcome_url)) # Stow env.safe_sudo("cd %s; stow proftpd" % env.install_dir) env.logger.debug("----- ProFTPd %s installed to %s -----" % (version, install_dir)) def install_sge(env): """Sun Grid Engine. """ out_dir = "ge6.2u5" url = "%s/ge62u5_lx24-amd64.tar.gz" % CDN_ROOT_URL install_dir = env.install_dir if env.safe_exists(os.path.join(install_dir, out_dir)): return with _make_tmp_dir() as work_dir: with contextlib.nested(cd(work_dir), settings(hide('stdout'))): env.safe_run("wget %s" % url) env.safe_sudo("chown %s %s" % (env.user, install_dir)) env.safe_run("tar -C %s -xvzf %s" % (install_dir, os.path.split(url)[1])) env.logger.debug("SGE setup") ================================================ FILE: cloudbio/custom/distributed.py ================================================ """Install instructions for distributed MapReduce style programs. """ import os from fabric.api import * from fabric.contrib.files import * from shared import (_if_not_python_lib, _pip_cmd, _is_anaconda) @_if_not_python_lib("pydoop") def install_pydoop(env): """pydoop; provides Hadoop access for Python. http://pydoop.sourceforge.net/docs/ """ java_home = env.java_home if "java_home" in env else os.environ["JAVA_HOME"] export_str = "export JAVA_HOME=%s" % (java_home) cmd = env.safe_run if _is_anaconda(env) else env.safe_sudo cmd("%s && %s install pydoop" % (export_str, _pip_cmd(env))) @_if_not_python_lib("bl.mr.seq.seqal") def install_seal(env): """Install seal: process high-throughput sequencing with Hadoop. http://biodoop-seal.sf.net/ """ install_pydoop(env) java_home = env.java_home if "java_home" in env else os.environ["JAVA_HOME"] export_str = "export JAVA_HOME=%s" % (java_home) cmd = env.safe_run if _is_anaconda(env) else env.safe_sudo cmd("%s && %s install --pre seal" % (export_str, _pip_cmd(env))) ================================================ FILE: cloudbio/custom/galaxy.py ================================================ """ Install any components that fall under 'galaxy' directive in main.yaml """ from cloudbio.galaxy import _setup_users from cloudbio.galaxy import _setup_galaxy_env_defaults from cloudbio.galaxy import _install_galaxy from cloudbio.galaxy import _configure_galaxy_options def install_galaxy_webapp(env): _prep_galaxy(env) _install_galaxy(env) _configure_galaxy_options(env) def _prep_galaxy(env): _setup_users(env) _setup_galaxy_env_defaults(env) ================================================ FILE: cloudbio/custom/galaxy_tools.py ================================================ """ Install any components that fall under 'galaxy_tools' directive in main.yaml """ from cloudbio.galaxy.tools import _install_tools from cloudbio.custom.galaxy import _prep_galaxy def install_cbl_galaxy_tools(env): _prep_galaxy(env) _install_tools(env) ================================================ FILE: cloudbio/custom/galaxyp.py ================================================ """ """ from cloudbio.galaxy.utils import _chown_galaxy from fabric.contrib.files import * from shared import _write_to_file def install_protvis(env): """ Installs Andrew Brock's proteomics visualize tool. https://bitbucket.org/Andrew_Brock/proteomics-visualise/ """ _setup_protvis_env(env) protvis_home = env["protvis_home"] env.safe_sudo("sudo apt-get -y --force-yes install libxml2-dev libxslt-dev") run("rm -rf protvis") run("git clone -b lorikeet https://github.com/jmchilton/protvis.git") with cd("protvis"): run("git submodule init") run("git submodule update") env.safe_sudo("rsync -avur --delete-after . %s" % (protvis_home)) _chown_galaxy(env, protvis_home) with cd(protvis_home): env.safe_sudo("./setup.sh", user=env.get("galaxy_user", "galaxy")) #default_revision = "8cc6af1c492c" # #revision = env.get("protvis_revision", default_revision) #url = _get_bitbucket_download_url(revision, "https://bitbucket.org/Andrew_Brock/proteomics-visualise") #def _make(env): #_get_install(url, env, _make) galaxy_data_dir = env.get('galaxy_data_dir', "/mnt/galaxyData/") protvis_converted_files_dir = env.get('protvis_converted_files_dir') _write_to_file('''GALAXY_ROOT = "%s" PATH_WHITELIST = ["%s/files/", "%s"] CONVERTED_FILES = "%s" ''' % (env.galaxy_home, galaxy_data_dir, protvis_converted_files_dir, protvis_converted_files_dir), "%s/conf.py" % protvis_home, "0755") _setup_protvis_service(env) def _setup_protvis_env(env): if not "protvis_home" in env: env["protvis_home"] = "%s/%s" % (env.galaxy_tools_dir, "protvis") if not "protvis_user" in env: env["protvis_user"] = "galaxy" if not "protvis_port" in env: env["protvis_port"] = "8500" if not "protvis_converted_files_dir" in env: galaxy_data_dir = env.get('galaxy_data_dir', "/mnt/galaxyData/") env['protvis_converted_files_dir'] = "%s/tmp/protvis" % galaxy_data_dir def _setup_protvis_service(env): _setup_conf_file(env, os.path.join("/etc/init.d/protvis"), "protvis_init", default_source="protvis_init") _setup_conf_file(env, os.path.join("/etc/default/protvis"), "protvis_default") _setup_simple_service("protvis") ================================================ FILE: cloudbio/custom/java.py ================================================ """Install instructions for non-packaged java programs. """ import os from fabric.api import * from fabric.contrib.files import * from shared import (_if_not_installed, _make_tmp_dir) from cloudbio.custom import shared @_if_not_installed("lein -v") def install_leiningen(env): """Clojure tool for project configuration and automation. http://github.com/technomancy/leiningen """ bin_dir = os.path.join(env.system_install, "bin") with _make_tmp_dir() as work_dir: with cd(work_dir): shared._remote_fetch(env, "https://raw.github.com/technomancy/leiningen/stable/bin/lein") env.safe_run("chmod a+rwx lein") env.safe_sudo("mv lein %s" % bin_dir) env.safe_run("%s/lein" % bin_dir) ================================================ FILE: cloudbio/custom/millstone.py ================================================ """Install instructions for non-packaged programs required by Millstone. """ from fabric.api import cd from cloudbio.custom.shared import _make_tmp_dir def install_unafold(env): """Required by optmage. """ # Since unafold is distributed as an .rpm, we need the program alien to # convert it into a .deb that can be installed on this system. env.safe_sudo("apt-get install -y alien") with _make_tmp_dir() as work_dir: with cd(work_dir): env.safe_run("wget http://dinamelt.rit.albany.edu/download/unafold-3.8-1.x86_64.rpm") env.safe_sudo("alien -i unafold-3.8-1.x86_64.rpm") ================================================ FILE: cloudbio/custom/phylogeny.py ================================================ """Install instructions for non-packaged phyologeny programs. """ import os from fabric.api import * from fabric.contrib.files import * from cloudbio.custom.shared import _if_not_installed, _make_tmp_dir def install_tracer(env): """A program for analysing results from Bayesian MCMC programs such as BEAST & MrBayes. http://tree.bio.ed.ac.uk/software/tracer/ """ version = "1.5" install_dir = os.path.join(env.system_install, "bioinf") final_exe = os.path.join(env.system_install, "bin", "tracer") if env.safe_exists(final_exe): return if not env.safe_exists(final_exe): with _make_tmp_dir() as work_dir: with cd(work_dir): env.safe_run("wget -O Tracer_v{0}.tgz 'http://tree.bio.ed.ac.uk/download.php?id=80&num=3'".format( version)) env.safe_run("tar xvzf Tracer_v{0}.tgz".format(version)) env.safe_run("chmod a+x Tracer_v{0}/bin/tracer".format(version)) env.safe_sudo("mkdir -p %s" % install_dir) env.safe_sudo("rm -rvf %s/tracer" % install_dir) env.safe_sudo("mv -f Tracer_v%s %s/tracer" % (version, install_dir)) env.safe_sudo("ln -sf %s/tracer/bin/tracer %s" % (install_dir, final_exe)) @_if_not_installed("beast -help") def install_beast(env): """BEAST: Bayesian MCMC analysis of molecular sequences. http://beast.bio.ed.ac.uk """ version = "1.7.4" install_dir = os.path.join(env.system_install, "bioinf") final_exe = os.path.join(env.system_install, "bin", "beast") if not env.safe_exists(final_exe): with _make_tmp_dir() as work_dir: with cd(work_dir): env.safe_run("wget -c http://beast-mcmc.googlecode.com/files/BEASTv%s.tgz" % version) env.safe_run("tar xvzf BEASTv%s.tgz" % version) env.safe_sudo("mkdir -p %s" % install_dir) env.safe_sudo("rm -rvf %s/beast" % install_dir) env.safe_sudo("mv -f BEASTv%s %s/beast" % (version, install_dir)) for l in ["beast","beauti","loganalyser","logcombiner","treeannotator","treestat"]: env.safe_sudo("ln -sf %s/beast/bin/%s %s/bin/%s" % (install_dir, l, env.system_install, l)) ================================================ FILE: cloudbio/custom/python.py ================================================ """Install instructions for python libraries not ready for easy_install. """ import os from fabric.api import * from fabric.contrib.files import * from shared import (_if_not_python_lib, _get_install, _python_make, _pip_cmd, _is_anaconda) @_if_not_python_lib("bx") def install_bx_python(env): """Tools for manipulating biological data, particularly multiple sequence alignments https://bitbucket.org/james_taylor/bx-python/wiki/Home """ version = "bitbucket" url = "https://bitbucket.org/james_taylor/bx-python/get/tip.tar.bz2" cmd = env.safe_run if _is_anaconda(env) else env.safe_sudo if not _is_anaconda(env): cmd("%s install --upgrade distribute" % _pip_cmd(env)) cmd("%s install --upgrade %s" % (_pip_cmd(env), url)) @_if_not_python_lib("rpy") def install_rpy(env): """RPy is a very simple, yet robust, Python interface to the R Programming Language. http://rpy.sourceforge.net/ """ version = "1.0.3" ext = "a" url = "http://downloads.sourceforge.net/project/rpy/rpy/" \ "%s/rpy-%s%s.zip" % (version, version, ext) def _fix_libraries(env): env.safe_run("""sed -i.bak -r -e "s/,'Rlapack'//g" setup.py""") with settings(hide('warnings', 'running', 'stdout', 'stderr'), warn_only=True): result = env.safe_run("R --version") if result.failed: return _get_install(url, env, _python_make, post_unpack_fn=_fix_libraries) @_if_not_python_lib("netsa") def install_netsa_python(env): """A suite of open source tools for monitoring large-scale networks using flow data. http://tools.netsa.cert.org/index.html """ version = "1.3" url = "http://tools.netsa.cert.org/releases/netsa-python-%s.tar.gz" % version cmd = env.safe_run if _is_anaconda(env) else env.safe_sudo cmd("%s install %s" % (_pip_cmd(env), url)) ================================================ FILE: cloudbio/custom/shared.py ================================================ """Reusable decorators and functions for custom installations. """ from __future__ import print_function from contextlib import contextmanager import functools import os import socket from string import Template import sys import tempfile from tempfile import NamedTemporaryFile import urllib import uuid import shutil import subprocess import time # Optional fabric imports, for back compatibility try: from fabric.api import * from fabric.contrib.files import * from cloudbio.fabutils import quiet, warn_only except ImportError: pass CBL_REPO_ROOT_URL = "https://raw.github.com/chapmanb/cloudbiolinux/master/" # -- decorators and context managers @contextmanager def chdir(new_dir): """Context manager to temporarily change to a new directory. http://lucentbeing.com/blog/context-managers-and-the-with-statement-in-python/ """ # On busy filesystems can have issues accessing main directory. Allow retries num_tries = 0 max_tries = 5 cur_dir = None while cur_dir is None: try: cur_dir = os.getcwd() except OSError: if num_tries > max_tries: raise num_tries += 1 time.sleep(2) safe_makedir(new_dir) os.chdir(new_dir) try: yield finally: os.chdir(cur_dir) def safe_makedir(dname): """Make a directory if it doesn't exist, handling concurrent race conditions. """ if not dname: return dname num_tries = 0 max_tries = 5 while not os.path.exists(dname): # we could get an error here if multiple processes are creating # the directory at the same time. Grr, concurrency. try: os.makedirs(dname) except OSError: if num_tries > max_tries: raise num_tries += 1 time.sleep(2) return dname def which(program, env=None): """ returns the path to an executable or None if it can't be found""" paths = os.environ["PATH"].split(os.pathsep) if env and hasattr(env, "system_install"): paths += [env.system_install, os.path.join(env.system_install, "anaconda")] def is_exe(fpath): return os.path.isfile(fpath) and os.access(fpath, os.X_OK) fpath, fname = os.path.split(program) if fpath: if is_exe(program): return program else: for path in paths: exe_file = os.path.join(path, program) if is_exe(exe_file): return exe_file return None def _if_not_installed(pname): """Decorator that checks if a callable program is installed. """ def argcatcher(func): functools.wraps(func) def decorator(*args, **kwargs): if _galaxy_tool_install(args): run_function = not _galaxy_tool_present(args) elif isinstance(pname, list): run_function = any([_executable_not_on_path(x) for x in pname]) else: run_function = _executable_not_on_path(pname) if run_function: return func(*args, **kwargs) return decorator return argcatcher def _all_cbl_paths(env, ext): """Add paths to other non-system directories installed by CloudBioLinux. """ return ":".join("%s/%s" % (p, ext) for p in [env.system_install, os.path.join(env.system_install, "anaconda")]) def _executable_not_on_path(pname): with settings(hide('warnings', 'running', 'stdout', 'stderr'), warn_only=True): result = env.safe_run("export PATH=%s:$PATH && " "export LD_LIBRARY_PATH=%s:$LD_LIBRARY_PATH && %s" % (_all_cbl_paths(env, "bin"), _all_cbl_paths(env, "lib"), pname)) return result.return_code == 127 def _galaxy_tool_install(args): try: return args[0]["galaxy_tool_install"] except: return False def _galaxy_tool_present(args): return env.safe_exists(os.path.join(args[0]["system_install"], "env.sh")) def _if_not_python_lib(library): """Decorator that checks if a python library is installed. """ def argcatcher(func): functools.wraps(func) def decorator(*args, **kwargs): with settings(warn_only=True): errcount = int(env.safe_run_output("%s -c 'import %s' 2>&1 | grep -c ImportError | cat" % (_python_cmd(env), library))) result = 0 if errcount >= 1 else 1 if result == 0: return func(*args, **kwargs) else: return result return decorator return argcatcher @contextmanager def make_tmp_dir_local(ext, work_dir): if ext: work_dir += ext safe_makedir(work_dir) yield work_dir shutil.rmtree(work_dir) @contextmanager def _make_tmp_dir(ext=None, work_dir=None): """ Setup a temporary working directory for building custom software. First checks fabric environment for a `work_dir` path, if that is not set it will use the remote path $TMPDIR/cloudbiolinux if $TMPDIR is defined remotely, finally falling back on remote $HOME/cloudbiolinux otherwise. `ext` allows creation of tool specific temporary directories to avoid conflicts using CloudBioLinux inside of CloudBioLinux. """ if not work_dir: work_dir = __work_dir() if ext: work_dir += ext use_sudo = False if not env.safe_exists(work_dir): with settings(warn_only=True): # Try to create this directory without using sudo, but # if needed fallback. result = env.safe_run("mkdir -p '%s'" % work_dir) if result.return_code != 0: use_sudo = True if use_sudo: env.safe_sudo("mkdir -p '%s'" % work_dir) env.safe_sudo("chown -R %s '%s'" % (env.user, work_dir)) yield work_dir if env.safe_exists(work_dir): run_func = env.safe_sudo if use_sudo else env.safe_run run_func("rm -rf %s" % work_dir) def __work_dir(): work_dir = env.get("work_dir", None) if not work_dir: with quiet(): tmp_dir = env.safe_run_output("echo $TMPDIR") if tmp_dir.failed or not tmp_dir.strip(): home_dir = env.safe_run_output("echo $HOME") tmp_dir = os.path.join(home_dir, "tmp") work_dir = os.path.join(tmp_dir.strip(), "cloudbiolinux") return work_dir # -- Standard build utility simplifiers def _get_expected_file(url, dir_name=None, safe_tar=False, tar_file_name=None): if tar_file_name: tar_file = tar_file_name else: tar_file = os.path.split(url.split("?")[0])[-1] safe_tar = "--pax-option='delete=SCHILY.*,delete=LIBARCHIVE.*'" if safe_tar else "" exts = {(".tar.gz", ".tgz"): "tar %s -xzpf" % safe_tar, (".tar",): "tar %s -xpf" % safe_tar, (".tar.bz2",): "tar %s -xjpf" % safe_tar, (".zip",): "unzip"} for ext_choices, tar_cmd in exts.iteritems(): for ext in ext_choices: if tar_file.endswith(ext): if dir_name is None: dir_name = tar_file[:-len(ext)] return tar_file, dir_name, tar_cmd raise ValueError("Did not find extract command for %s" % url) def _safe_dir_name(dir_name, need_dir=True): replace_try = ["", "-src", "_core"] for replace in replace_try: check = dir_name.replace(replace, "") if env.safe_exists(check): return check # still couldn't find it, it's a nasty one for check_part in (dir_name.split("-")[0].split("_")[0], dir_name.split("-")[-1].split("_")[-1], dir_name.split(".")[0], dir_name.lower().split(".")[0]): with settings(hide('warnings', 'running', 'stdout', 'stderr'), warn_only=True): dirs = env.safe_run_output("ls -d1 *%s*/" % check_part).split("\n") dirs = [x for x in dirs if "cannot access" not in x and "No such" not in x] if len(dirs) == 1 and dirs[0]: return dirs[0] dirs = env.safe_run_output("find * -type d -maxdepth 0").split("\n") if len(dirs) == 1 and dirs[0]: return dirs[0] if need_dir: raise ValueError("Could not find directory %s" % dir_name) def _remote_fetch(env, url, out_file=None, allow_fail=False, fix_fn=None, samedir=False): """Retrieve url using wget, performing download in a temporary directory. Provides a central location to handle retrieval issues and avoid using interrupted downloads. """ if out_file is None: out_file = os.path.basename(url) if not os.path.exists(out_file): if samedir and os.path.isabs(out_file): orig_dir = os.path.dirname(out_file) out_file = os.path.basename(out_file) else: orig_dir = os.getcwd() temp_ext = "/%s" % uuid.uuid3(uuid.NAMESPACE_URL, str("file://%s/%s/%s" % ("localhost", socket.gethostname(), out_file))) with make_tmp_dir_local(ext=temp_ext, work_dir=orig_dir) as tmp_dir: with chdir(tmp_dir): try: subprocess.check_call("wget --continue --no-check-certificate -O %s '%s'" % (out_file, url), shell=True) if fix_fn: out_file = fix_fn(env, out_file) subprocess.check_call("mv %s %s" % (out_file, orig_dir), shell=True) except subprocess.CalledProcessError: if allow_fail: out_file = None else: raise IOError("Failure to retrieve remote file: %s" % url) if samedir and out_file: out_file = os.path.join(orig_dir, out_file) return out_file def _fetch_and_unpack(url, need_dir=True, dir_name=None, revision=None, safe_tar=False, tar_file_name=None): if url.startswith(("git", "svn", "hg", "cvs")): base = os.path.splitext(os.path.basename(url.split()[-1]))[0] if env.safe_exists(base): env.safe_sudo("rm -rf {0}".format(base)) env.safe_run(url) if revision: if url.startswith("git"): env.safe_run("cd %s && git checkout %s" % (base, revision)) else: raise ValueError("Need to implement revision retrieval for %s" % url.split()[0]) return base else: # If tar_file_name is provided, use it instead of the inferred one tar_file, dir_name, tar_cmd = _get_expected_file(url, dir_name, safe_tar, tar_file_name=tar_file_name) tar_file = _remote_fetch(env, url, tar_file) env.safe_run("%s %s" % (tar_cmd, tar_file)) return _safe_dir_name(dir_name, need_dir) def _configure_make(env): env.safe_run("export PKG_CONFIG_PATH=$PKG_CONFIG_PATH:%s/lib/pkgconfig && " \ "./configure --disable-werror --prefix=%s " % (env.system_install, env.system_install)) lib_export = "export LD_LIBRARY_PATH=%s/lib:$LD_LIBRARY_PATH" % env.system_install env.safe_run("%s && make" % lib_export) env.safe_sudo("%s && make install" % lib_export) def _ac_configure_make(env): env.safe_run("autoreconf -i -f") _configure_make(env) def _make_copy(find_cmd=None, premake_cmd=None, do_make=True): def _do_work(env): if premake_cmd: premake_cmd() if do_make: env.safe_run("make") if find_cmd: install_dir = _get_bin_dir(env) for fname in env.safe_run_output(find_cmd).split("\n"): env.safe_sudo("cp -rf %s %s" % (fname.rstrip("\r"), install_dir)) return _do_work def _get_install(url, env, make_command, post_unpack_fn=None, revision=None, dir_name=None, safe_tar=False, tar_file_name=None): """Retrieve source from a URL and install in our system directory. """ with _make_tmp_dir() as work_dir: with cd(work_dir): dir_name = _fetch_and_unpack(url, revision=revision, dir_name=dir_name, safe_tar=safe_tar, tar_file_name=tar_file_name) with cd(os.path.join(work_dir, dir_name)): if post_unpack_fn: post_unpack_fn(env) make_command(env) def _apply_patch(env, url): patch = os.path.basename(url) cmd = "wget {url}; patch -p0 < {patch}".format(url=url, patch=patch) env.safe_run(cmd) def _get_install_local(url, env, make_command, dir_name=None, post_unpack_fn=None, safe_tar=False, tar_file_name=None): """Build and install in a local directory. """ (_, test_name, _) = _get_expected_file(url, safe_tar=safe_tar, tar_file_name=tar_file_name) test1 = os.path.join(env.local_install, test_name) if dir_name is not None: test2 = os.path.join(env.local_install, dir_name) elif "-" in test1: test2, _ = test1.rsplit("-", 1) else: test2 = os.path.join(env.local_install, test_name.split("_")[0]) if not env.safe_exists(test1) and not env.safe_exists(test2): with _make_tmp_dir() as work_dir: with cd(work_dir): dir_name = _fetch_and_unpack(url, dir_name=dir_name, safe_tar=safe_tar, tar_file_name=tar_file_name) if not env.safe_exists(os.path.join(env.local_install, dir_name)): with cd(dir_name): if post_unpack_fn: post_unpack_fn(env) make_command(env) # Copy instead of move because GNU mv does not have --parents flag. # The source dir will get cleaned up anyhow so just leave it. destination_dir = env.local_install env.safe_sudo("mkdir -p '%s'" % destination_dir) env.safe_sudo("cp --recursive %s %s" % (dir_name, destination_dir)) # --- Language specific utilities def _symlinked_install_dir(pname, version, env, extra_dir=None): if extra_dir: base_dir = os.path.join(env.system_install, "share", extra_dir, pname) else: base_dir = os.path.join(env.system_install, "share", pname) return base_dir, "%s-%s" % (base_dir, version) def _symlinked_dir_exists(pname, version, env, extra_dir=None): """Check if a symlinked directory exists and is non-empty. """ _, install_dir = _symlinked_install_dir(pname, version, env, extra_dir) if env.safe_exists(install_dir): items = env.safe_run_output("ls %s" % install_dir) if items.strip() != "": return True return False def _symlinked_shared_dir(pname, version, env, extra_dir=None): """Create a symlinked directory of files inside the shared environment. """ base_dir, install_dir = _symlinked_install_dir(pname, version, env, extra_dir) relative_install_dir = os.path.relpath(install_dir, os.path.dirname(base_dir)) # Does not exist, change symlink to new directory if not env.safe_exists(install_dir): env.safe_sudo("mkdir -p %s" % install_dir) if env.safe_exists(base_dir): env.safe_sudo("rm -f %s" % base_dir) env.safe_sudo("ln -sf %s %s" % (relative_install_dir, base_dir)) return install_dir items = env.safe_run_output("ls %s" % install_dir) # empty directory, change symlink and re-download if items.strip() == "": if env.safe_exists(base_dir): env.safe_sudo("rm -f %s" % base_dir) env.safe_sudo("ln -sf %s %s" % (relative_install_dir, base_dir)) return install_dir # Create symlink if previously deleted if not env.safe_exists(base_dir): env.safe_sudo("ln -sf %s %s" % (relative_install_dir, base_dir)) return None def _symlinked_java_version_dir(pname, version, env): return _symlinked_shared_dir(pname, version, env, extra_dir="java") def _java_install(pname, version, url, env, install_fn=None, pre_fetch_fn=None): """Download java jars into versioned input directories. pre_fetch_fn runs before URL retrieval, allowing insertion of manual steps like restricted downloads. """ install_dir = _symlinked_java_version_dir(pname, version, env) if install_dir: with _make_tmp_dir() as work_dir: with cd(work_dir): if pre_fetch_fn: out = pre_fetch_fn(env) if out is None: return dir_name = _fetch_and_unpack(url) with cd(dir_name): if install_fn is not None: install_fn(env, install_dir) else: env.safe_sudo("mv *.jar %s" % install_dir) def _python_cmd(env): """Retrieve python command, handling tricky situations on CentOS. """ anaconda_py = os.path.join(env.system_install, "anaconda", "bin", "python") if env.safe_exists(anaconda_py): return anaconda_py if "python_version_ext" in env and env.python_version_ext: major, minor = env.safe_run("python --version").split()[-1].split(".")[:2] check_major, check_minor = env.python_version_ext.split(".")[:2] if major != check_major or int(check_minor) > int(minor): return "python%s" % env.python_version_ext else: return "python" else: return "python" def _pip_cmd(env): """Retrieve pip command for installing python packages, allowing configuration. """ anaconda_pip = os.path.join(env.system_install, "anaconda", "bin", "pip") if env.safe_exists(anaconda_pip): to_check = [anaconda_pip] else: to_check = ["pip"] if "pip_cmd" in env and env.pip_cmd: to_check.append(env.pip_cmd) if not env.use_sudo: to_check.append(os.path.join(env.system_install, "bin", "pip")) if "python_version_ext" in env and env.python_version_ext: to_check.append("pip-{0}".format(env.python_version_ext)) for cmd in to_check: with quiet(): pip_version = env.safe_run("%s --version" % cmd) if pip_version.succeeded: return cmd raise ValueError("Could not find pip installer from: %s" % to_check) def _conda_cmd(env): if hasattr(env, "conda_cmd") and env.conda_cmd: return env.conda_cmd to_check = [] if env.hosts == ["localhost"]: to_check.append(os.path.join(os.path.dirname(os.path.realpath(sys.executable)), "conda")) to_check.extend([os.path.join(env.system_install, "anaconda", "bin", "conda"), "conda"]) for cmd in to_check: with quiet(): test = env.safe_run("%s --version" % cmd) if test.succeeded: return cmd return None def _is_anaconda(env): """Check if we have a conda command or are in an anaconda subdirectory. """ with quiet(): conda = _conda_cmd(env) has_conda = conda and env.safe_run_output("%s -h" % conda).startswith("usage: conda") with quiet(): try: full_pip = env.safe_run_output("which %s" % _pip_cmd(env)) except ValueError: full_pip = None in_anaconda_dir = full_pip and full_pip.succeeded and "/anaconda/" in full_pip return has_conda or in_anaconda_dir def _python_make(env): run_cmd = env.safe_run if _is_anaconda(env) else env.safe_sudo # Clean up previously failed builds env.safe_sudo("rm -rf /tmp/pip-build-%s" % env.user) env.safe_sudo("rm -rf /tmp/pip-*-build") run_cmd("%s install --upgrade ." % _pip_cmd(env)) for clean in ["dist", "build", "lib/*.egg-info"]: env.safe_sudo("rm -rf %s" % clean) def _get_installed_file(env, local_file): installed_files_dir = \ os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "..", "installed_files") path = os.path.join(installed_files_dir, local_file) if not os.path.exists(path): # If using cloudbiolinux as a library, this won't be available, # download the file from github instead f = NamedTemporaryFile(delete=False) cloudbiolinx_repo_url = env.get("cloudbiolinux_repo_url", CBL_REPO_ROOT_URL) url = os.path.join(cloudbiolinx_repo_url, 'installed_files', local_file) urllib.urlretrieve(url, f.name) path = f.name return path def _get_installed_file_contents(env, local_file): return open(_get_installed_file(env, local_file), "r").read() def _write_to_file(contents, path, mode): """ Use fabric to write string contents to remote file specified by path. """ fd, local_path = tempfile.mkstemp() try: os.write(fd, contents) tmp_path = os.path.join("/tmp", os.path.basename(path)) env.safe_put(local_path, tmp_path) env.safe_sudo("mv %s %s" % (tmp_path, path)) env.safe_sudo("chmod %s %s" % (mode, path)) os.close(fd) finally: os.unlink(local_path) def _get_bin_dir(env): """ When env.system_install is /usr this exists, but in the Galaxy it may not already exist. """ return _get_install_subdir(env, "bin") def _get_include_dir(env): return _get_install_subdir(env, "include") def _get_lib_dir(env): return _get_install_subdir(env, "lib") def _get_install_subdir(env, subdir): path = os.path.join(env.system_install, subdir) if not env.safe_exists(path): env.safe_sudo("mkdir -p '%s'" % path) return path def _set_default_config(env, install_dir, sym_dir_name="default"): """ Sets up default galaxy config directory symbolic link (if needed). Needed when it doesn't exists or when installing a new version of software. """ version = env["tool_version"] if env.safe_exists(install_dir): install_dir_root = "%s/.." % install_dir sym_dir = "%s/%s" % (install_dir_root, sym_dir_name) replace_default = False if not env.safe_exists(sym_dir): replace_default = True if not replace_default: default_version = env.safe_sudo("basename `readlink -f %s`" % sym_dir) if version > default_version: # Bug: Wouldn't work for 1.9 < 1.10 print("default version %s is older than version %s just installed, replacing..." % (default_version, version)) replace_default = True if replace_default: env.safe_sudo("rm -rf %s; ln -f -s %s %s" % (sym_dir, install_dir, sym_dir)) def _setup_simple_service(service_name): """ Very Ubuntu/Debian specific, will need to be modified if used on other archs. """ sudo("ln -f -s /etc/init.d/%s /etc/rc0.d/K01%s" % (service_name, service_name)) sudo("ln -f -s /etc/init.d/%s /etc/rc1.d/K01%s" % (service_name, service_name)) sudo("ln -f -s /etc/init.d/%s /etc/rc2.d/S99%s" % (service_name, service_name)) sudo("ln -f -s /etc/init.d/%s /etc/rc3.d/S99%s" % (service_name, service_name)) sudo("ln -f -s /etc/init.d/%s /etc/rc4.d/S99%s" % (service_name, service_name)) sudo("ln -f -s /etc/init.d/%s /etc/rc5.d/S99%s" % (service_name, service_name)) sudo("ln -f -s /etc/init.d/%s /etc/rc6.d/K01%s" % (service_name, service_name)) def _render_config_file_template(env, name, defaults={}, overrides={}, default_source=None): """ If ``name` is say ``nginx.conf``, check fabric environment for ``nginx_conf_path`` and then ``nginx_conf_template_path``. If ``nginx_conf_path`` is set, return the contents of that file. If nginx_conf_template_path is set, return the contents of that file but with variable interpolation performed. Variable interpolation is performed using a derivative of the fabric environment defined using the supplied ``defaults`` and ``overrides`` using the ``_extend_env`` function below. Finally, if neither ``nginx_conf_path`` or ``nginx_conf_template_path`` are set, check the ``installed_files`` directory for ``nginx.conf`` and finally ``nginx.conf.template``. """ param_prefix = name.replace(".", "_") # Deployer can specify absolute path for config file, check this first path_key_name = "%s_path" % param_prefix template_key_name = "%s_template_path" % param_prefix if env.get(path_key_name, None): source_path = env[path_key_name] source_template = False elif env.get(template_key_name, None): source_path = env[template_key_name] source_template = True elif default_source: source_path = _get_installed_file(env, default_source) source_template = source_path.endswith(".template") else: default_template_name = "%s.template" % name source_path = _get_installed_file(env, default_template_name) source_template = True if source_template: template = Template(open(source_path, "r").read()) template_params = _extend_env(env, defaults=defaults, overrides=overrides) contents = template.substitute(template_params) else: contents = open(source_path, "r").read() return contents def _extend_env(env, defaults={}, overrides={}): """ Create a new ``dict`` from fabric's ``env``, first adding defaults specified via ``defaults`` (if available). Finally, override anything in env, with values specified by ``overrides``. """ new_env = {} for key, value in defaults.iteritems(): new_env[key] = value for key, value in env.iteritems(): new_env[key] = value for key, value in overrides.iteritems(): new_env[key] = value return new_env def _setup_conf_file(env, dest, name, defaults={}, overrides={}, default_source=None, mode="0755"): conf_file_contents = _render_config_file_template(env, name, defaults, overrides, default_source) _write_to_file(conf_file_contents, dest, mode=mode) def _add_to_profiles(line, profiles=[], use_sudo=True): """ If it's not already there, append ``line`` to shell profiles files. By default, these are ``/etc/profile`` and ``/etc/bash.bashrc`` but can be overridden by providing a list of file paths to the ``profiles`` argument. """ if not profiles: profiles = ['/etc/bash.bashrc', '/etc/profile'] for profile in profiles: if not env.safe_contains(profile, line): env.safe_append(profile, line, use_sudo=use_sudo) def install_venvburrito(): """ If not already installed, install virtualenv-burrito (https://github.com/brainsik/virtualenv-burrito) as a convenient method for installing and managing Python virtualenvs. """ url = "https://raw.github.com/brainsik/virtualenv-burrito/master/virtualenv-burrito.sh" if not env.safe_exists("$HOME/.venvburrito/startup.sh"): env.safe_run("curl -sL {0} | $SHELL".format(url)) # Add the startup script into the ubuntu user's bashrc _add_to_profiles(". $HOME/.venvburrito/startup.sh", [env.shell_config], use_sudo=False) def _create_python_virtualenv(env, venv_name, reqs_file=None, reqs_url=None): """ Using virtual-burrito, create a new Python virtualenv named ``venv_name``. Do so only if the virtualenv of the given name does not already exist. virtual-burrito installs virtualenvs in ``$HOME/.virtualenvs``. By default, an empty virtualenv is created. Python libraries can be installed into the virutalenv at the time of creation by providing a path to the requirements.txt file (``reqs_file``). Instead of providing the file, a url to the file can be provided via ``reqs_url``, in which case the requirements file will first be downloaded. Note that if the ``reqs_url`` is provided, the downloaded file will take precedence over ``reqs_file``. """ # First make sure virtualenv-burrito is installed install_venvburrito() activate_vburrito = ". $HOME/.venvburrito/startup.sh" def create(): if "venv_directory" not in env: _create_global_python_virtualenv(env, venv_name, reqs_file, reqs_url) else: _create_local_python_virtualenv(env, venv_name, reqs_file, reqs_url) # TODO: Terrible hack here, figure it out and fix it. # prefix or vburrito does not work with is_local or at least deployer+is_local if env.is_local: create() else: with prefix(activate_vburrito): create() def _create_local_python_virtualenv(env, venv_name, reqs_file, reqs_url): """ Use virtualenv directly to setup virtualenv in specified directory. """ venv_directory = env.get("venv_directory") if not env.safe_exists(venv_directory): if reqs_url: _remote_fetch(env, reqs_url, reqs_file) env.logger.debug("Creating virtualenv in directory %s" % venv_directory) env.safe_sudo("virtualenv --no-site-packages '%s'" % venv_directory) env.logger.debug("Activating") env.safe_sudo(". %s/bin/activate; pip install -r '%s'" % (venv_directory, reqs_file)) def _create_global_python_virtualenv(env, venv_name, reqs_file, reqs_url): """ Use mkvirtualenv to setup this virtualenv globally for user. """ if venv_name in env.safe_run_output("bash -l -c lsvirtualenv | grep {0} || true" .format(venv_name)): env.logger.info("Virtualenv {0} already exists".format(venv_name)) else: with _make_tmp_dir(): if reqs_file or reqs_url: if not reqs_file: # This mean the url only is provided so 'standardize ' the file name reqs_file = 'requirements.txt' cmd = "bash -l -c 'mkvirtualenv -r {0} {1}'".format(reqs_file, venv_name) else: cmd = "bash -l -c 'mkvirtualenv {0}'".format(venv_name) if reqs_url: _remote_fetch(env, reqs_url, reqs_file) env.safe_run(cmd) env.logger.info("Finished installing virtualenv {0}".format(venv_name)) def _get_bitbucket_download_url(revision, default_repo): if revision.startswith("http"): url = revision else: url = "%s/get/%s.tar.gz" % (default_repo, revision) return url def _read_boolean(env, name, default): property_str = env.get(name, str(default)) return property_str.upper() in ["TRUE", "YES"] ================================================ FILE: cloudbio/custom/system.py ================================================ """ Install system programs not available from packages. """ import os from fabric.api import cd from cloudbio.custom import shared from cloudbio.custom.shared import _if_not_installed, _get_install, _configure_make from cloudbio.fabutils import quiet def install_homebrew(env): """Homebrew package manager for OSX and Linuxbrew for linux systems. https://github.com/mxcl/homebrew https://github.com/Homebrew/linuxbrew """ if env.distribution == "macosx": with quiet(): test_brewcmd = env.safe_run("brew --version") if not test_brewcmd.succeeded: env.safe_run('ruby -e "$(curl -fsSL https://raw.github.com/mxcl/homebrew/go/install)"') else: brewcmd = os.path.join(env.system_install, "bin", "brew") with quiet(): test_brewcmd = env.safe_run("%s --version" % brewcmd) if not test_brewcmd.succeeded or _linuxbrew_origin_problem(brewcmd): with shared._make_tmp_dir() as tmp_dir: with cd(tmp_dir): if env.safe_exists("linuxbrew"): env.safe_run("rm -rf linuxbrew") for cleandir in ["Library", ".git"]: if env.safe_exists("%s/%s" % (env.system_install, cleandir)): env.safe_run("rm -rf %s/%s" % (env.system_install, cleandir)) env.safe_run("git clone https://github.com/Homebrew/linuxbrew.git") with cd("linuxbrew"): if not env.safe_exists(env.system_install): env.safe_sudo("mkdir -p %s" % env.system_install) env.safe_sudo("chown %s %s" % (env.user, env.system_install)) paths = ["bin", "etc", "include", "lib", "lib/pkgconfig", "Library", "sbin", "share", "var", "var/log", "share/java", "share/locale", "share/man", "share/man/man1", "share/man/man2", "share/man/man3", "share/man/man4", "share/man/man5", "share/man/man6", "share/man/man7", "share/man/man8", "share/info", "share/doc", "share/aclocal", "lib/python2.7/site-packages", "lib/python2.6/site-packages", "lib/python3.2/site-packages", "lib/python3.3/site-packages", "lib/perl5", "lib/perl5/site_perl"] if not env.safe_exists("%s/bin" % env.system_install): env.safe_sudo("mkdir -p %s/bin" % env.system_install) for path in paths: if env.safe_exists("%s/%s" % (env.system_install, path)): env.safe_sudo("chown %s %s/%s" % (env.user, env.system_install, path)) if not env.safe_exists("%s/Library" % env.system_install): env.safe_run("mv Library %s" % env.system_install) if not env.safe_exists("%s/.git" % env.system_install): env.safe_run("mv .git %s" % env.system_install) man_dir = "share/man/man1" if not env.safe_exists("%s/%s" % (env.system_install, man_dir)): env.safe_run("mkdir -p %s/%s" % (env.system_install, man_dir)) env.safe_run("mv -f %s/brew.1 %s/%s" % (man_dir, env.system_install, man_dir)) env.safe_run("mv -f bin/brew %s/bin" % env.system_install) def _linuxbrew_origin_problem(brewcmd): """Check for linuxbrew origins which point to Homebrew instead of Linuxbrew. """ config_file = os.path.abspath(os.path.normpath((os.path.expanduser( os.path.join(os.path.dirname(brewcmd), os.pardir, ".git", "config"))))) if not os.path.exists(config_file): return True with open(config_file) as in_handle: return "linuxbrew" not in in_handle.read() @_if_not_installed("s3fs") def install_s3fs(env): """FUSE-based file system backed by Amazon S3. https://code.google.com/p/s3fs/ """ version = "1.61" url = "http://s3fs.googlecode.com/files/s3fs-{0}.tar.gz".format(version) _get_install(url, env, _configure_make) ================================================ FILE: cloudbio/custom/vcr.py ================================================ # # vcr.py # - Configures the environment for running the Viral Assembly (viral_assembly_pipeline.py) and VIGOR (VIGOR3.pl) pipelines (creating directory structure and installs software). # from __future__ import print_function import os.path, re, mmap from fabric.api import cd, env, hide, local, run, settings, sudo, task from fabric.network import disconnect_all # Common variables dependency_URL = "http://s3.amazonaws.com/VIGOR-GSC" # Galaxy VCR galaxy_central = "/mnt/galaxyTools/galaxy-central" galaxy_VCR_path = "/%s/tools/viral_assembly_annotation" % galaxy_central # Galaxy VCR - install method def install_galaxy_vcr(env): with cd("~"): print("Installing galaxy VCR tools (python and xml scripts).") sudo("git clone git://github.com/JCVI-Cloud/galaxy-tools-vcr.git") sudo("cp -R galaxy-tools-vcr/tools/viral_assembly_annotation %s" % galaxy_VCR_path) sudo("chown -R galaxy:galaxy %s" % galaxy_VCR_path) with cd(galaxy_central): print("Adding VCR to tool_conf.xml.") tcx_file = "tool_conf.xml" _set_pre_VCR(tcx_file,"galaxy","galaxy") tcx_string = _get_file_string(tcx_file,galaxy_central) vcr_header = "
\"" if (tcx_string.find(vcr_header) != -1): print("Galaxy VCR tools already included in tools_conf.xml!") else: sudo("sed -i '$d' %s/%s" % (galaxy_central,tcx_file)) sudo("echo -e '
' >> %s" % tcx_file) sudo("echo -e ' ' >> %s" % tcx_file) sudo("echo -e ' ' >> %s" % tcx_file) sudo("echo -e '
' >> %s" % tcx_file) sudo("echo -e '' >> %s" % tcx_file) print("Adding 'sanitize_all_html = False' to universe_wsgi.ini to enable JBrowse for VICVB.") uwi_file = "universe_wsgi.ini" _set_pre_VCR(uwi_file,"galaxy","galaxy") uwi_string = _get_file_string(uwi_file,galaxy_central) if (uwi_string.find("sanitize_all_html") != -1): print("Setting sanitize_all_html in %s to False." % uwi_file) sudo("sed -i '/^sanitize_all_html/c\sanitize_all_html = False' %s" % uwi_file) else: print("No sanitize_all_html present! Adding...") sudo("sed -i '/^\[app:main\]/a\\\nsanitize_all_html = False' %s" % uwi_file) # Viral Assembly viral_dirs = {} viral_urls = {} viral_tars = {} # Viral Assembly - install methods def install_viralassembly(env): try: _initialize_area_viral() _add_tools_viral() _add_refs() sudo("chmod -R 755 %(VIRAL_ROOT_DIR)s" % env) finally: disconnect_all() def install_viralassembly_cleanall(env): try: _initialize_env("viral") _remove_dir("%(VIRAL_ROOT_DIR)s" % env) print("Viral Assembly Removed\n") finally: disconnect_all() # Viral Assembly - utility methods def _initialize_area_viral(): _initialize_env("viral") env.VIRAL_SCRIPT = "%s/viral_assembly_pipeline.py" % dependency_URL viral_dirs["PROJECT_DIR"] = "%(VIRAL_ROOT_DIR)s/project" % env viral_dirs["REF_DIR"] = "%(VIRAL_ROOT_DIR)s/references" % env viral_dirs["TOOLS_DIR"] = "%(VIRAL_ROOT_DIR)s/tools" % env viral_dirs["TOOLS_BINARIES_DIR"] = "%s/BINARIES" % viral_dirs["TOOLS_DIR"] viral_dirs["TOOLS_PERL_DIR"] = "%s/PERL" % viral_dirs["TOOLS_DIR"] env.VIRAL_REF_FILES = "corona_virus,hadv,influenza_a_virus,jev,mpv,norv,rota_virus,rsv,veev,vzv,yfv" viral_urls["BIO_LINUX_URL"] = "http://nebc.nerc.ac.uk/bio-linux/" viral_tars["BINARIES_TARBALL"] = "BINARIES.tgz" viral_tars["PERL_TARBALL"] = "PERL.tgz" print("user: %(user)s" % env) print("host: %(host)s" % env) print("ROOT DIR: %(VIRAL_ROOT_DIR)s" % env) print("VIRAL ASSEMBLY SCRIPT: %(VIRAL_SCRIPT)s" % env) for name in sorted(viral_dirs.keys()): if not _path_is_dir(viral_dirs[name]): sudo("mkdir -p %s" % viral_dirs[name]) print("%s: %s" % (name,viral_dirs[name])) print("VIRAL ASSEMBLY REFS FILES: %(VIRAL_REF_FILES)s" % env) for name in sorted(viral_urls.keys()): print("%s: %s" % (name,viral_urls[name])) for name in sorted(viral_tars.keys()): print("%s: %s" % (name,viral_tars[name])) def _add_tools_viral(): with cd("/home/ubuntu/"): bashrc_file = ".bashrc" _set_pre_VCR(bashrc_file,"ubuntu","ubuntu") bashrc_string = _get_file_string(bashrc_file,"/home/ubuntu/") if (bashrc_string.find("DEBIAN_FRONTEND") != -1): print("Setting DEBIAN_FRONTEND in %s to noninteractive." % bashrc_file) sudo("sed -i \"/DEBIAN_FRONTEND/c\DEBIAN_FRONTEND=noninteractive\" %s/%s" % ("/home/ubuntu",bashrc_file)) else: print("No DEBIAN_FRONTEND present! Adding...") sudo("echo -e \"DEBIAN_FRONTEND=noninteractive\" >> %s" % bashrc_file) sudo("wget --no-check-certificate -O %s/viral_assembly_pipeline.py %s" % (env.VIRAL_ROOT_DIR,env.VIRAL_SCRIPT)) _add_package(dependency_URL,viral_tars["BINARIES_TARBALL"],viral_dirs["TOOLS_BINARIES_DIR"],"tar") _add_package(dependency_URL,viral_tars["PERL_TARBALL"],viral_dirs["TOOLS_PERL_DIR"],"tar") _apt_get_install("csh") _apt_get_install("gawk") _initialize_bio_linux() def _add_refs(): files = (env.VIRAL_REF_FILES).split(",") for file in files: _add_package(dependency_URL,"%s.tgz" % file,viral_dirs["REF_DIR"],"tar") def _initialize_bio_linux(): sudo("echo -e \"deb %s unstable bio-linux\" >> /etc/apt/sources.list" % viral_urls["BIO_LINUX_URL"]) sudo("sudo apt-get update") _apt_get_install("bio-linux-keyring") _apt_get_install("bwa") _apt_get_install("samtools") _apt_get_install("bio-linux-cap3") _apt_get_install("emboss") # VIGOR vigor_dirs = {} vigor_urls = {} vigor_tars = {} vigor_names = {} # VIGOR - install methods def install_viralvigor(env): try: _initialize_area_vigor() _initialize_host() _add_vigor() _add_tools_vigor() finally: disconnect_all() def install_viralvigor_test(env): try: _initialize_area_vigor() cmd = ("""%s/VIGOR3.pl \ -D yfv \ -i %s/westnile.fasta \ -O %s/westnile \ > %s/westnile_test_run.log 2>&1 \ """) % (vigor_dirs["VIGOR_RUNTIME_DIR"],vigor_dirs["VIGOR_SAMPLE_DATA_DIR"],vigor_dirs["VIGOR_TEST_OUTPUT_DIR"],env.VIGOR_SCRATCH_DIR) print("DEBUG: cmd[%s]" % cmd) run(cmd) finally: disconnect_all() def install_viralvigor_validate(env): try: _initialize_area_vigor() sudo("rm -f %s/westnile.rpt" % vigor_dirs["VIGOR_TEST_OUTPUT_DIR"]) sudo("rm -f %s/westnile.rpt" % vigor_dirs["VIGOR_SAMPLE_DATA_DIR"]) with settings(hide("running","stdout")): results = run("""diff -Bwr %s %s || echo 'VALIDATION FAILED'""" % (vigor_dirs["VIGOR_SAMPLE_DATA_DIR"],vigor_dirs["VIGOR_TEST_OUTPUT_DIR"])) if results: print("\n\nValidation Failed:\n\n%s\n" % results) finally: disconnect_all() def install_viralvigor_cleanall(env): try: _initialize_env("vigor") _remove_dir(env.VIGOR_ROOT_DIR) _remove_dir(env.VIGOR_SCRATCH_DIR) print("Vigor Removed\n") finally: disconnect_all() # VIGOR - utility methods def _initialize_area_vigor(): machine = run("uname -m") if machine.find('64')>0: env.ARCH = 'x64-linux' else: env.ARCH = 'ia32-linux' _initialize_env("vigor") vigor_dirs["TOOLS_DIR"] = "%s/tools" % env.VIGOR_ROOT_DIR vigor_dirs["VIGOR_STORED_DIR"] = "%s/vigor" % vigor_dirs["TOOLS_DIR"] vigor_dirs["VIGOR_RUNTIME_DIR"] = "%s/prod3" % vigor_dirs["VIGOR_STORED_DIR"] vigor_dirs["VIGOR_TEMPSPACE_DIR"] = "%s/tempspace" % env.VIGOR_SCRATCH_DIR vigor_dirs["VIGOR_SAMPLE_DATA_DIR"] = "%s/samples" % vigor_dirs["VIGOR_STORED_DIR"] vigor_dirs["VIGOR_TEST_OUTPUT_DIR"] = "%s/test" % vigor_dirs["VIGOR_STORED_DIR"] vigor_dirs["BLAST_DIR"] = "%s/blast" % vigor_dirs["TOOLS_DIR"] vigor_dirs["CLUSTALW_DIR"] = "%s/clustalw" % vigor_dirs["TOOLS_DIR"] vigor_dirs["EXE_DIR"] = vigor_dirs["VIGOR_RUNTIME_DIR"] vigor_names["BLAST_NAME"] = 'blast-2.2.15' vigor_names["CLUSTALW_NAME"] = 'clustalw-1.83' vigor_names["VIGOR_NAME"] = 'vigor-GSCcloud' vigor_tars["VIGOR_TAR_FILENAME"] = "%s.tgz" % vigor_names["VIGOR_NAME"] vigor_tars["BLAST_TAR_FILENAME"] = "%s-%s.tar.gz" % (vigor_names["BLAST_NAME"],env.ARCH) vigor_tars["CLUSTALW_TAR_FILENAME"] = "%s-%s.deb" % (vigor_names["CLUSTALW_NAME"],env.ARCH) print("user: %(user)s" % env) print("host: %(host)s" % env) print("ARCH: %(ARCH)s" % env) print("ROOT DIR: %(VIGOR_ROOT_DIR)s" % env) print("SCRATCH DIR: %(VIGOR_SCRATCH_DIR)s" % env) for name in sorted(vigor_dirs.keys()): print("%s: %s" % (name,vigor_dirs[name])) for name in sorted(vigor_urls.keys()): print("%s: %s" % (name,vigor_urls[name])) print("BLAST_NAME: %s" % vigor_names["BLAST_NAME"]) print("CLUSTALW_NAME: %s" % vigor_names["CLUSTALW_NAME"]) print("VIGOR_NAME: %s" % vigor_names["VIGOR_NAME"]) for name in sorted(vigor_tars.keys()): print("%s: %s" % (name,vigor_tars[name])) def _initialize_host(): local("ssh-keygen -R %(host)s" % env) _fix_etc_hosts() _create_vigor_scratch_dir() def _add_vigor(): print("Installing VIGOR...") _create_vigor_tempspace_dir() _create_vigor_scratch_dir() _add_package(dependency_URL, vigor_tars["VIGOR_TAR_FILENAME"], vigor_dirs["VIGOR_STORED_DIR"], "tar") sudo("chmod 755 %s" % os.path.join(vigor_dirs["VIGOR_RUNTIME_DIR"], "*.pl")) if not _path_exists(os.path.join(vigor_dirs["EXE_DIR"], "perl")): sudo("ln -sf %s %s" % ("/usr/bin/perl", vigor_dirs["EXE_DIR"])) sudo("ln -sf %s %s" % ("/usr/bin/perl", "/usr/local/bin")) if not _path_exists(os.path.join(vigor_dirs["EXE_DIR"], "vigorscratch")): sudo("ln -sf %s %s/vigorscratch" % (vigor_dirs["VIGOR_TEMPSPACE_DIR"], vigor_dirs["EXE_DIR"])) def _add_tools_vigor(): print("Install tools...") _create_tools_dir() _add_blast() _add_clustalw() _apt_get_install("libapache-dbi-perl") _apt_get_install("libclass-dbi-sqlite-perl") def _fix_etc_hosts(): internal_ip = sudo("hostname") print("internal_ip[%s]" % internal_ip) filespec = "/etc/hosts" sudo("echo '127.0.0.1 %s' >> %s" % (internal_ip, filespec)) def _create_vigor_tempspace_dir(): if not _path_is_dir(vigor_dirs["VIGOR_TEMPSPACE_DIR"]): sudo("mkdir -p %s" % vigor_dirs["VIGOR_TEMPSPACE_DIR"]) sudo("chown -R %s:%s %s" % (env.user, env.user, vigor_dirs["VIGOR_TEMPSPACE_DIR"])) sudo("find %s -type d -exec chmod 777 {} \;" % vigor_dirs["VIGOR_TEMPSPACE_DIR"]) def _create_vigor_scratch_dir(): if not _path_is_dir(env.VIGOR_SCRATCH_DIR): sudo("mkdir -p %s" % env.VIGOR_SCRATCH_DIR) sudo("find %s -type f -exec chmod 666 {} \;" % env.VIGOR_SCRATCH_DIR) sudo("find %s -type d -exec chmod 777 {} \;" % env.VIGOR_SCRATCH_DIR) def _create_tools_dir(): if not _path_is_dir(vigor_dirs["TOOLS_DIR"]): sudo("mkdir -p %s" % vigor_dirs["TOOLS_DIR"]) sudo("chown -R %s:%s %s" % (env.user,env.user,vigor_dirs["TOOLS_DIR"])) def _add_blast(): print(" Installing blast...") _create_tools_dir() _add_package(dependency_URL, vigor_tars["BLAST_TAR_FILENAME"], vigor_dirs["BLAST_DIR"], "tar") if not _path_exists(os.path.join(vigor_dirs["EXE_DIR"], "blastall")): sudo("ln -sf %s %s" % (os.path.join(vigor_dirs["BLAST_DIR"], vigor_names["BLAST_NAME"], "bin", "bl2seq"), vigor_dirs["EXE_DIR"])) sudo("ln -sf %s %s" % (os.path.join(vigor_dirs["BLAST_DIR"], vigor_names["BLAST_NAME"], "bin", "blastall"), vigor_dirs["EXE_DIR"])) sudo("ln -sf %s %s" % (os.path.join(vigor_dirs["BLAST_DIR"], vigor_names["BLAST_NAME"], "bin", "fastacmd"), vigor_dirs["EXE_DIR"])) sudo("ln -sf %s %s" % (os.path.join(vigor_dirs["BLAST_DIR"], vigor_names["BLAST_NAME"], "bin", "formatdb"), vigor_dirs["EXE_DIR"])) def _add_clustalw(): print(" Installing clustalw...") _create_tools_dir() _add_package(dependency_URL, vigor_tars["CLUSTALW_TAR_FILENAME"], vigor_dirs["CLUSTALW_DIR"], "deb") if not _path_exists(os.path.join(vigor_dirs["EXE_DIR"], "clustalw")): sudo("ln -sf %s %s" % (os.path.join(vigor_dirs["CLUSTALW_DIR"], vigor_names["CLUSTALW_NAME"], "clustalw"), vigor_dirs["EXE_DIR"])) # VICVB - install methods def install_vicvb(env): try: _initialize_env("vicvb") _apt_get_install("libperlio-gzip-perl") _apt_get_install("liblocal-lib-perl") tbl2asn_download_dir = "/usr/local/tbl2asn_download" tbl2asn_dir = "/usr/local/bin" if _path_exists(os.path.join(tbl2asn_dir,"tbl2asn")): sudo("mv %s/tbl2asn %s/tbl2asn_pre_VCR" % (tbl2asn_dir,tbl2asn_dir)) _add_package("ftp://ftp.ncbi.nih.gov/toolbox/ncbi_tools/converters/by_program/tbl2asn","linux64.tbl2asn.gz",tbl2asn_download_dir,"gzip") sudo("chmod 777 %s/linux64.tbl2asn" % tbl2asn_download_dir) sudo("mv %s/linux64.tbl2asn %s/tbl2asn" % (tbl2asn_download_dir,tbl2asn_dir)) _remove_dir(tbl2asn_download_dir) with cd("~"): sudo("git clone git://github.com/JCVI-Cloud/VICVB.git") with cd("~/VICVB"): sudo("lib/VICVB/data/install/install_to_dir_full.sh %s /mnt/galaxyTools/galaxy-central /" % (env.VICVB_LOCAL_DIR)) finally: disconnect_all() def install_vicvb_cleanall(env): try: _initialize_env("vicvb") _remove_dir(env.VICVB_LOCAL_DIR) _remove_dir(env.VICVB_GALAXY_DIR) with cd ("~"): sudo("rm -fr ~/VICVB") print("VICVB Removed\n") finally: disconnect_all() # Common methods def _initialize_env(pipeline): if pipeline == "viral": env.VIRAL_ROOT_DIR = "/usr/local/VHTNGS" if not _path_exists(env.VIRAL_ROOT_DIR): sudo("mkdir -p %s" % env.VIRAL_ROOT_DIR) elif pipeline == "vigor": env.VIGOR_ROOT_DIR = "/usr/local/VIGOR" if not _path_exists(env.VIGOR_ROOT_DIR): sudo("mkdir -p %s" % env.VIGOR_ROOT_DIR) env.VIGOR_SCRATCH_DIR = "/usr/local/scratch/vigor" if not _path_exists(env.VIGOR_SCRATCH_DIR): sudo("mkdir -p %s" % env.VIGOR_SCRATCH_DIR) sudo("find %s -type f -exec chmod 666 {} \;" % env.VIGOR_SCRATCH_DIR) sudo("find %s -type d -exec chmod 777 {} \;" % env.VIGOR_SCRATCH_DIR) else: env.VICVB_LOCAL_DIR = "/usr/local/VICVB"; env.VICVB_GALAXY_DIR = "/mnt/galaxyTools/galaxy-central/static/vicvb"; def _add_package(download_url, filename, install_dir, type): if not _path_is_dir(install_dir): sudo("mkdir -p %s" % install_dir) with cd(install_dir): if not _path_exists(os.path.join(install_dir, filename)): sudo("""wget --no-host-directories --cut-dirs=1 --directory-prefix=%s %s/%s""" % (install_dir, download_url, filename)) if type == "tar": sudo("tar xvfz %s" % filename) elif type == "bz2": sudo("tar xfj %s" % filename) elif type == "gzip": sudo("gunzip %s" % filename) else: sudo("dpkg -x %s %s" % (filename,install_dir)) sudo("mkdir %s/%s" % (install_dir, vigor_names["CLUSTALW_NAME"])) sudo("cp %s/usr/bin/* %s/%s" % (install_dir,install_dir,vigor_names["CLUSTALW_NAME"])) sudo("chown -R %s:%s %s" % (env.user, env.user, install_dir)) sudo("find %s -type d -exec chmod 755 {} \;" % install_dir) def _remove_dir(dirspec): if _path_is_dir(dirspec): _unlock_dir(dirspec) sudo("rm -rf %s" % dirspec) else: print("DEBUG: _remove_dir[%s] -- NOT FOUND" % dirspec) def _unlock_dir(dirspec): with settings(hide("running","stdout")): sudo("find %s -type d -exec chmod 755 {} \;" % dirspec) sudo("find %s -type d -exec chmod g+s {} \;" % dirspec) sudo("find %s -type f -exec chmod 644 {} \;" % dirspec) def _apt_get_install(tool): sudo("apt-get -q -y --force-yes install %s" % tool) def _path_exists(path): found = False with settings(hide("running","stdout")): result = sudo("test -e '%s' || echo 'FALSE'" % path) if result != "FALSE": found = True return found def _path_is_dir(path): found = False with settings(hide("running","stdout")): result = sudo("test -d '%s' || echo 'FALSE'" % path) if result != "FALSE": found = True return found def _set_pre_VCR(filename,user,group): sudo("cp %s %s_pre_VCR" %(filename,filename)) sudo("chown %s:%s %s_pre_VCR" % (user,group,filename)) def _get_file_string(filename,directory): fh = open("%s/%s" % (directory,filename)) string = mmap.mmap(fh.fileno(),0,access=mmap.ACCESS_READ) fh.close() return string ================================================ FILE: cloudbio/custom/versioncheck.py ================================================ """Tool specific version checking to identify out of date dependencies. This provides infrastructure to check version strings against installed tools, enabling re-installation if a version doesn't match. This is a lightweight way to avoid out of date dependencies. """ from __future__ import print_function from distutils.version import LooseVersion from cloudbio.custom import shared from cloudbio.fabutils import quiet def _parse_from_stdoutflag(out, flag, stdout_index=-1): """Extract version information from a flag in verbose stdout. flag -- text information to identify the line we should split for a version stdout_index -- Position of the version information in the split line. Defaults to the last item. """ for line in out.split("\n") + out.stderr.split("\n"): if line.find(flag) >= 0: parts = line.split() return parts[stdout_index].strip() print("Did not find version information with flag %s from: \n %s" % (flag, out)) return "" def _clean_version(x): if x.startswith("upstream/"): x = x.replace("upstream/", "") if x.startswith("("): x = x[1:].strip() if x.endswith(")"): x = x[:-1].strip() if x.startswith("v"): x = x[1:].strip() return x def up_to_date(env, cmd, version, args=None, stdout_flag=None, stdout_index=-1): iversion = get_installed_version(env, cmd, version, args, stdout_flag, stdout_index) if not iversion: return False else: return LooseVersion(iversion) >= LooseVersion(version) def is_version(env, cmd, version, args=None, stdout_flag=None, stdout_index=-1): iversion = get_installed_version(env, cmd, version, args, stdout_flag, stdout_index) if not iversion: return False else: return LooseVersion(iversion) == LooseVersion(version) def get_installed_version(env, cmd, version, args=None, stdout_flag=None, stdout_index=-1): """Check if the given command is up to date with the provided version. """ if shared._executable_not_on_path(cmd): return False if args: cmd = cmd + " " + " ".join(args) with quiet(): path_safe = ("export PKG_CONFIG_PATH=$PKG_CONFIG_PATH:{s}/lib/pkgconfig && " "export PATH=$PATH:{s}/bin && " "export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:{s}/lib && ".format(s=env.system_install)) out = env.safe_run_output(path_safe + cmd) if stdout_flag: iversion = _parse_from_stdoutflag(out, stdout_flag, stdout_index) else: iversion = out.strip() iversion = _clean_version(iversion) if " not found in the pkg-config search path" in iversion: return False return iversion ================================================ FILE: cloudbio/deploy/__init__.py ================================================ #!/usr/bin/env python from __future__ import print_function import os from tempfile import tempdir from subprocess import call from inspect import getargspec from cloudbio.utils import _setup_logging, _configure_fabric_environment, _parse_fabricrc from cloudbio.biodata.genomes import install_data, install_data_s3, install_data_rsync from cloudbio.galaxy import _setup_galaxy_env_defaults from cloudbio.galaxy.utils import _chown_galaxy from cloudbio.galaxy.tools import _install_tools from fabfile import _perform_install, _install_custom from .util import eval_template import cloudbio.deploy.plugins from fabric.main import load_settings from fabric.api import put, run, env, settings, sudo try: from .vmlauncher.transfer import FileTransferManager from .vmlauncher import build_vm_launcher except ImportError: build_vm_launcher = None FileTransferManager = None DEFAULT_CLOUDBIOLINUX_TARGET = None DEFAULT_CLOUDBIOLINUX_FLAVOR = None def deploy(options): _setup_logging(env) actions = _expand_actions(options.get("actions")) if options["vm_provider"] == "novm": vm_launcher = LocalVmLauncher(options) else: if not build_vm_launcher: raise ImportError("Require vmlauncher: https://github.com/jmchilton/vm-launcher") vm_launcher = build_vm_launcher(options) if _do_perform_action("list", actions): for node in vm_launcher.list(): print("Active node with uuid %s <%s>" % (node.uuid, node)) if _do_perform_action("destroy", actions): target_name = options["hostname"] for node in vm_launcher.list(): node_name = node.name if node_name == target_name: vm_launcher.destroy(node) __invoke_plugin_actions(env, actions, "local_actions", [vm_launcher, options]) # Do we have remaining actions requiring an vm? if len(actions) > 0: print('Setting up virtual machine') vm_launcher.boot_and_connect() _setup_vm(options, vm_launcher, actions) class LocalVmLauncher: """Provide a lightweight real machine, non-vm class for launching. """ def __init__(self, options): self.options = options def get_ip(self): specified_hostname = self.options.get("hostname", None) hostname = specified_hostname or "localhost" return hostname def get_key_file(self): return None def boot_and_connect(self): pass def destroy(self): pass def get_user(self): return env.user def list(self): return [] def _setup_vm(options, vm_launcher, actions): destroy_on_complete = get_boolean_option(options, 'destroy_on_complete', False) try: ip = vm_launcher.get_ip() _setup_fabric(vm_launcher, ip, options) with settings(host_string=ip): _setup_cloudbiolinux(options) if 'attach_volumes' in actions: from .volume import attach_volumes attach_volumes(vm_launcher, options) if 'max_lifetime' in options: seconds = options['max_lifetime'] # Unclear why the sleep is needed, but seems to be otherwise # this doesn't work. run("bash -c 'nohup sudo shutdown -h %d &'; sleep 2" % seconds) configure_instance(options, actions) if 'transfer' in actions: transfer_files(options) __invoke_plugin_actions(env, actions, "ready_actions", [vm_launcher, options]) if 'ssh' in actions: _interactive_ssh(vm_launcher) if 'attach_ip' in actions: vm_launcher.attach_public_ip() if 'snapshot_volumes' in actions: from .volume import make_snapshots make_snapshots(vm_launcher, options) if 'detach_volumes' in actions: from .volume import detach_volumes detach_volumes(vm_launcher, options) if 'package' in actions: name_template = vm_launcher.package_image_name() name = eval_template(env, name_template) vm_launcher.package(name=name) if not destroy_on_complete and hasattr(vm_launcher, "uuid"): print('Your instance (%s) is waiting at http://%s' % (vm_launcher.uuid, ip)) finally: if destroy_on_complete: vm_launcher.destroy() def _expand_actions(actions): unique_actions = set() for simple_action in _possible_actions(): if simple_action in actions: unique_actions.add(simple_action) compound_actions = __get_plugin_actions(env, "compound_actions") for compound_action in compound_actions.keys(): if compound_action in actions: for compound_action_part in compound_actions[compound_action]: unique_actions.add(compound_action_part) return unique_actions def _possible_actions(): possible_actions = [ "list", "destroy", "transfer", "purge_tools", "setup_tools", "setup_biodata", "setup_ssh_key", "package", "setup_image", "launch", # Dummy action justs launches image "install_biolinux", "install_custom", "ssh", "attach_ip", "snapshot_volumes", "attach_volumes", "detach_volumes", ] for action_type in ["local_actions", "configure_actions", "ready_action"]: for action in __get_plugin_actions(env, action_type): possible_actions.append(action) return possible_actions def _do_perform_action(action, action_list): do_perform = action in action_list if do_perform: action_list.remove(action) return do_perform def _setup_fabric(vm_launcher, ip, options): env.user = vm_launcher.get_user() env.hosts = [ip] env.key_filename = vm_launcher.get_key_file() env.disable_known_hosts = True def _setup_cloudbiolinux(options): def fabricrc_loader(env): _setup_cloudbiolinux_fabric_properties(env, options) flavor = get_main_options_string(options, "flavor", DEFAULT_CLOUDBIOLINUX_FLAVOR) need_distcheck = options.get("fabricrc_overrides", {}).get("use_sudo") _configure_fabric_environment(env, flavor, fabricrc_loader=fabricrc_loader, ignore_distcheck=not need_distcheck) _setup_image_user_data(env, options) def _setup_cloudbiolinux_fabric_properties(env, options): fabricrc_file = get_main_options_string(options, "fabricrc_file", None) env.config_dir = os.path.join(os.path.dirname(__file__), "..", "..", "config") env.tool_data_table_conf_file = os.path.join(env.config_dir, "..", "installed_files", "tool_data_table_conf.xml") if fabricrc_file: env.update(load_settings(fabricrc_file)) else: # Let cloudbiolinux find out default file based on flavor, dist, etc... _parse_fabricrc(env) overrides = options.get("fabricrc_overrides", {}) for key, value in overrides.iteritems(): # yaml parses bools, wouldn't be expected coming out of a fabricrc # file so replace everything with a string. if isinstance(value, bool): overrides[key] = str(value) env.update(overrides) _setup_galaxy_env_defaults(env) def _setup_image_user_data(env, options): if "image_user_data" in options: env["image_user_data_dict"] = options["image_user_data"] def purge_genomes(): sudo("rm -rf %s" % env.data_files) def configure_ssh_key(options): if "galaxy_ssh_key" in options: key_file = options["galaxy_ssh_key"] sudo("mkdir -p /home/%s/.ssh" % (env.galaxy_user)) sudo("chmod 700 /home/%s/.ssh" % (env.galaxy_user)) put(local_path=key_file, remote_path="/home/%s/.ssh/%s" % (env.galaxy_user, os.path.basename(key_file)), use_sudo=True, mode="0600") _chown_galaxy(env, "/home/%s/.ssh" % env.galaxy_user) def setup_biodata(options): install_proc = install_data genome_source = options.get("genome_source", "default") install_proc = { "default": install_data, "S3": install_data_s3, "rsync": install_data_rsync, }[genome_source] if genome_source == "default": install_proc(options["genomes"], ["ggd", "s3", "raw"]) else: install_proc(options["genomes"]) def configure_instance(options, actions): if "install_biolinux" in actions: install_biolinux(options) if "install_custom" in actions: install_custom(options) if "purge_tools" in actions: purge_tools() __invoke_plugin_actions(env, actions, "configure_actions", [options]) if "setup_tools" in actions: install_tools(options["tools"]) if "setup_biodata" in actions: setup_biodata(options) if "setup_ssh_key" in actions: configure_ssh_key(options) def install_custom(options): package = options.get("package") _install_custom(package) def install_biolinux(options): flavor = options.get("flavor", DEFAULT_CLOUDBIOLINUX_FLAVOR) target = options.get("target", DEFAULT_CLOUDBIOLINUX_TARGET) _perform_install(target=target, flavor=flavor, more_custom_add=options.get("custom_add", None)) def _interactive_ssh(vm_launcher): """ Launch an interactive SSH session to host described by vm_launcher object. """ host = vm_launcher.get_ip() user = vm_launcher.get_user() key_file = vm_launcher.get_key_file() cmd = "ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i '%s' -l '%s' '%s'" % (key_file, user, host) call(cmd, shell=True) def transfer_files(options): transfer_options = _build_transfer_options(options, "/mnt/uploaded_data", "galaxy") _do_transfer(transfer_options, options.get("files", []), options.get("compressed_files", [])) def _build_transfer_options(options, destination, user): transfer_options = {} transfer_options['compress'] = get_boolean_option(options, 'compress_transfers', True) transfer_options['num_compress_threads'] = int(get_main_options_string(options, 'num_compress_threads', '1')) transfer_options['num_transfer_threads'] = int(get_main_options_string(options, 'num_transfer_threads', '1')) transfer_options['num_decompress_threads'] = int(get_main_options_string(options, 'num_decompress_threads', '1')) transfer_options['chunk_size'] = int(get_main_options_string(options, 'transfer_chunk_size', '0')) transfer_options['transfer_retries'] = int(get_main_options_string(options, 'transfer_retries', '3')) transfer_options['local_temp'] = get_main_options_string(options, 'local_temp_dir', tempdir) transfer_options['destination'] = destination transfer_options['transfer_as'] = user return transfer_options def _do_transfer(transfer_options, files, compressed_files=[]): if not FileTransferManager: raise ImportError("Require vmlauncher: https://github.com/jmchilton/vm-launcher") FileTransferManager(**transfer_options).transfer_files(files, compressed_files) def purge_tools(): env.safe_sudo("rm -rf %s" % env.install_dir) def install_tools(tools_conf): """ """ _install_tools(env, tools_conf) def get_boolean_option(options, name, default=False): if name not in options: return default else: return options[name] def get_main_options_string(options, key, default=''): value = default if key in options: value = options[key] return value def __invoke_plugin_actions(env, actions, action_type, provided_args): possible_actions = __get_plugin_actions(env, action_type) for action in list(actions): if action in possible_actions: __invoke_plugin_action(env, possible_actions[action], provided_args) actions.remove(action) def __invoke_plugin_action(env, action_function, provided_args): arg_spec = getargspec(action_function).args args = [] if not arg_spec else provided_args action_function(*args) def __get_plugin_actions(env, action_type): actions = {} for plugin_module in __get_plugin_modules(env): if hasattr(plugin_module, action_type): for action_name, action_function in getattr(plugin_module, action_type).iteritems(): actions[action_name] = action_function return actions def __get_plugin_modules(env): if not "plugin_modules" in env: unsorted_module_names = __get_plugin_module_names( ) ## Load modules in reverse order to allow hierarchical overrides module_names = sorted(unsorted_module_names, reverse=True) modules = [] for plugin_module_name in module_names: try: module = __import__(plugin_module_name) for comp in plugin_module_name.split(".")[1:]: module = getattr(module, comp) modules.append(module) except BaseException as exception: exception_str = str(exception) message = "%s rule module could not be loaded: %s" % (plugin_module_name, exception_str) env.logger.warn(message) continue env.plugin_modules = modules return env.plugin_modules def __get_plugin_module_names(): plugin_module_dir = cloudbio.deploy.plugins.__path__[0] names = [] for fname in os.listdir(plugin_module_dir): if not(fname.startswith("_")) and fname.endswith(".py"): rule_module_name = "cloudbio.deploy.plugins.%s" % fname[:-len(".py")] names.append( rule_module_name ) return names ================================================ FILE: cloudbio/deploy/config.py ================================================ import inspect import os import yaml def parse_settings(name="deploy/settings.yaml"): return _read_yaml(_path_from_root(name)) def _path_from_root(name): root_path = os.path.join(os.path.dirname(inspect.getfile(inspect.currentframe())), "..", "..") file_path = os.path.join(root_path, name) return file_path def _read_yaml(yaml_file): with open(yaml_file) as in_handle: return yaml.safe_load(in_handle) ================================================ FILE: cloudbio/deploy/main.py ================================================ from argparse import ArgumentParser import yaml from cloudbio.deploy import deploy DESC = "Creates an on-demand cloud instance, sets up applications, and transfer files to it." ## Properties that may be specified as args or in settings file, ## argument takes precedence. ARG_PROPERTIES = [ # VM launcher options "files", "compressed_files", "actions", "runtime_properties", "vm_provider", "hostname", # CloudBioLinux options "target", "flavor", "package", # CloudMan options "target_bucket", # Galaxy options "galaxy_tool_version", "galaxy_tool_name", "galaxy_tool_dir", ] def main(): args = parse_args() options = parse_settings(args.settings) for property in ARG_PROPERTIES: _copy_arg_to_options(options, args, property) for fabric_property, fabric_value in zip(args.fabric_properties, args.fabric_values): if "fabricrc_overrides" not in options: options["fabricrc_overrides"] = {} options["fabricrc_overrides"][fabric_property] = fabric_value deploy(options) def _copy_arg_to_options(options, args, property): arg_property = getattr(args, property) if arg_property or not property in options: options[property] = arg_property def parse_args(): parser = ArgumentParser(DESC) parser.add_argument("--settings", dest="settings", default="settings.yaml") parser.add_argument('--action', dest="actions", action="append", default=[]) parser.add_argument('--runtime_property', dest="runtime_properties", action="append", default=[]) parser.add_argument('--compressed_file', dest="compressed_files", action="append", default=[], help="file to transfer to new instance and decompress") parser.add_argument('--file', dest="files", action="append", default=[], help="file to transfer to new instance") parser.add_argument("--vm_provider", dest="vm_provider", default=None, help="libcloud driver to use (or vagrant) (e.g. aws, openstack)") parser.add_argument("--hostname", dest="hostname", default=None, help="Newly created nodes are created with this specified hostname.") # CloudBioLinux options parser.add_argument("--target", dest="target", default=None, help="Specify a CloudBioLinux target, used with action install_biolinux action") parser.add_argument("--flavor", dest="flavor", default=None, help="Specify a CloudBioLinux flavor, used with action install_biolinux action") parser.add_argument("--package", dest="package", default=None, help="Specify a CloudBioLinux package, used with action install_custom") # CloudMan related options parser.add_argument("--target_bucket", dest="target_bucket", default=None, help="Specify a target bucket for CloudMan bucket related actions.") # Galaxy options parser.add_argument("--galaxy_tool_version", dest="galaxy_tool_version") parser.add_argument("--galaxy_tool_name", dest="galaxy_tool_name") parser.add_argument("--galaxy_tool_dir", dest="galaxy_tool_dir") parser.add_argument('--fabric_property', dest="fabric_properties", action="append", default=[]) parser.add_argument('--fabric_value', dest="fabric_values", action="append", default=[]) args = parser.parse_args() if len(args.actions) == 0: args.actions = ["transfer"] return args def parse_settings(name): if not name == "__none__": # Rather just die if settings.yaml does not exist or is not set, but would also # like to support pure command-line driven mode so make settings.yaml if # --settings=__none__ is passed to application. return _read_yaml(name) else: return {} def _read_yaml(yaml_file): with open(yaml_file) as in_handle: return yaml.safe_load(in_handle) if __name__ == "__main__": main() ================================================ FILE: cloudbio/deploy/plugins/__init__.py ================================================ """ Deploy plugins. local_actions: These actions occur before a VM has been created, if only local actions are specified, no VM is launched. configure_actions: Take in only options and use fabric to configure instance in some way. ready_actions: Once VM is configured, these actions are executed. compound_actions: Dictionary of list values. Key represents an short-cut action that is expanded to each action specified in corresponding value (a list of simple actions - standard or defined in the plugins). """ ================================================ FILE: cloudbio/deploy/plugins/cloudman.py ================================================ from __future__ import print_function from datetime import datetime from os.path import exists, join from os import listdir from tempfile import mkdtemp from cloudbio.deploy.util import eval_template from boto.exception import S3ResponseError from boto.s3.key import Key import yaml from fabric.api import local, lcd, env DEFAULT_BUCKET_NAME = 'cloudman' DEFAULT_CLOUDMAN_PASSWORD = 'adminpass' DEFAULT_CLOUDMAN_CLUSTER_NAME = 'cloudman' def bundle_cloudman(vm_launcher, options): cloudman_options = options.get('cloudman') cloudman_repository_path = cloudman_options['cloudman_repository'] delete_repository = False bucket_source = cloudman_options.get("bucket_source") if cloudman_repository_path.startswith("http"): # Not a local path, lets clone it out of a remote repostiroy, temp_directory = mkdtemp() if cloudman_repository_path.endswith(".git"): branch_opts = "" repository_branch = cloudman_options.get('repository_branch', None) if repository_branch: branch_opts = "-b '%s'" % repository_branch clone_command = "git clone " + branch_opts + " '%s' '%s'" else: clone_command = "hg clone '%s' '%s'" local(clone_command % (cloudman_repository_path, temp_directory)) cloudman_repository_path = temp_directory delete_repository = True try: with lcd(cloudman_repository_path): try: local("tar czvf cm.tar.gz *") local("mv cm.tar.gz '%s'" % bucket_source) finally: local("rm -f cm.tar.gz") finally: if delete_repository: local("rm -rf '%s'" % cloudman_repository_path) def cloudman_launch(vm_launcher, options): cloudman_options = options.get('cloudman') image_id = cloudman_options.get('image_id', None) if str(image_id).lower() == "__use_snaps__": # TODO: Make more flexible bucket_source = cloudman_options.get("bucket_source") snaps_path = join(bucket_source, "snaps.yaml") if not exists(snaps_path): raise Exception("CloudMan AMI set to __use_snaps__ but now snaps.yaml file could be found with path %s" % snaps_path) snaps = {} with open(snaps_path, "r") as in_handle: snaps = yaml.safe_load(in_handle) clouds = snaps["clouds"] if len(clouds) != 1: raise Exception("Exactly one cloud must be defined snaps.yaml for the deployer's CloudMan launch to work.") regions = clouds[0]["regions"] if len(regions) != 1: raise Exception("Exactly one region must be defined snaps.yaml for the deployer's CloudMan launch to work.") deployments = regions[0]["deployments"] if len(deployments) != 1: raise Exception("Exactly one deployment must be defined snaps.yaml for the deployer's CloudMan launch to work.") image_id = deployments[0]["default_mi"] size_id = cloudman_options.get('size_id', None) user_data = _prepare_user_data(vm_launcher, cloudman_options) vm_launcher.create_node('cloudman', image_id=image_id, size_id=size_id, ex_userdata=user_data) def sync_cloudman_bucket(vm_launcher, options): bucket = options.get("target_bucket", None) if not bucket: bucket = __get_bucket_default(options) bucket_source = options.get("cloudman", {}).get("bucket_source", None) if not bucket or not bucket_source: print("Warning: Failed to sync cloud bucket, bucket or bucket_source is undefined.") return conn = vm_launcher.boto_s3_connection() for file_name in listdir(bucket_source): _save_file_to_bucket(conn, bucket, file_name, join(bucket_source, file_name)) def _save_file_to_bucket(conn, bucket_name, remote_filename, local_file, **kwargs): """ Save the local_file to bucket_name as remote_filename. Also, any additional arguments passed as key-value pairs, are stored as file's metadata on S3.""" # print "Establishing handle with bucket '%s'..." % bucket_name b = _get_bucket(conn, bucket_name) if b is not None: # print "Establishing handle with key object '%s'..." % remote_filename k = Key( b, remote_filename ) print("Attempting to save file '%s' to bucket '%s'..." % (remote_filename, bucket_name)) try: # Store some metadata (key-value pairs) about the contents of the file being uploaded # Note that the metadata must be set *before* writing the file k.set_metadata('date_uploaded', str(datetime.utcnow())) for args_key in kwargs: print("Adding metadata to file '%s': %s=%s" % (remote_filename, args_key, kwargs[args_key])) k.set_metadata(args_key, kwargs[args_key]) print("Saving file '%s'" % local_file) k.set_contents_from_filename(local_file) print("Successfully added file '%s' to bucket '%s'." % (remote_filename, bucket_name)) make_public = True if make_public: k.make_public() except S3ResponseError as e: print("Failed to save file local file '%s' to bucket '%s' as file '%s': %s" % ( local_file, bucket_name, remote_filename, e )) return False return True else: return False def __get_bucket_default(options): cloudman_options = options.get("cloudman", {}) user_data = cloudman_options = cloudman_options.get('user_data', None) or {} bucket = user_data.get("bucket_default", None) return bucket def _prepare_user_data(vm_launcher, cloudman_options): cloudman_user_data = cloudman_options.get('user_data', None) or {} cluster_name = \ cloudman_options.get('cluster_name', DEFAULT_CLOUDMAN_CLUSTER_NAME) password = cloudman_options.get('password', DEFAULT_CLOUDMAN_PASSWORD) access_key = vm_launcher.access_id() secret_key = vm_launcher.secret_key() _set_property_if_needed(cloudman_user_data, 'access_key', access_key) _set_property_if_needed(cloudman_user_data, 'secret_key', secret_key) cluster_name = eval_template(env, cluster_name) _set_property_if_needed(cloudman_user_data, 'cluster_name', cluster_name) _set_property_if_needed(cloudman_user_data, 'password', password) return yaml.dump(cloudman_user_data) def _set_property_if_needed(user_data, property, value): if property not in user_data: user_data[property] = value def _get_bucket(s3_conn, bucket_name): b = None for i in range(0, 5): try: b = s3_conn.get_bucket(bucket_name) break except S3ResponseError: print("Bucket '%s' not found, attempt %s/5" % (bucket_name, i)) return None return b local_actions = { "cloudman_launch": cloudman_launch, "sync_cloudman_bucket": sync_cloudman_bucket, "bundle_cloudman": bundle_cloudman, } ================================================ FILE: cloudbio/deploy/plugins/galaxy.py ================================================ from cloudbio.galaxy.tools import _install_application def install_tool(options): version = options.get("galaxy_tool_version") name = options.get("galaxy_tool_name") install_dir = options.get("galaxy_tool_dir", None) _install_application(name, version, tool_install_dir=install_dir) configure_actions = { "install_galaxy_tool": install_tool, } ================================================ FILE: cloudbio/deploy/plugins/gvl.py ================================================ """ Deployer plugin containing actions related to older galaxy-vm-launcher functionality. """ from __future__ import print_function import os import time from cloudbio.biodata.genomes import install_data, install_data_s3 from cloudbio.deploy import get_main_options_string, _build_transfer_options, _do_transfer, transfer_files, get_boolean_option from cloudbio.deploy.util import wget, start_service, ensure_can_sudo_into, sudoers_append from cloudbio.galaxy.utils import _chown_galaxy from cloudbio.galaxy.tools import _setup_install_dir from cloudbio.custom.galaxy import install_galaxy_webapp from cloudbio.galaxy import _setup_users, _setup_xvfb, _install_nginx_standalone, _setup_postgresql from cloudbio.package import _configure_and_install_native_packages from cloudbio.package.deb import _apt_packages from fabric.api import put, run, env, sudo, get, cd from fabric.context_managers import prefix from fabric.contrib.files import append, contains, exists ## Deprecated galaxy-vm-launcher way of setting up biodata. def setup_genomes(options): install_proc = install_data sudo("mkdir -p %s" % env.data_files) sudo("chown -R %s:%s %s" % (env.user, env.user, env.data_files)) put("config/tool_data_table_conf.xml", "%s/tool_data_table_conf.xml" % env.galaxy_home) indexing_packages = ["bowtie", "bwa", "samtools"] path_extensions = ":".join(map(lambda package: "/opt/galaxyTools/tools/%s/default" % package, indexing_packages)) with prefix("PATH=$PATH:%s" % path_extensions): if 'S3' == options['genome_source']: install_proc = install_data_s3 install_proc(options["genomes"]) if options.get("setup_taxonomy_data", False): setup_taxonomy_data() stash_genomes_where = get_main_options_string(options, "stash_genomes") if stash_genomes_where: stash_genomes(stash_genomes_where) def setup_taxonomy_data(): """ Setup up taxonomy data required by Galaxy. Need to find another place to put this, it is useful. """ taxonomy_directory = os.path.join(env.data_files, "taxonomy") env.safe_sudo("mkdir -p '%s'" % taxonomy_directory, user=env.user) with cd(taxonomy_directory): taxonomy_url = "ftp://ftp.ncbi.nih.gov/pub/taxonomy/taxdump.tar.gz" gi_taxid_nucl = "ftp://ftp.ncbi.nih.gov/pub/taxonomy/gi_taxid_nucl.dmp.gz" gi_taxid_prot = "ftp://ftp.ncbi.nih.gov/pub/taxonomy/gi_taxid_prot.dmp.gz" wget(taxonomy_url) wget(gi_taxid_nucl) wget(gi_taxid_prot) run("gunzip -c taxdump.tar.gz | tar xvf -") run("gunzip gi_taxid_nucl.dmp.gz") run("gunzip gi_taxid_prot.dmp.gz") run("cat gi_taxid_nucl.dmp gi_taxid_prot.dmp > gi_taxid_all.dmp") run("sort -n -k 1 gi_taxid_all.dmp > gi_taxid_sorted.txt") run("rm gi_taxid_nucl.dmp gi_taxid_prot.dmp gi_taxid_all.dmp") run("cat names.dmp | sed s/[\\(\\)\\'\\\"]/_/g > names.temporary") run("mv names.dmp names.dmp.orig") run("mv names.temporary names.dmp") def stash_genomes(where): with _cd_indices_parent(): sudo("chown %s:%s ." % (env.user, env.user)) indices_dir_name = _indices_dir_name() remote_compressed_indices = "%s.tar.gz" % indices_dir_name run("tar czvf %s %s" % (remote_compressed_indices, indices_dir_name)) if where == 'download': get(remote_path=remote_compressed_indices, local_path="compressed_genomes.tar.gz") elif where == 'opt': sudo("cp %s /opt/compressed_genomes.tar.gz" % remote_compressed_indices) else: print("Invalid option specified for stash_genomes [%s] - valid values include download and opt." % where) def upload_genomes(options): with _cd_indices_parent(): sudo("chown %s:%s ." % (env.user, env.user)) indices_dir_name = _indices_dir_name() _transfer_genomes(options) run("rm -rf %s" % indices_dir_name) run("tar xzvfm compressed_genomes.tar.gz") sudo("/etc/init.d/galaxy restart") def purge_genomes(): sudo("rm -rf %s" % env.data_files) def _cd_indices_parent(): return cd(_indices_parent()) def _indices_parent(): parent_dir = os.path.abspath(os.path.join(env.data_files, "..")) return parent_dir def _indices_dir_name(): indices_dir = env.data_files if indices_dir.endswith("/"): indices_dir = indices_dir[0:(len(indices_dir) - 1)] indices_dir_name = os.path.basename(indices_dir) return indices_dir_name def galaxy_transfer(vm_launcher, options): transfer_files(options) # Upload local compressed genomes to the cloud image, obsecure option. do_upload_genomes = get_boolean_option(options, 'upload_genomes', False) if do_upload_genomes: upload_genomes(options) if not _seed_at_configure_time(options): seed_database() seed_workflows(options) wait_for_galaxy() create_data_library_for_uploads(options) def create_data_library_for_uploads(options): with cd(os.path.join(env.galaxy_home, "scripts", "api")): db_key_arg = get_main_options_string(options, 'db_key') transfer_history_name = get_main_options_string(options, 'transfer_history_name') transfer_history_api_key = get_main_options_string(options, 'transfer_history_api_key') cmd_template = 'python handle_uploads.py --api_key="%s" --db_key="%s" --history="%s" --history_api_key="%s" ' galaxy_data = options["galaxy"] admin_user_api_key = galaxy_data["users"][0]["api_key"] cmd = cmd_template % (admin_user_api_key, db_key_arg, transfer_history_name, transfer_history_api_key) sudo("bash -c 'export PYTHON_EGG_CACHE=eggs; %s'" % cmd, user="galaxy") def _seed_at_configure_time(options): if 'seed_galaxy' in options: return options['seed_galaxy'] == 'configure' else: return True def copy_runtime_properties(vm_launcher, options): fqdn = vm_launcher.get_ip() runtime_properties_raw = options.get("runtime_properties", {}) runtime_properties = {"FQDN": fqdn} for runtime_property_raw in runtime_properties_raw: (name, value) = runtime_property_raw.split(":") runtime_properties[name] = value export_file = "" for (name, value) in runtime_properties.iteritems(): export_file = "export %s=%s\n%s" % (name, value, export_file) sudo('mkdir -p %s' % env.galaxy_home) _chown_galaxy(env, env.galaxy_home) sudo("echo '%s' > %s/runtime_properties" % (export_file, env.galaxy_home), user=env.galaxy_user) def _transfer_genomes(options): # Use just transfer settings in YAML options = options['transfer'] transfer_options = _build_transfer_options(options, _indices_parent(), env.user) transfer_options["compress"] = False _do_transfer(transfer_options, ["compressed_genomes.tar.gz"]) def wait_for_galaxy(): while not "8080" in run("netstat -lant"): # Check if galaxy has started print("Waiting for galaxy to start.") time.sleep(10) def purge_galaxy(): sudo("/etc/init.d/galaxy stop") sudo("rm -rf %s" % env.galaxy_home) init_script = "postgresql" # if env.postgres_version[0] < '9': # # Postgres 8.4 had different name for script # init_script = "postgresql-%s" % env.postgres_version sudo("/etc/init.d/%s restart" % init_script) sudo('psql -c "drop database galaxy;"', user="postgres") sudo('psql -c "create database galaxy;"', user="postgres") def setup_galaxy(options): seed = _seed_at_configure_time(options) setup_galaxy(options, seed=seed) if seed: seed_workflows(options) def _setup_galaxy(options, seed=True): """Deploy a Galaxy server along with some tools. """ _setup_install_dir(env) # Still needed? -John install_galaxy_webapp(env) #_fix_galaxy_permissions() _setup_shed_tools_dir() _setup_galaxy_log_dir() _migrate_galaxy_database() if seed: seed_database(options["galaxy"]) _start_galaxy() def _migrate_galaxy_database(): with cd(env.galaxy_home): sudo("bash -c 'export PYTHON_EGG_CACHE=eggs; python ./scripts/build_universe_config.py conf.d; python -ES ./scripts/fetch_eggs.py; ./create_db.sh'", user="galaxy") def seed_database(galaxy_data): with cd(env.galaxy_home): sudo("rm -f seed.py") _setup_database_seed_file(galaxy_data) sudo("bash -c 'export PYTHON_EGG_CACHE=eggs; python ./scripts/build_universe_config.py conf.d; python -ES ./scripts/fetch_eggs.py; python seed.py'", user="galaxy") def seed_workflows(options): wait_for_galaxy() galaxy_data = options["galaxy"] with cd(os.path.join(env.galaxy_home, "workflows")): for user in galaxy_data["users"]: api_key = user["api_key"] workflows = None if "workflows" in user: workflows = user["workflows"] if not workflows: continue for workflow in workflows: sudo("bash -c 'export PYTHON_EGG_CACHE=eggs; bash import_all.sh %s %s'" % (api_key, workflow), user=env.galaxy_user) def _setup_database_seed_file(galaxy_data): _seed_append("""from scripts.db_shell import * from galaxy.util.bunch import Bunch from galaxy.security import GalaxyRBACAgent bunch = Bunch( **globals() ) bunch.engine = engine # model.flush() has been removed. bunch.session = db_session # For backward compatibility with "model.context.current" bunch.context = db_session security_agent = GalaxyRBACAgent( bunch ) security_agent.sa_session = sa_session def add_user(email, password, key=None): query = sa_session.query( User ).filter_by( email=email ) if query.count() > 0: return query.first() else: user = User(email) user.set_password_cleartext(password) sa_session.add(user) sa_session.flush() security_agent.create_private_user_role( user ) if not user.default_permissions: security_agent.user_set_default_permissions( user, history=True, dataset=True ) if key is not None: api_key = APIKeys() api_key.user_id = user.id api_key.key = key sa_session.add(api_key) sa_session.flush() return user def add_history(user, name): query = sa_session.query( History ).filter_by( user=user ).filter_by( name=name ) if query.count() == 0: history = History(user=user, name=name) sa_session.add(history) sa_session.flush() return history else: return query.first() """) i = 0 for user in galaxy_data["users"]: username = user["username"] password = user["password"] api_key = user["api_key"] histories = None if "histories" in user: histories = user["histories"] user_object = "user_%d" % i _seed_append("""%s = add_user("%s", "%s", "%s")""" % (user_object, username, password, api_key)) _import_histories(user_object, histories) i = i + 1 def _import_histories(user_object, histories): if not histories: return for history_name in histories: _import_history(user_object, history_name) def _import_history(user_object, history_name): history_name_stripped = history_name.strip() if history_name_stripped: _seed_append("""add_history(%s, "%s")""" % (user_object, history_name_stripped)) def _seed_append(text): append("%s/seed.py" % env.galaxy_home, text, use_sudo=True) def _start_galaxy(): # Create directory to store galaxy service's pid file. _make_dir_for_galaxy("/var/lib/galaxy") start_service("galaxy") def refresh_galaxy(target_galaxy_repo): _update_galaxy(target_galaxy_repo) sudo("/etc/init.d/galaxy restart", pty=False) def _setup_galaxy_log_dir(): _make_dir_for_galaxy("/var/log/galaxy") def _setup_shed_tools_dir(): _make_dir_for_galaxy("%s/../shed_tools" % env.galaxy_home) def _make_dir_for_galaxy(path): sudo("mkdir -p '%s'" % path) _chown_galaxy(env, path) def _update_galaxy(target_galaxy_repo): # Need to merge? -John hg_command = "hg pull %s; hg update" % target_galaxy_repo with cd(env.galaxy_home): sudo(hg_command, user=env.galaxy_user) def refresh_galaxy_action(vm_launcher, options): refresh_galaxy(env.galaxy_repository) def setup_image(options): _configure_package_holds(options) configure_MI(env) configure_smtp(options) configure_sudoers(options) def _configure_package_holds(options): # No longer respected. TODO: Implement. if 'package_holds' in options: env.package_holds = options['package_holds'] else: env.package_holds = None def configure_smtp(options): if 'smtp_server' in options: smtp_server = options['smtp_server'] username = options['smtp_user'] password = options['smtp_password'] conf_file_contents = """mailhub=%s UseSTARTTLS=YES AuthUser=%s AuthPass=%s FromLineOverride=YES """ % (smtp_server, username, password) _apt_packages(pkg_list=["ssmtp"]) sudo("""echo "%s" > /etc/ssmtp/ssmtp.conf""" % conf_file_contents) aliases = """root:%s:%s galaxy:%s:%s %s:%s:%s""" % (username, smtp_server, username, smtp_server, env.user, username, smtp_server) sudo("""echo "%s" > /etc/ssmtp/revaliases""" % aliases) def configure_sudoers(options): if "sudoers_additions" in options: for addition in options["sudoers_additions"]: sudoers_append(addition) def configure_MI(env): # Clean this next line up. _configure_and_install_native_packages(env, ["minimal", "cloudman", "galaxy"]) # _update_system() _setup_users(env) _setup_xvfb(env) _required_programs(env) # == required programs def _required_programs(env): """ Install required programs """ if not exists(env.install_dir): sudo("mkdir -p %s" % env.install_dir) sudo("chown %s %s" % (env.user, env.install_dir)) # Setup global environment for all users install_dir = os.path.split(env.install_dir)[0] exports = ["export PATH=%s/bin:%s/sbin:$PATH" % (install_dir, install_dir), "export LD_LIBRARY_PATH=%s/lib" % install_dir] for e in exports: _ensure_export(e) # Install required programs _install_nginx_standalone(env) _start_nginx(env) _deploy_setup_postgresql(env) # Verify this is not needed. # _install_samtools() def _ensure_export(command): if not contains('/etc/bash.bashrc', command): append('/etc/bash.bashrc', command, use_sudo=True) def _start_nginx(env): galaxy_data = env.galaxy_data_mount env.safe_sudo("mkdir -p '%s'" % env.galaxy_data) _chown_galaxy(env, galaxy_data) start_service("nginx") def _deploy_setup_postgresql(env): ensure_can_sudo_into("postgres") _setup_postgresql(env) configure_actions = {"setup_image": setup_image, "setup_genomes": setup_genomes, "purge_genomes": purge_genomes, "setup_galaxy": setup_galaxy, "purge_galaxy": purge_galaxy, } ready_actions = {"galaxy_transfer": galaxy_transfer, "refresh_galaxy": refresh_galaxy_action, "copy_runtime_properties": copy_runtime_properties, } compound_actions = {"configure": ["setup_image", "setup_tools", "setup_genomes", "setup_galaxy", "setup_ssh_key"], "reinstall_galaxy": ["purge_galaxy", "setup_galaxy"], "reinstall_genomes": ["purge_genomes", "setup_genomes"], "reinstall_tools": ["purge_tools", "setup_tools"] } ================================================ FILE: cloudbio/deploy/util.py ================================================ from string import Template from time import strftime import os from fabric.api import local, sudo, env, put, get from fabric.contrib.files import exists, append def setup_install_dir(): """Sets up install dir and ensures its owned by Galaxy""" if not exists(env.install_dir): sudo("mkdir -p %s" % env.install_dir) if not exists(env.jars_dir): sudo("mkdir -p %s" % env.jars_dir) # TODO: Fix bug here chown_galaxy(os.path.split(env.install_dir)[0]) def eval_template(env, template_str): props = { "env": env, "the_date": strftime('%Y%m%d'), "the_date_with_time": strftime('%Y%m%d_%H%M%S'), } return Template(template_str).safe_substitute(props) def ensure_can_sudo_into(user): sudoers_append("%admin ALL = (" + user + ") NOPASSWD: ALL") def sudoers_append(line): append("/etc/sudoers", line, use_sudo=True) def start_service(service_name): # For reasons I don't understand this doesn't work for galaxy init # script unless pty=False sudo("/etc/init.d/%s start" % service_name, pty=False) def wget(url, install_command=sudo, file_name=None): if not file_name: file_name = os.path.split(url)[-1] if '?' in file_name: file_name = file_name[0:file_name.index('?')] if ("cache_source_downloads" in env) and (not env.cache_source_downloads): install_command("wget %s -O %s" % (url, file_name)) else: cache_dir = env.source_cache_dir if not cache_dir: cache_dir = ".downloads" cached_file = os.path.join(cache_dir, file_name) if os.path.exists(cached_file): put(cached_file, file_name) else: install_command("wget %s -O %s" % (url, file_name)) local("mkdir -p '%s'" % cache_dir) get(file_name, cached_file) ================================================ FILE: cloudbio/deploy/vmlauncher/__init__.py ================================================ from __future__ import print_function import os import time from libcloud.compute.ssh import SSHClient from libcloud.compute.base import NodeImage, NodeSize from libcloud.compute.types import Provider from libcloud.compute.providers import get_driver import six # Ubuntu 10.04 LTS (Lucid Lynx) Daily Build [20120302] DEFAULT_AWS_IMAGE_ID = "ami-0bf6af4e" DEFAULT_AWS_SIZE_ID = "m1.large" DEFAULT_AWS_AVAILABILITY_ZONE = "us-west-1" from fabric.api import local, env, sudo, put, run class VmLauncher: def __init__(self, driver_options_key, options): self.driver_options_key = driver_options_key self.options = options self.__set_and_verify_key() def __set_and_verify_key(self): key_file = self.options.get('key_file', None) if not key_file: key_file = self._driver_options()['key_file'] # Expand tildes in path self.key_file = os.path.expanduser(key_file) if not os.path.exists(self.key_file): raise Exception("Invalid or unspecified key_file option: %s" % self.key_file) def _get_driver_options(self, driver_option_keys): driver_options = {} for key in driver_option_keys: if key in self._driver_options(): driver_options[key] = self._driver_options()[key] return driver_options def _driver_options(self): return self.options[self.driver_options_key] def get_key_file(self): return self.key_file def boot_and_connect(self): conn = self._connect_driver() node = self._boot() # Subclasses should implement this, and return libcloud node like object self.conn = conn self.node = node self.uuid = node.uuid self.connect(conn) def _connect_driver(self): if not getattr(self, 'conn', None): self.conn = self._get_connection() return self.conn def _wait_for_node_info(self, f): initial_value = f(self.node) if initial_value: return self._parse_node_info(initial_value) while True: time.sleep(10) refreshed_node = self._find_node() refreshed_value = f(refreshed_node) if refreshed_value and not refreshed_value == []: return self._parse_node_info(refreshed_value) def _parse_node_info(self, value): if isinstance(value, six.string_types): return value else: return value[0] def _find_node(self): nodes = self.conn.list_nodes() node_uuid = self.node.uuid for node in nodes: if node.uuid == node_uuid: return node def destroy(self, node=None): self._connect_driver() if node == None: node = self.node self.conn.destroy_node(node) def __get_ssh_client(self): ip = self.get_ip() # Subclasses should implement this key_file = self.get_key_file() ssh_client = SSHClient(hostname=ip, port=self.get_ssh_port(), username=self.get_user(), key=key_file) return ssh_client def get_user(self): return "ubuntu" def get_ssh_port(self): return 22 def connect(self, conn, tries=5): i = 0 while i < tries: try: ssh_client = self.__get_ssh_client() conn._ssh_client_connect(ssh_client=ssh_client, timeout=60) return except: i = i + 1 def list(self): self._connect_driver() return self.conn.list_nodes() def _boot(self): conn = self.conn boot_new = True last_instance_path = None if 'use_existing_instance' in self._driver_options(): boot_new = False instance_id = self._driver_options()['use_existing_instance'] if instance_id == "__auto__": last_instance_path = ".vmlauncher_last_instance_%s" % self.driver_options_key if not os.path.exists(last_instance_path): boot_new = True else: instance_id = open(last_instance_path, "r").read() if not boot_new: nodes = conn.list_nodes() nodes_with_id = [node for node in nodes if node.uuid == instance_id] if not nodes_with_id: err_msg_template = "Specified use_existing_instance with instance id %s, but no such instance found." raise Exception(err_msg_template % instance_id) node = nodes_with_id[0] if boot_new: node = self._boot_new(conn) if last_instance_path: open(last_instance_path, "w").write(node.uuid) return node def _image_from_id(self, image_id=None): image = NodeImage(id=image_id, name="", driver="") return image def _get_image_id(self, image_id=None): if not image_id: if 'image_id' in self._driver_options(): image_id = self._driver_options()['image_id'] else: image_id = self._default_image_id() return image_id def _default_image_id(self): return None def _get_default_size_id(self): return None def _get_size_id_option(self): return "size_id" def _size_from_id(self, size_id): size = NodeSize(id=size_id, name="", ram=None, disk=None, bandwidth=None, price=None, driver="") return size def _get_size_id(self, size_id=None): if not size_id: size_id_option = self._get_size_id_option() if size_id_option in self._driver_options(): size_id = self._driver_options()[size_id_option] else: size_id = self._get_default_size_id() return size_id def _boot_new(self, conn): hostname = self.options.get("hostname", "vm_launcher_instance") node = self.create_node(hostname) return node def access_id(self): return self._driver_options()["access_id"] def secret_key(self): return self._driver_options()["secret_key"] def package_image_name(self): name = self._driver_options()["package_image_name"] or "cloudbiolinux" return name def package_image_description(self, default=""): description = self._driver_options().get("package_image_description", default) return description class VagrantConnection: """'Fake' connection type to mimic libcloud's but for Vagrant""" def _ssh_client_connect(self, ssh_client): pass def destroy_node(self, node=None): local("vagrant halt") def list_nodes(self): return [VagrantNode()] class VagrantNode: def __init__(self): self.name = "vagrant" self.uuid = "vagrant" class VagrantVmLauncher(VmLauncher): """Launches vagrant VMs.""" def _get_connection(): return VagrantConnection() def __init__(self, driver_options_key, options): if not 'key_file' in options: options['key_file'] = os.path.join(os.environ["HOME"], ".vagrant.d", "insecure_private_key") VmLauncher.__init__(self, driver_options_key, options) self.uuid = "test" def _boot(self): local("vagrant up") return VagrantNode() def get_ip(self): return "33.33.33.11" def get_user(self): return "vagrant" def package(self, **kwds): local("vagrant package") class OpenstackVmLauncher(VmLauncher): """ Wrapper around libcloud's openstack API. """ def get_ip(self): return self._wait_for_node_info(lambda node: node.public_ips + node.private_ips) def _get_size_id_option(self): return "flavor_id" def create_node(self, hostname, image_id=None, size_id=None, **kwds): image_id = self._get_image_id() image = self._image_from_id(image_id) size_id = self._get_size_id() size = self._size_from_id(size_id) if 'ex_keyname' not in kwds: kwds['ex_keyname'] = self._driver_options()['ex_keyname'] node = self.conn.create_node(name=hostname, image=image, size=size, **kwds) return node def _get_connection(self): driver = get_driver(Provider.OPENSTACK) openstack_username = self._driver_options()['username'] openstack_api_key = self._driver_options()['password'] driver_option_keys = ['host', 'secure', 'port', 'ex_force_auth_url', 'ex_force_auth_version', 'ex_force_base_url', 'ex_tenant_name'] driver_options = self._get_driver_options(driver_option_keys) conn = driver(openstack_username, openstack_api_key, **driver_options) return conn def package(self, **kwds): name = kwds.get("name", self.package_image_name()) self.conn.ex_save_image(self.node, name) def attach_public_ip(self, public_ip=None): if not public_ip: public_ip = self._driver_options()["public_ip"] self.conn._node_action(self.node, "addFloatingIp", address=public_ip) class EucalyptusVmLauncher(VmLauncher): def get_ip(self): return self._wait_for_node_info(lambda node: node.public_ips) def _get_connection(self): driver = get_driver(Provider.EUCALYPTUS) driver_option_keys = ['secret', 'secure', 'port', 'host', 'path'] driver_options = self._get_driver_options(driver_option_keys) ec2_access_id = self.access_id() conn = driver(ec2_access_id, **driver_options) return conn def create_node(self, hostname, image_id=None, size_id=None, **kwds): image_id = self._get_image_id() image = self._image_from_id(image_id) size_id = self._get_size_id() size = self._size_from_id(size_id) if 'ex_keyname' not in kwds: kwds['ex_keyname'] = self._driver_options()["keypair_name"] node = self.conn.create_node(name=hostname, image=image, size=size, **kwds) return node class Ec2VmLauncher(VmLauncher): def get_ip(self): return self._wait_for_node_info(lambda node: node.extra['dns_name']) def boto_connection(self): """ Establish a boto library connection (for functionality not available in libcloud). """ import boto.ec2 region = boto.ec2.get_region(self._availability_zone()) ec2_access_id = self.access_id() ec2_secret_key = self.secret_key() return region.connect(aws_access_key_id=ec2_access_id, aws_secret_access_key=ec2_secret_key) def boto_s3_connection(self): from boto.s3.connection import S3Connection ec2_access_id = self.access_id() ec2_secret_key = self.secret_key() return S3Connection(ec2_access_id, ec2_secret_key) def _default_image_id(self): return DEFAULT_AWS_IMAGE_ID def package(self, **kwds): package_type = self._driver_options().get('package_type', 'default') if package_type == "create_image": self._create_image(**kwds) else: self._default_package(**kwds) def _create_image(self, **kwds): ec2_conn = self.boto_connection() instance_id = run("curl --silent http://169.254.169.254/latest/meta-data/instance-id") if "name" not in kwds: name = self.package_image_name() else: name = kwds["name"] if "description" not in kwds: description = self.package_image_description(default="") else: description = kwds["description"] image_id = ec2_conn.create_image(instance_id, name=name, description=description) if self._driver_options().get("make_public", False): ec2_conn.modify_image_attribute(image_id, attribute='launchPermission', operation='add', groups=['all']) def _default_package(self, **kwds): env.packaging_dir = "/mnt/packaging" sudo("mkdir -p %s" % env.packaging_dir) self._copy_keys() self._install_ec2_tools() self._install_packaging_scripts() def _install_ec2_tools(self): sudo("apt-add-repository ppa:awstools-dev/awstools") sudo("apt-get update") sudo('export DEBIAN_FRONTEND=noninteractive; sudo -E apt-get install ec2-api-tools ec2-ami-tools -y --force-yes') def _install_packaging_scripts(self): user_id = self._driver_options()["user_id"] bundle_cmd = "sudo ec2-bundle-vol -k %s/ec2_key -c%s/ec2_cert -u %s" % \ (env.packaging_dir, env.packaging_dir, user_id) self._write_script("%s/bundle_image.sh" % env.packaging_dir, bundle_cmd) bucket = self._driver_options()["package_bucket"] upload_cmd = "sudo ec2-upload-bundle -b %s -m /tmp/image.manifest.xml -a %s -s %s" % \ (bucket, self.access_id(), self.secret_key()) self._write_script("%s/upload_bundle.sh" % env.packaging_dir, upload_cmd) name = self.package_image_name() manifest = "image.manifest.xml" register_cmd = "sudo ec2-register -K %s/ec2_key -C %s/ec2_cert %s/%s -n %s" % (env.packaging_dir, env.packaging_dir, bucket, manifest, name) self._write_script("%s/register_bundle.sh" % env.packaging_dir, register_cmd) def _write_script(self, path, contents): full_contents = "#!/bin/bash\n%s" % contents sudo("echo '%s' > %s" % (full_contents, path)) sudo("chmod +x %s" % path) def _copy_keys(self): ec2_key_path = self._driver_options()["x509_key"] ec2_cert_path = self._driver_options()["x509_cert"] put(ec2_key_path, "%s/ec2_key" % env.packaging_dir, use_sudo=True) put(ec2_cert_path, "%s/ec2_cert" % env.packaging_dir, use_sudo=True) def _availability_zone(self): if "availability_zone" in self._driver_options(): availability_zone = self._driver_options()["availability_zone"] else: availability_zone = DEFAULT_AWS_AVAILABILITY_ZONE return availability_zone def _get_default_size_id(self): return DEFAULT_AWS_SIZE_ID def _get_location(self): availability_zone = self._availability_zone() locations = self.conn.list_locations() for location in locations: if location.availability_zone.name == availability_zone: break return location def create_node(self, hostname, image_id=None, size_id=None, location=None, **kwds): self._connect_driver() image_id = self._get_image_id(image_id) image = self._image_from_id(image_id) size_id = self._get_size_id(size_id) size = self._size_from_id(size_id) if not location: location = self._get_location() if not "ex_keyname" in kwds: keyname = self._driver_options()["keypair_name"] kwds["ex_keyname"] = keyname node = self.conn.create_node(name=hostname, image=image, size=size, location=location, **kwds) return node def attach_public_ip(self, public_ip=None): if not public_ip: public_ip = self._driver_options()["public_ip"] self.conn.ex_associate_addresses(self.node, public_ip) def _get_connection(self): driver = get_driver(Provider.EC2) ec2_access_id = self.access_id() ec2_secret_key = self.secret_key() conn = driver(ec2_access_id, ec2_secret_key) return conn def build_vm_launcher(options): provider_option_key = 'vm_provider' # HACK to maintain backward compatibity on vm_host option if not 'vm_provider' in options and 'vm_host' in options: print("Using deprecated 'vm_host' setting, please change this to 'vm_provider'") provider_option_key = 'vm_host' driver = options.get(provider_option_key, 'aws') # Will just fall back on EC2 driver_options_key = driver if driver in options: # Allow multiple sections or providers per driver (e.g. aws-project-1). # Assume the driver is just the provider name unless the provider # section sets an explict driver option. In above example, # the aws-project-1 would have to have a "driver: 'aws'" option # set. provider_options = options.get(driver) driver = provider_options.get('driver', driver) driver_classes = {'openstack': OpenstackVmLauncher, 'vagrant': VagrantVmLauncher, 'eucalyptus': EucalyptusVmLauncher} driver_class = driver_classes.get(driver, Ec2VmLauncher) vm_launcher = driver_class(driver_options_key, options) return vm_launcher ================================================ FILE: cloudbio/deploy/vmlauncher/config.md ================================================ # Configuring Cloud Parameters Currently four different virtual machine providers are implemented: `aws` (default), `openstack`, `eucalyptus` (partial support), and `vagrant`. Request for supporting additional cloud infrastructures can be created here https://github.com/jmchilton/vm-launcher/issues/new or pull requests are always welcome. The `vm-launcher` project is built heavily on Apache [libcloud], so support should be implemented at that level first, though dozens of cloud providers are currently implemented. ## aws This cloud supports the following compute parameters: `access_id`, `secret_key`, `size_id`, `image_id`, `availability_zone`. The aws driver supports two packaging modes, this is the code that is called when the `package` action is executed. By default, package will cause some scripts to be created on the remote server to aid in packaging, if however `package_type` parameter is set to `create_image`, the Amazon EC2 CreateImage (http://support.smartbear.com/viewarticle/22739/) operation will be used to automatically package the target instance. `create_image` mode can only be used for EBS backed instances, which is why the other more complex mode is the default. When `create_image` is enabled, the additional packaging parameters include `package_image_name`, `package_image_description`, and `make_public`. For the default packaging mode, many additional parameters related to S3 must be set including `x509_cert`, `x509_key`, `user_id`, and `package_bucket`. ## openstack OpenStack may be targetted using either the native OpenStack APIs or using the EC2 compatibility layer (e.g. what boto does). To target the EC2 compatibility use the `eucalyptus` driver, this `openstack` driver targets the native API. This driver allows the following parameters `username`, `password`, `host`, `secure` (boolean), `port`, `ex_force_uth_url`, `ex_force_base_url`, `ex_force_auth_version`, `ex_tenant_name`, `flavor_id`, `image_id`, `keypair_name`, and `package_image_name`. ## eucalyptus Support for the eucalyptus driver is somewhat experimental at this time and automated packaging is not available. This driver can be configured via the following options:: `secret`, `secure`, `port`, `host`, `path`, `size_id`. ## vagrant The vagrant driver supports no additional parameters, a precise64 box should be configured though this can be tweaks by adjusting the file `Vagrantfile`. [libcloud]: http://libcloud.apache.org/ ================================================ FILE: cloudbio/deploy/vmlauncher/transfer.py ================================================ from __future__ import print_function import os import gzip from operator import itemgetter from sys import exit from threading import Thread from threading import Condition from Queue import Queue from fabric.api import local, put, sudo, cd from fabric.colors import red class FileSplitter: """ Works like the UNIX split command break up a file into parts like: filename_aaaaaaaaa filename_aaaaaaaab etc... """ def __init__(self, chunk_size, destination_directory, callback): self.chunk_size = chunk_size * 1024 * 1024 self.destination_directory = destination_directory self.chunk_callback = callback def split_file(self, path, compress, transfer_target): basename = os.path.basename(path) file_size = os.path.getsize(path) total_bytes = 0 chunk_num = 0 suffix = '' if compress: suffix = '.gz' input = open(path, 'rb') while True: chunk_name = "%s_part%08d%s" % (basename, chunk_num, suffix) chunk_path = os.path.join(self.destination_directory, chunk_name) this_chunk_size = min(self.chunk_size, file_size - total_bytes) if this_chunk_size <= 0: break chunk = input.read(this_chunk_size) total_bytes += len(chunk) if compress: chunk_output = gzip.open(chunk_path, 'wb') else: chunk_output = file(chunk_path, 'wb') chunk_output.write(chunk) chunk_output.close() self.chunk_callback.handle_chunk(chunk_path, transfer_target) chunk_num += 1 class TransferTarget: def __init__(self, file, precompressed, transfer_manager): self.file = file self.precompressed = precompressed self.do_compress = transfer_manager.compress self.do_split = transfer_manager.chunk_size > 0 self.local_temp = transfer_manager.local_temp basename = os.path.basename(file) if len(basename) < 1: print(red(Exception("Invalid file specified - %s" % file))) exit(-1) self.basename = basename def should_compress(self): return not self.precompressed and self.do_compress def split_up(self): return self.do_split def clean(self): if self.should_compress(): local("rm -rf '%s'" % self.compressed_file()) def compressed_basename(self): if not self.precompressed: compressed_basename = "%s.gz" % self.basename else: compressed_basename = self.basename return compressed_basename def decompressed_basename(self): basename = self.basename if basename.endswith(".gz"): decompressed_basename = basename[:-len(".gz")] else: decompressed_basename = basename return decompressed_basename def compressed_file(self): compressed_file = "%s/%s.gz" % (self.local_temp, self.basename) return compressed_file def build_simple_chunk(self): if self.should_compress(): compressed_file = self.compressed_file() local("gzip -f -9 '%s' -c > '%s'" % (self.file, compressed_file)) return TransferChunk(compressed_file, self) else: return TransferChunk(self.file, self) class TransferChunk: def __init__(self, chunk_path, transfer_target): self.chunk_path = chunk_path self.transfer_target = transfer_target def clean_up(self): was_split = self.transfer_target.split_up() was_compressed = self.transfer_target.should_compress() if was_split or was_compressed: local("rm '%s'" % self.chunk_path) class FileTransferManager: def __init__(self, compress=True, num_compress_threads=1, num_transfer_threads=1, num_decompress_threads=1, chunk_size=0, transfer_retries=3, destination="/tmp", transfer_as="root", local_temp=None): self.compress = compress self.num_compress_threads = num_compress_threads self.num_transfer_threads = num_transfer_threads self.num_decompress_threads = num_decompress_threads self.chunk_size = chunk_size self.transfer_retries = transfer_retries self.destination = destination self.transfer_as = transfer_as self.local_temp = local_temp if not self.local_temp: self.local_temp = "/tmp" local("mkdir -p '%s'" % self.local_temp) self.file_splitter = FileSplitter(self.chunk_size, self.local_temp, self) def handle_chunk(self, chunk, transfer_target): self._enqueue_chunk(TransferChunk(chunk, transfer_target)) def transfer_files(self, files=[], compressed_files=[]): self.transfer_complete = False self.transfer_complete_condition = Condition() self._setup_destination_directory() self._setup_workers() self._enqueue_files(files, compressed_files) self._wait_for_completion() def _setup_workers(self): self._setup_compress_threads() self._setup_transfer_threads() self._setup_decompress_threads() def _setup_destination_directory(self): sudo("mkdir -p %s" % self.destination) self._chown(self.destination) def _setup_compress_threads(self): self.compress_queue = Queue() self._launch_threads(self.num_compress_threads, self._compress_files) def _setup_decompress_threads(self): self.decompress_queue = Queue() self._launch_threads(self.num_decompress_threads, self._decompress_files) def _setup_transfer_threads(self): self.transfer_queue = Queue() # For now just transfer one file at a time self._launch_threads(self.num_transfer_threads, self._put_files) def _launch_threads(self, num_threads, func): for thread_index in range(num_threads): t = Thread(target=func) t.daemon = True t.start() def _enqueue_files(self, files, compressed_files): transfer_targets = [] for file in files: transfer_target = TransferTarget(file, False, self) transfer_targets.append(transfer_target) for compressed_file in compressed_files: transfer_target = TransferTarget(compressed_file, True, self) transfer_targets.append(transfer_target) transfer_targets = self._sort_transfer_targets(transfer_targets) for transfer_target in transfer_targets: self.compress_queue.put(transfer_target) def _sort_transfer_targets(self, transfer_targets): for i in range(len(transfer_targets)): transfer_target = transfer_targets[i] transfer_targets[i] = transfer_target, os.stat(transfer_target.file).st_size transfer_targets.sort(key=itemgetter(1), reverse=True) return [transfer_target[0] for transfer_target in transfer_targets] def _wait_for_completion(self): self.compress_queue.join() self.transfer_queue.join() self.transfer_complete_condition.acquire() self.transfer_complete = True self.transfer_complete_condition.notifyAll() self.transfer_complete_condition.release() self.decompress_queue.join() def _compress_files(self): while True: try: transfer_target = self.compress_queue.get() file = transfer_target.file if self.chunk_size > 0: should_compress = transfer_target.should_compress() self.file_splitter.split_file(file, should_compress, transfer_target) self.decompress_queue.put(transfer_target) else: simple_chunk = transfer_target.build_simple_chunk() self._enqueue_chunk(simple_chunk) except Exception as e: print(red("Failed to compress a file to transfer")) print(red(e)) finally: self.compress_queue.task_done() def _decompress_files(self): if self.chunk_size > 0: self.transfer_complete_condition.acquire() while not self.transfer_complete: self.transfer_complete_condition.wait() self.transfer_complete_condition.release() while True: try: transfer_target = self.decompress_queue.get() basename = transfer_target.basename chunked = transfer_target.split_up() compressed = transfer_target.do_compress or transfer_target.precompressed with cd(self.destination): if compressed and chunked: destination = transfer_target.decompressed_basename() if transfer_target.precompressed: sudo("cat '%s_part'* | gunzip -c > %s" % (basename, destination), user=self.transfer_as) else: sudo("zcat '%s_part'* > %s" % (basename, destination), user=self.transfer_as) sudo("rm '%s_part'*" % (basename), user=self.transfer_as) elif compressed: sudo("gunzip -f '%s'" % transfer_target.compressed_basename(), user=self.transfer_as) elif chunked: sudo("cat '%s'_part* > '%s'" % (basename, basename), user=self.transfer_as) sudo("rm '%s_part'*" % (basename), user=self.transfer_as) except Exception as e: print(red("Failed to decompress or unsplit a transfered file.")) print(red(e)) finally: self.decompress_queue.task_done() def _put_files(self): while True: try: transfer_chunk = self.transfer_queue.get() transfer_target = transfer_chunk.transfer_target compressed_file = transfer_chunk.chunk_path basename = os.path.basename(compressed_file) self._put_as_user(compressed_file, "%s/%s" % (self.destination, basename)) if not transfer_target.split_up(): self.decompress_queue.put(transfer_target) except Exception as e: print(red("Failed to upload a file.")) print(red(e)) finally: transfer_chunk.clean_up() self.transfer_queue.task_done() def _chown(self, destination): sudo("chown %s:%s '%s'" % (self.transfer_as, self.transfer_as, destination)) def _put_as_user(self, source, destination): for attempt in range(self.transfer_retries): retry = False try: put(source, destination, use_sudo=True) self._chown(destination) except BaseException as e: retry = True print(red(e)) print(red("Failed to upload %s on attempt %d" % (source, attempt + 1))) except: # Should never get here, delete this block when more confident retry = True print(red("Failed to upload %s on attempt %d" % (source, attempt + 1))) finally: if not retry: return print(red("Failed to transfer file %s, exiting..." % source)) exit(-1) def _enqueue_chunk(self, transfer_chunk): self.transfer_queue.put(transfer_chunk) ================================================ FILE: cloudbio/deploy/volume.py ================================================ from __future__ import print_function from fabric.api import run, env from time import sleep from boto.exception import EC2ResponseError from .util import eval_template def attach_volumes(vm_launcher, options, format=False): """ """ volumes = options.get("volumes", []) if not volumes: return boto_connection = vm_launcher.boto_connection() instance_id = run("curl --silent http://169.254.169.254/latest/meta-data/instance-id") for volume in volumes: volume_id = volume['id'] device_id = volume['device'] if not _get_attached(boto_connection, instance_id, device_id, valid_states=["attached", "attaching"]): boto_connection.attach_volume(volume_id, instance_id, device_id) for volume in volumes: volume_id = volume['id'] device_id = volume['device'] path = volume.get("path") while True: if _get_attached(boto_connection, instance_id, device_id): break sleep(5) print("Waiting for volume corresponding to device %s to attach" % device_id) break # Don't mount if already mounted if _find_mounted_device_id(path): continue format = str(volume.get('format', "False")).lower() if format == "true": _format_device(device_id) env.safe_sudo("mkdir -p '%s'" % path) try: _mount(device_id, path) except: if format == "__auto__": print("Failed to mount device. format is set to __auto__ so will now format device and retry mount") _format_device(device_id) _mount(device_id, path) else: raise def _mount(device_id, path): env.safe_sudo("mount '%s' '%s'" % (device_id, path)) def _format_device(device_id): env.safe_sudo("mkfs -t ext3 %s" % device_id) def detach_volumes(vm_launcher, options): volumes = options.get("volumes", []) if not volumes: return boto_connection = vm_launcher.boto_connection() instance_id = run("curl --silent http://169.254.169.254/latest/meta-data/instance-id") for volume in volumes: volume_id = volume['id'] path = volume.get("path") env.safe_sudo("umount '%s'" % path) _detach(boto_connection, instance_id, volume_id) def make_snapshots(vm_launcher, options): volumes = options.get("volumes", []) for volume in volumes: path = volume.get("path") desc = volume.get("description", "Snapshot of path %s" % path) desc = eval_template(env, desc) # Allow volume to specify it should not be snapshotted, e.g. if # piggy backing on core teams snapshots for galaxyIndicies for instance. snapshot = volume.get("snapshot", True) if snapshot: _make_snapshot(vm_launcher, path, desc) def _get_attached(conn, instance_id, device_id, valid_states=['attached']): vol_list = conn.get_all_volumes() fs_vol = None for vol in vol_list: if vol.attach_data.instance_id == instance_id and vol.attach_data.device == device_id: if vol.attach_data.status in valid_states: fs_vol = vol break return fs_vol def _make_snapshot(vm_launcher, fs_path, desc): """ Create a snapshot of an existing volume that is currently attached to an instance, taking care of the unmounting and detaching. If you specify the optional argument (:galaxy), the script will pull the latest Galaxy code from bitbucket and perform an update before snapshotting. Else, the script will prompt for the file system path to be snapshoted. In order for this to work, an instance on EC2 needs to be running with a volume that wants to be snapshoted attached and mounted. The script will unmount the volume, create a snaphost and offer to reattach and mount the volume or create a new one from the freshly created snapshot. Except for potentially Galaxy, MAKE SURE there are no running processes using the volume and that no one is logged into the instance and sitting in the given directory. """ instance_id = run("curl --silent http://169.254.169.254/latest/meta-data/instance-id") availability_zone = run("curl --silent http://169.254.169.254/latest/meta-data/placement/availability-zone") instance_region = availability_zone[:-1] # Truncate zone letter to get region name # Find the device where the file system is mounted to # Find the EBS volume where the file system resides device_id = _find_mounted_device_id(fs_path) ec2_conn = vm_launcher.boto_connection() fs_vol = _get_attached(ec2_conn, instance_id, device_id) if fs_vol: env.safe_sudo("umount %s" % fs_path) _detach(ec2_conn, instance_id, fs_vol.id) snap_id = _create_snapshot(ec2_conn, fs_vol.id, desc) # TODO: Auto Update snaps? make_public = True if make_public: # Make option ec2_conn.modify_snapshot_attribute(snap_id, attribute='createVolumePermission', operation='add', groups=['all']) reattach = True if reattach: _attach(ec2_conn, instance_id, fs_vol.id, device_id) env.safe_sudo("mount %s %s" % (device_id, fs_path)) delete_old_volume = False if delete_old_volume: _delete_volume(ec2_conn, fs_vol.id) print("----- Done snapshoting volume '%s' for file system '%s' -----" % (fs_vol.id, fs_path)) else: print("ERROR: Failed to find require file system, is boto installed? Is it not actually mounted?") def _find_mounted_device_id(path): # Adding dollar sign to grep to distinguish between /mnt/galaxy and /mnt/galaxyIndices device_id = env.safe_sudo("df | grep '%s$' | awk '{print $1}'" % path) return device_id def _attach(ec2_conn, instance_id, volume_id, device): """ Attach EBS volume to the given device (using boto). Try it for some time. """ try: print("Attaching volume '%s' to instance '%s' as device '%s'" % (volume_id, instance_id, device)) volumestatus = ec2_conn.attach_volume(volume_id, instance_id, device) except EC2ResponseError as e: print("Attaching volume '%s' to instance '%s' as device '%s' failed. Exception: %s" % (volume_id, instance_id, device, e)) return False for counter in range(30): print("Attach attempt %s, volume status: %s" % (counter, volumestatus)) if volumestatus == 'attached': print("Volume '%s' attached to instance '%s' as device '%s'" % (volume_id, instance_id, device)) break if counter == 29: print("Volume '%s' FAILED to attach to instance '%s' as device '%s'. Aborting." % (volume_id, instance_id, device)) return False volumes = ec2_conn.get_all_volumes([volume_id]) volumestatus = volumes[0].attachment_state() sleep(3) return True def _detach(ec2_conn, instance_id, volume_id): """ Detach EBS volume from the given instance (using boto). Try it for some time. """ try: volumestatus = ec2_conn.detach_volume( volume_id, instance_id, force=True ) except EC2ResponseError as e: print("Detaching volume '%s' from instance '%s' failed. Exception: %s" % ( volume_id, instance_id, str(e) )) return False for counter in range( 30 ): print("Volume '%s' status '%s'" % ( volume_id, volumestatus )) if volumestatus == 'available': print("Volume '%s' successfully detached from instance '%s'." % ( volume_id, instance_id )) break if counter == 29: print("Volume '%s' FAILED to detach to instance '%s'." % ( volume_id, instance_id )) sleep(3) volumes = ec2_conn.get_all_volumes( [volume_id] ) volumestatus = volumes[0].status def _delete_volume(ec2_conn, vol_id): try: ec2_conn.delete_volume(vol_id) print("Deleted volume '%s'" % vol_id) except EC2ResponseError as e: print("ERROR deleting volume '%s': %s" % (vol_id, e)) def _create_snapshot(ec2_conn, volume_id, description=None): """ Create a snapshot of the EBS volume with the provided volume_id. Wait until the snapshot process is complete (note that this may take quite a while) """ snapshot = ec2_conn.create_snapshot(volume_id, description=description) if snapshot: while snapshot.status != 'completed': sleep(6) snapshot.update() print("Creation of snapshot for volume '%s' completed: '%s'" % (volume_id, snapshot)) return snapshot.id else: print("Could not create snapshot from volume with ID '%s'" % volume_id) return False ================================================ FILE: cloudbio/distribution.py ================================================ """Configuration details for specific server types. This module contains functions that help with initializing a Fabric environment for standard server types. """ import os import subprocess from fabric.api import env from cloudbio.fabutils import quiet from cloudbio.fabutils import configure_runsudo from cloudbio.custom import system def _setup_distribution_environment(ignore_distcheck=False): """Setup distribution environment. In low-level terms, this method attempts to populate various values in the fabric env data structure for use other places in CloudBioLinux. """ if "distribution" not in env: env.distribution = "__auto__" if "dist_name" not in env: env.dist_name = "__auto__" env.logger.info("Distribution %s" % env.distribution) if env.hosts == ["vagrant"]: _setup_vagrant_environment() elif env.hosts == ["localhost"]: _setup_local_environment() configure_runsudo(env) if env.distribution == "__auto__": env.distribution = _determine_distribution(env) if env.distribution == "ubuntu": ## TODO: Determine if dist_name check works with debian. if env.dist_name == "__auto__": env.dist_name = _ubuntu_dist_name(env) _setup_ubuntu() elif env.distribution == "centos": _setup_centos() elif env.distribution == "scientificlinux": _setup_scientificlinux() elif env.distribution == "debian": if env.dist_name == "__auto__": env.dist_name = _debian_dist_name(env) _setup_debian() elif env.distribution == "arch": pass # No package support for Arch yet elif env.distribution == "suse": pass # No package support for SUSE yet elif env.distribution == "macosx": _setup_macosx(env) ignore_distcheck = True else: raise ValueError("Unexpected distribution %s" % env.distribution) if not ignore_distcheck: _validate_target_distribution(env.distribution, env.get('dist_name', None)) _cloudman_compatibility(env) _setup_nixpkgs() _setup_fullpaths(env) # allow us to check for packages only available on 64bit machines machine = env.safe_run_output("uname -m") env.is_64bit = machine.find("_64") > 0 def _setup_fullpaths(env): home_dir = env.safe_run_output("echo $HOME") for attr in ["data_files", "galaxy_home", "local_install"]: if hasattr(env, attr): x = getattr(env, attr) if x.startswith("~"): x = x.replace("~", home_dir) setattr(env, attr, x) def _cloudman_compatibility(env): """Environmental variable naming for compatibility with CloudMan. """ env.install_dir = env.system_install def _validate_target_distribution(dist, dist_name=None): """Check target matches environment setting (for sanity) Throws exception on error """ env.logger.debug("Checking target distribution " + env.distribution) if dist in ["debian", "ubuntu"]: tag = env.safe_run_output("cat /proc/version") if tag.lower().find(dist) == -1: # hmmm, test issue file tag2 = env.safe_run_output("cat /etc/issue") if tag2.lower().find(dist) == -1: raise ValueError("Distribution does not match machine; are you using correct fabconfig for " + dist) if not dist_name: raise ValueError("Must specify a dist_name property when working with distribution %s" % dist) # Does this new method work with CentOS, do we need this. if dist == 'debian': actual_dist_name = _debian_dist_name(env) else: actual_dist_name = _ubuntu_dist_name(env) if actual_dist_name != dist_name: raise ValueError("Distribution does not match machine; are you using correct fabconfig <"+actual_dist_name+"> for " + dist) else: env.logger.debug("Unknown target distro") def _setup_ubuntu(): env.logger.info("Ubuntu setup") shared_sources = _setup_deb_general() # package information. This is ubuntu/debian based and could be generalized. sources = [ "deb http://us.archive.ubuntu.com/ubuntu/ %s universe", # unsupported repos "deb http://us.archive.ubuntu.com/ubuntu/ %s multiverse", "deb http://us.archive.ubuntu.com/ubuntu/ %s-updates universe", "deb http://us.archive.ubuntu.com/ubuntu/ %s-updates multiverse", "deb http://archive.canonical.com/ubuntu %s partner", # partner repositories "deb http://cran.fhcrc.org/bin/linux/ubuntu %s/", # lastest R versions "deb http://archive.canonical.com/ubuntu %s partner", # sun-java "deb http://ppa.launchpad.net/nebc/bio-linux/ubuntu trusty main", # Bio-Linux "deb [arch=amd64 trusted=yes] http://research.cs.wisc.edu/htcondor/debian/stable/ squeeze contrib", # HTCondor ] + shared_sources env.std_sources = _add_source_versions(env.dist_name, sources) def _setup_debian(): env.logger.info("Debian setup") unstable_remap = {"sid": "squeeze"} shared_sources = _setup_deb_general() sources = [ "deb http://cran.fhcrc.org/bin/linux/debian %s-cran/", # lastest R versions "deb http://nebc.nerc.ac.uk/bio-linux/ unstable bio-linux", # Bio-Linux ] + shared_sources # fill in %s dist_name = unstable_remap.get(env.dist_name, env.dist_name) env.std_sources = _add_source_versions(dist_name, sources) def _setup_deb_general(): """Shared settings for different debian based/derived distributions. """ env.logger.debug("Debian-shared setup") env.sources_file = "/etc/apt/sources.list.d/cloudbiolinux.list" env.global_sources_file = "/etc/apt/sources.list" env.apt_preferences_file = "/etc/apt/preferences" if not hasattr(env, "python_version_ext"): env.python_version_ext = "" if not hasattr(env, "ruby_version_ext"): env.ruby_version_ext = "1.9.1" if not env.has_key("java_home"): # Try to determine java location from update-alternatives java_home = "/usr/lib/jvm/java-7-openjdk-amd64" with quiet(): java_info = env.safe_run_output("update-alternatives --display java") for line in java_info.split("\n"): if line.strip().startswith("link currently points to"): java_home = line.split()[-1].strip() java_home = java_home.replace("/jre/bin/java", "") env.java_home = java_home shared_sources = [ "deb http://download.virtualbox.org/virtualbox/debian %s contrib", # virtualbox ] return shared_sources def _setup_centos(): env.logger.info("CentOS setup") if not hasattr(env, "python_version_ext"): # use installed anaconda version instead of package 2.6 #env.python_version_ext = "2.6" env.python_version_ext = "" #env.pip_cmd = "pip-python" if not hasattr(env, "ruby_version_ext"): env.ruby_version_ext = "" if not env.has_key("java_home"): env.java_home = "/etc/alternatives/java_sdk" def _setup_scientificlinux(): env.logger.info("ScientificLinux setup") if not hasattr(env, "python_version_ext"): env.python_version_ext = "" env.pip_cmd = "pip-python" if not env.has_key("java_home"): env.java_home = "/etc/alternatives/java_sdk" def _setup_macosx(env): # XXX Only framework in place; needs testing env.logger.info("MacOSX setup") # XXX Ensure XCode is installed and provide useful directions if not system.install_homebrew(env) # XXX find java correctly env.java_home = "" def _setup_nixpkgs(): # for now, Nix packages are only supported in Debian - it can # easily be done for others - just get Nix installed from the .rpm nixpkgs = False if env.has_key("nixpkgs"): if env.distribution in ["debian", "ubuntu"]: if env.nixpkgs == "True": nixpkgs = True else: nixpkgs = False else: env.logger.warn("NixPkgs are currently not supported for " + env.distribution) if nixpkgs: env.logger.info("NixPkgs: supported") else: env.logger.debug("NixPkgs: Ignored") env.nixpkgs = nixpkgs def _setup_local_environment(): """Setup a localhost environment based on system variables. """ env.logger.info("Get local environment") if not env.has_key("user"): env.user = os.environ["USER"] def _setup_vagrant_environment(): """Use vagrant commands to get connection information. https://gist.github.com/1d4f7c3e98efdf860b7e """ env.logger.info("Get vagrant environment") raw_ssh_config = subprocess.Popen(["vagrant", "ssh-config"], stdout=subprocess.PIPE).communicate()[0] env.logger.info(raw_ssh_config) ssh_config = dict([l.strip().split() for l in raw_ssh_config.split("\n") if l]) env.user = ssh_config["User"] env.hosts = [ssh_config["HostName"]] env.port = ssh_config["Port"] env.host_string = "%s@%s:%s" % (env.user, env.hosts[0], env.port) env.key_filename = ssh_config["IdentityFile"].replace('"', '') env.logger.debug("ssh %s" % env.host_string) def _add_source_versions(version, sources): """Patch package source strings for version, e.g. Debian 'stable' """ name = version env.logger.debug("Source=%s" % name) final = [] for s in sources: if s.find("%s") > 0: s = s % name final.append(s) return final def _ubuntu_dist_name(env): """ Determine Ubuntu dist name (e.g. precise or quantal). """ return env.safe_run_output("cat /etc/*release | grep DISTRIB_CODENAME | cut -f 2 -d =") def _debian_dist_name(env): """ Determine Debian dist name (e.g. squeeze). """ return env.safe_run_output("lsb_release -a | grep Codename | cut -f 2") def _determine_distribution(env): """ Attempt to automatically determine the distribution of the target machine. Currently works for Ubuntu, CentOS, Debian, Scientific Linux and Mac OS X. """ with quiet(): output = env.safe_run_output("cat /etc/*release").lower() if output.find("id=ubuntu") >= 0: return "ubuntu" elif output.find("centos release") >= 0: return "centos" elif output.find("centos linux release") >= 0: return "centos" elif output.find("red hat enterprise linux") >= 0: return "centos" elif output.find("fedora") >= 0: return "centos" # Amazon AMIs are Red-Hat based elif output.find("amzn") >= 0 or output.find("amazon") >= 0: return "centos" elif output.find("suse linux") >= 0: return "suse" elif output.find("opensuse") >= 0: return "suse" elif output.find("scientific linux") >= 0: return "scientificlinux" elif env.safe_exists("/etc/debian_version"): return "debian" elif output.find("id=arch") >= 0 or output.find('id_like="arch"') >=0: return "arch" elif output.find("antergos") >= 0: return "arch" # check for file used by Python's platform.mac_ver elif env.safe_exists("/System/Library/CoreServices/SystemVersion.plist"): return "macosx" else: raise Exception("Attempt to automatically determine Linux distribution of target machine failed:\n%s" % output) ================================================ FILE: cloudbio/fabutils.py ================================================ """Utilities to generalize usage of fabric for local and remote builds. Handles: - Providing a local equivalent of standard functions that avoid the need to ssh to a local machine. Adds generalized targets to the global `env` object which cleanly handle local and remote execution: - safe_run: Run a command - safe_run_output: Run a command, capturing the output - safe_sudo: Run a command as sudo user - safe_exists: Check for existence of a file. - safe_sed: Run sed command. """ import hashlib import os import re import shutil from fabric.api import env, run, sudo, local, settings, hide, put from fabric.contrib.files import exists, sed, contains, append, comment import six SUDO_ENV_KEEPS = [] # Environment variables passed through to sudo environment when using local sudo. SUDO_ENV_KEEPS += ["http_proxy", "https_proxy"] # Required for local sudo to work behind a proxy. # ## Local non-ssh access def local_exists(path, use_sudo=False): func = env.safe_sudo if use_sudo else env.safe_run cmd = 'test -e "$(echo %s)"' % path cmd_symbolic = 'test -h "$(echo %s)"' % path with settings(hide('everything'), warn_only=True): env.lcwd = env.cwd # We do not use cmd_symbolic so we avoid rescuing broken symlinks return not func(cmd).failed def run_local(use_sudo=False, capture=False): def _run(command, *args, **kwags): if use_sudo: sudo_env = " ".join(["%s=$%s" % (keep, keep) for keep in SUDO_ENV_KEEPS]) sudo_to = "" if "user" in kwags: sudo_to = "su - %s" % kwags["user"] sudo_prefix = "sudo %s %s bash -c " % (sudo_env, sudo_to) command = sudo_prefix + '"%s"' % command.replace('"', '\\"') env.lcwd = env.cwd return local(command, capture=capture) return _run def local_put(orig_file, new_file): shutil.copyfile(orig_file, new_file) def local_sed(filename, before, after, limit='', use_sudo=False, backup='.bak', flags='', shell=False): """ Run a search-and-replace on ``filename`` with given regex patterns. From main fabric contrib, modified to handle local. """ func = env.safe_sudo if use_sudo else env.safe_run # Characters to be escaped in both for char in "/'": before = before.replace(char, r'\%s' % char) after = after.replace(char, r'\%s' % char) # Characters to be escaped in replacement only (they're useful in regexen # in the 'before' part) for char in "()": after = after.replace(char, r'\%s' % char) if limit: limit = r'/%s/ ' % limit context = { 'script': r"'%ss/%s/%s/%sg'" % (limit, before, after, flags), 'filename': '"$(echo %s)"' % filename, 'backup': backup } # Test the OS because of differences between sed versions with hide('running', 'stdout'): platform = env.safe_run("uname") if platform in ('NetBSD', 'OpenBSD', 'QNX'): # Attempt to protect against failures/collisions hasher = hashlib.sha1() hasher.update(env.host_string) hasher.update(filename) context['tmp'] = "/tmp/%s" % hasher.hexdigest() # Use temp file to work around lack of -i expr = r"""cp -p %(filename)s %(tmp)s \ && sed -r -e %(script)s %(filename)s > %(tmp)s \ && cp -p %(filename)s %(filename)s%(backup)s \ && mv %(tmp)s %(filename)s""" else: context['extended_regex'] = '-E' if platform == 'Darwin' else '-r' expr = r"sed -i%(backup)s %(extended_regex)s -e %(script)s %(filename)s" command = expr % context return func(command, shell=shell) def local_comment(filename, regex, use_sudo=False, char='#', backup='.bak', shell=False): carot, dollar = '', '' if regex.startswith('^'): carot = '^' regex = regex[1:] if regex.endswith('$'): dollar = '$' regex = regex[:-1] regex = "%s(%s)%s" % (carot, regex, dollar) return local_sed( filename, before=regex, after=r'%s\1' % char, use_sudo=use_sudo, backup=backup, shell=shell ) def _escape_for_regex(text): """Escape ``text`` to allow literal matching using egrep""" regex = re.escape(text) # Seems like double escaping is needed for \ regex = regex.replace('\\\\', '\\\\\\') # Triple-escaping seems to be required for $ signs regex = regex.replace(r'\$', r'\\\$') # Whereas single quotes should not be escaped regex = regex.replace(r"\'", "'") return regex def _expand_path(path): return '"$(echo %s)"' % path def local_contains(filename, text, exact=False, use_sudo=False, escape=True, shell=False): func = use_sudo and env.safe_sudo or env.safe_run if escape: text = _escape_for_regex(text) if exact: text = "^%s$" % text with settings(hide('everything'), warn_only=True): egrep_cmd = 'egrep "%s" %s' % (text, _expand_path(filename)) return func(egrep_cmd, shell=shell).succeeded def local_append(filename, text, use_sudo=False, partial=False, escape=True, shell=False): func = use_sudo and env.safe_sudo or env.safe_run # Normalize non-list input to be a list if isinstance(text, six.string_types): text = [text] for line in text: regex = '^' + _escape_for_regex(line) + ('' if partial else '$') if (env.safe_exists(filename, use_sudo=use_sudo) and line and env.safe_contains(filename, regex, use_sudo=use_sudo, escape=False, shell=shell)): continue line = line.replace("'", r"'\\''") if escape else line func("echo '%s' >> %s" % (line, _expand_path(filename))) def run_output(*args, **kwargs): if not 'shell' in kwargs: kwargs['shell'] = False return run(*args, **kwargs) def configure_runsudo(env): """Setup env variable with safe_sudo and safe_run, supporting non-privileged users and local execution. """ env.is_local = env.hosts == ["localhost"] if env.is_local: env.safe_put = local_put env.safe_sed = local_sed env.safe_comment = local_comment env.safe_contains = local_contains env.safe_append = local_append env.safe_exists = local_exists env.safe_run = run_local() env.safe_run_output = run_local(capture=True) else: env.safe_put = put env.safe_sed = sed env.safe_comment = comment env.safe_contains = contains env.safe_append = append env.safe_exists = exists env.safe_run = run env.safe_run_output = run_output if isinstance(getattr(env, "use_sudo", "true"), six.string_types): if getattr(env, "use_sudo", "true").lower() in ["true", "yes"]: env.use_sudo = True if env.is_local: env.safe_sudo = run_local(True) else: env.safe_sudo = sudo else: env.use_sudo = False if env.is_local: env.safe_sudo = run_local() else: env.safe_sudo = run def find_cmd(env, cmd, args): """Retrieve location of a command, checking in installation directory. """ local_cmd = os.path.join(env.system_install, "bin", cmd) for cmd in [local_cmd, cmd]: with quiet(): test_version = env.safe_run("%s %s" % (cmd, args)) if test_version.succeeded: return cmd return None try: from fabric.api import quiet except ImportError: def quiet(): return settings(hide('warnings', 'running', 'stdout', 'stderr'), warn_only=True) try: from fabric.api import warn_only except ImportError: def warn_only(): return settings(warn_only=True) ================================================ FILE: cloudbio/flavor/__init__.py ================================================ """A Flavor reflects a specialization of a base install, the default being BioLinux. If you want to create a new specialization (say for your own server), the recommended procedure is to choose an existing base install (Edition) and write a Flavor. When your Flavor is of interest to other users, it may be a good idea to commit it to the main project (in ./contrib/flavor). Other (main) flavors can be found in this directory and in ./contrib/flavors """ class Flavor: """Base class. Every flavor derives from this """ def __init__(self, env): self.name = "Base Flavor - no overrides" # should override this self.short_name = "" self.env = env self.check_distribution() def rewrite_config_items(self, name, items): """Generic hook to rewrite a list of configured items. Can define custom dispatches based on name: packages, custom, python, ruby, perl """ return items def check_distribution(self): """Ensure the distribution matches an expected type for this edition. Base supports multiple distributions. """ pass def check_packages_source(self): """Override for check package definition file before updating """ pass def rewrite_apt_sources_list(self, sources): """Allows editions to modify the sources list """ return sources def rewrite_apt_preferences(self, preferences): """Allows editions to modify the apt preferences policy file """ return preferences def rewrite_apt_automation(self, package_info): """Allows editions to modify the apt automation list """ return package_info def rewrite_apt_keys(self, standalone, keyserver): """Allows editions to modify key list""" return standalone, keyserver def apt_upgrade_system(self, env=None): """Upgrade system through apt - so this behaviour can be overridden """ env.safe_sudo("apt-get -y --force-yes upgrade") def post_install(self): """Add scripts for starting FreeNX and CloudMan. """ pass class Minimal(Flavor): def __init__(self, env): Flavor.__init__(self, env) self.name = "Minimal Flavor" self.short_name = "minimal" def rewrite_config_items(self, name, items): """Generic hook to rewrite a list of configured items. Can define custom dispatches based on name: packages, custom, python, ruby, perl """ return items def post_install(self, pkg_install=None): """Add scripts for starting FreeNX and CloudMan. """ pass ================================================ FILE: cloudbio/flavor/config.py ================================================ """ Handle alternative configuration file locations for flavor customizations. """ import os import collections def _find_fname(env, fname): for dirname in [env.get("flavor_dir", None), env.config_dir]: if dirname: full_path = os.path.join(dirname, fname) if os.path.exists(full_path): return full_path return None def get_config_file(env, fname): """ Retrieve YAML configuration file from the default config directory or flavor directory. This combines all options for getting distribution or flavor specific customizations. """ base, ext = os.path.splitext(fname) distribution_fname = "{0}-{1}{2}".format(base, env.get("distribution", "notspecified"), ext) Config = collections.namedtuple("Config", "base dist") out = Config(base=_find_fname(env, fname), dist=_find_fname(env, distribution_fname)) env.logger.debug("Using config file {0}".format(out.base)) return out ================================================ FILE: cloudbio/galaxy/__init__.py ================================================ """ Adapted from Enis Afgan's mi-deployment code: https://bitbucket.org/afgane/mi-deployment """ from __future__ import print_function import os import contextlib from fabric.api import sudo, cd, settings, hide from fabric.colors import red from cloudbio.custom.shared import (_write_to_file, _setup_conf_file, _setup_simple_service, _make_tmp_dir, _add_to_profiles) from cloudbio.galaxy.tools import _install_tools from cloudbio.galaxy.utils import _chown_galaxy, _read_boolean, _dir_is_empty # -- Adjust this link if using content from another location CDN_ROOT_URL = "http://userwww.service.emory.edu/~eafgan/content" REPO_ROOT_URL = "https://bitbucket.org/afgane/mi-deployment/raw/tip" CM_REPO_ROOT_URL = "https://bitbucket.org/galaxy/cloudman/raw/tip/" def _setup_users(env): def _add_user(username, uid=None): """ Add user with username to the system """ if not env.safe_contains('/etc/passwd', "%s:" % username): uid_str = "--uid %s" % uid if uid else "" env.safe_sudo('useradd -d /home/%s --create-home --shell /bin/bash ' '-c"Galaxy-required user" %s --user-group %s' % (username, uid_str, username)) _add_user('galaxy') _add_user('sgeadmin') _add_user('postgres') env.logger.debug("Done setting up Galaxy/CloudMan users") def _setup_galaxy_env_defaults(env): if "galaxy_user" not in env: env.galaxy_user = "galaxy" if "galaxy_home" not in env: env.galaxy_home = "/mnt/galaxy/galaxy-app" if "galaxy_tools_dir" not in env: # Was called install_dir in tools_fabfile.py env.galaxy_tools_dir = "/mnt/galaxy/tools" if "galaxy_loc_files" not in env: indicies_dir = env.get("data_files", "/mnt/galaxyIndices") env.galaxy_loc_files = os.path.join(indicies_dir, "galaxy", "galaxy-data") if "galaxy_jars_dir" not in env: env.galaxy_jars_dir = os.path.join(env.galaxy_home, "tool-data", "shared", "jars") if "galaxy_update_default" not in env: env.galaxy_update_default = True if "python_version" not in env: env.python_version = "2.7" # Override in fabricrc if this is not the case. if "galaxy_indices_mount" not in env: indicies_dir = env.get("data_files", "/mnt/galaxyIndices") env.galaxy_indices_mount = indicies_dir if "galaxy_data_mount" not in env: env.galaxy_data_mount = "/mnt/galaxyData" if "galaxy_init_database" not in env: env.galaxy_init_database = False if "galaxy_len_files" not in env: env.galaxy_len_files = "/mnt/galaxy/configuration_data/len" def _install_galaxy(env): """ Used to install Galaxy and setup its environment, including tools. This method is somewhat targeted for the cloud deployment so some tweaking of the code may be desirable. This method cannot be used to update an existing Galaxy. """ _clone_galaxy_repo(env) _chown_galaxy(env, env.galaxy_home) # Make sure env.galaxy_user owns env.galaxy_home env.safe_sudo("chmod 755 %s" % os.path.split(env.galaxy_home)[0]) setup_db = _read_boolean(env, "galaxy_setup_database", False) if setup_db: _setup_galaxy_db(env) setup_service = _read_boolean(env, "galaxy_setup_service", False) if setup_service: _setup_service(env) _setup_trackster(env) _setup_shed_tools_dir(env) _install_tools(env) setup_xvfb = _read_boolean(env, "galaxy_setup_xvfb", False) if setup_xvfb: _setup_xvfb(env) _chown_galaxy(env, env.galaxy_home) # Make sure env.galaxy_user owns env.galaxy_home _chown_galaxy(env, env.galaxy_loc_files) # Make sure env.galaxy_user owns env.galaxy_loc_files return True def _clone_galaxy_repo(env): """ Clone Galaxy source code repository from ``env.galaxy_repository`` to ``env.galaxy_home``, setting the directory ownership to ``env.galaxy_user`` This method cannot be used to update an existing Galaxy installation. """ # Make sure ``env.galaxy_home`` dir exists but without Galaxy in it galaxy_exists = False if env.safe_exists(env.galaxy_home): if env.safe_exists(os.path.join(env.galaxy_home, '.hg')): env.logger.warning("Galaxy install dir {0} exists and seems to have " "a Mercurial repository already there. Galaxy already installed?" .format(env.galaxy_home)) galaxy_exists = True else: env.safe_sudo("mkdir -p '%s'" % env.galaxy_home) if not galaxy_exists: with cd(env.galaxy_home): # Needs to be done as non galaxy user, otherwise we have a # permissions problem. galaxy_repository = env.get("galaxy_repository", 'https://bitbucket.org/galaxy/galaxy-central/') env.safe_sudo('hg clone %s .' % galaxy_repository) # Make sure ``env.galaxy_home`` is owned by ``env.galaxy_user`` _chown_galaxy(env, env.galaxy_home) # Make sure env.galaxy_home root dir is also owned by env.galaxy_user so Galaxy # process can create necessary dirs (e.g., shed_tools, tmp) env.safe_sudo("chown {0}:{0} {1}".format(env.galaxy_user, os.path.split(env.galaxy_home)[0])) # If needed, custom-configure this freshly cloned Galaxy preconfigured = _read_boolean(env, "galaxy_preconfigured_repository", False) if not preconfigured: _configure_galaxy_repository(env) def _setup_galaxy_db(env): """ Create (if one already does not exist) and initialize a database for use by Galaxy. """ if not _galaxy_db_exists(env): _create_galaxy_db(env) _init_galaxy_db(env) def _get_galaxy_db_configs(env): """ Extract configuration options for Galaxy database and return those as a dictionary """ config = {} config['psql_data_dir'] = env.get('galaxy_database_path', '/mnt/galaxy/db') config['psql_conf_file'] = os.path.join(config['psql_data_dir'], 'postgresql.conf') config['psql_bin_dir'] = env.get('postgres_bin_dir', '/usr/lib/postgresql/9.1/bin') config['psql_user'] = env.get('postrges_user', 'postgres') config['psql_port'] = env.get('postgres_port', '5910') config['psql_log'] = '/tmp/pSQL.log' config['galaxy_db_name'] = env.get('galaxy_db_name', 'galaxy') config['galaxy_ftp_user_pwd'] = env.get('galaxy_ftp_user_password', 'fu5yOj2sn') # And a couple of useful command shortcuts config['pg_ctl_cmd'] = "{0} -D {2}".format(os.path.join(config['psql_bin_dir'], 'pg_ctl'), config['psql_port'], config['psql_data_dir']) config['pg_start_cmd'] = "{0} -w -l {1} start".format(config['pg_ctl_cmd'], config['psql_log']) config['pg_stop_cmd'] = "{0} -w stop".format(config['pg_ctl_cmd']) config['psql_cmd'] = "{0} -p {1}".format(os.path.join(config['psql_bin_dir'], 'psql'), config['psql_port']) return config def _galaxy_db_exists(env): """ Check if galaxy database already exists. Return ``True`` if it does, ``False`` otherwise. Note that this method does a best-effort attempt at starting the DB server if one is not already running to do a thorough test. It shuts the server down upon completion, but only it if also started it. """ db_exists = False started = False c = _get_galaxy_db_configs(env) if env.safe_exists(c['psql_data_dir']) and not _dir_is_empty(c['psql_data_dir']): env.safe_sudo("chown --recursive {0}:{0} {1}".format(c['psql_user'], c['psql_data_dir'])) env.logger.debug("Galaxy database directory {0} already exists.".format(c['psql_data_dir'])) # Check if PostgreSQL is already running and try to start the DB if not if not _postgres_running(env): with settings(warn_only=True): env.logger.debug("Trying to start DB server in {0}".format(c['psql_data_dir'])) env.safe_sudo("{0}".format(c['pg_start_cmd']), user=c['psql_user']) started = True # Check if galaxy DB already exists if 'galaxy' in sudo("{0} -P pager --list | grep {1} || true".format(c['psql_cmd'], c['galaxy_db_name']), user=c['psql_user']): env.logger.warning("Galaxy database {0} already exists in {1}! Not creating it." .format(c['galaxy_db_name'], c['psql_data_dir'])) db_exists = True if started: with settings(warn_only=True): env.safe_sudo("{0}".format(c['pg_stop_cmd']), user=c['psql_user']) return db_exists def _create_galaxy_db(env): """ Create a new PostgreSQL database for use by Galaxy """ c = _get_galaxy_db_configs(env) if not env.safe_exists(c['psql_data_dir']): env.safe_sudo("mkdir -p {0}".format(c['psql_data_dir'])) env.safe_sudo("chown --recursive {0}:{0} {1}".format(c['psql_user'], c['psql_data_dir'])) # Initialize a new database for Galaxy in ``psql_data_dir`` if _dir_is_empty(c['psql_data_dir']): env.safe_sudo("{0} -D {1}".format(os.path.join(c['psql_bin_dir'], 'initdb'), c['psql_data_dir']), user=c['psql_user']) # Set port for the database server env.safe_sed(c['psql_conf_file'], '#port = 5432', 'port = {0}'.format(c['psql_port']), use_sudo=True) env.safe_sudo("chown {0}:{0} {1}".format(c['psql_user'], c['psql_conf_file'])) # Start PostgreSQL server so a role for Galaxy user can be created if not _postgres_running(env): env.safe_sudo(c['pg_start_cmd'], user=c['psql_user']) started = True else: # Restart is required so port setting takes effect env.safe_sudo("{0} -D {1} -w -l {2} restart".format(c['pg_ctl_cmd']), c['psql_data_dir'], c['psql_log'], user=c['psql_user']) started = False # Create a role for env.galaxy_user env.safe_sudo('{0} -c"CREATE ROLE {1} LOGIN CREATEDB"'.format(c['psql_cmd'], env.galaxy_user), user=c['psql_user']) # Create a Galaxy database env.safe_sudo('{0} -p {1} {2}'.format(os.path.join(c['psql_bin_dir'], 'createdb'), c['psql_port'], c['galaxy_db_name']), user=env.galaxy_user) # Create a role for 'galaxyftp' user env.safe_sudo('{0} -c"CREATE ROLE galaxyftp LOGIN PASSWORD \'{1}\'"'.format(c['psql_cmd'], c['galaxy_ftp_user_pwd']), user=c['psql_user']) if started: with settings(warn_only=True): env.safe_sudo("{0}".format(c['pg_stop_cmd']), user=c['psql_user']) exp = "export PATH={0}:$PATH".format(c['psql_bin_dir']) _add_to_profiles(exp) def _init_galaxy_db(env): """ Initialize Galaxy's database with the tables and apply the migrations, fetching Galaxy eggs in the process. """ with cd(env.galaxy_home): universe_wsgi_url = env.get('galaxy_universe_wsgi_url', os.path.join(CM_REPO_ROOT_URL, 'universe_wsgi.ini.cloud')) env.safe_sudo("wget --output-document=universe_wsgi.ini {0}".format(universe_wsgi_url)) started = False if not _postgres_running(env): c = _get_galaxy_db_configs(env) env.safe_sudo(c['pg_start_cmd'], user=c['psql_user']) started = True sudo("bash -c 'export PYTHON_EGG_CACHE=eggs; python -ES ./scripts/fetch_eggs.py; ./create_db.sh'", user=env.galaxy_user) if started: with settings(warn_only=True): env.safe_sudo("{0}".format(c['pg_stop_cmd']), user=c['psql_user']) def _configure_galaxy_options(env, option_dict=None, prefix="galaxy_universe_"): """ Read through fabric's environment and make sure any property of the form galaxy_universe_XXX=YYY lands up in Galaxy's universe_wsgi.ini options as XXX=YYY using Galaxy configuration directory: """ galaxy_conf_directory = env.get("galaxy_conf_directory", None) if not galaxy_conf_directory: return False # By default just read the options from env (i.e. from fabricrc), but # allow override so the options can come from a YAML file (such as done # with galaxy-vm-launcher.) if option_dict is None: option_dict = env option_priority = env.get("galaxy_conf_priority", "200") for key, value in option_dict.iteritems(): if key.startswith(prefix): key = key[len(prefix):] conf_file_name = "%s_override_%s.ini" % (option_priority, key) conf_file = os.path.join(galaxy_conf_directory, conf_file_name) contents = "[app:main]\n%s=%s" % (key, value) _write_to_file(contents, conf_file, "0700") _chown_galaxy(env, conf_file) def _setup_shed_tools_dir(env): ts_dir = "%s/../shed_tools" % env.galaxy_home if not env.safe_exists(ts_dir): _make_dir_for_galaxy(env, ts_dir) env.logger.info("Setup Tool Shed directory {0}".format(ts_dir)) def _setup_trackster(env): """ Download .len files required by Trackster: http://wiki.galaxyproject.org/Learn/Visualization#Setup_for_Local_Instances """ if not env.safe_exists(env.galaxy_len_files): env.safe_sudo("mkdir -p {0}".format(env.galaxy_len_files)) with cd(env.galaxy_len_files): if not env.safe_exists(os.path.join(env.galaxy_len_files, "hg19.len")): local_fn = "len-files.tar.gz" env.safe_sudo('wget --output-document={0} ' 'https://s3.amazonaws.com/usegalaxy/len-files.tar.gz'.format(local_fn)) env.safe_sudo("tar xzf {0}".format(local_fn)) env.safe_sudo("rm {0}".format(local_fn)) _chown_galaxy(env, env.galaxy_len_files) def _configure_galaxy_repository(env): """ Custom-configure Galaxy repository. This is primarily targeted at a cloud deployment. mi-deployment would always edit the repository in this way, but galaxy-vm-launcher requires the configured Galaxy repository to pull in the changesets from https://bitbucket.org/jmchilton/cloud-galaxy-dist which prebakes these modifications in. """ _chown_galaxy(env, env.galaxy_home) # Make sure env.galaxy_user owns env.galaxy_home with cd(env.galaxy_home): # and settings(warn_only=True): # Make sure Galaxy runs in a new shell and does not # inherit the environment by adding the '-ES' flag # to all invocations of python within run.sh env.safe_sudo("sed -i 's/python .\//python -ES .\//g' %s/run.sh" % (env.galaxy_home), user=env.galaxy_user) if _read_boolean(env, "galaxy_cloud", True): # Append DRMAA_LIBRARY_PATH in run.sh as well (this file will exist # once SGE is installed - which happens at instance contextualization) env.safe_sudo("grep -q 'export DRMAA_LIBRARY_PATH=/opt/sge/lib/lx24-amd64/libdrmaa.so.1.0' %s/run.sh; if [ $? -eq 1 ]; then sed -i '2 a export DRMAA_LIBRARY_PATH=/opt/sge/lib/lx24-amd64/libdrmaa.so.1.0' run.sh; fi" % (env.galaxy_home), user=env.galaxy_user) # Upload the custom cloud welcome screen files if not env.safe_exists("%s/static/images/cloud.gif" % env.galaxy_home): env.safe_sudo("wget --output-document=%s/static/images/cloud.gif %s/cloud.gif" % (env.galaxy_home, CDN_ROOT_URL), user=env.galaxy_user) if not env.safe_exists("%s/static/images/cloud_txt.png" % env.galaxy_home): env.safe_sudo("wget --output-document=%s/static/images/cloud_text.png %s/cloud_text.png" % (env.galaxy_home, CDN_ROOT_URL), user=env.galaxy_user) env.safe_sudo("wget --output-document=%s/static/welcome.html %s/welcome.html" % (env.galaxy_home, CDN_ROOT_URL), user=env.galaxy_user) # Set up the symlink for SAMTOOLS (remove this code once SAMTOOLS is converted to data tables) if env.safe_exists("%s/tool-data/sam_fa_indices.loc" % env.galaxy_home): env.safe_sudo("rm %s/tool-data/sam_fa_indices.loc" % env.galaxy_home, user=env.galaxy_user) # TODO: Is this really necessary here? Shouldn't the tools do this themselves? # set up the jars directory for Java tools if not env.safe_exists(env.galaxy_jars_dir): env.safe_sudo("mkdir -p %s" % env.galaxy_jars_dir, user=env.galaxy_user) srma_dir = os.path.join(env.galaxy_tools_dir, 'srma', 'default') haploview_dir = os.path.join(env.galaxy_tools_dir, 'haploview', 'default') picard_dir = os.path.join(env.galaxy_tools_dir, 'picard', 'default') env.safe_sudo('ln -s -f %s/srma.jar %s' % (srma_dir, env.galaxy_jars_dir), user=env.galaxy_user) env.safe_sudo('ln -s -f %s/haploview.jar %s' % (haploview_dir, env.galaxy_jars_dir), user=env.galaxy_user) env.safe_sudo('ln -s -f %s/*.jar %s' % (picard_dir, env.galaxy_jars_dir), user=env.galaxy_user) return True def _setup_service(env): _setup_conf_file(env, "/etc/init.d/galaxy", "galaxy_init", default_source="galaxy_init") _setup_conf_file(env, "/etc/default/galaxy", "galaxy_default") _setup_simple_service("galaxy") def _setup_xvfb(env): _setup_conf_file(env, "/etc/init.d/xvfb", "xvfb_init", default_source="xvfb_init") _setup_conf_file(env, "/etc/default/xvfb", "xvfb_default", default_source="xvfb_default") _setup_simple_service("xvfb") env.safe_sudo("mkdir /var/lib/xvfb; chown root:root /var/lib/xvfb; chmod 0755 /var/lib/xvfb") display_export = "-v DISPLAY=:42" _add_to_profiles(display_export, profiles=["/home/%s/.sge_request" % env.get("galaxy_user", "galaxy")]) def _setup_nginx_service(env): # Setup system service for nginx, not needed for CloudMan but it is # useful if CloudMan is not being used (such as galaxy-vm-launcher work). _setup_conf_file(env, "/etc/init.d/nginx", "nginx_init", default_source="nginx_init") _setup_simple_service("nginx") def _install_nginx_standalone(env): _install_nginx(env) _setup_nginx_service(env) def _install_nginx_package(env): """ Install nginx from a custom package; also see https://github.com/afgane/gvl_flavor/tree/master/playbook """ version = "1.4.7-gvl20140507b01-precise" package_url = "http://cloudman-dev.s3.amazonaws.com/gvl-nginx-%s-amd64.deb" % version with _make_tmp_dir() as work_dir: with contextlib.nested(cd(work_dir), settings(hide('stdout'))): env.safe_run("wget %s" % package_url) env.safe_sudo("dpkg -i %s" % os.path.split(package_url)[1]) # The path for `nginx.conf` depends on how the package was built and # what's set in `nginx_remote_conf_path` must match that! defaults = {"galaxy_home": "/mnt/galaxy/galaxy-app"} _setup_conf_file(env, env.nginx_remote_conf_path, "nginx.conf", defaults=defaults) env.logger.debug("Nginx v{0} installed from package {1}".format(version, package_url)) def _install_nginx(env): """Nginx open source web server. http://www.nginx.org/ """ if "use_nginx_package" in env and env.use_nginx_package.upper() in ["TRUE", "YES"]: _install_nginx_package(env) return # Install nginx from directly version = "1.3.8" url = "http://nginx.org/download/nginx-%s.tar.gz" % version install_dir = os.path.join(env.install_dir, "nginx") remote_conf_dir = os.path.join(install_dir, "conf") # Skip install if already present if env.safe_exists(remote_conf_dir) and env.safe_contains(os.path.join(remote_conf_dir, "nginx.conf"), "/cloud"): env.logger.debug("Nginx already installed; not installing it again.") return with _make_tmp_dir() as work_dir: with contextlib.nested(cd(work_dir), settings(hide('stdout'))): modules = _get_nginx_modules(env) module_flags = " ".join(["--add-module=../%s" % x for x in modules]) env.safe_run("wget %s" % url) env.safe_run("tar xvzf %s" % os.path.split(url)[1]) with cd("nginx-%s" % version): env.safe_run("./configure --prefix=%s --with-ipv6 %s " "--user=galaxy --group=galaxy --with-debug " "--with-http_ssl_module --with-http_gzip_static_module " % (install_dir, module_flags)) env.safe_sed("objs/Makefile", "-Werror", "") env.safe_run("make") env.safe_sudo("make install") env.safe_sudo("cd %s; stow nginx" % env.install_dir) defaults = {"galaxy_home": "/mnt/galaxy/galaxy-app"} _setup_conf_file(env, os.path.join(remote_conf_dir, "nginx.conf"), "nginx.conf", defaults=defaults) nginx_errdoc_file = 'nginx_errdoc.tar.gz' url = os.path.join(REPO_ROOT_URL, nginx_errdoc_file) remote_errdoc_dir = os.path.join(install_dir, "html") with cd(remote_errdoc_dir): env.safe_sudo("wget --output-document=%s/%s %s" % (remote_errdoc_dir, nginx_errdoc_file, url)) env.safe_sudo('tar xvzf %s' % nginx_errdoc_file) env.safe_sudo("mkdir -p %s" % env.install_dir) if not env.safe_exists("%s/nginx" % env.install_dir): env.safe_sudo("ln -s %s/sbin/nginx %s/nginx" % (install_dir, env.install_dir)) # If the guessed symlinking did not work, force it now cloudman_default_dir = "/opt/galaxy/sbin" if not env.safe_exists(cloudman_default_dir): env.safe_sudo("mkdir -p %s" % cloudman_default_dir) if not env.safe_exists(os.path.join(cloudman_default_dir, "nginx")): env.safe_sudo("ln -s %s/sbin/nginx %s/nginx" % (install_dir, cloudman_default_dir)) env.logger.debug("Nginx {0} installed to {1}".format(version, install_dir)) def _get_nginx_modules(env): """Retrieve add-on modules compiled along with nginx. """ modules = { "upload": True, "chunk": True, "ldap": False } module_dirs = [] for module, enabled_by_default in modules.iteritems(): enabled = _read_boolean(env, "nginx_enable_module_%s" % module, enabled_by_default) if enabled: module_dirs.append(eval("_get_nginx_module_%s" % module)(env)) return module_dirs def _get_nginx_module_upload(env): upload_module_version = "2.2.0" upload_url = "http://www.grid.net.ru/nginx/download/" \ "nginx_upload_module-%s.tar.gz" % upload_module_version env.safe_run("wget %s" % upload_url) upload_fname = os.path.split(upload_url)[1] env.safe_run("tar -xvzpf %s" % upload_fname) return upload_fname.rsplit(".", 2)[0] def _get_nginx_module_chunk(env): chunk_module_version = "0.22" chunk_git_version = "b46dd27" chunk_url = "https://github.com/agentzh/chunkin-nginx-module/tarball/v%s" % chunk_module_version chunk_fname = "agentzh-chunkin-nginx-module-%s.tar.gz" % (chunk_git_version) env.safe_run("wget -O %s %s" % (chunk_fname, chunk_url)) env.safe_run("tar -xvzpf %s" % chunk_fname) return chunk_fname.rsplit(".", 2)[0] def _get_nginx_module_ldap(env): env.safe_run("rm -rf nginx-auth-ldap") # Delete it if its there or git won't clone env.safe_run("git clone https://github.com/kvspb/nginx-auth-ldap") return "nginx-auth-ldap" def _setup_postgresql(env): # Handled by CloudMan, but if configuring standalone galaxy, this # will need to be executed to create a postgres user for Galaxy. _configure_postgresql(env) _init_postgresql_data(env) def _configure_postgresql(env, delete_main_dbcluster=False): """ This method is intended for cleaning up the installation when PostgreSQL is installed from a package. Basically, when PostgreSQL is installed from a package, it creates a default database cluster and splits the config file away from the data. This method can delete the default database cluster that was automatically created when the package is installed. Deleting the main database cluster also has the effect of stopping the auto-start of the postmaster server at machine boot. The method adds all of the PostgreSQL commands to the PATH. """ pg_ver = sudo("dpkg -s postgresql | grep Version | cut -f2 -d':'") pg_ver = pg_ver.strip()[:3] # Get first 3 chars of the version since that's all that's used for dir name got_ver = False while(not got_ver): try: pg_ver = float(pg_ver) got_ver = True except Exception: print(red("Problems trying to figure out PostgreSQL version.")) pg_ver = raw_input(red("Enter the correct one (eg, 9.1; not 9.1.3): ")) if delete_main_dbcluster: env.safe_sudo('pg_dropcluster --stop %s main' % pg_ver, user='postgres') exp = "export PATH=/usr/lib/postgresql/%s/bin:$PATH" % pg_ver if not env.safe_contains('/etc/bash.bashrc', exp): env.safe_append('/etc/bash.bashrc', exp, use_sudo=True) def _init_postgresql_data(env): if "galaxy" not in env.safe_sudo("psql -P pager --list | grep galaxy || true", user="postgres"): env.safe_sudo("createdb galaxy", user="postgres") env.safe_sudo("psql -c 'create user galaxy; grant all privileges on database galaxy to galaxy;'", user="postgres") def _postgres_running(env): """ Return ``True`` if PostgreSQL is running, ``False`` otherwise. """ c = _get_galaxy_db_configs(env) if 'no server running' in sudo("{0} status || true".format(c['pg_ctl_cmd']), user=c['psql_user']): return False return True def _make_dir_for_galaxy(env, path): env.safe_sudo("mkdir -p '%s'" % path) _chown_galaxy(env, path) ================================================ FILE: cloudbio/galaxy/applications.py ================================================ """ This file is largely derived from a similar file in mi-deployment written Dr. Enis Afgan. https://bitbucket.org/afgane/mi-deployment/src/8cba95baf98f/tools_fabfile.py Long term it will be best to install these packages for Galaxy via the Tool Shed, however many of these tools are not yet in the tool shed and the tool shed installation is not currently available via the Galaxy API. Until such a time as that is available, Galaxy dependencies may be installed via these functions. I have taken a first crack at harmonizing this with the rest of CloudBioLinux. Wasn't able to reuse fastx_toolkit, tophat, cufflinks. """ import os from fabric.api import cd from cloudbio.custom.shared import _make_tmp_dir, _if_not_installed, _set_default_config from cloudbio.custom.shared import _get_install, _configure_make, _fetch_and_unpack, _get_bin_dir @_if_not_installed(None) def install_fastx_toolkit(env): version = env.tool_version gtext_version = "0.6.1" url_base = "http://hannonlab.cshl.edu/fastx_toolkit/" fastx_url = "%sfastx_toolkit-%s.tar.bz2" % (url_base, version) gtext_url = "%slibgtextutils-%s.tar.bz2" % (url_base, gtext_version) pkg_name = 'fastx_toolkit' install_dir = os.path.join(env.galaxy_tools_dir, pkg_name, version) with _make_tmp_dir() as work_dir: with cd(work_dir): env.safe_run("wget %s" % gtext_url) env.safe_run("tar -xjvpf %s" % (os.path.split(gtext_url)[-1])) install_cmd = env.safe_sudo if env.use_sudo else env.safe_run with cd("libgtextutils-%s" % gtext_version): env.safe_run("./configure --prefix=%s" % (install_dir)) env.safe_run("make") install_cmd("make install") env.safe_run("wget %s" % fastx_url) env.safe_run("tar -xjvpf %s" % os.path.split(fastx_url)[-1]) with cd("fastx_toolkit-%s" % version): env.safe_run("export PKG_CONFIG_PATH=%s/lib/pkgconfig; ./configure --prefix=%s" % (install_dir, install_dir)) env.safe_run("make") install_cmd("make install") ## TODO: Rework to use more of custom enhancements @_if_not_installed("maq") def install_maq(env): version = env["tool_version"] url = "http://downloads.sourceforge.net/project/maq/maq/%s/maq-%s.tar.bz2" \ % (version, version) _get_install(url, env, _configure_make) @_if_not_installed("macs14") def install_macs(env): from cloudbio.custom.bio_nextgen import install_macs as cbl_install_macs install_dir = env.system_install cbl_install_macs(env) env.safe_sudo("echo 'PATH=%s/bin:$PATH' > %s/env.sh" % (install_dir, install_dir)) env.safe_sudo("echo 'PYTHONPATH=%s/lib/python%s/site-packages:$PYTHONPATH' >> %s/env.sh" % (env.python_version, install_dir, install_dir)) _update_default(env, install_dir) @_if_not_installed("megablast") def install_megablast(env): version = env.tool_version url = 'ftp://ftp.ncbi.nlm.nih.gov/blast/executables/release/%s/blast-%s-x64-linux.tar.gz' % (version, version) install_dir = env.system_install install_cmd = env.safe_sudo if env.use_sudo else env.safe_run if not env.safe_exists(install_dir): install_cmd("mkdir -p %s" % install_dir) with _make_tmp_dir() as work_dir: with cd(work_dir): env.safe_run("wget %s" % url) env.safe_run("tar -xvzf %s" % os.path.split(url)[-1]) with cd('blast-%s/bin' % version): install_cmd("mv * %s" % install_dir) @_if_not_installed("blastn") def install_blast(env): version = env.tool_version url = 'ftp://ftp.ncbi.nlm.nih.gov/blast/executables/blast+/%s/ncbi-blast-%s-x64-linux.tar.gz' % (version[:-1], version) install_dir = env.system_install install_cmd = env.safe_sudo if env.use_sudo else env.safe_run if not env.safe_exists(install_dir): install_cmd("mkdir -p %s" % install_dir) with _make_tmp_dir() as work_dir: with cd(work_dir): env.safe_run("wget %s" % url) env.safe_run("tar -xvzf %s" % os.path.split(url)[-1]) with cd('ncbi-blast-%s/bin' % version): bin_dir = _get_bin_dir(env) install_cmd("mv * '%s'" % bin_dir) @_if_not_installed("sputnik") def install_sputnik(env): version = env.tool_version url = 'http://bitbucket.org/natefoo/sputnik-mononucleotide/downloads/sputnik_%s_linux2.6_x86_64' % version install_dir = env.system_install install_cmd = env.safe_sudo if env.use_sudo else env.safe_run if not env.safe_exists(install_dir): install_cmd("mkdir -p %s" % install_dir) with _make_tmp_dir() as work_dir: with cd(work_dir): env.safe_run("wget -O sputnik %s" % url) install_cmd("mv sputnik %s" % install_dir) @_if_not_installed("taxonomy2tree") def install_taxonomy(env): version = env.tool_version url = 'http://bitbucket.org/natefoo/taxonomy/downloads/taxonomy_%s_linux2.6_x86_64.tar.gz' % version install_dir = env.system_install install_cmd = env.safe_sudo if env.use_sudo else env.safe_run if not env.safe_exists(install_dir): install_cmd("mkdir -p %s" % install_dir) with _make_tmp_dir() as work_dir: with cd(work_dir): env.safe_run("wget %s" % url) env.safe_run("tar -xvzf %s" % os.path.split(url)[-1]) with cd(os.path.split(url)[-1].split('.tar.gz')[0]): install_cmd("mv * %s" % install_dir) @_if_not_installed("add_scores") def install_add_scores(env): version = env.tool_version url = 'http://bitbucket.org/natefoo/add_scores/downloads/add_scores_%s_linux2.6_x86_64' % version install_dir = env.system_install install_cmd = env.safe_sudo if env.use_sudo else env.safe_run if not env.safe_exists(install_dir): install_cmd("mkdir -p %s" % install_dir) with _make_tmp_dir() as work_dir: with cd(work_dir): env.safe_run("wget -O add_scores %s" % url) install_cmd("mv add_scores %s" % install_dir) @_if_not_installed("HYPHY") def install_hyphy(env): version = env.tool_version url = 'http://www.datam0nk3y.org/svn/hyphy' install_dir = env.system_install install_cmd = env.safe_sudo if env.use_sudo else env.safe_run if not env.safe_exists(install_dir): install_cmd("mkdir -p %s" % install_dir) with _make_tmp_dir() as work_dir: with cd(work_dir): env.safe_run("svn co -r %s %s src" % (version, url)) env.safe_run("mkdir -p build/Source/Link") env.safe_run("mkdir build/Source/SQLite") env.safe_run("cp src/trunk/Core/*.{h,cp,cpp} build/Source") env.safe_run("cp src/trunk/HeadlessLink/*.{h,cpp} build/Source/SQLite") env.safe_run("cp src/trunk/NewerFunctionality/*.{h,cpp} build/Source/") env.safe_run("cp src/SQLite/trunk/*.{c,h} build/Source/SQLite/") env.safe_run("cp src/trunk/Scripts/*.sh build/") env.safe_run("cp src/trunk/Mains/main-unix.cpp build/Source/main-unix.cxx") env.safe_run("cp src/trunk/Mains/hyphyunixutils.cpp build/Source/hyphyunixutils.cpp") env.safe_run("cp -R src/trunk/{ChartAddIns,DatapanelAddIns,GeneticCodes,Help,SubstitutionClasses,SubstitutionModels,TemplateBatchFiles,TopologyInference,TreeAddIns,UserAddins} build") env.safe_run("rm build/Source/preferences.cpp") with cd("build"): env.safe_run("bash build.sh SP") install_cmd("mv build/* %s" % install_dir) _update_default(env, install_dir) @_if_not_installed(None) def install_gatk(env): version = env.tool_version url = 'ftp://ftp.broadinstitute.org/pub/gsa/GenomeAnalysisTK/GenomeAnalysisTK-%s.tar.bz2' % version pkg_name = 'gatk' install_dir = os.path.join(env.galaxy_tools_dir, pkg_name, version) install_cmd = env.safe_sudo if env.use_sudo else env.safe_run if not env.safe_exists(install_dir): install_cmd("mkdir -p %s" % install_dir) install_cmd("mkdir -p %s/bin" % install_dir) with _make_tmp_dir() as work_dir: with cd(work_dir): env.safe_run("wget -O gatk.tar.bz2 %s" % url) env.safe_run("tar -xjf gatk.tar.bz2") install_cmd("cp GenomeAnalysisTK-%s/*.jar %s/bin" % (version, install_dir)) # Create shell script to wrap jar env.safe_sudo("echo '#!/bin/sh' > %s/bin/gatk" % (install_dir)) env.safe_sudo("echo 'java -jar %s/bin/GenomeAnalysisTK.jar $@' >> %s/bin/gatk" % (install_dir, install_dir)) env.safe_sudo("chmod +x %s/bin/gatk" % install_dir) # env file env.safe_sudo("echo 'PATH=%s/bin:$PATH' > %s/env.sh" % (install_dir, install_dir)) _update_default(env, install_dir) # Link jar to Galaxy's jar dir jar_dir = os.path.join(env.galaxy_jars_dir, pkg_name) if not env.safe_exists(jar_dir): install_cmd("mkdir -p %s" % jar_dir) tool_dir = os.path.join(env.galaxy_tools_dir, pkg_name, 'default', 'bin') install_cmd('ln --force --symbolic %s/*.jar %s/.' % (tool_dir, jar_dir)) install_cmd('chown --recursive %s:%s %s' % (env.galaxy_user, env.galaxy_user, jar_dir)) @_if_not_installed("srma.jar") def install_srma(env): version = env.tool_version mirror_info = "?use_mirror=voxel" url = 'http://downloads.sourceforge.net/project/srma/srma/%s/srma-%s.jar' \ % (version[:3], version) install_dir = env.system_install install_cmd = env.safe_sudo if env.use_sudo else env.safe_run if not env.safe_exists(install_dir): install_cmd("mkdir -p %s" % install_dir) with _make_tmp_dir() as work_dir: with cd(work_dir): env.safe_run("wget %s%s -O %s" % (url, mirror_info, os.path.split(url)[-1])) install_cmd("mv srma-%s.jar %s" % (version, install_dir)) install_cmd("ln -f -s srma-%s.jar %s/srma.jar" % (version, install_dir)) env.safe_sudo("touch %s/env.sh" % install_dir) _update_default(env, install_dir) @_if_not_installed("BEAM2") def install_beam(env): url = 'http://www.stat.psu.edu/~yuzhang/software/beam2.tar' install_dir = env.system_install install_cmd = env.safe_sudo if env.use_sudo else env.safe_run if not env.safe_exists(install_dir): install_cmd("mkdir -p %s" % install_dir) with _make_tmp_dir() as work_dir: with cd(work_dir): env.safe_run("wget %s -O %s" % (url, os.path.split(url)[-1])) env.safe_run("tar xf %s" % (os.path.split(url)[-1])) install_cmd("mv BEAM2 %s" % install_dir) env.safe_sudo("echo 'PATH=%s:$PATH' > %s/env.sh" % (install_dir, install_dir)) _update_default(env, install_dir) @_if_not_installed("pass2") def install_pass(env): url = 'http://www.stat.psu.edu/~yuzhang/software/pass2.tar' install_dir = env.system_install install_cmd = env.safe_sudo if env.use_sudo else env.safe_run if not env.safe_exists(install_dir): install_cmd("mkdir -p %s" % install_dir) with _make_tmp_dir() as work_dir: with cd(work_dir): env.safe_run("wget %s -O %s" % (url, os.path.split(url)[-1])) env.safe_run("tar xf %s" % (os.path.split(url)[-1])) install_cmd("mv pass2 %s" % install_dir) env.safe_sudo("echo 'PATH=%s:$PATH' > %s/env.sh" % (install_dir, install_dir)) _update_default(env, install_dir) @_if_not_installed("lps_tool") def install_lps_tool(env): version = env.tool_version url = 'http://www.bx.psu.edu/miller_lab/dist/lps_tool.%s.tar.gz' % version install_dir = env.system_install install_cmd = env.safe_sudo if env.use_sudo else env.safe_run if not env.safe_exists(install_dir): install_cmd("mkdir -p %s" % install_dir) with _make_tmp_dir() as work_dir: with cd(work_dir): env.safe_run("wget %s -O %s" % (url, os.path.split(url)[-1])) env.safe_run("tar zxf %s" % (os.path.split(url)[-1])) install_cmd("./lps_tool.%s/MCRInstaller.bin -P bean421.installLocation=\"%s/MCR\" -silent" % (version, install_dir)) install_cmd("mv lps_tool.%s/lps_tool %s" % (version, install_dir)) env.safe_sudo("echo 'PATH=%s:$PATH' > %s/env.sh" % (install_dir, install_dir)) env.safe_sudo("echo 'MCRROOT=%s/MCR/v711; export MCRROOT' >> %s/env.sh" % (install_dir, install_dir)) _update_default(env, install_dir) @_if_not_installed("plink") def install_plink(env): version = env.tool_version url = 'http://pngu.mgh.harvard.edu/~purcell/plink/dist/plink-%s-x86_64.zip' % version install_dir = env.system_install install_cmd = env.safe_sudo if env.use_sudo else env.safe_run if not env.safe_exists(install_dir): install_cmd("mkdir -p %s" % install_dir) with _make_tmp_dir() as work_dir: with cd(work_dir): env.safe_run("wget %s -O %s" % (url, os.path.split(url)[-1])) env.safe_run("unzip %s" % (os.path.split(url)[-1])) install_cmd("mv plink-%s-x86_64/plink %s" % (version, install_dir)) env.safe_sudo("echo 'PATH=%s:$PATH' > %s/env.sh" % (install_dir, install_dir)) _update_default(env, install_dir) @_if_not_installed(None) def install_fbat(env): version = env.tool_version url = 'http://www.biostat.harvard.edu/~fbat/software/fbat%s_linux64.tar.gz' % version.replace('.', '') install_dir = env.system_install install_cmd = env.safe_sudo if env.use_sudo else env.safe_run if not env.safe_exists(install_dir): install_cmd("mkdir -p %s" % install_dir) with _make_tmp_dir() as work_dir: with cd(work_dir): env.safe_run("wget %s -O %s" % (url, os.path.split(url)[-1])) env.safe_run("tar zxf %s" % (os.path.split(url)[-1])) install_cmd("mv fbat %s" % install_dir) env.safe_sudo("echo 'PATH=%s:$PATH' > %s/env.sh" % (install_dir, install_dir)) _update_default(env, install_dir) @_if_not_installed("Haploview_beta.jar") def install_haploview(env): url = 'http://www.broadinstitute.org/ftp/pub/mpg/haploview/Haploview_beta.jar' install_dir = env.system_install install_cmd = env.safe_sudo if env.use_sudo else env.safe_run if not env.safe_exists(install_dir): install_cmd("mkdir -p %s" % install_dir) with _make_tmp_dir() as work_dir: with cd(work_dir): env.safe_run("wget %s -O %s" % (url, os.path.split(url)[-1])) install_cmd("mv %s %s" % (os.path.split(url)[-1], install_dir)) install_cmd("ln -s %s %s/haploview.jar" % (os.path.split(url)[-1], install_dir)) _update_default(env, install_dir) @_if_not_installed("eigenstrat") def install_eigenstrat(env): version = env.tool_version url = 'http://www.hsph.harvard.edu/faculty/alkes-price/files/EIG%s.tar.gz' % version install_dir = env.system_install install_cmd = env.safe_sudo if env.use_sudo else env.safe_run if not env.safe_exists(install_dir): install_cmd("mkdir -p %s" % install_dir) with _make_tmp_dir() as work_dir: with cd(work_dir): env.safe_run("wget %s -O %s" % (url, os.path.split(url)[-1])) env.safe_run("tar zxf %s" % (os.path.split(url)[-1])) install_cmd("mv bin %s" % install_dir) env.safe_sudo("echo 'PATH=%s/bin:$PATH' > %s/env.sh" % (install_dir, install_dir)) _update_default(env, install_dir) @_if_not_installed("augustus") def install_augustus(env): default_version = "2.7" version = env.get('tool_version', default_version) url = "http://bioinf.uni-greifswald.de/augustus/binaries/augustus.%s.tar.gz" % version install_dir = env.system_install with _make_tmp_dir() as work_dir: with cd(work_dir): _fetch_and_unpack(url, need_dir=False) env.safe_sudo("mkdir -p '%s'" % install_dir) env.safe_sudo("mv augustus.%s/* '%s'" % (version, install_dir)) @_if_not_installed("SortSam.jar") def install_picard(env): version = env.tool_version mirror_info = "?use_mirror=voxel" url = 'http://downloads.sourceforge.net/project/picard/picard-tools/%s/picard-tools-%s.zip' % (version, version) pkg_name = 'picard' install_dir = env.system_install install_cmd = env.safe_sudo if env.use_sudo else env.safe_run if not env.safe_exists(install_dir): install_cmd("mkdir -p %s" % install_dir) with _make_tmp_dir() as work_dir: with cd(work_dir): env.safe_run("wget %s%s -O %s" % (url, mirror_info, os.path.split(url)[-1])) env.safe_run("unzip %s" % (os.path.split(url)[-1])) install_cmd("mv picard-tools-%s/*.jar %s" % (version, install_dir)) _update_default(env, install_dir) # set up the jars directory jar_dir = os.path.join(env.galaxy_jars_dir, 'picard') if not env.safe_exists(jar_dir): install_cmd("mkdir -p %s" % jar_dir) tool_dir = os.path.join(env.galaxy_tools_dir, pkg_name, 'default') install_cmd('ln --force --symbolic %s/*.jar %s/.' % (tool_dir, jar_dir)) install_cmd('chown --recursive %s:%s %s' % (env.galaxy_user, env.galaxy_user, jar_dir)) @_if_not_installed("fastqc") def install_fastqc(env): """ This tool is installed in Galaxy's jars dir """ version = env.tool_version url = 'http://www.bioinformatics.bbsrc.ac.uk/projects/fastqc/fastqc_v%s.zip' % version pkg_name = 'FastQC' install_dir = os.path.join(env.galaxy_jars_dir) install_cmd = env.safe_sudo if env.use_sudo else env.safe_run if not env.safe_exists(install_dir): install_cmd("mkdir -p %s" % install_dir) with cd(install_dir): install_cmd("wget %s -O %s" % (url, os.path.split(url)[-1])) install_cmd("unzip -u %s" % (os.path.split(url)[-1])) install_cmd("rm %s" % (os.path.split(url)[-1])) with cd(pkg_name): install_cmd('chmod 755 fastqc') install_cmd('chown --recursive %s:%s %s' % (env.galaxy_user, env.galaxy_user, pkg_name)) def _update_default(env, install_dir): env.safe_sudo("touch %s/env.sh" % install_dir) env.safe_sudo("chmod +x %s/env.sh" % install_dir) _set_default_config(env, install_dir) #@if_tool_not_found() #def install_emboss(env): # version = env.tool_version # url = 'ftp://emboss.open-bio.org/pub/EMBOSS/old/%s/EMBOSS-%s.tar.gz' % (version, version) # pkg_name = 'emboss' # install_dir = os.path.join(env.galaxy_tools_dir, pkg_name, version) # install_cmd = env.safe_sudo if env.use_sudo else env.safe_run # if not env.safe_exists(install_dir): # install_cmd("mkdir -p %s" % install_dir) # with _make_tmp_dir() as work_dir: # with cd(work_dir): # env.safe_run("wget %s" % url) # env.safe_run("tar -xvzf %s" % os.path.split(url)[-1]) # with cd(os.path.split(url)[-1].split('.tar.gz')[0]): # env.safe_run("./configure --prefix=%s" % install_dir) # env.safe_run("make") # install_cmd("make install") # phylip_version = '3.6b' # url = 'ftp://emboss.open-bio.org/pub/EMBOSS/old/%s/PHYLIP-%s.tar.gz' % (version, phylip_version) # with _make_tmp_dir() as work_dir: # with cd(work_dir): # env.safe_run("wget %s" % url) # env.safe_run("tar -xvzf %s" % os.path.split(url)[-1]) # with cd(os.path.split(url)[-1].split('.tar.gz')[0]): # env.safe_run("./configure --prefix=%s" % install_dir) # env.safe_run("make") # install_cmd("make install") ================================================ FILE: cloudbio/galaxy/r.py ================================================ import os import tempfile from cloudbio.custom.shared import _make_tmp_dir from fabric.api import sudo, put, cd r_packages_template = """ r <- getOption("repos"); r["CRAN"] <- "http://watson.nci.nih.gov/cran_mirror"; options(repos=r); install.packages( c( %s ), dependencies = TRUE); source("http://bioconductor.org/biocLite.R"); biocLite( c( %s ) ); """ def _install_r_packages(tools_conf): f = tempfile.NamedTemporaryFile() r_packages = tools_conf["r_packages"] bioconductor_packages = tools_conf["bioconductor_packages"] if not r_packages and not bioconductor_packages: return r_cmd = r_packages_template % (_concat_strings(r_packages), _concat_strings(bioconductor_packages)) f.write(r_cmd) f.flush() with _make_tmp_dir() as work_dir: put(f.name, os.path.join(work_dir, 'install_packages.r')) with cd(work_dir): sudo("R --vanilla --slave < install_packages.r") f.close() def _concat_strings(strings): if strings: return ", ".join(map(lambda x: '"%s"' % x, strings)) else: return "" ================================================ FILE: cloudbio/galaxy/tools.py ================================================ import os from string import Template import six import yaml from cloudbio.custom.bio_general import * from cloudbio.custom.bio_nextgen import * from cloudbio.custom.bio_proteomics import * from cloudbio.custom.shared import _set_default_config, _add_to_profiles from cloudbio.galaxy.applications import * from cloudbio.galaxy.r import _install_r_packages from cloudbio.galaxy.utils import _chown_galaxy, _read_boolean FAILED_INSTALL_MESSAGE = \ "Failed to install application %s as a Galaxy application. This may be a transient problem (e.g. mirror for download is currently unavailable) or misconfiguration. The contents of the CloudBioLinux temporary working directory may need to be deleted." def _install_tools(env, tools_conf=None): """ Install tools needed for Galaxy along with tool configuration directories needed by Galaxy. """ if not tools_conf: tools_conf = _load_tools_conf(env) if _read_boolean(env, "galaxy_install_dependencies", False): # Need to ensure the install dir exists and is owned by env.galaxy_user _setup_install_dir(env) _install_configured_applications(env, tools_conf) _chown_galaxy(env, env.galaxy_tools_dir) _chown_galaxy(env, env.galaxy_jars_dir) if _read_boolean(env, "galaxy_install_r_packages", False): _install_r_packages(tools_conf) def _tools_conf_path(env): """ Load path to galaxy_tools_conf file from env, allowing expansion of $__contrib_dir__. Default to $__contrib_dir__/flavor/cloudman/tools.yaml. """ contrib_dir = os.path.join(env.config_dir, os.pardir, "contrib") default_tools_conf_path = os.path.join(contrib_dir, "flavor", "cloudman", "tools.yaml") tools_conf_path = env.get("galaxy_tools_conf", default_tools_conf_path) ## Allow expansion of __config_dir__ in galaxy_tools_conf property. return Template(tools_conf_path).safe_substitute({"__contrib_dir__": contrib_dir}) def _load_tools_conf(env): with open(_tools_conf_path(env)) as in_handle: full_data = yaml.safe_load(in_handle) return full_data def _setup_install_dir(env): """Sets up install dir and ensures its owned by Galaxy""" if not env.safe_exists(env.galaxy_tools_dir): env.safe_sudo("mkdir -p %s" % env.galaxy_tools_dir) _chown_galaxy(env, env.galaxy_tools_dir) # Create a general-purpose ``bin`` directory under the galaxy_tools_dir # and put it on the PATH so users can more easily add custom tools bin_dir = os.path.join(env.galaxy_tools_dir, 'bin') if not env.safe_exists(bin_dir): env.safe_sudo("mkdir -p %s" % bin_dir) _chown_galaxy(env, bin_dir) line = "export PATH={0}:$PATH".format(bin_dir) _add_to_profiles(line) if not env.safe_exists(env.galaxy_jars_dir): env.safe_sudo("mkdir -p %s" % env.galaxy_jars_dir) _chown_galaxy(env, env.galaxy_jars_dir) def _install_configured_applications(env, tools_conf): """ Install external tools defined by YAML or dictionary data structure. Instead of installing in system_install (e.g. /usr), these custom tools will be installed as Galaxy dependency applications. """ applications = tools_conf["applications"] or {} # Changing the default behavior here to install all tools and # just record exceptions as they occur, but wait until the end # raise an exception out of this block. Disable this behavior # by setting galaxay_tool_defer_errors to False. defer_errors = env.get("galaxy_tool_defer_errors", True) exceptions = {} for (name, tool_conf) in applications.iteritems(): if not __check_conditional(tool_conf): continue try: _install_application(name, tool_conf) except BaseException as e: exceptions[name] = e if not defer_errors: break if exceptions: for name, exception in exceptions.iteritems(): env.logger.warn(FAILED_INSTALL_MESSAGE % name) first_exception = list(exceptions.values())[0] raise first_exception def _install_application(name, versions, tool_install_dir=None): """ Install single custom tool as Galaxy dependency application. TODO: Rename versions and document options. """ if type(versions) is str: versions = [versions] for version_info in versions: if type(version_info) is str: _install_tool(env, name, version=version_info, requirement_name=name, tool_install_dir=tool_install_dir) else: version = version_info["version"] bin_dirs = version_info.get("bin_dirs", ["bin"]) env_vars = version_info.get("env_vars", {}) provides = version_info.get("provides", []) if isinstance(provides, (str, unicode, six.string_types)): provides = [provides] for provide_conf in provides[:]: if isinstance(provide_conf, dict): provides.remove(provide_conf) if __check_conditional(provide_conf): provies.append(provide_conf["name"]) # Some requirements (e.g. blast+) maybe not have valid python # identifiers as name. Use install_blast to setup but override # requirement directory name with requirement_name field. requirement_name = version_info.get("requirement_name", name) tool_env = _install_tool(env, name, version, bin_dirs=bin_dirs, env_vars=env_vars, requirement_name=requirement_name, tool_install_dir=tool_install_dir) symlink_versions = version_info.get("symlink_versions", []) if type(symlink_versions) is str: symlink_versions = [symlink_versions] for symlink_version in symlink_versions: _set_default_config(tool_env, tool_env["system_install"], symlink_version) if provides: install_dir = tool_env["system_install"] ## Create additional symlinked packages from this one. tool_dir = "%s/.." % install_dir tools_dir = "%s/.." % tool_dir for package in provides: link_dir = "%s/%s" % (tools_dir, package) env.safe_sudo("ln -f -s '%s' '%s'" % (requirement_name, link_dir)) def _install_tool(env, name, version, requirement_name, bin_dirs=["bin"], env_vars={}, tool_install_dir=None): tool_env = _build_tool_env(env, requirement_name, version, tool_install_dir) env.logger.debug("Installing a Galaxy tool via 'install_%s'" % name) eval("install_%s" % name)(tool_env) _install_galaxy_config(tool_env, bin_dirs, env_vars=env_vars) return tool_env def _build_tool_env(env, name, version, tool_install_dir): """ Build new env to have tool installed for Galaxy instead of into /usr. """ tool_env = {"tool_version": version, "galaxy_tool_install": True} for key, value in env.iteritems(): tool_env[key] = value if not tool_install_dir: tool_install_dir = os.path.join(env.galaxy_tools_dir, name, version) tool_env["system_install"] = tool_install_dir tool_env["local_install"] = tool_install_dir tool_env["venv_directory"] = "%s/%s" % (tool_env["system_install"], "venv") return AttributeDict(tool_env) def __check_conditional(conf_dict): passes = True try: if "if" in conf_dict: value = conf_dict["if"] passes = _read_boolean(env, value, False) elif "unless" in conf_dict: value = conf_dict["unless"] passes = not _read_boolean(env, value, False) except TypeError: # configuration is not a dictionary, default to True pass return passes class AttributeDict(dict): """ Dictionary that allows attribute access to values. This is needed because cloudbio.custom.* accesses env extensively via attributes (e.g. env.system_install). http://stackoverflow.com/questions/4984647/accessing-dict-keys-like-an-attribute-in-python """ __getattr__ = dict.__getitem__ __setattr__ = dict.__setitem__ def _install_galaxy_config(tool_env, bin_dirs, env_vars): """ Setup galaxy tool config files (env.sh-es) and default version symbolic links. """ install_dir = tool_env["system_install"] env_path = os.path.join(install_dir, "env.sh") bin_paths = [os.path.join(install_dir, bin_dir) for bin_dir in bin_dirs] path_pieces = [bin_path for bin_path in bin_paths if env.safe_exists(bin_path)] if len(path_pieces) > 0 and not env.safe_exists(env_path): path_addtion = ":".join(path_pieces) # Standard bin install, just add it to path env.safe_sudo("echo 'PATH=%s:$PATH' > %s" % (path_addtion, env_path)) venv_path = "%s/%s" % (install_dir, "venv") if env.safe_exists(venv_path): # Have env.sh activate virtualdirectory env.safe_sudo("echo '. %s/bin/activate' >> %s" % (venv_path, env_path)) env.safe_sudo("chmod +x %s" % env_path) for env_var, env_var_value in env_vars.iteritems(): env_var_template = Template(env_var_value) expanded_env_var_value = env_var_template.substitute(tool_env) env.safe_sudo("echo 'export %s=%s' >> %s" % (env_var, expanded_env_var_value, env_path)) env.logger.debug("Added Galaxy env.sh file: %s" % env_path) # TODO: If a direct install (i.e. tool_install_dir specified instead of galaxy_tools_dir) # default is still setup. This is not really desired. _set_default_config(tool_env, install_dir) if _read_boolean(tool_env, "autoload_galaxy_tools", False) and env.safe_exists(env_path): # In this case, the web user (e.g. ubuntu) should auto-load all of # galaxy's default env.sh files so they are available for direct use # as well. _add_to_profiles(". %s" % env_path, profiles=["~/.bashrc"]) ================================================ FILE: cloudbio/galaxy/utils.py ================================================ from fabric.api import sudo from fabric.contrib.files import exists def _read_boolean(env, name, default): ## TODO: Replace calls to this with calls to cloudbio.custom.shared version property_str = env.get(name, str(default)) return property_str.upper() in ["TRUE", "YES"] def _chown_galaxy(env, path): """ Recursively change ownership of ``path``, first checking if ``path`` exists. """ chown_command = "chown --recursive %s:%s %s" galaxy_user = env.get("galaxy_user", "galaxy") if env.safe_exists(path): env.safe_sudo(chown_command % (galaxy_user, galaxy_user, path)) def _dir_is_empty(path): """ Return ``True`` is ``path`` directory has no files or folders in it. Return ``False`` otherwise. """ if "empty" in sudo('[ "$(ls -A {0})" ] || echo "empty"'.format(path)): return True return False ================================================ FILE: cloudbio/libraries.py ================================================ """Installers for programming language specific libraries. """ import os from fabric.api import env, cd, settings from cloudbio import fabutils from cloudbio.custom import shared def r_library_installer(config): """Install R libraries using CRAN and Bioconductor. """ if config.get("cran") or config.get("bioc") or config.get("github"): with shared._make_tmp_dir() as tmp_dir: with cd(tmp_dir): # Create an Rscript file with install details. out_file = os.path.join(tmp_dir, "install_packages.R") _make_install_script(out_file, config) # run the script and then get rid of it # try using either rlib_installed = False rscripts = [] conda_bin = shared._conda_cmd(env) if conda_bin: rscripts.append(fabutils.find_cmd(env, os.path.join(os.path.dirname(conda_bin), "Rscript"), "--version")) rscripts.append(fabutils.find_cmd(env, "Rscript", "--version")) for rscript in rscripts: if rscript: env.safe_run("%s %s" % (rscript, out_file)) rlib_installed = True break if not rlib_installed: env.logger.warn("Rscript not found; skipping install of R libraries.") env.safe_run("rm -f %s" % out_file) def _make_install_script(out_file, config): if env.safe_exists(out_file): env.safe_run("rm -f %s" % out_file) env.safe_run("touch %s" % out_file) lib_loc = os.path.join(env.system_install, "lib", "R", "site-library") env.safe_sudo("mkdir -p %s" % lib_loc) with settings(warn_only=True): env.safe_sudo("chown -R %s %s" % (env.user, lib_loc)) repo_info = """ .libPaths(c("%s")) library(methods) cran.repos <- getOption("repos") cran.repos["CRAN" ] <- "%s" options(repos=cran.repos) """ % (lib_loc, config["cranrepo"]) if config.get("biocrepo"): repo_info += """\nsource("%s")\n""" % config["biocrepo"] env.safe_append(out_file, repo_info) install_fn = """ repo.installer <- function(repos, install.fn, pkg_name_fn) { %s maybe.install <- function(pname) { if (!is.null(pkg_name_fn)) { pinfo <- pkg_name_fn(pname) ipkgs <- installed.packages()[,3][pinfo["pkg"]] if (is.na(ipkgs[pinfo["pkg"]]) || pinfo["version"] != ipkgs[pinfo["pkg"]]) try(install.fn(pinfo["pname"])) } else if (!(is.element(pname, installed.packages()[,1]))) install.fn(pname) } } """ if config.get("update_packages", True): update_str = """ update.packages(lib.loc="%s", repos=repos, ask=FALSE) """ % lib_loc else: update_str = "\n" env.safe_append(out_file, install_fn % update_str) if len(config.get("cran") or []) > 0: std_install = """ std.pkgs <- c(%s) std.installer = repo.installer(cran.repos, install.packages, NULL) lapply(std.pkgs, std.installer) """ % (", ".join('"%s"' % p for p in config['cran'])) env.safe_append(out_file, std_install) if len(config.get("bioc") or []) > 0: bioc_install = """ bioc.pkgs <- c(%s) bioc.installer = repo.installer(biocinstallRepos(), biocLite, NULL) lapply(bioc.pkgs, bioc.installer) """ % (", ".join('"%s"' % p for p in config['bioc'])) env.safe_append(out_file, bioc_install) if config.get("cran-after-bioc"): std2_install = """ std2.pkgs <- c(%s) lapply(std2.pkgs, std.installer) """ % (", ".join('"%s"' % p for p in config['cran-after-bioc'])) env.safe_append(out_file, std2_install) if config.get("github"): dev_install = """ library(devtools) github.pkgs <- c(%s) get_pkg_name <- function(orig) { c(pkg=unlist(strsplit(unlist(strsplit(orig, "/"))[2], "@"))[1], version=unlist(strsplit(orig, ";"))[2], pname=unlist(strsplit(orig, ";"))[1]) } gh_install <- function(name) { install_github(name, upgrade_dependencies=FALSE) } github_installer = repo.installer(NULL, gh_install, get_pkg_name) lapply(github.pkgs, github_installer) """ % (", ".join('"%s"' % p for p in config['github'])) env.safe_append(out_file, dev_install) ================================================ FILE: cloudbio/manifest.py ================================================ """Provide dump of software and libraries installed on CloudBioLinux image. This provides an output YAML file with package details, providing a complete dump of installed software and packages. The YAML output feeds into a BioGems style webpage that provides a more human friendly view of installed packages. The version information provides a reproducible dump of software on a system. """ import os import collections import json import inspect import subprocess import sys from six.moves import urllib import yaml try: import yolk.yolklib import yolk.metadata except ImportError: yolk = None def create(out_dir, tooldir="/usr/local", fetch_remote=False): """Create a manifest in the output directory with installed packages. """ if not os.path.exists(out_dir): os.makedirs(out_dir) #write_debian_pkg_info(out_dir, fetch_remote) write_python_pkg_info(out_dir) write_r_pkg_info(out_dir) #write_brew_pkg_info(out_dir, tooldir) #write_custom_pkg_info(out_dir, tooldir) # ## Custom packages def _get_custom_pkg_info(name, fn): """Retrieve information about the installed package from the install function. """ vals = dict((k, v) for k, v in inspect.getmembers(fn)) code = inspect.getsourcelines(fn) if vals["__name__"] == "decorator": fn = [x for x in fn.func_closure if not isinstance(x.cell_contents, str)][0].cell_contents vals = dict((k, v) for k, v in inspect.getmembers(fn)) code = inspect.getsourcelines(fn) version = "" for line in (l.strip() for l in code[0]): if line.find("version") >= 0 and line.find(" =") > 0: version = line.split()[-1].replace('"', '').replace("'", "") if version: break doc = vals.get("func_doc", "") descr, homepage = "", "" if doc is not None: descr = doc.split("\n")[0] for line in doc.split("\n"): if line.strip().startswith("http"): homepage = line.strip() return {"name": name.replace("install_", ""), "description": descr, "homepage_uri": homepage, "version": version} def write_custom_pkg_info(out_dir, tooldir): custom_names = ["bio_general", "bio_nextgen", "cloudman", "distributed", "java", "python", "phylogeny", "system"] out_file = os.path.join(out_dir, "custom-packages.yaml") if not os.path.exists(out_file): out = {} for modname in custom_names: try: mod = getattr(__import__("cloudbio.custom", globals(), locals(), [modname], -1), modname) except ImportError as msg: # Skip fabric import errors as we transition away from it if "fabric" in str(msg): mod = None else: raise except ValueError: mod = None if mod: for prog in [x for x in dir(mod) if x.startswith("install")]: pkg = _get_custom_pkg_info(prog, getattr(mod, prog)) out[pkg["name"]] = pkg with open(out_file, "w") as out_handle: yaml.safe_dump(out, out_handle, default_flow_style=False, allow_unicode=False) return out_file # ## Homebrew/Linuxbrew packages def write_brew_pkg_info(out_dir, tooldir): """Extract information for packages installed by homebrew/linuxbrew. """ out_file = os.path.join(out_dir, "brew-packages.yaml") if not os.path.exists(out_file): brew_cmd = os.path.join(tooldir, "bin", "brew") if tooldir else None if not brew_cmd or not os.path.exists(brew_cmd): brew_cmd = "brew" try: vout = subprocess.check_output([brew_cmd, "list", "--versions"]).decode() except (OSError, subprocess.CalledProcessError): # brew not installed/used vout = "" out = {} for vstr in vout.split("\n"): if vstr.strip(): parts = vstr.rstrip().split() name = parts[0] v = parts[-1] # remove any revisions from the version v = v.rsplit("_", 1)[0] out[name] = {"name": name, "version": v} with open(out_file, "w") as out_handle: yaml.safe_dump(out, out_handle, default_flow_style=False, allow_unicode=False) return out_file # ## R packages def get_r_pkg_info(): r_command = ("options(width=10000); subset(installed.packages(fields=c('Title', 'URL')), " "select=c('Version', 'Title','URL'))") try: out = subprocess.check_output(["Rscript", "-e", r_command]).decode() except (subprocess.CalledProcessError, OSError): out = "" pkg_raw_list = [] for line in out.split("\n")[1:]: pkg_raw_list.append(list(filter(None, [entry.strip(' ') for entry in line.split('"')]))) for pkg in pkg_raw_list: if len(pkg) > 2: yield {"name": pkg[0], "version": pkg[1], "description": pkg[2], "homepage_uri": (pkg[3], '')[pkg[3] == 'NA'] if len(pkg) > 3 else ""} def write_r_pkg_info(out_dir): out_file = os.path.join(out_dir, "r-packages.yaml") if not os.path.exists(out_file): out = {} for pkg in get_r_pkg_info(): out[pkg["name"]] = pkg with open(out_file, "w") as out_handle: yaml.safe_dump(out, out_handle, default_flow_style=False, allow_unicode=False) return out_file # ## Python packages def _get_conda_envs(conda_bin): info = json.loads(subprocess.check_output("{conda_bin} info --envs --json".format(**locals()), shell=True)) prefix = info["conda_prefix"] + "/envs/" return [e.replace(prefix, "") for e in info["envs"] if e.startswith(prefix)] def get_python_pkg_info(): if yolk: for dist in yolk.yolklib.Distributions().get_packages("all"): md = yolk.metadata.get_metadata(dist) yield {"name": md["Name"].lower(), "version": md["Version"], "description": md.get("Summary", ""), "homepage_uri": md.get("Home-page", "")} else: base_dir = os.path.dirname(os.path.realpath(sys.executable)) conda_bin = os.path.join(base_dir, "conda") if os.path.exists(conda_bin): for line in subprocess.check_output([conda_bin, "list"]).decode().split("\n"): if line.strip() and not line.startswith("#"): name, version = line.split()[:2] yield {"name": name.lower(), "version": version} for env in _get_conda_envs(conda_bin): for line in subprocess.check_output([conda_bin, "list", "-n", env]).decode().split("\n"): if line.strip() and not line.startswith("#"): name, version = line.split()[:2] yield {"name": name.lower(), "version": version} else: for line in subprocess.check_output([os.path.join(base_dir, "pip"), "list"]).decode().split("\n"): if line.strip() and not line.startswith("#"): name, version = line.split()[:2] yield {"name": name.lower(), "version": version[1:-1]} def _resolve_latest_pkg(pkgs): if len(pkgs) == 1 or not yolk: return pkgs[0] else: latest_version = yolk.yolklib.Distributions().get_highest_installed(pkgs[0]["name"]) return [x for x in pkgs if x["version"] == latest_version][0] def write_python_pkg_info(out_dir): out_file = os.path.join(out_dir, "python-packages.yaml") if not os.path.exists(out_file): pkgs_by_name = collections.defaultdict(list) for pkg in get_python_pkg_info(): pkgs_by_name[pkg["name"]].append(pkg) out = {} for name in sorted(pkgs_by_name.keys()): out[name] = _resolve_latest_pkg(pkgs_by_name[name]) with open(out_file, "w") as out_handle: yaml.safe_dump(out, out_handle, default_flow_style=False, allow_unicode=False) return out_file # ## Debian packages def _get_pkg_popcon(): """Retrieve popularity information for debian packages. """ url = "http://popcon.debian.org/by_vote" popcon = {} for line in (l for l in urllib.request.urlopen(url) if not l.startswith(("#", "--"))): parts = line.split() popcon[parts[1]] = int(parts[3]) return popcon def get_debian_pkg_info(fetch_remote=False): pkg_popcon = _get_pkg_popcon() if fetch_remote else {} cmd = ("dpkg-query --show --showformat " "'${Status}\t${Package}\t${Version}\t${Section}\t${Homepage}\t${binary:Summary}\n'") for pkg_line in [l for l in subprocess.check_output(cmd, shell=True).decode(errors="replace").split("\n") if l.startswith("install ok")]: parts = pkg_line.rstrip("\n").split("\t") if len(parts) > 5: pkg = {"name": parts[1], "version": parts[2], "section": parts[3], "homepage_uri": parts[4], "description": parts[5]} if pkg_popcon.get(pkg["name"]): pkg["downloads"] = pkg_popcon.get(pkg["name"], 0) yield pkg def write_debian_pkg_info(out_dir, fetch_remote=False): base_sections = set(["gnome", "admin", "utils", "web", "games", "sound", "devel", "kde", "x11", "net", "text", "graphics", "misc", "editors", "fonts", "doc", "mail", "otherosfs", "video", "kernel", "libs", "libdevel", "comm", "metapackages", "tex", "introspection"]) for s in list(base_sections): base_sections.add("universe/%s" % s) base_sections.add("partner/%s" % s) out_file = os.path.join(out_dir, "debian-packages.yaml") out_base_file = os.path.join(out_dir, "debian-base-packages.yaml") try: subprocess.check_call(["dpkg", "--help"], stdout=subprocess.PIPE) has_dpkg = True except (subprocess.CalledProcessError, OSError): has_dpkg = False if has_dpkg and (not os.path.exists(out_file) or not os.path.exists(out_base_file)): out = {} out_base = {} for pkg in get_debian_pkg_info(fetch_remote): if pkg.get("section") in base_sections: out_base[pkg["name"]] = pkg else: out[pkg["name"]] = pkg with open(out_file, "w") as out_handle: yaml.safe_dump(out, out_handle, default_flow_style=False, allow_unicode=False) with open(out_base_file, "w") as out_handle: yaml.safe_dump(out_base, out_handle, default_flow_style=False, allow_unicode=False) return out_file ================================================ FILE: cloudbio/package/__init__.py ================================================ """Install software and configure package managers. """ from __future__ import print_function import os def _configure_and_install_native_packages(env, pkg_install): """ Setups up native package repositories, determines list of native packages to install, and installs them. """ from fabric.api import env from cloudbio.package import brew from cloudbio.package.deb import (_apt_packages, _add_apt_gpg_keys, _setup_apt_automation, _setup_apt_sources) from cloudbio.package.rpm import (_yum_packages, _setup_yum_bashrc, _setup_yum_sources) home_dir = env.safe_run("echo $HOME") if home_dir: if env.shell_config.startswith("~"): nonhome = env.shell_config.split("~/", 1)[-1] env.shell_config = os.path.join(home_dir, nonhome) if env.distribution in ["debian", "ubuntu"]: _setup_apt_sources() _setup_apt_automation() _add_apt_gpg_keys() _apt_packages(pkg_install) elif env.distribution in ["centos", "scientificlinux"]: _setup_yum_sources() _yum_packages(pkg_install) _setup_yum_bashrc() elif env.distribution in ["arch", "suse"]: pass # No package support for Arch, SUSE yet elif env.distribution == "macosx": brew.install_packages(env, pkg_install) else: raise NotImplementedError("Unknown target distribution") def _connect_native_packages(env, pkg_install, lib_install): """Connect native installed packages to local versions. This helps setup a non-sudo environment to handle software that needs a local version in our non-root directory tree. """ from fabric.api import env bin_dir = os.path.join(env.system_install, "bin") exports = _get_shell_exports(env) path = env.safe_run_output("echo $PATH") comment_line = "# CloudBioLinux PATH updates" if not env.safe_contains(env.shell_config, comment_line): env.safe_append(env.shell_config, "\n" + comment_line) if bin_dir not in path and env.safe_exists(env.shell_config): if not env.safe_contains(env.shell_config, exports["path"]): env.safe_append(env.shell_config, exports["path"]) if "python" in pkg_install and "python" in lib_install: _create_local_virtualenv(env.system_install) def _get_shell_exports(env): bin_dir = os.path.join(env.system_install, "bin") return {"path": "export PATH=%s:$PATH" % bin_dir} def _print_shell_exports(env): """Print a set of exports to add to shell in isolated installations. """ exports = _get_shell_exports(env) print("\nIsolated installation: add the following to your shell to include installed files:") for e in ["path"]: print(exports[e]) def _create_local_virtualenv(target_dir): """Create virtualenv in target directory for non-sudo installs. """ from fabric.api import cd from cloudbio.custom.shared import _make_tmp_dir url = "https://raw.github.com/pypa/virtualenv/master/virtualenv.py" if not os.path.exists(os.path.join(target_dir, "bin", "python")): with _make_tmp_dir() as work_dir: with cd(work_dir): env.safe_run("wget --no-check-certificate %s" % url) env.safe_run("python virtualenv.py %s" % target_dir) ================================================ FILE: cloudbio/package/brew.py ================================================ """Install packages via the MacOSX Homebrew and Linux Linuxbrew package manager. https://github.com/mxcl/homebrew https://github.com/Homebrew/linuxbrew """ from __future__ import print_function import contextlib from distutils.version import LooseVersion import os import sys import yaml from cloudbio.custom import system, shared from cloudbio.flavor.config import get_config_file from cloudbio.fabutils import quiet, find_cmd from cloudbio.package import cpan from cloudbio.package.shared import _yaml_to_packages from fabric.api import cd, settings BOTTLE_URL = "https://s3.amazonaws.com/cloudbiolinux/brew_bottles/{pkg}-{version}.x86_64-linux.bottle.tar.gz" BOTTLE_SUPPORTED = set(["isaac-aligner", "isaac-variant-caller", "cmake"]) def install_packages(env, to_install=None, packages=None): """Install packages using the home brew package manager. Handles upgrading brew, tapping required repositories and installing or upgrading packages as appropriate. `to_install` is a CloudBioLinux compatible set of top level items to add, alternatively `packages` is a list of raw package names. """ config_file = get_config_file(env, "packages-homebrew.yaml") if to_install: (packages, _) = _yaml_to_packages(config_file.base, to_install, config_file.dist) # if we have no packages to install, do not try to install or update brew if len(packages) == 0: _remove_old(env, config_file.base) return system.install_homebrew(env) brew_cmd = _brew_cmd(env) formula_repos = ["homebrew/science", "chapmanb/cbl", "homebrew/dupes"] current_taps = set([x.strip() for x in env.safe_run_output("%s tap" % brew_cmd).split()]) _safe_update(env, brew_cmd, formula_repos, current_taps) current_taps = set([x.strip() for x in env.safe_run_output("%s tap" % brew_cmd).split()]) for repo in formula_repos: if repo not in current_taps: env.safe_run("%s tap %s" % (brew_cmd, repo)) env.safe_run("%s tap --repair" % brew_cmd) ipkgs = {"outdated": set([x.strip() for x in env.safe_run_output("%s outdated" % brew_cmd).split()]), "current": _get_current_pkgs(env, brew_cmd)} _install_brew_baseline(env, brew_cmd, ipkgs, packages) ipkgs = {"outdated": set([x.strip() for x in env.safe_run_output("%s outdated" % brew_cmd).split()]), "current": _get_current_pkgs(env, brew_cmd)} for pkg_str in packages: _install_pkg(env, pkg_str, brew_cmd, ipkgs) for pkg_str in ["pkg-config", "openssl", "cmake", "unzip"]: _safe_unlink_pkg(env, pkg_str, brew_cmd) with open(config_file.base) as in_handle: to_remove = yaml.safe_load(in_handle).get("to_remove", []) for pkg_str in ["curl"] + to_remove: _safe_uninstall_pkg(env, pkg_str, brew_cmd) def _remove_old(env, config_file): """Temporary approach to remove an old brew install migrated to conda packages. """ brew_cmd = os.path.join(env.system_install, "bin", "brew") if env.safe_exists(brew_cmd): baseline = ["pkg-config", "openssl", "cmake", "unzip", "curl"] with open(config_file) as in_handle: to_remove = yaml.safe_load(in_handle).get("to_remove", []) for pkg_str in baseline + to_remove: _safe_uninstall_pkg(env, pkg_str, brew_cmd) def _safe_update(env, brew_cmd, formula_repos, cur_taps): """Revert any taps if we fail to update due to local changes. """ with _git_stash(env, brew_cmd): with quiet(): with settings(warn_only=True): out = env.safe_run("%s update" % brew_cmd) if out.failed: for repo in formula_repos: if repo in cur_taps: env.safe_run("%s untap %s" % (brew_cmd, repo)) with settings(warn_only=True): out = env.safe_run("%s update" % brew_cmd) if out.failed: print("\n\nHomebrew update failed.") print("You might need to upgrade git by installing inside bcbio with:") print("'brew install git --env=inherit --ignore-dependences'\n\n") @contextlib.contextmanager def _git_stash(env, brew_cmd): """Perform a safe git stash around an update. This circumvents brews internal stash approach which doesn't work on older versions of git and is sensitive to missing config.emails. """ brew_prefix = env.safe_run_output("{brew_cmd} --prefix".format(**locals())) with cd(brew_prefix): with quiet(): with settings(warn_only=True): env.safe_run("git config user.email 'stash@brew.sh'") check_diff = env.safe_run("git diff --quiet") git_version = env.safe_run_output("git --version").strip().split()[-1] if git_version and LooseVersion(git_version) < LooseVersion("1.7"): if check_diff.return_code > 0: with cd(brew_prefix): with settings(warn_only=True): env.safe_run("git stash --quiet") try: yield None finally: if check_diff.return_code > 0: with cd(brew_prefix): with settings(warn_only=True): env.safe_run("git stash pop --quiet") else: yield None def _get_current_pkgs(env, brew_cmd): out = {} with quiet(): which_out = env.safe_run_output("{brew_cmd} list --versions".format(**locals())) for line in which_out.split("\n"): if line: parts = line.rstrip().split() if len(parts) == 2: pkg, version = line.rstrip().split() if pkg.endswith(":"): pkg = pkg[:-1] out[pkg] = version return out def _safe_unlink_pkg(env, pkg_str, brew_cmd): """Unlink packages which can cause issues with a Linux system. """ with settings(warn_only=True): with quiet(): env.safe_run("{brew_cmd} unlink {pkg_str}".format(**locals())) def _safe_link_pkg(env, pkg_str, brew_cmd): """Link packages required for builds, but not necessarily installed """ with settings(warn_only=True): with quiet(): env.safe_run("{brew_cmd} link --overwrite {pkg_str}".format(**locals())) def _safe_uninstall_pkg(env, pkg_str, brew_cmd): """Uninstall packages which get pulled in even when unlinked by brew. """ with settings(warn_only=True): with quiet(): env.safe_run("{brew_cmd} uninstall {pkg_str}".format(**locals())) def _install_pkg(env, pkg_str, brew_cmd, ipkgs): """Install a specific brew package, handling versioning and existing packages. """ pkg, version, args = _get_pkg_version_args(pkg_str) installed = False if version: _install_pkg_version(env, pkg, args, version, brew_cmd, ipkgs) installed = True if pkg in BOTTLE_SUPPORTED and not env.distribution == "macosx": installed = _install_bottle(env, brew_cmd, pkg, ipkgs) if not installed: _install_pkg_latest(env, pkg, args, brew_cmd, ipkgs) def _install_pkg_version(env, pkg, args, version, brew_cmd, ipkgs): """Install a specific version of a package by retrieving from git history. https://gist.github.com/gcatlin/1847248 Handles both global packages and those installed via specific taps. """ if ipkgs["current"].get(pkg.split("/")[-1]) == version: return if version == "HEAD": args = " ".join(args) brew_install = _get_brew_install_cmd(brew_cmd, env, pkg) env.safe_run("{brew_install} {args} --HEAD {pkg}".format(**locals())) else: raise ValueError("Cannot currently handle installing brew packages by version.") with _git_pkg_version(env, brew_cmd, pkg, version): if pkg.split("/")[-1] in ipkgs["current"]: with settings(warn_only=True): env.safe_run("{brew_cmd} unlink {pkg}".format( brew_cmd=brew_cmd, pkg=pkg.split("/")[-1])) # if we have a more recent version, uninstall that first cur_version_parts = env.safe_run_output("{brew_cmd} list --versions {pkg}".format( brew_cmd=brew_cmd, pkg=pkg.split("/")[-1])).strip().split() if len(cur_version_parts) > 1 and LooseVersion(cur_version_parts[1]) > LooseVersion(version): with settings(warn_only=True): env.safe_run("{brew_cmd} uninstall {pkg}".format(**locals())) env.safe_run("{brew_cmd} install {pkg}".format(**locals())) with settings(warn_only=True): env.safe_run("{brew_cmd} switch {pkg} {version}".format(**locals())) env.safe_run("%s link --overwrite %s" % (brew_cmd, pkg)) @contextlib.contextmanager def _git_pkg_version(env, brew_cmd, pkg, version): """Convert homebrew Git to previous revision to install a specific package version. """ git_cmd = _git_cmd_for_pkg_version(env, brew_cmd, pkg, version) git_fname = git_cmd.split()[-1] brew_prefix = env.safe_run_output("{brew_cmd} --prefix".format(**locals())) if git_fname.startswith("{brew_prefix}/Library/Taps/".format(**locals())): brew_prefix = os.path.dirname(git_fname) try: with cd(brew_prefix): if version != "HEAD": env.safe_run(git_cmd) yield finally: # reset Git back to latest with cd(brew_prefix): if version != "HEAD": cmd_parts = git_cmd.split() env.safe_run("%s reset HEAD %s" % (cmd_parts[0], cmd_parts[-1])) cmd_parts[2] = "--" env.safe_run(" ".join(cmd_parts)) def _git_cmd_for_pkg_version(env, brew_cmd, pkg, version): """Retrieve git command to check out a specific version from homebrew. """ git_cmd = None for git_line in env.safe_run_output("{brew_cmd} versions {pkg}".format(**locals())).split("\n"): if git_line.startswith(version): git_cmd = " ".join(git_line.rstrip().split()[1:]) break if git_cmd is None: raise ValueError("Did not find version %s for %s" % (version, pkg)) return git_cmd def _latest_pkg_version(env, brew_cmd, pkg, devel=False): """Retrieve the latest available version of a package and if it is linked. """ i = 0 version, is_linked = None, False with settings(warn_only=True): info_str = env.safe_run_output("{brew_cmd} info {pkg}".format(**locals())) for i, git_line in enumerate(info_str.split("\n")): if git_line.strip(): if i == 0: _, version_str = git_line.split(":") versions = version_str.split(",") if devel: dev_strs = [x for x in versions if x.strip().startswith("devel")] version = dev_strs[0].split()[-1].strip() else: version = versions[0].replace("(bottled)", "").split()[-1].strip() elif git_line.find("Cellar/%s" % pkg) > 0 and git_line.find(" files,") > 0: is_linked = git_line.strip().split()[-1] == "*" return version, is_linked def _get_brew_install_cmd(brew_cmd, env, pkg): perl_setup = "export PERL5LIB=%s/lib/perl5:${PERL5LIB}" % env.system_install compiler_setup = "export CC=${CC:-`which gcc`} && export CXX=${CXX:-`which g++`}" shell_setup = "export SHELL=${SHELL:-/bin/bash}" extra_args = "" if pkg in ["cmake"]: extra_args += " --without-docs" if pkg in ["lumpy-sv", "bamtools", "freebayes", "git"]: extra_args += " --ignore-dependencies" return "%s && %s && %s && %s install --env=inherit %s" % (compiler_setup, shell_setup, perl_setup, brew_cmd, extra_args) def _install_pkg_latest(env, pkg, args, brew_cmd, ipkgs): """Install the latest version of the given package. """ short_pkg = pkg.split("/")[-1] do_install = True is_linked = True remove_old = False if pkg in ipkgs["outdated"] or short_pkg in ipkgs["outdated"]: remove_old = True elif pkg in ipkgs["current"] or short_pkg in ipkgs["current"]: do_install = False pkg_version, is_linked = _latest_pkg_version(env, brew_cmd, pkg, devel="--devel" in args) cur_version = ipkgs["current"].get(pkg, ipkgs["current"][short_pkg]) if cur_version != pkg_version and cur_version.split("_")[0] != pkg_version: remove_old = True do_install = True if do_install: if remove_old: env.safe_run("{brew_cmd} remove --force {short_pkg}".format(**locals())) flags = " ".join(args) with settings(warn_only=True): cmd = "%s %s %s" % (_get_brew_install_cmd(brew_cmd, env, pkg), flags, pkg) with _custom_unlink(env, brew_cmd, pkg): result = env.safe_run_output(cmd) if result.failed and not result.find("Could not symlink") > 0: sys.tracebacklimit = 1 raise ValueError("Failed to install brew formula: %s\n" % pkg + "To debug, please try re-running the install command with verbose output:\n" + cmd.replace("brew install", "brew install -v")) env.safe_run("%s link --overwrite %s" % (brew_cmd, pkg)) # installed but not linked elif not is_linked: env.safe_run("%s link --overwrite %s" % (brew_cmd, pkg)) @contextlib.contextmanager def _custom_unlink(env, brew_cmd, pkg): """Handle custom unlinking of packages that can break builds of others. Does a temporary unlink and relink of packages while building. """ unlinks = {"lumpy-sv": ["bamtools"]} for upkg in unlinks.get(pkg, []): _safe_unlink_pkg(env, upkg, brew_cmd) try: yield None finally: for upkg in unlinks.get(pkg, []): with settings(warn_only=True): with quiet(): env.safe_run("%s link --overwrite %s" % (brew_cmd, upkg)) def _get_pkg_version_args(pkg_str): """Uses Python style package==0.1 version specifications and args separated with ';' """ arg_parts = pkg_str.split(";") if len(arg_parts) == 1: args = [] else: pkg_str = arg_parts[0] args = arg_parts[1:] parts = pkg_str.split("==") if len(parts) == 1: return parts[0], None, args else: assert len(parts) == 2 name, version = parts return name, version, args def _install_bottle(env, brew_cmd, pkg, ipkgs): """Install Linux bottles for brew packages that can be tricky to build. """ if env.distribution == "macosx": # Only Linux bottles, build away on Mac return False pkg_version, is_linked = _latest_pkg_version(env, brew_cmd, pkg) install_version = ipkgs["current"].get(pkg) if pkg_version == install_version: # Up to date if not is_linked: env.safe_run("%s link --overwrite %s" % (brew_cmd, pkg)) return True elif install_version or pkg in ipkgs["outdated"]: env.safe_run("{brew_cmd} remove --force {pkg}".format(**locals())) url = BOTTLE_URL.format(pkg=pkg, version=pkg_version) brew_cachedir = env.safe_run_output("%s --cache" % brew_cmd) brew_cellar = os.path.join(env.safe_run_output("%s --prefix" % brew_cmd), "Cellar") with quiet(): env.safe_run("mkdir -p %s" % brew_cellar) out_file = os.path.join(brew_cachedir, os.path.basename(url)) if env.safe_exists(out_file): env.safe_run("rm -f %s" % out_file) bottle_file = shared._remote_fetch(env, url, out_file=out_file, allow_fail=True, samedir=True) if bottle_file: with cd(brew_cellar): env.safe_run("tar -xf %s" % bottle_file) env.safe_run("%s link --overwrite %s" % (brew_cmd, pkg)) return True else: return False def _install_brew_baseline(env, brew_cmd, ipkgs, packages): """Install baseline brew components not handled by dependency system. - Installation of required Perl libraries. - Upgrades any package dependencies """ for dep in ["openssl"]: _safe_link_pkg(env, dep, brew_cmd) for dep in ["expat", "pkg-config", "xz", "unzip"]: _install_pkg(env, dep, brew_cmd, ipkgs) # check if we have an older git and need to install it from brew git_version = None with quiet(): with settings(warn_only=True): git_version = env.safe_run_output("git --version").strip().split()[-1] if git_version and LooseVersion(git_version) < LooseVersion("1.7"): _install_pkg(env, "git", brew_cmd, ipkgs) for dep in ["sambamba"]: # Avoid conflict with homebrew-science sambamba env.safe_run("{brew_cmd} remove --force {dep}".format(**locals())) for dependency in ["htslib"]: if dependency in packages: if (dependency in ipkgs["outdated"] or "chapmanb/cbl/%s" % dependency in ipkgs["outdated"] or dependency not in ipkgs["current"]): _install_pkg_latest(env, dependency, [], brew_cmd, ipkgs) if "cpanminus" in packages: _install_pkg_latest(env, "cpanminus", [], brew_cmd, ipkgs) _install_pkg_latest(env, "samtools-library-0.1", [], brew_cmd, ipkgs) cpan.install_packages(env) # Ensure paths we may have missed on install are accessible to regular user if env.use_sudo: paths = ["share", "share/java"] for path in paths: with quiet(): test_access = env.safe_run("test -d %s/%s && test -O %s/%s" % (env.system_install, path, env.system_install, path)) if test_access.failed and env.safe_exists("%s/%s" % (env.system_install, path)): env.safe_sudo("chown %s %s/%s" % (env.user, env.system_install, path)) def _brew_cmd(env): """Retrieve brew command for installing homebrew packages. """ cmd = find_cmd(env, "brew", "--version") if cmd is None: raise ValueError("Did not find working installation of Linuxbrew/Homebrew. " "Please check if you have ruby available.") else: return cmd ================================================ FILE: cloudbio/package/conda.py ================================================ """Install packages via the Conda package manager: http://conda.pydata.org/ """ from __future__ import print_function import collections import json import os import shutil import subprocess import yaml from cloudbio.package.shared import _yaml_to_packages ENV_PY_VERSIONS = collections.defaultdict(lambda: "python=3") ENV_PY_VERSIONS[None] = "python=3" ENV_PY_VERSIONS["python2"] = "python=2" ENV_PY_VERSIONS["python3.6"] = "python=3.6" ENV_PY_VERSIONS["dv"] = "python=3.6" ENV_PY_VERSIONS["samtools0"] = "python=3" ENV_PY_VERSIONS["r35"] = "python=3" ENV_PY_VERSIONS["rbcbiornaseq"] = "python=3" ENV_PY_VERSIONS["htslib1.9"] = "python=3" ENV_PY_VERSIONS["htslib1.10"] = "python=3" ENV_PY_VERSIONS["htslib1.11"] = "python=3" ENV_PY_VERSIONS["htslib1.12"] = "python=3" ENV_PY_VERSIONS["htslib1.12_py3.9"] = "python=3.9" ENV_PY_VERSIONS["java"] = "python=3" ENV_PY_VERSIONS["bwakit"] = "python=3" def install_packages(env, to_install=None, packages=None): """Old installation, based on pre-configured fabric inputs. """ from cloudbio.flavor.config import get_config_file from cloudbio.custom import shared if shared._is_anaconda(env): conda_bin = shared._conda_cmd(env) if hasattr(env, "conda_yaml"): Config = collections.namedtuple("Config", "base dist") config_file = Config(base=env.conda_yaml, dist=None) else: config_file = get_config_file(env, "packages-conda.yaml") install_in(conda_bin, env.system_install, config_file.base, packages) def _install_env_pkgs(env_name, env_packages, conda_bin, conda_envs, channels): """Install packages into the given environment. Uses mamba for initial install for speed, following by conda for completeness. TODO: currently duplicates mamba base code in _initial_base_install to make it easy to remove or roll back general mamba usage. We can refactor _initial_base_install in favor of this after further testing. conda_bin could refer to mamba """ mamba_bin = os.path.join(os.path.dirname(conda_bin), "mamba") conda_bin = os.path.join(os.path.dirname(mamba_bin), "conda") if env_name: assert env_name in conda_envs, (env_name, conda_envs) env_str = "-n %s" % env_name else: env_str = "" pkgs_str = " ".join(["'%s'" % x for x in sorted(env_packages)]) py_version = ENV_PY_VERSIONS[env_name] if "deepvariant" in env_packages: # Ignore /etc/boto.cfg which creates conflicts with conda gsutils # https://github.com/GoogleCloudPlatform/gsutil/issues/516 exports = "export BOTO_CONFIG=/ignoreglobal && " else: exports = "" if os.path.exists(mamba_bin): try: subprocess.check_call("{mamba_bin} install -q -y {env_str} {channels} " "{py_version} {pkgs_str}".format(**locals()), shell=True) except subprocess.CalledProcessError: # Fall back to standard conda install when we have system specific issues # https://github.com/bcbio/bcbio-nextgen/issues/2871 subprocess.check_call("{exports}{conda_bin} install -q -y {env_str} {channels} " "{py_version} {pkgs_str}".format(**locals()), shell=True) conda_pkg_list = json.loads(subprocess.check_output( "{conda_bin} list --json {env_str} -q".format(**locals()), shell=True)) return conda_pkg_list def install_in(conda_bin, system_installdir, config_file=None, packages=None): """Install packages inside a given anaconda directory. New approach, local only and not dependent on fabric. conda_bin could refer to mamba """ if config_file is None and packages is None: packages = [] check_channels = [] else: (packages, _) = _yaml_to_packages(config_file) with open(config_file) as in_handle: check_channels = yaml.safe_load(in_handle).get("channels", []) channels = " ".join(["-c %s" % x for x in check_channels]) conda_envs = _create_environments(conda_bin, packages) for env_dir in conda_envs.values(): _clean_environment(env_dir) conda_info = json.loads(subprocess.check_output("{conda_bin} info --json -q".format(**locals()), shell=True)) # Uninstall old R packages that clash with updated versions # Temporary fix to allow upgrades from older versions that have migrated # r-tximport is now bioconductor-tximport # py2cairo is incompatible with r 3.4.1+ problems = ["r-tximport", "py2cairo"] # Add packages migrated into separate environments, like python2 for env_name, env_packages in _split_by_condaenv(packages): if env_name: problems += env_packages if problems: print("Checking for problematic or migrated packages in default environment") cur_packages = [x["name"] for x in json.loads(subprocess.check_output("%s list --json -q" % (conda_bin), shell=True)) if x["name"] in problems and x["channel"] in check_channels] if cur_packages: print("Found packages that moved from default environment: %s" % ", ".join(cur_packages)) problems = " ".join(cur_packages) subprocess.check_call("{conda_bin} remove {channels} -y {problems}".format(**locals()), shell=True) _initial_base_install(conda_bin, [ps for (n, ps) in _split_by_condaenv(packages) if n is None][0], check_channels) # install our customized packages # don't link samtools from python3.6, use samtools from anaconda/bin # don't link tabix and bgzip dont_link = ["samtools", "htslib"] if len(packages) > 0: for env_name, env_packages in _split_by_condaenv(packages): print("# Installing into conda environment %s: %s" % (env_name or "default", ", ".join(env_packages))) conda_pkg_list = _install_env_pkgs(env_name, env_packages, conda_bin, conda_envs, channels) for package in env_packages: package_base = package.split("=")[0].split(">")[0] if not package_base in dont_link: _link_bin(package, system_installdir, conda_info, conda_bin, conda_pkg_list, conda_envdir=conda_envs.get(env_name)) conda_pkg_list = json.loads(subprocess.check_output("{conda_bin} list --json -q".format(**locals()), shell=True)) for pkg in ["python", "conda", "pip"]: _link_bin(pkg, system_installdir, conda_info, conda_bin, conda_pkg_list, files=[pkg], prefix="bcbio_") # link packages which avoided _link_bin exception_packages = {"python2": "python2", "vt": "python3.6"} for package in exception_packages: env_name = exception_packages[package] source_file = os.path.realpath(os.path.join(conda_envs.get(env_name), "bin", package)) # not realpath, because realpath would go to the source_file if exists dest_file = os.path.join(system_installdir, "bin", package) _do_link(source_file, dest_file) def _initial_base_install(conda_bin, env_packages, check_channels): """Provide a faster initial installation of base packages, avoiding dependency issues. Uses mamba (https://github.com/QuantStack/mamba) to provide quicker package resolution and avoid dependency conflicts with base install environment. Bootstraps the initial installation of all tools when key inputs that cause conflicts are missing. TODO: we could remove mamba package running code here in favor of _install_env_pkgs general mamba usage once that is further tested. """ initial_package_targets = {None: ["r-base"]} env_name = None env_str = "" channels = " ".join(["-c %s" % x for x in check_channels]) cur_ps = [x["name"] for x in json.loads(subprocess.check_output("{conda_bin} list --json {env_str} -q".format(**locals()), shell=True)) if x["channel"] in check_channels] have_package_targets = env_name in initial_package_targets and any([p for p in cur_ps if p in initial_package_targets[env_name]]) if not have_package_targets: print("Initalling initial set of packages for %s environment with mamba" % (env_name or "default")) py_version = ENV_PY_VERSIONS[env_name] pkgs_str = " ".join(["'%s'" % x for x in sorted(env_packages)]) if "mamba" not in cur_ps: subprocess.check_call("{conda_bin} install -y {env_str} {channels} " "{py_version} mamba".format(**locals()), shell=True) mamba_bin = os.path.join(os.path.dirname(conda_bin), "mamba") pkgs_str = " ".join(["'%s'" % x for x in sorted(env_packages)]) # Skip in favor of _install_env_pkgs, can be removed later after testing if False: try: subprocess.check_call("{mamba_bin} install -y {env_str} {channels} " "{py_version} {pkgs_str}".format(**locals()), shell=True) except subprocess.CalledProcessError: # Fall back to standard conda install when we have system specific issues # https://github.com/bcbio/bcbio-nextgen/issues/2871 pass def _link_bin(package, system_installdir, conda_info, conda_bin, conda_pkg_list, files=None, prefix="", conda_env=None, conda_envdir=None): """Link files installed in the bin directory into the install directory. This is imperfect but we're trying not to require injecting everything in the anaconda directory into a user's path. """ package = package.split("=")[0].split(">")[0] final_bindir = os.path.join(system_installdir, "bin") if conda_envdir: base_bindir = os.path.join(conda_envdir, "bin") else: base_bindir = os.path.dirname(conda_bin) # resolve any symlinks in the final and base heirarchies final_bindir = subprocess.check_output("cd %s && pwd -P" % final_bindir, shell=True).decode().strip() base_bindir = subprocess.check_output("cd %s && pwd -P" % base_bindir, shell=True).decode().strip() for pkg_subdir in [x for x in conda_pkg_list if x["name"] == package]: pkg_subdir = pkg_subdir["dist_name"].split("::")[-1] for pkg_dir in conda_info["pkgs_dirs"]: pkg_bindir = os.path.join(os.path.realpath(pkg_dir), pkg_subdir, "bin") python_bindir = os.path.join(os.path.dirname(pkg_bindir), "python-scripts") if (os.path.commonprefix([pkg_bindir, base_bindir]).find("anaconda") > 0 and (os.path.exists(python_bindir) or os.path.exists(pkg_bindir))): if not files: if not os.path.exists(python_bindir): python_bindir = "" if not os.path.exists(pkg_bindir): pkg_bindir = "" files = subprocess.check_output("ls -1 {pkg_bindir} {python_bindir}" .format(**locals()), shell=True).decode().split() for fname in files: # symlink to the original file in the /anaconda/bin directory # this could be a hard or soft link base_fname = os.path.join(base_bindir, fname) if os.path.exists(base_fname) and os.path.lexists(base_fname): _do_link(base_fname, os.path.join(final_bindir, "%s%s" % (prefix, fname))) def _do_link(orig_file, final_file): """Perform a soft link of the original file into the final location. We need the symlink to point to /anaconda/bin directory, not the real location in the pkgs directory so conda can resolve LD_LIBRARY_PATH and the interpreters. """ needs_link = True # working symlink, check if already in the right place or remove it if os.path.exists(final_file): if (os.path.realpath(final_file) == os.path.realpath(orig_file) and orig_file == os.path.normpath(os.path.join(os.path.dirname(final_file), os.readlink(final_file)))): needs_link = False else: os.remove(final_file) # broken symlink elif os.path.lexists(final_file): os.unlink(final_file) if needs_link: os.symlink(os.path.relpath(orig_file, os.path.dirname(final_file)), final_file) def _split_by_condaenv(packages): """Split packages into those requiring special conda environments. """ out = collections.defaultdict(list) envs = set([]) for p in packages: parts = p.split(";") name = parts[0] metadata = parts[1:] condaenv = None for k, v in [x.split("=") for x in metadata]: if k == "env": condaenv = v envs.add(condaenv) out[condaenv].append(name) envs = [None] + sorted([x for x in list(envs) if x]) return [(e, out[e]) for e in envs] def _get_conda_envs(conda_bin): info = json.loads(subprocess.check_output("{conda_bin} info --envs --json -q".format(**locals()), shell=True)) return [e for e in info["envs"] if e.startswith(info["conda_prefix"])] def _create_environments(conda_bin, packages): """Creates custom local environments that conflict with global dependencies. Available environments: - python2 -- tools that require python 2 and are not compatible with python3. The goal is to move all other installs into a default python 3 base environment. - python3 -- support tools that require python 3. This will get deprecated but for and removed as we move to an all python3 install. For now it collects tools that require 3 or some other specific requirements. - samtools0 -- For tools that require older samtools 0.1.19 - dv -- DeepVariant, which requires a specific version of numpy and tensorflow - r36 -- R3.6 for PureCN - htslib1.10 -- htslib 1.10 for mosdepth and other packages that require it until bioconda switches off of 1.9. - htslib1.11 -- htslib 1.11 for scramble until bioconda switched off of 1.9 - bwakit -- requires a specific (old) version of samblaster """ env_names = set([e for e, ps in _split_by_condaenv(packages) if e]) out = {} conda_envs = _get_conda_envs(conda_bin) for addenv in ["python3.6", "samtools0", "dv", "python2", "r35", "htslib1.9", "htslib1.10", "htslib1.11", "htslib1.12", "htslib1.12_py3.9", "bwakit", "java", "rbcbiornaseq"]: if addenv in env_names: if not any(x.endswith("/%s" % addenv) for x in conda_envs): print("Creating conda environment: %s" % addenv) py_version = ENV_PY_VERSIONS[addenv] subprocess.check_call("{conda_bin} create -q --no-default-packages -y --name {addenv} {py_version} nomkl" .format(**locals()), shell=True) conda_envs = _get_conda_envs(conda_bin) out[addenv] = [x for x in conda_envs if x.endswith("/%s" % addenv)][0] return out def _clean_environment(env_dir): """Remove problem elements in environmental directories. - Get rid of old history comment lines that cause parsing failures: https://github.com/bcbio/bcbio-nextgen/issues/2431 """ history_file = os.path.join(env_dir, "conda-meta", "history") if os.path.exists(history_file): has_problem = False cleaned_lines = [] with open(history_file) as in_handle: for line in in_handle: # Remove lines like `# create specs:` which have no information after colon if line.startswith("#") and len([x for x in line.strip().split(":") if x]) == 1: has_problem = True else: cleaned_lines.append(line) if has_problem: shutil.copy(history_file, history_file + ".orig") with open(history_file, "w") as out_handle: for line in cleaned_lines: out_handle.write(line) ================================================ FILE: cloudbio/package/cpan.py ================================================ """Install perl packages using CPAN and cpanminus (cpanm). """ import os from fabric.api import cd, settings from cloudbio.flavor.config import get_config_file from cloudbio.fabutils import find_cmd from cloudbio.package.shared import _yaml_to_packages from cloudbio.custom import shared as cshared def install_packages(env): config_file = get_config_file(env, "perl-libs.yaml") (packages, _) = _yaml_to_packages(config_file.base, subs_yaml_file=config_file.dist, namesort=False) cpanm_cmd = find_cmd(env, "cpanm", "--version") for package in packages: if package.count("==") > 1: _install_from_url(env, cpanm_cmd, package) else: _install_from_cpan(env, cpanm_cmd, package) def _install_from_cpan(env, cpanm_cmd, package): """Install from CPAN using cpanm, handling special arguments. The simplest input string is just a package to install (like XML::Simple) but users can also specify build arguments and exports as additional items separated by ';' """ parts = package.split(";") if len(parts) == 1: perl_lib = parts[0] args = "" exports = [] elif len(parts) == 2: perl_lib, args = parts exports = [] else: perl_lib, args = parts[:2] exports = parts[2:] export_strs = [] for export in exports: export_strs.append("export " + export.format(system_install=env.system_install)) export = " && ".join(export_strs) + " && " if export_strs else "" build_args = ("--build-args='%s'" % args) if args else "" env.safe_run("%s %s -i --notest --local-lib=%s %s '%s'" % (export, cpanm_cmd, env.system_install, build_args, perl_lib)) def _install_from_url(env, cpanm_cmd, package): """Check version of a dependency and download and install with cpanm if not up to date. Packages installed via URL have the package name, target version and URL separated with '=='. They can also optionally have a build directory or dependency to remove. """ parts = package.split("==") package, target_version, url = parts[:3] args = {} if len(parts) > 3: for key, value in (x.split("=") for x in parts[3:]): args[key] = value with settings(warn_only=True): cur_version = env.safe_run_output("export PERL5LIB=%s/lib/perl5:${PERL5LIB} && " % env.system_install + """perl -le 'eval "require $ARGV[0]" and print $ARGV[0]->VERSION' %s""" % package) if cur_version != target_version: with cshared._make_tmp_dir() as work_dir: with cd(work_dir): dl_dir = cshared._fetch_and_unpack(url) if args.get("build"): dl_dir = os.path.join(dl_dir, args["build"]) with cd(dl_dir): if args.get("depremove"): for fname in ["Makefile.PL", "MYMETA.json", "MYMETA.yml"]: env.safe_run(r"""sed -i.bak -e '/^.*%s.*/s/^/#/' %s""" % (args["depremove"], fname)) env.safe_run("%s -i --notest --local-lib=%s ." % (cpanm_cmd, env.system_install)) ================================================ FILE: cloudbio/package/deb.py ================================================ """ Automated installation on debian package systems with apt. """ from fabric.api import * from fabric.contrib.files import * from cloudbio.package.shared import _yaml_to_packages from cloudbio.flavor.config import get_config_file def _apt_packages(to_install=None, pkg_list=None): """ Install packages available via apt-get. Note that ``to_install`` and ``pkg_list`` arguments cannot be used simultaneously. :type to_install: list :param to_install: A list of strings (ie, groups) present in the ``main.yaml`` config file that will be used to filter out the specific packages to be installed. :type pkg_list: list :param pkg_list: An explicit list of packages to install. No other files, flavors are considered. """ if "minimal" not in env.flavor.short_name: env.logger.info("Update the system") with settings(warn_only=True): env.safe_sudo("apt-get update") if to_install is not None: config_file = get_config_file(env, "packages.yaml") if "minimal" not in env.flavor.name and "minimal" not in env.flavor.short_name: env.flavor.apt_upgrade_system(env=env) (packages, _) = _yaml_to_packages(config_file.base, to_install, config_file.dist) packages = env.flavor.rewrite_config_items("packages", packages) elif pkg_list is not None: env.logger.info("Will install specific packages: {0}".format(pkg_list)) packages = pkg_list else: raise ValueError("Need a file with packages or a list of packages") # A single line install is much faster - note that there is a max # for the command line size, so we do 30 at a time group_size = 30 i = 0 env.logger.info("Installing %i packages" % len(packages)) while i < len(packages): env.logger.info("Package install progress: {0}/{1}".format(i, len(packages))) env.safe_sudo("apt-get -y --force-yes install %s" % " ".join(packages[i:i + group_size])) i += group_size env.safe_sudo("apt-get clean") def _add_apt_gpg_keys(): """Adds GPG keys from all repositories """ env.logger.info("Update GPG keys for repositories") standalone = [ "http://archive.cloudera.com/debian/archive.key", 'http://download.virtualbox.org/virtualbox/debian/oracle_vbox.asc', 'https://www.postgresql.org/media/keys/ACCC4CF8.asc', ] keyserver = [ ("keyserver.ubuntu.com", "7F0CEB10"), ("keyserver.ubuntu.com", "E084DAB9"), ("keyserver.ubuntu.com", "D018A4CE"), ("keyserver.ubuntu.com", "D67FC6EAE2A11821"), ] standalone, keyserver = env.flavor.rewrite_apt_keys(standalone, keyserver) for key in standalone: with settings(warn_only=True): env.safe_sudo("wget -q -O- %s | apt-key add -" % key) for url, key in keyserver: with settings(warn_only=True): env.safe_sudo("apt-key adv --keyserver %s --recv %s" % (url, key)) with settings(warn_only=True): env.safe_sudo("apt-get update") env.safe_sudo("sudo apt-get install -y --force-yes bio-linux-keyring") def _setup_apt_automation(): """Setup the environment to be fully automated for tricky installs. Sun Java license acceptance: http://www.davidpashley.com/blog/debian/java-license MySQL root password questions; install with empty root password: http://snowulf.com/archives/540-Truly-non-interactive-unattended-apt-get-install.html Postfix, setup for no configuration. See more on issues here: http://www.uluga.ubuntuforums.org/showthread.php?p=9120196 """ interactive_cmd = "export DEBIAN_FRONTEND=noninteractive" if not env.safe_contains(env.shell_config, interactive_cmd): env.safe_append(env.shell_config, interactive_cmd) # Remove interactive checks in .bashrc which prevent # bash customizations env.safe_comment(env.shell_config, "^[ ]+\*\) return;;$") package_info = [ "postfix postfix/not_configured boolean true", "postfix postfix/main_mailer_type select 'No configuration'", "mysql-server-5.1 mysql-server/root_password string '(password omitted)'", "mysql-server-5.1 mysql-server/root_password_again string '(password omitted)'", "mysql-server-5.5 mysql-server/root_password string '(password omitted)'", "mysql-server-5.5 mysql-server/root_password_again string '(password omitted)'", "sun-java6-jdk shared/accepted-sun-dlj-v1-1 select true", "sun-java6-jre shared/accepted-sun-dlj-v1-1 select true", "sun-java6-bin shared/accepted-sun-dlj-v1-1 select true", "grub-pc grub2/linux_cmdline string ''", "grub-pc grub-pc/install_devices_empty boolean true", "acroread acroread/default-viewer boolean false", "rabbitmq-server rabbitmq-server/upgrade_previous note", "condor condor/wantdebconf boolean false", "ttf-mscorefonts-installer msttcorefonts/accepted-mscorefonts-eula boolean true", "ttf-mscorefonts-installer msttcorefonts/present-mscorefonts-eula note", "gdm shared/default-x-display-manager select gdm", "lightdm shared/default-x-display-manager select gdm", "postfix postfix/mailname string notusedexample.org", # Work harder to avoid gdm dialogs # https://bugs.launchpad.net/ubuntu/+source/gdm/+bug/1020770 "debconf debconf/priority select critical" ] package_info = env.flavor.rewrite_apt_automation(package_info) cmd = "" for l in package_info: cmd += 'echo "%s" | /usr/bin/debconf-set-selections;' % l env.safe_sudo(cmd) def _setup_apt_sources(): """Add sources for retrieving library packages. Using add-apt-repository allows processing PPAs (on Ubuntu) This method modifies the apt sources file. Uses python-software-properties, which provides an abstraction of apt repositories """ # It may be sudo is not installed - which has fab fail - therefor # we'll try to install it by default, assuming we have root access # already (e.g. on EC2). Fab will fail anyway, otherwise. if not env.safe_exists('/usr/bin/sudo') or not env.safe_exists('/usr/bin/curl'): env.safe_sudo('apt-get update') env.safe_sudo('apt-get -y --force-yes install sudo curl') env.logger.debug("_setup_apt_sources " + env.sources_file) env.flavor.check_packages_source() comment = "# This file was modified for " + env.flavor.name # Setup apt download policy (default is None) # (see also https://help.ubuntu.com/community/PinningHowto) preferences = env.flavor.rewrite_apt_preferences([]) if len(preferences): # make sure it exists, and is empty env.safe_sudo("rm -f %s" % env.apt_preferences_file) env.safe_sudo("touch %s" % env.apt_preferences_file) env.safe_append(env.apt_preferences_file, comment, use_sudo=True) lines = "\n".join(preferences) env.logger.debug("Policy %s" % lines) # append won't duplicate, so we use echo env.safe_sudo("/bin/echo -e \"%s\" >> %s" % (lines, env.apt_preferences_file)) # check there is no error parsing the file env.logger.debug(env.safe_sudo("apt-cache policy")) # Make sure a source file exists if not env.safe_exists(env.sources_file): env.safe_sudo("touch %s" % env.sources_file) # Add a comment if not env.safe_contains(env.sources_file, comment): env.safe_append(env.sources_file, comment, use_sudo=True) for source in env.flavor.rewrite_apt_sources_list(env.std_sources): env.logger.debug("Source %s" % source) if source.startswith("ppa:"): env.safe_sudo("apt-get install -y --force-yes python-software-properties") env.safe_sudo("add-apt-repository '%s'" % source) elif (not env.safe_contains(env.sources_file, source) and not env.safe_contains(env.global_sources_file, source)): env.safe_append(env.sources_file, source, use_sudo=True) ================================================ FILE: cloudbio/package/nix.py ================================================ """Install software with the Nix package manager. """ from fabric.api import * from fabric.contrib.files import * from cloudbio.package.shared import _yaml_to_packages from cloudbio.flavor.config import get_config_file def _setup_nix_sources(): if env.nixpkgs: target_info = run("uname -a") env.logger.info("Target: "+target_info) # find the target architecture, if not preset if not env.has_key("arch"): env.arch = run("uname -m") # first override the path append("/root/.bashrc", "export PATH=$HOME/.nix-profile/bin:$PATH", use_sudo=True) env.logger.info("Checking NixPkgs") if not exists("/nix/store"): # first time installation if not exists("/usr/bin/nix-env"): # install Nix (standard Debian release) nix_deb = "nix_0.16-1_"+env.arch+".deb" if not exists(nix_deb): # run("wget http://hydra.nixos.org/build/565031/download/1/nix_0.16-1_i386.deb") run("wget http://hydra.nixos.org/build/565048/download/1/"+nix_deb) sudo("dpkg -i "+nix_deb) run("nix-channel --list") if run("nix-channel --list") == "": # Setup channel sudo("nix-channel --add http://nixos.org/releases/nixpkgs/channels/nixpkgs-unstable") sudo("nix-channel --update") # upgrade Nix to latest (and remove the older version, as it is much slower) sudo("nix-env -b -i nix") if exists("/usr/bin/nix-env"): env.logger.info("uninstall older Nix (Debian release)") sudo("dpkg -r nix") def _nix_packages(to_install): """Install packages available via nixpkgs (optional) """ if env.nixpkgs: env.logger.info("Update and install NixPkgs packages") pkg_config_file = get_config_file(env, "packages-nix.yaml").base sudo("nix-channel --update") # Retrieve final package names (packages, _) = _yaml_to_packages(pkg_config_file, to_install) packages = env.flavor.rewrite_config_items("packages", packages) for p in packages: sudo("nix-env -b -i %s" % p) ================================================ FILE: cloudbio/package/rpm.py ================================================ """Automated installation on RPM systems with the yum package manager. """ import itertools from fabric.api import * from fabric.contrib.files import * from cloudbio.package.shared import _yaml_to_packages from cloudbio.flavor.config import get_config_file def _yum_packages(to_install): """Install rpm packages available via yum. """ if env.distribution == "scientificlinux": package_file = "packages-scientificlinux.yaml" else: package_file = "packages-yum.yaml" pkg_config = get_config_file(env, package_file).base with settings(warn_only=True): env.safe_sudo("yum -y update") env.safe_sudo("yum clean all") env.safe_sudo("yum makecache") env.safe_sudo("yum check-update") # Retrieve packages to get and install each of them (packages, _) = _yaml_to_packages(pkg_config, to_install) # At this point allow the Flavor to rewrite the package list packages = env.flavor.rewrite_config_items("packages", packages) for package_group in _partition_all(20, packages): env.safe_sudo("yum -y install %s" % " ".join(package_group)) def _partition_all(n, iterable): """http://stackoverflow.com/questions/5129102/python-equivalent-to-clojures-partition-all """ it = iter(iterable) while True: chunk = list(itertools.islice(it, n)) if not chunk: break yield chunk def _setup_yum_bashrc(): """Fix the user bashrc to update compilers. """ if env.distribution in ["centos"]: to_include = ["export PKG_CONFIG_PATH=${PKG_CONFIG_PATH}:/usr/lib/pkgconfig"] # gcc fixes no longer necessary on recent CentOS versions #"export CC=gcc44", "export CXX=g++44", "export FC=gfortran44", fname = env.safe_run_output("ls %s" % env.shell_config) for line in to_include: if not env.safe_contains(fname, line.split("=")[0]): env.safe_append(fname, line) def _setup_yum_sources(): """Add additional useful yum repositories. """ repos = [ "http://dl.fedoraproject.org/pub/epel/6/x86_64/epel-release-6-8.noarch.rpm", "http://archive.cloudera.com/redhat/6/x86_64/cdh/cdh3-repository-1.0-1.noarch.rpm" ] for repo in repos: with settings(warn_only=True): env.safe_sudo("rpm -Uvh %s" % repo) ================================================ FILE: cloudbio/package/shared.py ================================================ """Shared functionality useful for multiple package managers. """ from __future__ import print_function import yaml def _yaml_to_packages(yaml_file, to_install=None, subs_yaml_file=None, namesort=True, env=None): """Read a list of packages from a nested YAML configuration file. """ print("Reading packages from %s" % yaml_file) with open(yaml_file) as in_handle: full_data = yaml.safe_load(in_handle) if full_data is None: full_data = {} if subs_yaml_file is not None: with open(subs_yaml_file) as in_handle: subs = yaml.safe_load(in_handle) else: subs = {} # filter the data based on what we have configured to install data = [(k, v) for (k, v) in full_data.items() if (to_install is None or k in to_install) and k not in ["channels"]] data.sort() packages = [] pkg_to_group = dict() while len(data) > 0: cur_key, cur_info = data.pop(0) if cur_info: if isinstance(cur_info, (list, tuple)): packages.extend(_filter_subs_packages(cur_info, subs, namesort)) for p in cur_info: pkg_to_group[p] = cur_key elif isinstance(cur_info, dict): for key, val in cur_info.items(): # if we are okay, propagate with the top level key if env and key == 'needs_64bit': if env.is_64bit: data.insert(0, (cur_key, val)) elif env and key.startswith(env.distribution): if key.endswith(env.dist_name): data.insert(0, (cur_key, val)) else: data.insert(0, (cur_key, val)) else: raise ValueError(cur_info) return packages, pkg_to_group def _filter_subs_packages(initial, subs, namesort=True): """Rename and filter package list with subsitutions; for similar systems. """ final = [] for p in initial: try: new_p = subs[p] except KeyError: new_p = p if new_p: final.append(new_p) if namesort: final.sort() return final ================================================ FILE: cloudbio/utils.py ================================================ """Utilities for logging and progress tracking. """ import logging import os import sys from fabric.main import load_settings from fabric.colors import yellow, red, green, magenta from fabric.api import settings, hide, cd, run from fabric.contrib.files import exists from cloudbio.distribution import _setup_distribution_environment from cloudbio.flavor import Flavor from cloudbio.flavor.config import get_config_file class ColorFormatter(logging.Formatter): """ Format log message based on the message level http://stackoverflow.com/questions/1343227/can-pythons-logging-format-be-modified-depending-on-the-message-log-level """ # Setup formatters for each of the levels err_fmt = red("ERR [%(filename)s(%(lineno)d)] %(msg)s") warn_fmt = magenta("WARN [%(filename)s(%(lineno)d)]: %(msg)s") dbg_fmt = yellow("DBG [%(filename)s]: %(msg)s") info_fmt = green("INFO: %(msg)s") def __init__(self, fmt="%(name)s %(levelname)s: %(msg)s"): logging.Formatter.__init__(self, fmt) def format(self, record): # Save the original format configured by the user # when the logger formatter was instantiated format_orig = self._fmt # Replace the original format with one customized by logging level if record.levelno == 10: # DEBUG self._fmt = ColorFormatter.dbg_fmt elif record.levelno == 20: # INFO self._fmt = ColorFormatter.info_fmt elif record.levelno == 30: # WARN self._fmt = ColorFormatter.warn_fmt elif record.levelno == 40: # ERROR self._fmt = ColorFormatter.err_fmt # Call the original formatter class to do the grunt work result = logging.Formatter.format(self, record) # Restore the original format configured by the user self._fmt = format_orig return result def _setup_logging(env): env.logger = logging.getLogger("cloudbiolinux") env.logger.setLevel(logging.DEBUG) ch = logging.StreamHandler() ch.setLevel(logging.DEBUG) # Use custom formatter ch.setFormatter(ColorFormatter()) env.logger.addHandler(ch) def _update_biolinux_log(env, target, flavor): """Updates the VM so it contains information on the latest BioLinux update in /var/log/biolinux.log. The latest information is appended to the file and can be used to see if an installation/update has completed (see also ./test/test_vagrant). """ if not target: target = env.get("target", None) if not target: target = "unknown" else: target = target.name if not flavor: flavor = env.get("flavor", None) if not flavor: flavor = "unknown" else: flavor = flavor.name logfn = "/var/log/biolinux.log" info = "Target="+target+"; Flavor="+flavor env.logger.info(info) if env.use_sudo: env.safe_sudo("date +\"%D %T - Updated "+info+"\" >> "+logfn) def _configure_fabric_environment(env, flavor=None, fabricrc_loader=None, ignore_distcheck=False): if not fabricrc_loader: fabricrc_loader = _parse_fabricrc _setup_flavor(env, flavor) fabricrc_loader(env) # get parameters for distro, packages etc. _setup_distribution_environment(ignore_distcheck=ignore_distcheck) _create_local_paths(env) def _setup_flavor(env, flavor): """Setup a flavor, providing customization hooks to modify CloudBioLinux installs. Specify flavor as a name, in which case we look it up in the standard flavor directory (contrib/flavor/your_flavor), or as an absolute path to a flavor directory outside of cloudbiolinux. """ env.flavor = Flavor(env) env.flavor_dir = None if flavor: # set the directory for flavor customizations if os.path.isabs(flavor) or os.path.exists(flavor): flavor_dir = flavor else: flavor_dir = os.path.join(os.path.dirname(__file__), '..', 'contrib', 'flavor', flavor) assert os.path.exists(flavor_dir), \ "Did not find directory {0} for flavor {1}".format(flavor_dir, flavor) env.flavor_dir = flavor_dir flavor_name = os.path.split(flavor_dir)[-1] # Reinstantiate class if custom defined import cloudbio.flavor try: env.flavor = getattr(cloudbio.flavor, flavor_name.capitalize())(env) except AttributeError: pass env.flavor.name = flavor_name # Load python customizations to base configuration if present for ext in ["", "flavor"]: py_flavor = os.path.split(os.path.realpath(flavor_dir))[1] + ext flavor_custom_py = os.path.join(flavor_dir, "{0}.py".format(py_flavor)) if os.path.exists(flavor_custom_py): sys.path.append(flavor_dir) mod = __import__(py_flavor, fromlist=[py_flavor]) env.logger.info(env.flavor) env.logger.info("This is a %s flavor" % env.flavor.name) def _parse_fabricrc(env): """Defaults from fabricrc.txt file; loaded if not specified at commandline. """ env.config_dir = os.path.join(os.path.dirname(__file__), "..", "config") env.tool_data_table_conf_file = os.path.join(env.config_dir, "..", "installed_files", "tool_data_table_conf.xml") if not env.has_key("distribution") and not env.has_key("system_install"): env.logger.info("Reading default fabricrc.txt") env.update(load_settings(get_config_file(env, "fabricrc.txt").base)) if "shell_config" not in env: env.shell_config = "~/.bashrc" if "shell" not in env: env.shell = "/bin/bash -i -c" def _create_local_paths(env): """Expand any paths defined in terms of shell shortcuts (like ~). """ with settings(hide('warnings', 'running', 'stdout', 'stderr'), warn_only=True): # This is the first point we call into a remote host - make sure # it does not fail silently by calling a dummy run env.logger.info("Now, testing connection to host...") test = env.safe_run("pwd") # If there is a connection failure, the rest of the code is (sometimes) not # reached - for example with Vagrant the program just stops after above run # command. if test != None: env.logger.info("Connection to host appears to work!") else: raise NotImplementedError("Connection to host failed") env.logger.debug("Expand paths") if "local_install" in env: if not env.safe_exists(env.local_install): env.safe_sudo("mkdir -p %s" % env.local_install) user = env.safe_run_output("echo $USER") env.safe_sudo("chown -R %s %s" % (user, env.local_install)) with cd(env.local_install): result = env.safe_run_output("pwd") env.local_install = result ================================================ FILE: config/README.md ================================================ This directory contains details of the software installed with [CloudBioLinux][1]. This is the right place to dig around if you are interested in adding packages to the image, or would like to get an overview of what is installed. The configuration files are written in easily readable [YAML format][2]. * [main.yaml][4] -- High level category view of packages and libraries that are installed with CloudBioLinux. * [packages.yaml][5] -- A full list of operating system packages that are included, organized by category. The names are standard [Ubuntu APT package names][3]. * [custom.yaml][6] -- Installed software that is not included in the standard package repository. These are often specialized biological packages that have not yet been cleanly packaged. Actual installation code is in the `custom` sub-directory. * [python-libs.yaml][7], [r-libs.yaml][8], [perl-libs.yaml][9], [ruby-libs.yaml][10],-- Libraries installed for a number of programming languages. These are installed by the language specific library managers (easy\_install for Python, cpan for Perl, gem for Ruby). [1]: http://cloudbiolinux.com/ [2]: http://en.wikipedia.org/wiki/YAML [3]: https://help.ubuntu.com/community/AptGet/Howto [4]: https://github.com/chapmanb/cloudbiolinux/blob/master/config/main.yaml [5]: https://github.com/chapmanb/cloudbiolinux/blob/master/config/packages.yaml [6]: https://github.com/chapmanb/cloudbiolinux/blob/master/config/custom.yaml [7]: https://github.com/chapmanb/cloudbiolinux/blob/master/config/python-libs.yaml [8]: https://github.com/chapmanb/cloudbiolinux/blob/master/config/r-libs.yaml [9]: https://github.com/chapmanb/cloudbiolinux/blob/master/config/perl-libs.yaml [10]: https://github.com/chapmanb/cloudbiolinux/blob/master/config/ruby-libs.yaml ================================================ FILE: config/biodata.yaml ================================================ --- # Configuration file defining biological data to retrieve and install. # These are stored in an Amazon S3 buckets: # https://s3.amazonaws.com/biodata # https://s3.amazonaws.com/bcbio-nextgen # and retrieved using the data_fabfile Fabric script. # ## Genome data -- Next generation sequencing and Galaxy # Details about the genomes you want to include. # Required genome fields (corresponding to Galaxy's tool_data_table_conf.xml columns): # dbkey - globally unique identifier for the genome (e.g., hg19) # name - descriptive name for the given genome (to be displayed in Galaxy, e.g., Hsapiens) # Optional genome fields (corresponding to Galaxy's tool_data_table_conf.xml columns): # formats, species, dbkey1, dbkey2, value, path, index # Additional genome fields specific to data deployment: # genome_indexes - list of tool indexes specific to the associated genome (overrides global 'genome_indexes') genomes: - dbkey: phix name: phiX174 - dbkey: hg19 name: Human (hg19) indexes: [seq, twobit] annotations: [GA4GH_problem_regions, capture_regions, rmsk, MIG, prioritize, dbsnp, hapmap, 1000g_omni_snps, 1000g_snps, mills_indels, cosmic, ancestral, clinvar, qsignature, ACMG56_genes, transcripts, RADAR, mirbase, genesplicer, effects_transcripts, vcfanno, viral, exac, gnomad_exome, esp, 1000g, varpon, topmed, genotype2phenotype, fusion-blacklist, simple_repeat, purecn_mappability, af_only_gnomad] annotations_available: [battenberg, dbnsfp, dbscsnv, ericscript, gnomad, topmed] validation: [giab-NA12878, platinum-genome-NA12878, giab-NA24385, giab-NA24631, giab-NA24143, giab-NA24149] - dbkey: GRCh37 name: Human (GRCh37) indexes: [seq, twobit] annotations: [GA4GH_problem_regions, capture_regions, MIG, prioritize, dbsnp, hapmap, 1000g_omni_snps, 1000g_snps, mills_indels, cosmic, ancestral, clinvar, qsignature, ACMG56_genes, transcripts, RADAR, mirbase, genesplicer, effects_transcripts, vcfanno, viral, exac, gnomad_exome, esp, 1000g, varpon, topmed, genotype2phenotype, fusion-blacklist] annotations_available: [battenberg, dbnsfp, dbscsnv, ericscript, gnomad, topmed] validation: [giab-NA12878, giab-NA24385, giab-NA24631, dream-syn3, dream-syn4, giab-NA12878-NA24385-somatic, giab-NA24143, giab-NA24149, giab-NA24694, giab-NA24695] - dbkey: hg38 name: Human (hg38) full indexes: [seq, twobit, bwa, hisat2] annotations: [ccds, coverage, capture_regions, rmsk, prioritize, dbsnp, hapmap_snps, 1000g_omni_snps, 1000g_snps, 1000g_indels, mills_indels, clinvar, qsignature, ACMG56_genes, transcripts, genesplicer, effects_transcripts, vcfanno, esp, exac, gnomad_exome, viral, RADAR, mirbase, varpon, topmed, genotype2phenotype, salmon-decoys, fusion-blacklist, purecn_mappability, simple_repeat, af_only_gnomad] annotations_available: [dbnsfp, dbscsnv, ericscript, gnomad, topmed] validation: [giab-NA12878, giab-NA24385, giab-NA24631, platinum-genome-NA12878, giab-NA12878-remap, giab-NA12878-crossmap, dream-syn4-crossmap, dream-syn3-crossmap, giab-NA12878-NA24385-somatic, giab-NA24143, giab-NA24149, giab-NA24694, giab-NA24695] - dbkey: hg38-noalt name: Human (hg38) without alternative alleles annotations: [coverage, dbsnp, hapmap_snps, 1000g_omni_snps, 1000g_snps, 1000g_indels, mills_indels, clinvar, transcripts, mirbase, genotype2phenotype] annotations_available: [dbnsfp, dbscsnv] - dbkey: mm9 name: Mouse (mm9) - dbkey: mm10 name: Mouse (mm10) indexes: [seq, twobit] annotations: [problem_regions, dbsnp, transcripts, mirbase, rmsk, vcfanno, prioritize] - dbkey: rn5 name: Rat (rn5) - dbkey: rn6 name: Rat (rn6) indexes: [seq, twobit] annotations: [transcripts, mirbase] - dbkey: canFam3 name: Dog (canFam3) indexes: [twobit] annotations: [dbsnp, transcripts, mirbase] - dbkey: galGal4 name: Chicken (galGal4) - dbkey: Sscrofa11.1 name: Pig (Sscrofa11.1) indexes: [seq, twobit] annotations: [transcripts] - dbkey: dm3 name: D melangogaster (dm3) - dbkey: BDGP6 name: D melangogaster (BDGP6) indexes: [seq] annotations: [transcripts, mirbase] - dbkey: TAIR10 name: Arabidopsis thaliana (TAIR10) annotations: [mirbase] - dbkey: xenTro3 name: X tropicalis (xenTro3) - dbkey: GRCz11 name: Zebrafish (GRCz11) indexes: [seq, twobit] annotations: [transcripts] - dbkey: GRCz10 name: Zebrafish (GRCz10) - dbkey: Zv9 name: Zebrafish (Zv9) - dbkey: sacCer3 indexes: [seq] annotations: [transcripts] name: S cerevisiae (sacCer3) - dbkey: WBcel235 name: C elegans (WBcel235) - dbkey: pseudomonas_aeruginosa_ucbpp_pa14 name: Pseudomonas aeruginosa UCBPP-PA14 # High level targets for specifying annotations annotation_groups: variation: [ccds, problem_regions, GA4GH_problem_regions, capture_regions, MIG, coverage, prioritize, dbsnp, hapmap, hapmap_snps, 1000g_omni_snps, ACMG56_genes, 1000g_snps, mills_indels, 1000g_indels, clinvar, cosmic, ancestral, qsignature, genesplicer, effects_transcripts, varpon, vcfanno, viral, purecn_mappability, simple_repeat, af_only_gnomad] rnaseq: [transcripts, RADAR, rmsk, salmon-decoys, fusion-blacklist] smallrna: [mirbase] gemini: [esp, exac, gnomad_exome, 1000g] # Global set of indexes to include for each genome. # Available choices are in GENOME_INDEXES_SUPPORTED in cloudbio/biodata/genomes.py genome_indexes: - bwa - twobit # Additional data targets install_liftover: false install_uniref: false ================================================ FILE: config/chef/cookbooks/.gitkeep ================================================ ================================================ FILE: config/chef_recipes.yaml ================================================ minimal: globus: - globus::client-tools - globus::myproxy - globus::gridftp-default ================================================ FILE: config/custom.yaml ================================================ --- # List of custom programs to install that are not packaged. These each require # a install_name function in the appropriate place in the 'custom' directory to # define the steps for downloading, building, and installing the package. bio_general: #- bio4j # no easy download for 0.8 - anaconda - emboss - pgdspider bio_nextgen: alignment: - bowtie - gmap - lastz - mosaik # - perm # difficult build process with directory names - snap #- stampy # no version names on tarballs utilities: - dwgsim - fastq_screen - ogap - rnaseqc - varianttools - disambiguate - alientrimmer analysis: # - echo # build problems on ubuntu 12.10 - picard - shrec - tophat - featurecounts assembly: - abyss - cortex_var - ray - transabyss - trinity - velvet sv: - hydra - freec variant: - bcbio_variation - crisp - grabix - mutect - tassel - stacks - varscan chip: - macs needs_64bit: - ucsc_tools - bfast bio_proteomics: - transproteomic_pipeline - omssa - mzmine - myrimatch - directag - tagrecon - idpqonvert - pepitome - percolator bio_proteomics_wine: - proteomics_wine_env #- multiplierz - proteowizard - morpheus cloudman: - cloudman - novnc galaxy: - galaxy_webapp galaxy_tools: - cbl_galaxy_tools galaxyp: - galaxy_protk - protvis # Pydoop and seal need update and should only be installed on cloud systems #distributed: # - pydoop # - seal system: - homebrew - s3fs python: - bx-python - netsa-python - rpy java: - leiningen phylogeny: - tracer - beast #Viral Cloud Resource (VCR) - JCVI's Viral Genomic Pipelines on the cloud vcr: - galaxy_vcr - viralassembly - viralassembly_cleanall - viralvigor - viralvigor_test - viralvigor_validate - viralvigor_cleanall - vicvb - vicvb_cleanall # ToDo -- test on 64bit with bfast #- dnaa #- srma ================================================ FILE: config/fabricrc.txt ================================================ # Configuration file for Fabric specifying filesystem information # This can be adjusted and passed to fab on the commandline as '-c' # # For example # # fab -f data_fabfile.py -H your_machine -c your_fabricrc.txt install_data_s3:your_biodata.yaml # -- Details about the operating system ## If target machine is Ubuntu or CentOS, the following defaults should allow CloudBioLinux to ## automatically determine the correct settings. For other operating systems (e.g. Debian or ScientificLinux) ## please override distribution (and dist_name in case of Debian). distribution = __auto__ dist_name = __auto__ ## Ubuntu #distribution = ubuntu #dist_name = quantal ## Centos #distribution = centos ## Debian #distribution = debian #dist_name = squeeze # -- Installation details # Whether to clean the system for EC2 AMI preparation; # do not set on vagrant or local machines is_ec2_image = false # Global installation directory for packages and standard programs system_install = /usr/local # Local install directory for versioned software that will not # be included in the path by default local_install = /usr/local/share # Shell to be used by CBL scripts during runtime shell_config = ~/.bashrc shell = /bin/bash -i -c # Global setting for using sudo; allows installation of custom packages # by non-privileged users. # *Note*: ``system_install`` needs to point to a user-writeable directory if # ``use_sudo`` is set to ``False`` use_sudo = True # -- Details about reference data installation # Path where biological reference data files should be retrieved to data_files = /mnt/biodata # -- Details about installing Galaxy and its dependencies. Values behind the # comments are the defaults. # Specify as which user should Galaxy be installed #galaxy_user = galaxy # Path where Galaxy application is installed #galaxy_home = /mnt/galaxyTools/galaxy-central # URL for the Galaxy application source code repository #galaxy_repository = https://bitbucket.org/galaxy/galaxy-central/ # A path to the binaries for interacting with PostgreSQL #postgres_bin_dir = /usr/lib/postgresql/9.1/bin #postrges_user = postgres #postgres_port = 5840 # If ``True``, setup and initialize Postgres database for use by Galaxy # according to the following options #galaxy_setup_database = False #galaxy_db_name = galaxy #galaxy_database_path = /mnt/galaxy/db #galaxy_ftp_user_password = fu5yOj2sn #galaxy_universe_wsgi_url = https://bitbucket.org/galaxy/cloudman/raw/tip/universe_wsgi.ini.cloud # If ``True``, Galaxy repository will not be custom-configured after being cloned # The customization details are primarily targeted at a cloud deployment; for # details, see ``cbl.galaxy.__init__ -> _configure_galaxy_repository`` #galaxy_preconfigured_repository = False # If ``True``, setup and initialize Postgres database for use by Galaxy #galaxy_setup_database = False # If ``True``, setup Galaxy as a system-level service controlled via init.d #galaxy_setup_service = False # If ``True``, setup Xvfb as a system-level service via init.d #galaxy_setup_xvfb = False # Location of Galaxy's .loc files #galaxy_loc_files = /mnt/biodata/galaxy/galaxy-data # Location of files required by Galaxy Trackster # This path needs to match what will be set in Galaxy's universe_wsgi.ini #galaxy_len_files = /mnt/galaxy/configuration_data/len # Galaxy requires Java tools to have their jar files places in a specific # directory, so specify it here. #galaxy_jars_dir = /mnt/galaxyTools/galaxy-central/tool-data/shared/jars # If ``True``, install a set of tools considered as Galaxy dependencies #galaxy_install_dependencies = False # Specify a custom location to a YAML file containing a list of tools to # install as Galaxy dependencies #galaxy_tools_conf = $__contrib_dir__/cloudman/tools.yaml # Location where (some?!) Galaxy dependency tools should be installed #galaxy_tools_dir = /mnt/galaxyTools/tools # If ``True``, install R packages listed in contrib/cloudman/tools.yaml #galaxy_install_r_packages = False # Paths for respective file systems that will be added to ``/etc/exports`` # and thus exported via NFS when the operating system boots #galaxy_data_mount = /mnt/galaxyData #galaxy_tools_mount = /mnt/galaxyTools #galaxy_indices_mount = /mnt/biodata #extra_nfs_exports = comma,separate,desired,paths ## Defaults for CloudMan LWR puppet_lwr_user = galaxy puppet_lwr_destination = /mnt/galaxy/lwr # -- Tool-specific configurations # A location of a custom nginx.conf or a comparable template, respectively. #nginx_conf_template_path = installed_files/nginx.conf.template #nginx_conf_path = path/to/custom/nginx.conf # Picard home picard_home = /usr/share/java/picard # -- novnc configuration options # Configure novnc along with CloudMan, disabled # by default for now. #configure_novnc = False # Where to install novnc #novnc_install_dir = /opt/novnc # Password used by novnc to connect to vncserver. #vnc_password = cl0udbi0l1nux # Set vnc_user if different than env.user #vnc_user=ubuntu #vnc_display=1 #vnc_depth=16 #vnc_geometry=1024x768 ================================================ FILE: config/haskell-libs.yaml ================================================ --- # Define haskell libraries to be installed via cabal. cabal: # base - network - cabal-install - cabal-dev # libraries - mtl - parsec2 - parsec3 - quickcheck - tagsoup - utf8-string - x11 - x11-xft # bioinformatics - bio - flower # FIXME: Fails in Lucid, if the source is downloaded separately and installed according to the README the issue won't appread - flowsim #- samtools ================================================ FILE: config/main.yaml ================================================ --- # Top level configuration file that specifies which groups of programs # should be installed. New sections that are added to individual config # files should go here. Comment out any groups you don't want to have # installed. packages: - minimal - desktop - programming - distributed - documentation - libraries - amazon - python - r - ruby - perl - java - java_extras - erlang - haskell - databases - math - viz - web - bio_general - bio_search - bio_alignment - bio_nextgen - bio_sequencing - bio_annotation - bio_microarray - bio_visualization - bio_utils - phylogeny # Do not install by default to avoid errors with local installs # Injected by setting is_ec2_image in fabricrc.txt # - galaxy # - galaxy_tools # - cloudman libraries: #- haskell-libs - perl-libs - python-libs - r-libs - ruby-libs ================================================ FILE: config/node_extra.json ================================================ // Properties used by chef to configure node. Merged with fabric // properties to produce final node.json file used by Chef. { "globus": { "repository_deb_url" : "http://www.globus.org/ftppub/gt5/5.2/5.2.4/installers/repo/globus-repository-5.2-stable-quantal_0.0.3_all.deb" } } ================================================ FILE: config/packages-debian.yaml ================================================ --- # Debian packages use the standard packages.yaml list with naming substitutions # specified in this file. clojure: "" ec2-ami-tools: "" ec2-api-tools: "" firefox: iceweasel freenx: "" freenx-server: "" gdm: gdm3 mongodb-10gen: mongodb python-vm-builder-ec2: "" ubuntu-desktop: "" ugene: "" unity-2d: "" # Problem Bio-Linux builds on debian bio-linux-arb: "" bio-linux-assembly-conversion-tools: "" bio-linux-emboss: "" # emboss-data is broken in Squeeze emboss: "" emboss-data: "" emboss-lib: "" ================================================ FILE: config/packages-homebrew.yaml ================================================ # Packages available in the Homebrew and Linuxbrew package manager --- bio_nextgen: alignment: - bwa - bowtie2 - novoalign - rna-star utilities: - bamtools - bedtools - cramtools - biobambam - fastqc - fastx_toolkit - qualimap - sambamba-binary - staden_io_lib analysis: - cufflinks-binary - samtools - htslib - bcftools #- tophat variant: - delly;--with-binary - freebayes - gatk-framework - lofreq - lumpy-sv - pindel - platypus-variant - rtg-tools - scalpel - snpeff - theta2 - vardict - vcflib - vcftools - vep - wham # - yaha The Principal Investigator has moved; the software does not download ATM integration: - biointerchange distributed: - parallel ================================================ FILE: config/packages-nix.yaml ================================================ # These are packages defined in Nix packages. Nix is an independent and # distribution agnostic packaging system, with many interesting features. # # For more information see http://nixos.org/nixpkgs/ minimal: - nix editors: - vim-7.3 # a recent vim, just for the test phylogeny: - paml ================================================ FILE: config/packages-scientificlinux.yaml ================================================ minimal: version_control: - git-core - subversion - mercurial build: - cmake - gcc - gcc-c++ - gcc-gfortran - make util: - screen - unzip - tar - less network: - curl - wget libraries: - atlas - atlas-devel - blas - blas-devel - bzip2-devel - freetype-devel - gsl-devel - libpng-devel - openssl-devel - ncurses-devel documentation: programming: - protobuf-c - protobuf-compiler - protobuf-python java: - ant - ant-junit - ant-nodeps - java-1.7.0-openjdk - java-1.7.0-openjdk-devel - junit4 python: - python - python-devel - boost-python-devel - numpy - python-pip r: - R - R-core - libRmath - libRmath-devel distributed: - hadoop-0.20 - hadoop-0.20-native - hadoop-0.20-pipes - hadoop-0.20-source - hadoop-0.20-libhdfs - mpich2 - mpich2-devel galaxy: all: - postgresql - gcc-gfortran - java-1.7.0-openjdk - xorg-x11-server-Xvfb - cmake - zlib-devel - ncurses-devel - xz-devel - tcl-devel - tk-devel - ImageMagick # - scipy ## Available in CentOS but not SL, install some prequirements instead including numpy (likely many tools just require this.) - numpy - numpy-f2py - blas-devel - atlas-devel # Following requrie EPEL repository (setup by CloudBioLinux) - pdfjam - sparsehash-devel # cran packages for CentOS are not maintained, do what we can with EPEL. - R - R-devel # Missing relative to ubuntu install: All missing in CentOS (e.g. axel, python-rpy, many R modules) # Missing relative to CentOS install: scipy. # Information on setting up scipy and rpy can be found at http://getgalaxyp.org/install.html bio_proteomics: # Needed for TPP - libxslt - perl-libxml-perl - libXpm-devel - bzip2-devel - gnuplot # Needed for percolator - xerces-c-devel - xerces-c - xsd - sqlite-devel # - libboost-serialization-dev, libboost-system-dev, libboost-filesystem-dev Installed by boost-devel - zlib-devel - boost-devel # needed for building OpenMS - qt-devel - qtwebkit-devel # Needed for PSM-eval - libyaml-devel # Needed for RVM (in turn required for ProtK) - bison - libffi-devel - libtool - libyaml-devel - openssl-devel - patch - readline-devel galaxyp: # Needed for TPP - libxslt - perl-libxml-perl - libXpm-devel - bzip2-devel - gnuplot # Needed for percolator - xerces-c-devel - xerces-c - xsd - sqlite-devel # - libboost-serialization-dev, libboost-system-dev, libboost-filesystem-dev Installed by boost-devel - zlib-devel - boost-devel # Needed for nbic galaxy database utilities - perl-Log-Log4perl # - libswiss-perl NOT AVAILABLE, ONE TOOL WON'T WORK. # needed for building OpenMS - qt-devel - qtwebkit-devel # Needed for PSM-eval - libyaml-devel # Needed for RVM (in turn required for ProtK) - bison - libffi-devel - libtool - libyaml-devel - openssl-devel - patch - readline-devel ================================================ FILE: config/packages-yum.yaml ================================================ --- # List of packages to be installed via the yum package manager. minimal: version_control: - git-core - subversion - mercurial build: - cmake - gcc - gcc-c++ - gcc-gfortran - make - patch util: - screen - unzip - tar - less network: - curl - wget libraries: - atlas - atlas-devel - blas - blas-devel - bzip2 - bzip2-devel - freetype-devel - gsl-devel - lapack - lapack-devel - libgfortran - libpng-devel - openssl-devel - ncurses-devel - libxml2-devel - zlib-devel documentation: desktop: - gnome-panel - freenx - gnome-terminal - firefox - evince - xorg-x11-server-Xvfb programming: editors: - emacs - emacs-common - emacs-el - vim-X11 - ctags version_control: - cvs - bzr build: - swig - curl-devel - libxslt - libxslt-devel - xfsprogs - glibc - glibc-devel - zip - zziplib-devel - xz - xz-devel - gtk2 - gtk2-devel - globus-openssl-module - globus-openssl-module-devel - pcre - pcre-devel - readline - readline-devel - rsync - librsync - librsync-devel libraries: - libuuid-devel - tcl-devel - tk-devel - ImageMagick - boost - boost-devel - openmpi - openmpi-devel - gd - gd-devel - libgnomecanvas - libgnomecanvas-devel - gnome-vfs2 - gnome-vfs2-devel - libgnomeui - libgnomeui-devel - pango - pango-devel virtual: util: - tree lang: - aspell distributed: amazon: python: - python - python-devel - boost-python-devel - numpy ruby: - ruby - ruby-devel - rubygems r: - R - R-core - R-core-devel - R-java - libRmath - libRmath-devel perl: - perl - perl-devel - perl-core java: - ant - java-1.7.0-openjdk - java-1.7.0-openjdk-devel erlang: - erlang # - erlang-devel ## Not available, erlang seems to install quite a bit. haskell: - ghc databases: mysql: - mysql - mysql-devel - mysql-server postgres: - postgresql - postgresql-devel - postgresql-server sqlite: - sqlite - sqlite-devel document: - mongodb - mongodb-devel - couchdb - tokyocabinet - tokyocabinet-devel hdf: - hdf5 #- hdf5-devel math: # - prover9 # No longer available? Cannot find. - octave - octave-devel viz: - gnuplot - ImageMagick - ImageMagick-devel - graphviz - graphviz-devel web: - httpd - httpd-devel - nginx bio_general: bio_search: bio_alignment: bio_nextgen: bio_sequencing: bio_annotation: bio_microarray: bio_visualization: bio_utils: phylogeny: all: needs_64bit: galaxy: all: - postgresql - gcc-gfortran - java-1.7.0-openjdk - xorg-x11-server-Xvfb - cmake - zlib-devel - ncurses-devel - xz-devel - tcl-devel - tk-devel - ImageMagick - scipy # Following requrie EPEL repository (setup by CloudBioLinux) - pdfjam - sparsehash-devel # cran packages for CentOS are not maintained, do what we can with EPEL. - R - R-devel # Missing relative to ubuntu install: axel, python-rpy, all R modules # Information on setting up rpy can be found at http://getgalaxyp.org/install.html bio_proteomics: # Needed for TPP - libxslt - perl-libxml-perl - libXpm-devel - bzip2-devel - gnuplot # Needed for percolator - xerces-c-devel - xerces-c - xsd - sqlite-devel # - libboost-serialization-dev, libboost-system-dev, libboost-filesystem-dev Installed by boost-devel - zlib-devel - boost-devel # needed for building OpenMS - qt-devel - qtwebkit-devel # Needed for PSM-eval - libyaml-devel # Needed for RVM (in turn required for ProtK) - bison - libffi-devel - libtool - libyaml-devel - openssl-devel - patch - readline-devel galaxyp: # Needed for TPP - libxslt - perl-libxml-perl - libXpm-devel - bzip2-devel - gnuplot # Needed for percolator - xerces-c-devel - xerces-c - xsd - sqlite-devel # - libboost-serialization-dev, libboost-system-dev, libboost-filesystem-dev Installed by boost-devel - zlib-devel - boost-devel # Needed for nbic galaxy database utilities - perl-Log-Log4perl # - libswiss-perl NOT AVAILABLE, ONE TOOL WON'T WORK. # needed for building OpenMS - qt-devel - qtwebkit-devel # Needed for PSM-eval - libyaml-devel # Needed for RVM (in turn required for ProtK) - bison - libffi-devel - libtool - libyaml-devel - openssl-devel - patch - readline-devel ================================================ FILE: config/packages.yaml ================================================ --- # List of packages to be installed via the package manager. # # This is derived from the Infochimps machetEC2 package: # # http://github.com/infochimps/machetec2 # # and merged with packages from bioperl-max: # # http://fortinbras.us/bioperl-max/ # # and biolinux: # # http://www.jcvi.org/cms/research/projects/jcvi-cloud-biolinux/included-software/ # # Package names are the terminal symbols in the tree (the text on any # line which begins with whitespace followed by a hypen and a space). # # The package list is organized taxonomically so that parts of it can # be selectively installed/ignored. See main.yaml for top level configuration. minimal: #Packages to support a minimal installation (no X!) administration: - sudo - htop editors: - vim # don't put in emacs, as it pulls in X version_control: - git-core - cvs - subversion - mercurial - bzr #- darcs # pulls in postfix mail configuration. build: - bsdmainutils - clang - cmake - gcc - g++ - gfortran - libtool - make - patch - pkg-config util: - axel - gawk - openssh-server - pigz - rsync - screen - unzip - less network: - curl - wget libraries: - libatlas-base-dev #- libatlas-dev - libblas-dev - liblapack-dev - libeigen3-dev - libc6-dev - libzip-dev - libbz2-dev - zlib1g-dev - libcurl4-openssl-dev - libgsl-dev - libgsl2 - gsl-bin - mime-support - libncurses5-dev - libpcre3-dev - libpng12-dev - libreadline-dev - libssl-dev - uuid - uuid-dev - libxml2 - libxml2-dev - libxslt1-dev - libxslt1.1 - xz-utils - liblzma-dev - zlib1g-dev - libnetcdf-dev documentation: - texlive-latex-base - texlive-latex-recommended - texlive-latex-extra desktop: #Packages to support a remote desktop all: - gnome-panel - gnome-system-tools - synaptic - software-center #- freenx-server # Gone on 14.04 - gnome-backgrounds - gnome-terminal - firefox - evince - ubuntu-desktop - xvfb - gdm # NX compatible window managers on Ubuntu 11.10+ ubuntu_oneiric: - gnome-session-fallback - unity-2d ubuntu_lucid: - rubygems desktop-cloud: # some special purpose VMs, such as BOINC, only require a minimal desktop - lxde - vnc4server programming: editors: - emacs - emacs-goodies-el - emacsen-common - exuberant-ctags - gedit build: - libgtk2.0-dev - protobuf-compiler - python-software-properties - stow - swig - rsync - xfsprogs filesystem: - libfuse-dev libraries: - tcl-dev - tk-dev - imagemagick - pdfjam - libsparsehash-dev - xvfb - libboost-dev - libboost-python-dev - libboost-thread-dev - libgd2-xpm-dev - libgnomecanvas2-0 - libgnomecanvas2-dev - libgnomevfs2-0 - libgnomevfs2-dev - libgnomeui-0 - libgnomeui-dev - libpango1.0-0 - libpango1.0-dev - libx11-dev packaging: - build-essential - devscripts - debhelper - dh-make - pbuilder - unzip virtual: #- virtualbox-4.2 - dkms util: - mc - tree - screen - time - fdupes network: - lynx - traceroute lang: - aspell - dictionaries-common - libaspell-dev distributed: all: - rabbitmq-server - nfs-kernel-server - nfs-common - libopenmpi-dev - openmpi-bin - libzmq-dev - libfuse-dev # only for oneiric 11.10 ubuntu_oneiric: - libmpich2-3 - libmpich2-dev - mpich2 amazon: - s3cmd - euca2ools python: - python - python-dev - python-distutils-extra - python-pip - python-setuptools - python-virtualenv - python3 - python3-dev - python-rpy ruby: - ruby1.9.1 # default Ruby install - ruby1.9.1-dev r: - r-base - r-base-core - r-base-dev perl: - perl - libperl-dev - perl-modules - perl-tk - parrot - parrot-devel - cpanminus - libextutils-autoinstall-perl java: # - sun-java6-bin # Sun Java removed from Ubuntu package repositories # - sun-java6-jre # Package 'sun-java6-jre' has no installation candidate # - sun-java6-jdk Package 'sun-java6-jdk' has no installation candidate - openjdk-7-jdk - openjdk-7-jre java_extras: - openjdk-6-jdk - openjdk-6-jre - ant - libbiojava-java - libbiojava-java-demos #- eclipse - maven - libjline-java - libjhdf5-java - jruby - jython - scala - scala-library erlang: - erlang - erlang-base - erlang-dev - erlang-examples - erlang-manpages - erlang-mode haskell: - ghc - cabal-install - c2hs databases: mysql: - mysql-client - mysql-common - mysql-server - libmysqlclient-dev postgres: - postgresql - postgresql-client - postgresql-plpython - postgresql-plperl - libpq-dev sqlite: - libsqlite0 - libsqlite0-dev - libsqlite3-0 - libsqlite3-dev - sqlite3 document: - libdb-dev - mongodb - couchdb - tokyocabinet-bin - libtokyocabinet-dev hdf: - libhdf5-serial-dev math: - prover9 - octave viz: - imagemagick - x11-apps - mayavi2 - mtasc # for modest maps - gnuplot - graphviz - libgraphviz-dev - libgl1-mesa-dev - libgl1-mesa-dri - libgl1-mesa-glx - libglu1-mesa - libglu1-mesa-dev web: - apache2 boinc: - boinc-client - boinc-manager bio_general: - primer3 - ugene bio_search: - blast2 - ncbi-tools-bin - cdbfasta bio_alignment: - exonerate - mafft - muscle - probcons - wise bio_nextgen: - fastx-toolkit - maq - plink - last-align - tabix # - samtools # only on 0.18, prefer custom build bio_sequencing: bio_annotation: - mcl - predictprotein - tigr-glimmer bio_markergenes: - ampliconnoise - mothur - rdp-classifier - qiime bio_microarray: bio_visualization: - rasmol bio_proteomics: all: # Required for protk - libxml2-dev - libreadline-dev - libxslt-dev # Required for protvis - python-virtualenv - python-setuptools - libpng12-dev - python-dev - libexpat1-dev - libboost-dev # Needed for TPP - xsltproc - libxml-sax-expat-perl - libgd2-xpm-dev - libbz2-dev - gnuplot # Needed for percolator - libxerces-c3.1 - libxerces-c-dev - xsdcxx - libboost-dev - libboost-filesystem-dev - libboost-system-dev - libsqlite3-0 - libboost-serialization-dev - zlib1g-dev - libboost-dev # needed for 32bit execs distributed with searchgui. # - ia32-libs ## Doesn't play well with dependencies of gnuplot, TODO: Find more specific lib. # needed for building OpenMS - libtool - qt4-dev-tools - libqtwebkit-dev - libqtwebkit4 # Needed for psm-eval - libyaml-dev # Needed for RVM - libbison-dev - ncurses-dev - automake - libtool - bison - subversion - libgdbm-dev - libffi-dev - libsqlite3-dev - sqlite3 bio_proteomics_wine: - wine bio_utils: phylogeny_testing: # new packages in testing (Debian 'wheezy') all: - debian-archive-keyring - raxml # - mrbayes-mpi broken, should eventually replace biolinux-mrbayes-multi phylogeny_experimental: # new packages in experimental all: - paml # should replace biolinux-paml phylogeny: all: - phyml - mrbayes - njplot - tree-puzzle - treeviewx needs_64bit: - fasttree ## Package lists imported from mi-deployment. cloudman: all: # Required to install CloudMan-required Python libraries - python-pip # Required for communication between instances - rabbitmq-server # Required to handle data volumes (including LVM support) - lvm2 - dmsetup # - mdadm - reiserfsprogs - xfsprogs # Required to compile nginx - build-essential #- gcc ## included in minimal - libpcre3-dev - libssl-dev - stow # Required for cluster-wide NFS file system - nfs-kernel-server - nfs-common # Enable OpenMPI - libopenmpi-dev - libopenmpi1.3 - openmpi-bin - openmpi-common # Needed to enable updates for Galaxy # - mercurial ## Included in minimal # Required for compiling ProFTPd (must match installed PostgreSQL version!) - postgresql-server-dev-9.5 # Actually not sure what of the following is really needed - libboost-all-dev - libreadline6-dev # Required for s3fs - libfuse-dev - libfuse2 - pkg-config - libcurl4-openssl-dev - libxml2-dev - mime-support # Required for novnc - vnc4server - glusterfs-client ubuntu_quantal: # Required to get HTCondor, not available on 12.04 (latest LTS release). - condor galaxy: all: - postgresql - gfortran - default-jdk - python-rpy - xvfb # Required by Freebayes - cmake # Required by BWA - zlib1g-dev # Required by Samtools - libncurses5-dev # Required by Compute q-values - liblzma-dev # Parallel file download (used by the Galaxy ObjectStore) - axel # Required by various R modules - tcl-dev # Required by various R modules - tk-dev # Required by RGalaxy - imagemagick - pdfjam - python-scipy - libsparsehash-dev # Latest python-rpy not currently compatible with R 3.0.1 at least on precise (presumably # quantal as well), hack below is to specify an older version of R compabile with python-rpy. #- r-base #- r-base-core #- r-base-dev - r-cran-lattice - r-cran-qvalue # Required by HVIS - r-bioc-hilbertvis ubuntu_quantal: - "r-base-core=3.0.0-2quantal" - "r-base-dev=3.0.0-2quantal" ubuntu_precise: - "r-base-core=3.0.0-2precise" - "r-base-dev=3.0.0-2precise" galaxyp: all: # Required for protk - libxml2-dev - libreadline-dev - libxslt-dev # Required for protvis - python-virtualenv - python-setuptools - libpng12-dev - python-dev - libexpat1-dev - libboost-dev # Needed for TPP - xsltproc - libxml-sax-expat-perl - libgd2-xpm-dev - libbz2-dev - gnuplot # Needed for percolator - libxerces-c3.1 - libxerces-c-dev - xsdcxx - libboost-dev - libboost-filesystem-dev - libboost-system-dev - libsqlite3-0 - libboost-serialization-dev - zlib1g-dev - libboost-dev # needed for 32bit execs distributed with searchgui. # - ia32-libs ## Doesn't play well with dependencies of gnuplot, TODO: Find more specific lib. # needed for building OpenMS - libtool - qt4-dev-tools - libqtwebkit-dev - libqtwebkit4 # Needed for nbic galaxy database utilities - liblog-log4perl-perl - libswiss-perl # Needed for psm-eval - libyaml-dev # Needed for RVM - libbison-dev - ncurses-dev - automake - libtool - bison - subversion - libgdbm-dev - libffi-dev - libsqlite3-dev - sqlite3 nginx: - nginx rabbitmq: - rabbitmq-server supervisor: - supervisor coffeescript: - coffeescript ================================================ FILE: config/perl-libs.yaml ================================================ --- # Configuration file defining perl libraries to install via CPAN cpan: # build - 'Locale::Maketext::Lexicon' - 'Module::Signature' - 'Build' # dependencies - 'Algorithm::C3' - 'Algorithm::Diff' - 'Algorithm::Munkres' - 'Array::Compare' - 'Bit::Vector' - 'Bundle::LWP' - 'Cache' - 'Cairo' - 'Carp::Assert' - 'Carp::Clan' - 'CGI::Application' - 'CGI::Application::Plugin::Authentication' - 'CGI::Application::Plugin::Authorization' - 'CGI::Application::Plugin::TT' - 'CGI::Application::Plugin::HTDot' - 'CGI::Session' - 'Class::Accessor' - 'Class::C3' - 'Class::C3::XS' - 'Class::Data::Inheritable' - 'Class::Inspector' - 'Class::ISA' - 'Class::MOP' - 'Clone' - 'Config::Auto' - 'Config::IniFiles' - 'Config::Simple' - 'Convert::Binary::C' - 'Crypt::SSLeay' - 'Data::FormValidator' - 'Data::OptList' - 'Data::Stag' - 'Date::Calc' - 'Date::Manip' - 'DateTime::TimeZone' - 'DateTime::Locale' - 'DateTime' - 'Devel::GlobalDestruction' - 'Devel::StackTrace' - 'Digest::MD5' - 'Digest::HMAC' - 'Digest::SHA1' - 'Email::Valid' - 'Error' - 'Exception::Class' - 'Exception::Class::TryCatch' - 'ExtUtils::Depends' - 'ExtUtils::Manifest' - 'ExtUtils::ModuleMaker' - 'ExtUtils::PkgConfig' - 'FCGI' - 'File::Copy::Recursive' - 'File::Listing' - 'File::MMagic' - 'File::Slurp' - 'File::Which' - 'Font::AFM' - 'FreezeThaw' - 'GD::Graph' - 'GD::Text' - 'Glib' - 'Gtk2' - 'Gnome2::Canvas' - 'Gnome2::VFS' - 'Gnome2' - 'Graph' - 'GraphViz' - 'Heap' - 'HTML::Entities' - 'HTML::FillInForm' - 'HTML::FormatData' - 'HTML::HeadParser' - 'HTML::Parser' - 'HTML::Tagset' - 'HTML::Template' - 'HTML::Tree' - 'Image::Size' - 'Inline' - 'IO::Compress::Base' - 'IO::Compress::Bzip2' - 'IO::Compress::Lzf' - 'IO::Pty' - 'IO::Socket::SSL' - 'IO::String' - 'IO::Stringy' - 'IO::Unread' - 'IPC::Run' - 'IPC::ShareLite' - 'IPC::Signal' - 'Jcode' - 'List::MoreUtils' - 'Locale::gettext' - 'Log::Dispatch' - 'LWP::Simple' - 'Mail::Internet' - 'Math::Derivative' - 'Math::Random' - 'Math::Spline' - 'MIME::Tools' - 'MIME::Types' - 'MLDBM' - 'Module::Build' - 'MRO::Compat' - 'Net::Daemon' - 'Net::DBus' - 'Net::DNS' - 'Net::Domain::TLD' - 'Net::IP' - 'Net::LibIDN' - 'Net::SSLeay' - 'OLE::Storage' - 'Pango' - 'Params::Util' - 'Params::Validate' - 'Parse::RecDescent' - 'Perl6::Junction' - 'Pod::Xhtml' - 'PostScript::File' - 'PostScript::TextBlock' - 'Proc::InvokeEditor' - 'Proc::WaitStat' - 'Regexp::Common' - 'RPC::XML' - 'Set::Scalar' - 'Scalar::Util' - 'Scope::Guard' - 'SOAP::Lite' - 'Sort::Naturally' - 'Spreadsheet::ParseExcel' - 'Spreadsheet::ParseExcel::Simple' - 'Spreadsheet::WriteExcel' - 'Storable' - 'SQL::Statement' - 'Statistics::Descriptive' - 'Sub::Exporter' - 'Sub::Install' - 'Sub::Name' - 'Sub::Uplevel' - 'SVG::Graph' - 'SVG' - 'Syntax::Highlight::Perl::Improved' - 'Task::Weaken' - 'Term::ReadKey' - 'Test::Exception' - 'Text::CharWidth' - 'Text::CSV' - 'Text::CSV_XS' - 'Text::Iconv' - 'Text::ParseWords' - 'Test::Harness' - 'Test::More' - 'Tie::IxHash' - 'Tree::DAG_Node' - 'Try::Tiny' - 'Unicode::Map' - 'UNIVERSAL::require' - 'URI::Escape' - 'UUID' - 'WWW::RobotRules' - 'XML::DOM' - 'XML::DOM::XPath' - 'XML::Filter::BufferText' - 'XML::LibXML ' - 'XML::NamespaceSupport' - 'XML::Parser' - 'XML::Parser::PerlSAX' - 'XML::RegExp' - 'XML::SAX' - 'XML::SAX::Expat' - 'XML::SAX::Writer' - 'XML::Simple' - 'XML::Twig' - 'XML::Writer' - 'XML::XPathEngine' - 'XML::XPath' - 'YAML' - 'YAML::Syck' # Database drivers - 'DB_File' - 'DBI' - 'DBD::AnyData' - 'DBD::CSV' - 'DBD::mysql' - 'DBD::Pg' - 'DBD::SQLite' # Biome : these will definitely be added to over time - 'Moose' - 'MooseX::ClassAttribute' - 'MooseX::Types' - 'MooseX::Role::Parameterized' - 'MooseX::Aliases' # Biology - 'Ace' - 'Bio::ASN1::EntrezGene' - 'Bio::Graphics' - 'Bio::Phylo' - 'http://www.cpan.org/authors/id/C/CJ/CJFIELDS/BioPerl-1.6.901.tar.gz' - 'Bio::Das' - 'Bio::GFF3::LowLevel' # Currently have problems building on CentOS # - 'AnyData' # - 'Convert::BinHex' # - 'XML' # - 'XML::LibXSLT' # - 'Modern::Perl' # These need samtools and Jim Kent's source trees so need to be done by hand # - 'Bio::DB::Sam' # - 'Bio::DB::BigFile' ================================================ FILE: config/puppet/modules/.gitkeep ================================================ ================================================ FILE: config/puppet_classes.yaml ================================================ minimal: biocloudcentral: # Idea is still being developed, I wonder if actually defining a puppet class # like cloudbiolinux::biocloudcentral that in turn brings in these classes might # be a better idea. - "concat::setup" # Required for apache - "apache" - "apache::mod::wsgi" - "biocloudcentral" - "biocloudcentral::apache" ## Uncomment following lines to add LWR to CloudMan meta-package. #cloudmn: # - lwr ================================================ FILE: config/python-libs.yaml ================================================ --- # Configuration file defining python specific libraries on pypi to install # using easy install pypi: # infrastructure - setuptools - amqplib - BeautifulSoup - fabric - Logbook - lxml - mako - paste - pastescript - pycrypto - python-cjson - pyOpenSSL - routes - simplejson - sphinx - StarCluster - webhelpers - webob - xlrd # build - Pyrex-real - cython # needed for installing pybedtools # concurrency/web - Twisted # science - cutadapt - ruffus - DendroPy - HTSeq - pybedtools - pygr - PyVCF - rpy2 - ramp # databases - CouchDB - MySQL-python - pymongo - pysqlite - psycopg2 # visualization - gnuplot-py - PIL - pypdf - reportlab # cloudman - oca - hoover - Mako conda: - boto - biopython - cython - h5py - ipython - matplotlib - networkx - nose - numpy - pandas - pysam - pyyaml - pyzmq - scikit-learn - scipy - SQLAlchemy - statsmodels - tornado ## compile problems #- cogent #- tables #- pygraphviz #- pysparse #- sympy #- pydot # references python 3-only pyparsing #- pygooglechart #- rpy2-bioconductor-extensions ================================================ FILE: config/r-libs.yaml ================================================ --- # Configuration file defining R specific libraries that are installed # via CRAN and Bioconductor. cranrepo: http://cran.fhcrc.org/ cran: - abind - ape - aplpack - biglm - BiodiversityR - bitops - boot - car - caret - caTools - chron - class - cluster - codetools - coin - colorspace - date - DBI - devtools - digest - effects - FactoMineR - flashClust - foreign - gdata - gee - graph - ggplot2 - gplots - gsalib - gtools - hdf5 - hexbin - hmisc - HH - hwriter - kernlab - kernsmooth - knitr - lattice - latticeExtra - leaps - lmtest - maptree - markdown - mass - matrix - mclust - mgcv - modeltools - multcomp - multicore - nlme - NMF - nnet - pamr - pi0 - plotrix - plyr - proto - qvalue - rcmdr - relimp - reshape - reshape2 - rgl - RColorBrewer - RCurl - rpart - rjson - Rmpi - RMySQL - RSQLite - sandwich - sm - snowfall - spatial - sqldf - statmod - stringr - strucchange - survival - testthat - vegan - vgam - VennDiagram - vr - XML - xtable - yacca - yaml - zoo biocrepo: http://bioconductor.org/biocLite.R bioc: - affxparser - affy - affydata - affyio - affyPLM - affyQCReport - annaffy - annotate - AnnotationDbi - aroma.light - ArrayExpress - arrayQualityMetrics - ArrayTools - BayesPeak - baySeq - beadarray - Biobase - BiocInstaller - bioDist - biomaRt - Biostrings - BSgenome #- BSgenome.Athaliana.TAIR.TAIR9 #- BSgenome.Celegans.UCSC.ce6 #- BSgenome.Dmelanogaster.UCSC.dm3 #- BSgenome.Hsapiens.UCSC.hg19 #- BSgenome.Mmusculus.UCSC.mm9 #- BSgenome.Scerevisiae.UCSC.sacCer3 - ChIPpeakAnno - chipseq - ChIPseqR - ChIPsim - CNAnorm - cn.mops - CSAR - cummeRbund - DEGseq - DESeq - DEXSeq - DiffBind - DNAcopy - DynDoc - edgeR - EDASeq - exomeCopy - gage - gcrma - genefilter - geneplotter - GenomeGraphs - genomeIntervals - GenomicFeatures - GenomicRanges - Genominator - GEOquery - GGBase - GGtools - ggbio - girafe - globaltest #- GO.db - GOFunction - goseq - GOstats - GSEABase - GWASTools - hgu95av2.db - HilbertVis - htSeqTools - impute - IRanges - KEGG.db - limma - lumi - makecdfenv - marray - MEDIPS - multtest - MotIV - mzR - oneChannelGUI #- org.At.tair.db #- org.Ce.eg.db #- org.Dm.eg.db #- org.Hs.eg.db #- org.Mm.eg.db #- org.Sc.sgd.db #- org.Xl.eg.db - PAnnBuilder - preprocessCore - PICS - qpgraph - qrqc - rGADEM - R453Plus1Toolbox - RBGL - Repitools - rGADEM - Rgraphviz - Ringo - rnaSeqMap - Rolexa - Rsamtools - Rsubread - rtracklayer - segmentSeq - seqbias - seqLogo - ShortRead - siggenes - simpleaffy - snpMatrix - snpStats - splots - SRAdb - tkWidgets - VariantAnnotation - vsn - widgetTools - xcms - zlibbioc ================================================ FILE: config/ruby-libs.yaml ================================================ --- # Configuration file defining ruby libraries to install # using gem install (avoiding the distribution package manager) # # Gems are installed for Ruby1.9 gems: - addressable - cake - cheat - fastercsv - hpricot - htmlentities - json - kwalify - libxml-ruby - uuidtools - xml-simple # science - bio - bio-bwa - bio-gff3 - biomart #- bio-biolinux # error: uninitialized constant Syck::Syck # databases - couchdb-ruby - data_objects - dm-adjust - dm-aggregates - dm-ar-finders - dm-cli - dm-constraints - dm-core - dm-is-list - dm-is-nested_set - dm-is-remixable - dm-is-searchable - dm-is-state_machine - dm-is-tree - dm-is-versioned - dm-migrations - dm-observer - dm-tags - dm-timestamps - dm-validations - do_sqlite3 - datamapper - dm-rest-adapter - dm-serializer - dm-types # Require up to date ruby. Fail on Centos default - vagrant - mysql - do_mysql # sqlite3-ruby is renamed with sqlite3 - sqlite3 # Problems building on 1.9.1 #- dm-ferret-adapter #- dm-sweatshop #- bio-blastxmlparser #- ruby-ensembl-api ================================================ FILE: contrib/__init__.py ================================================ """Module containing user contributions """ ================================================ FILE: contrib/flavor/__init__.py ================================================ """BioLinux flavors""" ================================================ FILE: contrib/flavor/biocloudcentral/main.yaml ================================================ packages: - minimal - biocloudcentral ================================================ FILE: contrib/flavor/biopython/custom.yaml ================================================ --- bio_general: - anaconda bio_nextgen: ================================================ FILE: contrib/flavor/biopython/fabricrc.txt ================================================ # Configuration file for Fabric specifying filesystem information # This can be adjusted and passed to fab on the commandline as '-c' # # For example # # fab -f data_fabfile.py -H your_machine -c your_fabricrc.txt install_data_s3:your_biodata.yaml # -- Details about the operating system distribution = __auto__ dist_name = __auto__ # -- Installation details # Global installation directory for packages and standard programs system_install = ~/biopython # Local install directory for versioned software that will not # be included in the path by default local_install = ~/biopython/share # Global setting for using sudo; allows installation of custom packages # by non-privileged users. # *Note*: ``system_install`` needs to point to a user-writeable directory if # ``use_sudo`` is set to ``False`` use_sudo = True keep_isolated = True # Set to get CloudBioLinux customizations like setup scripts; # or set to different edition to get specific customizations #edition = cloudbiolinux # Whether to clean the system for EC2 AMI preparation; # do not set on vagrant or local machines is_ec2_image = false # Shell to be used by CBL scripts during runtime shell_config = ~/.bashrc shell = /bin/bash -i -c ================================================ FILE: contrib/flavor/biopython/main.yaml ================================================ --- # Flavor containing with minimal instructions to install tools for # running next-generation sequencing pipelines. packages: - minimal - libraries - python - ruby - bio_general - bio_alignment - bio_search - bio_nextgen libraries: - python-libs ================================================ FILE: contrib/flavor/biopython/packages-homebrew.yaml ================================================ # Packages available in the Homebrew and Linuxbrew package manager --- bio_alignment: - clustal-w - clustal-omega - mafft bio_nextgen: utilities: - bedtools analysis: - samtools ================================================ FILE: contrib/flavor/biopython/packages.yaml ================================================ --- minimal: build: - cmake - gcc - g++ - gfortran - libtool - make - patch - pkg-config util: - unzip network: - curl - wget libraries: - libzip-dev - libbz2-dev - zlib1g-dev - libcurl4-openssl-dev - libgsl0-dev - libgsl0ldbl - gsl-bin - libncurses5-dev - libpcre3-dev - libpng12-dev - libreadline-dev - libssl-dev - libxml2 - libxml2-dev - zlib1g-dev python: ruby: - ruby1.9.1 # default Ruby install - ruby1.9.1-dev bio_general: - primer3 - emboss # Broken on 13.10 - emboss-data # Broken on 13.10 - emboss-lib # broken on 13.10 bio_search: - ncbi-blast+ bio_alignment: bio_nextgen: ================================================ FILE: contrib/flavor/biopython/python-libs.yaml ================================================ --- pypi: - MySQL-python - psycopg2 - rdflib - reportlab conda: - ipython - matplotlib - numpy - pandas ================================================ FILE: contrib/flavor/boinc/__init__.py ================================================ """Boinc flavor Copyright (C) 2011 Pjotr Prins and Steffen Moeller """ ================================================ FILE: contrib/flavor/boinc/boincflavor.py ================================================ from fabric.api import * from fabric.contrib.files import * from cloudbio.flavor import Flavor from cloudbio.custom.shared import (_fetch_and_unpack) class BoincFlavor(Flavor): """A VM flavor for running Boinc """ def __init__(self, env): Flavor.__init__(self,env) self.name = "Boinc Flavor" def rewrite_config_items(self, name, packages): if name == 'packages': packages += [ 'openssh-server', 'unzip', 'tar', 'sudo' ] for package in packages: env.logger.info("Selected: "+name+" "+package) return packages def post_install(self): env.logger.info("Starting post-install") pass env.flavor = BoincFlavor(env) ================================================ FILE: contrib/flavor/boinc/fabricrc_debian.txt ================================================ edition = minimal edition_version = 0.10 flavor_path = boinc.boincflavor distribution = debian dist_name = squeeze # Update to the target's nearest repository: debian_repository=http://ftp.nl.debian.org/debian/ # Global installation directory for packages and standard programs system_install = /usr local_install = ~/install shell_config = ~/.bashrc shell = /bin/bash -l -c # data_files = /mnt/biodata ================================================ FILE: contrib/flavor/boinc/main.yaml ================================================ --- packages: # - minimal : skip, as we don't need the build system - boinc - desktop-cloud libraries: ================================================ FILE: contrib/flavor/cloudman/README.md ================================================ # CloudMan Flavors This document briefly describes the CloudMan/[Galaxy][10] flavors of CloudBioLinux available and how to configure them. For quick, small modifications to CloudMan your best bet is probably to modify an existing instance, documentation on how to do this can be found [here][9]. Documentation for building a CloudMan image and corresponding volumes and bucket from scratch using the CloudBioLinux deployer can be found [here][8]. ## Core Flavors This directory contains package lists used to setup [CloudMan][1] and/or CloudMan and [Galaxy][2]. A minimal CloudMan machine can be configured with the following command: fab -f fabfile.py -i -H ubuntu@ install_custom:cloudman A slightly more flushed out instance can be installed via the command: fab -f fabfile.py -i -H ubuntu@ install_biolinux:flavor=cloudman/cloudman Finally, CloudMan and Galaxy can be installed together via the command: fab -f fabfile.py -i -H ubuntu@ install_biolinux:flavor=cloudman/cloudman_and_galaxy You can additionally configure your CloudMan and Galaxy instance by specifying a configuration file: "-c " in the above command. A subset of the parameters you may override via this mechanism includes (see ``config/fabricrc.txt`` for a full list): * `galaxy_user` (default `galaxy`): User of Galaxy webapp * `galaxy_home` (default `/mnt/galaxyTools/galaxy-central`): Galaxy installation directory * `galaxy_tools_dir` (default `/mnt/galaxyTools/tools`): Galaxy tool's directory, formally called `install_dir` in mi-deployment's `tools_fabfile.py` file. * `galaxy_loc_files` (default `/mnt/galaxyIndices`): Directory for installation of Galaxy loc files. * `galaxy_repository` (default `https://bitbucket.org/galaxy/galaxy-central/`): This is the mercurial repository of Galaxy to install into. * `galaxy_preconfigured_repository` (default `False`): If this is `True`, CloudBioLinux will not tweak the repository pulled in via mercurial for CloudMan. This is for applications which prepackage the needed changes to Galaxy directly into the Mercurial repository. * `galaxy_cloud` (default `True`): If this is `True`, the Galaxy installation will be preconfigured for SGE and contain CloudMan branding. ## Galaxy Extensions ### Galaxy-P Flavors There are two additional flavors assembled for the [Galaxy-P][4] project. Below are some recommended fabricrc.txt overrides for Galaxy-P. * `dist_name = precise` * `galaxy_repository = https://bitbucket.org/galaxyp/galaxyp-central/` * `galaxy_tool_conf = /path/to/cloudbiolinux/contrib/flavor/proteomics/galaxyp/galaxy_tools.yaml` Side Note: Galaxy-P is actively developed, tested, and deployed with Ubuntu 12.04 LTS, so this is what to target for best results. Feel free to let me (jmchilton@gmail.com) know if there are issues or if you would like help deploying in other environments and I will attempt to help in whatever way I can. #### The Galaxy-P Server Flavor This flavor is used to build the internal and [public][5] Galaxy-P servers hosted on the OpenStack cloud at the Minnesota Supercomputing Institute, though can of course be used to build your own Galaxy-P environment. This flavor is a sort of stripped down proteomics environment for Galaxy. fab -f fabfile.py -i -H ubuntu@ install_biolinux:flavor=cloudman/cloudman_and_galaxyp This flavor additionally requires a wine environment to be packaged using the [proteomics-wine-env][7] project and made available to CloudBioLinux via the `setup_proteomics_wine_env_script` fabricrc property. #### The Galaxy-P Desktop Flavor The Galaxy-P desktop flavor builds on the server flavor with additional desktop tools and programming libraries designed to make it a broadly useful environment for mass spec data analysis even outside the context of Galaxy while also stripping out the wine related tools to avoid potential legal concerns associated with redistributing such tools. An Amazon AMI of this flavor will be available for launch on MSI's [BioCloudCentral instance][6]. fab -f fabfile.py -i -H ubuntu@ install_biolinux:flavor=cloudman/cloudman_desktop_and_galaxyp [1]: http://usecloudman.org/ [2]: http://usegalaxy.org/ [3]: http://cloudbiolinux.org/ [4]: http://getgalaxyp.org/ [5]: https://usegalaxyp.org/ [6]: https://biocloudcentral.msi.umn.edu/ [7]: https://github.com/jmchilton/proteomics-wine-env [8]: https://github.com/chapmanb/cloudbiolinux/blob/master/deploy/cloudman.md [9]: http://wiki.galaxyproject.org/CloudMan/CustomizeGalaxyCloud [10]: http://galaxyproject.org ================================================ FILE: contrib/flavor/cloudman/cloudman/main.yaml ================================================ --- packages: - minimal - cloudman libraries: ================================================ FILE: contrib/flavor/cloudman/cloudman_and_galaxy/main.yaml ================================================ --- packages: - minimal - cloudman - galaxy libraries: ================================================ FILE: contrib/flavor/cloudman/cloudman_and_galaxyp/main.yaml ================================================ --- packages: - minimal - cloudman - galaxy - galaxyp - bio_proteomics_wine libraries: ================================================ FILE: contrib/flavor/cloudman/cloudman_desktop_and_galaxyp/main.yaml ================================================ --- packages: - minimal - ruby # - python # - databases - cloudman - galaxy - galaxyp - java - desktop libraries: - ruby-libs - r-libs ================================================ FILE: contrib/flavor/cloudman/cloudman_desktop_and_galaxyp/ruby-libs.yaml ================================================ --- # Configuration file defining ruby libraries to install # using gem install (avoiding the distribution package manager) # # Gems are installed for Ruby1.9 gems: # Library for working with mass spectrometry data - mspire ================================================ FILE: contrib/flavor/cloudman/migration_checklist.md ================================================ This document is meant to layout work that is to be done and/or that has been done in merging mi-deployment and galaxy-vm-launcher into cloudbiolinux. ## Concrete TODO List: ### mi-deployment migration: * *TODETERMINE*: Is SGE configuration in cloudbiolinux up-to-date with mi-deployment? * *TODETERMINE*: Is setuptools install in mi-deployment needed with cloudbiolinux? (Seems no. -John) * *TODO*: Port mi-deployment s3fs install functionality to cloudbiolinux. * *TODETERMINE*: Is _configure_postgresql functionality needed? Seems to work without it. * *TODO*: Port mi-deployment _configure_galaxy_env functionality to cloudbiolinux. * *TODO*: Port mi-deployment _save_image_conf_support functionality to cloudbiolinx * *TODO*: Move required files for _configure_logrotate to cloudbiolinux installed_files and update setup procedure accordingly. * *TODO*: Move required files for proftpd to cloudbiolinux installed_files and update setup procedure accordingly. * *TODO*: Move nginx_errdoc.tar.gz to cloudbiolinux installed_files and update setup procedure accordingly. * *TODO*: Move required files for vimrc to cloudbiolinux installed_files and update setup procedure accordingly. * *TODO*: Port mi-deployment volume_manipulations_fab.py functionality to cloudbiolinux (if makes sense ). * *TODO*: Port mi-deployment instance-to-ebs-ami.sh functionality to cloudbiolinux (if makes sense ). * *TODO*: Port mi-deployment copy_snap/local_to_ebs_fab.py functionality to cloudbiolinux (if makes sense ). * *TODO*: Port mi-deployment wrf, graphlab, and tools/* functionality to cloudbiolinux (if makes sense ). ### galaxy-vm-launcher migration: * *TODO*: Move these procedures into cloudbiolinux - start_nginx (all disabled by default to ensure cloudman compat.). * *TODO*: Refactor deploy.py to use cloudbio/galaxy/tools instead of gvl/lib/tools.py * *TODO*: Refactor galaxy.py to install galaxy via cloudbiolinux methods. * *TODO*: Move this functionality into cloudbiolinx - galaxy database migrations (disabled by default to ensure cloudman compat.) * *TODETERMINE*: Could we move setup_taxonomy_data from gvl/lib/deploy.py into cloudbio/biodata somewhere? How has cloudman been configuring this data? ### Completed TODOs from original checklist * *TODO*: Migrate _init_postgresql_data * *TODO*: Migrate _configure_nginx_service * *TODETERMINE*: Is install_nginx in cloudbiolinux up-to-date with mi-deployment? * *TODO*: Refactor install_nginx in cloudbio.custom.cloudman so it can be used by galaxy-vm-launcher * *TODO*: Determine and implement good way to make cloudman specific parts of nginx.conf optional. * *TODO*: Port mi-deployment _configure_xvfb functionality to cloudbiolinx * *TODO*: Move this functionality into cloudbiolinx - setting up galaxy init service, log (disabled by default to ensure cloudman compat.) ## mi-deployment * data_fabfile.py - Seems all the work was already been done. * ec2autorun.py - Ported with the same default behavior as of 8/12/12, but optional user data extensions to handle more galaxy-vm-launcher style use cases. * instance-to-ebs-ami.sh - No work done? * mi-fabile.py - * apt package installations - Ported over and update to date as of 8/12/12. * setting up users - Ported over and update to date as of 8/12/12. * install nginx - Ported over at some point. TODO: Determine if the cloudbiolinux stuff is up-to-date. Extension: The cloudbiolinux version has been extended to allow parameterization of the Galaxy webapp installation directory. * Configure SGE - Ported over. TODO: Determine if the cloudbiolinux stuff is up-to-date. * Install setuptools - This doesn't seem to be needed with cloudbiolinux, at least with Ubuntu 12.04. * Install s3fs - TODO: Port this functionality. * _configure_postgresql - Is this really needed? Have the same code in galaxy-vm-launcher, but Cloudman seems to working without this cleanup. * _install_proftpd - Ported over, up-to-date as of 8/12/12. Should move configuration files to installed_files instead of mi-deployment. * _configure_ec2_autorun - Ported over and up-to-date as of 8/12/12. * _configure_sge - Ported over and up-to-date as of 8/12/12. * _configure_galaxy_env - TODO: Port this functionality. * _configure_nfs - Ported over and seems functionally equivalent as of 8/12/12. * _configure_logrotate - Files should be moved into cloudbiolinux/installed_files, but this is functionally equivalent as of 8/12/12. * _save_image_conf_support - TODO: Port this functionality. * _configure_xvfb - TODO: Port this functionality. * _configure_bash - Ported over as part of cloudbio.cloudman's _setup_env method and up-to-date * nginx.conf - Ported over (8/12/12), extended to allow parameterization of galaxy_home path. * nginx_errdoc.tar.gz - Right now this is fetched from mi-deployment, should be stored in cloudbiolinux. * tools_fabfile.py - Ported over and up-to-date as of 8/12/12. Functionality split into installing Galaxy and installing actual tool dependencies. To install Galaxy the way tools_fabfile does, install cloudman via: fab -f fabfile.py -i -H ubuntu@ install_biolinux:packagelist=./contrib/cloudman/cloudman_and_galaxy.yaml instead of : fab -f fabfile.py -i -H ubuntu@ install_custom:cloudman Several extension points on top of mi-deployment have been added (configured via fabricrc options), but the default behavior should be the same. These extensions include, allowing deployer to set: * galaxy_repository: Which repo to point at. * galaxy_preconfigured_repository: Override default of true to skip the tweaking of the installed galaxy repository that tools_fabfile does. The galaxy-vm-launcher approach is to include these changesets https://bitbucket.org/jmchilton/cloud-galaxy-dist in the repository Galaxy being installed from. * galaxy_conf_directory: Use this work https://bitbucket.org/galaxy/galaxy-central/pull-request/44/allow-usage-of-directory-of-configuration to allow overridding specific galaxy universe_wsgi.ini properties at image configuration time. Any fabric environment properties of the form: galaxy_universe_XXXX=YYYY will end up at runtime as Galaxy universe_wsgi.ini properties of the form XXXX=YYYY inside [app:main]. My extensions to CloudMan also allow setting properties via this mechanism from user data at startup time. By default, cloudbiolinux properties will have a priority of 200 and cloudman properties a priority of 400, so if there are conflicts, CloudMan's startup time properties will override those of CloudBioLinux's. The remaining functionality of installing Galaxy application and R/Bioconductor dependencies is available, but turned off by default. To enable these, update the fabric enviornment to set galaxy_install_dependencies=true and galaxy_install_r_packages. This functionality has been extended to dynamically read which packages and versions to install from the file: contrib/cloudman/tools.yaml. Multiple versions of the same tool can be installed via this mechanism. I have updated various software versions and tweaked the install methods to work under Ubuntu 12.04 for all tools. * volume_manipulations_fab.py - No work has been done on this. This file has not been updated in a while, is it still useful? * xvfb_default and xvfb_init - TODO: Port these files and mi-deployments' configure_xvfb functionality over. * conf_files/proftp.conf, conf_files/proftp.initd - Right now this is fetched from mi- deployment, should be stored in cloudbiolinux. * conf_files/vimrc - Right now this is fetched from mi-deployment, should be stored in cloudbiolinux. * conf_files/apps.yaml, conf_files/config.yaml - The cloudman and galaxy dependencies have been ported over and will be installed using cloudbiolinux's similar package management configuration mechanisms. The extensions (wrf, graphlab) have not been. * conf_files/cloudman.logrotate - File still be fetched from mi-deployment, this should be read from installed_files instead. * copy_snap/local_to_ebs_fab.py - No work has been done on this. This file has not been updated in a while, is it still useful? * tools/ - No work on the custom tools has been done. Are there more cloudbiolinux-y ways to handle these? ## galaxy-vm-launcher * lib/genomes.py - Uses cloudbiolinux alread. TODO: Just delete out of galaxy-vm-launcher. * lib/image.py - Setting up users and installing packages is good to go. install_nginx in cloudbio/custom/cloudman should be refactored to a common shared place, but still invoked in install_cloudman. A seperate nginx conf file should be optional for galaxy-vm-launcher or we should find a good way to make the cloudman paths optional (and enabled by default). _configure_postgresql - I don't think this is needed except maybe allowing overridding of postgresql.conf etc file. _init_postgresql_data, _configure_nginx_service, start_nginx - This should all be ported over but disabled by default (for cloudman compatiablity). configure_xvfb - TODO: Configure this. * lib/tools.py - Mostly merged already. galaxy-vm-launcher should be refactored to use cloudbio/galaxy/tools. * lib/galaxy.py - * Setting up galaxy and options (functionality mostly available in cloudbiolinux now gvl needs to be refactored to use it.) * Setting up galaxy service, log, and database migrations. Functionality could be move to cloudbio/galaxy and disabled by default (not compatiable with cloudman). * Seeding galaxy with data. Functionality should remain in the galaxy-vm-launcher, not really compatiable with cloudman. galaxy-vm-launcher should be refactored to use blend though and possibily some fork of blend with additional methods that directly interact with the database the way galaxy-vm-launcher does. * lib/deploy.py - File should largely remain as is. setup_taxonomy_data - Could be moved into cloudbio/biodata. Does cloudman use the metagenomics tools? How is this currently being configured? ================================================ FILE: contrib/flavor/cloudman/tools.yaml ================================================ --- applications: ## Coming from bio_nextgen.py # bwa: "0.5.9" # ToolShed installed ucsc_tools: "default" # bowtie: "0.12.7" # ToolShed installed samtools: - version: "0.1.16" # abyss: "1.3.3" # Compile error bfast: "0.7.0a" velvet: "1.2.07" # perm: "3.6" # Compile error lastz: "1.02.00" mosaik: "default" # Version ignored now, CBL pulls from trunk # freebayes: "default" # ToolShed installed macs: "1.4.2-1" ## Coming from bio_general.py emboss: - version: "6.5.7" symlink_versions: "5.0.0" # Setup a symbolic link, so when Galaxy looks for 5.0.0 it finds newest version ## Coming from applications.py ## Tried with bio_nextgen didn't work. [ubuntu@10.1.0.3] out: No package 'gtextutils' found fastx_toolkit: "0.0.13.2" ## Tried with bio_nextgen Eventually got bam to work, not eigen dependencies... ## switching to version 2.0.0 didn't help cufflinks: - "1.3.0" - "2.0.2" tophat: - "1.3.3" - "2.0.6" picard: "1.56" fastqc: "0.10.0" gatk: "2.3-9-gdcdccbb" # maq: "0.7.1" # Compile error megablast: "2.2.22" blast: "2.2.25+" sputnik: "r1" taxonomy: "r2" add_scores: "r1" # hyphy: "r418" # Compile error srma: "0.1.15" beam: "2" pass: "2.0" lps_tool: "2010.09.30" plink: "1.07" haploview: "4.2b" # eigenstrat: "3.0" # Compile error r_packages: - DBI - RColorBrewer - RCurl - RSQLite - XML - biglm - bitops - digest - ggplot2 - graph - hexbin - hwriter - kernlab - latticeExtra - leaps - pamr - plyr - proto - qvalue - reshape - statmod - xtable - yacca bioconductor_packages: - AnnotationDbi - ArrayExpress - ArrayTools - Biobase - Biostrings - DynDoc - GEOquery - GGBase - GGtools - GSEABase - IRanges - affy - affyPLM - affyQCReport - affydata - affyio - annaffy - annotate - arrayQualityMetrics - beadarray - biomaRt - gcrma - genefilter - geneplotter - globaltest - hgu95av2.db - limma - lumi - makecdfenv - marray - preprocessCore - ShortRead - siggenes - simpleaffy - snpMatrix - vsn ================================================ FILE: contrib/flavor/cwl_dockers/packages-bcbio-alignment.yaml ================================================ bcbio-process_alignment: - bwa - bwakit - novoalign - snap-aligner - samtools - sambamba - fgbio - umis - biobambam channels: - bioconda - conda-forge ================================================ FILE: contrib/flavor/demo/README.md ================================================ Small example flavor to demonstrate ability to provide local installations ================================================ FILE: contrib/flavor/demo/custom.yaml ================================================ --- bio_nextgen: ================================================ FILE: contrib/flavor/demo/fabricrc.txt ================================================ # -- Installation details # Global installation directory for packages and standard programs system_install = ~/tmp/cbl_demo # Local install directory for versioned software that will not # be included in the path by default local_install = ~/tmp/cbl_demo/share # Global setting for using sudo; allows installation of custom packages # by non-privileged users. # *Note*: ``system_install`` needs to point to a user-writeable directory if # ``use_sudo`` is set to ``False`` use_sudo = False keep_isolated = True # -- Details about reference data installation # Path where biological reference data files should be retrieved to data_files = ~/tmp/cbl_demo/biodata ================================================ FILE: contrib/flavor/demo/main.yaml ================================================ --- # Demonstration flavor for learning use packages: - minimal - libraries - python - java - ruby - bio_nextgen ================================================ FILE: contrib/flavor/demo/packages-homebrew.yaml ================================================ # Packages available in the Homebrew and Linuxbrew package manager --- bio_nextgen: alignment: - bwa utilities: - bedtools analysis: - samtools variant: - gatk-framework ================================================ FILE: contrib/flavor/edx_course/custom.yaml ================================================ --- bio_nextgen: - picard ================================================ FILE: contrib/flavor/edx_course/edx_setup.sh ================================================ #!/bin/bash set -eu -o pipefail # Vagrant installation script for creating VM to use in the # edX variant analysis course, PH525.6x: # https://www.edx.org/course/case-study-variant-discovery-genotyping-harvardx-ph525-6x # Base system sudo apt-get update sudo apt-get install -y build-essential zlib1g-dev wget curl python-setuptools git sudo apt-get install -y openjdk-7-jdk openjdk-7-jre ruby libncurses5-dev libcurl4-openssl-dev libbz2-dev unzip pigz bsdmainutils sudo apt-get install -y python-pip python-dev sudo pip install fabric # CloudBioLinux cd /vagrant sudo chown -R vagrant:vagrant /usr/local git clone https://github.com/chapmanb/cloudbiolinux.git fab -H localhost install_biolinux:flavor=edx_course # Need to hack cmake (to build 32bit): brew uninstall --force cmake && brew install cmake --without-docs # # GEMINI wget https://raw.github.com/arq5x/gemini/master/gemini/scripts/gemini_install.py python gemini_install.py /usr/local /usr/local/share/gemini --nosudo --nodata --notools # Copy snpEff hg19 file from existing bcbio installation # cleanup rm -rf ~/.cache/Homebrew rm -rf /usr/local/share/gemini/gemini /usr/local/share/gemini/anaconda/bin/conda clean --yes --tarballs sudo apt-get clean sudo rm -rf /var/lib/apt/lists/* # On VM -- Compact space: http://andrewdeponte.com/2013/10/29/shrinking-vagrant-linux-boxes.html sudo dd if=/dev/zero of=wipefile bs=1024x1024; rm wipefile # Afterwards, clone and export: # vagrant halt # virtualbox # - Select vagrant image # - Machine -> Clone to new VirtualMachine # - File -> Export as OVA ================================================ FILE: contrib/flavor/edx_course/fabricrc.txt ================================================ # Configuration file for Fabric specifying filesystem information # This can be adjusted and passed to fab on the commandline as '-c' # # For example # # fab -f data_fabfile.py -H your_machine -c your_fabricrc.txt install_data_s3:your_biodata.yaml # -- Details about the operating system ## If target machine is Ubuntu or CentOS, the following defaults should allow CloudBioLinux to ## automatically determine the correct settings. For other operating systems (e.g. Debian or ScientificLinux) ## please override distribution (and dist_name in case of Debian). distribution = __auto__ dist_name = __auto__ # -- Installation details # Whether to clean the system for EC2 AMI preparation; # do not set on vagrant or local machines is_ec2_image = false # Global installation directory for packages and standard programs system_install = /usr/local # Local install directory for versioned software that will not # be included in the path by default local_install = /usr/local/share # Shell to be used by CBL scripts during runtime shell_config = ~/.bashrc shell = /bin/bash -i -c use_sudo = False keep_isolated = True ================================================ FILE: contrib/flavor/edx_course/main.yaml ================================================ --- # Flavor containing with minimal instructions to install tools for # running next-generation sequencing pipelines. packages: - minimal - libraries - perl - python - java - r - ruby - bio_nextgen libraries: - python-libs ================================================ FILE: contrib/flavor/edx_course/packages-homebrew.yaml ================================================ # Packages available in the Homebrew and Linuxbrew package manager --- minimal: - cmake perl: bio_nextgen: alignment: - bwa utilities: - bedtools - fastqc - grabix - sambamba-binary - samblaster analysis: - samtools;--without-curses - htslib - bcftools variant: - freebayes - gatk-framework - metal-gwas - plink - snpeff - vcflib ================================================ FILE: contrib/flavor/edx_course/python-libs.yaml ================================================ --- pypi: - cutadapt ================================================ FILE: contrib/flavor/globus/main.yaml ================================================ --- packages: - minimal - globus libraries: ================================================ FILE: contrib/flavor/millstone/README.md ================================================ The Millstone flavor of cloudbiolinux eases deploying the Church Lab's [Millstone](http://churchlab.github.io/millstone/) software, a platform for genome design and analysis. This document is intended for developers. Most users will want to clone an existing, pre-packaged Amazon Machine Image (AMI), Those instructions are available at the main Millstone webpage here: . The following instructions provided steps for deploying a (possibly) modified version of Millstone to AWS. The instructions are specific to Amazon, but the steps can be modified for development in different VM environments. This document describes how to deploy an instance and create an AMI. See the main cloudbiolinux README.md for general usage instructions for cloudbiolinux. ### Usage The following steps provide instructions for configuring a master, worker, or hybrid instance of Millstone. The MASTER instance runs the main Django web server as well as the Postgresql database and hosts the central RabbitMQ messaging system for distributing computation. A WORKER instance runs a minimal web server to allow the user to connect it to the master instance, and then run asynchronous tasks as they become available. When executing a fabric install, be sure to set either `MASTER=1` or `WORKER=1`, or both in environment variables to specify the role (Master or Worker) of the VM. Setting both runs the web server and background workers on the same machine. See below for example commands. *NOTE*: The split master/worker implementation is currenlty under development. Only the combined setup has been tested. #### Steps ##### Local requirements On your local machine, install Python-Fabric (and its dep PyYAML), which are used by cloudbiolinux to manage deploying to a remote machine. pip install fabric pip install pyyaml ##### Commands to load VMs The following commands are run from the top-level cloudbiolinux directory on your local machine and convert a clean EC2 instance to running a combined, master, or worker configuration of Millstone, respectively. *NOTE*: Remember to replace both `SSH_KEY_FILE` and `ec2-xx-xx-xx-xx.compute-1.amazonaws.com` with your ec2 private key and the correct instance url. ###### Master/Worker combined VM: MASTER=1 WORKER=1 fab -i SSH_KEY_FILE -u ubuntu -H ec2-xx-xx-xx-xx.compute-1.amazonaws.com install_biolinux:flavor=millstone ###### Master VM only (IN PROGRESS) MASTER=1 fab -i SSH_KEY_FILE -u ubuntu -H ec2-xx-xx-xx-xx.compute-1.amazonaws.com install_biolinux:flavor=millstone ###### Worker VM (IN PROGRESS) WORKER=1 fab -i SSH_KEY_FILE -u ubuntu -H ec2-xx-xx-xx-xx.compute-1.amazonaws.com install_biolinux:flavor=millstone *DO NOT RESTART* if you want to package an Amazon Machine Image (AMI). Restarting will bootstrap the software and data, resulting in potentially insecure/malfunctional configurations. Immediately follow the steps in the section on Building an AMI below. ##### Connecting Workers to Master *NOTE*: Not yet supported. Draft: If you configured separate master and worker instances, you will need to tell the worker instances about the master instance. The master info is located on the master instance at: http://ec2-xx-xx-xx-xx.compute-1.amazonaws.com/ec2/info This shows its hostname and password, two important information that need to put into the worker instance. On worker instance, add the above information here: http://ec2-xx-xx-xx-xx.compute-1.amazonaws.com/worker You need to update hostname and password from the master instance, so that the worker processes can connect properly. No need to change other fields. That page then allows you to test the connection. The master will automatically put async tasks on its RabbitMQ and registered workers will consume them. Workers will write back to the master's Postgresql database. ##### Building an Amazon Machine Image (AMI) First, clean up remaining SSH keys and history files. fab -c contrib/flavor/millstone/ec2.txt -i SSH_KEY_FILE -u ubuntu -H ec2-xx-xx-xx-xx.compute-1.amazonaws.com install_biolinux:flavor=millstone,target=clean Go to EC2 Dashboard inside AWS Console <console.aws.amazon.com>. Find the instance on which we just installed our software, and select "Create Image" in "Actions" tab. ##### Creating VM from AMI Inside EC2 Dashboard, select create instance and choose AMI built in previous section. Initial bootstrap will take about five minutes for master VM and one minute for worker VM, before the VM becomes responsive. If you instantiate separate master and worker instances, follow the directions above for connecting workers to the master. ### How it works Cloudbiolinux executes a sequence of installation targets listed in `fabfile.py`. Specifically: Target `packages` installs system-wide dependencies, defined in `main.yaml` in this directory. Target `libraries` installs packages for different languages. We only use it for python packages in `python-libs.yaml`. Target `post_install` invokes `installer.py`, which defines installation procedures for millstone. The script clones the millstone repository from Github, sets up jbrowse, and writes a one-time bootstrap script `bootstrap.sh` located in clone directory set by `INSTALLATION_PATH`. `bootstrap.sh` will not run until first reboot, and will run on startup after an instance is created from AMI. It generates a new password for PostgreSQL and RabbitMQ if running on master VM. This password is saved to `password.txt` along with `bootstrap.sh`. To reconfigure, remove `BOOTSTRAPPED` file and re-run `bootstrap.sh`. Note this will generate a new password. ### Thanks Big thanks to Changping Chen (https://github.com/ccp0101) for designing and implementing the Millstone integration with cloudbiolinux. ================================================ FILE: contrib/flavor/millstone/__init__.py ================================================ ================================================ FILE: contrib/flavor/millstone/custom.yaml ================================================ --- java: [] python: [] millstone: - unafold ================================================ FILE: contrib/flavor/millstone/installer.py ================================================ from fabric.api import * from fabric.contrib.files import * import os.path import os INSTALLATION_PATH = "$HOME/millstone" REPO_URL = "git@github.com:churchlab/millstone.git" REPO_STABLE_COMMIT = "634db63de2fb275719868839bf44bd9b4b9f016e" BOOTSTRAP_SCRIPT = """#!/bin/bash set -x export RUN_MASTER=%d export RUN_WORKER=%d export PROJECT_DIR="%s" export MILLSTONE_PATH="%s" export CELERY_MANAGER_PATH="%s" export JBROWSE_PATH="%s" export TEMP_FILE_PATH="%s" export PASSWORD_FILE_PATH="%s" export BOOTSTRAP_FINISH_PATH="%s" export RUN_AS_USER="%s" export EC2_HOSTNAME=$(curl http://169.254.169.254/latest/meta-data/public-hostname) export MILLSTONE_WEB_PORT=8000 export CELERY_MANAGER_WEB_PORT=8001 genpw() { python -c "import string,random;print ''.join(random.choice(string.letters + string.digits) for x in range(10))" } cpucount() { grep -c ^processor /proc/cpuinfo } export NUM_CPU=$(cpucount) export TIMEOUT=3600 if [ "$RUN_MASTER" == "1" ]; then echo "Configuring master..." export PASSWORD=$(genpw) export RABBITMQ_USER="millstone" export POSTGRES_DB="millstone" export POSTGRES_USER="millstone" echo ${PASSWORD} > ${PASSWORD_FILE_PATH} echo "Using '${PASSWORD}' as password for PostgresSQL and RabbitMQ" /etc/init.d/rabbitmq-server stop cat > /etc/rabbitmq/rabbitmq-env.conf << EOF NODE_IP_ADDRESS=0.0.0.0 NODE_PORT=5672 NODENAME="rabbit@localhost" EOF /etc/init.d/rabbitmq-server start # Setup user in RabbitMQ and make it public. rabbitmqctl change_password guest $(genpw) RABBITMQ_USERS=$(sudo rabbitmqctl list_users -q) if [[ "${RABBITMQ_USERS}" =~ "${RABBITMQ_USER}" ]] then echo "Deleting existing ${RABBITMQ_USER} user in RabbitMQ." rabbitmqctl delete_user ${RABBITMQ_USER} fi rabbitmqctl add_user ${RABBITMQ_USER} ${PASSWORD} rabbitmqctl set_permissions -p / ${RABBITMQ_USER} ".*" ".*" ".*" /etc/init.d/rabbitmq-server restart # Setup user and database in PostgresSQL sudo -u postgres psql -U postgres -d postgres -c "DROP DATABASE IF EXISTS ${POSTGRES_DB};" sudo -u postgres psql -U postgres -d postgres -c "DROP USER IF EXISTS ${POSTGRES_USER};" sudo -u postgres psql -U postgres -d postgres -c "CREATE USER ${POSTGRES_USER} WITH PASSWORD '${PASSWORD}';" sudo -u postgres psql -U postgres -d postgres -c "CREATE DATABASE ${POSTGRES_DB};" sudo -u postgres psql -U postgres -d postgres -c "GRANT ALL PRIVILEGES ON DATABASE ${POSTGRES_DB} to ${POSTGRES_USER};" sudo -u postgres psql -U postgres -d postgres -c "ALTER USER ${POSTGRES_USER} CREATEDB;" POSTGRES_CONF=$(find /etc/postgresql -name "postgresql.conf" | head -1) echo "listen_addresses='*'" >> ${POSTGRES_CONF} PG_HDA_CONF=$(find /etc/postgresql -name "pg_hba.conf" | head -1) echo "host all all 0.0.0.0/0 md5" >> ${PG_HDA_CONF} /etc/init.d/postgresql restart LOCAL_SETTINGS_PATH="${MILLSTONE_PATH}/conf/local_settings.py" if [[ ! -e ${LOCAL_SETTINGS_PATH} ]] then echo "${LOCAL_SETTINGS_PATH} does not exist!" fi cat > ${LOCAL_SETTINGS_PATH} << EOF DATABASES = { 'default': { 'ENGINE': 'django.db.backends.postgresql_psycopg2', 'NAME': '${POSTGRES_DB}', 'USER': '${POSTGRES_USER}', 'PASSWORD': '${PASSWORD}', 'HOST': '127.0.0.1', 'PORT': '5432', 'OS_USER': 'postgres', } } BROKER_URL = 'amqp://${RABBITMQ_USER}:${PASSWORD}@127.0.0.1:5672//' EOF chown ${RUN_AS_USER} ${LOCAL_SETTINGS_PATH} echo "New local settings in ${LOCAL_SETTINGS_PATH}:" cat ${LOCAL_SETTINGS_PATH} fi chown -R ${RUN_AS_USER} ${PROJECT_DIR} echo "Reconfiguring nginx and supervisor..." /etc/init.d/supervisor stop /etc/init.d/nginx stop rm -f /etc/nginx/sites-enabled/default rm -f /etc/nginx/sites-available/millstone rm -f /etc/nginx/sites-enabled/millstone rm -f /etc/supervisor/supervisord.conf cat > /etc/nginx/sites-available/millstone << EOF server { server_name localhost; location /jbrowse { alias ${JBROWSE_PATH}; } location /tmp { alias ${TEMP_FILE_PATH}; } location /static { alias ${MILLSTONE_PATH}/main/static; } location / { proxy_pass http://127.0.0.1:${MILLSTONE_WEB_PORT}/; } location /worker/ { proxy_pass http://127.0.0.1:${CELERY_MANAGER_WEB_PORT}/; } location /worker/static { alias ${CELERY_MANAGER_PATH}/celery_manager/static; } # Override timeouts. Especially relevant to upload requests. proxy_connect_timeout ${TIMEOUT}; proxy_read_timeout ${TIMEOUT}; # No limit on upload size. client_max_body_size 0; } EOF if [ "$RUN_WORKER" == "1" ]; then echo "Configuring worker..." fi ln -s /etc/nginx/sites-available/millstone /etc/nginx/sites-enabled/millstone /etc/init.d/nginx start cat > "/etc/supervisor/supervisord.conf" << EOF [unix_http_server] file=/var/run//supervisor.sock ; (the path to the socket file) chmod=0700 ; sockef file mode (default 0700) [inet_http_server] port=127.0.0.1:9001 [supervisord] logfile=/var/log/supervisor/supervisord.log ; (main log file;default $CWD/supervisord.log) pidfile=/var/run/supervisord.pid ; (supervisord pidfile;default supervisord.pid) childlogdir=/var/log/supervisor ; ('AUTO' child log dir, default $TEMP) [rpcinterface:supervisor] supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface [supervisorctl] serverurl=http://127.0.0.1:9001 [program:millstone] command=gunicorn_django -b 127.0.0.1:${MILLSTONE_WEB_PORT} --workers=$(expr ${NUM_CPU} + 1) --timeout=${TIMEOUT} directory=${MILLSTONE_PATH} autostart=${RUN_MASTER} autorestart=True redirect_stderr=True user=${RUN_AS_USER} [program:celery_manager] command=gunicorn_django -b 127.0.0.1:${CELERY_MANAGER_WEB_PORT} --workers=$(expr ${NUM_CPU} + 1) --timeout=${TIMEOUT} directory=${CELERY_MANAGER_PATH} autostart=${RUN_WORKER} autorestart=True redirect_stderr=True user=${RUN_AS_USER} [program:celery] command=python manage.py celery worker --loglevel=info directory=${MILLSTONE_PATH} autostart=${RUN_WORKER} autorestart=True redirect_stderr=True user=${RUN_AS_USER} EOF /etc/init.d/supervisor start update-rc.d nginx defaults update-rc.d supervisor defaults if [ "$RUN_MASTER" == "1" ]; then pushd ${MILLSTONE_PATH} sudo -u ${RUN_AS_USER} python manage.py syncdb --noinput sudo -u ${RUN_AS_USER} python manage.py migrate echo "y" | sudo -u ${RUN_AS_USER} python scripts/bootstrap_data.py popd fi touch ${BOOTSTRAP_FINISH_PATH} echo "Bootstrap finished!" """ BOOTSTRAP_INVOKER_SCRIPT = """#!/bin/bash ### BEGIN INIT INFO # Provides: millstone_setup # Required-Start: $all # Required-Stop: $remote_fs $syslog # Default-Start: 2 3 4 5 # Default-Stop: 0 1 6 ### END INIT INFO export PROJECT_DIR="%s" export BOOTSTRAP_FINISH_PATH="%s" export BOOTSTRAP_SCRIPT_PATH="%s" # Carry out specific functions when asked to by the system case "$1" in start) if [[ ! -e "${BOOTSTRAP_FINISH_PATH}" ]]; then /bin/bash ${BOOTSTRAP_SCRIPT_PATH} 2>&1 > "${PROJECT_DIR}/bootstrap.log" fi ;; stop) echo "/etc/init.d/millstone_setup stop" ;; *) echo "Usage: /etc/init.d/millstone_setup {start|stop}" exit 1 ;; esac exit 0 """ def install_millstone(env): current_user = env.safe_run("echo $USER").strip() home_dir = env.safe_run("echo $HOME").strip() installation_dir = env.safe_run("echo %s" % INSTALLATION_PATH).strip() VM_MODE = { 'MASTER': os.getenv("MASTER") is not None, 'WORKER': os.getenv("WORKER") is not None, } env.logger.info("VM_MODE: %r" % VM_MODE) if env.safe_exists(installation_dir): env.logger.warning("%s already exists. Removing the directory. " % installation_dir) with cd(installation_dir): env.safe_sudo("rm -rf %s" % installation_dir) env.safe_run("mkdir -p %s" % installation_dir) env.logger.info("Installing Millstone to %s" % installation_dir) env.logger.debug("Configure SSH to ignore host checking for Github...") env.safe_run("mkdir -p %s" % os.path.join(home_dir, ".ssh")) env.safe_run("chmod 700 ~/.ssh") env.safe_run("chmod 600 ~/.ssh/*") append("~/.ssh/config", "Host github.com\n\tStrictHostKeyChecking no\n") with cd(installation_dir): env.safe_run("git clone %s %s" % (REPO_URL, installation_dir)) env.safe_run("git reset --hard %s" % REPO_STABLE_COMMIT) project_dir = installation_dir jbrowse_dir = os.path.join(project_dir, "jbrowse") genome_designer_dir = os.path.join(project_dir, "genome_designer") temp_file_dir = os.path.join(genome_designer_dir, "temp_data/tmp") celery_manager_dir = os.path.join(project_dir, "celery_manager") config_dir = os.path.join(project_dir, "config") assert env.safe_exists(project_dir) assert env.safe_exists(jbrowse_dir) assert env.safe_exists(genome_designer_dir) assert env.safe_exists(celery_manager_dir) assert env.safe_exists(config_dir) with cd(project_dir): # clone JBrowse env.safe_run("git submodule update --init --recursive") with cd(jbrowse_dir): # Setup JBrowse. env.safe_run("./setup.sh") # Install millstone python requirements. python_requirements_file = os.path.join(installation_dir, 'requirements', 'deploy.txt') env.safe_sudo("pip install -r %s" % python_requirements_file) with cd(genome_designer_dir): env.safe_run("ln -s ../jbrowse jbrowse") env.safe_run("./millstone_setup.py") run_worker = 1 if VM_MODE['WORKER'] else 0 run_master = 1 if VM_MODE['MASTER'] else 0 bootstrap_script_path = os.path.join(project_dir, "bootstrap.sh") bootstrap_finish_path = os.path.join(project_dir, "BOOTSTRAPPED") bootstrap_script = BOOTSTRAP_SCRIPT % (run_master, run_worker, project_dir, genome_designer_dir, celery_manager_dir, jbrowse_dir, temp_file_dir, os.path.join(project_dir, "password.txt"), bootstrap_finish_path, current_user) append(bootstrap_script_path, bootstrap_script) env.safe_run("chmod +x %s" % bootstrap_script_path) """ /etc/init.d/millstone_setup will check if bootstrap script has run before, and execute bootstrap if not. """ append("/etc/init.d/millstone_setup", BOOTSTRAP_INVOKER_SCRIPT % (project_dir, bootstrap_finish_path, bootstrap_script_path), use_sudo=True) env.safe_sudo("sudo chmod +x /etc/init.d/millstone_setup") env.safe_sudo("sudo update-rc.d millstone_setup defaults") ================================================ FILE: contrib/flavor/millstone/main.yaml ================================================ --- # NOTE: rabbitmq and postgres installed manually. packages: - minimal - libraries - amazon - python - ruby - perl - java - nginx - supervisor - coffeescript - millstone libraries: - python-libs ================================================ FILE: contrib/flavor/millstone/millstoneflavor.py ================================================ """ Millstone flavor. """ from fabric.api import * from fabric.contrib.files import * from cloudbio.custom import shared from cloudbio.flavor import Flavor from installer import install_millstone class MillstoneFlavor(Flavor): def __init__(self, env): Flavor.__init__(self, env) self.name = "Millstone Flavor" def rewrite_config_items(self, name, packages): return packages def post_install(self): # Manually install logrotate, postgresql-9.3, rabbitmq-server. # postgresql-9.3 and rabbitmq-server require incompatible versions of # logrotate by default, so we force install the working version. env.safe_sudo("apt-get -y --force-yes install logrotate=3.7.8-6ubuntu5") env.safe_sudo("apt-get -y --force-yes install postgresql-9.3") env.safe_sudo("apt-get -y --force-yes install libpq-dev") env.safe_sudo("apt-get -y --force-yes install pgdg-keyring") env.safe_sudo("apt-get -y --force-yes install rabbitmq-server") # Now install psycogpg2, which requires postgresql-9.3 and related # to have been installed first. env.safe_sudo("{0} install --upgrade {1}".format(shared._pip_cmd(env), 'psycopg2')) # Finally, install Millstone. install_millstone(self.env) env.flavor = MillstoneFlavor(env) ================================================ FILE: contrib/flavor/millstone/python-libs.yaml ================================================ --- # Millstone libaries installed from millstone's requirements/deploy.txt pypi: [] ================================================ FILE: contrib/flavor/minimal/fabricrc_debian.txt ================================================ # BioNode configuration file for Fabric specifying filesystem information # # For example, to configure a minimal install with vagrant (for VirtualBox), # assuming you have a configured ./Vagrantfile # # fab -H vagrant -f $source/fabfile.py -c $source/contrib/flavor/minimal/fabricrc.txt install_biolinux:packagelist=$source/contrib/flavor/minimal/main.yaml # # Updated by Pjotr Prins # edition_version = 0.93 distribution = __auto__ dist_name = __auto__ nixpkgs = False # Update to the target's nearest repository: debian_repository=http://ftp.nl.debian.org/debian/ # Global installation directory for packages and standard programs system_install = /usr/local local_install = /usr/local/share shell_config = ~/.bashrc shell = /bin/bash -i -c use_sudo = True is_ec2_image = false # data_files = /mnt/biodata ================================================ FILE: contrib/flavor/minimal/main.yaml ================================================ --- # Top level configuration file that specifies which groups of programs # should be installed. New sections that are added to individual config # files should go here. Comment out any groups you don't want to have # installed. # # This is the minimal version to kick-off development, including editors, # version control, and basic build support (GNU tools) packages: - minimal - ruby libraries: ================================================ FILE: contrib/flavor/neuro/__init__.py ================================================ """Neuroinformatics flavor """ ================================================ FILE: contrib/flavor/neuro/custom.yaml ================================================ --- bio_nextgen: ================================================ FILE: contrib/flavor/neuro/fabricrc.txt ================================================ edition = minimal flavor_path = neuro.neuro distribution = ubuntu user = vagrant # Global installation directory for packages and standard programs system_install = /usr/local local_install = ~/install shell_config = ~/.bashrc shell = /bin/bash -l -c # data_files = /mnt/biodata ================================================ FILE: contrib/flavor/neuro/main.yaml ================================================ --- packages: - neuro libraries: ================================================ FILE: contrib/flavor/neuro/neuro.py ================================================ from fabric.api import * from fabric.contrib.files import * from cloudbio.flavor import Flavor # This flavour installs several neuroinformatics software from NeuroDebian # # http://neuro.debian.net/ # # Author: Roman Valls Guimera class NeuroFlavor(Flavor): """ A flavour for installing NeuroDebian. A debian neuroinformatics repository of software and datasets. """ def __init__(self, env): self.name = "Neuroinformatics Flavor" def rewrite_config_items(self, name, packages): if name == 'packages': packages.extend([ "git" ]) return packages def rewrite_apt_sources_list(self, name, sources): sources = [ 'deb http://neurodeb.pirsquared.org data main contrib non-free', '#deb-src http://neurodeb.pirsquared.org data main contrib non-free', 'deb http://neurodeb.pirsquared.org saucy main contrib non-free', '#deb-src http://neurodeb.pirsquared.org saucy main contrib non-free' ] #wget -O- http://neuro.debian.net/lists/saucy.us-ca.full | sudo tee /etc/apt/sources.list.d/neurodebian.sources.list #sudo apt-key adv --recv-keys --keyserver pgp.mit.edu 2649A5A9 #env.flavor = NeuroFlavor(env) ================================================ FILE: contrib/flavor/neuro/packages.yaml ================================================ --- neuro: - afni - afni-atlases - afni-common - afni-dbg - afni-dev - aghermann - ants - arno-iptables-firewall - autotools-dev - biosig-tools - caret - caret-data - cde - cgroup-bin - cmtk - cnrun - condor - condor-dbg - condor-dev - condor-doc - connectomeviewer - coop-computing-tools - coop-computing-tools-dev - coop-computing-tools-doc - cython - cython-dbg - debhelper - debian-handbook - debootstrap - debootstrap-udeb - debruijn - dh-autoreconf - dh-python - dh-systemd - dicomnifti - dmtcp - dmtcp-dbg - eatmydata - edac-utils - eegdev-plugins-free - eeglab11-sampledata - eegview - fail2ban - fis-gtm - fis-gtm-6.0-003 - freeipmi - freeipmi-bmc-watchdog - freeipmi-common - freeipmi-ipmidetect - freeipmi-tools - freenect - fsl - fsl-5.0 - fsl-5.0-complete - fsl-5.0-core - fsl-5.0-doc - fsl-5.0-doc-wikiattachments - fsl-5.0-gpu - fsl-atlases - fsl-bangor-cerebellar-atlas - fsl-complete - fsl-core - fsl-doc-5.0 - fsl-feeds - fsl-first-data - fsl-harvard-oxford-atlases - fsl-harvard-oxford-cortical-lateralized-atlas - fsl-jhu-dti-whitematter-atlas - fsl-juelich-histological-atlas - fsl-mni-structural-atlas - fsl-mni152-templates - fsl-neurosynth-atlas - fsl-neurosynth-top100-atlas - fsl-oxford-striatal-atlas - fsl-oxford-thalamic-connectivity-atlas - fsl-possum-data - fsl-subthalamic-nucleus-atlas - fsl-talairach-daemon-atlas - fslview - fslview-doc - gdf-tools - gifti-bin - glew-utils - gmsl - guacamole - guacamole-tomcat - guacd - haxby2001-faceobject - haxby2001-faceobject-subject1 - haxby2001-faceobject-subject2 - haxby2001-faceobject-subject3 - haxby2001-faceobject-subject4 - haxby2001-faceobject-subject5 - haxby2001-faceobject-subject6 - impressive - incf-nidash-oneclick-clients - init-system-helpers - insighttoolkit4-examples - ipython01x - ipython01x-doc - ipython01x-notebook - ipython01x-parallel - ipython01x-qtconsole - ipython1x - ipython1x-doc - ipython1x-notebook - ipython1x-parallel - ipython1x-qtconsole - isis-utils - itksnap - jist - kbibtex - klustakwik - libbiosig-dev - libbiosig1 - libbiosig1-dbg - libcgroup-dev - libcgroup1 - libclassad-dev - libclassad3 - libdmtcpaware-dev - libdmtcpaware1 - libdouble-conversion-dbg - libdouble-conversion-dev - libdouble-conversion1 - libdrawtk-dev - libdrawtk0 - libdrawtk0-dbg - libedac-dev - libedac1 - libedac1-dbg - libeegdev-dev - libeegdev0 - libeegdev0-dbg - libeigen3-dev - libeigen3-doc - libfreeipmi-dev - libfreeipmi12 - libfreenect-bin - libfreenect-demos - libfreenect-dev - libfreenect-doc - libfreenect0.1 - libgdf-dev - libgdf0 - libgdf0-dbg - libgiftiio-dev - libgiftiio0 - libglew1.9 - libglew1.9-dbg - libglew1.9-dev - libglewmx1.9 - libglewmx1.9-dbg - libglewmx1.9-dev - libgoogle-glog-dev - libgoogle-glog-doc - libgoogle-glog0 - libguac-client-rdp0 - libguac-client-ssh0 - libguac-client-vnc0 - libguac-dev - libguac3 - libguac5 - libinsighttoolkit4-dev - libinsighttoolkit4.2 - libinsighttoolkit4.5 - libipmiconsole-dev - libipmiconsole2 - libipmidetect-dev - libipmidetect0 - libipmimonitoring-dev - libipmimonitoring5 - libisis-core-dev - libisis-core0 - libisis-ioplugins-common - libisis-ioplugins-dicom - libisis-qt4-0 - libisis-qt4-dev - libjs-underscore - libmcpanel-dev - libmcpanel0 - libmcpanel0-dbg - libmia-2.0-8 - libmia-2.0-8-dbg - libmia-2.0-dev - libmia-2.0-doc - libmialm-dev - libmialm-doc - libmialm3 - libmialm3-dbg - libmtcp-dev - libmtcp1 - libnifti-dev - libnifti-doc - libnifti2 - libnlopt-dev - libnlopt-guile0 - libnlopt0 - libodin-dev - libopenmeeg-dev - libopenmeeg1 - libopenwalnut1 - libopenwalnut1-dev - libopenwalnut1-doc - liborthancclient-dev - liborthancclient-doc - liborthancclient0.7 - libpam-cgroup - libpgm-5.1-0 - libpgm-dbg - libpgm-dev - librtfilter-dev - librtfilter1 - librtfilter1-dbg - libshogun-dev - libshogun11 - libslicer3 - libslicer3-dev - libsvm-dev - libsvm-java - libsvm-tools - libsvm3 - libsvm3-java - libusb-1.0-0 - libusb-1.0-0-dev - libusb-1.0-0-udeb - libvia-dev - libvia-doc - libvia2 - libvistaio-dev - libvistaio14 - libvistaio14-dbg - libvrpn-dev - libvrpn0 - libvrpnserver0 - libvtk-java - libvtk5-dev - libvtk5-qt4-dev - libvtk5.8 - libvtk5.8-qt4 - libvw-dev - libvw0 - libxdffileio-dev - libxdffileio0 - libxdffileio0-dbg - libzmq-dbg - libzmq-dev - libzmq1 - lipsia - lipsia-doc - matlab-eeglab11 - matlab-gdf - matlab-psychtoolbox-3 - matlab-psychtoolbox-3-nonfree - matlab-spm8 - matlab-support - matlab-support-dev - mclaren-rhesus-macaque-atlas - mia-tools - mia-tools-dbg - mia-tools-doc - mialmpick - mialmpick-dbg - mipav - mitools - mni-colin27-minc - mni-colin27-nifti - mni-icbm152-nlin-2009a - mni-icbm152-nlin-2009b - mni-icbm152-nlin-2009c - mriconvert - mricron - mricron-data - mricron-doc - mrtrix - mrtrix-doc - mwrap - neurodebian-desktop - neurodebian-dev - neurodebian-guest-additions - neurodebian-keyring - neurodebian-popularity-contest - neurosynth-dataset - nifti-bin - nifti2dicom - nifti2dicom-data - nipy-suite - nipy-suite-doc - nuitka - numdiff - octave-biosig - octave-gdf - octave-nlopt - octave-psychtoolbox-3 - octave-psychtoolbox-3-nonfree - odin - openelectrophy - openmeeg-tools - opensesame - openvibe-bin - openvibe-data - openvibe-dev - openvibe-libs - openvibe-plugins - openwalnut-modules - openwalnut-qt4 - orthanc - packaging-tutorial - psychopy - psychtoolbox-3-common - psychtoolbox-3-dbg - psychtoolbox-3-lib - python-biosig - python-brian - python-brian-doc - python-brian-lib - python-cfflib - python-dicom - python-dipy - python-dipy-doc - python-dipy-lib - python-expyriment - python-freenect - python-isis - python-joblib - python-lazyarray - python-libsvm - python-mdp - python-mlpy - python-mlpy-doc - python-mlpy-lib - python-mne - python-mpi4py - python-mpi4py-dbg - python-mpi4py-doc - python-mvpa - python-mvpa-doc - python-mvpa-lib - python-mvpa2 - python-mvpa2-doc - python-mvpa2-lib - python-mvpa2-tutorialdata - python-neo - python-networkx - python-networkx-doc - python-neuroshare - python-neurosynth - python-nibabel - python-nibabel-doc - python-nibabel-snapshot - python-nifti - python-nipy - python-nipy-doc - python-nipy-lib - python-nipy-lib-dbg - python-nipype - python-nipype-doc - python-nitime - python-nitime-doc - python-nlopt - python-numexpr - python-numpydoc - python-openmeeg - python-openopt - python-openpyxl - python-pandas - python-pandas-lib - python-patsy - python-patsy-doc - python-pp - python-pprocess - python-pyentropy - python-pyepl - python-pyepl-common - python-pyglet - python-pymc - python-pymc-doc - python-pynn - python-pyo - python-pyoptical - python-pypsignifit - python-pyssdh - python-pyxid - python-pyxnat - python-quantities - python-scikits-learn - python-scikits-statsmodels - python-scikits-statsmodels-doc - python-scikits.statsmodels - python-scipy - python-scipy-dbg - python-simplegeneric - python-skimage - python-skimage-doc - python-skimage-lib - python-sklearn - python-sklearn-doc - python-sklearn-lib - python-sphinx - python-spyderlib - python-spykeutils - python-statsmodels - python-statsmodels-doc - python-statsmodels-lib - python-stfio - python-surfer - python-sympy - python-tornado - python-traits4 - python-tz - python-visionegg - python-vtk - python-workqueue - python-zmq - python-zmq-dbg - python3-dateutil - python3-mdp - python3-mpi4py - python3-mpi4py-dbg - python3-pandas - python3-pandas-lib - python3-patsy - python3-skimage - python3-skimage-lib - python3-tz - qlandkarte - qlandkartegt - qlandkartegt-garmin - qnifti2dicom - rdfind - remake - rorden-mri-tutorial - rorden-mri-tutorial-data - shogun-cmdline-static - shogun-csharp-modular - shogun-dbg - shogun-doc-cn - shogun-doc-en - shogun-elwms-static - shogun-java-modular - shogun-lua-modular - shogun-python-modular - shogun-python-static - shogun-r-static - shogun-ruby-modular - sigviewer - slicer - slicer-data - spm8-common - spm8-data - spm8-doc - spyder - spykeviewer - sri24-atlas - stabilitycalc - stimfit - stimfit-dbg - svgtune - tcl-vtk - testkraut - tigervnc-common - tigervnc-scraping-server - tigervnc-standalone-server - tigervnc-viewer - tigervnc-xorg-extension - ubuntu-keyring - ubuntu-keyring-udeb - via-bin - virtual-mri-nonfree - vowpal-wabbit - vowpal-wabbit-dbg - vowpal-wabbit-doc - voxbo - vrpn - vrpn-dbg - vtk-doc - vtk-examples - xmhtml1 - xmhtml1-dev - xppaut libraries: ================================================ FILE: contrib/flavor/ngs_pipeline_minimal/custom.yaml ================================================ --- bio_nextgen: alignment: utilities: analysis: variant: ================================================ FILE: contrib/flavor/ngs_pipeline_minimal/main.yaml ================================================ --- # Flavor containing with minimal instructions to install tools for # running next-generation sequencing pipelines. packages: - minimal - libraries - perl - python - java - r - ruby - bio_nextgen libraries: - r-libs ================================================ FILE: contrib/flavor/ngs_pipeline_minimal/packages-conda.yaml ================================================ --- # some ideas how to debug solving issues # not pinning is best - easier to solve, assuming latest possible versions # sometimes pinning means the latest version, say we update the old one by pinning package>=2.0 # in 3 years that pinning pushes the installation down (it is still calculated as lower boundary) # don't duplicate packages - if a package is in bcbio bioconda recipe, delete it from here # mind r-base / python / htslib / java differences - their versions create separate solution spaces # the goal is to make one big main enviroment and isolate problematic packages channels: - conda-forge - bioconda minimal: - anaconda-client - awscli - bzip2 - ncurses - nodejs - p7zip - readline - s3gof3r - xz # currently not included to reduce total size of Docker images # - pandoc bio_nextgen: # default: python3.7.10 htslib1.13 # htslib1.14 is too fresh - solving issues - age-metasv - atropos - bamtools - bamutil # installs samtools, a lot of xorg, openjdk11.0.9.1 - a candidate for isolation - bbmap # wants to update htslib if unrestricted # installs perl 5.32.1 - bcftools=1.13 - bedops # installed by bcbio bioconda recipe # 2.30.0 + pybedtools # - bedtools # installs boost, curl, libmaus # see issue 3650 - biobambam=2.0.87 - bowtie - break-point-inspector - bwa # installs ruby - bio-vcf - cage # downgrades joblib - cnvkit - coincbc - cramtools # cyvcf2 is installed from bcbio-nextgen.yaml # debugpy=1.4.1 may cause solve issues # - debugpy=1.4.0=py37he226ad3_0 - deeptools - express - fastp # tmp pin adapt the wrapper to the new version - fastqc=0.11.9 - geneimpacts # installs libgcc 7.20 - genesplicer - gffcompare - goleft - grabix - gsort - gsutil - gvcfgenotyper - h5py=3.3 - hdf5=1.10 - hmmlearn - hisat2 # installed by bcbio bioconda recipe # - htslib=1.13 - htseq - impute2 # kallisto 0.46 downgrades h5py to 3.3.0, to hdf5 1.10.6, kallisto is not working with 3.4.0/1.12.1 - kallisto=0.46 # downgrades perl to 5.26.2 - kraken - ldc # installed by bcbio bioconda recipe = 7.79.1 required by pysam and mamba # - libcurl - macs2 - maxentscan - mbuffer - minimap2 - mintmap - mirdeep2 - mirtop - moreutils - multiqc - multiqc-bcbio - ngs-disambiguate - novoalign - oncofuse - pandoc - parallel - pbgzip - peddy # installed by bcbio conda recipe #- pysam>=0.14.0 - pizzly # TODO: re-enable preseq but it pulls in openblas which causes problems with other blas libraries # - preseq - pythonpy - qsignature - rapmap # installed by optitype # - razers3 - rtg-tools - sailfish - salmon - samblaster - samtools=1.13 - scalpel # old versions numbered as 2015.x new versions - 1.3 - seq2c<2016 - seqbuster - seqcluster - seqtk - sickle-trim - simple_sv_annotation - singlecell-barcodes - snap-aligner=1.0dev.97 - snpeff=5.0 - solvebio # downgrades openmp - spades # installed #- staden_io_lib - star=2.6.1d - stringtie - subread - survivor - tophat-recondition - tdrmapper - trim-galore - ucsc-bedtobigbed - ucsc-bigbedinfo - ucsc-bigbedsummary - ucsc-bigbedtobed - ucsc-bedgraphtobigwig - ucsc-bigwiginfo - ucsc-bigwigsummary - ucsc-bigwigtobedgraph - ucsc-bigwigtowig - ucsc-fatotwobit - ucsc-gtftogenepred - ucsc-liftover - ucsc-wigtobigwig - umis - wham # old versions numbered as 2015.x new versions - 1.8.2 # ok with r installed, without r installs r3.2.2 - vardict<=2015 - vardict-java - variantbam - varscan - vcfanno - viennarna # a lot of dependencies including QT - separate env? - vqsr_cnn # corrects bcbio install, lift up to bcbio conda recipe ipython: # noarch - ipython-cluster-helper=0.6.4=py_0 - ipython_genutils=0.2.0=py37_0 # 5x fails - traitlets=4.3.3 # 7x has different interface - ipyparallel=6.3.0 # <6.0.0 pin in requirements isn't applied - ipython=7.29.0 #installs many more perl dependencies perl: - perl-app-cpanminus - perl-archive-extract - perl-archive-zip # perl-bio-db-sam is deprecated - perl-bio-samtools - perl-cgi - perl-dbi - perl-encode-locale - perl-file-fetch - perl-file-sharedir - perl-file-sharedir-install - perl-ipc-system-simple - perl-lwp-simple - perl-lwp-protocol-https # removed due to the perl upgrade # needs to be upgraded from https://github.com/cancerit/cgpBattenberg # perl-sanger-cgp-battenberg - perl-statistics-descriptive - perl-time-hires - perl-vcftools-vcf r: # default r=4.1 SI=1.22 VA=1.38 htslib1.13 # r4.0 pushes htslibto 1.10 - r=4.1=r41hd8ed1ab_1004 # yes r is two digit, rbase is three # another 4.1.1 rbase pushed down htslib - r-base=4.1.1=hb67fd72_0 - xorg-libxt - r-chbutils - r-devtools - r-dplyr - r-dt - r-janitor - r-knitr - r-ggdendro - r-ggplot2 - r-ggrepel - r-gplots - r-gsalib - r-pheatmap - r-plyr - r-pscbs - r-reshape - r-rmarkdown - r-rsqlite - r-sleuth - r-snow - r-stringi - r-viridis - r-wasabi - bioconductor-annotate - bioconductor-apeglm - bioconductor-biocgenerics - bioconductor-biocinstaller - bioconductor-biocstyle - bioconductor-biostrings # 1.22.0 requires r4.1 which gives more solve issues # 1.20.0 = r40_0 - bioconductor-summarizedexperiment - bioconductor-biovizbase - bioconductor-bsgenome.hsapiens.ucsc.hg19 - bioconductor-bsgenome.hsapiens.ucsc.hg38 - bioconductor-bubbletree - bioconductor-cn.mops - bioconductor-copynumber - bioconductor-dnacopy - bioconductor-degreport - bioconductor-deseq2 - bioconductor-dexseq - bioconductor-genomeinfodbdata - bioconductor-genomicranges - bioconductor-iranges - bioconductor-limma # 2.38.0 breaks wasabi! - bioconductor-rhdf5 - bioconductor-rtracklayer # no bioconductor-snpchip in bioconda anymore # bioconductor-snpchip needed for the TitanCNA wrapper but not listed as a depedency in the formula - bioconductor-titancna - bioconductor-vsn #- r-optparse=1.7.1=r40hc72bb7e_0 - r-optparse - bioconductor-purecn>=2.0.1 - bioconductor-txdb.hsapiens.ucsc.hg38.knowngene - bioconductor-org.hs.eg.db - bioconductor-genomeinfodb - bioconductor-genomeinfodbdata - bioconductor-rtracklayer - bioconductor-txdb.hsapiens.ucsc.hg19.knowngene - r-tidyverse - bioconductor-tximport - r-deconstructsigs r35: # for arriba, old r dependencies htslib1.9, # libtiff -> libdeflate nightmare - r=3.5.1;env=r35 - r-base=3.5.1;env=r35 - arriba=1.2.0;env=r35 rbcbiornaseq: - r-bcbiornaseq>=0.5.1;env=rbcbiornaseq # openjdk 8 - waiting for gatk 4.4.0 release in conda to switch to java17 java: - fgbio;env=java - gatk;env=java # fixes log4j - gatk4=4.2.5.0;env=java - picard=2.27.5;env=java # lots of dependencies # try removing gridss and hmftools # - gridss;env=java # - hmftools-amber;env=java # - hmftools-cobalt;env=java # - hmftools-purple;env=java - qualimap;env=java - bcbio-prioritize;env=java - bcbio-variation;env=java # does not work with java11 - bcbio-variation-recall;env=java htslib1.12: - htslib=1.12;env=htslib1.12 - octopus>=0.7.4;env=htslib1.12 htslib1.11: - htslib=1.11;env=htslib1.11 - scramble;env=htslib1.11 htslib1.10: - htslib=1.10;env=htslib1.10 - ataqv;env=htslib1.10 - mosdepth;env=htslib1.10 - hts-nim-tools;env=htslib1.10 - pindel;env=htslib1.10 - ensembl-vep=104;env=htslib1.10 - perl-bio-db-hts>=3.01;env=htslib1.10 - cancerit-allelecount;env=htslib1.10 - delly;env=htslib1.10 - freebayes;env=htslib1.10 - vcflib;env=htslib1.10 - verifybamid2;env=htslib1.10 # this isn't working for vt normalize #- vt;env=htslib1.10 htslib1.12_py3.9: - htslib=1.12;env=htslib1.12_py3.9 - lofreq;env=htslib1.12_py3.9 - sambamba;env=htslib1.12_py3.9 - chipseq-greylist;env=htslib1.12_py3.9 htslib1.9: - htslib=1.9;env=htslib1.9 - duphold;env=htslib1.9 - extract-sv-reads;env=htslib1.9 # stall packages python3.6: # pushes down mamba, libtiff, but not hts - bowtie2;env=python3.6 # uses specific pysam - crossmap;env=python3.6 - cufflinks;env=python3.6 - pyloh;env=python3.6 # vt is tricky, the only combination that works for vt normalize is - samtools=1.7;env=python3.6 - htslib=1.7;env=python3.6 - vt=2015.11.10=he941832_3;env=python3.6 # not working in the base py3.7 - optitype;env=python3.6 - razers3;env=python3.6 - pysam=0.15.4;env=python3.6 - pandas=0.25.3;env=python3.6 #htslib/samtools 1.10 python2: - bismark;env=python2 - cpat;env=python2 - cutadapt=1.16;env=python2 # by default 0.30.11 is installed - causes issues with blob gts fields in vcf2db.py/gemini - cyvcf2=0.10.0;env=python2 - dkfz-bias-filter;env=python2 - gemini;env=python2 - gvcf-regions;env=python2 - hap.py;env=python2 - htslib=1.10;env=python2 - lumpy-sv;env=python2 - manta;env=python2 - metasv;env=python2 - mirge;env=python2 - phylowgs;env=python2 - platypus-variant;env=python2 - samtools=1.10;env=python2 - sentieon;env=python2 - smcounter2;env=python2 - smoove;env=python2 - strelka;env=python2 - svtools;env=python2 - svtyper;env=python2 - theta2;env=python2 - tophat;env=python2 - vawk;env=python2 - vcf2db;env=python2 - numpy=1.16;env=python2 # for dkfz filter # downgrades metasv0.4.0 and libdeflate1.3 - pysam=0.15.3;env=python2 samtools0: # rbase4.1.1 ncurses5.9. samtools0.1.19 - ericscript;env=samtools0 # samtools 1.7 bwakit: - bwakit;env=bwakit # python3.6 dv: - deepvariant;env=dv ================================================ FILE: contrib/flavor/ngs_pipeline_minimal/packages-homebrew.yaml ================================================ # Packages available in the Homebrew and Linuxbrew package manager --- # Migrated to use conda/bioconda builds to_remove: - age-metasv - bamtools - bcbio-variation-recall - bcftools - bedtools - biobambam - bowtie2 - bwa - bwakit - cancerit-allelecount - cramtools - cufflinks-binary - delly - express-binary - fastqc - freebayes - gatk-framework - grabix - hall-lab-sv-tools - htslib - impute2 - kraken - lofreq - lumpy-sv - manta - novoalign - oncofuse - p7zip - picard-tools - pindel - platypus-variant - qualimap - sambamba-binary - samtools-library-0.1 - samblaster - samtools - sickle - snpeff - spades-binary - theta2 - tophat-binary - qsignature - rtg-tools - scalpel - seqbuster - seqtk - staden_io_lib - wham - vardict - vardict-java - vcflib - vep - vt minimal: perl: bio_nextgen: old-unused: # - staden_io_lib # Migrated to samtools indexing # - express-binary # No longer used with Sailfish # - glia # Needs updates to fsom # - qsnp # Issues with download and reliability of qcmg site # - vcftools # Build problems with default libz on Ubuntu 12.04 ================================================ FILE: contrib/flavor/ngs_pipeline_minimal/perl-libs.yaml ================================================ --- cpan: # mirdeep2 # - PDF::API2 # ## VEP dependencies # don't need with conda perl # - parent # - Archive::Tar # - Module::Load::Conditional # - IPC::Cmd # - Params::Check # ## Migrated to conda # - Encode::Locale # - Archive::Extract # - Archive::Zip # - Statistics::Descriptive # - Compress::Raw::Zlib # - DBI # - LWP::Simple # - LWP::Protocol::https # - File::Fetch # - File::ShareDir # - File::ShareDir::Install # - Time::HiRes # - IPC::System::Simple # ## Sanger Cancer library dependencies # - Bio::DB::Sam;--config lddlflags=-shared;SAMTOOLS={system_install}/share/samtools-0.1 # - Vcf==0.953==https://github.com/chapmanb/vcftools-cpan/archive/v0.953.tar.gz # Sanger Cancer library dependencies -- to migrate #- Sanger::CGP::Vcf==1.2.3==https://github.com/cancerit/cgpVcf/archive/v1.2.3.tar.gz #- Sanger::CGP::AlleleCount==2.1.0==https://github.com/cancerit/alleleCount/archive/v2.1.0.tar.gz==build=perl #- PCAP==1.8.1==https://github.com/ICGC-TCGA-PanCancer/PCAP-core/archive/v1.8.1.tar.gz==depremove=GD #- Sanger::CGP::Battenberg==1.3.2_07==https://github.com/cancerit/cgpBattenberg/archive/92cd3a2.tar.gz==build=perl ================================================ FILE: contrib/flavor/ngs_pipeline_minimal/r-libs.yaml ================================================ --- cranrepo: http://cran.fhcrc.org/ cran: cran-after-bioc: #biocrepo: http://bioconductor.org/biocLite.R bioc: github: #github: # - genome/bmm@b49c57a # - chapmanb/sciClone@e8ed9d0 ================================================ FILE: contrib/flavor/phylogeny/__init__.py ================================================ """Phylogeny flavor Copyright (C) 2011 Pjotr Prins """ ================================================ FILE: contrib/flavor/phylogeny/fabricrc_debian.txt ================================================ # Flavor targetting runningy Phylogeny programs in the Cloud, # such as recent edititions of mrbayes-mpi, raxml, BEAST and # tracer. # # This VM is used for a practical Phylogeny course. # # Copyright (C) 2011 Pjotr Prins # edition = bionode edition_version = 0.93 flavor_path = phylogeny.phylogenyflavor distribution = debian dist_name = squeeze nixpkgs = True # Update to the target's nearest repository: debian_repository=http://ftp.nl.debian.org/debian/ # Global installation directory for packages and standard programs system_install = /usr local_install = ~/install shell_config = ~/.bashrc shell = /bin/bash -l -c # data_files = /mnt/biodata ================================================ FILE: contrib/flavor/phylogeny/fabricrc_ubuntu.txt ================================================ # Flavor targetting runningy Phylogeny programs in the Cloud, # such as recent edititions of mrbayes-mpi, raxml, BEAST and # tracer. # # This VM is used for a practical Phylogeny course. # # Copyright (C) 2011 Pjotr Prins # edition = bionode edition_version = 0.93 flavor_path = phylogeny.phylogenyflavor distribution = ubuntu dist_name = natty # nixpkgs = True # Update to the target's nearest repository: # debian_repository=http://ftp.nl.debian.org/debian/ # Global installation directory for packages and standard programs system_install = /usr local_install = ~/install shell_config = ~/.bashrc shell = /bin/bash -l -c # data_files = /mnt/biodata ================================================ FILE: contrib/flavor/phylogeny/install_debian.sh ================================================ #! /bin/sh # # Install the biolinux-phylogeny on a host # # Usage: # # ./contrib/flavor/phylogeny/install_host.sh user@hostname # # where # # user: the password-less login name (see doc/private_cloud.md) # hostname: the hostname, or IP # # # Copyright (C) 2011 Pjotr Prins host=$1 if [ ! -e fabfile.py -o -z $host ]; then echo Usage: echo echo ./contrib/flavor/phylogeny/install_host.sh user@hostname fi source=`pwd` fabricrc=$source/contrib/flavor/phylogeny/fabricrc_debian.txt packagelist=$source/contrib/flavor/phylogeny/main.yaml fab -f $source/fabfile.py -H $host -c $fabricrc install_biolinux:packagelist=$packagelist ================================================ FILE: contrib/flavor/phylogeny/install_ubuntu.sh ================================================ #! /bin/sh # # Install the biolinux-phylogeny on a host # # Usage: # # ./contrib/flavor/phylogeny/install_host.sh user@hostname # # where # # user: the password-less login name (see doc/private_cloud.md) # hostname: the hostname, or IP # # # Copyright (C) 2011 Pjotr Prins host=$1 if [ ! -e fabfile.py -o -z $host ]; then echo Usage: echo echo ./contrib/flavor/phylogeny/install_host.sh user@hostname fi source=`pwd` fabricrc=$source/contrib/flavor/phylogeny/fabricrc_ubuntu.txt packagelist=$source/contrib/flavor/phylogeny/main.yaml fab -f $source/fabfile.py -H $host -c $fabricrc install_biolinux:packagelist=$packagelist ================================================ FILE: contrib/flavor/phylogeny/main.yaml ================================================ --- packages: - minimal - desktop-cloud - editors - phylogeny # - phylogeny_testing - Debian only - r libraries: ================================================ FILE: contrib/flavor/phylogeny/phylogenyflavor.py ================================================ from fabric.api import * from fabric.contrib.files import * from fabfile import _freenx_scripts from cloudbio.flavor import Flavor from cloudbio.custom.shared import (_fetch_and_unpack) class PhylogenyFlavor(Flavor): """A VM flavor for running Phylogeny """ def __init__(self, env): Flavor.__init__(self,env) self.name = "Phylogeny Flavor" def rewrite_config_items(self, name, packages): if name == 'packages': packages += [ 'openssh-server', 'unzip', 'tar', 'sudo', 'openjdk-6-jre'] packages += [ 'openmpi-bin' ] # required for MrBayes-MPI # if 'bio-linux-mrbayes-multi' in packages: # (Debian version is still not OK) # packages.remove('bio-linux-mrbayes-multi') for package in packages: env.logger.info("Selected: "+name+" "+package) return packages def post_install(self): env.logger.info("Starting post-install") _freenx_scripts() pass env.flavor = PhylogenyFlavor(env) ================================================ FILE: contrib/flavor/phylogeny/virtualbox.md ================================================ = Running the Phylogeny VM in Virtualbox == Install Virtualbox Download and install Virtualbox from https://www.virtualbox.org/ Start Virtualbox. == Download and install the BioLinux image Fetch a BioLinux for virtualbox using the provided link, e.g. wget http://hostname/biolinux-phylogeny-virtualbox.vmdk Add this file to VirtualBox by selecting 'New', choose a name and select Debian Linux. After setting RAM, select 'Use existing hard disk' and select the downloaded .vmdk file. == Start up the image Select the image and press 'Start'. == Using the VM After starting the VM you get a desktop. The user name is 'vagrant', as well as the password. You can get root with the same password. Open a terminal with LXterminal (from the menu). Run tools from the command line, e.g. raxmlHPC mb-mpi you can have root, so installing new software is possible using apt-get. == Modifying the virtual hardware You can configure your image to run on multiple CPUs. Right-click on the image icon in virtualbox and 'Settings'. Configure the hardware. == Use ssh/scp Openssh-server should be running on the VM apt-get install openssh-server The VM is visible on a local network inside your PC. The VM network card is eth1. Start a terminal as root and initialize dhcpclient -v eth1 /sbin/ifconfig from your PC use ssh to the listed IP address. When DHCP fails, reconfigure networking in VirtualBox. Right-click on the image icon of the VM, click 'Settings', and configure the network to use 'bridge'. And again run DHCP. == Copying files to the image With scp files can be copied to the image. An alternative approach, since the VM has network, is to provide files through a webserver, or Amazon S3. E.g. wget http://webserver/file.tgz ================================================ FILE: contrib/flavor/pjotrp/__init__.py ================================================ """Pjotr's flavors""" ================================================ FILE: contrib/flavor/pjotrp/biotest/__init__.py ================================================ """BioTest flavor Copyright (C) 2011 Pjotr Prins """ ================================================ FILE: contrib/flavor/pjotrp/biotest/biotestflavor.py ================================================ from fabric.api import * from fabric.contrib.files import * from cloudbio.flavor import Flavor from cloudbio.custom.shared import (_fetch_and_unpack) class BioTestFlavor(Flavor): """A Flavor for cross Bio* tests """ def __init__(self, env): Flavor.__init__(self,env) self.name = "Bio* cross-lang flavor" def rewrite_config_items(self, name, items): if name == "packages": # list.remove('screen') # list.append('test') return items elif name == "python": return [ 'biopython' ] elif name == "perl": return [ 'bioperl' ] elif name == "ruby": return [ 'bio' ] elif name == "custom": return [] else: return items def post_install(self): env.logger.info("Starting post-install") env.logger.info("Load Scalability tests") if exists('Scalability'): with cd('Scalability'): run('git pull') else: _fetch_and_unpack("git clone git://github.com/pjotrp/Scalability.git") # Now run a post installation routine (for the heck of it) run('./Scalability/scripts/hello.sh') env.logger.info("Load Cross-language tests") if exists('Cross-language-interfacing'): with cd('Cross-language-interfacing'): run('git pull') else: _fetch_and_unpack("git clone git://github.com/pjotrp/Cross-language-interfacing.git") # Special installs for the tests with cd('Cross-language-interfacing'): sudo('./scripts/install-packages-root.sh ') run('./scripts/install-packages.sh') run('./scripts/create_test_files.rb') env.flavor = BioTestFlavor(env) ================================================ FILE: contrib/flavor/pjotrp/biotest/fabricrc_debian.txt ================================================ edition = minimal edition_version = 0.10 flavor_path = pjotrp.biotest.biotestflavor distribution = debian dist_name = squeeze # Update to the target's nearest repository: debian_repository=http://ftp.nl.debian.org/debian/ # Global installation directory for packages and standard programs system_install = /usr local_install = ~/install shell_config = ~/.bashrc shell = /bin/bash -l -c # data_files = /mnt/biodata ================================================ FILE: contrib/flavor/pjotrp/biotest/main.yaml ================================================ --- # Top level configuration file that specifies which groups of programs # should be installed. New sections that are added to individual config # files should go here. Comment out any groups you don't want to have # installed. # # This is the minimal version to kick-off development, including editors, # version control, and basic build support (GNU tools) packages: - minimal - ruby - r - python libraries: - ruby-libs ================================================ FILE: contrib/flavor/proteomics/galaxyp/README.md ================================================ This directory contains a stand-alone (no CloudMan) Galaxy-P flavor for installing Galaxy-P in desktop or cluster environments. If configuring Galaxy-P for cloud based enviornments, the cloudman/cloudman_and_galaxyp flavor is likely a more appropriate target. For more information on using CloudBioLinux to install Galaxy-P, please see http://getgalaxyp.org/install.html. ================================================ FILE: contrib/flavor/proteomics/galaxyp/main.yaml ================================================ --- packages: - minimal - galaxy - galaxyp libraries: ================================================ FILE: contrib/flavor/proteomics/galaxyp/settings-sample-galaxyp.yaml ================================================ --- vm_provider: novm fabricrc_overrides: # Directory to install Galaxy dependencies into. Be sure to update Galaxy's universe_wsgi.ini file # so that tool_dependency_dir in the app:main section points to this same directory. galaxy_tools_dir: /opt/galaxy/tools # System user Galaxy will run as. galaxy_user: galaxy galaxy_tools_conf: $__contrib_dir__/flavor/proteomics/galaxyp/tools.yaml galaxy_install_dependencies: True ================================================ FILE: contrib/flavor/proteomics/galaxyp/tools.yaml ================================================ --- # This file contains the install procedures for Galaxy tools used by # the stock Galaxy-P configuration (hosted on usegalaxyp.org). # Override the property `galaxy_tools_conf` in fabricrc to reference this # file and set the property `install_galaxy_dependencies` to True and # install with cbl_galaxy_tools custom install function directly using # # % fab -f fabfile.py -i -H ubuntu@ install_custom:galaxy_tools # # Or as part of a full flavor install that include galaxy, for instance using # the cloudman/cloudman_and_galaxyp flavor. # # % fab -f fabfile.py -i -H ubuntu@ install_biolinux:flavor=cloudman/cloudman_and_galaxyp # applications: galaxy_protk: "1.2.2" ## Bioinformatics and NGS Applications emboss: - version: "6.6.0" symlink_versions: "5.0.0" augustus: - version: "2.7" bin_dirs: - bin - scripts env_vars: AUGUSTUS_CONFIG_PATH: $system_install/config fastx_toolkit: "0.0.13.2" ucsc_tools: "latest" kent_tools: "latest" blast: - version: "2.2.26+" requirement_name: "blast+" - version: "2.2.28+" requirement_name: "blast+" bowtie: "0.12.9" tophat: - version: "1.3.3" bin_dirs: # tophat requires bowtie so make sure to include it in it's env.sh - bin - ../../bowtie/0.12.9/bin bowtie2: "2.1.0" tophat2: - version: "2.0.9" bin_dirs: - bin - ../../bowtie2/2.1.0/bin trinity: - version: "r2012-10-05" bin_dirs: - . cufflinks: "2.1.1" samtools: "0.1.16" bwa: "0.5.9" gatk: "1.4-9-g1f1233b" fastqc: "0.10.0" picard: "1.56" ## Mass Spec Applications: peptide_to_gff: "master" transproteomic_pipeline: - version: "4.6.1-occupy" bin_dirs: - bin - cgi-bin provides: - "xtandem" - "trans_proteomic_pipeline" - name: "proteowizard" unless: "using_wine" openms: - version: "1.10.0" env_vars: LD_LIBRARY_PATH: "${system_install}/lib:$$LD_LIBRARY_PATH" omssa: "2.1.9" tint_proteomics_scripts: - version: "1.19.20" symlink_versions: "1.19.19" superhirn: "0.03" mzmine: "2.10" percolator: "2_04" myrimatch: "2.1.131" directag: "1.3.62" idpqonvert: "3.0.475" pepitome: "1.0.45" tagrecon: "1.4.63" crux: "1.39" fido: "2011" pride_inspector: "1.3.0" pride_converter2: "2.0.17" ms2preproc: "2009" mayu: "1.06" pepnovo: "20120423" ipig: "r5" searchgui: "1.14.1" peptide_shaker: - version: "0.20.1" env_vars: # If DISPLAY set keep it the same, otherwise use XVFB DISPLAY: "$${DISPLAY:-:42}" psm_eval: "0.1.0" ================================================ FILE: contrib/flavor/proteomics/swift/custom.yaml ================================================ --- bio_proteomics_wine: - proteomics_wine_env - proteowizard bio_proteomics: - idpqonvert - myrimatch # - transproteomic_pipeline # - omssa # - mzmine # - directag # - tagrecon # - pepitome # - percolator ================================================ FILE: contrib/flavor/proteomics/swift/main.yaml ================================================ --- packages: # Editors, utils, build systems, system administration, etc.. - minimal # Brings in JDK and maven - java # R packages determined in r-libs.yaml in this directory. - r # See custom.yaml for what subsets of these packages are imported. - bio_proteomics_wine - bio_proteomics ================================================ FILE: contrib/flavor/proteomics/swift/r-libs.yaml ================================================ --- # Configuration file defining R specific libraries that are installed # via CRAN and Bioconductor. cranrepo: http://watson.nci.nih.gov/cran_mirror/ cran: - caTools - gam - png - bitops - akima ================================================ FILE: contrib/flavor/seal/__init__.py ================================================ """Seal flavor """ ================================================ FILE: contrib/flavor/seal/fabricrc_sl.txt ================================================ edition = minimal #edition_version = 0.10 flavor_path = seal.sealflavor distribution = scientificlinux #dist_name = # Global installation directory for packages and standard programs system_install = /usr local_install = ~/install shell_config = ~/.bashrc shell = /bin/bash -l -c # data_files = /mnt/biodata # galaxy_base = /mnt/biodata/galaxy ================================================ FILE: contrib/flavor/seal/main.yaml ================================================ --- packages: - distributed - java - programming - python libraries: ================================================ FILE: contrib/flavor/seal/sealflavor.py ================================================ from fabric.api import * from fabric.contrib.files import * from cloudbio.flavor import Flavor from cloudbio.custom.shared import (_fetch_and_unpack) import sys # This flavour installs the Seal toolkit for processing high-throughput # sequencing data on Hadoop. # http://biodoop-seal.sf.net/ # # It pulls in quite a few dependencies, including Hadoop itself and # Pydoop (http://pydoop.sf.net/). # # The dependencies it pulls into Cloudbiolinux are structured as follows: # # contrib/flavor/seal/main.yaml # sealdist # customsealdist # # config/packages-yum.yaml # sealdist (metapackage) # config/custom.yaml # customsealdist (metapackage) # - pydoop # - seal # # The components of the customsealdist metapackage are installed through # the functions in cloudbio/custom/customsealdist.py # # # This flavour has only been installed on Scientific Linux and has not # yet been well tested. # # To try installing it run the following: # cd # fab -f ./fabfile.py -H root@ -c ./contrib/flavor/seal/fabricrc_sl.txt install_biolinux:packagelist=contrib/flavor/seal/main.yaml # # Authors: Roman Valls Guimera # Luca Pireddu class SealFlavor(Flavor): """A flavour for installing Seal """ def __init__(self, env): Flavor.__init__(self,env) self.name = "Seal Flavor" def rewrite_config_items(self, name, packages): if name == 'packages': if sys.version_info < (2,7): # for versions of Python prior to 2.7 we need to add importlib # and argparse packages.extend([ "python-importlib", "python-argparse" ]) return packages def post_install(self): env.logger.info("Starting post-install") pass env.flavor = SealFlavor(env) ================================================ FILE: contrib/flavor/variantviz/custom.yaml ================================================ --- # List of custom programs to install that are not packaged. These each require # a install_name function in the appropriate place in the 'custom' directory to # define the steps for downloading, building, and installing the package. bio_nextgen: - vep java: - leinengin ================================================ FILE: contrib/flavor/variantviz/fabricrc.txt ================================================ # Configuration file for Fabric specifying filesystem information # This can be adjusted and passed to fab on the commandline as '-c' # # For example # # fab -f data_fabfile.py -H your_machine -c your_fabricrc.txt install_data_s3:your_biodata.yaml # details about the operating system ## Ubuntu #distribution = ubuntu #dist_name = precise ## Centos distribution = centos ## Debian #distribution = debian #dist_name = squeeze # Unset to avoid CloudBioLinux customizations like setup scripts; # or set to different edition to get specific customizations #edition = cloudbiolinux # Whether to clean the system for EC2 AMI preparation; # do not set on vagrant or local machines is_ec2_image = false # Global installation directory for packages and standard programs system_install = /usr # Local install directory for versioned software that will not # be included in the path by default local_install = ~/install shell_config = ~/.bashrc shell = /bin/bash -l -c # Global setting for using sudo; allows installation of custom packages # by non-privileged users. # system_install needs to point to a user writeable directory if set to False use_sudo = true # details for biological data files and Galaxy data_files = /mnt/biodata galaxy_home = /mnt/biodata/galaxy # Picard home picard_home = /usr/share/java/picard ================================================ FILE: contrib/flavor/variantviz/main.yaml ================================================ --- # Top level configuration file that specifies which groups of programs # should be installed. New sections that are added to individual config # files should go here. Comment out any groups you don't want to have # installed. packages: - programming - python - perl - ruby - java - web - bio_nextgen libraries: ================================================ FILE: contrib/flavor/variantviz/packages-yum.yaml ================================================ --- # List of packages to be installed via the yum package manager. programming: build: - gcc - gcc-c++ - gcc-gfortran libraries: - atlas-devel - blas - blas-devel - lapack - lapack-devel python: - python - python-devel - python-distribute - python-pip - scipy - numpy perl: - perl - perl-DBI - perl-DBD-MySQL ruby: - ruby - ruby-devel - rubygems java: - java-1.7.0-openjdk - java-1.7.0-openjdk-devel - ant web: - nginx bio_nextgen: ================================================ FILE: contributors.mkd ================================================ ## Contributors An open source project is nothing without the great people who generously contribute time and expertise. - Enis Afgan - Sébastien Boisvert - Tim Booth - Brad Chapman - John Chilton - Michael Heuer - Richard Holland - Mark Jensen - Justin Johnson - Ntino Krampis - Tristan J. Lubinski - Hervé Ménager - Dave Messina - Steffen Möller - Pjotr Prins - Mitch Skinner - Bela Tiwari - Roman Valls - Luca Pireddu ================================================ FILE: data_fabfile.py ================================================ """Fabric deployment file to install genomic data on remote instances. Designed to automatically download and manage biologically associated data on cloud instances like Amazon EC2. Fabric (http://docs.fabfile.org) manages automation of remote servers. Usage: fab -i key_file -H servername -f data_fabfile.py install_data """ import os import sys from fabric.main import load_settings from fabric.api import * from fabric.contrib.files import * from fabric.context_managers import path try: import boto except ImportError: boto = None # preferentially use local cloudbio directory for to_remove in [p for p in sys.path if p.find("cloudbiolinux-") > 0]: sys.path.remove(to_remove) sys.path.append(os.path.dirname(__file__)) from cloudbio.utils import _setup_logging, _configure_fabric_environment from cloudbio.biodata import genomes # -- Host specific setup env.remove_old_genomes = False def setup_environment(): """Setup environment with required data file locations. """ _setup_logging(env) _add_defaults() _configure_fabric_environment(env, ignore_distcheck=True) def _add_defaults(): """Defaults from fabricrc.txt file; loaded if not specified at commandline. """ env.config_dir = os.path.join(os.path.dirname(__file__), "config") conf_file = "tool_data_table_conf.xml" env.tool_data_table_conf_file = os.path.join(os.path.dirname(__file__), "installed_files", conf_file) if not env.has_key("distribution"): config_file = os.path.join(env.config_dir, "fabricrc.txt") if os.path.exists(config_file): env.update(load_settings(config_file)) CONFIG_FILE = os.path.join(os.path.dirname(__file__), "config", "biodata.yaml") def install_data(config_source=CONFIG_FILE): """Main entry point for installing useful biological data. """ setup_environment() genomes.install_data(config_source) def install_data_raw(config_source=CONFIG_FILE): """Installing useful biological data building from scratch. Useful for debugging. """ setup_environment() genomes.install_data(config_source, approaches=["raw"]) def install_data_s3(config_source=CONFIG_FILE, do_setup_environment=True): """Install data using pre-existing genomes present on Amazon s3. """ setup_environment() genomes.install_data_s3(config_source) def install_data_rsync(config_source=CONFIG_FILE): """Install data using Galaxy rsync data servers. """ setup_environment() genomes.install_data_rsync(config_source) def install_data_ggd(recipe, organism): """Install data using Get Genomics Data (GGD) recipes. """ setup_environment() from cloudbio.biodata import ggd, genomes genome_dir = os.path.join(genomes._make_genome_dir(), organism) recipe_file = os.path.join(os.path.dirname(__file__), "ggd-recipes", organism, "%s.yaml" % recipe) ggd.install_recipe(genome_dir, env, recipe_file, organism) def upload_s3(config_source=CONFIG_FILE): """Upload prepared genome files by identifier to Amazon s3 buckets. """ setup_environment() genomes.upload_s3(config_source) ================================================ FILE: deploy/README.md ================================================ # CloudBioLinux Deployer This CloudBioLinux deployer has grown out of the galaxy-vm-launcher and can be used to launch cloud virtual machines, configure them with Galaxy, and seed it with input data, genomes, workflows, etc.... More recently, actions for installing CloudBioLinux and launching CloudMan have been added. ## Prerequisites The `deploy.sh` script should install the needed dependencies in a Python virtual environment using venvburrito and doesn't require special permissions as long as `python`, `easy_install`, and `git` are available. ## Specify settings All deploy actions first require the existence of a setting file. cp settings-sample-oldgalaxyvmlauncher.yaml settings.yaml This file has numerous settings to customize how the deployer acts. Be default, the deployer will target Amazon Web Services and `key_file`, `access_id`, and `private_key` in the aws section of of this file must be specified. The argument `--settings=/path/to/custom_settings.yaml` may be passed to `deploy.sh` to specify a custom path for this settings file. ## Configuring Galaxy ./deploy.sh --action=configure --action=transfer file1 file2 file3 When called this way, deploy.sh will launch a VM, configure Galaxy, tools, and genomes. Once Galaxy is ready, it will transfer each of provided input files to the newly launched VM, and use the Galaxy REST API to add them to a Galaxy data library (and optionally a history). Once all of that is complete, it will print a URL to the screen telling the operator where to find the new Galaxy instance. This does not install CloudMan, Galaxy is configured to run at startup by an init script. A more traditional CloudMan workflow can be achieved using the `install_biolinux` action described next. ## Installing CloudBioLinux ./deploy.sh --action=install_biolinux --action=package This mode will launch an instance, install CloudBioLinux (a flavor can be specified in setting.yaml), and `package` (see settings.yaml for more details) the resulting virtual image. ## Additional Actions The actions show above can be combined in different manners, for instance `configure` and `package` can be used to configure a Galaxy instance and package so that later `transfer` can be used without requiring a full configure. Alternatively, `install_biolinux` can be followed up with `transfer` to install CloudBioLinux and start analyzing data without requiring Galaxy (be sure to set `use_galaxy: False` in settings.yaml in this case). You can see running instances on target cloud with this command: ./deploy.sh --action=list You can also destroy all running instances with this command: ./deploy.sh --action=destroy If an existing CloudBioLinux image bundled with CloudMan has been created and its image id set as `image_id` in the `cloudman` section of `setting.yaml`, then this image can be launched for testing with: ./deploy.sh --action=launch_cloudman The full list of actions can be found in `cloudbio/deploy/__init__.py` and includes: * `list` * `destroy` * `transfer` * `destroy` * `transfer`, * `purge_galaxy` * `setup_galaxy` * `purge_tools` * `setup_tools` * `purge_genomes` * `setup_genomes` * `setup_ssh_key` * `package` * `setup_image` * `launch` - Dummy action justs launches instance * `install_biolinux` * `cloudman_launch` Additional composite actions are shortcuts for multiple actions - these include: * `configure` - `setup_image`, `setup_tools`, `setup_genomes`, `setup_ssh_key` * `reinstall_galaxy` - `purge_galaxy` and `setup_galaxy` * `reinstall_genomes` - `purge_genomes` and `setup_genomes` * `reinstall_tools` - `purge_tools` and `setup_tools` ## Configuring Cloud Provider Cloud interactions are managed via the [vm-launcher] project, full information on configuring different cloud providers can be found [here][vm-launcher-config] In brief, there are few different options for where to create the VMs. Amazon EC2 is the default target, but it can also target Eucalyptus or OpenStack based clouds. The ruby package `vagrant` can be used to target virtual instances on your own machine. [vm-launcher-config]: https://github.com/jmchilton/vm-launcher/blob/master/config.md ================================================ FILE: deploy/TODO ================================================ TODO: Refactor use_existing_instance out of cloud specific config, make command-line option TODO: Allow specification of clouds by name instead of type. TODO: Implement alternate transfer mechanisms (fabric-based transfer has not proven very robust). TODO: Improve documentation. ================================================ FILE: deploy/Vagrantfile ================================================ # -*- mode: ruby -*- # vi: set ft=ruby : # Vagrantfile API/syntax version. Don't touch unless you know what you're doing! VAGRANTFILE_API_VERSION = "2" Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| # All Vagrant configuration is done here. The most common configuration # options are documented and commented below. For a complete reference, # please see the online documentation at vagrantup.com. # Every Vagrant virtual environment requires a box to build off of. config.vm.box = "base" # Disable automatic box update checking. If you disable this, then # boxes will only be checked for updates when the user runs # `vagrant box outdated`. This is not recommended. # config.vm.box_check_update = false # Create a forwarded port mapping which allows access to a specific port # within the machine from a port on the host machine. In the example below, # accessing "localhost:8080" will access port 80 on the guest machine. # config.vm.network "forwarded_port", guest: 80, host: 8080 # Create a private network, which allows host-only access to the machine # using a specific IP. # config.vm.network "private_network", ip: "192.168.33.10" # Create a public network, which generally matched to bridged network. # Bridged networks make the machine appear as another physical device on # your network. # config.vm.network "public_network" # If true, then any SSH connections made will enable agent forwarding. # Default value: false # config.ssh.forward_agent = true # Share an additional folder to the guest VM. The first argument is # the path on the host to the actual folder. The second argument is # the path on the guest to mount the folder. And the optional third # argument is a set of non-required options. # config.vm.synced_folder "../data", "/vagrant_data" # Provider-specific configuration so you can fine-tune various # backing providers for Vagrant. These expose provider-specific options. # Example for VirtualBox: # # config.vm.provider "virtualbox" do |vb| # # Don't boot with headless mode # vb.gui = true # # # Use VBoxManage to customize the VM. For example to change memory: # vb.customize ["modifyvm", :id, "--memory", "1024"] # end # # View the documentation for the provider you're using for more # information on available options. # Enable provisioning with CFEngine. CFEngine Community packages are # automatically installed. For example, configure the host as a # policy server and optionally a policy file to run: # # config.vm.provision "cfengine" do |cf| # cf.am_policy_hub = true # # cf.run_file = "motd.cf" # end # # You can also configure and bootstrap a client to an existing # policy server: # # config.vm.provision "cfengine" do |cf| # cf.policy_server_address = "10.0.2.15" # end # Enable provisioning with Puppet stand alone. Puppet manifests # are contained in a directory path relative to this Vagrantfile. # You will need to create the manifests directory and a manifest in # the file default.pp in the manifests_path directory. # # config.vm.provision "puppet" do |puppet| # puppet.manifests_path = "manifests" # puppet.manifest_file = "site.pp" # end # Enable provisioning with chef solo, specifying a cookbooks path, roles # path, and data_bags path (all relative to this Vagrantfile), and adding # some recipes and/or roles. # # config.vm.provision "chef_solo" do |chef| # chef.cookbooks_path = "../my-recipes/cookbooks" # chef.roles_path = "../my-recipes/roles" # chef.data_bags_path = "../my-recipes/data_bags" # chef.add_recipe "mysql" # chef.add_role "web" # # # You may also specify custom JSON attributes: # chef.json = { mysql_password: "foo" } # end # Enable provisioning with chef server, specifying the chef server URL, # and the path to the validation key (relative to this Vagrantfile). # # The Opscode Platform uses HTTPS. Substitute your organization for # ORGNAME in the URL and validation key. # # If you have your own Chef Server, use the appropriate URL, which may be # HTTP instead of HTTPS depending on your configuration. Also change the # validation key to validation.pem. # # config.vm.provision "chef_client" do |chef| # chef.chef_server_url = "https://api.opscode.com/organizations/ORGNAME" # chef.validation_key_path = "ORGNAME-validator.pem" # end # # If you're using the Opscode platform, your validator client is # ORGNAME-validator, replacing ORGNAME with your organization name. # # If you have your own Chef Server, the default validation client name is # chef-validator, unless you changed the configuration. # # chef.validation_client_name = "ORGNAME-validator" end ================================================ FILE: deploy/cloudman.html ================================================

CloudBioLinux Deployer CloudMan QuickStart

As far as I can determine there is no current documentation on how to build CloudMan instances from scratch. Thus I am collecting my unofficial notes on how to do this here - specifically using the CloudBioLinux deployer.

You will need to navigate the AWS management console and obtain the following information.

  • access_id
  • secret_key
  • Ubuntu EBS-backed AMI ID to target. This writeup was tested with ami-9b85eef2 (12.04.2 (64-bit) in us-east-1)
  • Image size to use (e.g. m1-small)
  • Availability zone (e.g. us-east-1)
  • You will need to setup a bucket to store your snaps file, here you will need the bucket name.
  • You will need to setup two volumes in your target availability zone, one for Galaxy tools and data (perhaps 20Gb for testing) and one for galaxyIndices. Here you will need the volume ids.
  • Generate a private a key (e.g. galaxy1.pem) and copy it into keys directory (or anywhere really), also note the keypair_name corresponding to the key.

Create a directory (e.g. /home/mary/marys_cloudman_bucket_contents). Copy the files from an existing CloudMan bucket here (e.g. http://s3.amazonaws.com /cloudman-dev).

It is not really important how you download these files, but one quick option is to use s3cmd tool:

% sudo apt-get install s3cmd  # Or your OS's package manager
% mkdir /home/mary/marys_cloudman_bucket_contents
% s3cmd -r get s3://cloudman-dev marys_cloudman_bucket_contents

Here you can update cm.tar.gz or any of these files to match the customized setup you would like. In particular you are going to want to create a custom snaps.yaml file. Here is a simple outline that we will fill out as we good.

version: 1
clouds:
  - name: amazon
    regions:
    - deployments:
      - name: GalaxyCloud
        filesystems:
        - name: galaxy
          roles: galaxyTools,galaxyData
          snap_id: snap-XXXXXXXXXXX
        - name: galaxyIndices
          roles: galaxyIndices
          snap_id: snap-XXXXXXXXXXXX
        default_mi: ami-XXXXXXXXXXXXX
        bucket: marys_cloudman_bucket
      name: us-east-1

Immediately this template can be updated to reflect the bucket created above and the availability zone you are targetting. We can update the snapid's and the defaultmi after creating them.

Copy and modify settings.yaml:

% cp settings-sample-cm.yaml settings.yaml
% vim settings.yaml # or your favorite editor

Carefully scan through that file and change the properties marked as requiring change. The keyword UPDATE indicates properties of special interest that either don't have reasonable defaults or have reasonable defaults but that I have deemed highly likely to be overridden.

Now you can use the CloudBioLinux deployer to launch an image, attach volumes, install biolinux, take needed snapshots, and package the whole thing up:

% ./deploy.sh --action=launch
% ./deploy.sh --action=attach_volumes
% ./deploy.sh --action=install_biolinux --flavor=cloudman/cloudman_and_galaxy
% ./deploy.sh --action=snapshot_volumes
% ./deploy.sh --action=detach_volumes
% ./deploy.sh --action=package

Finally, update snaps.yaml in your bucket directory (e.g. /home/mary/marys_cloudman_bucket_contents) to reflect the snap_ids and AMI's just created. These should all be available via the AWS console or by reviewing the output of the steps above.

Finally, you can upload your new bucket and launch a test CloudMan instance:

% ./deploy.sh --action=sync_cloudman_bucket
% ./deploy.sh --action=cloudman_launch
================================================ FILE: deploy/cloudman.md ================================================ # CloudBioLinux Deployer CloudMan QuickStart As far as I can determine there is no current documentation on how to build CloudMan instances from scratch. Thus I am collecting my unofficial notes on how to do this here - specifically using the CloudBioLinux deployer. You will need to navigate the AWS management console and obtain the following information. * Your AWS Access ID and secret key (`access_id`, `secret_key`) * Ubuntu EBS-backed AMI ID to target. This writeup was tested with ami-9b85eef2 (12.04.2 (64-bit) in us-east-1) * Image size to use (e.g. m1-small) * Availability zone (e.g. us-east-1) * You will need to setup a bucket to store your snaps file, here you will need the bucket name. * You will need to setup two volumes in your target availability zone, one for Galaxy tools and data (perhaps 20Gb for testing) and one for galaxyIndices. Here you will need the volume ids. * Generate a private a key (e.g. galaxy1.pem) and copy it into keys directory (or anywhere really), also note the keypair_name corresponding to the key. Create a directory (e.g. `/home/mary/marys_cloudman_bucket_contents`). Copy the files from an existing CloudMan bucket here (e.g. http://s3.amazonaws.com /cloudman-dev). It is not really important how you download these files, but one quick option is to use `s3cmd` tool: % sudo apt-get install s3cmd # Or your OS's package manager % mkdir /home/mary/marys_cloudman_bucket_contents % s3cmd -r get s3://cloudman-dev /home/mary/marys_cloudman_bucket_contents Here you can replace the CloudMan source (i.e. `cm.tar.gz`) or any of these files to match the customized setup you would like. In particular you are going to want to create a custom snaps.yaml file. Here is a simple outline that we will fill out as we good. version: 1 clouds: - name: amazon regions: - deployments: - name: GalaxyCloud filesystems: - name: galaxy roles: galaxyTools,galaxyData snap_id: snap-XXXXXXXXXXX mount_point: /mnt/galaxy - name: galaxyIndices roles: galaxyIndices snap_id: snap-XXXXXXXXXXXX mount_point: /mnt/galaxyIndices default_mi: ami-XXXXXXXXXXXXX bucket: marys_cloudman_bucket name: us-east-1 Immediately this template can be updated to reflect the bucket created above and the availability zone you are targetting. We can update the snap_id's and the default_mi after creating them. Copy and modify `settings-sample-cm.yaml` to `settings.yaml`: % git clone git://github.com/chapmanb/cloudbiolinux.git % cd cloudbiolinux/deploy % cp settings-sample-cm.yaml settings.yaml % vim settings.yaml # or your favorite editor Carefully scan through `settings.yaml` and change the properties marked as requiring change. The word `UPDATE` in the comments indicates properties of special interest that either don't have reasonable defaults or have reasonable defaults but that I have deemed highly likely to be overridden. Now you can use the CloudBioLinux deployer to launch an image, attach volumes, install biolinux, take needed snapshots, and package the whole thing up: % ./deploy.sh --action=launch % ./deploy.sh --action=attach_volumes % ./deploy.sh --action=install_biolinux --flavor=cloudman/cloudman_and_galaxy % ./deploy.sh --action=snapshot_volumes % ./deploy.sh --action=detach_volumes % ./deploy.sh --action=package If at any point in the above process you need to interactively inspect the state of the instance being configured you can do this via the following command: % ./deploy.sh --action=ssh Once a CloudMan AMI has been created, update `snaps.yaml` in your bucket directory (e.g. `/home/mary/marys_cloudman_bucket_contents`) to reflect the `snap_id`s and AMI created. These should all be available via the AWS management console or by reviewing the output of the steps above. Finally, you can upload your new bucket and launch a test CloudMan instance: % ./deploy.sh --action=sync_cloudman_bucket % ./deploy.sh --action=cloudman_launch ## Customizing The above example uses the `cloudman/cloudman_and_galaxy` CloudBioLinux flavor, but there are additional flavors of CloudBioLinux available. Please consult [this page][1] and choose the most appropriate flavor: ### Customizing Galaxy Installing a customized Galaxy is as simple as overriding the `galaxy_repository` variable in the `fabricrc_overrides` section of the `settings.yaml`. ### Customizing Tools Out of the box, CloudBioLinux can be configured to install dozens of bioinformatic packages out of the box and adding additional packages is fairly straight forward. One simply need to create a CloudBioLinux flavor that configures which such packages are installed and specify that flavor (either in the command-line as shown above or in `settings.yaml`). Your custom flavor should include the `cloudman` packages. If your flavor additionally includes `galaxy` (as the flavor `cloudman_and_galaxy` shown above) packages and `install_tool_dependencies` is set to `True` in `settings.yaml` - CloudBioLinux will setup a tool dependencies directory for Galaxy. This allows multiple versions of an application to be installed in isolation. When enabled, the list of tools and versions that is installed can be found in ``cloudbiolinux/contrib/flavor/cloudman/tools.yaml ``. One can modify that file directly or specify an entirely new file by setting the ``galaxy_tools_conf`` property in the `fabric_overrides` section of `settings.yaml`. ### Customizing CloudMan CloudMan is downloaded from the bucket you specify and installed at system startup. Hence one can simply place a customized version of CloudMan (tarred up and named `cm.tar.gz`) in the bucket. If `cloudman_repository`, `bucket_source`, and `bucket_default` are set in the `cloudman` section of `settings.yaml`, then one can execute the following command to quickly tar up the local copy of CloudMan (in `cloudman_repository`) and update your target bucket. % ./deploy.sh --action=bundle_cloudman --action=sync_cloudman_bucket [1]: https://github.com/chapmanb/cloudbiolinux/tree/master/contrib/flavor/cloudman ================================================ FILE: deploy/config/tool_data_table_conf.xml ================================================ value, dbkey, name, path
value, dbkey, formats, name, path
value, name, path
value, name, path
value, dbkey, name, path
value, dbkey, name, path
value, dbkey, name, path
value, dbkey, name, path
name, value, dbkey, species
value, dbkey, name, path
value, name, path
value, name, path
value, name, path
line_type, value, path
value, dbkey, name, path
value, dbkey, name, path
value, name, gatk_value, tools_valid_for
value, dbkey, name, path
value, dbkey, name, path
value, dbkey, name, path
value, name, path
value, dbkey, name, path
================================================ FILE: deploy/deploy.sh ================================================ #!/bin/bash # Name of virtualenv to create using virtualenvwrapper VIRTUALENV_NAME=cbl_deploy # Ensure working directory is cloudbiolinux/deploy. PROJECT_DIRECTORY="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" cd $PROJECT_DIRECTORY # Ensure virtualenv-burrito has been installed. if [ ! -e $HOME/.venvburrito/startup.sh ]; then wget -qO- https://raw.github.com/brainsik/virtualenv-burrito/master/virtualenv-burrito.sh | $SHELL fi # Configure virtualenv and virtualenvwrapper with virtualenv-burrito . $HOME/.venvburrito/startup.sh # If no cbl_deploy virtualenv exists, create it and populate # requirements. if [ ! `lsvirtualenv | grep $VIRTUALENV_NAME` ]; then mkvirtualenv -r requirements.txt $VIRTUALENV_NAME fi # Use cbl_deploy virtualenv workon $VIRTUALENV_NAME sh $PROJECT_DIRECTORY/deploy_no_deps.sh "$@" ================================================ FILE: deploy/deploy_bourne.sh ================================================ #!/bin/sh VIRTUALENV_VERSION=${VIRTUALENV_VERSION:-1.10.1} cd `dirname $0` PROJECT_DIRECTORY=${PROJECT_DIRECTORY:-`pwd`} VENV_DIRECTORY=$PROJECT_DIRECTORY/.venv if [ ! -e $VENV_DIRECTORY ]; then VIRTUALENV_URL="https://pypi.python.org/packages/source/v/virtualenv/virtualenv-${VIRTUALENV_VERSION}.tar.gz" DOWNLOAD_TAR_BALL="$PROJECT_DIRECTORY/virtualenv.tar.gz" PYTHON_VERSION=${PYTHON_VERSION:-`python -c "import sys; rev = sys.version_info; str = '%d.%d' % (rev[0], rev[1]); print str"`} LOCAL_PYTHON=$PROJECT_DIRECTORY/.virtualenv VIRTUALENV_PACKAGES_DIR="$LOCAL_PYTHON/lib/python${PYTHON_VERSION}/site-packages" VIRTUALENV_SOURCE_DIR=$PROJECT_DIRECTORY/.virtualenv_source wget -O "$DOWNLOAD_TAR_BALL" ${VIRTUALENV_URL} tar xzvf "$DOWNLOAD_TAR_BALL" mv virtualenv-${VIRTUALENV_VERSION} $VIRTUALENV_SOURCE_DIR mkdir -p $VIRTUALENV_PACKAGES_DIR export PYTHONPATH=$VIRTUALENV_PACKAGES_DIR:$PYTHONPATH export PATH=$LOCAL_PYTHON/bin:$PATH cd $VIRTUALENV_SOURCE_DIR python setup.py install --prefix="$LOCAL_PYTHON" cd $PROJECT_DIRECTORY virtualenv --no-site-packages $VENV_DIRECTORY . $VENV_DIRECTORY/bin/activate pip install -r $PROJECT_DIRECTORY/requirements.txt fi sh $PROJECT_DIRECTORY/deploy_no_deps.sh "$@" ================================================ FILE: deploy/deploy_no_deps.sh ================================================ #!/bin/sh export PROJECT_DIRECTORY="." # Add cloudbiolinux to python path and run deployment. export PYTHONPATH=..:$PYTHONPATH python $PROJECT_DIRECTORY/../cloudbio/deploy/main.py "$@" ================================================ FILE: deploy/requirements.txt ================================================ fabric paramiko==2.10.1 argparse pyyaml apache-libcloud==0.11.3 boto ================================================ FILE: deploy/settings-sample-cm.yaml ================================================ --- vm_provider: aws hostname: cloudman fabricrc_overrides: ## CloudBioLinux fabric property overrides ## Mercurial repository for Galaxy (default is https://bitbucket.org/galaxy/galaxy-central/) ## e.g. for galaxyp #galaxy_repository: https://bitbucket.org/galaxyp/galaxyp-central # Location to install galaxy galaxy_home: /mnt/galaxy/galaxy-app # Location to setup galaxy tool dependencies galaxy_tools_dir: /mnt/galaxy/tools galaxy_jars_dir: /mnt/galaxy/jars # Configure biodata location data_files: "/mnt/galaxyIndices" galaxy_base: "/mnt/galaxyIndices/galaxy" galaxy_data_mount: "/mnt/galaxy" galaxy_indices_mount: "/mnt/galaxyIndices" galaxy_install_dependencies: True # Setup an init service for Galaxy and virtual frame buffer galaxy_setup_xvfb: True distribution: ubuntu # UPDATE: Change this if you are targetting quantal or even newer Ubuntu. dist_name: precise ## UPDATE: Override tools that galaxy will install, defaults to *cloudbiolinux*/contrib/flavor/cloudman/tools.yaml # galaxy_tools_conf: /home/gxadmin/cloud_manage/tools.yaml aws: # UPDATE: Path to your galaxy keys file key_file: keys/galaxy1.pem # UPDATE: Set this to the name of the keypair corresponding to the above key_file. keypair_name: "galaxy1" # UPDATE: Set this to you access_id access_id: "XXXXXXXXXXXXXXXXXXXXXX" # UPDATE: Set this to your secret key. secret_key: "XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX" # UPDATE: Set this to the size you want. size_id: "m1.small" # UPDATE: set this to the Ubuntu AMI you would like to target. image_id: ami-9b85eef2 # EBS backed 12.04.2 (64-bit) # UPDATE: Set this to the availability zone to target. availability_zone: us-east-1 # Create a new instnace on first use, then reuses same instance for subsequent calls. use_existing_instance: __auto__ ## Packing Options # Use Amazon's create image API, only works with EBS volumes and AWS. package_type: create_image # Make new AMI public make_public: True # UPDATE: Name to give AMI image. package_image_name: "marys_cloudman_${the_date_with_time}" volumes: - id: vol-XXXXXXX # UPDATE with volume id created via AWS console. path: /mnt/galaxyIndices device: /dev/xvdj description: "Galaxy Indices [${the_date_with_time}]" format: __auto__ # Attempt to auto-format new partitions if needed, deactivate by setting to False. - id: vol-XXXXXX # UPDATE with volume id created via AWS console. path: /mnt/galaxy device: /dev/xvdk description: "Galaxy Tools and Data [${the_date_with_time}]" format: __auto__ # Attempt to auto-format new partitions if needed, deactivate by setting to False. ## CloudMan Options (mostly used after image is created for launching new CloudMan instance) cloudman: bucket_source: /home/mary/marys_cloudman_bucket_contents ## Id of image containing cloudman to launch ## Can set an AMI here, but __use_snaps__ will look in the local bucket ## contents of bucket_source for a snaps.yaml file and load that AMI instead. image_id: __use_snaps__ ## Size or flavor id of machine to launch. Defaults to ## cloud-specific option above. # size_id: ## access_key & secret_key to for use by CloudMan. Defaults ## to access_id & secret_key specified in cloud-spefic options above. # access_key: # secret_key: ## Name of cloudman cluster to create. cluster_name: "cloud_test_${the_date_with_time}" ## Password for CloudMan admin interface. # password: adminpass ## CloudMan source (local_path, mercurial repo (e.g. https://bitbucket.org/galaxy/cloudman), ## or git repo (https://github.com/jmchilton/cloudman.git). ## It can be packaged and inserted into bucket_source using: ## ./deploy.sh --action=bundle_cloudman # cloudman_repository: /path/to/cloudman_source user_data: # Update bucket to target. bucket_default: marys_cloudman_bucket ## Uncomment and update this to specify default user data properties for this ## AMI. ## I suppose all custom CloudMan AMIs should at least specify a new bucket ## default right? -John #image_user_data: # bucket_default: marys_cloudman_bucket # Advanced option sections ignored for CloudMan workflow outlined in cloudman.md galaxy: genomes: tools: ================================================ FILE: deploy/settings-sample-minimal.yaml ================================================ --- vm_provider: aws hostname: minimal fabricrc_overrides: distribution: ubuntu # UPDATE: Change this if you are targetting quantal or even newer Ubuntu. dist_name: precise aws: # UPDATE: Path to your galaxy keys file key_file: keys/galaxy1.pem # UPDATE: Set this to the name of the keypair corresponding to the above key_file. keypair_name: "galaxy1" # UPDATE: Set this to you access_id access_id: "XXXXXXXXXXXXXXXXXXXXXX" # UPDATE: Set this to your secret key. secret_key: "XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX" # UPDATE: Set this to the size you want. size_id: "m1.small" # UPDATE: set this to the Ubuntu AMI you would like to target. image_id: ami-9b85eef2 # EBS backed 12.04.2 (64-bit) # UPDATE: Set this to the availability zone to target. availability_zone: us-east-1 # Create a new instnace on first use, then reuses same instance for subsequent calls. use_existing_instance: __auto__ ## Packing Options # Use Amazon's create image API, only works with EBS volumes and AWS. package_type: create_image # Make new AMI public make_public: True # UPDATE: Name to give AMI image. package_image_name: "marys_minimal_cbl_${the_date_with_time}" # Advanced option sections ignored for CloudMan workflow outlined in cloudman.md galaxy: genomes: tools: ================================================ FILE: deploy/settings-sample-oldgalaxyvmlauncher.yaml ================================================ --- ## Set VM provider, default is aws. Other options include ## vagrant, openstack, and eucalyptus. Specify connection ## details in corresponding sections below. # vm_provider: aws hostname: galaxy ## Uncomment to destroy VM after the script is complete (for testing). #destroy_on_complete: True ## Pull in galaxy update and restart galaxy service at runtime. refresh_galaxy: False ## Seed galaxy with API users, API keys, workflows etc... at 'configure' time or 'runtime' seed_galaxy: configure genome_source: default # can also set to 'S3' to download pre-existing genomes tool_data_table_conf_file: config/tool_data_table_conf.xml ## Uncomment to compress and download genomes locally during the ## configure or setup_genomes step. # stash_genomes: download ## Uncomment to compress genomes and store in /opt of the VM during ## the configure or setup_genomes_step, when galaxy boots in a fresh ## VM this file will be decompressed into /mnt if needed. # stash_genomes: opt ## Uncomment to upload compressed genomes previously downloaded during ## a configure step. # upload_genomes: True ## Add sudoers_addition lines to add options to the /etc/sudoers ## file. Use with caution! sudoers_additions: # This line is needed to implement max_lifetime option below. - "galaxy ALL=NOPASSWD: /sbin/shutdown -h now,/sbin/reboot,/sbin/halt" ## Configure max lifetime for VM in minutes. # max_lifetime: 10200 ## Configure ssmtp (right now only gmail is really supported). # smtp_server: smtp.gmail.com:587 # smtp_user: @gmail.com # smtp_password: ## Configure genome for initially uploaded files. # db_key: hg19 ## Configure history name for initially uploaded files. This history ## name and corresponding user api key must match data found in ## galaxy section below. # transfer_history_name: # transfer_history_api_key: ## Uncomment to setup taxonomy data required by Galaxy metagenomics ## tools during the setup_genomes action. # setup_taxonomy_data: True ## Path to ssh key to install for galaxy user (optional). (We ## use this to automate transfers of results back to home base ## from the Cloud. -John) #galaxy_ssh_key: /path/to/ssh_private_key ## Specify flavor and target for CloudBioLinux to install with ## when configuring environmet and install (if install_biolinux ## action is used). # flavor: cloudman/cloudman_and_galaxy # target: ## Default CloudBioLinux properties to load, by default CloudBioLinux ## will just its own defaults (either ../config/fabrircr.txt or whatever ## is specified by the CloudBioLinux flavor). # fabricrc_file: ../config/fabricrc.txt ## CloudMan Options (does not work with Vagrant driver) cloudman: ## Id of image containing cloudman to launch image_id: XXXXXXXXXX ## Size or flavor id of machine to launch. Defaults to ## cloud-specific option below. # size_id: ## access_key & secret_key to for use by CloudMan. Defaults ## to access_id & secret_key specified in cloud-spefic options below. # access_key: # secret_key: ## Name of cloudman cluster to create. # cluster_name: cloudman ## Password for CloudMan admin interface. # password: adminpass ## Set fabric env properties once virtual machine has been created, ## these are mostly used to modify CloudBioLinux behavior and will ## override the properties loaded by fabricrc_file defind above. fabricrc_overrides: ## Mercurial repository for galaxy. If use_galaxy is True above, ## this should be fork of or contain changesets in ## https://bitbucket.org/jmchilton/cloud-galaxy-dist. For a more ## traditional CloudMan workflow this can be any galaxy repository. galaxy_repository: https://bitbucket.org/jmchilton/cloud-galaxy-dist ## Set to true because above galaxy repo preconfigures Galaxy ## properly. Comment this out for a more traditional CloudMan ## workflow. galaxy_preconfigured_repository: True ## These defaults reflect installing galaxy and tools on instance ## disk. These defaults should be changed for more traditional ## CloudMan workflow to reflect defaults in config/fabricrc.txt. # Location to install galaxy galaxy_home: /opt/galaxy/web # Location to setup galaxy tool dependencies galaxy_tools_dir: /opt/galaxy/tools galaxy_jars_dir: /opt/galaxy/jars # Configure biodata location data_files: "/mnt/galaxyIndices" galaxy_base: "/mnt/galaxyIndices/galaxy" ## Setup Galaxy init service. For more traditional cloudman workflow, ## this should be set to False. galaxy_setup_service: True galaxy_setup_xvfb: True postgres_version: 9.1 ## Transferring large files is tough, have fabric try multiple ## times if needed and lengthen timeouts. keepalive: 30 timeout: 60 connection_attempts: 5 tool_data_table_conf_file: config/tool_data_table_conf.xml ## Define Amazon connection information. This is used if `vm_provider: aws` ## (the default) is set can also specify `eucalytus` or `openstack` sections ## for other cloud environments. aws: # Path to private key used to connect to machine image key_file: XXXXXXXXXXXXX access_id: XXXXXXXXXXXXX secret_key: XXXXXXXXXXXX keypair_name: galaxy_key size_id: m1.medium ## Fresh Ubuntu Instance image_id: ami-5c9b4935 availability_zone: us-east-1 ## Set this to a specific instance UUID to have deploy.sh commands use an ## existing VM instead launching a new one # use_existing_instance: XXXXXXXXXXXXXXXXXXXXXXXXXXXX ## If package_type is default, manual packaging scripts are setup on the VM ## that are appropriate for instance or EBS backed instances. If package_type ## is create_image, Amazon's CreateImage instance command is used, this is ## only appropriate for EBS backed instances (requires boto). # package_type: default ## Packaging related options, all optional unless --action=package is ## specified. ## AWS -> Your Account -> Security Credentials -> X.509 Certificates # x509_cert: /path/to/cert-XXXXXXXXXXXXXXXX.pem # x509_key: /path/to/pk-XXXXXXXXXXXXXX.pem ## AWS -> Your Account -> Security Credentials -> Account Identifiers -> AWS Account ID # user_id: 123456789 ## S3 bucket to store your image into # package_bucket: test1 ## Name of packaged instance # package_image_name: Test1 ## Make the resulting image available to the public (currently only available ## for package_type: create_image). # make_public: False # Vagrant VM provider options, these are used if vm_provider: vagrant is set. vagrant: key_file: ~/.vagrant.d/insecure_private_key galaxy: ## In order to create data libraries. First user should be admin@localhost ## and an API key must be specified, be sure to change API keys and passwords users: - username: admin@localhost password: adminpass api_key: 1234556789 - username: user1@example.com password: pass1 api_key: 987654321 ## Histories to create for this user histories: - ExampleHistory ## Workflows to automatically import for this user. workflows: # - /path/to/workflow_file ## We have had all sorts of trouble getting fabric to reliably SFTP ## really large files, so we have made all sorts of options available ## to tweak this process. transfer: ## Override what local temp directory is used on this machine if ## chunking files or compression is used in transfer # local_temp_dir: /tmp/ ## Compress transferred files (default is False) # compress_transfers: True ## If the following parameter is set, files will be split into ## chunks of this size (in Mb) and recombined on remote host. # transfer_chunk_size: 1 genomes: # Details about the genomes you want to include. # Required genome fields (corresponding to Galaxy's tool_data_table_conf.xml columns): # dbkey - globally unique identifier for the genome (e.g., hg19) # name - descriptive name for the given genome (to be displayed in Galaxy, e.g., Hsapiens) # Optional genome fields (corresponding to Galaxy's tool_data_table_conf.xml columns): # formats, species, dbkey1, dbkey2, value, path, index # Additional genome fields specific to data deployment: # genome_indexes - list of tool indexes specific to the associated genome (overrides global 'genome_indexes') genomes: - dbkey: phix name: phiX174 # - dbkey: hg19 # name: Human (hg19) # dbsnp: false # rnaseq: false # Global set of indexes to include for each genome. # Available choices are in GENOMES_INDEXES_SUPPORTED in cloudbiolinux/biodata/genomes.py genome_indexes: - bwa # Additional data targets install_liftover: false install_uniref: false ## This tool selection was up-to-date for the galaxy-vm-launcher, but ## this probably should be updated based on the latest contents of ## contrib/flavor/cloudman/tools.yaml. tools: applications: bwa: "0.5.9" ucsc_tools: "default" bowtie: "0.12.7" samtools: - "0.1.16" - "0.1.18" abyss: "1.3.3" bfast: "0.7.0a" velvet: "1.2.07" perm: "3.6" lastz: "1.02.00" mosaik: "default" freebayes: "default" macs: "1.4.2-1" emboss: - version: "6.5.7" symlink_versions: "5.0.0" # Setup a symbolic link, so when Galaxy looks for 5.0.0 it finds newest version fastx_toolkit: "0.0.13.2" cufflinks: "1.3.0" tophat: "1.3.3" picard: "1.56" fastqc: "0.10.0" gatk: "1.4-9-g1f1233b" maq: "0.7.1" megablast: "2.2.22" blast: "2.2.25+" sputnik: "r1" taxonomy: "r2" add_scores: "r1" hyphy: "r418" srma: "0.1.15" beam: "2" pass: "2.0" lps_tool: "2010.09.30" plink: "1.07" haploview: "4.2b" eigenstrat: "3.0" r_packages: - DBI - RColorBrewer - RCurl - RSQLite - XML - biglm - bitops - digest - ggplot2 - graph - hexbin - hwriter - kernlab - latticeExtra - leaps - pamr - plyr - proto - qvalue - reshape - statmod - xtable - yacca bioconductor_packages: - AnnotationDbi - ArrayExpress - ArrayTools - Biobase - Biostrings - DynDoc - GEOquery - GGBase - GGtools - GSEABase - IRanges - affy - affyPLM - affyQCReport - affydata - affyio - annaffy - annotate - arrayQualityMetrics - beadarray - biomaRt - gcrma - genefilter - geneplotter - globaltest - hgu95av2.db - limma - lumi - makecdfenv - marray - preprocessCore - ShortRead - siggenes - simpleaffy - snpMatrix - vsn ================================================ FILE: deploy/test_install_galaxy_tool.py ================================================ """ Test script for building Python API for installing Galaxy tools using CBL without any dependencies (i.e. it clones down CBL and utilizes it like bcbio-nextgen's installer). Goal is to ultimately fold something like this to Galaxy tool shed client code to provide high-level support for easy CloudBioLinux based tool installations as @chapmanb described at the 2013 BOSC Codefest. """ import os from subprocess import check_call from tempfile import mkdtemp from getpass import getuser DEFAULT_CBL_URL = "https://github.com/chapmanb/cloudbiolinux.git" def __clone_cloudbiolinux(cbl_config): """Clone CloudBioLinux to a temporary directory. TODO: Support particular revision. """ cbl_url = cbl_config.get("repository", DEFAULT_CBL_URL) cbl_dir = mkdtemp(suffix="cbl") check_call(["git", "clone", cbl_url, cbl_dir]) revision = cbl_config.get("revision", None) if revision: git_dir = os.path.join(cbl_dir, ".git") check_call(["git", "--work-tree", cbl_dir, "--git-dir", git_dir, "checkout", revision]) return cbl_dir def install_cbl_tool(tool_name, tool_version, install_dir, cbl_config={}): cbl_dir = __clone_cloudbiolinux(cbl_config) cbl_install_command = [os.path.join(cbl_dir, "deploy", "deploy.sh"), "--action", "install_galaxy_tool"] deployer_args = {"vm_provider": "novm", "galaxy_tool_name": tool_name, "galaxy_tool_version": tool_version, "galaxy_tool_dir": install_dir, "settings": "__none__"} for prop, val in deployer_args.iteritems(): cbl_install_command.append("--%s" % prop) cbl_install_command.append(val) fabric_properties = {"use_sudo": "False", "galaxy_user": getuser()} for prop, val in fabric_properties.iteritems(): cbl_install_command.append("--fabric_property") cbl_install_command.append(prop) cbl_install_command.append("--fabric_value") cbl_install_command.append(val) check_call(cbl_install_command) cbl_config = {"repository": "https://github.com/jmchilton/cloudbiolinux.git"} install_cbl_tool("tint_proteomics_scripts", "1.19.20", os.path.abspath("test_tool_dir"), cbl_config) ================================================ FILE: deploy/update_dependencies.sh ================================================ #!/bin/bash # Name of virtualenv to create using virtualenvwrapper VIRTUALENV_NAME=cbl_deploy # Configure virtualenv and virtualenvwrapper with virtualenv-burrito . $HOME/.venvburrito/startup.sh # Upgrade dependencies mkvirtualenv -r requirements.txt $VIRTUALENV_NAME ================================================ FILE: doc/Makefile ================================================ # Makefile for Sphinx documentation # # You can set these variables from the command line. SPHINXOPTS = SPHINXBUILD = sphinx-build PAPER = BUILDDIR = build # Internal variables. PAPEROPT_a4 = -D latex_paper_size=a4 PAPEROPT_letter = -D latex_paper_size=letter ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) source # the i18n builder cannot share the environment and doctrees with the others I18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) source .PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest gettext help: @echo "Please use \`make ' where is one of" @echo " html to make standalone HTML files" @echo " dirhtml to make HTML files named index.html in directories" @echo " singlehtml to make a single large HTML file" @echo " pickle to make pickle files" @echo " json to make JSON files" @echo " htmlhelp to make HTML files and a HTML help project" @echo " qthelp to make HTML files and a qthelp project" @echo " devhelp to make HTML files and a Devhelp project" @echo " epub to make an epub" @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" @echo " latexpdf to make LaTeX files and run them through pdflatex" @echo " text to make text files" @echo " man to make manual pages" @echo " texinfo to make Texinfo files" @echo " info to make Texinfo files and run them through makeinfo" @echo " gettext to make PO message catalogs" @echo " changes to make an overview of all changed/added/deprecated items" @echo " linkcheck to check all external links for integrity" @echo " doctest to run all doctests embedded in the documentation (if enabled)" clean: -rm -rf $(BUILDDIR)/* html: $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html @echo @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." dirhtml: $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml @echo @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." singlehtml: $(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml @echo @echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml." pickle: $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle @echo @echo "Build finished; now you can process the pickle files." json: $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json @echo @echo "Build finished; now you can process the JSON files." htmlhelp: $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp @echo @echo "Build finished; now you can run HTML Help Workshop with the" \ ".hhp project file in $(BUILDDIR)/htmlhelp." qthelp: $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp @echo @echo "Build finished; now you can run "qcollectiongenerator" with the" \ ".qhcp project file in $(BUILDDIR)/qthelp, like this:" @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/CloudBioLinux.qhcp" @echo "To view the help file:" @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/CloudBioLinux.qhc" devhelp: $(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp @echo @echo "Build finished." @echo "To view the help file:" @echo "# mkdir -p $$HOME/.local/share/devhelp/CloudBioLinux" @echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/CloudBioLinux" @echo "# devhelp" epub: $(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub @echo @echo "Build finished. The epub file is in $(BUILDDIR)/epub." latex: $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex @echo @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." @echo "Run \`make' in that directory to run these through (pdf)latex" \ "(use \`make latexpdf' here to do that automatically)." latexpdf: $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex @echo "Running LaTeX files through pdflatex..." $(MAKE) -C $(BUILDDIR)/latex all-pdf @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." text: $(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text @echo @echo "Build finished. The text files are in $(BUILDDIR)/text." man: $(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man @echo @echo "Build finished. The manual pages are in $(BUILDDIR)/man." texinfo: $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo @echo @echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo." @echo "Run \`make' in that directory to run these through makeinfo" \ "(use \`make info' here to do that automatically)." info: $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo @echo "Running Texinfo files through makeinfo..." make -C $(BUILDDIR)/texinfo info @echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo." gettext: $(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale @echo @echo "Build finished. The message catalogs are in $(BUILDDIR)/locale." changes: $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes @echo @echo "The overview file is in $(BUILDDIR)/changes." linkcheck: $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck @echo @echo "Link check complete; look for any errors in the above output " \ "or in $(BUILDDIR)/linkcheck/output.txt." doctest: $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest @echo "Testing of doctests in the sources finished, look at the " \ "results in $(BUILDDIR)/doctest/output.txt." ================================================ FILE: doc/hacking.md ================================================ # Hacking BioLinux tips and tricks The BioLinux tools allow building a full environment for Bioinformatics. The design allows for flexible targets (Flavors). VirtualBox + Vagrant make an ideal toying environment for building and testing targets. The BioLinux regression test system (in ./test/) uses that combo too. Please read the README and ./doc/vagrant documentation that come with the BioLinux source tree first (see http://github.com/chapmanb/cloudbiolinux). ## Start with the Minimal Minimal is the smallest common denominator of all, as it installs the minimum of packages to bootstrap a full install. Once the vagrant box is up and running, Minimal is invoked from the desktop by fab -f $source/fabfile.py -H target_hostname -c $source/contrib/flavor/minimal/fabricrc_debian.txt install_biolinux:flavor=$source/contrib/flavor/minimal where $source points to your biolinux source tree (replace 'target_hostname' with 'vagrant' when using that). In fact, the testing script in ./test/test_vagrant does exactly this! Try: cd $my_vms $source/test/test_vagrant --help and the actual run: $source/test/test_vagrant (also read $source/test/README) The main.yaml file ascertains the major editors are included, as well remote access, version control, and the basic build system (gcc and friends). Note the Minimal overwrites the (apt) sources file to make sure there are no conflicts with user settings. ## Adding install packages To expand on the package list you can define your own main.yaml, and pass that in. In your main.yaml file add the meta-packages listed in config/packages.yaml. Invoke your new package list with fab -f $source/fabfile.py -H target_hostname -c $source/contrib/flavor/minimal/fabricrc_debian.txt install_biolinux:flavor=/path/to/myproject where the `myproject` directory contains your main.yaml. It is that simple! If packages.yaml is not complete, you may suggest changing its contents in the main repository. The alternative is to create your own flavor, which we will do in a minute. The same strategy holds for the other definitions in the ./config directory, such as for Ruby gems, Python eggs, Perl CPAN, R-CRAN etc. ## Define a Flavor For a cross language Bio* project performance test I needed to create a special version of BioLinux that would pull in a list of scripts and some additional packages. Starting from an existing edition (in this case the Minimum edition, but it also works on top of BioNode and BioLinux editions), I created a new flavor in ./contrib/flavor/pjotrp/biotest/biotestflavor.py, named BioTestFlavor (note you also need an empty __init__.py file). A Flavor class overrides the Flavor methods defined in ./cloudbio/flavor/__init__.py, in particular rewrite_config_items, a generic hook to rewrite a list of configured items (the package lists), and post_install, a post installation hook. To see what packages your Flavor wants to install, simply override rewrite_config_items, and add a print statement. For example: def rewrite_config_items(self, name, packages): for package in packages: env.logger.info("Selected: "+name+" "+package) return packages The flavor is itself is found through a fabricrc.txt file. The main package list may be in a new main.yaml file. Kicking it into submission: fab -f $source/fabfile.py -H target_hostname -c $source/contrib/flavor/pjotrp/biotest/fabricrc_debian.txt install_biolinux:flavor=$source/contrib/flavor/pjotrp/biotest The flavor module itself sets env.flavor on loading the module (this can only happen once). For more examples see the files in ./contrib/flavor. ## Distribute a VirtualBox With vagrant a box can be exported with vagrant package To extract the contained VirtualBox vmdk: tar xvf package.box ## Flavor: change default sources (apt, yum, rpm) Note: NYI BioLinux creates a default list of package sources. These sources can be overridden by the Flavor.rewrite_apt_sources_list method - which should return a new list. ## Flavor: install additional packages The primary way of adding new packages is by creating a new main.yaml file, as discussed above in ''Define a flavor''. In addition a flavor can define a method: BioLinux creates a list of packages. These sources can be overridden by the Flavor.rewrite_config_list method - which should return a new list. ## Flavor: filter packages To filter/remove packages from the default list, use rewrite_config_list to filter existing meta packages. ## Flavor: rewrite Ruby gem, Perl CPAN, Python egg, R CRAN lists The function rewrite_config_list also allows rewriting package lists for Ruby, Python, R, Perl etc. The general idea is that they define the inclusion of the main languages, and pull in Bio* related packages. To override this behaviour use the rewrite functions, e.g. def rewrite_config_list(self, name, list): if name == 'ruby': return [ 'bio' ] return list only allows the BioRuby 'bio' gem to be installed. This happens at the time your meta main.yaml reads libraries: - ruby-libs and pulls in ruby-libs.yaml. One ruby-libs.yaml is shared to make sure all editions are up-to-date. Likewise for all the other yaml files. The configuration options are at the main.yaml (meta-package) level, and by using rewrite methods at Edition and Flavor levels. ## Flavor: install special software BioLinux comes with a bag of tricks to install special software outside the main package system. There are methods for checking out source repositories, and building software. There are methods for accessing public data resources (such as Amazon S3). These are so called custom installs which are defined in custom.yaml. Each of these can be pulled in and are configured by code in the ./cloudbio/custom/ directory. The method fetches names from custom.yaml that delegate to a method in the custom/name.py program. These mechanisms are shared between BioLinux editions. But, importantly, it is easy to role your own custom methods using a Flavor! This mechanism can also be used to automatically run post-install software, such as puppet, cfruby and chef. For example, you can tell your flavor to clone a git repository, and execute a script by adding a post_install method to your flavor. E.g. def post_install(self): env.logger.info("Starting post-install") if exists('Scalability'): with cd('Scalability'): run('git pull') else: _fetch_and_unpack("git clone git://github.com/pjotrp/Scalability.git") # Now run a post installation routine run('./Scalability/scripts/hello.sh') You can run post_install on its own (convenient for testing!) using the finalize target, e.g. fab -H hostname -f $source/fabfile.py -c $flavor/fabricrc_debian.txt install_biolinux:flavor=$flavor,target=finalize (Note: finalize may become post-install in the future) For a full Flavor example see https://github.com/pjotrp/cloudbiolinux/blob/master/contrib/flavor/pjotrp/biotest/biotestflavor.py ## Individualize a Flavor with env.environment Sometimes it may be useful to have post-install one-offs, for individual purposes (say you want to define a user account for yourself). Rather than create a full Flavor for every possibility, you could add a parameter to the fabricrc file. Even better, add a command line parameter named 'environment' to the install_biolinux parameter list. E.g. fab -H hostname -f $source/fabfile.py -c $flavor/fabricrc_debian.txt install_biolinux:flavor=$flavor,environment=special which automatically becomes part of the Flavor environment state as 'env.environment'. Use this parameter to distinguish between targets. We use it for distinguishing dev,test and production environments. # More tips and tricks ## Using VNC To have a remote desktop, login with ssh to the VM and start VNC vnc4server Enter a password. Note the output pointing to the VNC viewer (IP:1). on the client (your desktop) use vncviewer IP:1 ## Tip for checking BioLinux installation effects To see the what a BioLinux install does to your system, store the settings of the original (untouched) state of a VM: 1. Make a dump of the current installed package list dpkg -l > dpkg-original-list.txt 2. Store the /etc tree - one way is to use git in /etc After running BioLinux you can see what has been done to your system by diffing against the package list, and checking /etc. ## Use the testing framework to create new Flavors BioLinux comes with a testing framework in ./test. The frame work creates a new VM on a local machine. You can add tests, to check if a VM is complete. See the main README file for more information. ## Testing a ggd recipe reference=hg38, or GRCh37, or another. system_install=/path/bcbio without bin ``` mkdir recipe_test cp recipe.yaml recipe_test cd recipe_test git clone https://github.com/chapmanb/cloudbiolinux ln -s /path/bcbio/genomes/Hsapiens/[reference]/seq seq export PYTHONPATH=/path/cloudbiolinux:$PYTHONPATH python -c 'from cloudbio.biodata.ggd import install_recipe; install_recipe("/path/where_to_install", "/path/bcbio", "recipe.yaml", "[reference]")' ``` In bcbio the alternative option is to: - start bcbio_nextge.py upgrade [options] - interrupt it - modify the recipe in tmpbcbio-install/cloudbiolinux/ggd-recipes (not in ggd-run.sh!) - rerun bcbio_nextgen.py upgrade [options]. ## Testing conda environments with bcbio before pushing a PR: When bcbio is doing an update `bcbio_nextgen.py upgrade -u development --tools`, it pulls cloudbiolinux from github. To use your copy of cloudbiolinux with modified conda yaml: Before running bcbio upgrade do ``` mkdir tmpbcbio-install ln -s /your/test/copy/of/cloudbiolinux tmpbcbio-install/cloudbiolinux ``` and bcbio will skip that step and use the link instead, so you can make the change locally and test it out before pushing it. ================================================ FILE: doc/intro/FAQ.tex ================================================ Are there online or local user groups for Amazon Web Services? User groups are listed on the \href{https://aws.amazon.com/about-aws/events/}{AWS Events page}. ================================================ FILE: doc/intro/README ================================================ These files make up an introductory tutorial about running CloudBioLinux on Amazon EC2. The controlling file is called gettingStarted_Cloud-Bio-Linux.tex and mostly just has include statements for the other tex files. The pdf was made using pdflatex and calling it on gettingStarted_Cloud-Bio-Linux.tex ================================================ FILE: doc/intro/basicTerminology.aux ================================================ \relax \@writefile{toc}{\contentsline {section}{\numberline {A.2}Basic terminology}{29}{section.A.2}} \@writefile{toc}{\contentsline {paragraph}{}{29}{section*.80}} \@writefile{toc}{\contentsline {paragraph}{Amazon Machine Image}{29}{section*.81}} \@writefile{toc}{\contentsline {paragraph}{Amazon Simple Storage Service}{29}{section*.82}} \@writefile{toc}{\contentsline {paragraph}{AMI}{29}{section*.83}} \@writefile{toc}{\contentsline {paragraph}{Availability Regions and Zones}{29}{section*.84}} \@writefile{toc}{\contentsline {paragraph}{AWS}{29}{section*.85}} \@writefile{toc}{\contentsline {paragraph}{AWS console}{29}{section*.86}} \@writefile{toc}{\contentsline {paragraph}{EBS}{29}{section*.87}} \@writefile{toc}{\contentsline {paragraph}{EC2}{29}{section*.88}} \@writefile{toc}{\contentsline {paragraph}{Elastic Block Storage}{29}{section*.89}} \@writefile{toc}{\contentsline {paragraph}{Elastic Compute Cloud}{30}{section*.90}} \@writefile{toc}{\contentsline {paragraph}{Image}{30}{section*.91}} \@writefile{toc}{\contentsline {paragraph}{Instance}{30}{section*.92}} \@writefile{toc}{\contentsline {paragraph}{NX}{30}{section*.93}} \@writefile{toc}{\contentsline {paragraph}{Putty}{30}{section*.94}} \@writefile{toc}{\contentsline {paragraph}{S3}{30}{section*.95}} \@writefile{toc}{\contentsline {paragraph}{SSH}{30}{section*.96}} \@writefile{toc}{\contentsline {paragraph}{Snapshot}{30}{section*.97}} \@writefile{toc}{\contentsline {paragraph}{Volume}{30}{section*.98}} \@setckpt{basicTerminology}{ \setcounter{page}{31} \setcounter{equation}{0} \setcounter{enumi}{4} \setcounter{enumii}{0} \setcounter{enumiii}{0} \setcounter{enumiv}{0} \setcounter{footnote}{0} \setcounter{mpfootnote}{0} \setcounter{part}{0} \setcounter{chapter}{1} \setcounter{section}{2} \setcounter{subsection}{0} \setcounter{subsubsection}{0} \setcounter{paragraph}{0} \setcounter{subparagraph}{0} \setcounter{figure}{3} \setcounter{table}{0} \setcounter{SC@C}{5} \setcounter{Item}{32} \setcounter{Hfootnote}{9} \setcounter{section@level}{4} } ================================================ FILE: doc/intro/basicTerminology.tex ================================================ \section{Basic terminology} \paragraph{}One of the most daunting parts of starting to work on the cloud is the abundance of new words and acronyms that litter the documentation. Here are a few of the most common terms you'll come across, and what they mean when used in the cloud computing context. \paragraph{Amazon Machine Image} - Pre-configured machine images that you can start up and work on. Most commonly referred to as AMI, or just "image" in documentation. \paragraph{Amazon Simple Storage Service} - More commonly known as "S3", this is a storage type offered as a web service. This document is primarily concerned with EBS Volumes for storage, rather than S3. However, if you take a snapshot of a volume, it is stored in S3. There are many \href{http://www.cloudiquity.com/2009/03/differences-between-s3-and-ebs/}{differences between EBS and S3}, which you should find out about if you plan to work on the cloud seriously. \paragraph{AMI} - see "Amazon Machine Image". \paragraph{Availability Regions and Zones} - Amazon have a number of data centres around the world. A Region in this context described centres in different geographic areas (e.g. the U.S. and Europe). An Availability Zone describes distinct locations within a region. For a new user, the most likely thing to look out for is that you create volumes in the same region and zone as your running instance(s). For more detailed information about regions and zones, check out the \href{http://docs.amazonwebservices.com/AWSEC2/latest/UserGuide/concepts-regions-availability-zones.html}{EC2 Userguide}, including the \href{http://docs.amazonwebservices.com/AWSEC2/latest/UserGuide/index.html?FAQ_Regions_Availability_Zones.html}{Region and Availability Zone FAQ}. \paragraph{AWS} - Stands for "Amazon Web Services". An umbrella name for the myriad of web services offered by Amazon. These services include the provision of compute power and storage, which are the focus of this document. \paragraph{AWS console} - \href{http://aws.amazon.com/console/}{A graphical web interface to your AWS account}. From this interface, you can create, manage and delete Amazon resources such as system instances and data volumes. \paragraph{EBS} - See "Elastic Block Storage". \paragraph{EC2} - See "Elastic Compute Cloud" \paragraph{Elastic Block Storage} - Also referred to as "volumes". A type of storage that was designed for Amazon EC2 instances. You can create EBS volumes and mount these as devices, as you might with a hard drive on a standard system. EBS volumes are particularly useful if you plan to store data for multiple uses. \paragraph{Elastic Compute Cloud} - Also known as EC2, this is an Amazon web service that makes computational capacity available as a virtual computing environment. When you start up an image, you do so from within an EC2 account, from the EC2 section of the AWS Console. \paragraph{Image} - See "Amazon Machine Image" \paragraph{Instance} - See "Amazon Machine Image" \paragraph{NX} - At a user level, NX is a method allowing you to experience a full, graphical desktop session on a machine you are logged into remotely. \href{http://en.wikipedia.org/wiki/NX_technology}{Documentation outlining how this is achieved is available}. \paragraph{Putty} - In the context of this document, Putty is a program that can be used on Windows as an SSH client. \paragraph{S3} - See "Amazon Simple Storage Service" \paragraph{SSH} - Stands for Secure Shell. A network protocol that allows data to be exchanged using a secure channel. Information sent over an SSH connection is encrypted. The standard port for SSH connections is port 22. \paragraph{Snapshot} - A snapshot is essentially a copy of a volume. Taking a snapshot of an EBS volume means that you save the state of your volume, as a snapshot, in Amazon S3. This snapshot is then replicated across multiple Availability Zones. This means that the information on your EBS volume is saved in a durable manner. Snapshots can then be used as the starting point for new EBS volumes. For example, if you select a snapshot in the AWS Console, you can choose to make a volume of it. Once you have attached and mounted that volume to a running image, you have access to all the information you saved when you took the snapshot. Snapshots can also be used when sharing data or images. See the official documentation for more. \paragraph{Volume} - See "Elastic Block Storage". ================================================ FILE: doc/intro/cloudbl_desktopIntro.aux ================================================ \relax \@writefile{toc}{\contentsline {section}{\numberline {A.1}The CloudBioLinux Desktop}{27}{section.A.1}} \newlabel{section:cloudblDesktop}{{A.1}{27}{The CloudBioLinux Desktop\relax }{section.A.1}{}} \@writefile{toc}{\contentsline {paragraph}{}{27}{section*.75}} \@writefile{toc}{\contentsline {subsection}{\numberline {A.1.1}Bioinformatics documentation}{27}{subsection.A.1.1}} \@writefile{toc}{\contentsline {paragraph}{}{27}{section*.76}} \@writefile{toc}{\contentsline {paragraph}{}{27}{section*.77}} \@writefile{lof}{\contentsline {figure}{\numberline {A.1}{\ignorespaces Bioinformatics docs}}{27}{figure.A.1}} \newlabel{fig:bldocs}{{A.1}{27}{Bioinformatics docs\relax }{figure.A.1}{}} \newlabel{SC@4}{{A.1.2}{28}{\relax }{section*.78}{}} \@writefile{lof}{\contentsline {figure}{\numberline {A.2}{\ignorespaces Bioinformatics menu}}{28}{figure.A.2}} \newlabel{fig:bioinfmenu}{{A.2}{28}{\SC@OPTCAPtext \relax }{figure.A.2}{}} \newlabel{SC@5}{{A.1.3}{28}{\relax }{section*.79}{}} \@writefile{lof}{\contentsline {figure}{\numberline {A.3}{\ignorespaces Open terminal}}{28}{figure.A.3}} \newlabel{fig:NXterminal}{{A.3}{28}{\SC@OPTCAPtext \relax }{figure.A.3}{}} \@writefile{toc}{\contentsline {subsection}{\numberline {A.1.2}The bioinformatics menu}{28}{subsection.A.1.2}} \@writefile{toc}{\contentsline {paragraph}{}{28}{section*.78}} \@writefile{toc}{\contentsline {subsection}{\numberline {A.1.3}Opening a terminal}{28}{subsection.A.1.3}} \@writefile{toc}{\contentsline {paragraph}{}{28}{section*.79}} \@setckpt{cloudbl_desktopIntro}{ \setcounter{page}{29} \setcounter{equation}{0} \setcounter{enumi}{4} \setcounter{enumii}{0} \setcounter{enumiii}{0} \setcounter{enumiv}{0} \setcounter{footnote}{0} \setcounter{mpfootnote}{0} \setcounter{part}{0} \setcounter{chapter}{1} \setcounter{section}{1} \setcounter{subsection}{3} \setcounter{subsubsection}{0} \setcounter{paragraph}{0} \setcounter{subparagraph}{0} \setcounter{figure}{3} \setcounter{table}{0} \setcounter{SC@C}{5} \setcounter{Item}{32} \setcounter{Hfootnote}{9} \setcounter{section@level}{4} } ================================================ FILE: doc/intro/cloudbl_desktopIntro.tex ================================================ \section{The CloudBioLinux Desktop} \label{section:cloudblDesktop} \paragraph{}This section provides only a few tips about the CloudBioLinux desktop. As CloudBioLinux is based on Ubuntu, we recommend referring to online Ubuntu documentation for further details about the system in general. \subsection{Bioinformatics documentation} \paragraph{}CloudBioLinux comes with a categorised bioinformatics documentation system, which provides information about the bioinformatics software on the system. This is the easiest place to find out about what bioinformatics programs are available, and where to find out more about how to use them. \paragraph{}At the moment, the easiest way to get to the installed documents is to open up a web browser (just click on the little firefox icon in the top taskbar), and then enter the address: \textbf{file:///var/www/bioinformatics/index.html}. See figure \ref{fig:bldocs}. \begin{figure}[!hd] \fbox { \begin{minipage}{13cm} \includegraphics[width=\maxwidth]{"images/blbioinfDocumentation"} \caption[Bioinformatics docs]{\label{fig:bldocs}A categorised bioinformatics documentation system. Perusing the information in these pages is probably the easiest way to start finding out what is available on your CloudBioLinux image.} \end{minipage} } \end{figure} \subsection{The bioinformatics menu} \paragraph{}Some of the bioinformatics software on the system can be accessed via the graphical bioinformatics menu. See figure \ref{fig:bioinfmenu}. \begin{SCfigure}[][t] \includegraphics[width=40mm]{"images/bioinfMenuNX2"} \caption[Bioinformatics menu]{\label{fig:bioinfmenu}Some bioinformatics programs can be launched from the Bioinformatics menu. For a more comprehensive listing of the bioinformatics software on the system, check out the installed bioinformatics documentation pages. See figure \ref{fig:bldocs}.} \end{SCfigure} \subsection{Opening a terminal} \paragraph{}Much of the bioinformatics software on CloudBioLinux systems can only be launched from the command line. This includes command line tools, and also some graphical software. In addition, much of the power of Linux comes through using the command line. You can easily open a terminal, in which you can type commands, when logged into a full desktop session. Just go to the \textbf{Accessories submenu} under the Applications menu in the top taskbar, and choose \textbf{Terminal}. See figure \ref{fig:NXterminal}. \begin{SCfigure}[][t] \includegraphics[width=40mm]{"images/openTerminalNX"} \caption[Open terminal]{\label{fig:NXterminal}You can open a command line terminal from under the Accessories submenu of the Applications menu in the top taskbar.} \end{SCfigure} ================================================ FILE: doc/intro/getReady.aux ================================================ \relax \newlabel{SC@1}{{1.1}{3}{\relax }{section*.2}{}} \@writefile{lof}{\contentsline {figure}{\numberline {1.1}{\ignorespaces Sign up for AWS}}{3}{figure.1.1}} \newlabel{fig:aws_signup}{{1.1}{3}{\SC@OPTCAPtext \relax }{figure.1.1}{}} \@writefile{toc}{\contentsline {section}{\numberline {1.1}Get an Amazon AWS account}{3}{section.1.1}} \newlabel{section:getaccount}{{1.1}{3}{Get an Amazon AWS account\relax }{section.1.1}{}} \@writefile{toc}{\contentsline {paragraph}{}{3}{section*.2}} \@writefile{toc}{\contentsline {paragraph}{The rest of this document assumes you have an AWS account and you are logged into it.}{3}{section*.3}} \@writefile{toc}{\contentsline {section}{\numberline {1.2}Get an Amazon EC2 Account}{3}{section.1.2}} \@writefile{toc}{\contentsline {paragraph}{}{3}{section*.4}} \@writefile{toc}{\contentsline {paragraph}{}{3}{section*.5}} \@writefile{toc}{\contentsline {paragraph}{To get your EC2 account}{3}{section*.6}} \@writefile{toc}{\contentsline {section}{\numberline {1.3}Get an EC2 key pair}{3}{section.1.3}} \@writefile{toc}{\contentsline {paragraph}{}{3}{section*.7}} \@writefile{lof}{\contentsline {figure}{\numberline {1.2}{\ignorespaces EC2 Homepage}}{4}{figure.1.2}} \newlabel{fig:ec2homepage}{{1.2}{4}{EC2 Homepage\relax }{figure.1.2}{}} \@writefile{toc}{\contentsline {paragraph}{To create a key pair:}{4}{section*.8}} \@writefile{toc}{\contentsline {paragraph}{}{4}{section*.9}} \newlabel{SC@2}{{1.3}{5}{\relax }{figure.1.2}{}} \@writefile{lof}{\contentsline {figure}{\numberline {1.3}{\ignorespaces Keypair creation}}{5}{figure.1.3}} \newlabel{fig:keypaircreate}{{1.3}{5}{\SC@OPTCAPtext \relax }{figure.1.3}{}} \@writefile{toc}{\contentsline {paragraph}{}{5}{section*.10}} \@writefile{toc}{\contentsline {paragraph}{}{5}{section*.11}} \@writefile{toc}{\contentsline {paragraph}{How the EC2 key pairs work}{5}{section*.12}} \@writefile{toc}{\contentsline {paragraph}{}{5}{section*.13}} \@writefile{toc}{\contentsline {section}{\numberline {1.4}Install an NX Client on your local machine}{5}{section.1.4}} \@writefile{toc}{\contentsline {paragraph}{}{5}{section*.14}} \@writefile{toc}{\contentsline {paragraph}{You only need to install an NX Client on your local system if you wish to run a full graphical desktop session for working on CloudBioLinux.}{5}{section*.15}} \@setckpt{getReady}{ \setcounter{page}{6} \setcounter{equation}{0} \setcounter{enumi}{5} \setcounter{enumii}{0} \setcounter{enumiii}{0} \setcounter{enumiv}{0} \setcounter{footnote}{2} \setcounter{mpfootnote}{0} \setcounter{part}{0} \setcounter{chapter}{1} \setcounter{section}{4} \setcounter{subsection}{0} \setcounter{subsubsection}{0} \setcounter{paragraph}{0} \setcounter{subparagraph}{0} \setcounter{figure}{3} \setcounter{table}{0} \setcounter{SC@C}{2} \setcounter{Item}{9} \setcounter{Hfootnote}{2} \setcounter{section@level}{4} } ================================================ FILE: doc/intro/getReady.tex ================================================ \section{Get an Amazon AWS account} \label{section:getaccount} \paragraph{}Anyone can set up an account with Amazon to access their computer cloud. Just go to \href{http://aws.amazon.com}{http://aws.amazon.com} and sign up for an account. \begin{SCfigure}[][t] \includegraphics[width=40mm]{"images/aws_signup_button"} \caption[Sign up for AWS]{\label{fig:aws_signup}Signing up for an AWS account starts with the click of a button.} \end{SCfigure} \paragraph{The rest of this document assumes you have an AWS account and you are logged into it.} \section{Get an Amazon EC2 Account} \paragraph{}There are various ways you can access the power of the Amazon cloud. In this document, we describe using EC2. \paragraph{}If you do not already have one, you need to sign up for an EC2 account. This is in addition to the general Amazon aws account you have if you followed the instructions in section \ref{section:getaccount}. \paragraph{To get your EC2 account}, \begin{enumerate} \item click on the Products tab on the \href{http://aws.amazon.com}{Amazon aws page}, \item click on the \href{http://aws.amazon.com/ec2/}{Amazon Elastic Compute Cloud (EC2) link} that appears in the Compute section of the listing, and \item click on the button in the right hand pane that says Sign up for Amazon EC2. \item Complete the registration process\footnote{Signing up for Amazon EC2 also automatically signs you up for Amazon Simple Storage Service and Amazon Virtual Private Cloud. You will not be charged for any service unless you use it.}. \end{enumerate} \section{Get an EC2 key pair} \paragraph{}After you've signed up for your account, Amazon will send you an email with a link in it to the Access Identifiers section of your account. Amazon provides a list of \href{http://docs.amazonwebservices.com/AWSSecurityCredentials/1.0/AboutAWSCredentials.html#EC2Credentials}{which credentials you need to do particular tasks}. If all you will be doing is starting up Bio-Linux using the Amazon (graphical) console, then you only need an \href{http://docs.amazonwebservices.com/AWSSecurityCredentials/1.0/AboutAWSCredentials.html#EC2KeyPairs}{Amazon EC2 Key Pair}. \begin{figure}[!hd] \fbox { \begin{minipage}{13cm} %the maxwidth variable is defined in the summary page: gettingStarted_CloudBioLinux.tex \includegraphics[width=\maxwidth]{"images/EC2Homepage2"} \caption[EC2 Homepage]{\label{fig:ec2homepage}Your AWS home are will look something like this. Here, we are looking at the information under the EC2 tab (Red Circle). Blue Circle: your Key Pairs - if you don't have any, click on the link to create some. Purple Circle: You can easily launch any publicly available EC2 image from Amazon by clicking on this button. Green Circle: The number of running instances you have. If you have any, you can find out more about them by clicking on this link. Pink Circle: If you add Key Pairs, or start up instances, you may need to hit the refresh button to see changes on your EC2 Dashboard.} \end{minipage} } \end{figure} \begin{SCfigure}[][t] %the maxwidth variable is defined in the summary page: gettingStarted_CloudBioLinux.tex \includegraphics[width=60mm]{"images/keypairCreateButton"} \caption[Keypair creation]{\label{fig:keypaircreate}Keypair creation is simple - just click on the button and follow the instructions on screen.} \end{SCfigure} \paragraph{To create a key pair:} \begin{enumerate} \item Go to the the \href{https://console.aws.amazon.com/ec2/home}{EC2 area on Amazon https://console.aws.amazon.com/ec2/home}. \item Click on the \emph{Key Pairs} link under My Resources in the right hand area of the window. See the blue circle in figure~\ref{fig:ec2homepage}. \item Click on the \emph{Create Key Pair} button near the top of the Key Pairs section of the window. See figure \ref{fig:keypaircreate} on page \pageref{fig:keypaircreate}. \item Give your key pair a memorable name when prompted. Save your private key to a safe location. See the further information below about this. \item Click on the link in the left hand pane to go back to the \emph{EC2 Dashboard} and then click on the \emph{Refresh button} at the far right hand side of the window (see pink circle in figure~\ref{fig:ec2homepage}). \end{enumerate} \paragraph{}You should now see that you have a key pair registered in the \emph{My Resources} section. \paragraph{}Each EC2 key pair includes a private key file and a public key file. \textbf{Save your private key to a secure and memorable location}. Don't lose it or share it. (Amazon does not make a copy of it.) If you're working on Linux, adjust the permissions on your key file so it is readable only by you. \paragraph{}If you plan to use the command line tools to start up an instance, you will also need to get your \href{http://docs.amazonwebservices.com/AWSSecurityCredentials/1.0/AboutAWSCredentials.html#X509Credentials}{X509 certificates}. This document assumes that you will only be using the graphical console, so this is not covered further here. \paragraph{How the EC2 key pairs work} \paragraph{}When you launch a CloudBioLinux instance from Amazon, you will specify a particular EC2 key pair. The Amazon system puts a copy of your public key, which it has a record of, on the instance. You, as the (only!) holder of the private key will be the only one able to access the CloudBioLinux instance you just started up. \section{Install an NX Client on your local machine} \paragraph{}To run a full desktop session, you need have NX client software on the machine you will be connecting to the Amazon Cloud from. If you don't already have NX Client software installed, we recommend the free clients from \href{http://www.nomachine.com/download.php}{NoMachine}\footnote{CloudBioLinux already has the FreeNXServer software installed}. %put that server line in a footnote, because for anyone not familiar with "server" and "client", it might be easy to get confused. \paragraph{You only need to install an NX Client on your local system if you wish to run a full graphical desktop session for working on CloudBioLinux.} ================================================ FILE: doc/intro/gettingStarted_CloudBioLinux.aux ================================================ \relax \ifx\hyper@anchor\@undefined \global \let \oldcontentsline\contentsline \gdef \contentsline#1#2#3#4{\oldcontentsline{#1}{#2}{#3}} \global \let \oldnewlabel\newlabel \gdef \newlabel#1#2{\newlabelxx{#1}#2} \gdef \newlabelxx#1#2#3#4#5#6{\oldnewlabel{#1}{{#2}{#3}}} \AtEndDocument{\let \contentsline\oldcontentsline \let \newlabel\oldnewlabel} \else \global \let \hyper@last\relax \fi \@writefile{toc}{\contentsline {chapter}{\numberline {1}Preparing to work on the Amazon Cloud}{2}{chapter.1}} \@writefile{lof}{\addvspace {10\p@ }} \@writefile{lot}{\addvspace {10\p@ }} \@input{getReady.aux} \@writefile{toc}{\contentsline {chapter}{\numberline {2}Running a CloudBioLinux instance}{6}{chapter.2}} \@writefile{lof}{\addvspace {10\p@ }} \@writefile{lot}{\addvspace {10\p@ }} \@input{workingOnCloudBL.aux} \@writefile{toc}{\contentsline {chapter}{\numberline {3}Working with data on the cloud}{17}{chapter.3}} \@writefile{lof}{\addvspace {10\p@ }} \@writefile{lot}{\addvspace {10\p@ }} \@input{workingWithData.aux} \@writefile{toc}{\contentsline {chapter}{\numberline {A}Appendices}{26}{appendix.A}} \@writefile{lof}{\addvspace {10\p@ }} \@writefile{lot}{\addvspace {10\p@ }} \@input{cloudbl_desktopIntro.aux} \@input{basicTerminology.aux} ================================================ FILE: doc/intro/gettingStarted_CloudBioLinux.out ================================================ \BOOKMARK [0][-]{chapter.1}{Preparing to work on the Amazon Cloud}{} \BOOKMARK [1][-]{section.1.1}{Get an Amazon AWS account}{chapter.1} \BOOKMARK [1][-]{section.1.2}{Get an Amazon EC2 Account}{chapter.1} \BOOKMARK [1][-]{section.1.3}{Get an EC2 key pair}{chapter.1} \BOOKMARK [1][-]{section.1.4}{Install an NX Client on your local machine}{chapter.1} \BOOKMARK [0][-]{chapter.2}{Running a CloudBioLinux instance}{} \BOOKMARK [1][-]{section.2.1}{The process in a nutshell}{chapter.2} \BOOKMARK [1][-]{section.2.2}{Starting up a CloudBioLinux instance}{chapter.2} \BOOKMARK [1][-]{section.2.3}{Connecting to and logging into your CloudBioLinux instance}{chapter.2} \BOOKMARK [2][-]{subsection.2.3.1}{Graphical, or command line?}{section.2.3} \BOOKMARK [2][-]{subsection.2.3.2}{Find out the address of your instance}{section.2.3} \BOOKMARK [2][-]{subsection.2.3.3}{Logging into a terminal using ssh}{section.2.3} \BOOKMARK [2][-]{subsection.2.3.4}{Logging into graphical desktop using NX}{section.2.3} \BOOKMARK [2][-]{subsection.2.3.5}{The logic of the NX setup}{section.2.3} \BOOKMARK [1][-]{section.2.4}{Logging out of your CloudBioLinux instance}{chapter.2} \BOOKMARK [1][-]{section.2.5}{Terminating your CloudBioLinux instance}{chapter.2} \BOOKMARK [0][-]{chapter.3}{Working with data on the cloud}{} \BOOKMARK [1][-]{section.3.1}{Introduction}{chapter.3} \BOOKMARK [1][-]{section.3.2}{Copying data onto your CloudBioLinux instance}{chapter.3} \BOOKMARK [1][-]{section.3.3}{Using EBS volumes for data}{chapter.3} \BOOKMARK [1][-]{section.3.4}{Accesing public datasets on Amazon}{chapter.3} \BOOKMARK [0][-]{appendix.A}{Appendices}{} \BOOKMARK [1][-]{section.A.1}{The CloudBioLinux Desktop}{appendix.A} \BOOKMARK [2][-]{subsection.A.1.1}{Bioinformatics documentation}{section.A.1} \BOOKMARK [2][-]{subsection.A.1.2}{The bioinformatics menu}{section.A.1} \BOOKMARK [2][-]{subsection.A.1.3}{Opening a terminal}{section.A.1} \BOOKMARK [1][-]{section.A.2}{Basic terminology}{appendix.A} ================================================ FILE: doc/intro/gettingStarted_CloudBioLinux.tex ================================================ \documentclass[pdftex]{report} \usepackage[pdftex]{graphicx} %\usepackage{graphicx} %\usepackage{floatflt} \usepackage{sidecap} \usepackage{hyperref} \begin{document} \title{Getting started with CloudBioLinux} \author{Konstantinos Krampis, Bela Tiwari} \maketitle \makeatletter \def\maxwidth { \ifdim\Gin@nat@width>\linewidth \linewidth \else \Gin@nat@width \fi } \makeatother \tableofcontents \chapter{Preparing to work on the Amazon Cloud} \include{getReady} \chapter{Running a CloudBioLinux instance} \include{workingOnCloudBL} \chapter{Working with data on the cloud} \include{workingWithData} \appendix \chapter{Appendices} \include{cloudbl_desktopIntro} \include{basicTerminology} %\include{FAQ} \end{document} ================================================ FILE: doc/intro/gettingStarted_CloudBioLinux.toc ================================================ \contentsline {chapter}{\numberline {1}Preparing to work on the Amazon Cloud}{2}{chapter.1} \contentsline {section}{\numberline {1.1}Get an Amazon AWS account}{3}{section.1.1} \contentsline {paragraph}{}{3}{section*.2} \contentsline {paragraph}{The rest of this document assumes you have an AWS account and you are logged into it.}{3}{section*.3} \contentsline {section}{\numberline {1.2}Get an Amazon EC2 Account}{3}{section.1.2} \contentsline {paragraph}{}{3}{section*.4} \contentsline {paragraph}{}{3}{section*.5} \contentsline {paragraph}{To get your EC2 account}{3}{section*.6} \contentsline {section}{\numberline {1.3}Get an EC2 key pair}{3}{section.1.3} \contentsline {paragraph}{}{3}{section*.7} \contentsline {paragraph}{To create a key pair:}{4}{section*.8} \contentsline {paragraph}{}{4}{section*.9} \contentsline {paragraph}{}{5}{section*.10} \contentsline {paragraph}{}{5}{section*.11} \contentsline {paragraph}{How the EC2 key pairs work}{5}{section*.12} \contentsline {paragraph}{}{5}{section*.13} \contentsline {section}{\numberline {1.4}Install an NX Client on your local machine}{5}{section.1.4} \contentsline {paragraph}{}{5}{section*.14} \contentsline {paragraph}{You only need to install an NX Client on your local system if you wish to run a full graphical desktop session for working on CloudBioLinux.}{5}{section*.15} \contentsline {chapter}{\numberline {2}Running a CloudBioLinux instance}{6}{chapter.2} \contentsline {section}{\numberline {2.1}The process in a nutshell}{7}{section.2.1} \contentsline {paragraph}{}{7}{section*.16} \contentsline {paragraph}{}{8}{section*.17} \contentsline {paragraph}{A note about charging:}{8}{section*.18} \contentsline {paragraph}{}{8}{section*.19} \contentsline {paragraph}{A couple of things to note when starting out:}{8}{section*.20} \contentsline {section}{\numberline {2.2}Starting up a CloudBioLinux instance}{8}{section.2.2} \contentsline {paragraph}{This document focusses on using the \href {http://console.aws.amazon.com/ec2/home}{AWS Management Console}, a web-interface, for managing Amazon Web Services resources.}{8}{figure.2.2} \contentsline {paragraph}{}{10}{section*.22} \contentsline {section}{\numberline {2.3}Connecting to and logging into your CloudBioLinux instance}{10}{section.2.3} \contentsline {subsection}{\numberline {2.3.1}Graphical, or command line?}{10}{subsection.2.3.1} \contentsline {paragraph}{For those wanting to work in a graphical computing environment}{10}{section*.23} \contentsline {paragraph}{For those comfortable in text-only environments, including Linux users who wish to run graphical programs, without a full desktop,}{10}{section*.24} \contentsline {paragraph}{}{10}{section*.25} \contentsline {subsection}{\numberline {2.3.2}Find out the address of your instance}{10}{subsection.2.3.2} \contentsline {paragraph}{}{10}{section*.26} \contentsline {subsection}{\numberline {2.3.3}Logging into a terminal using ssh}{11}{subsection.2.3.3} \contentsline {paragraph}{}{11}{section*.27} \contentsline {paragraph}{ssh -i mykey.pem root@ec2-184-72-144-209.compute-1.amazonaws.com}{11}{section*.28} \contentsline {paragraph}{}{11}{section*.29} \contentsline {paragraph}{ssh -i mykey.pem ubuntu@ec2-184-72-144-209.compute-1.amazonaws.com}{11}{section*.30} \contentsline {paragraph}{}{11}{section*.31} \contentsline {paragraph}{Note:}{12}{section*.32} \contentsline {paragraph}{ssh -i /home/mydirectory/keys/mykey.pem ubuntu@ec2-184-72-144-209.compute-1.amazonaws.com}{12}{section*.33} \contentsline {paragraph}{}{12}{section*.34} \contentsline {paragraph}{}{12}{section*.35} \contentsline {subsection}{\numberline {2.3.4}Logging into graphical desktop using NX}{12}{subsection.2.3.4} \contentsline {paragraph}{}{12}{section*.36} \contentsline {paragraph}{}{12}{section*.37} \contentsline {paragraph}{}{12}{section*.38} \contentsline {subsection}{\numberline {2.3.5}The logic of the NX setup}{12}{subsection.2.3.5} \contentsline {paragraph}{}{12}{section*.39} \contentsline {paragraph}{}{13}{section*.40} \contentsline {section}{\numberline {2.4}Logging out of your CloudBioLinux instance}{13}{section.2.4} \contentsline {paragraph}{From an NX connection}{13}{section*.41} \contentsline {paragraph}{From an ssh command line (or Putty) connection}{13}{section*.42} \contentsline {section}{\numberline {2.5}Terminating your CloudBioLinux instance}{14}{section.2.5} \contentsline {paragraph}{}{14}{section*.43} \contentsline {paragraph}{}{15}{section*.44} \contentsline {chapter}{\numberline {3}Working with data on the cloud}{17}{chapter.3} \contentsline {section}{\numberline {3.1}Introduction}{18}{section.3.1} \contentsline {paragraph}{}{18}{section*.45} \contentsline {paragraph}{}{18}{section*.46} \contentsline {section}{\numberline {3.2}Copying data onto your CloudBioLinux instance}{18}{section.3.2} \contentsline {paragraph}{}{18}{section*.47} \contentsline {paragraph}{}{18}{section*.48} \contentsline {paragraph}{}{19}{section*.49} \contentsline {paragraph}{}{19}{section*.50} \contentsline {paragraph}{}{19}{section*.51} \contentsline {section}{\numberline {3.3}Using EBS volumes for data}{20}{section.3.3} \contentsline {paragraph}{}{20}{section*.52} \contentsline {paragraph}{}{20}{section*.53} \contentsline {paragraph}{A note on charging:}{20}{section*.54} \contentsline {paragraph}{Creating your volume}{20}{figure.3.2} \contentsline {paragraph}{}{20}{section*.56} \contentsline {paragraph}{}{20}{section*.57} \contentsline {paragraph}{}{21}{section*.58} \contentsline {paragraph}{}{22}{section*.59} \contentsline {paragraph}{Accessing your volume}{22}{section*.60} \contentsline {paragraph}{}{22}{section*.61} \contentsline {paragraph}{}{23}{section*.62} \contentsline {paragraph}{Unmounting your volume}{24}{figure.3.6} \contentsline {paragraph}{This is a simple but vital step to avoid the possibility of data corruption.}{24}{section*.64} \contentsline {paragraph}{}{24}{section*.65} \contentsline {paragraph}{Detaching your volume}{24}{section*.66} \contentsline {paragraph}{}{24}{section*.67} \contentsline {paragraph}{Backing up or sharing your volume}{25}{section*.68} \contentsline {paragraph}{}{25}{section*.69} \contentsline {paragraph}{Deleting your volume}{25}{section*.70} \contentsline {paragraph}{}{25}{section*.71} \contentsline {section}{\numberline {3.4}Accesing public datasets on Amazon}{25}{section.3.4} \contentsline {paragraph}{}{25}{section*.72} \contentsline {paragraph}{}{25}{section*.73} \contentsline {paragraph}{}{25}{section*.74} \contentsline {chapter}{\numberline {A}Appendices}{26}{appendix.A} \contentsline {section}{\numberline {A.1}The CloudBioLinux Desktop}{27}{section.A.1} \contentsline {paragraph}{}{27}{section*.75} \contentsline {subsection}{\numberline {A.1.1}Bioinformatics documentation}{27}{subsection.A.1.1} \contentsline {paragraph}{}{27}{section*.76} \contentsline {paragraph}{}{27}{section*.77} \contentsline {subsection}{\numberline {A.1.2}The bioinformatics menu}{28}{subsection.A.1.2} \contentsline {paragraph}{}{28}{section*.78} \contentsline {subsection}{\numberline {A.1.3}Opening a terminal}{28}{subsection.A.1.3} \contentsline {paragraph}{}{28}{section*.79} \contentsline {section}{\numberline {A.2}Basic terminology}{29}{section.A.2} \contentsline {paragraph}{}{29}{section*.80} \contentsline {paragraph}{Amazon Machine Image}{29}{section*.81} \contentsline {paragraph}{Amazon Simple Storage Service}{29}{section*.82} \contentsline {paragraph}{AMI}{29}{section*.83} \contentsline {paragraph}{Availability Regions and Zones}{29}{section*.84} \contentsline {paragraph}{AWS}{29}{section*.85} \contentsline {paragraph}{AWS console}{29}{section*.86} \contentsline {paragraph}{EBS}{29}{section*.87} \contentsline {paragraph}{EC2}{29}{section*.88} \contentsline {paragraph}{Elastic Block Strorage}{29}{section*.89} \contentsline {paragraph}{Elastic Compute Cloud}{30}{section*.90} \contentsline {paragraph}{Image}{30}{section*.91} \contentsline {paragraph}{Instance}{30}{section*.92} \contentsline {paragraph}{NX}{30}{section*.93} \contentsline {paragraph}{Putty}{30}{section*.94} \contentsline {paragraph}{S3}{30}{section*.95} \contentsline {paragraph}{SSH}{30}{section*.96} \contentsline {paragraph}{Snapshot}{30}{section*.97} \contentsline {paragraph}{Volume}{30}{section*.98} ================================================ FILE: doc/intro/tips.tex ================================================ \section{Tips} \paragraph{This page contains links and tips that I found useful when starting out.} The \href{https://aws.amazon.com/about-aws/events/}{AWS Events} page contains a listing of things like webinars, conferences and user groups - both online and real world. ================================================ FILE: doc/intro/usefulLinks.tex ================================================ \section{Useful Links} The \href{http://docs.amazonwebservices.com}{main Amazon documentation webpage} contains links to oodles of documentation for the Amazon web services. \href{http://docs.amazonwebservices.com/AWSEC2/latest/UserGuide/}{Amazon User Guide} \href{http://docs.amazonwebservices.com/AWSEC2/latest/GettingStartedGuide/}{EC2 Userguide}. The \href{https://aws.amazon.com/about-aws/events/}{AWS Events} page contains a listing of things like webinars, conferences and user groups - both online and real world. ================================================ FILE: doc/intro/workingOnCloudBL.aux ================================================ \relax \@writefile{toc}{\contentsline {section}{\numberline {2.1}The process in a nutshell}{7}{section.2.1}} \@writefile{lof}{\contentsline {figure}{\numberline {2.1}{\ignorespaces Start an Instance}}{7}{figure.2.1}} \newlabel{fig:nutshell}{{2.1}{7}{Start an Instance\relax }{figure.2.1}{}} \@writefile{toc}{\contentsline {paragraph}{}{7}{section*.16}} \@writefile{toc}{\contentsline {paragraph}{}{8}{section*.17}} \@writefile{toc}{\contentsline {paragraph}{A note about charging:}{8}{section*.18}} \@writefile{toc}{\contentsline {paragraph}{}{8}{section*.19}} \@writefile{toc}{\contentsline {paragraph}{A couple of things to note when starting out:}{8}{section*.20}} \@writefile{toc}{\contentsline {section}{\numberline {2.2}Starting up a CloudBioLinux instance}{8}{section.2.2}} \@writefile{toc}{\contentsline {paragraph}{This document focusses on using the \href {http://console.aws.amazon.com/ec2/home}{AWS Management Console}, a web-interface, for managing Amazon Web Services resources.}{8}{figure.2.2}} \@writefile{lof}{\contentsline {figure}{\numberline {2.2}{\ignorespaces Start an Instance}}{9}{figure.2.2}} \newlabel{fig:requestInstance}{{2.2}{9}{Start an Instance\relax }{figure.2.2}{}} \@writefile{toc}{\contentsline {paragraph}{}{10}{section*.22}} \@writefile{toc}{\contentsline {section}{\numberline {2.3}Connecting to and logging into your CloudBioLinux instance}{10}{section.2.3}} \@writefile{toc}{\contentsline {subsection}{\numberline {2.3.1}Graphical, or command line?}{10}{subsection.2.3.1}} \@writefile{toc}{\contentsline {paragraph}{For those wanting to work in a graphical computing environment}{10}{section*.23}} \@writefile{toc}{\contentsline {paragraph}{For those comfortable in text-only environments, including Linux users who wish to run graphical programs, without a full desktop,}{10}{section*.24}} \@writefile{toc}{\contentsline {paragraph}{}{10}{section*.25}} \@writefile{toc}{\contentsline {subsection}{\numberline {2.3.2}Find out the address of your instance}{10}{subsection.2.3.2}} \@writefile{toc}{\contentsline {paragraph}{}{10}{section*.26}} \@writefile{lof}{\contentsline {figure}{\numberline {2.3}{\ignorespaces Start an Instance}}{11}{figure.2.3}} \newlabel{fig:instancesOptions}{{2.3}{11}{Start an Instance\relax }{figure.2.3}{}} \@writefile{toc}{\contentsline {subsection}{\numberline {2.3.3}Logging into a terminal using ssh}{11}{subsection.2.3.3}} \@writefile{toc}{\contentsline {paragraph}{}{11}{section*.27}} \@writefile{toc}{\contentsline {paragraph}{ssh -i mykey.pem root@ec2-184-72-144-209.compute-1.amazonaws.com}{11}{section*.28}} \@writefile{toc}{\contentsline {paragraph}{}{11}{section*.29}} \@writefile{toc}{\contentsline {paragraph}{ssh -i mykey.pem ubuntu@ec2-184-72-144-209.compute-1.amazonaws.com}{11}{section*.30}} \@writefile{toc}{\contentsline {paragraph}{}{11}{section*.31}} \@writefile{toc}{\contentsline {paragraph}{Note:}{12}{section*.32}} \@writefile{toc}{\contentsline {paragraph}{ssh -i /home/mydirectory/keys/mykey.pem ubuntu@ec2-184-72-144-209.compute-1.amazonaws.com}{12}{section*.33}} \@writefile{toc}{\contentsline {paragraph}{}{12}{section*.34}} \@writefile{toc}{\contentsline {paragraph}{}{12}{section*.35}} \newlabel{section:nx}{{2.3.3}{12}{\relax }{section*.35}{}} \@writefile{toc}{\contentsline {subsection}{\numberline {2.3.4}Logging into graphical desktop using NX}{12}{subsection.2.3.4}} \@writefile{toc}{\contentsline {paragraph}{}{12}{section*.36}} \@writefile{toc}{\contentsline {paragraph}{}{12}{section*.37}} \@writefile{toc}{\contentsline {paragraph}{}{12}{section*.38}} \@writefile{toc}{\contentsline {subsection}{\numberline {2.3.5}The logic of the NX setup}{12}{subsection.2.3.5}} \@writefile{toc}{\contentsline {paragraph}{}{12}{section*.39}} \@writefile{lof}{\contentsline {figure}{\numberline {2.4}{\ignorespaces NX Client start}}{13}{figure.2.4}} \newlabel{fig:NX-menuOnUbuntu}{{2.4}{13}{NX Client start\relax }{figure.2.4}{}} \@writefile{toc}{\contentsline {paragraph}{}{13}{section*.40}} \@writefile{toc}{\contentsline {section}{\numberline {2.4}Logging out of your CloudBioLinux instance}{13}{section.2.4}} \@writefile{toc}{\contentsline {paragraph}{From an NX connection}{13}{section*.41}} \@writefile{toc}{\contentsline {paragraph}{From an ssh command line (or Putty) connection}{13}{section*.42}} \@writefile{lof}{\contentsline {figure}{\numberline {2.5}{\ignorespaces NX Client start}}{14}{figure.2.5}} \newlabel{fig:NXConnectionWizard}{{2.5}{14}{NX Client start\relax }{figure.2.5}{}} \@writefile{toc}{\contentsline {section}{\numberline {2.5}Terminating your CloudBioLinux instance}{14}{section.2.5}} \@writefile{toc}{\contentsline {paragraph}{}{14}{section*.43}} \@writefile{lof}{\contentsline {figure}{\numberline {2.6}{\ignorespaces CloudBL desktop}}{15}{figure.2.6}} \newlabel{fig:bldesktop}{{2.6}{15}{CloudBL desktop\relax }{figure.2.6}{}} \@writefile{toc}{\contentsline {paragraph}{}{15}{section*.44}} \newlabel{section:cloudblDesktop}{{2.5}{15}{\relax }{section*.44}{}} \newlabel{SC@3}{{2.5}{16}{\relax }{section*.44}{}} \@writefile{lof}{\contentsline {figure}{\numberline {2.7}{\ignorespaces Logging out of NX}}{16}{figure.2.7}} \newlabel{fig:nxshutdown}{{2.7}{16}{\SC@OPTCAPtext \relax }{figure.2.7}{}} \@setckpt{workingOnCloudBL}{ \setcounter{page}{17} \setcounter{equation}{0} \setcounter{enumi}{11} \setcounter{enumii}{0} \setcounter{enumiii}{0} \setcounter{enumiv}{0} \setcounter{footnote}{5} \setcounter{mpfootnote}{0} \setcounter{part}{0} \setcounter{chapter}{2} \setcounter{section}{5} \setcounter{subsection}{0} \setcounter{subsubsection}{0} \setcounter{paragraph}{0} \setcounter{subparagraph}{0} \setcounter{figure}{7} \setcounter{table}{0} \setcounter{SC@C}{3} \setcounter{Item}{25} \setcounter{Hfootnote}{7} \setcounter{section@level}{4} } ================================================ FILE: doc/intro/workingOnCloudBL.tex ================================================ \section{The process in a nutshell} \begin{figure}[!hd] \fbox { \begin{minipage}{13cm} %the maxwidth variable is defined in the summary page: gettingStarted_CloudBioLinux.tex \includegraphics[width=\maxwidth]{"images/nutshell"} \caption[Start an Instance]{\label{fig:nutshell}Schematic of the basic process of working on CloudBioLinux on Amazon EC2. You can log in and out of your instance as often as you like; you continue to be charged whether you are logged in or not. You can also stop and re-start instances (not shown here). When you are finished with an instance, you should terminate it. Once terminated, the system and all data on it are deleted. This is the simplest setup. There are additional easy steps you can take to store data and even whole systems, at a fraction of the price of a running image.} \end{minipage} } \end{figure} \paragraph{}The general process you will follow when working with CloudBioLinux is outlined in figure~\ref{fig:nutshell}: \begin{enumerate} \item \textbf{Start up} a CloudBioLinux instance \item \textbf{Log into} your CloudBioLinux instance \item \textbf{Log out of} the CloudBioLinux instance \item Still want to work on this instance? You can log into it and out of it as often as you like, or you can stop and start the instance, which can work out slightly cheaper. \item When you're really finished, and don't need the CloudBioLinux instance anymore, \textbf{terminate} the instance. You will stop being charged for this instance when it is terminated.\footnote{\href{http://support.rightscale.com/06-FAQs/FAQ\_0149\_-_What\%27s_the_difference_between_Terminating_and_Stopping_an_EC2_Instance\%3F}{Stopping and terminating are different.}} \end{enumerate} \paragraph{}This chapter focusses on starting and logging into a full graphical Bio-Linux desktop on the cloud. Of course, there are other things that you may wish to do, like save your system and data for use again later, or share it with others. These things and more are covered in the {http://aws.amazon.com/ebs/}{official documentation for Elastic Block Storage}. %It would be good to write a basic intro, as the page linked here is hardly an easy newbie read. However, this is %lower priority than getting the rest of this guide done. \paragraph{A note about charging:} \paragraph{}The charging structure for Amazon EC2 is well defined and quite detailed. It is important to understand what you are being charged for, so you can make good decisions about when using the cloud is a cost effective option, and when it is not. You will be charged for running instances, and also for things like bandwidth when transferring data on and off Amazon systems, and data volumes you wish to use later. Please read the \href{http://aws.amazon.com/ec2/pricing/}{Amazon pricing documentation} so you don't get surprised when you next see your credit card bill. \paragraph{A couple of things to note when starting out:} \begin{itemize} \item \textbf{You will be charged for the time your instance is running.} It's not about when you're logged into it that counts. Charging for the instance terminates when you terminate the instance. You can just transfer your files off the system onto your local machine - but be aware that you may be charged for the bandwidth you use. Alternatively, you could consider using \href{http://aws.amazon.com/ebs/}{Elastic Block Storage}. \item \textbf{You are charged by the time-hour.} This means that if you start up an instance at 1:55pm and use it until 2:05pm, you are charged for two hours - because your instance was running in two different hours of the clock. \end{itemize} \section{Starting up a CloudBioLinux instance} \paragraph{This document focusses on using the \href{http://console.aws.amazon.com/ec2/home}{AWS Management Console}, a web-interface, for managing Amazon Web Services resources.} \begin{figure}[!hd] \fbox { \begin{minipage}{13cm} %the maxwidth variable is defined in the summary page: gettingStarted_CloudBioLinux.tex \includegraphics[width=\maxwidth]{"images/requestInstance"} \caption[Start an Instance]{\label{fig:requestInstance}Search for the term "biolinux" in the Community instances. You are likely to find a number of different images available. Those listed here were available on July 20, 2010. We recommend you pick the one with the system you want (e.g. 32 bit or 64 bit) that has \textbf{the most recent date in the text within the Manifest column}.} \end{minipage} } \end{figure} %\begin{figure}[!hd] %%the maxwidth variable is defined in the summary page: gettingStarted_CloudBioLinux.tex %\includegraphics[width=\maxwidth]{"images/openssh"} %\caption[Start an Instance]{\label{fig:openssh}XXX text here} %\end{figure} \begin{enumerate} \item Go to the \href{http://console.aws.amazon.com/ec2/home}{EC2 Management Console URL: http://console.aws.amazon.com/ec2/home} \item You should see a button saying \textbf{Launch instance}. Click on this. \item You are presented with a window called \textbf{Request Instances Wizard}. \item To start up CloudBioLinux, go to the \textbf{Community AMIs tab} and search All Images for the term \textbf{biolinux}. This will bring up a list of available CloudBioLinux images. See figure~\ref{fig:requestInstance}. \item Click on the image you wish to run to highlight it. Then click on the Select button on the right hand side. \item Now click on the \textbf{Launch Instances} in the next window presented to you. \item Click on the Continue Button at the bottom of the next window. \item Leave the Advanced options on the next page as they are. \emph{Note that this would be the time to alter these settings if you wanted to do so; you cannot change them in a running instance.} %The above item needs editing once the NX version is available, as you will likely have to provide %user data to get that to work. \item In the next window, you'll need to provide the name of your Key Pair. \emph{If you created a key pair earlier, but are not offered the option of using it, and if you created your keys in the same session you are currently logged into, try logging out of Amazon and logging back in again.} \item Once you have provided a key pair name, the next window will ask about your preferred security settings. This is analogous to setting up a machine firewall. \textbf{At a minimum you will need to enable ssh access;} ssh is the protocol you need to use to connect to your instance, whether you do so via the command line or via a graphical NX connection. If you want to access web pages provided by your instance, then you also need to open a port for http. You will want to do this if, say, you wish to refer to the Bio-Linux documentation pages on your instance. If you will be running MySQL or postgreSQL for example, you'll need to enable access to these also. \item Once you've done all this, you should be able to review the information you've provided, and if you're happy click on the \textbf{Launch instance} button. \end{enumerate} \paragraph{}If you go back to your \href{http://console.aws.amazon.com/ec2/home}{Amazon EC2 home area} and click on the Instances link in the left hand pane, you should see your CloudBioLinux instance starting up. When you see a green icon with the word running beside it, your instance is ready to log into. \section{Connecting to and logging into your CloudBioLinux instance} \subsection{Graphical, or command line?} \paragraph{For those wanting to work in a graphical computing environment}, as opposed to working from the command line, we recommend that you set up an NX connection. This provides you with a full graphical CloudBioLinux desktop. For a given instance that you have launched, you must go through the steps in the following two sections once. After that, you will be able to connect to a graphical desktop session for your launched instance as often as you like. \paragraph{For those comfortable in text-only environments, including Linux users who wish to run graphical programs, without a full desktop,} you need only follow the instructions in the next section one time. Then using the instructions in the following section, you can log into a terminal using ssh as often as you like. \paragraph{}For Windows users who wish to have access to graphical programs, it is easiest to run an NX connection. \subsection{Find out the address of your instance} \paragraph{}You need to know the address that's been assigned to your image, so that you can tell ssh or NX which machine you are trying to connect to. \begin{figure}[!hd] \fbox { \begin{minipage}{13cm} \includegraphics[width=\maxwidth]{"images/instancesOptions"} \caption[Start an Instance]{\label{fig:instancesOptions}Click the Instance Actions button (pink circle) to bring up a menu with options including connecting to an instance you have already started up. This menu is also used to terminate a running instance.} \end{minipage} } \end{figure} \begin{itemize} \item Assuming you have already clicked on the Instances link on the left side of your \href{https://console.aws.amazon.com/ec2/home}{EC2 Dashboard}, click on the \textbf{Instance Actions} button near the top of the Instances page. See figure~\ref{fig:instancesOptions}. \item Choose \textbf{Connect}. A window will open containing directions about how to connect to your CloudBioLinux instance using ssh. \emph{You need to make a couple of changes to the suggested connection instructions}, described below. \end{itemize} \subsection{Logging into a terminal using ssh} \paragraph{}After you clicked on the Connect option to connect to your instance, you should have seen a small window pop up. The instructions in that window should have text similar to the following in it: \paragraph{ssh -i mykey.pem root@ec2-184-72-144-209.compute-1.amazonaws.com} \paragraph{}The text after the @ symbol is the address of your running instance. If you are working on Linux, or you have an ssh program on Windows with a terminal, the information in the window is \emph{similar to} the command you could use to connect to your instance \footnote{If you are logging in using Putty on Windows, you will need to \href{http://www.ualberta.ca/CNS/RESEARCH/LinuxClusters/pka-putty.html}{enter the relevant information into the Putty system} in order to connect.}. CloudBioLinux is based on Ubuntu. \textbf{To log into the instance, you need to use the \emph{ubuntu} user}, not the root user\footnote{The default for most systems on Amazon EC2 is to log in as the root user.}. So, an example command you might run in a terminal to connect to your instance is: \paragraph{ssh -i mykey.pem ubuntu@ec2-184-72-144-209.compute-1.amazonaws.com} \paragraph{}where you have used the ubuntu username (instead of root), and you include your machine address after the @ symbol. \paragraph{Note:} If you get a warning when you try to connect that suggests that your key cannot be found, it may mean that you have saved your key to a non-standard location and/or given it a non-standard name. In this case add \href{http://nebc.nerc.ac.uk/tools/bio-linux/bio-linux-faq\#path}{path} information for your key to the command line so that the private key can be found from where you run the ssh connection command. For example, if your key is stored in a subdirectory of your home directory called \emph{keys}, and you want to log in as the \emph{ubuntu} user, you could log in using ssh and the command, you need to \paragraph{ssh -i /home/mydirectory/keys/mykey.pem ubuntu@ec2-184-72-144-209.compute-1.amazonaws.com} \paragraph{}The first time you connect to your running CloudBioLinux instance, you should be offered the opportunity to set up NX on your instance. At this point, you can also provide a username other than "ubuntu", and this new user will be created for you. \paragraph{}You can continue at this point to run programs on the command line, or if you are working on a Linux system, you can launch graphical applications using the command line. However, we generally recommend connecting to your CloudBioLinux instance using an NX client instead of a text-based ssh client. This is because many people find a graphical desktop environment easier to work on, and the menus and desktop links help people take full advantage of the facilities of the CloudBioLinux system. To log in via NX, just follow the instructions in the next section. You may wish to log out of your current ssh session (although this is not necessary). \label{section:nx} \subsection{Logging into graphical desktop using NX} \paragraph{}Start up your NX client software\footnote{Images shown here refer to the Nomachine NX client for Linux, running on a standard NEBC Bio-Linux machine, but the process should be similar no matter what type of system you are working on (Linux, Windows, Mac).} \begin{figure}[!hd] \fbox { \begin{minipage}{13cm} \includegraphics[width=\maxwidth]{"images/NX-menuOnUbuntu"} \caption[NX Client start]{\label{fig:NX-menuOnUbuntu}If an NX client is already installed, you should be able launch it from under your Applications menu on Ubuntu-based machines, or from your programs listing on Windows machines.} \end{minipage} } \end{figure} \paragraph{}If you are using a Nomachine NX client, you should now see an NX connection wizard. Here, you need to enter the address of your launched instance (the same address you used to log into the terminal earlier), and you should change the desktop type to Gnome. See figure \ref{fig:NXConnectionWizard}. \begin{figure}[!hd] \fbox { \begin{minipage}{13cm} \includegraphics[width=\maxwidth]{"images/NXConnectionWizardandGnome"} \caption[NX Client start]{\label{fig:NXConnectionWizard}Enter the same machine address you used for your terminal login earlier. Remember to use the same username you provided earlier - this may be the default username, "ubuntu", or it may be something else that you chose when you set the NX password. Please choose the gnome desktop type.} \end{minipage} } \end{figure} \paragraph{}When you've logged in, you should see a desktop similar to that in figure \ref{fig:bldesktop}. \subsection{The logic of the NX setup} \paragraph{}It may initially seem strange that you need to log into a command terminal before you can log into your EC2 instance using an NX client. The reason that this needs to be done is that key auth support, which is used for the ssh connections into the EC2 instances, is not supported by NX. Thus, for any given instance that is started, \emph{you need a password to be able to log in using NX}. \paragraph{}When you set a password during your terminal session, you are ensuring that only you have that password. You also have the opportunity to create another user on the system, just by providing a username other than "ubuntu" when prompted. \section{Logging out of your CloudBioLinux instance} \paragraph{From an NX connection} you need to go to the System menu and choose the option \textbf{Shut Down...}. See figure~\ref{fig:nxshutdown}. \paragraph{From an ssh command line (or Putty) connection} you need to type \textbf{exit} at the command prompt. \section{Terminating your CloudBioLinux instance} \paragraph{}Highlight the instance you wish to terminate in the list on your Instances page. Click on the \textbf{Instance Actions} button (see figure~\ref{fig:instancesOptions}) and choose \textbf{Terminate} under the Instance Action section of the menu. In basic terms, terminating results in the system and all the files and data on it being deleted \footnote{You will still be charged a fee if you have only stopped your instance, as opposed to terminating it, and \href{http://docs.amazonwebservices.com/AWSEC2/latest/UserGuide/index.html?Concepts\_BootFromEBS.html\#Stop\_Start} {your data may still be deleted depending on how you have set things up}. \href{http://support.rightscale.com/06-FAQs/FAQ\_0149\_-_What\%27s\_the\_difference\_between\_Terminating\_and\_Stopping\_an\_EC2\_Instance\%3F}{Stopping and terminating are different.}}. If you have work you wish to save before terminating, or if you wish to keep a copy of this image so that you can use it later, without paying as much as you would for a running instance, please check out the Amazon documentation on EBS Volumes and taking snapshots of instances. \begin{figure}[!hd] \fbox { \begin{minipage}{13cm} \includegraphics[width=\maxwidth]{"images/initalNXdesktop"} \caption[CloudBL desktop]{\label{fig:bldesktop}The CloudBioLinux desktop. It includes a number of menus, as well as icons for folders containing sample data relevant for the bioinformatics software installed, and a link to this document.} \end{minipage} } \end{figure} \paragraph{}We provide a short introduction to the Cloud Bio-Linux desktop in Appendix A.2 on page \label{section:cloudblDesktop}. \begin{SCfigure}[][t] \includegraphics[width=40mm]{"images/nxshutdown"} \caption[Logging out of NX]{\label{fig:nxshutdown}Choosing the \textbf{Shut Down...} option in an NX session logs you out. Logging out is not the same as terminating your CloudBioLinux instance. You will still be charged while the instance is running - whether you are logged into it or not.} \end{SCfigure} ================================================ FILE: doc/intro/workingWithData.aux ================================================ \relax \@writefile{toc}{\contentsline {section}{\numberline {3.1}Introduction}{18}{section.3.1}} \newlabel{section:data}{{3.1}{18}{Introduction\relax }{section.3.1}{}} \@writefile{toc}{\contentsline {paragraph}{}{18}{section*.45}} \@writefile{toc}{\contentsline {paragraph}{}{18}{section*.46}} \@writefile{toc}{\contentsline {section}{\numberline {3.2}Copying data onto your CloudBioLinux instance}{18}{section.3.2}} \newlabel{section:directcopy}{{3.2}{18}{Copying data onto your CloudBioLinux instance\relax }{section.3.2}{}} \@writefile{toc}{\contentsline {paragraph}{}{18}{section*.47}} \@writefile{toc}{\contentsline {paragraph}{}{18}{section*.48}} \@writefile{toc}{\contentsline {paragraph}{}{19}{section*.49}} \@writefile{lof}{\contentsline {figure}{\numberline {3.1}{\ignorespaces Drag and drop from remote machine}}{19}{figure.3.1}} \newlabel{fig:graphicalftp}{{3.1}{19}{Drag and drop from remote machine\relax }{figure.3.1}{}} \@writefile{toc}{\contentsline {paragraph}{}{19}{section*.50}} \@writefile{toc}{\contentsline {paragraph}{}{19}{section*.51}} \@writefile{toc}{\contentsline {section}{\numberline {3.3}Using EBS volumes for data}{20}{section.3.3}} \newlabel{section:ebscopy}{{3.3}{20}{Using EBS volumes for data\relax }{section.3.3}{}} \@writefile{toc}{\contentsline {paragraph}{}{20}{section*.52}} \@writefile{toc}{\contentsline {paragraph}{}{20}{section*.53}} \@writefile{toc}{\contentsline {paragraph}{A note on charging:}{20}{section*.54}} \@writefile{toc}{\contentsline {paragraph}{Creating your volume}{20}{figure.3.2}} \@writefile{toc}{\contentsline {paragraph}{}{20}{section*.56}} \@writefile{toc}{\contentsline {paragraph}{}{20}{section*.57}} \@writefile{lof}{\contentsline {figure}{\numberline {3.2}{\ignorespaces Attaching a volume}}{21}{figure.3.2}} \newlabel{fig:createandmountvolume}{{3.2}{21}{Attaching a volume\relax }{figure.3.2}{}} \@writefile{toc}{\contentsline {paragraph}{}{21}{section*.58}} \@writefile{lof}{\contentsline {figure}{\numberline {3.3}{\ignorespaces Create volume in console}}{22}{figure.3.3}} \newlabel{fig:createvolume}{{3.3}{22}{Create volume in console\relax }{figure.3.3}{}} \@writefile{lof}{\contentsline {figure}{\numberline {3.4}{\ignorespaces Attaching a volume}}{22}{figure.3.4}} \newlabel{fig:attachvolume1}{{3.4}{22}{Attaching a volume\relax }{figure.3.4}{}} \@writefile{toc}{\contentsline {paragraph}{}{22}{section*.59}} \@writefile{toc}{\contentsline {paragraph}{Accessing your volume}{22}{section*.60}} \newlabel{text:mounting}{{3.3}{22}{\relax }{section*.61}{}} \@writefile{toc}{\contentsline {paragraph}{}{22}{section*.61}} \@writefile{lof}{\contentsline {figure}{\numberline {3.5}{\ignorespaces Attaching a volume}}{23}{figure.3.5}} \newlabel{fig:attachvolume2}{{3.5}{23}{Attaching a volume\relax }{figure.3.5}{}} \@writefile{toc}{\contentsline {paragraph}{}{23}{section*.62}} \@writefile{lof}{\contentsline {figure}{\numberline {3.6}{\ignorespaces Attaching a volume}}{24}{figure.3.6}} \newlabel{fig:unmountdetach}{{3.6}{24}{Attaching a volume\relax }{figure.3.6}{}} \@writefile{toc}{\contentsline {paragraph}{Unmounting your volume}{24}{figure.3.6}} \@writefile{toc}{\contentsline {paragraph}{This is a simple but vital step to avoid the possibility of data corruption.}{24}{section*.64}} \@writefile{toc}{\contentsline {paragraph}{}{24}{section*.65}} \newlabel{text:unmounting}{{3.3}{24}{\relax }{section*.65}{}} \@writefile{toc}{\contentsline {paragraph}{Detaching your volume}{24}{section*.66}} \@writefile{toc}{\contentsline {paragraph}{}{24}{section*.67}} \@writefile{toc}{\contentsline {paragraph}{Backing up or sharing your volume}{25}{section*.68}} \@writefile{toc}{\contentsline {paragraph}{}{25}{section*.69}} \@writefile{toc}{\contentsline {paragraph}{Deleting your volume}{25}{section*.70}} \@writefile{toc}{\contentsline {paragraph}{}{25}{section*.71}} \@writefile{toc}{\contentsline {section}{\numberline {3.4}Accesing public datasets on Amazon}{25}{section.3.4}} \newlabel{section:publiccopy}{{3.4}{25}{Accesing public datasets on Amazon\relax }{section.3.4}{}} \@writefile{lof}{\contentsline {figure}{\numberline {3.7}{\ignorespaces List public data}}{25}{figure.3.7}} \newlabel{fig:ensembl}{{3.7}{25}{List public data\relax }{figure.3.7}{}} \@writefile{toc}{\contentsline {paragraph}{}{25}{section*.72}} \@writefile{toc}{\contentsline {paragraph}{}{25}{section*.73}} \@writefile{toc}{\contentsline {paragraph}{}{25}{section*.74}} \@setckpt{workingWithData}{ \setcounter{page}{26} \setcounter{equation}{0} \setcounter{enumi}{4} \setcounter{enumii}{0} \setcounter{enumiii}{0} \setcounter{enumiv}{0} \setcounter{footnote}{2} \setcounter{mpfootnote}{0} \setcounter{part}{0} \setcounter{chapter}{3} \setcounter{section}{4} \setcounter{subsection}{0} \setcounter{subsubsection}{0} \setcounter{paragraph}{0} \setcounter{subparagraph}{0} \setcounter{figure}{7} \setcounter{table}{0} \setcounter{SC@C}{3} \setcounter{Item}{32} \setcounter{Hfootnote}{9} \setcounter{section@level}{4} } ================================================ FILE: doc/intro/workingWithData.tex ================================================ \section{Introduction}\label{section:data} \paragraph{}For many bioinformatics tasks, you will want to work on your own data and files – for example, perhaps your own sequence data and blast databases. To do this, you will need to \emph{upload your files onto a machine that your CloudBioLinux instance can access}. Three options are covered in this chapter: \begin{enumerate} \item Copy your data directly onto the CloudBioLinux instance you are running. See section \ref{section:directcopy} This would be alright if you were going to use this data only on this running instance and you're happy for it to be deleted when you terminate the instance. \item Copy your date onto a separate EBS volume. This would be useful if you wish to store your files for use in other sessions, but you do not plan to keep the same running instance. (EBS Volumes are cheaper than running instances.) See section \ref{section:ebscopy} \item If the data you want to use is already available on Amazon EBS volumes (for example, ENSEMBL data), you can access this easily, with no data transfer costs. \ref{section:publiccopy}. \end{enumerate} \paragraph{}\href{http://docs.amazonwebservices.com/AWSEC2/latest/UserGuide/creating-snapshot-ebs.html}{Snapshots} are not discussed in this document, but are an important topic to understand if you plan to use the Amazon Cloud, especially if you will be working with your own data over a prolonged period or making customisations you do not wish to lose. \section{Copying data onto your CloudBioLinux instance} \label{section:directcopy} \paragraph{}If you only need your data for a single CloudBioLinux instance, then you can just copy your data onto that instance directly. Once you are logged into your CloudBioLinux instance, there are a number of ways to do this. For example, there are command line tools like \textbf{scp}, for copying files from a machine you have an account on, or \textbf{wget} to bring in data from public websites or ftp sites. \paragraph{}Alternatively, if you are logged into the full graphical desktop using NX (information on page \pageref{section:nx}), you can use the file browser to connect to a remote site and \textbf{drag and drop} your files to your running Bio-Linux instance. This is the method we focus on here. \begin{itemize} \item Go to the \textbf{Places} menu in the top taskbar and open up a file browser, for example by clicking on your Home Folder. \item Now go to the \textbf{Go} menu and click on Location... (or just type Ctrl-L). \item If you are going to copy files from a machine that you have login permissions on, then in the box next to the word \emph{Location} that appears in your file browser, type: \textbf{ssh://your.machine.com}, replacing your.machine.com with the address of the machine your files are on. Alternatively, if you wanted to copy files from a public ftp server, say, then you would enter something like the following in the Location box: \\\textbf{ftp://ftp.someother.database.site} \end{itemize} \paragraph{}As a specific example, if I want to copy fasta files from the EMBL database sections, I would type the following into the Location box: \\\textbf{ftp://ftp.ebi.ac.uk/pub/databases/fastafiles/emblrelease} \begin{figure}[!hd] \fbox { \begin{minipage}{13cm} \includegraphics[width=\maxwidth]{"images/graphicalFTP_full"} \caption[Drag and drop from remote machine]{\label{fig:graphicalftp}Copying files from remote machine is easy using the graphical File Browser, which can be launched from under the \textbf{Places} menu in the top taskbar. Choosing the Locations option under the Go menu of the file browser will allow you to type in a protocol (e.g. ftp, ssh) and a location. Here, a section of EMBL from the EBI is copied to my system using drag and drop between the two file browser windows.} \end{minipage} } \end{figure} \paragraph{}Now open another file browser by going to the \textbf{Places} menu. Navigate to the folder you wish to store the files in. Now you can just drag and drop your files from the remote machine onto your CloudBioLinux instance. See figure \ref{fig:graphicalftp} on page \pageref{fig:graphicalftp}. \paragraph{}This process is simple, and for one-off jobs, is perfectly adequate. Note that you will generally pay for the network traffic you generate in transferring the data\footnote{Until November 1, 2010, data transfer onto Amazon is free. The first Gb per month of transfer off is also free. (Information taken from the \href{http://aws.amazon.com/ec2/pricing/}{Amazon website} on July 21, 2010, with no guarantees to be correct at the time you are reading this document. Please check the \href{http://aws.amazon.com/ec2/}{official pricing list}.} So if you are going to use the same dataset numerous times, it is worth considering setting up an EBS volume rather than transferring data onto new instances. Even if this transfer is free, it will still generally take more time than mounting an EBS volume that already has your data on it. \section{Using EBS volumes for data} \label{section:ebscopy} \paragraph{}An Amazon EBS volume is what you need if \begin{itemize} \item you are going to use a dataset a number of times, with gaps in time between uses, or \item you want to store your data such that you can connect to it from different CloudBioLinux (or other Amazon EC2) images, or \item if you wish to share your data with other people also working on Amazon EC2 systems. \end{itemize} \paragraph{}This guide presents only a small part of what is possible with EBS volumes. Please check out the \href{http://docs.amazonwebservices.com/AWSEC2/latest/UserGuide/ebs-api-overview.html}{EBS volume documentation on the Amazon website} for further information. \paragraph{A note on charging:}You will be \href{http://aws.amazon.com/ec2/}{charged for your Amazon EBS volume} as long as it is in existence, and you will be \emph{charged for the space you request}, not the space you are really using. So if you ask for 1Gb, you are paying for 1Gb, even if you only use 100Kb. \paragraph{Creating your volume} \begin{figure}[!ht] % \centering \fbox { \begin{minipage}{13cm} \includegraphics[width=\maxwidth]{"images/createAndMountVol-1"} \caption[Attaching a volume]{\label{fig:createandmountvolume}To create a new volume and get access to it from your running instance involves 4 steps. Here we suggest that the first two are done using the AWS Console. The second two steps are done within your running instance. If you want to work with a volume you (or someone else) has created earlier, then you need only carry out the steps in the grey boxes. The commands for carrying out the two steps in your running instance are provided in the text on page \pageref{text:mounting}} \end{minipage} } \end{figure} \paragraph{}The instructions in this section assume you have already started up and have logged into an EC2 instance. \paragraph{}The steps involved in creating and mounting a volume are illustrated in figure~\ref{fig:createandmountvolume}. \begin{figure}[!hd] \fbox { \begin{minipage}{13cm} \includegraphics[width=\maxwidth]{"images/createVolume-1"} \caption[Create volume in console]{\label{fig:createvolume}The volumes pane of the AWS Console is brought up by clicking on the Volumes link in the Navigation pane (Green oval). Red circle: Just click the Create Volume button - it does what it says. Purple circle: the status of your volume creation is reported to you. When the circle turns blue and the word says \emph{available}, you can proceed to attach and mount your volume.} \end{minipage} } \end{figure} \begin{itemize} \item In the Navigation pane (left side) of the AWS Management Console, go to the Elastic Block Store area and choose \textbf{Volumes}. See figure~\ref{fig:createvolume}. \item Click on the \textbf{Create Volume} button. \item Choose the same availability zone as the images you plan to use this volume with. \item After changing any other settings in this window, press the \textbf{Create} button. \item Wait until the yellow circle beside the word \emph{creating} is replaced by a blue circle beside the word \emph{available}. \end{itemize} \paragraph{}You now have an EBS volume attached to your running instance. This is analagous to plugging in a new piece of hardware to a server. This means that the volume \textbf{is not yet ready for copying data onto}. You first must mount the volume on your instance - this makes the volume accessible to you when you are logged into your instance \footnote{For those used to user-friendly plug 'n play systems, where you just plug a pre-formatted hard drive into a machine and copy to it directly, the steps outlined here might come as a bit of long-winded shock. In fact, all you are doing is carrying out the steps that, between your hardware vendor and your computer operating system, are often done for you. In particular, the equivalent to a disk drive here is the EBS volume, which is initally unformatted and needs to be formatted. To do this, you attach it to your running instance, then you format it. Now, to work with the filesystem on your volume, you need to mount in onto your image file system, so the whole thing - image and volume, can be interacted with as one. Until you unmount the volume of course.}. \begin{figure}[!hd] \fbox { \begin{minipage}{13cm} \includegraphics[width=\maxwidth]{"images/attachVolume-1"} \caption[Attaching a volume]{\label{fig:attachvolume1}Click on the Attach Volume button (Orange circle) to Attach a volume to an instance.} \end{minipage} } \end{figure} \begin{figure}[!hd] \fbox { \begin{minipage}{13cm} \includegraphics[width=\maxwidth]{"images/attachVolume-2"} \caption[Attaching a volume]{\label{fig:attachvolume2}Fill in the information requested. Note that both the instance and the EBS volume are in the same region. You just need to provide an unused device to attach your volume to. If you don't know what this means, and are starting up a CloudBioLinux instance, then just use any of the suggested locations: /dev/sdf, /dev/sdg, /dev/sdh....up to /dev/sdp for now and check out \href{http://it.toolbox.com/wiki/index.php/Mount_point}{some documentation to find out what it means later}.} \end{minipage} } \end{figure} \begin{itemize} \item Click on the \textbf{Attach Volume} button on the Volumes page. See figure~\ref{fig:attachvolume1}. \item To attach a volume using the AWS Console interface, you just need to fill in the requested information. See figure~\ref{fig:attachvolume2}. \end{itemize} \paragraph{}Note that selecting any of your volumes in the AWS Management Console will bring up details of that volume at the bottom of the page. \paragraph{Accessing your volume} \paragraph{}\label{text:mounting}This is where it gets a bit ugly, as you need to log into your machine and use the command line for the next couple of steps. The first of these steps, \textbf{formatting your disk}, only needs to be done \emph{the first time} you use a particular volume. The second step, \textbf{mounting the volume}, needs to be done \emph{each time} you want to access data on your volume from a new instance. There are ways to automate mounting a volume onto new instances, but these are not covered here. \begin{enumerate} \item Log into your instance. If you are logged in using NX, start up a terminal window. \item \textbf{The first time you mount a volume for use:} Type the following command to create an ext3 filesystem on your volume. Here I assume you have mounted it to /dev/sdf. See figure \ref{fig:attachvolume2} on page \pageref{fig:attachvolume2}. \\\textbf{sudo mkfs -t ext3 /dev/sdf } \item Now make an empty directory. This will act as a place where you can mount your new filesystem. By convention, mount points tend to be put in directories such as /mnt or /media. For example, the command below creates a directory called /mnt/datasets: \\\textbf{sudo mkdir /mnt/datasets} \item Now you can mount your volume onto this: \\\textbf{sudo mount /dev/sdf /mnt/datasets} \end{enumerate} \paragraph{}You will now be able to put data under the folder /mnt/datasets. All files under that directory are on your EBS volume and will not be lost when your instance terminates. Ensure you read the section on page \pageref{text:unmounting} on \textbf{unmounting your volume} as failure to unmount before detaching your volume (or terminating your instance) could lead to data corruption. \paragraph{Unmounting your volume} \begin{figure}[!hd] \fbox { \begin{minipage}{13cm} \includegraphics[width=\maxwidth]{"images/unmountDetach-1"} \caption[Attaching a volume]{\label{fig:unmountdetach}You must unmount a volume before detaching it. Not doing so may result in data corruption on the volume. You can then either detach the volume from your instance by using the \textbf{Detach Volume} button on the AWS Console, or the volume will be detached for you when you terminate your instance. The commands for unmounting a volume from within your running instance are provided in the text on page \pageref{text:unmounting}} \end{minipage} } \end{figure} \paragraph{This is a simple but vital step to avoid the possibility of data corruption.} Do this \emph{before you detach} your volume or terminate your instance. \paragraph{}If you had attached your device to /dev/sdf, then you simply need to type: \\\textbf{umount -d /dev/sdf} \label{text:unmounting} \paragraph{Detaching your volume} \paragraph{}You can detach your volume from your instance using the AWS Management Console using the Detach Volume button on the console Navigation pane. Alternatively, your volumes will be detached automatically when you terminate your instance. \paragraph{Backing up or sharing your volume} \paragraph{}Check out the Userguide information on \href{http://docs.amazonwebservices.com/AWSEC2/latest/UserGuide/creating-snapshot-ebs.html}{creating snapshots} and on \href{http://docs.amazonwebservices.com/AWSEC2/latest/UserGuide/modifying-snapshot-permissions-ebs.html}{modifying permisisons on snapshots}. \paragraph{Deleting your volume} \paragraph{}You can delete your volume using the Delete button on the console Navigation pane. \section{Accesing public datasets on Amazon} \label{section:publiccopy} \begin{figure}[!hd] \fbox { \begin{minipage}{13cm} \includegraphics[width=\maxwidth]{"images/public-ensembl"} \caption[List public data]{\label{fig:ensembl}Click the Snapshot link in the Navigation pane. Then select Public Snapshots in the Viewing menu. Searching for a term such as \emph{ensembl} brings up all the public snapshots that contain ensembl in their description. } \end{minipage} } \end{figure} \paragraph{}Amazon makes some \href{http://aws.amazon.com/publicdatasets/}{public data sets available as snapshots}. You can just attach and mount these (see grey boxes in figure~\ref{fig:createandmountvolume}. No data transfer is necessary. Finding datasets is easy: just search through the public snapshots for relevant terms. See figure~\ref{fig:ensembl}. Check out the \href{http://developer.amazonwebservices.com/connect/kbcategory.jspa?categoryID=243}{full public data set listing}. \paragraph{}Amazon provides \href{http://docs.amazonwebservices.com/AWSEC2/latest/UserGuide/using-public-data-sets.html}{documentation on how to make use of these public data resources}. \paragraph{} ================================================ FILE: doc/linux_kvm.md ================================================ # CloudBioLinux and Linux KVM This document gives some additional information on using BioLinux on Linux KVM. KVM is a great virtualization environment, it is part of the Linux effort, will work with the default Linux kernel. A running 64-bit Linux kernel can run both 32-bit and 64-bit VMs. With most Linux distributions KVM comes out of the box... Together with the BioLinux fabric environment, any KMV VM can be bootstrapped. Here we start from a bare Debian/Ubuntu installation. With Debian you can install from a fresh download of [stable](http://www.debian.org/releases/stable/) version burning it on a CDROM. The default install will do on, say, a 10GB partition - so the rest can be used for LVM partitioning. Select ssh and the standard system utilities. # Install KVM There are many web resources for installing KVM. On Debian derived systems: apt-get install kvm libvirt-bin virtinst bridge-utils \ qemu-kvm virt-manager libvirt-bin and add your user to the kvm group. E.g. adduser username kvm For non-Debian systems see, for example, [OpenSuse](http://doc.opensuse.org/documentation/html/openSUSE/opensuse-kvm/cha.kvm.requires.html). Note that not all machines support virtualization, and if they do you may need to switch it on in the BIOS (especially true on older hardware). Check for CPU support with: egrep --color '(vmx|svm)' /proc/cpuinfo Also note that some older Linux installations may need a kernel upgrade. Start KVM with /etc/init.d/qemu-kvm start or with later versions /etc/init.d/qemu-system-x86 start # Create a bare VM Download a live installation image file. For example fetch a standard or network image from [[http://www.debian.org/][Debian]]. Reserve space on the disk partition - this should be enough for a Debian install and updates. qemu-img create hda.img -opreallocation=metadata -ocluster_size=2M -f qcow2 10G (settings suggested by Red Hat) and fire up the VM kvm debian-live-$(VER).img -hda hda.img -curses -no-reboot -serial pty Alternatively use the netinstall. The CloudBioLinux integration test system does something similar, starting from the smaller net install of Debian Linux: qemu-system-x86_64 -enable-kvm -cdrom debian-$(VER)-amd64-netinst.iso -hda hda.img hit ESC and optionally type 'install fb=false' to disable the frame buffer. Fire up the installer. Note that the file system of the installer can be slow, that speed is not representative for an installed VM later (with -enable-kvm). With the base install, boot the new system qemu-system-x86_64 -enable-kvm -redir tcp:2222::22 -hda hda.img and install ssh on the VM (it comes already on netinst) apt-get install openssh-server so that ssh login works ssh -p 2222 biolinux@localhost on user biolinux without a password (preferably using a key with empty password). And give that user 'sudo bash'. This ssh and sudo configuration is described in ./doc/private_cloud.md. After generating the key ssh -i ~/.ssh/biolinux -p 2222 biolinux@localhost test run sudo without a password sudo bash Not much to installing Linux with KVM! From this point onwards you can install CloudBioLinux using the fabric file. Make a copy of the hda.img file, so you can have the same starting point every time cp hda.img kvm_with_biolinux_login.img The CloudBioLinux test script also starts from here. Try the ./test/test_biolinux script to test drive the VM. test_biolinux will install a CloudBioLinux flavor, and check whether the installation is complete. Essentially with a running instance: ./test/test_biolinux -p 2222 -u biolinux -i ~/.ssh/biolinux 127.0.0.1 (note the use of 127.0.0.1 over localhost - this is because of a bug in fabric - is this still true?). # KVM tips KVM is nice and powerful. It is used in many Cloud service providers. For fast performance, it pays to install on a raw (LVM) partition, get bridging sorted, and make sure hardware acceleration is in place. Interesting goodies are the monitor (Crtl-Alt-2), virtsh, etc. See also [http://www.linux-kvm.org/page/FAQ][kvm tips]. [kvm tips]: http://www.linux-kvm.org/page/FAQ ================================================ FILE: doc/private_cloud.md ================================================ # Private Cloud and CloudBioLinux CloudBioLinux can be used to create a private Cloud for Bioinformatics. Essentially, all you need is ssh access to a VM running somewhere. This VM should be a clean install of Linux. With CloudBioLinux Debian and Ubuntu distributions are supported best. ## Start a VM Start a VM and make sure there is a network defined, and ssh running. On the VM dhclient -v ifconfig ps xa|grep ssh ssh localhost Check the network (e.g. with Debian) apt-get update apt-get install vim You should be able to use the IP address to login from your desktop ssh biolinux@VM_IP_address ## Get password free ssh access The CloudBioLinux fabric tools work best when you have password free login. If you can login to the remote with ssh biolinux@VM_IP_address you are set. Otherwise, create a password free ssh key. To achieve this, see the many Internet resources, e.g. http://www.mtu.net/~engstrom/ssh-agent.php. Combine username, key, hostname and port in ./ssh/config as Host biolinux hostname localhost user biolinux port 2222 IdentityFile ~/.ssh/biolinux so you can login with ssh biolinux If this works it is possible cloudbiolinux comes with a fab error saying Fatal error: Low level socket error connecting to host localhost on port 2222 One possibility is that it tries IPv6 to connect to localhost. You may have to comment out the line '::1 localhost ' in /etc/hosts to run fab (it has bitten me several times). ## Install sudo without password Install the sudo program. Next, edit /etc/sudoers with the 'visudo' command, and add the line biolinux ALL=NOPASSWD: /bin/bash where biolinux is your VM user login name. Alternatively add biolinux to the sudo group. Now try: sudo bash and you should be root, without a password. ## Install CloudBioLinux See the README for installing CloudBioLinux and fabric. ## Run fabric Now you should be set! To install BioLinux fab -f $source/fabfile.py -H biolinux@$VM_IP_address -c $fabricrc install_biolinux:packagelist=$packagelist Where source points to the checked out source tree, e.g. export source=$HOME/izip/git/opensource/debian/biolinux For example, to install the Minimal flavor on Debian stable on a VM running on IP 192.168.64.105: fab -f $source/fabfile.py -H biolinux@192.168.64.105 \ -c $source/contrib/flavor/minimal/fabricrc_debian.txt \ install_biolinux:packagelist=$source/contrib/flavor/minimal/main.yaml CloudBioLinux shows the following output. First it sets up the environment [192.168.64.105] Executing task 'install_biolinux' cloudbiolinux WARNING: Skipping fabricrc.txt as distribution is already defined cloudbiolinux DEBUG: Minimal Edition 1.0.1 cloudbiolinux INFO: This is a minimal cloudbiolinux INFO: This is a Base Flavor - no overrides cloudbiolinux INFO: Distribution debian cloudbiolinux INFO: Debian setup cloudbiolinux DEBUG: Debian-shared setup cloudbiolinux DEBUG: Source=squeeze cloudbiolinux DEBUG: Checking target distribution debian [192.168.64.105] run: cat /proc/version [192.168.64.105] out: Linux version 2.6.32-5-amd64 (Debian 2.6.32-31) (ben@decadent.org.uk) (gcc version 4.3.5 (Debian 4.3.5-4) ) #1 SMP Mon Mar 7 21:35:22 UTC [192.168.64.105] out: [192.168.64.105] out: cloudbiolinux INFO: Now, testing connection to host... cloudbiolinux INFO: Connection to host appears to work! cloudbiolinux DEBUG: Expand paths cloudbiolinux INFO: packagelist=/home/biolinux/izip/git/opensource/debian/biolinux/contrib/flavor/minimal/main.yaml cloudbiolinux INFO: Meta-package information cloudbiolinux INFO: minimal,ruby cloudbiolinux INFO: cloudbiolinux INFO: Target=None Here it modifies the source file for apt-get, as well as keys: cloudbiolinux DEBUG: _setup_apt_sources /etc/apt/sources.list.d/cloudbiolinux.list Minimal Edition [192.168.64.105] sudo: touch /etc/apt/sources.list.d/cloudbiolinux.list [192.168.64.105] sudo: echo '# This file was modified for Minimal Edition' >> /etc/apt/sources.list.d/cloudbiolinux.list cloudbiolinux DEBUG: Source deb http://ftp.nl.debian.org/debian/ squeeze main contrib non-free [192.168.64.105] sudo: echo 'deb http://ftp.nl.debian.org/debian/ squeeze main contrib non-free' >> /etc/apt/sources.list.d/cloudbiolinux.list cloudbiolinux DEBUG: Source deb http://ftp.nl.debian.org/debian/ squeeze-updates main contrib non-free [192.168.64.105] sudo: echo 'deb http://ftp.nl.debian.org/debian/ squeeze-updates main contrib non-free' >> /etc/apt/sources.list.d/cloudbiolinux.list [192.168.64.105] sudo: cloudbiolinux INFO: Update GPG keys for repositories cloudbiolinux INFO: Update and install all packages [192.168.64.105] sudo: apt-get update [192.168.64.105] out: Hit http://ftp.nl.debian.org squeeze Release.gpg and starts installing packages cloudbiolinux INFO: Updating 26 packages [192.168.64.105] sudo: apt-get -y --force-yes install ruby1.8 ruby1.8-dev ruby1.9.1 ruby1.9.1-dev axel less openssh-server rsync screen sudo tar unzip bzr cvs darcs git-core mercurial subversion vim cmake g++ gcc gfortran make patch swig [192.168.64.105] out: Reading package lists... Done [192.168.64.105] out: Building dependency tree [192.168.64.105] out: Reading state information... Done [192.168.64.105] out: gcc is already the newest version. [192.168.64.105] out: gcc set to manually installed. [192.168.64.105] out: less is already the newest version. [192.168.64.105] out: less set to manually installed. [192.168.64.105] out: make is already the newest version. (etc, etc) Finally some clean ups [192.168.64.105] sudo: apt-get clean cloudbiolinux INFO: Target=unknown; Edition=Minimal Edition; Flavor=Base Flavor - no overrides write an entry in the log file [192.168.64.105] sudo: date +"%D %T - Updated Target=unknown; Edition=Minimal Edition; Flavor=Base Flavor - no overrides" >> /var/log/biolinux.log [192.168.64.105] run: uname -m [192.168.64.105] out: x86_64 [192.168.64.105] out: cloudbiolinux INFO: Reading /home/biolinux/izip/git/opensource/debian/biolinux/config/custom.yaml cloudbiolinux DEBUG: Packages: cloudbiolinux DEBUG: cloudbiolinux INFO: Cleaning up space from package builds [192.168.64.105] sudo: rm -rf .cpanm [192.168.64.105] sudo: rm -f /var/crash/* And it is done. Minimal has no post-installation configuration, but that is easy to add. ================================================ FILE: doc/remote_gui.md ================================================ # BioLinux Remote X access BioLinux supports both VNC and freenx GUI X-windows access to a remote VM. And you can use X programs through ssh, naturally. ## VNC VNC is a ubiquitous remote access tool - always there, and easy to install/use. In a nutshell: Make sure vnc4server is installed on the VM. Enable ports 5900, 5901 and 5902 on the VM. Run the server vnc4server -depth 24 (set password) Run the client on your desktop vncviewer -FullColor=1 HostIP:1 Where HostIP is the reachable host IP address or DNS name. Next, it may be necessary to start an X desktop, such as LXDE: startlxde ### Amazon EC2 ports Create a security group for your instance that allows at least ports 22,5900,5901 and 5902. ### Vagrant ports You may need to add port forwarding to vagrant - as the testing system does. I.e. add to the Vagrantfile: config.vm.forward_port('vnc0', 5900, 5900) config.vm.forward_port('vnc1', 5901, 5901) config.vm.forward_port('vnc2', 5902, 5902) This is for testing, mostly. You do not need VNC on Vagrant/VirtualBox. Fire up the GUI directly! ### VNC Security Please note that VNC is not very secure - it has no proper key protection. You can tunnel over ssh for improved security. Or use freenx instead. ## FreeNX FreeNX is a fast version of the X protocol. Make sure freenx is installed on the VM. CloudBioLinux comes with scripts for setting up freenx. (to be filled in) ## X over ssh Normally you have ssh access to the remote VM. You can use X-windows programs remotely, provided you have a local X server (always on Linux and OSX). Just login with the -X switch ssh -X user@$hostIP in the terminal type an X program, e.g. firefox and the program should display locally (running remotely). ================================================ FILE: doc/source/conf.py ================================================ # -*- coding: utf-8 -*- # # CloudBioLinux documentation build configuration file, created by # sphinx-quickstart on Wed Jul 17 09:14:27 2013. # # This file is execfile()d with the current directory set to its containing dir. # # Note that not all possible configuration values are present in this # autogenerated file. # # All configuration values have a default; values that are commented out # serve to show the default. import sys, os # If extensions (or modules to document with autodoc) are in another directory, # add these directories to sys.path here. If the directory is relative to the # documentation root, use os.path.abspath to make it absolute, like shown here. #sys.path.insert(0, os.path.abspath('.')) # -- General configuration ----------------------------------------------------- # If your documentation needs a minimal Sphinx version, state it here. #needs_sphinx = '1.0' # Add any Sphinx extension module names here, as strings. They can be extensions # coming with Sphinx (named 'sphinx.ext.*') or your custom ones. extensions = ['sphinx.ext.autodoc', 'sphinx.ext.todo', 'sphinx.ext.viewcode'] # Add any paths that contain templates here, relative to this directory. templates_path = ['_templates'] # The suffix of source filenames. source_suffix = '.rst' # The encoding of source files. #source_encoding = 'utf-8-sig' # The master toctree document. master_doc = 'index' # General information about the project. project = u'CloudBioLinux' copyright = u'2013, CloudBioLinux contributors' # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the # built documents. # # The short X.Y version. version = '1.0' # The full version, including alpha/beta/rc tags. release = '1.0' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. #language = None # There are two options for replacing |today|: either, you set today to some # non-false value, then it is used: #today = '' # Else, today_fmt is used as the format for a strftime call. #today_fmt = '%B %d, %Y' # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. exclude_patterns = [] # The reST default role (used for this markup: `text`) to use for all documents. #default_role = None # If true, '()' will be appended to :func: etc. cross-reference text. #add_function_parentheses = True # If true, the current module name will be prepended to all description # unit titles (such as .. function::). #add_module_names = True # If true, sectionauthor and moduleauthor directives will be shown in the # output. They are ignored by default. #show_authors = False # The name of the Pygments (syntax highlighting) style to use. pygments_style = 'sphinx' # A list of ignored prefixes for module index sorting. #modindex_common_prefix = [] # -- Options for HTML output --------------------------------------------------- # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. html_theme = 'default' # Theme options are theme-specific and customize the look and feel of a theme # further. For a list of options available for each theme, see the # documentation. #html_theme_options = {} # Add any paths that contain custom themes here, relative to this directory. #html_theme_path = [] # The name for this set of Sphinx documents. If None, it defaults to # " v documentation". #html_title = None # A shorter title for the navigation bar. Default is the same as html_title. #html_short_title = None # The name of an image file (relative to this directory) to place at the top # of the sidebar. #html_logo = None # The name of an image file (within the static path) to use as favicon of the # docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 # pixels large. #html_favicon = None # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". html_static_path = ['_static'] # If not '', a 'Last updated on:' timestamp is inserted at every page bottom, # using the given strftime format. #html_last_updated_fmt = '%b %d, %Y' # If true, SmartyPants will be used to convert quotes and dashes to # typographically correct entities. #html_use_smartypants = True # Custom sidebar templates, maps document names to template names. #html_sidebars = {} # Additional templates that should be rendered to pages, maps page names to # template names. #html_additional_pages = {} # If false, no module index is generated. #html_domain_indices = True # If false, no index is generated. #html_use_index = True # If true, the index is split into individual pages for each letter. #html_split_index = False # If true, links to the reST sources are added to the pages. #html_show_sourcelink = True # If true, "Created using Sphinx" is shown in the HTML footer. Default is True. #html_show_sphinx = True # If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. #html_show_copyright = True # If true, an OpenSearch description file will be output, and all pages will # contain a tag referring to it. The value of this option must be the # base URL from which the finished HTML is served. #html_use_opensearch = '' # This is the file name suffix for HTML files (e.g. ".xhtml"). #html_file_suffix = None # Output file base name for HTML help builder. htmlhelp_basename = 'CloudBioLinuxdoc' # -- Options for LaTeX output -------------------------------------------------- latex_elements = { # The paper size ('letterpaper' or 'a4paper'). #'papersize': 'letterpaper', # The font size ('10pt', '11pt' or '12pt'). #'pointsize': '10pt', # Additional stuff for the LaTeX preamble. #'preamble': '', } # Grouping the document tree into LaTeX files. List of tuples # (source start file, target name, title, author, documentclass [howto/manual]). latex_documents = [ ('index', 'CloudBioLinux.tex', u'CloudBioLinux Documentation', u'CloudBioLinux contributors', 'manual'), ] # The name of an image file (relative to this directory) to place at the top of # the title page. #latex_logo = None # For "manual" documents, if this is true, then toplevel headings are parts, # not chapters. #latex_use_parts = False # If true, show page references after internal links. #latex_show_pagerefs = False # If true, show URL addresses after external links. #latex_show_urls = False # Documents to append as an appendix to all manuals. #latex_appendices = [] # If false, no module index is generated. #latex_domain_indices = True # -- Options for manual page output -------------------------------------------- # One entry per manual page. List of tuples # (source start file, name, description, authors, manual section). man_pages = [ ('index', 'cloudbiolinux', u'CloudBioLinux Documentation', [u'CloudBioLinux contributors'], 1) ] # If true, show URL addresses after external links. #man_show_urls = False # -- Options for Texinfo output ------------------------------------------------ # Grouping the document tree into Texinfo files. List of tuples # (source start file, target name, title, author, # dir menu entry, description, category) texinfo_documents = [ ('index', 'CloudBioLinux', u'CloudBioLinux Documentation', u'CloudBioLinux contributors', 'CloudBioLinux', 'One line description of project.', 'Miscellaneous'), ] # Documents to append as an appendix to all manuals. #texinfo_appendices = [] # If false, no module index is generated. #texinfo_domain_indices = True # How to display URL addresses: 'footnote', 'no', or 'inline'. #texinfo_show_urls = 'footnote' ================================================ FILE: doc/source/framework.rst ================================================ Using the CloudBioLinux Build Framework --------------------------------------- ----------------------- Obtaining CloudBioLinux ----------------------- CloudBioLinux can be obtained using `git `_. :: % git clone https://github.com/chapmanb/cloudbiolinux.git % cd cloudbiolinux ------------------------- Overview of the Framework ------------------------- ------------------------ Building Cloud Instances ------------------------ When building CloudBioLinux instances for the cloud, ``deploy/deploy.sh`` contains a script to automate cloud interactions and the installation of CloudBioLinux. This script will require a system Python be available on your system, but should otherwise install its own dependencies. Before using the deployer you will need to create a settings file describing your cloud credentials and connection information as well as any tweaks you would like to make to the: :: % cd deploy % cp settings-sample-minimal.yaml settings.yaml Before updating settings.yaml you will need to navigate the AWS management console and obtain the following information. * Your AWS Access ID and secret key (`access_id`, `secret_key`) * Ubuntu EBS-backed AMI ID to target. This writeup was tested with `ami-9b85eef2` (12.04.2 (64-bit) in us-east-1). * Image size to use (e.g. m1-small) * Availability zone (e.g. us-east-1) Carefully scan through `settings.yaml` and change the properties marked as requiring change. The word `UPDATE` in the comments indicates properties of special interest that either don't have reasonable defaults or have reasonable defaults but that I have deemed highly likely to be overridden. For this simple example the only changes you will need to make are in the ``aws`` section. Once you have updated ``settings.yaml``, launch and cloud instance and configure it with the following command: :: % ./deploy.sh --action=install_biolinux --flavor=minimal This command will configure CloudBioLinux with a minimal set of CloudBioLinux packages. The set of packages that is installed is controlled by the ``--flavor`` command. More sophisticated setups that require using Amazon EBS volumes and S3 buckets such as CloudMan clusters require additional configuration as outlined below. You can SSH into the newly created cloud instance with the command: :: % ./deploy.sh --action=ssh ----------------------------------- Building CloudMan Enabled Instances ----------------------------------- Before continuing, delete your previous instance and the file ``.vmlauncher_last_instance_aws``. TODO: Add action for this. Building a more sophisticated CloudBioLinux image integrating tools such as CloudMan requires additional settings. Please start by copying ``settings- sample-cm.yaml`` to ``settings.yaml`` and repopulating the ``aws`` section options. To fill out the remaining options found in this file, you will need to return the AWS management console and do the following: * You will need to setup a bucket to store your snaps file, here you will need the bucket name. * You will need to setup two volumes in your target availability zone, one for Galaxy tools and data (perhaps 20Gb for testing) and one for galaxyIndices. Here you will need the volume ids. * Generate a private a key (e.g. galaxy1.pem) and copy it into keys directory (or anywhere really), also note the keypair_name corresponding to the key. Next you will want to setup a directory to contain the S3 bucket contents that will eventually be used by CloudMan to configure your cluster. Create a directory (e.g. `/home/mary/marys_cloudman_bucket_contents`). Copy the files from an existing CloudMan bucket here (e.g. http://s3.amazonaws.com /cloudman- dev). It is not really important how you download these files, but one quick option is to use `s3cmd` tool: :: % sudo apt-get install s3cmd # Or your OS's package manager % mkdir /home/mary/marys_cloudman_bucket_contents % s3cmd -r get s3://cloudman-dev /home/mary/marys_cloudman_bucket_contents Here you can replace the CloudMan source (i.e. `cm.tar.gz`) or any of these files to match the customized setup you would like. In particular you are going to want to create a custom snaps.yaml file. Here is a simple outline that we will fill out as we good. :: version: 1 clouds: - name: amazon regions: - deployments: - name: GalaxyCloud filesystems: - name: galaxy roles: galaxyTools,galaxyData snap_id: snap-XXXXXXXXXXX mount_point: /mnt/galaxy - name: galaxyIndices roles: galaxyIndices snap_id: snap-XXXXXXXXXXXX mount_point: /mnt/galaxyIndices default_mi: ami-XXXXXXXXXXXXX bucket: marys_cloudman_bucket name: us-east-1 Immediately this template can be updated to reflect the bucket created above and the availability zone you are targetting. We can update the snap_id's and the default_mi after creating them. Reopen ``settings.yaml`` and fill out the remaining properties, including the volume ids you just created and the name of the bucket you used. The following set of commands will now launch a new cloud instance, attach and format tool and data volumes for CloudMan, build CloudBioLinux, snapshot these volumes, and package the image. :: % ./deploy.sh --action=launch % ./deploy.sh --action=attach_volumes % ./deploy.sh --action=install_biolinux --flavor=cloudman/cloudman_and_galaxy % ./deploy.sh --action=snapshot_volumes % ./deploy.sh --action=detach_volumes % ./deploy.sh --action=package Once a CloudMan AMI has been created, update `snaps.yaml` in your bucket directory (e.g. `/home/mary/marys_cloudman_bucket_contents`) to reflect the `snap_id`s and AMI created. These should all be available via the AWS management console or by reviewing the output of the steps above. Finally, you can upload your new bucket and launch a test CloudMan instance: :: % ./deploy.sh --action=sync_cloudman_bucket % ./deploy.sh --action=cloudman_launch This last action (``cloudman_launch``) requires uncommenting the following lines and updating the bucket name: :: #image_user_data: # bucket_default: marys_cloudman_bucket ================================================ FILE: doc/source/index.rst ================================================ .. CloudBioLinux documentation master file, created by sphinx-quickstart on Wed Jul 17 09:14:27 2013. You can adapt this file completely to your liking, but it should at least contain the root `toctree` directive. Welcome to CloudBioLinux's documentation! ========================================= .. include:: ../../README.rst .. include:: framework.rst Indices and tables ================== * :ref:`genindex` * :ref:`modindex` * :ref:`search` ================================================ FILE: doc/virtualbox.md ================================================ # CloudBioLinux, VirtualBox and Vagrant This document gives some additional information on using Vagrant with BioLinux. [Vagrant][v1] is a convenient command line manager for VirtualBox. In conjunction with the BioLinux fabric environment, any VirtualBox VM can be bootstrapped. Note the current version of vagrant needs at least VirtualBox version 4.1.x. ## VirtualBox with vagrant Add a base image to vagrant, and boot it up; community Vagrant boxes are available from [http://vagrantbox.es][v3] and [BioLinux flavors][v4]: vagrant box add box_name http://path_to_the_image.box mkdir tmp/biolinux cd tmp/biolinux vagrant init box_name vagrant up Run the fabfile, building CloudBioLinux: fab -H vagrant -f /path/to/cloudbiolinux/fabfile.py install_biolinux Then build the box, renaming package.box to `cloudbiolinux_date` and move it to a public webserver, such as Amazon S3: vagrant package mv package.box biolinux_20110122.box s3cmd put --acl-public --guess-mime-type biolinux_20110122.box s3://chapmanb/biolinux_20110122.box [v3]: http://vagrantbox.es/ [v4]: http://biobeat.org/bionode # Rolling your own ## Start from a BioLinux box See the main README file for firing up a pre-installed BioLinux box. ## Start from scratch Despite the extra work, starting from scratch may have advantages. For one you have more control of the base install. Say for a different version of Linux, a BSD kernel, or for install less software (do you really need X/KDE/Gnome?), so you do not end up with an 8 GB VM, or for more software and/or data pre-installed on a VM. The BioLinux setup is designed to be modular, to support multiple flavors (see the main README for an explanation of terms). Start with a standard downloadable prepared Vagrant box. For example a Debian 32-bits box prepared for Vagrant, or create one from scratch as is explained on the [Vagrant web site][v1]. Next add the virtualbox to vagrant using a URL, or box file: vagrant box add debian_squeeze_32 debian_squeeze_32.box (boxes are available form [http://vagrantbox.es][v3] and [http://biobeat.org/bionode][BioLinux flavors]) and create your own version mkdir myflavor cd myflavor Creates a ./Vagrantfile describing the VM. vagrant init debian_squeeze_32 Have a look inside the Vagrantfile. The default should be fine now. Start the VM (which gets copied the first time, which may take a while): vagrant up and login vagrant ssh # no password needed make sure you have enough disk space (twice the box size) for the dir ~/VirtualBox\ VMs and ~/.vagrant, as this is where VMs are copied from the original box file. At this point a bare VM is running that will accept BioLinux installations. The next step is to pull the BioLinux tree on your local system, and to run fab using the vagrant host, using a minimal install target. E.g. export source=/path/to/cloudbiolinux and fab -f $source/fabfile.py -H vagrant -c $source/contrib/flavor/minimal/fabricrc_debian.txt install_biolinux:packagelist=$source/contrib/flavor/minimal/main.yaml which uses the information from the local ./Vagrantfile. The first time the minimal fabfile is run it updates the /etc/apt/sources (on Debian-based systems), and a number of basic packages, including sudo, python, chef. It may be the Linux kernel and support libraries get upgraded, if they are in the dependency tree. Starting from a minimalistic Debian Vagrant box, the BioLinux minimal install has an unpacked size under 1Gb. E.g. vagrant@vagrant-debian-squeeze:~$ df -h Filesystem Size Used Avail Use% Mounted on /dev/sda1 39G 804M 36G 3% / tmpfs 188M 0 188M 0% /lib/init/rw udev 184M 116K 184M 1% /dev tmpfs 188M 0 188M 0% /dev/shm v-root 51G 24G 27G 47% /vagrant Despite the fact that running fabfile.py is destructive, i.e. it overwrites the current install, it is reasonably safe as it mostly uses the underlying package management system and dependency resolution. Rerunning a BioLinux fabfile can be fast. Minimal runs the second time in under 20 seconds on a basic laptop, as we do with a 'Minimal' install: ./test/test_vagrant --continue For completeness, after a minimal install you can still install a full BioLinux execute fab -H vagrant -f $source/fabfile.py install_biolinux Once you have a working Virtual Box VM with vagrant, you can package it with vagrant package and make the resulting .box file available for others to use. Read the README for further information. [v1]: http://vagrantup.com/docs/base_boxes.html ## Trouble shooting ### Guest additions You may see an error [default] The guest additions on this VM do not match the install version of VirtualBox! This may cause things such as forwarded ports, shared folders, and more to not work properly. If any of those things fail on this machine, please update the guest additions and repackage the box. Guest Additions Version: 4.0.4 VirtualBox Version: 4.1.0 this error may actually be caused by the Vbox Linux kernel drivers not having been loaded! Fix modprobe vboxdrv # Converting Vagrant images to VirtualBox and Eucalyptus images (protocol steps tested in Ubuntu Natty) ## software pre-requisite sudo gem install vagrant sudo apt-get install cloud-utils ## Importing cloud biolinux VM to your system vagrant box add base https://s3.amazonaws.com/cloudbiolinux/cbl_ubuntu_11_4_32_20110628.box vagrant init base vagrant up ## adding some missing components to the vagrant VMs vagrant ssh sudo apt-get install gdm cloud-utils openssh sudo useradd -d /home/ubuntu -m ubuntusudo passwd ubuntu sudo shutdown -r now in the graphical login after reboot get in with user:ubuntu / pass:ubuntu go to System--->Administration--->Login Window to enable autologin ## VirtualBox Appliance Virtual Appliances are pre-assemblied VM images configured for various purposes. Open the Virtualbox GUI, you should see the VM added by vagrant - you can rename it to "Cloud BioLinux 32" File->Export Appliance and distribute the .ova. Anyone in any OS running Virtualbox can import the .ova with File->Import Appliance. # Making a Eucalyptus image from VirtualBox Start with the Cloud BioLinux Virtualbox .vmdk (its location is in the VM properties from the Virtualbox GUI). Resize the vmdk, since the size may be 40G, and the Eucalyptus image will have that size. According to http://mtnbike.org/blog/?p=29 and the same here: http://www.my-guides.net/en/content/view/122/26/ convert to raw .img qemu-img convert -O raw CloudBioLinux-32bit-disk1.vmdk CloudBioLinux-32bit-disk1.img deploy to Eucalyptus via uec-publish-img CloudBioLinux-32bit-disk1.img # VirtualBox, KVM or XEN? There are more ways than one to virtualize machines on Linux. Despite the attractions of vagrant and Virtualbox, as displayed here, we note that Linux KVM may be a better choice for virtualization and testing of CloudBioLinux, as Linux distributions support KVM out of the box, and KVM has more Unix-like control. See also the information for using KVM in ./doc/linux_kvm.md. For production environments check out XEN virtualization (XEN runs Amazon EC2). ================================================ FILE: fabfile.py ================================================ """Main Fabric deployment file for CloudBioLinux distribution. This installs a standard set of useful biological applications on a remote server. It is designed for bootstrapping a machine from scratch, as with new Amazon EC2 instances. Usage: fab -H hostname -i private_key_file install_biolinux which will call into the 'install_biolinux' method below. See the README for more examples. hostname can be a named host in ~/.ssh/config Requires: Fabric http://docs.fabfile.org PyYAML http://pyyaml.org/wiki/PyYAMLDocumentation """ import os import sys from datetime import datetime from fabric.api import * from fabric.contrib.files import * import yaml # use local cloudbio directory for to_remove in [p for p in sys.path if p.find("cloudbiolinux-") > 0]: sys.path.remove(to_remove) sys.path.append(os.path.dirname(__file__)) import cloudbio from cloudbio import libraries from cloudbio.utils import _setup_logging, _configure_fabric_environment from cloudbio.cloudman import _cleanup_ec2, _configure_cloudman from cloudbio.cloudbiolinux import _cleanup_space, _freenx_scripts from cloudbio.custom import shared from cloudbio.package.shared import _yaml_to_packages from cloudbio.package import brew, conda from cloudbio.package import (_configure_and_install_native_packages, _connect_native_packages, _print_shell_exports) from cloudbio.package.nix import _setup_nix_sources, _nix_packages from cloudbio.flavor.config import get_config_file from cloudbio.config_management.puppet import _puppet_provision from cloudbio.config_management.chef import _chef_provision, chef, _configure_chef # ### Shared installation targets for all platforms def install_biolinux(target=None, flavor=None): """Main entry point for installing BioLinux on a remote server. `flavor` allows customization of CloudBioLinux behavior. It can either be a flavor name that maps to a corresponding directory in contrib/flavor or the path to a custom directory. This can contain: - alternative package lists (main.yaml, packages.yaml, custom.yaml) - custom python code (nameflavor.py) that hooks into the build machinery `target` allows running only particular parts of the build process. Valid choices are: - packages Install distro packages - custom Install custom packages - chef_recipes Provision chef recipes - libraries Install programming language libraries - post_install Setup CloudMan, FreeNX and other system services - cleanup Remove downloaded files and prepare images for AMI builds """ _setup_logging(env) time_start = _print_time_stats("Config", "start") _check_fabric_version() if env.ssh_config_path and os.path.isfile(os.path.expanduser(env.ssh_config_path)): env.use_ssh_config = True _configure_fabric_environment(env, flavor, ignore_distcheck=(target is not None and target in ["libraries", "custom"])) env.logger.debug("Target is '%s'" % target) env.logger.debug("Flavor is '%s'" % flavor) _perform_install(target, flavor) _print_time_stats("Config", "end", time_start) if hasattr(env, "keep_isolated") and env.keep_isolated: _print_shell_exports(env) def _perform_install(target=None, flavor=None, more_custom_add=None): """ Once CBL/fabric environment is setup, this method actually runs the required installation procedures. See `install_biolinux` for full details on arguments `target` and `flavor`. """ pkg_install, lib_install, custom_ignore, custom_add = _read_main_config() if more_custom_add: if custom_add is None: custom_add = {} for k, vs in more_custom_add.iteritems(): if k in custom_add: custom_add[k].extend(vs) else: custom_add[k] = vs if target is None or target == "packages": env.keep_isolated = getattr(env, "keep_isolated", "false").lower() in ["true", "yes"] # Only touch system information if we're not an isolated installation if not env.keep_isolated: # can only install native packages if we have sudo access or are root if env.use_sudo or env.safe_run_output("whoami").strip() == "root": _configure_and_install_native_packages(env, pkg_install) else: _connect_native_packages(env, pkg_install, lib_install) if env.nixpkgs: # ./doc/nixpkgs.md _setup_nix_sources() _nix_packages(pkg_install) if target is None or target == "custom": _custom_installs(pkg_install, custom_ignore, custom_add) if target is None or target == "chef_recipes": _provision_chef_recipes(pkg_install, custom_ignore) if target is None or target == "puppet_classes": _provision_puppet_classes(pkg_install, custom_ignore) if target is None or target == "brew": install_brew(flavor=flavor, automated=True) if target is None or target == "conda": install_conda(flavor=flavor, automated=True) if target is None or target == "libraries": _do_library_installs(lib_install) if target is None or target == "post_install": env.flavor.post_install() if "is_ec2_image" in env and env.is_ec2_image.upper() in ["TRUE", "YES"]: _freenx_scripts(env) if pkg_install is not None and 'cloudman' in pkg_install: _configure_cloudman(env) if target is None or target == "cleanup": if env.use_sudo: _cleanup_space(env) if "is_ec2_image" in env and env.is_ec2_image.upper() in ["TRUE", "YES"]: _cleanup_ec2(env) def _print_time_stats(action, event, prev_time=None): """ A convenience method for displaying time event during configuration. :type action: string :param action: Indicates type of action (eg, Config, Lib install, Pkg install) :type event: string :param event: The monitoring event (eg, start, stop) :type prev_time: datetime :param prev_time: A timeststamp of a previous event. If provided, duration between the time the method is called and the time stamp is included in the printout :rtype: datetime :return: A datetime timestamp of when the method was called """ time = datetime.utcnow() s = "{0} {1} time: {2}".format(action, event, time) if prev_time: s += "; duration: {0}".format(str(time-prev_time)) env.logger.info(s) return time def _check_fabric_version(): """Checks for fabric version installed """ version = env.version if int(version.split(".")[0]) < 1: raise NotImplementedError("Please install fabric version 1 or higher") def _custom_installs(to_install, ignore=None, add=None): if not env.safe_exists(env.local_install) and env.local_install: env.safe_run("mkdir -p %s" % env.local_install) pkg_config = get_config_file(env, "custom.yaml").base packages, pkg_to_group = _yaml_to_packages(pkg_config, to_install) packages = [p for p in packages if ignore is None or p not in ignore] if add is not None: for key, vals in add.iteritems(): for v in vals: pkg_to_group[v] = key packages.append(v) for p in env.flavor.rewrite_config_items("custom", packages): install_custom(p, True, pkg_to_group) def _provision_chef_recipes(to_install, ignore=None): """ Much like _custom_installs, read config file, determine what to install, and install it. """ pkg_config = get_config_file(env, "chef_recipes.yaml").base packages, _ = _yaml_to_packages(pkg_config, to_install) packages = [p for p in packages if ignore is None or p not in ignore] recipes = [recipe for recipe in env.flavor.rewrite_config_items("chef_recipes", packages)] if recipes: # Don't bother running chef if nothing to configure install_chef_recipe(recipes, True) def _provision_puppet_classes(to_install, ignore=None): """ Much like _custom_installs, read config file, determine what to install, and install it. """ pkg_config = get_config_file(env, "puppet_classes.yaml").base packages, _ = _yaml_to_packages(pkg_config, to_install) packages = [p for p in packages if ignore is None or p not in ignore] classes = [recipe for recipe in env.flavor.rewrite_config_items("puppet_classes", packages)] if classes: # Don't bother running chef if nothing to configure install_puppet_class(classes, True) def install_chef_recipe(recipe, automated=False, flavor=None): """Install one or more chef recipes by name. Usage: fab [-i key] [-u user] -H host install_chef_recipe:recipe :type recipe: string or list :param recipe: TODO :type automated: bool :param automated: If set to True, the environment is not loaded. """ _setup_logging(env) if not automated: _configure_fabric_environment(env, flavor) time_start = _print_time_stats("Chef provision for recipe(s) '{0}'".format(recipe), "start") _configure_chef(env, chef) recipes = recipe if isinstance(recipe, list) else [recipe] for recipe_to_add in recipes: chef.add_recipe(recipe_to_add) _chef_provision(env, recipes) _print_time_stats("Chef provision for recipe(s) '%s'" % recipe, "end", time_start) def install_puppet_class(classes, automated=False, flavor=None): """Install one or more puppet classes by name. Usage: fab [-i key] [-u user] -H host install_puppet_class:class :type classes: string or list :param classes: TODO :type automated: bool :param automated: If set to True, the environment is not loaded. """ _setup_logging(env) if not automated: _configure_fabric_environment(env, flavor) time_start = _print_time_stats("Puppet provision for class(es) '{0}'".format(classes), "start") classes = classes if isinstance(classes, list) else [classes] _puppet_provision(env, classes) _print_time_stats("Puppet provision for classes(s) '%s'" % classes, "end", time_start) def install_custom(p, automated=False, pkg_to_group=None, flavor=None): """ Install a single custom program or package by name. This method fetches program name from ``config/custom.yaml`` and delegates to a method in ``custom/*name*.py`` to proceed with the installation. Alternatively, if a program install method is defined in the appropriate package, it will be called directly (see param ``p``). Usage: fab [-i key] [-u user] -H host install_custom:program_name :type p: string :param p: A name of the custom program to install. This has to be either a name that is listed in ``custom.yaml`` as a subordinate to a group name or a program name whose install method is defined in either ``cloudbio`` or ``custom`` packages (e.g., ``cloudbio/custom/cloudman.py -> install_cloudman``). :type automated: bool :param automated: If set to True, the environment is not loaded and reading of the ``custom.yaml`` is skipped. """ p = p.lower() # All packages listed in custom.yaml are in lower case if not automated: _setup_logging(env) _configure_fabric_environment(env, flavor, ignore_distcheck=True) pkg_config = get_config_file(env, "custom.yaml").base packages, pkg_to_group = _yaml_to_packages(pkg_config, None) time_start = _print_time_stats("Custom install for '{0}'".format(p), "start") fn = _custom_install_function(env, p, pkg_to_group) fn(env) ## TODO: Replace the previous 4 lines with the following one, barring ## objections. Slightly different behavior because pkg_to_group will be ## loaded regardless of automated if it is None, but IMO this shouldn't ## matter because the following steps look like they would fail if ## automated is True and pkg_to_group is None. # _install_custom(p, pkg_to_group) _print_time_stats("Custom install for '%s'" % p, "end", time_start) def _install_custom(p, pkg_to_group=None): if pkg_to_group is None: pkg_config = get_config_file(env, "custom.yaml").base packages, pkg_to_group = _yaml_to_packages(pkg_config, None) fn = _custom_install_function(env, p, pkg_to_group) fn(env) def install_brew(p=None, version=None, flavor=None, automated=False): """Top level access to homebrew/linuxbrew packages. p is a package name to install, or all configured packages if not specified. """ if not automated: _setup_logging(env) _configure_fabric_environment(env, flavor, ignore_distcheck=True) if p is not None: if version: p = "%s==%s" % (p, version) brew.install_packages(env, packages=[p]) else: pkg_install = _read_main_config()[0] brew.install_packages(env, to_install=pkg_install) def install_conda(p=None, flavor=None, automated=False): if not automated: _setup_logging(env) _configure_fabric_environment(env, flavor, ignore_distcheck=True) if p is not None: conda.install_packages(env, packages=[p]) else: pkg_install = _read_main_config()[0] conda.install_packages(env, to_install=pkg_install) def _custom_install_function(env, p, pkg_to_group): """ Find custom install function to execute based on package name to pkg_to_group dict. """ try: # Allow direct calling of a program install method, even if the program # is not listed in the custom list (ie, not contained as a key value in # pkg_to_group). For an example, see 'install_cloudman' or use p=cloudman. mod_name = pkg_to_group[p] if p in pkg_to_group else p env.logger.debug("Importing module cloudbio.custom.%s" % mod_name) mod = __import__("cloudbio.custom.%s" % mod_name, fromlist=["cloudbio", "custom"]) except ImportError: raise ImportError("Need to write module cloudbio.custom.%s" % pkg_to_group[p]) replace_chars = ["-"] try: for to_replace in replace_chars: p = p.replace(to_replace, "_") env.logger.debug("Looking for custom install function %s.install_%s" % (mod.__name__, p)) fn = getattr(mod, "install_%s" % p) except AttributeError: raise ImportError("Need to write a install_%s function in custom.%s" % (p, pkg_to_group[p])) return fn def _read_main_config(): """Pull a list of groups to install based on our main configuration YAML. Reads 'main.yaml' and returns packages and libraries """ yaml_file = get_config_file(env, "main.yaml").base with open(yaml_file) as in_handle: full_data = yaml.safe_load(in_handle) packages = full_data.get('packages', []) packages = env.flavor.rewrite_config_items("main_packages", packages) libraries = full_data.get('libraries', []) custom_ignore = full_data.get('custom_ignore', []) custom_add = full_data.get("custom_additional") if packages is None: packages = [] if libraries is None: libraries = [] if custom_ignore is None: custom_ignore = [] env.logger.info("Meta-package information from {2}\n- Packages: {0}\n- Libraries: " "{1}".format(",".join(packages), ",".join(libraries), yaml_file)) return packages, sorted(libraries), custom_ignore, custom_add # ### Library specific installation code def _python_library_installer(config): """Install python specific libraries using pip, conda and easy_install. Handles using isolated anaconda environments. """ if shared._is_anaconda(env): conda_bin = shared._conda_cmd(env) for pname in env.flavor.rewrite_config_items("python", config.get("conda", [])): env.safe_run("{0} install --yes {1}".format(conda_bin, pname)) cmd = env.safe_run with settings(warn_only=True): cmd("%s -U distribute" % os.path.join(os.path.dirname(conda_bin), "easy_install")) else: pip_bin = shared._pip_cmd(env) ei_bin = pip_bin.replace("pip", "easy_install") env.safe_sudo("%s -U pip" % ei_bin) with settings(warn_only=True): env.safe_sudo("%s -U distribute" % ei_bin) cmd = env.safe_sudo for pname in env.flavor.rewrite_config_items("python", config['pypi']): cmd("{0} install --upgrade {1} --allow-unverified {1} --allow-external {1}".format(shared._pip_cmd(env), pname)) # fixes problem with packages not being in pypi def _ruby_library_installer(config): """Install ruby specific gems. """ gem_ext = getattr(env, "ruby_version_ext", "") def _cur_gems(): with settings( hide('warnings', 'running', 'stdout', 'stderr')): gem_info = env.safe_run_output("gem%s list --no-versions" % gem_ext) return [l.rstrip("\r") for l in gem_info.split("\n") if l.rstrip("\r")] installed = _cur_gems() for gem in env.flavor.rewrite_config_items("ruby", config['gems']): # update current gems only to check for new installs if gem not in installed: installed = _cur_gems() if gem in installed: env.safe_sudo("gem%s update %s" % (gem_ext, gem)) else: env.safe_sudo("gem%s install %s" % (gem_ext, gem)) def _perl_library_installer(config): """Install perl libraries from CPAN with cpanminus. """ with shared._make_tmp_dir() as tmp_dir: with cd(tmp_dir): env.safe_run("wget --no-check-certificate -O cpanm " "https://raw.github.com/miyagawa/cpanminus/master/cpanm") env.safe_run("chmod a+rwx cpanm") env.safe_sudo("mv cpanm %s/bin" % env.system_install) sudo_str = "--sudo" if env.use_sudo else "" for lib in env.flavor.rewrite_config_items("perl", config['cpan']): # Need to hack stdin because of some problem with cpanminus script that # causes fabric to hang # http://agiletesting.blogspot.com/2010/03/getting-past-hung-remote-processes-in.html env.safe_run("cpanm %s --skip-installed --notest %s < /dev/null" % (sudo_str, lib)) def _haskell_library_installer(config): """Install haskell libraries using cabal. """ run("cabal update") for lib in config["cabal"]: sudo_str = "--root-cmd=sudo" if env.use_sudo else "" env.safe_run("cabal install %s --global %s" % (sudo_str, lib)) lib_installers = { "r-libs" : libraries.r_library_installer, "python-libs" : _python_library_installer, "ruby-libs" : _ruby_library_installer, "perl-libs" : _perl_library_installer, "haskell-libs": _haskell_library_installer, } def install_libraries(language): """High level target to install libraries for a specific language. """ _setup_logging(env) _check_fabric_version() _configure_fabric_environment(env, ignore_distcheck=True) _do_library_installs(["%s-libs" % language]) def _do_library_installs(to_install): for iname in to_install: yaml_file = get_config_file(env, "%s.yaml" % iname).base with open(yaml_file) as in_handle: config = yaml.safe_load(in_handle) lib_installers[iname](config) ================================================ FILE: ggd-recipes/BDGP6/gtf.yaml ================================================ # Ensembl GTF file distributed by Ensembl for BDGP6 --- attributes: name: gtf version: 95 recipe: full: recipe_type: bash recipe_cmds: - | url=http://ftp.ensembl.org/pub/release-95/gtf/drosophila_melanogaster/Drosophila_melanogaster.BDGP6.95.gtf.gz mkdir -p rnaseq wget --no-check-certificate -qO- $url | gunzip -c | rnaseq/BDGP6.gtf recipe_outfiles: - rnaseq/BDGP6.gtf ================================================ FILE: ggd-recipes/BDGP6/mirbase.yaml ================================================ # small RNA seq annotation from different sources --- attributes: name: srnaseq version: 20200302 recipe: full: recipe_type: bash recipe_cmds: - | mkdir -p srnaseq cd srnaseq wget --random-wait --retry-connrefused -nv -c -O tmp.gtf.gz ftp://ftp.ncbi.nlm.nih.gov/genomes/all/GCF/000/001/215/GCF_000001215.4_Release_6_plus_ISO1_MT/GCF_000001215.4_Release_6_plus_ISO1_MT_genomic.gff.gz zgrep -v exon tmp.gtf.gz | grep -v region | sed 's/Name/name/g' | sed -e 's/NC_004353.4/chr4/g; s/NC_004354.4/chrX/g; s/NC_024511.2/chrM/g; s/NC_024512.1/chrY/g; s/NT_033777.3/chr3R/g; s/NT_033778.4/chr2R/g; s/NT_033779.5/chr2L/g; s/NT_037436.4/chr3L/g; s/NW_007931121.1/CP007120.1/g' | sed 's/=/ /g' > srna-transcripts.gtf # mirbase wget --random-wait --retry-connrefused -nv -c -O hairpin.t.fa https://mirbase.org/download/CURRENT/hairpin.fa cat hairpin.t.fa | awk '{if ($0~/>dme/){name=$0; print name} else if ($0~/^>/){name=0};if (name!=0 && $0!~/^>/){print $0;}}' | sed 's/U/T/g' > hairpin.fa rm hairpin.t.fa wget --random-wait --retry-connrefused -nv -c -O mature.t.fa https://mirbase.org/download/CURRENT/mature.fa cat mature.t.fa | awk '{if ($0~/>dme/){name=$0; print name} else if ($0~/^>/){name=0};if (name!=0 && $0!~/^>/){print $0;}}' | sed 's/U/T/g' > mature.fa rm mature.t.fa wget --random-wait --retry-connrefused -nv -c -O miRNA.t.str https://mirbase.org/download/CURRENT/miRNA.dat cat miRNA.t.str | awk '{if ($0~/dme/)print $0}' > miRNA.str rm miRNA.t.str wget --random-wait --retry-connrefused -nv -c -O mirbase.gff3 https://mirbase.org/download/dme.gff3 wget --no-check-certificate -c -O Rfam_for_miRDeep.fa.gz https://github.com/lpantano/mirdeep2_core/raw/data/Rfam_for_miRDeep.fa.gz && gunzip -f Rfam_for_miRDeep.fa.gz # targetscan analysis wget --random-wait --retry-connrefused --no-check-certificate -nv -c -O Summary_Counts.txt.zip http://www.targetscan.org/fly_72/fly_72_data_download/Summary_Counts.all_predictions.txt.zip && unzip Summary_Counts.txt.zip wget --random-wait --retry-connrefused --no-check-certificate -nv -c -O miR_Family_Info.txt.zip http://www.targetscan.org/fly_72/fly_72_data_download/miR_Family_Info.txt.zip && unzip miR_Family_Info.txt.zip wget --random-wait --retry-connrefused --no-check-certificate -nv -N -c ftp://mirbase.org/pub/mirbase/22/database_files/mirna_mature.txt.gz recipe_outfiles: - srnaseq/srna-transcripts.gtf - srnaseq/hairpin.fa - srnaseq/mature.fa - srnaseq/miRNA.str - srnaseq/mirbase.gff3 - srnaseq/Rfam_for_miRDeep.fa - srnaseq/Summary_Counts.all_predictions.txt - srnaseq/miR_Family_Info.txt - srnaseq/mirna_mature.txt.gz ================================================ FILE: ggd-recipes/BDGP6/seq.yaml ================================================ # Drosophila BDGP6 genome distributed by Ensembl --- attributes: name: seq version: BDGP6 recipe: full: recipe_type: bash recipe_cmds: - | mkdir -p seq url=http://ftp.ensembl.org/pub/release-95/fasta/drosophila_melanogaster/dna/Drosophila_melanogaster.BDGP6.dna.toplevel.fa.gz wget -c -O seq/BDGP6.fa.gz ${url} gunzip -c seq/BDGP6.fa.gz > seq/BDGP6.fa recipe_outfiles: - seq/BDGP6.fa - seq/BDGP6.fa.gz ================================================ FILE: ggd-recipes/BDGP6/transcripts.yaml ================================================ # Transcript, annotation and indices for RNA-seq analysis # Prepared from Ensembl transcripts using: # https://github.com/chapmanb/cloudbiolinux/blob/master/utils/prepare_tx_gff.py # Prepared against Ensembl gene build 95 --- attributes: name: transcripts version: 2019-03-26_95 recipe: full: recipe_type: bash recipe_cmds: - | baseurl=https://s3.amazonaws.com/biodata/annotation/BDGP6-rnaseq-2019-03-26_95.tar.xz wget -c -N --no-check-certificate $baseurl xz -dc *-rnaseq-*.tar.xz | tar -xpf - mv */rnaseq-* rnaseq recipe_outfiles: - rnaseq/ref-transcripts.bed - rnaseq/ref-transcripts.dexseq.gff3 - rnaseq/ref-transcripts.fa - rnaseq/ref-transcripts.genePred - rnaseq/ref-transcripts.gtf - rnaseq/ref-transcripts.gtf.db - rnaseq/ref-transcripts-mask.gtf - rnaseq/ref-transcripts.refFlat - rnaseq/ref-transcripts-splicesites.txt - rnaseq/rRNA.gtf - rnaseq/rRNA.gtf.db - rnaseq/rRNA.interval_list - rnaseq/tx2gene.csv - rnaseq/version.txt ================================================ FILE: ggd-recipes/GRCh37/1000g.yaml ================================================ # 1000 genomes --- attributes: name: 1000g version: phase3_shapeit2_mvncall_integrated_v5a.20130502 recipe: full: recipe_type: bash recipe_cmds: - | baseurl=https://s3.amazonaws.com/gemini-annotations url=$baseurl/ALL.wgs.phase3_shapeit2_mvncall_integrated_v5a.20130502.sites.tidy.vcf.gz mkdir -p variation wget -c -O variation/1000g.vcf.gz $url wget -c -O variation/1000g.vcf.gz.tbi $url.tbi recipe_outfiles: - variation/1000g.vcf.gz - variation/1000g.vcf.gz.tbi ================================================ FILE: ggd-recipes/GRCh37/1000g_omni_snps.yaml ================================================ --- attributes: name: 1000g_omni_snps version: 2.5 recipe: full: recipe_type: bash recipe_cmds: - | baseurl=ftp://gsapubftp-anonymous:none@ftp.broadinstitute.org/bundle/b37/1000G_omni2.5.b37.vcf.gz mkdir -p variation cd variation wget -O - $baseurl | gunzip -c | bgzip -c > 1000G_omni2.5.vcf.gz tabix -f -p vcf 1000G_omni2.5.vcf.gz recipe_outfiles: - variation/1000G_omni2.5.vcf.gz - variation/1000G_omni2.5.vcf.gz.tbi ================================================ FILE: ggd-recipes/GRCh37/1000g_snps.yaml ================================================ --- attributes: name: 1000g_snps version: 2.8 recipe: full: recipe_type: bash recipe_cmds: - | baseurl=ftp://gsapubftp-anonymous:none@ftp.broadinstitute.org/bundle/b37/1000G_phase1.snps.high_confidence.b37.vcf.gz mkdir -p variation cd variation wget -O - $baseurl | gunzip -c | bgzip -c > 1000G_phase1.snps.high_confidence.vcf.gz tabix -f -p vcf 1000G_phase1.snps.high_confidence.vcf.gz recipe_outfiles: - variation/1000G_phase1.snps.high_confidence.vcf.gz - variation/1000G_phase1.snps.high_confidence.vcf.gz.tbi ================================================ FILE: ggd-recipes/GRCh37/ACMG56_genes.yaml ================================================ # ACMG regions: http://iobio.io/2016/03/29/acmg56/ # https://github.com/lpantano/custom_genome_annotation --- attributes: name: ACMG56_genes version: 20160810 recipe: full: recipe_type: bash recipe_cmds: - | baseurl=https://raw.githubusercontent.com/lpantano/custom_genome_annotation/master/acmg56.GRCh37.bed dir=coverage/prioritize/actionable mkdir -p $dir cd $dir wget --no-check-certificate -c -O ACMG56.bed $baseurl recipe_outfiles: - coverage/prioritize/actionable/ACMG56.bed ================================================ FILE: ggd-recipes/GRCh37/GA4GH_problem_regions.yaml ================================================ # GA4GH problematic region files: https://drive.google.com/folderview?id=0B7Ao1qqJJDHQUjVIN3liUUZNWjg&usp=sharing#list # Cached mappability regions # wget --no-check-certificate -O - http://hgdownload.cse.ucsc.edu/goldenPath/hg19/encodeDCC/wgEncodeMapability/wgEncodeDacMapabilityConsensusExcludable.bed.gz | gunzip -c | sed "s/^chrM/MT/g" | sed "s/^chr//g" | bgzip -c > $encode/wgEncodeDacMapabilityConsensusExcludable.bed.gz # tabix -p bed -f $encode/wgEncodeDacMapabilityConsensusExcludable.bed.gz # GC regions for Hartwig tools: https://resources.hartwigmedicalfoundation.nl/ --- attributes: name: GA4GH_problem_regions version: 20181016 recipe: full: recipe_type: bash recipe_cmds: - | baseurl=http://bcbio_nextgen.s3.amazonaws.com/GA4GH_problem_regions.zip mkdir -p coverage/problem_regions/GA4GH cd coverage/problem_regions/GA4GH wget --no-check-certificate -c -O GA4GH_problem_regions.zip $baseurl unzip -o GA4GH_problem_regions.zip for file in *.bed do sed 's/^chr//g' $file > $file.tmp mv $file.tmp $file bgzip $file tabix -p bed $file.gz done cd ../../.. encode=coverage/problem_regions/ENCODE mkdir -p $encode cd $encode wget --no-check-certificate http://s3.amazonaws.com/biodata/coverage/problem_regions/ENCODE/GRCh37/wgEncodeDacMapabilityConsensusExcludable.bed.gz wget --no-check-certificate http://s3.amazonaws.com/biodata/coverage/problem_regions/ENCODE/GRCh37/wgEncodeDacMapabilityConsensusExcludable.bed.gz.tbi cd ../../.. repeats=coverage/problem_regions/repeats mkdir -p $repeats wget --no-check-certificate -O - https://raw.githubusercontent.com/chapmanb/delly/master/human.hg19.excl.tsv | grep -v ^chr > $repeats/sv_repeat_telomere_centromere.bed wget --no-check-certificate -O - https://github.com/lh3/varcmp/raw/master/scripts/LCR-hs37d5.bed.gz | gunzip -c | bgzip -c > $repeats/LCR.bed.gz tabix -p bed -f $repeats/LCR.bed.gz wget --no-check-certificate -O $repeats/polyx.bed.gz http://s3.amazonaws.com/biodata/coverage/problem_regions/polyx/polyX-GRCh37.bed.gz wget --no-check-certificate -O $repeats/polyx.bed.gz.tbi http://s3.amazonaws.com/biodata/coverage/problem_regions/polyx/polyX-GRCh37.bed.gz.tbi gc=coverage/gc mkdir -p $gc wget --no-check-certificate -O - https://s3.amazonaws.com/biodata/collections/GRCh37/coverage/gc/GC_profile.1000bp.cnp > $gc/GC_profile.1000bp.cnp recipe_outfiles: - coverage/problem_regions/GA4GH/README.md - coverage/problem_regions/GA4GH/bad_promoter.bed.gz - coverage/problem_regions/GA4GH/bad_promoter.bed.gz.tbi - coverage/problem_regions/GA4GH/gc15.bed.gz - coverage/problem_regions/GA4GH/gc15.bed.gz.tbi - coverage/problem_regions/GA4GH/gc15to20.bed.gz - coverage/problem_regions/GA4GH/gc15to20.bed.gz.tbi - coverage/problem_regions/GA4GH/gc20to25.bed.gz - coverage/problem_regions/GA4GH/gc20to25.bed.gz.tbi - coverage/problem_regions/GA4GH/gc25to30.bed.gz - coverage/problem_regions/GA4GH/gc25to30.bed.gz.tbi - coverage/problem_regions/GA4GH/gc65to70.bed.gz - coverage/problem_regions/GA4GH/gc65to70.bed.gz.tbi - coverage/problem_regions/GA4GH/gc70to75.bed.gz - coverage/problem_regions/GA4GH/gc70to75.bed.gz.tbi - coverage/problem_regions/GA4GH/gc75to80.bed.gz - coverage/problem_regions/GA4GH/gc75to80.bed.gz.tbi - coverage/problem_regions/GA4GH/gc80to85.bed.gz - coverage/problem_regions/GA4GH/gc80to85.bed.gz.tbi - coverage/problem_regions/GA4GH/gc85.bed.gz - coverage/problem_regions/GA4GH/gc85.bed.gz.tbi - coverage/problem_regions/GA4GH/heng_um75-hs37d5.bed.gz - coverage/problem_regions/GA4GH/heng_um75-hs37d5.bed.gz.tbi - coverage/problem_regions/GA4GH/low_complexity_51to200bp.bed.gz - coverage/problem_regions/GA4GH/low_complexity_51to200bp.bed.gz.tbi - coverage/problem_regions/GA4GH/low_complexity_gt200bp.bed.gz - coverage/problem_regions/GA4GH/low_complexity_gt200bp.bed.gz.tbi - coverage/problem_regions/GA4GH/self_chain.bed.gz - coverage/problem_regions/GA4GH/self_chain.bed.gz.tbi - coverage/problem_regions/ENCODE/wgEncodeDacMapabilityConsensusExcludable.bed.gz - coverage/problem_regions/ENCODE/wgEncodeDacMapabilityConsensusExcludable.bed.gz.tbi - coverage/problem_regions/repeats/sv_repeat_telomere_centromere.bed - coverage/problem_regions/repeats/LCR.bed.gz - coverage/problem_regions/repeats/LCR.bed.gz.tbi - coverage/problem_regions/repeats/polyx.bed.gz - coverage/problem_regions/repeats/polyx.bed.gz.tbi - coverage/gc/GC_profile.1000bp.cnp ================================================ FILE: ggd-recipes/GRCh37/GRCh37_NCBI2ensembl.txt ================================================ NC_000001.10 1 NC_000002.11 2 NC_000003.11 3 NC_000004.11 4 NC_000005.9 5 NC_000006.11 6 NC_000007.13 7 NC_000008.10 8 NC_000009.11 9 NC_000010.10 10 NC_000011.9 11 NC_000012.11 12 NC_000013.10 13 NC_000014.8 14 NC_000015.9 15 NC_000016.9 16 NC_000017.10 17 NC_000018.9 18 NC_000019.9 19 NC_000020.10 20 NC_000021.8 21 NC_000022.10 22 NC_000023.10 X NC_000024.9 Y NC_012920.1 MT ================================================ FILE: ggd-recipes/GRCh37/MIG.yaml ================================================ # Medically Interpretable Genes, compiled by Personalis: http://www.genomemedicine.com/content/pdf/s13073-015-0197-4.pdf --- attributes: name: MIG version: 20150730 recipe: full: recipe_type: bash recipe_cmds: - | baseurl=http://bcbio_nextgen.s3.amazonaws.com/MIG.zip mkdir -p prioritization cd prioritization wget -c -O MIG.zip $baseurl unzip MIG.zip file=MIG.bed sed 's/^chr//g' $file > $file.tmp mv $file.tmp $file cd .. recipe_outfiles: - prioritization/MIG.bed ================================================ FILE: ggd-recipes/GRCh37/RADAR.yaml ================================================ # RADAR RNA editing events: # (http://lilab.stanford.edu/GokulR/database/Human_AG_all_hg19_v2.txt) # installs a BED file of the RADAR events, converted to GRCh37 chromosome # coordinates --- attributes: name: RADAR version: v2-20180202 recipe: full: recipe_type: bash recipe_cmds: - | url=https://s3.amazonaws.com/biodata/annotation/RADAR/GRCh37/RADAR.bed.gz mkdir -p editing cd editing wget --no-check-certificate -q $url wget --no-check-certificate -q $url.tbi recipe_outfiles: - editing/RADAR.bed.gz - editing/RADAR.bed.gz.tbi ================================================ FILE: ggd-recipes/GRCh37/af_only_gnomad.yaml ================================================ # af_only_gnomad file for t-only variant calling with mutect2 # From the Broad bundle: # ftp://gsapubftp-anonymous@ftp.broadinstitute.org/bundle/Mutect2/af-only-gnomad.raw.sites.b37.vcf.gz --- attributes: name: af_only_gnomad version: 20170602 recipe: full: recipe_type: bash recipe_cmds: - | url=ftp://gsapubftp-anonymous@ftp.broadinstitute.org/bundle/Mutect2/ base=af-only-gnomad.raw.sites.b37 new=af_only_gnomad mkdir -p variation for suffix in .vcf.gz .vcf.gz.tbi do [[ -f variation/$new$suffix ]] || wget --no-check-certificate -c -O variation/$new$suffix $url/$base$suffix done recipe_outfiles: - variation/af_only_gnomad.vcf.gz - variation/af_only_gnomad.vcf.gz.tbi ================================================ FILE: ggd-recipes/GRCh37/ancestral.yaml ================================================ --- attributes: name: ancestral version: 20141010 recipe: full: recipe_type: bash recipe_cmds: - | baseurl=https://s3.amazonaws.com/bcbio_nextgen/human_ancestor.fa.gz mkdir -p variation cd variation wget --no-check-certificate -c $baseurl wget --no-check-certificate -c $baseurl.fai wget --no-check-certificate -c $baseurl.gzi recipe_outfiles: - variation/human_ancestor.fa.gz - variation/human_ancestor.fa.gz.fai - variation/human_ancestor.fa.gz.gzi ================================================ FILE: ggd-recipes/GRCh37/battenberg.yaml ================================================ # Battenberg CNV caller input data: # https://github.com/cancerit/cgpBattenberg --- attributes: name: battenberg version: v3-20150610 recipe: full: required: - download_generate_bberg_ref_files.pl recipe_type: bash recipe_cmds: - | proburl=https://github.com/cancerit/cgpBattenberg/raw/dev/perl/share/battenberg/probloci.txt.gz impute_info=battenberg/impute/impute_info.txt mkdir -p battenberg [[ -f $impute_info ]] || download_generate_bberg_ref_files.pl -c -o `pwd`/battenberg # Change references to point to final directory sed -i 's/txtmp\///g' $impute_info wget --no-check-certificate -c -O battenberg/probloci.txt.gz $proburl gunzip battenberg/probloci.txt.gz recipe_outfiles: - battenberg/probloci.txt - battenberg/1000genomesloci - battenberg/impute ================================================ FILE: ggd-recipes/GRCh37/capture_regions.yaml ================================================ # Common sets of capture region BED files for different platforms --- attributes: name: capture_regions version: 20161202 recipe: full: recipe_type: bash recipe_cmds: - | baseurl=https://raw.githubusercontent.com/AstraZeneca-NGS/reference_data/master/GRCh37/bed files="Exome-Agilent_V5_Clinical.bed Exome-Agilent_V6.bed Exome-IDT_V1.bed Exome-MedExome.bed Exome-NGv3.bed" mkdir -p coverage/capture_regions cd coverage/capture_regions for fname in $files do wget --no-check-certificate -c $baseurl/$fname done recipe_outfiles: - coverage/capture_regions/Exome-Agilent_V5_Clinical.bed - coverage/capture_regions/Exome-Agilent_V6.bed - coverage/capture_regions/Exome-IDT_V1.bed - coverage/capture_regions/Exome-MedExome.bed - coverage/capture_regions/Exome-NGv3.bed ================================================ FILE: ggd-recipes/GRCh37/clinvar.yaml ================================================ # ClinVar: http://www.clinvar.com/ # UCSFify name sed magic from: https://github.com/mmarchin/utilities/blob/master/ucscify.sh --- attributes: name: clinvar version: 20210110 recipe: full: recipe_type: bash recipe_cmds: - | release=20210110 # the latest version baseurl=https://ftp.ncbi.nlm.nih.gov/pub/clinvar/vcf_GRCh37/clinvar.vcf.gz mkdir -p variation wget --no-check-certificate -c -O variation/clinvar-orig.vcf.gz $baseurl # remove old files rm -f variation/clinvar.vcf.gz rm -f variation/clinvar.vcf.gz.tbi zcat variation/clinvar-orig.vcf.gz | bgzip -c > variation/clinvar.vcf.gz tabix -f -p vcf variation/clinvar.vcf.gz recipe_outfiles: - variation/clinvar.vcf.gz - variation/clinvar.vcf.gz.tbi ================================================ FILE: ggd-recipes/GRCh37/cosmic.yaml ================================================ --- attributes: name: cosmic version: 68-20180114 recipe: full: recipe_type: bash recipe_cmds: - | baseurl=https://s3.amazonaws.com/biodata/variants/cosmic-v68-GRCh37.vcf.gz mkdir -p variation cd variation wget --no-check-certificate -c -O cosmic.vcf.gz $baseurl wget --no-check-certificate -c -O cosmic.vcf.gz.tbi $baseurl.tbi recipe_outfiles: - variation/cosmic.vcf.gz - variation/cosmic.vcf.gz.tbi ================================================ FILE: ggd-recipes/GRCh37/dbnsfp.yaml ================================================ # google drive downloads are unreliable - too many requests, try again in 24h # using ftp ftp://dbnsfp:dbnsfp@dbnsfp.softgenetics.com/dbNSFP4.1a.zip # bumped version because 3.5a is not supported anymore - 25G instead of 16G --- attributes: name: dbnsfp version: 4.1a recipe: full: recipe_type: bash recipe_cmds: - | mkdir -p variation cd variation wget -c ftp://dbnsfp:dbnsfp@dbnsfp.softgenetics.com/dbNSFP4.1a.zip UNPACK_DIR=`pwd`/tmpunpack if [ ! -f dbNSFP.txt.gz ]; then mkdir -p $UNPACK_DIR 7z e dbNSFP*.zip "dbNSFP*_variant.chrM.gz" gunzip dbNSFP*_variant.chrM.gz head -n1 dbNSFP*_variant.chrM > $UNPACK_DIR/header.txt rm dbNSFP*_variant.chrM # unzip only files with chromosomal info, eg. skip genes and readme. # [awk] check if hg19 position exists, if so, move data to col 1 and 2, then print # and sort on first and second column unzip -p dbNSFP*.zip "dbNSFP*_variant.chr*" | gunzip -c | grep -v '^#chr' | awk -F $'\t' 'BEGIN {OFS = FS} {if ($8 != "." && $9 != ".") {$1=$8;$2=$9;print}}' | sort -T $UNPACK_DIR -k1,1 -k2,2n | cat $UNPACK_DIR/header.txt - | bgzip -c > dbNSFP.txt.gz #extract readme file, used by VEP plugin to add vcf header info unzip -p dbNSFP*.zip "*readme.txt" > dbNSFP.readme.txt fi # index in position 1 and 2 tabix -f -s 1 -b 2 -e 2 -c '#' dbNSFP.txt.gz rm -f $UNPACK_DIR/* && rmdir $UNPACK_DIR rm -f dbNSFP*.zip recipe_outfiles: - variation/dbNSFP.txt.gz - variation/dbNSFP.txt.gz.tbi - variation/dbNSFP.readme.txt ================================================ FILE: ggd-recipes/GRCh37/dbscsnv.yaml ================================================ --- attributes: name: dbscsnv version: 1.1 recipe: full: recipe_type: bash recipe_cmds: - | baseurl=ftp://dbnsfp:dbnsfp@dbnsfp.softgenetics.com/dbscSNV1.1.zip mkdir -p variation cd variation wget -c -N $baseurl if [ ! -f dbscSNV.txt.gz ]; then UNPACK_DIR=`pwd`/tmp-unpack mkdir -p $UNPACK_DIR unzip -p dbscSNV*.zip "dbscSNV*.chr*" | \ awk -F $'\t' ' BEGIN {OFS = FS} (FNR==1){print} ($1~/^chr/){next} {print | "sort -T "UNPACK_DIR" -k1,1 -k2,2n"} ' "UNPACK_DIR=$UNPACK_DIR" | \ bgzip -c > dbscSNV.txt.gz fi tabix -s 1 -b 2 -e 2 -S 1 dbscSNV.txt.gz rm -f $UNPACK_DIR/* && rmdir $UNPACK_DIR rm -f dbscSNV*.zip recipe_outfiles: - variation/dbscSNV.txt.gz - variation/dbscSNV.txt.gz.tbi ================================================ FILE: ggd-recipes/GRCh37/dbsnp.yaml ================================================ # UCSFify name sed magic from: https://github.com/mmarchin/utilities/blob/master/ucscify.sh # https://github.com/dpryan79/ChromosomeMappings/ --- attributes: name: dbsnp version: 156-20231016 recipe: full: recipe_type: bash recipe_cmds: - | build=156 version=GCF_000001405.25 url=https://ftp.ncbi.nih.gov/snp/archive/b$build/VCF/$version.gz mkdir -p variation cd variation wget -c -O dbsnp-${build}-orig.vcf.gz $url wget -c -O dbsnp-${build}-orig.vcf.gz.tbi $url.tbi remap_url=https://raw.githubusercontent.com/chapmanb/cloudbiolinux/master/ggd-recipes/GRCh37/GRCh37_NCBI2ensembl.txt wget --no-check-certificate -q -O remap.tsv $remap_url export TMPDIR=`pwd` # remove NW_ and NT_ contigs absent in hg19 and in GRCh37 gunzip -c dbsnp-${build}-orig.vcf.gz | grep -v "^NW_" | grep -v "^NT_" | bgzip -c > dbsnp-${build}.noNW.vcf.gz tabix dbsnp-${build}.noNW.vcf.gz bcftools annotate -Ou --rename-chrs remap.tsv dbsnp-${build}.noNW.vcf.gz | \ bcftools sort -m 1G -Oz -T . -o dbsnp-${build}.vcf.gz tabix -f -p vcf -C dbsnp-${build}.vcf.gz tabix -f -p vcf dbsnp-${build}.vcf.gz ln -sf dbsnp-${build}.vcf.gz dbsnp.vcf.gz ln -sf dbsnp-${build}.vcf.gz.tbi dbsnp.vcf.gz.tbi ln -sf dbsnp-${build}.vcf.gz.csi dbsnp.vcf.gz.csi cd .. recipe_outfiles: - variation/dbsnp.vcf.gz - variation/dbsnp.vcf.gz.csi - variation/dbsnp.vcf.gz.tbi - variation/dbsnp-156.vcf.gz - variation/dbsnp-156.vcf.gz.csi - variation/dbsnp-156.vcf.gz.tbi ================================================ FILE: ggd-recipes/GRCh37/dream-syn3.yaml ================================================ # Truth sets from the DREAM genomic challenge 3 # https://www.synapse.org/#!Synapse:syn312572/wiki/62018 # # Originally retrieved from https://www.synapse.org/#!Synapse:syn2177211 # and prepared with this script: # https://github.com/chapmanb/bcbio-nextgen/blob/master/scripts/utils/format_dream_truthset.py --- attributes: name: dream-syn3 version: 2014-08-04 recipe: full: recipe_type: bash recipe_cmds: - | dir=validation/dream-syn3 orig=synthetic_challenge_set3_tumor_20pctmasked_truth mkdir -p $dir wget --no-check-certificate https://s3.amazonaws.com/bcbio_nextgen/dream/synthetic_challenge_set3_tumor_20pctmasked_truth.tar.gz tar -xzvpf synthetic_challenge_set3_tumor_20pctmasked_truth.tar.gz cp ${orig}.vcf.gz $dir/truth_small_variants.vcf.gz cp ${orig}.vcf.gz.tbi $dir/truth_small_variants.vcf.gz.tbi cp ${orig}_regions.bed $dir/truth_regions.bed cp ${orig}_sv_DEL.bed $dir/truth_DEL.bed cp ${orig}_sv_DUP.bed $dir/truth_DUP.bed cp ${orig}_sv_INS.bed $dir/truth_INS.bed cp ${orig}_sv_INV.bed $dir/truth_INV.bed recipe_outfiles: - validation/dream-syn3/truth_small_variants.vcf.gz - validation/dream-syn3/truth_small_variants.vcf.gz.tbi - validation/dream-syn3/truth_regions.bed - validation/dream-syn3/truth_DEL.bed - validation/dream-syn3/truth_DUP.bed - validation/dream-syn3/truth_INS.bed - validation/dream-syn3/truth_INV.bed ================================================ FILE: ggd-recipes/GRCh37/dream-syn4.yaml ================================================ # Truth sets from the DREAM genomic challenge 4 # https://www.synapse.org/#!Synapse:syn312572/wiki/62018 # # Originally retrieved from https://www.synapse.org/#!Synapse:syn2177211 # and prepared with this script: # https://github.com/chapmanb/bcbio-nextgen/blob/master/scripts/utils/format_dream_truthset.py --- attributes: name: dream-syn4 version: 2016-06-11 recipe: full: recipe_type: bash recipe_cmds: - | dir=validation/dream-syn4 orig=synthetic_challenge_set4_tumour_25pctmasked_truth mkdir -p $dir wget --no-check-certificate -c https://s3.amazonaws.com/bcbio_nextgen/dream/synthetic_challenge_set4_tumour_25pctmasked_truth.tar.gz tar -xzvpf ${orig}.tar.gz cp ${orig}.vcf.gz $dir/truth_small_variants.vcf.gz cp ${orig}.vcf.gz.tbi $dir/truth_small_variants.vcf.gz.tbi cp ${orig}_regions.bed $dir/truth_regions.bed cp ${orig}_sv_DEL.bed $dir/truth_DEL.bed cp ${orig}_sv_DUP.bed $dir/truth_DUP.bed cp ${orig}_sv_INV.bed $dir/truth_INV.bed recipe_outfiles: - validation/dream-syn4/truth_small_variants.vcf.gz - validation/dream-syn4/truth_small_variants.vcf.gz.tbi - validation/dream-syn4/truth_regions.bed - validation/dream-syn4/truth_DEL.bed - validation/dream-syn4/truth_DUP.bed - validation/dream-syn4/truth_INV.bed ================================================ FILE: ggd-recipes/GRCh37/ericscript.yaml ================================================ --- attributes: name: ericscript version: 73 recipe: full: recipe_type: bash recipe_cmds: - | # Uses Google Drive for faster download with tricks from # http://stackoverflow.com/a/38937732/252589 ggID='0B9s__vuJPvIibDRIb0RFdHFlQmM' ggURL='https://drive.google.com/uc?export=download' mkdir -p rnaseq/ericscript cd rnaseq/ericscript filename="$(curl -k -sc tmp-gcokie "${ggURL}&id=${ggID}" | grep -o '="uc-name.*' | sed 's/.*">//;s/<.a> .*//')" getcode="$(awk '/_warning_/ {print $NF}' tmp-gcokie)" curl -k -Lb tmp-gcokie "${ggURL}&confirm=${getcode}&id=${ggID}" -o "${filename}" -C - tar -xjvpf ericscript_db_homosapiens_ensembl73.tar.bz2 recipe_outfiles: - rnaseq/ericscript/ericscript_db_homosapiens_ensembl73/data/homo_sapiens/EnsemblGene.Reference.fa.amb - rnaseq/ericscript/ericscript_db_homosapiens_ensembl73/data/homo_sapiens/allseq.fa.pac - rnaseq/ericscript/ericscript_db_homosapiens_ensembl73/data/homo_sapiens/allseq.fa.bwt - rnaseq/ericscript/ericscript_db_homosapiens_ensembl73/data/homo_sapiens/allseq.fa - rnaseq/ericscript/ericscript_db_homosapiens_ensembl73/data/homo_sapiens/EnsemblGene.Sequences.RData - rnaseq/ericscript/ericscript_db_homosapiens_ensembl73/data/homo_sapiens/EnsemblGene.GeneInfo.RData - rnaseq/ericscript/ericscript_db_homosapiens_ensembl73/data/homo_sapiens/EnsemblGene.Reference.fa - rnaseq/ericscript/ericscript_db_homosapiens_ensembl73/data/homo_sapiens/EnsemblGene.GenePosition.RData - rnaseq/ericscript/ericscript_db_homosapiens_ensembl73/data/homo_sapiens/allseq.fa.sa - rnaseq/ericscript/ericscript_db_homosapiens_ensembl73/data/homo_sapiens/allseq.fa.amb - rnaseq/ericscript/ericscript_db_homosapiens_ensembl73/data/homo_sapiens/EnsemblGene.Paralogs.RData - rnaseq/ericscript/ericscript_db_homosapiens_ensembl73/data/homo_sapiens/EnsemblGene.GeneNames.RData - rnaseq/ericscript/ericscript_db_homosapiens_ensembl73/data/homo_sapiens/allseq.fa.ann - rnaseq/ericscript/ericscript_db_homosapiens_ensembl73/data/homo_sapiens/EnsemblGene.Reference.fa.bwt - rnaseq/ericscript/ericscript_db_homosapiens_ensembl73/data/homo_sapiens/EnsemblGene.Reference.fa.sa - rnaseq/ericscript/ericscript_db_homosapiens_ensembl73/data/homo_sapiens/EnsemblGene.Transcripts.RData - rnaseq/ericscript/ericscript_db_homosapiens_ensembl73/data/homo_sapiens/EnsemblGene.Reference.fa.ann - rnaseq/ericscript/ericscript_db_homosapiens_ensembl73/data/homo_sapiens/EnsemblGene.Reference.fa.pac - rnaseq/ericscript/ericscript_db_homosapiens_ensembl73/data/homo_sapiens/EnsemblGene.Structures.RData ================================================ FILE: ggd-recipes/GRCh37/esp.yaml ================================================ # Exome Sequencing Project (ESP): http://evs.gs.washington.edu/EVS/ --- attributes: name: esp version: ESP6500SI-V2 recipe: full: recipe_type: bash recipe_cmds: - | baseurl=https://s3.amazonaws.com/gemini-annotations url=$baseurl/ESP6500SI.all.snps_indels.tidy.v2.vcf.gz mkdir -p variation wget -c -O variation/esp.vcf.gz $url wget -c -O variation/esp.vcf.gz.tbi $url.tbi recipe_outfiles: - variation/esp.vcf.gz - variation/esp.vcf.gz.tbi ================================================ FILE: ggd-recipes/GRCh37/exac.yaml ================================================ # ExAC VCF sites: http://exac.broadinstitute.org/ --- attributes: name: exac version: 0.3 recipe: full: recipe_type: bash recipe_cmds: - | baseurl=https://s3.amazonaws.com/gemini-annotations url=$baseurl/ExAC.r0.3.sites.vep.tidy.vcf.gz mkdir -p variation wget -c -O variation/exac.vcf.gz $url wget -c -O variation/exac.vcf.gz.tbi $url.tbi recipe_outfiles: - variation/exac.vcf.gz - variation/exac.vcf.gz.tbi ================================================ FILE: ggd-recipes/GRCh37/fusion-blacklist.yaml ================================================ # fusion blacklist regions from arriba --- attributes: name: fusion-blacklist version: 2 recipe: full: recipe_type: bash recipe_cmds: - | version=v1.1.0 url=https://github.com/suhrig/arriba/releases/download/${version}/arriba_${version}.tar.gz arriba_tarball=arriba_${version}.tar.gz outdir=rnaseq/fusion-blacklist mkdir -p ${outdir} wget --no-check-certificate -c -O ${arriba_tarball} ${url} tar zxvf ${arriba_tarball} mv arriba_${version}/database/blacklist_hg19_hs37d5_GRCh37_2018-11-04.tsv.gz ${outdir}/arriba-blacklist.tsv.gz mv arriba_${version}/database/cytobands_hg19_hs37d5_GRCh37_2018-02-23.tsv ${outdir}/arriba-cytobands.tsv mv arriba_${version}/database/protein_domains_hg19_hs37d5_GRCh37_2018-03-06.gff3 ${outdir}/arriba-protein-domains.gff3 mv arriba_${version}/database/protein_domains_hg19_hs37d5_GRCh37_2018-03-06.gff3.idx ${outdir}/arriba-protein-domains.gff3.idx rm -r arriba_${version} rm ${arriba_tarball} recipe_outfiles: - rnaseq/fusion-blacklist/arriba-blacklist.tsv.gz - rnaseq/fusion-blacklist/arriba-cytobands.tsv - rnaseq/fusion-blacklist/arriba-protein-domains.gff3 - rnaseq/fusion-blacklist/arriba-protein-domains.gff3.idx ================================================ FILE: ggd-recipes/GRCh37/genesplicer.yaml ================================================ --- attributes: name: genesplicer version: 2004.04.03 recipe: full: recipe_type: bash recipe_cmds: - | mkdir -p variation/genesplicer wget ftp://ftp.ccb.jhu.edu/pub/software/genesplicer/GeneSplicer.tar.gz tar -C variation/genesplicer --strip-components=2 -xvf GeneSplicer.tar.gz GeneSplicer/human rm GeneSplicer.tar.gz recipe_outfiles: - variation/genesplicer/acc1.mar - variation/genesplicer/acc14 - variation/genesplicer/acc20 - variation/genesplicer/acc26 - variation/genesplicer/acc33 - variation/genesplicer/acc7 - variation/genesplicer/chgd.fasta - variation/genesplicer/don1.mar - variation/genesplicer/don15 - variation/genesplicer/don19 - variation/genesplicer/don20 - variation/genesplicer/don24 - variation/genesplicer/embls.seq - variation/genesplicer/exons.list - variation/genesplicer/f.don - variation/genesplicer/outin - variation/genesplicer/score_ex.don - variation/genesplicer/score_in.don - variation/genesplicer/acc13 - variation/genesplicer/acc19 - variation/genesplicer/acc25 - variation/genesplicer/acc32 - variation/genesplicer/acc34 - variation/genesplicer/acc8 - variation/genesplicer/config_file - variation/genesplicer/don10 - variation/genesplicer/don16 - variation/genesplicer/don2 - variation/genesplicer/don23 - variation/genesplicer/don9 - variation/genesplicer/exons.dat - variation/genesplicer/f.acc - variation/genesplicer/outex - variation/genesplicer/score_ex.acc - variation/genesplicer/score_in.acc - variation/genesplicer/seqs ================================================ FILE: ggd-recipes/GRCh37/giab-NA12878-NA24385-somatic.yaml ================================================ # Truth set for NA12878/NA24385 somatic mixture # http://ftp-trace.ncbi.nlm.nih.gov/giab/ftp/use_cases/mixtures/UMCUTRECHT_NA12878_NA24385_mixture_10052016/ --- attributes: name: giab-NA12878-NA24385-somatic version: v3_3_2-2018-07-05 recipe: full: recipe_type: bash recipe_cmds: - | dir=validation/giab-NA12878-NA24385-somatic url=https://s3.amazonaws.com/bcbio/giab/NA12878-NA24385/2018-07-05 calls=na12878-na24385-somatic-GRCh37-truth.vcf.gz regions=na12878-na24385-somatic-GRCh37-truth-regions.bed mkdir -p $dir wget -c $url/$calls -O $dir/truth_small_variants.vcf.gz wget -c $url/$calls.tbi -O $dir/truth_small_variants.vcf.gz.tbi wget -c $url/$regions -O $dir/truth_regions.bed recipe_outfiles: - validation/giab-NA12878-NA24385-somatic/truth_small_variants.vcf.gz - validation/giab-NA12878-NA24385-somatic/truth_small_variants.vcf.gz.tbi - validation/giab-NA12878-NA24385-somatic/truth_regions.bed ================================================ FILE: ggd-recipes/GRCh37/giab-NA12878.yaml ================================================ # Truth sets from Genome in a Bottle for NA12878 -- http://genomeinabottle.org/ --- attributes: name: giab-NA12878 version: v3_3_2 recipe: full: recipe_type: bash recipe_cmds: - | dir=validation/giab-NA12878 url=http://ftp-trace.ncbi.nlm.nih.gov/giab/ftp/release/NA12878_HG001/NISTv3.3.2/GRCh37/ calls=HG001_GRCh37_GIAB_highconf_CG-IllFB-IllGATKHC-Ion-10X-SOLID_CHROM1-X_v.3.3.2_highconf_PGandRTGphasetransfer.vcf.gz regions=HG001_GRCh37_GIAB_highconf_CG-IllFB-IllGATKHC-Ion-10X-SOLID_CHROM1-X_v.3.3.2_highconf_nosomaticdel.bed mkdir -p $dir wget -c -O $dir/truth_small_variants.vcf.gz $url/$calls tabix -f -p vcf $dir/truth_small_variants.vcf.gz wget -c -O $dir/truth_regions.bed $url/$regions wget -O - http://ftp.ncbi.nih.gov/giab/ftp/technical/svclassify_Manuscript/Supplementary_Information/Personalis_1000_Genomes_deduplicated_deletions.bed | grep -v ^Chr > $dir/truth_DEL.bed wget -O - http://ftp.ncbi.nih.gov/giab/ftp/technical/svclassify_Manuscript/Supplementary_Information/Spiral_Genetics_insertions.bed | grep -v ^Chr > $dir/truth_INS.bed recipe_outfiles: - validation/giab-NA12878/truth_small_variants.vcf.gz - validation/giab-NA12878/truth_small_variants.vcf.gz.tbi - validation/giab-NA12878/truth_regions.bed - validation/giab-NA12878/truth_DEL.bed - validation/giab-NA12878/truth_INS.bed ================================================ FILE: ggd-recipes/GRCh37/giab-NA24143.yaml ================================================ # Truth sets from Genome in a Bottle for NA24143 -- http://genomeinabottle.org/ --- attributes: name: giab-NA24143 version: v3_3_2 recipe: full: recipe_type: bash recipe_cmds: - | dir=validation/giab-NA24143 url=https://ftp-trace.ncbi.nlm.nih.gov/giab/ftp/release/AshkenazimTrio/HG004_NA24143_mother/NISTv3.3.2/GRCh37/ calls=HG004_GRCh37_GIAB_highconf_CG-IllFB-IllGATKHC-Ion-10X_CHROM1-22_v.3.3.2_highconf.vcf.gz regions=HG004_GRCh37_GIAB_highconf_CG-IllFB-IllGATKHC-Ion-10X_CHROM1-22_v.3.3.2_highconf_noinconsistent.bed mkdir -p $dir wget -c -O $dir/truth_small_variants.vcf.gz $url/$calls tabix -f -p vcf $dir/truth_small_variants.vcf.gz wget -c -O $dir/truth_regions.bed $url/$regions recipe_outfiles: - validation/giab-NA24143/truth_small_variants.vcf.gz - validation/giab-NA24143/truth_small_variants.vcf.gz.tbi - validation/giab-NA24143/truth_regions.bed ================================================ FILE: ggd-recipes/GRCh37/giab-NA24149.yaml ================================================ # Truth sets from Genome in a Bottle for NA24149 -- http://genomeinabottle.org/ --- attributes: name: giab-NA24149 version: v3_3_2 recipe: full: recipe_type: bash recipe_cmds: - | dir=validation/giab-NA24149 url=https://ftp-trace.ncbi.nlm.nih.gov/giab/ftp/release/AshkenazimTrio/HG003_NA24149_father/NISTv3.3.2/GRCh37/ calls=HG003_GRCh37_GIAB_highconf_CG-IllFB-IllGATKHC-Ion-10X_CHROM1-22_v.3.3.2_highconf.vcf.gz regions=HG003_GRCh37_GIAB_highconf_CG-IllFB-IllGATKHC-Ion-10X_CHROM1-22_v.3.3.2_highconf_noinconsistent.bed mkdir -p $dir wget -c -O $dir/truth_small_variants.vcf.gz $url/$calls tabix -f -p vcf $dir/truth_small_variants.vcf.gz wget -c -O $dir/truth_regions.bed $url/$regions recipe_outfiles: - validation/giab-NA24149/truth_small_variants.vcf.gz - validation/giab-NA24149/truth_small_variants.vcf.gz.tbi - validation/giab-NA24149/truth_regions.bed ================================================ FILE: ggd-recipes/GRCh37/giab-NA24385.yaml ================================================ # Truth sets from Genome in a Bottle for NA24385 -- http://genomeinabottle.org/ # Structural variants from # ftp://ftp-trace.ncbi.nlm.nih.gov/giab/ftp/data/AshkenazimTrio/analysis/NIST_UnionSVs_05092017/Preliminary_Integrations_v0.4.0/ --- attributes: name: giab-NA24385 version: v3_3_2-sv_v0.6 recipe: full: recipe_type: bash recipe_cmds: - | dir=validation/giab-NA24385 url=https://ftp-trace.ncbi.nlm.nih.gov/giab/ftp/release/AshkenazimTrio/HG002_NA24385_son/NISTv3.3.2/GRCh37/ calls=HG002_GRCh37_GIAB_highconf_CG-IllFB-IllGATKHC-Ion-10X-SOLID_CHROM1-22_v.3.3.2_highconf_triophased.vcf.gz regions=HG002_GRCh37_GIAB_highconf_CG-IllFB-IllGATKHC-Ion-10X-SOLID_CHROM1-22_v.3.3.2_highconf_noinconsistent.bed mkdir -p $dir wget -c -O $dir/truth_small_variants.vcf.gz $url/$calls tabix -f -p vcf $dir/truth_small_variants.vcf.gz wget -c -O $dir/truth_regions.bed $url/$regions delurl=https://s3.amazonaws.com/bcbio/giab/NA24385/NA24385-GIAB-2tech-dels-v0_1_8-GRCh37.bed.gz wget -c -O $dir/truth_DEL_giab.bed.gz $delurl wget -c -O $dir/truth_DEL_giab.bed.gz.tbi $delurl.tbi cnvurl=https://s3.amazonaws.com/bcbio/giab/NA24385/NA24385-crowd-dels-GRCh37.bed.gz wget -c -O $dir/truth_DEL_crowd.bed.gz $cnvurl wget -c -O $dir/truth_DEL_crowd.bed.gz.tbi $cnvurl.tbi svurl=https://ftp-trace.ncbi.nlm.nih.gov/giab/ftp/data/AshkenazimTrio/analysis/NIST_SVs_Integration_v0.6/HG002_SVs_Tier1_v0.6.vcf.gz svbedurl=https://ftp-trace.ncbi.nlm.nih.gov/giab/ftp/data/AshkenazimTrio/analysis/NIST_SVs_Integration_v0.6/HG002_SVs_Tier1_v0.6.bed wget -c -O $dir/truth_sv.vcf.gz $svurl wget -c -O $dir/truth_sv.vcf.gz.tbi $svurl.tbi wget -c -O $dir/truth_sv_regions.bed $svbedurl recipe_outfiles: - validation/giab-NA24385/truth_small_variants.vcf.gz - validation/giab-NA24385/truth_small_variants.vcf.gz.tbi - validation/giab-NA24385/truth_regions.bed - validation/giab-NA24385/truth_DEL_giab.bed.gz - validation/giab-NA24385/truth_DEL_giab.bed.gz.tbi - validation/giab-NA24385/truth_DEL_crowd.bed.gz - validation/giab-NA24385/truth_DEL_crowd.bed.gz.tbi - validation/giab-NA24385/truth_sv.vcf.gz - validation/giab-NA24385/truth_sv.vcf.gz.tbi ================================================ FILE: ggd-recipes/GRCh37/giab-NA24631.yaml ================================================ # Truth sets from Genome in a Bottle for NA24631 -- http://genomeinabottle.org/ --- attributes: name: giab-NA24631 version: v3_3_2 recipe: full: recipe_type: bash recipe_cmds: - | dir=validation/giab-NA24631 url=http://ftp-trace.ncbi.nlm.nih.gov/giab/ftp/release/ChineseTrio/HG005_NA24631_son/NISTv3.3.2/GRCh37/ calls=HG005_GRCh37_highconf_CG-IllFB-IllGATKHC-Ion-SOLID_CHROM1-22_v.3.3.2_highconf.vcf.gz regions=HG005_GRCh37_highconf_CG-IllFB-IllGATKHC-Ion-SOLID_CHROM1-22_v.3.3.2_highconf_noMetaSV.bed mkdir -p $dir wget -c -O $dir/truth_small_variants.vcf.gz $url/$calls tabix -f -p vcf $dir/truth_small_variants.vcf.gz wget -c -O $dir/truth_regions.bed $url/$regions recipe_outfiles: - validation/giab-NA24631/truth_small_variants.vcf.gz - validation/giab-NA24631/truth_small_variants.vcf.gz.tbi - validation/giab-NA24631/truth_regions.bed ================================================ FILE: ggd-recipes/GRCh37/giab-NA24694.yaml ================================================ # Truth sets from Genome in a Bottle for NA24694 -- http://genomeinabottle.org/ --- attributes: name: giab-NA24694 version: v3_3_2 recipe: full: recipe_type: bash recipe_cmds: - | dir=validation/giab-NA24694 url=https://ftp-trace.ncbi.nlm.nih.gov/giab/ftp/release/ChineseTrio/HG006_NA24694_father/NISTv3.3.2/GRCh37/ calls=HG006_GIAB_GRCh37_highconf_CG-IllFB-IllSNT-10X_CHROM1-22_v.3.3.2_highconf.vcf.gz regions=HG006_GIAB_GRCh37_highconf_CG-IllFB-IllSNT-10X_CHROM1-22_v.3.3.2_highconf_noinconsistent.bed mkdir -p $dir wget -c $url/$calls -O $dir/truth_small_variants.vcf.gz tabix -f -p vcf $dir/truth_small_variants.vcf.gz wget -c $url/$regions -O $dir/truth_regions.bed recipe_outfiles: - validation/giab-NA24694/truth_small_variants.vcf.gz - validation/giab-NA24694/truth_small_variants.vcf.gz.tbi - validation/giab-NA24694/truth_regions.bed ================================================ FILE: ggd-recipes/GRCh37/giab-NA24695.yaml ================================================ # Truth sets from Genome in a Bottle for NA24695 -- http://genomeinabottle.org/ --- attributes: name: giab-NA24695 version: v3_3_2 recipe: full: recipe_type: bash recipe_cmds: - | dir=validation/giab-NA24695 url=https://ftp-trace.ncbi.nlm.nih.gov/giab/ftp/release/ChineseTrio/HG007_NA24695_mother/NISTv3.3.2/GRCh37/ calls=HG007_GIAB_GRCh37_highconf_CG-IllFB-IllSNT-10X_CHROM1-22_v.3.3.2_highconf.vcf.gz regions=HG007_GIAB_GRCh37_highconf_CG-IllFB-IllSNT-10X_CHROM1-22_v.3.3.2_highconf_noinconsistent.bed mkdir -p $dir wget -c $url/$calls -O $dir/truth_small_variants.vcf.gz tabix -f -p vcf $dir/truth_small_variants.vcf.gz wget -c $url/$regions -O $dir/truth_regions.bed recipe_outfiles: - validation/giab-NA24695/truth_small_variants.vcf.gz - validation/giab-NA24695/truth_small_variants.vcf.gz.tbi - validation/giab-NA24695/truth_regions.bed ================================================ FILE: ggd-recipes/GRCh37/gnomad.yaml ================================================ # GnomAD VCF sites: # - https://gnomad.broadinstitute.org/downloads # - https://macarthurlab.org/2018/10/17/gnomad-v2-1/ # - http://ftp.ensemblorg.ebi.ac.uk/pub/data_files/homo_sapiens/GRCh37/variation_genotype/gnomad/r2.1/genomes/ # Script # - sorts according to the reference file # - no need in decomposing (it is done in gnomad genome 2.1), but normalization and uniq are needed # - filters only PASS variants (segdup, decoy are retained) # - removes many INFO fields to reduce file size: controls_, hist, non_, using bcftools annotate instead of vt rminfo, because it failed randomly on chr1 # - merges all chr into a single file # wget is separated from the processing because wget in the pipe fails randomly (on some systems?): it comes out when downloading many files. --- attributes: name: gnomad version: 2.1.1 recipe: full: recipe_type: bash recipe_cmds: - | url_prefix=https://storage.googleapis.com/gnomad-public/release/2.1.1/vcf/genomes/ vcf_file=gnomad.genomes.r2.1.1.sites.vcf.bgz ref=../seq/GRCh37.fa mkdir -p variation export TMPDIR=`pwd` gnomad_fields_to_keep_url=https://gist.githubusercontent.com/naumenko-sa/d20db928b915a87bba4012ba1b89d924/raw/cf343b105cb3347e966cc95d049e364528c86880/gnomad_fields_to_keep.txt wget --no-check-certificate -c $gnomad_fields_to_keep_url vcf=${url_prefix}${vcf_file} wget --no-check-certificate -c $vcf wget --no-check-certificate -c $vcf.tbi fields_to_keep="INFO/"$(cat gnomad_fields_to_keep.txt | paste -s | sed s/"\t"/",INFO\/"/g) bcftools view -f PASS $vcf_file | bcftools annotate -x "^$fields_to_keep" -Ov | vt decompose -s - | vt normalize -r $ref -n - | vt uniq - | bgzip -c > variation/gnomad_genome.vcf.gz tabix -f -p vcf variation/gnomad_genome.vcf.gz tabix -f -p vcf --csi variation/gnomad_genome.vcf.gz rm $vcf_file $vcf_file.tbi recipe_outfiles: - variation/gnomad_genome.vcf.gz - variation/gnomad_genome.vcf.gz.tbi - variation/gnomad_genome.vcf.gz.csi ================================================ FILE: ggd-recipes/GRCh37/gnomad_exome.yaml ================================================ # GnomAD VCF sites: # - https://gnomad.broadinstitute.org/downloads # - https://macarthurlab.org/2018/10/17/gnomad-v2-1/ # Script # - no need in decomposing, normalizing,sort, and uniq anymore - it is done in gnomad 2.1.1 exomes # - filters only PASS variants (segdup, decoy are retained) # - removes many INFO fields to reduce file size: controls_, hist, non_, using bcftools annotate instead of vt rminfo, because it failed randomly on chr1 # recipe test: https://github.com/chapmanb/cloudbiolinux/blob/master/doc/hacking.md#testing-a-ggd-recipe # much faster from gnomad in google cloud - downloads in 20-25 min on O2 # vt is 3x faster than bcftools for removing INFO tags: in a test with 4444 variants t(bcftools -x) = t(bcftools -x "^") = 27s, time(vt rminfo) = 8s, # but vt rminfo is failing after 148M, using bcftools # overall processing is still slow: 87G to G in --- attributes: name: gnomad_exome version: 2.1.1 recipe: full: recipe_type: bash recipe_cmds: - | vcf_prefix=https://storage.googleapis.com/gnomad-public/release/2.1.1/vcf/exomes/ vcf_file=gnomad.exomes.r2.1.1.sites.vcf.bgz ref=../seq/GRCh37.fa mkdir -p variation export TMPDIR=`pwd` gnomad_fields_to_keep_url=https://gist.githubusercontent.com/naumenko-sa/d20db928b915a87bba4012ba1b89d924/raw/cf343b105cb3347e966cc95d049e364528c86880/gnomad_fields_to_keep.txt wget --no-check-certificate -c $gnomad_fields_to_keep_url vcf=${vcf_prefix}${vcf_file} wget --no-check-certificate -c $vcf wget --no-check-certificate -c $vcf.tbi fields_to_keep="INFO/"$(cat gnomad_fields_to_keep.txt | paste -s | sed s/"\t"/",INFO\/"/g) bcftools view -f PASS $vcf_file | bcftools annotate -x "^$fields_to_keep" -Ov | bgzip -c > variation/gnomad_exome.vcf.gz tabix -f -p vcf variation/gnomad_exome.vcf.gz tabix -f -p vcf --csi variation/gnomad_exome.vcf.gz rm $vcf_file $vcf_file.tbi recipe_outfiles: - variation/gnomad_exome.vcf.gz - variation/gnomad_exome.vcf.gz.tbi - variation/gnomad_exome.vcf.gz.csi ================================================ FILE: ggd-recipes/GRCh37/gnomad_sv.yaml ================================================ --- attributes: name: gnomad_sv version: 2 recipe: full: recipe_type: bash recipe_cmds: - | baseurl=https://storage.googleapis.com/gnomad-public/papers/2019-sv/gnomad_v2_sv.sites mkdir -p variation cd variation wget -O - $baseurl.vcf.gz | gunzip -c | bgzip -c > gnomad_sv.vcf.gz tabix -f -p vcf gnomad_sv.vcf.gz wget -O - $baseurl.bed.gz | gunzip -c | bgzip -c > gnomad_sv.bed.gz tabix -f -p bed gnomad_sv.bed.gz recipe_outfiles: - variation/gnomad_sv.vcf.gz - variation/gnomad_sv.vcf.gz.tbi - variation/gnomad_sv.bed.gz - variation/gnomad_sv.bed.gz.tbi ================================================ FILE: ggd-recipes/GRCh37/hapmap.yaml ================================================ --- attributes: name: hapmap version: 3.3 recipe: full: recipe_type: bash recipe_cmds: - | baseurl=ftp://gsapubftp-anonymous:none@ftp.broadinstitute.org/bundle/b37/hapmap_3.3.b37.vcf.gz mkdir -p variation cd variation wget -O - $baseurl | gunzip -c | bgzip -c > hapmap_3.3.vcf.gz tabix -f -p vcf hapmap_3.3.vcf.gz recipe_outfiles: - variation/hapmap_3.3.vcf.gz - variation/hapmap_3.3.vcf.gz.tbi ================================================ FILE: ggd-recipes/GRCh37/mills_indels.yaml ================================================ --- attributes: name: mills_indels version: 2.8 recipe: full: recipe_type: bash recipe_cmds: - | baseurl=ftp://gsapubftp-anonymous:none@ftp.broadinstitute.org/bundle/b37/Mills_and_1000G_gold_standard.indels.b37.vcf.gz mkdir -p variation cd variation wget -O - $baseurl | gunzip -c | bgzip -c > Mills_and_1000G_gold_standard.indels.vcf.gz tabix -f -p vcf Mills_and_1000G_gold_standard.indels.vcf.gz recipe_outfiles: - variation/Mills_and_1000G_gold_standard.indels.vcf.gz - variation/Mills_and_1000G_gold_standard.indels.vcf.gz.tbi ================================================ FILE: ggd-recipes/GRCh37/prioritize.yaml ================================================ # Regions for prioritization: # cancer # - civic -- known cancer genes from the CIViC database # - az300 -- list of 300 cancer genes of interest from AstraZeneca Oncology # - az300-with-fusion -- az300 list plus known interacting fusions # from FusionCatcher https://github.com/ndaniel/fusioncatcher/blob/master/bin/generate_known.py#L102 # - az-cancer-panel -- az300 genes plus additional useful panel genes --- attributes: name: prioritize version: 20181227 recipe: full: recipe_type: bash recipe_cmds: - | baseurl=https://s3.amazonaws.com/biodata/coverage/prioritize/prioritize-cancer-GRCh37-20181227.tar.gz outdir=coverage/prioritize mkdir -p $outdir cd $outdir wget --no-check-certificate -c -O cancer.tar.gz $baseurl tar -xzvpf cancer.tar.gz recipe_outfiles: - coverage/prioritize/cancer/az300.bed.gz - coverage/prioritize/cancer/az300.bed.gz.tbi - coverage/prioritize/cancer/az300-with-fusion.bed.gz - coverage/prioritize/cancer/az300-with-fusion.bed.gz.tbi - coverage/prioritize/cancer/az-cancer-panel.bed.gz - coverage/prioritize/cancer/az-cancer-panel.bed.gz.tbi - coverage/prioritize/cancer/civic-2018-12-27.bed.gz - coverage/prioritize/cancer/civic-2018-12-27.bed.gz.tbi ================================================ FILE: ggd-recipes/GRCh37/qsignature.yaml ================================================ --- attributes: name: qsignature version: 20140703 recipe: full: recipe_type: bash recipe_cmds: - | baseurl=http://downloads.sourceforge.net/project/adamajava/qsignature.tar.bz2 mkdir -p variation cd variation wget -N -c $baseurl tar -xjf qsignature.tar.bz2 qsignature_positions.txt mv qsignature_positions.txt qsignature.vcf recipe_outfiles: - variation/qsignature.vcf ================================================ FILE: ggd-recipes/GRCh37/seq.yaml ================================================ # GRCh37 reference genome from Broad bundle # ftp://gsapubftp-anonymous:none@ftp.broadinstitute.org/bundle/b37 # Include bgzipped fasta file for Ensembl and other tools that support it --- attributes: name: seq version: broad-20120813 recipe: full: recipe_type: bash recipe_cmds: - | wget --no-check-certificate -c https://s3.amazonaws.com/biodata/genomes/GRCh37-seq.tar.gz tar -xzvpf GRCh37-seq.tar.gz gunzip -c seq/GRCh37.fa.gz > seq/GRCh37.fa touch seq/GRCh37.fa.fai touch seq/GRCh37.dict recipe_outfiles: - seq/GRCh37.fa - seq/GRCh37.fa.fai - seq/GRCh37.fa.gz - seq/GRCh37.fa.gz.fai - seq/GRCh37.fa.gz.gzi - seq/GRCh37.dict ================================================ FILE: ggd-recipes/GRCh37/topmed.yaml ================================================ # Topmed WGS allele frequencies: https://bravo.sph.umich.edu/freeze3a/hg19/ #url=ftp://ftp.ensembl.org/pub/data_files/homo_sapiens/GRCh37/variation_genotype/TOPMED_GRCh37.vcf.gz #ref=GRCh37.fa #mkdir -p variation #export TMPDIR=`pwd` #gunzip -c TOPMED_GRCh37.vcf.gz | sed s/Number=\./Number=A/g | vt normalize -r $ref -n - | vt uniq - | gsort -m 3000 /dev/stdin $ref.fai | bgzip -c > variation/topmed-freeze3a.vcf.gz #tabix -f -p vcf variation/topmed-freeze3a.vcf.gz --- attributes: name: topmed version: freeze3a recipe: full: recipe_type: bash recipe_cmds: - | url=http://s3.amazonaws.com/bcbio-nextgen/genomes/Hsapiens/GRCh37/variation/topmed-freeze3a.vcf.gz mkdir -p variation wget -c -O - ${url} > variation/topmed-freeze3a.vcf.gz url=http://s3.amazonaws.com/bcbio-nextgen/genomes/Hsapiens/GRCh37/variation/topmed-freeze3a.vcf.gz.tbi wget -c -O - ${url} > variation/topmed-freeze3a.vcf.gz.tbi rm -f variation/topmed.vcf.gz rm -f variation/topmed.vcf.gz.tbi cd variation ln -s topmed-freeze3a.vcf.gz topmed.vcf.gz ln -s topmed-freeze3a.vcf.gz.tbi topmed.vcf.gz.tbi recipe_outfiles: - variation/topmed.vcf.gz - variation/topmed.vcf.gz.tbi - variation/topmed-freeze3a.vcf.gz - variation/topmed-freeze3a.vcf.gz.tbi ================================================ FILE: ggd-recipes/GRCh37/transcripts.yaml ================================================ # Transcript, annotation and indices for RNA-seq analysis # Prepared from Ensembl transcripts using: # https://github.com/chapmanb/cloudbiolinux/blob/master/utils/prepare_tx_gff.py --- attributes: name: transcripts version: 2015-12-01 recipe: full: recipe_type: bash recipe_cmds: - | baseurl=https://s3.amazonaws.com/biodata/annotation/GRCh37-rnaseq-2015-12-01.tar.xz wget -c -N --random-wait --retry-connrefused --no-check-certificate $baseurl xz -dc *-rnaseq-*.tar.xz | tar --no-same-owner -xpf - mv */rnaseq-* rnaseq recipe_outfiles: - rnaseq/kallisto - rnaseq/tophat - rnaseq/ref-transcripts.bed - rnaseq/ref-transcripts.dexseq.gff3 - rnaseq/ref-transcripts.fa - rnaseq/ref-transcripts.genePred - rnaseq/ref-transcripts.gtf - rnaseq/ref-transcripts.gtf.db - rnaseq/ref-transcripts-mask.gtf - rnaseq/ref-transcripts.refFlat - rnaseq/ref-transcripts-splicesites.txt - rnaseq/rRNA.gtf - rnaseq/rRNA.gtf.db - rnaseq/rRNA.interval_list ================================================ FILE: ggd-recipes/GRCh37/twobit.yaml ================================================ # GRCh37 UCSC 2bit reference file --- attributes: name: twobit version: broad-20120813 recipe: full: recipe_type: bash recipe_cmds: - | mkdir -p ucsc cd ucsc faToTwoBit ../../seq/GRCh37.fa GRCh37.2bit recipe_outfiles: - ucsc/GRCh37.2bit ================================================ FILE: ggd-recipes/GRCh37/varpon.yaml ================================================ # Panels of normals (PONs) for variant calling # germline_het_pon -- Locations of likely germline heterogygote sites --- attributes: name: varpon version: 20181105 recipe: full: recipe_type: bash recipe_cmds: - | mkdir -p variation wget --no-check-certificate -O - 'https://s3.amazonaws.com/biodata/collections/GRCh37/variation/germline_het_pon.bed.gz' > variation/germline_het_pon.bed.gz wget --no-check-certificate -O - 'https://s3.amazonaws.com/biodata/collections/GRCh37/variation/germline_het_pon.bed.gz.tbi' > variation/germline_het_pon.bed.gz.tbi recipe_outfiles: - variation/germline_het_pon.bed.gz - variation/germline_het_pon.bed.gz.tbi ================================================ FILE: ggd-recipes/GRCh37/vcfanno.yaml ================================================ # Annotation files for vcfanno variant annotation # Feeds into vcf2db GEMINI database creation and other annotation tasks --- attributes: name: vcfanno version: 20210204 recipe: full: recipe_type: bash recipe_cmds: - | mkdir -p config/vcfanno cd config/vcfanno bcbio_baseurl=https://raw.githubusercontent.com/bcbio/bcbio-nextgen/master/config/vcfanno wget --no-check-certificate -O gemini.conf -c $bcbio_baseurl/GRCh37-gemini.conf wget --no-check-certificate -O gemini.lua -c $bcbio_baseurl/GRCh37-gemini.lua wget --no-check-certificate -O rnaedit.conf -c $bcbio_baseurl/rnaedit.conf wget --no-check-certificate -O rnaedit.lua -c $bcbio_baseurl/rnaedit.lua wget --no-check-certificate -O dbnsfp.conf -c $bcbio_baseurl/GRCh37-dbnsfp.conf wget --no-check-certificate -O dbscsnv.conf -c $bcbio_baseurl/GRCh37-dbscsnv.conf wget --no-check-certificate -O gnomad_genome.conf -c $bcbio_baseurl/GRCh37-gnomad_genome.conf wget --no-check-certificate -O somatic.conf -c $bcbio_baseurl/somatic.conf recipe_outfiles: - config/vcfanno/gemini.lua - config/vcfanno/gemini.conf - config/vcfanno/rnaedit.lua - config/vcfanno/rnaedit.conf - config/vcfanno/dbnsfp.conf - config/vcfanno/dbscsnv.conf - config/vcfanno/gnomad_genome.conf - config/vcfanno/somatic.conf ================================================ FILE: ggd-recipes/GRCh37/viral.yaml ================================================ # Virus reference genomes, prepared for bwa alignment # Taken from TCGA GDC reference genome # https://gdc.cancer.gov/about-data/data-harmonization-and-generation/gdc-reference-files --- attributes: name: viral version: 2019.12.03 recipe: full: recipe_type: bash recipe_cmds: - | url=http://s3.amazonaws.com/biodata/viral/GRCh37-gdc-viral.tar.gz mkdir -p viral wget --no-check-certificate -c $url tar -xzvpf *.tar.gz mv *-gdc-viral/* viral recipe_outfiles: - viral/gdc-viral.fa - viral/gdc-viral.dict - viral/gdc-viral.fa.ann - viral/gdc-viral.fa.fai - viral/gdc-viral.fa.pac - viral/gdc-viral.fa.bwt - viral/gdc-viral.fa.amb - viral/gdc-viral.fa.sa ================================================ FILE: ggd-recipes/GRCz11/seq.yaml ================================================ --- attributes: name: seq version: 92 recipe: full: recipe_type: bash recipe_cmds: - | BUILD=GRCz11 url=http://ftp.ensembl.org/pub/release-92/fasta/danio_rerio/dna/Danio_rerio.GRCz11.dna.toplevel.fa.gz mkdir -p seq wget --no-check-certificate -O - $url | gunzip -c > seq/$BUILD.fa samtools faidx seq/$BUILD.fa picard -Xms1g -Xmx3g CreateSequenceDictionary REFERENCE=seq/$BUILD.fa OUTPUT=seq/$BUILD.dict SPECIES=$BUILD recipe_outfiles: - seq/GRCz11.fa - seq/GRCz11.fa.fai - seq/GRCz11.dict ================================================ FILE: ggd-recipes/GRCz11/transcripts.yaml ================================================ # Transcript, annotation and indices for RNA-seq analysis # Prepared from Ensembl transcripts using: # https://github.com/chapmanb/cloudbiolinux/blob/master/utils/prepare_tx_gff.py --- attributes: name: transcripts version: 2018-05-15_92 recipe: full: recipe_type: bash recipe_cmds: - | baseurl=https://s3.amazonaws.com/biodata/annotation/GRCz11-rnaseq-2018-05-15_92.tar.xz wget -c -N --no-check-certificate $baseurl xz -dc *-rnaseq-*.tar.xz | tar -xpf - mv */rnaseq-* rnaseq recipe_outfiles: - rnaseq/ref-transcripts.bed - rnaseq/ref-transcripts.dexseq.gff3 - rnaseq/ref-transcripts.fa - rnaseq/ref-transcripts.genePred - rnaseq/ref-transcripts.gtf - rnaseq/ref-transcripts.gtf.db - rnaseq/ref-transcripts-mask.gtf - rnaseq/ref-transcripts.refFlat - rnaseq/ref-transcripts-splicesites.txt - rnaseq/rRNA.gtf - rnaseq/rRNA.gtf.db - rnaseq/rRNA.interval_list - rnaseq/tx2gene.csv - rnaseq/version.txt ================================================ FILE: ggd-recipes/GRCz11/twobit.yaml ================================================ --- attributes: name: twobit version: 92 recipe: full: recipe_type: bash recipe_cmds: - | mkdir -p ucsc cd ucsc faToTwoBit -long ../../seq/GRCz11.fa GRCz11.2bit recipe_outfiles: - ucsc/GRCz11.2bit ================================================ FILE: ggd-recipes/README.md ================================================ Recipes for downloading data Idea developed from Aaron Quinlan's GGD (Get Genomic Data) https://github.com/arq5x/ggd https://github.com/arq5x/ggd-recipes ================================================ FILE: ggd-recipes/Sscrofa11.1/seq.yaml ================================================ --- attributes: name: seq version: 92 recipe: full: recipe_type: bash recipe_cmds: - | BUILD=Sscrofa11.1 url=http://ftp.ensembl.org/pub/release-92/fasta/sus_scrofa/dna/Sus_scrofa.Sscrofa11.1.dna_rm.toplevel.fa.gz mkdir -p seq wget --no-check-certificate -O - $url | gunzip -c > seq/$BUILD.fa samtools faidx seq/$BUILD.fa picard -Xms1g -Xmx3g CreateSequenceDictionary REFERENCE=seq/$BUILD.fa OUTPUT=seq/$BUILD.dict SPECIES=$BUILD recipe_outfiles: - seq/Sscrofa11.1.fa - seq/Sscrofa11.1.fa.fai - seq/Sscrofa11.1.dict ================================================ FILE: ggd-recipes/Sscrofa11.1/transcripts.yaml ================================================ # Transcript, annotation and indices for RNA-seq analysis # Prepared from Ensembl transcripts using: # https://github.com/chapmanb/cloudbiolinux/blob/master/utils/prepare_tx_gff.py --- attributes: name: transcripts version: 2018-05-14_92 recipe: full: recipe_type: bash recipe_cmds: - | baseurl=https://s3.amazonaws.com/biodata/annotation/Sscrofa11.1-rnaseq-2018-05-14_92.tar.xz wget -c -N --no-check-certificate $baseurl xz -dc *-rnaseq-*.tar.xz | tar -xpf - mv */rnaseq-* rnaseq recipe_outfiles: - rnaseq/ref-transcripts.bed - rnaseq/ref-transcripts.dexseq.gff3 - rnaseq/ref-transcripts.fa - rnaseq/ref-transcripts.genePred - rnaseq/ref-transcripts.gtf - rnaseq/ref-transcripts.gtf.db - rnaseq/ref-transcripts-mask.gtf - rnaseq/ref-transcripts.refFlat - rnaseq/ref-transcripts-splicesites.txt - rnaseq/rRNA.gtf - rnaseq/rRNA.gtf.db - rnaseq/rRNA.interval_list - rnaseq/tx2gene.csv - rnaseq/version.txt ================================================ FILE: ggd-recipes/Sscrofa11.1/twobit.yaml ================================================ --- attributes: name: twobit version: 92 recipe: full: recipe_type: bash recipe_cmds: - | mkdir -p ucsc cd ucsc faToTwoBit ../../seq/Sscrofa11.1.fa Sscrofa11.1.2bit recipe_outfiles: - ucsc/Sscrofa11.1.2bit ================================================ FILE: ggd-recipes/TAIR10/mirbase.yaml ================================================ # small RNA seq annotation from different sources --- attributes: name: srnaseq version: 20160603 recipe: full: recipe_type: bash recipe_cmds: - | mkdir -p srnaseq cd srnaseq wget -c -O tmp.gtf.gz ftp://ftp.ncbi.nlm.nih.gov/genomes/all/GCF/000/001/735/GCF_000001735.3_TAIR10/GCF_000001735.3_TAIR10_genomic.gff.gz zgrep -v exon tmp.gtf.gz | grep -v region | sed 's/Name/name/g' | sed 's/=/ /g' > srna-transcripts.gtf wget -c -O hairpin.t.fa.gz ftp://mirbase.org/pub/mirbase/21/hairpin.fa.gz && gunzip -f hairpin.t.fa.gz cat hairpin.t.fa | awk '{if ($0~/>ath/){name=$0; print name} else if ($0~/^>/){name=0};if (name!=0 && $0!~/^>/){print $0;}}' | sed 's/U/T/g' > hairpin.fa wget -c -O mature.t.fa.gz ftp://mirbase.org/pub/mirbase/21/mature.fa.gz && gunzip -f mature.t.fa.gz cat mature.t.fa | awk '{if ($0~/>ath/){name=$0; print name} else if ($0~/^>/){name=0};if (name!=0 && $0!~/^>/){print $0;}}' | sed 's/U/T/g' > mature.fa wget -c -O miRNA.t.str.gz ftp://mirbase.org/pub/mirbase/21/miRNA.str.gz && gunzip -f miRNA.t.str.gz cat miRNA.t.str | awk '{if ($0~/ath/)print $0}' > miRNA.str wget --no-check-certificate -c -O Rfam_for_miRDeep.fa.gz https://github.com/lpantano/mirdeep2_core/raw/data/Rfam_for_miRDeep.fa.gz && gunzip -f Rfam_for_miRDeep.fa.gz recipe_outfiles: - srnaseq/srna-transcripts.gtf - srnaseq/hairpin.fa - srnaseq/mature.fa - srnaseq/miRNA.str - srnaseq/Rfam_for_miRDeep.fa ================================================ FILE: ggd-recipes/canFam3/dbsnp.yaml ================================================ --- attributes: name: dbsnp version: 20140510 recipe: full: recipe_type: bash recipe_cmds: - | baseurl=https://s3.amazonaws.com/biodata/variants/canFam3-dbSNP-2014-05-10.vcf.gz mkdir -p variation cd variation wget -c -N $baseurl wget -c -N $baseurl.tbi recipe_outfiles: - variation/canFam3-dbSNP-2014-05-10.vcf.gz - variation/canFam3-dbSNP-2014-05-10.vcf.gz.tbi ================================================ FILE: ggd-recipes/canFam3/mirbase.yaml ================================================ # small RNA seq annotation from different sources --- attributes: name: srnaseq version: 20181024 recipe: full: recipe_type: bash recipe_cmds: - | mkdir -p srnaseq cd srnaseq wget --random-wait --retry-connrefused -q -c -O cfa.gff3 https://mirbase.org/download/cfa.gff3 awk '$3=="miRNA"' cfa.gff3 | sed 's/=/ /g' > srna-transcripts.gtf wget --random-wait --retry-connrefused -q -c -O rmsk.txt.gz http://hgdownload.soe.ucsc.edu/goldenPath/canFam3/database/rmsk.txt.gz zcat rmsk.txt.gz | awk '{print $6"\t.\trepeat\t"$7+1"\t"$8+1"\t.\t"$10"\t.\tname "$12";"}' >> srna-transcripts.gtf wget --random-wait --retry-connrefused -q -c -O refGene.txt.gz http://hgdownload.soe.ucsc.edu/goldenPath/canFam3/database/refGene.txt.gz zcat refGene.txt.gz | awk '{print $3"\t.\tgene\t"$5"\t"$6"\t.\t"$4"\t.\tname "$13";"}' >> srna-transcripts.gtf # mirbase wget --random-wait --retry-connrefused -q -c -O hairpin.fa.raw https://mirbase.org/download/CURRENT/hairpin.fa cat hairpin.fa.raw | awk '{if ($0~/>cfa/){name=$0; print name} else if ($0~/^>/){name=0};if (name!=0 && $0!~/^>/){print $0;}}' | sed 's/U/T/g' > hairpin.fa rm hairpin.fa.raw wget --random-wait --retry-connrefused -q -c -O mature.fa.raw https://mirbase.org/download/CURRENT/mature.fa cat mature.fa.raw | awk '{if ($0~/>cfa/){name=$0; print name} else if ($0~/^>/){name=0};if (name!=0 && $0!~/^>/){print $0;}}' | sed 's/U/T/g' > mature.fa rm mature.fa.raw wget --random-wait --retry-connrefused -q -c -O miRNA.dat.raw https://mirbase.org/download/CURRENT/miRNA.dat cat miRNA.dat.raw | awk '{if ($0~/cfa/)print $0}' > miRNA.str rm miRNA.dat.raw wget --random-wait --retry-connrefused -q -c -O mirbase.gff3 https://mirbase.org/download/cfa.gff3 # mintmap # mirdeep2 wget --random-wait --retry-connrefused --no-check-certificate -q -c -O Rfam_for_miRDeep.fa.gz https://github.com/lpantano/mirdeep2_core/raw/data/Rfam_for_miRDeep.fa.gz && gunzip Rfam_for_miRDeep.fa.gz # targetscan analysis recipe_outfiles: - srnaseq/srna-transcripts.gtf - srnaseq/hairpin.fa - srnaseq/mature.fa - srnaseq/miRNA.str - srnaseq/mirbase.gff3 - srnaseq/Rfam_for_miRDeep.fa ================================================ FILE: ggd-recipes/canFam3/transcripts.yaml ================================================ # Transcript, annotation and indices for RNA-seq analysis # Prepared from Ensembl transcripts using: # https://github.com/chapmanb/cloudbiolinux/blob/master/utils/prepare_tx_gff.py --- attributes: name: transcripts version: 2018-03-22_91 recipe: full: recipe_type: bash recipe_cmds: - | baseurl=https://s3.amazonaws.com/biodata/annotation/canFam3-rnaseq-2018-03-22_91.tar.xz wget -c -N --no-check-certificate $baseurl xz -dc *-rnaseq-*.tar.xz | tar -xpf - mv */rnaseq-* rnaseq recipe_outfiles: - rnaseq/ref-transcripts.bed - rnaseq/ref-transcripts.dexseq.gff3 - rnaseq/ref-transcripts.fa - rnaseq/ref-transcripts.genePred - rnaseq/ref-transcripts.gtf - rnaseq/ref-transcripts.gtf.db - rnaseq/ref-transcripts-mask.gtf - rnaseq/ref-transcripts.refFlat - rnaseq/ref-transcripts-splicesites.txt - rnaseq/rRNA.gtf - rnaseq/rRNA.gtf.db - rnaseq/rRNA.interval_list - rnaseq/tx2gene.csv - rnaseq/version.txt ================================================ FILE: ggd-recipes/canFam3/twobit.yaml ================================================ # UCSC twobit reference --- attributes: name: twobit version: canFam3 recipe: full: recipe_type: bash recipe_cmds: - | mkdir -p ucsc cd ucsc faToTwoBit ../../seq/canFam3.fa canFam3.2bit recipe_outfiles: - ucsc/canFam3.2bit ================================================ FILE: ggd-recipes/hg19/1000g.yaml ================================================ # 1000 genomes --- attributes: name: 1000g version: phase3_shapeit2_mvncall_integrated_v5a.20130502 recipe: full: recipe_type: bash recipe_cmds: - | baseurl=https://s3.amazonaws.com/gemini-annotations url=$baseurl/ALL.wgs.phase3_shapeit2_mvncall_integrated_v5a.20130502.sites.tidy.vcf.gz remap_url=https://raw.githubusercontent.com/dpryan79/ChromosomeMappings/master/GRCh37_ensembl2UCSC.txt mkdir -p variation wget --no-check-certificate -qO- $remap_url | awk '{if($1!=$2) print "s/^"$1"/"$2"/g"}' > remap.sed export TMPDIR=`pwd` wget -c -O - $url | gunzip -c | sed -f remap.sed | bgzip -c > variation/1000g.vcf.gz tabix -f -p vcf variation/1000g.vcf.gz recipe_outfiles: - variation/1000g.vcf.gz - variation/1000g.vcf.gz.tbi ================================================ FILE: ggd-recipes/hg19/1000g_omni_snps.yaml ================================================ --- attributes: name: 1000g_omni_snps version: 2.5 recipe: full: recipe_type: bash recipe_cmds: - | baseurl=ftp://gsapubftp-anonymous:none@ftp.broadinstitute.org/bundle/hg19/1000G_omni2.5.hg19.sites.vcf.gz mkdir -p variation cd variation wget -O - $baseurl | gunzip -c | bgzip -c > 1000G_omni2.5.vcf.gz tabix -f -p vcf 1000G_omni2.5.vcf.gz recipe_outfiles: - variation/1000G_omni2.5.vcf.gz - variation/1000G_omni2.5.vcf.gz.tbi ================================================ FILE: ggd-recipes/hg19/1000g_snps.yaml ================================================ --- attributes: name: 1000g_snps version: 2.8 recipe: full: recipe_type: bash recipe_cmds: - | baseurl=ftp://gsapubftp-anonymous:none@ftp.broadinstitute.org/bundle/hg19/1000G_phase1.snps.high_confidence.hg19.sites.vcf.gz mkdir -p variation cd variation wget -O - $baseurl | gunzip -c | bgzip -c > 1000G_phase1.snps.high_confidence.vcf.gz tabix -f -p vcf 1000G_phase1.snps.high_confidence.vcf.gz recipe_outfiles: - variation/1000G_phase1.snps.high_confidence.vcf.gz - variation/1000G_phase1.snps.high_confidence.vcf.gz.tbi ================================================ FILE: ggd-recipes/hg19/ACMG56_genes.yaml ================================================ # ACMG regions: http://iobio.io/2016/03/29/acmg56/ # https://github.com/lpantano/custom_genome_annotation --- attributes: name: ACMG56_genes version: 20160629 recipe: full: recipe_type: bash recipe_cmds: - | baseurl=https://raw.githubusercontent.com/lpantano/custom_genome_annotation/master/acmg56.ucsc.bed dir=coverage/prioritize/actionable mkdir -p $dir cd $dir wget --no-check-certificate -c -O ACMG56.bed $baseurl recipe_outfiles: - coverage/prioritize/actionable/ACMG56.bed ================================================ FILE: ggd-recipes/hg19/GA4GH_problem_regions.yaml ================================================ # GA4GH problematic region files: https://drive.google.com/folderview?id=0B7Ao1qqJJDHQUjVIN3liUUZNWjg&usp=sharing#list # Cached mappability regions # wget --no-check-certificate -O - http://hgdownload.cse.ucsc.edu/goldenPath/hg19/encodeDCC/wgEncodeMapability/wgEncodeDacMapabilityConsensusExcludable.bed.gz | gunzip -c | bgzip -c > $encode/wgEncodeDacMapabilityConsensusExcludable.bed.gz # tabix -p bed -f $encode/wgEncodeDacMapabilityConsensusExcludable.bed.gz --- attributes: name: GA4GH_problem_regions version: 20181016 recipe: full: recipe_type: bash recipe_cmds: - | baseurl=http://bcbio_nextgen.s3.amazonaws.com/GA4GH_problem_regions.zip mkdir -p coverage/problem_regions/GA4GH cd coverage/problem_regions/GA4GH wget --no-check-certificate -c -O GA4GH_problem_regions.zip $baseurl unzip -o GA4GH_problem_regions.zip for file in *.bed do bgzip $file tabix -p bed $file.gz done cd ../../.. encode=coverage/problem_regions/ENCODE mkdir -p $encode cd $encode wget --no-check-certificate http://s3.amazonaws.com/biodata/coverage/problem_regions/ENCODE/hg19/wgEncodeDacMapabilityConsensusExcludable.bed.gz wget --no-check-certificate http://s3.amazonaws.com/biodata/coverage/problem_regions/ENCODE/hg19/wgEncodeDacMapabilityConsensusExcludable.bed.gz.tbi cd ../../.. repeats=coverage/problem_regions/repeats mkdir -p $repeats wget --no-check-certificate -O - https://raw.githubusercontent.com/chapmanb/delly/master/human.hg19.excl.tsv | grep ^chr > $repeats/sv_repeat_telomere_centromere.bed wget --no-check-certificate -O - https://github.com/lh3/varcmp/raw/master/scripts/LCR-hs37d5.bed.gz | gunzip -c | grep -v ^GL | grep -v ^NC | grep -v ^hs | sed 's/^/chr/' | bgzip -c > $repeats/LCR.bed.gz tabix -p bed -f $repeats/LCR.bed.gz wget --no-check-certificate -O $repeats/polyx.bed.gz http://s3.amazonaws.com/biodata/coverage/problem_regions/polyx/polyX-hg19.bed.gz wget --no-check-certificate -O $repeats/polyx.bed.gz.tbi http://s3.amazonaws.com/biodata/coverage/problem_regions/polyx/polyX-hg19.bed.gz.tbi gc=coverage/gc mkdir -p $gc wget --no-check-certificate -O - https://s3.amazonaws.com/biodata/collections/hg19/coverage/gc/GC_profile.1000bp.cnp > $gc/GC_profile.1000bp.cnp recipe_outfiles: - coverage/problem_regions/GA4GH/README.md - coverage/problem_regions/GA4GH/bad_promoter.bed.gz - coverage/problem_regions/GA4GH/bad_promoter.bed.gz.tbi - coverage/problem_regions/GA4GH/gc15.bed.gz - coverage/problem_regions/GA4GH/gc15.bed.gz.tbi - coverage/problem_regions/GA4GH/gc15to20.bed.gz - coverage/problem_regions/GA4GH/gc15to20.bed.gz.tbi - coverage/problem_regions/GA4GH/gc20to25.bed.gz - coverage/problem_regions/GA4GH/gc20to25.bed.gz.tbi - coverage/problem_regions/GA4GH/gc25to30.bed.gz - coverage/problem_regions/GA4GH/gc25to30.bed.gz.tbi - coverage/problem_regions/GA4GH/gc65to70.bed.gz - coverage/problem_regions/GA4GH/gc65to70.bed.gz.tbi - coverage/problem_regions/GA4GH/gc70to75.bed.gz - coverage/problem_regions/GA4GH/gc70to75.bed.gz.tbi - coverage/problem_regions/GA4GH/gc75to80.bed.gz - coverage/problem_regions/GA4GH/gc75to80.bed.gz.tbi - coverage/problem_regions/GA4GH/gc80to85.bed.gz - coverage/problem_regions/GA4GH/gc80to85.bed.gz.tbi - coverage/problem_regions/GA4GH/gc85.bed.gz - coverage/problem_regions/GA4GH/gc85.bed.gz.tbi - coverage/problem_regions/GA4GH/heng_um75-hs37d5.bed.gz - coverage/problem_regions/GA4GH/heng_um75-hs37d5.bed.gz.tbi - coverage/problem_regions/GA4GH/low_complexity_51to200bp.bed.gz - coverage/problem_regions/GA4GH/low_complexity_51to200bp.bed.gz.tbi - coverage/problem_regions/GA4GH/low_complexity_gt200bp.bed.gz - coverage/problem_regions/GA4GH/low_complexity_gt200bp.bed.gz.tbi - coverage/problem_regions/GA4GH/self_chain.bed.gz - coverage/problem_regions/GA4GH/self_chain.bed.gz.tbi - coverage/problem_regions/ENCODE/wgEncodeDacMapabilityConsensusExcludable.bed.gz - coverage/problem_regions/ENCODE/wgEncodeDacMapabilityConsensusExcludable.bed.gz.tbi - coverage/problem_regions/repeats/sv_repeat_telomere_centromere.bed - coverage/problem_regions/repeats/LCR.bed.gz - coverage/problem_regions/repeats/LCR.bed.gz.tbi - coverage/problem_regions/repeats/polyx.bed.gz - coverage/problem_regions/repeats/polyx.bed.gz.tbi - coverage/gc/GC_profile.1000bp.cnp ================================================ FILE: ggd-recipes/hg19/MIG.yaml ================================================ # Medically Interpretable Genes, compiled by Personalis: http://www.genomemedicine.com/content/pdf/s13073-015-0197-4.pdf --- attributes: name: MIG version: 20150730 recipe: full: recipe_type: bash recipe_cmds: - | baseurl=http://bcbio_nextgen.s3.amazonaws.com/MIG.zip mkdir -p prioritization cd prioritization wget -c -O MIG.zip $baseurl unzip MIG.zip recipe_outfiles: - prioritization/MIG.bed ================================================ FILE: ggd-recipes/hg19/RADAR.yaml ================================================ # RADAR RNA editing events: (http://www.stanford.edu/~gokulr/database/Human_AG_all_hg19_v2.txt) # installs a BED file of the RADAR events --- attributes: name: RADAR version: v2-20180202 recipe: full: recipe_type: bash recipe_cmds: - | url=https://s3.amazonaws.com/biodata/annotation/RADAR/hg19/RADAR.bed.gz mkdir -p editing cd editing wget --no-check-certificate -q $url wget --no-check-certificate -q $url.tbi recipe_outfiles: - editing/RADAR.bed.gz - editing/RADAR.bed.gz.tbi ================================================ FILE: ggd-recipes/hg19/af_only_gnomad.yaml ================================================ # af_only_gnomad file for t-only variant calling with mutect2 # From the Broad bundle: # ftp://gsapubftp-anonymous@ftp.broadinstitute.org/bundle/Mutect2/af-only-gnomad.raw.sites.b37.vcf.gz --- attributes: name: af_only_gnomad version: 20170602 recipe: full: recipe_type: bash recipe_cmds: - | url=ftp://gsapubftp-anonymous@ftp.broadinstitute.org/bundle/Mutect2/ base=af-only-gnomad.raw.sites.b37 mkdir -p variation for suffix in .vcf.gz .vcf.gz.tbi do wget --no-check-certificate -c ${url}/${base}${suffix} done remap_url=https://raw.githubusercontent.com/dpryan79/ChromosomeMappings/master/GRCh37_ensembl2UCSC.txt ref=../seq/hg19.fa export TMPDIR=`pwd` wget --no-check-certificate -qO- $remap_url | awk '{if($1!=$2) print "s/^"$1"/"$2"/g"}' > remap.sed # remove grch37 contig names in the header bcftools view -f PASS $base.vcf.gz | sed -f remap.sed | grep -v "##contig=" | vt decompose -s - | vt normalize -r $ref -n - | vt uniq - | bgzip -c > variation/af_only_gnomad.vcf.gz tabix -f -p vcf variation/af_only_gnomad.vcf.gz tabix -f -p vcf --csi variation/af_only_gnomad.vcf.gz rm $base.vcf.gz $base.vcf.gz.tbi recipe_outfiles: - variation/af_only_gnomad.vcf.gz - variation/af_only_gnomad.vcf.gz.tbi - variation/af_only_gnomad.vcf.gz.csi ================================================ FILE: ggd-recipes/hg19/battenberg.yaml ================================================ # Battenberg CNV caller input data: # https://github.com/cancerit/cgpBattenberg --- attributes: name: battenberg version: v3-20150610 recipe: full: required: - download_generate_bberg_ref_files.pl recipe_type: bash recipe_cmds: - | proburl=https://github.com/cancerit/cgpBattenberg/raw/dev/perl/share/battenberg/probloci.txt.gz impute_info=battenberg/impute/impute_info.txt mkdir -p battenberg [[ -f $impute_info ]] || download_generate_bberg_ref_files.pl -c -o `pwd`/battenberg # Change references to point to final directory sed -i 's/txtmp\///g' $impute_info wget --no-check-certificate -c -O battenberg/probloci.txt.gz $proburl gunzip battenberg/probloci.txt.gz # Convert chromosomes to hg19 names sed -i "s/^\([0-9]\+\)\t/chr\1\t/g" battenberg/probloci.txt sed -i "s/^X/chrX/g" battenberg/probloci.txt sed -i "s/^\([0-9]\+\)\t/chr\1\t/g" $impute_info sed -i "s/^X/chrX/g" $impute_info sed -i "s/^\([0-9]\+\)\t/chr\1\t/g" battenberg/1000genomesloci/1000genomesloci2012_chr* sed -i "s/^X/chrX/g" battenberg/1000genomesloci/1000genomesloci2012_chr23.txt recipe_outfiles: - battenberg/probloci.txt - battenberg/1000genomesloci - battenberg/impute ================================================ FILE: ggd-recipes/hg19/capture_regions.yaml ================================================ # Common sets of capture region BED files for different platforms --- attributes: name: capture_regions version: 20161202 recipe: full: recipe_type: bash recipe_cmds: - | baseurl=https://raw.githubusercontent.com/AstraZeneca-NGS/reference_data/master/hg19/bed files="Exome-Agilent_V5_Clinical.bed Exome-Agilent_V6.bed Exome-AZ_V2.bed Exome-IDT_V1.bed Exome-MedExome.bed Exome-NGv3.bed" mkdir -p coverage/capture_regions cd coverage/capture_regions for fname in $files do wget --no-check-certificate -c $baseurl/$fname done recipe_outfiles: - coverage/capture_regions/Exome-Agilent_V5_Clinical.bed - coverage/capture_regions/Exome-Agilent_V6.bed - coverage/capture_regions/Exome-AZ_V2.bed - coverage/capture_regions/Exome-IDT_V1.bed - coverage/capture_regions/Exome-MedExome.bed - coverage/capture_regions/Exome-NGv3.bed ================================================ FILE: ggd-recipes/hg19/clinvar.yaml ================================================ # ClinVar: http://www.clinvar.com/ # UCSFify name sed magic from: https://github.com/mmarchin/utilities/blob/master/ucscify.sh --- attributes: name: clinvar version: 20210110 recipe: full: recipe_type: bash recipe_cmds: - | release=20210110 # clinvar.vcf.gz is the latest one baseurl=https://ftp.ncbi.nlm.nih.gov/pub/clinvar/vcf_GRCh37/clinvar.vcf.gz mkdir -p variation wget --no-check-certificate -c -O variation/clinvar-orig.vcf.gz $baseurl rm -f variation/clinvar.vcf.gz rm -f variation/clinvar.vcf.gz.tbi zcat variation/clinvar-orig.vcf.gz | sed "s/^\([0-9]\+\)\t/chr\1\t/g" | sed "s/^MT/chrM/g" | sed "s/^X/chrX/g" | sed "s/^Y/chrY/g" | bgzip -c > variation/clinvar.vcf.gz tabix -f -p vcf variation/clinvar.vcf.gz recipe_outfiles: - variation/clinvar.vcf.gz - variation/clinvar.vcf.gz.tbi ================================================ FILE: ggd-recipes/hg19/cosmic.yaml ================================================ --- attributes: name: cosmic version: 68-20180114 recipe: full: recipe_type: bash recipe_cmds: - | baseurl=https://s3.amazonaws.com/biodata/variants/cosmic-v68-hg19.vcf.gz mkdir -p variation cd variation wget --no-check-certificate -c -O cosmic.vcf.gz $baseurl wget --no-check-certificate -c -O cosmic.vcf.gz.tbi $baseurl.tbi recipe_outfiles: - variation/cosmic.vcf.gz - variation/cosmic.vcf.gz.tbi ================================================ FILE: ggd-recipes/hg19/dbsnp.yaml ================================================ # recipe_outfiles: move symlinks first, not otherwise --- attributes: name: dbsnp version: 156-20230320 recipe: full: recipe_type: bash recipe_cmds: - | build=156 version=GCF_000001405.25 url=https://ftp.ncbi.nih.gov/snp/archive/b$build/VCF/$version.gz mkdir -p variation cd variation wget -c -O dbsnp-${build}-orig.vcf.gz $url wget -c -O dbsnp-${build}-orig.vcf.gz.tbi $url.tbi remap_url=https://raw.githubusercontent.com/dpryan79/ChromosomeMappings/master/GRCh37_NCBI2UCSC.txt # remove chromosomes not included in hg19 from remap wget --no-check-certificate -qO- $remap_url | sed 's/\r//' | awk '{if(NF==2) print $0}'> remap.tsv export TMPDIR=`pwd` # remove NW_ contigs absent in hg19 gunzip -c dbsnp-${build}-orig.vcf.gz | grep -v "NW_" | bgzip -c > dbsnp-${build}.noNW.vcf.gz tabix dbsnp-${build}.noNW.vcf.gz bcftools annotate -Ou --rename-chrs remap.tsv dbsnp-${build}.noNW.vcf.gz | \ bcftools sort -m 1G -Oz -T . -o dbsnp-${build}.vcf.gz tabix -f -p vcf dbsnp-${build}.vcf.gz tabix -f -p vcf -C dbsnp-${build}.vcf.gz ln -sf dbsnp-${build}.vcf.gz dbsnp.vcf.gz ln -sf dbsnp-${build}.vcf.gz.tbi dbsnp.vcf.gz.tbi ln -sf dbsnp-${build}.vcf.gz.csi dbsnp.vcf.gz.csi cd .. recipe_outfiles: - variation/dbsnp.vcf.gz - variation/dbsnp.vcf.gz.csi - variation/dbsnp.vcf.gz.tbi - variation/dbsnp-156.vcf.gz - variation/dbsnp-156.vcf.gz.csi - variation/dbsnp-156.vcf.gz.tbi ================================================ FILE: ggd-recipes/hg19/effects_transcripts.yaml ================================================ # Custom transcripts for effects prediction # Tweaks canonical transcripts in effects predictors like snpEff. --- attributes: name: effects_transcript version: 2017-03-16 recipe: full: recipe_type: bash recipe_cmds: - | COMMIT=93d7d3794e6afe39d92e1abc0c56406373f98501 mkdir -p snpeff/transcripts cd snpeff/transcripts wget --no-check-certificate -O canonical_cancer.txt https://raw.githubusercontent.com/AstraZeneca-NGS/reference_data/$COMMIT/transcripts/canon_cancer_replacement.txt recipe_outfiles: - snpeff/transcripts/canonical_cancer.txt ================================================ FILE: ggd-recipes/hg19/esp.yaml ================================================ # Exome Sequencing Project (ESP): http://evs.gs.washington.edu/EVS/ --- attributes: name: esp version: ESP6500SI-V2 recipe: full: recipe_type: bash recipe_cmds: - | baseurl=https://s3.amazonaws.com/gemini-annotations url=$baseurl/ESP6500SI.all.snps_indels.tidy.v2.vcf.gz remap_url=https://raw.githubusercontent.com/dpryan79/ChromosomeMappings/master/GRCh37_ensembl2UCSC.txt mkdir -p variation wget --no-check-certificate -qO- $remap_url | awk '{if($1!=$2) print "s/^"$1"/"$2"/g"}' > remap.sed export TMPDIR=`pwd` wget -c -O - $url | gunzip -c | sed -f remap.sed | bgzip -c > variation/esp.vcf.gz tabix -f -p vcf variation/esp.vcf.gz recipe_outfiles: - variation/esp.vcf.gz - variation/esp.vcf.gz.tbi ================================================ FILE: ggd-recipes/hg19/exac.yaml ================================================ # ExAC VCF sites: http://exac.broadinstitute.org/ # Script cleans up to use chr-style naming and sort by ref file --- attributes: name: exac version: 0.3 recipe: full: recipe_type: bash recipe_cmds: - | baseurl=https://s3.amazonaws.com/gemini-annotations url=$baseurl/ExAC.r0.3.sites.vep.tidy.vcf.gz remap_url=https://raw.githubusercontent.com/dpryan79/ChromosomeMappings/master/GRCh37_ensembl2UCSC.txt mkdir -p variation wget --no-check-certificate -qO- $remap_url | awk '{if($1!=$2) print "s/^"$1"/"$2"/g"}' > remap.sed export TMPDIR=`pwd` wget -c -O - $url | gunzip -c | sed -f remap.sed | bgzip -c > variation/exac.vcf.gz tabix -f -p vcf variation/exac.vcf.gz recipe_outfiles: - variation/exac.vcf.gz - variation/exac.vcf.gz.tbi ================================================ FILE: ggd-recipes/hg19/fusion-blacklist.yaml ================================================ # fusion blacklist regions from arriba --- attributes: name: fusion-blacklist version: 2 recipe: full: recipe_type: bash recipe_cmds: - | version=v1.1.0 url=https://github.com/suhrig/arriba/releases/download/${version}/arriba_${version}.tar.gz arriba_tarball=arriba_${version}.tar.gz outdir=rnaseq/fusion-blacklist mkdir -p ${outdir} wget --no-check-certificate -c -O ${arriba_tarball} ${url} tar zxvf ${arriba_tarball} mv arriba_${version}/database/blacklist_hg19_hs37d5_GRCh37_2018-11-04.tsv.gz ${outdir}/arriba-blacklist.tsv.gz mv arriba_${version}/database/cytobands_hg19_hs37d5_GRCh37_2018-02-23.tsv ${outdir}/arriba-cytobands.tsv mv arriba_${version}/database/protein_domains_hg19_hs37d5_GRCh37_2018-03-06.gff3 ${outdir}/arriba-protein-domains.gff3 mv arriba_${version}/database/protein_domains_hg19_hs37d5_GRCh37_2018-03-06.gff3.idx ${outdir}/arriba-protein-domains.gff3.idx rm -r arriba_${version} rm ${arriba_tarball} recipe_outfiles: - rnaseq/fusion-blacklist/arriba-blacklist.tsv.gz - rnaseq/fusion-blacklist/arriba-cytobands.tsv - rnaseq/fusion-blacklist/arriba-protein-domains.gff3 - rnaseq/fusion-blacklist/arriba-protein-domains.gff3.idx ================================================ FILE: ggd-recipes/hg19/giab-NA12878.yaml ================================================ # Truth sets from Genome in a Bottle for NA12878 -- http://genomeinabottle.org/ # Converts coordinates to UCSC 'chr' style naming with sed scripts # Removes mitochondrial chromosome to avoid comparisons there --- attributes: name: giab-NA12878 version: v3_3_2 recipe: full: recipe_type: bash recipe_cmds: - | dir=validation/giab-NA12878 remapurl=https://raw.githubusercontent.com/dpryan79/ChromosomeMappings/master/GRCh37_ensembl2UCSC.txt url=http://ftp-trace.ncbi.nlm.nih.gov/giab/ftp/release/NA12878_HG001/NISTv3.3.2/GRCh37/ calls=HG001_GRCh37_GIAB_highconf_CG-IllFB-IllGATKHC-Ion-10X-SOLID_CHROM1-X_v.3.3.2_highconf_PGandRTGphasetransfer.vcf.gz regions=HG001_GRCh37_GIAB_highconf_CG-IllFB-IllGATKHC-Ion-10X-SOLID_CHROM1-X_v.3.3.2_highconf_nosomaticdel.bed mkdir -p $dir wget -qO- $remapurl | awk '{if($1!=$2) print "s/^"$1"/"$2"/g"}' > remap.sed wget -c -O - $url/$calls | gunzip -c | sed -f remap.sed | grep -v "##contig=" | bgzip -c > $dir/truth_small_variants.vcf.gz tabix -f -p vcf $dir/truth_small_variants.vcf.gz wget -c -O - $url/$regions | sed -f remap.sed > $dir/truth_regions.bed wget -O - http://ftp.ncbi.nih.gov/giab/ftp/technical/svclassify_Manuscript/Supplementary_Information/Personalis_1000_Genomes_deduplicated_deletions.bed | grep -v ^Chr | sed "s/^\([0-9]\+\)\t/chr\1\t/g" | sed "s/^MT/chrM/g" | sed "s/^X/chrX/g" | sed "s/^Y/chrY/g" > $dir/truth_DEL.bed wget -O - http://ftp.ncbi.nih.gov/giab/ftp/technical/svclassify_Manuscript/Supplementary_Information/Spiral_Genetics_insertions.bed | grep -v ^Chr | sed "s/^\([0-9]\+\)\t/chr\1\t/g" | sed "s/^MT/chrM/g" | sed "s/^X/chrX/g" | sed "s/^Y/chrY/g" > $dir/truth_INS.bed recipe_outfiles: - validation/giab-NA12878/truth_small_variants.vcf.gz - validation/giab-NA12878/truth_small_variants.vcf.gz.tbi - validation/giab-NA12878/truth_regions.bed - validation/giab-NA12878/truth_DEL.bed - validation/giab-NA12878/truth_INS.bed ================================================ FILE: ggd-recipes/hg19/giab-NA24143.yaml ================================================ # Truth sets from Genome in a Bottle for NA24143 -- http://genomeinabottle.org/ --- attributes: name: giab-NA24143 version: v3_3_2 recipe: full: recipe_type: bash recipe_cmds: - | dir=validation/giab-NA24143 url=https://ftp-trace.ncbi.nlm.nih.gov/giab/ftp/release/AshkenazimTrio/HG004_NA24143_mother/NISTv3.3.2/GRCh37/ calls=HG004_GRCh37_GIAB_highconf_CG-IllFB-IllGATKHC-Ion-10X_CHROM1-22_v.3.3.2_highconf.vcf.gz regions=HG004_GRCh37_GIAB_highconf_CG-IllFB-IllGATKHC-Ion-10X_CHROM1-22_v.3.3.2_highconf_noinconsistent.bed remapurl=https://raw.githubusercontent.com/dpryan79/ChromosomeMappings/master/GRCh37_ensembl2UCSC.txt mkdir -p $dir wget -qO- $remapurl | awk '{if($1!=$2) print "s/^"$1"/"$2"/g"}' > remap.sed wget -c -O - $url/$calls | gunzip -c | sed -f remap.sed | grep -v "##contig=" | bgzip -c > $dir/truth_small_variants.vcf.gz tabix -f -p vcf $dir/truth_small_variants.vcf.gz wget -c -O - $url/$regions | sed -f remap.sed > $dir/truth_regions.bed recipe_outfiles: - validation/giab-NA24143/truth_small_variants.vcf.gz - validation/giab-NA24143/truth_small_variants.vcf.gz.tbi - validation/giab-NA24143/truth_regions.bed ================================================ FILE: ggd-recipes/hg19/giab-NA24149.yaml ================================================ # Truth sets from Genome in a Bottle for NA24149 -- http://genomeinabottle.org/ --- attributes: name: giab-NA24149 version: v3_3_2 recipe: full: recipe_type: bash recipe_cmds: - | dir=validation/giab-NA24149 url=https://ftp-trace.ncbi.nlm.nih.gov/giab/ftp/release/AshkenazimTrio/HG003_NA24149_father/NISTv3.3.2/GRCh37/ calls=HG003_GRCh37_GIAB_highconf_CG-IllFB-IllGATKHC-Ion-10X_CHROM1-22_v.3.3.2_highconf.vcf.gz regions=HG003_GRCh37_GIAB_highconf_CG-IllFB-IllGATKHC-Ion-10X_CHROM1-22_v.3.3.2_highconf_noinconsistent.bed remapurl=https://raw.githubusercontent.com/dpryan79/ChromosomeMappings/master/GRCh37_ensembl2UCSC.txt mkdir -p $dir wget -qO- $remapurl | awk '{if($1!=$2) print "s/^"$1"/"$2"/g"}' > remap.sed wget -c -O - $url/$calls | gunzip -c | sed -f remap.sed | grep -v "##contig=" | bgzip -c > $dir/truth_small_variants.vcf.gz tabix -f -p vcf $dir/truth_small_variants.vcf.gz wget -c -O - $url/$regions | sed -f remap.sed > $dir/truth_regions.bed recipe_outfiles: - validation/giab-NA24149/truth_small_variants.vcf.gz - validation/giab-NA24149/truth_small_variants.vcf.gz.tbi - validation/giab-NA24149/truth_regions.bed ================================================ FILE: ggd-recipes/hg19/giab-NA24385.yaml ================================================ # Truth sets from Genome in a Bottle for NA24385 -- http://genomeinabottle.org/ --- attributes: name: giab-NA24385 version: v3_3_2-sv_v0.6 recipe: full: recipe_type: bash recipe_cmds: - | dir=validation/giab-NA24385 url=http://ftp-trace.ncbi.nlm.nih.gov/giab/ftp/release/AshkenazimTrio/HG002_NA24385_son/NISTv3.3.2/GRCh37/ calls=HG002_GRCh37_GIAB_highconf_CG-IllFB-IllGATKHC-Ion-10X-SOLID_CHROM1-22_v.3.3.2_highconf_triophased.vcf.gz regions=HG002_GRCh37_GIAB_highconf_CG-IllFB-IllGATKHC-Ion-10X-SOLID_CHROM1-22_v.3.3.2_highconf_noinconsistent.bed remapurl=https://raw.githubusercontent.com/dpryan79/ChromosomeMappings/master/GRCh37_ensembl2UCSC.txt mkdir -p $dir wget -qO- $remapurl | awk '{if($1!=$2) print "s/^"$1"/"$2"/g"}' > remap.sed wget -c -O - $url/$calls | gunzip -c | sed -f remap.sed | grep -v "##contig=" | bgzip -c > $dir/truth_small_variants.vcf.gz tabix -f -p vcf $dir/truth_small_variants.vcf.gz wget -c -O - $url/$regions | sed -f remap.sed > $dir/truth_regions.bed delurl=https://s3.amazonaws.com/bcbio/giab/NA24385/NA24385-GIAB-2tech-dels-v0_1_8-hg19.bed.gz wget -c -O $dir/truth_DEL_giab.bed.gz $delurl wget -c -O $dir/truth_DEL_giab.bed.gz.tbi $delurl.tbi cnvurl=https://s3.amazonaws.com/bcbio/giab/NA24385/NA24385-crowd-dels-hg19.bed.gz wget -c -O $dir/truth_DEL_crowd.bed.gz $cnvurl wget -c -O $dir/truth_DEL_crowd.bed.gz.tbi $cnvurl.tbi svurl=ftp://ftp-trace.ncbi.nlm.nih.gov/giab/ftp/data/AshkenazimTrio/analysis/NIST_SVs_Integration_v0.6/HG002_SVs_Tier1_v0.6.vcf.gz svbedurl=ftp://ftp-trace.ncbi.nlm.nih.gov/giab/ftp/data/AshkenazimTrio/analysis/NIST_SVs_Integration_v0.6/HG002_SVs_Tier1_v0.6.bed wget -c -O - $svurl | gunzip -c | sed -f remap.sed | grep -v "##contig=" | bgzip -c > $dir/truth_sv.vcf.gz tabix -f -p vcf $dir/truth_sv.vcf.gz wget -c -O - $svbedurl | sed -f remap.sed > $dir/truth_sv_regions.bed recipe_outfiles: - validation/giab-NA24385/truth_small_variants.vcf.gz - validation/giab-NA24385/truth_small_variants.vcf.gz.tbi - validation/giab-NA24385/truth_regions.bed - validation/giab-NA24385/truth_DEL_giab.bed.gz - validation/giab-NA24385/truth_DEL_giab.bed.gz.tbi - validation/giab-NA24385/truth_DEL_crowd.bed.gz - validation/giab-NA24385/truth_DEL_crowd.bed.gz.tbi - validation/giab-NA24385/truth_sv.vcf.gz - validation/giab-NA24385/truth_sv.vcf.gz.tbi ================================================ FILE: ggd-recipes/hg19/giab-NA24631.yaml ================================================ # Truth sets from Genome in a Bottle for NA24631 -- http://genomeinabottle.org/ --- attributes: name: giab-NA24631 version: v3_3_2 recipe: full: recipe_type: bash recipe_cmds: - | dir=validation/giab-NA24631 url=http://ftp-trace.ncbi.nlm.nih.gov/giab/ftp/release/ChineseTrio/HG005_NA24631_son/NISTv3.3.2/GRCh37/ calls=HG005_GRCh37_highconf_CG-IllFB-IllGATKHC-Ion-SOLID_CHROM1-22_v.3.3.2_highconf.vcf.gz regions=HG005_GRCh37_highconf_CG-IllFB-IllGATKHC-Ion-SOLID_CHROM1-22_v.3.3.2_highconf_noMetaSV.bed remapurl=https://raw.githubusercontent.com/dpryan79/ChromosomeMappings/master/GRCh37_ensembl2UCSC.txt mkdir -p $dir wget -qO- $remapurl | awk '{if($1!=$2) print "s/^"$1"/"$2"/g"}' > remap.sed wget -c -O - $url/$calls | gunzip -c | sed -f remap.sed | grep -v "##contig=" | bgzip -c > $dir/truth_small_variants.vcf.gz tabix -f -p vcf $dir/truth_small_variants.vcf.gz wget -c -O - $url/$regions | sed -f remap.sed > $dir/truth_regions.bed recipe_outfiles: - validation/giab-NA24631/truth_small_variants.vcf.gz - validation/giab-NA24631/truth_small_variants.vcf.gz.tbi - validation/giab-NA24631/truth_regions.bed ================================================ FILE: ggd-recipes/hg19/gnomad.yaml ================================================ # GnomAD VCF sites: # - https://gnomad.broadinstitute.org/downloads # - https://macarthurlab.org/2018/10/17/gnomad-v2-1/ # - http://ftp.ensemblorg.ebi.ac.uk/pub/data_files/homo_sapiens/GRCh37/variation_genotype/gnomad/r2.1/genomes/ # Script # - cleans up to use chr-style naming, # - sorts according to the reference file # - normalizes (input is decomposed but not normalized in gnomad genome 2.1) # - filters only PASS variants (segdup, decoy are retained) # - removes many INFO fields to reduce file size: controls_, hist, non_, using bcftools annotate instead of vt rminfo, because it failed randomly on chr1 # - merges all chr into a single file # wget is separated from the processing because wget in the pipe fails randomly (on some systems?): it comes out when downloading many files. # recipe test: https://github.com/chapmanb/cloudbiolinux/blob/master/doc/hacking.md#testing-a-ggd-recipe --- attributes: name: gnomad version: 2.1.1 recipe: full: recipe_type: bash recipe_cmds: - | url_prefix=https://storage.googleapis.com/gnomad-public/release/2.1.1/vcf/genomes/ vcf_file=gnomad.genomes.r2.1.1.sites.vcf.bgz remap_url=https://raw.githubusercontent.com/dpryan79/ChromosomeMappings/master/GRCh37_ensembl2UCSC.txt ref=../seq/hg19.fa mkdir -p variation export TMPDIR=`pwd` wget --no-check-certificate -qO- $remap_url | awk '{if($1!=$2) print "s/^"$1"/"$2"/g"}' > remap.sed gnomad_fields_to_keep_url=https://gist.githubusercontent.com/naumenko-sa/d20db928b915a87bba4012ba1b89d924/raw/cf343b105cb3347e966cc95d049e364528c86880/gnomad_fields_to_keep.txt wget --no-check-certificate -c $gnomad_fields_to_keep_url vcf=${url_prefix}${vcf_file} wget --no-check-certificate -c $vcf wget --no-check-certificate -c $vcf.tbi fields_to_keep="INFO/"$(cat gnomad_fields_to_keep.txt | paste -s | sed s/"\t"/",INFO\/"/g) bcftools view -f PASS $vcf_file | bcftools annotate -x "^$fields_to_keep" -Ov | sed -f remap.sed | grep -v "##config=" | vt decompose -s - | vt normalize -r $ref -n - | vt uniq - | bgzip -c > variation/gnomad_genome.vcf.gz tabix -f -p vcf variation/gnomad_genome.vcf.gz tabix -f -p vcf --csi variation/gnomad_genome.vcf.gz rm $vcf_file $vcf_file.tbi recipe_outfiles: - variation/gnomad_genome.vcf.gz - variation/gnomad_genome.vcf.gz.tbi - variation/gnomad_genome.vcf.gz.csi ================================================ FILE: ggd-recipes/hg19/gnomad_exome.yaml ================================================ # GnomAD VCF sites: # - https://gnomad.broadinstitute.org/downloads # - https://macarthurlab.org/2018/10/17/gnomad-v2-1/ # Script # - no need in decomposing, normalizing,sort, and uniq anymore - it is done in gnomad 2.1.1 exomes # - filters only PASS variants (segdup, decoy are retained) # - removes many INFO fields to reduce file size: controls_, hist, non_, using bcftools annotate instead of vt rminfo, because it failed randomly on chr1 # recipe test: https://github.com/chapmanb/cloudbiolinux/blob/master/doc/hacking.md#testing-a-ggd-recipe # much faster from gnomad in google cloud - downloads in 20-25 min on O2 # vt is 3x faster than bcftools for removing INFO tags: in a test with 4444 variants t(bcftools -x) = t(bcftools -x "^") = 27s, time(vt rminfo) = 8s, # but vt rminfo is failing after 148M, using bcftools # overall processing is still slow: 87G to G in --- attributes: name: gnomad_exome version: 2.1.1 recipe: full: recipe_type: bash recipe_cmds: - | remap_url=https://raw.githubusercontent.com/dpryan79/ChromosomeMappings/master/GRCh37_ensembl2UCSC.txt ref=../seq/hg19.fa mkdir -p variation export TMPDIR=`pwd` wget --no-check-certificate -qO- $remap_url | awk '{if($1!=$2) print "s/^"$1"/"$2"/g"}' > remap.sed vcf_prefix=https://storage.googleapis.com/gnomad-public/release/2.1.1/vcf/exomes/ vcf_file=gnomad.exomes.r2.1.1.sites.vcf.bgz ref=../seq/GRCh37.fa mkdir -p variation export TMPDIR=`pwd` gnomad_fields_to_keep_url=https://gist.githubusercontent.com/naumenko-sa/d20db928b915a87bba4012ba1b89d924/raw/cf343b105cb3347e966cc95d049e364528c86880/gnomad_fields_to_keep.txt wget --no-check-certificate -c $gnomad_fields_to_keep_url vcf=${vcf_prefix}${vcf_file} wget --no-check-certificate -c $vcf wget --no-check-certificate -c $vcf.tbi fields_to_keep="INFO/"$(cat gnomad_fields_to_keep.txt | paste -s | sed s/"\t"/",INFO\/"/g) bcftools view -f PASS $vcf_file | bcftools annotate -x "^$fields_to_keep" -Ov | sed -f remap.sed | grep -v "##contig=" | bgzip -c > variation/gnomad_exome.vcf.gz tabix -f -p vcf variation/gnomad_exome.vcf.gz tabix -f -p vcf --csi variation/gnomad_exome.vcf.gz rm $vcf_file $vcf_file.tbi recipe_outfiles: - variation/gnomad_exome.vcf.gz - variation/gnomad_exome.vcf.gz.tbi - variation/gnomad_exome.vcf.gz.csi ================================================ FILE: ggd-recipes/hg19/gnomad_genome.grch37_to_hg19.sh ================================================ #!/bin/bash #PBS -l walltime=230:00:00,nodes=1:ppn=1 #PBS -joe . #PBS -d . #PBS -l vmem=10g,mem=10g date export PATH=/hpf/largeprojects/ccmbio/naumenko/tools/bcbio_1.1.5/bin:$PATH remap_url=https://raw.githubusercontent.com/dpryan79/ChromosomeMappings/master/GRCh37_ensembl2UCSC.txt ref=seq/hg19.fa export TMPDIR=`pwd` #wget --no-check-certificate -qO- $remap_url | awk '{if($1!=$2) print "s/^"$1"/"$2"/g"}' > remap.sed gunzip -c gnomad_genome.grch37.vcf.gz | sed -f remap.sed | gsort -m 3000 /dev/stdin $ref.fai | vt normalize -r $ref -n - | vt uniq - | bgzip -c > gnomad_genome.vcf.gz tabix -f -p vcf gnomad_genome.vcf.gz tabix -f -p vcf --csi gnomad_genome.vcf.gz date ================================================ FILE: ggd-recipes/hg19/gtf.yaml ================================================ --- attributes: name: gtf version: 87 recipe: full: recipe_type: bash recipe_cmds: - | url=ftp://ftp.ensembl.org/pub/grch37/current/gtf/homo_sapiens/Homo_sapiens.GRCh37.87.gtf.gz mkdir -p rnaseq remap_url=http://raw.githubusercontent.com/dpryan79/ChromosomeMappings/master/GRCh37_ensembl2UCSC.txt wget --no-check-certificate -qO- $remap_url | awk '{if($1!=$2) print "s/^"$1"/"$2"/g"}' > remap.sed wget --no-check-certificate -qO- $url | gunzip -c > hg19.gtf sed -f remap.sed hg19.gtf | grep -v _hap > rnaseq/hg19.gtf rm remap.sed rm hg19.gtf recipe_outfiles: - rnaseq/hg19.gtf ================================================ FILE: ggd-recipes/hg19/hapmap.yaml ================================================ --- attributes: name: hapmap version: 3.3 recipe: full: recipe_type: bash recipe_cmds: - | baseurl=ftp://gsapubftp-anonymous:none@ftp.broadinstitute.org/bundle/hg19/hapmap_3.3.hg19.sites.vcf.gz mkdir -p variation cd variation wget -O - $baseurl | gunzip -c | bgzip -c > hapmap_3.3.vcf.gz tabix -f -p vcf hapmap_3.3.vcf.gz recipe_outfiles: - variation/hapmap_3.3.vcf.gz - variation/hapmap_3.3.vcf.gz.tbi ================================================ FILE: ggd-recipes/hg19/mills_indels.yaml ================================================ --- attributes: name: mills_indels version: 2.8 recipe: full: recipe_type: bash recipe_cmds: - | baseurl=ftp://gsapubftp-anonymous:none@ftp.broadinstitute.org/bundle/hg19/Mills_and_1000G_gold_standard.indels.hg19.sites.vcf.gz mkdir -p variation cd variation wget -O - $baseurl | gunzip -c | bgzip -c > Mills_and_1000G_gold_standard.indels.vcf.gz tabix -f -p vcf Mills_and_1000G_gold_standard.indels.vcf.gz recipe_outfiles: - variation/Mills_and_1000G_gold_standard.indels.vcf.gz - variation/Mills_and_1000G_gold_standard.indels.vcf.gz.tbi ================================================ FILE: ggd-recipes/hg19/mirbase.yaml ================================================ # small RNA seq annotation from different sources --- attributes: name: srnaseq version: 20211104 recipe: full: recipe_type: bash recipe_cmds: - | mkdir -p srnaseq cd srnaseq wget --random-wait --retry-connrefused -nv -c -O hsahg19.gff3 https://www.mirbase.org/ftp/20/genomes/hsa.gff3 awk '$3=="miRNA"' hsahg19.gff3 | sed 's/=/ /g' > srna-transcripts.gtf wget --random-wait --retry-connrefused -nv -c -O wgRna.txt.gz http://hgdownload.soe.ucsc.edu/goldenPath/hg19/database/wgRna.txt.gz zgrep -v 'hsa-' wgRna.txt.gz | awk '{print $2"\t.\tncrna\t"$3"\t"$4"\t.\t"$7"\t.\tname "$5";"}' >> srna-transcripts.gtf wget --random-wait --retry-connrefused -nv -c -O tRNAs.txt.gz http://hgdownload.soe.ucsc.edu/goldenPath/hg19/database/tRNAs.txt.gz zcat tRNAs.txt.gz | awk '{print $2"\t.\ttRNA\t"$3"\t"$4"\t.\t"$7"\t.\tname "$5";"}' >> srna-transcripts.gtf wget --random-wait --retry-connrefused -nv -c -O rmsk.txt.gz http://hgdownload.soe.ucsc.edu/goldenPath/hg19/database/rmsk.txt.gz zcat rmsk.txt.gz | awk '{print $6"\t.\trepeat\t"$7+1"\t"$8+1"\t.\t"$10"\t.\tname "$12";"}' >> srna-transcripts.gtf wget --random-wait --retry-connrefused -nv -c -O refGene.txt.gz http://hgdownload.soe.ucsc.edu/goldenPath/hg19/database/refGene.txt.gz zcat refGene.txt.gz | awk '{print $3"\t.\tgene\t"$5"\t"$6"\t.\t"$4"\t.\tname "$13";"}' >> srna-transcripts.gtf # wget --random-wait --retry-connrefused -nv -c -O piR_hg19_v1.0.bed.gz http://www.regulatoryrna.org/database/piRNA/download/archive/v1.0/bed/piR_hg19_v1.0.bed.gz # zcat piR_hg19_v1.0.bed.gz | awk '{print $1"\t.\tpiRNA\t"$2"\t"$3"\t.\t"$6"\t.\tname "$4";"}' >> srna-transcripts.gtf # mirbase wget --random-wait --retry-connrefused -nv -c -O hairpin.fa.raw https://mirbase.org/download/CURRENT/hairpin.fa cat hairpin.fa.raw | awk '{if ($0~/>hsa/){name=$0; print name} else if ($0~/^>/){name=0};if (name!=0 && $0!~/^>/){print $0;}}' | sed 's/U/T/g' > hairpin.fa rm hairpin.fa.raw wget --random-wait --retry-connrefused -nv -c -O mature.fa.raw https://mirbase.org/download/CURRENT/mature.fa cat mature.fa.raw | awk '{if ($0~/>hsa/){name=$0; print name} else if ($0~/^>/){name=0};if (name!=0 && $0!~/^>/){print $0;}}' | sed 's/U/T/g' > mature.fa rm mature.fa.raw wget --random-wait --retry-connrefused -nv -c -O miRNA.dat.raw https://mirbase.org/download/CURRENT/miRNA.dat cat miRNA.dat.raw | awk '{if ($0~/hsa/)print $0}' > miRNA.str rm miRNA.dat.raw wget --random-wait --retry-connrefused -nv -c -O mirbase.gff3 https://mirbase.org/download/hsa.gff3 # tdrmapper wget --random-wait --retry-connrefused --no-check-certificate -nv -c -O trna_mature_pre.fa https://github.com/sararselitsky/tDRmapper/raw/master/hg19_mature_and_pre.fa # mintmap wget --random-wait --retry-connrefused -nv -c https://github.com/TJU-CMC-Org/MINTmap/archive/refs/tags/MINTmap_Standard_v1.0.tar.gz tar xzf MINTmap_Standard_v1.0.tar.gz MINTmap-MINTmap_Standard_v1.0/LookupTable.tRFs.MINTmap_v1.txt --strip-components=1 tar xzf MINTmap_Standard_v1.0.tar.gz MINTmap-MINTmap_Standard_v1.0/tRNAspace.Spliced.Sequences.MINTmap_v1.fa --strip-components=1 tar xzf MINTmap_Standard_v1.0.tar.gz MINTmap-MINTmap_Standard_v1.0/OtherAnnotations.MINTmap_v1.txt --strip-components=1 # mirdeep2 wget --random-wait --retry-connrefused --no-check-certificate -nv -c -O Rfam_for_miRDeep.fa.gz https://github.com/lpantano/mirdeep2_core/raw/data/Rfam_for_miRDeep.fa.gz && gunzip Rfam_for_miRDeep.fa.gz # targetscan analysis wget --random-wait --retry-connrefused --no-check-certificate -nv -c -O Summary_Counts.txt.zip http://www.targetscan.org/vert_71/vert_71_data_download/Summary_Counts.all_predictions.txt.zip && unzip Summary_Counts.txt.zip wget --random-wait --retry-connrefused --no-check-certificate -nv -c -O miR_Family_Info.txt.zip http://www.targetscan.org/vert_71/vert_71_data_download/miR_Family_Info.txt.zip && unzip miR_Family_Info.txt.zip wget --random-wait --retry-connrefused --no-check-certificate -nv -N -c https://www.mirbase.org/ftp/21/database_files/mirna_mature.txt.gz recipe_outfiles: - srnaseq/srna-transcripts.gtf - srnaseq/hairpin.fa - srnaseq/mature.fa - srnaseq/miRNA.str - srnaseq/mirbase.gff3 - srnaseq/trna_mature_pre.fa - srnaseq/LookupTable.tRFs.MINTmap_v1.txt - srnaseq/tRNAspace.Spliced.Sequences.MINTmap_v1.fa - srnaseq/OtherAnnotations.MINTmap_v1.txt - srnaseq/Rfam_for_miRDeep.fa - srnaseq/Summary_Counts.all_predictions.txt - srnaseq/miR_Family_Info.txt - srnaseq/mirna_mature.txt.gz ================================================ FILE: ggd-recipes/hg19/platinum-genome-NA12878.yaml ================================================ # Truth sets from Illumina Platinum Genomes project # http://www.illumina.com/platinumgenomes/ --- attributes: name: platinum-genome-NA12878 version: v8_0_1 recipe: full: recipe_type: bash recipe_cmds: - | version=v8_0_1 dir=validation/platinum-genome-NA12878 mkdir -p $dir wget --no-check-certificate -c -O $dir/truth_small_variants.vcf.gz https://s3.amazonaws.com/cloudbiolinux/cache/platinum-genome/platinum-genome-NA12878-hg19-$version.vcf.gz wget --no-check-certificate -c -O $dir/truth_small_variants.vcf.gz.tbi https://s3.amazonaws.com/cloudbiolinux/cache/platinum-genome/platinum-genome-NA12878-hg19-$version.vcf.gz.tbi wget --no-check-certificate -c -O - https://s3.amazonaws.com/cloudbiolinux/cache/platinum-genome/platinum-genome-NA12878-hg19-regions-$version.bed.gz | gunzip -c > $dir/truth_regions.bed recipe_outfiles: - validation/platinum-genome-NA12878/truth_small_variants.vcf.gz - validation/platinum-genome-NA12878/truth_small_variants.vcf.gz.tbi - validation/platinum-genome-NA12878/truth_regions.bed ================================================ FILE: ggd-recipes/hg19/prioritize.yaml ================================================ # Regions for prioritization: # cancer # - civic -- known cancer genes from the CIViC database # - az300 -- list of 300 cancer genes of interest from AstraZeneca Oncology # - az300-with-fusion -- az300 list plus known interacting fusions # from FusionCatcher https://github.com/ndaniel/fusioncatcher/blob/master/bin/generate_known.py#L102 # - az-cancer-panel -- az300 genes plus additional useful panel genes --- attributes: name: prioritize version: 20181227 recipe: full: recipe_type: bash recipe_cmds: - | baseurl=https://s3.amazonaws.com/biodata/coverage/prioritize/prioritize-cancer-hg19-20181227.tar.gz outdir=coverage/prioritize mkdir -p $outdir cd $outdir wget --no-check-certificate -c -O cancer.tar.gz $baseurl tar -xzvpf cancer.tar.gz recipe_outfiles: - coverage/prioritize/cancer/az300.bed.gz - coverage/prioritize/cancer/az300.bed.gz.tbi - coverage/prioritize/cancer/az300-with-fusion.bed.gz - coverage/prioritize/cancer/az300-with-fusion.bed.gz.tbi - coverage/prioritize/cancer/az-cancer-panel.bed.gz - coverage/prioritize/cancer/az-cancer-panel.bed.gz.tbi - coverage/prioritize/cancer/civic-2018-12-27.bed.gz - coverage/prioritize/cancer/civic-2018-12-27.bed.gz.tbi ================================================ FILE: ggd-recipes/hg19/purecn_mappability.yaml ================================================ # mappability regions for purecn analyses: # http://bioconductor.org/packages/release/bioc/vignettes/PureCN/inst/doc/Quick.html # file size: 99M --- attributes: name: purecn_mappability version: 20200928 recipe: full: recipe_type: bash recipe_cmds: - | mkdir -p coverage/mappability cd coverage/mappability wget --no-check-certificate -c http://hgdownload.cse.ucsc.edu/goldenPath/hg19/encodeDCC/wgEncodeMapability/wgEncodeCrgMapabilityAlign100mer.bigWig cd .. recipe_outfiles: - coverage/mappability/wgEncodeCrgMapabilityAlign100mer.bigWig ================================================ FILE: ggd-recipes/hg19/rmsk.yaml ================================================ # repeat elements location on the genome --- attributes: name: rmsk version: 20180319 recipe: full: recipe_type: bash recipe_cmds: - | mkdir -p coverage/problem_regions/repeats cd coverage/problem_regions/repeats wget --random-wait --retry-connrefused -q -c -O rmsk.txt.gz http://hgdownload.soe.ucsc.edu/goldenPath/hg19/database/rmsk.txt.gz zcat rmsk.txt.gz | awk '{print $6"\t.\trepeat\t"$7+1"\t"$8+1"\t.\t"$10"\t.\tname "$12";"}' | gzip > rmsk.gtf.gz recipe_outfiles: - coverage/problem_regions/repeats/rmsk.gtf.gz ================================================ FILE: ggd-recipes/hg19/seq.yaml ================================================ # hg19 reference genome from Broad bundle # ftp://gsapubftp-anonymous:none@ftp.broadinstitute.org/bundle/hg19 # Include bgzipped fasta file for Ensembl and other tools that support it --- attributes: name: seq version: broad-20120813 recipe: full: recipe_type: bash recipe_cmds: - | wget --no-check-certificate -c https://s3.amazonaws.com/biodata/genomes/hg19-seq.tar.gz tar -xzvpf hg19-seq.tar.gz gunzip -c seq/hg19.fa.gz > seq/hg19.fa touch seq/hg19.fa.fai touch seq/hg19.dict recipe_outfiles: - seq/hg19.fa - seq/hg19.fa.fai - seq/hg19.fa.gz - seq/hg19.fa.gz.fai - seq/hg19.fa.gz.gzi - seq/hg19.dict ================================================ FILE: ggd-recipes/hg19/simple_repeat.yaml ================================================ # UCSC repeats track for purecn variant filtration: --- attributes: name: simple_repeat version: 20200928 recipe: full: recipe_type: bash recipe_cmds: - | mkdir -p coverage/problem_regions/repeats cd coverage/problem_regions/repeats wget --random-wait --retry-connrefused -q -c -O simple_repeat.txt.gz http://hgdownload.soe.ucsc.edu/goldenPath/hg19/database/simpleRepeat.txt.gz zcat simple_repeat.txt.gz | awk -F '\t' '{print $2"\t"$3"\t"$4"\t"$5"\t"$11"\t."}' > simple_repeat.bed recipe_outfiles: - coverage/problem_regions/repeats/simple_repeat.bed ================================================ FILE: ggd-recipes/hg19/topmed.yaml ================================================ # Topmed WGS allele frequencies: https://bravo.sph.umich.edu/freeze3a/hg19/ --- attributes: name: topmed version: freeze3a recipe: full: recipe_type: bash recipe_cmds: - | url=ftp://ftp.ensembl.org/pub/data_files/homo_sapiens/GRCh37/variation_genotype/TOPMED_GRCh37.vcf.gz remap_url=https://raw.githubusercontent.com/dpryan79/ChromosomeMappings/master/GRCh37_ensembl2UCSC.txt ref=../seq/hg19.fa mkdir -p variation export TMPDIR=`pwd` wget --no-check-certificate -qO- $remap_url | awk '{if($1!=$2) print "s/^"$1"/"$2"/g"}' > remap.sed wget -c -O - $url | gunzip -c | sed -f remap.sed | gsort -m 3000 /dev/stdin $ref.fai | vt decompose -s - | vt normalize -r $ref -n - | vt uniq - | bgzip -c > variation/topmed.vcf.gz tabix -f -p vcf variation/topmed.vcf.gz recipe_outfiles: - variation/topmed.vcf.gz - variation/topmed.vcf.gz.tbi ================================================ FILE: ggd-recipes/hg19/transcripts.yaml ================================================ # Transcript, annotation and indices for RNA-seq analysis # Prepared from Ensembl transcripts using: # https://github.com/chapmanb/cloudbiolinux/blob/master/utils/prepare_tx_gff.py # Built from ftp://ftp.ensembl.org/pub/release-67/gtf/homo_sapiens/Homo_sapiens.GRCh37.67.gtf.gz # translated to UCSC chromosomes, with genes in haplotype patches discarded. # See gtf.yaml for details. --- attributes: name: transcripts version: 2019-03-01_87 recipe: full: recipe_type: bash recipe_cmds: - | baseurl=https://s3.amazonaws.com/biodata/annotation/hg19-rnaseq-2019-03-01_75.tar.xz wget -c -N --no-check-certificate $baseurl xz -dc *-rnaseq-*.tar.xz | tar -xpf - mv */rnaseq-* rnaseq recipe_outfiles: - rnaseq/ref-transcripts.bed - rnaseq/ref-transcripts.fa - rnaseq/ref-transcripts.genePred - rnaseq/ref-transcripts.gtf - rnaseq/ref-transcripts.gtf.db - rnaseq/ref-transcripts-mask.gtf - rnaseq/ref-transcripts.refFlat - rnaseq/ref-transcripts-splicesites.txt - rnaseq/rRNA.gtf - rnaseq/rRNA.gtf.db - rnaseq/rRNA.interval_list - rnaseq/tx2gene.csv ================================================ FILE: ggd-recipes/hg19/twobit.yaml ================================================ # UCSC twobit reference --- attributes: name: twobit version: broad-20120813 recipe: full: recipe_type: bash recipe_cmds: - | mkdir -p ucsc cd ucsc faToTwoBit ../../seq/hg19.fa hg19.2bit recipe_outfiles: - ucsc/hg19.2bit ================================================ FILE: ggd-recipes/hg19/varpon.yaml ================================================ # Panels of normals (PONs) for variant calling # germline_het_pon -- Locations of likely germline heterogygote sites --- attributes: name: varpon version: 20181105 recipe: full: recipe_type: bash recipe_cmds: - | mkdir -p variation wget --no-check-certificate -O - 'https://s3.amazonaws.com/biodata/collections/hg19/variation/germline_het_pon.bed.gz' > variation/germline_het_pon.bed.gz wget --no-check-certificate -O - 'https://s3.amazonaws.com/biodata/collections/hg19/variation/germline_het_pon.bed.gz.tbi' > variation/germline_het_pon.bed.gz.tbi recipe_outfiles: - variation/germline_het_pon.bed.gz - variation/germline_het_pon.bed.gz.tbi ================================================ FILE: ggd-recipes/hg19/viral.yaml ================================================ # Virus reference genomes, prepared for bwa alignment # Taken from TCGA GDC reference genome # https://gdc.cancer.gov/about-data/data-harmonization-and-generation/gdc-reference-files --- attributes: name: viral version: 2019.12.03 recipe: full: recipe_type: bash recipe_cmds: - | url=http://s3.amazonaws.com/biodata/viral/GRCh37-gdc-viral.tar.gz mkdir -p viral wget --no-check-certificate -c $url tar -xzvpf *.tar.gz mv *-gdc-viral/* viral recipe_outfiles: - viral/gdc-viral.fa - viral/gdc-viral.dict - viral/gdc-viral.fa.ann - viral/gdc-viral.fa.fai - viral/gdc-viral.fa.pac - viral/gdc-viral.fa.bwt - viral/gdc-viral.fa.amb - viral/gdc-viral.fa.sa ================================================ FILE: ggd-recipes/hg38/1000g_indels.yaml ================================================ # 1000 genomes phase 1 indel calls # # From the Broad v2.8 resource bundle # https://www.broadinstitute.org/gatk/guide/article.php?id=1213 # lifted over to hg38 coordinates by Alison Meynert # https://github.com/chapmanb/bcbio-nextgen/issues/817 --- attributes: name: 1000g_indels version: 2.8_hg38_20150522 recipe: full: recipe_type: bash recipe_cmds: - | url=https://s3.amazonaws.com/biodata/hg38_bundle base=1000G_phase1.indels.b38.primary_assembly new=1000G_phase1.indels mkdir -p variation for suffix in .vcf.gz .vcf.gz.tbi do [[ -f variation/$new$suffix ]] || wget -c -O variation/$new$suffix $url/$base$suffix done recipe_outfiles: - variation/1000G_phase1.indels.vcf.gz - variation/1000G_phase1.indels.vcf.gz.tbi ================================================ FILE: ggd-recipes/hg38/1000g_omni_snps.yaml ================================================ # 1000 genomes OMNI 2.5 array calls # http://www.1000genomes.org/category/frequently-asked-questions/omni # # From the Broad hg38 bundle: # ftp://gsapubftp-anonymous:none@ftp.broadinstitute.org/bundle/ # https://www.broadinstitute.org/gatk/guide/article.php?id=1213 --- attributes: name: 1000g_omni_snps version: 20160105 recipe: full: recipe_type: bash recipe_cmds: - | url=ftp://gsapubftp-anonymous:none@ftp.broadinstitute.org/bundle/hg38/ new=1000G_omni2.5 base=$new.hg38 mkdir -p variation for suffix in .vcf.gz .vcf.gz.tbi do [[ -f variation/$new$suffix ]] || wget --no-check-certificate -c -O variation/$new$suffix $url/$base$suffix done recipe_outfiles: - variation/1000G_omni2.5.vcf.gz - variation/1000G_omni2.5.vcf.gz.tbi ================================================ FILE: ggd-recipes/hg38/1000g_snps.yaml ================================================ # 1000 genomes phase 1 SNP calls # # From the Broad hg38 bundle: # ftp://gsapubftp-anonymous:none@ftp.broadinstitute.org/bundle/ # https://www.broadinstitute.org/gatk/guide/article.php?id=1213 --- attributes: name: 1000g_snps version: 20160105 recipe: full: recipe_type: bash recipe_cmds: - | url=ftp://gsapubftp-anonymous:none@ftp.broadinstitute.org/bundle/hg38/ new=1000G_phase1.snps.high_confidence base=$new.hg38 mkdir -p variation for suffix in .vcf.gz .vcf.gz.tbi do [[ -f variation/$new$suffix ]] || wget --no-check-certificate -c -O variation/$new$suffix $url/$base$suffix done recipe_outfiles: - variation/1000G_phase1.snps.high_confidence.vcf.gz - variation/1000G_phase1.snps.high_confidence.vcf.gz.tbi ================================================ FILE: ggd-recipes/hg38/ACMG56_genes.yaml ================================================ # ACMG regions: http://iobio.io/2016/03/29/acmg56/ # https://github.com/lpantano/custom_genome_annotation --- attributes: name: ACMG56_genes version: 20160726 recipe: full: recipe_type: bash recipe_cmds: - | baseurl=https://raw.githubusercontent.com/lpantano/custom_genome_annotation/master/acmg56.hg38.bed dir=coverage/prioritize/actionable mkdir -p $dir cd $dir wget --no-check-certificate -c -O ACMG56.bed $baseurl recipe_outfiles: - coverage/prioritize/actionable/ACMG56.bed ================================================ FILE: ggd-recipes/hg38/RADAR.yaml ================================================ # RADAR RNA editing events: (http://www.stanford.edu/~gokulr/database/Human_AG_all_hg19_v2.txt) # installs a BED file of the RADAR events, preconverted to hg38 # how this file was created: # # 1. Download the hg19 RADAR file and converted to BED format. # url=http://www.stanford.edu/~gokulr/database/Human_AG_all_hg19_v2.txt # mkdir -p editing # cd editing # wget -qO- $url | awk 'BEGIN{OFS="\t"} {print $1,$2,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11}' | sed "s/position position/start end/" > RADAR-hg19.bed # cd ../ # 2. Convert RADAR BED file to BED3 format. # cut -f1,2,3 RADAR-hg19.bed > RADAR-hg19.tmp.bed # 3. Get the Crossmap hg19 to hg38 chain file: # wget http://hgdownload.soe.ucsc.edu/goldenPath/hg19/liftOver/hg19ToHg38.over.chain.gz # 4. Run Crossmap: # Crossmap.py bed hg19ToHg38.over.chain.gz RADAR-hg19.tmp.bed > conversions.bed # 5. Merge original RADAR file with the Crossmap output # wget https://raw.githubusercontent.com/roryk/seqscripts/master/radar-convert-annotations.py # python radar-convert-annotations.py RADAR-hg19.bed conversions.bed RADAR-hg38.bed.gz --- attributes: name: RADAR version: v2-20180202 recipe: full: recipe_type: bash recipe_cmds: - | url=https://s3.amazonaws.com/biodata/annotation/RADAR/hg38/RADAR.bed.gz mkdir -p editing cd editing wget --no-check-certificate -q $url wget --no-check-certificate -q $url.tbi recipe_outfiles: - editing/RADAR.bed.gz - editing/RADAR.bed.gz.tbi ================================================ FILE: ggd-recipes/hg38/README.md ================================================ Full hg38/GRCh38 reference genome distributed by 1000 genomes Derived from NCBI set with HLA and decoy alternative alleles http://ftp.1000genomes.ebi.ac.uk/vol1/ftp/technical/reference/GRCh38_reference_genome/ ================================================ FILE: ggd-recipes/hg38/af_only_gnomad.yaml ================================================ # af_only_gnomad file for t-only variant calling with mutect2 # From the Broad hg38 bundle: # ftp://gsapubftp-anonymous@ftp.broadinstitute.org/bundle/Mutect2/af-only-gnomad.hg38.vcf.gz --- attributes: name: af_only_gnomad version: 20180129 recipe: full: recipe_type: bash recipe_cmds: - | url=ftp://gsapubftp-anonymous@ftp.broadinstitute.org/bundle/Mutect2/ base=af-only-gnomad.hg38 new=af_only_gnomad mkdir -p variation for suffix in .vcf.gz .vcf.gz.tbi do [[ -f variation/$new$suffix ]] || wget --no-check-certificate -c -O variation/$new$suffix $url/$base$suffix done recipe_outfiles: - variation/af_only_gnomad.vcf.gz - variation/af_only_gnomad.vcf.gz.tbi ================================================ FILE: ggd-recipes/hg38/bwa.yaml ================================================ # Full hg38/GRCh38 reference genome distributed by 1000 genomes # Derived from NCBI set with HLA and decoy alternative alleles # http://ftp.1000genomes.ebi.ac.uk/vol1/ftp/technical/reference/GRCh38_reference_genome/ # Pre-build bwa indices --- attributes: name: bwa version: 1000g-20150219 recipe: full: recipe_type: bash recipe_cmds: - | url=ftp://ftp.1000genomes.ebi.ac.uk/vol1/ftp/technical/reference/GRCh38_reference_genome base=GRCh38_full_analysis_set_plus_decoy_hla.fa new=hg38.fa mkdir -p bwa for suffix in .bwt .amb .ann .pac .sa .alt do [[ -f bwa/$new$suffix ]] || wget -c -O bwa/$new$suffix $url/$base$suffix done recipe_outfiles: - bwa ================================================ FILE: ggd-recipes/hg38/canonical_cancer_99.txt ================================================ ENSG00000007312 ENST00000006750 ENSG00000012048 ENST00000357654 ENSG00000049618 ENST00000636930 ENSG00000066468 ENST00000358487 ENSG00000068078 ENST00000440486 ENSG00000077782 ENST00000447712 ENSG00000085999 ENST00000371975 ENSG00000087460 ENST00000371085 ENSG00000091831 ENST00000206249 ENSG00000105976 ENST00000397752 ENSG00000115392 ENST00000233741 ENSG00000139144 ENST00000538779 ENSG00000141510 ENST00000269305 ENSG00000142208 ENST00000649815 ENSG00000147889 ENST00000304494 ENSG00000149554 ENST00000438015 ENSG00000168769 ENST00000380013 ENSG00000172936 ENST00000650905 ENSG00000183765 ENST00000404276 ENSG00000185379 ENST00000345365 ENSG00000221914 ENST00000380737 ENSG00000245848 ENST00000498907 ================================================ FILE: ggd-recipes/hg38/capture_regions.yaml ================================================ # Common sets of capture region BED files for different platforms --- attributes: name: capture_regions version: 20161202 recipe: full: recipe_type: bash recipe_cmds: - | baseurl=https://raw.githubusercontent.com/AstraZeneca-NGS/reference_data/master/hg38/bed files="Exome-Agilent_V5_Clinical.bed Exome-Agilent_V6.bed Exome-AZ_V2.bed Exome-IDT_V1.bed Exome-MedExome.bed Exome-NGv3.bed" mkdir -p coverage/capture_regions cd coverage/capture_regions for fname in $files do wget --no-check-certificate -c $baseurl/$fname done recipe_outfiles: - coverage/capture_regions/Exome-Agilent_V5_Clinical.bed - coverage/capture_regions/Exome-Agilent_V6.bed - coverage/capture_regions/Exome-AZ_V2.bed - coverage/capture_regions/Exome-IDT_V1.bed - coverage/capture_regions/Exome-MedExome.bed - coverage/capture_regions/Exome-NGv3.bed ================================================ FILE: ggd-recipes/hg38/ccds.yaml ================================================ # Consensus CDS (CCDS https://www.ncbi.nlm.nih.gov/projects/CCDS/CcdsBrowse.cgi) regions with two adjustments: # # - 2 bps added to internal introns to capture canonical splice acceptor/donor sites # - Multiple transcripts from a single gene are merged into a single all inclusive gene entry. --- attributes: name: ccds version: r20 recipe: full: recipe_type: bash recipe_cmds: - | baseurl=https://s3.amazonaws.com/biodata/coverage/ccds-hg38-r20.bed.gz outdir=coverage/coding mkdir -p $outdir cd $outdir wget --no-check-certificate -c -O ccds.bed.gz $baseurl wget --no-check-certificate -c -O ccds.bed.gz.tbi $baseurl.tbi recipe_outfiles: - coverage/coding/ccds.bed.gz - coverage/coding/ccds.bed.gz.tbi ================================================ FILE: ggd-recipes/hg38/clinvar.yaml ================================================ # ClinVar: http://www.clinvar.com/ # UCSFify name sed magic from: https://github.com/mmarchin/utilities/blob/master/ucscify.sh --- attributes: name: clinvar version: 20210110 recipe: full: recipe_type: bash recipe_cmds: - | release=20210110 # the latest release baseurl=https://ftp.ncbi.nlm.nih.gov/pub/clinvar/vcf_GRCh38/clinvar.vcf.gz mkdir -p variation wget --no-check-certificate -c -O variation/clinvar-orig.vcf.gz $baseurl rm -f variation/clinvar.vcf.gz rm -f variation/clinvar.vcf.gz.tbi zcat variation/clinvar-orig.vcf.gz | sed "s/^\([0-9]\+\)\t/chr\1\t/g" | sed "s/^MT/chrM/g" | sed "s/^X/chrX/g" | sed "s/^Y/chrY/g" | bgzip -c > variation/clinvar.vcf.gz tabix -f -p vcf variation/clinvar.vcf.gz recipe_outfiles: - variation/clinvar.vcf.gz - variation/clinvar.vcf.gz.tbi ================================================ FILE: ggd-recipes/hg38/coverage.yaml ================================================ # Problematic and other coverage regions of interest # # Low complexity regions from supplemental data of Heng Li's work on # reducing variant calling artifacts: # # http://bioinformatics.oxfordjournals.org/content/30/20/2843 # https://github.com/lh3/varcmp/tree/master/scripts # # Centromere and other exclusion regions for SV calling from: # https://github.com/tobiasrausch/delly/blob/master/excludeTemplates/human.hg38.excl.tsv --- attributes: name: coverage version: 2018-10-16 recipe: full: recipe_type: bash recipe_cmds: - | repeats=coverage/problem_regions/repeats mkdir -p $repeats url=https://gist.githubusercontent.com/chapmanb/4c40f961b3ac0a4a22fd/raw/2025f3912a477edc597e61d911bd1044dc943440/sv_repeat_telomere_centromere.bed wget --no-check-certificate -O - $url > $repeats/sv_repeat_telomere_centromere.bed url=https://github.com/lh3/varcmp/raw/bb5b616526c5c3ecb46abfd9877e1bd6d50d1802/scripts/LCR-hs38.bed.gz out=$repeats/LCR.bed.gz wget --no-check-certificate -O $out -c $url tabix -f -p bed $out wget --no-check-certificate -O $repeats/polyx.bed.gz http://s3.amazonaws.com/biodata/coverage/problem_regions/polyx/polyX-hg38.bed.gz wget --no-check-certificate -O $repeats/polyx.bed.gz.tbi http://s3.amazonaws.com/biodata/coverage/problem_regions/polyx/polyX-hg38.bed.gz.tbi gc=coverage/gc mkdir -p $gc wget --no-check-certificate -O - https://s3.amazonaws.com/biodata/collections/hg38/coverage/gc/GC_profile.1000bp.cnp > $gc/GC_profile.1000bp.cnp recipe_outfiles: - coverage/problem_regions/repeats/sv_repeat_telomere_centromere.bed - coverage/problem_regions/repeats/LCR.bed.gz - coverage/problem_regions/repeats/LCR.bed.gz.tbi - coverage/problem_regions/repeats/polyx.bed.gz - coverage/problem_regions/repeats/polyx.bed.gz.tbi - coverage/gc/GC_profile.1000bp.cnp ================================================ FILE: ggd-recipes/hg38/dbnsfp.yaml ================================================ # google drive downloads are unreliable - too many requests, try again in 24h # using ftp ftp://dbnsfp:dbnsfp@dbnsfp.softgenetics.com/dbNSFP4.1a.zip # bumped version because 3.5a is not supported anymore - 25G download instead of 16G --- attributes: name: dbnsfp version: 4.1a recipe: full: recipe_type: bash recipe_cmds: - | mkdir -p variation cd variation wget -c ftp://dbnsfp:dbnsfp@dbnsfp.softgenetics.com/dbNSFP4.1a.zip if [ ! -f dbNSFP.txt.gz ]; then UNPACK_DIR=`pwd`/tmpunpack mkdir -p $UNPACK_DIR 7z e dbNSFP*.zip "dbNSFP*_variant.chrM.gz" gunzip dbNSFP*_variant.chrM.gz head -n1 dbNSFP*_variant.chrM > $UNPACK_DIR/header.txt rm dbNSFP*_variant.chrM # unzip only files with chromosomal info, eg. skip genes and readme. cat $UNPACK_DIR/header.txt > dbNSFP.txt 7z e dbNSFP*.zip "dbNSFP*_variant.chr*.gz" -so | gunzip -c | grep -v '^#chr' | sort -T $UNPACK_DIR -k1,1 -k2,2n >> dbNSFP.txt bgzip dbNSFP.txt #extract readme file, used by VEP plugin to add vcf header info 7z e dbNSFP*.zip "*readme.txt" -so > dbNSFP.readme.txt fi # index in position 1 and 2 tabix -f -s 1 -b 2 -e 2 -c '#' dbNSFP.txt.gz rm -f ${UNPACK_DIR}/* && rmdir $UNPACK_DIR rm -f dbNSFP*.zip recipe_outfiles: - variation/dbNSFP.txt.gz - variation/dbNSFP.txt.gz.tbi - variation/dbNSFP.readme.txt ================================================ FILE: ggd-recipes/hg38/dbscsnv.yaml ================================================ --- attributes: name: dbscsnv version: 1.1 recipe: full: recipe_type: bash recipe_cmds: - | baseurl=ftp://dbnsfp:dbnsfp@dbnsfp.softgenetics.com/dbscSNV1.1.zip mkdir -p variation cd variation UNPACK_DIR=`pwd`/tmp-unpack mkdir -p $UNPACK_DIR if [ ! -f dbscSNV.txt.gz.tbi ]; then wget -c -N $baseurl # [awk] check if hg38 position exists, if so, move data to col 1 and 2, then print # and sort on first and second column unzip -p dbscSNV*.zip "dbscSNV*.chr*" | awk -F $'\t' 'BEGIN {OFS = FS} (FNR==1){print} ($1~/^chr/){next} ($5 != "." && $6 != "."){$1=$5;$2=$6;print | "sort -T "UNPACK_DIR" -k1,1 -k2,2n"}' "UNPACK_DIR=$UNPACK_DIR" | bgzip -c > dbscSNV.txt.gz tabix -s 1 -b 2 -e 2 -S 1 dbscSNV.txt.gz fi rm -f $UNPACK_DIR/* && rm -rf $UNPACK_DIR rm -f dbscSNV*.zip recipe_outfiles: - variation/dbscSNV.txt.gz - variation/dbscSNV.txt.gz.tbi ================================================ FILE: ggd-recipes/hg38/dbsnp.yaml ================================================ # install time: 1h20min # install size: 16G # order of files in recipe_outfiles is important # first come symlinks, than real files. # otherwise real files get copied first, and symlinks become corrupt --- attributes: name: dbsnp version: 156-20231017 recipe: full: recipe_type: bash recipe_cmds: - | build=156 version=GCF_000001405.40 url=http://ftp.ncbi.nih.gov/snp/archive/b$build/VCF/$version.gz remap_url=https://gist.githubusercontent.com/matthdsm/f833aedd2d67e28013ff1d171c70f4ee/raw/442a45ed3ddc6e85c66c5e58e0fa78e16a0821c8/refseq2ucsc.tsv mkdir -p variation cd variation wget -c -O dbsnp-${build}-orig.vcf.gz $url wget -c -O dbsnp-${build}-orig.vcf.gz.tbi $url.tbi [[ -f dbsnp-$build.vcf.gz ]] || bcftools annotate -Ou --rename-chrs $remap_url dbsnp-$build-orig.vcf.gz |\ bcftools sort -m 1G -Oz -T . -o dbsnp-$build.vcf.gz && \ tabix -f -p vcf -C dbsnp-${build}.vcf.gz tabix -f -p vcf dbsnp-${build}.vcf.gz ln -sf dbsnp-${build}.vcf.gz dbsnp.vcf.gz ln -sf dbsnp-${build}.vcf.gz.tbi dbsnp.vcf.gz.tbi ln -sf dbsnp-${build}.vcf.gz.csi dbsnp.vcf.gz.csi cd .. recipe_outfiles: - variation/dbsnp.vcf.gz - variation/dbsnp.vcf.gz.csi - variation/dbsnp.vcf.gz.tbi - variation/dbsnp-156.vcf.gz - variation/dbsnp-156.vcf.gz.csi - variation/dbsnp-156.vcf.gz.tbi ================================================ FILE: ggd-recipes/hg38/dream-syn3-crossmap.yaml ================================================ # Truth sets from the DREAM genomic challenge 3 # https://www.synapse.org/#!Synapse:syn312572/wiki/62018 # # Originally retrieved from https://www.synapse.org/#!Synapse:syn2177211 # and prepared with this script: # https://github.com/chapmanb/bcbio-nextgen/blob/master/scripts/utils/format_dream_truthset.py # # Converted to hg38 coordinates with CrossMap: # https://github.com/hbc/giab_remap_38 --- attributes: name: dream-syn3-crossmap version: 2014-08-04 recipe: full: recipe_type: bash recipe_cmds: - | dir=validation/dream-syn3-crossmap orig=synthetic_challenge_set3_tumor_20pctmasked_truth-crossmap-hg38 mkdir -p $dir wget --no-check-certificate -c https://s3.amazonaws.com/bcbio_nextgen/dream/$orig.tar.gz tar -xzvpf ${orig}.tar.gz cp $orig/${orig}.vcf.gz $dir/truth_small_variants.vcf.gz cp $orig/${orig}.vcf.gz.tbi $dir/truth_small_variants.vcf.gz.tbi cp $orig/${orig}_regions.bed $dir/truth_regions.bed cp $orig/${orig}_sv_DEL.bed $dir/truth_DEL.bed cp $orig/${orig}_sv_DUP.bed $dir/truth_DUP.bed cp $orig/${orig}_sv_INV.bed $dir/truth_INV.bed cp $orig/${orig}_sv_INV.bed $dir/truth_INS.bed recipe_outfiles: - validation/dream-syn3-crossmap/truth_small_variants.vcf.gz - validation/dream-syn3-crossmap/truth_small_variants.vcf.gz.tbi - validation/dream-syn3-crossmap/truth_regions.bed - validation/dream-syn3-crossmap/truth_DEL.bed - validation/dream-syn3-crossmap/truth_DUP.bed - validation/dream-syn3-crossmap/truth_INV.bed - validation/dream-syn3-crossmap/truth_INS.bed ================================================ FILE: ggd-recipes/hg38/dream-syn4-crossmap.yaml ================================================ # Truth sets from the DREAM genomic challenge 4 # https://www.synapse.org/#!Synapse:syn312572/wiki/62018 # # Originally retrieved from https://www.synapse.org/#!Synapse:syn2177211 # and prepared with this script: # https://github.com/chapmanb/bcbio-nextgen/blob/master/scripts/utils/format_dream_truthset.py # # Converted to hg38 coordinates with CrossMap: # https://github.com/hbc/giab_remap_38 --- attributes: name: dream-syn4-crossmap version: 2014-08-09 recipe: full: recipe_type: bash recipe_cmds: - | dir=validation/dream-syn4-crossmap orig=synthetic_challenge_set4_tumour_25pctmasked_truth-crossmap-hg38 mkdir -p $dir wget --no-check-certificate -c https://s3.amazonaws.com/bcbio_nextgen/dream/synthetic_challenge_set4_tumour_25pctmasked_truth-crossmap-hg38.tar.gz tar -xzvpf ${orig}.tar.gz cp $orig/${orig}.vcf.gz $dir/truth_small_variants.vcf.gz cp $orig/${orig}.vcf.gz.tbi $dir/truth_small_variants.vcf.gz.tbi cp $orig/${orig}_regions.bed $dir/truth_regions.bed cp $orig/${orig}_sv_DEL.bed $dir/truth_DEL.bed cp $orig/${orig}_sv_DUP.bed $dir/truth_DUP.bed cp $orig/${orig}_sv_INV.bed $dir/truth_INV.bed recipe_outfiles: - validation/dream-syn4-crossmap/truth_small_variants.vcf.gz - validation/dream-syn4-crossmap/truth_small_variants.vcf.gz.tbi - validation/dream-syn4-crossmap/truth_regions.bed - validation/dream-syn4-crossmap/truth_DEL.bed - validation/dream-syn4-crossmap/truth_DUP.bed - validation/dream-syn4-crossmap/truth_INV.bed ================================================ FILE: ggd-recipes/hg38/effects_transcripts.yaml ================================================ # Custom transcripts for effects prediction # Tweaks canonical transcripts in effects predictors like snpEff. # the new table was created by extracted canonical trascripts from biomaRt: # https://github.com/naumenko-sa/bioscripts/blob/master/gene_panels/genes.R --- attributes: name: effects_transcript version: 2021-12-08 recipe: full: recipe_type: bash recipe_cmds: - | mkdir -p snpeff/transcripts cd snpeff/transcripts wget --no-check-certificate -O canonical_cancer.txt https://raw.githubusercontent.com/chapmanb/cloudbiolinux/master/ggd-recipes/hg38/canonical_cancer_99.txt recipe_outfiles: - snpeff/transcripts/canonical_cancer.txt ================================================ FILE: ggd-recipes/hg38/ericscript.yaml ================================================ --- attributes: name: ericscript version: 84 recipe: full: recipe_type: bash recipe_cmds: - | # Uses Google Drive for faster download with tricks from # http://stackoverflow.com/a/38937732/252589 ggID='0B9s__vuJPvIiUGt1SnFMZFg4TlE' ggURL='https://drive.google.com/uc?export=download' mkdir -p rnaseq/ericscript cd rnaseq/ericscript filename="$(curl -k -sc tmp-gcokie "${ggURL}&id=${ggID}" | grep -o '="uc-name.*' | sed 's/.*">//;s/<.a> .*//')" getcode="$(awk '/_warning_/ {print $NF}' tmp-gcokie)" curl -k -Lb tmp-gcokie "${ggURL}&confirm=${getcode}&id=${ggID}" -o "${filename}" -C - tar -xjvpf ericscript_db_homosapiens_ensembl84.tar.bz2 recipe_outfiles: - rnaseq/ericscript/ericscript_db_homosapiens_ensembl84/data/homo_sapiens/EnsemblGene.Reference.fa.amb - rnaseq/ericscript/ericscript_db_homosapiens_ensembl84/data/homo_sapiens/allseq.fa.pac - rnaseq/ericscript/ericscript_db_homosapiens_ensembl84/data/homo_sapiens/allseq.fa.bwt - rnaseq/ericscript/ericscript_db_homosapiens_ensembl84/data/homo_sapiens/allseq.fa - rnaseq/ericscript/ericscript_db_homosapiens_ensembl84/data/homo_sapiens/EnsemblGene.Sequences.RData - rnaseq/ericscript/ericscript_db_homosapiens_ensembl84/data/homo_sapiens/EnsemblGene.GeneInfo.RData - rnaseq/ericscript/ericscript_db_homosapiens_ensembl84/data/homo_sapiens/EnsemblGene.Reference.fa - rnaseq/ericscript/ericscript_db_homosapiens_ensembl84/data/homo_sapiens/EnsemblGene.GenePosition.RData - rnaseq/ericscript/ericscript_db_homosapiens_ensembl84/data/homo_sapiens/allseq.fa.sa - rnaseq/ericscript/ericscript_db_homosapiens_ensembl84/data/homo_sapiens/allseq.fa.amb - rnaseq/ericscript/ericscript_db_homosapiens_ensembl84/data/homo_sapiens/EnsemblGene.Paralogs.RData - rnaseq/ericscript/ericscript_db_homosapiens_ensembl84/data/homo_sapiens/EnsemblGene.GeneNames.RData - rnaseq/ericscript/ericscript_db_homosapiens_ensembl84/data/homo_sapiens/allseq.fa.ann - rnaseq/ericscript/ericscript_db_homosapiens_ensembl84/data/homo_sapiens/EnsemblGene.Reference.fa.bwt - rnaseq/ericscript/ericscript_db_homosapiens_ensembl84/data/homo_sapiens/EnsemblGene.Reference.fa.sa - rnaseq/ericscript/ericscript_db_homosapiens_ensembl84/data/homo_sapiens/EnsemblGene.Transcripts.RData - rnaseq/ericscript/ericscript_db_homosapiens_ensembl84/data/homo_sapiens/EnsemblGene.Reference.fa.ann - rnaseq/ericscript/ericscript_db_homosapiens_ensembl84/data/homo_sapiens/EnsemblGene.Reference.fa.pac - rnaseq/ericscript/ericscript_db_homosapiens_ensembl84/data/homo_sapiens/EnsemblGene.Structures.RData ================================================ FILE: ggd-recipes/hg38/esp.yaml ================================================ # Exome Sequencing Project (ESP): http://evs.gs.washington.edu/EVS/ # http://evs.gs.washington.edu/evs_bulk_data/ESP6500SI-V2-SSA137.GRCh38-liftover.snps_indels.vcf.tar.gz # Lifted over to hg38 and prepped with utils/prep_esp_hg38.py --- attributes: name: esp version: ESP6500SI-V2-fix recipe: full: recipe_type: bash recipe_cmds: - | url=https://s3.amazonaws.com/biodata/hg38_bundle/ESP6500SI-V2-hg38.vcf.gz mkdir -p variation wget -c -O variation/esp.vcf.gz $url wget -c -O variation/esp.vcf.gz.tbi $url.tbi recipe_outfiles: - variation/esp.vcf.gz - variation/esp.vcf.gz.tbi ================================================ FILE: ggd-recipes/hg38/exac.yaml ================================================ # ExAC VCF sites: http://exac.broadinstitute.org/ # Lifted over to hg38 by the Ensembl variation team # Script cleans up to use chr-style naming and sort by ref file --- attributes: name: exac version: 0.3 recipe: full: recipe_type: bash recipe_cmds: - | url=http://ftp.ensembl.org/pub/data_files/homo_sapiens/GRCh38/variation_genotype/ExAC.0.3.GRCh38.vcf.gz remap_url=http://raw.githubusercontent.com/dpryan79/ChromosomeMappings/master/GRCh38_ensembl2UCSC.txt ref=../seq/hg38.fa mkdir -p variation wget --no-check-certificate -qO- $remap_url | awk '{if($1!=$2) print "s/^"$1"/"$2"/g"}' > remap.sed export TMPDIR=`pwd` wget -c -O - $url | gunzip -c | sed -f remap.sed | grep -v "##contig=" | gsort -m 500 /dev/stdin $ref.fai | vt decompose -s - | vt normalize -r $ref -n - | vt uniq - | bgzip -c > variation/exac.vcf.gz tabix -f -p vcf variation/exac.vcf.gz recipe_outfiles: - variation/exac.vcf.gz - variation/exac.vcf.gz.tbi ================================================ FILE: ggd-recipes/hg38/fusion-blacklist.yaml ================================================ # fusion blacklist regions from arriba --- attributes: name: fusion-blacklist version: 2 recipe: full: recipe_type: bash recipe_cmds: - | version=v1.1.0 url=https://github.com/suhrig/arriba/releases/download/${version}/arriba_${version}.tar.gz arriba_tarball=arriba_${version}.tar.gz outdir=rnaseq/fusion-blacklist mkdir -p ${outdir} wget --no-check-certificate -c -O ${arriba_tarball} ${url} tar zxvf ${arriba_tarball} mv arriba_${version}/database/blacklist_hg38_GRCh38_2018-11-04.tsv.gz ${outdir}/arriba-blacklist.tsv.gz mv arriba_${version}/database/cytobands_hg38_GRCh38_2018-02-23.tsv ${outdir}/arriba-cytobands.tsv mv arriba_${version}/database/protein_domains_hg38_GRCh38_2018-03-06.gff3 ${outdir}/arriba-protein-domains.gff3 mv arriba_${version}/database/protein_domains_hg38_GRCh38_2018-03-06.gff3.idx ${outdir}/arriba-protein-domains.gff3.idx rm -r arriba_${version} rm ${arriba_tarball} recipe_outfiles: - rnaseq/fusion-blacklist/arriba-blacklist.tsv.gz - rnaseq/fusion-blacklist/arriba-cytobands.tsv - rnaseq/fusion-blacklist/arriba-protein-domains.gff3 - rnaseq/fusion-blacklist/arriba-protein-domains.gff3.idx ================================================ FILE: ggd-recipes/hg38/genesplicer.yaml ================================================ --- attributes: name: genesplicer version: 2004.04.03 recipe: full: recipe_type: bash recipe_cmds: - | mkdir -p variation/genesplicer wget ftp://ftp.ccb.jhu.edu/pub/software/genesplicer/GeneSplicer.tar.gz tar -C variation/genesplicer --strip-components=2 -xvf GeneSplicer.tar.gz GeneSplicer/human rm GeneSplicer.tar.gz recipe_outfiles: - variation/genesplicer/acc1.mar - variation/genesplicer/acc14 - variation/genesplicer/acc20 - variation/genesplicer/acc26 - variation/genesplicer/acc33 - variation/genesplicer/acc7 - variation/genesplicer/chgd.fasta - variation/genesplicer/don1.mar - variation/genesplicer/don15 - variation/genesplicer/don19 - variation/genesplicer/don20 - variation/genesplicer/don24 - variation/genesplicer/embls.seq - variation/genesplicer/exons.list - variation/genesplicer/f.don - variation/genesplicer/outin - variation/genesplicer/score_ex.don - variation/genesplicer/score_in.don - variation/genesplicer/acc13 - variation/genesplicer/acc19 - variation/genesplicer/acc25 - variation/genesplicer/acc32 - variation/genesplicer/acc34 - variation/genesplicer/acc8 - variation/genesplicer/config_file - variation/genesplicer/don10 - variation/genesplicer/don16 - variation/genesplicer/don2 - variation/genesplicer/don23 - variation/genesplicer/don9 - variation/genesplicer/exons.dat - variation/genesplicer/f.acc - variation/genesplicer/outex - variation/genesplicer/score_ex.acc - variation/genesplicer/score_in.acc - variation/genesplicer/seqs ================================================ FILE: ggd-recipes/hg38/genotype2phenotype.yaml ================================================ # G2P is a publicly-accessible online system designed to facilitate the development, validation, # curation and distribution of large-scale, evidence-based datasets for use in diagnostic variant filtering. # Each G2P entry associates an allelic requirement and a mutational consequence at a defined locus with a disease entity. # A confidence level and evidence link are assigned to each entry. # https://www.ebi.ac.uk/gene2phenotype/about --- attributes: name: genotype2phenotype version: 20180802 recipe: full: recipe_type: bash recipe_cmds: - | cancer_gene_url="https://www.ebi.ac.uk/gene2phenotype/downloads/CancerG2P.csv.gz" DD_gene_url="https://www.ebi.ac.uk/gene2phenotype/downloads/DDG2P.csv.gz" wget $cancer_gene_url wget $cancer_gene_url zcat *csv.gz > G2P.csv recipe_outfiles: - variation/G2P.csv ================================================ FILE: ggd-recipes/hg38/giab-NA12878-NA24385-somatic.yaml ================================================ # Truth set for NA12878/NA24385 somatic mixture # http://ftp-trace.ncbi.nlm.nih.gov/giab/ftp/use_cases/mixtures/UMCUTRECHT_NA12878_NA24385_mixture_10052016/ --- attributes: name: giab-NA12878-NA24385-somatic version: v3_3_2-2018-07-05 recipe: full: recipe_type: bash recipe_cmds: - | dir=validation/giab-NA12878-NA24385-somatic url=https://s3.amazonaws.com/bcbio/giab/NA12878-NA24385/2018-07-05 calls=na12878-na24385-somatic-hg38-truth.vcf.gz regions=na12878-na24385-somatic-hg38-truth-regions.bed mkdir -p $dir wget -c $url/$calls -O $dir/truth_small_variants.vcf.gz wget -c $url/$calls.tbi -O $dir/truth_small_variants.vcf.gz.tbi wget -c $url/$regions -O $dir/truth_regions.bed recipe_outfiles: - validation/giab-NA12878-NA24385-somatic/truth_small_variants.vcf.gz - validation/giab-NA12878-NA24385-somatic/truth_small_variants.vcf.gz.tbi - validation/giab-NA12878-NA24385-somatic/truth_regions.bed ================================================ FILE: ggd-recipes/hg38/giab-NA12878-crossmap.yaml ================================================ # Truth sets from the Genome in a Bottle project (https://sites.stanford.edu/abms/giab) # Converted to hg38 coordinates using scripts from: # https://github.com/hbc/giab_remap_38 # These are created using CrossMap with UCSC chain files. --- attributes: name: giab-NA12878-crossmap version: v2_19_1 recipe: full: recipe_type: bash recipe_cmds: - | dir=validation/giab-NA12878-crossmap mkdir -p $dir wget --no-check-certificate -c -O $dir/truth_small_variants.vcf.gz https://biodata.s3.amazonaws.com/giab_hg38_remap/GiaB_v2_19-38_crossmap.vcf.gz wget --no-check-certificate -c -O $dir/truth_small_variants.vcf.gz.tbi https://biodata.s3.amazonaws.com/giab_hg38_remap/GiaB_v2_19-38_crossmap.vcf.gz.tbi wget --no-check-certificate -c -O - https://biodata.s3.amazonaws.com/giab_hg38_remap/GiaB_v2_19-38_crossmap-regions.bed | grep -v ^chrM > $dir/truth_regions.bed wget --no-check-certificate -c -O - https://biodata.s3.amazonaws.com/giab_hg38_remap/giab-svclassify-deletions-2015-05-22-crossmap-hg38.bed.gz | gunzip -c > $dir/truth_DEL.bed recipe_outfiles: - validation/giab-NA12878-crossmap/truth_small_variants.vcf.gz - validation/giab-NA12878-crossmap/truth_small_variants.vcf.gz.tbi - validation/giab-NA12878-crossmap/truth_regions.bed - validation/giab-NA12878-crossmap/truth_DEL.bed ================================================ FILE: ggd-recipes/hg38/giab-NA12878-remap.yaml ================================================ # Truth sets from the Genome in a Bottle project (https://sites.stanford.edu/abms/giab) # Converted to hg38 coordinates using scripts from: # https://github.com/hbc/giab_remap_38 # These are created using NCBI's remap --- attributes: name: giab-NA12878-remap version: v2_19_1 recipe: full: recipe_type: bash recipe_cmds: - | dir=validation/giab-NA12878-remap mkdir -p $dir wget --no-check-certificate -c -O $dir/truth_small_variants.vcf.gz https://biodata.s3.amazonaws.com/giab_hg38_remap/GiaB_v2_19-38_remap.vcf.gz wget --no-check-certificate -c -O $dir/truth_small_variants.vcf.gz.tbi https://biodata.s3.amazonaws.com/giab_hg38_remap/GiaB_v2_19-38_remap.vcf.gz.tbi wget --no-check-certificate -c -O - https://biodata.s3.amazonaws.com/giab_hg38_remap/GiaB_v2_19-38_remap-regions.bed | grep -v ^chrM > $dir/truth_regions.bed recipe_outfiles: - validation/giab-NA12878-remap/truth_small_variants.vcf.gz - validation/giab-NA12878-remap/truth_small_variants.vcf.gz.tbi - validation/giab-NA12878-remap/truth_regions.bed ================================================ FILE: ggd-recipes/hg38/giab-NA12878.yaml ================================================ # Truth sets from Genome in a Bottle for NA12878 -- http://genomeinabottle.org/ --- attributes: name: giab-NA12878 version: v3_3_2 recipe: full: recipe_type: bash recipe_cmds: - | dir=validation/giab-NA12878 url=http://ftp-trace.ncbi.nlm.nih.gov/giab/ftp/release/NA12878_HG001/NISTv3.3.2/GRCh38/ calls=HG001_GRCh38_GIAB_highconf_CG-IllFB-IllGATKHC-Ion-10X-SOLID_CHROM1-X_v.3.3.2_highconf_PGandRTGphasetransfer.vcf.gz regions=HG001_GRCh38_GIAB_highconf_CG-IllFB-IllGATKHC-Ion-10X-SOLID_CHROM1-X_v.3.3.2_highconf_nosomaticdel_noCENorHET7.bed mkdir -p $dir wget -c $url/$calls -O $dir/truth_small_variants.vcf.gz tabix -f -p vcf $dir/truth_small_variants.vcf.gz wget -c $url/$regions -O $dir/truth_regions.bed recipe_outfiles: - validation/giab-NA12878/truth_small_variants.vcf.gz - validation/giab-NA12878/truth_small_variants.vcf.gz.tbi - validation/giab-NA12878/truth_regions.bed ================================================ FILE: ggd-recipes/hg38/giab-NA24143.yaml ================================================ # Truth sets from Genome in a Bottle for NA24143 -- http://genomeinabottle.org/ --- attributes: name: giab-NA24143 version: v3_3_2 recipe: full: recipe_type: bash recipe_cmds: - | dir=validation/giab-NA24143 url=https://ftp-trace.ncbi.nlm.nih.gov/giab/ftp/release/AshkenazimTrio/HG004_NA24143_mother/NISTv3.3.2/GRCh38/ calls=HG004_GRCh38_GIAB_highconf_CG-Illfb-IllsentieonHC-Ion-10XsentieonHC_CHROM1-22_v.3.3.2_highconf.vcf.gz regions=HG004_GRCh38_GIAB_highconf_CG-Illfb-IllsentieonHC-Ion-10XsentieonHC_CHROM1-22_v.3.3.2_highconf_noinconsistent.bed mkdir -p $dir wget -c $url/$calls -O $dir/truth_small_variants.vcf.gz tabix -f -p vcf $dir/truth_small_variants.vcf.gz wget -c $url/$regions -O $dir/truth_regions.bed recipe_outfiles: - validation/giab-NA24143/truth_small_variants.vcf.gz - validation/giab-NA24143/truth_small_variants.vcf.gz.tbi - validation/giab-NA24143/truth_regions.bed ================================================ FILE: ggd-recipes/hg38/giab-NA24149.yaml ================================================ # Truth sets from Genome in a Bottle for NA24149 -- http://genomeinabottle.org/ --- attributes: name: giab-NA24149 version: v3_3_2 recipe: full: recipe_type: bash recipe_cmds: - | dir=validation/giab-NA24149 url=https://ftp-trace.ncbi.nlm.nih.gov/giab/ftp/release/AshkenazimTrio/HG003_NA24149_father/NISTv3.3.2/GRCh38/ calls=HG003_GRCh38_GIAB_highconf_CG-Illfb-IllsentieonHC-Ion-10XsentieonHC_CHROM1-22_v.3.3.2_highconf.vcf.gz regions=HG003_GRCh38_GIAB_highconf_CG-Illfb-IllsentieonHC-Ion-10XsentieonHC_CHROM1-22_v.3.3.2_highconf_noinconsistent.bed mkdir -p $dir wget -c $url/$calls -O $dir/truth_small_variants.vcf.gz tabix -f -p vcf $dir/truth_small_variants.vcf.gz wget -c $url/$regions -O $dir/truth_regions.bed recipe_outfiles: - validation/giab-NA24149/truth_small_variants.vcf.gz - validation/giab-NA24149/truth_small_variants.vcf.gz.tbi - validation/giab-NA24149/truth_regions.bed ================================================ FILE: ggd-recipes/hg38/giab-NA24385.yaml ================================================ # Truth sets from Genome in a Bottle for NA24385 -- http://genomeinabottle.org/ --- attributes: name: giab-NA24385 version: v3_3_2 recipe: full: recipe_type: bash recipe_cmds: - | dir=validation/giab-NA24385 url=http://ftp-trace.ncbi.nlm.nih.gov/giab/ftp/release/AshkenazimTrio/HG002_NA24385_son/NISTv3.3.2/GRCh38/ calls=HG002_GRCh38_GIAB_highconf_CG-Illfb-IllsentieonHC-Ion-10XsentieonHC-SOLIDgatkHC_CHROM1-22_v.3.3.2_highconf_triophased.vcf.gz regions=HG002_GRCh38_GIAB_highconf_CG-Illfb-IllsentieonHC-Ion-10XsentieonHC-SOLIDgatkHC_CHROM1-22_v.3.3.2_highconf_noinconsistent.bed mkdir -p $dir wget -c $url/$calls -O $dir/truth_small_variants.vcf.gz tabix -f -p vcf $dir/truth_small_variants.vcf.gz wget -c $url/$regions -O $dir/truth_regions.bed recipe_outfiles: - validation/giab-NA24385/truth_small_variants.vcf.gz - validation/giab-NA24385/truth_small_variants.vcf.gz.tbi - validation/giab-NA24385/truth_regions.bed ================================================ FILE: ggd-recipes/hg38/giab-NA24631.yaml ================================================ # Truth sets from Genome in a Bottle for NA24631 -- http://genomeinabottle.org/ --- attributes: name: giab-NA24631 version: v3_3_2 recipe: full: recipe_type: bash recipe_cmds: - | dir=validation/giab-NA24631 url=http://ftp-trace.ncbi.nlm.nih.gov/giab/ftp/release/ChineseTrio/HG005_NA24631_son/NISTv3.3.2/GRCh38/ calls=HG005_GRCh38_GIAB_highconf_CG-Illfb-IllsentieonHC-Ion-10XsentieonHC-SOLIDgatkHC_CHROM1-22_v.3.3.2_highconf.vcf.gz regions=HG005_GRCh38_GIAB_highconf_CG-Illfb-IllsentieonHC-Ion-10XsentieonHC-SOLIDgatkHC_CHROM1-22_v.3.3.2_highconf.bed mkdir -p $dir wget -c $url/$calls -O $dir/truth_small_variants.vcf.gz tabix -f -p vcf $dir/truth_small_variants.vcf.gz wget -c $url/$regions -O $dir/truth_regions.bed recipe_outfiles: - validation/giab-NA24631/truth_small_variants.vcf.gz - validation/giab-NA24631/truth_small_variants.vcf.gz.tbi - validation/giab-NA24631/truth_regions.bed ================================================ FILE: ggd-recipes/hg38/giab-NA24694.yaml ================================================ # Truth sets from Genome in a Bottle for NA24631 -- http://genomeinabottle.org/ --- attributes: name: giab-NA24694 version: v3_3_2 recipe: full: recipe_type: bash recipe_cmds: - | dir=validation/giab-NA24694 url=https://ftp-trace.ncbi.nlm.nih.gov/giab/ftp/release/ChineseTrio/HG006_NA24694_father/NISTv3.3.2/GRCh38/ calls=HG006_GIAB_GRCh38_highconf_CG-IllFB-IllSNT-10X_CHROM1-22_v.3.3.2_highconf.vcf.gz regions=HG006_GIAB_GRCh38_highconf_CG-IllFB-IllSNT-10X_CHROM1-22_v.3.3.2_highconf_noinconsistent.bed mkdir -p $dir wget -c $url/$calls -O $dir/truth_small_variants.vcf.gz tabix -f -p vcf $dir/truth_small_variants.vcf.gz wget -c $url/$regions -O $dir/truth_regions.bed recipe_outfiles: - validation/giab-NA24694/truth_small_variants.vcf.gz - validation/giab-NA24694/truth_small_variants.vcf.gz.tbi - validation/giab-NA24694/truth_regions.bed ================================================ FILE: ggd-recipes/hg38/giab-NA24695.yaml ================================================ # Truth sets from Genome in a Bottle for NA24631 -- http://genomeinabottle.org/ --- attributes: name: giab-NA24695 version: v3_3_2 recipe: full: recipe_type: bash recipe_cmds: - | dir=validation/giab-NA24695 url=https://ftp-trace.ncbi.nlm.nih.gov/giab/ftp/release/ChineseTrio/HG007_NA24695_mother/NISTv3.3.2/GRCh38/ calls=HG007_GIAB_GRCh38_highconf_CG-IllFB-IllSNT-10X_CHROM1-22_v.3.3.2_highconf.vcf.gz regions=HG007_GIAB_GRCh38_highconf_CG-IllFB-IllSNT-10X_CHROM1-22_v.3.3.2_highconf_noinconsistent.bed mkdir -p $dir wget -c $url/$calls -O $dir/truth_small_variants.vcf.gz tabix -f -p vcf $dir/truth_small_variants.vcf.gz wget -c $url/$regions -O $dir/truth_regions.bed recipe_outfiles: - validation/giab-NA24695/truth_small_variants.vcf.gz - validation/giab-NA24695/truth_small_variants.vcf.gz.tbi - validation/giab-NA24695/truth_regions.bed ================================================ FILE: ggd-recipes/hg38/gnomad.yaml ================================================ # GnomAD VCF sites: # https://gnomad.broadinstitute.org/downloads # Using only PASS variants, removing some fields to make the vcf smaller # recipe test: https://github.com/chapmanb/cloudbiolinux/blob/master/doc/hacking.md#testing-a-ggd-recipe # using gnomad v3 which has WGS only bug aligned to hg38 # gnomad v2 is WES aligned to hg19 + liftover - not using it # download from google cloud with gsutil # processes 241G vcf.gz to 132G vcf.gz in < 6 days - old estimate # gnomad fields changes in 2023 - no version update to avoid triggering gnomad reinstalls --- attributes: name: gnomad version: 4 recipe: full: recipe_type: bash recipe_cmds: - | vcf_prefix=gnomad.genomes.v3.1.sites mkdir -p variation export TMPDIR=`pwd` gnomad_fields_to_keep_url=https://gist.githubusercontent.com/naumenko-sa/7b83cb718ffeadb75ae61676f2158de9/raw/e6d4e39fa2744e1271d3fce0613808d167c158e4/gnomad_fields_to_keep_2023.txt wget $gnomad_fields_to_keep_url gsutil cp gs://gcp-public-data--gnomad/release/3.1/vcf/genomes/*sites.chr*.vcf.bgz . gsutil cp gs://gcp-public-data--gnomad/release/3.1/vcf/genomes/*sites.chr*.vcf.bgz.tbi . fields_to_keep="INFO/"$(cat gnomad_fields_to_keep_2023.txt | paste -s | sed s/"\t"/",INFO\/"/g) ref=../seq/hg38.fa for f in `seq 1 22` X Y M do vcf_file=${vcf_prefix}.chr${f}.vcf.bgz bcftools view -f PASS $vcf_file | bcftools annotate -x "^$fields_to_keep" -Ov | vt decompose -s - | vt normalize -r $ref -n - | vt uniq - | bgzip -c > gg.chr${f}.vcf.gz tabix gg.chr${f}.vcf.gz echo gg.chr${f}.vcf.gz >> gnomad_to_merge.txt done bcftools merge -l gnomad_to_merge.txt -Oz > variation/gnomad_genome.vcf.gz tabix -f -p vcf variation/gnomad_genome.vcf.gz tabix -f -p vcf --csi variation/gnomad_genome.vcf.gz rm ${vcf_prefix}*.vcf.bgz rm ${vcf_prefix}*.vcf.bgz.tbi rm gg.chr*.vcf.gz rm gg.chr*.vcf.gz.tbi recipe_outfiles: - variation/gnomad_genome.vcf.gz - variation/gnomad_genome.vcf.gz.tbi - variation/gnomad_genome.vcf.gz.csi ================================================ FILE: ggd-recipes/hg38/gnomad_exome.yaml ================================================ # GnomAD VCF sites: # - https://gnomad.broadinstitute.org/downloads # - https://macarthurlab.org/2018/10/17/gnomad-v2-1/ # Lifted over to hg38 by Gnomad team # Script # - filters only PASS variants (segdup, decoy are retained) # - removes many INFO fields to reduce file size: controls_, hist, non_, using bcftools annotate instead of vt rminfo, because it failed randomly on chr1 # recipe test: https://github.com/chapmanb/cloudbiolinux/blob/master/doc/hacking.md#testing-a-ggd-recipe # much faster from gnomad in the google cloud - download in 20-25 min on O2 # vt is 3x faster than bcftools for removing INFO tags: in a test with 4444 variants t(bcftools -x) = t(bcftools -x "^") = 27s, time(vt rminfo) = 8s, # but vt rminfo is failing after 148M, using bcftools # processing time: 87G to 4G in 7h # there are no native hg38 exome calls in gnomad.v3 - only for genomes, so exome variants are still lifted over from grch37, # and they need processing because liftover creates duplicated variants # receipe version is the same - the old installations won't be reinstalled --- attributes: name: gnomad_exome version: 2.1.2 recipe: full: recipe_type: bash recipe_cmds: - | gsutil cp gs://gcp-public-data--gnomad/release/2.1.1/liftover_grch38/vcf/exomes/gnomad.exomes.r2.1.1.sites.liftover_grch38.vcf.bgz . gsutil cp gs://gcp-public-data--gnomad/release/2.1.1/liftover_grch38/vcf/exomes/gnomad.exomes.r2.1.1.sites.liftover_grch38.vcf.bgz.tbi . vcf_file=gnomad.exomes.r2.1.1.sites.liftover_grch38.vcf.bgz mkdir -p variation export TMPDIR=`pwd` gnomad_fields_to_keep_url=https://raw.githubusercontent.com/chapmanb/cloudbiolinux/master/ggd-recipes/hg38/gnomad_fields_to_keep.txt wget --no-check-certificate -c $gnomad_fields_to_keep_url fields_to_keep="INFO/"$(cat gnomad_fields_to_keep.txt | paste -s | sed s/"\t"/",INFO\/"/g) ref=../seq/hg38.fa bcftools view -f PASS $vcf_file | bcftools annotate -x "^$fields_to_keep" -Ov | vt decompose -s - | vt normalize -r $ref -n - | vt uniq - | bgzip -c > variation/gnomad_exome.vcf.gz tabix -f -p vcf variation/gnomad_exome.vcf.gz tabix -f -p vcf --csi variation/gnomad_exome.vcf.gz rm $vcf_file $vcf_file.tbi recipe_outfiles: - variation/gnomad_exome.vcf.gz - variation/gnomad_exome.vcf.gz.tbi - variation/gnomad_exome.vcf.gz.csi ================================================ FILE: ggd-recipes/hg38/gnomad_fields_to_keep.txt ================================================ AC AN AF FS InbreedingCoeff MQ MQRankSum QD ReadPosRankSum SOR BaseQRankSum ClippingRankSum DP variant_type allele_type n_alt_alleles AC_nfe_seu AN_nfe_seu AF_nfe_seu nhomalt_nfe_seu AC_raw AN_raw AF_raw nhomalt_raw AC_fin_female AN_fin_female AF_fin_female nhomalt_fin_female AC_nfe_bgr AN_nfe_bgr AF_nfe_bgr nhomalt_nfe_bgr AC_sas_male AN_sas_male AF_sas_male nhomalt_sas_male AC_afr_male AN_afr_male AF_afr_male nhomalt_afr_male AC_afr AN_afr AF_afr nhomalt_afr AC_eas_female AN_eas_female AF_eas_female nhomalt_eas_female AC_afr_female AN_afr_female AF_afr_female nhomalt_afr_female AC_sas AN_sas AF_sas nhomalt_sas AC_nfe_onf AN_nfe_onf AF_nfe_onf nhomalt_nfe_onf AC_fin_male AN_fin_male AF_fin_male nhomalt_fin_male AC_nfe_female AN_nfe_female AF_nfe_female nhomalt_nfe_female AC_amr AN_amr AF_amr nhomalt_amr AC_eas AN_eas AF_eas nhomalt_eas nhomalt AC_asj_male AN_asj_male AF_asj_male nhomalt_asj_male AC_oth_female AN_oth_female AF_oth_female nhomalt_oth_female AC_nfe_swe AN_nfe_swe AF_nfe_swe nhomalt_nfe_swe AC_nfe_nwe AN_nfe_nwe AF_nfe_nwe nhomalt_nfe_nwe AC_eas_jpn AN_eas_jpn AF_eas_jpn nhomalt_eas_jpn AC_female AN_female AF_female nhomalt_female AC_eas_kor AN_eas_kor AF_eas_kor nhomalt_eas_kor AC_eas_oea AN_eas_oea AF_eas_oea nhomalt_eas_oea AC_nfe_est AN_nfe_est AF_nfe_est nhomalt_nfe_est AC_eas_male AN_eas_male AF_eas_male nhomalt_eas_male AC_nfe AN_nfe AF_nfe nhomalt_nfe AC_fin AN_fin AF_fin nhomalt_fin AC_nfe_male AN_nfe_male AF_nfe_male nhomalt_nfe_male AC_sas_female AN_sas_female AF_sas_female nhomalt_sas_female AC_asj_female AN_asj_female AF_asj_female nhomalt_asj_female AC_asj AN_asj AF_asj nhomalt_asj AC_oth AN_oth AF_oth nhomalt_oth AC_male AN_male AF_male nhomalt_male AC_amr_male AN_amr_male AF_amr_male nhomalt_amr_male AC_amr_female AN_amr_female AF_amr_female nhomalt_amr_female AC_oth_male AN_oth_male AF_oth_male nhomalt_oth_male faf95 faf99 popmax AC_popmax AN_popmax AF_popmax nhomalt_popmax ================================================ FILE: ggd-recipes/hg38/gtf.yaml ================================================ # Ensembl GTF file distributed by Ensembl for hg38 # Cleans GTF file by converting chromosome names to standard names # Uses https://github.com/dpryan79/ChromosomeMappings to remap the chromosome names --- attributes: name: gtf version: 78 recipe: full: recipe_type: bash recipe_cmds: - | url=http://ftp.ensembl.org/pub/release-78/gtf/homo_sapiens/Homo_sapiens.GRCh38.78.gtf.gz mkdir -p rnaseq remap_url=http://raw.githubusercontent.com/dpryan79/ChromosomeMappings/master/GRCh38_ensembl2UCSC.txt wget --no-check-certificate -qO- $remap_url | awk '{if($1!=$2) print "s/^"$1"/"$2"/g"}' > remap.sed wget --no-check-certificate -qO- $url | gunzip | sed -f remap.sed | grep -v "*_*_alt" > rnaseq/hg38.gtf rm remap.sed recipe_outfiles: - rnaseq/hg38.gtf ================================================ FILE: ggd-recipes/hg38/hapmap_snps.yaml ================================================ # HapMap v3.3 SNP calls # # From the Broad hg38 bundle: # ftp://gsapubftp-anonymous:none@ftp.broadinstitute.org/bundle/ # https://www.broadinstitute.org/gatk/guide/article.php?id=1213 --- attributes: name: hapmap_snps version: 20160105 recipe: full: recipe_type: bash recipe_cmds: - | url=ftp://gsapubftp-anonymous:none@ftp.broadinstitute.org/bundle/hg38/ new=hapmap_3.3 base=$new.hg38 mkdir -p variation for suffix in .vcf.gz .vcf.gz.tbi do [[ -f variation/$new$suffix ]] || wget --no-check-certificate -c -O variation/$new$suffix $url/$base$suffix done recipe_outfiles: - variation/hapmap_3.3.vcf.gz - variation/hapmap_3.3.vcf.gz.tbi ================================================ FILE: ggd-recipes/hg38/hisat2.yaml ================================================ # hisat2 (v2.0.1) reference with Ensembl release 78 exon and splicesite annotations # Derived from NCBI set with HLA and decoy alternative alleles # http://ftp.1000genomes.ebi.ac.uk/vol1/ftp/technical/reference/GRCh38_reference_genome/ # README.md has an in depth description of how this was created --- attributes: name: hisat2 version: 12-07-2015 recipe: full: recipe_type: bash recipe_cmds: - | url=https://s3.amazonaws.com/biodata/genomes/hg38-hisat2-12-07-2015.tar.xz wget -qO- $url > hg38-hisat2-tar.xz tar Jxvf hg38-hisat2-tar.xz rm hg38-hisat2-tar.xz recipe_outfiles: - hisat2/hg38.1.ht2 - hisat2/hg38.2.ht2 - hisat2/hg38.3.ht2 - hisat2/hg38.4.ht2 - hisat2/hg38.5.ht2 - hisat2/hg38.6.ht2 - hisat2/hg38.7.ht2 - hisat2/hg38.8.ht2 - hisat2/README.md ================================================ FILE: ggd-recipes/hg38/mills_indels.yaml ================================================ # Curated training indels from Mills et al: # # http://genome.cshlp.org/content/21/6/830.full # # From the Broad hg38 bundle: # ftp://gsapubftp-anonymous:none@ftp.broadinstitute.org/bundle/ # https://www.broadinstitute.org/gatk/guide/article.php?id=1213 --- attributes: name: mills_indels version: 20160105 recipe: full: recipe_type: bash recipe_cmds: - | url=ftp://gsapubftp-anonymous:none@ftp.broadinstitute.org/bundle/hg38/ base=Mills_and_1000G_gold_standard.indels.hg38 new=Mills_and_1000G_gold_standard.indels mkdir -p variation for suffix in .vcf.gz .vcf.gz.tbi do [[ -f variation/$new$suffix ]] || wget --no-check-certificate -c -O variation/$new$suffix $url/$base$suffix done recipe_outfiles: - variation/Mills_and_1000G_gold_standard.indels.vcf.gz - variation/Mills_and_1000G_gold_standard.indels.vcf.gz.tbi ================================================ FILE: ggd-recipes/hg38/mirbase.yaml ================================================ # small RNA seq annotation from different sources --- attributes: name: srnaseq version: 20211104 recipe: full: recipe_type: bash recipe_cmds: - | mkdir -p srnaseq cd srnaseq wget --random-wait --retry-connrefused -q -c -O hsa.gff3 https://mirbase.org/download/hsa.gff3 awk '$3=="miRNA"' hsa.gff3 | sed 's/=/ /g' > srna-transcripts.gtf wget --random-wait --retry-connrefused -q -c -O wgRna.txt.gz http://hgdownload.soe.ucsc.edu/goldenPath/hg38/database/wgRna.txt.gz zgrep -v 'hsa-' wgRna.txt.gz | awk '{print $2"\t.\tncrna\t"$3"\t"$4"\t.\t"$7"\t.\tname "$5";"}' >> srna-transcripts.gtf wget --random-wait --retry-connrefused -q -c -O tRNAs.txt.gz http://hgdownload.soe.ucsc.edu/goldenPath/hg38/database/tRNAs.txt.gz zcat tRNAs.txt.gz | awk '{print $2"\t.\ttRNA\t"$3"\t"$4"\t.\t"$7"\t.\tname "$5";"}' >> srna-transcripts.gtf wget --random-wait --retry-connrefused -q -c -O rmsk.txt.gz http://hgdownload.soe.ucsc.edu/goldenPath/hg38/database/rmsk.txt.gz zcat rmsk.txt.gz | awk '{print $6"\t.\trepeat\t"$7+1"\t"$8+1"\t.\t"$10"\t.\tname "$12";"}' >> srna-transcripts.gtf wget --random-wait --retry-connrefused -q -c -O refGene.txt.gz http://hgdownload.soe.ucsc.edu/goldenPath/hg38/database/refGene.txt.gz zcat refGene.txt.gz | awk '{print $3"\t.\tgene\t"$5"\t"$6"\t.\t"$4"\t.\tname "$13";"}' >> srna-transcripts.gtf sed -i 's/\t0\t/\t1\t/' srna-transcripts.gtf # wget http://www.regulatoryrna.org/database/piRNA/download/archive/v1.0/bed/piR_hg19_v1.0.bed.gz # zcat piR_hg19_v1.0.bed.gz | awk '{print $1"\t.\tpiRNA\t"$2"\t"$3"\t.\t"$6"\t.\tname "$4";"}' >> srna-transcripts.gtf # mirbase wget --random-wait --retry-connrefused -q -c -O hairpin.fa.raw https://mirbase.org/download/CURRENT/hairpin.fa cat hairpin.fa.raw | awk '{if ($0~/>hsa/){name=$0; print name} else if ($0~/^>/){name=0};if (name!=0 && $0!~/^>/){print $0;}}' | sed 's/U/T/g' > hairpin.fa rm hairpin.fa.raw wget --random-wait --retry-connrefused -q -c -O mature.fa.raw https://mirbase.org/download/CURRENT/mature.fa cat mature.fa.raw | awk '{if ($0~/>hsa/){name=$0; print name} else if ($0~/^>/){name=0};if (name!=0 && $0!~/^>/){print $0;}}' | sed 's/U/T/g' > mature.fa rm mature.fa.raw wget --random-wait --retry-connrefused -q -c -O miRNA.dat.raw https://mirbase.org/download/CURRENT/miRNA.dat cat miRNA.dat.raw | awk '{if ($0~/hsa/)print $0}' > miRNA.str rm miRNA.dat.raw wget --random-wait --retry-connrefused -q -c -O mirbase.gff3 https://mirbase.org/download/hsa.gff3 #tdrmapper wget --random-wait --retry-connrefused --no-check-certificate -q -c -O trna_mature_pre.fa https://github.com/sararselitsky/tDRmapper/raw/master/hg19_mature_and_pre.fa # mintmap wget --random-wait --retry-connrefused -nv -c https://github.com/TJU-CMC-Org/MINTmap/archive/refs/tags/MINTmap_Standard_v1.0.tar.gz tar xzf MINTmap_Standard_v1.0.tar.gz MINTmap-MINTmap_Standard_v1.0/LookupTable.tRFs.MINTmap_v1.txt --strip-components=1 tar xzf MINTmap_Standard_v1.0.tar.gz MINTmap-MINTmap_Standard_v1.0/tRNAspace.Spliced.Sequences.MINTmap_v1.fa --strip-components=1 tar xzf MINTmap_Standard_v1.0.tar.gz MINTmap-MINTmap_Standard_v1.0/OtherAnnotations.MINTmap_v1.txt --strip-components=1 # mirdeep2 wget --random-wait --retry-connrefused --no-check-certificate -q -c -O Rfam_for_miRDeep.fa.gz https://github.com/lpantano/mirdeep2_core/raw/data/Rfam_for_miRDeep.fa.gz && gunzip Rfam_for_miRDeep.fa.gz # targetscan analysis recipe_outfiles: - srnaseq/srna-transcripts.gtf - srnaseq/hairpin.fa - srnaseq/mature.fa - srnaseq/miRNA.str - srnaseq/mirbase.gff3 - srnaseq/trna_mature_pre.fa - srnaseq/LookupTable.tRFs.MINTmap_v1.txt - srnaseq/tRNAspace.Spliced.Sequences.MINTmap_v1.fa - srnaseq/OtherAnnotations.MINTmap_v1.txt - srnaseq/Rfam_for_miRDeep.fa ================================================ FILE: ggd-recipes/hg38/platinum-genome-NA12878.yaml ================================================ # Truth sets from Illumina Platinum Genomes project # http://www.illumina.com/platinumgenomes/ --- attributes: name: platinum-genome-NA12878 version: v2_0_1 recipe: full: recipe_type: bash recipe_cmds: - | version=v2_0_1 dir=validation/platinum-genome-NA12878 mkdir -p $dir wget --no-check-certificate -c -O $dir/truth_small_variants.vcf.gz https://s3.amazonaws.com/cloudbiolinux/cache/platinum-genome/platinum-genome-NA12878-hg38-$version.vcf.gz wget --no-check-certificate -c -O $dir/truth_small_variants.vcf.gz.tbi https://s3.amazonaws.com/cloudbiolinux/cache/platinum-genome/platinum-genome-NA12878-hg38-$version.vcf.gz.tbi wget --no-check-certificate -c -O - https://s3.amazonaws.com/cloudbiolinux/cache/platinum-genome/platinum-genome-NA12878-hg38-regions-$version.bed.gz | gunzip -c > $dir/truth_regions.bed recipe_outfiles: - validation/platinum-genome-NA12878/truth_small_variants.vcf.gz - validation/platinum-genome-NA12878/truth_small_variants.vcf.gz.tbi - validation/platinum-genome-NA12878/truth_regions.bed ================================================ FILE: ggd-recipes/hg38/prioritize.yaml ================================================ # Regions for prioritization: # cancer # - civic -- known cancer genes from the CIViC database # - az300 -- list of 300 cancer genes of interest from AstraZeneca Oncology # - az300-with-fusion -- az300 list plus known interacting fusions # from FusionCatcher https://github.com/ndaniel/fusioncatcher/blob/master/bin/generate_known.py#L102 # - az-cancer-panel -- az300 genes plus additional useful panel genes --- attributes: name: prioritize version: 20181227 recipe: full: recipe_type: bash recipe_cmds: - | baseurl=https://s3.amazonaws.com/biodata/coverage/prioritize/prioritize-cancer-hg38-20181227.tar.gz outdir=coverage/prioritize mkdir -p $outdir cd $outdir wget --no-check-certificate -c -O cancer.tar.gz $baseurl tar -xzvpf cancer.tar.gz recipe_outfiles: - coverage/prioritize/cancer/az300.bed.gz - coverage/prioritize/cancer/az300.bed.gz.tbi - coverage/prioritize/cancer/az300-with-fusion.bed.gz - coverage/prioritize/cancer/az300-with-fusion.bed.gz.tbi - coverage/prioritize/cancer/az-cancer-panel.bed.gz - coverage/prioritize/cancer/az-cancer-panel.bed.gz.tbi - coverage/prioritize/cancer/civic-2018-12-27.bed.gz - coverage/prioritize/cancer/civic-2018-12-27.bed.gz.tbi ================================================ FILE: ggd-recipes/hg38/purecn_mappability.yaml ================================================ # mappability regions for purecn analyses: # http://bioconductor.org/packages/release/bioc/vignettes/PureCN/inst/doc/Quick.html # file size: 1.5G --- attributes: name: purecn_mappability version: 20200928 recipe: full: recipe_type: bash recipe_cmds: - | mkdir -p coverage/mappability cd coverage/mappability wget --no-check-certificate -c https://s3.amazonaws.com/purecn/GCA_000001405.15_GRCh38_no_alt_analysis_set_100.bw cd .. recipe_outfiles: - coverage/mappability/GCA_000001405.15_GRCh38_no_alt_analysis_set_100.bw ================================================ FILE: ggd-recipes/hg38/qsignature.yaml ================================================ --- attributes: name: qsignature version: 20160526 recipe: full: recipe_type: bash recipe_cmds: - | baseurl=http://github.com/lpantano/custom_genome_annotation/raw/master/qsignature.hg38.tbz mkdir -p variation cd variation wget -N -c --no-check-certificate $baseurl tar xjvf qsignature.hg38.tbz mv qsignature.hg38.txt qsignature.txt recipe_outfiles: - variation/qsignature.txt ================================================ FILE: ggd-recipes/hg38/rmsk.yaml ================================================ # repeat elements location on the genome --- attributes: name: rmsk version: 20180319 recipe: full: recipe_type: bash recipe_cmds: - | mkdir -p coverage/problem_regions/repeats cd coverage/problem_regions/repeats wget --random-wait --retry-connrefused -q -c -O rmsk.txt.gz http://hgdownload.soe.ucsc.edu/goldenPath/hg38/database/rmsk.txt.gz zcat rmsk.txt.gz | awk '{print $6"\t.\trepeat\t"$7+1"\t"$8+1"\t.\t"$10"\t.\tname "$12";"}' | bgzip > rmsk.gtf.gz tabix rmsk.gtf.gz recipe_outfiles: - coverage/problem_regions/repeats/rmsk.gtf.gz - coverage/problem_regions/repeats/rmsk.gtf.gz.tbi ================================================ FILE: ggd-recipes/hg38/salmon-decoys.yaml ================================================ # Decoys for SA mode of Salmon # built against hg38 with the alt alleles # Ensembl gene build 94 --- attributes: name: salmon-decoys version: 94 recipe: full: recipe_type: bash recipe_cmds: - | url=https://s3.amazonaws.com/biodata/annotation/hg38-build94-salmon-decoys.tar.gz outdir=rnaseq/salmon-decoys mkdir -p ${outdir} wget -c -O decoys.tar.gz ${url} tar zxvf decoys.tar.gz mv decoys.fa ${outdir}/. mv decoys.txt ${outdir}/. rm decoys.tar.gz recipe_outfiles: - rnaseq/salmon-decoys/decoys.fa - rnaseq/salmon-decoys/decoys.txt ================================================ FILE: ggd-recipes/hg38/seq.yaml ================================================ # Full hg38/GRCh38 reference genome distributed by 1000 genomes # Derived from NCBI set with HLA and decoy alternative alleles # ftp://ftp.1000genomes.ebi.ac.uk/vol1/ftp/technical/reference/GRCh38_reference_genome/ --- attributes: name: seq version: 1000g-20150219_1 recipe: full: recipe_type: bash recipe_cmds: - | url=ftp://ftp.1000genomes.ebi.ac.uk/vol1/ftp/technical/reference/GRCh38_reference_genome base=GRCh38_full_analysis_set_plus_decoy_hla new=hg38 mkdir -p seq for suffix in .dict .fa.fai do [[ -f seq/$new$suffix ]] || wget -c -O seq/$new$suffix $url/$base$suffix done # gzipped references for use with ensembl-vep HGVS urlgz=https://s3.amazonaws.com/biodata/hg38_bundle for suffix in .fa.gz .fa.gz.fai .fa.gz.gzi do [[ -f seq/$new$suffix ]] || wget --no-check-certificate -c -O seq/$new$suffix $urlgz/$new$suffix done gunzip -c seq/hg38.fa.gz > seq/hg38.fa touch seq/hg38.fa.fai touch seq/hg38.dict recipe_outfiles: - seq/hg38.fa - seq/hg38.fa.fai - seq/hg38.fa.gz - seq/hg38.fa.gz.fai - seq/hg38.fa.gz.gzi - seq/hg38.dict ================================================ FILE: ggd-recipes/hg38/simple_repeat.yaml ================================================ # UCSC repeats track for purecn variant filtration: --- attributes: name: simple_repeat version: 20200928 recipe: full: recipe_type: bash recipe_cmds: - | mkdir -p coverage/problem_regions/repeats cd coverage/problem_regions/repeats wget --random-wait --retry-connrefused -q -c -O simple_repeat.txt.gz http://hgdownload.soe.ucsc.edu/goldenPath/hg38/database/simpleRepeat.txt.gz zcat simple_repeat.txt.gz | awk -F '\t' '{print $2"\t"$3"\t"$4"\t"$5"\t"$11"\t."}' > simple_repeat.bed recipe_outfiles: - coverage/problem_regions/repeats/simple_repeat.bed ================================================ FILE: ggd-recipes/hg38/topmed.yaml ================================================ # Topmed WGS allele frequencies: https://bravo.sph.umich.edu/freeze5/hg38/ # topmed vcf from ensembl does not have contig names # # made via: # url=ftp://ftp.ensembl.org/pub/data_files/homo_sapiens/GRCh38/variation_genotype/TOPMED_GRCh38_20180418.vcf.gz # remap_url=http://raw.githubusercontent.com/dpryan79/ChromosomeMappings/master/GRCh38_ensembl2UCSC.txt # ref=hg38.fa # wget --no-check-certificate -qO- $remap_url | awk '{ print length, $0 }' | sort -n -s -r | cut -d" " -f2- | awk '{if(!$2) print "/^"$1"/d"; else if($1!=$2) print "s/^"$1"/"$2"/g";}' > remap.sed # mkdir -p variation # export TMPDIR=`pwd` # wget --no-check-certificate -qO- ${url} > topmed.vcf.gz # gunzip -c topmed.vcf.gz | sed -f remap.sed | sed s/Number=\./Number=A/g | vt normalize -r $ref -n - | vt uniq - | gsort -m 5000 /dev/stdin $ref.fai | bgzip -c > variation/topmed.vcf.gz # tabix -f -p vcf variation/topmed.vcf.gz --- attributes: name: topmed version: freeze5 recipe: full: recipe_type: bash recipe_cmds: - | url=http://s3.amazonaws.com/bcbio-nextgen/genomes/Hsapiens/hg38/variation/topmed-freeze5.vcf.gz mkdir -p variation wget -c -O - ${url} > variation/topmed-freeze5.vcf.gz url=http://s3.amazonaws.com/bcbio-nextgen/genomes/Hsapiens/hg38/variation/topmed-freeze5.vcf.gz.tbi wget -c -O - ${url} > variation/topmed-freeze5.vcf.gz.tbi rm -f variation/topmed.vcf.gz rm -f variation/topmed.vcf.gz.tbi cd variation ln -s topmed-freeze5.vcf.gz topmed.vcf.gz ln -s topmed-freeze5.vcf.gz.tbi topmed.vcf.gz.tbi recipe_outfiles: - variation/topmed.vcf.gz - variation/topmed.vcf.gz.tbi - variation/topmed-freeze5.vcf.gz - variation/topmed-freeze5.vcf.gz.tbi ================================================ FILE: ggd-recipes/hg38/transcripts.yaml ================================================ # Transcript, annotation and indices for RNA-seq analysis # Prepared from Ensembl transcripts using: # https://github.com/chapmanb/cloudbiolinux/blob/master/utils/prepare_tx_gff.py # Prepared against Ensembl gene build 94 --- attributes: name: transcripts version: 2018-10-10_92 recipe: full: recipe_type: bash recipe_cmds: - | baseurl=https://s3.amazonaws.com/biodata/annotation/hg38-rnaseq-2018-10-10_92.tar.xz wget -c -N --no-check-certificate $baseurl xz -dc *-rnaseq-*.tar.xz | tar -xpf - mv */rnaseq-* rnaseq recipe_outfiles: - rnaseq/ref-transcripts.bed - rnaseq/ref-transcripts.dexseq.gff3 - rnaseq/ref-transcripts.fa - rnaseq/ref-transcripts.genePred - rnaseq/ref-transcripts.gtf - rnaseq/ref-transcripts.gtf.db - rnaseq/ref-transcripts-mask.gtf - rnaseq/ref-transcripts.refFlat - rnaseq/ref-transcripts-splicesites.txt - rnaseq/rRNA.gtf - rnaseq/rRNA.gtf.db - rnaseq/rRNA.interval_list - rnaseq/tx2gene.csv - rnaseq/version.txt ================================================ FILE: ggd-recipes/hg38/twobit.yaml ================================================ # UCSC 2bit reference --- attributes: name: twobit version: 1000g-20150219_1 recipe: full: recipe_type: bash recipe_cmds: - | mkdir -p ucsc cd ucsc faToTwoBit ../../seq/hg38.fa hg38.2bit recipe_outfiles: - ucsc/hg38.2bit ================================================ FILE: ggd-recipes/hg38/varpon.yaml ================================================ # Panels of normals (PONs) for variant calling # germline_het_pon -- Locations of likely germline heterogygote sites --- attributes: name: varpon version: 20181105 recipe: full: recipe_type: bash recipe_cmds: - | ref=../seq/hg38.fa mkdir -p variation wget --no-check-certificate -O - 'https://s3.amazonaws.com/biodata/collections/hg38/variation/germline_het_pon.bed.gz' > variation/germline_het_pon.bed.gz wget --no-check-certificate -O - 'https://s3.amazonaws.com/biodata/collections/hg38/variation/germline_het_pon.bed.gz.tbi' > variation/germline_het_pon.bed.gz.tbi recipe_outfiles: - variation/germline_het_pon.bed.gz - variation/germline_het_pon.bed.gz.tbi ================================================ FILE: ggd-recipes/hg38/vcfanno.yaml ================================================ # Annotation files for vcfanno variant annotation # Feeds into vcf2db GEMINI database creation and other annotation tasks --- attributes: name: vcfanno version: 20210204 recipe: full: recipe_type: bash recipe_cmds: - | mkdir -p config/vcfanno cd config/vcfanno bcbio_baseurl=https://raw.githubusercontent.com/bcbio/bcbio-nextgen/master/config/vcfanno wget --no-check-certificate -O gemini.conf -c $bcbio_baseurl/hg38-gemini.conf wget --no-check-certificate -O gemini.lua -c $bcbio_baseurl/hg38-gemini.lua wget --no-check-certificate -O dbnsfp.conf -c $bcbio_baseurl/hg38-dbnsfp.conf wget --no-check-certificate -O dbscsnv.conf -c $bcbio_baseurl/hg38-dbscsnv.conf wget --no-check-certificate -O gnomad_genome.conf -c $bcbio_baseurl/hg38-gnomad_genome.conf wget --no-check-certificate -O rnaedit.conf -c $bcbio_baseurl/rnaedit.conf wget --no-check-certificate -O rnaedit.lua -c $bcbio_baseurl/rnaedit.lua wget --no-check-certificate -O somatic.conf -c $bcbio_baseurl/somatic.conf recipe_outfiles: - config/vcfanno/gemini.lua - config/vcfanno/gemini.conf - config/vcfanno/dbnsfp.conf - config/vcfanno/dbscsnv.conf - config/vcfanno/gnomad_genome.conf - config/vcfanno/rnaedit.lua - config/vcfanno/rnaedit.conf - config/vcfanno/somatic.conf ================================================ FILE: ggd-recipes/hg38/viral.yaml ================================================ # Virus reference genomes, prepared for bwa alignment # Taken from TCGA GDC reference genome # https://gdc.cancer.gov/about-data/data-harmonization-and-generation/gdc-reference-files --- attributes: name: viral version: 2017.02.04 recipe: full: recipe_type: bash recipe_cmds: - | url=http://s3.amazonaws.com/biodata/viral/hg38-gdc-viral.tar.gz mkdir -p viral wget --no-check-certificate -c $url tar -xzvpf *.tar.gz mv *-gdc-viral/* viral recipe_outfiles: - viral/gdc-viral.fa - viral/gdc-viral.dict - viral/gdc-viral.fa.ann - viral/gdc-viral.fa.fai - viral/gdc-viral.fa.pac - viral/gdc-viral.fa.bwt - viral/gdc-viral.fa.amb - viral/gdc-viral.fa.sa ================================================ FILE: ggd-recipes/hg38-noalt/RADAR.yaml ================================================ # RADAR RNA editing events: (http://www.stanford.edu/~gokulr/database/Human_AG_all_hg19_v2.txt) # installs a BED file of the RADAR events, preconverted to hg38 # how this file was created: # # 1. Download the hg19 RADAR file and converted to BED format. # url=http://www.stanford.edu/~gokulr/database/Human_AG_all_hg19_v2.txt # mkdir -p editing # cd editing # wget -qO- $url | awk 'BEGIN{OFS="\t"} {print $1,$2,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11}' | sed "s/position position/start end/" > RADAR-hg19.bed # cd ../ # 2. Convert RADAR BED file to BED3 format. # cut -f1,2,3 RADAR-hg19.bed > RADAR-hg19.tmp.bed # 3. Get the Crossmap hg19 to hg38 chain file: # wget http://hgdownload.soe.ucsc.edu/goldenPath/hg19/liftOver/hg19ToHg38.over.chain.gz # 4. Run Crossmap: # Crossmap.py bed hg19ToHg38.over.chain.gz RADAR-hg19.tmp.bed > conversions.bed # 5. Merge original RADAR file with the Crossmap output # wget https://raw.githubusercontent.com/roryk/seqscripts/master/radar-convert-annotations.py # python radar-convert-annotations.py RADAR-hg19.bed conversions.bed RADAR-hg38.bed.gz --- attributes: name: RADAR-hg38 version: 1 recipe: full: recipe_type: bash recipe_cmds: - | url=https://biodata.s3.amazonaws.com/annotation/RADAR-hg38.bed.gz mkdir -p editing cd editing wget -qO- $url > RADAR-hg38.bed.gz cd ../ recipe_outfiles: - editing/RADAR-hg38.bed.gz ================================================ FILE: ggd-recipes/hg38-noalt/README.md ================================================ Human reference genome: GRCh38/hg38 without any alternative reference contigs http://ftp.ncbi.nlm.nih.gov/genomes/genbank/vertebrate_mammalian/Homo_sapiens/all_assembly_versions/GCA_000001405.15_GRCh38/seqs_for_alignment_pipelines.ucsc_ids/ ================================================ FILE: ggd-recipes/hg38-noalt/bowtie2.yaml ================================================ # pre-built bowtie2 indices --- attributes: name: bowtie2 version: 0.1 recipe: full: recipe_type: bash recipe_cmds: - | base=GCA_000001405.15_GRCh38_no_alt_analysis_set.fna.bowtie_index new=hg38-noalt.fa ncbiurl=http://ftp.ncbi.nlm.nih.gov/genomes/genbank/vertebrate_mammalian/Homo_sapiens/all_assembly_versions/GCA_000001405.15_GRCh38/seqs_for_alignment_pipelines.ucsc_ids wget -c $ncbiurl/$base.tar.gz [[ -f $base.1.bt2 ]] || tar -xzvpf $base.tar.gz mkdir -p bowtie2 for suffix in .1.bt2 .2.bt2 .3.bt2 .4.bt2 .rev.1.bt2 .rev.2.bt2 do [[ -f bowtie2/$new$suffix ]] || mv -f $base$suffix bowtie2/$new$suffix done recipe_outfiles: - bowtie2 ================================================ FILE: ggd-recipes/hg38-noalt/bwa.yaml ================================================ # pre-built bwa indices from NCBIs hg38 distribution with no alternative alleles --- attributes: name: bwa version: 0.1 recipe: full: recipe_type: bash recipe_cmds: - | base=GCA_000001405.15_GRCh38_no_alt_analysis_set.fna new=hg38-noalt.fa ncbiurl=http://ftp.ncbi.nlm.nih.gov/genomes/genbank/vertebrate_mammalian/Homo_sapiens/all_assembly_versions/GCA_000001405.15_GRCh38/seqs_for_alignment_pipelines.ucsc_ids wget -c $ncbiurl/$base.bwa_index.tar.gz [[ -f $base.fna.bwt ]] || tar -xzvpf $base.bwa_index.tar.gz mkdir -p bwa for suffix in .bwt .amb .ann .pac .sa do [[ -f bwa/$new$suffix ]] || mv -f $base$suffix bwa/$new$suffix done recipe_outfiles: - bwa ================================================ FILE: ggd-recipes/hg38-noalt/gtf.yaml ================================================ # Ensembl GTF file distributed by Ensembl for hg38-noalt # Cleans GTF file by converting chromosome names to standard names # Uses https://github.com/dpryan79/ChromosomeMappings to remap the chromosome names --- attributes: name: gtf version: 78 recipe: full: recipe_type: bash recipe_cmds: - | url=http://ftp.ensembl.org/pub/release-78/gtf/homo_sapiens/Homo_sapiens.GRCh38.78.gtf.gz mkdir -p rnaseq remap_url=http://raw.githubusercontent.com/dpryan79/ChromosomeMappings/master/GRCh38_ensembl2UCSC.txt wget -qO- $remap_url | awk '{if($1!=$2) print "s/^"$1"/"$2"/g"}' > remap.sed wget -qO- $url | gunzip | sed -f remap.sed > rnaseq/hg38-noalt.gtf rm remap.sed recipe_outfiles: - rnaseq/hg38-noalt.gtf ================================================ FILE: ggd-recipes/hg38-noalt/mirbase.yaml ================================================ # small RNA seq annotation from different sources --- attributes: name: srnaseq version: 20211104 recipe: full: recipe_type: bash recipe_cmds: - | mkdir -p srnaseq cd srnaseq wget --random-wait --retry-connrefused -q -c -O hsa.gff3 https://mirbase.org/download/hsa.gff3 awk '$3=="miRNA"' hsa.gff3 | sed 's/=/ /g' > srna-transcripts.gtf wget --random-wait --retry-connrefused -q -c -O wgRna.txt.gz http://hgdownload.soe.ucsc.edu/goldenPath/hg38/database/wgRna.txt.gz zgrep -v 'hsa-' wgRna.txt.gz | awk '{print $2"\t.\tncrna\t"$3"\t"$4"\t.\t"$7"\t.\tname "$5";"}' >> srna-transcripts.gtf wget --random-wait --retry-connrefused -q -c -O tRNAs.txt.gz http://hgdownload.soe.ucsc.edu/goldenPath/hg38/database/tRNAs.txt.gz zcat tRNAs.txt.gz | awk '{print $2"\t.\ttRNA\t"$3"\t"$4"\t.\t"$7"\t.\tname "$5";"}' >> srna-transcripts.gtf wget --random-wait --retry-connrefused -q -c -O rmsk.txt.gz http://hgdownload.soe.ucsc.edu/goldenPath/hg38/database/rmsk.txt.gz zcat rmsk.txt.gz | awk '{print $6"\t.\trepeat\t"$7+1"\t"$8+1"\t.\t"$10"\t.\tname "$12";"}' >> srna-transcripts.gtf wget --random-wait --retry-connrefused -q -c -O refGene.txt.gz http://hgdownload.soe.ucsc.edu/goldenPath/hg38/database/refGene.txt.gz zcat refGene.txt.gz | awk '{print $3"\t.\tgene\t"$5"\t"$6"\t.\t"$4"\t.\tname "$13";"}' >> srna-transcripts.gtf sed -i 's/\t0\t/\t1\t/' srna-transcripts.gtf # wget http://www.regulatoryrna.org/database/piRNA/download/archive/v1.0/bed/piR_hg19_v1.0.bed.gz # zcat piR_hg19_v1.0.bed.gz | awk '{print $1"\t.\tpiRNA\t"$2"\t"$3"\t.\t"$6"\t.\tname "$4";"}' >> srna-transcripts.gtf # mirbase wget --random-wait --retry-connrefused -q -c -O hairpin.fa.raw https://mirbase.org/download/CURRENT/hairpin.fa cat hairpin.fa.raw | awk '{if ($0~/>hsa/){name=$0; print name} else if ($0~/^>/){name=0};if (name!=0 && $0!~/^>/){print $0;}}' | sed 's/U/T/g' > hairpin.fa rm hairpin.fa.raw wget --random-wait --retry-connrefused -q -c -O mature.fa.raw https://mirbase.org/download/CURRENT/mature.fa cat mature.fa.raw| awk '{if ($0~/>hsa/){name=$0; print name} else if ($0~/^>/){name=0};if (name!=0 && $0!~/^>/){print $0;}}' | sed 's/U/T/g' > mature.fa rm mature.fa.raw wget --random-wait --retry-connrefused -q -c -O miRNA.dat.raw https://mirbase.org/download/CURRENT/miRNA.dat cat miRNA.dat.raw | awk '{if ($0~/hsa/)print $0}' > miRNA.str rm miRNA.dat.raw wget --random-wait --retry-connrefused -q -c -O mirbase.gff3 https://mirbase.org/download/hsa.gff3 #tdrmapper wget --random-wait --retry-connrefused --no-check-certificate -q -c -O trna_mature_pre.fa https://github.com/sararselitsky/tDRmapper/raw/master/hg19_mature_and_pre.fa # mintmap wget --random-wait --retry-connrefused -nv -c https://github.com/TJU-CMC-Org/MINTmap/archive/refs/tags/MINTmap_Standard_v1.0.tar.gz tar xzf MINTmap_Standard_v1.0.tar.gz MINTmap-MINTmap_Standard_v1.0/LookupTable.tRFs.MINTmap_v1.txt --strip-components=1 tar xzf MINTmap_Standard_v1.0.tar.gz MINTmap-MINTmap_Standard_v1.0/tRNAspace.Spliced.Sequences.MINTmap_v1.fa --strip-components=1 tar xzf MINTmap_Standard_v1.0.tar.gz MINTmap-MINTmap_Standard_v1.0/OtherAnnotations.MINTmap_v1.txt --strip-components=1 # mirdeep2 wget --random-wait --retry-connrefused --no-check-certificate -q -c -O Rfam_for_miRDeep.fa.gz https://github.com/lpantano/mirdeep2_core/raw/data/Rfam_for_miRDeep.fa.gz && gunzip Rfam_for_miRDeep.fa.gz # targetscan analysis recipe_outfiles: - srnaseq/srna-transcripts.gtf - srnaseq/hairpin.fa - srnaseq/mature.fa - srnaseq/miRNA.str - srnaseq/mirbase.gff3 - srnaseq/trna_mature_pre.fa - srnaseq/LookupTable.tRFs.MINTmap_v1.txt - srnaseq/tRNAspace.Spliced.Sequences.MINTmap_v1.fa - srnaseq/OtherAnnotations.MINTmap_v1.txt - srnaseq/Rfam_for_miRDeep.fa ================================================ FILE: ggd-recipes/hg38-noalt/seq.yaml ================================================ # Reference genome distributed by NCBI for GRCh38, without alternative reference contigs --- attributes: name: seq version: 0.1 recipe: full: recipe_type: bash recipe_cmds: - | mkdir -p seq ncbidir=ftp://ftp.ncbi.nlm.nih.gov/genomes/all/GCA/000/001/405/GCA_000001405.15_GRCh38/seqs_for_alignment_pipelines.ucsc_ids filebase=GCA_000001405.15_GRCh38_no_alt_analysis_set.fna wget -c -O - $ncbidir/${filebase}.gz | gunzip -c > seq/hg38-noalt.fa - wget -c -O seq/hg38-noalt.fa.fai $ncbidir/${filebase}.fai - > [[ -f seq/hg38-noalt.dict ]] || picard CreateSequenceDictionary REFERENCE=seq/hg38-noalt.fa OUTPUT=seq/hg38-noalt.dict SPECIES=hg38-noalt recipe_outfiles: - seq/hg38-noalt.fa - seq/hg38-noalt.fa.fai - seq/hg38-noalt.dict ================================================ FILE: ggd-recipes/hg38-noalt/transcripts.yaml ================================================ # Transcript, annotation and indices for RNA-seq analysis # Prepared from Ensembl transcripts using: # https://github.com/chapmanb/cloudbiolinux/blob/master/utils/prepare_tx_gff.py --- attributes: name: transcripts version: 2015-11-22 recipe: full: recipe_type: bash recipe_cmds: - | baseurl=https://s3.amazonaws.com/biodata/annotation/hg38-noalt-rnaseq-2015-11-22.tar.xz wget -c -N --no-check-certificate $baseurl xz -dc *-rnaseq-*.tar.xz | tar -xpf - mv */rnaseq-* rnaseq recipe_outfiles: - rnaseq/tophat - rnaseq/ref-transcripts.bed - rnaseq/ref-transcripts.dexseq.gff3 - rnaseq/ref-transcripts.fa - rnaseq/ref-transcripts.genePred - rnaseq/ref-transcripts.gtf - rnaseq/ref-transcripts.gtf.db - rnaseq/ref-transcripts-mask.gtf - rnaseq/ref-transcripts.refFlat - rnaseq/ref-transcripts-splicesites.txt - rnaseq/rRNA.gtf - rnaseq/rRNA.gtf.db - rnaseq/rRNA.interval_list ================================================ FILE: ggd-recipes/mm10/dbsnp.yaml ================================================ # dbsnp for mouse was discontinued in 2017, we are using the latest release # https://www.animalgenome.org/community/angenmap/hmail/archive/5936.html --- attributes: name: dbsnp version: 20130912 recipe: full: recipe_type: bash recipe_cmds: - | baseurl=https://s3.amazonaws.com/biodata/variants/mm10-dbSNP-2013-09-12.vcf.gz mkdir -p variation cd variation wget -c -N $baseurl wget -c -N $baseurl.tbi recipe_outfiles: - variation/mm10-dbSNP-2013-09-12.vcf.gz - variation/mm10-dbSNP-2013-09-12.vcf.gz.tbi ================================================ FILE: ggd-recipes/mm10/mirbase.yaml ================================================ # small RNA seq annotation from different sources --- attributes: name: srnaseq version: 20211203 recipe: full: recipe_type: bash recipe_cmds: - | mkdir -p srnaseq cd srnaseq wget --random-wait --retry-connrefused -q -N -c http://hgdownload.soe.ucsc.edu/goldenPath/mm10/database/wgEncodeGencodeBasicVM11.txt.gz zcat wgEncodeGencodeBasicVM11.txt.gz | awk '{print $3"\t.\tencode\t"$5+1"\t"$6"\t.\t"$4"\t.\tname "$13";"}' | awk '$5-$4 < 500' > srna-transcripts.gtf wget --random-wait --retry-connrefused -q -N -c http://hgdownload.soe.ucsc.edu/goldenPath/mm10/database/tRNAs.txt.gz zcat tRNAs.txt.gz | awk '{print $2"\t.\ttRNA\t"$3+1"\t"$4"\t.\t"$7"\t.\tname "$5";"}' >> srna-transcripts.gtf wget --random-wait --retry-connrefused -q -N -c http://hgdownload.soe.ucsc.edu/goldenPath/mm10/database/rmsk.txt.gz zcat rmsk.txt.gz | awk '{print $6"\t.\trepeat\t"$7+1"\t"$8+1"\t.\t"$10"\t.\tname "$12";"}' >> srna-transcripts.gtf wget --random-wait --retry-connrefused -q -N -c https://www.mirbase.org/ftp/20/genomes/mmu.gff3 awk '$3=="miRNA"' mmu.gff3 | sed 's/=/ /g' >> srna-transcripts.gtf wget --random-wait --retry-connrefused -q -c -O hairpin.fa.raw https://mirbase.org/download/CURRENT/hairpin.fa cat hairpin.fa.raw | awk '{if ($0~/>mmu/){name=$0; print name} else if ($0~/^>/){name=0};if (name!=0 && $0!~/^>/){print $0;}}' | sed 's/U/T/g' > hairpin.fa rm hairpin.fa.raw wget --random-wait --retry-connrefused -q -c -O mature.fa.raw https://mirbase.org/download/CURRENT/mature.fa cat mature.fa.raw | awk '{if ($0~/>mmu/){name=$0; print name} else if ($0~/^>/){name=0};if (name!=0 && $0!~/^>/){print $0;}}' | sed 's/U/T/g' > mature.fa rm mature.fa.raw wget --random-wait --retry-connrefused -q -c -O miRNA.dat.raw https://mirbase.org/download/CURRENT/miRNA.dat cat miRNA.dat.raw | awk '{if ($0~/mmu/)print $0}' > miRNA.str rm miRNA.dat.raw wget --random-wait --retry-connrefused -nv -c -O mirbase.gff3 https://mirbase.org/download/mmu.gff3 wget --random-wait --retry-connrefused --no-check-certificate -q -c -O Rfam_for_miRDeep.fa.gz https://github.com/lpantano/mirdeep2_core/raw/data/Rfam_for_miRDeep.fa.gz && gunzip Rfam_for_miRDeep.fa.gz # targetscan analysis # wget --no-check-certificate -q -c -O Summary_Counts.txt.zip http://www.targetscan.org/mmu_71/mmu_71_data_download/Summary_Counts.all_predictions.txt.zip && unzip Summary_Counts.txt.zip # wget --no-check-certificate -q -c -O miR_Family_Info.txt.zip http://www.targetscan.org/mmu_71/mmu_71_data_download/miR_Family_Info.txt.zip && unzip miR_Family_Info.txt.zip # wget --no-check-certificate -q -N -c ftp://mirbase.org/pub/mirbase/21/database_files/mirna_mature.txt.gz # tdrmapper wget --random-wait --retry-connrefused --no-check-certificate -c -O trna_mature_pre.fa https://github.com/sararselitsky/tDRmapper/raw/master/mm10_mature_pre_for_tdrMapper.fa recipe_outfiles: - srnaseq/srna-transcripts.gtf - srnaseq/hairpin.fa - srnaseq/mature.fa - srnaseq/mirbase.gff3 - srnaseq/miRNA.str - srnaseq/Rfam_for_miRDeep.fa - srnaseq/trna_mature_pre.fa ================================================ FILE: ggd-recipes/mm10/prioritize.yaml ================================================ # Regions for prioritization: # cancer # - az-cancer-panel -- az300 genes plus additional useful panel genes # mm10 version contributed by Ramiya Gurrapu --- attributes: name: prioritize version: 20210329 recipe: full: recipe_type: bash recipe_cmds: - | baseurl=https://s3.amazonaws.com/bcbio-nextgen/coverage/prioritize/prioritize-cancer-mm10-20210329.tar.gz outdir=coverage/prioritize mkdir -p $outdir cd $outdir wget --no-check-certificate -c -O cancer.tar.gz $baseurl tar -xzvpf cancer.tar.gz recipe_outfiles: - coverage/prioritize/cancer/az-cancer-panel.bed.gz - coverage/prioritize/cancer/az-cancer-panel.bed.gz.tbi ================================================ FILE: ggd-recipes/mm10/problem_regions.yaml ================================================ # Problematic genomic regions for annotations and improving variant call comparisons # mm10 LCRs from Brent Pedersen: http://figshare.com/articles/LCR_mm10_bed_gz/1180124 --- attributes: name: problem_regions version: 20151028 recipe: full: recipe_type: bash recipe_cmds: - | repeats=coverage/problem_regions/repeats mkdir -p $repeats wget --no-check-certificate -O - http://files.figshare.com/1688228/LCR_mm10.bed.gz | gunzip -c | bgzip -c > $repeats/LCR.bed.gz tabix -p vcf -f $repeats/LCR.bed.gz recipe_outfiles: - coverage/problem_regions/repeats/LCR.bed.gz - coverage/problem_regions/repeats/LCR.bed.gz.tbi ================================================ FILE: ggd-recipes/mm10/rmsk.yaml ================================================ # repeat elements location on the genome --- attributes: name: rmsk version: 20180319 recipe: full: recipe_type: bash recipe_cmds: - | mkdir -p coverage/problem_regions/repeats cd coverage/problem_regions/repeats wget --random-wait --retry-connrefused -q -c -O rmsk.txt.gz http://hgdownload.soe.ucsc.edu/goldenPath/mm10/database/rmsk.txt.gz zcat rmsk.txt.gz | awk '{print $6"\t.\trepeat\t"$7+1"\t"$8+1"\t.\t"$10"\t.\tname "$12";"}' | gzip > rmsk.gtf.gz recipe_outfiles: - coverage/problem_regions/repeats/rmsk.gtf.gz ================================================ FILE: ggd-recipes/mm10/seq.yaml ================================================ --- attributes: name: seq version: ucsc-201112 recipe: full: recipe_type: bash recipe_cmds: - | wget --no-check-certificate -c https://s3.amazonaws.com/biodata/genomes/mm10-seq.tar.xz tar -xJvpf mm10-seq.tar.xz recipe_outfiles: - seq/mm10.fa - seq/mm10.fa.fai - seq/mm10.dict ================================================ FILE: ggd-recipes/mm10/transcripts.yaml ================================================ # Transcript, annotation and indices for RNA-seq analysis # Prepared from Ensembl transcripts using: # https://github.com/chapmanb/cloudbiolinux/blob/master/utils/prepare_tx_gff.py # Ensembl genebuild 94 --- attributes: name: transcripts version: 2018-10-10_92 recipe: full: recipe_type: bash recipe_cmds: - | baseurl=https://s3.amazonaws.com/biodata/annotation/mm10-rnaseq-2018-10-10_92.tar.xz wget -c -N --no-check-certificate $baseurl xz -dc *-rnaseq-*.tar.xz | tar -xpf - mv */rnaseq-* rnaseq recipe_outfiles: - rnaseq/ref-transcripts.bed - rnaseq/ref-transcripts.dexseq.gff3 - rnaseq/ref-transcripts.fa - rnaseq/ref-transcripts.genePred - rnaseq/ref-transcripts.gtf - rnaseq/ref-transcripts.gtf.db - rnaseq/ref-transcripts-mask.gtf - rnaseq/ref-transcripts.refFlat - rnaseq/ref-transcripts-splicesites.txt - rnaseq/rRNA.gtf - rnaseq/rRNA.gtf.db - rnaseq/rRNA.interval_list - rnaseq/tx2gene.csv - rnaseq/version.txt ================================================ FILE: ggd-recipes/mm10/twobit.yaml ================================================ # UCSC twobit reference --- attributes: name: twobit version: mm10 recipe: full: recipe_type: bash recipe_cmds: - | mkdir -p ucsc cd ucsc faToTwoBit ../../seq/mm10.fa mm10.2bit recipe_outfiles: - ucsc/mm10.2bit ================================================ FILE: ggd-recipes/mm10/vcfanno.yaml ================================================ # Annotation files for vcfanno variant annotation --- attributes: name: vcfanno version: 20180427 recipe: full: recipe_type: bash recipe_cmds: - | githash=b9bc5cc34dbedaf49ff41ca6249d49b3bb74e67a mkdir -p config/vcfanno cd config/vcfanno bcbio_baseurl=https://github.com/chapmanb/bcbio-nextgen/raw/$githash/config/vcfanno/ wget --no-check-certificate -O gemini.conf -c $bcbio_baseurl/mm10-gemini.conf recipe_outfiles: - config/vcfanno/gemini.conf ================================================ FILE: ggd-recipes/rn6/mirbase.yaml ================================================ # small RNA seq annotation from different sources --- attributes: name: srnaseq version: 20161103 recipe: full: recipe_type: bash recipe_cmds: - | mkdir -p srnaseq cd srnaseq wget -q -N -c ftp://ftp.ensembl.org/pub/release-86/gtf/rattus_norvegicus/Rattus_norvegicus.Rnor_6.0.86.gtf.gz zcat Rattus_norvegicus.Rnor_6.0.86.gtf.gz | awk '$5-$4 < 500' | grep -v "^#"> srna-transcripts.gtf wget -q -N -c http://hgdownload.soe.ucsc.edu/goldenPath/rn6/database/rmsk.txt.gz zcat rmsk.txt.gz | awk '{print $6"\t.\trepeat\t"$7+1"\t"$8+1"\t.\t"$10"\t.\tname "$12";"}' | sed 's/^chr//' | sed 's/^M/MT/' >> srna-transcripts.gtf # wget -q -N -c ftp://mirbase.org/pub/mirbase/20/genomes/rno.gff3 # awk '$3=="miRNA"' mmu.gff3 | sed 's/=/ /g' >> srna-transcripts.gtf wget -q -c -O hairpin.fa.gz ftp://mirbase.org/pub/mirbase/21/hairpin.fa.gz zcat hairpin.fa.gz | awk '{if ($0~/>rno/){name=$0; print name} else if ($0~/^>/){name=0};if (name!=0 && $0!~/^>/){print $0;}}' | sed 's/U/T/g' > hairpin.fa wget -q -c -O mature.fa.gz ftp://mirbase.org/pub/mirbase/21/mature.fa.gz zcat mature.fa.gz | awk '{if ($0~/>rno/){name=$0; print name} else if ($0~/^>/){name=0};if (name!=0 && $0!~/^>/){print $0;}}' | sed 's/U/T/g' > mature.fa wget -q -c -O miRNA.str.gz ftp://mirbase.org/pub/mirbase/21/miRNA.str.gz zcat miRNA.str.gz | awk '{if ($0~/rno/)print $0}' > miRNA.str wget --no-check-certificate -q -c -O Rfam_for_miRDeep.fa.gz https://github.com/lpantano/mirdeep2_core/raw/data/Rfam_for_miRDeep.fa.gz && gunzip Rfam_for_miRDeep.fa.gz # targetscan analysis # tdrmapper recipe_outfiles: - srnaseq/srna-transcripts.gtf - srnaseq/hairpin.fa - srnaseq/mature.fa - srnaseq/miRNA.str - srnaseq/Rfam_for_miRDeep.fa ================================================ FILE: ggd-recipes/rn6/seq.yaml ================================================ # Full hg38/GRCh38 reference genome distributed by 1000 genomes # Derived from NCBI set with HLA and decoy alternative alleles # http://ftp.1000genomes.ebi.ac.uk/vol1/ftp/technical/reference/GRCh38_reference_genome/ --- attributes: name: seq version: 85 recipe: full: recipe_type: bash recipe_cmds: - | url=http://ftp.ensembl.org/pub/release-85/fasta/rattus_norvegicus/dna/Rattus_norvegicus.Rnor_6.0.dna.toplevel.fa.gz mkdir -p seq wget --no-check-certificate -O - $url | gunzip -c > seq/rn6.fa recipe_outfiles: - seq/rn6.fa ================================================ FILE: ggd-recipes/rn6/transcripts.yaml ================================================ # Transcript, annotation and indices for RNA-seq analysis # Prepared from Ensembl transcripts using: # https://github.com/chapmanb/cloudbiolinux/blob/master/utils/prepare_tx_gff.py --- attributes: name: transcripts version: 2018-03-22_91 recipe: full: recipe_type: bash recipe_cmds: - | baseurl=https://s3.amazonaws.com/biodata/annotation/rn6-rnaseq-2018-03-22_91.tar.xz wget -c -N --no-check-certificate $baseurl xz -dc *-rnaseq-*.tar.xz | tar -xpf - mv */rnaseq-* rnaseq recipe_outfiles: - rnaseq/ref-transcripts.bed - rnaseq/ref-transcripts.dexseq.gff3 - rnaseq/ref-transcripts.fa - rnaseq/ref-transcripts.genePred - rnaseq/ref-transcripts.gtf - rnaseq/ref-transcripts.gtf.db - rnaseq/ref-transcripts-mask.gtf - rnaseq/ref-transcripts.refFlat - rnaseq/ref-transcripts-splicesites.txt - rnaseq/rRNA.gtf - rnaseq/rRNA.gtf.db - rnaseq/rRNA.interval_list - rnaseq/tx2gene.csv - rnaseq/version.txt ================================================ FILE: ggd-recipes/rn6/twobit.yaml ================================================ # UCSC twobit reference --- attributes: name: twobit version: 85 recipe: full: recipe_type: bash recipe_cmds: - | mkdir -p ucsc cd ucsc faToTwoBit ../../seq/rn6.fa rn6.2bit recipe_outfiles: - ucsc/rn6.2bit ================================================ FILE: ggd-recipes/sacCer3/seq.yaml ================================================ --- attributes: name: seq version: 94 recipe: full: recipe_type: bash recipe_cmds: - | RELEASE=94 BUILD=sacCer3 url=ftp://ftp.ensembl.org/pub/release-$RELEASE/fasta/saccharomyces_cerevisiae/dna/Saccharomyces_cerevisiae.R64-1-1.dna.toplevel.fa.gz mkdir -p seq wget -O - --no-check-certificate https://gist.githubusercontent.com/chapmanb/a973558c2ec67a57eb4592c709ec0d09/raw/81d398fd7f3c901f89753d4f9f39c5a87d05a7fa/sacCer3.chrom.sizes | cut -f 1 > sacCer3.chrom.sizes wget -c --no-check-certificate https://raw.githubusercontent.com/kwongj/sort-contigs/master/sort-contigs.py wget --no-check-certificate -O - $url | gunzip -c | python -c 'import sys; [sys.stdout.write(">chr%s\n" % (l[1:].split()[0].replace("Mito", "M")) if l.startswith(">") else l) for l in sys.stdin]' > seq/$BUILD.fa.orig python2 sort-contigs.py seq/$BUILD.fa.orig --out seq/$BUILD.fa --fmt fasta --order sacCer3.chrom.sizes samtools faidx seq/$BUILD.fa picard -Xms1g -Xmx3g CreateSequenceDictionary REFERENCE=seq/$BUILD.fa OUTPUT=seq/$BUILD.dict SPECIES=$BUILD recipe_outfiles: - seq/sacCer3.fa - seq/sacCer3.fa.fai - seq/sacCer3.dict ================================================ FILE: ggd-recipes/sacCer3/transcripts.yaml ================================================ # Transcript, annotation and indices for RNA-seq analysis # Prepared from Ensembl transcripts using: # https://github.com/chapmanb/cloudbiolinux/blob/master/utils/prepare_tx_gff.py --- attributes: name: transcripts version: 2018-11-20_94 recipe: full: recipe_type: bash recipe_cmds: - | baseurl=https://s3.amazonaws.com/biodata/annotation/sacCer3-rnaseq-2018-11-20_94.tar.xz wget -c -N --no-check-certificate $baseurl xz -dc *-rnaseq-*.tar.xz | tar -xpf - mv */rnaseq-* rnaseq recipe_outfiles: - rnaseq/ref-transcripts.bed - rnaseq/ref-transcripts.dexseq.gff3 - rnaseq/ref-transcripts.fa - rnaseq/ref-transcripts.genePred - rnaseq/ref-transcripts.gtf - rnaseq/ref-transcripts.gtf.db - rnaseq/ref-transcripts-mask.gtf - rnaseq/ref-transcripts.refFlat - rnaseq/ref-transcripts-splicesites.txt - rnaseq/rRNA.gtf - rnaseq/rRNA.gtf.db - rnaseq/rRNA.interval_list - rnaseq/tx2gene.csv - rnaseq/version.txt ================================================ FILE: installed_files/bash_history ================================================ sudo -i sudo su galaxy tail -444f /mnt/cm/paster.log ================================================ FILE: installed_files/bash_login ================================================ #!/bin/bash # Here is a suitable bash_login to drop into /home/ubuntu/.bash_login # Ensures this is an interactive ssh login, then asks the user if they # want to enable NX functionality. flagfile=~/.nx_setup_done setupscript=setupnx.sh cont=yes # Do I go ahead? [ -e "$flagfile" ] && cont=no [ -n "$SSH_CONNECTION" ] || cont=no tty -s || cont=no if [ "$cont" = yes ] ; then cat <'." else echo "Quitting." fi fi ================================================ FILE: installed_files/ec2autorun.py ================================================ #!/usr/bin/env python """ This is a contextualization script required by CloudMan; it is automatically run at instance startup (via an upstart job). Requires: PyYAML http://pyyaml.org/wiki/PyYAMLDocumentation (easy_install pyyaml) boto http://code.google.com/p/boto/ (easy_install boto) Assumptions: DEFAULT_BUCKET_NAME and DEFAULT_BOOT_SCRIPT_NAME are publicly accessible and do not require any form of authentication """ import os, sys, yaml, urllib2, logging, hashlib, time, subprocess, random from urlparse import urlparse from boto.s3.key import Key from boto.s3.connection import S3Connection from boto.exception import S3ResponseError from boto.s3.connection import OrdinaryCallingFormat logging.getLogger('boto').setLevel(logging.INFO) # Only log boto messages >=INFO log = None USER_DATA_URL = 'http://169.254.169.254/latest/user-data' # USER_DATA_URL = 'http://userwww.service.emory.edu/~eafgan/content/userData.yaml.sample' # used for testing # USER_DATA_URL = 'http://userwww.service.emory.edu/~eafgan/content/url_ud.txt' # used for testing LOCAL_PATH = '/tmp/cm' # Local path destination used for storing/reading any files created by this script USER_DATA_FILE_NAME = 'userData.yaml' # Local file with user data formatted by this script USER_DATA_FILE = os.path.join(LOCAL_PATH, USER_DATA_FILE_NAME) # The final/processed UD file # Local file containing UD in its original format USER_DATA_ORIG = os.path.join(LOCAL_PATH, 'original_%s' % USER_DATA_FILE_NAME) SERVICE_ROOT = 'http://s3.amazonaws.com/' # Obviously, customized for Amazon's S3 DEFAULT_BUCKET_NAME = 'cloudman' # Ensure this bucket is accessible to anyone! DEFAULT_BOOT_SCRIPT_NAME = 'cm_boot.py' # Ensure this file is accessible to anyone in the public bucket! CLOUDMAN_HOME = '/mnt/cm' # ====================== Utility methods ====================== def _setup_logging(): # Logging setup formatter = logging.Formatter("[%(levelname)s] %(module)s:%(lineno)d %(asctime)s: %(message)s") console = logging.StreamHandler() # log to console - used during testing # console.setLevel(logging.INFO) # accepts >INFO levels console.setFormatter(formatter) # log_file = logging.FileHandler(os.path.join(LOCAL_PATH, "%s.log" % os.path.splitext(sys.argv[0])[0]), 'w') # log_file.setLevel(logging.DEBUG) # accepts all levels # log_file.setFormatter(formatter) log = logging.root log.addHandler(console) # log.addHandler(log_file) log.setLevel(logging.DEBUG) return log def _get_user_data(): ud = '' for i in range(0, 5): try: log.info("Getting user data from '%s', attempt %s" % (USER_DATA_URL, i)) fp = urllib2.urlopen(USER_DATA_URL) ud = fp.read() fp.close() log.debug("Saving user data in its original format to file '%s'" % USER_DATA_ORIG) with open(USER_DATA_ORIG, 'w') as ud_orig: ud_orig.write(ud) if ud: log.debug("Got user data") return ud except IOError: log.info("User data not found. Setting it to empty.") return '' # Used for testing # return 'http://s3.amazonaws.com/cloudman/cm_boot' # return '' # return "gc_dev1|||somePWD" # with open('sample.yaml') as ud_yaml: # ud = ud_yaml.read() if ud == '': log.debug("Received empty/no user data") return ud def _get_bucket_name(cluster_name, access_key): """Compose bucket name based on the user-provided cluster name and user access key""" m = hashlib.md5() m.update( cluster_name + access_key ) return "cm-" + m.hexdigest() def _isurl(path): """Test if path is a net location. Tests the scheme and netloc.""" # BUG : URLs require a scheme string ('http://') to be used. # www.google.com will fail. # Should we prepend the scheme for those that don't have it and # test that also? scheme, netloc, upath, uparams, uquery, ufrag = urlparse(path) return bool(scheme and netloc) def _get_s3_conn(ud): try: if 'cloud_type' in ud and ud['cloud_type'] != 'ec2': # If the user has specified a cloud type other than EC2, # create an s3 connection using the info from their user data log.debug('Establishing boto S3 connection to a custom Object Store') try: s3_conn = S3Connection(aws_access_key_id=ud['access_key'], aws_secret_access_key=ud['secret_key'], is_secure=ud.get('is_secure', True), host=ud.get('s3_host', ''), port=ud.get('s3_port', 8888), calling_format=OrdinaryCallingFormat(), path=ud.get('s3_conn_path', '/')) except S3ResponseError, e: log.error("Trouble connecting to a custom Object Store. User data: {0}; Exception: {1}"\ .format(ud, e)) else: # Use the default Amazon S3 connection log.debug('Establishing boto S3 connection to Amazon') s3_conn = S3Connection(ud['access_key'], ud['secret_key']) except Exception, e: log.error("Exception getting S3 connection: %s" % e) return None return s3_conn def _bucket_exists(s3_conn, bucket_name): bucket = None for i in range(1, 6): try: # log.debug("Looking for bucket '%s'" % bucket_name) bucket = s3_conn.lookup(bucket_name) break except S3ResponseError: log.error ("Bucket '%s' not found, attempt %s/5" % (bucket_name, i+1)) time.sleep(2) if bucket is not None: log.debug("Cluster bucket '%s' found." % bucket_name) return True else: log.debug("Cluster bucket '%s' not found." % bucket_name) return False def _remote_file_exists(s3_conn, bucket_name, remote_filename): b = None for i in range(0, 5): try: b = s3_conn.get_bucket(bucket_name) break except S3ResponseError: log.error ("Problem connecting to bucket '%s', attempt %s/5" % (bucket_name, i)) time.sleep(2) if b is not None: k = Key(b, remote_filename) if k.exists(): return True return False def _save_file_to_bucket(s3_conn, bucket_name, remote_filename, local_file, force=False): local_file = os.path.join(LOCAL_PATH, local_file) # log.debug( "Establishing handle with bucket '%s'..." % bucket_name) b = None for i in range(0, 5): try: b = s3_conn.get_bucket(bucket_name) break except S3ResponseError, e: log.error ("Problem connecting to bucket '%s', attempt %s/5" % (bucket_name, i)) time.sleep(2) if b is not None: # log.debug("Establishing handle with key object '%s'..." % remote_filename) k = Key(b, remote_filename) if k.exists() and not force: log.debug("Remote file '%s' already exists. Not overwriting it." % remote_filename) return True log.debug( "Attempting to save local file '%s' to bucket '%s' as '%s'" % (local_file, bucket_name, remote_filename)) try: k.set_contents_from_filename(local_file) log.info( "Successfully saved file '%s' to bucket '%s'." % (remote_filename, bucket_name)) return True except S3ResponseError, e: log.error("Failed to save file local file '%s' to bucket '%s' as file '%s': %s" % (local_file, bucket_name, remote_filename, e)) return False else: return False def _get_file_from_bucket(s3_conn, bucket_name, remote_filename, local_filename): local_filename = os.path.join(LOCAL_PATH, local_filename) try: # log.debug("Establishing handle with bucket '%s'" % bucket_name) b = s3_conn.get_bucket(bucket_name) # log.debug("Establishing handle with file object '%s'" % remote_filename) k = Key(b, remote_filename) log.debug("Attempting to retrieve file '%s' from bucket '%s'" % (remote_filename, bucket_name)) if k.exists(): k.get_contents_to_filename(local_filename) log.info("Successfully retrieved file '%s' from bucket '%s' to '%s'." % (remote_filename, bucket_name, local_filename)) return True else: log.error("File '%s' in bucket '%s' not found." % (remote_filename, bucket_name)) return False except S3ResponseError, e: log.error("Failed to get file '%s' from bucket '%s': %s" % (remote_filename, bucket_name, e)) return False def _get_file_from_url(url): local_filename = os.path.join(LOCAL_PATH, os.path.split(url)[1]) log.info("Getting boot script from '%s' and saving it locally to '%s'" % (url, local_filename)) try: f = urllib2.urlopen(url) with open(local_filename, 'w') as local_file: local_file.write(f.read()) os.chmod(local_filename, 0744) if f: log.debug("Got boot script from '%s'" % url) return True return False except IOError: log.error("Boot script at '%s' not found." % url) return False def _get_boot_script(ud): # Test if cluster bucket exists; if it does not, resort to the default # bucket for downloading the boot script use_default_bucket = ud.get("use_default_bucket", False) if ud.has_key('bucket_default'): default_bucket_name = ud['bucket_default'] else: default_bucket_name = DEFAULT_BUCKET_NAME if not use_default_bucket and ud.has_key('bucket_cluster') and ud['access_key'] is not None and ud['secret_key'] is not None: s3_conn = _get_s3_conn(ud) # Check if cluster bucket exists or use the default one if not _bucket_exists(s3_conn, ud['bucket_cluster']) or \ not _remote_file_exists(s3_conn, ud['bucket_cluster'], ud['boot_script_name']): log.debug("Using default bucket '%s'" % default_bucket_name) use_default_bucket = True else: log.debug("Using cluster bucket '%s'" % ud['bucket_cluster']) use_default_bucket = False else: log.debug("bucket_clutser not specified or no credentials provided; defaulting to bucket '%s'" % default_bucket_name) use_default_bucket = True # If using cluster bucket, use credentials because the boot script may not be accessible to everyone got_boot_script = False if use_default_bucket is False: log.debug("Trying to get boot script '%s' from cluster bucket '%s'" % (ud['boot_script_name'], ud.get('bucket_cluster', None))) got_boot_script = _get_file_from_bucket(s3_conn, ud['bucket_cluster'], ud['boot_script_name'], DEFAULT_BOOT_SCRIPT_NAME) if got_boot_script: os.chmod(os.path.join(LOCAL_PATH, DEFAULT_BOOT_SCRIPT_NAME), 0744) # If did not get the boot script, fall back on the publicly available one if not got_boot_script or use_default_bucket: boot_script_url = os.path.join(_get_default_bucket_url(ud), ud.get('boot_script_name', DEFAULT_BOOT_SCRIPT_NAME)) log.debug("Could not get boot script '%s' from cluster bucket '%s'; " "retrieving the public one from bucket url '%s'" \ % (ud['boot_script_name'], ud.get('bucket_cluster', None), boot_script_url)) got_boot_script = _get_file_from_url(boot_script_url) if got_boot_script: log.debug("Saved boot script to '%s'" % os.path.join(LOCAL_PATH, DEFAULT_BOOT_SCRIPT_NAME)) # Save the downloaded boot script to cluster bucket for future invocations use_object_store = ud.get("use_object_store", True) if use_object_store and ud.has_key('bucket_cluster') and ud['bucket_cluster']: s3_conn = _get_s3_conn(ud) if _bucket_exists(s3_conn, ud['bucket_cluster']) and \ not _remote_file_exists(s3_conn, ud['bucket_cluster'], ud['boot_script_name']): _save_file_to_bucket(s3_conn, ud['bucket_cluster'], ud['boot_script_name'], \ DEFAULT_BOOT_SCRIPT_NAME) return True log.debug("**Could not get the boot script**") return False def _run_boot_script(boot_script_name): script = os.path.join(LOCAL_PATH, boot_script_name) log.info("Running boot script '%s'" % script) process = subprocess.Popen(script, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) stdout, stderr = process.communicate() if process.returncode == 0: log.debug("Successfully ran boot script '%s'" % script) return True else: log.error("Error running boot script '%s'. Process returned code '%s' and following stderr: %s" % (script, process.returncode, stderr)) return False def _create_basic_user_data_file(): # Create a basic YAML file that is expected by CloudMan with open(USER_DATA_FILE, 'w') as ud_file: ud_formatted = {'access_key': None, 'boot_script_name': DEFAULT_BOOT_SCRIPT_NAME, 'boot_script_path': LOCAL_PATH, 'bucket_default': DEFAULT_BUCKET_NAME, 'bucket_cluster': None, 'cloudman_home': CLOUDMAN_HOME, 'cluster_name': 'aGalaxyCloudManCluster_%s' % random.randrange(1, 9999999), 'role': 'master', 'secret_key': None} yaml.dump(ud_formatted, ud_file, default_flow_style=False) return ud_formatted def _get_default_bucket_url(ud=None): if ud and ud.has_key('bucket_default'): default_bucket_name = ud['bucket_default'] else: default_bucket_name = DEFAULT_BUCKET_NAME # TODO: Check if th bucket 'default_bucket_name' is accessible to everyone # because it is being accessed as a URL if ud: bucket_url = ud.get("default_bucket_url", None) else: bucket_url = None if not bucket_url: bucket_url = os.path.join(SERVICE_ROOT, default_bucket_name) log.debug("Default bucket url: %s" % bucket_url) return bucket_url def _user_exists(username): """ Check if the given username exists as a system user """ with open('/etc/passwd', 'r') as f: ep = f.read() return ep.find(username) > 0 def _allow_password_logins(passwd): for user in ["ubuntu", "galaxy"]: if _user_exists(user): log.info("Setting up password-based login for user '{0}'".format(user)) p1 = subprocess.Popen(["echo", "%s:%s" % (user, passwd)], stdout=subprocess.PIPE) p2 = subprocess.Popen(["chpasswd"], stdin=p1.stdout, stdout=subprocess.PIPE) p1.stdout.close() p2.communicate()[0] cl = ["sed", "-i", "s/^PasswordAuthentication .*/PasswordAuthentication yes/", "/etc/ssh/sshd_config"] subprocess.check_call(cl) cl = ["/usr/sbin/service", "ssh", "reload"] subprocess.check_call(cl) def _handle_freenx(passwd): # Check if FreeNX is installed on the image before trying to configure it cl = "/usr/bin/dpkg --get-selections | /bin/grep freenx" retcode = subprocess.call(cl, shell=True) if retcode == 0: log.info("Setting up FreeNX") cl = ["dpkg-reconfigure", "-pcritical", "freenx-server"] # On slower/small instance types, there can be a conflict when running # debconf so try this a few times for i in range(5): retcode = subprocess.call(cl) if retcode == 0: break else: time.sleep(5) else: log.info("freenx-server is not installed; not configuring it") # ====================== Actions methods ====================== def _handle_empty(): log.info("Received empty user data; assuming default contextualization") _create_basic_user_data_file() # This file is expected by CloudMan # Get & run boot script file_url = os.path.join(_get_default_bucket_url(), DEFAULT_BOOT_SCRIPT_NAME) log.debug("Resorting to the default bucket to get the boot script: %s" % file_url) _get_file_from_url(file_url) _run_boot_script(DEFAULT_BOOT_SCRIPT_NAME) def _handle_url(url): log.info("Handling user data provided URL: '%s'" % url) _get_file_from_url(url) boot_script_name = os.path.split(url)[1] _run_boot_script(boot_script_name) #http://stackoverflow.com/questions/823196/yaml-merge-in-python def _merge(specific, default): """ Recursively merges two yaml produced data structures, a more specific input (`specific`) and defaults (`default`). """ if isinstance(specific, dict) and isinstance(default, dict): for k, v in default.iteritems(): if k not in specific: specific[k] = v else: specific[k] = _merge(specific[k], v) return specific def _load_user_data(user_data): """ Loads user data into dict (using pyyaml). If machine image contains default data this is loaded and populated in resulting data structure as well. These separate options are merged using the `_merge` function above and priority is always given to user supplied options. """ ud = yaml.safe_load(user_data) if ud == user_data: # Bad user data, cannot merge default return ud default_user_data_path = \ os.path.join(os.path.dirname(os.path.abspath(__file__)), 'IMAGE_USER_DATA') if os.path.exists(default_user_data_path): image_ud = yaml.safe_load(open(default_user_data_path, 'r').read()) if image_ud: ud = _merge(ud, image_ud) return ud def _handle_yaml(user_data): """ Process user data in YAML format""" log.info("Handling user data in YAML format.") ud = _load_user_data(user_data) # Handle bad user data as a string if ud == user_data: return _handle_empty() # Allow password based logins. Do so also in case only NX is being setup. if "freenxpass" in ud or "password" in ud: passwd = ud.get("freenxpass", None) or ud.get("password", None) _allow_password_logins(passwd) # Handle freenx passwords and the case with only a NX password sent if "freenxpass" in ud: _handle_freenx(ud["freenxpass"]) if len(ud) == 1: return _handle_empty() # Create a YAML file from user data and store it as USER_DATA_FILE # This code simply ensures fields required by CloudMan are in the # created file. Any other fields that might be included as user data # are also included in the created USER_DATA_FILE if ud.get('no_start', None) is not None: log.info("Received 'no_start' user data option. Not doing anything else.") return if not ud.has_key('cluster_name'): log.warning("The provided user data should contain cluster_name field.") ud['cluster_name'] = 'aCloudManCluster_%s' % random.randrange(1, 9999999) elif ud['cluster_name'] == '': log.warning("The cluster_name field of user data should not be empty.") ud['cluster_name'] = 'aCloudManCluster_%s' % random.randrange(1, 9999999) if not ud.has_key('access_key'): log.info("The provided user data does not contain access_key field; setting it to None..") ud['access_key'] = None elif ud['access_key'] == '' or ud['access_key'] is None: log.warning("The access_key field of user data should not be empty; setting it to None.") ud['access_key'] = None if not ud.has_key('secret_key'): log.info("The provided user data does not contain secret_key field; setting it to None.") ud['secret_key'] = None elif ud['secret_key'] == '' or ud['secret_key'] is None: log.warning("The secret_key field of user data should not be empty; setting it to None.") ud['secret_key'] = None if not ud.has_key('password'): log.warning("The provided user data should contain password field.") elif ud['password'] == '': log.warning("The password field of user data should not be empty.") else: # ensure the password is a string ud['password'] = str(ud['password']) if not ud.has_key('bucket_default'): log.debug("The provided user data does not contain bucket_default field; setting it to '%s'." % DEFAULT_BUCKET_NAME) ud['bucket_default'] = DEFAULT_BUCKET_NAME elif ud['bucket_default'] == '': log.warning("The bucket_default field of user data was empty; setting it to '%s'." % DEFAULT_BUCKET_NAME) ud['bucket_default'] = DEFAULT_BUCKET_NAME if not ud.has_key('bucket_cluster'): if ud['access_key'] is not None and ud['secret_key'] is not None: ud['bucket_cluster'] = _get_bucket_name(ud['cluster_name'], ud['access_key']) if not ud.has_key('role'): ud['role'] = 'master' if not ud.has_key('cloudman_home'): ud['cloudman_home'] = CLOUDMAN_HOME if not ud.has_key('boot_script_name'): ud['boot_script_name'] = DEFAULT_BOOT_SCRIPT_NAME ud['boot_script_path'] = LOCAL_PATH # Marks where boot script was saved log.debug("Composed user data: %s" % ud) with open(USER_DATA_FILE, 'w') as ud_yaml: yaml.dump(ud, ud_yaml, default_flow_style=False) # Get & run boot script if _get_boot_script(ud): _run_boot_script(DEFAULT_BOOT_SCRIPT_NAME) # ====================== Driver code ====================== def _parse_user_data(ud): if ud == '': _handle_empty() elif _isurl(ud): _handle_url(ud) else: # default to yaml _handle_yaml(ud) def main(): if not os.path.exists(LOCAL_PATH): os.mkdir(LOCAL_PATH) global log log = _setup_logging() ud = _get_user_data() _parse_user_data(ud) log.info("---> %s done <---" % sys.argv[0]) if __name__ == "__main__": main() ================================================ FILE: installed_files/galaxy_default.template ================================================ export GALAXY_USER=galaxy export GALAXY_HOME=$galaxy_home export PIDFILE=/var/lib/galaxy/galaxy.pid export GALAXY_INDICES=$galaxy_indices_mount export GALAXY_DATA=$galaxy_data_mount ================================================ FILE: installed_files/galaxy_init ================================================ #!/bin/sh ### BEGIN INIT INFO # Provides: galaxy # Required-Start: # Required-Stop: # Default-Start: 2 3 4 5 # Default-Stop: 0 1 6 # Short-Description: Start Galaxy daemon ### END INIT INFO PATH=/sbin:/bin:/usr/sbin:/usr/bin . /lib/lsb/init-functions . /etc/default/galaxy NAME=galaxy case $1 in start) if [ ! -d $GALAXY_DATA ]; then log_daemon_msg "Creating directory for galaxy data" mkdir -p $GALAXY_DATA chown $GALAXY_USER:$GALAXY_USER $GALAXY_DATA fi if [ ! -d $GALAXY_INDICES -a -f /opt/compressed_genomes.tar.gz ]; then # Galaxy VM launcher has option to stash compressed genomes right # in image in compressed state. Look for such a file in /opt/compressed_genomes.tar.gz # and decompress in appropriate directory if needed. log_daemon_msg "Decompressing genomes for galaxy" cp /opt/compressed_genomes.tar.gz $GALAXY_INDICES/.. tar xzvf $GALAXY_INDICES/../compressed_genomes.tar.gz -C $GALAXY_INDICES/.. fi log_daemon_msg "Starting Galaxy" "Galaxy" su - $GALAXY_USER /bin/sh -c "$GALAXY_HOME/run.sh --pid-file=$PIDFILE --log-file /var/log/galaxy/galaxy.log --daemon" status=$? log_end_msg $status ;; stop) log_daemon_msg "Stopping Galaxy" "Galaxy" su - $GALAXY_USER /bin/sh -c "$GALAXY_HOME/run.sh --pid-file=$PIDFILE --stop-daemon" log_end_msg $? rm -f $PIDFILE ;; restart|force-reload) $0 stop && sleep 2 && $0 start ;; try-restart) if $0 status >/dev/null; then $0 restart else exit 0 fi ;; reload) exit 3 ;; *) echo "Usage: $0 {start|stop|restart|try-restart|force-reload}" exit 2 ;; esac ================================================ FILE: installed_files/galaxyp_nginx.conf.template ================================================ worker_processes 2; events { worker_connections 1024; } http { include mime.types; default_type application/octet-stream; sendfile on; keepalive_timeout 65; gzip on; gzip_http_version 1.1; gzip_vary on; gzip_comp_level 4; gzip_proxied any; gzip_types text/plain text/css application/x-javascript text/xml application/xml text/javascript application/json; gzip_buffers 16 8k; gzip_disable "MSIE [1-6].(?!.*SV1)"; upstream galaxy_app { server localhost:8080; } upstream cm_app { server localhost:42284; } upstream protvis-site { server 127.0.0.1:$protvis_port; } server { listen 80; client_max_body_size 2048m; server_name localhost; proxy_read_timeout 600; location /cloud { proxy_pass http://cm_app; proxy_set_header X-Forwarded-Host $$host; proxy_set_header X-Forwarded-For $$proxy_add_x_forwarded_for; error_page 502 /errdoc/cm_502.html; } location /cloud/static { alias /mnt/cm/static; expires 24h; } location /cloud/static/style { alias /mnt/cm/static/style; expires 24h; } location /cloud/static/scripts { alias /mnt/cm/static/scripts; expires 24h; } location / { proxy_pass http://galaxy_app; proxy_set_header X-Forwarded-Host $$host; proxy_set_header X-Forwarded-For $$proxy_add_x_forwarded_for; } location /static { alias $galaxy_home/static; expires 24h; } location /static/style { alias $galaxy_home/static/june_2007_style/blue; expires 24h; } location /static/scripts { alias $galaxy_home/static/scripts/packed; expires 24h; } location /robots.txt { alias $galaxy_home/static/robots.txt; } location /favicon.ico { alias $galaxy_home/static/favicon.ico; } location /_x_accel_redirect/ { internal; alias /; } location /_upload { upload_store /mnt/galaxyData/upload_store; upload_pass_form_field ""; upload_set_form_field "__$${upload_field_name}__is_composite" "true"; upload_set_form_field "__$${upload_field_name}__keys" "name path"; upload_set_form_field "$${upload_field_name}_name" "$$upload_file_name"; upload_set_form_field "$${upload_field_name}_path" "$$upload_tmp_path"; upload_pass_args on; upload_pass /_upload_done; } location /_upload_done { set $$dst /tool_runner/index; if ($$args ~ nginx_redir=([^&]+)) { set $$dst $$1; } rewrite "" $$dst; } error_page 502 /errdoc/502.html; location /errdoc { root html; } location /protvis { rewrite ^/protvis/(.*) /\$1 break; proxy_set_header Host $host; proxy_set_header X-Real-IP \$remote_addr; proxy_set_header X-Forwarded-For \$proxy_add_x_forwarded_for; proxy_set_header X-Forwarded-Proto \$scheme; client_max_body_size 16m; client_body_buffer_size 128k; proxy_connect_timeout 60s; proxy_send_timeout 90s; proxy_read_timeout 90s; proxy_buffering off; proxy_temp_file_write_size 64k; proxy_pass http://protvis-site; proxy_redirect off; } location /protvis/res/ { alias $protvis_home/res/; expires 30d; add_header Cache-Control public; access_log off; } } } ================================================ FILE: installed_files/image_user_data ================================================ # YAML properties placed in this file will act as defaults # for CloudMan user data. User supplied data at launch time # will override these properties. ================================================ FILE: installed_files/ipython_config.py ================================================ # Configuration file for ipython. c = get_config() c.InteractiveShell.autoindent = True c.InteractiveShell.colors = 'Linux' c.InteractiveShell.confirm_exit = False c.AliasManager.user_aliases = [ ('ll', 'ls -l'), ('lt', 'ls -ltr'), ] #------------------------------------------------------------------------------ # InteractiveShellApp configuration #------------------------------------------------------------------------------ # A Mixin for applications that start InteractiveShell instances. # # Provides configurables for loading extensions and executing files as part of # configuring a Shell environment. # # Provides init_extensions() and init_code() methods, to be called after # init_shell(), which must be implemented by subclasses. # Execute the given command string. # c.InteractiveShellApp.code_to_run = '' # lines of code to run at IPython startup. # c.InteractiveShellApp.exec_lines = [] # If true, an 'import *' is done from numpy and pylab, when using pylab # c.InteractiveShellApp.pylab_import_all = True # A list of dotted module names of IPython extensions to load. # c.InteractiveShellApp.extensions = [] # dotted module name of an IPython extension to load. # c.InteractiveShellApp.extra_extension = '' # List of files to run at IPython startup. # c.InteractiveShellApp.exec_files = [] # A file to be run # c.InteractiveShellApp.file_to_run = '' #------------------------------------------------------------------------------ # TerminalIPythonApp configuration #------------------------------------------------------------------------------ # TerminalIPythonApp will inherit config from: BaseIPythonApplication, # Application, InteractiveShellApp # Execute the given command string. # c.TerminalIPythonApp.code_to_run = '' # The IPython profile to use. # c.TerminalIPythonApp.profile = u'default' # Set the log level by value or name. # c.TerminalIPythonApp.log_level = 30 # lines of code to run at IPython startup. # c.TerminalIPythonApp.exec_lines = [] # Enable GUI event loop integration ('qt', 'wx', 'gtk', 'glut', 'pyglet'). # c.TerminalIPythonApp.gui = None # Pre-load matplotlib and numpy for interactive use, selecting a particular # matplotlib backend and loop integration. # c.TerminalIPythonApp.pylab = None # Suppress warning messages about legacy config files # c.TerminalIPythonApp.ignore_old_config = False # Create a massive crash report when IPython enconters what may be an internal # error. The default is to append a short message to the usual traceback # c.TerminalIPythonApp.verbose_crash = False # If a command or file is given via the command-line, e.g. 'ipython foo.py # c.TerminalIPythonApp.force_interact = False # If true, an 'import *' is done from numpy and pylab, when using pylab # c.TerminalIPythonApp.pylab_import_all = True # The name of the IPython directory. This directory is used for logging # configuration (through profiles), history storage, etc. The default is usually # $HOME/.ipython. This options can also be specified through the environment # variable IPYTHON_DIR. # c.TerminalIPythonApp.ipython_dir = u'/home/ubuntu/.ipython' # Whether to display a banner upon starting IPython. # c.TerminalIPythonApp.display_banner = True # Start IPython quickly by skipping the loading of config files. # c.TerminalIPythonApp.quick = False # A list of dotted module names of IPython extensions to load. # c.TerminalIPythonApp.extensions = [] # Whether to install the default config files into the profile dir. If a new # profile is being created, and IPython contains config files for that profile, # then they will be staged into the new directory. Otherwise, default config # files will be automatically generated. # c.TerminalIPythonApp.copy_config_files = False # dotted module name of an IPython extension to load. # c.TerminalIPythonApp.extra_extension = '' # List of files to run at IPython startup. # c.TerminalIPythonApp.exec_files = [] # Whether to overwrite existing config files when copying # c.TerminalIPythonApp.overwrite = False # A file to be run # c.TerminalIPythonApp.file_to_run = '' #------------------------------------------------------------------------------ # TerminalInteractiveShell configuration #------------------------------------------------------------------------------ # TerminalInteractiveShell will inherit config from: InteractiveShell # auto editing of files with syntax errors. # c.TerminalInteractiveShell.autoedit_syntax = False # Use colors for displaying information about objects. Because this information # is passed through a pager (like 'less'), and some pagers get confused with # color codes, this capability can be turned off. # c.TerminalInteractiveShell.color_info = True # # c.TerminalInteractiveShell.history_length = 10000 # Don't call post-execute functions that have failed in the past. # c.TerminalInteractiveShell.disable_failing_post_execute = False # Show rewritten input, e.g. for autocall. # c.TerminalInteractiveShell.show_rewritten_input = True # Set the color scheme (NoColor, Linux, or LightBG). # c.TerminalInteractiveShell.colors = 'LightBG' # Autoindent IPython code entered interactively. # c.TerminalInteractiveShell.autoindent = True # # c.TerminalInteractiveShell.separate_in = '\n' # Deprecated, use PromptManager.in2_template # c.TerminalInteractiveShell.prompt_in2 = ' .\\D.: ' # # c.TerminalInteractiveShell.separate_out = '' # Deprecated, use PromptManager.in_template # c.TerminalInteractiveShell.prompt_in1 = 'In [\\#]: ' # Enable deep (recursive) reloading by default. IPython can use the deep_reload # module which reloads changes in modules recursively (it replaces the reload() # function, so you don't need to change anything to use it). deep_reload() # forces a full reload of modules whose code may have changed, which the default # reload() function does not. When deep_reload is off, IPython will use the # normal reload(), but deep_reload will still be available as dreload(). # c.TerminalInteractiveShell.deep_reload = False # Make IPython automatically call any callable object even if you didn't type # explicit parentheses. For example, 'str 43' becomes 'str(43)' automatically. # The value can be '0' to disable the feature, '1' for 'smart' autocall, where # it is not applied if there are no more arguments on the line, and '2' for # 'full' autocall, where all callable objects are automatically called (even if # no arguments are present). # c.TerminalInteractiveShell.autocall = 0 # Number of lines of your screen, used to control printing of very long strings. # Strings longer than this number of lines will be sent through a pager instead # of directly printed. The default value for this is 0, which means IPython # will auto-detect your screen size every time it needs to print certain # potentially long strings (this doesn't change the behavior of the 'print' # keyword, it's only triggered internally). If for some reason this isn't # working well (it needs curses support), specify it yourself. Otherwise don't # change the default. # c.TerminalInteractiveShell.screen_length = 0 # Set the editor used by IPython (default to $EDITOR/vi/notepad). # c.TerminalInteractiveShell.editor = 'vi' # Deprecated, use PromptManager.justify # c.TerminalInteractiveShell.prompts_pad_left = True # The part of the banner to be printed before the profile # c.TerminalInteractiveShell.banner1 = 'Python 2.7.1 (r271:86832, Jun 25 2011, 05:09:01) \nType "copyright", "credits" or "license" for more information.\n\nIPython 0.12 -- An enhanced Interactive Python.\n? -> Introduction and overview of IPython\'s features.\n%quickref -> Quick reference.\nhelp -> Python\'s own help system.\nobject? -> Details about \'object\', use \'object??\' for extra details.\n' # # c.TerminalInteractiveShell.readline_parse_and_bind = ['tab: complete', '"\\C-l": clear-screen', 'set show-all-if-ambiguous on', '"\\C-o": tab-insert', '"\\C-r": reverse-search-history', '"\\C-s": forward-search-history', '"\\C-p": history-search-backward', '"\\C-n": history-search-forward', '"\\e[A": history-search-backward', '"\\e[B": history-search-forward', '"\\C-k": kill-line', '"\\C-u": unix-line-discard'] # The part of the banner to be printed after the profile # c.TerminalInteractiveShell.banner2 = '' # # c.TerminalInteractiveShell.separate_out2 = '' # # c.TerminalInteractiveShell.wildcards_case_sensitive = True # # c.TerminalInteractiveShell.debug = False # Set to confirm when you try to exit IPython with an EOF (Control-D in Unix, # Control-Z/Enter in Windows). By typing 'exit' or 'quit', you can force a # direct exit without any confirmation. # c.TerminalInteractiveShell.confirm_exit = True # # c.TerminalInteractiveShell.ipython_dir = '' # # c.TerminalInteractiveShell.readline_remove_delims = '-/~' # Start logging to the default log file. # c.TerminalInteractiveShell.logstart = False # The name of the logfile to use. # c.TerminalInteractiveShell.logfile = '' # The shell program to be used for paging. # c.TerminalInteractiveShell.pager = 'less' # Enable magic commands to be called without the leading %. # c.TerminalInteractiveShell.automagic = True # Save multi-line entries as one entry in readline history # c.TerminalInteractiveShell.multiline_history = True # # c.TerminalInteractiveShell.readline_use = True # Start logging to the given file in append mode. # c.TerminalInteractiveShell.logappend = '' # # c.TerminalInteractiveShell.xmode = 'Context' # # c.TerminalInteractiveShell.quiet = False # Enable auto setting the terminal title. # c.TerminalInteractiveShell.term_title = False # # c.TerminalInteractiveShell.object_info_string_level = 0 # Deprecated, use PromptManager.out_template # c.TerminalInteractiveShell.prompt_out = 'Out[\\#]: ' # Set the size of the output cache. The default is 1000, you can change it # permanently in your config file. Setting it to 0 completely disables the # caching system, and the minimum value accepted is 20 (if you provide a value # less than 20, it is reset to 0 and a warning is issued). This limit is # defined because otherwise you'll spend more time re-flushing a too small cache # than working # c.TerminalInteractiveShell.cache_size = 1000 # Automatically call the pdb debugger after every exception. # c.TerminalInteractiveShell.pdb = False #------------------------------------------------------------------------------ # PromptManager configuration #------------------------------------------------------------------------------ # This is the primary interface for producing IPython's prompts. # Output prompt. '\#' will be transformed to the prompt number # c.PromptManager.out_template = 'Out[\\#]: ' # Continuation prompt. # c.PromptManager.in2_template = ' .\\D.: ' # If True (default), each prompt will be right-aligned with the preceding one. # c.PromptManager.justify = True # Input prompt. '\#' will be transformed to the prompt number # c.PromptManager.in_template = 'In [\\#]: ' # # c.PromptManager.color_scheme = 'Linux' #------------------------------------------------------------------------------ # ProfileDir configuration #------------------------------------------------------------------------------ # An object to manage the profile directory and its resources. # # The profile directory is used by all IPython applications, to manage # configuration, logging and security. # # This object knows how to find, create and manage these directories. This # should be used by any code that wants to handle profiles. # Set the profile location directly. This overrides the logic used by the # `profile` option. # c.ProfileDir.location = u'' #------------------------------------------------------------------------------ # PlainTextFormatter configuration #------------------------------------------------------------------------------ # The default pretty-printer. # # This uses :mod:`IPython.external.pretty` to compute the format data of the # object. If the object cannot be pretty printed, :func:`repr` is used. See the # documentation of :mod:`IPython.external.pretty` for details on how to write # pretty printers. Here is a simple example:: # # def dtype_pprinter(obj, p, cycle): # if cycle: # return p.text('dtype(...)') # if hasattr(obj, 'fields'): # if obj.fields is None: # p.text(repr(obj)) # else: # p.begin_group(7, 'dtype([') # for i, field in enumerate(obj.descr): # if i > 0: # p.text(',') # p.breakable() # p.pretty(field) # p.end_group(7, '])') # PlainTextFormatter will inherit config from: BaseFormatter # # c.PlainTextFormatter.type_printers = {} # # c.PlainTextFormatter.newline = '\n' # # c.PlainTextFormatter.float_precision = '' # # c.PlainTextFormatter.verbose = False # # c.PlainTextFormatter.deferred_printers = {} # # c.PlainTextFormatter.pprint = True # # c.PlainTextFormatter.max_width = 79 # # c.PlainTextFormatter.singleton_printers = {} #------------------------------------------------------------------------------ # IPCompleter configuration #------------------------------------------------------------------------------ # Extension of the completer class with IPython-specific features # IPCompleter will inherit config from: Completer # Instruct the completer to omit private method names # # Specifically, when completing on ``object.``. # # When 2 [default]: all names that start with '_' will be excluded. # # When 1: all 'magic' names (``__foo__``) will be excluded. # # When 0: nothing will be excluded. # c.IPCompleter.omit__names = 2 # Whether to merge completion results into a single list # # If False, only the completion results from the first non-empty completer will # be returned. # c.IPCompleter.merge_completions = True # Activate greedy completion # # This will enable completion on elements of lists, results of function calls, # etc., but can be unsafe because the code is actually evaluated on TAB. # c.IPCompleter.greedy = False ================================================ FILE: installed_files/jwmrc.xml ================================================ x-terminal-emulator x-www-broser xscreensaver-command -activate Pidgin gkrellm xterm root:1 showdesktop xload -nolabel -bg black -fg red -hl white xclock -*-fixed-*-r-*-*-10-*-*-*-*-*-*-* 4 20 white gray30:gray60 black 1.0 #aaaaaa #808488:#303438 black 0.5:0.9:0.1 -*-fixed-*-r-*-*-10-*-*-*-*-*-*-* black gray80:gray90 black gray90:gray80 -*-fixed-*-r-*-*-10-*-*-*-*-*-*-* gray90 black 0.75 black gray90 #808488 #70849d #2e3a67 -*-fixed-*-r-*-*-10-*-*-*-*-*-*-* black gray90 white #70849d:#2e3a67 0.85 -*-fixed-*-r-*-*-10-*-*-*-*-*-*-* black black yellow /usr/share/icons/wm-icons/32x32-gant /export0/images/formulae.jpg 400 2 sloppy border opaque opaque gkrellm up down right left left down up right select escape next close desktop# root:1 window maximize rdesktop ldesktop udesktop ddesktop ================================================ FILE: installed_files/nginx.conf.template ================================================ worker_processes 2; events { worker_connections 1024; } http { include mime.types; default_type application/octet-stream; sendfile on; keepalive_timeout 65; gzip on; gzip_http_version 1.1; gzip_vary on; gzip_comp_level 4; gzip_proxied any; gzip_types text/plain text/css application/x-javascript text/xml application/xml text/javascript application/json; gzip_buffers 16 8k; gzip_disable "MSIE [1-6].(?!.*SV1)"; upstream galaxy_app { server localhost:8080; } upstream cm_app { server localhost:42284; } upstream galaxy_reports_app { server localhost:9001; } server { listen 80; client_max_body_size 2048m; server_name localhost; proxy_read_timeout 600; location /cloud { proxy_pass http://cm_app; proxy_set_header X-Forwarded-Host $$host; proxy_set_header X-Forwarded-For $$proxy_add_x_forwarded_for; error_page 502 /errdoc/cm_502.html; } location /cloud/static { alias /mnt/cm/static; expires 24h; } location /cloud/static/style { alias /mnt/cm/static/style; expires 24h; } location /cloud/static/scripts { alias /mnt/cm/static/scripts; expires 24h; } location /reports { rewrite ^/reports/(.*) /$$1 break; proxy_pass http://galaxy_reports_app; proxy_set_header X-Forwarded-Host $$host; proxy_set_header X-Forwarded-For $$proxy_add_x_forwarded_for; } location / { proxy_pass http://galaxy_app; proxy_set_header X-Forwarded-Host $$host; proxy_set_header X-Forwarded-For $$proxy_add_x_forwarded_for; } location /static { alias $galaxy_home/static; expires 24h; } location /static/style { alias $galaxy_home/static/june_2007_style/blue; expires 24h; } location /static/scripts { alias $galaxy_home/static/scripts/packed; expires 24h; } location /robots.txt { alias $galaxy_home/static/robots.txt; } location /favicon.ico { alias $galaxy_home/static/favicon.ico; } location /_x_accel_redirect/ { internal; alias /; } location /_upload { upload_store /mnt/galaxyData/upload_store; upload_pass_form_field ""; upload_set_form_field "__$${upload_field_name}__is_composite" "true"; upload_set_form_field "__$${upload_field_name}__keys" "name path"; upload_set_form_field "$${upload_field_name}_name" "$$upload_file_name"; upload_set_form_field "$${upload_field_name}_path" "$$upload_tmp_path"; upload_pass_args on; upload_pass /_upload_done; } location /_upload_done { set $$dst /tool_runner/index; if ($$args ~ nginx_redir=([^&]+)) { set $$dst $$1; } rewrite "" $$dst; } error_page 502 /errdoc/502.html; location /errdoc { root html; } } } ================================================ FILE: installed_files/nginx_init ================================================ #! /bin/sh ### BEGIN INIT INFO # Provides: nginx # Required-Start: $local_fs $remote_fs $network $syslog # Required-Stop: $local_fs $remote_fs $network $syslog # Default-Start: 2 3 4 5 # Default-Stop: 0 1 6 # Short-Description: starts the nginx web server # Description: starts nginx using start-stop-daemon ### END INIT INFO PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin:/opt/galaxy/sbin/ DAEMON=/opt/galaxy/sbin/nginx NAME=nginx DESC=nginx test -x $DAEMON || exit 0 # Include nginx defaults if available if [ -f /etc/default/nginx ] ; then . /etc/default/nginx fi set -e . /lib/lsb/init-functions test_nginx_config() { if nginx -t $DAEMON_OPTS then return 0 else return $? fi } case "$1" in start) echo -n "Starting $DESC: " test_nginx_config start-stop-daemon --start --quiet --pidfile /var/run/$NAME.pid \ --exec $DAEMON -- $DAEMON_OPTS || true echo "$NAME." ;; stop) echo -n "Stopping $DESC: " start-stop-daemon --stop --quiet --pidfile /var/run/$NAME.pid \ --exec $DAEMON || true echo "$NAME." ;; restart|force-reload) echo -n "Restarting $DESC: " start-stop-daemon --stop --quiet --pidfile \ /var/run/$NAME.pid --exec $DAEMON || true sleep 1 test_nginx_config start-stop-daemon --start --quiet --pidfile \ /var/run/$NAME.pid --exec $DAEMON -- $DAEMON_OPTS || true echo "$NAME." ;; reload) echo -n "Reloading $DESC configuration: " test_nginx_config start-stop-daemon --stop --signal HUP --quiet --pidfile /var/run/$NAME.pid \ --exec $DAEMON || true echo "$NAME." ;; configtest) echo -n "Testing $DESC configuration: " if test_nginx_config then echo "$NAME." else exit $? fi ;; status) status_of_proc -p /var/run/$NAME.pid "$DAEMON" nginx && exit 0 || exit $? ;; *) echo "Usage: $NAME {start|stop|restart|reload|force-reload|status|configtest}" >&2 exit 1 ;; esac exit 0 ================================================ FILE: installed_files/novnc_default.template ================================================ NOVNC_USER=$vnc_user NOVNC_HOME=$novnc_install_dir PORT="0.0.0.0:6080" VNC_DEST="localhost:590$vnc_display" ================================================ FILE: installed_files/novnc_init ================================================ #! /bin/sh ### BEGIN INIT INFO # Provides: novnc # Required-Start: $remote_fs $syslog # Required-Stop: $remote_fs $syslog # Default-Start: 2 3 4 5 # Default-Stop: 0 1 6 # Short-Description: novnc init script. # Description: novnc init script. ### END INIT INFO # Author: John Chilton # # TODO: This script doesn't work to stop/kill novnc, # this should be fixed. PATH=/sbin:/usr/sbin:/bin:/usr/bin DESC="novnc proxy web server" NAME=novnc # Read configuration variable file if it is present [ -r /etc/default/$NAME ] && . /etc/default/$NAME DAEMON=/usr/bin/env DAEMON_ARGS="python $NOVNC_HOME/utils/wsproxy.py --web $NOVNC_HOME $PORT $VNC_DEST" PIDFILE=/var/run/$NAME.pid SCRIPTNAME=/etc/init.d/$NAME # Exit if the package is not installed [ -x "$DAEMON" ] || exit 0 # Load the VERBOSE setting and other rcS variables . /lib/init/vars.sh # Define LSB log_* functions. # Depend on lsb-base (>= 3.2-14) to ensure that this file is present # and status_of_proc is working. . /lib/lsb/init-functions # # Function that starts the daemon/service # do_start() { # Return # 0 if daemon has been started # 1 if daemon was already running # 2 if daemon could not be started start-stop-daemon --start --quiet --pidfile $PIDFILE --user $NOVNC_USER --chuid $NOVNC_USER --exec $DAEMON --test > /dev/null \ || return 1 start-stop-daemon --start --quiet --make-pidfile --pidfile $PIDFILE --user $NOVNC_USER --chuid $NOVNC_USER --background --exec $DAEMON -- \ $DAEMON_ARGS \ || return 2 # Add code here, if necessary, that waits for the process to be ready # to handle requests from services started subsequently which depend # on this one. As a last resort, sleep for some time. } # # Function that stops the daemon/service # do_stop() { # Return # 0 if daemon has been stopped # 1 if daemon was already stopped # 2 if daemon could not be stopped # other if a failure occurred start-stop-daemon --stop --quiet --retry=TERM/30/KILL/5 --pidfile $PIDFILE --name $NAME RETVAL="$?" [ "$RETVAL" = 2 ] && return 2 # Wait for children to finish too if this is a daemon that forks # and if the daemon is only ever run from this initscript. # If the above conditions are not satisfied then add some other code # that waits for the process to drop all resources that could be # needed by services started subsequently. A last resort is to # sleep for some time. start-stop-daemon --stop --quiet --oknodo --retry=0/30/KILL/5 --exec $DAEMON [ "$?" = 2 ] && return 2 # Many daemons don't delete their pidfiles when they exit. rm -f $PIDFILE return "$RETVAL" } # # Function that sends a SIGHUP to the daemon/service # do_reload() { # # If the daemon can reload its configuration without # restarting (for example, when it is sent a SIGHUP), # then implement that here. # start-stop-daemon --stop --signal 1 --quiet --pidfile $PIDFILE --name $NAME return 0 } case "$1" in start) [ "$VERBOSE" != no ] && log_daemon_msg "Starting $DESC" "$NAME" do_start case "$?" in 0|1) [ "$VERBOSE" != no ] && log_end_msg 0 ;; 2) [ "$VERBOSE" != no ] && log_end_msg 1 ;; esac ;; stop) [ "$VERBOSE" != no ] && log_daemon_msg "Stopping $DESC" "$NAME" do_stop case "$?" in 0|1) [ "$VERBOSE" != no ] && log_end_msg 0 ;; 2) [ "$VERBOSE" != no ] && log_end_msg 1 ;; esac ;; status) status_of_proc "$DAEMON" "$NAME" && exit 0 || exit $? ;; #reload|force-reload) # # If do_reload() is not implemented then leave this commented out # and leave 'force-reload' as an alias for 'restart'. # #log_daemon_msg "Reloading $DESC" "$NAME" #do_reload #log_end_msg $? #;; restart|force-reload) # # If the "reload" option is implemented then remove the # 'force-reload' alias # log_daemon_msg "Restarting $DESC" "$NAME" do_stop case "$?" in 0|1) do_start case "$?" in 0) log_end_msg 0 ;; 1) log_end_msg 1 ;; # Old process is still running *) log_end_msg 1 ;; # Failed to start esac ;; *) # Failed to stop log_end_msg 1 ;; esac ;; *) #echo "Usage: $SCRIPTNAME {start|stop|restart|reload|force-reload}" >&2 echo "Usage: $SCRIPTNAME {start|stop|status|restart|force-reload}" >&2 exit 3 ;; esac : ================================================ FILE: installed_files/pg_ctl ================================================ #!/bin/bash # # This script is required because of CloudMan - CloudMan's path resolution # code would result in using the latest version of PostgreSQL installed via # a package manager. However, if a user switches images, the database data # may not match the version of PostgreSQL being invoked so this script # is used to differentiate which PostgreSQL version is actually invoked. # This had to be done here because it was the only variable for old # clusters being invoked with a new image. Also note that this script # is image dependent (because PostgreSQL v9.1 will be the default version). # Recompose args so pg_ctl will work args="" # If an argument contains whitespace, add double quotes around it whitespace="[[:space:]]" for arg in "$@"; do if [[ $arg =~ $whitespace ]]; then arg=\"$arg\" fi args="$args $arg" done # If the -D option is specified (path to data dir), extract PG_VERSION from the data dir while getopts ":D:" opt; do case $opt in D) # Get current Postgres DB version PG_VERSION=`cat "$OPTARG"/PG_VERSION` ;; esac done if [ "$PG_VERSION" == "8.4" ]; then CMD="/usr/lib/postgresql/8.4/bin/pg_ctl $args" else CMD="/usr/lib/postgresql/9.1/bin/pg_ctl_orig $args" fi # Must eval here; otherwise quotes in the args don’t get passed through properly eval $CMD ================================================ FILE: installed_files/proftpd.conf.template ================================================ ServerName "Galaxy CloudMan FTP" ServerType standalone DefaultServer on # Port 21 is the standard FTP port. Port 21 # Don't use IPv6 support by default. UseIPv6 on # Umask 022 is a good standard umask to prevent new dirs and files # from being group and world writable. Umask 022 SyslogFacility DAEMON SyslogLevel debug PidFile /var/run/proftpd.pid # To prevent DoS attacks, set the maximum number of child processes # to 30. If you need to allow more than 30 concurrent connections # at once, simply increase this value. Note that this ONLY works # in standalone mode, in inetd mode you should use an inetd server # that allows you to limit maximum number of processes per service # (such as xinetd). MaxInstances 30 # Set the user and group under which the server will run. User nobody Group nogroup PassivePorts 30000 30100 DisplayConnect REPLACE_THIS_WITH_CUSTOM_INSTALL_DIR/etc/welcome_msg.txt # To cause every FTP user to be "jailed" (chrooted) into their home # directory, uncomment this line. DefaultRoot ~ # Normally, we want files to be overwriteable. AllowOverwrite on AllowStoreRestart on # Bar use of SITE CHMOD by default DenyAll DenyAll FactsAdvertise off DenyAll # AuthPAM off CreateHome on dirmode 700 SQLPasswordEngine on SQLPasswordEncoding hex SQLEngine on SQLLogFile /var/log/proftpd-sql.log SQLBackend postgres SQLConnectInfo galaxy@localhost:$postgres_port galaxyftp $galaxy_ftp_user_password SQLAuthTypes SHA1 SQLAuthenticate users SQLDefaultUID $galaxy_uid SQLDefaultGID $galaxy_uid SQLDefaultHomedir $install_dir/var SQLUserInfo custom:/LookupGalaxyUser SQLNamedQuery LookupGalaxyUser SELECT "email,password,'$galaxy_uid','$galaxy_uid','$galaxy_fs/tmp/ftp/%U','/bin/bash' from galaxy_user where email='%U'" ================================================ FILE: installed_files/protvis_default.template ================================================ export TMPDIR=$protvis_converted_files_dir export PORT=$protvis_port export ADDRESS="0.0.0.0" export PROTVIS_HOME=$protvis_home export PROTVIS_USER=$protvis_user ================================================ FILE: installed_files/protvis_init ================================================ #!/bin/sh ### BEGIN INIT INFO # Provides: protvis # Required-Start: # Required-Stop: # Default-Start: 2 3 4 5 # Default-Stop: 0 1 6 # Short-Description: Start protvis daemon ### END INIT INFO PATH=/sbin:/bin:/usr/sbin:/usr/bin NAME="protvis" . /lib/lsb/init-functions . /etc/default/$NAME case $1 in start) DAEMON_ARGS="--daemon --pid-file=$PROTVIS_HOME/daemon.pid $PROTVIS_HOME/production.ini http_port=$PORT http_address=$ADDRESS" if [ ! -f "$PROTVIS_HOME/production.ini" ]; then log_daemon_msg "Copying over default settings for protvis." su - $PROTVIS_USER /bin/bash -c "cd $PROTVIS_HOME; cp production.ini.sample production.ini" fi log_daemon_msg "Starting $NAME" "$NAME" su - $PROTVIS_USER /bin/bash -c "export TMPDIR=$TMPDIR; cd $PROTVIS_HOME; env/bin/pserve $DAEMON_ARGS" log_end_msg $? ;; stop) log_daemon_msg "Stopping $NAME" "$NAME" DAEMON_ARGS="--stop-daemon --pid-file=$PROTVIS_HOME/daemon.pid" su - $PROTVIS_USER /bin/bash -c "cd $PROTVIS_HOME; env/bin/pserve $DAEMON_ARGS" log_end_msg $? ;; restart) $0 stop && sleep 2 && $0 start ;; *) echo "Usage: $0 {start|stop|restart}" exit 2 ;; esac ================================================ FILE: installed_files/psql ================================================ #!/bin/bash # # This script is required because of CloudMan - CloudMan's path resolution # code would result in using the latest version of PostgreSQL installed via # a package manager. However, if a user switches images, the database data # may not match the version of PostgreSQL being invoked so this script # is used to differentiate which PostgreSQL version is actually invoked. # This had to be done here because it was the only variable for old # clusters being invoked with a new image. Also note that this script # is image dependent (because PostgreSQL v9.1 will be the default version). versions=( "9.1" "8.4" ) args="" for arg in "$@"; do # If there are spaces in an argument, surround it with quotes or psql won’t work [[ "$arg" = "${arg% *}" ]] && args="$args $arg" || args="$args \"$arg\"" done for ver in "${versions[@]}"; do if [ "$ver" == "9.1" ]; then cmd="/usr/lib/postgresql/$ver/bin/psql_orig $args" else cmd="/usr/lib/postgresql/$ver/bin/psql $args" fi echo “Running command: $cmd” # Must eval here; otherwise quotes in the args don’t get passed through properly eval $cmd ec=$? if [ $ec -eq 0 ]; then echo "Version $ver seems to work" exit $ec fi done # Didn’t find a working solution it seems exit 1 ================================================ FILE: installed_files/setupnx.sh ================================================ #!/bin/bash # Set up NX to work with password access. # This can be run manually or triggered from ~/.bash_login if [ `id -u` != 0 ] ; then echo "This script must be run as root." exit 1 fi CURRENTUSER=${1:-$USER} cat < /dev/null ; then passwd "$USERTOMAKE" else useradd -m -s /bin/bash "$USERTOMAKE" passwd "$USERTOMAKE" fi #force SSH to allow password logins echo "Reconfiguring SSH to allow password login." sed -i 's/^PasswordAuthentication .*/PasswordAuthentication yes/' /etc/ssh/sshd_config /etc/init.d/ssh reload echo "Setting up NX." #force freenx-server to fix itself in any case if ! dpkg -s freenx-server >&/dev/null ; then apt-get -y install freenx-server fi dpkg-reconfigure -pcritical freenx-server >&/dev/null echo "DONE! You can find clients for NX here: " echo " http://www.nomachine.com/download.php" ================================================ FILE: installed_files/tool_data_table_conf.xml ================================================ name, value, dbkey, species
value, dbkey, name, path
value, dbkey, name, path
value, dbkey, name, path
value, dbkey, name, path
value, dbkey, name, path
value, dbkey, name, path
index, value, path
value, dbkey, name, path
value, dbkey, name, path
index, dbkey, value
dbkey, value
dbkey1, dbkey2, value
value, dbkey, name, path
value, dbkey, name, path
================================================ FILE: installed_files/vncserver_default.template ================================================ # The Username:Group that will run VNC export USER="$vnc_user" # The display that VNC will use export DISPLAY="$vnc_display" # Color depth (between 8 and 32) export DEPTH="$vnc_depth" # The Desktop geometry to use. export GEOMETRY="$vnc_geometry" # The name that the VNC Desktop will have. export DESKTOP_NAME="cloudbiolinux-vnc-server" ================================================ FILE: installed_files/vncserver_init ================================================ #!/bin/sh -e ### BEGIN INIT INFO # Provides: vncserver # Required-Start: networking # Default-Start: 3 4 5 # Default-Stop: 0 6 ### END INIT INFO PATH="$PATH:/usr/X11R6/bin/" NAME="vncserver" # Read configuration variable file if it is present [ -r /etc/default/$NAME ] && . /etc/default/$NAME OPTIONS="-name ${DESKTOP_NAME} -depth ${DEPTH} -geometry ${GEOMETRY} :${DISPLAY}" . /lib/lsb/init-functions case "$1" in start) log_action_begin_msg "Starting vncserver for user '${USER}' on localhost:${DISPLAY}" su ${USER} -c "/usr/bin/vncserver ${OPTIONS}" ;; stop) log_action_begin_msg "Stoping vncserver for user '${USER}' on localhost:${DISPLAY}" su ${USER} -c "/usr/bin/vncserver -kill :${DISPLAY}" ;; restart) $0 stop $0 start ;; esac exit 0 ================================================ FILE: installed_files/xstartup ================================================ [ -x /etc/vnc/xstartup ] && exec /etc/vnc/xstartup [ -r $HOME/.Xresources ] && xrdb $HOME/.Xresources xsetroot -solid grey vncconfig -iconic & x-terminal-emulator -geometry 80x24+10+10 -ls -title "$VNCDESKTOP Desktop" & # jwm & startxfce4 & ================================================ FILE: installed_files/xvfb_default ================================================ XVFB_OPTS=":42 -auth /var/lib/xvfb/auth -ac -nolisten tcp -shmem -screen 0 800x600x24" ================================================ FILE: installed_files/xvfb_init ================================================ #!/bin/sh ### BEGIN INIT INFO # Provides: xvfb # Required-Start: $syslog # Required-Stop: $syslog # Default-Start: 2 3 4 5 # Default-Stop: 0 1 6 # Short-Description: Start Xvfb daemon ### END INIT INFO PATH=/sbin:/bin:/usr/sbin:/usr/bin . /lib/lsb/init-functions NAME=xvfb DAEMON=/usr/bin/Xvfb PIDFILE=/var/run/Xvfb.pid test -x $DAEMON || exit 5 if [ -r /etc/default/$NAME ]; then . /etc/default/$NAME fi case $1 in start) log_daemon_msg "Starting Virtual Framebuffer" "Xvfb" start-stop-daemon --start --quiet --background --make-pidfile --pidfile $PIDFILE --startas $DAEMON -- $XVFB_OPTS status=$? log_end_msg $status ;; stop) log_daemon_msg "Stopping Virtual Framebuffer" "Xvfb" start-stop-daemon --stop --quiet --pidfile $PIDFILE log_end_msg $? rm -f $PIDFILE ;; restart|force-reload) $0 stop && sleep 2 && $0 start ;; try-restart) if $0 status >/dev/null; then $0 restart else exit 0 fi ;; reload) exit 3 ;; status) pidofproc -p $PIDFILE $DAEMON >/dev/null status=$? if [ $status -eq 0 ]; then log_success_msg "Xvfb server is running." else log_failure_msg "Xvfb server is not running." fi exit $status ;; *) echo "Usage: $0 {start|stop|restart|try-restart|force-reload|status}" exit 2 ;; esac ================================================ FILE: manifest/custom-packages.yaml ================================================ abyss: description: Assembly By Short Sequences - a de novo, parallel, paired-end sequence assembler. homepage_uri: http://www.bcgsc.ca/platform/bioinfo/software/abyss name: abyss version: 1.3.3 beast: description: 'BEAST: Bayesian MCMC analysis of molecular sequences.' homepage_uri: http://beast.bio.ed.ac.uk/main_page name: beast version: '' bedtools: description: A flexible suite of utilities for comparing genomic features. homepage_uri: https://code.google.com/p/bedtools/ name: bedtools version: github bfast: description: 'BFAST: Blat-like Fast Accurate Search Tool.' homepage_uri: http://sourceforge.net/apps/mediawiki/bfast/index.php?title=Main_Page name: bfast version: 0.7.0 bio4j: description: Bio4j graph based database built on Neo4j with UniProt, GO, RefSeq and more. homepage_uri: http://www.bio4j.com/ name: bio4j version: '0.7' bowtie: description: The bowtie short read aligner. homepage_uri: http://bowtie-bio.sourceforge.net/index.shtml name: bowtie version: 0.12.7 bowtie2: description: bowtie2 short read aligner, with gap support. homepage_uri: http://bowtie-bio.sourceforge.net/bowtie2/index.shtml name: bowtie2 version: 2.0.0-beta6 bwa: description: 'BWA: aligns short nucleotide sequences against a long reference sequence.' homepage_uri: http://bio-bwa.sourceforge.net/ name: bwa version: 0.5.9 bx_python: description: Tools for manipulating biological data, particularly multiple sequence alignments homepage_uri: https://bitbucket.org/james_taylor/bx-python/wiki/Home name: bx_python version: bitbucket cljr: description: Clojure package manager, cljr. homepage_uri: http://github.com/liebke/cljr name: cljr version: '' cloudman: description: ' A meta method for installing all of CloudMan components.' homepage_uri: '' name: cloudman version: '' crisp: description: Detect SNPs and short indels from pooled sequencing data. homepage_uri: https://sites.google.com/site/vibansal/software/crisp/ name: crisp version: '5' cufflinks: description: Cufflinks assembles transcripts, estimates their abundances, and tests for differential expression and regulation in RNA-Seq samples. homepage_uri: http://cufflinks.cbcb.umd.edu/ name: cufflinks version: 2.0.0 dwgsim: description: 'DWGSIM: simulating NGS data and evaluating mappings and variant calling.' homepage_uri: http://sourceforge.net/apps/mediawiki/dnaa/index.php?title=Main_Page name: dwgsim version: 0.1.8 echo: description: 'ECHO: A reference-free short-read error correction algorithm' homepage_uri: http://uc-echo.sourceforge.net/ name: echo version: '1_11' emboss: description: 'EMBOSS: A high-quality package of free, Open Source software for molecular biology.' homepage_uri: http://emboss.sourceforge.net/ name: emboss version: 6.3.1 fastqc: description: A quality control tool for high throughput sequence data. homepage_uri: http://www.bioinformatics.babraham.ac.uk/projects/fastqc/ name: fastqc version: 0.10.1 fastx_toolkit: description: 'FASTX-Toolkit: collection of command line tools for Short-Reads FASTA/FASTQ files preprocessing.' homepage_uri: http://hannonlab.cshl.edu/fastx_toolkit/ name: fastx_toolkit version: 0.0.13 freebayes: description: Bayesian haplotype-based polymorphism discovery and genotyping. homepage_uri: https://github.com/ekg/freebayes name: freebayes version: github gatk: description: 'GATK: library for writing efficient analysis tools using next-generation sequencing data ' homepage_uri: http://www.broadinstitute.org/gsa/wiki/index.php/Home_Page name: gatk version: 1.6-11-g3b2fab9 gatk_queue: description: Command-line scripting framework for defining multi-stage genomic analysis pipelines. homepage_uri: http://www.broadinstitute.org/gsa/wiki/index.php/GATK-Queue name: gatk_queue version: 1.6-11-g3b2fab9 gmap: description: 'GMAP and GSNAP: A Genomic Mapping and Alignment Program for mRNA EST and short reads.' homepage_uri: http://research-pub.gene.com/gmap/ name: gmap version: '2011-11-12' hydra: description: Hydra detects structural variation breakpoints in both unique and duplicated genomic regions. homepage_uri: https://code.google.com/p/hydra-sv/ name: hydra version: 0.5.3 kraken: description: kraken homepage_uri: 'http://ccb.jhu.edu/software/kraken/' name: kraken version: 0.10.3-beta lastz: description: LASTZ sequence alignment program. homepage_uri: http://www.bx.psu.edu/miller_lab/dist/README.lastz-1.02.00/README.lastz-1.02.00a.html name: lastz version: 1.02.00 leinengin: description: Clojure tool for project configuration and automation. homepage_uri: http://github.com/technomancy/leiningen name: leinengin version: '' macs: description: Model-based Analysis for ChIP-Seq. homepage_uri: http://liulab.dfci.harvard.edu/MACS/ name: macs version: 1.4.2 matplotlib: description: matplotlib is a python 2D plotting library which produces publication quality figures homepage_uri: http://matplotlib.sourceforge.net/ name: matplotlib version: 1.0.1 mosaik: description: 'MOSAIK: reference-guided aligner for next-generation sequencing technologies' homepage_uri: http://code.google.com/p/mosaik-aligner/ name: mosaik version: github nginx: description: Nginx open source web server. homepage_uri: http://www.nginx.org/ name: nginx version: 1.2.0 novoalign: description: Novoalign short read aligner using Needleman-Wunsch algorithm with affine gap penalties. homepage_uri: http://www.novocraft.com/main/index.php name: novoalign version: V2.08.01 perm: description: Efficient mapping of short sequences accomplished with periodic full sensitive spaced seeds. homepage_uri: https://code.google.com/p/perm/ name: perm version: '3.6' pgdspider: description: PGDSpider format conversion for population genetics programs. homepage_uri: http://www.cmpg.unibe.ch/software/PGDSpider/ name: pgdspider version: 2.0.1.2 picard: description: Command-line utilities that manipulate BAM files with a Java API. homepage_uri: http://picard.sourceforge.net/ name: picard version: '1.68' plink_seq: description: A toolset for working with human genetic variation data. homepage_uri: http://atgu.mgh.harvard.edu/plinkseq/ name: plink_seq version: '0.08' proftpd: description: Highly configurable GPL-licensed FTP server software. homepage_uri: http://proftpd.org/ name: proftpd version: 1.3.4a pydoop: description: pydoop; provides Hadoop access for Python. homepage_uri: http://pydoop.sourceforge.net/docs/ name: pydoop version: '' rpy: description: RPy is a very simple, yet robust, Python interface to the R Programming Language. homepage_uri: http://rpy.sourceforge.net/ name: rpy version: 1.0.3 s3fs: description: FUSE-based file system backed by Amazon S3. homepage_uri: https://code.google.com/p/s3fs/ name: s3fs version: '1.61' samtools: description: SAM Tools provide various utilities for manipulating alignments in the SAM format. homepage_uri: http://samtools.sourceforge.net/ name: samtools version: 0.1.18 seal: description: 'Install seal: process high-throughput sequencing with Hadoop.' homepage_uri: http://biodoop-seal.sf.net/ name: seal version: '' sge: description: Sun Grid Engine. homepage_uri: '' name: sge version: '' shrec: description: Shrec is a bioinformatics tool for error correction of HTS read data. homepage_uri: http://sourceforge.net/projects/shrec-ec/ name: shrec version: '2.2' snpeff: description: Variant annotation and effect prediction tool. homepage_uri: http://snpeff.sourceforge.net/ name: snpeff version: '2_0_5' solexaqa: description: SolexaQA creates visual representations of data quality from FASTQ files. homepage_uri: http://solexaqa.sourceforge.net/ name: solexaqa version: '1.4' stacks: description: 'Stacks: build loci out of a set of short-read sequenced samples.' homepage_uri: http://creskolab.uoregon.edu/stacks/ name: stacks version: '0.998' stampy: description: 'Stampy: mapping of short reads from illumina sequencing machines onto a reference genome.' homepage_uri: http://www.well.ox.ac.uk/project-stampy name: stampy version: 1.0.15 tassel: description: 'TASSEL: evaluate traits associations, evolutionary patterns, and linkage disequilibrium.' homepage_uri: http://www.maizegenetics.net/index.php?option=com_content&task=view&id=89&Itemid=119 name: tassel version: '3.0' tophat: description: TopHat is a fast splice junction mapper for RNA-Seq reads homepage_uri: http://tophat.cbcb.umd.edu/ name: tophat version: 2.0.0 tracer: description: A program for analysing results from Bayesian MCMC programs such as BEAST & MrBayes. homepage_uri: http://beast.bio.ed.ac.uk/main_page name: tracer version: '' transabyss: description: Analyze ABySS multi-k-assembled shotgun transcriptome data. homepage_uri: http://www.bcgsc.ca/platform/bioinfo/software/trans-abyss name: transabyss version: 1.3.2 trinity: description: Efficient and robust de novo reconstruction of transcriptomes from RNA-seq data. homepage_uri: http://trinityrnaseq.sourceforge.net/ name: trinity version: r2012-05-18 ucsc_tools: description: Useful executables from UCSC. homepage_uri: http://hgdownload.cse.ucsc.edu/admin/jksrc.zip name: ucsc_tools version: '' varianttools: description: Annotation, selection, and analysis of variants in the context of next-gen sequencing analysis. homepage_uri: http://varianttools.sourceforge.net/ name: varianttools version: 1.0.1 vcftools: description: Work with VCF files, such as those generated by the 1000 Genomes Project. homepage_uri: http://vcftools.sourceforge.net/ name: vcftools version: 0.1.7 velvet: description: Sequence assembler for very short reads. homepage_uri: http://www.ebi.ac.uk/~zerbino/velvet/ name: velvet version: 1.2.05 ================================================ FILE: manifest/debian-packages.yaml ================================================ accountsservice: description: query and manipulate user account information downloads: 8774 homepage_uri: http://cgit.freedesktop.org/accountsservice/ name: accountsservice section: gnome version: 0.6.15-2ubuntu9 acl: description: Access control list utilities downloads: 12836 homepage_uri: http://savannah.nongnu.org/projects/acl/ name: acl section: utils version: 2.2.51-5ubuntu1 acpi-support: description: scripts for handling many ACPI events downloads: 131 homepage_uri: '' name: acpi-support section: admin version: '0.140' acpid: description: Advanced Configuration and Power Interface event daemon downloads: 65108 homepage_uri: http://www.tedfelix.com/linux/acpid-netlink.html name: acpid section: admin version: 1:2.0.10-1ubuntu3 activity-log-manager-common: description: blacklist configuration for Zeitgeist (assets) downloads: 0 homepage_uri: https://launchpad.net/activity-log-manager name: activity-log-manager-common section: utils version: 0.9.4-0ubuntu3 activity-log-manager-control-center: description: blacklist configuration for Zeitgeist (control center integration) downloads: 0 homepage_uri: https://launchpad.net/activity-log-manager name: activity-log-manager-control-center section: utils version: 0.9.4-0ubuntu3 adduser: description: add and remove users and groups downloads: 108947 homepage_uri: http://alioth.debian.org/projects/adduser/ name: adduser section: admin version: 3.113ubuntu2 adium-theme-ubuntu: description: Adium message style for Ubuntu downloads: 1 homepage_uri: http://launchpad.net/adium-theme-ubuntu name: adium-theme-ubuntu section: gnome version: 0.3.2-0ubuntu1 aisleriot: description: Solitaire card games downloads: 751 homepage_uri: https://live.gnome.org/Aisleriot name: aisleriot section: games version: 1:3.2.3.2-0ubuntu1 alacarte: description: easy GNOME menu editing tool downloads: 4661 homepage_uri: '' name: alacarte section: universe/utils version: 0.13.2-2ubuntu4 alsa-base: description: ALSA driver configuration files downloads: 2273 homepage_uri: http://www.alsa-project.org/ name: alsa-base section: sound version: 1.0.25+dfsg-0ubuntu1 alsa-utils: description: Utilities for configuring and using ALSA downloads: 43508 homepage_uri: http://www.alsa-project.org/ name: alsa-utils section: sound version: 1.0.25-1ubuntu5 anacron: description: cron-like program that doesn't go by time downloads: 52784 homepage_uri: '' name: anacron section: admin version: 2.3-14ubuntu1 ant: description: Java based build tool like make downloads: 3787 homepage_uri: http://ant.apache.org/ name: ant section: java version: 1.8.2-4build1 ant-optional: description: Java based build tool like make - optional libraries downloads: 1806 homepage_uri: http://ant.apache.org/ name: ant-optional section: java version: 1.8.2-4build1 antlr3: description: language tool for constructing recognizers, compilers etc downloads: 845 homepage_uri: '' name: antlr3 section: universe/devel version: 3.2.is.3.2-0ubuntu1 apache2: description: Apache HTTP Server metapackage downloads: 22309 homepage_uri: http://httpd.apache.org/ name: apache2 section: web version: 2.2.22-1ubuntu1 apache2-mpm-worker: description: Apache HTTP Server - high speed threaded model downloads: 7746 homepage_uri: http://httpd.apache.org/ name: apache2-mpm-worker section: net version: 2.2.22-1ubuntu1 apache2-utils: description: utility programs for webservers downloads: 52222 homepage_uri: http://httpd.apache.org/ name: apache2-utils section: net version: 2.2.22-1ubuntu1 apache2.2-bin: description: Apache HTTP Server common binary files downloads: 54017 homepage_uri: http://httpd.apache.org/ name: apache2.2-bin section: httpd version: 2.2.22-1ubuntu1 apache2.2-common: description: Apache HTTP Server common files downloads: 52101 homepage_uri: http://httpd.apache.org/ name: apache2.2-common section: web version: 2.2.22-1ubuntu1 apg: description: Automated Password Generator - Standalone version downloads: 2576 homepage_uri: '' name: apg section: admin version: 2.2.3.dfsg.1-2 app-install-data: description: Ubuntu applications (data files) downloads: 11372 homepage_uri: '' name: app-install-data section: gnome version: 0.12.04.4 app-install-data-partner: description: Application Installer (data files for partner applications/repositories) downloads: 0 homepage_uri: '' name: app-install-data-partner section: x11 version: 12.12.04.1 apparmor: description: User-space parser utility for AppArmor downloads: 80 homepage_uri: http://apparmor.net/ name: apparmor section: admin version: 2.7.102-0ubuntu3.1 appmenu-gtk: description: Export GTK menus over DBus downloads: 3 homepage_uri: https://launchpad.net/appmenu-gtk name: appmenu-gtk section: libs version: 0.3.92-0ubuntu1 appmenu-gtk3: description: Export GTK menus over DBus downloads: 2 homepage_uri: https://launchpad.net/appmenu-gtk name: appmenu-gtk3 section: libs version: 0.3.92-0ubuntu1 appmenu-qt: description: appmenu support for Qt downloads: 3 homepage_uri: https://launchpad.net/appmenu-qt name: appmenu-qt section: x11 version: 0.2.6-0ubuntu1 apport: description: automatically generate crash reports for debugging downloads: 18 homepage_uri: https://wiki.ubuntu.com/Apport name: apport section: utils version: 2.0.1-0ubuntu8 apport-gtk: description: GTK+ frontend for the apport crash report system downloads: 0 homepage_uri: https://wiki.ubuntu.com/Apport name: apport-gtk section: gnome version: 2.0.1-0ubuntu8 apport-symptoms: description: symptom scripts for apport downloads: 0 homepage_uri: https://wiki.ubuntu.com/Apport name: apport-symptoms section: utils version: '0.16' apt: description: commandline package manager downloads: 109951 homepage_uri: '' name: apt section: admin version: 0.8.16~exp12ubuntu10.2 apt-transport-https: description: https download transport for APT downloads: 140 homepage_uri: '' name: apt-transport-https section: admin version: 0.8.16~exp12ubuntu10.2 apt-utils: description: package managment related utility programs downloads: 51631 homepage_uri: '' name: apt-utils section: admin version: 0.8.16~exp12ubuntu10.2 apt-xapian-index: description: maintenance and search tools for a Xapian index of Debian packages downloads: 46570 homepage_uri: http://www.enricozini.org/sw/apt-xapian-index/ name: apt-xapian-index section: admin version: 0.44ubuntu5 aptdaemon: description: transaction based package management service downloads: 20343 homepage_uri: https://launchpad.net/aptdaemon name: aptdaemon section: admin version: 0.43+bzr805-0ubuntu1 aptdaemon-data: description: data files for clients downloads: 844 homepage_uri: https://launchpad.net/aptdaemon name: aptdaemon-data section: admin version: 0.43+bzr805-0ubuntu1 aptitude: description: terminal-based package manager (terminal interface only) downloads: 28616 homepage_uri: http://aptitude.alioth.debian.org/ name: aptitude section: admin version: 0.6.6-1ubuntu1 apturl: description: install packages using the apt protocol - GTK+ frontend downloads: 16 homepage_uri: '' name: apturl section: admin version: 0.5.1ubuntu3 apturl-common: description: install packages using the apt protocol - common data downloads: 14 homepage_uri: '' name: apturl-common section: admin version: 0.5.1ubuntu3 aspell: description: GNU Aspell spell-checker downloads: 18130 homepage_uri: http://aspell.net/ name: aspell section: text version: 0.60.7~20110707-1 aspell-en: description: English dictionary for GNU Aspell downloads: 10099 homepage_uri: http://aspell.net/ name: aspell-en section: text version: 6.0-0-6ubuntu2 at: description: Delayed job execution and batch processing downloads: 61794 homepage_uri: '' name: at section: admin version: 3.1.13-1ubuntu1 at-spi2-core: description: Assistive Technology Service Provider Interface (dbus core) downloads: 352 homepage_uri: http://live.gnome.org/Accessibility/GNOME3 name: at-spi2-core section: misc version: 2.4.2-0ubuntu0.1 autoconf: description: automatic configure script builder downloads: 4443 homepage_uri: http://www.gnu.org/software/autoconf name: autoconf section: devel version: 2.68-1ubuntu2 automake: description: Tool for generating GNU Standards-compliant Makefiles downloads: 3850 homepage_uri: http://www.gnu.org/software/automake/ name: automake section: devel version: 1:1.11.3-1ubuntu2 autotools-dev: description: Update infrastructure for config.{guess,sub} files downloads: 4445 homepage_uri: http://savannah.gnu.org/projects/config/ name: autotools-dev section: devel version: 20120210.1ubuntu1 avahi-autoipd: description: Avahi IPv4LL network address configuration daemon downloads: 5901 homepage_uri: http://avahi.org/ name: avahi-autoipd section: net version: 0.6.30-5ubuntu2 avahi-daemon: description: Avahi mDNS/DNS-SD daemon downloads: 48215 homepage_uri: http://avahi.org/ name: avahi-daemon section: net version: 0.6.30-5ubuntu2 avahi-utils: description: Avahi browsing, publishing and discovery utilities downloads: 632 homepage_uri: http://avahi.org/ name: avahi-utils section: net version: 0.6.30-5ubuntu2 axel: description: light download accelerator - console version downloads: 200 homepage_uri: http://axel.alioth.debian.org/ name: axel section: universe/web version: 2.4-1 bamfdaemon: description: Window matching library - daemon downloads: 18 homepage_uri: https://launchpad.net/bamf name: bamfdaemon section: libs version: 0.2.118-0ubuntu0.1 baobab: description: GNOME disk usage analyzer downloads: 4281 homepage_uri: http://live.gnome.org/GnomeUtils name: baobab section: gnome version: 3.4.1-0ubuntu1 base-files: description: Debian base system miscellaneous files downloads: 110597 homepage_uri: '' name: base-files section: admin version: 6.5ubuntu6 base-passwd: description: Debian base system master password and group files downloads: 110291 homepage_uri: '' name: base-passwd section: admin version: 3.5.24 bash: description: GNU Bourne Again SHell downloads: 110784 homepage_uri: http://tiswww.case.edu/php/chet/bash/bashtop.html name: bash section: shells version: 4.2-2ubuntu2 bash-completion: description: programmable completion for the bash shell downloads: 6280 homepage_uri: http://bash-completion.alioth.debian.org name: bash-completion section: shells version: 1:1.3-1ubuntu8 bc: description: The GNU bc arbitrary precision calculator language downloads: 18413 homepage_uri: '' name: bc section: math version: 1.06.95-2 bind9-host: description: Version of 'host' bundled with BIND 9.X downloads: 53926 homepage_uri: '' name: bind9-host section: net version: 1:9.8.1.dfsg.P1-4ubuntu0.1 binfmt-support: description: Support for extra binary formats downloads: 18676 homepage_uri: '' name: binfmt-support section: admin version: 2.0.8 binutils: description: GNU assembler, linker and binary utilities downloads: 69573 homepage_uri: '' name: binutils section: devel version: 2.22-6ubuntu1 bio-linux-assembly-conversion-tools: description: Conversion tools for handling 454 assemblies. downloads: 0 homepage_uri: '' name: bio-linux-assembly-conversion-tools section: science version: 0.01-9 bio-linux-base-directories: description: Essential Bio-Linux configuration files. downloads: 0 homepage_uri: '' name: bio-linux-base-directories section: science version: 1.0-37 bio-linux-big-blast: description: The big-blast script for annotation of long sequences downloads: 0 homepage_uri: '' name: bio-linux-big-blast section: '' version: 1.0.3-4 bio-linux-blast: description: Rapid searching of nucleotide and protein databases. downloads: 0 homepage_uri: '' name: bio-linux-blast section: science version: 1:1bl-6 bio-linux-blast+: description: Blast+ contains new builds of the popular blast family of programs from the NCBI. downloads: 0 homepage_uri: '' name: bio-linux-blast+ section: science version: 1:1bl1 bio-linux-bldp-files: description: Documentation for bioinformatics software on Bio-Linux. downloads: 0 homepage_uri: '' name: bio-linux-bldp-files section: science version: 1.1-96 bio-linux-blixem: description: Interactive browser for viewing pairwise Blast results. downloads: 0 homepage_uri: '' name: bio-linux-blixem section: science version: 3.1.1-1 bio-linux-cap3: description: A base calling system downloads: 0 homepage_uri: '' name: bio-linux-cap3 section: science version: 20080520-1 bio-linux-cd-hit: description: a suite of programs designed to quickly group sequences downloads: 0 homepage_uri: '' name: bio-linux-cd-hit section: science version: 2009-1020-1 bio-linux-clcsequenceviewer: description: CLC Free Sequence Viewer downloads: 1 homepage_uri: '' name: bio-linux-clcsequenceviewer section: science version: 6.4-1 bio-linux-clustal: description: Installs clustalw and clustalx for multiple sequence and profile alignments. downloads: 0 homepage_uri: '' name: bio-linux-clustal section: science version: 1:1bl-6 bio-linux-coalesce: description: Population estimate modelling. downloads: 0 homepage_uri: '' name: bio-linux-coalesce section: science version: 1.5b-1 bio-linux-das-prep: description: Sets up dependencies necessary for running the DAS Proserver on Bio-Linux downloads: 0 homepage_uri: '' name: bio-linux-das-prep section: science version: 0.1-4 bio-linux-dendroscope: description: Interactive viewer for large phylogenetic trees. downloads: 0 homepage_uri: '' name: bio-linux-dendroscope section: science version: 1.4.0-5 bio-linux-dotter: description: Graphical dotplot program for detailed comparison of two sequences downloads: 0 homepage_uri: '' name: bio-linux-dotter section: science version: 3.1-1 bio-linux-dust: description: Dust - repetitive sequence masker downloads: 0 homepage_uri: '' name: bio-linux-dust section: science version: 1+0.1+20020708-4 bio-linux-estscan: description: Program for detecting coding regions of DNA sequences. downloads: 0 homepage_uri: '' name: bio-linux-estscan section: science version: 2.1-11 bio-linux-exchanger: description: Exchanger lets you store, annotate and share files via an omixed server. downloads: 0 homepage_uri: '' name: bio-linux-exchanger section: science version: 1.0-4 bio-linux-fasta: description: Collection of programs for searching DNA and protein databases. downloads: 1 homepage_uri: '' name: bio-linux-fasta section: science version: 36t2d5-0 bio-linux-fastdnaml: description: fastDNAml is a program for estimating maximum likelihood phylogenetic trees from nucleotide sequences. downloads: 0 homepage_uri: '' name: bio-linux-fastdnaml section: science version: 1:1bl-0 bio-linux-fasttree: description: FastTree infers approximately-maximum-likelihood phylogenetic trees from alignments of nucleotide or protein sequences. FastTree can handle alignments with up to a million of sequences in a reasonable amount of time and memory. downloads: 0 homepage_uri: '' name: bio-linux-fasttree section: science version: 2.1.3-0 bio-linux-fluctuate: description: Population modelling software. downloads: 1 homepage_uri: '' name: bio-linux-fluctuate section: science version: 1.4-0 bio-linux-forester: description: ATV (A Tree Viewer) is a Java tool for the visualization of annotated phylogenetic trees. downloads: 0 homepage_uri: '' name: bio-linux-forester section: science version: 1.92-10 bio-linux-gap2caf: description: Conversion tools for handling conversion of gap format assemblies to caf format. downloads: 0 homepage_uri: '' name: bio-linux-gap2caf section: science version: 0.20100408-1 bio-linux-genquery: description: GenQuery is a set of Perl libraries for managing SQL query templates and making web-based query forms. downloads: 0 homepage_uri: '' name: bio-linux-genquery section: science version: 2.2.4-3 bio-linux-glimmer3: description: Glimmer is a system for finding genes in microbial DNA. downloads: 0 homepage_uri: '' name: bio-linux-glimmer3 section: science version: 3.02-0 bio-linux-happy: description: 'HAPPY: a software package for Multipoint QTL Mapping in Genetically Heterogeneous Animals' downloads: 0 homepage_uri: '' name: bio-linux-happy section: science version: 1.2-1 bio-linux-jalview: description: Jalview is a Java multiple alignment editor downloads: 1 homepage_uri: '' name: bio-linux-jalview section: science version: 1:2.6.1-1 bio-linux-keyring: description: Public key package for the Bio-Linux package repository downloads: 0 homepage_uri: '' name: bio-linux-keyring section: misc version: '4' bio-linux-lucy: description: Preparation of raw DNA sequence fragments for sequence assembly. downloads: 1 homepage_uri: '' name: bio-linux-lucy section: science version: 1.20-0 bio-linux-maxd: description: maxd is a data warehouse and visualisation environment for genomic expression data. downloads: 1 homepage_uri: '' name: bio-linux-maxd section: science version: 2.3.0-5 bio-linux-mesquite: description: Mesquite is software for evolutionary biology. downloads: 0 homepage_uri: '' name: bio-linux-mesquite section: science version: 2.74-1 bio-linux-migrate: description: Estimates effective population sizes and past migration rates between n populations. downloads: 0 homepage_uri: '' name: bio-linux-migrate section: science version: 3.1.2-1 bio-linux-mrbayes-multi: description: Multiprocessor version of MrBayes downloads: 2 homepage_uri: '' name: bio-linux-mrbayes-multi section: science version: 3.1.2-4 bio-linux-mspcrunch: description: MSPcrunch is a BLAST post-processing filter. downloads: 0 homepage_uri: '' name: bio-linux-mspcrunch section: science version: 2.5-0 bio-linux-mview: description: Multiple alignment viewer for sequence database search results. downloads: 0 homepage_uri: '' name: bio-linux-mview section: science version: 1.51-2 bio-linux-ncbi-tools-x11: description: NCBI libraries for biology applications (X-based utilities) downloads: 2 homepage_uri: '' name: bio-linux-ncbi-tools-x11 section: science version: 1:1bl-2 bio-linux-nrdb: description: nrdb can be used to generate quasi-nonredundant protein and nucleotide sequence databases. downloads: 0 homepage_uri: '' name: bio-linux-nrdb section: science version: 2-0 bio-linux-ocount: description: Ocount - oligonucleotide frequency counter. downloads: 0 homepage_uri: '' name: bio-linux-ocount section: science version: 0.5-0 bio-linux-oligoarray: description: Computation of gene-specific oligonucleotides for genome-scale microarray construction. downloads: 0 homepage_uri: '' name: bio-linux-oligoarray section: science version: 2.1-8 bio-linux-oligoarrayaux: description: OligoArrayAux 3.7 is free software that is required for the OligoArray2.1 software. downloads: 0 homepage_uri: '' name: bio-linux-oligoarrayaux section: science version: 3.7-0 bio-linux-omegamap: description: OmegaMap is a program for detecting natural selection and recombination in DNA or RNA sequences. downloads: 0 homepage_uri: '' name: bio-linux-omegamap section: science version: 0.5-0 bio-linux-paml: description: Phylogenetic analysis tool for DNA and protein sequences using maximum likelihood. downloads: 0 homepage_uri: '' name: bio-linux-paml section: science version: 4.4c-1 bio-linux-partigene: description: Cluster and anotation tool for sequence data. downloads: 0 homepage_uri: '' name: bio-linux-partigene section: science version: 3.05-8 bio-linux-pfaat: description: Sequence alignment editor designed for protein family analysis. downloads: 0 homepage_uri: '' name: bio-linux-pfaat section: science version: 1.0.034-5 bio-linux-prank: description: Multiple alignment software. downloads: 0 homepage_uri: '' name: bio-linux-prank section: science version: 20100318-0 bio-linux-prevent-upgrade: description: Stop Bio-Linux from upgrading on release of 12.04 LTS downloads: 0 homepage_uri: '' name: bio-linux-prevent-upgrade section: science version: '0.2' bio-linux-prot4est: description: Translation of error-prone nucleotide sequences producing robust polypeptide predictions downloads: 0 homepage_uri: '' name: bio-linux-prot4est section: science version: 2.2-10 bio-linux-qtlcart: description: Suite of programs to map quantitative traits using a map of molecular markers. downloads: 0 homepage_uri: '' name: bio-linux-qtlcart section: science version: 1.17f-0 bio-linux-rbs-finder: description: A program to find Ribosomal binding sites. downloads: 0 homepage_uri: '' name: bio-linux-rbs-finder section: '' version: 1.0-3 bio-linux-recombine: description: Population modelling software. downloads: 0 homepage_uri: '' name: bio-linux-recombine section: science version: 1.41-0 bio-linux-sampledata: description: Sample data for Bio-Linux packages downloads: 0 homepage_uri: '' name: bio-linux-sampledata section: science version: 20041011-9 bio-linux-seaview: description: Multiple sequence alignment editor. downloads: 1 homepage_uri: '' name: bio-linux-seaview section: science version: 4.2.8-2 bio-linux-sequin: description: Transitional package that provides bio-linux-ncbi-tools-x11 downloads: 0 homepage_uri: '' name: bio-linux-sequin section: science version: 1:1bl-0 bio-linux-splitstree: description: Analysis and visualisation tool for distance data from biological sequences. downloads: 0 homepage_uri: '' name: bio-linux-splitstree section: science version: 4.0-11 bio-linux-squint: description: Squint is a java-based multiple alignment program and editor downloads: 0 homepage_uri: '' name: bio-linux-squint section: science version: 1.02-2 bio-linux-staden: description: A suite of tools for sequence analysis. downloads: 1 homepage_uri: '' name: bio-linux-staden section: science version: 2006.1.7.0+1.11.6-5 bio-linux-stars: description: STARS is an alternative interface to Staden. downloads: 0 homepage_uri: '' name: bio-linux-stars section: science version: 1.2a-11 bio-linux-taxinspector: description: TaxInspector is a browser for entries in the NCBI taxonomy database. downloads: 0 homepage_uri: '' name: bio-linux-taxinspector section: science version: 1.0b3-10 bio-linux-tetra: description: TETRA - tetranucleotide frequency calculator with GUI. downloads: 0 homepage_uri: '' name: bio-linux-tetra section: science version: 1.0b3 bio-linux-themes-v5: description: Graphics, icons, KDE splash screen and a GDM theme for Bio-Linux 5. downloads: 0 homepage_uri: '' name: bio-linux-themes-v5 section: science version: '1.2' bio-linux-transterm-hp: description: Transterm finds rho-independent transcription terminators in bacterial genomes. downloads: 0 homepage_uri: '' name: bio-linux-transterm-hp section: science version: 2.07-1 bio-linux-treeview: description: TreeView is a simple program for displaying phylogenies. downloads: 1 homepage_uri: '' name: bio-linux-treeview section: science version: 1:1bl-0 bio-linux-trnascan: description: tRNAscan-SE searches for tRNA genes in genomic sequences. downloads: 0 homepage_uri: '' name: bio-linux-trnascan section: science version: 1.23-0 bio-linux-tutorials: description: Tutorials and documentation for Bio-Linux users. downloads: 0 homepage_uri: '' name: bio-linux-tutorials section: science version: 1-18 bio-linux-wise2: description: Tool for comparison of biopolymers, DNA, and protein sequence. downloads: 0 homepage_uri: '' name: bio-linux-wise2 section: science version: 2.1.20-0 bio-linux-xcut: description: A re-write of the GNU cut command with many new features. downloads: 0 homepage_uri: '' name: bio-linux-xcut section: science version: 1.1-6 bio-linux-zsh: description: Z-Shell wrapper package for Bio-Linux. downloads: 0 homepage_uri: '' name: bio-linux-zsh section: science version: 1-18 blast2: description: Basic Local Alignment Search Tool downloads: 64 homepage_uri: http://www.ncbi.nih.gov/BLAST/ name: blast2 section: science version: 1:2.2.25.20110713-3ubuntu2 blcr-util: description: Userspace tools to Checkpoint and Restart Linux processes downloads: 214 homepage_uri: https://ftg.lbl.gov/CheckpointRestart/CheckpointRestart.shtml name: blcr-util section: universe/misc version: 0.8.2-15ubuntu2.1 bluez: description: Bluetooth tools and daemons downloads: 36308 homepage_uri: http://www.bluez.org name: bluez section: admin version: 4.98-2ubuntu7 bluez-alsa: description: Bluetooth ALSA support downloads: 197 homepage_uri: http://www.bluez.org name: bluez-alsa section: admin version: 4.98-2ubuntu7 bluez-alsa:i386: description: Bluetooth ALSA support downloads: 0 homepage_uri: http://www.bluez.org name: bluez-alsa:i386 section: admin version: 4.98-2ubuntu7 bluez-cups: description: Bluetooth printer driver for CUPS downloads: 2273 homepage_uri: http://www.bluez.org name: bluez-cups section: admin version: 4.98-2ubuntu7 bluez-gstreamer: description: Bluetooth GStreamer support downloads: 5056 homepage_uri: http://www.bluez.org name: bluez-gstreamer section: admin version: 4.98-2ubuntu7 branding-ubuntu: description: Replacement artwork with Ubuntu branding downloads: 0 homepage_uri: https://wiki.ubuntu.com/branding name: branding-ubuntu section: graphics version: '0.7' brasero: description: CD/DVD burning application for GNOME downloads: 26866 homepage_uri: http://www.gnome.org/projects/brasero/ name: brasero section: gnome version: 3.4.1-0ubuntu1 brasero-cdrkit: description: cdrkit extensions for the Brasero burning application downloads: 624 homepage_uri: http://www.gnome.org/projects/brasero/ name: brasero-cdrkit section: gnome version: 3.4.1-0ubuntu1 brasero-common: description: Common files for the Brasero CD burning application and library downloads: 12343 homepage_uri: http://www.gnome.org/projects/brasero/ name: brasero-common section: gnome version: 3.4.1-0ubuntu1 bridge-utils: description: Utilities for configuring the Linux Ethernet bridge downloads: 6694 homepage_uri: '' name: bridge-utils section: net version: 1.5-2ubuntu6 brltty: description: Access software for a blind person using a braille display downloads: 2336 homepage_uri: http://mielke.cc/brltty/ name: brltty section: admin version: 4.3-1ubuntu5 bsdmainutils: description: collection of more utilities from FreeBSD downloads: 105954 homepage_uri: '' name: bsdmainutils section: utils version: 8.2.3ubuntu1 bsdutils: description: Basic utilities from 4.4BSD-Lite downloads: 100492 homepage_uri: http://userweb.kernel.org/~kzak/util-linux/ name: bsdutils section: utils version: 1:2.20.1-1ubuntu3 bsh: description: Java scripting environment (BeanShell) Version 2 downloads: 2545 homepage_uri: http://www.beanshell.org/ name: bsh section: devel version: 2.0b4-12build1 bsh-gcj: description: Java scripting environment (BeanShell) Version 2 (native code) downloads: 642 homepage_uri: http://www.beanshell.org/ name: bsh-gcj section: libs version: 2.0b4-12build1 build-essential: description: Informational list of build-essential packages downloads: 1485 homepage_uri: '' name: build-essential section: devel version: 11.5ubuntu2 busybox-initramfs: description: Standalone shell setup for initramfs downloads: 29 homepage_uri: http://www.busybox.net name: busybox-initramfs section: shells version: 1:1.18.5-1ubuntu4 busybox-static: description: Standalone rescue shell with tons of builtin utilities downloads: 119 homepage_uri: http://www.busybox.net name: busybox-static section: shells version: 1:1.18.5-1ubuntu4 byobu: description: powerful, text based window manager and shell multiplexer downloads: 475 homepage_uri: http://launchpad.net/byobu name: byobu section: misc version: 5.17-0ubuntu1 bzip2: description: high-quality block-sorting file compressor - utilities downloads: 61069 homepage_uri: http://www.bzip.org/ name: bzip2 section: utils version: 1.0.6-1 bzr: description: easy to use distributed version control system downloads: 1101 homepage_uri: http://bazaar-vcs.org name: bzr section: devel version: 2.5.0-2ubuntu2 c2hs: description: C->Haskell Interface Generator downloads: 36 homepage_uri: http://hackage.haskell.org/package/c2hs name: c2hs section: universe/devel version: 0.16.3-2 ca-certificates: description: Common CA certificates downloads: 32260 homepage_uri: '' name: ca-certificates section: misc version: '20111211' ca-certificates-java: description: Common CA certificates (JKS keystore) downloads: 12396 homepage_uri: '' name: ca-certificates-java section: misc version: 20110912ubuntu6 cabal-install: description: command-line interface for Cabal and Hackage downloads: 145 homepage_uri: http://www.haskell.org/cabal/ name: cabal-install section: universe/haskell version: 0.14.0-2 cdbs: description: common build system for Debian packages downloads: 620 homepage_uri: '' name: cdbs section: devel version: 0.4.100ubuntu2 cgroup-lite: description: Light-weight package to set up cgroups at system boot downloads: 0 homepage_uri: '' name: cgroup-lite section: admin version: '1.1' checkbox: description: System testing application downloads: 0 homepage_uri: '' name: checkbox section: python version: 0.13.7 checkbox-qt: description: QT4 interface for checkbox downloads: 0 homepage_uri: '' name: checkbox-qt section: utils version: 0.13.7 clojure: description: a Lisp dialect for the JVM downloads: 20 homepage_uri: http://clojure.org name: clojure section: universe/devel version: 1.1.0+dfsg-3 cloud-init: description: Init scripts for cloud instances downloads: 1 homepage_uri: '' name: cloud-init section: admin version: 0.6.3-0ubuntu1 cloud-initramfs-growroot: description: automatically resize the root partition on first boot downloads: 0 homepage_uri: '' name: cloud-initramfs-growroot section: admin version: 0.4ubuntu1 cloud-initramfs-rescuevol: description: boot off a rescue volume rather than root filesystem downloads: 0 homepage_uri: '' name: cloud-initramfs-rescuevol section: admin version: 0.4ubuntu1 cloud-utils: description: cloud image management utilities downloads: 1 homepage_uri: '' name: cloud-utils section: admin version: 0.25-0ubuntu5 cloudbl-desktop: description: Adds useful links to the remote desktop for Cloud BL downloads: 0 homepage_uri: '' name: cloudbl-desktop section: science version: '1.4' clustalw: description: global multiple nucleotide or peptide sequence alignment downloads: 40 homepage_uri: http://www.clustal.org/ name: clustalw section: non-free/science version: 2.1+lgpl-2 clustalx: description: Multiple alignment of nucleic acid and protein sequences (graphical interface) downloads: 96 homepage_uri: ftp://ftp.ebi.ac.uk/pub/software/unix/clustalx/ name: clustalx section: non-free/science version: 2.1+lgpl-2 cmake: description: cross-platform, open-source make system downloads: 1799 homepage_uri: http://cmake.org/ name: cmake section: devel version: 2.8.7-0ubuntu4 cmake-data: description: CMake data files (modules, templates and documentation) downloads: 1153 homepage_uri: http://cmake.org/ name: cmake-data section: devel version: 2.8.7-0ubuntu4 cmap-adobe-japan1: description: CMaps for Adobe-Japan1 downloads: 1870 homepage_uri: http://opensource.adobe.com/wiki/display/cmap/CMap+Resources name: cmap-adobe-japan1 section: misc version: 0+20090930-2 cmap-adobe-japan2: description: CMaps for Adobe-Japan2 downloads: 0 homepage_uri: http://opensource.adobe.com/wiki/display/cmap/CMap+Resources name: cmap-adobe-japan2 section: misc version: 0+20090930-2 colord: description: system service to manage device colour profiles -- system daemon downloads: 12797 homepage_uri: http://www.freedesktop.org/software/colord/ name: colord section: graphics version: 0.1.16-2 comerr-dev: description: common error description library - headers and static libraries downloads: 12043 homepage_uri: http://e2fsprogs.sourceforge.net name: comerr-dev section: libdevel version: 2.1-1.42-1ubuntu2 command-not-found: description: Suggest installation of packages in interactive bash sessions downloads: 640 homepage_uri: '' name: command-not-found section: admin version: 0.2.46ubuntu6 command-not-found-data: description: Set of data files for command-not-found. downloads: 0 homepage_uri: '' name: command-not-found-data section: admin version: 0.2.46ubuntu6 compiz: description: OpenGL window and compositing manager downloads: 1 homepage_uri: '' name: compiz section: x11 version: 1:0.9.7.8-0ubuntu1 compiz-core: description: OpenGL window and compositing manager downloads: 2258 homepage_uri: '' name: compiz-core section: x11 version: 1:0.9.7.8-0ubuntu1 compiz-gnome: description: OpenGL window and compositing manager - GNOME window decorator downloads: 465 homepage_uri: '' name: compiz-gnome section: x11 version: 1:0.9.7.8-0ubuntu1 compiz-plugins-default: description: OpenGL window and compositing manager - default plugins downloads: 1 homepage_uri: '' name: compiz-plugins-default section: x11 version: 1:0.9.7.8-0ubuntu1 compiz-plugins-main-default: description: Compiz plugins - main default collection downloads: 1 homepage_uri: '' name: compiz-plugins-main-default section: x11 version: 1:0.9.7.0~bzr19-0ubuntu10 compizconfig-backend-gconf: description: Compiz Fusion configuration system - gconf backend downloads: 617 homepage_uri: '' name: compizconfig-backend-gconf section: libs version: 0.9.5.92-0ubuntu5 console-setup: description: console font and keymap setup program downloads: 49561 homepage_uri: '' name: console-setup section: utils version: 1.70ubuntu5 consolekit: description: framework for defining and tracking users, sessions and seats downloads: 52291 homepage_uri: http://www.freedesktop.org/wiki/Software/ConsoleKit name: consolekit section: admin version: 0.4.5-2 coreutils: description: GNU core utilities downloads: 110992 homepage_uri: http://gnu.org/software/coreutils name: coreutils section: utils version: 8.13-3ubuntu3 couchdb: description: RESTful document oriented database, system DB downloads: 207 homepage_uri: http://couchdb.apache.org/ name: couchdb section: universe/misc version: 1.0.1-0ubuntu18 couchdb-bin: description: RESTful document oriented database, programs downloads: 8 homepage_uri: http://couchdb.apache.org/ name: couchdb-bin section: universe/misc version: 1.0.1-0ubuntu18 cpio: description: GNU cpio -- a program to manage archives of files downloads: 51114 homepage_uri: http://www.gnu.org/software/cpio/ name: cpio section: utils version: 2.11-7ubuntu3 cpp: description: GNU C preprocessor (cpp) downloads: 48670 homepage_uri: '' name: cpp section: interpreters version: 4:4.6.3-1ubuntu5 cpp-4.6: description: GNU C preprocessor downloads: 13427 homepage_uri: http://gcc.gnu.org/ name: cpp-4.6 section: interpreters version: 4.6.3-1ubuntu5 cpu-checker: description: tools to help evaluate certain CPU (or BIOS) features downloads: 22 homepage_uri: https://launchpad.net/cpu-checker name: cpu-checker section: utils version: 0.7-0ubuntu1 crda: description: wireless Central Regulatory Domain Agent downloads: 9531 homepage_uri: http://wireless.kernel.org/en/developers/Regulatory/#CRDA name: crda section: net version: 1.1.2-1ubuntu1 cron: description: process scheduling daemon downloads: 110008 homepage_uri: http://ftp.isc.org/isc/cron/ name: cron section: admin version: 3.0pl1-120ubuntu3 cryptsetup-bin: description: disk encryption support - command line tools downloads: 2319 homepage_uri: http://code.google.com/p/cryptsetup/ name: cryptsetup-bin section: admin version: 2:1.4.1-2ubuntu4 cups: description: Common UNIX Printing System(tm) - server downloads: 44997 homepage_uri: http://www.cups.org name: cups section: net version: 1.5.3-0ubuntu1 cups-bsd: description: Common UNIX Printing System(tm) - BSD commands downloads: 9338 homepage_uri: http://www.cups.org name: cups-bsd section: net version: 1.5.3-0ubuntu1 cups-client: description: Common UNIX Printing System(tm) - client programs (SysV) downloads: 18378 homepage_uri: http://www.cups.org name: cups-client section: net version: 1.5.3-0ubuntu1 cups-common: description: Common UNIX Printing System(tm) - common files downloads: 17924 homepage_uri: http://www.cups.org name: cups-common section: net version: 1.5.3-0ubuntu1 cups-filters: description: OpenPrinting CUPS Filters downloads: 5104 homepage_uri: http://www.openprinting.org/ name: cups-filters section: net version: 1.0.18-0ubuntu0.1 cups-pk-helper: description: PolicyKit helper to configure cups with fine-grained privileges downloads: 3113 homepage_uri: http://www.freedesktop.org/software/cups-pk-helper/ name: cups-pk-helper section: universe/gnome version: 0.2.1.2-1 cups-ppdc: description: Common UNIX Printing System(tm) - PPD manipulation utilities downloads: 7732 homepage_uri: http://www.cups.org name: cups-ppdc section: net version: 1.5.3-0ubuntu1 curl: description: Get a file from an HTTP, HTTPS or FTP server downloads: 10344 homepage_uri: http://curl.haxx.se name: curl section: web version: 7.22.0-3ubuntu4 cvs: description: Concurrent Versions System downloads: 2449 homepage_uri: http://www.nongnu.org/cvs/ name: cvs section: devel version: 2:1.12.13+real-8 darcs: description: distributed, interactive, smart revision control system downloads: 243 homepage_uri: http://darcs.net/ name: darcs section: universe/devel version: 2.7.99.1-1 dash: description: POSIX-compliant shell downloads: 92934 homepage_uri: http://gondor.apana.org.au/~herbert/dash/ name: dash section: shells version: 0.5.7-2ubuntu2 dbus: description: simple interprocess messaging system (daemon and utilities) downloads: 66602 homepage_uri: http://dbus.freedesktop.org/ name: dbus section: devel version: 1.4.18-1ubuntu1 dbus-x11: description: simple interprocess messaging system (X11 deps) downloads: 48441 homepage_uri: http://dbus.freedesktop.org/ name: dbus-x11 section: x11 version: 1.4.18-1ubuntu1 dc: description: The GNU dc arbitrary precision reverse-polish calculator downloads: 14520 homepage_uri: '' name: dc section: math version: 1.06.95-2 dconf-gsettings-backend: description: simple configuration storage system - GSettings back-end downloads: 14720 homepage_uri: http://live.gnome.org/dconf name: dconf-gsettings-backend section: libs version: 0.12.0-0ubuntu1 dconf-service: description: simple configuration storage system - D-Bus service downloads: 13671 homepage_uri: http://live.gnome.org/dconf name: dconf-service section: libs version: 0.12.0-0ubuntu1 dctrl-tools: description: Command-line tools to process Debian package information downloads: 4785 homepage_uri: '' name: dctrl-tools section: utils version: 2.18ubuntu1 debconf: description: Debian configuration management system downloads: 111132 homepage_uri: '' name: debconf section: admin version: 1.5.42ubuntu1 debconf-i18n: description: full internationalization support for debconf downloads: 864 homepage_uri: '' name: debconf-i18n section: admin version: 1.5.42ubuntu1 debhelper: description: helper programs for debian/rules downloads: 5177 homepage_uri: http://kitenet.net/~joey/code/debhelper/ name: debhelper section: devel version: 9.20120115ubuntu3 debianutils: description: Miscellaneous utilities specific to Debian downloads: 116783 homepage_uri: '' name: debianutils section: utils version: 4.2.1ubuntu2 debootstrap: description: Bootstrap a basic Debian system downloads: 1603 homepage_uri: '' name: debootstrap section: admin version: 1.0.39ubuntu0.1 default-jdk: description: Standard Java or Java compatible Development Kit downloads: 45 homepage_uri: '' name: default-jdk section: devel version: 1:1.6-43ubuntu2 default-jre: description: Standard Java or Java compatible Runtime downloads: 12183 homepage_uri: '' name: default-jre section: interpreters version: 1:1.6-43ubuntu2 default-jre-headless: description: Standard Java or Java compatible Runtime (headless) downloads: 13149 homepage_uri: '' name: default-jre-headless section: interpreters version: 1:1.6-43ubuntu2 deja-dup: description: Back up your files downloads: 345 homepage_uri: https://launchpad.net/deja-dup name: deja-dup section: utils version: 22.0-0ubuntu2 desktop-file-utils: description: Utilities for .desktop files downloads: 37457 homepage_uri: '' name: desktop-file-utils section: devel version: 0.20-0ubuntu2 devscripts: description: scripts to make the life of a Debian Package maintainer easier downloads: 1921 homepage_uri: '' name: devscripts section: devel version: 2.11.6ubuntu1.2 dh-apparmor: description: AppArmor debhelper routines downloads: 1 homepage_uri: http://apparmor.net/ name: dh-apparmor section: devel version: 2.7.102-0ubuntu3.1 dh-make: description: tool that converts source archives into Debian package source downloads: 520 homepage_uri: '' name: dh-make section: devel version: 0.59ubuntu1 dh-translations: description: debhelper extension for translation support downloads: 1 homepage_uri: '' name: dh-translations section: devel version: '116' dialign: description: Segment-based multiple sequence alignment downloads: 31 homepage_uri: http://dialign.gobics.de/ name: dialign section: science version: 2.2.1-5 dict: description: dictionary client downloads: 892 homepage_uri: http://sourceforge.net/projects/dict/ name: dict section: text version: 1.12.0+dfsg-4build1 dictionaries-common: description: Common utilities for spelling dictionary tools downloads: 18912 homepage_uri: http://dict-common.alioth.debian.org name: dictionaries-common section: text version: 1.12.1ubuntu2 diffstat: description: produces graph of changes introduced by a diff file downloads: 2366 homepage_uri: http://invisible-island.net/diffstat/ name: diffstat section: devel version: 1.54-1 diffutils: description: File comparison utilities downloads: 86627 homepage_uri: http://www.gnu.org/software/diffutils/ name: diffutils section: utils version: 1:3.2-1ubuntu1 dkms: description: Dynamic Kernel Module Support Framework downloads: 9246 homepage_uri: http://linux.dell.com/dkms name: dkms section: admin version: 2.2.0.3-1ubuntu3 dmidecode: description: SMBIOS/DMI table decoder downloads: 14886 homepage_uri: http://dmidecode.nongnu.org/ name: dmidecode section: utils version: 2.11-4 dmsetup: description: The Linux Kernel Device Mapper userspace library downloads: 63207 homepage_uri: http://sources.redhat.com/lvm2/ name: dmsetup section: admin version: 2:1.02.48-4ubuntu7.1 dmz-cursor-theme: description: Style neutral, scalable cursor theme downloads: 0 homepage_uri: '' name: dmz-cursor-theme section: gnome version: 0.4.3 dnsmasq-base: description: Small caching DNS proxy and DHCP/TFTP server downloads: 3857 homepage_uri: '' name: dnsmasq-base section: net version: 2.59-4 dnsutils: description: Clients provided with BIND downloads: 14967 homepage_uri: '' name: dnsutils section: net version: 1:9.8.1.dfsg.P1-4ubuntu0.1 doc-base: description: utilities to manage online documentation downloads: 7963 homepage_uri: '' name: doc-base section: doc version: 0.10.3 docbook-xml: description: standard XML documentation system for software and systems downloads: 679 homepage_uri: http://docbook.org/schemas/4x.html name: docbook-xml section: text version: 4.5-7ubuntu1 dosfstools: description: utilities for making and checking MS-DOS FAT filesystems downloads: 5523 homepage_uri: http://www.daniel-baumann.ch/software/dosfstools/ name: dosfstools section: otherosfs version: 3.0.12-1ubuntu1 dpatch: description: patch maintenance system for Debian source packages downloads: 869 homepage_uri: '' name: dpatch section: devel version: 2.0.34ubuntu1 dpkg: description: Debian package management system downloads: 116788 homepage_uri: http://wiki.debian.org/Teams/Dpkg name: dpkg section: admin version: 1.16.1.2ubuntu7 dpkg-dev: description: Debian package development tools downloads: 7479 homepage_uri: http://wiki.debian.org/Teams/Dpkg name: dpkg-dev section: utils version: 1.16.1.2ubuntu7 dput: description: Debian package upload tool downloads: 1149 homepage_uri: '' name: dput section: devel version: 0.9.6.2ubuntu1 duplicity: description: encrypted bandwidth-efficient backup downloads: 907 homepage_uri: http://duplicity.nongnu.org/ name: duplicity section: utils version: 0.6.18-0ubuntu3 dvd+rw-tools: description: DVD+-RW/R tools downloads: 12960 homepage_uri: http://fy.chalmers.se/~appro/linux/DVD+RW/ name: dvd+rw-tools section: utils version: 7.1-10 e2fslibs: description: ext2/ext3/ext4 file system libraries downloads: 19465 homepage_uri: http://e2fsprogs.sourceforge.net name: e2fslibs section: libs version: 1.42-1ubuntu2 e2fsprogs: description: ext2/ext3/ext4 file system utilities downloads: 73393 homepage_uri: http://e2fsprogs.sourceforge.net name: e2fsprogs section: admin version: 1.42-1ubuntu2 ebtables: description: Ethernet bridge frame table administration downloads: 3094 homepage_uri: http://ebtables.sourceforge.net name: ebtables section: net version: 2.0.9.2-2ubuntu2 ec2-ami-tools: description: Amazon EC2 AMI tools downloads: 2 homepage_uri: http://aws.amazon.com/developertools/368 name: ec2-ami-tools section: multiverse/admin version: 1.4.0.2-0ubuntu1 ec2-api-tools: description: Amazon EC2 API tools downloads: 12 homepage_uri: http://aws.amazon.com/developertools/351 name: ec2-api-tools section: multiverse/admin version: 1.5.0.0-0ubuntu1 eclipse: description: Extensible Tool Platform and Java IDE downloads: 15 homepage_uri: http://www.eclipse.org/ name: eclipse section: universe/devel version: 3.7.2-1 eclipse-jdt: description: Eclipse Java Development Tools (JDT) downloads: 967 homepage_uri: http://www.eclipse.org/ name: eclipse-jdt section: universe/devel version: 3.7.2-1 eclipse-pde: description: Eclipse Plug-in Development Environment (PDE) downloads: 985 homepage_uri: http://www.eclipse.org/ name: eclipse-pde section: universe/devel version: 3.7.2-1 eclipse-platform: description: Eclipse platform without development plug-ins downloads: 1691 homepage_uri: http://www.eclipse.org/ name: eclipse-platform section: universe/devel version: 3.7.2-1 eclipse-platform-data: description: Eclipse platform without development plug-ins (data) downloads: 1681 homepage_uri: http://www.eclipse.org/ name: eclipse-platform-data section: universe/java version: 3.7.2-1 eclipse-rcp: description: Eclipse Rich Client Platform (RCP) downloads: 1651 homepage_uri: http://www.eclipse.org/ name: eclipse-rcp section: universe/devel version: 3.7.2-1 ed: description: classic UNIX line editor downloads: 8485 homepage_uri: http://www.gnu.org/software/ed/ name: ed section: editors version: 1.5-3 eject: description: ejects CDs and operates CD-Changers under Linux downloads: 19162 homepage_uri: http://www.pobox.com/~tranter/eject.html name: eject section: utils version: 2.1.5+deb1+cvs20081104-9 emacs: description: The GNU Emacs editor (metapackage) downloads: 205 homepage_uri: http://www.gnu.org/software/emacs/ name: emacs section: editors version: 23.3+1-1ubuntu9 emacs-goodies-el: description: Miscellaneous add-ons for Emacs downloads: 319 homepage_uri: '' name: emacs-goodies-el section: editors version: 35.2ubuntu1.1 emacs23: description: The GNU Emacs editor (with GTK+ user interface) downloads: 6479 homepage_uri: http://www.gnu.org/software/emacs/ name: emacs23 section: editors version: 23.3+1-1ubuntu9 emacs23-bin-common: description: The GNU Emacs editor's shared, architecture dependent files downloads: 7524 homepage_uri: http://www.gnu.org/software/emacs/ name: emacs23-bin-common section: editors version: 23.3+1-1ubuntu9 emacs23-common: description: The GNU Emacs editor's shared, architecture independent infrastructure downloads: 2045 homepage_uri: http://www.gnu.org/software/emacs/ name: emacs23-common section: editors version: 23.3+1-1ubuntu9 emacsen-common: description: Common facilities for all emacsen downloads: 6504 homepage_uri: '' name: emacsen-common section: editors version: 1.4.22ubuntu1 emboss: description: the european molecular biology open software suite downloads: 45 homepage_uri: http://emboss.sourceforge.net name: emboss section: science version: 6.3.1-6ubuntu3 emboss-data: description: data files for the EMBOSS package downloads: 1 homepage_uri: http://emboss.sourceforge.net name: emboss-data section: science version: 6.3.1-6ubuntu3 emboss-lib: description: EMBOSS Libraries downloads: 46 homepage_uri: http://emboss.sourceforge.net name: emboss-lib section: libs version: 6.3.1-6ubuntu3 empathy: description: GNOME multi-protocol chat and call client downloads: 4159 homepage_uri: http://live.gnome.org/Empathy name: empathy section: gnome version: 3.4.2-0ubuntu1 empathy-common: description: GNOME multi-protocol chat and call client (common files) downloads: 0 homepage_uri: http://live.gnome.org/Empathy name: empathy-common section: gnome version: 3.4.2-0ubuntu1 enchant: description: Wrapper for various spell checker engines (binary programs) downloads: 1688 homepage_uri: http://www.abisource.com/projects/enchant/ name: enchant section: text version: 1.6.0-7 eog: description: Eye of GNOME graphics viewer program downloads: 17636 homepage_uri: http://projects.gnome.org/eog name: eog section: gnome version: 3.4.1-0ubuntu1 erlang: description: Concurrent, real-time, distributed functional language downloads: 0 homepage_uri: http://www.erlang.org/ name: erlang section: universe/interpreters version: 1:14.b.4-dfsg-1ubuntu1 erlang-appmon: description: Erlang/OTP application monitor downloads: 36 homepage_uri: http://www.erlang.org/ name: erlang-appmon section: universe/interpreters version: 1:14.b.4-dfsg-1ubuntu1 erlang-asn1: description: Erlang/OTP modules for ASN.1 support downloads: 494 homepage_uri: http://www.erlang.org/ name: erlang-asn1 section: interpreters version: 1:14.b.4-dfsg-1ubuntu1 erlang-base: description: Erlang/OTP virtual machine and base applications downloads: 1098 homepage_uri: http://www.erlang.org/ name: erlang-base section: interpreters version: 1:14.b.4-dfsg-1ubuntu1 erlang-common-test: description: Erlang/OTP application for automated testing downloads: 71 homepage_uri: http://www.erlang.org/ name: erlang-common-test section: universe/interpreters version: 1:14.b.4-dfsg-1ubuntu1 erlang-corba: description: Erlang/OTP applications for CORBA support downloads: 95 homepage_uri: http://www.erlang.org/ name: erlang-corba section: interpreters version: 1:14.b.4-dfsg-1ubuntu1 erlang-crypto: description: Erlang/OTP cryptographic modules downloads: 714 homepage_uri: http://www.erlang.org/ name: erlang-crypto section: interpreters version: 1:14.b.4-dfsg-1ubuntu1 erlang-debugger: description: Erlang/OTP application for debugging and testing downloads: 31 homepage_uri: http://www.erlang.org/ name: erlang-debugger section: universe/interpreters version: 1:14.b.4-dfsg-1ubuntu1 erlang-dev: description: Erlang/OTP development libraries and headers downloads: 136 homepage_uri: http://www.erlang.org/ name: erlang-dev section: interpreters version: 1:14.b.4-dfsg-1ubuntu1 erlang-dialyzer: description: Erlang/OTP discrepancy analyzer application downloads: 72 homepage_uri: http://www.erlang.org/ name: erlang-dialyzer section: universe/interpreters version: 1:14.b.4-dfsg-1ubuntu1 erlang-diameter: description: Erlang/OTP implementation of RFC 3588 protocol downloads: 61 homepage_uri: http://www.erlang.org/ name: erlang-diameter section: interpreters version: 1:14.b.4-dfsg-1ubuntu1 erlang-docbuilder: description: Erlang/OTP application for building HTML documentation downloads: 32 homepage_uri: http://www.erlang.org/ name: erlang-docbuilder section: interpreters version: 1:14.b.4-dfsg-1ubuntu1 erlang-edoc: description: Erlang/OTP module for generating documentation downloads: 103 homepage_uri: http://www.erlang.org/ name: erlang-edoc section: interpreters version: 1:14.b.4-dfsg-1ubuntu1 erlang-erl-docgen: description: Erlang/OTP documentation stylesheets downloads: 95 homepage_uri: http://www.erlang.org/ name: erlang-erl-docgen section: interpreters version: 1:14.b.4-dfsg-1ubuntu1 erlang-et: description: Erlang/OTP event tracer application downloads: 32 homepage_uri: http://www.erlang.org/ name: erlang-et section: interpreters version: 1:14.b.4-dfsg-1ubuntu1 erlang-eunit: description: Erlang/OTP module for unit testing downloads: 120 homepage_uri: http://www.erlang.org/ name: erlang-eunit section: interpreters version: 1:14.b.4-dfsg-1ubuntu1 erlang-examples: description: Erlang/OTP application examples downloads: 27 homepage_uri: http://www.erlang.org/ name: erlang-examples section: interpreters version: 1:14.b.4-dfsg-1ubuntu1 erlang-gs: description: Erlang/OTP graphics system downloads: 41 homepage_uri: http://www.erlang.org/ name: erlang-gs section: interpreters version: 1:14.b.4-dfsg-1ubuntu1 erlang-ic: description: Erlang/OTP IDL compiler downloads: 94 homepage_uri: http://www.erlang.org/ name: erlang-ic section: interpreters version: 1:14.b.4-dfsg-1ubuntu1 erlang-ic-java: description: Erlang/OTP IDL compiler (Java classes) downloads: 22 homepage_uri: http://www.erlang.org/ name: erlang-ic-java section: universe/interpreters version: 1:14.b.4-dfsg-1ubuntu1 erlang-inets: description: Erlang/OTP Internet clients and servers downloads: 426 homepage_uri: http://www.erlang.org/ name: erlang-inets section: interpreters version: 1:14.b.4-dfsg-1ubuntu1 erlang-inviso: description: Erlang/OTP trace tool downloads: 92 homepage_uri: http://www.erlang.org/ name: erlang-inviso section: interpreters version: 1:14.b.4-dfsg-1ubuntu1 erlang-jinterface: description: Java communication tool to Erlang downloads: 25 homepage_uri: http://www.erlang.org/ name: erlang-jinterface section: universe/interpreters version: 1:14.b.4-dfsg-1ubuntu1 erlang-manpages: description: Erlang/OTP manual pages downloads: 0 homepage_uri: http://www.erlang.org/ name: erlang-manpages section: universe/interpreters version: 1:14.b.4-dfsg-1ubuntu1 erlang-megaco: description: Erlang/OTP implementation of Megaco/H.248 protocol downloads: 32 homepage_uri: http://www.erlang.org/ name: erlang-megaco section: interpreters version: 1:14.b.4-dfsg-1ubuntu1 erlang-mnesia: description: Erlang/OTP distributed relational/object hybrid database downloads: 530 homepage_uri: http://www.erlang.org/ name: erlang-mnesia section: interpreters version: 1:14.b.4-dfsg-1ubuntu1 erlang-mode: description: Erlang major editing mode for Emacs downloads: 60 homepage_uri: http://www.erlang.org/ name: erlang-mode section: universe/interpreters version: 1:14.b.4-dfsg-1ubuntu1 erlang-nox: description: Erlang/OTP applications that don't require X Window System downloads: 213 homepage_uri: http://www.erlang.org/ name: erlang-nox section: interpreters version: 1:14.b.4-dfsg-1ubuntu1 erlang-observer: description: Erlang/OTP application for investigating distributed systems downloads: 34 homepage_uri: http://www.erlang.org/ name: erlang-observer section: universe/interpreters version: 1:14.b.4-dfsg-1ubuntu1 erlang-odbc: description: Erlang/OTP interface to SQL databases downloads: 131 homepage_uri: http://www.erlang.org/ name: erlang-odbc section: interpreters version: 1:14.b.4-dfsg-1ubuntu1 erlang-os-mon: description: Erlang/OTP operating system monitor downloads: 246 homepage_uri: http://www.erlang.org/ name: erlang-os-mon section: interpreters version: 1:14.b.4-dfsg-1ubuntu1 erlang-parsetools: description: Erlang/OTP parsing tools downloads: 94 homepage_uri: http://www.erlang.org/ name: erlang-parsetools section: interpreters version: 1:14.b.4-dfsg-1ubuntu1 erlang-percept: description: Erlang/OTP concurrency profiling tool downloads: 92 homepage_uri: http://www.erlang.org/ name: erlang-percept section: interpreters version: 1:14.b.4-dfsg-1ubuntu1 erlang-pman: description: Erlang/OTP process manager downloads: 32 homepage_uri: http://www.erlang.org/ name: erlang-pman section: universe/interpreters version: 1:14.b.4-dfsg-1ubuntu1 erlang-public-key: description: Erlang/OTP public key infrastructure downloads: 528 homepage_uri: http://www.erlang.org/ name: erlang-public-key section: interpreters version: 1:14.b.4-dfsg-1ubuntu1 erlang-reltool: description: Erlang/OTP release management tool downloads: 31 homepage_uri: http://www.erlang.org/ name: erlang-reltool section: universe/interpreters version: 1:14.b.4-dfsg-1ubuntu1 erlang-runtime-tools: description: Erlang/OTP runtime tracing/debugging tools downloads: 176 homepage_uri: http://www.erlang.org/ name: erlang-runtime-tools section: interpreters version: 1:14.b.4-dfsg-1ubuntu1 erlang-snmp: description: Erlang/OTP SNMP applications downloads: 221 homepage_uri: http://www.erlang.org/ name: erlang-snmp section: interpreters version: 1:14.b.4-dfsg-1ubuntu1 erlang-src: description: Erlang/OTP applications sources downloads: 36 homepage_uri: http://www.erlang.org/ name: erlang-src section: universe/interpreters version: 1:14.b.4-dfsg-1ubuntu1 erlang-ssh: description: Erlang/OTP implementation of SSH protocol downloads: 93 homepage_uri: http://www.erlang.org/ name: erlang-ssh section: interpreters version: 1:14.b.4-dfsg-1ubuntu1 erlang-ssl: description: Erlang/OTP implementation of SSL downloads: 462 homepage_uri: http://www.erlang.org/ name: erlang-ssl section: interpreters version: 1:14.b.4-dfsg-1ubuntu1 erlang-syntax-tools: description: Erlang/OTP modules for handling abstract Erlang syntax trees downloads: 204 homepage_uri: http://www.erlang.org/ name: erlang-syntax-tools section: interpreters version: 1:14.b.4-dfsg-1ubuntu1 erlang-test-server: description: Erlang/OTP server for automated application testing downloads: 32 homepage_uri: http://www.erlang.org/ name: erlang-test-server section: universe/interpreters version: 1:14.b.4-dfsg-1ubuntu1 erlang-toolbar: description: Erlang/OTP graphical toolbar downloads: 31 homepage_uri: http://www.erlang.org/ name: erlang-toolbar section: universe/interpreters version: 1:14.b.4-dfsg-1ubuntu1 erlang-tools: description: Erlang/OTP various tools downloads: 232 homepage_uri: http://www.erlang.org/ name: erlang-tools section: interpreters version: 1:14.b.4-dfsg-1ubuntu1 erlang-tv: description: Erlang/OTP table viewer downloads: 32 homepage_uri: http://www.erlang.org/ name: erlang-tv section: universe/interpreters version: 1:14.b.4-dfsg-1ubuntu1 erlang-typer: description: Erlang/OTP code type annotator downloads: 66 homepage_uri: http://www.erlang.org/ name: erlang-typer section: universe/interpreters version: 1:14.b.4-dfsg-1ubuntu1 erlang-webtool: description: Erlang/OTP helper for web-based tools downloads: 161 homepage_uri: http://www.erlang.org/ name: erlang-webtool section: interpreters version: 1:14.b.4-dfsg-1ubuntu1 erlang-xmerl: description: Erlang/OTP XML tools downloads: 236 homepage_uri: http://www.erlang.org/ name: erlang-xmerl section: interpreters version: 1:14.b.4-dfsg-1ubuntu1 esound-common: description: Enlightened Sound Daemon - Common files downloads: 8826 homepage_uri: '' name: esound-common section: sound version: 0.2.41-10build3 espeak: description: Multi-lingual software speech synthesizer downloads: 2513 homepage_uri: http://espeak.sourceforge.net/ name: espeak section: sound version: 1.46.02-0ubuntu1 espeak-data: description: 'Multi-lingual software speech synthesizer: speech data files' downloads: 46 homepage_uri: http://espeak.sourceforge.net/ name: espeak-data section: sound version: 1.46.02-0ubuntu1 euca2ools: description: managing cloud instances for Eucalyptus downloads: 17 homepage_uri: http://open.eucalyptus.com/open/ name: euca2ools section: utils version: 2.0.0~bzr516-0ubuntu3.1 evince: description: Document (PostScript, PDF) viewer downloads: 33662 homepage_uri: http://www.gnome.org/projects/evince/ name: evince section: gnome version: 3.4.0-0ubuntu1 evince-common: description: Document (PostScript, PDF) viewer - common files downloads: 0 homepage_uri: http://www.gnome.org/projects/evince/ name: evince-common section: gnome version: 3.4.0-0ubuntu1 evolution-data-server: description: evolution database backend server downloads: 21994 homepage_uri: http://www.gnome.org/projects/evolution/ name: evolution-data-server section: gnome version: 3.2.3-0ubuntu7 evolution-data-server-common: description: architecture independent files for Evolution Data Server downloads: 72 homepage_uri: http://www.gnome.org/projects/evolution/ name: evolution-data-server-common section: gnome version: 3.2.3-0ubuntu7 example-content: description: Ubuntu example content downloads: 0 homepage_uri: https://launchpad.net/example-content name: example-content section: x11 version: '46' exonerate: description: generic tool for pairwise sequence comparison downloads: 27 homepage_uri: http://www.ebi.ac.uk/~guy/exonerate/ name: exonerate section: universe/science version: 2.2.0-6 expect: description: Automates interactive applications downloads: 4238 homepage_uri: http://sourceforge.net/projects/expect/ name: expect section: interpreters version: 5.45-2 exuberant-ctags: description: build tag file indexes of source code definitions downloads: 1238 homepage_uri: http://ctags.sourceforge.net/ name: exuberant-ctags section: editors version: 1:5.9~svn20110310-3 fakeroot: description: tool for simulating superuser privileges downloads: 6533 homepage_uri: '' name: fakeroot section: utils version: 1.18.2-1 fastdnaml: description: Tool for construction of phylogenetic trees of DNA sequences downloads: 20 homepage_uri: http://web.archive.org/web/20061017161001/http://geta.life.uiuc.edu/~gary/programs/fastDNAml.html name: fastdnaml section: science version: 1.2.2-10 fastjar: description: Jar creation utility downloads: 1799 homepage_uri: '' name: fastjar section: misc version: 2:0.98-3 fastx-toolkit: description: FASTQ/A short nucleotide reads pre-processing tools downloads: 9 homepage_uri: http://hannonlab.cshl.edu/fastx_toolkit/ name: fastx-toolkit section: universe/science version: 0.0.13.1-1 fdupes: description: identifies duplicate files within given directories downloads: 309 homepage_uri: http://code.google.com/p/fdupes/ name: fdupes section: utils version: 1.50-PR2-3 file: description: Determines file type using "magic" numbers downloads: 34695 homepage_uri: http://www.darwinsys.com/file/ name: file section: utils version: 5.09-2 file-roller: description: archive manager for GNOME downloads: 29747 homepage_uri: http://fileroller.sourceforge.net/ name: file-roller section: gnome version: 3.4.1-0ubuntu1 findutils: description: utilities for finding files--find, xargs downloads: 110950 homepage_uri: http://savannah.gnu.org/projects/findutils/ name: findutils section: utils version: 4.4.2-4ubuntu1 firefox: description: Safe and easy web browser from Mozilla downloads: 393 homepage_uri: '' name: firefox section: web version: 13.0+build1-0ubuntu0.12.04.1 firefox-globalmenu: description: Unity appmenu integration for Firefox downloads: 4 homepage_uri: '' name: firefox-globalmenu section: web version: 13.0+build1-0ubuntu0.12.04.1 firefox-gnome-support: description: Safe and easy web browser from Mozilla - GNOME support downloads: 1 homepage_uri: '' name: firefox-gnome-support section: gnome version: 13.0.1+build1-0ubuntu0.12.04.1 folks-common: description: library to aggregates people into metacontacts (common files) downloads: 0 homepage_uri: http://telepathy.freedesktop.org/wiki/Folks name: folks-common section: libs version: 0.6.8-2 fontconfig: description: generic font configuration library - support binaries downloads: 55585 homepage_uri: '' name: fontconfig section: utils version: 2.8.0-3ubuntu9 fontconfig-config: description: generic font configuration library - configuration downloads: 54331 homepage_uri: '' name: fontconfig-config section: libs version: 2.8.0-3ubuntu9 fonts-kacst: description: KACST free TrueType Arabic fonts downloads: 0 homepage_uri: '' name: fonts-kacst section: fonts version: 2.01+mry-3 fonts-kacst-one: description: TrueType font designed for Arabic language downloads: 0 homepage_uri: http://projects.arabeyes.org name: fonts-kacst-one section: fonts version: 5.0+svn11846-2 fonts-khmeros-core: description: KhmerOS Unicode fonts for the Khmer language of Cambodia downloads: 0 homepage_uri: http://www.khmeros.info name: fonts-khmeros-core section: fonts version: 5.0-5ubuntu1 fonts-lao: description: TrueType font for Lao language downloads: 0 homepage_uri: '' name: fonts-lao section: fonts version: 0.0.20060226-8 fonts-liberation: description: Fonts with the same metrics as Times, Arial and Courier downloads: 708 homepage_uri: https://fedorahosted.org/liberation-fonts/ name: fonts-liberation section: fonts version: 1.07.0-2 fonts-nanum: description: Nanum Korean fonts downloads: 0 homepage_uri: http://hangeul.naver.com/nanum.nhn name: fonts-nanum section: fonts version: 3.010-2 fonts-opensymbol: description: OpenSymbol TrueType font downloads: 0 homepage_uri: http://www.libreoffice.org name: fonts-opensymbol section: fonts version: 2:102.2+LibO3.5.3-0ubuntu1 fonts-takao-pgothic: description: Japanese TrueType font set, Takao P Gothic Fonts downloads: 0 homepage_uri: https://launchpad.net/takao-fonts name: fonts-takao-pgothic section: fonts version: 003.02.01-5ubuntu1 fonts-thai-tlwg: description: Thai fonts maintained by TLWG (meta package) downloads: 52 homepage_uri: http://linux.thai.net/projects/thaifonts-scalable name: fonts-thai-tlwg section: fonts version: 1:0.4.17-1ubuntu1 fonts-tlwg-garuda: description: Thai Garuda font downloads: 0 homepage_uri: http://linux.thai.net/projects/thaifonts-scalable name: fonts-tlwg-garuda section: fonts version: 1:0.4.17-1ubuntu1 fonts-tlwg-kinnari: description: Thai Kinnari font downloads: 0 homepage_uri: http://linux.thai.net/projects/thaifonts-scalable name: fonts-tlwg-kinnari section: fonts version: 1:0.4.17-1ubuntu1 fonts-tlwg-loma: description: Thai Loma font downloads: 0 homepage_uri: http://linux.thai.net/projects/thaifonts-scalable name: fonts-tlwg-loma section: fonts version: 1:0.4.17-1ubuntu1 fonts-tlwg-mono: description: Thai TlwgMono font downloads: 0 homepage_uri: http://linux.thai.net/projects/thaifonts-scalable name: fonts-tlwg-mono section: fonts version: 1:0.4.17-1ubuntu1 fonts-tlwg-norasi: description: Thai Norasi font downloads: 0 homepage_uri: http://linux.thai.net/projects/thaifonts-scalable name: fonts-tlwg-norasi section: fonts version: 1:0.4.17-1ubuntu1 fonts-tlwg-purisa: description: Thai Purisa font downloads: 0 homepage_uri: http://linux.thai.net/projects/thaifonts-scalable name: fonts-tlwg-purisa section: fonts version: 1:0.4.17-1ubuntu1 fonts-tlwg-sawasdee: description: Thai Sawasdee font downloads: 0 homepage_uri: http://linux.thai.net/projects/thaifonts-scalable name: fonts-tlwg-sawasdee section: fonts version: 1:0.4.17-1ubuntu1 fonts-tlwg-typewriter: description: Thai TlwgTypewriter font downloads: 0 homepage_uri: http://linux.thai.net/projects/thaifonts-scalable name: fonts-tlwg-typewriter section: fonts version: 1:0.4.17-1ubuntu1 fonts-tlwg-typist: description: Thai TlwgTypist font downloads: 0 homepage_uri: http://linux.thai.net/projects/thaifonts-scalable name: fonts-tlwg-typist section: fonts version: 1:0.4.17-1ubuntu1 fonts-tlwg-typo: description: Thai TlwgTypo font downloads: 0 homepage_uri: http://linux.thai.net/projects/thaifonts-scalable name: fonts-tlwg-typo section: fonts version: 1:0.4.17-1ubuntu1 fonts-tlwg-umpush: description: Thai Umpush font downloads: 0 homepage_uri: http://linux.thai.net/projects/thaifonts-scalable name: fonts-tlwg-umpush section: fonts version: 1:0.4.17-1ubuntu1 fonts-tlwg-waree: description: Thai Waree font downloads: 0 homepage_uri: http://linux.thai.net/projects/thaifonts-scalable name: fonts-tlwg-waree section: fonts version: 1:0.4.17-1ubuntu1 fonts-ubuntu-font-family-console: description: Ubuntu Font Family Linux console fonts, sans-serif monospace downloads: 0 homepage_uri: http://font.ubuntu.com/ name: fonts-ubuntu-font-family-console section: fonts version: 0.80-0ubuntu2 foomatic-db-compressed-ppds: description: OpenPrinting printer support - Compressed PPDs derived from the database downloads: 4921 homepage_uri: http://www.openprinting.org/ name: foomatic-db-compressed-ppds section: text version: 20120322-0ubuntu1 foomatic-db-engine: description: OpenPrinting printer support - programs downloads: 11594 homepage_uri: http://www.openprinting.org/ name: foomatic-db-engine section: text version: 4.0.8-2ubuntu1 foomatic-filters: description: OpenPrinting printer support - filters downloads: 16910 homepage_uri: http://www.openprinting.org/ name: foomatic-filters section: text version: 4.0.15-0ubuntu1 fop: description: XML formatter driven by XSL Formatting Objects (XSL-FO.) downloads: 805 homepage_uri: http://xmlgraphics.apache.org/fop/ name: fop section: text version: 1:1.0.dfsg2-6 freenx-server: description: Remote desktop/application/thin-client server core features downloads: 113 homepage_uri: '' name: freenx-server section: x11 version: 0.7.3.zgit.120322.977c28d-0~ppa11 friendly-recovery: description: Make recovery more user-friendly downloads: 1 homepage_uri: '' name: friendly-recovery section: admin version: 0.2.25 ftp: description: classical file transfer client downloads: 9907 homepage_uri: '' name: ftp section: net version: 0.17-25 fuse: description: Filesystem in Userspace downloads: 10674 homepage_uri: http://fuse.sourceforge.net/ name: fuse section: utils version: 2.8.6-2ubuntu2 fuse-utils: description: Filesystem in Userspace (transitional package) downloads: 9727 homepage_uri: http://fuse.sourceforge.net/ name: fuse-utils section: utils version: 2.8.6-2ubuntu2 g++: description: GNU C++ compiler downloads: 10176 homepage_uri: '' name: g++ section: devel version: 4:4.6.3-1ubuntu5 g++-4.6: description: GNU C++ compiler downloads: 5740 homepage_uri: http://gcc.gnu.org/ name: g++-4.6 section: devel version: 4.6.3-1ubuntu5 gawk: description: GNU awk, a pattern scanning and processing language downloads: 35792 homepage_uri: http://www.gnu.org/software/gawk/ name: gawk section: interpreters version: 1:3.1.8+dfsg-0.1ubuntu1 gcalctool: description: GNOME desktop calculator downloads: 11016 homepage_uri: http://live.gnome.org/Gcalctool name: gcalctool section: math version: 6.4.1.1-0ubuntu2 gcc: description: GNU C compiler downloads: 23226 homepage_uri: '' name: gcc section: devel version: 4:4.6.3-1ubuntu5 gcc-4.6: description: GNU C compiler downloads: 10350 homepage_uri: http://gcc.gnu.org/ name: gcc-4.6 section: devel version: 4.6.3-1ubuntu5 gcc-4.6-base: description: GCC, the GNU Compiler Collection (base package) downloads: 13433 homepage_uri: http://gcc.gnu.org/ name: gcc-4.6-base section: libs version: 4.6.3-1ubuntu5 gcc-4.6-base:i386: description: GCC, the GNU Compiler Collection (base package) downloads: 0 homepage_uri: http://gcc.gnu.org/ name: gcc-4.6-base:i386 section: libs version: 4.6.3-1ubuntu5 gccxml: description: XML output extension to GCC downloads: 341 homepage_uri: http://www.gccxml.org/ name: gccxml section: devel version: 0.9.0+cvs20111013-1 gcj-4.6-base: description: GCC, the GNU Compiler Collection (gcj base package) downloads: 3645 homepage_uri: http://gcc.gnu.org/ name: gcj-4.6-base section: libs version: 4.6.3-1ubuntu2 gcj-4.6-jre-lib: description: Java runtime library for use with gcj (jar files) downloads: 205 homepage_uri: http://gcc.gnu.org/ name: gcj-4.6-jre-lib section: java version: 4.6.3-1ubuntu2 gconf-service: description: GNOME configuration database system (D-Bus service) downloads: 15018 homepage_uri: http://projects.gnome.org/gconf/ name: gconf-service section: libs version: 3.2.5-0ubuntu2 gconf-service-backend: description: GNOME configuration database system (D-Bus service) downloads: 1 homepage_uri: http://projects.gnome.org/gconf/ name: gconf-service-backend section: libs version: 3.2.5-0ubuntu2 gconf2: description: GNOME configuration database system (support tools) downloads: 45482 homepage_uri: http://projects.gnome.org/gconf/ name: gconf2 section: libs version: 3.2.5-0ubuntu2 gconf2-common: description: GNOME configuration database system (common files) downloads: 15781 homepage_uri: http://projects.gnome.org/gconf/ name: gconf2-common section: libs version: 3.2.5-0ubuntu2 gdb: description: The GNU Debugger downloads: 5081 homepage_uri: '' name: gdb section: devel version: 7.4-2012.04-0ubuntu2 gdm: description: GNOME Display Manager downloads: 7248 homepage_uri: '' name: gdm section: universe/gnome version: 3.0.4-0ubuntu15 gedit: description: official text editor of the GNOME desktop environment downloads: 19078 homepage_uri: http://www.gnome.org/projects/gedit/ name: gedit section: gnome version: 3.4.1-0ubuntu1 gedit-common: description: official text editor of the GNOME desktop environment (support files) downloads: 0 homepage_uri: http://www.gnome.org/projects/gedit/ name: gedit-common section: gnome version: 3.4.1-0ubuntu1 genisoimage: description: Creates ISO-9660 CD-ROM filesystem images downloads: 15424 homepage_uri: '' name: genisoimage section: otherosfs version: 9:1.1.11-2ubuntu2 geoclue: description: Geographic information framework downloads: 944 homepage_uri: http://www.freedesktop.org/wiki/Software/GeoClue name: geoclue section: utils version: 0.12.0-1ubuntu12 geoclue-ubuntu-geoip: description: Provide positioning for GeoClue via Ubuntu GeoIP services downloads: 0 homepage_uri: https://launchpad.net/ubuntu-geoip name: geoclue-ubuntu-geoip section: gnome version: 0.0.2-0ubuntu6 geoip-database: description: IP lookup command line tools that use the GeoIP library (country database) downloads: 908 homepage_uri: http://www.maxmind.com/ name: geoip-database section: net version: 20111220-1 gettext: description: GNU Internationalization utilities downloads: 8557 homepage_uri: http://www.gnu.org/software/gettext/ name: gettext section: devel version: 0.18.1.1-5ubuntu3 gettext-base: description: GNU Internationalization utilities for the base system downloads: 26115 homepage_uri: http://www.gnu.org/software/gettext/ name: gettext-base section: utils version: 0.18.1.1-5ubuntu3 gfortran: description: GNU Fortran 95 compiler downloads: 1171 homepage_uri: '' name: gfortran section: devel version: 4:4.6.3-1ubuntu5 gfortran-4.6: description: GNU Fortran compiler downloads: 480 homepage_uri: http://gcc.gnu.org/ name: gfortran-4.6 section: devel version: 4.6.3-1ubuntu5 ghc: description: The Glasgow Haskell Compilation system downloads: 420 homepage_uri: http://haskell.org/ghc/ name: ghc section: universe/haskell version: 7.4.1-1ubuntu2 ghc6: description: transitional dummy package downloads: 338 homepage_uri: http://haskell.org/ghc/ name: ghc6 section: universe/devel version: 7.4.1-1ubuntu2 ghostscript: description: interpreter for the PostScript language and for PDF downloads: 25415 homepage_uri: http://www.ghostscript.com/ name: ghostscript section: text version: 9.05~dfsg-0ubuntu4 ghostscript-cups: description: interpreter for the PostScript language and for PDF - CUPS filters downloads: 6081 homepage_uri: http://www.ghostscript.com/ name: ghostscript-cups section: text version: 9.05~dfsg-0ubuntu4 ghostscript-x: description: interpreter for the PostScript language and for PDF - X11 support downloads: 8197 homepage_uri: http://www.ghostscript.com/ name: ghostscript-x section: text version: 9.05~dfsg-0ubuntu4 ginn: description: 'Gesture Injector: No-GEIS, No-Toolkits' downloads: 0 homepage_uri: https://launchpad.net/ginn name: ginn section: misc version: 0.2.4-0ubuntu1 gir1.2-appindicator3-0.1: description: Typelib files for libappindicator3-1. downloads: 3 homepage_uri: https://launchpad.net/libappindicator name: gir1.2-appindicator3-0.1 section: libs version: 0.4.92-0ubuntu1 gir1.2-atk-1.0: description: ATK accessibility toolkit (GObject introspection) downloads: 11420 homepage_uri: http://www.gtk.org/ name: gir1.2-atk-1.0 section: libs version: 2.4.0-0ubuntu1 gir1.2-atspi-2.0: description: Assistive Technology Service Provider (GObject introspection) downloads: 267 homepage_uri: http://live.gnome.org/Accessibility/GNOME3 name: gir1.2-atspi-2.0 section: misc version: 2.4.2-0ubuntu0.1 gir1.2-dbusmenu-glib-0.4: description: typelib file for libdbusmenu-glib4 downloads: 9 homepage_uri: https://launchpad.net/dbusmenu name: gir1.2-dbusmenu-glib-0.4 section: libs version: 0.6.1-0ubuntu3 gir1.2-dbusmenu-gtk-0.4: description: typelib file for libdbusmenu-gtk4 downloads: 4 homepage_uri: https://launchpad.net/dbusmenu name: gir1.2-dbusmenu-gtk-0.4 section: libs version: 0.6.1-0ubuntu3 gir1.2-dee-1.0: description: GObject introspection data for the Dee library downloads: 1 homepage_uri: https://launchpad.net/dee name: gir1.2-dee-1.0 section: libs version: 1.0.10-0ubuntu1 gir1.2-freedesktop: description: Introspection data for some FreeDesktop components downloads: 10913 homepage_uri: http://live.gnome.org/GObjectIntrospection name: gir1.2-freedesktop section: libs version: 1.32.0-1 gir1.2-gconf-2.0: description: GNOME configuration database system (GObject-Introspection) downloads: 3376 homepage_uri: http://projects.gnome.org/gconf/ name: gir1.2-gconf-2.0 section: libs version: 3.2.5-0ubuntu2 gir1.2-gdkpixbuf-2.0: description: GDK Pixbuf library - GObject-Introspection downloads: 11411 homepage_uri: http://www.gtk.org/ name: gir1.2-gdkpixbuf-2.0 section: libs version: 2.26.1-1 gir1.2-glib-2.0: description: Introspection data for GLib, GObject, Gio and GModule downloads: 13556 homepage_uri: http://live.gnome.org/GObjectIntrospection name: gir1.2-glib-2.0 section: libs version: 1.32.0-1 gir1.2-gmenu-3.0: description: GObject introspection data for the GNOME menu library downloads: 7880 homepage_uri: '' name: gir1.2-gmenu-3.0 section: libs version: 3.4.0-0ubuntu1 gir1.2-gnomebluetooth-1.0: description: Introspection data for GnomeBluetooth downloads: 9491 homepage_uri: http://live.gnome.org/GnomeBluetooth name: gir1.2-gnomebluetooth-1.0 section: gnome version: 3.2.2-0ubuntu5 gir1.2-gnomekeyring-1.0: description: GNOME keyring services library - introspection data downloads: 422 homepage_uri: http://live.gnome.org/GnomeKeyring name: gir1.2-gnomekeyring-1.0 section: introspection version: 3.2.2-2 gir1.2-gst-plugins-base-0.10: description: 'Description: GObject introspection data for the GStreamer Plugins Base library' downloads: 2071 homepage_uri: http://gstreamer.freedesktop.org name: gir1.2-gst-plugins-base-0.10 section: libs version: 0.10.36-1 gir1.2-gstreamer-0.10: description: 'Description: GObject introspection data for the GStreamer library' downloads: 2489 homepage_uri: http://gstreamer.freedesktop.org name: gir1.2-gstreamer-0.10 section: libs version: 0.10.36-1ubuntu1 gir1.2-gtk-2.0: description: GTK+ graphical user interface library -- gir bindings downloads: 737 homepage_uri: http://www.gtk.org/ name: gir1.2-gtk-2.0 section: libs version: 2.24.10-0ubuntu6 gir1.2-gtk-3.0: description: GTK+ graphical user interface library -- gir bindings downloads: 10691 homepage_uri: http://www.gtk.org/ name: gir1.2-gtk-3.0 section: libs version: 3.4.2-0ubuntu0.2 gir1.2-gtksource-3.0: description: gir files for the GTK+ syntax highlighting widget downloads: 5493 homepage_uri: http://projects.gnome.org/gtksourceview/ name: gir1.2-gtksource-3.0 section: libs version: 3.4.2-0ubuntu1 gir1.2-gudev-1.0: description: libgudev-1.0 introspection data downloads: 1 homepage_uri: http://www.kernel.org/pub/linux/utils/kernel/hotplug/udev.html name: gir1.2-gudev-1.0 section: libs version: 175-0ubuntu9 gir1.2-indicate-0.7: description: Typelib file for libindicate5 downloads: 3 homepage_uri: https://launchpad.net/libindicate name: gir1.2-indicate-0.7 section: libs version: 0.6.92-0ubuntu1 gir1.2-javascriptcoregtk-3.0: description: GObject introspection data for the GTK+-based JavaScriptCore library downloads: 1241 homepage_uri: http://webkitgtk.org/ name: gir1.2-javascriptcoregtk-3.0 section: libs version: 1.8.0-0ubuntu2 gir1.2-launchpad-integration-3.0: description: library for launchpad integration (gir files) downloads: 0 homepage_uri: '' name: gir1.2-launchpad-integration-3.0 section: libs version: 0.1.56 gir1.2-notify-0.7: description: sends desktop notifications to a notification daemon (Introspection files) downloads: 484 homepage_uri: '' name: gir1.2-notify-0.7 section: libs version: 0.7.5-1 gir1.2-panelapplet-4.0: description: GObject introspection for the GNOME Panel Applet library downloads: 197 homepage_uri: '' name: gir1.2-panelapplet-4.0 section: universe/libs version: 1:3.4.1-0ubuntu1 gir1.2-pango-1.0: description: Layout and rendering of internationalized text - gir bindings downloads: 11358 homepage_uri: '' name: gir1.2-pango-1.0 section: libs version: 1.30.0-0ubuntu3 gir1.2-peas-1.0: description: Application plugin library (introspection files) downloads: 8059 homepage_uri: http://live.gnome.org/Libpeas name: gir1.2-peas-1.0 section: libs version: 1.2.0-1ubuntu1 gir1.2-rb-3.0: description: GObject introspection data for the rhythmbox music player downloads: 1405 homepage_uri: http://projects.gnome.org/rhythmbox/ name: gir1.2-rb-3.0 section: libs version: 2.96-0ubuntu4 gir1.2-soup-2.4: description: GObject introspection data for the libsoup HTTP library downloads: 8431 homepage_uri: '' name: gir1.2-soup-2.4 section: libs version: 2.38.1-1 gir1.2-totem-1.0: description: GObject introspection data for Totem media player downloads: 4634 homepage_uri: http://www.gnome.org/projects/totem/ name: gir1.2-totem-1.0 section: libs version: 3.0.1-0ubuntu21 gir1.2-totem-plparser-1.0: description: GObject introspection data for the Totem Playlist Parser library downloads: 4566 homepage_uri: '' name: gir1.2-totem-plparser-1.0 section: libs version: 3.4.1-1 gir1.2-ubuntuoneui-3.0: description: Ubuntu One widget library downloads: 1 homepage_uri: https://launchpad.net/libubuntuone name: gir1.2-ubuntuoneui-3.0 section: introspection version: 3.0.1-0ubuntu1 gir1.2-unity-5.0: description: GObject introspection data for the Unity library downloads: 0 homepage_uri: https://launchpad.net/libunity name: gir1.2-unity-5.0 section: libs version: 5.12.0-0ubuntu1 gir1.2-vte-2.90: description: GObject introspection data for the VTE library downloads: 4537 homepage_uri: '' name: gir1.2-vte-2.90 section: libs version: 1:0.32.1-0ubuntu1 gir1.2-webkit-3.0: description: GObject introspection data for the WebKit library downloads: 1920 homepage_uri: http://webkitgtk.org/ name: gir1.2-webkit-3.0 section: libs version: 1.8.0-0ubuntu2 gir1.2-wnck-3.0: description: GObject introspection data for the WNCK library downloads: 461 homepage_uri: '' name: gir1.2-wnck-3.0 section: libs version: 3.4.0-0ubuntu1 git: description: fast, scalable, distributed revision control system downloads: 13988 homepage_uri: http://git-scm.com/ name: git section: vcs version: 1:1.7.9.5-1 git-core: description: fast, scalable, distributed revision control system (obsolete) downloads: 1481 homepage_uri: http://git-scm.com/ name: git-core section: devel version: 1:1.7.9.5-1 git-man: description: fast, scalable, distributed revision control system (manual pages) downloads: 0 homepage_uri: http://git-scm.com/ name: git-man section: vcs version: 1:1.7.9.5-1 gksu: description: graphical frontend to su downloads: 29768 homepage_uri: http://www.nongnu.org/gksu name: gksu section: admin version: 2.0.2-6ubuntu1 glib-networking: description: network-related giomodules for GLib downloads: 2203 homepage_uri: '' name: glib-networking section: libs version: 2.32.1-1ubuntu1 glib-networking-common: description: network-related giomodules for GLib - data files downloads: 2362 homepage_uri: '' name: glib-networking-common section: libs version: 2.32.1-1ubuntu1 glib-networking-services: description: network-related giomodules for GLib - D-Bus services downloads: 2361 homepage_uri: '' name: glib-networking-services section: libs version: 2.32.1-1ubuntu1 glib-networking:i386: description: network-related giomodules for GLib downloads: 0 homepage_uri: '' name: glib-networking:i386 section: libs version: 2.32.1-1ubuntu1 gnome-accessibility-themes: description: accessibility themes for the GNOME desktop downloads: 0 homepage_uri: '' name: gnome-accessibility-themes section: gnome version: 3.4.1-0ubuntu1 gnome-applets: description: Various applets for the GNOME panel - binary files downloads: 21647 homepage_uri: '' name: gnome-applets section: universe/gnome version: 3.4.1-0ubuntu1 gnome-applets-data: description: Various applets for the GNOME panel - data files downloads: 22572 homepage_uri: '' name: gnome-applets-data section: universe/gnome version: 3.4.1-0ubuntu1 gnome-backgrounds: description: Set of backgrounds packaged with the GNOME desktop downloads: 0 homepage_uri: '' name: gnome-backgrounds section: universe/gnome version: 3.4.1-1 gnome-bluetooth: description: GNOME Bluetooth tools downloads: 28843 homepage_uri: http://live.gnome.org/GnomeBluetooth name: gnome-bluetooth section: gnome version: 3.2.2-0ubuntu5 gnome-control-center: description: utilities to configure the GNOME desktop downloads: 17783 homepage_uri: '' name: gnome-control-center section: gnome version: 1:3.4.2-0ubuntu0.2 gnome-control-center-data: description: configuration applets for GNOME - data files downloads: 0 homepage_uri: '' name: gnome-control-center-data section: gnome version: 1:3.4.2-0ubuntu0.2 gnome-desktop3-data: description: Common files for GNOME desktop apps downloads: 0 homepage_uri: '' name: gnome-desktop3-data section: gnome version: 3.4.1-0ubuntu1 gnome-disk-utility: description: manage and configure disk drives and media downloads: 26660 homepage_uri: http://git.gnome.org/cgit/gnome-disk-utility/ name: gnome-disk-utility section: admin version: 3.0.2-2ubuntu7 gnome-font-viewer: description: font viewer for GNOME downloads: 1378 homepage_uri: http://live.gnome.org/GnomeUtils name: gnome-font-viewer section: gnome version: 3.4.0-1 gnome-games-data: description: data files for the GNOME games downloads: 1226 homepage_uri: http://live.gnome.org/GnomeGames name: gnome-games-data section: gnome version: 1:3.4.1-0ubuntu2 gnome-icon-theme: description: GNOME Desktop icon theme (small subset) downloads: 36553 homepage_uri: '' name: gnome-icon-theme section: gnome version: 3.4.0-0ubuntu1 gnome-icon-theme-symbolic: description: GNOME desktop icon theme (symbolic icons) downloads: 8456 homepage_uri: '' name: gnome-icon-theme-symbolic section: gnome version: 3.4.0-1 gnome-keyring: description: GNOME keyring services (daemon and tools) downloads: 37949 homepage_uri: http://live.gnome.org/GnomeKeyring name: gnome-keyring section: gnome version: 3.2.2-2ubuntu4 gnome-media: description: GNOME media utilities downloads: 8792 homepage_uri: http://live.gnome.org/GnomeMedia name: gnome-media section: gnome version: 3.4.0-0ubuntu2 gnome-menus: description: GNOME implementation of the freedesktop menu specification downloads: 22903 homepage_uri: '' name: gnome-menus section: gnome version: 3.4.0-0ubuntu1 gnome-nettool: description: network information tool for GNOME downloads: 3412 homepage_uri: http://projects.gnome.org/gnome-network/ name: gnome-nettool section: gnome version: 3.2.0-0ubuntu1 gnome-online-accounts: description: GNOME Online Accounts downloads: 1009 homepage_uri: https://live.gnome.org/OnlineAccounts name: gnome-online-accounts section: gnome version: 3.4.0-0ubuntu1 gnome-orca: description: Scriptable screen reader downloads: 2177 homepage_uri: http://live.gnome.org/Orca name: gnome-orca section: gnome version: 3.4.2-0ubuntu0.1 gnome-panel: description: launcher and docking facility for GNOME downloads: 25163 homepage_uri: '' name: gnome-panel section: universe/gnome version: 1:3.4.1-0ubuntu1 gnome-panel-data: description: common files for the GNOME Panel downloads: 22503 homepage_uri: '' name: gnome-panel-data section: universe/gnome version: 1:3.4.1-0ubuntu1 gnome-power-manager: description: power management tool for the GNOME desktop downloads: 24959 homepage_uri: http://www.gnome.org/projects/gnome-power-manager/ name: gnome-power-manager section: gnome version: 3.4.0-0ubuntu1 gnome-screensaver: description: GNOME screen saver and locker downloads: 30960 homepage_uri: http://live.gnome.org/GnomeScreensaver name: gnome-screensaver section: gnome version: 3.4.1-0ubuntu1 gnome-screenshot: description: screenshot application for GNOME downloads: 8739 homepage_uri: http://live.gnome.org/GnomeUtils name: gnome-screenshot section: gnome version: 3.4.1-0ubuntu1 gnome-session: description: GNOME Session Manager - GNOME 3 session downloads: 20285 homepage_uri: '' name: gnome-session section: gnome version: 3.2.1-0ubuntu8 gnome-session-bin: description: GNOME Session Manager - Minimal runtime downloads: 32359 homepage_uri: '' name: gnome-session-bin section: gnome version: 3.2.1-0ubuntu8 gnome-session-canberra: description: GNOME session log in and log out sound events downloads: 2866 homepage_uri: http://0pointer.de/lennart/projects/libcanberra/ name: gnome-session-canberra section: libs version: 0.28-3ubuntu3 gnome-session-common: description: GNOME Session Manager - common files downloads: 22194 homepage_uri: '' name: gnome-session-common section: gnome version: 3.2.1-0ubuntu8 gnome-session-fallback: description: GNOME Session Manager - GNOME fallback session downloads: 3923 homepage_uri: '' name: gnome-session-fallback section: universe/gnome version: 3.2.1-0ubuntu8 gnome-settings-daemon: description: daemon handling the GNOME session settings downloads: 33233 homepage_uri: '' name: gnome-settings-daemon section: gnome version: 3.4.1-0ubuntu1.1 gnome-sudoku: description: Sudoku puzzle game for GNOME downloads: 480 homepage_uri: http://live.gnome.org/GnomeGames name: gnome-sudoku section: games version: 1:3.4.1-0ubuntu2 gnome-system-log: description: system log viewer for GNOME downloads: 2978 homepage_uri: http://live.gnome.org/GnomeUtils name: gnome-system-log section: gnome version: 3.4.1-0ubuntu1 gnome-system-monitor: description: Process viewer and system resource monitor for GNOME downloads: 8633 homepage_uri: '' name: gnome-system-monitor section: gnome version: 3.4.1-0ubuntu1 gnome-system-tools: description: Cross-platform configuration utilities for GNOME downloads: 6907 homepage_uri: '' name: gnome-system-tools section: universe/gnome version: 3.0.0-2ubuntu1 gnome-terminal: description: GNOME terminal emulator application downloads: 25270 homepage_uri: '' name: gnome-terminal section: gnome version: 3.4.1.1-0ubuntu1 gnome-terminal-data: description: Data files for the GNOME terminal emulator downloads: 1753 homepage_uri: '' name: gnome-terminal-data section: gnome version: 3.4.1.1-0ubuntu1 gnome-user-guide: description: GNOME user's guide downloads: 1 homepage_uri: '' name: gnome-user-guide section: gnome version: 3.4.1-1 gnome-user-share: description: User level public file sharing via WebDAV or ObexFTP downloads: 24693 homepage_uri: '' name: gnome-user-share section: gnome version: 3.0.2-0ubuntu1 gnomine: description: popular minesweeper puzzle game for GNOME downloads: 403 homepage_uri: http://live.gnome.org/GnomeGames name: gnomine section: games version: 1:3.4.1-0ubuntu2 gnupg: description: GNU privacy guard - a free PGP replacement downloads: 49882 homepage_uri: http://www.gnupg.org name: gnupg section: utils version: 1.4.11-3ubuntu2 gnuplot: description: A command-line driven interactive plotting program downloads: 213 homepage_uri: http://gnuplot.sourceforge.net/ name: gnuplot section: universe/math version: 4.4.3-0ubuntu3 gnuplot-nox: description: A command-line driven interactive plotting program downloads: 1961 homepage_uri: http://gnuplot.sourceforge.net/ name: gnuplot-nox section: universe/math version: 4.4.3-0ubuntu3 gnuplot-x11: description: A command-line driven interactive plotting program downloads: 2498 homepage_uri: http://gnuplot.sourceforge.net/ name: gnuplot-x11 section: universe/math version: 4.4.3-0ubuntu3 gpgv: description: GNU privacy guard - signature verification tool downloads: 68978 homepage_uri: http://www.gnupg.org name: gpgv section: utils version: 1.4.11-3ubuntu2 graphviz: description: rich set of graph drawing tools downloads: 2333 homepage_uri: http://www.graphviz.org/ name: graphviz section: graphics version: 2.26.3-10ubuntu1 grep: description: GNU grep, egrep and fgrep downloads: 110992 homepage_uri: http://www.gnu.org/software/grep/ name: grep section: utils version: 2.10-1 groff: description: GNU troff text-formatting system downloads: 1090 homepage_uri: http://groff.ffii.org/ name: groff section: text version: 1.21-7 groff-base: description: GNU troff text-formatting system (base system components) downloads: 105546 homepage_uri: http://groff.ffii.org/ name: groff-base section: text version: 1.21-7 growisofs: description: DVD+-RW/R recorder downloads: 4117 homepage_uri: http://fy.chalmers.se/~appro/linux/DVD+RW/ name: growisofs section: video version: 7.1-10 grub-common: description: GRand Unified Bootloader (common files) downloads: 17653 homepage_uri: http://www.gnu.org/software/grub/ name: grub-common section: admin version: 1.99-21ubuntu3.1 grub-gfxpayload-lists: description: GRUB gfxpayload blacklist downloads: 1 homepage_uri: '' name: grub-gfxpayload-lists section: admin version: '0.6' grub-legacy-ec2: description: Handles update-grub for ec2 instances downloads: 1 homepage_uri: '' name: grub-legacy-ec2 section: admin version: 0.6.3-0ubuntu1 grub-pc: description: GRand Unified Bootloader, version 2 (PC/BIOS version) downloads: 10228 homepage_uri: http://www.gnu.org/software/grub/ name: grub-pc section: admin version: 1.99-21ubuntu3.1 grub-pc-bin: description: GRand Unified Bootloader, version 2 (PC/BIOS binaries) downloads: 1709 homepage_uri: http://www.gnu.org/software/grub/ name: grub-pc-bin section: admin version: 1.99-21ubuntu3.1 grub2-common: description: GRand Unified Bootloader (common files for version 2) downloads: 6525 homepage_uri: http://www.gnu.org/software/grub/ name: grub2-common section: admin version: 1.99-21ubuntu3.1 gs-cjk-resource: description: Resource files for gs-cjk, ghostscript CJK-TrueType extension downloads: 3135 homepage_uri: '' name: gs-cjk-resource section: text version: 1.20100103-3 gsettings-desktop-schemas: description: GSettings deskop-wide schemas downloads: 10179 homepage_uri: http://www.gnome.org/ name: gsettings-desktop-schemas section: libs version: 3.4.1-0ubuntu1 gsfonts: description: Fonts for the Ghostscript interpreter(s) downloads: 25634 homepage_uri: http://www.ghostscript.com/ name: gsfonts section: text version: 1:8.11+urwcyr1.0.7~pre44-4.2ubuntu1 gsl-bin: description: GNU Scientific Library (GSL) -- binary package downloads: 108 homepage_uri: http://www.gnu.org/software/gsl name: gsl-bin section: universe/math version: 1.15+dfsg-1build1 gstreamer0.10-alsa: description: GStreamer plugin for ALSA downloads: 24770 homepage_uri: http://gstreamer.freedesktop.org name: gstreamer0.10-alsa section: libs version: 0.10.36-1 gstreamer0.10-gconf: description: GStreamer plugin for getting the sink/source information from GConf downloads: 2698 homepage_uri: '' name: gstreamer0.10-gconf section: sound version: 0.10.31-1ubuntu1 gstreamer0.10-nice: description: ICE library (GStreamer plugin) downloads: 4907 homepage_uri: http://nice.freedesktop.org/ name: gstreamer0.10-nice section: net version: 0.1.1-2ubuntu1 gstreamer0.10-plugins-base: description: GStreamer plugins from the "base" set downloads: 34634 homepage_uri: http://gstreamer.freedesktop.org name: gstreamer0.10-plugins-base section: libs version: 0.10.36-1 gstreamer0.10-plugins-base-apps: description: GStreamer helper programs from the "base" set downloads: 170 homepage_uri: http://gstreamer.freedesktop.org name: gstreamer0.10-plugins-base-apps section: utils version: 0.10.36-1 gstreamer0.10-plugins-base:i386: description: GStreamer plugins from the "base" set downloads: 0 homepage_uri: http://gstreamer.freedesktop.org name: gstreamer0.10-plugins-base:i386 section: libs version: 0.10.36-1 gstreamer0.10-plugins-good: description: GStreamer plugins from the "good" set downloads: 33874 homepage_uri: '' name: gstreamer0.10-plugins-good section: libs version: 0.10.31-1ubuntu1 gstreamer0.10-plugins-good:i386: description: GStreamer plugins from the "good" set downloads: 0 homepage_uri: '' name: gstreamer0.10-plugins-good:i386 section: libs version: 0.10.31-1ubuntu1 gstreamer0.10-pulseaudio: description: GStreamer plugin for PulseAudio downloads: 5700 homepage_uri: '' name: gstreamer0.10-pulseaudio section: sound version: 0.10.31-1ubuntu1 gstreamer0.10-tools: description: Tools for use with GStreamer downloads: 3295 homepage_uri: http://gstreamer.freedesktop.org name: gstreamer0.10-tools section: utils version: 0.10.36-1ubuntu1 gstreamer0.10-x: description: GStreamer plugins for X11 and Pango downloads: 30682 homepage_uri: http://gstreamer.freedesktop.org name: gstreamer0.10-x section: libs version: 0.10.36-1 gstreamer0.10-x:i386: description: GStreamer plugins for X11 and Pango downloads: 0 homepage_uri: http://gstreamer.freedesktop.org name: gstreamer0.10-x:i386 section: libs version: 0.10.36-1 gtk2-engines: description: theme engines for GTK+ 2.x downloads: 27638 homepage_uri: '' name: gtk2-engines section: gnome version: 1:2.20.2-1ubuntu1 gtk2-engines-murrine: description: cairo-based gtk+-2.0 theme engine downloads: 963 homepage_uri: '' name: gtk2-engines-murrine section: x11 version: 0.98.2-0ubuntu1 gtk2-engines-murrine:i386: description: cairo-based gtk+-2.0 theme engine downloads: 0 homepage_uri: '' name: gtk2-engines-murrine:i386 section: x11 version: 0.98.2-0ubuntu1 gtk2-engines-oxygen:i386: description: Oxygen widget theme for GTK+-based applications downloads: 0 homepage_uri: https://projects.kde.org/projects/playground/artwork/oxygen-gtk name: gtk2-engines-oxygen:i386 section: x11 version: 1.2.2-0ubuntu1.1 gtk2-engines-pixbuf:i386: description: pixbuf-based theme for GTK+ 2.x downloads: 0 homepage_uri: http://www.gtk.org/ name: gtk2-engines-pixbuf:i386 section: universe/graphics version: 2.24.10-0ubuntu6 gtk2-engines:i386: description: theme engines for GTK+ 2.x downloads: 0 homepage_uri: '' name: gtk2-engines:i386 section: gnome version: 1:2.20.2-1ubuntu1 gtk3-engines-unico: description: Unico Gtk+ 3 theme engine downloads: 55 homepage_uri: https://launchpad.net/unico name: gtk3-engines-unico section: x11 version: 1.0.2-0ubuntu1 gucharmap: description: Unicode character picker and font browser downloads: 3104 homepage_uri: http://live.gnome.org/Gucharmap name: gucharmap section: x11 version: 1:3.4.1.1-0ubuntu1 guile-1.8-libs: description: Core Guile libraries downloads: 2341 homepage_uri: http://www.gnu.org/software/guile/ name: guile-1.8-libs section: libs version: 1.8.8+1-6ubuntu2 gvfs: description: userspace virtual filesystem - GIO module downloads: 38669 homepage_uri: '' name: gvfs section: libs version: 1.12.1-0ubuntu1 gvfs-backends: description: userspace virtual filesystem - backends downloads: 33369 homepage_uri: '' name: gvfs-backends section: libs version: 1.12.1-0ubuntu1 gvfs-bin: description: userspace virtual filesystem - binaries downloads: 11948 homepage_uri: '' name: gvfs-bin section: libs version: 1.12.1-0ubuntu1 gvfs-common: description: userspace virtual filesystem - common data files downloads: 14841 homepage_uri: '' name: gvfs-common section: libs version: 1.12.1-0ubuntu1 gvfs-daemons: description: userspace virtual filesystem - servers downloads: 14827 homepage_uri: '' name: gvfs-daemons section: libs version: 1.12.1-0ubuntu1 gvfs-fuse: description: userspace virtual filesystem - fuse server downloads: 3866 homepage_uri: '' name: gvfs-fuse section: libs version: 1.12.1-0ubuntu1 gvfs-libs: description: userspace virtual filesystem - private libraries downloads: 14834 homepage_uri: '' name: gvfs-libs section: libs version: 1.12.1-0ubuntu1 gvfs-libs:i386: description: userspace virtual filesystem - private libraries downloads: 0 homepage_uri: '' name: gvfs-libs:i386 section: libs version: 1.12.1-0ubuntu1 gvfs:i386: description: userspace virtual filesystem - GIO module downloads: 0 homepage_uri: '' name: gvfs:i386 section: libs version: 1.12.1-0ubuntu1 gwibber: description: Open source social networking client for GNOME downloads: 131 homepage_uri: https://launchpad.net/gwibber name: gwibber section: misc version: 3.4.1-0ubuntu1 gwibber-service: description: Open source social networking client for GNOME downloads: 149 homepage_uri: https://launchpad.net/gwibber name: gwibber-service section: misc version: 3.4.1-0ubuntu1 gwibber-service-facebook: description: Facebook plugin for Gwibber downloads: 0 homepage_uri: https://launchpad.net/gwibber name: gwibber-service-facebook section: misc version: 3.4.1-0ubuntu1 gwibber-service-identica: description: Identi.ca plugin for Gwibber downloads: 0 homepage_uri: https://launchpad.net/gwibber name: gwibber-service-identica section: misc version: 3.4.1-0ubuntu1 gwibber-service-twitter: description: Twitter plugin for Gwibber downloads: 0 homepage_uri: https://launchpad.net/gwibber name: gwibber-service-twitter section: misc version: 3.4.1-0ubuntu1 gzip: description: GNU compression utilities downloads: 110479 homepage_uri: '' name: gzip section: utils version: 1.4-1ubuntu2 hadoop-0.20: description: A software platform for processing vast amounts of data downloads: 32 homepage_uri: http://hadoop.apache.org/core/ name: hadoop-0.20 section: misc version: 0.20.2+923.256-1~maverick-cdh3 hadoop-0.20-pipes: description: Interface to author Hadoop MapReduce jobs in C++ downloads: 0 homepage_uri: http://hadoop.apache.org/core/ name: hadoop-0.20-pipes section: misc version: 0.20.2+923.256-1~maverick-cdh3 hadoop-0.20-source: description: Source code for Hadoop downloads: 0 homepage_uri: http://hadoop.apache.org/core/ name: hadoop-0.20-source section: misc version: 0.20.2+923.256-1~maverick-cdh3 hadoop-hive: description: A data warehouse infrastructure built on top of Hadoop downloads: 2 homepage_uri: http://hadoop.apache.org/hive/ name: hadoop-hive section: misc version: 0.7.1+42.43-2~maverick-cdh3 hadoop-pig: description: A platform for analyzing large data sets using Hadoop downloads: 3 homepage_uri: http://hadoop.apache.org/pig/ name: hadoop-pig section: misc version: 0.8.1+28.32-1~maverick-cdh3 hdparm: description: tune hard disk parameters for high performance downloads: 23393 homepage_uri: http://sourceforge.net/projects/hdparm/ name: hdparm section: admin version: 9.37-0ubuntu3.1 hicolor-icon-theme: description: default fallback theme for FreeDesktop.org icon themes downloads: 33911 homepage_uri: '' name: hicolor-icon-theme section: misc version: 0.12-1ubuntu2 hmmer: description: profile hidden Markov models for protein sequence analysis downloads: 30 homepage_uri: http://hmmer.janelia.org/ name: hmmer section: science version: 3.0-3 hostname: description: utility to set/show the host name or domain name downloads: 73725 homepage_uri: '' name: hostname section: admin version: 3.06ubuntu1 hplip: description: HP Linux Printing and Imaging System (HPLIP) downloads: 17838 homepage_uri: http://hplipopensource.com/hplip-web/index.html name: hplip section: utils version: 3.12.2-1ubuntu3 hplip-data: description: HP Linux Printing and Imaging - data files downloads: 140 homepage_uri: http://hplipopensource.com/hplip-web/index.html name: hplip-data section: utils version: 3.12.2-1ubuntu3 html2text: description: advanced HTML to text converter downloads: 15442 homepage_uri: http://www.mbayer.de/html2text/ name: html2text section: web version: 1.3.2a-15 htop: description: interactive processes viewer downloads: 8845 homepage_uri: http://htop.sourceforge.net name: htop section: universe/utils version: 1.0.1-1 humanity-icon-theme: description: Humanity Icon theme downloads: 40 homepage_uri: '' name: humanity-icon-theme section: gnome version: 0.5.3.11 hunspell-en-us: description: English_american dictionary for hunspell downloads: 0 homepage_uri: '' name: hunspell-en-us section: text version: 20070829-4ubuntu3 hwdata: description: hardware identification / configuration data downloads: 3 homepage_uri: http://git.fedorahosted.org/git/hwdata.git name: hwdata section: misc version: 0.233-1ubuntu1 ia32-libs: description: ia32 shared libraries - transitional package downloads: 701 homepage_uri: '' name: ia32-libs section: universe/libs version: 20090808ubuntu36 ia32-libs-multiarch:i386: description: Multi-arch versions of former ia32-libraries downloads: 0 homepage_uri: '' name: ia32-libs-multiarch:i386 section: universe/libs version: 20090808ubuntu36 ibus: description: Intelligent Input Bus - core downloads: 679 homepage_uri: http://code.google.com/p/ibus/ name: ibus section: utils version: 1.4.1-3ubuntu1 ibus-gtk: description: Intelligent Input Bus - GTK+2 support downloads: 452 homepage_uri: http://code.google.com/p/ibus/ name: ibus-gtk section: utils version: 1.4.1-3ubuntu1 ibus-gtk3: description: Intelligent Input Bus - GTK+3 support downloads: 48 homepage_uri: http://code.google.com/p/ibus/ name: ibus-gtk3 section: utils version: 1.4.1-3ubuntu1 ibus-gtk:i386: description: Intelligent Input Bus - GTK+2 support downloads: 0 homepage_uri: http://code.google.com/p/ibus/ name: ibus-gtk:i386 section: utils version: 1.4.1-3ubuntu1 ibus-pinyin: description: Pinyin engine for IBus downloads: 178 homepage_uri: http://code.google.com/p/ibus name: ibus-pinyin section: utils version: 1.4.0-1 ibus-pinyin-db-android: description: Pinyin engine for IBus - Android database downloads: 0 homepage_uri: http://code.google.com/p/ibus name: ibus-pinyin-db-android section: utils version: 1.4.0-1 ibus-pinyin-db-open-phrase: description: Pinyin engine for IBus - open-phrase database downloads: 173 homepage_uri: http://code.google.com/p/ibus name: ibus-pinyin-db-open-phrase section: universe/utils version: 1.4.0-1 ibus-table: description: table engine for IBus downloads: 119 homepage_uri: http://code.google.com/p/ibus name: ibus-table section: utils version: 1.3.9.20110827-1ubuntu1 icedtea-6-jre-cacao: description: Alternative JVM for OpenJDK, using Cacao downloads: 91 homepage_uri: http://openjdk.java.net/ name: icedtea-6-jre-cacao section: interpreters version: 6b24-1.11.1-4ubuntu3 icedtea-6-jre-jamvm: description: Alternative JVM for OpenJDK, using JamVM downloads: 33 homepage_uri: http://openjdk.java.net/ name: icedtea-6-jre-jamvm section: java version: 6b24-1.11.1-4ubuntu3 icedtea-7-jre-cacao: description: Alternative JVM for OpenJDK, using Cacao downloads: 2 homepage_uri: http://openjdk.java.net/ name: icedtea-7-jre-cacao section: universe/java version: 7~u3-2.1.1~pre1-1ubuntu3 icedtea-7-jre-jamvm: description: Alternative JVM for OpenJDK, using JamVM downloads: 1 homepage_uri: http://openjdk.java.net/ name: icedtea-7-jre-jamvm section: universe/java version: 7~u3-2.1.1~pre1-1ubuntu3 icedtea-netx: description: NetX - implementation of the Java Network Launching Protocol (JNLP) downloads: 343 homepage_uri: http://icedtea.classpath.org/wiki/IcedTea-Web name: icedtea-netx section: java version: 1.2-2ubuntu1 icedtea-netx-common: description: NetX - implementation of the Java Network Launching Protocol (JNLP) downloads: 427 homepage_uri: http://icedtea.classpath.org/wiki/IcedTea-Web name: icedtea-netx-common section: java version: 1.2-2ubuntu1 ifupdown: description: high level tools to configure network interfaces downloads: 71985 homepage_uri: '' name: ifupdown section: admin version: 0.7~beta2ubuntu8 im-switch: description: Input method switch framework downloads: 191 homepage_uri: '' name: im-switch section: x11 version: 1.20ubuntu5 imagemagick: description: image manipulation programs downloads: 23864 homepage_uri: http://www.imagemagick.org/ name: imagemagick section: graphics version: 8:6.6.9.7-5ubuntu3.1 imagemagick-common: description: image manipulation programs -- infrastructure downloads: 0 homepage_uri: http://www.imagemagick.org/ name: imagemagick-common section: graphics version: 8:6.6.9.7-5ubuntu3.1 indicator-applet-complete: description: Clone of the GNOME panel indicator applet downloads: 95 homepage_uri: https://launchpad.net/indicator-applet name: indicator-applet-complete section: universe/gnome version: 0.5.0-0ubuntu1 indicator-application: description: Application Indicators downloads: 39 homepage_uri: https://launchpad.net/indicator-application name: indicator-application section: gnome version: 0.5.0-0ubuntu1 indicator-appmenu: description: Indicator for application menus. downloads: 0 homepage_uri: http://launchpad.net/indicator-appmenu name: indicator-appmenu section: gnome version: 0.3.97-0ubuntu1 indicator-datetime: description: Simple clock downloads: 0 homepage_uri: https://launchpad.net/indicator-datetime name: indicator-datetime section: misc version: 0.3.94-0ubuntu2 indicator-messages: description: indicator that collects messages that need a response downloads: 132 homepage_uri: https://launchpad.net/indicator-messages name: indicator-messages section: gnome version: 0.6.0-0ubuntu1 indicator-power: description: Indicator showing power state. downloads: 0 homepage_uri: https://launchpad.net/indicator-power name: indicator-power section: gnome version: 2.0-0ubuntu1 indicator-printers: description: indicator showing active print jobs downloads: 0 homepage_uri: https://launchpad.net/indicator-printers name: indicator-printers section: gnome version: 0.1.6-0ubuntu1 indicator-session: description: indicator showing session management, status and user switching downloads: 45 homepage_uri: https://launchpad.net/indicator-session name: indicator-session section: gnome version: 0.3.96-0ubuntu1 indicator-sound: description: System sound indicator. downloads: 19 homepage_uri: https://launchpad.net/indicator-sound name: indicator-sound section: sound version: 0.8.5.0-0ubuntu2.1 indicator-status-provider-mc5: description: indicator-messages status provider for telepathy mission-control-5 downloads: 6 homepage_uri: https://launchpad.net/indicator-messages name: indicator-status-provider-mc5 section: gnome version: 0.6.0-0ubuntu1 info: description: Standalone GNU Info documentation browser downloads: 10379 homepage_uri: '' name: info section: doc version: 4.13a.dfsg.1-8ubuntu2 initramfs-tools: description: tools for generating an initramfs downloads: 52983 homepage_uri: '' name: initramfs-tools section: utils version: 0.99ubuntu13 initramfs-tools-bin: description: binaries used by initramfs-tools downloads: 28 homepage_uri: '' name: initramfs-tools-bin section: utils version: 0.99ubuntu13 initscripts: description: scripts for initializing and shutting down the system downloads: 110492 homepage_uri: http://savannah.nongnu.org/projects/sysvinit name: initscripts section: admin version: 2.88dsf-13.10ubuntu11 inputattach: description: utility to connect serial-attached peripherals to the input subsystem downloads: 366 homepage_uri: https://sourceforge.net/projects/linuxconsole/ name: inputattach section: utils version: 1:1.4.2-1 insserv: description: Tool to organize boot sequence using LSB init.d script dependencies downloads: 85368 homepage_uri: http://savannah.nongnu.org/projects/sysvinit name: insserv section: misc version: 1.14.0-2.1ubuntu2 install-info: description: Manage installed documentation in info format downloads: 34204 homepage_uri: '' name: install-info section: doc version: 4.13a.dfsg.1-8ubuntu2 intel-gpu-tools: description: tools for debugging the Intel graphics driver downloads: 58 homepage_uri: http://www.intellinuxgraphics.org/ name: intel-gpu-tools section: x11 version: 1.2-1 intltool: description: Utility scripts for internationalizing XML downloads: 1275 homepage_uri: https://launchpad.net/intltool name: intltool section: devel version: 0.50.2-2 intltool-debian: description: Help i18n of RFC822 compliant config files downloads: 6883 homepage_uri: '' name: intltool-debian section: devel version: 0.35.0+20060710.1 iproute: description: networking and traffic control tools downloads: 79306 homepage_uri: http://www.linux-foundation.org/en/Net:Iproute2 name: iproute section: net version: 20111117-1ubuntu2 iptables: description: administration tools for packet filtering and NAT downloads: 31867 homepage_uri: http://www.netfilter.org/ name: iptables section: net version: 1.4.12-1ubuntu4 iputils-arping: description: Tool to send ICMP echo requests to an ARP address downloads: 561 homepage_uri: '' name: iputils-arping section: net version: 3:20101006-1ubuntu1 iputils-ping: description: Tools to test the reachability of network hosts downloads: 44649 homepage_uri: '' name: iputils-ping section: net version: 3:20101006-1ubuntu1 iputils-tracepath: description: Tools to trace the network path to a remote host downloads: 1408 homepage_uri: '' name: iputils-tracepath section: net version: 3:20101006-1ubuntu1 irqbalance: description: Daemon to balance interrupts for SMP systems downloads: 832 homepage_uri: http://irqbalance.org/ name: irqbalance section: utils version: 0.56-1ubuntu4 isc-dhcp-client: description: ISC DHCP client downloads: 52500 homepage_uri: '' name: isc-dhcp-client section: net version: 4.1.ESV-R4-0ubuntu5.1 isc-dhcp-common: description: common files used by all the isc-dhcp* packages downloads: 48869 homepage_uri: '' name: isc-dhcp-common section: net version: 4.1.ESV-R4-0ubuntu5.1 iso-codes: description: ISO language, territory, currency, script codes and their translations downloads: 7814 homepage_uri: http://pkg-isocodes.alioth.debian.org/ name: iso-codes section: libs version: 3.31-1 iw: description: tool for configuring Linux wireless devices downloads: 962 homepage_uri: http://wireless.kernel.org/en/users/Documentation/iw name: iw section: net version: 3.2-1 jarwrapper: description: Run executable Java .jar files downloads: 1853 homepage_uri: '' name: jarwrapper section: universe/libs version: 0.40ubuntu1 java-common: description: Base of all Java packages downloads: 24761 homepage_uri: '' name: java-common section: misc version: 0.43ubuntu2 java-wrappers: description: wrappers for java executables downloads: 2660 homepage_uri: '' name: java-wrappers section: misc version: 0.1.24 jockey-common: description: user interface and desktop integration for driver management downloads: 7 homepage_uri: '' name: jockey-common section: admin version: 0.9.7-0ubuntu7 jockey-gtk: description: GNOME user interface and desktop integration for driver management downloads: 5 homepage_uri: '' name: jockey-gtk section: admin version: 0.9.7-0ubuntu7 jruby: description: 100% pure-Java implementation of Ruby downloads: 21 homepage_uri: http://jruby.org name: jruby section: universe/ruby version: 1.5.6-2 junit: description: Automated testing framework for Java downloads: 1687 homepage_uri: http://www.junit.org name: junit section: devel version: 3.8.2-8 junit4: description: JUnit regression test framework for Java downloads: 1460 homepage_uri: http://www.junit.org/ name: junit4 section: devel version: 4.8.2-2 jython: description: Python seamlessly integrated with Java downloads: 96 homepage_uri: http://www.jython.org/ name: jython section: universe/python version: 2.5.1-2ubuntu2 kbd: description: Linux console font and keytable utilities downloads: 47118 homepage_uri: '' name: kbd section: utils version: 1.15.2-3ubuntu4 kerneloops-daemon: description: kernel oops tracker downloads: 7220 homepage_uri: http://www.kerneloops.org/ name: kerneloops-daemon section: utils version: 0.12+git20090217-1ubuntu19 keyboard-configuration: description: system-wide keyboard preferences downloads: 48584 homepage_uri: '' name: keyboard-configuration section: utils version: 1.70ubuntu5 klibc-utils: description: small utilities built with klibc for early boot downloads: 32706 homepage_uri: '' name: klibc-utils section: libs version: 1.5.25-1ubuntu2 kpartx: description: create device mappings for partitions downloads: 1378 homepage_uri: http://christophe.varoqui.free.fr/ name: kpartx section: admin version: 0.4.9-3ubuntu5 krb5-locales: description: Internationalization support for MIT Kerberos downloads: 1 homepage_uri: http://web.mit.edu/kerberos/ name: krb5-locales section: localization version: 1.10+dfsg~beta1-2ubuntu0.1 krb5-multidev: description: Development files for MIT Kerberos without Heimdal conflict downloads: 11860 homepage_uri: http://web.mit.edu/kerberos/ name: krb5-multidev section: libdevel version: 1.10+dfsg~beta1-2ubuntu0.1 kvm-ipxe: description: PXE ROM's for KVM downloads: 0 homepage_uri: http://ipxe.org/ name: kvm-ipxe section: admin version: 1.0.0+git-3.55f6c88-0ubuntu1 lacheck: description: Simple syntax checker for LaTeX downloads: 1365 homepage_uri: http://www.ctan.org/tex-archive/support/lacheck/ name: lacheck section: tex version: 1.26-14 landscape-client: description: The Landscape administration system client downloads: 1 homepage_uri: '' name: landscape-client section: admin version: 12.04.3-0ubuntu1 landscape-client-ui-install: description: The Landscape administration system client - UI installer downloads: 0 homepage_uri: '' name: landscape-client-ui-install section: admin version: 12.04.3-0ubuntu1 landscape-common: description: The Landscape administration system client - Common files downloads: 3 homepage_uri: '' name: landscape-common section: admin version: 12.04.3-0ubuntu1 language-selector-common: description: Language selector for Ubuntu downloads: 6 homepage_uri: '' name: language-selector-common section: admin version: '0.79' language-selector-gnome: description: Language selector for Ubuntu downloads: 0 homepage_uri: '' name: language-selector-gnome section: admin version: '0.79' laptop-detect: description: attempt to detect a laptop downloads: 7447 homepage_uri: '' name: laptop-detect section: utils version: 0.13.7ubuntu2 last-align: description: genome-scale comparison of biological sequences downloads: 20 homepage_uri: http://last.cbrc.jp/ name: last-align section: universe/science version: 189-1 latex-beamer: description: LaTeX class to produce presentations downloads: 0 homepage_uri: '' name: latex-beamer section: tex version: 3.10-1 latex-xcolor: description: Easy driver-independent TeX class for color downloads: 106 homepage_uri: '' name: latex-xcolor section: tex version: 2.11-1 launchpad-integration: description: launchpad integration downloads: 6 homepage_uri: '' name: launchpad-integration section: libs version: 0.1.56 less: description: pager program similar to more downloads: 54882 homepage_uri: http://www.greenwoodsoftware.com/less/ name: less section: text version: 444-1ubuntu1 lesstif2: description: OSF/Motif 2.1 implementation released under LGPL downloads: 6241 homepage_uri: http://www.lesstif.org/ name: lesstif2 section: universe/libs version: 1:0.95.2-1 libaa1: description: ASCII art library downloads: 38610 homepage_uri: http://aa-project.sourceforge.net/aalib/ name: libaa1 section: libs version: 1.4p5-39ubuntu1 libaa1:i386: description: ASCII art library downloads: 0 homepage_uri: http://aa-project.sourceforge.net/aalib/ name: libaa1:i386 section: libs version: 1.4p5-39ubuntu1 libaccountsservice0: description: query and manipulate user account information - shared libraries downloads: 8882 homepage_uri: http://cgit.freedesktop.org/accountsservice/ name: libaccountsservice0 section: libs version: 0.6.15-2ubuntu9 libacl1: description: Access control list shared library downloads: 64410 homepage_uri: http://savannah.nongnu.org/projects/acl/ name: libacl1 section: libs version: 2.2.51-5ubuntu1 libacl1:i386: description: Access control list shared library downloads: 0 homepage_uri: http://savannah.nongnu.org/projects/acl/ name: libacl1:i386 section: libs version: 2.2.51-5ubuntu1 libaio1: description: Linux kernel AIO access library - shared library downloads: 8133 homepage_uri: http://www.kernel.org/pub/linux/libs/aio/ name: libaio1 section: libs version: 0.3.109-2ubuntu1 libaio1:i386: description: Linux kernel AIO access library - shared library downloads: 0 homepage_uri: http://www.kernel.org/pub/linux/libs/aio/ name: libaio1:i386 section: libs version: 0.3.109-2ubuntu1 libalgorithm-diff-perl: description: module to find differences between files downloads: 2116 homepage_uri: http://search.cpan.org/dist/Algorithm-Diff/ name: libalgorithm-diff-perl section: perl version: 1.19.02-2 libalgorithm-diff-xs-perl: description: module to find differences between files (XS accelerated) downloads: 1639 homepage_uri: http://search.cpan.org/dist/Algorithm-Diff-XS/ name: libalgorithm-diff-xs-perl section: perl version: 0.04-2build2 libalgorithm-merge-perl: description: Perl module for three-way merge of textual data downloads: 1540 homepage_uri: http://search.cpan.org/dist/Algorithm-Merge/ name: libalgorithm-merge-perl section: perl version: 0.08-2 libamd2.2.0: description: approximate minimum degree ordering library for sparse matrices downloads: 1188 homepage_uri: http://www.cise.ufl.edu/research/sparse/SuiteSparse/ name: libamd2.2.0 section: universe/libs version: 1:3.4.0-2ubuntu3 libantlr-java: description: language tool for constructing recognizers, compilers etc (java library) downloads: 1621 homepage_uri: http://www.antlr2.org/ name: libantlr-java section: devel version: 2.7.7+dfsg-3 libao-common: description: Cross Platform Audio Output Library (Common files) downloads: 20028 homepage_uri: http://www.xiph.org/ao/ name: libao-common section: libs version: 1.1.0-1ubuntu2 libao4:i386: description: Cross Platform Audio Output Library downloads: 0 homepage_uri: http://www.xiph.org/ao/ name: libao4:i386 section: libs version: 1.1.0-1ubuntu2 libapache-pom-java: description: Maven metadata for all Apache Software projects downloads: 414 homepage_uri: http://maven.apache.org/pom/asf/ name: libapache-pom-java section: java version: 10-2 libapparmor1: description: changehat AppArmor library downloads: 23 homepage_uri: http://apparmor.net/ name: libapparmor1 section: libs version: 2.7.102-0ubuntu3.1 libappindicator1: description: Application Indicators downloads: 92 homepage_uri: https://launchpad.net/libappindicator name: libappindicator1 section: libs version: 0.4.92-0ubuntu1 libappindicator3-1: description: Application Indicators downloads: 23 homepage_uri: https://launchpad.net/libappindicator name: libappindicator3-1 section: libs version: 0.4.92-0ubuntu1 libapr1: description: Apache Portable Runtime Library downloads: 69515 homepage_uri: http://apr.apache.org/ name: libapr1 section: libs version: 1.4.6-1 libaprutil1: description: Apache Portable Runtime Utility Library downloads: 63932 homepage_uri: http://apr.apache.org/ name: libaprutil1 section: libs version: 1.3.12+dfsg-3 libaprutil1-dbd-sqlite3: description: Apache Portable Runtime Utility Library - SQLite3 Driver downloads: 50226 homepage_uri: http://apr.apache.org/ name: libaprutil1-dbd-sqlite3 section: libs version: 1.3.12+dfsg-3 libaprutil1-ldap: description: Apache Portable Runtime Utility Library - LDAP Driver downloads: 51271 homepage_uri: http://apr.apache.org/ name: libaprutil1-ldap section: libs version: 1.3.12+dfsg-3 libapt-inst1.4: description: deb package format runtime library downloads: 3 homepage_uri: '' name: libapt-inst1.4 section: admin version: 0.8.16~exp12ubuntu10.2 libapt-pkg-perl: description: Perl interface to libapt-pkg downloads: 12808 homepage_uri: '' name: libapt-pkg-perl section: perl version: 0.1.25build2 libapt-pkg4.12: description: package managment runtime library downloads: 15503 homepage_uri: '' name: libapt-pkg4.12 section: admin version: 0.8.16~exp12ubuntu10.2 libarchive12: description: Multi-format archive and compression library (shared library) downloads: 12762 homepage_uri: http://libarchive.github.com/ name: libarchive12 section: libs version: 3.0.3-6ubuntu1 libarpack2: description: Fortran77 subroutines to solve large scale eigenvalue problems downloads: 53 homepage_uri: http://forge.scilab.org/index.php/p/arpack-ng/ name: libarpack2 section: universe/libs version: 3.0.2-3 libart-2.0-2: description: Library of functions for 2D graphics - runtime files downloads: 35944 homepage_uri: '' name: libart-2.0-2 section: libs version: 2.3.21-1 libart-2.0-dev: description: Library of functions for 2D graphics - development files downloads: 403 homepage_uri: '' name: libart-2.0-dev section: libdevel version: 2.3.21-1 libasm3-java: description: Java bytecode manipulation framework downloads: 1474 homepage_uri: http://asm.objectweb.org/ name: libasm3-java section: devel version: 3.3.2-1 libasn1-8-heimdal: description: Heimdal Kerberos - ASN.1 library downloads: 470 homepage_uri: http://www.h5l.org/ name: libasn1-8-heimdal section: libs version: 1.6~git20120311.dfsg.1-2 libasn1-8-heimdal:i386: description: Heimdal Kerberos - ASN.1 library downloads: 0 homepage_uri: http://www.h5l.org/ name: libasn1-8-heimdal:i386 section: libs version: 1.6~git20120311.dfsg.1-2 libasound2: description: shared library for ALSA applications downloads: 53087 homepage_uri: http://www.alsa-project.org/ name: libasound2 section: libs version: 1.0.25-1ubuntu10.1 libasound2-plugins: description: ALSA library additional plugins downloads: 7008 homepage_uri: http://www.alsa-project.org/ name: libasound2-plugins section: libs version: 1.0.25-1ubuntu1 libasound2-plugins:i386: description: ALSA library additional plugins downloads: 0 homepage_uri: http://www.alsa-project.org/ name: libasound2-plugins:i386 section: libs version: 1.0.25-1ubuntu1 libasound2:i386: description: shared library for ALSA applications downloads: 0 homepage_uri: http://www.alsa-project.org/ name: libasound2:i386 section: libs version: 1.0.25-1ubuntu10.1 libaspell-dev: description: Development files for applications with GNU Aspell support downloads: 124 homepage_uri: http://aspell.net/ name: libaspell-dev section: libdevel version: 0.60.7~20110707-1 libaspell15: description: GNU Aspell spell-checker runtime library downloads: 43298 homepage_uri: http://aspell.net/ name: libaspell15 section: libs version: 0.60.7~20110707-1 libasyncns0: description: Asynchronous name service query library downloads: 30522 homepage_uri: http://0pointer.de/lennart/projects/libasyncns/ name: libasyncns0 section: libs version: 0.8-4 libasyncns0:i386: description: Asynchronous name service query library downloads: 0 homepage_uri: http://0pointer.de/lennart/projects/libasyncns/ name: libasyncns0:i386 section: libs version: 0.8-4 libatasmart4: description: ATA S.M.A.R.T. reading and parsing library downloads: 46706 homepage_uri: http://0pointer.de/blog/projects/being-smart.html name: libatasmart4 section: libs version: 0.18-3 libatk-adaptor: description: at-spi2-atk downloads: 490 homepage_uri: http://live.gnome.org/Accessibility/GNOME3 name: libatk-adaptor section: misc version: 2.4.0-1ubuntu2 libatk-adaptor-schemas: description: at-spi2-atk GSettings schemas downloads: 0 homepage_uri: http://live.gnome.org/Accessibility/GNOME3 name: libatk-adaptor-schemas section: misc version: 2.4.0-1ubuntu2 libatk-wrapper-java: description: An ATK implementation for Java using JNI downloads: 693 homepage_uri: http://ftp.gnome.org/pub/GNOME/sources/java-atk-wrapper/ name: libatk-wrapper-java section: java version: 0.30.4-0ubuntu2 libatk-wrapper-java-jni: description: An ATK implementation for Java using JNI (jni bindings) downloads: 692 homepage_uri: http://ftp.gnome.org/pub/GNOME/sources/java-atk-wrapper/ name: libatk-wrapper-java-jni section: java version: 0.30.4-0ubuntu2 libatk1.0-0: description: ATK accessibility toolkit downloads: 53350 homepage_uri: http://www.gtk.org/ name: libatk1.0-0 section: libs version: 2.4.0-0ubuntu1 libatk1.0-0:i386: description: ATK accessibility toolkit downloads: 0 homepage_uri: http://www.gtk.org/ name: libatk1.0-0:i386 section: libs version: 2.4.0-0ubuntu1 libatk1.0-data: description: Common files for the ATK accessibility toolkit downloads: 45828 homepage_uri: http://www.gtk.org/ name: libatk1.0-data section: misc version: 2.4.0-0ubuntu1 libatk1.0-dev: description: Development files for the ATK accessibility toolkit downloads: 3161 homepage_uri: http://www.gtk.org/ name: libatk1.0-dev section: libdevel version: 2.4.0-0ubuntu1 libatkmm-1.6-1: description: C++ wrappers for ATK accessibility toolkit (shared libraries) downloads: 6986 homepage_uri: http://www.gtkmm.org/ name: libatkmm-1.6-1 section: libs version: 2.22.6-1ubuntu1 libatlas-base-dev: description: Automatically Tuned Linear Algebra Software, generic static downloads: 737 homepage_uri: http://math-atlas.sourceforge.net/ name: libatlas-base-dev section: universe/devel version: 3.8.4-3build1 libatlas-dev: description: Automatically Tuned Linear Algebra Software, C header files downloads: 713 homepage_uri: http://math-atlas.sourceforge.net/ name: libatlas-dev section: universe/libdevel version: 3.8.4-3build1 libatlas3gf-base: description: Automatically Tuned Linear Algebra Software, generic shared downloads: 2645 homepage_uri: http://math-atlas.sourceforge.net/ name: libatlas3gf-base section: universe/libs version: 3.8.4-3build1 libatspi2.0-0: description: Assistive Technology Service Provider Interface - shared library downloads: 654 homepage_uri: http://live.gnome.org/Accessibility/GNOME3 name: libatspi2.0-0 section: misc version: 2.4.2-0ubuntu0.1 libattr1: description: Extended attribute shared library downloads: 65923 homepage_uri: http://savannah.nongnu.org/projects/attr/ name: libattr1 section: libs version: 1:2.4.46-5ubuntu1 libattr1:i386: description: Extended attribute shared library downloads: 0 homepage_uri: http://savannah.nongnu.org/projects/attr/ name: libattr1:i386 section: libs version: 1:2.4.46-5ubuntu1 libaudio2: description: Network Audio System - shared libraries downloads: 23866 homepage_uri: '' name: libaudio2 section: libs version: 1.9.3-4 libaudio2:i386: description: Network Audio System - shared libraries downloads: 0 homepage_uri: '' name: libaudio2:i386 section: libs version: 1.9.3-4 libaudiofile1:i386: description: Open-source version of SGI's audiofile library downloads: 0 homepage_uri: http://audiofile.68k.org/ name: libaudiofile1:i386 section: libs version: 0.3.3-2 libavahi-client-dev: description: Development files for the Avahi client library downloads: 1442 homepage_uri: http://avahi.org/ name: libavahi-client-dev section: net version: 0.6.30-5ubuntu2 libavahi-client3: description: Avahi client library downloads: 52737 homepage_uri: http://avahi.org/ name: libavahi-client3 section: libs version: 0.6.30-5ubuntu2 libavahi-client3:i386: description: Avahi client library downloads: 0 homepage_uri: http://avahi.org/ name: libavahi-client3:i386 section: libs version: 0.6.30-5ubuntu2 libavahi-common-data: description: Avahi common data files downloads: 34905 homepage_uri: http://avahi.org/ name: libavahi-common-data section: libs version: 0.6.30-5ubuntu2 libavahi-common-data:i386: description: Avahi common data files downloads: 0 homepage_uri: http://avahi.org/ name: libavahi-common-data:i386 section: libs version: 0.6.30-5ubuntu2 libavahi-common-dev: description: Development files for the Avahi common library downloads: 1321 homepage_uri: http://avahi.org/ name: libavahi-common-dev section: net version: 0.6.30-5ubuntu2 libavahi-common3: description: Avahi common library downloads: 55373 homepage_uri: http://avahi.org/ name: libavahi-common3 section: libs version: 0.6.30-5ubuntu2 libavahi-common3:i386: description: Avahi common library downloads: 0 homepage_uri: http://avahi.org/ name: libavahi-common3:i386 section: libs version: 0.6.30-5ubuntu2 libavahi-core7: description: Avahi's embeddable mDNS/DNS-SD library downloads: 43012 homepage_uri: http://avahi.org/ name: libavahi-core7 section: libs version: 0.6.30-5ubuntu2 libavahi-glib-dev: description: Development headers for the Avahi glib integration library downloads: 371 homepage_uri: http://avahi.org/ name: libavahi-glib-dev section: net version: 0.6.30-5ubuntu2 libavahi-glib1: description: Avahi glib integration library downloads: 40876 homepage_uri: http://avahi.org/ name: libavahi-glib1 section: libs version: 0.6.30-5ubuntu2 libavahi-gobject0: description: Avahi GObject library downloads: 16653 homepage_uri: http://avahi.org/ name: libavahi-gobject0 section: libs version: 0.6.30-5ubuntu2 libavahi-ui-gtk3-0: description: Avahi GTK+ User interface library for GTK3 downloads: 7410 homepage_uri: http://avahi.org/ name: libavahi-ui-gtk3-0 section: libs version: 0.6.30-5ubuntu2 libavalon-framework-java: description: Common framework for Java server applications downloads: 669 homepage_uri: http://excalibur.apache.org/ name: libavalon-framework-java section: libs version: 4.2.0-8 libavc1394-0: description: control IEEE 1394 audio/video devices downloads: 33336 homepage_uri: '' name: libavc1394-0 section: libs version: 0.5.3-1ubuntu2 libavc1394-0:i386: description: control IEEE 1394 audio/video devices downloads: 0 homepage_uri: '' name: libavc1394-0:i386 section: libs version: 0.5.3-1ubuntu2 libavcodec53: description: Libav codec library downloads: 14915 homepage_uri: http://libav.org/ name: libavcodec53 section: libs version: 4:0.8.1-0ubuntu1 libavformat53: description: Libav file format library downloads: 3444 homepage_uri: http://libav.org/ name: libavformat53 section: libs version: 4:0.8.1-0ubuntu1 libavutil51: description: Libav utility library downloads: 10200 homepage_uri: http://libav.org/ name: libavutil51 section: libs version: 4:0.8.1-0ubuntu1 libbackport-util-concurrent-java: description: backport of java.util.concurrent to Java 1.4 downloads: 246 homepage_uri: http://backport-jsr166.sourceforge.net/ name: libbackport-util-concurrent-java section: libs version: 3.1-3 libbamf0: description: Window matching library - shared library downloads: 1 homepage_uri: https://launchpad.net/bamf name: libbamf0 section: libs version: 0.2.118-0ubuntu0.1 libbamf3-0: description: Window matching library - shared library downloads: 35 homepage_uri: https://launchpad.net/bamf name: libbamf3-0 section: libs version: 0.2.118-0ubuntu0.1 libbatik-java: description: xml.apache.org SVG Library downloads: 692 homepage_uri: http://xml.apache.org/batik/ name: libbatik-java section: libs version: 1.7.ubuntu-8ubuntu1 libbind9-80: description: BIND9 Shared Library used by BIND downloads: 15273 homepage_uri: '' name: libbind9-80 section: libs version: 1:9.8.1.dfsg.P1-4ubuntu0.1 libbiojava-java: description: Java API to biological data and applications downloads: 19 homepage_uri: http://www.biojava.org name: libbiojava-java section: universe/science version: 1:1.7.1-1 libbiojava-java-demos: description: Example programs for BioJava downloads: 0 homepage_uri: http://www.biojava.org name: libbiojava-java-demos section: universe/science version: 1:1.7.1-1 libbiojava1.7-java: description: Java API to biological data and applications downloads: 4 homepage_uri: http://www.biojava.org name: libbiojava1.7-java section: universe/science version: 1:1.7.1-1 libblas-dev: description: Basic Linear Algebra Subroutines 3, static library downloads: 1678 homepage_uri: http://www.netlib.org/blas/ name: libblas-dev section: libdevel version: 1.2.20110419-2ubuntu1 libblas3gf: description: Basic Linear Algebra Reference implementations, shared library downloads: 31268 homepage_uri: http://www.netlib.org/blas/ name: libblas3gf section: libs version: 1.2.20110419-2ubuntu1 libblkid1: description: block device id library downloads: 39432 homepage_uri: http://userweb.kernel.org/~kzak/util-linux/ name: libblkid1 section: libs version: 2.20.1-1ubuntu3 libbluetooth3: description: Library to use the BlueZ Linux Bluetooth stack downloads: 37668 homepage_uri: http://www.bluez.org name: libbluetooth3 section: libs version: 4.98-2ubuntu7 libbonobo2-0: description: Bonobo CORBA interfaces library downloads: 35551 homepage_uri: '' name: libbonobo2-0 section: libs version: 2.32.1-0ubuntu1 libbonobo2-common: description: Bonobo CORBA interfaces library -- support files downloads: 33480 homepage_uri: '' name: libbonobo2-common section: devel version: 2.32.1-0ubuntu1 libbonobo2-dev: description: Bonobo CORBA interfaces library -- development files downloads: 196 homepage_uri: '' name: libbonobo2-dev section: libdevel version: 2.32.1-0ubuntu1 libbonoboui2-0: description: The Bonobo UI library downloads: 30699 homepage_uri: '' name: libbonoboui2-0 section: libs version: 2.24.5-0ubuntu1 libbonoboui2-common: description: The Bonobo UI library -- common files downloads: 10 homepage_uri: '' name: libbonoboui2-common section: libs version: 2.24.5-0ubuntu1 libbonoboui2-dev: description: The Bonobo UI library - development files downloads: 279 homepage_uri: '' name: libbonoboui2-dev section: libdevel version: 2.24.5-0ubuntu1 libboost-all-dev: description: Boost C++ Libraries development files (ALL, default version) downloads: 0 homepage_uri: http://www.boost.org/ name: libboost-all-dev section: universe/libdevel version: 1.48.0.2 libboost-date-time-dev: description: date-time libraries based on generic programming (default version) downloads: 31 homepage_uri: http://www.boost.org/libs/date_time/ name: libboost-date-time-dev section: universe/libdevel version: 1.48.0.2 libboost-date-time1.46-dev: description: set of date-time libraries based on generic programming concepts downloads: 7 homepage_uri: http://www.boost.org/libs/date_time/ name: libboost-date-time1.46-dev section: universe/libdevel version: 1.46.1-7ubuntu3 libboost-date-time1.46.1: description: set of date-time libraries based on generic programming concepts downloads: 0 homepage_uri: http://www.boost.org/libs/date_time/ name: libboost-date-time1.46.1 section: universe/libs version: 1.46.1-7ubuntu3 libboost-dev: description: Boost C++ Libraries development files (default version) downloads: 109 homepage_uri: http://www.boost.org/ name: libboost-dev section: libdevel version: 1.48.0.2 libboost-filesystem-dev: description: filesystem operations in C++ (default version) downloads: 22 homepage_uri: http://boost.org/libs/filesystem/ name: libboost-filesystem-dev section: universe/libdevel version: 1.48.0.2 libboost-filesystem1.46-dev: description: filesystem operations (portable paths, iteration over directories, etc) in C++ downloads: 5 homepage_uri: http://boost.org/libs/filesystem/ name: libboost-filesystem1.46-dev section: universe/libdevel version: 1.46.1-7ubuntu3 libboost-filesystem1.46.1: description: filesystem operations (portable paths, iteration over directories, etc) in C++ downloads: 14 homepage_uri: http://boost.org/libs/filesystem/ name: libboost-filesystem1.46.1 section: universe/libs version: 1.46.1-7ubuntu3 libboost-graph-dev: description: generic graph components and algorithms in C++ (default version) downloads: 8 homepage_uri: http://www.boost.org/libs/graph/ name: libboost-graph-dev section: universe/libdevel version: 1.48.0.2 libboost-graph-parallel-dev: description: generic graph components and algorithms in C++ (default version) downloads: 0 homepage_uri: http://www.boost.org/libs/graph_parallel/ name: libboost-graph-parallel-dev section: universe/libdevel version: 1.48.0.2 libboost-graph-parallel1.46-dev: description: generic graph components and algorithms in C++ downloads: 4 homepage_uri: http://www.boost.org/libs/graph_parallel/ name: libboost-graph-parallel1.46-dev section: universe/libdevel version: 1.46.1-7ubuntu3 libboost-graph-parallel1.46.1: description: generic graph components and algorithms in C++ downloads: 0 homepage_uri: http://www.boost.org/libs/graph_parallel/ name: libboost-graph-parallel1.46.1 section: universe/libs version: 1.46.1-7ubuntu3 libboost-graph1.46-dev: description: generic graph components and algorithms in C++ downloads: 6 homepage_uri: http://www.boost.org/libs/graph/ name: libboost-graph1.46-dev section: libdevel version: 1.46.1-7ubuntu3 libboost-graph1.46.1: description: generic graph components and algorithms in C++ downloads: 0 homepage_uri: http://www.boost.org/libs/graph/ name: libboost-graph1.46.1 section: libs version: 1.46.1-7ubuntu3 libboost-iostreams-dev: description: Boost.Iostreams Library development files (default version) downloads: 12 homepage_uri: http://www.boost.org/libs/iostreams/ name: libboost-iostreams-dev section: libdevel version: 1.48.0.2 libboost-iostreams1.46-dev: description: Boost.Iostreams Library development files downloads: 5 homepage_uri: http://www.boost.org/libs/iostreams/ name: libboost-iostreams1.46-dev section: libdevel version: 1.46.1-7ubuntu3 libboost-iostreams1.46.1: description: Boost.Iostreams Library downloads: 0 homepage_uri: http://www.boost.org/libs/iostreams/ name: libboost-iostreams1.46.1 section: libs version: 1.46.1-7ubuntu3 libboost-math-dev: description: Boost.Math Library development files (default version) downloads: 1 homepage_uri: http://www.boost.org/libs/math/ name: libboost-math-dev section: universe/libdevel version: 1.48.0.2 libboost-math1.46-dev: description: Boost.Math Library development files downloads: 5 homepage_uri: http://www.boost.org/libs/math/ name: libboost-math1.46-dev section: universe/libdevel version: 1.46.1-7ubuntu3 libboost-math1.46.1: description: Boost.Math Library downloads: 0 homepage_uri: http://www.boost.org/libs/math/ name: libboost-math1.46.1 section: universe/libs version: 1.46.1-7ubuntu3 libboost-mpi-dev: description: C++ interface to the Message Passing Interface (MPI) (default version) downloads: 1 homepage_uri: http://www.boost.org/doc/html/mpi.html name: libboost-mpi-dev section: universe/libdevel version: 1.48.0.2 libboost-mpi-python-dev: description: Python interface to the Message Passing Interface (MPI) (default version) downloads: 0 homepage_uri: http://www.boost.org/doc/html/mpi.html name: libboost-mpi-python-dev section: universe/libdevel version: 1.48.0.2 libboost-mpi1.46-dev: description: C++ interface to the Message Passing Interface (MPI) downloads: 5 homepage_uri: http://www.boost.org/doc/html/mpi.html name: libboost-mpi1.46-dev section: universe/libdevel version: 1.46.1-7ubuntu3 libboost-mpi1.46.1: description: C++ interface to the Message Passing Interface (MPI) downloads: 0 homepage_uri: http://www.boost.org/doc/html/mpi.html name: libboost-mpi1.46.1 section: universe/libs version: 1.46.1-7ubuntu3 libboost-program-options-dev: description: program options library for C++ (default version) downloads: 56 homepage_uri: http://www.boost.org/libs/program_options/ name: libboost-program-options-dev section: libdevel version: 1.48.0.2 libboost-program-options1.46-dev: description: program options library for C++ downloads: 5 homepage_uri: http://www.boost.org/libs/program_options/ name: libboost-program-options1.46-dev section: libdevel version: 1.46.1-7ubuntu3 libboost-program-options1.46.1: description: program options library for C++ downloads: 0 homepage_uri: http://www.boost.org/libs/program_options/ name: libboost-program-options1.46.1 section: libs version: 1.46.1-7ubuntu3 libboost-python-dev: description: Boost.Python Library development files (default version) downloads: 17 homepage_uri: http://www.boost.org/libs/python/ name: libboost-python-dev section: libdevel version: 1.48.0.2 libboost-python1.46-dev: description: Boost.Python Library development files downloads: 22 homepage_uri: http://www.boost.org/libs/python/ name: libboost-python1.46-dev section: libdevel version: 1.46.1-7ubuntu3 libboost-python1.46.1: description: Boost.Python Library downloads: 0 homepage_uri: http://www.boost.org/libs/python/ name: libboost-python1.46.1 section: python version: 1.46.1-7ubuntu3 libboost-regex-dev: description: regular expression library for C++ (default version) downloads: 15 homepage_uri: http://www.boost.org/libs/regex/ name: libboost-regex-dev section: universe/libdevel version: 1.48.0.2 libboost-regex1.46-dev: description: regular expression library for C++ downloads: 5 homepage_uri: http://www.boost.org/libs/regex/ name: libboost-regex1.46-dev section: libdevel version: 1.46.1-7ubuntu3 libboost-regex1.46.1: description: regular expression library for C++ downloads: 0 homepage_uri: http://www.boost.org/libs/regex/ name: libboost-regex1.46.1 section: libs version: 1.46.1-7ubuntu3 libboost-serialization-dev: description: serialization library for C++ (default version) downloads: 14 homepage_uri: http://www.boost.org/libs/serialization/ name: libboost-serialization-dev section: libdevel version: 1.48.0.2 libboost-serialization1.46-dev: description: serialization library for C++ downloads: 9 homepage_uri: http://www.boost.org/libs/serialization/ name: libboost-serialization1.46-dev section: libdevel version: 1.46.1-7ubuntu3 libboost-serialization1.46.1: description: serialization library for C++ downloads: 0 homepage_uri: http://www.boost.org/libs/serialization/ name: libboost-serialization1.46.1 section: libs version: 1.46.1-7ubuntu3 libboost-signals-dev: description: managed signals and slots library for C++ (default version) downloads: 12 homepage_uri: http://www.boost.org/libs/signals/ name: libboost-signals-dev section: universe/libdevel version: 1.48.0.2 libboost-signals1.46-dev: description: managed signals and slots library for C++ downloads: 6 homepage_uri: http://www.boost.org/libs/signals/ name: libboost-signals1.46-dev section: universe/libdevel version: 1.46.1-7ubuntu3 libboost-signals1.46.1: description: managed signals and slots library for C++ downloads: 0 homepage_uri: http://www.boost.org/libs/signals/ name: libboost-signals1.46.1 section: universe/libs version: 1.46.1-7ubuntu3 libboost-system-dev: description: Operating system (e.g. diagnostics support) library (default version) downloads: 6 homepage_uri: http://www.boost.org/libs/system/ name: libboost-system-dev section: universe/libdevel version: 1.48.0.2 libboost-system1.46-dev: description: Operating system (e.g. diagnostics support) library downloads: 6 homepage_uri: http://www.boost.org/libs/system/ name: libboost-system1.46-dev section: universe/libdevel version: 1.46.1-7ubuntu3 libboost-system1.46.1: description: Operating system (e.g. diagnostics support) library downloads: 14 homepage_uri: http://www.boost.org/libs/system/ name: libboost-system1.46.1 section: universe/libs version: 1.46.1-7ubuntu3 libboost-test-dev: description: components for writing and executing test suites (default version) downloads: 7 homepage_uri: http://www.boost.org/libs/test/ name: libboost-test-dev section: libdevel version: 1.48.0.2 libboost-test1.46-dev: description: components for writing and executing test suites downloads: 7 homepage_uri: http://www.boost.org/libs/test/ name: libboost-test1.46-dev section: libdevel version: 1.46.1-7ubuntu3 libboost-test1.46.1: description: components for writing and executing test suites downloads: 0 homepage_uri: http://www.boost.org/libs/test/ name: libboost-test1.46.1 section: libs version: 1.46.1-7ubuntu3 libboost-thread-dev: description: portable C++ multi-threading (default version) downloads: 82 homepage_uri: http://www.boost.org/libs/thread/ name: libboost-thread-dev section: universe/libdevel version: 1.48.0.2 libboost-thread1.46-dev: description: portable C++ multi-threading downloads: 7 homepage_uri: http://www.boost.org/libs/thread/ name: libboost-thread1.46-dev section: universe/libdevel version: 1.46.1-7ubuntu3 libboost-thread1.46.1: description: portable C++ multi-threading downloads: 1 homepage_uri: http://www.boost.org/libs/thread/ name: libboost-thread1.46.1 section: universe/libs version: 1.46.1-7ubuntu3 libboost-wave-dev: description: C99/C++ preprocessor library (default version) downloads: 7 homepage_uri: http://www.boost.org/libs/wave/ name: libboost-wave-dev section: universe/libdevel version: 1.48.0.2 libboost-wave1.46-dev: description: C99/C++ preprocessor library downloads: 5 homepage_uri: http://www.boost.org/libs/wave/ name: libboost-wave1.46-dev section: universe/libdevel version: 1.46.1-7ubuntu3 libboost-wave1.46.1: description: C99/C++ preprocessor library downloads: 0 homepage_uri: http://www.boost.org/libs/wave/ name: libboost-wave1.46.1 section: universe/libs version: 1.46.1-7ubuntu3 libboost1.46-dev: description: Boost C++ Libraries development files downloads: 39 homepage_uri: http://www.boost.org/ name: libboost1.46-dev section: libdevel version: 1.46.1-7ubuntu3 libbrasero-media3-1: description: CD/DVD burning library for GNOME - runtime downloads: 6990 homepage_uri: http://www.gnome.org/projects/brasero/ name: libbrasero-media3-1 section: libs version: 3.4.1-0ubuntu1 libbrlapi0.5: description: braille display access via BRLTTY - shared library downloads: 9379 homepage_uri: http://mielke.cc/brltty/ name: libbrlapi0.5 section: libs version: 4.3-1ubuntu5 libbsd-dev: description: utility functions from BSD systems - development files downloads: 391 homepage_uri: http://libbsd.freedesktop.org/ name: libbsd-dev section: libdevel version: 0.3.0-2 libbsd0: description: utility functions from BSD systems - shared library downloads: 39621 homepage_uri: http://libbsd.freedesktop.org/ name: libbsd0 section: libs version: 0.3.0-2 libbsf-java: description: Bean Scripting Framework to support scripting languages in Java downloads: 725 homepage_uri: http://jakarta.apache.org/bsf/ name: libbsf-java section: libs version: 1:2.4.0-5 libburn4: description: library to provide CD/DVD writing functions downloads: 12911 homepage_uri: http://libburnia-project.org name: libburn4 section: libs version: 1.1.8-1 libbytecode-java: description: Java bytecode manipulation library downloads: 19 homepage_uri: '' name: libbytecode-java section: universe/devel version: 0.92.svn.20090106-1 libbz2-1.0: description: high-quality block-sorting file compressor library - runtime downloads: 104583 homepage_uri: http://www.bzip.org/ name: libbz2-1.0 section: libs version: 1.0.6-1 libbz2-1.0:i386: description: high-quality block-sorting file compressor library - runtime downloads: 0 homepage_uri: http://www.bzip.org/ name: libbz2-1.0:i386 section: libs version: 1.0.6-1 libbz2-dev: description: high-quality block-sorting file compressor library - development downloads: 1473 homepage_uri: http://www.bzip.org/ name: libbz2-dev section: libdevel version: 1.0.6-1 libc-bin: description: 'Embedded GNU C Library: Binaries' downloads: 93979 homepage_uri: http://www.eglibc.org name: libc-bin section: libs version: 2.15-0ubuntu10 libc-dev-bin: description: 'Embedded GNU C Library: Development binaries' downloads: 33548 homepage_uri: http://www.eglibc.org name: libc-dev-bin section: libdevel version: 2.15-0ubuntu10 libc6: description: 'Embedded GNU C Library: Shared libraries' downloads: 117876 homepage_uri: http://www.eglibc.org name: libc6 section: libs version: 2.15-0ubuntu10 libc6-dev: description: 'Embedded GNU C Library: Development Libraries and Header Files' downloads: 52134 homepage_uri: http://www.eglibc.org name: libc6-dev section: libdevel version: 2.15-0ubuntu10 libc6:i386: description: 'Embedded GNU C Library: Shared libraries' downloads: 0 homepage_uri: http://www.eglibc.org name: libc6:i386 section: libs version: 2.15-0ubuntu10 libcaca0: description: colour ASCII art library downloads: 32988 homepage_uri: http://caca.zoy.org/wiki/libcaca name: libcaca0 section: libs version: 0.99.beta17-2.1ubuntu2 libcaca0:i386: description: colour ASCII art library downloads: 0 homepage_uri: http://caca.zoy.org/wiki/libcaca name: libcaca0:i386 section: libs version: 0.99.beta17-2.1ubuntu2 libcache-cache-perl: description: Managed caches of persistent information downloads: 700 homepage_uri: http://search.cpan.org/dist/Cache-Cache/ name: libcache-cache-perl section: universe/perl version: 1.06-2 libcairo-gobject2: description: The Cairo 2D vector graphics library (GObject library) downloads: 16269 homepage_uri: http://cairographics.org/ name: libcairo-gobject2 section: libs version: 1.10.2-6.1ubuntu3 libcairo-gobject2:i386: description: The Cairo 2D vector graphics library (GObject library) downloads: 0 homepage_uri: http://cairographics.org/ name: libcairo-gobject2:i386 section: libs version: 1.10.2-6.1ubuntu3 libcairo-perl: description: Perl interface to the Cairo graphics library downloads: 17618 homepage_uri: http://gtk2-perl.sourceforge.net/ name: libcairo-perl section: perl version: 1.081-1build2 libcairo-script-interpreter2: description: The Cairo 2D vector graphics library (script interpreter) downloads: 1611 homepage_uri: http://cairographics.org/ name: libcairo-script-interpreter2 section: libs version: 1.10.2-6.1ubuntu3 libcairo2: description: The Cairo 2D vector graphics library downloads: 59418 homepage_uri: http://cairographics.org/ name: libcairo2 section: libs version: 1.10.2-6.1ubuntu3 libcairo2-dev: description: Development files for the Cairo 2D graphics library downloads: 3292 homepage_uri: http://cairographics.org/ name: libcairo2-dev section: libdevel version: 1.10.2-6.1ubuntu3 libcairo2:i386: description: The Cairo 2D vector graphics library downloads: 0 homepage_uri: http://cairographics.org/ name: libcairo2:i386 section: libs version: 1.10.2-6.1ubuntu3 libcairomm-1.0-1: description: C++ wrappers for Cairo (shared libraries) downloads: 18118 homepage_uri: http://cairographics.org/cairomm/ name: libcairomm-1.0-1 section: libs version: 1.10.0-1ubuntu1 libcamel-1.2-29: description: Evolution MIME message handling library downloads: 0 homepage_uri: http://www.gnome.org/projects/evolution/ name: libcamel-1.2-29 section: libs version: 3.2.3-0ubuntu7 libcanberra-dev: description: simple abstract interface for playing event sounds downloads: 200 homepage_uri: http://0pointer.de/lennart/projects/libcanberra/ name: libcanberra-dev section: libdevel version: 0.28-3ubuntu3 libcanberra-gtk-module: description: translates GTK+ widgets signals to event sounds downloads: 26234 homepage_uri: http://0pointer.de/lennart/projects/libcanberra/ name: libcanberra-gtk-module section: libs version: 0.28-3ubuntu3 libcanberra-gtk-module:i386: description: translates GTK+ widgets signals to event sounds downloads: 0 homepage_uri: http://0pointer.de/lennart/projects/libcanberra/ name: libcanberra-gtk-module:i386 section: libs version: 0.28-3ubuntu3 libcanberra-gtk0: description: GTK+ helper for playing widget event sounds with libcanberra downloads: 36287 homepage_uri: http://0pointer.de/lennart/projects/libcanberra/ name: libcanberra-gtk0 section: libs version: 0.28-3ubuntu3 libcanberra-gtk0:i386: description: GTK+ helper for playing widget event sounds with libcanberra downloads: 0 homepage_uri: http://0pointer.de/lennart/projects/libcanberra/ name: libcanberra-gtk0:i386 section: libs version: 0.28-3ubuntu3 libcanberra-gtk3-0: description: GTK+ 3.0 helper for playing widget event sounds with libcanberra downloads: 11593 homepage_uri: http://0pointer.de/lennart/projects/libcanberra/ name: libcanberra-gtk3-0 section: libs version: 0.28-3ubuntu3 libcanberra-gtk3-module: description: translates GTK3 widgets signals to event sounds downloads: 2502 homepage_uri: http://0pointer.de/lennart/projects/libcanberra/ name: libcanberra-gtk3-module section: debug version: 0.28-3ubuntu3 libcanberra-pulse: description: PulseAudio backend for libcanberra downloads: 2027 homepage_uri: http://0pointer.de/lennart/projects/libcanberra/ name: libcanberra-pulse section: libdevel version: 0.28-3ubuntu3 libcanberra0: description: simple abstract interface for playing event sounds downloads: 41526 homepage_uri: http://0pointer.de/lennart/projects/libcanberra/ name: libcanberra0 section: libs version: 0.28-3ubuntu3 libcanberra0:i386: description: simple abstract interface for playing event sounds downloads: 0 homepage_uri: http://0pointer.de/lennart/projects/libcanberra/ name: libcanberra0:i386 section: libs version: 0.28-3ubuntu3 libcap-ng0: description: An alternate POSIX capabilities library downloads: 43995 homepage_uri: http://people.redhat.com/sgrubb/libcap-ng name: libcap-ng0 section: libs version: 0.6.6-1ubuntu1 libcap2: description: support for getting/setting POSIX.1e capabilities downloads: 87126 homepage_uri: http://sites.google.com/site/fullycapable/ name: libcap2 section: libs version: 1:2.22-1ubuntu3 libcap2-bin: description: basic utility programs for using capabilities downloads: 21861 homepage_uri: http://sites.google.com/site/fullycapable/ name: libcap2-bin section: utils version: 1:2.22-1ubuntu3 libcap2:i386: description: support for getting/setting POSIX.1e capabilities downloads: 0 homepage_uri: http://sites.google.com/site/fullycapable/ name: libcap2:i386 section: libs version: 1:2.22-1ubuntu3 libcapi20-3:i386: description: libraries for CAPI support downloads: 0 homepage_uri: '' name: libcapi20-3:i386 section: universe/libs version: 1:3.12.20071127-0ubuntu11 libcbf0: description: shared library supporting CBFlib downloads: 284 homepage_uri: http://www.bernstein-plus-sons.com/software/CBF/ name: libcbf0 section: universe/libs version: 0.7.9.1-3build1 libccolamd2.7.1: description: constrained column approximate library for sparse matrices downloads: 0 homepage_uri: http://www.cise.ufl.edu/research/sparse/SuiteSparse/ name: libccolamd2.7.1 section: universe/libs version: 1:3.4.0-2ubuntu3 libcdio-cdda1: description: library to read and control digital audio CDs downloads: 6433 homepage_uri: http://www.gnu.org/software/libcdio/ name: libcdio-cdda1 section: libs version: 0.83-1 libcdio-paranoia1: description: library to read digital audio CDs with error correction downloads: 6427 homepage_uri: http://www.gnu.org/software/libcdio/ name: libcdio-paranoia1 section: libs version: 0.83-1 libcdio13: description: library to read and control CD-ROM downloads: 6890 homepage_uri: http://www.gnu.org/software/libcdio/ name: libcdio13 section: libs version: 0.83-1 libcdparanoia0: description: audio extraction tool for sampling CDs (library) downloads: 27393 homepage_uri: http://www.xiph.org/paranoia/ name: libcdparanoia0 section: sound version: 3.10.2+debian-10ubuntu1 libcdparanoia0:i386: description: audio extraction tool for sampling CDs (library) downloads: 0 homepage_uri: http://www.xiph.org/paranoia/ name: libcdparanoia0:i386 section: sound version: 3.10.2+debian-10ubuntu1 libcdt4: description: rich set of graph drawing tools - cdt library downloads: 3832 homepage_uri: http://www.graphviz.org/ name: libcdt4 section: libs version: 2.26.3-10ubuntu1 libcgi-application-perl: description: framework for building reusable web-applications downloads: 37 homepage_uri: http://search.cpan.org/dist/CGI-Application/ name: libcgi-application-perl section: universe/perl version: 4.50-1 libcgraph5: description: rich set of graph drawing tools - cgraph library downloads: 2227 homepage_uri: http://www.graphviz.org/ name: libcgraph5 section: libs version: 2.26.3-10ubuntu1 libcholmod1.7.1: description: sparse Cholesky factorization library for sparse matrices downloads: 9 homepage_uri: http://www.cise.ufl.edu/research/sparse/SuiteSparse/ name: libcholmod1.7.1 section: universe/libs version: 1:3.4.0-2ubuntu3 libck-connector0: description: ConsoleKit libraries downloads: 52459 homepage_uri: http://www.freedesktop.org/wiki/Software/ConsoleKit name: libck-connector0 section: libs version: 0.4.5-2 libclass-accessor-perl: description: Perl module that automatically generates accessors downloads: 3682 homepage_uri: http://search.cpan.org/dist/Class-Accessor/ name: libclass-accessor-perl section: perl version: 0.34-1 libclass-inspector-perl: description: Perl module that provides information about classes downloads: 1859 homepage_uri: http://search.cpan.org/dist/Class-Inspector/ name: libclass-inspector-perl section: perl version: 1.25-1 libclass-isa-perl: description: report the search path for a class's ISA tree downloads: 16211 homepage_uri: http://search.cpan.org/dist/Class-ISA/ name: libclass-isa-perl section: perl version: 0.36-3 libclassworlds-java: description: framework for container developers requiring manipulation of ClassLoaders downloads: 93 homepage_uri: http://classworlds.codehaus.org/ name: libclassworlds-java section: universe/libs version: 1.1-final-5 libclone-perl: description: recursively copy Perl datatypes downloads: 1427 homepage_uri: http://search.cpan.org/dist/Clone/ name: libclone-perl section: perl version: 0.31-1build3 libcmis-0.2-0: description: CMIS protocol client library downloads: 3 homepage_uri: '' name: libcmis-0.2-0 section: libs version: 0.1.0-1 libcolamd2.7.1: description: column approximate minimum degree ordering library for sparse matrices downloads: 4010 homepage_uri: http://www.cise.ufl.edu/research/sparse/SuiteSparse/ name: libcolamd2.7.1 section: universe/libs version: 1:3.4.0-2ubuntu3 libcolord1: description: system service to manage device colour profiles -- runtime downloads: 15526 homepage_uri: http://www.freedesktop.org/software/colord/ name: libcolord1 section: libs version: 0.1.16-2 libcomerr2: description: common error description library downloads: 104955 homepage_uri: http://e2fsprogs.sourceforge.net name: libcomerr2 section: libs version: 1.42-1ubuntu2 libcomerr2:i386: description: common error description library downloads: 0 homepage_uri: http://e2fsprogs.sourceforge.net name: libcomerr2:i386 section: libs version: 1.42-1ubuntu2 libcommon-sense-perl: description: module that implements some sane defaults for Perl programs downloads: 2068 homepage_uri: http://search.cpan.org/dist/common-sense/ name: libcommon-sense-perl section: perl version: 3.4-1 libcommons-beanutils-java: description: utility for manipulating JavaBeans downloads: 92 homepage_uri: http://commons.apache.org/beanutils/ name: libcommons-beanutils-java section: universe/libs version: 1.8.3-2 libcommons-cli-java: description: API for working with the command line arguments and options downloads: 2015 homepage_uri: http://commons.apache.org/cli/ name: libcommons-cli-java section: libs version: 1.2-3 libcommons-codec-java: description: encoder and decoders such as Base64 and hexadecimal codec downloads: 2317 homepage_uri: http://commons.apache.org/codec/ name: libcommons-codec-java section: libs version: 1.5-1 libcommons-collections-java: description: set of abstract data type interfaces and implementations downloads: 92 homepage_uri: http://commons.apache.org/collections/ name: libcommons-collections-java section: libs version: 2.1.1-10 libcommons-collections3-java: description: A set of abstract data type interfaces and implementations downloads: 67 homepage_uri: http://commons.apache.org/collections/ name: libcommons-collections3-java section: libs version: 3.2.1-5 libcommons-compress-java: description: Java API for working with tar, zip and bzip2 files downloads: 94 homepage_uri: http://commons.apache.org/compress/ name: libcommons-compress-java section: libs version: 1.2-1ubuntu1 libcommons-configuration-java: description: Java based library providing a generic configuration interface downloads: 28 homepage_uri: http://commons.apache.org/configuration/ name: libcommons-configuration-java section: universe/libs version: 1.7-1 libcommons-dbcp-java: description: Database Connection Pooling Services downloads: 778 homepage_uri: http://commons.apache.org/dbcp/ name: libcommons-dbcp-java section: libs version: 1.4-1ubuntu1 libcommons-digester-java: description: Rule based XML Java object mapping tool downloads: 19 homepage_uri: http://commons.apache.org/digester/ name: libcommons-digester-java section: universe/libs version: 1.8.1-3 libcommons-el-java: description: Implementation of the JSP2.0 Expression Language interpreter downloads: 9 homepage_uri: http://commons.apache.org/el/ name: libcommons-el-java section: universe/libs version: 1.0-7 libcommons-httpclient-java: description: A Java(TM) library for creating HTTP clients downloads: 2067 homepage_uri: http://hc.apache.org/httpclient-3.x name: libcommons-httpclient-java section: libs version: 3.1-10 libcommons-io-java: description: Common useful IO related classes downloads: 708 homepage_uri: http://commons.apache.org/io/ name: libcommons-io-java section: libs version: 1.4-4 libcommons-jexl-java: description: expression language engine downloads: 15 homepage_uri: http://commons.apache.org/jexl name: libcommons-jexl-java section: universe/libs version: 1.1-3 libcommons-jxpath-java: description: manipulate javabean using XPath syntax downloads: 0 homepage_uri: http://commons.apache.org/jxpath/ name: libcommons-jxpath-java section: universe/libs version: 1.3-5 libcommons-lang-java: description: Extension of the java.lang package downloads: 1741 homepage_uri: http://commons.apache.org/lang/ name: libcommons-lang-java section: libs version: 2.6-3ubuntu1 libcommons-logging-java: description: commmon wrapper interface for several logging APIs downloads: 3368 homepage_uri: http://commons.apache.org/logging name: libcommons-logging-java section: libs version: 1.1.1-9 libcommons-net2-java: description: internet protocol suite Java library downloads: 23 homepage_uri: http://commons.apache.org/net/ name: libcommons-net2-java section: java version: 2.2-1ubuntu1 libcommons-parent-java: description: Maven metadata for Apache Commons project downloads: 1527 homepage_uri: http://commons.apache.org/ name: libcommons-parent-java section: java version: 22-2 libcommons-pool-java: description: pooling implementation for Java objects downloads: 779 homepage_uri: http://commons.apache.org/pool/ name: libcommons-pool-java section: libs version: 1.5.6-1 libcommons-vfs-java: description: Java API for accessing various filesystems downloads: 0 homepage_uri: http://commons.apache.org/vfs/ name: libcommons-vfs-java section: libs version: 2.0-1ubuntu1 libcompizconfig0: description: Settings library for plugins - OpenCompositing Project downloads: 2041 homepage_uri: '' name: libcompizconfig0 section: libs version: 0.9.7.0~bzr428-0ubuntu6 libconfig-inifiles-perl: description: Read .ini-style configuration files downloads: 1804 homepage_uri: http://search.cpan.org/dist/Config-IniFiles/ name: libconfig-inifiles-perl section: perl version: 2.68-1 libconstantine-java: description: platform constants for Java downloads: 87 homepage_uri: http://github.com/wmeissner/jnr-constants/ name: libconstantine-java section: universe/java version: 0.7-3 libconvert-binhex-perl: description: Perl5 module for extracting data from macintosh BinHex files downloads: 3721 homepage_uri: http://search.cpan.org/dist/Convert-BinHex/ name: libconvert-binhex-perl section: perl version: 1.119+pristine-3 libcr0: description: Libraries to Checkpoint and Restart Linux processes downloads: 248 homepage_uri: https://ftg.lbl.gov/CheckpointRestart/CheckpointRestart.shtml name: libcr0 section: universe/libs version: 0.8.2-15ubuntu2.1 libcroco3: description: Cascading Style Sheet (CSS) parsing and manipulation toolkit downloads: 28507 homepage_uri: '' name: libcroco3 section: libs version: 0.6.5-1 libcroco3:i386: description: Cascading Style Sheet (CSS) parsing and manipulation toolkit downloads: 0 homepage_uri: '' name: libcroco3:i386 section: libs version: 0.6.5-1 libcrypt-passwdmd5-perl: description: interoperable MD5-based crypt() for perl downloads: 1454 homepage_uri: http://search.cpan.org/dist/Crypt-PasswdMD5/ name: libcrypt-passwdmd5-perl section: perl version: 1.3-10 libcrypt-ssleay-perl: description: Support for https protocol in LWP downloads: 3613 homepage_uri: http://search.cpan.org/dist/Crypt-SSLeay/ name: libcrypt-ssleay-perl section: universe/perl version: 0.57-2ubuntu1 libcryptsetup4: description: disk encryption support - shared library downloads: 2655 homepage_uri: http://code.google.com/p/cryptsetup/ name: libcryptsetup4 section: libs version: 2:1.4.1-2ubuntu4 libcups2: description: Common UNIX Printing System(tm) - Core library downloads: 56561 homepage_uri: http://www.cups.org name: libcups2 section: libs version: 1.5.3-0ubuntu1 libcups2:i386: description: Common UNIX Printing System(tm) - Core library downloads: 0 homepage_uri: http://www.cups.org name: libcups2:i386 section: libs version: 1.5.3-0ubuntu1 libcupscgi1: description: Common UNIX Printing System(tm) - CGI library downloads: 41402 homepage_uri: http://www.cups.org name: libcupscgi1 section: libs version: 1.5.3-0ubuntu1 libcupsdriver1: description: Common UNIX Printing System(tm) - Driver library downloads: 255 homepage_uri: http://www.cups.org name: libcupsdriver1 section: libs version: 1.5.3-0ubuntu1 libcupsfilters1: description: OpenPrinting CUPS Filters - Shared library downloads: 5114 homepage_uri: http://www.openprinting.org/ name: libcupsfilters1 section: libs version: 1.0.18-0ubuntu0.1 libcupsimage2: description: Common UNIX Printing System(tm) - Raster image library downloads: 45416 homepage_uri: http://www.cups.org name: libcupsimage2 section: libs version: 1.5.3-0ubuntu1 libcupsimage2:i386: description: Common UNIX Printing System(tm) - Raster image library downloads: 0 homepage_uri: http://www.cups.org name: libcupsimage2:i386 section: libs version: 1.5.3-0ubuntu1 libcupsmime1: description: Common UNIX Printing System(tm) - MIME library downloads: 41415 homepage_uri: http://www.cups.org name: libcupsmime1 section: libs version: 1.5.3-0ubuntu1 libcupsppdc1: description: Common UNIX Printing System(tm) - PPD manipulation library downloads: 41493 homepage_uri: http://www.cups.org name: libcupsppdc1 section: libs version: 1.5.3-0ubuntu1 libcurl3: description: Multi-protocol file transfer library (OpenSSL) downloads: 27070 homepage_uri: http://curl.haxx.se name: libcurl3 section: libs version: 7.22.0-3ubuntu4 libcurl3-gnutls: description: Multi-protocol file transfer library (GnuTLS) downloads: 46882 homepage_uri: http://curl.haxx.se name: libcurl3-gnutls section: libs version: 7.22.0-3ubuntu4 libcurl3-nss: description: Multi-protocol file transfer library (NSS) downloads: 1287 homepage_uri: http://curl.haxx.se name: libcurl3-nss section: libs version: 7.22.0-3ubuntu4 libcurl3:i386: description: Multi-protocol file transfer library (OpenSSL) downloads: 0 homepage_uri: http://curl.haxx.se name: libcurl3:i386 section: libs version: 7.22.0-3ubuntu4 libcurl4-openssl-dev: description: Development files and documentation for libcurl (OpenSSL) downloads: 887 homepage_uri: http://curl.haxx.se name: libcurl4-openssl-dev section: libdevel version: 7.22.0-3ubuntu4 libcvector2: description: ANSI C implementation of dynamic arrays downloads: 281 homepage_uri: http://sourceforge.net/projects/cvector/ name: libcvector2 section: universe/libs version: 1.0.3-1 libcwidget3: description: high-level terminal interface library for C++ (runtime files) downloads: 28117 homepage_uri: http://cwidget.alioth.debian.org name: libcwidget3 section: libs version: 0.5.16-3.1ubuntu1 libcxsparse2.2.3: description: concise sparse matrix library (complex, int and long int support) downloads: 0 homepage_uri: http://www.cise.ufl.edu/research/sparse/SuiteSparse/ name: libcxsparse2.2.3 section: universe/libs version: 1:3.4.0-2ubuntu3 libcxxtools7: description: library of unrelated but useful C++ classes downloads: 0 homepage_uri: http://www.tntnet.org/cxxtools.html name: libcxxtools7 section: universe/libs version: 2.0-4ubuntu2 libdaemon0: description: lightweight C library for daemons - runtime library downloads: 48519 homepage_uri: http://0pointer.de/lennart/projects/libdaemon/ name: libdaemon0 section: libs version: 0.14-2 libdatrie1: description: Double-array trie library downloads: 41416 homepage_uri: http://linux.thai.net/projects/libthai name: libdatrie1 section: libs version: 0.2.5-3 libdatrie1:i386: description: Double-array trie library downloads: 0 homepage_uri: http://linux.thai.net/projects/libthai name: libdatrie1:i386 section: libs version: 0.2.5-3 libdb-dev: description: Berkeley Database Libraries [development] downloads: 98 homepage_uri: '' name: libdb-dev section: libdevel version: 5.1.4ubuntu1 libdb-java: description: Berkeley Database Libraries for Java downloads: 0 homepage_uri: '' name: libdb-java section: universe/java version: 5.1.4ubuntu1 libdb-je-java: description: Oracle Berkeley Database Java Edition downloads: 12 homepage_uri: http://www.oracle.com/database/berkeley-db/je/index.html name: libdb-je-java section: universe/libs version: 3.3.98-1 libdb4.8: description: Berkeley v4.8 Database Libraries [runtime] downloads: 6016 homepage_uri: http://www.oracle.com/technology/software/products/berkeley-db/index.html name: libdb4.8 section: libs version: 4.8.30-11ubuntu1 libdb5.1: description: Berkeley v5.1 Database Libraries [runtime] downloads: 21287 homepage_uri: http://www.oracle.com/technology/software/products/berkeley-db/index.html name: libdb5.1 section: libs version: 5.1.25-11build1 libdb5.1-dev: description: Berkeley v5.1 Database Libraries [development] downloads: 216 homepage_uri: http://www.oracle.com/technology/software/products/berkeley-db/index.html name: libdb5.1-dev section: libdevel version: 5.1.25-11build1 libdb5.1-java: description: Berkeley v5.1 Database Libraries for Java downloads: 161 homepage_uri: http://www.oracle.com/technology/software/products/berkeley-db/index.html name: libdb5.1-java section: java version: 5.1.25-11build1 libdb5.1-java-gcj: description: Berkeley v5.1 Database Libraries for Java (native code) downloads: 137 homepage_uri: http://www.oracle.com/technology/software/products/berkeley-db/index.html name: libdb5.1-java-gcj section: java version: 5.1.25-11build1 libdb5.1:i386: description: Berkeley v5.1 Database Libraries [runtime] downloads: 0 homepage_uri: http://www.oracle.com/technology/software/products/berkeley-db/index.html name: libdb5.1:i386 section: libs version: 5.1.25-11build1 libdbd-mysql-perl: description: Perl5 database interface to the MySQL database downloads: 30179 homepage_uri: http://search.cpan.org/dist/DBD-mysql/ name: libdbd-mysql-perl section: perl version: 4.020-1build2 libdbd-pg-perl: description: Perl DBI driver for the PostgreSQL database server downloads: 1756 homepage_uri: http://search.cpan.org/dist/DBD-Pg/ name: libdbd-pg-perl section: universe/perl version: 2.19.0-1 libdbi-perl: description: Perl Database Interface (DBI) downloads: 32158 homepage_uri: http://dbi.perl.org/ name: libdbi-perl section: perl version: 1.616-1build2 libdbus-1-3: description: simple interprocess messaging system (library) downloads: 71064 homepage_uri: http://dbus.freedesktop.org/ name: libdbus-1-3 section: libs version: 1.4.18-1ubuntu1 libdbus-1-3:i386: description: simple interprocess messaging system (library) downloads: 0 homepage_uri: http://dbus.freedesktop.org/ name: libdbus-1-3:i386 section: libs version: 1.4.18-1ubuntu1 libdbus-1-dev: description: simple interprocess messaging system (development headers) downloads: 2521 homepage_uri: http://dbus.freedesktop.org/ name: libdbus-1-dev section: libdevel version: 1.4.18-1ubuntu1 libdbus-glib-1-2: description: simple interprocess messaging system (GLib-based shared library) downloads: 58809 homepage_uri: http://www.freedesktop.org/wiki/Software/DBusBindings name: libdbus-glib-1-2 section: libs version: 0.98-1ubuntu1 libdbus-glib-1-2:i386: description: simple interprocess messaging system (GLib-based shared library) downloads: 0 homepage_uri: http://www.freedesktop.org/wiki/Software/DBusBindings name: libdbus-glib-1-2:i386 section: libs version: 0.98-1ubuntu1 libdbusmenu-glib4: description: library for passing menus over DBus downloads: 583 homepage_uri: https://launchpad.net/dbusmenu name: libdbusmenu-glib4 section: libs version: 0.6.1-0ubuntu3 libdbusmenu-gtk3-4: description: library for passing menus over DBus - GTK+ version downloads: 228 homepage_uri: https://launchpad.net/dbusmenu name: libdbusmenu-gtk3-4 section: libs version: 0.6.1-0ubuntu3 libdbusmenu-gtk4: description: library for passing menus over DBus - GTK+ version downloads: 80 homepage_uri: https://launchpad.net/dbusmenu name: libdbusmenu-gtk4 section: libs version: 0.6.1-0ubuntu3 libdbusmenu-qt2: description: Qt implementation of the DBusMenu protocol downloads: 1797 homepage_uri: https://launchpad.net/libdbusmenu-qt name: libdbusmenu-qt2 section: libs version: 0.9.2-0ubuntu1 libdconf-dbus-1-0: description: simple configuration storage system - D-Bus library downloads: 0 homepage_uri: http://live.gnome.org/dconf name: libdconf-dbus-1-0 section: libs version: 0.12.0-0ubuntu1 libdconf-qt0: description: dconf Qt bindings (library) downloads: 0 homepage_uri: https://gitorious.org/dconf-qt name: libdconf-qt0 section: libs version: 0.0.0.110722-0ubuntu4 libdconf0: description: simple configuration storage system - runtime library downloads: 13821 homepage_uri: http://live.gnome.org/dconf name: libdconf0 section: libs version: 0.12.0-0ubuntu1 libdecoration0: description: Compiz window decoration library downloads: 2134 homepage_uri: '' name: libdecoration0 section: x11 version: 1:0.9.7.8-0ubuntu1 libdee-1.0-4: description: model to synchronize mutiple instances over DBus - shared lib downloads: 338 homepage_uri: https://launchpad.net/dee name: libdee-1.0-4 section: libs version: 1.0.10-0ubuntu1 libdevmapper-event1.02.1: description: The Linux Kernel Device Mapper userspace library downloads: 3829 homepage_uri: http://sources.redhat.com/lvm2/ name: libdevmapper-event1.02.1 section: libs version: 2:1.02.48-4ubuntu7.1 libdevmapper1.02.1: description: The Linux Kernel Device Mapper userspace library downloads: 70680 homepage_uri: http://sources.redhat.com/lvm2/ name: libdevmapper1.02.1 section: libs version: 2:1.02.48-4ubuntu7.1 libdigest-hmac-perl: description: module for creating standard message integrity checks downloads: 9900 homepage_uri: http://search.cpan.org/dist/Digest-HMAC/ name: libdigest-hmac-perl section: perl version: 1.03+dfsg-1 libdiscid0: description: Library for creating MusicBrainz DiscIDs downloads: 19854 homepage_uri: http://musicbrainz.org/doc/libdiscid name: libdiscid0 section: libs version: 0.2.2-3 libdjvulibre-text: description: Linguistic support files for libdjvulibre downloads: 0 homepage_uri: http://djvu.sourceforge.net/ name: libdjvulibre-text section: libs version: 3.5.24-9 libdjvulibre21: description: Runtime support for the DjVu image format downloads: 30294 homepage_uri: http://djvu.sourceforge.net/ name: libdjvulibre21 section: libs version: 3.5.24-9 libdmapsharing-3.0-2: description: DMAP client and server library - runtime downloads: 4900 homepage_uri: http://www.flyn.org/projects/libdmapsharing/ name: libdmapsharing-3.0-2 section: libs version: 2.9.14-1 libdns81: description: DNS Shared Library used by BIND downloads: 1908 homepage_uri: '' name: libdns81 section: libs version: 1:9.8.1.dfsg.P1-4ubuntu0.1 libdom4j-java: description: flexible XML framework for Java downloads: 229 homepage_uri: http://sourceforge.net/projects/dom4j/ name: libdom4j-java section: libs version: 1.6.1+dfsg.2-5 libdotconf1.0: description: Configuration file parser library - runtime files downloads: 1656 homepage_uri: '' name: libdotconf1.0 section: libs version: 1.0.13-3 libdoxia-java: description: powerful content generation framework downloads: 93 homepage_uri: http://maven.apache.org/doxia/ name: libdoxia-java section: universe/devel version: 1.1.4-1ubuntu3 libdpkg-perl: description: Dpkg perl modules downloads: 7768 homepage_uri: http://wiki.debian.org/Teams/Dpkg name: libdpkg-perl section: perl version: 1.16.1.2ubuntu7 libdrm-dev: description: Userspace interface to kernel DRM services -- development files downloads: 294 homepage_uri: '' name: libdrm-dev section: libdevel version: 2.4.32-1ubuntu1 libdrm-intel1: description: Userspace interface to intel-specific kernel DRM services -- runtime downloads: 29506 homepage_uri: '' name: libdrm-intel1 section: libs version: 2.4.32-1ubuntu1 libdrm-intel1:i386: description: Userspace interface to intel-specific kernel DRM services -- runtime downloads: 0 homepage_uri: '' name: libdrm-intel1:i386 section: libs version: 2.4.32-1ubuntu1 libdrm-nouveau1a: description: Userspace interface to nouveau-specific kernel DRM services -- runtime downloads: 3676 homepage_uri: '' name: libdrm-nouveau1a section: libs version: 2.4.32-1ubuntu1 libdrm-nouveau1a:i386: description: Userspace interface to nouveau-specific kernel DRM services -- runtime downloads: 0 homepage_uri: '' name: libdrm-nouveau1a:i386 section: libs version: 2.4.32-1ubuntu1 libdrm-radeon1: description: Userspace interface to radeon-specific kernel DRM services -- runtime downloads: 27911 homepage_uri: '' name: libdrm-radeon1 section: libs version: 2.4.32-1ubuntu1 libdrm-radeon1:i386: description: Userspace interface to radeon-specific kernel DRM services -- runtime downloads: 0 homepage_uri: '' name: libdrm-radeon1:i386 section: libs version: 2.4.32-1ubuntu1 libdrm2: description: Userspace interface to kernel DRM services -- runtime downloads: 44737 homepage_uri: '' name: libdrm2 section: libs version: 2.4.32-1ubuntu1 libdrm2:i386: description: Userspace interface to kernel DRM services -- runtime downloads: 0 homepage_uri: '' name: libdrm2:i386 section: libs version: 2.4.32-1ubuntu1 libdv4: description: software library for DV format digital video (runtime lib) downloads: 27909 homepage_uri: http://libdv.sourceforge.net/ name: libdv4 section: libs version: 1.0.0-3ubuntu1 libdv4:i386: description: software library for DV format digital video (runtime lib) downloads: 0 homepage_uri: http://libdv.sourceforge.net/ name: libdv4:i386 section: libs version: 1.0.0-3ubuntu1 libeasymock-java: description: Java library to generate Mock Objects for given interfaces downloads: 95 homepage_uri: http://www.easymock.org name: libeasymock-java section: libs version: 2.4+ds1-6 libebackend-1.2-1: description: Utility library for evolution data servers downloads: 0 homepage_uri: http://www.gnome.org/projects/evolution/ name: libebackend-1.2-1 section: libs version: 3.2.3-0ubuntu7 libebook-1.2-12: description: Client library for evolution address books downloads: 0 homepage_uri: http://www.gnome.org/projects/evolution/ name: libebook-1.2-12 section: libs version: 3.2.3-0ubuntu7 libecal-1.2-10: description: Client library for evolution calendars downloads: 0 homepage_uri: http://www.gnome.org/projects/evolution/ name: libecal-1.2-10 section: libs version: 3.2.3-0ubuntu7 libecj-java: description: Eclipse Java compiler (library) downloads: 1270 homepage_uri: '' name: libecj-java section: libs version: 3.5.1-3 libedata-book-1.2-11: description: Backend library for evolution address books downloads: 0 homepage_uri: http://www.gnome.org/projects/evolution/ name: libedata-book-1.2-11 section: libs version: 3.2.3-0ubuntu7 libedata-cal-1.2-13: description: Backend library for evolution calendars downloads: 0 homepage_uri: http://www.gnome.org/projects/evolution/ name: libedata-cal-1.2-13 section: libs version: 3.2.3-0ubuntu7 libedataserver-1.2-15: description: Utility library for evolution data servers downloads: 0 homepage_uri: http://www.gnome.org/projects/evolution/ name: libedataserver-1.2-15 section: libs version: 3.2.3-0ubuntu7 libedataserverui-3.0-1: description: GUI utility library for evolution data servers downloads: 9215 homepage_uri: http://www.gnome.org/projects/evolution/ name: libedataserverui-3.0-1 section: libs version: 3.2.3-0ubuntu7 libedit2: description: BSD editline and history libraries downloads: 74951 homepage_uri: http://ftp.netbsd.org/pub/NetBSD/NetBSD-current/src/lib/libedit/ name: libedit2 section: libs version: 2.11-20080614-3ubuntu2 libeigen3-dev: description: lightweight C++ template library for linear algebra downloads: 46 homepage_uri: http://eigen.tuxfamily.org name: libeigen3-dev section: universe/libdevel version: 3.0.5-1 libelf1: description: library to read and write ELF files downloads: 15227 homepage_uri: https://fedorahosted.org/elfutils/ name: libelf1 section: libs version: 0.152-1ubuntu3 libemail-valid-perl: description: Perl module for checking the validity of Internet email addresses downloads: 1365 homepage_uri: http://search.cpan.org/dist/Email-Valid/ name: libemail-valid-perl section: perl version: 0.185-1 libenchant1c2a: description: Wrapper library for various spell checker engines (runtime libs) downloads: 40880 homepage_uri: http://www.abisource.com/projects/enchant/ name: libenchant1c2a section: libs version: 1.6.0-7 libencode-locale-perl: description: utility to determine the locale encoding downloads: 5317 homepage_uri: http://search.cpan.org/dist/Encode-Locale/ name: libencode-locale-perl section: perl version: 1.02-2 libept1.4.12: description: High-level library for managing Debian package information downloads: 9660 homepage_uri: '' name: libept1.4.12 section: libs version: 1.0.6~exp1ubuntu1 libequinox-osgi-java: description: Equinox OSGi framework downloads: 1625 homepage_uri: http://www.eclipse.org/ name: libequinox-osgi-java section: universe/java version: 3.7.2-1 liberror-perl: description: Perl module for error/exception handling in an OO-ish way downloads: 8317 homepage_uri: '' name: liberror-perl section: perl version: 0.17-1 libesd0:i386: description: Enlightened Sound Daemon - Shared libraries downloads: 0 homepage_uri: '' name: libesd0:i386 section: libs version: 0.2.41-10build3 libespeak1: description: 'Multi-lingual software speech synthesizer: shared library' downloads: 22011 homepage_uri: http://espeak.sourceforge.net/ name: libespeak1 section: libs version: 1.46.02-0ubuntu1 libevent-2.0-5: description: Asynchronous event notification library downloads: 14054 homepage_uri: http://www.monkey.org/~provos/libevent/ name: libevent-2.0-5 section: libs version: 2.0.16-stable-1 libevince3-3: description: Document (PostScript, PDF) rendering library downloads: 2194 homepage_uri: http://www.gnome.org/projects/evince/ name: libevince3-3 section: libs version: 3.4.0-0ubuntu1 libexempi3: description: library to parse XMP metadata (Library) downloads: 32109 homepage_uri: http://libopenraw.freedesktop.org/wiki/Exempi name: libexempi3 section: libs version: 2.2.0-1 libexif12: description: library to parse EXIF files downloads: 42348 homepage_uri: http://libexif.sourceforge.net/ name: libexif12 section: libs version: 0.6.20-2 libexif12:i386: description: library to parse EXIF files downloads: 0 homepage_uri: http://libexif.sourceforge.net/ name: libexif12:i386 section: libs version: 0.6.20-2 libexiv2-11: description: EXIF/IPTC metadata manipulation library downloads: 0 homepage_uri: http://www.exiv2.org name: libexiv2-11 section: libs version: 0.22-2 libexpat1: description: XML parsing C library - runtime library downloads: 87687 homepage_uri: http://expat.sourceforge.net name: libexpat1 section: libs version: 2.0.1-7.2ubuntu1 libexpat1-dev: description: XML parsing C library - development kit downloads: 5258 homepage_uri: http://expat.sourceforge.net name: libexpat1-dev section: libdevel version: 2.0.1-7.2ubuntu1 libexpat1:i386: description: XML parsing C library - runtime library downloads: 0 homepage_uri: http://expat.sourceforge.net name: libexpat1:i386 section: libs version: 2.0.1-7.2ubuntu1 libexporter-lite-perl: description: lightweight subset of Exporter downloads: 794 homepage_uri: http://search.cpan.org/dist/Exporter-Lite/ name: libexporter-lite-perl section: perl version: 0.02-2 libexttextcat-data: description: Language detection library - data files downloads: 7 homepage_uri: http://software.wise-guys.nl/libtextcat/ name: libexttextcat-data section: text version: 3.2.0-1ubuntu1 libexttextcat0: description: Language detection library downloads: 7 homepage_uri: http://software.wise-guys.nl/libtextcat/ name: libexttextcat0 section: libs version: 3.2.0-1ubuntu1 libfarstream-0.1-0: description: 'Audio/Video communications framework: core library' downloads: 2769 homepage_uri: http://www.freedesktop.org/software/farstream/releases/farstream/ name: libfarstream-0.1-0 section: libs version: 0.1.2-0ubuntu1 libfcgi-perl: description: helper module for FastCGI downloads: 2141 homepage_uri: http://search.cpan.org/dist/FCGI/ name: libfcgi-perl section: universe/perl version: 0.74-1build1 libffi-dev: description: Foreign Function Interface library (development files) downloads: 1263 homepage_uri: '' name: libffi-dev section: libdevel version: 3.0.11~rc1-5 libffi6: description: Foreign Function Interface library runtime downloads: 7 homepage_uri: '' name: libffi6 section: libs version: 3.0.11~rc1-5 libffi6:i386: description: Foreign Function Interface library runtime downloads: 0 homepage_uri: '' name: libffi6:i386 section: libs version: 3.0.11~rc1-5 libfftw3-3: description: library for computing Fast Fourier Transforms downloads: 18319 homepage_uri: http://fftw.org name: libfftw3-3 section: libs version: 3.3-1ubuntu1 libfile-basedir-perl: description: Perl module to use the freedesktop basedir specification downloads: 2537 homepage_uri: http://search.cpan.org/dist/File-BaseDir/ name: libfile-basedir-perl section: perl version: 0.03-1fakesync1 libfile-copy-recursive-perl: description: Perl extension for recursively copying files and directories downloads: 17283 homepage_uri: http://search.cpan.org/dist/File-Copy-Recursive/ name: libfile-copy-recursive-perl section: perl version: 0.38-1 libfile-desktopentry-perl: description: Perl module to handle freedesktop .desktop files downloads: 2479 homepage_uri: http://search.cpan.org/dist/File-DesktopEntry/ name: libfile-desktopentry-perl section: perl version: 0.04-3 libfile-listing-perl: description: module to parse directory listings downloads: 5271 homepage_uri: http://search.cpan.org/dist/File-Listing/ name: libfile-listing-perl section: perl version: 6.03-1 libfile-mimeinfo-perl: description: Perl module to determine file types downloads: 2474 homepage_uri: http://search.cpan.org/dist/File-MimeInfo/ name: libfile-mimeinfo-perl section: perl version: 0.15-2 libflac8: description: Free Lossless Audio Codec - runtime C library downloads: 40618 homepage_uri: http://flac.sourceforge.net/ name: libflac8 section: libs version: 1.2.1-6 libflac8:i386: description: Free Lossless Audio Codec - runtime C library downloads: 0 homepage_uri: http://flac.sourceforge.net/ name: libflac8:i386 section: libs version: 1.2.1-6 libfltk1.1: description: Fast Light Toolkit - shared libraries downloads: 1663 homepage_uri: http://www.fltk.org/ name: libfltk1.1 section: libs version: 1.1.10-10 libfolks-eds25: description: Evolution-data-server backend for libfolks downloads: 5262 homepage_uri: http://telepathy.freedesktop.org/wiki/Folks name: libfolks-eds25 section: libs version: 0.6.8-2 libfolks-telepathy25: description: Telepathy backend for libfolks downloads: 5215 homepage_uri: http://telepathy.freedesktop.org/wiki/Folks name: libfolks-telepathy25 section: libs version: 0.6.8-2 libfolks25: description: library to aggregates people into metacontacts downloads: 7939 homepage_uri: http://telepathy.freedesktop.org/wiki/Folks name: libfolks25 section: libs version: 0.6.8-2 libfont-afm-perl: description: Font::AFM - Interface to Adobe Font Metrics files downloads: 2415 homepage_uri: http://search.cpan.org/dist/Font-AFM/ name: libfont-afm-perl section: perl version: 1.20-1 libfontconfig1: description: generic font configuration library - runtime downloads: 66704 homepage_uri: '' name: libfontconfig1 section: libs version: 2.8.0-3ubuntu9 libfontconfig1-dev: description: generic font configuration library - development downloads: 4320 homepage_uri: '' name: libfontconfig1-dev section: libdevel version: 2.8.0-3ubuntu9 libfontconfig1:i386: description: generic font configuration library - runtime downloads: 0 homepage_uri: '' name: libfontconfig1:i386 section: libs version: 2.8.0-3ubuntu9 libfontenc1: description: X11 font encoding library downloads: 36105 homepage_uri: '' name: libfontenc1 section: x11 version: 1:1.1.0-1 libfop-java: description: XML formatter driven by XSL Formatting Objects (XSL-FO.) downloads: 499 homepage_uri: http://xmlgraphics.apache.org/fop/ name: libfop-java section: java version: 1:1.0.dfsg2-6 libfreerdp-plugins-standard: description: RDP client for Windows Terminal Services (plugins) downloads: 2465 homepage_uri: http://www.freerdp.com/ name: libfreerdp-plugins-standard section: x11 version: 1.0.1-1ubuntu2 libfreerdp1: description: RDP client for Windows Terminal Services (library) downloads: 665 homepage_uri: http://www.freerdp.com/ name: libfreerdp1 section: libs version: 1.0.1-1ubuntu2 libfreetype6: description: FreeType 2 font engine, shared library files downloads: 77130 homepage_uri: http://www.freetype.org name: libfreetype6 section: libs version: 2.4.8-1ubuntu2 libfreetype6-dev: description: FreeType 2 font engine, development files downloads: 5105 homepage_uri: http://www.freetype.org name: libfreetype6-dev section: libdevel version: 2.4.8-1ubuntu2 libfreetype6:i386: description: FreeType 2 font engine, shared library files downloads: 0 homepage_uri: http://www.freetype.org name: libfreetype6:i386 section: libs version: 2.4.8-1ubuntu2 libfribidi0: description: Free Implementation of the Unicode BiDi algorithm downloads: 11609 homepage_uri: http://www.fribidi.org/ name: libfribidi0 section: libs version: 0.19.2-1 libfs6: description: X11 Font Services library downloads: 5546 homepage_uri: '' name: libfs6 section: libs version: 2:1.0.3-1 libfuse-dev: description: Filesystem in Userspace (development) downloads: 138 homepage_uri: http://fuse.sourceforge.net/ name: libfuse-dev section: libdevel version: 2.8.6-2ubuntu2 libfuse2: description: Filesystem in Userspace (library) downloads: 23958 homepage_uri: http://fuse.sourceforge.net/ name: libfuse2 section: libs version: 2.8.6-2ubuntu2 libgail-3-0: description: GNOME Accessibility Implementation Library -- shared libraries downloads: 11162 homepage_uri: http://www.gtk.org/ name: libgail-3-0 section: libs version: 3.4.2-0ubuntu0.2 libgail-common: description: GNOME Accessibility Implementation Library -- common modules downloads: 18796 homepage_uri: http://www.gtk.org/ name: libgail-common section: libs version: 2.24.10-0ubuntu6 libgail-common:i386: description: GNOME Accessibility Implementation Library -- common modules downloads: 0 homepage_uri: http://www.gtk.org/ name: libgail-common:i386 section: libs version: 2.24.10-0ubuntu6 libgail-dev: description: GNOME Accessibility Implementation Library -- development files downloads: 311 homepage_uri: http://www.gtk.org/ name: libgail-dev section: libdevel version: 2.24.10-0ubuntu6 libgail18: description: GNOME Accessibility Implementation Library -- shared libraries downloads: 27619 homepage_uri: http://www.gtk.org/ name: libgail18 section: libs version: 2.24.10-0ubuntu6 libgail18:i386: description: GNOME Accessibility Implementation Library -- shared libraries downloads: 0 homepage_uri: http://www.gtk.org/ name: libgail18:i386 section: libs version: 2.24.10-0ubuntu6 libganymed-ssh2-java: description: pure Java implementation of the SSH-2 protocol downloads: 2 homepage_uri: http://www.cleondris.ch/ssh2/ name: libganymed-ssh2-java section: universe/libs version: 250-2 libgc1c2: description: conservative garbage collector for C and C++ downloads: 29447 homepage_uri: http://www.hpl.hp.com/personal/Hans_Boehm/gc/ name: libgc1c2 section: libs version: 1:7.1-8build1 libgcc1: description: GCC support library downloads: 118025 homepage_uri: http://gcc.gnu.org/ name: libgcc1 section: libs version: 1:4.6.3-1ubuntu5 libgcc1:i386: description: GCC support library downloads: 0 homepage_uri: http://gcc.gnu.org/ name: libgcc1:i386 section: libs version: 1:4.6.3-1ubuntu5 libgcj-bc: description: Link time only library for use with gcj downloads: 579 homepage_uri: '' name: libgcj-bc section: java version: 4.6.3-1ubuntu5 libgcj-common: description: Java runtime library (common files) downloads: 13885 homepage_uri: '' name: libgcj-common section: libs version: 1:4.6.3-1ubuntu5 libgcj12: description: Java runtime library for use with gcj downloads: 3640 homepage_uri: http://gcc.gnu.org/ name: libgcj12 section: libs version: 4.6.3-1ubuntu2 libgck-1-0: description: Glib wrapper library for PKCS#11 - runtime downloads: 12242 homepage_uri: http://live.gnome.org/GnomeKeyring name: libgck-1-0 section: libs version: 3.2.2-2ubuntu4 libgconf-2-4: description: GNOME configuration database system (shared libraries) downloads: 15055 homepage_uri: http://projects.gnome.org/gconf/ name: libgconf-2-4 section: libs version: 3.2.5-0ubuntu2 libgconf-2-4:i386: description: GNOME configuration database system (shared libraries) downloads: 0 homepage_uri: http://projects.gnome.org/gconf/ name: libgconf-2-4:i386 section: libs version: 3.2.5-0ubuntu2 libgconf2-4: description: GNOME configuration database system (dummy package) downloads: 43270 homepage_uri: http://projects.gnome.org/gconf/ name: libgconf2-4 section: libs version: 3.2.5-0ubuntu2 libgconf2-dev: description: GNOME configuration database system (development) downloads: 819 homepage_uri: http://projects.gnome.org/gconf/ name: libgconf2-dev section: libdevel version: 3.2.5-0ubuntu2 libgconf2-doc: description: GNOME configuration database system (API reference) downloads: 0 homepage_uri: http://projects.gnome.org/gconf/ name: libgconf2-doc section: doc version: 3.2.5-0ubuntu2 libgcr-3-1: description: Library for Crypto UI related task - runtime downloads: 12252 homepage_uri: http://live.gnome.org/GnomeKeyring name: libgcr-3-1 section: libs version: 3.2.2-2ubuntu4 libgcr-3-common: description: Library for Crypto UI related task - common files downloads: 11822 homepage_uri: http://live.gnome.org/GnomeKeyring name: libgcr-3-common section: libs version: 3.2.2-2ubuntu4 libgcrypt11: description: LGPL Crypto library - runtime library downloads: 85078 homepage_uri: http://directory.fsf.org/project/libgcrypt/ name: libgcrypt11 section: libs version: 1.5.0-3ubuntu0.1 libgcrypt11-dev: description: LGPL Crypto library - development files downloads: 3027 homepage_uri: http://directory.fsf.org/project/libgcrypt/ name: libgcrypt11-dev section: libdevel version: 1.5.0-3ubuntu0.1 libgcrypt11:i386: description: LGPL Crypto library - runtime library downloads: 0 homepage_uri: http://directory.fsf.org/project/libgcrypt/ name: libgcrypt11:i386 section: libs version: 1.5.0-3ubuntu0.1 libgd-gd2-perl: description: Perl module wrapper for libgd - gd2 variant downloads: 965 homepage_uri: http://search.cpan.org/dist/GD/ name: libgd-gd2-perl section: perl version: 1:2.46-3build1 libgd-graph-perl: description: Graph Plotting Module for Perl 5 downloads: 595 homepage_uri: http://search.cpan.org/dist/GDGraph/ name: libgd-graph-perl section: perl version: 1.44-6 libgd-text-perl: description: Text utilities for use with GD downloads: 603 homepage_uri: http://search.cpan.org/dist/GDTextUtil/ name: libgd-text-perl section: perl version: 0.86-8 libgd2-xpm: description: GD Graphics Library version 2 downloads: 39833 homepage_uri: http://www.libgd.org/ name: libgd2-xpm section: libs version: 2.0.36~rc1~dfsg-6ubuntu2 libgd2-xpm-dev: description: GD Graphics Library version 2 (development version) downloads: 225 homepage_uri: http://www.libgd.org/ name: libgd2-xpm-dev section: libdevel version: 2.0.36~rc1~dfsg-6ubuntu2 libgd2-xpm:i386: description: GD Graphics Library version 2 downloads: 0 homepage_uri: http://www.libgd.org/ name: libgd2-xpm:i386 section: libs version: 2.0.36~rc1~dfsg-6ubuntu2 libgdata-common: description: Library for accessing GData webservices - common data files downloads: 97 homepage_uri: http://live.gnome.org/libgdata name: libgdata-common section: libs version: 0.12.0-1 libgdata13: description: Library for accessing GData webservices - shared libraries downloads: 7312 homepage_uri: http://live.gnome.org/libgdata name: libgdata13 section: libs version: 0.12.0-1 libgdbm3: description: GNU dbm database routines (runtime version) downloads: 109796 homepage_uri: http://directory.fsf.org/project/gdbm/ name: libgdbm3 section: libs version: 1.8.3-10 libgdbm3:i386: description: GNU dbm database routines (runtime version) downloads: 0 homepage_uri: http://directory.fsf.org/project/gdbm/ name: libgdbm3:i386 section: libs version: 1.8.3-10 libgdk-pixbuf2.0-0: description: GDK Pixbuf library downloads: 17771 homepage_uri: http://www.gtk.org/ name: libgdk-pixbuf2.0-0 section: libs version: 2.26.1-1 libgdk-pixbuf2.0-0:i386: description: GDK Pixbuf library downloads: 0 homepage_uri: http://www.gtk.org/ name: libgdk-pixbuf2.0-0:i386 section: libs version: 2.26.1-1 libgdk-pixbuf2.0-common: description: GDK Pixbuf library - data files downloads: 15887 homepage_uri: http://www.gtk.org/ name: libgdk-pixbuf2.0-common section: libs version: 2.26.1-1 libgdk-pixbuf2.0-dev: description: GDK Pixbuf library (development files) downloads: 2027 homepage_uri: http://www.gtk.org/ name: libgdk-pixbuf2.0-dev section: libdevel version: 2.26.1-1 libgdu-gtk0: description: GTK+ standard dialog library for libgdu downloads: 26659 homepage_uri: http://git.gnome.org/cgit/gnome-disk-utility/ name: libgdu-gtk0 section: libs version: 3.0.2-2ubuntu7 libgdu0: description: GObject based Disk Utility Library downloads: 34679 homepage_uri: http://git.gnome.org/cgit/gnome-disk-utility/ name: libgdu0 section: libs version: 3.0.2-2ubuntu7 libgee2: description: GObject based collection library downloads: 13375 homepage_uri: http://live.gnome.org/Libgee name: libgee2 section: libs version: 0.6.4-1 libgeoclue0: description: C API for GeoClue downloads: 24563 homepage_uri: http://www.freedesktop.org/wiki/Software/GeoClue name: libgeoclue0 section: libs version: 0.12.0-1ubuntu12 libgeoip1: description: non-DNS IP-to-country resolver library downloads: 6506 homepage_uri: http://www.maxmind.com/ name: libgeoip1 section: libs version: 1.4.8+dfsg-2 libgettextpo0: description: GNU Internationalization library downloads: 2211 homepage_uri: http://www.gnu.org/software/gettext/ name: libgettextpo0 section: devel version: 0.18.1.1-5ubuntu3 libgettextpo0:i386: description: GNU Internationalization library downloads: 0 homepage_uri: http://www.gnu.org/software/gettext/ name: libgettextpo0:i386 section: devel version: 0.18.1.1-5ubuntu3 libgexiv2-1: description: GObject-based wrapper around the Exiv2 library downloads: 1286 homepage_uri: http://redmine.yorba.org/projects/gexiv2/wiki name: libgexiv2-1 section: libs version: 0.4.1-1build1 libgfortran3: description: Runtime library for GNU Fortran applications downloads: 3850 homepage_uri: http://gcc.gnu.org/ name: libgfortran3 section: libs version: 4.6.3-1ubuntu5 libgif4: description: library for GIF images (library) downloads: 22730 homepage_uri: http://giflib.sourceforge.net/ name: libgif4 section: libs version: 4.1.6-9ubuntu1 libgirepository-1.0-1: description: Library for handling GObject introspection data (runtime library) downloads: 13188 homepage_uri: http://live.gnome.org/GObjectIntrospection name: libgirepository-1.0-1 section: libs version: 1.32.0-1 libgksu2-0: description: library providing su and sudo functionality downloads: 29787 homepage_uri: http://www.nongnu.org/gksu name: libgksu2-0 section: libs version: 2.0.13~pre1-5ubuntu2 libgl1-mesa-dev: description: free implementation of the OpenGL API -- GLX development files downloads: 2647 homepage_uri: http://mesa3d.sourceforge.net/ name: libgl1-mesa-dev section: libdevel version: 8.0.2-0ubuntu3.1 libgl1-mesa-dri: description: free implementation of the OpenGL API -- DRI modules downloads: 26165 homepage_uri: http://mesa3d.sourceforge.net/ name: libgl1-mesa-dri section: libs version: 8.0.2-0ubuntu3.1 libgl1-mesa-dri:i386: description: free implementation of the OpenGL API -- DRI modules downloads: 0 homepage_uri: http://mesa3d.sourceforge.net/ name: libgl1-mesa-dri:i386 section: libs version: 8.0.2-0ubuntu3.1 libgl1-mesa-glx: description: free implementation of the OpenGL API -- GLX runtime downloads: 44268 homepage_uri: http://mesa3d.sourceforge.net/ name: libgl1-mesa-glx section: libs version: 8.0.2-0ubuntu3.1 libgl1-mesa-glx:i386: description: free implementation of the OpenGL API -- GLX runtime downloads: 0 homepage_uri: http://mesa3d.sourceforge.net/ name: libgl1-mesa-glx:i386 section: libs version: 8.0.2-0ubuntu3.1 libgl2ps0: description: Lib providing high quality vector output for OpenGL application downloads: 72 homepage_uri: http://geuz.org/gl2ps/ name: libgl2ps0 section: universe/libs version: 1.3.6-1 libglade2-0: description: library to load .glade files at runtime downloads: 36287 homepage_uri: '' name: libglade2-0 section: libs version: 1:2.6.4-1ubuntu1.1 libglapi-mesa: description: free implementation of the GL API -- shared library downloads: 2749 homepage_uri: http://mesa3d.sourceforge.net/ name: libglapi-mesa section: libs version: 8.0.2-0ubuntu3.1 libglapi-mesa:i386: description: free implementation of the GL API -- shared library downloads: 0 homepage_uri: http://mesa3d.sourceforge.net/ name: libglapi-mesa:i386 section: libs version: 8.0.2-0ubuntu3.1 libglew1.6: description: OpenGL Extension Wrangler - runtime environment downloads: 120 homepage_uri: http://glew.sourceforge.net/ name: libglew1.6 section: libs version: 1.6.0-4 libglewmx1.6: description: OpenGL Extension Wrangler - runtime environment downloads: 10 homepage_uri: http://glew.sourceforge.net/ name: libglewmx1.6 section: libs version: 1.6.0-4 libglib-perl: description: interface to the GLib and GObject libraries downloads: 17559 homepage_uri: http://gtk2-perl.sourceforge.net/ name: libglib-perl section: perl version: 2:1.241-1 libglib2.0-0: description: GLib library of C routines downloads: 75544 homepage_uri: http://www.gtk.org/ name: libglib2.0-0 section: libs version: 2.32.3-0ubuntu1 libglib2.0-0:i386: description: GLib library of C routines downloads: 0 homepage_uri: http://www.gtk.org/ name: libglib2.0-0:i386 section: libs version: 2.32.3-0ubuntu1 libglib2.0-bin: description: Programs for the GLib library downloads: 13149 homepage_uri: http://www.gtk.org/ name: libglib2.0-bin section: misc version: 2.32.3-0ubuntu1 libglib2.0-data: description: Common files for GLib library downloads: 33562 homepage_uri: http://www.gtk.org/ name: libglib2.0-data section: misc version: 2.32.3-0ubuntu1 libglib2.0-dev: description: Development files for the GLib library downloads: 4917 homepage_uri: http://www.gtk.org/ name: libglib2.0-dev section: libdevel version: 2.32.3-0ubuntu1 libglibmm-2.4-1c2a: description: C++ wrapper for the GLib toolkit (shared libraries) downloads: 815 homepage_uri: http://www.gtkmm.org/ name: libglibmm-2.4-1c2a section: libs version: 2.32.0-0ubuntu1 libglpk0: description: linear programming kit with integer (MIP) support downloads: 155 homepage_uri: http://www.gnu.org/software/glpk/glpk.html name: libglpk0 section: universe/math version: 4.45-1build1 libglu1-mesa: description: Mesa OpenGL utility library (GLU) downloads: 13895 homepage_uri: http://mesa3d.sourceforge.net/ name: libglu1-mesa section: libs version: 8.0.2-0ubuntu3.1 libglu1-mesa-dev: description: Mesa OpenGL utility library -- development files downloads: 2574 homepage_uri: http://mesa3d.sourceforge.net/ name: libglu1-mesa-dev section: libdevel version: 8.0.2-0ubuntu3.1 libglu1-mesa:i386: description: Mesa OpenGL utility library (GLU) downloads: 0 homepage_uri: http://mesa3d.sourceforge.net/ name: libglu1-mesa:i386 section: libs version: 8.0.2-0ubuntu3.1 libgmime-2.6-0: description: MIME message parser and creator library - runtime downloads: 2714 homepage_uri: http://spruce.sourceforge.net/gmime/ name: libgmime-2.6-0 section: libs version: 2.6.7-1 libgmp-dev: description: Multiprecision arithmetic library developers tools downloads: 937 homepage_uri: http://gmplib.org/ name: libgmp-dev section: libdevel version: 2:5.0.2+dfsg-2ubuntu1 libgmp10: description: Multiprecision arithmetic library downloads: 17100 homepage_uri: http://gmplib.org/ name: libgmp10 section: libs version: 2:5.0.2+dfsg-2ubuntu1 libgmpxx4ldbl: description: Multiprecision arithmetic library (C++ bindings) downloads: 246 homepage_uri: http://gmplib.org/ name: libgmpxx4ldbl section: libs version: 2:5.0.2+dfsg-2ubuntu1 libgnome-bluetooth8: description: GNOME Bluetooth tools - support library downloads: 0 homepage_uri: http://live.gnome.org/GnomeBluetooth name: libgnome-bluetooth8 section: libs version: 3.2.2-0ubuntu5 libgnome-control-center1: description: utilities to configure the GNOME desktop downloads: 0 homepage_uri: '' name: libgnome-control-center1 section: libs version: 1:3.4.2-0ubuntu0.2 libgnome-desktop-3-2: description: Utility library for loading .desktop files - runtime files downloads: 10541 homepage_uri: '' name: libgnome-desktop-3-2 section: libs version: 3.4.1-0ubuntu1 libgnome-keyring-common: description: GNOME keyring services library - data files downloads: 9471 homepage_uri: http://live.gnome.org/GnomeKeyring name: libgnome-keyring-common section: libs version: 3.2.2-2 libgnome-keyring-dev: description: Development files for GNOME keyring service downloads: 359 homepage_uri: http://live.gnome.org/GnomeKeyring name: libgnome-keyring-dev section: libdevel version: 3.2.2-2 libgnome-keyring0: description: GNOME keyring services library downloads: 41606 homepage_uri: http://live.gnome.org/GnomeKeyring name: libgnome-keyring0 section: libs version: 3.2.2-2 libgnome-keyring0:i386: description: GNOME keyring services library downloads: 0 homepage_uri: http://live.gnome.org/GnomeKeyring name: libgnome-keyring0:i386 section: libs version: 3.2.2-2 libgnome-media-profiles-3.0-0: description: GNOME Media Profiles library downloads: 1362 homepage_uri: '' name: libgnome-media-profiles-3.0-0 section: libs version: 3.0.0-1 libgnome-menu-3-0: description: GNOME implementation of the freedesktop menu specification downloads: 9037 homepage_uri: '' name: libgnome-menu-3-0 section: gnome version: 3.4.0-0ubuntu1 libgnome-menu2: description: GNOME implementation of the freedesktop menu specification downloads: 4142 homepage_uri: '' name: libgnome-menu2 section: gnome version: 3.0.1-0ubuntu7 libgnome2-0: description: The GNOME library - runtime files downloads: 34342 homepage_uri: '' name: libgnome2-0 section: libs version: 2.32.1-2ubuntu1 libgnome2-common: description: The GNOME library - common files downloads: 12 homepage_uri: '' name: libgnome2-common section: libs version: 2.32.1-2ubuntu1 libgnome2-dev: description: The GNOME library - development files downloads: 173 homepage_uri: '' name: libgnome2-dev section: libdevel version: 2.32.1-2ubuntu1 libgnomecanvas2-0: description: powerful object-oriented display engine - runtime files downloads: 29139 homepage_uri: '' name: libgnomecanvas2-0 section: libs version: 2.30.3-1ubuntu1 libgnomecanvas2-common: description: powerful object-oriented display engine - common files downloads: 3 homepage_uri: '' name: libgnomecanvas2-common section: x11 version: 2.30.3-1ubuntu1 libgnomecanvas2-dev: description: powerful object-oriented display engine - development files downloads: 195 homepage_uri: '' name: libgnomecanvas2-dev section: libdevel version: 2.30.3-1ubuntu1 libgnomekbd-common: description: GNOME library to manage keyboard configuration - common files downloads: 7367 homepage_uri: '' name: libgnomekbd-common section: libs version: 3.4.0.2-1 libgnomekbd7: description: GNOME library to manage keyboard configuration - shared library downloads: 10484 homepage_uri: '' name: libgnomekbd7 section: libs version: 3.4.0.2-1 libgnomeui-0: description: GNOME user interface library - runtime files downloads: 28160 homepage_uri: '' name: libgnomeui-0 section: libs version: 2.24.5-2ubuntu2 libgnomeui-common: description: GNOME user interface library - common files downloads: 5 homepage_uri: '' name: libgnomeui-common section: libs version: 2.24.5-2ubuntu2 libgnomeui-dev: description: GNOME user interface library - development files downloads: 155 homepage_uri: '' name: libgnomeui-dev section: libdevel version: 2.24.5-2ubuntu2 libgnomevfs2-0: description: GNOME Virtual File System (runtime libraries) downloads: 37623 homepage_uri: '' name: libgnomevfs2-0 section: libs version: 1:2.24.4-1ubuntu2 libgnomevfs2-common: description: GNOME Virtual File System (common files) downloads: 16 homepage_uri: '' name: libgnomevfs2-common section: devel version: 1:2.24.4-1ubuntu2 libgnomevfs2-dev: description: GNOME Virtual File System library (development files) downloads: 262 homepage_uri: '' name: libgnomevfs2-dev section: libdevel version: 1:2.24.4-1ubuntu2 libgnutls-dev: description: GNU TLS library - development files downloads: 2912 homepage_uri: http://www.gnutls.org/ name: libgnutls-dev section: libdevel version: 2.12.14-5ubuntu3 libgnutls-openssl27: description: GNU TLS library - OpenSSL wrapper downloads: 2421 homepage_uri: http://www.gnutls.org/ name: libgnutls-openssl27 section: libs version: 2.12.14-5ubuntu3 libgnutls26: description: GNU TLS library - runtime library downloads: 94484 homepage_uri: http://www.gnutls.org/ name: libgnutls26 section: libs version: 2.12.14-5ubuntu3 libgnutls26:i386: description: GNU TLS library - runtime library downloads: 0 homepage_uri: http://www.gnutls.org/ name: libgnutls26:i386 section: libs version: 2.12.14-5ubuntu3 libgnutlsxx27: description: GNU TLS library - C++ runtime library downloads: 1575 homepage_uri: http://www.gnutls.org/ name: libgnutlsxx27 section: libs version: 2.12.14-5ubuntu3 libgoa-1.0-0: description: library for GNOME Online Accounts downloads: 8078 homepage_uri: https://live.gnome.org/OnlineAccounts name: libgoa-1.0-0 section: libs version: 3.4.0-0ubuntu1 libgoa-1.0-common: description: library for GNOME Online Accounts - common files downloads: 7857 homepage_uri: https://live.gnome.org/OnlineAccounts name: libgoa-1.0-common section: libs version: 3.4.0-0ubuntu1 libgomp1: description: GCC OpenMP (GOMP) support library downloads: 37166 homepage_uri: http://gcc.gnu.org/ name: libgomp1 section: libs version: 4.6.3-1ubuntu5 libgomp1:i386: description: GCC OpenMP (GOMP) support library downloads: 0 homepage_uri: http://gcc.gnu.org/ name: libgomp1:i386 section: libs version: 4.6.3-1ubuntu5 libgoogle-collections-java: description: suite of collections and related goodies for Java 5.0 downloads: 25 homepage_uri: http://code.google.com/p/google-collections/ name: libgoogle-collections-java section: universe/libs version: 1.0-2 libgpg-error-dev: description: library for common error values and messages in GnuPG components (development) downloads: 3104 homepage_uri: http://www.gnupg.org/related_software/libgpg-error/ name: libgpg-error-dev section: libdevel version: 1.10-2ubuntu1 libgpg-error0: description: library for common error values and messages in GnuPG components downloads: 82490 homepage_uri: http://www.gnupg.org/related_software/libgpg-error/ name: libgpg-error0 section: libs version: 1.10-2ubuntu1 libgpg-error0:i386: description: library for common error values and messages in GnuPG components downloads: 0 homepage_uri: http://www.gnupg.org/related_software/libgpg-error/ name: libgpg-error0:i386 section: libs version: 1.10-2ubuntu1 libgpgme11: description: GPGME - GnuPG Made Easy downloads: 39396 homepage_uri: '' name: libgpgme11 section: libs version: 1.2.0-1.4ubuntu2 libgphoto2-2: description: gphoto2 digital camera library downloads: 40261 homepage_uri: http://www.gphoto.org/proj/libgphoto2/ name: libgphoto2-2 section: libs version: 2.4.13-1ubuntu1 libgphoto2-2:i386: description: gphoto2 digital camera library downloads: 0 homepage_uri: http://www.gphoto.org/proj/libgphoto2/ name: libgphoto2-2:i386 section: libs version: 2.4.13-1ubuntu1 libgphoto2-l10n: description: gphoto2 digital camera library - localized messages downloads: 1 homepage_uri: http://www.gphoto.org/proj/libgphoto2/ name: libgphoto2-l10n section: localization version: 2.4.13-1ubuntu1 libgphoto2-port0: description: gphoto2 digital camera port library downloads: 40174 homepage_uri: http://www.gphoto.org/proj/libgphoto2/ name: libgphoto2-port0 section: libs version: 2.4.13-1ubuntu1 libgphoto2-port0:i386: description: gphoto2 digital camera port library downloads: 0 homepage_uri: http://www.gphoto.org/proj/libgphoto2/ name: libgphoto2-port0:i386 section: libs version: 2.4.13-1ubuntu1 libgpm2: description: General Purpose Mouse - shared library downloads: 37225 homepage_uri: http://unix.schottelius.org/gpm/ name: libgpm2 section: libs version: 1.20.4-4 libgpm2:i386: description: General Purpose Mouse - shared library downloads: 0 homepage_uri: http://unix.schottelius.org/gpm/ name: libgpm2:i386 section: libs version: 1.20.4-4 libgpod-common: description: common files for libgpod downloads: 14508 homepage_uri: http://www.gtkpod.org/wiki/Libgpod name: libgpod-common section: libs version: 0.8.2-4 libgpod4: description: library to read and write songs and artwork to an iPod downloads: 20074 homepage_uri: http://www.gtkpod.org/wiki/Libgpod name: libgpod4 section: libs version: 0.8.2-4 libgraph4: description: rich set of graph drawing tools - graph library downloads: 3762 homepage_uri: http://www.graphviz.org/ name: libgraph4 section: libs version: 2.26.3-10ubuntu1 libgraphicsmagick++3: description: format-independent image processing - C++ shared library downloads: 64 homepage_uri: '' name: libgraphicsmagick++3 section: universe/libs version: 1.3.12-1.1build1 libgraphicsmagick3: description: format-independent image processing - C shared library downloads: 6346 homepage_uri: '' name: libgraphicsmagick3 section: universe/libs version: 1.3.12-1.1build1 libgraphviz-dev: description: graphviz libs and headers against which to build applications downloads: 259 homepage_uri: http://www.graphviz.org/ name: libgraphviz-dev section: libdevel version: 2.26.3-10ubuntu1 libgrip0: description: Shared library providing multitouch gestures to GTK+ apps. downloads: 0 homepage_uri: https://launchpad.net/libgrip name: libgrip0 section: libs version: 0.3.4-0ubuntu2~ubuntu12.04.1 libgs9: description: interpreter for the PostScript language and for PDF - Library downloads: 12326 homepage_uri: http://www.ghostscript.com/ name: libgs9 section: libs version: 9.05~dfsg-0ubuntu4 libgs9-common: description: interpreter for the PostScript language and for PDF - common files downloads: 0 homepage_uri: http://www.ghostscript.com/ name: libgs9-common section: libs version: 9.05~dfsg-0ubuntu4 libgsl0-dev: description: GNU Scientific Library (GSL) -- development package downloads: 496 homepage_uri: http://www.gnu.org/software/gsl name: libgsl0-dev section: libdevel version: 1.15+dfsg-1build1 libgsl0ldbl: description: GNU Scientific Library (GSL) -- library package downloads: 5499 homepage_uri: http://www.gnu.org/software/gsl name: libgsl0ldbl section: math version: 1.15+dfsg-1build1 libgsm1: description: Shared libraries for GSM speech compressor downloads: 9419 homepage_uri: '' name: libgsm1 section: libs version: 1.0.13-3 libgssapi-krb5-2: description: MIT Kerberos runtime libraries - krb5 GSS-API Mechanism downloads: 83523 homepage_uri: http://web.mit.edu/kerberos/ name: libgssapi-krb5-2 section: libs version: 1.10+dfsg~beta1-2ubuntu0.1 libgssapi-krb5-2:i386: description: MIT Kerberos runtime libraries - krb5 GSS-API Mechanism downloads: 0 homepage_uri: http://web.mit.edu/kerberos/ name: libgssapi-krb5-2:i386 section: libs version: 1.10+dfsg~beta1-2ubuntu0.1 libgssapi3-heimdal: description: Heimdal Kerberos - GSSAPI support library downloads: 135 homepage_uri: http://www.h5l.org/ name: libgssapi3-heimdal section: libs version: 1.6~git20120311.dfsg.1-2 libgssapi3-heimdal:i386: description: Heimdal Kerberos - GSSAPI support library downloads: 0 homepage_uri: http://www.h5l.org/ name: libgssapi3-heimdal:i386 section: libs version: 1.6~git20120311.dfsg.1-2 libgssdp-1.0-3: description: GObject-based library for SSDP downloads: 3568 homepage_uri: http://www.gupnp.org name: libgssdp-1.0-3 section: libs version: 0.12.1-2 libgssglue1: description: mechanism-switch gssapi library downloads: 13251 homepage_uri: http://www.citi.umich.edu/projects/nfsv4/linux/ name: libgssglue1 section: libs version: 0.3-4 libgssrpc4: description: MIT Kerberos runtime libraries - GSS enabled ONCRPC downloads: 11945 homepage_uri: http://web.mit.edu/kerberos/ name: libgssrpc4 section: libs version: 1.10+dfsg~beta1-2ubuntu0.1 libgstreamer-plugins-base0.10-0: description: GStreamer libraries from the "base" set downloads: 44173 homepage_uri: http://gstreamer.freedesktop.org name: libgstreamer-plugins-base0.10-0 section: libs version: 0.10.36-1 libgstreamer-plugins-base0.10-0:i386: description: GStreamer libraries from the "base" set downloads: 0 homepage_uri: http://gstreamer.freedesktop.org name: libgstreamer-plugins-base0.10-0:i386 section: libs version: 0.10.36-1 libgstreamer0.10-0: description: Core GStreamer libraries and elements downloads: 45345 homepage_uri: http://gstreamer.freedesktop.org name: libgstreamer0.10-0 section: libs version: 0.10.36-1ubuntu1 libgstreamer0.10-0:i386: description: Core GStreamer libraries and elements downloads: 0 homepage_uri: http://gstreamer.freedesktop.org name: libgstreamer0.10-0:i386 section: libs version: 0.10.36-1ubuntu1 libgtextutils0: description: Gordon Text_utils library downloads: 1 homepage_uri: http://hannonlab.cshl.edu/fastx_toolkit/ name: libgtextutils0 section: universe/libs version: 0.6.2-1 libgtk-3-0: description: GTK+ graphical user interface library downloads: 15534 homepage_uri: http://www.gtk.org/ name: libgtk-3-0 section: libs version: 3.4.2-0ubuntu0.2 libgtk-3-bin: description: programs for the GTK+ graphical user interface library downloads: 13404 homepage_uri: http://www.gtk.org/ name: libgtk-3-bin section: misc version: 3.4.2-0ubuntu0.2 libgtk-3-common: description: common files for the GTK+ graphical user interface library downloads: 15279 homepage_uri: http://www.gtk.org/ name: libgtk-3-common section: misc version: 3.4.2-0ubuntu0.2 libgtk2-perl: description: Perl interface to the 2.x series of the Gimp Toolkit library downloads: 17546 homepage_uri: http://gtk2-perl.sourceforge.net/ name: libgtk2-perl section: perl version: 2:1.223-1build3 libgtk2.0-0: description: GTK+ graphical user interface library downloads: 52473 homepage_uri: http://www.gtk.org/ name: libgtk2.0-0 section: libs version: 2.24.10-0ubuntu6 libgtk2.0-0:i386: description: GTK+ graphical user interface library downloads: 0 homepage_uri: http://www.gtk.org/ name: libgtk2.0-0:i386 section: libs version: 2.24.10-0ubuntu6 libgtk2.0-bin: description: programs for the GTK+ graphical user interface library downloads: 31537 homepage_uri: http://www.gtk.org/ name: libgtk2.0-bin section: misc version: 2.24.10-0ubuntu6 libgtk2.0-common: description: common files for the GTK+ graphical user interface library downloads: 51408 homepage_uri: http://www.gtk.org/ name: libgtk2.0-common section: misc version: 2.24.10-0ubuntu6 libgtk2.0-dev: description: development files for the GTK+ library downloads: 2991 homepage_uri: http://www.gtk.org/ name: libgtk2.0-dev section: libdevel version: 2.24.10-0ubuntu6 libgtkmm-3.0-1: description: C++ wrappers for GTK+ (shared libraries) downloads: 173 homepage_uri: http://www.gtkmm.org/ name: libgtkmm-3.0-1 section: libs version: 3.4.0-0ubuntu1 libgtksourceview-3.0-0: description: shared libraries for the GTK+ syntax highlighting widget downloads: 5166 homepage_uri: http://projects.gnome.org/gtksourceview/ name: libgtksourceview-3.0-0 section: libs version: 3.4.2-0ubuntu1 libgtksourceview-3.0-common: description: common files for the GTK+ syntax highlighting widget downloads: 4194 homepage_uri: http://projects.gnome.org/gtksourceview/ name: libgtksourceview-3.0-common section: libs version: 3.4.2-0ubuntu1 libgtkspell-3-0: description: spell-checking addon for GTK's TextView widget downloads: 52 homepage_uri: http://gtkspell.sourceforge.net/ name: libgtkspell-3-0 section: libs version: 3.0.0~hg20110814-1 libgtop2-7: description: gtop system monitoring library (shared) downloads: 35432 homepage_uri: '' name: libgtop2-7 section: libs version: 2.28.4-2 libgtop2-common: description: gtop system monitoring library (common) downloads: 29978 homepage_uri: '' name: libgtop2-common section: libs version: 2.28.4-2 libgucharmap-2-90-7: description: Unicode browser widget library (shared library) downloads: 3886 homepage_uri: http://live.gnome.org/Gucharmap name: libgucharmap-2-90-7 section: libs version: 1:3.4.1.1-0ubuntu1 libgudev-1.0-0: description: GObject-based wrapper library for libudev downloads: 44090 homepage_uri: http://www.kernel.org/pub/linux/utils/kernel/hotplug/udev.html name: libgudev-1.0-0 section: libs version: 1:175-0ubuntu9 libgudev-1.0-0:i386: description: GObject-based wrapper library for libudev downloads: 0 homepage_uri: http://www.kernel.org/pub/linux/utils/kernel/hotplug/udev.html name: libgudev-1.0-0:i386 section: libs version: 1:175-0ubuntu9 libgupnp-1.0-4: description: GObject-based library for UPnP downloads: 3564 homepage_uri: http://www.gupnp.org name: libgupnp-1.0-4 section: libs version: 0.18.1-2 libgupnp-igd-1.0-4: description: library to handle UPnP IGD port mapping downloads: 3098 homepage_uri: http://www.gupnp.org/ name: libgupnp-igd-1.0-4 section: libs version: 0.2.1-2 libgutenprint2: description: runtime for the Gutenprint printer driver library downloads: 11599 homepage_uri: '' name: libgutenprint2 section: libs version: 5.2.8~pre1-0ubuntu2 libgvc5: description: rich set of graph drawing tools - gvc library downloads: 3789 homepage_uri: http://www.graphviz.org/ name: libgvc5 section: libs version: 2.26.3-10ubuntu1 libgvpr1: description: rich set of graph drawing tools - gvpr library downloads: 2163 homepage_uri: http://www.graphviz.org/ name: libgvpr1 section: libs version: 2.26.3-10ubuntu1 libgweather-3-0: description: GWeather shared library downloads: 7369 homepage_uri: '' name: libgweather-3-0 section: libs version: 3.4.1-0ubuntu1 libgweather-common: description: GWeather common files downloads: 6027 homepage_uri: '' name: libgweather-common section: libs version: 3.4.1-0ubuntu1 libgwibber-gtk2: description: Gwibber GTK Widgets downloads: 0 homepage_uri: https://launchpad.net/gwibber name: libgwibber-gtk2 section: libs version: 3.4.1-0ubuntu1 libgwibber2: description: Gwibber - shared library downloads: 0 homepage_uri: https://launchpad.net/gwibber name: libgwibber2 section: libs version: 3.4.1-0ubuntu1 libhal1: description: Hardware Abstraction Layer - shared library downloads: 24663 homepage_uri: http://hal.freedesktop.org/ name: libhal1 section: universe/libs version: 0.5.14-8 libhamcrest-java: description: library of matchers for building test expressions downloads: 1000 homepage_uri: http://code.google.com/p/hamcrest/ name: libhamcrest-java section: libs version: 1.1-8 libhawtjni-runtime-java: description: Java library that provide JNI code generation downloads: 145 homepage_uri: http://fusesource.com/forge/sites/hawtjni/ name: libhawtjni-runtime-java section: universe/java version: 1.0~+git0c502e20c4-3 libhcrypto4-heimdal: description: Heimdal Kerberos - crypto library downloads: 156 homepage_uri: http://www.h5l.org/ name: libhcrypto4-heimdal section: libs version: 1.6~git20120311.dfsg.1-2 libhcrypto4-heimdal:i386: description: Heimdal Kerberos - crypto library downloads: 0 homepage_uri: http://www.h5l.org/ name: libhcrypto4-heimdal:i386 section: libs version: 1.6~git20120311.dfsg.1-2 libhdf5-serial-1.8.4: description: Hierarchical Data Format 5 (HDF5) - runtime files - serial version downloads: 0 homepage_uri: http://hdfgroup.org/HDF5/ name: libhdf5-serial-1.8.4 section: universe/libs version: 1.8.4-patch1-3ubuntu2 libhdf5-serial-dev: description: Hierarchical Data Format 5 (HDF5) - development files - serial version downloads: 123 homepage_uri: http://hdfgroup.org/HDF5/ name: libhdf5-serial-dev section: universe/libdevel version: 1.8.4-patch1-3ubuntu2 libhdfs0: description: JNI Bindings to access Hadoop HDFS from C downloads: 0 homepage_uri: http://hadoop.apache.org/core/ name: libhdfs0 section: misc version: 0.20.2+923.256-1~maverick-cdh3 libhdfs0-dev: description: Development support for libhdfs0 downloads: 0 homepage_uri: http://hadoop.apache.org/core/ name: libhdfs0-dev section: libdevel version: 0.20.2+923.256-1~maverick-cdh3 libheimbase1-heimdal: description: Heimdal Kerberos - Base library downloads: 124 homepage_uri: http://www.h5l.org/ name: libheimbase1-heimdal section: libs version: 1.6~git20120311.dfsg.1-2 libheimbase1-heimdal:i386: description: Heimdal Kerberos - Base library downloads: 0 homepage_uri: http://www.h5l.org/ name: libheimbase1-heimdal:i386 section: libs version: 1.6~git20120311.dfsg.1-2 libheimntlm0-heimdal: description: Heimdal Kerberos - NTLM support library downloads: 159 homepage_uri: http://www.h5l.org/ name: libheimntlm0-heimdal section: libs version: 1.6~git20120311.dfsg.1-2 libheimntlm0-heimdal:i386: description: Heimdal Kerberos - NTLM support library downloads: 0 homepage_uri: http://www.h5l.org/ name: libheimntlm0-heimdal:i386 section: libs version: 1.6~git20120311.dfsg.1-2 libhpdf-2.2.1: description: C library for generating pdf files downloads: 30 homepage_uri: http://www.libharu.org name: libhpdf-2.2.1 section: universe/libs version: 2.2.1-1 libhpmud0: description: HP Multi-Point Transport Driver (hpmud) run-time libraries downloads: 22962 homepage_uri: http://hplipopensource.com/hplip-web/index.html name: libhpmud0 section: libs version: 3.12.2-1ubuntu3 libhsqldb-java: description: Java SQL database engine downloads: 769 homepage_uri: '' name: libhsqldb-java section: libs version: 1.8.0.10-9ubuntu2 libhtml-form-perl: description: module that represents an HTML form element downloads: 869 homepage_uri: http://search.cpan.org/dist/HTML-Form/ name: libhtml-form-perl section: perl version: 6.00-1 libhtml-format-perl: description: module for transforming HTML into various formats downloads: 2414 homepage_uri: http://search.cpan.org/dist/HTML-Format/ name: libhtml-format-perl section: perl version: 2.10-1 libhtml-parser-perl: description: collection of modules that parse HTML text documents downloads: 25434 homepage_uri: http://search.cpan.org/dist/HTML-Parser/ name: libhtml-parser-perl section: perl version: 3.69-1build1 libhtml-tagset-perl: description: Data tables pertaining to HTML downloads: 25059 homepage_uri: http://search.cpan.org/dist/HTML-Tagset/ name: libhtml-tagset-perl section: perl version: 3.20-2 libhtml-template-perl: description: module for using HTML Templates with Perl downloads: 5445 homepage_uri: http://search.cpan.org/dist/HTML-Template/ name: libhtml-template-perl section: perl version: 2.10-1 libhtml-tree-perl: description: Perl module to represent and create HTML syntax trees downloads: 22482 homepage_uri: http://search.cpan.org/dist/HTML-Tree/ name: libhtml-tree-perl section: perl version: 4.2-1 libhttp-cookies-perl: description: HTTP cookie jars downloads: 5256 homepage_uri: http://search.cpan.org/dist/HTTP-Cookies/ name: libhttp-cookies-perl section: perl version: 6.00-2 libhttp-daemon-perl: description: simple http server class downloads: 1195 homepage_uri: http://search.cpan.org/dist/HTTP-Daemon/ name: libhttp-daemon-perl section: perl version: 6.00-1 libhttp-date-perl: description: module of date conversion routines downloads: 5326 homepage_uri: http://search.cpan.org/dist/HTTP-Date/ name: libhttp-date-perl section: perl version: 6.00-1 libhttp-message-perl: description: perl interface to HTTP style messages downloads: 5299 homepage_uri: http://search.cpan.org/dist/HTTP-Message/ name: libhttp-message-perl section: perl version: 6.01-1 libhttp-negotiate-perl: description: implementation of content negotiation downloads: 5255 homepage_uri: http://search.cpan.org/dist/HTTP-Negotiate/ name: libhttp-negotiate-perl section: perl version: 6.00-2 libhttpclient-java: description: HTTP/1.1 compliant HTTP agent implementation downloads: 116 homepage_uri: http://hc.apache.org/httpcomponents-client/index.html name: libhttpclient-java section: universe/java version: 4.1.1-1 libhttpcore-java: description: set of low level HTTP transport components for Java downloads: 93 homepage_uri: http://hc.apache.org/httpcomponents-core/index.html name: libhttpcore-java section: universe/java version: 4.1.4-1 libhunspell-1.3-0: description: spell checker and morphological analyzer (shared library) downloads: 3059 homepage_uri: http://hunspell.sourceforge.net/ name: libhunspell-1.3-0 section: libs version: 1.3.2-4 libhx509-5-heimdal: description: Heimdal Kerberos - X509 support library downloads: 339 homepage_uri: http://www.h5l.org/ name: libhx509-5-heimdal section: libs version: 1.6~git20120311.dfsg.1-2 libhx509-5-heimdal:i386: description: Heimdal Kerberos - X509 support library downloads: 0 homepage_uri: http://www.h5l.org/ name: libhx509-5-heimdal:i386 section: libs version: 1.6~git20120311.dfsg.1-2 libhyphen0: description: ALTLinux hyphenation library - shared library downloads: 41 homepage_uri: '' name: libhyphen0 section: libs version: 2.8.3-1 libibus-1.0-0: description: Intelligent Input Bus - shared library downloads: 401 homepage_uri: http://code.google.com/p/ibus/ name: libibus-1.0-0 section: libs version: 1.4.1-3ubuntu1 libibus-1.0-0:i386: description: Intelligent Input Bus - shared library downloads: 0 homepage_uri: http://code.google.com/p/ibus/ name: libibus-1.0-0:i386 section: libs version: 1.4.1-3ubuntu1 libibverbs-dev: description: Development files for the libibverbs library downloads: 1254 homepage_uri: http://www.openfabrics.org/ name: libibverbs-dev section: libdevel version: 1.1.5-1ubuntu1 libibverbs1: description: Library for direct userspace use of RDMA (InfiniBand/iWARP) downloads: 1349 homepage_uri: http://www.openfabrics.org/ name: libibverbs1 section: libs version: 1.1.5-1ubuntu1 libical0: description: iCalendar library implementation in C (runtime) downloads: 30310 homepage_uri: http://freeassociation.sourceforge.net name: libical0 section: libs version: 0.48-1ubuntu3 libice-dev: description: X11 Inter-Client Exchange library (development headers) downloads: 3671 homepage_uri: '' name: libice-dev section: libdevel version: 2:1.0.7-2build1 libice6: description: X11 Inter-Client Exchange library downloads: 56047 homepage_uri: '' name: libice6 section: libs version: 2:1.0.7-2build1 libice6:i386: description: X11 Inter-Client Exchange library downloads: 0 homepage_uri: '' name: libice6:i386 section: libs version: 2:1.0.7-2build1 libicu-dev: description: Development files for International Components for Unicode downloads: 762 homepage_uri: http://www.icu-project.org name: libicu-dev section: libdevel version: 4.8.1.1-3 libicu48: description: International Components for Unicode downloads: 3750 homepage_uri: http://www.icu-project.org name: libicu48 section: libs version: 4.8.1.1-3 libicu4j-4.4-java: description: Library for Unicode support and internalisation downloads: 691 homepage_uri: http://www.icu-project.org/ name: libicu4j-4.4-java section: universe/java version: 4.4.2.2-1 libicu4j-java: description: Library for unicode support and internalisation downloads: 0 homepage_uri: http://www.icu-project.org/ name: libicu4j-java section: universe/devel version: 4.2.1.1-1 libidl-common: description: library for parsing CORBA IDL files (common files) downloads: 0 homepage_uri: '' name: libidl-common section: libs version: 0.8.14-0.2ubuntu2 libidl-dev: description: development files for programs that use libIDL downloads: 531 homepage_uri: '' name: libidl-dev section: libdevel version: 0.8.14-0.2ubuntu2 libidl0: description: library for parsing CORBA IDL files downloads: 26535 homepage_uri: '' name: libidl0 section: libs version: 0.8.14-0.2ubuntu2 libidn11: description: GNU Libidn library, implementation of IETF IDN specifications downloads: 62674 homepage_uri: http://www.gnu.org/software/libidn/ name: libidn11 section: libs version: 1.23-2 libidn11-dev: description: Development files for GNU Libidn, an IDN library downloads: 2178 homepage_uri: http://www.gnu.org/software/libidn/ name: libidn11-dev section: libdevel version: 1.23-2 libidn11:i386: description: GNU Libidn library, implementation of IETF IDN specifications downloads: 0 homepage_uri: http://www.gnu.org/software/libidn/ name: libidn11:i386 section: libs version: 1.23-2 libido3-0.1-0: description: Shared library providing extra gtk menu items for display in downloads: 141 homepage_uri: https://launchpad.net/ido name: libido3-0.1-0 section: libs version: 0.3.4-0ubuntu1 libiec61883-0: description: an partial implementation of IEC 61883 downloads: 61 homepage_uri: '' name: libiec61883-0 section: libs version: 1.2.0-0.1ubuntu1 libiec61883-0:i386: description: an partial implementation of IEC 61883 downloads: 0 homepage_uri: '' name: libiec61883-0:i386 section: libs version: 1.2.0-0.1ubuntu1 libieee1284-3: description: cross-platform library for parallel port access downloads: 23399 homepage_uri: '' name: libieee1284-3 section: libs version: 0.2.11-10build1 libieee1284-3:i386: description: cross-platform library for parallel port access downloads: 0 homepage_uri: '' name: libieee1284-3:i386 section: libs version: 0.2.11-10build1 libijs-0.35: description: 'IJS raster image transport protocol: shared library' downloads: 8343 homepage_uri: http://www.linuxprinting.org/ijs/ name: libijs-0.35 section: libs version: 0.35-8 libilmbase6: description: several utility libraries from ILM used by OpenEXR downloads: 288 homepage_uri: http://www.openexr.com name: libilmbase6 section: libs version: 1.0.1-3build2 libimobiledevice2: description: Library for communicating with the iPhone and iPod Touch downloads: 14843 homepage_uri: http://libimobiledevice.org/ name: libimobiledevice2 section: libs version: 1.1.1-4 libindicate-gtk3: description: library for raising indicators via DBus - GTK+ bindings downloads: 284 homepage_uri: https://launchpad.net/libindicate name: libindicate-gtk3 section: libs version: 0.6.92-0ubuntu1 libindicate5: description: library for raising indicators via DBus downloads: 452 homepage_uri: https://launchpad.net/libindicate name: libindicate5 section: libs version: 0.6.92-0ubuntu1 libindicator-messages-status-provider1: description: indicator status provider - shared library downloads: 81 homepage_uri: https://launchpad.net/indicator-messages name: libindicator-messages-status-provider1 section: libs version: 0.6.0-0ubuntu1 libindicator3-7: description: panel indicator applet - shared library downloads: 219 homepage_uri: https://launchpad.net/libindicator name: libindicator3-7 section: libs version: 0.5.0-0ubuntu1 libindicator7: description: panel indicator applet - shared library downloads: 70 homepage_uri: https://launchpad.net/libindicator name: libindicator7 section: libs version: 0.5.0-0ubuntu1 libio-pty-perl: description: Perl module for pseudo tty IO downloads: 4862 homepage_uri: '' name: libio-pty-perl section: perl version: 1:1.08-1build2 libio-socket-inet6-perl: description: object interface for AF_INET6 domain sockets downloads: 9137 homepage_uri: http://search.cpan.org/dist/IO-Socket-INET6/ name: libio-socket-inet6-perl section: perl version: 2.69-2 libio-socket-ssl-perl: description: Perl module implementing object oriented interface to SSL sockets downloads: 5741 homepage_uri: http://search.cpan.org/dist/IO-Socket-SSL/ name: libio-socket-ssl-perl section: perl version: 1.53-1 libio-string-perl: description: Emulate IO::File interface for in-core strings downloads: 3238 homepage_uri: '' name: libio-string-perl section: perl version: 1.08-2 libio-stringy-perl: description: Perl modules for IO from scalars and arrays downloads: 6820 homepage_uri: http://search.cpan.org/dist/IO-stringy/ name: libio-stringy-perl section: perl version: 2.110-5 libipc-run-perl: description: Perl module for running processes downloads: 1669 homepage_uri: http://search.cpan.org/dist/IPC-Run/ name: libipc-run-perl section: perl version: 0.90-1 libipc-sharelite-perl: description: Perl module that provides a simple interface to shared memory downloads: 639 homepage_uri: http://search.cpan.org/dist/IPC-ShareLite/ name: libipc-sharelite-perl section: perl version: 0.17-1+build4 libisc83: description: ISC Shared Library used by BIND downloads: 1908 homepage_uri: '' name: libisc83 section: libs version: 1:9.8.1.dfsg.P1-4ubuntu0.1 libisccc80: description: Command Channel Library used by BIND downloads: 1200 homepage_uri: '' name: libisccc80 section: libs version: 1:9.8.1.dfsg.P1-4ubuntu0.1 libisccfg82: description: Config File Handling Library used by BIND downloads: 1907 homepage_uri: '' name: libisccfg82 section: libs version: 1:9.8.1.dfsg.P1-4ubuntu0.1 libisofs6: description: library to create ISO9660 images downloads: 12871 homepage_uri: http://libburnia-project.org name: libisofs6 section: libs version: 1.1.6-1ubuntu1 libitext1-java: description: Java Library to generate PDF on the Fly downloads: 8 homepage_uri: http://www.lowagie.com/iText/ name: libitext1-java section: universe/java version: 1.4-5 libiw30: description: Wireless tools - library downloads: 10789 homepage_uri: http://www.hpl.hp.com/personal/Jean_Tourrilhes/Linux/Tools.html name: libiw30 section: libs version: 30~pre9-5ubuntu2 libjack-jackd2-0: description: JACK Audio Connection Kit (libraries) downloads: 5632 homepage_uri: http://jackaudio.org/ name: libjack-jackd2-0 section: libs version: 1.9.8~dfsg.1-1ubuntu1 libjack-jackd2-0:i386: description: JACK Audio Connection Kit (libraries) downloads: 0 homepage_uri: http://jackaudio.org/ name: libjack-jackd2-0:i386 section: libs version: 1.9.8~dfsg.1-1ubuntu1 libjansi-java: description: Java library for generating and interpreting ANSI escape sequences downloads: 310 homepage_uri: http://jansi.fusesource.org/ name: libjansi-java section: universe/java version: 1.4-3 libjansi-native-java: description: Java library that allows to use ANSI escape sequences - Native downloads: 145 homepage_uri: http://jansi.fusesource.org/ name: libjansi-native-java section: universe/java version: 1.0-3 libjasper-java: description: Implementation of the JSP Container downloads: 0 homepage_uri: http://tomcat.apache.org name: libjasper-java section: universe/java version: 5.5.33-2 libjasper1: description: JasPer JPEG-2000 runtime library downloads: 26321 homepage_uri: '' name: libjasper1 section: libs version: 1.900.1-13 libjasper1:i386: description: JasPer JPEG-2000 runtime library downloads: 0 homepage_uri: '' name: libjasper1:i386 section: libs version: 1.900.1-13 libjavascriptcoregtk-1.0-0: description: Javascript engine library for GTK+ downloads: 2263 homepage_uri: http://webkitgtk.org/ name: libjavascriptcoregtk-1.0-0 section: libs version: 1.8.0-0ubuntu2 libjavascriptcoregtk-3.0-0: description: Javascript engine library for GTK+ downloads: 10097 homepage_uri: http://webkitgtk.org/ name: libjavascriptcoregtk-3.0-0 section: libs version: 1.8.0-0ubuntu2 libjaxen-java: description: Java XPath engine downloads: 256 homepage_uri: http://jaxen.org name: libjaxen-java section: libs version: 1.1.3-1 libjaxme-java: description: implementation of the JAXB specification for Java/XML binding downloads: 215 homepage_uri: http://ws.apache.org/jaxme/ name: libjaxme-java section: libs version: 0.5.2+dfsg-6 libjaxp1.3-java: description: Java XML parser and transformer APIs (DOM, SAX, JAXP, TrAX) downloads: 8312 homepage_uri: http://java.sun.com/xml/jaxp/index.jsp name: libjaxp1.3-java section: libs version: 1.3.05-2ubuntu2 libjbig2dec0: description: JBIG2 decoder library - shared libraries downloads: 3554 homepage_uri: http://jbig2dec.sourceforge.net/ name: libjbig2dec0 section: libs version: 0.11-1ubuntu1 libjdom1-java: description: lightweight and fast library using XML downloads: 239 homepage_uri: http://jdom.org/ name: libjdom1-java section: libs version: 1.1.2+dfsg-2 libjetty-java: description: Java servlet engine and webserver -- core libraries downloads: 356 homepage_uri: http://jetty.mortbay.com/ name: libjetty-java section: universe/java version: 6.1.24-6ubuntu0.12.04.1 libjffi-java: description: Java Foreign Function Interface downloads: 38 homepage_uri: http://github.com/wmeissner/jffi name: libjffi-java section: universe/java version: 1.0.2-8ubuntu1 libjffi-jni: description: Java Foreign Function Interface (JNI library) downloads: 50 homepage_uri: http://github.com/wmeissner/jffi name: libjffi-jni section: universe/java version: 1.0.2-8ubuntu1 libjhdf5-java: description: Java HDF5 Object Package downloads: 39 homepage_uri: http://www.hdfgroup.org/hdf-java-html/ name: libjhdf5-java section: universe/java version: 2.6.1-2build1 libjhdf5-jni: description: Java HDF5 Object Package (Java JNI library) downloads: 69 homepage_uri: http://www.hdfgroup.org/hdf-java-html/ name: libjhdf5-jni section: universe/libs version: 2.6.1-2build1 libjline-java: description: Java library for handling console input downloads: 3048 homepage_uri: http://jline.sourceforge.net/ name: libjline-java section: libs version: 1.0-1 libjna-java: description: Dynamic access of native libraries from Java without JNI downloads: 573 homepage_uri: http://jna.dev.java.net/ name: libjna-java section: universe/libs version: 3.2.7-4 libjna-posix-java: description: basic POSIX-like functions for Java downloads: 12 homepage_uri: http://kenai.com/projects/jna-posix name: libjna-posix-java section: universe/java version: 1.0.1-1 libjpeg-dev: description: Independent JPEG Group's JPEG runtime library (dependency package) downloads: 2 homepage_uri: '' name: libjpeg-dev section: libdevel version: 8c-2ubuntu7 libjpeg-turbo8: description: IJG JPEG compliant runtime library. downloads: 1 homepage_uri: http://libjpeg-turbo.virtualgl.org/ name: libjpeg-turbo8 section: libs version: 1.1.90+svn733-0ubuntu4 libjpeg-turbo8-dev: description: Development files for the IJG JPEG library downloads: 1 homepage_uri: http://libjpeg-turbo.virtualgl.org/ name: libjpeg-turbo8-dev section: libdevel version: 1.1.90+svn733-0ubuntu4 libjpeg-turbo8:i386: description: IJG JPEG compliant runtime library. downloads: 0 homepage_uri: http://libjpeg-turbo.virtualgl.org/ name: libjpeg-turbo8:i386 section: libs version: 1.1.90+svn733-0ubuntu4 libjpeg8: description: Independent JPEG Group's JPEG runtime library (dependency package) downloads: 30858 homepage_uri: '' name: libjpeg8 section: libs version: 8c-2ubuntu7 libjpeg8-dev: description: Independent JPEG Group's JPEG runtime library (dependency package) downloads: 1545 homepage_uri: '' name: libjpeg8-dev section: libdevel version: 8c-2ubuntu7 libjpeg8:i386: description: Independent JPEG Group's JPEG runtime library (dependency package) downloads: 0 homepage_uri: '' name: libjpeg8:i386 section: libs version: 8c-2ubuntu7 libjs-jquery: description: JavaScript library for dynamic web applications downloads: 3979 homepage_uri: http://jquery.com/ name: libjs-jquery section: web version: 1.7.1-1ubuntu1 libjsch-java: description: pure Java implementation of the SSH2 protocol downloads: 1770 homepage_uri: http://www.jcraft.com/jsch/ name: libjsch-java section: devel version: 0.1.42-2fakesync1 libjson-glib-1.0-0: description: GLib JSON manipulation library downloads: 25681 homepage_uri: http://live.gnome.org/JsonGlib name: libjson-glib-1.0-0 section: libs version: 0.14.2-1 libjson-perl: description: module for manipulating JSON-formatted data downloads: 1823 homepage_uri: http://search.cpan.org/dist/JSON/ name: libjson-perl section: perl version: 2.53-1 libjson-xs-perl: description: module for manipulating JSON-formatted data (C/XS-accelerated) downloads: 2339 homepage_uri: http://search.cpan.org/dist/JSON-XS/ name: libjson-xs-perl section: perl version: 2.320-1build1 libjson0: description: JSON manipulation library - shared library downloads: 575 homepage_uri: http://oss.metaparadigm.com/json-c/ name: libjson0 section: libs version: 0.9-1ubuntu1 libjson0:i386: description: JSON manipulation library - shared library downloads: 0 homepage_uri: http://oss.metaparadigm.com/json-c/ name: libjson0:i386 section: libs version: 0.9-1ubuntu1 libjsoup-java: description: Java HTML parser that makes sense of real-world HTML soup downloads: 9 homepage_uri: http://jsoup.org/ name: libjsoup-java section: universe/java version: 1.6.1-2 libjsr305-java: description: Java library that provides annotations for software defect detection downloads: 3 homepage_uri: http://code.google.com/p/jsr-305/ name: libjsr305-java section: universe/java version: 0.1~+svn49-4 libjte1: description: Jigdo Template Export - runtime library downloads: 138 homepage_uri: http://www.einval.com/~steve/software/JTE/ name: libjte1 section: libs version: 1.19-1 libjtidy-java: description: JTidy downloads: 26 homepage_uri: http://jtidy.sourceforge.net/ name: libjtidy-java section: universe/libs version: 7+svn20110807-3 libk5crypto3: description: MIT Kerberos runtime libraries - Crypto Library downloads: 77684 homepage_uri: http://web.mit.edu/kerberos/ name: libk5crypto3 section: libs version: 1.10+dfsg~beta1-2ubuntu0.1 libk5crypto3:i386: description: MIT Kerberos runtime libraries - Crypto Library downloads: 0 homepage_uri: http://web.mit.edu/kerberos/ name: libk5crypto3:i386 section: libs version: 1.10+dfsg~beta1-2ubuntu0.1 libkadm5clnt-mit8: description: MIT Kerberos runtime libraries - Administration Clients downloads: 1838 homepage_uri: http://web.mit.edu/kerberos/ name: libkadm5clnt-mit8 section: libs version: 1.10+dfsg~beta1-2ubuntu0.1 libkadm5srv-mit8: description: MIT Kerberos runtime libraries - KDC and Admin Server downloads: 1835 homepage_uri: http://web.mit.edu/kerberos/ name: libkadm5srv-mit8 section: libs version: 1.10+dfsg~beta1-2ubuntu0.1 libkdb5-6: description: MIT Kerberos runtime libraries - Kerberos database downloads: 1388 homepage_uri: http://web.mit.edu/kerberos/ name: libkdb5-6 section: libs version: 1.10+dfsg~beta1-2ubuntu0.1 libkeyutils1: description: Linux Key Management Utilities (library) downloads: 73600 homepage_uri: http://people.redhat.com/~dhowells/keyutils/ name: libkeyutils1 section: misc version: 1.5.2-2 libkeyutils1:i386: description: Linux Key Management Utilities (library) downloads: 0 homepage_uri: http://people.redhat.com/~dhowells/keyutils/ name: libkeyutils1:i386 section: misc version: 1.5.2-2 libklibc: description: minimal libc subset for use with initramfs downloads: 25583 homepage_uri: '' name: libklibc section: libs version: 1.5.25-1ubuntu2 libkms1: description: Userspace interface to kernel DRM buffer management downloads: 315 homepage_uri: '' name: libkms1 section: libs version: 2.4.32-1ubuntu1 libkpathsea5: description: 'TeX Live: path search library for TeX (runtime part)' downloads: 9 homepage_uri: http://www.tug.org/texlive name: libkpathsea5 section: libs version: 2009-11ubuntu2 libkrb5-26-heimdal: description: Heimdal Kerberos - libraries downloads: 438 homepage_uri: http://www.h5l.org/ name: libkrb5-26-heimdal section: libs version: 1.6~git20120311.dfsg.1-2 libkrb5-26-heimdal:i386: description: Heimdal Kerberos - libraries downloads: 0 homepage_uri: http://www.h5l.org/ name: libkrb5-26-heimdal:i386 section: libs version: 1.6~git20120311.dfsg.1-2 libkrb5-3: description: MIT Kerberos runtime libraries downloads: 83479 homepage_uri: http://web.mit.edu/kerberos/ name: libkrb5-3 section: libs version: 1.10+dfsg~beta1-2ubuntu0.1 libkrb5-3:i386: description: MIT Kerberos runtime libraries downloads: 0 homepage_uri: http://web.mit.edu/kerberos/ name: libkrb5-3:i386 section: libs version: 1.10+dfsg~beta1-2ubuntu0.1 libkrb5-dev: description: Headers and development libraries for MIT Kerberos downloads: 3792 homepage_uri: http://web.mit.edu/kerberos/ name: libkrb5-dev section: libdevel version: 1.10+dfsg~beta1-2ubuntu0.1 libkrb5support0: description: MIT Kerberos runtime libraries - Support library downloads: 72904 homepage_uri: http://web.mit.edu/kerberos/ name: libkrb5support0 section: libs version: 1.10+dfsg~beta1-2ubuntu0.1 libkrb5support0:i386: description: MIT Kerberos runtime libraries - Support library downloads: 0 homepage_uri: http://web.mit.edu/kerberos/ name: libkrb5support0:i386 section: libs version: 1.10+dfsg~beta1-2ubuntu0.1 libladr4: description: the LADR deduction library downloads: 4 homepage_uri: http://www.cs.unm.edu/~mccune/mace4/ name: libladr4 section: universe/libs version: 0.0.200902a-2 liblapack-dev: description: library of linear algebra routines 3 - static version downloads: 816 homepage_uri: http://www.netlib.org/lapack/ name: liblapack-dev section: devel version: 3.3.1-1 liblapack3gf: description: library of linear algebra routines 3 - shared version downloads: 4966 homepage_uri: http://www.netlib.org/lapack/ name: liblapack3gf section: libs version: 3.3.1-1 liblaunchpad-integration-3.0-1: description: library for launchpad integration downloads: 0 homepage_uri: '' name: liblaunchpad-integration-3.0-1 section: libs version: 0.1.56 liblaunchpad-integration-common: description: library for launchpad integration common data downloads: 0 homepage_uri: '' name: liblaunchpad-integration-common section: libs version: 0.1.56 liblcms1: description: Little CMS color management library downloads: 48150 homepage_uri: '' name: liblcms1 section: libs version: 1.19.dfsg-1ubuntu3 liblcms1:i386: description: Little CMS color management library downloads: 0 homepage_uri: '' name: liblcms1:i386 section: libs version: 1.19.dfsg-1ubuntu3 liblcms2-2: description: Little CMS 2 color management library downloads: 15581 homepage_uri: http://www.littlecms.com/ name: liblcms2-2 section: libs version: 2.2+git20110628-2ubuntu3 libldap-2.4-2: description: OpenLDAP libraries downloads: 82688 homepage_uri: http://www.openldap.org/ name: libldap-2.4-2 section: libs version: 2.4.28-1.1ubuntu4 libldap-2.4-2:i386: description: OpenLDAP libraries downloads: 0 homepage_uri: http://www.openldap.org/ name: libldap-2.4-2:i386 section: libs version: 2.4.28-1.1ubuntu4 libldap2-dev: description: OpenLDAP development libraries downloads: 2501 homepage_uri: http://www.openldap.org/ name: libldap2-dev section: libdevel version: 2.4.28-1.1ubuntu4 liblightdm-gobject-1-0: description: LightDM GObject client library downloads: 1061 homepage_uri: https://launchpad.net/lightdm name: liblightdm-gobject-1-0 section: x11 version: 1.2.1-0ubuntu1 liblircclient0: description: infra-red remote control support - client library downloads: 26561 homepage_uri: http://lirc.org name: liblircclient0 section: libs version: 0.9.0-0ubuntu1 libllvm3.0: description: Low-Level Virtual Machine (LLVM), runtime library downloads: 35 homepage_uri: http://www.llvm.org/ name: libllvm3.0 section: libs version: 3.0-4ubuntu1 libllvm3.0:i386: description: Low-Level Virtual Machine (LLVM), runtime library downloads: 0 homepage_uri: http://www.llvm.org/ name: libllvm3.0:i386 section: libs version: 3.0-4ubuntu1 liblocale-gettext-perl: description: module using libc functions for internationalization in Perl downloads: 84667 homepage_uri: http://search.cpan.org/dist/gettext/gettext.pm name: liblocale-gettext-perl section: perl version: 1.05-7build1 liblockfile-bin: description: support binaries for and cli utilities based on liblockfile downloads: 10314 homepage_uri: '' name: liblockfile-bin section: utils version: 1.09-3 liblockfile1: description: NFS-safe locking library downloads: 59233 homepage_uri: '' name: liblockfile1 section: libs version: 1.09-3 liblog4j1.2-java: description: Logging library for java downloads: 1166 homepage_uri: http://logging.apache.org/log4j/1.2/ name: liblog4j1.2-java section: libs version: 1.2.16-3ubuntu1 liblouis-data: description: Braille translation library - data downloads: 1065 homepage_uri: http://code.google.com/p/liblouis/ name: liblouis-data section: text version: 2.3.0-3 liblouis2: description: Braille translation library - shared libs downloads: 1323 homepage_uri: http://code.google.com/p/liblouis/ name: liblouis2 section: libs version: 2.3.0-3 liblqr-1-0: description: converts plain array images into multi-size representation downloads: 110 homepage_uri: http://liblqr.wikidot.com/ name: liblqr-1-0 section: libs version: 0.4.1-1.1 libltdl7: description: A system independent dlopen wrapper for GNU libtool downloads: 56813 homepage_uri: http://www.gnu.org/software/libtool/ name: libltdl7 section: libs version: 2.4.2-1ubuntu1 libltdl7:i386: description: A system independent dlopen wrapper for GNU libtool downloads: 0 homepage_uri: http://www.gnu.org/software/libtool/ name: libltdl7:i386 section: libs version: 2.4.2-1ubuntu1 liblua5.1-0: description: Shared library for the Lua interpreter version 5.1 downloads: 7810 homepage_uri: http://www.lua.org name: liblua5.1-0 section: libs version: 5.1.4-12ubuntu1 liblucene2-java: description: Full-text search engine library for Java(TM) downloads: 5270 homepage_uri: http://lucene.apache.org/java/ name: liblucene2-java section: universe/text version: 2.9.4+ds1-4 liblvm2app2.2: description: LVM2 application library downloads: 28 homepage_uri: http://sources.redhat.com/lvm2/ name: liblvm2app2.2 section: libs version: 2.02.66-4ubuntu7.1 liblwp-mediatypes-perl: description: module to guess media type for a file or a URL downloads: 5298 homepage_uri: http://search.cpan.org/dist/LWP-MediaTypes/ name: liblwp-mediatypes-perl section: perl version: 6.01-1 liblwp-protocol-https-perl: description: https driver for LWP::UserAgent downloads: 5216 homepage_uri: http://search.cpan.org/dist/LWP-Protocol-https/ name: liblwp-protocol-https-perl section: perl version: 6.02-1 liblwres80: description: Lightweight Resolver Library used by BIND downloads: 1 homepage_uri: '' name: liblwres80 section: libs version: 1:9.8.1.dfsg.P1-4ubuntu0.1 liblzma-dev: description: XZ-format compression library - development files downloads: 103 homepage_uri: http://tukaani.org/xz/ name: liblzma-dev section: libdevel version: 5.1.1alpha+20110809-3 liblzma5: description: XZ-format compression library downloads: 18267 homepage_uri: http://tukaani.org/xz/ name: liblzma5 section: libs version: 5.1.1alpha+20110809-3 libm17n-0: description: multilingual text processing library - runtime downloads: 1253 homepage_uri: http://www.m17n.org/ name: libm17n-0 section: libs version: 1.6.3-1 libmaa3: description: maa programming library downloads: 779 homepage_uri: http://sourceforge.net/projects/dict/ name: libmaa3 section: libs version: 1.3.1-1 libmad0:i386: description: MPEG audio decoder library downloads: 0 homepage_uri: '' name: libmad0:i386 section: libs version: 0.15.1b-7ubuntu1 libmagic1: description: File type determination library using "magic" numbers downloads: 54754 homepage_uri: http://www.darwinsys.com/file/ name: libmagic1 section: libs version: 5.09-2 libmagickcore4: description: low-level image manipulation library downloads: 318 homepage_uri: http://www.imagemagick.org/ name: libmagickcore4 section: libs version: 8:6.6.9.7-5ubuntu3.1 libmagickcore4-extra: description: low-level image manipulation library - extra codecs downloads: 37 homepage_uri: http://www.imagemagick.org/ name: libmagickcore4-extra section: libs version: 8:6.6.9.7-5ubuntu3.1 libmagickwand4: description: image manipulation library downloads: 0 homepage_uri: http://www.imagemagick.org/ name: libmagickwand4 section: libs version: 8:6.6.9.7-5ubuntu3.1 libmail-sendmail-perl: description: Send email from a perl script downloads: 1915 homepage_uri: http://search.cpan.org/dist/Mail-Sendmail/ name: libmail-sendmail-perl section: perl version: 0.79.16-1 libmailtools-perl: description: Manipulate email in perl programs downloads: 7806 homepage_uri: http://search.cpan.org/dist/MailTools/ name: libmailtools-perl section: perl version: 2.08-1 libmaven-archiver-java: description: Archiver component for Maven downloads: 1 homepage_uri: http://maven.apache.org/shared/maven-archiver/ name: libmaven-archiver-java section: universe/devel version: 2.4-4 libmaven-clean-plugin-java: description: Maven clean plugin downloads: 10 homepage_uri: http://maven.apache.org/plugins/maven-clean-plugin/ name: libmaven-clean-plugin-java section: universe/java version: 2.3-5 libmaven-compiler-plugin-java: description: Maven Compiler plugin downloads: 10 homepage_uri: http://maven.apache.org/plugins/maven-compiler-plugin/ name: libmaven-compiler-plugin-java section: universe/java version: 2.0.2-6 libmaven-dependency-tree-java: description: Maven Dependency Tree downloads: 0 homepage_uri: http://maven.apache.org name: libmaven-dependency-tree-java section: universe/java version: 1.2-4 libmaven-file-management-java: description: Maven File Management API downloads: 0 homepage_uri: http://maven.apache.org/shared/file-management/ name: libmaven-file-management-java section: universe/devel version: 1.2.1-3 libmaven-filtering-java: description: Maven Filtering downloads: 0 homepage_uri: http://maven.apache.org/shared/maven-filtering/ name: libmaven-filtering-java section: universe/devel version: 1.0~beta-2-5 libmaven-install-plugin-java: description: Maven install plugin downloads: 0 homepage_uri: http://maven.apache.org/plugins/maven-install-plugin/ name: libmaven-install-plugin-java section: universe/java version: 2.3-4 libmaven-jar-plugin-java: description: Maven Jar Plugin downloads: 10 homepage_uri: http://maven.apache.org/plugins/maven-jar-plugin/ name: libmaven-jar-plugin-java section: universe/java version: 2.2-6 libmaven-resources-plugin-java: description: Maven resources plugin downloads: 10 homepage_uri: http://maven.apache.org/plugins/maven-resources-plugin/ name: libmaven-resources-plugin-java section: universe/java version: 2.3-7 libmaven-scm-java: description: Maven SCM provides a common API for doing SCM operations downloads: 10 homepage_uri: http://maven.apache.org/scm/ name: libmaven-scm-java section: universe/java version: 1.3-4 libmaven-shade-plugin-java: description: Maven shade plugin downloads: 0 homepage_uri: http://maven.apache.org/plugins/maven-shade-plugin/ name: libmaven-shade-plugin-java section: universe/java version: 1.2.1-4 libmaven-shared-io-java: description: Maven API for I/O support downloads: 0 homepage_uri: http://maven.apache.org/shared/maven-shared-io/ name: libmaven-shared-io-java section: universe/devel version: 1.1-4 libmaven2-core-java: description: Core libraries for Maven2 downloads: 363 homepage_uri: http://maven.apache.org/ name: libmaven2-core-java section: universe/java version: 2.2.1-8 libmeanwhile1: description: open implementation of the Lotus Sametime Community Client protocol downloads: 5641 homepage_uri: http://meanwhile.sourceforge.net/ name: libmeanwhile1 section: libs version: 1.0.2-4ubuntu1 libmetacity-private0: description: library for the Metacity window manager downloads: 20 homepage_uri: '' name: libmetacity-private0 section: libs version: 1:2.34.1-1ubuntu11 libmhash2: description: Library for cryptographic hashing and message authentication downloads: 768 homepage_uri: '' name: libmhash2 section: libs version: 0.9.9.9-1 libmikmod2:i386: description: Portable sound library downloads: 0 homepage_uri: http://mikmod.raphnet.net/ name: libmikmod2:i386 section: universe/libs version: 3.1.12-2 libmime-tools-perl: description: Perl5 modules for MIME-compliant messages downloads: 5595 homepage_uri: http://search.cpan.org/dist/MIME-tools/ name: libmime-tools-perl section: perl version: 5.502-1 libminiupnpc8: description: UPnP IGD client lightweight library downloads: 0 homepage_uri: http://miniupnp.free.fr/ name: libminiupnpc8 section: net version: 1.6-3ubuntu1 libmission-control-plugins0: description: management daemon for Telepathy (library for plugins) downloads: 7784 homepage_uri: http://telepathy.freedesktop.org/wiki/ name: libmission-control-plugins0 section: libs version: 1:5.12.0-0ubuntu2 libmng1: description: Multiple-image Network Graphics library downloads: 19461 homepage_uri: http://www.libmng.com name: libmng1 section: libs version: 1.0.10-3 libmng1:i386: description: Multiple-image Network Graphics library downloads: 0 homepage_uri: http://www.libmng.com name: libmng1:i386 section: libs version: 1.0.10-3 libmodello-java: description: a Data Model toolkit in use by the Maven 2 Project downloads: 93 homepage_uri: http://modello.codehaus.org name: libmodello-java section: universe/libs version: 1.1-2 libmount1: description: block device id library downloads: 3160 homepage_uri: http://userweb.kernel.org/~kzak/util-linux/ name: libmount1 section: libs version: 2.20.1-1ubuntu3 libmozjs185-1.0: description: Spidermonkey javascript engine downloads: 7970 homepage_uri: http://www.mozilla.org/js/spidermonkey/ name: libmozjs185-1.0 section: universe/libs version: 1.8.5-1.0.0-0ubuntu8 libmpc2: description: multiple precision complex floating-point library downloads: 16388 homepage_uri: http://www.multiprecision.org/mpc/ name: libmpc2 section: libs version: 0.9-4 libmpfr4: description: multiple precision floating-point computation downloads: 64013 homepage_uri: http://www.mpfr.org/ name: libmpfr4 section: libs version: 3.1.0-3ubuntu2 libmpg123-0:i386: description: MPEG layer 1/2/3 audio decoder -- runtime library downloads: 0 homepage_uri: '' name: libmpg123-0:i386 section: universe/libs version: 1.12.1-3.2ubuntu1 libmtdev1: description: Multitouch Protocol Translation Library - shared library downloads: 5764 homepage_uri: http://bitmath.org/code/mtdev/ name: libmtdev1 section: libs version: 1.1.0-2ubuntu1 libmtp-common: description: Media Transfer Protocol (MTP) common files downloads: 12505 homepage_uri: http://libmtp.sourceforge.net/ name: libmtp-common section: libs version: 1.1.3-1 libmtp-runtime: description: Media Transfer Protocol (MTP) runtime tools downloads: 10087 homepage_uri: http://libmtp.sourceforge.net/ name: libmtp-runtime section: libs version: 1.1.3-1 libmtp9: description: Media Transfer Protocol (MTP) library downloads: 10955 homepage_uri: http://libmtp.sourceforge.net/ name: libmtp9 section: libs version: 1.1.3-1 libmusicbrainz3-6: description: library to access the MusicBrainz.org database downloads: 2658 homepage_uri: '' name: libmusicbrainz3-6 section: libs version: 3.0.2-2.1 libmysqlclient-dev: description: MySQL database development files downloads: 1256 homepage_uri: http://dev.mysql.com/ name: libmysqlclient-dev section: libdevel version: 5.5.24-0ubuntu0.12.04.1 libmysqlclient18: description: MySQL database client library downloads: 3752 homepage_uri: http://dev.mysql.com/ name: libmysqlclient18 section: libs version: 5.5.24-0ubuntu0.12.04.1 libmysqlclient18:i386: description: MySQL database client library downloads: 0 homepage_uri: http://dev.mysql.com/ name: libmysqlclient18:i386 section: libs version: 5.5.24-0ubuntu0.12.04.1 libmythes-1.2-0: description: simple thesaurus library downloads: 9 homepage_uri: '' name: libmythes-1.2-0 section: libs version: 2:1.2.2-1 libnautilus-extension1a: description: libraries for nautilus components - runtime version downloads: 9552 homepage_uri: http://www.gnome.org/projects/nautilus/ name: libnautilus-extension1a section: libs version: 1:3.4.2-0ubuntu2 libncbi6: description: NCBI libraries for biology applications downloads: 162 homepage_uri: http://www.ncbi.nlm.nih.gov/IEB/ToolBox/ name: libncbi6 section: libs version: 6.1.20110713-3ubuntu2 libncurses5: description: shared libraries for terminal handling downloads: 107369 homepage_uri: http://invisible-island.net/ncurses/ name: libncurses5 section: libs version: 5.9-4 libncurses5-dev: description: developer's libraries for ncurses downloads: 3263 homepage_uri: http://invisible-island.net/ncurses/ name: libncurses5-dev section: libdevel version: 5.9-4 libncurses5:i386: description: shared libraries for terminal handling downloads: 0 homepage_uri: http://invisible-island.net/ncurses/ name: libncurses5:i386 section: libs version: 5.9-4 libncursesw5: description: shared libraries for terminal handling (wide character support) downloads: 103990 homepage_uri: http://invisible-island.net/ncurses/ name: libncursesw5 section: libs version: 5.9-4 libncursesw5:i386: description: shared libraries for terminal handling (wide character support) downloads: 0 homepage_uri: http://invisible-island.net/ncurses/ name: libncursesw5:i386 section: libs version: 5.9-4 libneon27-gnutls: description: HTTP and WebDAV client library (GnuTLS enabled) downloads: 2323 homepage_uri: http://www.webdav.org/neon/ name: libneon27-gnutls section: libs version: 0.29.6-1 libnet-daemon-perl: description: Perl module for building portable Perl daemons easily downloads: 3747 homepage_uri: http://search.cpan.org/dist/Net-Daemon/ name: libnet-daemon-perl section: perl version: 0.48-1 libnet-dbus-perl: description: Extension for the DBus bindings downloads: 9344 homepage_uri: http://search.cpan.org/dist/Net-DBus/ name: libnet-dbus-perl section: universe/perl version: 1.0.0-1build1 libnet-dns-perl: description: Perform DNS queries from a Perl script downloads: 9096 homepage_uri: http://www.net-dns.org/ name: libnet-dns-perl section: perl version: 0.66-2ubuntu3 libnet-domain-tld-perl: description: Perl module for retrieving a list of currently available TLDs downloads: 1063 homepage_uri: http://search.cpan.org/dist/Net-Domain-TLD/ name: libnet-domain-tld-perl section: perl version: 1.69-1 libnet-http-perl: description: module providing low-level HTTP connection client downloads: 5071 homepage_uri: http://search.cpan.org/dist/Net-HTTP/ name: libnet-http-perl section: perl version: 6.02-1 libnet-ip-perl: description: Perl extension for manipulating IPv4/IPv6 addresses downloads: 10893 homepage_uri: http://search.cpan.org/dist/Net-IP/ name: libnet-ip-perl section: perl version: 1.25-3 libnet-ssleay-perl: description: Perl module for Secure Sockets Layer (SSL) downloads: 9569 homepage_uri: http://search.cpan.org/dist/Net-SSLeay/ name: libnet-ssleay-perl section: perl version: 1.42-1build1 libnetbeans-cvsclient-java: description: NetBeans CVS Client library downloads: 76 homepage_uri: http://javacvs.netbeans.org/library/ name: libnetbeans-cvsclient-java section: universe/java version: 6.5-2 libnetcdf6: description: An interface for scientific data access to large binary data downloads: 0 homepage_uri: http://www.unidata.ucar.edu/software/netcdf/ name: libnetcdf6 section: universe/libs version: 1:4.1.1-6 libnetfilter-conntrack3: description: Netfilter netlink-conntrack library downloads: 1549 homepage_uri: http://www.netfilter.org/projects/libnetfilter_conntrack/ name: libnetfilter-conntrack3 section: libs version: 0.9.1-1ubuntu1 libnetpbm10: description: Graphics conversion tools shared libraries downloads: 5717 homepage_uri: http://netpbm.alioth.debian.org name: libnetpbm10 section: libs version: 2:10.0-15 libnettle4: description: low level cryptographic library (symmetric and one-way cryptos) downloads: 2272 homepage_uri: http://www.lysator.liu.se/~nisse/nettle/ name: libnettle4 section: libs version: 2.4-1 libnewt0.52: description: Not Erik's Windowing Toolkit - text mode windowing with slang downloads: 26597 homepage_uri: https://fedorahosted.org/newt/ name: libnewt0.52 section: libs version: 0.52.11-2ubuntu10 libnfnetlink0: description: Netfilter netlink library downloads: 25624 homepage_uri: '' name: libnfnetlink0 section: libs version: 1.0.0-1 libnfsidmap2: description: NFS idmapping library downloads: 9998 homepage_uri: http://www.citi.umich.edu/projects/nfsv4/linux/ name: libnfsidmap2 section: libs version: 0.25-1ubuntu2 libnice10: description: ICE library (shared library) downloads: 3184 homepage_uri: http://nice.freedesktop.org/ name: libnice10 section: libs version: 0.1.1-2ubuntu1 libnih-dbus1: description: NIH D-Bus Bindings Library downloads: 131 homepage_uri: https://launchpad.net/libnih name: libnih-dbus1 section: libs version: 1.0.3-4ubuntu9 libnih1: description: NIH Utility Library downloads: 132 homepage_uri: https://launchpad.net/libnih name: libnih1 section: libs version: 1.0.3-4ubuntu9 libnl-3-200: description: library for dealing with netlink sockets downloads: 15141 homepage_uri: http://people.suug.ch/~tgr/libnl/ name: libnl-3-200 section: libs version: 3.2.3-2ubuntu2 libnl-genl-3-200: description: library for dealing with netlink sockets - generic netlink downloads: 14658 homepage_uri: http://people.suug.ch/~tgr/libnl/ name: libnl-genl-3-200 section: libs version: 3.2.3-2ubuntu2 libnl-route-3-200: description: library for dealing with netlink sockets - route interface downloads: 12982 homepage_uri: http://people.suug.ch/~tgr/libnl/ name: libnl-route-3-200 section: libs version: 3.2.3-2ubuntu2 libnm-glib-vpn1: description: network management framework (GLib VPN shared library) downloads: 26545 homepage_uri: http://www.gnome.org/projects/NetworkManager/ name: libnm-glib-vpn1 section: libs version: 0.9.4.0-0ubuntu4.1 libnm-glib4: description: network management framework (GLib shared library) downloads: 13402 homepage_uri: http://www.gnome.org/projects/NetworkManager/ name: libnm-glib4 section: libs version: 0.9.4.0-0ubuntu4.1 libnm-gtk-common: description: network management framework (common files for wifi and mobile) downloads: 7534 homepage_uri: http://www.gnome.org/projects/NetworkManager/ name: libnm-gtk-common section: gnome version: 0.9.4.1-0ubuntu2 libnm-gtk0: description: network management framework (GNOME dialogs for wifi and mobile) downloads: 9228 homepage_uri: http://www.gnome.org/projects/NetworkManager/ name: libnm-gtk0 section: libs version: 0.9.4.1-0ubuntu2 libnm-util2: description: network management framework (shared library) downloads: 13493 homepage_uri: http://www.gnome.org/projects/NetworkManager/ name: libnm-util2 section: libs version: 0.9.4.0-0ubuntu4.1 libnotify-bin: description: sends desktop notifications to a notification daemon (Utilities) downloads: 1371 homepage_uri: '' name: libnotify-bin section: utils version: 0.7.5-1 libnotify4: description: sends desktop notifications to a notification daemon downloads: 15455 homepage_uri: '' name: libnotify4 section: libs version: 0.7.5-1 libnspr4: description: NetScape Portable Runtime Library downloads: 12211 homepage_uri: http://www.mozilla.org/projects/nspr/ name: libnspr4 section: libs version: 4.8.9-1ubuntu2 libnspr4:i386: description: NetScape Portable Runtime Library downloads: 0 homepage_uri: http://www.mozilla.org/projects/nspr/ name: libnspr4:i386 section: libs version: 4.8.9-1ubuntu2 libnss-mdns: description: NSS module for Multicast DNS name resolution downloads: 259 homepage_uri: '' name: libnss-mdns section: admin version: 0.10-3.2 libnss3: description: Network Security Service libraries downloads: 4646 homepage_uri: http://www.mozilla.org/projects/security/pki/nss/ name: libnss3 section: libs version: 3.13.1.with.ckbi.1.88-1ubuntu6 libnss3-1d: description: Network Security Service libraries downloads: 31079 homepage_uri: http://www.mozilla.org/projects/security/pki/nss/ name: libnss3-1d section: libs version: 3.13.1.with.ckbi.1.88-1ubuntu6 libnss3:i386: description: Network Security Service libraries downloads: 0 homepage_uri: http://www.mozilla.org/projects/security/pki/nss/ name: libnss3:i386 section: libs version: 3.13.1.with.ckbi.1.88-1ubuntu6 libnuma1: description: Libraries for controlling NUMA policy downloads: 120 homepage_uri: '' name: libnuma1 section: libs version: 2.0.8~rc3-1 libnux-2.0-0: description: Visual rendering toolkit for real-time applications - shared lib downloads: 0 homepage_uri: http://launchpad.net/nux name: libnux-2.0-0 section: libs version: 2.12.0-0ubuntu1 libnux-2.0-common: description: Visual rendering toolkit for real-time applications - common files downloads: 0 homepage_uri: http://launchpad.net/nux name: libnux-2.0-common section: libs version: 2.12.0-0ubuntu1 libnx-xorg: description: NX support libraries downloads: 0 homepage_uri: '' name: libnx-xorg section: x11 version: 1:3.5.0-9-2-1-2-0~ppa1~precise2 liboauth0: description: C library for implementing OAuth 1.0 downloads: 3467 homepage_uri: http://liboauth.sourceforge.net/ name: liboauth0 section: libs version: 0.9.4-3 libodbc1: description: ODBC library for Unix downloads: 4181 homepage_uri: http://www.unixodbc.org/ name: libodbc1 section: libs version: 2.2.14p2-5ubuntu3 libodbc1:i386: description: ODBC library for Unix downloads: 0 homepage_uri: http://www.unixodbc.org/ name: libodbc1:i386 section: libs version: 2.2.14p2-5ubuntu3 libogg0: description: Ogg bitstream library downloads: 44811 homepage_uri: http://xiph.org/ogg/ name: libogg0 section: libs version: 1.2.2~dfsg-1ubuntu1 libogg0:i386: description: Ogg bitstream library downloads: 0 homepage_uri: http://xiph.org/ogg/ name: libogg0:i386 section: libs version: 1.2.2~dfsg-1ubuntu1 liboobs-1-5: description: GObject based interface to system-tools-backends - shared library downloads: 964 homepage_uri: '' name: liboobs-1-5 section: universe/libs version: 3.0.0-1 libopenal-data: description: Software implementation of the OpenAL API (data files) downloads: 719 homepage_uri: http://kcat.strangesoft.net/openal.html name: libopenal-data section: libs version: 1:1.13-4ubuntu3 libopenal1:i386: description: Software implementation of the OpenAL API (shared library) downloads: 0 homepage_uri: http://kcat.strangesoft.net/openal.html name: libopenal1:i386 section: libs version: 1:1.13-4ubuntu3 libopencc1: description: simplified-traditional chinese conversion library - runtime downloads: 30 homepage_uri: http://code.google.com/p/opencc name: libopencc1 section: libs version: 0.3.0-1 libopenexr6: description: runtime files for the OpenEXR image library downloads: 256 homepage_uri: http://www.openexr.com name: libopenexr6 section: libs version: 1.6.1-4.1 libopenmpi-dev: description: high performance message passing library -- header files downloads: 1188 homepage_uri: http://www.open-mpi.org/ name: libopenmpi-dev section: universe/libdevel version: 1.4.3-2.1ubuntu3 libopenmpi1.3: description: high performance message passing library -- shared library downloads: 1715 homepage_uri: http://www.open-mpi.org/ name: libopenmpi1.3 section: universe/libs version: 1.4.3-2.1ubuntu3 libopenobex1: description: OBEX protocol library downloads: 260 homepage_uri: http://www.openobex.org name: libopenobex1 section: libs version: 1.5-2build1 liborbit2: description: libraries for ORBit2 - a CORBA ORB downloads: 33873 homepage_uri: '' name: liborbit2 section: libs version: 1:2.14.19-0.1ubuntu1 liborbit2-dev: description: development files for ORBit2 - a CORBA ORB downloads: 543 homepage_uri: '' name: liborbit2-dev section: libdevel version: 1:2.14.19-0.1ubuntu1 liborc-0.4-0: description: Library of Optimized Inner Loops Runtime Compiler downloads: 37029 homepage_uri: http://code.entropywave.com/projects/orc/ name: liborc-0.4-0 section: libs version: 1:0.4.16-1ubuntu2 liborc-0.4-0:i386: description: Library of Optimized Inner Loops Runtime Compiler downloads: 0 homepage_uri: http://code.entropywave.com/projects/orc/ name: liborc-0.4-0:i386 section: libs version: 1:0.4.16-1ubuntu2 libossp-uuid-perl: description: perl OSSP::UUID - OSSP uuid Perl Binding downloads: 1796 homepage_uri: http://www.ossp.org/pkg/lib/uuid/ name: libossp-uuid-perl section: universe/libs version: 1.6.2-1.1build3 libossp-uuid16: description: OSSP uuid ISO-C and C++ - shared library downloads: 1299 homepage_uri: http://www.ossp.org/pkg/lib/uuid/ name: libossp-uuid16 section: libs version: 1.6.2-1.1build3 libotf0: description: A Library for handling OpenType Font - runtime downloads: 66 homepage_uri: http://www.m17n.org/ name: libotf0 section: libs version: 0.9.12-1 liboverlay-scrollbar-0.2-0: description: Scrollbar overlayed widget - shared lib downloads: 0 homepage_uri: http://launchpad.net/ayatana-scrollbar name: liboverlay-scrollbar-0.2-0 section: libs version: 0.2.16-0ubuntu1 liboverlay-scrollbar3-0.2-0: description: Scrollbar overlayed widget - shared lib downloads: 0 homepage_uri: http://launchpad.net/ayatana-scrollbar name: liboverlay-scrollbar3-0.2-0 section: libs version: 0.2.16-0ubuntu1 libp11-kit-dev: description: Library for loading and coordinating access to PKCS#11 modules - development downloads: 1565 homepage_uri: http://p11-glue.freedesktop.org/p11-kit.html name: libp11-kit-dev section: libdevel version: 0.12-2ubuntu1 libp11-kit0: description: Library for loading and coordinating access to PKCS#11 modules - runtime downloads: 19102 homepage_uri: http://p11-glue.freedesktop.org/p11-kit.html name: libp11-kit0 section: libs version: 0.12-2ubuntu1 libp11-kit0:i386: description: Library for loading and coordinating access to PKCS#11 modules - runtime downloads: 0 homepage_uri: http://p11-glue.freedesktop.org/p11-kit.html name: libp11-kit0:i386 section: libs version: 0.12-2ubuntu1 libpackagekit-glib2-14: description: Library for accessing PackageKit using GLib downloads: 10162 homepage_uri: http://www.packagekit.org name: libpackagekit-glib2-14 section: libs version: 0.7.2-4ubuntu3 libpam-cap: description: PAM module for implementing capabilities downloads: 59 homepage_uri: http://sites.google.com/site/fullycapable/ name: libpam-cap section: libs version: 1:2.22-1ubuntu3 libpam-ck-connector: description: ConsoleKit PAM module downloads: 35908 homepage_uri: http://www.freedesktop.org/wiki/Software/ConsoleKit name: libpam-ck-connector section: admin version: 0.4.5-2 libpam-gnome-keyring: description: PAM module to unlock the GNOME keyring upon login downloads: 46702 homepage_uri: http://live.gnome.org/GnomeKeyring name: libpam-gnome-keyring section: admin version: 3.2.2-2ubuntu4 libpam-modules: description: Pluggable Authentication Modules for PAM downloads: 108815 homepage_uri: http://pam.sourceforge.net/ name: libpam-modules section: admin version: 1.1.3-7ubuntu2 libpam-modules-bin: description: Pluggable Authentication Modules for PAM - helper binaries downloads: 10192 homepage_uri: http://pam.sourceforge.net/ name: libpam-modules-bin section: admin version: 1.1.3-7ubuntu2 libpam-runtime: description: Runtime support for the PAM library downloads: 86700 homepage_uri: http://pam.sourceforge.net/ name: libpam-runtime section: admin version: 1.1.3-7ubuntu2 libpam0g: description: Pluggable Authentication Modules library downloads: 107338 homepage_uri: http://pam.sourceforge.net/ name: libpam0g section: libs version: 1.1.3-7ubuntu2 libpanel-applet-4-0: description: library for GNOME Panel applets downloads: 4761 homepage_uri: '' name: libpanel-applet-4-0 section: universe/libs version: 1:3.4.1-0ubuntu1 libpango-perl: description: Perl module to layout and render international text downloads: 17349 homepage_uri: http://gtk2-perl.sourceforge.net/ name: libpango-perl section: perl version: 1.222-1build1 libpango1.0-0: description: Layout and rendering of internationalized text downloads: 58689 homepage_uri: '' name: libpango1.0-0 section: libs version: 1.30.0-0ubuntu3 libpango1.0-0:i386: description: Layout and rendering of internationalized text downloads: 0 homepage_uri: '' name: libpango1.0-0:i386 section: libs version: 1.30.0-0ubuntu3 libpango1.0-dev: description: Development files for the Pango downloads: 3243 homepage_uri: '' name: libpango1.0-dev section: libdevel version: 1.30.0-0ubuntu3 libpangomm-1.4-1: description: C++ Wrapper for pango (shared libraries) downloads: 219 homepage_uri: http://gtkmm.org name: libpangomm-1.4-1 section: libs version: 2.28.4-1ubuntu1 libpaper-utils: description: library for handling paper characteristics (utilities) downloads: 27405 homepage_uri: '' name: libpaper-utils section: utils version: 1.1.24+nmu1build1 libpaper1: description: library for handling paper characteristics downloads: 28396 homepage_uri: '' name: libpaper1 section: libs version: 1.1.24+nmu1build1 libparams-validate-perl: description: Perl module to validate parameters to Perl method/function calls downloads: 2612 homepage_uri: http://search.cpan.org/dist/Params-Validate/ name: libparams-validate-perl section: perl version: 0.97-1build2 libparpack2: description: Parallel subroutines to solve large scale eigenvalue problems downloads: 15 homepage_uri: http://forge.scilab.org/index.php/p/arpack-ng/ name: libparpack2 section: universe/libs version: 3.0.2-3 libparrot-dev: description: Parrot shared library development files downloads: 18 homepage_uri: http://parrot.org name: libparrot-dev section: universe/libdevel version: 3.6.0-1 libparrot3.6.0: description: Parrot shared library downloads: 0 homepage_uri: http://parrot.org name: libparrot3.6.0 section: universe/libs version: 3.6.0-1 libparse-debcontrol-perl: description: parser for debian control-like files downloads: 766 homepage_uri: http://search.cpan.org/dist/Parse-DebControl/ name: libparse-debcontrol-perl section: perl version: 2.005-3 libparse-debianchangelog-perl: description: parse Debian changelogs and output them in other formats downloads: 1924 homepage_uri: '' name: libparse-debianchangelog-perl section: perl version: 1.2.0-1ubuntu1 libparted0debian1: description: disk partition manipulator - shared library downloads: 4406 homepage_uri: http://www.gnu.org/software/parted name: libparted0debian1 section: libs version: 2.3-8ubuntu5 libpathplan4: description: rich set of graph drawing tools - pathplan library downloads: 3685 homepage_uri: http://www.graphviz.org/ name: libpathplan4 section: libs version: 2.26.3-10ubuntu1 libpcap0.8: description: system interface for user-level packet capture downloads: 14933 homepage_uri: http://www.tcpdump.org/ name: libpcap0.8 section: libs version: 1.1.1-10 libpci3: description: Linux PCI Utilities (shared library) downloads: 23202 homepage_uri: http://atrey.karlin.mff.cuni.cz/~mj/pciutils.shtml name: libpci3 section: libs version: 1:3.1.8-2ubuntu5 libpciaccess0: description: Generic PCI access library for X downloads: 27790 homepage_uri: '' name: libpciaccess0 section: libs version: 0.12.902-1 libpciaccess0:i386: description: Generic PCI access library for X downloads: 0 homepage_uri: '' name: libpciaccess0:i386 section: libs version: 0.12.902-1 libpcre3: description: Perl 5 Compatible Regular Expression Library - runtime files downloads: 102180 homepage_uri: '' name: libpcre3 section: libs version: 8.12-4 libpcre3-dev: description: Perl 5 Compatible Regular Expression Library - development files downloads: 4209 homepage_uri: '' name: libpcre3-dev section: libdevel version: 8.12-4 libpcre3:i386: description: Perl 5 Compatible Regular Expression Library - runtime files downloads: 0 homepage_uri: '' name: libpcre3:i386 section: libs version: 8.12-4 libpcrecpp0: description: Perl 5 Compatible Regular Expression Library - C++ runtime files downloads: 206 homepage_uri: '' name: libpcrecpp0 section: libs version: 8.12-4 libpcsclite1: description: Middleware to access a smart card using PC/SC (library) downloads: 40799 homepage_uri: http://pcsclite.alioth.debian.org/ name: libpcsclite1 section: libs version: 1.7.4-2ubuntu2 libpeas-1.0-0: description: Application plugin library downloads: 9305 homepage_uri: http://live.gnome.org/Libpeas name: libpeas-1.0-0 section: libs version: 1.2.0-1ubuntu1 libpeas-common: description: Application plugin library (common files) downloads: 8459 homepage_uri: http://live.gnome.org/Libpeas name: libpeas-common section: libs version: 1.2.0-1ubuntu1 libperl-dev: description: 'Perl library: development files' downloads: 365 homepage_uri: '' name: libperl-dev section: libdevel version: 5.14.2-6ubuntu2 libperl5.14: description: shared Perl library downloads: 9519 homepage_uri: '' name: libperl5.14 section: libs version: 5.14.2-6ubuntu2 libpgm-5.1-0: description: OpenPGM shared library downloads: 2 homepage_uri: http://code.google.com/p/openpgm/ name: libpgm-5.1-0 section: universe/libs version: 5.1.116~dfsg-2 libpipeline1: description: pipeline manipulation library downloads: 20210 homepage_uri: http://libpipeline.nongnu.org/ name: libpipeline1 section: libs version: 1.2.1-1 libpixman-1-0: description: pixel-manipulation library for X and cairo downloads: 54407 homepage_uri: '' name: libpixman-1-0 section: libs version: 0.24.4-1 libpixman-1-0:i386: description: pixel-manipulation library for X and cairo downloads: 0 homepage_uri: '' name: libpixman-1-0:i386 section: libs version: 0.24.4-1 libpixman-1-dev: description: pixel-manipulation library for X and cairo (development files) downloads: 2924 homepage_uri: '' name: libpixman-1-dev section: libdevel version: 0.24.4-1 libplexus-ant-factory-java: description: Plexus Ant Factory downloads: 93 homepage_uri: http://plexus.codehaus.org/ name: libplexus-ant-factory-java section: universe/java version: 1.0~alpha2.1-3 libplexus-archiver-java: description: The archiver plugin for the Plexus compiler system downloads: 93 homepage_uri: http://plexus.codehaus.org/plexus-components/plexus-archiver/ name: libplexus-archiver-java section: universe/libs version: 1.0~alpha12-3 libplexus-bsh-factory-java: description: Plexus Beanshell Factory downloads: 93 homepage_uri: http://plexus.codehaus.org/ name: libplexus-bsh-factory-java section: universe/java version: 1.0~alpha7-3 libplexus-build-api-java: description: Incremental build API for Plexus components. downloads: 20 homepage_uri: https://github.com/sonatype/sisu-build-api name: libplexus-build-api-java section: universe/java version: 0.0.4-4 libplexus-cipher-java: description: Plexus Cipher Component used by Maven downloads: 5 homepage_uri: https://github.com/sonatype/plexus-cipher name: libplexus-cipher-java section: universe/java version: 1.5-2 libplexus-classworlds-java: description: Class loading utilities for the Plexus framework downloads: 39 homepage_uri: http://plexus.codehaus.org/ name: libplexus-classworlds-java section: universe/libs version: 1.5.0-3 libplexus-compiler-api-java: description: API for the Plexus compiler system (dummy package) downloads: 0 homepage_uri: http://plexus.codehaus.org/plexus-components/plexus-compiler/ name: libplexus-compiler-api-java section: universe/libs version: 1.6-2 libplexus-compiler-java: description: API for the Plexus compiler system downloads: 0 homepage_uri: http://plexus.codehaus.org/plexus-components/plexus-compiler/ name: libplexus-compiler-java section: universe/java version: 1.6-2 libplexus-compiler-javac-java: description: javac plugin for the Plexus compiler system (dummy package) downloads: 0 homepage_uri: http://plexus.codehaus.org/plexus-components/plexus-compiler/ name: libplexus-compiler-javac-java section: universe/libs version: 1.6-2 libplexus-compiler-manager-java: description: manager for the Plexus compiler system (dummy package) downloads: 0 homepage_uri: http://plexus.codehaus.org/plexus-components/plexus-compiler/ name: libplexus-compiler-manager-java section: universe/libs version: 1.6-2 libplexus-container-default-java: description: utilities for the Plexus framework downloads: 93 homepage_uri: http://plexus.codehaus.org/ name: libplexus-container-default-java section: universe/libs version: 1.0-alpha-9-stable-1-6 libplexus-containers-java: description: utilities for the Plexus framework downloads: 33 homepage_uri: http://plexus.codehaus.org name: libplexus-containers-java section: universe/libs version: 1.0~beta3.0.7-5 libplexus-digest-java: description: Digest utilities for the Plexus system downloads: 0 homepage_uri: http://plexus.codehaus.org/plexus-components/plexus-digest/ name: libplexus-digest-java section: universe/java version: 1.1-3 libplexus-i18n-java: description: a component to support internationalization of applications using Plexus downloads: 9 homepage_uri: http://plexus.codehaus.org/ name: libplexus-i18n-java section: universe/libs version: 1.0-beta-10-3 libplexus-interactivity-api-java: description: interactivity API for the Plexus framework downloads: 93 homepage_uri: http://plexus.codehaus.org/plexus-components/plexus-interactivity/ name: libplexus-interactivity-api-java section: universe/libs version: 1.0-alpha-6-6 libplexus-interpolation-java: description: Plexus Interpolation API downloads: 93 homepage_uri: http://plexus.codehaus.org/plexus-components/plexus-interpolation/ name: libplexus-interpolation-java section: universe/libs version: 1.11-3 libplexus-io-java: description: Plexus IO Components downloads: 1 homepage_uri: http://plexus.codehaus.org/plexus-components/plexus-io/ name: libplexus-io-java section: universe/libs version: 1.0~alpha5-2 libplexus-sec-dispatcher-java: description: Plexus Security Dispatcher Component used by Maven downloads: 93 homepage_uri: https://github.com/sonatype/plexus-sec-dispatcher name: libplexus-sec-dispatcher-java section: universe/java version: 1.3.1-5 libplexus-utils-java: description: utilities for the Plexus framework downloads: 98 homepage_uri: http://plexus.codehaus.org/ name: libplexus-utils-java section: universe/libs version: 1:1.5.15-4 libplist1: description: Library for handling Apple binary and XML property lists downloads: 38812 homepage_uri: http://www.libimobiledevice.org/ name: libplist1 section: libs version: 1.8-1 libplrpc-perl: description: Perl extensions for writing PlRPC servers and clients downloads: 23806 homepage_uri: http://search.cpan.org/search?query=PlRPC&mode=module name: libplrpc-perl section: perl version: 0.2020-2 libplymouth2: description: graphical boot animation and logger - shared libraries downloads: 0 homepage_uri: '' name: libplymouth2 section: libs version: 0.8.2-2ubuntu30 libpng12-0: description: PNG library - runtime downloads: 74882 homepage_uri: http://libpng.org/pub/png/libpng.html name: libpng12-0 section: libs version: 1.2.46-3ubuntu4 libpng12-0:i386: description: PNG library - runtime downloads: 0 homepage_uri: http://libpng.org/pub/png/libpng.html name: libpng12-0:i386 section: libs version: 1.2.46-3ubuntu4 libpng12-dev: description: PNG library - development downloads: 5284 homepage_uri: http://libpng.org/pub/png/libpng.html name: libpng12-dev section: libdevel version: 1.2.46-3ubuntu4 libpolkit-agent-1-0: description: PolicyKit Authentication Agent API downloads: 41384 homepage_uri: http://hal.freedesktop.org/docs/PolicyKit/ name: libpolkit-agent-1-0 section: libs version: 0.104-1ubuntu1 libpolkit-backend-1-0: description: PolicyKit backend API downloads: 40928 homepage_uri: http://hal.freedesktop.org/docs/PolicyKit/ name: libpolkit-backend-1-0 section: libs version: 0.104-1ubuntu1 libpolkit-gobject-1-0: description: PolicyKit Authorization API downloads: 51739 homepage_uri: http://hal.freedesktop.org/docs/PolicyKit/ name: libpolkit-gobject-1-0 section: libs version: 0.104-1ubuntu1 libpoppler-glib8: description: PDF rendering library (GLib-based shared library) downloads: 7196 homepage_uri: http://poppler.freedesktop.org/ name: libpoppler-glib8 section: libs version: 0.18.4-1ubuntu2 libpoppler19: description: PDF rendering library downloads: 214 homepage_uri: http://poppler.freedesktop.org/ name: libpoppler19 section: libs version: 0.18.4-1ubuntu2 libpopt-dev: description: lib for parsing cmdline parameters - development files downloads: 759 homepage_uri: http://rpm5.org/ name: libpopt-dev section: libdevel version: 1.16-3ubuntu1 libpopt0: description: lib for parsing cmdline parameters downloads: 110129 homepage_uri: http://rpm5.org/ name: libpopt0 section: libs version: 1.16-3ubuntu1 libportaudio2: description: Portable audio I/O - shared library downloads: 2126 homepage_uri: http://www.portaudio.com/ name: libportaudio2 section: libs version: 19+svn20111121-1 libpq-dev: description: header files for libpq5 (PostgreSQL library) downloads: 1258 homepage_uri: http://www.postgresql.org/ name: libpq-dev section: libdevel version: 9.1.4-0ubuntu12.04 libpq5: description: PostgreSQL C client library downloads: 16324 homepage_uri: http://www.postgresql.org/ name: libpq5 section: libs version: 9.1.4-0ubuntu12.04 libprotobuf7: description: protocol buffers C++ library downloads: 52 homepage_uri: http://code.google.com/p/protobuf/ name: libprotobuf7 section: devel version: 2.4.1-1ubuntu2 libprotoc7: description: protocol buffers compiler library downloads: 3 homepage_uri: http://code.google.com/p/protobuf/ name: libprotoc7 section: devel version: 2.4.1-1ubuntu2 libproxy1: description: automatic proxy configuration management library (shared) downloads: 1 homepage_uri: http://code.google.com/p/libproxy/ name: libproxy1 section: libs version: 0.4.7-0ubuntu4 libproxy1-plugin-gsettings: description: automatic proxy configuration management library (GSettings plugin) downloads: 0 homepage_uri: http://code.google.com/p/libproxy/ name: libproxy1-plugin-gsettings section: libs version: 0.4.7-0ubuntu4 libproxy1-plugin-networkmanager: description: automatic proxy configuration management library (Network Manager plugin) downloads: 0 homepage_uri: http://code.google.com/p/libproxy/ name: libproxy1-plugin-networkmanager section: libs version: 0.4.7-0ubuntu4 libproxy1:i386: description: automatic proxy configuration management library (shared) downloads: 0 homepage_uri: http://code.google.com/p/libproxy/ name: libproxy1:i386 section: libs version: 0.4.7-0ubuntu4 libpth20: description: The GNU Portable Threads downloads: 39911 homepage_uri: http://www.gnu.org/software/pth/ name: libpth20 section: libs version: 2.0.7-16ubuntu3 libpthread-stubs0: description: pthread stubs not provided by native libc downloads: 969 homepage_uri: '' name: libpthread-stubs0 section: libs version: 0.3-3 libpthread-stubs0-dev: description: pthread stubs not provided by native libc, development files downloads: 4850 homepage_uri: '' name: libpthread-stubs0-dev section: libdevel version: 0.3-3 libpulse-mainloop-glib0: description: PulseAudio client libraries (glib support) downloads: 17470 homepage_uri: http://www.pulseaudio.org name: libpulse-mainloop-glib0 section: sound version: 1:1.1-0ubuntu15.1 libpulse-mainloop-glib0:i386: description: PulseAudio client libraries (glib support) downloads: 0 homepage_uri: http://www.pulseaudio.org name: libpulse-mainloop-glib0:i386 section: sound version: 1:1.1-0ubuntu15.1 libpulse0: description: PulseAudio client libraries downloads: 32941 homepage_uri: http://www.pulseaudio.org name: libpulse0 section: libs version: 1:1.1-0ubuntu15.1 libpulse0:i386: description: PulseAudio client libraries downloads: 0 homepage_uri: http://www.pulseaudio.org name: libpulse0:i386 section: libs version: 1:1.1-0ubuntu15.1 libpulsedsp: description: PulseAudio OSS pre-load library downloads: 0 homepage_uri: http://www.pulseaudio.org name: libpulsedsp section: sound version: 1:1.1-0ubuntu15.1 libpulsedsp:i386: description: PulseAudio OSS pre-load library downloads: 0 homepage_uri: http://www.pulseaudio.org name: libpulsedsp:i386 section: sound version: 1:1.1-0ubuntu15.1 libpurple-bin: description: multi-protocol instant messaging library - extra utilities downloads: 1289 homepage_uri: http://www.pidgin.im name: libpurple-bin section: net version: 1:2.10.3-0ubuntu1 libpurple0: description: multi-protocol instant messaging library downloads: 6069 homepage_uri: http://www.pidgin.im name: libpurple0 section: net version: 1:2.10.3-0ubuntu1 libpvm3: description: Parallel Virtual Machine - shared libraries downloads: 261 homepage_uri: '' name: libpvm3 section: universe/libs version: 3.4.5-12.3 libpython2.7: description: Shared Python runtime library (version 2.7) downloads: 13671 homepage_uri: '' name: libpython2.7 section: libs version: 2.7.3-0ubuntu3 libpython3.2: description: Shared Python runtime library (version 3.2) downloads: 97 homepage_uri: '' name: libpython3.2 section: libs version: 3.2.3-0ubuntu3 libqhull5: description: calculate convex hulls and related structures (shared library) downloads: 428 homepage_uri: http://www.qhull.org name: libqhull5 section: universe/libs version: 2009.1-3ubuntu1 libqrupdate1: description: Fast updates of QR and Cholesky decompositions downloads: 2 homepage_uri: http://qrupdate.sf.net name: libqrupdate1 section: universe/libs version: 1.1.1-1 libqt4-dbus: description: Qt 4 D-Bus module downloads: 6741 homepage_uri: http://qt.nokia.com/ name: libqt4-dbus section: libs version: 4:4.8.1-0ubuntu4.1 libqt4-dbus:i386: description: Qt 4 D-Bus module downloads: 0 homepage_uri: http://qt.nokia.com/ name: libqt4-dbus:i386 section: libs version: 4:4.8.1-0ubuntu4.1 libqt4-declarative: description: Qt 4 Declarative module downloads: 448 homepage_uri: http://qt.nokia.com/ name: libqt4-declarative section: libs version: 4:4.8.1-0ubuntu4.1 libqt4-declarative:i386: description: Qt 4 Declarative module downloads: 0 homepage_uri: http://qt.nokia.com/ name: libqt4-declarative:i386 section: libs version: 4:4.8.1-0ubuntu4.1 libqt4-designer: description: Qt 4 designer module downloads: 301 homepage_uri: http://qt.nokia.com/ name: libqt4-designer section: libs version: 4:4.8.1-0ubuntu4.1 libqt4-designer:i386: description: Qt 4 designer module downloads: 0 homepage_uri: http://qt.nokia.com/ name: libqt4-designer:i386 section: libs version: 4:4.8.1-0ubuntu4.1 libqt4-help: description: Qt 4 help module downloads: 524 homepage_uri: http://qt.nokia.com/ name: libqt4-help section: libs version: 4:4.8.1-0ubuntu4.1 libqt4-network: description: Qt 4 network module downloads: 1418 homepage_uri: http://qt.nokia.com/ name: libqt4-network section: libs version: 4:4.8.1-0ubuntu4.1 libqt4-network:i386: description: Qt 4 network module downloads: 0 homepage_uri: http://qt.nokia.com/ name: libqt4-network:i386 section: libs version: 4:4.8.1-0ubuntu4.1 libqt4-opengl: description: Qt 4 OpenGL module downloads: 1235 homepage_uri: http://qt.nokia.com/ name: libqt4-opengl section: libs version: 4:4.8.1-0ubuntu4.1 libqt4-opengl:i386: description: Qt 4 OpenGL module downloads: 0 homepage_uri: http://qt.nokia.com/ name: libqt4-opengl:i386 section: libs version: 4:4.8.1-0ubuntu4.1 libqt4-qt3support:i386: description: Qt 3 compatibility library for Qt 4 downloads: 0 homepage_uri: http://qt.nokia.com/ name: libqt4-qt3support:i386 section: libs version: 4:4.8.1-0ubuntu4.1 libqt4-script: description: Qt 4 script module downloads: 1282 homepage_uri: http://qt.nokia.com/ name: libqt4-script section: libs version: 4:4.8.1-0ubuntu4.1 libqt4-script:i386: description: Qt 4 script module downloads: 0 homepage_uri: http://qt.nokia.com/ name: libqt4-script:i386 section: libs version: 4:4.8.1-0ubuntu4.1 libqt4-scripttools: description: Qt 4 script tools module downloads: 78 homepage_uri: http://qt.nokia.com/ name: libqt4-scripttools section: libs version: 4:4.8.1-0ubuntu4.1 libqt4-scripttools:i386: description: Qt 4 script tools module downloads: 0 homepage_uri: http://qt.nokia.com/ name: libqt4-scripttools:i386 section: libs version: 4:4.8.1-0ubuntu4.1 libqt4-sql: description: Qt 4 SQL module downloads: 659 homepage_uri: http://qt.nokia.com/ name: libqt4-sql section: libs version: 4:4.8.1-0ubuntu4.1 libqt4-sql-mysql: description: Qt 4 MySQL database driver downloads: 3031 homepage_uri: http://qt.nokia.com/ name: libqt4-sql-mysql section: libs version: 4:4.8.1-0ubuntu4.1 libqt4-sql-mysql:i386: description: Qt 4 MySQL database driver downloads: 0 homepage_uri: http://qt.nokia.com/ name: libqt4-sql-mysql:i386 section: libs version: 4:4.8.1-0ubuntu4.1 libqt4-sql-sqlite: description: Qt 4 SQLite 3 database driver downloads: 4678 homepage_uri: http://qt.nokia.com/ name: libqt4-sql-sqlite section: libs version: 4:4.8.1-0ubuntu4.1 libqt4-sql:i386: description: Qt 4 SQL module downloads: 0 homepage_uri: http://qt.nokia.com/ name: libqt4-sql:i386 section: libs version: 4:4.8.1-0ubuntu4.1 libqt4-svg: description: Qt 4 SVG module downloads: 14224 homepage_uri: http://qt.nokia.com/ name: libqt4-svg section: libs version: 4:4.8.1-0ubuntu4.1 libqt4-svg:i386: description: Qt 4 SVG module downloads: 0 homepage_uri: http://qt.nokia.com/ name: libqt4-svg:i386 section: libs version: 4:4.8.1-0ubuntu4.1 libqt4-test: description: Qt 4 test module downloads: 334 homepage_uri: http://qt.nokia.com/ name: libqt4-test section: libs version: 4:4.8.1-0ubuntu4.1 libqt4-test:i386: description: Qt 4 test module downloads: 0 homepage_uri: http://qt.nokia.com/ name: libqt4-test:i386 section: libs version: 4:4.8.1-0ubuntu4.1 libqt4-xml: description: Qt 4 XML module downloads: 920 homepage_uri: http://qt.nokia.com/ name: libqt4-xml section: libs version: 4:4.8.1-0ubuntu4.1 libqt4-xml:i386: description: Qt 4 XML module downloads: 0 homepage_uri: http://qt.nokia.com/ name: libqt4-xml:i386 section: libs version: 4:4.8.1-0ubuntu4.1 libqt4-xmlpatterns: description: Qt 4 XML patterns module downloads: 438 homepage_uri: http://qt.nokia.com/ name: libqt4-xmlpatterns section: libs version: 4:4.8.1-0ubuntu4.1 libqt4-xmlpatterns:i386: description: Qt 4 XML patterns module downloads: 0 homepage_uri: http://qt.nokia.com/ name: libqt4-xmlpatterns:i386 section: libs version: 4:4.8.1-0ubuntu4.1 libqtassistantclient4: description: Qt Assistant client library (runtime) downloads: 43 homepage_uri: http://doc.qt.nokia.com/4.6/qassistantclient.html name: libqtassistantclient4 section: libs version: 4.6.3-3ubuntu2 libqtbamf1: description: Qt binding and QML plugin for bamf - shared library downloads: 0 homepage_uri: '' name: libqtbamf1 section: libs version: 0.2.4-0ubuntu1 libqtcore4: description: Qt 4 core module downloads: 6540 homepage_uri: http://qt.nokia.com/ name: libqtcore4 section: libs version: 4:4.8.1-0ubuntu4.1 libqtcore4:i386: description: Qt 4 core module downloads: 0 homepage_uri: http://qt.nokia.com/ name: libqtcore4:i386 section: libs version: 4:4.8.1-0ubuntu4.1 libqtdee2: description: Qt binding and QML plugin for Dee - shared library downloads: 0 homepage_uri: '' name: libqtdee2 section: libs version: 0.2.4-0ubuntu1 libqtgconf1: description: Qt binding and QML plugin for GConf - shared library downloads: 0 homepage_uri: '' name: libqtgconf1 section: libs version: 0.1-0ubuntu5 libqtgui4: description: Qt 4 GUI module downloads: 17821 homepage_uri: http://qt.nokia.com/ name: libqtgui4 section: libs version: 4:4.8.1-0ubuntu4.1 libqtgui4:i386: description: Qt 4 GUI module downloads: 0 homepage_uri: http://qt.nokia.com/ name: libqtgui4:i386 section: libs version: 4:4.8.1-0ubuntu4.1 libqtwebkit4: description: Web content engine library for Qt downloads: 363 homepage_uri: http://trac.webkit.org/wiki/QtWebKit name: libqtwebkit4 section: libs version: 2.2.1-1ubuntu4 libqtwebkit4:i386: description: Web content engine library for Qt downloads: 0 homepage_uri: http://trac.webkit.org/wiki/QtWebKit name: libqtwebkit4:i386 section: libs version: 2.2.1-1ubuntu4 libquadmath0: description: GCC Quad-Precision Math Library downloads: 1010 homepage_uri: http://gcc.gnu.org/ name: libquadmath0 section: libs version: 4.6.3-1ubuntu5 libquvi-scripts: description: library for parsing video download links (Lua scripts) downloads: 1740 homepage_uri: http://quvi.sourceforge.net name: libquvi-scripts section: libs version: 0.4.2-1 libquvi7: description: library for parsing video download links (runtime libraries) downloads: 2780 homepage_uri: http://quvi.sourceforge.net name: libquvi7 section: libs version: 0.4.0-1 librados2: description: RADOS distributed object store client library downloads: 2776 homepage_uri: http://ceph.newdream.net/ name: librados2 section: libs version: 0.41-1ubuntu2 libraptor2-0: description: Raptor 2 RDF syntax library downloads: 172 homepage_uri: http://librdf.org/raptor/ name: libraptor2-0 section: libs version: 2.0.6-1 librarian0: description: Documentation meta-data library (library package) downloads: 4 homepage_uri: http://rarian.freedesktop.org/ name: librarian0 section: libs version: 0.8.1-5 librasqal3: description: Rasqal RDF query library downloads: 96 homepage_uri: http://librdf.org/rasqal/ name: librasqal3 section: libs version: 0.9.28-1 libraw1394-11: description: library for direct access to IEEE 1394 bus (aka FireWire) downloads: 32778 homepage_uri: https://ieee1394.wiki.kernel.org/ name: libraw1394-11 section: libs version: 2.0.7-1ubuntu1 libraw1394-11:i386: description: library for direct access to IEEE 1394 bus (aka FireWire) downloads: 0 homepage_uri: https://ieee1394.wiki.kernel.org/ name: libraw1394-11:i386 section: libs version: 2.0.7-1ubuntu1 libraw5: description: raw image decoder library downloads: 1309 homepage_uri: http://www.libraw.org/ name: libraw5 section: libs version: 0.14.4-0ubuntu2 librbd1: description: RADOS block device client library downloads: 2913 homepage_uri: http://ceph.newdream.net/ name: librbd1 section: libs version: 0.41-1ubuntu2 librdf0: description: Redland Resource Description Framework (RDF) library downloads: 468 homepage_uri: http://librdf.org/ name: librdf0 section: libs version: 1.0.14-1 libreadline-dev: description: GNU readline and history libraries, development files downloads: 629 homepage_uri: '' name: libreadline-dev section: libdevel version: 6.2-8 libreadline-java: description: GNU readline and BSD editline wrappers for Java downloads: 102 homepage_uri: http://java-readline.sourceforge.net/ name: libreadline-java section: universe/libs version: 0.8.0.1+dfsg-2 libreadline5: description: GNU readline and history libraries, run-time libraries downloads: 12279 homepage_uri: '' name: libreadline5 section: libs version: 5.2-11 libreadline6: description: GNU readline and history libraries, run-time libraries downloads: 72242 homepage_uri: '' name: libreadline6 section: libs version: 6.2-8 libreadline6-dev: description: GNU readline and history libraries, development files downloads: 1582 homepage_uri: '' name: libreadline6-dev section: libdevel version: 6.2-8 librecode0: description: Shared library on which recode is based downloads: 5741 homepage_uri: '' name: librecode0 section: libs version: 3.6-18 libregexp-java: description: Regular expression library for Java downloads: 456 homepage_uri: http://jakarta.apache.org/regexp/ name: libregexp-java section: libs version: 1.5-3 libreoffice-base-core: description: office productivity suite -- shared library downloads: 1442 homepage_uri: http://www.libreoffice.org name: libreoffice-base-core section: editors version: 1:3.5.3-0ubuntu1 libreoffice-calc: description: office productivity suite -- spreadsheet downloads: 14504 homepage_uri: http://www.libreoffice.org name: libreoffice-calc section: editors version: 1:3.5.3-0ubuntu1 libreoffice-common: description: office productivity suite -- arch-independent files downloads: 15266 homepage_uri: http://www.libreoffice.org name: libreoffice-common section: editors version: 1:3.5.3-0ubuntu1 libreoffice-core: description: office productivity suite -- arch-dependent files downloads: 14658 homepage_uri: http://www.libreoffice.org name: libreoffice-core section: editors version: 1:3.5.3-0ubuntu1 libreoffice-draw: description: office productivity suite -- drawing downloads: 14326 homepage_uri: http://www.libreoffice.org name: libreoffice-draw section: editors version: 1:3.5.3-0ubuntu1 libreoffice-emailmerge: description: office productivity suite -- email mail merge downloads: 281 homepage_uri: http://www.libreoffice.org name: libreoffice-emailmerge section: editors version: 1:3.5.3-0ubuntu1 libreoffice-gnome: description: office productivity suite -- GNOME integration downloads: 4712 homepage_uri: http://www.libreoffice.org name: libreoffice-gnome section: gnome version: 1:3.5.3-0ubuntu1 libreoffice-gtk: description: office productivity suite -- GTK+ integration downloads: 5457 homepage_uri: http://www.libreoffice.org name: libreoffice-gtk section: gnome version: 1:3.5.3-0ubuntu1 libreoffice-help-en-us: description: office productivity suite -- English_american help downloads: 0 homepage_uri: http://www.libreoffice.org name: libreoffice-help-en-us section: doc version: 1:3.5.3-0ubuntu1 libreoffice-impress: description: office productivity suite -- presentation downloads: 14275 homepage_uri: http://www.libreoffice.org name: libreoffice-impress section: editors version: 1:3.5.3-0ubuntu1 libreoffice-math: description: office productivity suite -- equation editor downloads: 14474 homepage_uri: http://www.libreoffice.org name: libreoffice-math section: editors version: 1:3.5.3-0ubuntu1 libreoffice-style-human: description: office productivity suite -- Human symbol style downloads: 0 homepage_uri: http://www.libreoffice.org name: libreoffice-style-human section: editors version: 1:3.5.3-0ubuntu1 libreoffice-style-tango: description: office productivity suite -- Tango symbol style downloads: 0 homepage_uri: http://www.libreoffice.org name: libreoffice-style-tango section: editors version: 1:3.5.3-0ubuntu1 libreoffice-writer: description: office productivity suite -- word processor downloads: 14740 homepage_uri: http://www.libreoffice.org name: libreoffice-writer section: editors version: 1:3.5.3-0ubuntu1 librest-0.7-0: description: REST service access library downloads: 7833 homepage_uri: http://www.gnome.org/ name: librest-0.7-0 section: libs version: 0.7.12-1ubuntu2 librhino-java: description: Libraries for rhino Java Script Engine downloads: 569 homepage_uri: http://www.mozilla.org/rhino/ name: librhino-java section: java version: 1.7R3-5 librhythmbox-core5: description: support library for the rhythmbox music player downloads: 0 homepage_uri: http://projects.gnome.org/rhythmbox/ name: librhythmbox-core5 section: libs version: 2.96-0ubuntu4 libroken18-heimdal: description: Heimdal Kerberos - roken support library downloads: 462 homepage_uri: http://www.h5l.org/ name: libroken18-heimdal section: libs version: 1.6~git20120311.dfsg.1-2 libroken18-heimdal:i386: description: Heimdal Kerberos - roken support library downloads: 0 homepage_uri: http://www.h5l.org/ name: libroken18-heimdal:i386 section: libs version: 1.6~git20120311.dfsg.1-2 librsvg2-2: description: SAX-based renderer library for SVG files (runtime) downloads: 42164 homepage_uri: http://librsvg.sourceforge.net/ name: librsvg2-2 section: libs version: 2.36.1-0ubuntu1 librsvg2-2:i386: description: SAX-based renderer library for SVG files (runtime) downloads: 0 homepage_uri: http://librsvg.sourceforge.net/ name: librsvg2-2:i386 section: libs version: 2.36.1-0ubuntu1 librsvg2-common: description: SAX-based renderer library for SVG files (extra runtime) downloads: 44086 homepage_uri: http://librsvg.sourceforge.net/ name: librsvg2-common section: libs version: 2.36.1-0ubuntu1 librsvg2-common:i386: description: SAX-based renderer library for SVG files (extra runtime) downloads: 0 homepage_uri: http://librsvg.sourceforge.net/ name: librsvg2-common:i386 section: libs version: 2.36.1-0ubuntu1 librsync1: description: rsync remote-delta algorithm library downloads: 2601 homepage_uri: http://librsync.sourceforge.net/ name: librsync1 section: libs version: 0.9.7-8build1 librtmp-dev: description: toolkit for RTMP streams (development files) downloads: 1148 homepage_uri: http://rtmpdump.mplayerhq.hu/ name: librtmp-dev section: libdevel version: 2.4~20110711.gitc28f1bab-1 librtmp0: description: toolkit for RTMP streams (shared library) downloads: 23234 homepage_uri: http://rtmpdump.mplayerhq.hu/ name: librtmp0 section: libs version: 2.4~20110711.gitc28f1bab-1 librtmp0:i386: description: toolkit for RTMP streams (shared library) downloads: 0 homepage_uri: http://rtmpdump.mplayerhq.hu/ name: librtmp0:i386 section: libs version: 2.4~20110711.gitc28f1bab-1 libruby: description: Transitional package for libruby1.8 downloads: 92 homepage_uri: http://www.ruby-lang.org/ name: libruby section: libs version: '4.8' libruby1.8: description: Libraries necessary to run Ruby 1.8 downloads: 11284 homepage_uri: http://www.ruby-lang.org/ name: libruby1.8 section: libs version: 1.8.7.352-2ubuntu1 libruby1.9.1: description: Libraries necessary to run Ruby 1.9.1 downloads: 1918 homepage_uri: http://www.ruby-lang.org/ name: libruby1.9.1 section: libs version: 1.9.3.0-1ubuntu1 libsamplerate0: description: Audio sample rate conversion library downloads: 29750 homepage_uri: http://www.mega-nerd.com/SRC/ name: libsamplerate0 section: libs version: 0.1.8-4 libsamplerate0:i386: description: Audio sample rate conversion library downloads: 0 homepage_uri: http://www.mega-nerd.com/SRC/ name: libsamplerate0:i386 section: libs version: 0.1.8-4 libsane: description: API library for scanners downloads: 23907 homepage_uri: http://www.sane-project.org name: libsane section: libs version: 1.0.22-7ubuntu1 libsane-common: description: API library for scanners -- documentation and support files downloads: 13894 homepage_uri: http://www.sane-project.org name: libsane-common section: libs version: 1.0.22-7ubuntu1 libsane-hpaio: description: HP SANE backend for multi-function peripherals downloads: 13127 homepage_uri: http://hplipopensource.com/hplip-web/index.html name: libsane-hpaio section: libs version: 3.12.2-1ubuntu3 libsane:i386: description: API library for scanners downloads: 0 homepage_uri: http://www.sane-project.org name: libsane:i386 section: libs version: 1.0.22-7ubuntu1 libsasl2-2: description: Cyrus SASL - authentication abstraction library downloads: 70343 homepage_uri: http://www.cyrusimap.org/ name: libsasl2-2 section: libs version: 2.1.25.dfsg1-3ubuntu0.1 libsasl2-2:i386: description: Cyrus SASL - authentication abstraction library downloads: 0 homepage_uri: http://www.cyrusimap.org/ name: libsasl2-2:i386 section: libs version: 2.1.25.dfsg1-3ubuntu0.1 libsasl2-modules: description: Cyrus SASL - pluggable authentication modules downloads: 16494 homepage_uri: http://www.cyrusimap.org/ name: libsasl2-modules section: devel version: 2.1.25.dfsg1-3ubuntu0.1 libsasl2-modules:i386: description: Cyrus SASL - pluggable authentication modules downloads: 0 homepage_uri: http://www.cyrusimap.org/ name: libsasl2-modules:i386 section: devel version: 2.1.25.dfsg1-3ubuntu0.1 libsaxon-java: description: Saxon XSLT Processor downloads: 501 homepage_uri: http://saxon.sourceforge.net/ name: libsaxon-java section: libs version: 1:6.5.5-8 libschroedinger-1.0-0: description: library for encoding/decoding of Dirac video streams downloads: 1181 homepage_uri: '' name: libschroedinger-1.0-0 section: libs version: 1.0.11-1 libsctp1: description: user-space access to Linux Kernel SCTP - shared library downloads: 294 homepage_uri: http://lksctp.sf.net/ name: libsctp1 section: libs version: 1.0.11+dfsg-2 libsdl-image1.2:i386: description: image loading library for Simple DirectMedia Layer 1.2 downloads: 0 homepage_uri: http://www.libsdl.org/projects/SDL_image/ name: libsdl-image1.2:i386 section: universe/libs version: 1.2.10-3 libsdl-mixer1.2:i386: description: Mixer library for Simple DirectMedia Layer 1.2, libraries downloads: 0 homepage_uri: http://www.libsdl.org/projects/SDL_mixer/ name: libsdl-mixer1.2:i386 section: universe/libs version: 1.2.11-7 libsdl-net1.2:i386: description: Network library for Simple DirectMedia Layer 1.2, libraries downloads: 0 homepage_uri: http://www.libsdl.org/projects/SDL_net/ name: libsdl-net1.2:i386 section: universe/libs version: 1.2.7-5 libsdl-ttf2.0-0: description: ttf library for Simple DirectMedia Layer with FreeType 2 support downloads: 3157 homepage_uri: '' name: libsdl-ttf2.0-0 section: universe/libs version: 2.0.9-1.1ubuntu1 libsdl-ttf2.0-0:i386: description: ttf library for Simple DirectMedia Layer with FreeType 2 support downloads: 0 homepage_uri: '' name: libsdl-ttf2.0-0:i386 section: universe/libs version: 2.0.9-1.1ubuntu1 libsdl1.2debian: description: Simple DirectMedia Layer downloads: 12691 homepage_uri: http://www.libsdl.org/ name: libsdl1.2debian section: libs version: 1.2.14-6.4ubuntu3 libsdl1.2debian:i386: description: Simple DirectMedia Layer downloads: 0 homepage_uri: http://www.libsdl.org/ name: libsdl1.2debian:i386 section: libs version: 1.2.14-6.4ubuntu3 libselinux1: description: SELinux runtime shared libraries downloads: 110664 homepage_uri: '' name: libselinux1 section: libs version: 2.1.0-4.1ubuntu1 libselinux1-dev: description: SELinux development headers downloads: 706 homepage_uri: '' name: libselinux1-dev section: libs version: 2.1.0-4.1ubuntu1 libselinux1:i386: description: SELinux runtime shared libraries downloads: 0 homepage_uri: '' name: libselinux1:i386 section: libs version: 2.1.0-4.1ubuntu1 libsensors4: description: library to read temperature/voltage/fan sensors downloads: 48392 homepage_uri: http://www.lm-sensors.org name: libsensors4 section: libs version: 1:3.3.1-2ubuntu1 libsepol1: description: SELinux library for manipulating binary security policies downloads: 108108 homepage_uri: '' name: libsepol1 section: libs version: 2.1.0-1.2 libsepol1-dev: description: SELinux binary policy maniulation library and development files downloads: 711 homepage_uri: '' name: libsepol1-dev section: libdevel version: 2.1.0-1.2 libservlet2.4-java: description: Servlet 2.4 and JSP 2.0 Java library downloads: 15 homepage_uri: http://jakarta.apache.org/tomcat/index.html name: libservlet2.4-java section: universe/libs version: 5.5.33-1 libservlet2.5-java: description: Servlet 2.5 and JSP 2.1 Java API classes downloads: 1271 homepage_uri: http://tomcat.apache.org name: libservlet2.5-java section: web version: 6.0.35-1ubuntu3 libsgutils2-2: description: utilities for devices using the SCSI command set (shared libraries) downloads: 14688 homepage_uri: http://sg.danny.cz/sg/ name: libsgutils2-2 section: libs version: 1.33-1 libshout3: description: MP3/Ogg Vorbis broadcast streaming library downloads: 1266 homepage_uri: http://www.icecast.org/ name: libshout3 section: libs version: 2.2.2-7ubuntu1 libshout3:i386: description: MP3/Ogg Vorbis broadcast streaming library downloads: 0 homepage_uri: http://www.icecast.org/ name: libshout3:i386 section: libs version: 2.2.2-7ubuntu1 libsigc++-2.0-0c2a: description: type-safe Signal Framework for C++ - runtime downloads: 339 homepage_uri: '' name: libsigc++-2.0-0c2a section: libs version: 2.2.10-0ubuntu2 libsigsegv2: description: Library for handling page faults in a portable way downloads: 10017 homepage_uri: http://libsigsegv.sourceforge.net/ name: libsigsegv2 section: libs version: 2.9-4ubuntu2 libslang2: description: S-Lang programming library - runtime version downloads: 40802 homepage_uri: '' name: libslang2 section: libs version: 2.2.4-3ubuntu1 libslang2:i386: description: S-Lang programming library - runtime version downloads: 0 homepage_uri: '' name: libslang2:i386 section: libs version: 2.2.4-3ubuntu1 libslf4j-java: description: Simple Logging Facade for Java downloads: 400 homepage_uri: http://www.slf4j.org/ name: libslf4j-java section: universe/libs version: 1.6.4-1 libslp1: description: OpenSLP libraries downloads: 3525 homepage_uri: http://www.openslp.org/ name: libslp1 section: libs version: 1.2.1-7.8ubuntu1 libsm-dev: description: X11 Session Management library (development headers) downloads: 4318 homepage_uri: '' name: libsm-dev section: libdevel version: 2:1.2.0-2build1 libsm6: description: X11 Session Management library downloads: 11640 homepage_uri: '' name: libsm6 section: libs version: 2:1.2.0-2build1 libsm6:i386: description: X11 Session Management library downloads: 0 homepage_uri: '' name: libsm6:i386 section: libs version: 2:1.2.0-2build1 libsmbclient: description: shared library for communication with SMB/CIFS servers downloads: 33987 homepage_uri: http://www.samba.org name: libsmbclient section: libs version: 2:3.6.3-2ubuntu2.2 libsndfile1: description: Library for reading/writing audio files downloads: 11542 homepage_uri: http://www.mega-nerd.com/libsndfile/ name: libsndfile1 section: libs version: 1.0.25-4 libsndfile1:i386: description: Library for reading/writing audio files downloads: 0 homepage_uri: http://www.mega-nerd.com/libsndfile/ name: libsndfile1:i386 section: libs version: 1.0.25-4 libsnmp-base: description: SNMP (Simple Network Management Protocol) MIBs and documentation downloads: 18974 homepage_uri: http://net-snmp.sourceforge.net/ name: libsnmp-base section: libs version: 5.4.3~dfsg-2.4ubuntu1.1 libsnmp15: description: SNMP (Simple Network Management Protocol) library downloads: 27082 homepage_uri: http://net-snmp.sourceforge.net/ name: libsnmp15 section: libs version: 5.4.3~dfsg-2.4ubuntu1.1 libsoap-lite-perl: description: Perl implementation of a SOAP client and server downloads: 1802 homepage_uri: http://sourceforge.net/projects/soaplite/ name: libsoap-lite-perl section: universe/perl version: 0.714-1 libsocket6-perl: description: Perl extensions for IPv6 downloads: 11051 homepage_uri: '' name: libsocket6-perl section: perl version: 0.23-1build2 libsonic0: description: Simple library to speed up or slow down speech downloads: 132 homepage_uri: http://dev.vinux-project.org/sonic name: libsonic0 section: libs version: 0.1.17-1.1 libsoup-gnome2.4-1: description: HTTP library implementation in C -- GNOME support library downloads: 26762 homepage_uri: '' name: libsoup-gnome2.4-1 section: libs version: 2.38.1-1 libsoup-gnome2.4-1:i386: description: HTTP library implementation in C -- GNOME support library downloads: 0 homepage_uri: '' name: libsoup-gnome2.4-1:i386 section: libs version: 2.38.1-1 libsoup2.4-1: description: HTTP library implementation in C -- Shared library downloads: 35798 homepage_uri: '' name: libsoup2.4-1 section: libs version: 2.38.1-1 libsoup2.4-1:i386: description: HTTP library implementation in C -- Shared library downloads: 0 homepage_uri: '' name: libsoup2.4-1:i386 section: libs version: 2.38.1-1 libsparsehash-dev: description: Google's extremely memory-efficient C++ hash_map implementation downloads: 10 homepage_uri: http://code.google.com/p/google-sparsehash name: libsparsehash-dev section: universe/libdevel version: 1.10-1build1 libspectre1: description: Library for rendering PostScript documents downloads: 51 homepage_uri: http://libspectre.freedesktop.org name: libspectre1 section: libs version: 0.2.6-1build1 libspeechd2: description: 'Speech Dispatcher: Shared libraries' downloads: 1663 homepage_uri: http://devel.freebsoft.org/speechd name: libspeechd2 section: libs version: 0.7.1-6ubuntu3 libspeex1: description: The Speex codec runtime library downloads: 21855 homepage_uri: http://www.speex.org/ name: libspeex1 section: libs version: 1.2~rc1-3ubuntu2 libspeex1:i386: description: The Speex codec runtime library downloads: 0 homepage_uri: http://www.speex.org/ name: libspeex1:i386 section: libs version: 1.2~rc1-3ubuntu2 libspeexdsp1: description: The Speex extended runtime library downloads: 5580 homepage_uri: http://www.speex.org/ name: libspeexdsp1 section: libs version: 1.2~rc1-3ubuntu2 libspeexdsp1:i386: description: The Speex extended runtime library downloads: 0 homepage_uri: http://www.speex.org/ name: libspeexdsp1:i386 section: libs version: 1.2~rc1-3ubuntu2 libsprng2: description: SPRNG Scalable Parallel RNG library -- library package downloads: 71 homepage_uri: '' name: libsprng2 section: universe/math version: 2.0a-8ubuntu1 libsqlite0: description: SQLite shared library downloads: 1414 homepage_uri: '' name: libsqlite0 section: universe/libs version: 2.8.17-7fakesync1build1 libsqlite0-dev: description: SQLite development files downloads: 142 homepage_uri: '' name: libsqlite0-dev section: universe/libdevel version: 2.8.17-7fakesync1build1 libsqlite3-0: description: SQLite 3 shared library downloads: 81378 homepage_uri: '' name: libsqlite3-0 section: libs version: 3.7.9-2ubuntu1 libsqlite3-0:i386: description: SQLite 3 shared library downloads: 0 homepage_uri: '' name: libsqlite3-0:i386 section: libs version: 3.7.9-2ubuntu1 libsqlite3-dev: description: SQLite 3 development files downloads: 1843 homepage_uri: '' name: libsqlite3-dev section: libdevel version: 3.7.9-2ubuntu1 libss2: description: command-line interface parsing library downloads: 1052 homepage_uri: http://e2fsprogs.sourceforge.net name: libss2 section: libs version: 1.42-1ubuntu2 libssh-4: description: tiny C SSH library downloads: 3905 homepage_uri: http://www.libssh.org/ name: libssh-4 section: libs version: 0.5.2-1 libssl-dev: description: SSL development libraries, header files and documentation downloads: 5911 homepage_uri: '' name: libssl-dev section: libdevel version: 1.0.1-4ubuntu5.2 libssl-doc: description: SSL development documentation documentation downloads: 101 homepage_uri: '' name: libssl-doc section: doc version: 1.0.1-4ubuntu5.2 libssl0.9.8:i386: description: SSL shared libraries downloads: 0 homepage_uri: '' name: libssl0.9.8:i386 section: universe/libs version: 0.9.8o-7ubuntu3.1 libssl1.0.0: description: SSL shared libraries downloads: 22162 homepage_uri: '' name: libssl1.0.0 section: libs version: 1.0.1-4ubuntu5.2 libssl1.0.0:i386: description: SSL shared libraries downloads: 0 homepage_uri: '' name: libssl1.0.0:i386 section: libs version: 1.0.1-4ubuntu5.2 libstartup-notification0: description: library for program launch feedback (shared library) downloads: 36041 homepage_uri: '' name: libstartup-notification0 section: libs version: 0.12-1ubuntu1 libstdc++5:i386: description: The GNU Standard C++ Library v3 downloads: 0 homepage_uri: '' name: libstdc++5:i386 section: universe/libs version: 1:3.3.6-25ubuntu1 libstdc++6: description: GNU Standard C++ Library v3 downloads: 73251 homepage_uri: http://gcc.gnu.org/ name: libstdc++6 section: libs version: 4.6.3-1ubuntu5 libstdc++6-4.6-dev: description: GNU Standard C++ Library v3 (development files) downloads: 5740 homepage_uri: http://gcc.gnu.org/ name: libstdc++6-4.6-dev section: libdevel version: 4.6.3-1ubuntu5 libstdc++6:i386: description: GNU Standard C++ Library v3 downloads: 0 homepage_uri: http://gcc.gnu.org/ name: libstdc++6:i386 section: libs version: 4.6.3-1ubuntu5 libstringtemplate-java: description: StringTemplate templating engine for Java downloads: 842 homepage_uri: http://www.stringtemplate.org/ name: libstringtemplate-java section: universe/devel version: 3.2.1-1 libsub-name-perl: description: module for assigning a new name to referenced sub downloads: 2975 homepage_uri: http://search.cpan.org/dist/Sub-Name/ name: libsub-name-perl section: perl version: 0.05-1build2 libsvn1: description: Shared libraries used by Subversion downloads: 12515 homepage_uri: http://subversion.apache.org/ name: libsvn1 section: libs version: 1.6.17dfsg-3ubuntu3 libswitch-perl: description: switch statement for Perl downloads: 16209 homepage_uri: http://search.cpan.org/dist/Switch/ name: libswitch-perl section: perl version: 2.16-2 libswscale2: description: Libav video scaling library downloads: 2242 homepage_uri: http://libav.org/ name: libswscale2 section: libs version: 4:0.8.1-0ubuntu1 libswt-cairo-gtk-3-jni: description: Standard Widget Toolkit for GTK+ Cairo JNI library downloads: 927 homepage_uri: http://www.eclipse.org/swt/ name: libswt-cairo-gtk-3-jni section: universe/libs version: 3.7.2-2 libswt-glx-gtk-3-jni: description: Standard Widget Toolkit for GTK+ GLX JNI library downloads: 48 homepage_uri: http://www.eclipse.org/swt/ name: libswt-glx-gtk-3-jni section: universe/libs version: 3.7.2-2 libswt-gnome-gtk-3-jni: description: Standard Widget Toolkit for GTK+ GNOME JNI library downloads: 244 homepage_uri: http://www.eclipse.org/swt/ name: libswt-gnome-gtk-3-jni section: universe/libs version: 3.7.2-2 libswt-gtk-3-java: description: Standard Widget Toolkit for GTK+ Java library downloads: 971 homepage_uri: http://www.eclipse.org/swt/ name: libswt-gtk-3-java section: universe/java version: 3.7.2-2 libswt-gtk-3-jni: description: Standard Widget Toolkit for GTK+ JNI library downloads: 942 homepage_uri: http://www.eclipse.org/swt/ name: libswt-gtk-3-jni section: universe/libs version: 3.7.2-2 libswt-webkit-gtk-3-jni: description: Standard Widget Toolkit for GTK+ WebKit JNI library downloads: 916 homepage_uri: http://www.eclipse.org/swt/ name: libswt-webkit-gtk-3-jni section: universe/libs version: 3.7.2-2 libsyncdaemon-1.0-1: description: Ubuntu One synchronization daemon library downloads: 0 homepage_uri: https://one.ubuntu.com name: libsyncdaemon-1.0-1 section: libs version: 3.0.1-0ubuntu1.0.1 libsys-hostname-long-perl: description: Figure out the long (fully-qualified) hostname downloads: 7705 homepage_uri: http://search.cpan.org/dist/Sys-Hostname-Long/ name: libsys-hostname-long-perl section: perl version: 1.4-2 libsysfs2: description: interface library to sysfs downloads: 1267 homepage_uri: '' name: libsysfs2 section: libs version: 2.1.0+repack-1 libt1-5: description: Type 1 font rasterizer library - runtime downloads: 328 homepage_uri: ftp://sunsite.unc.edu/pub/Linux/libs/graphics/ name: libt1-5 section: libs version: 5.1.2-3.4ubuntu1 libtag1-vanilla: description: audio meta-data library - vanilla flavour downloads: 3317 homepage_uri: http://developer.kde.org/~wheeler/taglib.html name: libtag1-vanilla section: libs version: 1.7-1ubuntu5 libtag1-vanilla:i386: description: audio meta-data library - vanilla flavour downloads: 0 homepage_uri: http://developer.kde.org/~wheeler/taglib.html name: libtag1-vanilla:i386 section: libs version: 1.7-1ubuntu5 libtag1c2a: description: audio meta-data library downloads: 2589 homepage_uri: http://developer.kde.org/~wheeler/taglib.html name: libtag1c2a section: libs version: 1.7-1ubuntu5 libtag1c2a:i386: description: audio meta-data library downloads: 0 homepage_uri: http://developer.kde.org/~wheeler/taglib.html name: libtag1c2a:i386 section: libs version: 1.7-1ubuntu5 libtalloc2: description: hierarchical pool based memory allocator downloads: 42627 homepage_uri: http://talloc.samba.org/ name: libtalloc2 section: libs version: 2.0.7-3 libtask-weaken-perl: description: Ensure that a platform has weaken support downloads: 1721 homepage_uri: http://search.cpan.org/dist/Task-Weaken/ name: libtask-weaken-perl section: perl version: 1.03-1 libtasn1-3: description: Manage ASN.1 structures (runtime) downloads: 63506 homepage_uri: http://www.gnu.org/software/libtasn1/ name: libtasn1-3 section: libs version: 2.10-1ubuntu1.1 libtasn1-3-dev: description: Manage ASN.1 structures (development) downloads: 2279 homepage_uri: http://www.gnu.org/software/libtasn1/ name: libtasn1-3-dev section: libdevel version: 2.10-1ubuntu1.1 libtasn1-3:i386: description: Manage ASN.1 structures (runtime) downloads: 0 homepage_uri: http://www.gnu.org/software/libtasn1/ name: libtasn1-3:i386 section: libs version: 2.10-1ubuntu1.1 libtdb1: description: Trivial Database - shared library downloads: 43875 homepage_uri: http://tdb.samba.org/ name: libtdb1 section: libs version: 1.2.9-4 libtdb1:i386: description: Trivial Database - shared library downloads: 0 homepage_uri: http://tdb.samba.org/ name: libtdb1:i386 section: libs version: 1.2.9-4 libtelepathy-farstream2: description: Glue library between telepathy and farstream downloads: 16 homepage_uri: http://telepathy.freedesktop.org/ name: libtelepathy-farstream2 section: libs version: 0.4.0-0ubuntu1 libtelepathy-glib0: description: Telepathy framework - GLib library downloads: 12519 homepage_uri: http://telepathy.freedesktop.org/wiki/ name: libtelepathy-glib0 section: libs version: 0.18.0-1ubuntu1 libtelepathy-logger2: description: Telepathy logger service - utility library downloads: 8063 homepage_uri: http://telepathy.freedesktop.org/wiki/ name: libtelepathy-logger2 section: libs version: 0.4.0-0ubuntu1 libterm-readkey-perl: description: A perl module for simple terminal control downloads: 5229 homepage_uri: http://search.cpan.org/dist/TermReadKey/ name: libterm-readkey-perl section: perl version: 2.30-4build3 libtext-charwidth-perl: description: get display widths of characters on the terminal downloads: 56874 homepage_uri: http://search.cpan.org/search?module=Text::CharWidth name: libtext-charwidth-perl section: perl version: 0.04-7build1 libtext-csv-perl: description: comma-separated values manipulator (using XS or PurePerl) downloads: 379 homepage_uri: http://search.cpan.org/dist/Text-CSV/ name: libtext-csv-perl section: perl version: 1.21-1 libtext-csv-xs-perl: description: Perl C/XS module to process Comma-Separated Value files downloads: 547 homepage_uri: http://search.cpan.org/dist/Text-CSV_XS/ name: libtext-csv-xs-perl section: perl version: 0.85-1build1 libtext-iconv-perl: description: converts between character sets in Perl downloads: 55885 homepage_uri: http://search.cpan.org/search?module=Text::Iconv name: libtext-iconv-perl section: perl version: 1.7-5 libtext-wrapi18n-perl: description: internationalized substitute of Text::Wrap downloads: 56781 homepage_uri: http://search.cpan.org/search?module=Text::WrapI18N name: libtext-wrapi18n-perl section: perl version: 0.06-7 libthai-data: description: Data files for Thai language support library downloads: 44645 homepage_uri: http://linux.thai.net/projects/libthai name: libthai-data section: libs version: 0.1.16-3 libthai0: description: Thai language support library downloads: 53443 homepage_uri: http://linux.thai.net/projects/libthai name: libthai0 section: libs version: 0.1.16-3 libthai0:i386: description: Thai language support library downloads: 0 homepage_uri: http://linux.thai.net/projects/libthai name: libthai0:i386 section: libs version: 0.1.16-3 libtheora0: description: The Theora Video Compression Codec downloads: 2207 homepage_uri: http://www.theora.org/ name: libtheora0 section: libs version: 1.1.1+dfsg.1-3ubuntu2 libtheora0:i386: description: The Theora Video Compression Codec downloads: 0 homepage_uri: http://www.theora.org/ name: libtheora0:i386 section: libs version: 1.1.1+dfsg.1-3ubuntu2 libtie-ixhash-perl: description: ordered associative arrays for Perl downloads: 1420 homepage_uri: '' name: libtie-ixhash-perl section: perl version: 1.21-2 libtiff4: description: Tag Image File Format (TIFF) library downloads: 39340 homepage_uri: http://libtiff.maptools.org name: libtiff4 section: libs version: 3.9.5-2ubuntu1 libtiff4:i386: description: Tag Image File Format (TIFF) library downloads: 0 homepage_uri: http://libtiff.maptools.org name: libtiff4:i386 section: libs version: 3.9.5-2ubuntu1 libtimedate-perl: description: collection of modules to manipulate date/time information downloads: 15372 homepage_uri: http://search.cpan.org/dist/TimeDate/ name: libtimedate-perl section: perl version: 1.2000-1 libtimezonemap1: description: GTK+3 timezone map widget downloads: 0 homepage_uri: '' name: libtimezonemap1 section: libs version: 0.3.2 libtinfo-dev: description: developer's library for the low-level terminfo library downloads: 2346 homepage_uri: http://invisible-island.net/ncurses/ name: libtinfo-dev section: libdevel version: 5.9-4 libtinfo5: description: shared low-level terminfo library for terminal handling downloads: 21285 homepage_uri: http://invisible-island.net/ncurses/ name: libtinfo5 section: libs version: 5.9-4 libtinfo5:i386: description: shared low-level terminfo library for terminal handling downloads: 0 homepage_uri: http://invisible-island.net/ncurses/ name: libtinfo5:i386 section: libs version: 5.9-4 libtirpc1: description: transport-independent RPC library downloads: 13634 homepage_uri: '' name: libtirpc1 section: libs version: 0.2.2-5 libtntnet9: description: Tntnet libraries downloads: 0 homepage_uri: http://www.tntnet.org/ name: libtntnet9 section: universe/libs version: 2.0+dfsg1-2 libtokyocabinet-dev: description: Tokyo Cabinet Database Libraries [development] downloads: 38 homepage_uri: '' name: libtokyocabinet-dev section: libdevel version: 1.4.37-6.1build1 libtokyocabinet8: description: Tokyo Cabinet Database Libraries [runtime] downloads: 6 homepage_uri: '' name: libtokyocabinet8 section: libs version: 1.4.37-6.1build1 libtorque2: description: shared library for Torque client and server downloads: 77 homepage_uri: http://www.clusterresources.com/pages/products/torque/ name: libtorque2 section: universe/libs version: 2.4.16+dfsg-1build1 libtotem-plparser17: description: Totem Playlist Parser library - runtime files downloads: 30386 homepage_uri: '' name: libtotem-plparser17 section: libs version: 3.4.1-1 libtotem0: description: Main library for the Totem media player downloads: 8049 homepage_uri: http://www.gnome.org/projects/totem/ name: libtotem0 section: video version: 3.0.1-0ubuntu21 libubuntuoneui-3.0-1: description: Ubuntu One widget library downloads: 0 homepage_uri: https://launchpad.net/libubuntuone name: libubuntuoneui-3.0-1 section: libs version: 3.0.1-0ubuntu1 libudev0: description: udev library downloads: 72455 homepage_uri: http://www.kernel.org/pub/linux/utils/kernel/hotplug/udev.html name: libudev0 section: admin version: 175-0ubuntu9 libudev0:i386: description: udev library downloads: 0 homepage_uri: http://www.kernel.org/pub/linux/utils/kernel/hotplug/udev.html name: libudev0:i386 section: admin version: 175-0ubuntu9 libumfpack5.4.0: description: sparse LU factorization library downloads: 0 homepage_uri: http://www.cise.ufl.edu/research/sparse/SuiteSparse/ name: libumfpack5.4.0 section: universe/libs version: 1:3.4.0-2ubuntu3 libunique-3.0-0: description: Library for writing single instance applications - shared libraries downloads: 7505 homepage_uri: http://live.gnome.org/LibUnique name: libunique-3.0-0 section: libs version: 3.0.2-1 libunistring0: description: Unicode string library for C downloads: 14108 homepage_uri: http://www.gnu.org/software/libunistring/ name: libunistring0 section: libs version: 0.9.3-5 libunistring0:i386: description: Unicode string library for C downloads: 0 homepage_uri: http://www.gnu.org/software/libunistring/ name: libunistring0:i386 section: libs version: 0.9.3-5 libunity-2d-private0: description: Unity 2D shared library downloads: 0 homepage_uri: '' name: libunity-2d-private0 section: libs version: 5.12.0-0ubuntu1.1 libunity-core-5.0-5: description: Core library for the Unity interface. downloads: 0 homepage_uri: https://launchpad.net/unity name: libunity-core-5.0-5 section: libs version: 5.12-0ubuntu1.1 libunity-misc4: description: Miscellaneous functions for Unity - shared library downloads: 0 homepage_uri: https://launchpad.net/libunity-misc name: libunity-misc4 section: libs version: 4.0.4-0ubuntu2 libunity9: description: binding to get places into the launcher - shared library downloads: 0 homepage_uri: https://launchpad.net/libunity name: libunity9 section: libs version: 5.12.0-0ubuntu1 libupower-glib1: description: abstraction for power management - shared library downloads: 35721 homepage_uri: http://upower.freedesktop.org/ name: libupower-glib1 section: libs version: 0.9.15-3git1 liburi-perl: description: module to manipulate and access URI strings downloads: 40880 homepage_uri: http://search.cpan.org/dist/URI/ name: liburi-perl section: perl version: 1.59-1 libusb-0.1-4: description: userspace USB programming library downloads: 59696 homepage_uri: http://www.linux-usb.org/ name: libusb-0.1-4 section: libs version: 2:0.1.12-20 libusb-0.1-4:i386: description: userspace USB programming library downloads: 0 homepage_uri: http://www.linux-usb.org/ name: libusb-0.1-4:i386 section: libs version: 2:0.1.12-20 libusb-1.0-0: description: userspace USB programming library downloads: 43040 homepage_uri: http://www.linux-usb.org/ name: libusb-1.0-0 section: libs version: 2:1.0.9~rc3-2ubuntu1 libusbmuxd1: description: USB multiplexor daemon for iPhone and iPod Touch devices - library downloads: 14591 homepage_uri: http://marcansoft.com/blog/iphonelinux/usbmuxd/ name: libusbmuxd1 section: libs version: 1.0.7-2 libutempter0: description: A privileged helper for utmp/wtmp updates (runtime) downloads: 13636 homepage_uri: http://freshmeat.net/projects/libutempter name: libutempter0 section: libs version: 1.1.5-4 libutouch-evemu1: description: KernelInput Event Device Emulation Library downloads: 0 homepage_uri: https://launchpad.net/utouch-evemu name: libutouch-evemu1 section: libs version: 1.0.9-0ubuntu1 libutouch-frame1: description: Touch Frame Library downloads: 0 homepage_uri: https://launchpad.net/utouch-frame name: libutouch-frame1 section: libs version: 2.2.3-0ubuntu1 libutouch-geis1: description: Gesture engine interface support downloads: 0 homepage_uri: https://launchpad.net/utouch-geis name: libutouch-geis1 section: libs version: 2.2.9-0ubuntu3 libutouch-grail1: description: Gesture Recognition And Instantiation Library downloads: 0 homepage_uri: https://launchpad.net/utouch-grail name: libutouch-grail1 section: libs version: 3.0.5-0ubuntu1 libuuid-perl: description: Perl extension for using UUID interfaces as defined in e2fsprogs downloads: 37705 homepage_uri: http://search.cpan.org/dist/UUID/ name: libuuid-perl section: perl version: 0.02-4ubuntu1 libuuid1: description: Universally Unique ID library downloads: 73488 homepage_uri: http://userweb.kernel.org/~kzak/util-linux/ name: libuuid1 section: libs version: 2.20.1-1ubuntu3 libuuid1:i386: description: Universally Unique ID library downloads: 0 homepage_uri: http://userweb.kernel.org/~kzak/util-linux/ name: libuuid1:i386 section: libs version: 2.20.1-1ubuntu3 libv4l-0: description: Collection of video4linux support libraries downloads: 4781 homepage_uri: http://linuxtv.org/downloads/v4l-utils/ name: libv4l-0 section: libs version: 0.8.6-1ubuntu2 libv4l-0:i386: description: Collection of video4linux support libraries downloads: 0 homepage_uri: http://linuxtv.org/downloads/v4l-utils/ name: libv4l-0:i386 section: libs version: 0.8.6-1ubuntu2 libv4lconvert0: description: Video4linux frame format conversion library downloads: 3177 homepage_uri: http://linuxtv.org/downloads/v4l-utils/ name: libv4lconvert0 section: libs version: 0.8.6-1ubuntu2 libv4lconvert0:i386: description: Video4linux frame format conversion library downloads: 0 homepage_uri: http://linuxtv.org/downloads/v4l-utils/ name: libv4lconvert0:i386 section: libs version: 0.8.6-1ubuntu2 libva1: description: Video Acceleration (VA) API for Linux -- runtime downloads: 2614 homepage_uri: http://www.freedesktop.org/wiki/Software/vaapi name: libva1 section: libs version: 1.0.15-4 libvibrant6a: description: NCBI libraries for graphic biology applications downloads: 120 homepage_uri: http://www.ncbi.nlm.nih.gov/IEB/ToolBox/ name: libvibrant6a section: libs version: 6.1.20110713-3ubuntu2 libvirt-bin: description: programs for the libvirt library downloads: 3171 homepage_uri: http://libvirt.org name: libvirt-bin section: devel version: 0.9.8-2ubuntu17.1 libvirt0: description: library for interfacing with different virtualization systems downloads: 532 homepage_uri: http://libvirt.org name: libvirt0 section: devel version: 0.9.8-2ubuntu17.1 libvisual-0.4-0: description: Audio visualization framework downloads: 335 homepage_uri: '' name: libvisual-0.4-0 section: libs version: 0.4.0-4 libvisual-0.4-0:i386: description: Audio visualization framework downloads: 0 homepage_uri: '' name: libvisual-0.4-0:i386 section: libs version: 0.4.0-4 libvisual-0.4-plugins: description: Audio visualization framework plugins downloads: 2341 homepage_uri: '' name: libvisual-0.4-plugins section: sound version: 0.4.0.dfsg.1-7 libvisual-0.4-plugins:i386: description: Audio visualization framework plugins downloads: 0 homepage_uri: '' name: libvisual-0.4-plugins:i386 section: sound version: 0.4.0.dfsg.1-7 libvncserver0: description: API to write one's own vnc server downloads: 1953 homepage_uri: http://libvncserver.sourceforge.net name: libvncserver0 section: libs version: 0.9.8.2-2ubuntu1 libvorbis0a: description: The Vorbis General Audio Compression Codec (Decoder library) downloads: 42690 homepage_uri: '' name: libvorbis0a section: libs version: 1.3.2-1ubuntu3 libvorbis0a:i386: description: The Vorbis General Audio Compression Codec (Decoder library) downloads: 0 homepage_uri: '' name: libvorbis0a:i386 section: libs version: 1.3.2-1ubuntu3 libvorbisenc2: description: The Vorbis General Audio Compression Codec (Encoder library) downloads: 15005 homepage_uri: '' name: libvorbisenc2 section: libs version: 1.3.2-1ubuntu3 libvorbisenc2:i386: description: The Vorbis General Audio Compression Codec (Encoder library) downloads: 0 homepage_uri: '' name: libvorbisenc2:i386 section: libs version: 1.3.2-1ubuntu3 libvorbisfile3: description: The Vorbis General Audio Compression Codec (High Level API) downloads: 41932 homepage_uri: '' name: libvorbisfile3 section: libs version: 1.3.2-1ubuntu3 libvorbisfile3:i386: description: The Vorbis General Audio Compression Codec (High Level API) downloads: 0 homepage_uri: '' name: libvorbisfile3:i386 section: libs version: 1.3.2-1ubuntu3 libvpx1: description: VP8 video codec (shared library) downloads: 2840 homepage_uri: http://www.webmproject.org name: libvpx1 section: libs version: 1.0.0-1 libvte-2.90-9: description: Terminal emulator widget for GTK+ 3.0 - runtime files downloads: 6293 homepage_uri: '' name: libvte-2.90-9 section: libs version: 1:0.32.1-0ubuntu1 libvte-2.90-common: description: Terminal emulator widget for GTK+ 3.0 - common files downloads: 1568 homepage_uri: '' name: libvte-2.90-common section: libs version: 1:0.32.1-0ubuntu1 libvte-common: description: Terminal emulator widget for GTK+ 2.x - common files downloads: 23330 homepage_uri: '' name: libvte-common section: libs version: 1:0.28.2-3ubuntu2 libvte9: description: Terminal emulator widget for GTK+ 2.0 - runtime files downloads: 27115 homepage_uri: '' name: libvte9 section: libs version: 1:0.28.2-3ubuntu2 libvtk5.8: description: Visualization Toolkit - A high level 3D visualization library - runtime downloads: 116 homepage_uri: http://www.vtk.org/ name: libvtk5.8 section: universe/libs version: 5.8.0-5 libwacom-common: description: Wacom model feature query library (common files) downloads: 5066 homepage_uri: '' name: libwacom-common section: libs version: 0.4-1ubuntu1 libwacom2: description: Wacom model feature query library downloads: 5120 homepage_uri: '' name: libwacom2 section: libs version: 0.4-1ubuntu1 libwagon-java: description: tools to manage Maven artifacts and deployment downloads: 22 homepage_uri: http://maven.apache.org/wagon/ name: libwagon-java section: universe/libs version: 1.0.0-2ubuntu2 libwavpack1: description: audio codec (lossy and lossless) - library downloads: 8327 homepage_uri: http://www.wavpack.com name: libwavpack1 section: libs version: 4.60.1-2 libwavpack1:i386: description: audio codec (lossy and lossless) - library downloads: 0 homepage_uri: http://www.wavpack.com name: libwavpack1:i386 section: libs version: 4.60.1-2 libwbclient0: description: Samba winbind client library downloads: 43887 homepage_uri: http://www.samba.org name: libwbclient0 section: libs version: 2:3.6.3-2ubuntu2.2 libwebkitgtk-1.0-0: description: Web content engine library for GTK+ downloads: 6789 homepage_uri: http://webkitgtk.org/ name: libwebkitgtk-1.0-0 section: libs version: 1.8.0-0ubuntu2 libwebkitgtk-1.0-common: description: Web content engine library for GTK+ - data files downloads: 0 homepage_uri: http://webkitgtk.org/ name: libwebkitgtk-1.0-common section: libs version: 1.8.0-0ubuntu2 libwebkitgtk-3.0-0: description: Web content engine library for GTK+ downloads: 9072 homepage_uri: http://webkitgtk.org/ name: libwebkitgtk-3.0-0 section: libs version: 1.8.0-0ubuntu2 libwebkitgtk-3.0-common: description: Web content engine library for GTK+ - data files downloads: 0 homepage_uri: http://webkitgtk.org/ name: libwebkitgtk-3.0-common section: libs version: 1.8.0-0ubuntu2 libwind0-heimdal: description: Heimdal Kerberos - stringprep implementation downloads: 367 homepage_uri: http://www.h5l.org/ name: libwind0-heimdal section: libs version: 1.6~git20120311.dfsg.1-2 libwind0-heimdal:i386: description: Heimdal Kerberos - stringprep implementation downloads: 0 homepage_uri: http://www.h5l.org/ name: libwind0-heimdal:i386 section: libs version: 1.6~git20120311.dfsg.1-2 libwmf0.2-7: description: Windows metafile conversion library downloads: 37381 homepage_uri: '' name: libwmf0.2-7 section: libs version: 0.2.8.4-10ubuntu1 libwmf0.2-7-gtk: description: Windows metafile conversion library downloads: 21 homepage_uri: '' name: libwmf0.2-7-gtk section: libs version: 0.2.8.4-10ubuntu1 libwnck-3-0: description: Window Navigator Construction Kit - runtime files downloads: 729 homepage_uri: '' name: libwnck-3-0 section: libs version: 3.4.0-0ubuntu1 libwnck-3-common: description: Window Navigator Construction Kit - common files downloads: 74 homepage_uri: '' name: libwnck-3-common section: libs version: 3.4.0-0ubuntu1 libwnck-common: description: Window Navigator Construction Kit - common files downloads: 5349 homepage_uri: '' name: libwnck-common section: libs version: 1:2.30.7-0ubuntu1 libwnck22: description: Window Navigator Construction Kit - runtime files downloads: 8940 homepage_uri: '' name: libwnck22 section: libs version: 1:2.30.7-0ubuntu1 libwpd-0.9-9: description: Library for handling WordPerfect documents (shared library) downloads: 17 homepage_uri: '' name: libwpd-0.9-9 section: libs version: 0.9.4-1 libwpg-0.2-2: description: WordPerfect graphics import/convert library (shared library) downloads: 19 homepage_uri: '' name: libwpg-0.2-2 section: libs version: 0.2.1-1 libwps-0.2-2: description: Works text file format import filter library (shared library) downloads: 6 homepage_uri: '' name: libwps-0.2-2 section: libs version: 0.2.4-1 libwrap0: description: Wietse Venema's TCP wrappers library downloads: 99797 homepage_uri: '' name: libwrap0 section: libs version: 7.6.q-21 libwrap0:i386: description: Wietse Venema's TCP wrappers library downloads: 0 homepage_uri: '' name: libwrap0:i386 section: libs version: 7.6.q-21 libwww-perl: description: simple and consistent interface to the world-wide web downloads: 39604 homepage_uri: http://search.cpan.org/dist/libwww-perl/ name: libwww-perl section: perl version: 6.03-1 libwww-robotrules-perl: description: database of robots.txt-derived permissions downloads: 5254 homepage_uri: http://search.cpan.org/dist/WWW-RobotRules/ name: libwww-robotrules-perl section: perl version: 6.01-1 libwxbase2.8-0: description: wxBase library (runtime) - non-GUI support classes of wxWidgets toolkit downloads: 652 homepage_uri: http://www.wxwidgets.org/ name: libwxbase2.8-0 section: universe/libs version: 2.8.12.1-6ubuntu2 libwxgtk2.8-0: description: wxWidgets Cross-platform C++ GUI toolkit (GTK+ runtime) downloads: 759 homepage_uri: http://www.wxwidgets.org/ name: libwxgtk2.8-0 section: universe/libs version: 2.8.12.1-6ubuntu2 libx11-6: description: X11 client-side library downloads: 80016 homepage_uri: '' name: libx11-6 section: libs version: 2:1.4.99.1-0ubuntu2 libx11-6:i386: description: X11 client-side library downloads: 0 homepage_uri: '' name: libx11-6:i386 section: libs version: 2:1.4.99.1-0ubuntu2 libx11-data: description: X11 client-side library downloads: 76574 homepage_uri: '' name: libx11-data section: x11 version: 2:1.4.99.1-0ubuntu2 libx11-dev: description: X11 client-side library (development headers) downloads: 6788 homepage_uri: '' name: libx11-dev section: libdevel version: 2:1.4.99.1-0ubuntu2 libx11-doc: description: X11 client-side library (development documentation) downloads: 0 homepage_uri: '' name: libx11-doc section: libdevel version: 2:1.4.99.1-0ubuntu2 libx11-xcb1: description: Xlib/XCB interface library downloads: 33058 homepage_uri: '' name: libx11-xcb1 section: libs version: 2:1.4.99.1-0ubuntu2 libx11-xcb1:i386: description: Xlib/XCB interface library downloads: 0 homepage_uri: '' name: libx11-xcb1:i386 section: libs version: 2:1.4.99.1-0ubuntu2 libx86-1: description: x86 real-mode library downloads: 6098 homepage_uri: http://www.codon.org.uk/~mjg59/libx86/ name: libx86-1 section: libs version: 1.1+ds1-7ubuntu1 libxalan2-java: description: XSL Transformations (XSLT) processor in Java downloads: 7172 homepage_uri: http://xml.apache.org/xalan-j/ name: libxalan2-java section: libs version: 2.7.1-7 libxapian22: description: Search engine library downloads: 24429 homepage_uri: http://xapian.org/ name: libxapian22 section: libs version: 1.2.8-1 libxatracker1: description: X acceleration library -- runtime downloads: 0 homepage_uri: http://mesa3d.sourceforge.net/ name: libxatracker1 section: libs version: 8.0.2-0ubuntu3.1 libxau-dev: description: X11 authorisation library (development headers) downloads: 4925 homepage_uri: '' name: libxau-dev section: libdevel version: 1:1.0.6-4 libxau6: description: X11 authorisation library downloads: 60200 homepage_uri: '' name: libxau6 section: libs version: 1:1.0.6-4 libxau6:i386: description: X11 authorisation library downloads: 0 homepage_uri: '' name: libxau6:i386 section: libs version: 1:1.0.6-4 libxaw7: description: X11 Athena Widget library downloads: 47931 homepage_uri: '' name: libxaw7 section: libs version: 2:1.0.9-3ubuntu1 libxaw7:i386: description: X11 Athena Widget library downloads: 0 homepage_uri: '' name: libxaw7:i386 section: libs version: 2:1.0.9-3ubuntu1 libxbean-java: description: plugin based Java application server downloads: 16 homepage_uri: http://geronimo.apache.org/xbean/ name: libxbean-java section: universe/libs version: 3.7-4 libxcb-dri2-0: description: X C Binding, dri2 extension downloads: 14482 homepage_uri: http://xcb.freedesktop.org name: libxcb-dri2-0 section: libs version: 1.8.1-1 libxcb-glx0: description: X C Binding, glx extension downloads: 295 homepage_uri: http://xcb.freedesktop.org name: libxcb-glx0 section: libs version: 1.8.1-1 libxcb-glx0:i386: description: X C Binding, glx extension downloads: 0 homepage_uri: http://xcb.freedesktop.org name: libxcb-glx0:i386 section: libs version: 1.8.1-1 libxcb-render0: description: X C Binding, render extension downloads: 54187 homepage_uri: http://xcb.freedesktop.org name: libxcb-render0 section: libs version: 1.8.1-1 libxcb-render0-dev: description: X C Binding, render extension, development files downloads: 2736 homepage_uri: http://xcb.freedesktop.org name: libxcb-render0-dev section: libdevel version: 1.8.1-1 libxcb-render0:i386: description: X C Binding, render extension downloads: 0 homepage_uri: http://xcb.freedesktop.org name: libxcb-render0:i386 section: libs version: 1.8.1-1 libxcb-shape0: description: X C Binding, shape extension downloads: 23089 homepage_uri: http://xcb.freedesktop.org name: libxcb-shape0 section: libs version: 1.8.1-1 libxcb-shm0: description: X C Binding, shm extension downloads: 37273 homepage_uri: http://xcb.freedesktop.org name: libxcb-shm0 section: libs version: 1.8.1-1 libxcb-shm0-dev: description: X C Binding, shm extension, development files downloads: 2045 homepage_uri: http://xcb.freedesktop.org name: libxcb-shm0-dev section: libdevel version: 1.8.1-1 libxcb-shm0:i386: description: X C Binding, shm extension downloads: 0 homepage_uri: http://xcb.freedesktop.org name: libxcb-shm0:i386 section: libs version: 1.8.1-1 libxcb-util0: description: utility libraries for X C Binding -- atom, aux and event downloads: 8018 homepage_uri: http://xcb.freedesktop.org name: libxcb-util0 section: libs version: 0.3.8-2 libxcb1: description: X C Binding downloads: 75811 homepage_uri: http://xcb.freedesktop.org name: libxcb1 section: libs version: 1.8.1-1 libxcb1-dev: description: X C Binding, development files downloads: 4930 homepage_uri: http://xcb.freedesktop.org name: libxcb1-dev section: libdevel version: 1.8.1-1 libxcb1:i386: description: X C Binding downloads: 0 homepage_uri: http://xcb.freedesktop.org name: libxcb1:i386 section: libs version: 1.8.1-1 libxcomp3: description: NX X compression library downloads: 79 homepage_uri: '' name: libxcomp3 section: universe/x11 version: 3.5.0-2-0ubuntu1~precise libxcompext3: description: NX X11 protocol compression extensions library downloads: 55 homepage_uri: '' name: libxcompext3 section: x11 version: 1:3.5.0-9-2-1-2-0~ppa1~precise2 libxcomposite-dev: description: X11 Composite extension library (development headers) downloads: 2348 homepage_uri: '' name: libxcomposite-dev section: libdevel version: 1:0.4.3-2build1 libxcomposite1: description: X11 Composite extension library downloads: 51862 homepage_uri: '' name: libxcomposite1 section: libs version: 1:0.4.3-2build1 libxcomposite1:i386: description: X11 Composite extension library downloads: 0 homepage_uri: '' name: libxcomposite1:i386 section: libs version: 1:0.4.3-2build1 libxcompshad3: description: NX shadowing library downloads: 55 homepage_uri: '' name: libxcompshad3 section: universe/x11 version: 1:3.5.0-9-2-1-2-0~ppa1~precise2 libxcursor-dev: description: X cursor management library (development files) downloads: 2576 homepage_uri: '' name: libxcursor-dev section: libdevel version: 1:1.1.12-1 libxcursor1: description: X cursor management library downloads: 33654 homepage_uri: '' name: libxcursor1 section: libs version: 1:1.1.12-1 libxcursor1:i386: description: X cursor management library downloads: 0 homepage_uri: '' name: libxcursor1:i386 section: libs version: 1:1.1.12-1 libxdamage-dev: description: X11 damaged region extension library (development headers) downloads: 2256 homepage_uri: '' name: libxdamage-dev section: libdevel version: 1:1.1.3-2build1 libxdamage1: description: X11 damaged region extension library downloads: 7362 homepage_uri: '' name: libxdamage1 section: libs version: 1:1.1.3-2build1 libxdamage1:i386: description: X11 damaged region extension library downloads: 0 homepage_uri: '' name: libxdamage1:i386 section: libs version: 1:1.1.3-2build1 libxdmcp-dev: description: X11 authorisation library (development headers) downloads: 4435 homepage_uri: '' name: libxdmcp-dev section: libdevel version: 1:1.1.0-4 libxdmcp6: description: X11 Display Manager Control Protocol library downloads: 39868 homepage_uri: '' name: libxdmcp6 section: libs version: 1:1.1.0-4 libxdmcp6:i386: description: X11 Display Manager Control Protocol library downloads: 0 homepage_uri: '' name: libxdmcp6:i386 section: libs version: 1:1.1.0-4 libxdot4: description: rich set of graph drawing tools - xdot library downloads: 3684 homepage_uri: http://www.graphviz.org/ name: libxdot4 section: libs version: 2.26.3-10ubuntu1 libxenstore3.0: description: Xenstore communications library for Xen downloads: 4494 homepage_uri: '' name: libxenstore3.0 section: libs version: 4.1.2-2ubuntu2 libxerces2-java: description: Validating XML parser for Java with DOM level 3 support downloads: 9370 homepage_uri: http://xerces.apache.org/xerces2-j/ name: libxerces2-java section: libs version: 2.11.0-4 libxext-dev: description: X11 miscellaneous extensions library (development headers) downloads: 4591 homepage_uri: '' name: libxext-dev section: libdevel version: 2:1.3.0-3build1 libxext6: description: X11 miscellaneous extension library downloads: 56111 homepage_uri: '' name: libxext6 section: libs version: 2:1.3.0-3build1 libxext6:i386: description: X11 miscellaneous extension library downloads: 0 homepage_uri: '' name: libxext6:i386 section: libs version: 2:1.3.0-3build1 libxfixes-dev: description: X11 miscellaneous 'fixes' extension library (development headers) downloads: 2659 homepage_uri: '' name: libxfixes-dev section: libdevel version: 1:5.0-4ubuntu4 libxfixes3: description: X11 miscellaneous 'fixes' extension library downloads: 36634 homepage_uri: '' name: libxfixes3 section: libs version: 1:5.0-4ubuntu4 libxfixes3:i386: description: X11 miscellaneous 'fixes' extension library downloads: 0 homepage_uri: '' name: libxfixes3:i386 section: libs version: 1:5.0-4ubuntu4 libxfont1: description: X11 font rasterisation library downloads: 18569 homepage_uri: '' name: libxfont1 section: libs version: 1:1.4.4-1 libxft-dev: description: FreeType-based font drawing library for X (development files) downloads: 3744 homepage_uri: '' name: libxft-dev section: libdevel version: 2.2.0-3ubuntu2 libxft2: description: FreeType-based font drawing library for X downloads: 54321 homepage_uri: '' name: libxft2 section: libs version: 2.2.0-3ubuntu2 libxft2:i386: description: FreeType-based font drawing library for X downloads: 0 homepage_uri: '' name: libxft2:i386 section: libs version: 2.2.0-3ubuntu2 libxi-dev: description: X11 Input extension library (development headers) downloads: 2720 homepage_uri: '' name: libxi-dev section: libdevel version: 2:1.6.0-0ubuntu2 libxi6: description: X11 Input extension library downloads: 27373 homepage_uri: '' name: libxi6 section: libs version: 2:1.6.0-0ubuntu2 libxi6:i386: description: X11 Input extension library downloads: 0 homepage_uri: '' name: libxi6:i386 section: libs version: 2:1.6.0-0ubuntu2 libxinerama-dev: description: X11 Xinerama extension library (development headers) downloads: 2958 homepage_uri: '' name: libxinerama-dev section: libdevel version: 2:1.1.1-3build1 libxinerama1: description: X11 Xinerama extension library downloads: 16279 homepage_uri: '' name: libxinerama1 section: libs version: 2:1.1.1-3build1 libxinerama1:i386: description: X11 Xinerama extension library downloads: 0 homepage_uri: '' name: libxinerama1:i386 section: libs version: 2:1.1.1-3build1 libxkbfile1: description: X11 keyboard file manipulation library downloads: 46629 homepage_uri: '' name: libxkbfile1 section: libs version: 1:1.0.7-1 libxklavier16: description: X Keyboard Extension high-level API downloads: 30355 homepage_uri: http://www.freedesktop.org/wiki/Software/LibXklavier name: libxklavier16 section: libs version: 5.2.1-1ubuntu1 libxml-commons-external-java: description: XML Commons external code - DOM, SAX, and JAXP, etc downloads: 1552 homepage_uri: http://xml.apache.org/commons/ name: libxml-commons-external-java section: libs version: 1.4.01-2 libxml-commons-resolver1.1-java: description: XML entity and URI resolver library downloads: 1269 homepage_uri: http://xml.apache.org/commons/ name: libxml-commons-resolver1.1-java section: libs version: 1.2-7 libxml-namespacesupport-perl: description: Perl module for supporting simple generic namespaces downloads: 7761 homepage_uri: '' name: libxml-namespacesupport-perl section: perl version: 1.09-3 libxml-parser-perl: description: Perl module for parsing XML files downloads: 13643 homepage_uri: http://www.libexpat.org/ name: libxml-parser-perl section: perl version: 2.41-1build1 libxml-sax-base-perl: description: base class for SAX drivers and filters downloads: 3649 homepage_uri: http://search.cpan.org/dist/XML-SAX-Base/ name: libxml-sax-base-perl section: perl version: 1.07-1 libxml-sax-expat-perl: description: Perl module for a SAX2 driver for Expat (XML::Parser) downloads: 3318 homepage_uri: http://search.cpan.org/dist/XML-SAX-Expat/ name: libxml-sax-expat-perl section: perl version: 0.40-2 libxml-sax-perl: description: Perl module for using and building Perl SAX2 XML processors downloads: 6399 homepage_uri: http://search.cpan.org/dist/XML-SAX/ name: libxml-sax-perl section: perl version: 0.99+dfsg-1 libxml-simple-perl: description: Perl module for reading and writing XML downloads: 4698 homepage_uri: http://search.cpan.org/dist/XML-Simple/ name: libxml-simple-perl section: perl version: 2.18-3 libxml-twig-perl: description: Perl module for processing huge XML documents in tree mode downloads: 10124 homepage_uri: http://www.xmltwig.com/ name: libxml-twig-perl section: perl version: 1:3.39-1ubuntu1 libxml-writer-perl: description: Perl module for writing XML documents downloads: 671 homepage_uri: http://search.cpan.org/dist/XML-Writer/ name: libxml-writer-perl section: universe/perl version: 0.612-1 libxml-xpath-perl: description: Perl module for processing XPath downloads: 579 homepage_uri: http://search.cpan.org/dist/XML-XPath/ name: libxml-xpath-perl section: perl version: 1.13-7 libxml2: description: GNOME XML library downloads: 82994 homepage_uri: http://xmlsoft.org/ name: libxml2 section: libs version: 2.7.8.dfsg-5.1ubuntu4.1 libxml2-dev: description: Development files for the GNOME XML library downloads: 3497 homepage_uri: http://xmlsoft.org/ name: libxml2-dev section: libdevel version: 2.7.8.dfsg-5.1ubuntu4.1 libxml2-utils: description: XML utilities downloads: 4698 homepage_uri: http://xmlsoft.org/ name: libxml2-utils section: text version: 2.7.8.dfsg-5.1ubuntu4.1 libxml2:i386: description: GNOME XML library downloads: 0 homepage_uri: http://xmlsoft.org/ name: libxml2:i386 section: libs version: 2.7.8.dfsg-5.1ubuntu4.1 libxmlgraphics-commons-java: description: reusable components used by Batik and FOP downloads: 45 homepage_uri: http://xmlgraphics.apache.org/commons/ name: libxmlgraphics-commons-java section: libs version: 1.4.dfsg-4ubuntu1 libxmlrpc-core-c3: description: lightweight RPC library based on XML and HTTP [C runtime libraries] downloads: 32 homepage_uri: http://xmlrpc-c.sourceforge.net name: libxmlrpc-core-c3 section: libs version: 1.16.33-3.1ubuntu5 libxmu6: description: X11 miscellaneous utility library downloads: 36776 homepage_uri: '' name: libxmu6 section: libs version: 2:1.1.0-3 libxmu6:i386: description: X11 miscellaneous utility library downloads: 0 homepage_uri: '' name: libxmu6:i386 section: libs version: 2:1.1.0-3 libxmuu1: description: X11 miscellaneous micro-utility library downloads: 39923 homepage_uri: '' name: libxmuu1 section: libs version: 2:1.1.0-3 libxom-java: description: A new XML object model for Java downloads: 49 homepage_uri: http://www.xom.nu/ name: libxom-java section: libs version: 1.2.1-3 libxp6: description: X Printing Extension (Xprint) client library downloads: 617 homepage_uri: '' name: libxp6 section: x11 version: 1:1.0.1-2 libxp6:i386: description: X Printing Extension (Xprint) client library downloads: 0 homepage_uri: '' name: libxp6:i386 section: x11 version: 1:1.0.1-2 libxpm-dev: description: X11 pixmap library (development headers) downloads: 1066 homepage_uri: '' name: libxpm-dev section: libdevel version: 1:3.5.9-4 libxpm4: description: X11 pixmap library downloads: 57681 homepage_uri: '' name: libxpm4 section: libs version: 1:3.5.9-4 libxpm4:i386: description: X11 pixmap library downloads: 0 homepage_uri: '' name: libxpm4:i386 section: libs version: 1:3.5.9-4 libxpp2-java: description: XML pull parser library for java V2 downloads: 214 homepage_uri: http://www.extreme.indiana.edu/xgws/xsoap/xpp/xpp2/ name: libxpp2-java section: libs version: 2.1.10-7 libxpp3-java: description: XML pull parser library for java downloads: 605 homepage_uri: http://www.extreme.indiana.edu/xgws/xsoap/xpp name: libxpp3-java section: libs version: 1.1.4c-2 libxrandr-dev: description: X11 RandR extension library (development headers) downloads: 2706 homepage_uri: '' name: libxrandr-dev section: libdevel version: 2:1.3.2-2 libxrandr2: description: X11 RandR extension library downloads: 38647 homepage_uri: '' name: libxrandr2 section: libs version: 2:1.3.2-2 libxrandr2:i386: description: X11 RandR extension library downloads: 0 homepage_uri: '' name: libxrandr2:i386 section: libs version: 2:1.3.2-2 libxrender-dev: description: X Rendering Extension client library (development files) downloads: 4211 homepage_uri: '' name: libxrender-dev section: libdevel version: 1:0.9.6-2build1 libxrender1: description: X Rendering Extension client library downloads: 56401 homepage_uri: '' name: libxrender1 section: libs version: 1:0.9.6-2build1 libxrender1:i386: description: X Rendering Extension client library downloads: 0 homepage_uri: '' name: libxrender1:i386 section: libs version: 1:0.9.6-2build1 libxres1: description: X11 Resource extension library downloads: 5965 homepage_uri: '' name: libxres1 section: libs version: 2:1.0.5-1 libxslt1-dev: description: XSLT 1.0 processing library - development kit downloads: 968 homepage_uri: http://xmlsoft.org/xslt/ name: libxslt1-dev section: libdevel version: 1.1.26-8ubuntu1 libxslt1.1: description: XSLT 1.0 processing library - runtime library downloads: 17000 homepage_uri: http://xmlsoft.org/xslt/ name: libxslt1.1 section: libs version: 1.1.26-8ubuntu1 libxslt1.1:i386: description: XSLT 1.0 processing library - runtime library downloads: 0 homepage_uri: http://xmlsoft.org/xslt/ name: libxslt1.1:i386 section: libs version: 1.1.26-8ubuntu1 libxss-dev: description: X11 Screen Saver extension library (development headers) downloads: 798 homepage_uri: '' name: libxss-dev section: libdevel version: 1:1.2.1-2 libxss1: description: X11 Screen Saver extension library downloads: 10752 homepage_uri: '' name: libxss1 section: libs version: 1:1.2.1-2 libxss1:i386: description: X11 Screen Saver extension library downloads: 0 homepage_uri: '' name: libxss1:i386 section: libs version: 1:1.2.1-2 libxt-dev: description: X11 toolkit intrinsics library (development headers) downloads: 3984 homepage_uri: '' name: libxt-dev section: libdevel version: 1:1.1.1-2build1 libxt6: description: X11 toolkit intrinsics library downloads: 48065 homepage_uri: '' name: libxt6 section: libs version: 1:1.1.1-2build1 libxt6:i386: description: X11 toolkit intrinsics library downloads: 0 homepage_uri: '' name: libxt6:i386 section: libs version: 1:1.1.1-2build1 libxtst6: description: X11 Testing -- Record extension library downloads: 37093 homepage_uri: '' name: libxtst6 section: libs version: 2:1.2.0-4 libxtst6:i386: description: X11 Testing -- Record extension library downloads: 0 homepage_uri: '' name: libxtst6:i386 section: libs version: 2:1.2.0-4 libxv1: description: X11 Video extension library downloads: 30977 homepage_uri: '' name: libxv1 section: libs version: 2:1.0.6-2build1 libxv1:i386: description: X11 Video extension library downloads: 0 homepage_uri: '' name: libxv1:i386 section: libs version: 2:1.0.6-2build1 libxvmc1: description: X11 Video extension library downloads: 20191 homepage_uri: '' name: libxvmc1 section: libs version: 2:1.0.6-1ubuntu2 libxxf86dga1: description: X11 Direct Graphics Access extension library downloads: 884 homepage_uri: '' name: libxxf86dga1 section: libs version: 2:1.1.2-1 libxxf86vm1: description: X11 XFree86 video mode extension library downloads: 36992 homepage_uri: '' name: libxxf86vm1 section: libs version: 1:1.1.1-2build1 libxxf86vm1:i386: description: X11 XFree86 video mode extension library downloads: 0 homepage_uri: '' name: libxxf86vm1:i386 section: libs version: 1:1.1.1-2build1 libyajl1: description: Yet Another JSON Library downloads: 432 homepage_uri: http://lloyd.github.com/yajl/ name: libyajl1 section: libs version: 1.0.12-2 libyaml-0-2: description: Fast YAML 1.1 parser and emitter library downloads: 2551 homepage_uri: http://pyyaml.org/wiki/LibYAML name: libyaml-0-2 section: libs version: 0.1.4-2 libyaml-libyaml-perl: description: Perl interface to libyaml, a YAML implementation downloads: 175 homepage_uri: http://search.cpan.org/dist/YAML-LibYAML/ name: libyaml-libyaml-perl section: perl version: 0.38-2 libyaml-perl: description: YAML Ain't Markup Language downloads: 1151 homepage_uri: http://search.cpan.org/dist/YAML/ name: libyaml-perl section: perl version: 0.77-1 libyaml-tiny-perl: description: Perl module for reading and writing YAML files downloads: 3633 homepage_uri: http://search.cpan.org/~adamk/YAML-Tiny/ name: libyaml-tiny-perl section: perl version: 1.50-1 libyelp0: description: Library for the GNOME help browser downloads: 3015 homepage_uri: http://live.gnome.org/Yelp name: libyelp0 section: libs version: 3.4.1-0ubuntu1 libzeitgeist-1.0-1: description: library to access Zeitgeist - shared library downloads: 737 homepage_uri: https://launchpad.net/libzeitgeist name: libzeitgeist-1.0-1 section: libs version: 0.3.18-1ubuntu1 libzephyr4: description: Project Athena's notification service - non-Kerberos libraries downloads: 104 homepage_uri: '' name: libzephyr4 section: libs version: 3.0.1-1 libzip-dev: description: library for reading, creating, and modifying zip archives (development) downloads: 74 homepage_uri: http://www.nih.at/libzip/ name: libzip-dev section: libdevel version: 0.10-1ubuntu1 libzip2: description: library for reading, creating, and modifying zip archives (runtime) downloads: 482 homepage_uri: http://www.nih.at/libzip/ name: libzip2 section: libs version: 0.10-1ubuntu1 libzmq-dev: description: ZeroMQ lightweight messaging kernel (development libraries and header files) downloads: 36 homepage_uri: http://www.zeromq.org/ name: libzmq-dev section: universe/libdevel version: 2.1.11-1ubuntu1 libzmq1: description: ZeroMQ lightweight messaging kernel (shared library) downloads: 192 homepage_uri: http://www.zeromq.org/ name: libzmq1 section: universe/libs version: 2.1.11-1ubuntu1 light-themes: description: Light Themes (Ambiance and Radiance) downloads: 0 homepage_uri: https://launchpad.net/light-themes name: light-themes section: gnome version: 0.1.9.1-0ubuntu1 lightdm: description: Display Manager downloads: 1096 homepage_uri: https://launchpad.net/lightdm name: lightdm section: x11 version: 1.2.1-0ubuntu1 lintian: description: Debian package checker downloads: 1270 homepage_uri: '' name: lintian section: devel version: 2.5.6ubuntu0.1 linux-firmware: description: Firmware for Linux kernel drivers downloads: 22 homepage_uri: '' name: linux-firmware section: misc version: '1.79' linux-headers-3.2.0-25: description: Header files related to Linux kernel version 3.2.0 downloads: 0 homepage_uri: '' name: linux-headers-3.2.0-25 section: devel version: 3.2.0-25.40 linux-headers-3.2.0-25-generic: description: Linux kernel headers for version 3.2.0 on 64 bit x86 SMP downloads: 0 homepage_uri: '' name: linux-headers-3.2.0-25-generic section: devel version: 3.2.0-25.40 linux-headers-generic: description: Generic Linux kernel headers downloads: 0 homepage_uri: '' name: linux-headers-generic section: devel version: 3.2.0.25.27 linux-image-3.2.0-23-virtual: description: Linux kernel image for version 3.2.0 on 64 bit x86 Virtual Guests downloads: 0 homepage_uri: '' name: linux-image-3.2.0-23-virtual section: kernel version: 3.2.0-23.36 linux-image-virtual: description: Linux kernel image for virtual machines downloads: 0 homepage_uri: '' name: linux-image-virtual section: metapackages version: 3.2.0.23.25 linux-libc-dev: description: Linux Kernel Headers for development downloads: 34885 homepage_uri: '' name: linux-libc-dev section: devel version: 3.2.0-25.40 linux-sound-base: description: base package for ALSA and OSS sound systems downloads: 37280 homepage_uri: http://www.alsa-project.org/ name: linux-sound-base section: sound version: 1.0.25+dfsg-0ubuntu1 linux-virtual: description: Complete Linux kernel for virtual machines downloads: 0 homepage_uri: '' name: linux-virtual section: metapackages version: 3.2.0.23.25 lksctp-tools: description: user-space access to Linux Kernel SCTP - commandline tools downloads: 267 homepage_uri: http://lksctp.sf.net/ name: lksctp-tools section: net version: 1.0.11+dfsg-2 lmodern: description: scalable PostScript and OpenType fonts based on Computer Modern downloads: 1972 homepage_uri: http://www.gust.org.pl/projects/e-foundry/latin-modern/ name: lmodern section: tex version: 2.004.1-3.1ubuntu1 locales: description: common files for locale support downloads: 28005 homepage_uri: '' name: locales section: libs version: 2.13+git20120306-3 lockfile-progs: description: Programs for locking and unlocking files and mailboxes downloads: 44611 homepage_uri: '' name: lockfile-progs section: misc version: 0.1.16 login: description: system login tools downloads: 110986 homepage_uri: http://pkg-shadow.alioth.debian.org/ name: login section: admin version: 1:4.1.4.2+svn3283-3ubuntu5 logrotate: description: Log rotation utility downloads: 110453 homepage_uri: '' name: logrotate section: admin version: 3.7.8-6ubuntu5 lsb-base: description: Linux Standard Base 4.0 init script functionality downloads: 110574 homepage_uri: http://www.linux-foundation.org/en/LSB name: lsb-base section: misc version: 4.0-0ubuntu20 lsb-release: description: Linux Standard Base version reporting utility downloads: 55189 homepage_uri: http://www.linux-foundation.org/en/LSB name: lsb-release section: misc version: 4.0-0ubuntu20 lshw: description: information about hardware configuration downloads: 1433 homepage_uri: '' name: lshw section: utils version: 02.15-2 lsof: description: List open files downloads: 19260 homepage_uri: '' name: lsof section: utils version: 4.81.dfsg.1-1build1 ltrace: description: Tracks runtime library calls in dynamically linked programs downloads: 399 homepage_uri: '' name: ltrace section: utils version: 0.5.3-2.1ubuntu2 luatex: description: next generation TeX engine downloads: 9397 homepage_uri: http://www.luatex.org name: luatex section: tex version: 0.70.1-1ubuntu1 lvm2: description: The Linux Logical Volume Manager downloads: 17090 homepage_uri: http://sources.redhat.com/lvm2/ name: lvm2 section: admin version: 2.02.66-4ubuntu7.1 lynx: description: Text-mode WWW Browser (transitional package) downloads: 530 homepage_uri: http://lynx.isc.org/ name: lynx section: web version: 2.8.8dev.9-2 lynx-cur: description: Text-mode WWW Browser with NLS support (development version) downloads: 8607 homepage_uri: http://lynx.isc.org/ name: lynx-cur section: web version: 2.8.8dev.9-2 m17n-contrib: description: multilingual text processing library - contributed database downloads: 0 homepage_uri: http://www.m17n.org/ name: m17n-contrib section: text version: 1.1.13-1 m17n-db: description: multilingual text processing library - database downloads: 1137 homepage_uri: http://www.m17n.org/ name: m17n-db section: text version: 1.6.3-1 m4: description: a macro processing language downloads: 12727 homepage_uri: http://www.gnu.org/software/m4/ name: m4 section: interpreters version: 1.4.16-2ubuntu1 mafft: description: Multiple alignment program for amino acid or nucleotide sequences downloads: 31 homepage_uri: http://align.bmr.kyushu-u.ac.jp/mafft/software/ name: mafft section: universe/science version: 6.850-1 mahjongg: description: classic Eastern tile game for GNOME downloads: 480 homepage_uri: http://live.gnome.org/GnomeGames name: mahjongg section: games version: 1:3.4.1-0ubuntu2 make: description: An utility for Directing compilation. downloads: 24471 homepage_uri: http://www.gnu.org/software/make/ name: make section: devel version: 3.81-8.1ubuntu1 makedev: description: creates device files in /dev downloads: 4954 homepage_uri: '' name: makedev section: admin version: 2.3.1-89ubuntu2 man-db: description: on-line manual pager downloads: 105783 homepage_uri: http://man-db.nongnu.org/ name: man-db section: doc version: 2.6.1-2 manpages: description: Manual pages about using a GNU/Linux system downloads: 1 homepage_uri: http://www.kernel.org/doc/man-pages/ name: manpages section: doc version: 3.35-0.1ubuntu1 manpages-dev: description: Manual pages about using GNU/Linux for development downloads: 1 homepage_uri: http://www.kernel.org/doc/man-pages/ name: manpages-dev section: doc version: 3.35-0.1ubuntu1 maq: description: maps short fixed-length polymorphic DNA sequence reads to reference sequences downloads: 21 homepage_uri: http://maq.sourceforge.net/ name: maq section: science version: 0.7.1-4 maven2: description: Java software project management and comprehension tool downloads: 369 homepage_uri: http://maven.apache.org/ name: maven2 section: universe/devel version: 2.2.1-10 mawk: description: a pattern scanning and text processing language downloads: 78500 homepage_uri: '' name: mawk section: utils version: 1.3.3-17 mayavi2: description: scientific visualization package for 2-D and 3-D data downloads: 103 homepage_uri: http://code.enthought.com/projects/mayavi/ name: mayavi2 section: universe/science version: 4.0.0-3build1 mc: description: Midnight Commander - a powerful file manager downloads: 12038 homepage_uri: http://www.midnight-commander.org name: mc section: universe/utils version: 3:4.8.1-2ubuntu1 mc-data: description: Midnight Commander - a powerful file manager -- data files downloads: 1928 homepage_uri: http://www.midnight-commander.org name: mc-data section: universe/utils version: 3:4.8.1-2ubuntu1 mcl: description: Markov Cluster algorithm downloads: 17 homepage_uri: http://micans.org/mcl/ name: mcl section: math version: 1:11-294-1 mdadm: description: tool to administer Linux MD arrays (software RAID) downloads: 12014 homepage_uri: http://neil.brown.name/blog/mdadm name: mdadm section: admin version: 3.2.3-2ubuntu1 media-player-info: description: Media player identification files downloads: 10387 homepage_uri: http://www.freedesktop.org/software/media-player-info/ name: media-player-info section: admin version: 16-1 memtest86+: description: thorough real-mode memory tester downloads: 479 homepage_uri: http://www.memtest.org/ name: memtest86+ section: misc version: 4.20-1.1ubuntu1 menu: description: generates programs menu for all menu-aware applications downloads: 34177 homepage_uri: '' name: menu section: universe/admin version: 2.1.46ubuntu1 mercurial: description: easy-to-use, scalable distributed version control system downloads: 2401 homepage_uri: http://mercurial.selenic.com/ name: mercurial section: universe/devel version: 2.0.2-1ubuntu1 mercurial-common: description: easy-to-use, scalable distributed version control system (common files) downloads: 2451 homepage_uri: http://mercurial.selenic.com/ name: mercurial-common section: universe/devel version: 2.0.2-1ubuntu1 mesa-common-dev: description: Developer documentation for Mesa downloads: 2670 homepage_uri: http://mesa3d.sourceforge.net/ name: mesa-common-dev section: devel version: 8.0.2-0ubuntu3.1 metacity: description: lightweight GTK+ window manager downloads: 32103 homepage_uri: '' name: metacity section: x11 version: 1:2.34.1-1ubuntu11 metacity-common: description: shared files for the Metacity window manager downloads: 10 homepage_uri: '' name: metacity-common section: misc version: 1:2.34.1-1ubuntu11 mime-support: description: MIME files 'mime.types' & 'mailcap', and support programs downloads: 92780 homepage_uri: '' name: mime-support section: net version: 3.51-1ubuntu1 mlocate: description: quickly find files on the filesystem based on their name downloads: 88285 homepage_uri: http://carolina.mff.cuni.cz/~trmac/blog/mlocate name: mlocate section: utils version: 0.23.1-1ubuntu2 mobile-broadband-provider-info: description: database of mobile broadband service providers downloads: 2 homepage_uri: http://live.gnome.org/NetworkManager/MobileBroadband/ServiceProviders name: mobile-broadband-provider-info section: admin version: 20120410-0ubuntu1 modemmanager: description: D-Bus service for managing modems downloads: 32673 homepage_uri: '' name: modemmanager section: net version: 0.5.2.0-0ubuntu2 module-init-tools: description: tools for managing Linux kernel modules downloads: 88323 homepage_uri: http://www.kerneltools.org/ name: module-init-tools section: admin version: 3.16-1ubuntu2 mongodb-10gen: description: An object/document-oriented database downloads: 136 homepage_uri: http://www.mongodb.org name: mongodb-10gen section: devel version: 2.0.6 mount: description: Tools for mounting and manipulating filesystems downloads: 110417 homepage_uri: http://userweb.kernel.org/~kzak/util-linux/ name: mount section: admin version: 2.20.1-1ubuntu3 mountall: description: filesystem mounting tool downloads: 2 homepage_uri: '' name: mountall section: admin version: '2.36' mousetweaks: description: mouse accessibility enhancements for the GNOME desktop downloads: 19462 homepage_uri: http://live.gnome.org/Mousetweaks/Home name: mousetweaks section: gnome version: 3.4.1-0ubuntu1 mpi-default-dev: description: Standard MPI development files (metapackage) downloads: 204 homepage_uri: '' name: mpi-default-dev section: universe/libdevel version: 1.0.1 mrbayes: description: Bayesian Inference of Phylogeny downloads: 8 homepage_uri: http://mrbayes.csit.fsu.edu/ name: mrbayes section: math version: 3.1.2-2 mscompress: description: Microsoft "compress.exe/expand.exe" compatible (de)compressor downloads: 1442 homepage_uri: '' name: mscompress section: otherosfs version: 0.3-3.1 msr-tools: description: Utilities for modifying MSRs from userspace downloads: 7 homepage_uri: '' name: msr-tools section: admin version: 1.2-3 mtasc: description: ActionScript 2 to Flash (SWF) compiler downloads: 19 homepage_uri: http://www.mtasc.org name: mtasc section: universe/devel version: 1.14-2build2 mtools: description: Tools for manipulating MSDOS files downloads: 6177 homepage_uri: http://www.mtools.linux.lu/ name: mtools section: otherosfs version: 4.0.12-1 mtr-tiny: description: Full screen ncurses traceroute tool downloads: 1966 homepage_uri: http://www.bitwizard.nl/mtr/ name: mtr-tiny section: net version: 0.80-1ubuntu1 multiarch-support: description: Transitional package to ensure multiarch compatibility downloads: 1 homepage_uri: http://www.eglibc.org name: multiarch-support section: libs version: 2.15-0ubuntu10 mummer: description: Efficient sequence alignment of full genomes downloads: 26 homepage_uri: http://mummer.sourceforge.net/ name: mummer section: science version: 3.22~dfsg-2 muscle: description: Multiple alignment program of protein sequences downloads: 37 homepage_uri: http://www.drive5.com/muscle/ name: muscle section: science version: 1:3.8.31-1 mysql-client: description: MySQL database client (metapackage depending on the latest version) downloads: 1336 homepage_uri: http://dev.mysql.com/ name: mysql-client section: database version: 5.5.24-0ubuntu0.12.04.1 mysql-client-5.5: description: MySQL database client binaries downloads: 4207 homepage_uri: http://dev.mysql.com/ name: mysql-client-5.5 section: database version: 5.5.24-0ubuntu0.12.04.1 mysql-client-core-5.5: description: MySQL database core client binaries downloads: 33 homepage_uri: http://dev.mysql.com/ name: mysql-client-core-5.5 section: database version: 5.5.24-0ubuntu0.12.04.1 mysql-common: description: MySQL database common files, e.g. /etc/mysql/my.cnf downloads: 27558 homepage_uri: http://dev.mysql.com/ name: mysql-common section: database version: 5.5.24-0ubuntu0.12.04.1 mysql-server: description: MySQL database server (metapackage depending on the latest version) downloads: 42 homepage_uri: http://dev.mysql.com/ name: mysql-server section: database version: 5.5.24-0ubuntu0.12.04.1 mysql-server-5.5: description: MySQL database server binaries and system database setup downloads: 2790 homepage_uri: http://dev.mysql.com/ name: mysql-server-5.5 section: database version: 5.5.24-0ubuntu0.12.04.1 mysql-server-core-5.5: description: MySQL database server binaries downloads: 3847 homepage_uri: http://dev.mysql.com/ name: mysql-server-core-5.5 section: database version: 5.5.24-0ubuntu0.12.04.1 nano: description: small, friendly text editor inspired by Pico downloads: 23959 homepage_uri: http://www.nano-editor.org/ name: nano section: editors version: 2.2.6-1 nautilus: description: file manager and graphical shell for GNOME downloads: 30390 homepage_uri: http://www.gnome.org/projects/nautilus/ name: nautilus section: gnome version: 1:3.4.2-0ubuntu2 nautilus-data: description: data files for nautilus downloads: 32249 homepage_uri: http://www.gnome.org/projects/nautilus/ name: nautilus-data section: gnome version: 1:3.4.2-0ubuntu2 nautilus-sendto: description: integrates Evolution and Pidgin into the Nautilus file manager downloads: 19647 homepage_uri: '' name: nautilus-sendto section: gnome version: 3.0.1-2ubuntu2 nautilus-sendto-empathy: description: GNOME multi-protocol chat and call client (nautilus-sendto plugin) downloads: 1583 homepage_uri: http://live.gnome.org/Empathy name: nautilus-sendto-empathy section: gnome version: 3.4.2-0ubuntu1 nautilus-share: description: Nautilus extension to share folder using Samba downloads: 1213 homepage_uri: http://gentoo.ovibes.net/nautilus-share/ name: nautilus-share section: gnome version: 0.7.3-1ubuntu2 ncbi-blast+: description: next generation suite of BLAST sequence search tools downloads: 22 homepage_uri: http://www.ncbi.nlm.nih.gov/IEB/ToolBox/CPP_DOC/ name: ncbi-blast+ section: science version: 2.2.25-7 ncbi-blast+-legacy: description: NCBI Blast legacy call script downloads: 2 homepage_uri: http://www.ncbi.nlm.nih.gov/IEB/ToolBox/CPP_DOC/ name: ncbi-blast+-legacy section: science version: 2.2.25-7 ncbi-data: description: Platform-independent data for the NCBI toolkit downloads: 154 homepage_uri: http://www.ncbi.nlm.nih.gov/IEB/ToolBox/ name: ncbi-data section: libs version: 6.1.20110713-3ubuntu2 ncbi-tools-bin: description: NCBI libraries for biology applications (text-based utilities) downloads: 26 homepage_uri: http://www.ncbi.nlm.nih.gov/IEB/ToolBox/ name: ncbi-tools-bin section: science version: 6.1.20110713-3ubuntu2 ncbi-tools-x11: description: NCBI libraries for biology applications (X-based utilities) downloads: 120 homepage_uri: http://www.ncbi.nlm.nih.gov/IEB/ToolBox/ name: ncbi-tools-x11 section: science version: 6.1.20110713-3ubuntu2 ncftp: description: User-friendly and well-featured FTP client downloads: 1288 homepage_uri: http://www.ncftpd.com/ncftp/ name: ncftp section: universe/net version: 2:3.2.5-1 ncurses-base: description: basic terminal type definitions downloads: 90113 homepage_uri: http://invisible-island.net/ncurses/ name: ncurses-base section: utils version: 5.9-4 ncurses-bin: description: terminal-related programs and man pages downloads: 108834 homepage_uri: http://invisible-island.net/ncurses/ name: ncurses-bin section: utils version: 5.9-4 net-tools: description: The NET-3 networking toolkit downloads: 86718 homepage_uri: http://net-tools.berlios.de/ name: net-tools section: net version: 1.60-24.1ubuntu2 netbase: description: Basic TCP/IP networking system downloads: 90312 homepage_uri: '' name: netbase section: admin version: 4.47ubuntu1 netcat-openbsd: description: TCP/IP swiss army knife downloads: 1157 homepage_uri: '' name: netcat-openbsd section: net version: 1.89-4ubuntu1 netpbm: description: Graphics conversion tools between image formats downloads: 5843 homepage_uri: http://netpbm.alioth.debian.org name: netpbm section: graphics version: 2:10.0-15 network-manager: description: network management framework (daemon and userspace tools) downloads: 37981 homepage_uri: http://www.gnome.org/projects/NetworkManager/ name: network-manager section: net version: 0.9.4.0-0ubuntu4.1 network-manager-gnome: description: network management framework (GNOME frontend) downloads: 27132 homepage_uri: http://www.gnome.org/projects/NetworkManager/ name: network-manager-gnome section: net version: 0.9.4.1-0ubuntu2 network-manager-pptp: description: network management framework (PPTP plugin core) downloads: 894 homepage_uri: http://www.gnome.org/projects/NetworkManager/ name: network-manager-pptp section: net version: 0.9.4.0-0ubuntu1 network-manager-pptp-gnome: description: network management framework (PPTP plugin GNOME GUI) downloads: 650 homepage_uri: http://www.gnome.org/projects/NetworkManager/ name: network-manager-pptp-gnome section: net version: 0.9.4.0-0ubuntu1 nfs-common: description: NFS support files common to client and server downloads: 59352 homepage_uri: http://nfs.sourceforge.net/ name: nfs-common section: net version: 1:1.2.5-3ubuntu3 nfs-kernel-server: description: support for NFS kernel server downloads: 12883 homepage_uri: http://nfs.sourceforge.net/ name: nfs-kernel-server section: net version: 1:1.2.5-3ubuntu3 njplot: description: phylogenetic tree drawing program downloads: 120 homepage_uri: http://pbil.univ-lyon1.fr/software/njplot.html name: njplot section: science version: 2.3-4 notification-daemon: description: daemon for displaying passive pop-up notifications downloads: 32826 homepage_uri: http://www.galago-project.org/specs/notification/index.php name: notification-daemon section: x11 version: 0.7.3-1 notify-osd: description: daemon that displays passive pop-up notifications downloads: 300 homepage_uri: https://launchpad.net/notify-osd name: notify-osd section: x11 version: 0.9.34-0ubuntu2 notify-osd-icons: description: Notify-OSD icons downloads: 0 homepage_uri: '' name: notify-osd-icons section: x11 version: '0.7' ntfs-3g: description: read/write NTFS driver for FUSE downloads: 18895 homepage_uri: http://www.tuxera.com/community/ntfs-3g-advanced/ name: ntfs-3g section: otherosfs version: 1:2012.1.15AR.1-1ubuntu1 ntpdate: description: client for setting system time from NTP servers downloads: 23778 homepage_uri: http://support.ntp.org/ name: ntpdate section: net version: 1:4.2.6.p3+dfsg-1ubuntu3.1 nux-tools: description: Visual rendering toolkit for real-time applications - tools downloads: 0 homepage_uri: http://launchpad.net/nux name: nux-tools section: x11 version: 2.12.0-0ubuntu1 nvidia-common: description: Find obsolete NVIDIA drivers downloads: 19 homepage_uri: '' name: nvidia-common section: x11 version: 1:0.2.44 nx-common: description: NX server common package. downloads: 0 homepage_uri: '' name: nx-common section: x11 version: 0.7.3.zgit.120322.977c28d-0~ppa11 nxagent: description: X server for remote access downloads: 173 homepage_uri: '' name: nxagent section: x11 version: 1:3.5.0-9-2-1-2-0~ppa1~precise2 obex-data-server: description: D-Bus service for OBEX client and server side functionality downloads: 29398 homepage_uri: http://wiki.muiline.com/obex-data-server name: obex-data-server section: admin version: 0.4.6-0ubuntu1 obexd-client: description: D-Bus OBEX client downloads: 27934 homepage_uri: http://www.bluez.org name: obexd-client section: admin version: 0.44-0ubuntu1 octave3.2: description: GNU Octave language for numerical computations (3.2 branch) downloads: 664 homepage_uri: http://www.octave.org/ name: octave3.2 section: universe/math version: 3.2.4-12 octave3.2-common: description: architecture-independent files for octave3.2 downloads: 0 homepage_uri: http://www.octave.org/ name: octave3.2-common section: universe/devel version: 3.2.4-12 odbcinst: description: Helper program for accessing odbc ini files downloads: 17237 homepage_uri: http://www.unixodbc.org/ name: odbcinst section: libs version: 2.2.14p2-5ubuntu3 odbcinst1debian2: description: Support library for accessing odbc ini files downloads: 17275 homepage_uri: http://www.unixodbc.org/ name: odbcinst1debian2 section: libs version: 2.2.14p2-5ubuntu3 odbcinst1debian2:i386: description: Support library for accessing odbc ini files downloads: 0 homepage_uri: http://www.unixodbc.org/ name: odbcinst1debian2:i386 section: libs version: 2.2.14p2-5ubuntu3 onboard: description: Simple On-screen Keyboard downloads: 6 homepage_uri: https://launchpad.net/onboard name: onboard section: gnome version: 0.97.0-0ubuntu3 oneconf: description: synchronize your configuration data over the network downloads: 0 homepage_uri: '' name: oneconf section: python version: 0.2.8.1 openjdk-6-jdk: description: OpenJDK Development Kit (JDK) downloads: 1485 homepage_uri: http://openjdk.java.net/ name: openjdk-6-jdk section: devel version: 6b24-1.11.1-4ubuntu3 openjdk-6-jre: description: OpenJDK Java runtime, using Hotspot JIT downloads: 9734 homepage_uri: http://openjdk.java.net/ name: openjdk-6-jre section: interpreters version: 6b24-1.11.1-4ubuntu3 openjdk-6-jre-headless: description: OpenJDK Java runtime, using Hotspot JIT (headless) downloads: 14227 homepage_uri: http://openjdk.java.net/ name: openjdk-6-jre-headless section: interpreters version: 6b24-1.11.1-4ubuntu3 openjdk-6-jre-lib: description: OpenJDK Java runtime (architecture independent libraries) downloads: 12439 homepage_uri: http://openjdk.java.net/ name: openjdk-6-jre-lib section: interpreters version: 6b24-1.11.1-4ubuntu3 openjdk-7-jdk: description: OpenJDK Development Kit (JDK) downloads: 59 homepage_uri: http://openjdk.java.net/ name: openjdk-7-jdk section: universe/java version: 7~u3-2.1.1~pre1-1ubuntu3 openjdk-7-jre: description: OpenJDK Java runtime, using Hotspot JIT downloads: 307 homepage_uri: http://openjdk.java.net/ name: openjdk-7-jre section: universe/java version: 7~u3-2.1.1~pre1-1ubuntu3 openjdk-7-jre-headless: description: OpenJDK Java runtime, using Hotspot JIT (headless) downloads: 373 homepage_uri: http://openjdk.java.net/ name: openjdk-7-jre-headless section: universe/java version: 7~u3-2.1.1~pre1-1ubuntu3 openjdk-7-jre-lib: description: OpenJDK Java runtime (architecture independent libraries) downloads: 369 homepage_uri: http://openjdk.java.net/ name: openjdk-7-jre-lib section: universe/java version: 7~u3-2.1.1~pre1-1ubuntu3 openmpi-bin: description: high performance message passing library -- binaries downloads: 789 homepage_uri: http://www.open-mpi.org/ name: openmpi-bin section: universe/net version: 1.4.3-2.1ubuntu3 openmpi-checkpoint: description: high performance message passing library -- checkpoint support downloads: 130 homepage_uri: http://www.open-mpi.org/ name: openmpi-checkpoint section: universe/net version: 1.4.3-2.1ubuntu3 openmpi-common: description: high performance message passing library -- common files downloads: 1252 homepage_uri: http://www.open-mpi.org/ name: openmpi-common section: universe/net version: 1.4.3-2.1ubuntu3 openprinting-ppds: description: OpenPrinting printer support - PostScript PPD files downloads: 2364 homepage_uri: http://www.openprinting.org/ name: openprinting-ppds section: text version: 20120322-0ubuntu1 openssh-client: description: secure shell (SSH) client, for secure access to remote machines downloads: 67813 homepage_uri: http://www.openssh.org/ name: openssh-client section: net version: 1:5.9p1-5ubuntu1 openssh-server: description: secure shell (SSH) server, for secure access from remote machines downloads: 82562 homepage_uri: http://www.openssh.org/ name: openssh-server section: net version: 1:5.9p1-5ubuntu1 openssl: description: Secure Socket Layer (SSL) binary and related cryptographic tools downloads: 90115 homepage_uri: '' name: openssl section: utils version: 1.0.1-4ubuntu5.2 orbit2: description: a CORBA ORB downloads: 367 homepage_uri: '' name: orbit2 section: devel version: 1:2.14.19-0.1ubuntu1 os-prober: description: utility to detect other OSes on a set of drives downloads: 18333 homepage_uri: '' name: os-prober section: utils version: 1.51ubuntu3 oss-compat: description: Open Sound System (OSS) compatibility package downloads: 12880 homepage_uri: '' name: oss-compat section: universe/sound version: '1' overlay-scrollbar: description: Scrollbar overlayed widget downloads: 0 homepage_uri: http://launchpad.net/ayatana-scrollbar name: overlay-scrollbar section: libs version: 0.2.16-0ubuntu1 parrot: description: virtual machine for dynamic languages downloads: 23 homepage_uri: http://parrot.org name: parrot section: universe/interpreters version: 3.6.0-1 parrot-devel: description: Parrot development tools downloads: 18 homepage_uri: http://parrot.org name: parrot-devel section: universe/devel version: 3.6.0-1 parrot-minimal: description: minimal install of the Parrot VM downloads: 30 homepage_uri: http://parrot.org name: parrot-minimal section: universe/interpreters version: 3.6.0-1 parted: description: disk partition manipulator downloads: 1565 homepage_uri: http://www.gnu.org/software/parted name: parted section: admin version: 2.3-8ubuntu5 passwd: description: change and administer password and group data downloads: 95504 homepage_uri: http://pkg-shadow.alioth.debian.org/ name: passwd section: admin version: 1:4.1.4.2+svn3283-3ubuntu5 patch: description: Apply a diff file to an original downloads: 16773 homepage_uri: '' name: patch section: utils version: 2.6.1-3 patchutils: description: Utilities to work with patches downloads: 1938 homepage_uri: http://cyberelk.net/tim/patchutils/index.html name: patchutils section: text version: 0.3.2-1.1 pbuilder: description: personal package builder for Debian packages downloads: 635 homepage_uri: '' name: pbuilder section: devel version: 0.208ubuntu1 pciutils: description: Linux PCI Utilities downloads: 19276 homepage_uri: http://atrey.karlin.mff.cuni.cz/~mj/pciutils.shtml name: pciutils section: admin version: 1:3.1.8-2ubuntu5 pcmciautils: description: PCMCIA utilities for Linux 2.6 downloads: 5559 homepage_uri: http://www.kernel.org/pub/linux/utils/kernel/pcmcia/pcmcia.html name: pcmciautils section: admin version: 018-6 pdfjam: description: collection of PDF document handling utilities downloads: 171 homepage_uri: '' name: pdfjam section: universe/text version: 2.05-2 perl: description: Larry Wall's Practical Extraction and Report Language downloads: 96842 homepage_uri: '' name: perl section: perl version: 5.14.2-6ubuntu2 perl-base: description: minimal Perl system downloads: 118139 homepage_uri: '' name: perl-base section: perl version: 5.14.2-6ubuntu2 perl-doc: description: Perl documentation downloads: 1353 homepage_uri: '' name: perl-doc section: doc version: 5.14.2-6ubuntu2 perl-modules: description: Core Perl modules downloads: 92959 homepage_uri: '' name: perl-modules section: perl version: 5.14.2-6ubuntu2 perl-tk: description: Perl module providing the Tk graphics library downloads: 1051 homepage_uri: http://www.perltk.org/ name: perl-tk section: universe/perl version: 1:804.029-1.1ubuntu2 pgf: description: TeX Portable Graphic Format downloads: 104 homepage_uri: '' name: pgf section: tex version: 2.10-1 phylip: description: package of programs for inferring phylogenies downloads: 12 homepage_uri: http://evolution.genetics.washington.edu/phylip.html name: phylip section: non-free/science version: 1:3.69-1 phyml: description: Phylogenetic estimation using Maximum Likelihood downloads: 32 homepage_uri: http://www.atgc-montpellier.fr/phyml name: phyml section: science version: 2:20110919-1 pinyin-database: description: PinYin database used by ibus-pinyin downloads: 5 homepage_uri: http://code.google.com/p/ibus name: pinyin-database section: universe/utils version: 1.2.99-3 pkg-config: description: manage compile and link flags for libraries downloads: 11087 homepage_uri: http://pkg-config.freedesktop.org name: pkg-config section: devel version: 0.26-1ubuntu1 plink: description: whole-genome association analysis toolset downloads: 22 homepage_uri: http://pngu.mgh.harvard.edu/~purcell/plink/ name: plink section: universe/science version: 1.07-1 plymouth: description: graphical boot animation and logger - main package downloads: 1105 homepage_uri: '' name: plymouth section: x11 version: 0.8.2-2ubuntu30 plymouth-label: description: graphical boot animation and logger - label control downloads: 20 homepage_uri: '' name: plymouth-label section: x11 version: 0.8.2-2ubuntu30 plymouth-theme-ubuntu-logo: description: graphical boot animation and logger - ubuntu-logo theme downloads: 19 homepage_uri: '' name: plymouth-theme-ubuntu-logo section: x11 version: 0.8.2-2ubuntu30 plymouth-theme-ubuntu-text: description: graphical boot animation and logger - ubuntu-logo theme downloads: 2 homepage_uri: '' name: plymouth-theme-ubuntu-text section: x11 version: 0.8.2-2ubuntu30 pm-utils: description: utilities and scripts for power management downloads: 45158 homepage_uri: http://pm-utils.freedesktop.org/ name: pm-utils section: admin version: 1.4.1-9 po-debconf: description: tool for managing templates file translations with gettext downloads: 7107 homepage_uri: '' name: po-debconf section: devel version: 1.0.16+nmu2ubuntu1 policykit-1: description: framework for managing administrative policies and privileges downloads: 41365 homepage_uri: http://hal.freedesktop.org/docs/PolicyKit/ name: policykit-1 section: admin version: 0.104-1ubuntu1 policykit-1-gnome: description: GNOME authentication agent for PolicyKit-1 downloads: 32793 homepage_uri: http://hal.freedesktop.org/docs/PolicyKit-gnome/ name: policykit-1-gnome section: gnome version: 0.105-1ubuntu3 policykit-desktop-privileges: description: run common desktop actions without password downloads: 0 homepage_uri: '' name: policykit-desktop-privileges section: admin version: '0.10' poppler-utils: description: PDF utilities (based on Poppler) downloads: 9981 homepage_uri: http://poppler.freedesktop.org/ name: poppler-utils section: utils version: 0.18.4-1ubuntu2 popularity-contest: description: Vote for your favourite packages automatically downloads: 110453 homepage_uri: http://popcon.ubuntu.com/ name: popularity-contest section: misc version: 1.53ubuntu1 postfix: description: High-performance mail transport agent downloads: 21151 homepage_uri: http://www.postfix.org name: postfix section: mail version: 2.9.1-5 postgresql: description: object-relational SQL database (supported version) downloads: 88 homepage_uri: '' name: postgresql section: database version: 9.1+129 postgresql-8.4: description: object-relational SQL database, version 8.4 server downloads: 6325 homepage_uri: http://www.postgresql.org/ name: postgresql-8.4 section: universe/database version: 8.4.11-1 postgresql-9.1: description: object-relational SQL database, version 9.1 server downloads: 2092 homepage_uri: http://www.postgresql.org/ name: postgresql-9.1 section: database version: 9.1.4-0ubuntu12.04 postgresql-client: description: front-end programs for PostgreSQL (supported version) downloads: 280 homepage_uri: '' name: postgresql-client section: database version: 9.1+129 postgresql-client-8.4: description: front-end programs for PostgreSQL 8.4 downloads: 7112 homepage_uri: http://www.postgresql.org/ name: postgresql-client-8.4 section: universe/database version: 8.4.11-1 postgresql-client-9.1: description: front-end programs for PostgreSQL 9.1 downloads: 2248 homepage_uri: http://www.postgresql.org/ name: postgresql-client-9.1 section: database version: 9.1.4-0ubuntu12.04 postgresql-client-common: description: manager for multiple PostgreSQL client versions downloads: 13611 homepage_uri: '' name: postgresql-client-common section: misc version: '129' postgresql-common: description: PostgreSQL database-cluster manager downloads: 12529 homepage_uri: '' name: postgresql-common section: misc version: '129' postgresql-plperl-8.4: description: PL/Perl procedural language for PostgreSQL 8.4 downloads: 27 homepage_uri: http://www.postgresql.org/ name: postgresql-plperl-8.4 section: universe/database version: 8.4.11-1 postgresql-plpython-8.4: description: PL/Python procedural language for PostgreSQL 8.4 downloads: 20 homepage_uri: http://www.postgresql.org/ name: postgresql-plpython-8.4 section: universe/database version: 8.4.11-1 postgresql-server-dev-9.1: description: development files for PostgreSQL 9.1 server-side programming downloads: 36 homepage_uri: http://www.postgresql.org/ name: postgresql-server-dev-9.1 section: libdevel version: 9.1.4-0ubuntu12.04 powermgmt-base: description: Common utils and configs for power management downloads: 57470 homepage_uri: '' name: powermgmt-base section: utils version: '1.31' ppp: description: Point-to-Point Protocol (PPP) - daemon downloads: 8073 homepage_uri: '' name: ppp section: admin version: 2.4.5-5ubuntu1 pppconfig: description: A text menu based utility for configuring ppp downloads: 184 homepage_uri: '' name: pppconfig section: admin version: 2.3.18+nmu3ubuntu1 pppoeconf: description: configures PPPoE/ADSL connections downloads: 315 homepage_uri: '' name: pppoeconf section: net version: 1.20ubuntu1 pptp-linux: description: Point-to-Point Tunneling Protocol (PPTP) Client downloads: 1254 homepage_uri: '' name: pptp-linux section: net version: 1.7.2-6 preview-latex-style: description: extraction of elements from LaTeX documents as graphics downloads: 3559 homepage_uri: http://www.gnu.org/software/auctex/preview-latex.html name: preview-latex-style section: tex version: 11.86-2ubuntu1 primer3: description: Tool to design flanking oligo nucleotides for DNA amplification downloads: 35 homepage_uri: http://primer3.sourceforge.net name: primer3 section: science version: 2.2.3-1 printer-driver-c2esp: description: printer driver for Kodak ESP AiO color inkjet Series downloads: 195 homepage_uri: http://cupsdriverkodak.sf.net/ name: printer-driver-c2esp section: text version: 23-1 printer-driver-foo2zjs: description: printer driver for ZjStream-based printers downloads: 9013 homepage_uri: http://foo2zjs.rkkda.com/ name: printer-driver-foo2zjs section: text version: 20111202dfsg0-1ubuntu1 printer-driver-gutenprint: description: printer drivers for CUPS downloads: 7282 homepage_uri: '' name: printer-driver-gutenprint section: graphics version: 5.2.8~pre1-0ubuntu2 printer-driver-hpcups: description: HP Linux Printing and Imaging - CUPS Raster driver (hpcups) downloads: 1067 homepage_uri: http://hplipopensource.com/hplip-web/index.html name: printer-driver-hpcups section: text version: 3.12.2-1ubuntu3 printer-driver-hpijs: description: HP Linux Printing and Imaging - gs IJS driver (hpijs) downloads: 1096 homepage_uri: http://hplipopensource.com/hplip-web/index.html name: printer-driver-hpijs section: text version: 3.12.2-1ubuntu3 printer-driver-min12xxw: description: printer driver for KonicaMinolta PagePro 1[234]xxW downloads: 1376 homepage_uri: http://www.hinterbergen.de/mala/min12xxw/ name: printer-driver-min12xxw section: text version: 0.0.9-6ubuntu1 printer-driver-pnm2ppa: description: printer driver for HP-GDI printers downloads: 1341 homepage_uri: http://pnm2ppa.sf.net/ name: printer-driver-pnm2ppa section: text version: 1.13+nondbs-0ubuntu1 printer-driver-ptouch: description: printer driver Brother P-touch label printers downloads: 5686 homepage_uri: http://www.diku.dk/hjemmesider/ansatte/panic/P-touch/ name: printer-driver-ptouch section: text version: 1.3-3ubuntu0.1 printer-driver-pxljr: description: printer driver for HP Color LaserJet 35xx/36xx downloads: 5858 homepage_uri: http://sourceforge.net/projects/hp-pxl-jetready/ name: printer-driver-pxljr section: text version: 1.3+repack0-2 printer-driver-sag-gdi: description: printer driver for Ricoh Aficio SP 1000s/SP 1100s downloads: 190 homepage_uri: http://www.openprinting.org/driver/rastertosag-gdi/ name: printer-driver-sag-gdi section: text version: 0.1-3 printer-driver-splix: description: Driver for Samsung and Xerox SPL2 and SPLc laser printers downloads: 329 homepage_uri: http://splix.ap2c.org/ name: printer-driver-splix section: text version: 2.0.0+svn300-1.1ubuntu2 probcons: description: PROBabilistic CONSistency-based multiple sequence alignment downloads: 25 homepage_uri: http://probcons.stanford.edu/ name: probcons section: universe/science version: 1.12-8 procps: description: /proc file system utilities downloads: 102900 homepage_uri: http://procps.sf.net/ name: procps section: admin version: 1:3.2.8-11ubuntu6 prosper: description: LaTeX class for writing transparencies downloads: 0 homepage_uri: '' name: prosper section: tex version: 1.00.4+cvs.2007.05.01-4 protobuf-compiler: description: compiler for protocol buffer definition files downloads: 142 homepage_uri: http://code.google.com/p/protobuf/ name: protobuf-compiler section: devel version: 2.4.1-1ubuntu2 prover9: description: theorem prover and countermodel generator downloads: 5 homepage_uri: http://www.cs.unm.edu/~mccune/mace4/ name: prover9 section: universe/math version: 0.0.200902a-2 ps2eps: description: convert PostScript to EPS (Encapsulated PostScript) files downloads: 1481 homepage_uri: http://tm.uka.de/~bless/ps2eps name: ps2eps section: text version: 1.68-1 psmisc: description: utilities that use the proc file system downloads: 82235 homepage_uri: http://psmisc.sf.net/ name: psmisc section: admin version: 22.15-2ubuntu1 psutils: description: PostScript document handling utilities downloads: 2573 homepage_uri: http://www.tardis.ed.ac.uk/~ajcd/psutils/ name: psutils section: text version: 1.17-31 pulseaudio: description: PulseAudio sound server downloads: 16800 homepage_uri: http://www.pulseaudio.org name: pulseaudio section: sound version: 1:1.1-0ubuntu15.1 pulseaudio-module-bluetooth: description: Bluetooth module for PulseAudio sound server downloads: 881 homepage_uri: http://www.pulseaudio.org name: pulseaudio-module-bluetooth section: sound version: 1:1.1-0ubuntu15.1 pulseaudio-module-gconf: description: GConf module for PulseAudio sound server downloads: 2195 homepage_uri: http://www.pulseaudio.org name: pulseaudio-module-gconf section: sound version: 1:1.1-0ubuntu15.1 pulseaudio-module-x11: description: X11 module for PulseAudio sound server downloads: 13497 homepage_uri: http://www.pulseaudio.org name: pulseaudio-module-x11 section: sound version: 1:1.1-0ubuntu15.1 pulseaudio-utils: description: Command line tools for the PulseAudio sound server downloads: 14931 homepage_uri: http://www.pulseaudio.org name: pulseaudio-utils section: sound version: 1:1.1-0ubuntu15.1 pvm: description: Parallel Virtual Machine - binaries downloads: 261 homepage_uri: '' name: pvm section: universe/devel version: 3.4.5-12.3 python: description: interactive high-level object-oriented language (default version) downloads: 85476 homepage_uri: http://www.python.org/ name: python section: python version: 2.7.3-0ubuntu2 python-appindicator: description: Python bindings for libappindicator downloads: 22 homepage_uri: https://launchpad.net/libappindicator name: python-appindicator section: python version: 0.4.92-0ubuntu1 python-apport: description: apport crash report handling library downloads: 35 homepage_uri: https://wiki.ubuntu.com/Apport name: python-apport section: python version: 2.0.1-0ubuntu8 python-apptools: description: ETS Application Tools downloads: 193 homepage_uri: http://pypi.python.org/pypi/apptools name: python-apptools section: universe/python version: 4.0.0-1build1 python-apt: description: Python interface to libapt-pkg downloads: 65417 homepage_uri: '' name: python-apt section: python version: 0.8.3ubuntu7 python-apt-common: description: Python interface to libapt-pkg (locales) downloads: 1 homepage_uri: '' name: python-apt-common section: python version: 0.8.3ubuntu7 python-aptdaemon: description: Python module for the server and client of aptdaemon downloads: 20325 homepage_uri: https://launchpad.net/aptdaemon name: python-aptdaemon section: python version: 0.43+bzr805-0ubuntu1 python-aptdaemon.gtk3widgets: description: Python GTK+ 3 widgets to run an aptdaemon client downloads: 1585 homepage_uri: https://launchpad.net/aptdaemon name: python-aptdaemon.gtk3widgets section: python version: 0.43+bzr805-0ubuntu1 python-aptdaemon.pkcompat: description: PackageKit compatibilty for AptDaemon downloads: 5 homepage_uri: https://launchpad.net/aptdaemon name: python-aptdaemon.pkcompat section: python version: 0.43+bzr805-0ubuntu1 python-boto: description: Python interface to Amazon's Web Services downloads: 102 homepage_uri: http://code.google.com/p/boto/ name: python-boto section: python version: 2.2.2-0ubuntu2 python-brlapi: description: Python bindings for BrlAPI downloads: 2122 homepage_uri: http://mielke.cc/brltty/ name: python-brlapi section: python version: 4.3-1ubuntu5 python-bzrlib: description: distributed version control system - python library downloads: 547 homepage_uri: http://bazaar-vcs.org name: python-bzrlib section: python version: 2.5.0-2ubuntu2 python-cairo: description: Python bindings for the Cairo vector graphics library downloads: 38358 homepage_uri: http://cairographics.org/pycairo/ name: python-cairo section: python version: 1.8.8-1ubuntu3 python-central: description: register and build utility for Python packages downloads: 45028 homepage_uri: '' name: python-central section: universe/python version: 0.6.17ubuntu2 python-chardet: description: universal character encoding detector downloads: 28442 homepage_uri: http://chardet.feedparser.org/ name: python-chardet section: python version: 2.0.1-2build1 python-cheetah: description: text-based template engine and Python code generator downloads: 698 homepage_uri: http://www.cheetahtemplate.org/ name: python-cheetah section: text version: 2.4.4-2ubuntu3 python-configglue: description: Glues together optparse.OptionParser and ConfigParser.ConfigParser downloads: 13 homepage_uri: '' name: python-configglue section: python version: 1.0-1build1 python-configobj: description: simple but powerful config file reader and writer for Python downloads: 12308 homepage_uri: http://www.voidspace.org.uk/python/configobj.html name: python-configobj section: python version: 4.7.2+ds-3build1 python-crypto: description: cryptographic algorithms and protocols for Python downloads: 5955 homepage_uri: http://www.pycrypto.org/ name: python-crypto section: python version: 2.4.1-1 python-cups: description: Python bindings for CUPS downloads: 28915 homepage_uri: '' name: python-cups section: python version: 1.9.61-0ubuntu2 python-cupshelpers: description: Python modules for printer configuration with CUPS downloads: 26679 homepage_uri: '' name: python-cupshelpers section: python version: 1.3.8+20120201-0ubuntu8.1 python-dateutil: description: powerful extensions to the standard datetime module downloads: 2412 homepage_uri: '' name: python-dateutil section: python version: 1.5-1 python-dbus: description: simple interprocess messaging system (Python interface) downloads: 37449 homepage_uri: http://www.freedesktop.org/wiki/Software/DBusBindings#Python name: python-dbus section: python version: 1.0.0-1ubuntu1 python-dbus-dev: description: main loop integration development files for python-dbus downloads: 8985 homepage_uri: http://www.freedesktop.org/wiki/Software/DBusBindings#Python name: python-dbus-dev section: python version: 1.0.0-1ubuntu1 python-debian: description: Python modules to work with Debian-related data formats downloads: 47296 homepage_uri: '' name: python-debian section: devel version: 0.1.21ubuntu1 python-debtagshw: description: 'Match debtags hardware:: tags against the actual hardware' downloads: 2 homepage_uri: http://wiki.debian.org/Debtags name: python-debtagshw section: python version: 1.9+git20120320 python-defer: description: Small framework for asynchronous programming downloads: 3413 homepage_uri: https://launchpad.net/python-defer name: python-defer section: python version: 1.0.2+bzr481-1 python-dev: description: header files and a static library for Python (default) downloads: 1433 homepage_uri: http://www.python.org/ name: python-dev section: python version: 2.7.3-0ubuntu2 python-dirspec: description: Python User Folders Specification Library downloads: 1 homepage_uri: '' name: python-dirspec section: python version: 3.0.0-0ubuntu1 python-distutils-extra: description: enhancements to the Python build system downloads: 93 homepage_uri: '' name: python-distutils-extra section: python version: 2.32-2 python-egenix-mxdatetime: description: date and time handling routines for Python downloads: 1759 homepage_uri: '' name: python-egenix-mxdatetime section: python version: 3.2.1-1ubuntu1 python-egenix-mxtools: description: collection of additional builtins for Python downloads: 1669 homepage_uri: '' name: python-egenix-mxtools section: python version: 3.2.1-1ubuntu1 python-envisage: description: Extensible Application Framework downloads: 35 homepage_uri: http://pypi.python.org/pypi/envisage name: python-envisage section: universe/python version: 4.0.0-1build1 python-gconf: description: Python bindings for the GConf configuration database system downloads: 26968 homepage_uri: http://www.pygtk.org/ name: python-gconf section: python version: 2.28.1+dfsg-1 python-gdbm: description: GNU dbm database support for Python downloads: 20429 homepage_uri: '' name: python-gdbm section: python version: 2.7.3-1ubuntu1 python-gi: description: Python 2.x bindings for gobject-introspection libraries downloads: 12254 homepage_uri: '' name: python-gi section: python version: 3.2.2-1~precise python-gi-cairo: description: Python Cairo bindings for the GObject library downloads: 2665 homepage_uri: '' name: python-gi-cairo section: python version: 3.2.2-1~precise python-gmenu: description: GNOME implementation of the freedesktop menu specification downloads: 2743 homepage_uri: '' name: python-gmenu section: gnome version: 3.0.1-0ubuntu7 python-gnomekeyring: description: Python bindings for the GNOME keyring library downloads: 21841 homepage_uri: '' name: python-gnomekeyring section: python version: 2.32.0+dfsg-1 python-gnupginterface: description: Python interface to GnuPG (GPG) downloads: 2296 homepage_uri: '' name: python-gnupginterface section: python version: 0.3.2-9.1ubuntu3 python-gobject: description: Python 2.x bindings for GObject - transitional package downloads: 43657 homepage_uri: '' name: python-gobject section: python version: 3.2.2-1~precise python-gobject-2: description: deprecated static Python bindings for the GObject library downloads: 17303 homepage_uri: '' name: python-gobject-2 section: oldlibs version: 2.28.6-10ubuntu1 python-gst0.10: description: generic media-playing framework (Python bindings) downloads: 38116 homepage_uri: http://gstreamer.freedesktop.org name: python-gst0.10 section: python version: 0.10.22-3 python-gtk2: description: Python bindings for the GTK+ widget set downloads: 39940 homepage_uri: http://www.pygtk.org/ name: python-gtk2 section: python version: 2.24.0-3 python-httplib2: description: comprehensive HTTP client library written for Python downloads: 16810 homepage_uri: http://code.google.com/p/httplib2/ name: python-httplib2 section: libs version: 0.7.2-1ubuntu2 python-ibus: description: Intelligent Input Bus - Python support downloads: 678 homepage_uri: http://code.google.com/p/ibus/ name: python-ibus section: python version: 1.4.1-3ubuntu1 python-imaging: description: Python Imaging Library downloads: 45621 homepage_uri: '' name: python-imaging section: python version: 1.1.7-4 python-keyring: description: store and access your passwords safely downloads: 386 homepage_uri: http://home.python-keyring.org/ name: python-keyring section: python version: 0.7.1-1fakesync1 python-launchpadlib: description: Launchpad web services client library downloads: 320 homepage_uri: https://launchpad.net/launchpadlib name: python-launchpadlib section: python version: 1.9.12-1 python-lazr.restfulclient: description: client for lazr.restful-based web services downloads: 3852 homepage_uri: https://launchpad.net/lazr.restfulclient name: python-lazr.restfulclient section: python version: 0.12.0-1ubuntu1 python-lazr.uri: description: library for parsing, manipulating, and generating URIs downloads: 3856 homepage_uri: https://launchpad.net/lazr.uri name: python-lazr.uri section: python version: 1.0.3-1 python-libproxy: description: automatic proxy configuration management library (python) downloads: 120 homepage_uri: http://code.google.com/p/libproxy/ name: python-libproxy section: python version: 0.4.7-0ubuntu4 python-libvirt: description: libvirt Python bindings downloads: 1251 homepage_uri: http://libvirt.org name: python-libvirt section: devel version: 0.9.8-2ubuntu17.1 python-libxml2: description: Python bindings for the GNOME XML library downloads: 42392 homepage_uri: http://xmlsoft.org/ name: python-libxml2 section: python version: 2.7.8.dfsg-5.1ubuntu4.1 python-louis: description: Python bindings for liblouis downloads: 2247 homepage_uri: http://code.google.com/p/liblouis/ name: python-louis section: python version: 2.3.0-3 python-m2crypto: description: a crypto and SSL toolkit for Python downloads: 469 homepage_uri: http://chandlerproject.org/Projects/MeTooCrypto name: python-m2crypto section: libs version: 0.21.1-2ubuntu2 python-magic: description: File type determination library using "magic" numbers (Python bindings) downloads: 433 homepage_uri: http://www.darwinsys.com/file/ name: python-magic section: python version: 5.09-2 python-mako: description: fast and lightweight templating for the Python platform downloads: 4062 homepage_uri: http://www.makotemplates.org/ name: python-mako section: python version: 0.5.0-1 python-markupsafe: description: XML/HTML/XHTML Markup safe string for Python downloads: 19492 homepage_uri: http://pypi.python.org/pypi/MarkupSafe name: python-markupsafe section: python version: 0.15-1 python-minimal: description: minimal subset of the Python language (default version) downloads: 85145 homepage_uri: http://www.python.org/ name: python-minimal section: python version: 2.7.3-0ubuntu2 python-newt: description: A NEWT module for Python downloads: 1059 homepage_uri: https://fedorahosted.org/newt/ name: python-newt section: python version: 0.52.11-2ubuntu10 python-notify: description: Python bindings for libnotify downloads: 32518 homepage_uri: http://www.galago-project.org name: python-notify section: python version: 0.1.1-3 python-numpy: description: Numerical Python adds a fast array facility to the Python language downloads: 38123 homepage_uri: http://numpy.scipy.org/ name: python-numpy section: python version: 1:1.6.1-6ubuntu1 python-oauth: description: Python library implementing of the OAuth protocol downloads: 4026 homepage_uri: http://code.google.com/p/oauth name: python-oauth section: python version: 1.0.1-3build1 python-openssl: description: Python wrapper around the OpenSSL library downloads: 6669 homepage_uri: http://launchpad.net/pyopenssl name: python-openssl section: python version: 0.12-1ubuntu2 python-packagekit: description: PackageKit Python bindings downloads: 6292 homepage_uri: http://www.packagekit.org name: python-packagekit section: python version: 0.7.2-4ubuntu3 python-pam: description: A Python interface to the PAM library downloads: 930 homepage_uri: '' name: python-pam section: python version: 0.4.2-12.2ubuntu4 python-paramiko: description: Make ssh v2 connections with Python downloads: 493 homepage_uri: http://www.lag.net/paramiko/ name: python-paramiko section: python version: 1.7.7.1-2 python-pexpect: description: Python module for automating interactive applications downloads: 1651 homepage_uri: '' name: python-pexpect section: python version: 2.3-1ubuntu2 python-pip: description: alternative Python package installer downloads: 546 homepage_uri: http://pip.openplans.org/ name: python-pip section: universe/python version: 1.0-1build1 python-piston-mini-client: description: library for writing clients for Django's Piston REST APIs downloads: 0 homepage_uri: https://launchpad.net/piston-mini-client name: python-piston-mini-client section: python version: 0.7.2-0ubuntu1 python-pkg-resources: description: Package Discovery and Resource Access using pkg_resources downloads: 31151 homepage_uri: http://packages.python.org/distribute name: python-pkg-resources section: python version: 0.6.24-1ubuntu1 python-problem-report: description: Python library to handle problem reports downloads: 15 homepage_uri: https://wiki.ubuntu.com/Apport name: python-problem-report section: python version: 2.0.1-0ubuntu8 python-protobuf: description: Python bindings for protocol buffers downloads: 12 homepage_uri: http://code.google.com/p/protobuf/ name: python-protobuf section: python version: 2.4.1-1ubuntu2 python-pyatspi2: description: Assistive Technology Service Provider Interface - Python bindings downloads: 413 homepage_uri: http://live.gnome.org/Accessibility/GNOME3 name: python-pyatspi2 section: python version: 2.4.0+dfsg-0ubuntu3 python-pycurl: description: Python bindings to libcurl downloads: 10373 homepage_uri: http://pycurl.sourceforge.net name: python-pycurl section: python version: 7.19.0-4ubuntu3 python-pyface: description: traits-capable windowing framework downloads: 96 homepage_uri: http://pypi.python.org/pypi/pyface name: python-pyface section: universe/python version: 4.0.0-1build1 python-pyinotify: description: simple Linux inotify Python bindings downloads: 285 homepage_uri: https://github.com/seb-m/pyinotify name: python-pyinotify section: python version: 0.9.2-1 python-qt4: description: Python bindings for Qt4 downloads: 6076 homepage_uri: http://www.riverbankcomputing.co.uk/software/pyqt/ name: python-qt4 section: python version: 4.9.1-2ubuntu1 python-renderpm: description: python low level render interface downloads: 366 homepage_uri: '' name: python-renderpm section: python version: 2.5-1.1build1 python-reportlab: description: ReportLab library to create PDF documents using Python downloads: 1997 homepage_uri: '' name: python-reportlab section: python version: 2.5-1.1build1 python-reportlab-accel: description: C coded extension accelerator for the ReportLab Toolkit downloads: 1559 homepage_uri: '' name: python-reportlab-accel section: python version: 2.5-1.1build1 python-rpy: description: Python interface to the GNU R language and environment downloads: 29 homepage_uri: '' name: python-rpy section: python version: 1.0.3-21precise0 python-scour: description: SVG scrubber and optimizer downloads: 11 homepage_uri: http://codedread.com/scour/ name: python-scour section: python version: 0.26-3 python-serial: description: pyserial - module encapsulating access for the serial port downloads: 612 homepage_uri: '' name: python-serial section: python version: 2.5-2.1build1 python-setuptools: description: Python Distutils Enhancements (setuptools compatibility) downloads: 8371 homepage_uri: http://packages.python.org/distribute name: python-setuptools section: python version: 0.6.24-1ubuntu1 python-simplejson: description: simple, fast, extensible JSON encoder/decoder for Python downloads: 5862 homepage_uri: http://undefined.org/python/#simplejson name: python-simplejson section: python version: 2.3.2-1 python-sip: description: Python/C++ bindings generator runtime library downloads: 4615 homepage_uri: http://www.riverbankcomputing.co.uk/software/sip/ name: python-sip section: python version: 4.13.2-1 python-smbc: description: Python bindings for Samba clients (libsmbclient) downloads: 53 homepage_uri: http://cyberelk.net/tim/software/pysmbc/ name: python-smbc section: python version: 1.0.13-0ubuntu1 python-software-properties: description: manage the repositories that you install software from downloads: 18287 homepage_uri: '' name: python-software-properties section: gnome version: 0.82.7.1 python-speechd: description: Python interface to Speech Dispatcher downloads: 546 homepage_uri: http://devel.freebsoft.org/speechd name: python-speechd section: python version: 0.7.1-6ubuntu3 python-support: description: automated rebuilding support for Python modules downloads: 80317 homepage_uri: '' name: python-support section: universe/python version: 1.0.14ubuntu2 python-traits: description: Manifest typing and reactive programming for Python downloads: 176 homepage_uri: http://pypi.python.org/pypi/traits name: python-traits section: universe/python version: 4.0.0-1build1 python-traitsui: description: traits-capable user interfaces downloads: 111 homepage_uri: http://pypi.python.org/pypi/traitsui name: python-traitsui section: universe/python version: 4.0.1-1build1 python-twisted-bin: description: Event-based framework for internet applications downloads: 6286 homepage_uri: '' name: python-twisted-bin section: python version: 11.1.0-1ubuntu2 python-twisted-core: description: Event-based framework for internet applications downloads: 6118 homepage_uri: '' name: python-twisted-core section: python version: 11.1.0-1ubuntu2 python-twisted-names: description: DNS protocol implementation with client and server downloads: 297 homepage_uri: '' name: python-twisted-names section: python version: 11.1.0-1 python-twisted-web: description: HTTP protocol implementation together with clients and servers downloads: 4233 homepage_uri: '' name: python-twisted-web section: python version: 11.1.0-1 python-ubuntu-sso-client: description: Ubuntu Single Sign-On client - Python library downloads: 10 homepage_uri: '' name: python-ubuntu-sso-client section: python version: 3.0.0-0ubuntu2 python-ubuntuone-client: description: Ubuntu One client Python libraries downloads: 17 homepage_uri: https://one.ubuntu.com name: python-ubuntuone-client section: python version: 3.0.1-0ubuntu1.0.1 python-ubuntuone-control-panel: description: Ubuntu One Control Panel - Python Libraries downloads: 6 homepage_uri: '' name: python-ubuntuone-control-panel section: python version: 3.0.1-0ubuntu1 python-ubuntuone-storageprotocol: description: Python library for Ubuntu One file storage and sharing service downloads: 18 homepage_uri: https://one.ubuntu.com name: python-ubuntuone-storageprotocol section: python version: 3.0.0-0ubuntu1.1 python-uno: description: Python-UNO bridge downloads: 9177 homepage_uri: http://udk.openoffice.org/python/python-bridge.html name: python-uno section: python version: 1:3.5.3-0ubuntu1 python-virtkey: description: Library to emulate keyboard keypresses. downloads: 4 homepage_uri: https://launchpad.net/python-virtkey name: python-virtkey section: python version: 0.60.0-0ubuntu5 python-vm-builder: description: VM builder downloads: 0 homepage_uri: '' name: python-vm-builder section: universe/utils version: 0.12.4+bzr477-0ubuntu1 python-vm-builder-ec2: description: EC2 Ubuntu VM builder downloads: 0 homepage_uri: '' name: python-vm-builder-ec2 section: universe/utils version: 0.12.4+bzr477-0ubuntu1 python-vtk: description: Python bindings for VTK downloads: 170 homepage_uri: http://www.vtk.org/ name: python-vtk section: universe/python version: 5.8.0-5 python-wadllib: description: Python library for navigating WADL files downloads: 3825 homepage_uri: https://launchpad.net/wadllib name: python-wadllib section: python version: 1.3.0-2 python-wxgtk2.8: description: wxWidgets Cross-platform C++ GUI toolkit (wxPython binding) downloads: 9076 homepage_uri: http://www.wxpython.org/ name: python-wxgtk2.8 section: universe/python version: 2.8.12.1-6ubuntu2 python-wxversion: description: wxWidgets Cross-platform C++ GUI toolkit (wxPython version selector) downloads: 10382 homepage_uri: http://www.wxpython.org/ name: python-wxversion section: universe/python version: 2.8.12.1-6ubuntu2 python-xapian: description: Xapian search engine interface for Python downloads: 46065 homepage_uri: http://xapian.org/ name: python-xapian section: python version: 1.2.8-1 python-xdg: description: Python library to access freedesktop.org standards downloads: 26576 homepage_uri: http://www.freedesktop.org/wiki/Software/pyxdg name: python-xdg section: python version: 0.19-3ubuntu2 python-xkit: description: library for the manipulation of the xorg.conf downloads: 0 homepage_uri: '' name: python-xkit section: python version: 0.4.2.3build1 python-yaml: description: YAML parser and emitter for Python downloads: 794 homepage_uri: http://pyyaml.org/ name: python-yaml section: python version: 3.10-2 python-zeitgeist: description: event logging framework - Python bindings downloads: 431 homepage_uri: http://zeitgeist-project.com/ name: python-zeitgeist section: python version: 0.9.0-1ubuntu1 python-zope.interface: description: Interfaces for Python downloads: 19516 homepage_uri: http://pypi.python.org/pypi/zope.interface name: python-zope.interface section: python version: 3.6.1-1ubuntu3 python2.7: description: Interactive high-level object-oriented language (version 2.7) downloads: 19008 homepage_uri: '' name: python2.7 section: python version: 2.7.3-0ubuntu3 python2.7-dev: description: Header files and a static library for Python (v2.7) downloads: 1098 homepage_uri: '' name: python2.7-dev section: python version: 2.7.3-0ubuntu3 python2.7-minimal: description: Minimal subset of the Python language (version 2.7) downloads: 19404 homepage_uri: '' name: python2.7-minimal section: python version: 2.7.3-0ubuntu3 python3: description: interactive high-level object-oriented language (default python3 version) downloads: 1037 homepage_uri: http://www.python.org/ name: python3 section: python version: 3.2.3-0ubuntu1 python3-dev: description: header files and a static library for Python (default) downloads: 55 homepage_uri: http://www.python.org/ name: python3-dev section: python version: 3.2.3-0ubuntu1 python3-minimal: description: minimal subset of the Python language (default python3 version) downloads: 1202 homepage_uri: http://www.python.org/ name: python3-minimal section: python version: 3.2.3-0ubuntu1 python3.2: description: Interactive high-level object-oriented language (version 3.2) downloads: 1125 homepage_uri: '' name: python3.2 section: python version: 3.2.3-0ubuntu3 python3.2-dev: description: Header files and a static library for Python (v3.2) downloads: 65 homepage_uri: '' name: python3.2-dev section: python version: 3.2.3-0ubuntu3 python3.2-minimal: description: Minimal subset of the Python language (version 3.2) downloads: 1324 homepage_uri: '' name: python3.2-minimal section: python version: 3.2.3-0ubuntu3 qdbus: description: Qt 4 D-Bus tool downloads: 5325 homepage_uri: http://qt.nokia.com/ name: qdbus section: libs version: 4:4.8.1-0ubuntu4.1 qemu-common: description: qemu common functionality (bios, documentation, etc) downloads: 0 homepage_uri: http://www.linux-kvm.org name: qemu-common section: misc version: 1.0+noroms-0ubuntu13 qemu-kvm: description: Full virtualization on i386 and amd64 hardware downloads: 3566 homepage_uri: http://www.linux-kvm.org name: qemu-kvm section: misc version: 1.0+noroms-0ubuntu13 qemu-utils: description: qemu utilities downloads: 1069 homepage_uri: http://www.linux-kvm.org name: qemu-utils section: misc version: 1.0+noroms-0ubuntu13 qt-at-spi: description: accessibility plugin for Qt downloads: 0 homepage_uri: '' name: qt-at-spi section: misc version: 0.2.0+git20120411-0ubuntu1 r-base: description: GNU R statistical computation and graphics system downloads: 1 homepage_uri: http://www.r-project.org/ name: r-base section: universe/math version: 2.15.0-1precise0 r-base-core: description: GNU R core of statistical computation and graphics system downloads: 1501 homepage_uri: http://www.r-project.org/ name: r-base-core section: universe/math version: 2.15.0-1precise0 r-base-dev: description: GNU R installation of auxiliary GNU R packages downloads: 1 homepage_uri: http://www.r-project.org/ name: r-base-dev section: universe/devel version: 2.15.0-1precise0 r-base-html: description: GNU R html docs for statistical computing system functions downloads: 77 homepage_uri: http://www.r-project.org/ name: r-base-html section: universe/math version: 2.15.0-1precise0 r-cran-boot: description: GNU R package for bootstrapping functions from Davison and Hinkley downloads: 319 homepage_uri: '' name: r-cran-boot section: universe/math version: 1.3-4-1precise0 r-cran-class: description: GNU R package for classification downloads: 323 homepage_uri: http://cran.r-project.org/web/packages/class/index.html name: r-cran-class section: universe/gnu-r version: 7.3-3-1precise0 r-cran-cluster: description: GNU R package for cluster analysis by Rousseeuw et al downloads: 544 homepage_uri: '' name: r-cran-cluster section: universe/math version: 1.14.2-1precise0 r-cran-codetools: description: GNU R package providing code analysis tools downloads: 345 homepage_uri: '' name: r-cran-codetools section: universe/math version: 0.2-8-1precise0 r-cran-foreign: description: GNU R package to read/write data from other stat. systems downloads: 368 homepage_uri: '' name: r-cran-foreign section: universe/math version: 0.8.50-1precise0 r-cran-kernsmooth: description: GNU R package for kernel smoothing and density estimation downloads: 329 homepage_uri: '' name: r-cran-kernsmooth section: universe/math version: 2.23-7-1precise0 r-cran-lattice: description: GNU R package for 'Trellis' graphics downloads: 636 homepage_uri: '' name: r-cran-lattice section: universe/math version: 0.20-6-1precise0 r-cran-mass: description: GNU R package of Venables and Ripley's MASS downloads: 421 homepage_uri: http://cran.r-project.org/web/packages/MASS/index.html name: r-cran-mass section: universe/gnu-r version: 7.3-18-1precise0 r-cran-matrix: description: GNU R package of classes for dense and sparse matrices downloads: 585 homepage_uri: '' name: r-cran-matrix section: universe/math version: 1.0-6-1precise0 r-cran-mgcv: description: GNU R package for multiple parameter smoothing estimation downloads: 599 homepage_uri: '' name: r-cran-mgcv section: universe/math version: 1.7-18-1precise0 r-cran-nlme: description: GNU R package for (non-)linear mixed effects models downloads: 595 homepage_uri: '' name: r-cran-nlme section: universe/math version: 3.1.104-1precise0 r-cran-nnet: description: GNU R package for feed-forward neural networks downloads: 574 homepage_uri: http://cran.r-project.org/web/packages/nnet/index.html name: r-cran-nnet section: universe/gnu-r version: 7.3-1-2precise0 r-cran-rpart: description: GNU R package for recursive partitioning and regression trees downloads: 317 homepage_uri: '' name: r-cran-rpart section: universe/math version: 3.1.52-1precise0 r-cran-spatial: description: GNU R package for spatial statistics downloads: 312 homepage_uri: http://cran.r-project.org/web/packages/spatial/index.html name: r-cran-spatial section: universe/gnu-r version: 7.3-3-1precise0 r-cran-survival: description: GNU R package for survival analysis downloads: 608 homepage_uri: '' name: r-cran-survival section: universe/math version: 2.36-14-1precise0 r-doc-html: description: GNU R html manuals for statistical computing system downloads: 0 homepage_uri: http://www.r-project.org/ name: r-doc-html section: universe/doc version: 2.15.0-1precise0 r-recommended: description: GNU R collection of recommended packages [metapackage] downloads: 33 homepage_uri: http://www.r-project.org/ name: r-recommended section: universe/math version: 2.15.0-1precise0 rabbitmq-server: description: An AMQP server written in Erlang downloads: 197 homepage_uri: http://www.rabbitmq.com/ name: rabbitmq-server section: net version: 2.7.1-0ubuntu4 radeontool: description: utility to control ATI Radeon backlight functions on laptops downloads: 1434 homepage_uri: '' name: radeontool section: utils version: 1.6.2-1.1 rarian-compat: description: Documentation meta-data library (compatibility tools) downloads: 11693 homepage_uri: http://rarian.freedesktop.org/ name: rarian-compat section: doc version: 0.8.1-5 rasmol: description: Visualize biological macromolecules downloads: 284 homepage_uri: http://rasmol.org name: rasmol section: universe/science version: 2.7.5-3 readline-common: description: GNU readline and history libraries, common files downloads: 18065 homepage_uri: '' name: readline-common section: utils version: 6.2-8 readseq: description: Conversion between sequence formats downloads: 22 homepage_uri: http://iubio.bio.indiana.edu/soft/molbio/readseq/ name: readseq section: science version: 1-9 recode: description: Character set conversion utility downloads: 1243 homepage_uri: '' name: recode section: text version: 3.6-18 reiserfsprogs: description: User-level tools for ReiserFS filesystems downloads: 1437 homepage_uri: '' name: reiserfsprogs section: admin version: 1:3.6.21-1build1 remmina: description: remote desktop client for GNOME desktop environment downloads: 3739 homepage_uri: http://remmina.sourceforge.net/ name: remmina section: gnome version: 1.0.0-1ubuntu6.1 remmina-common: description: common files for remmina remote desktop client downloads: 0 homepage_uri: http://remmina.sourceforge.net/ name: remmina-common section: gnome version: 1.0.0-1ubuntu6.1 remmina-plugin-rdp: description: RDP plugin for remmina remote desktop client downloads: 2412 homepage_uri: http://remmina.sourceforge.net/ name: remmina-plugin-rdp section: gnome version: 1.0.0-1ubuntu6.1 remmina-plugin-vnc: description: VNC plugin for remmina remote desktop client downloads: 2396 homepage_uri: http://remmina.sourceforge.net/ name: remmina-plugin-vnc section: gnome version: 1.0.0-1ubuntu6.1 resolvconf: description: name server information handler downloads: 4595 homepage_uri: http://alioth.debian.org/projects/resolvconf/ name: resolvconf section: net version: 1.63ubuntu14 rfkill: description: tool for enabling and disabling wireless devices downloads: 398 homepage_uri: http://wireless.kernel.org/en/users/Documentation/rfkill name: rfkill section: utils version: 0.4-1ubuntu2 rhino: description: JavaScript engine written in Java downloads: 1015 homepage_uri: http://www.mozilla.org/rhino/ name: rhino section: interpreters version: 1.7R3-5 rhythmbox: description: music player and organizer for GNOME downloads: 15854 homepage_uri: http://projects.gnome.org/rhythmbox/ name: rhythmbox section: gnome version: 2.96-0ubuntu4 rhythmbox-data: description: data files for rhythmbox downloads: 0 homepage_uri: http://projects.gnome.org/rhythmbox/ name: rhythmbox-data section: gnome version: 2.96-0ubuntu4 rhythmbox-mozilla: description: Rhythmbox Mozilla plugin downloads: 0 homepage_uri: http://projects.gnome.org/rhythmbox/ name: rhythmbox-mozilla section: web version: 2.96-0ubuntu4 rhythmbox-plugin-cdrecorder: description: burning plugin for rhythmbox music player downloads: 4072 homepage_uri: http://projects.gnome.org/rhythmbox/ name: rhythmbox-plugin-cdrecorder section: gnome version: 2.96-0ubuntu4 rhythmbox-plugin-magnatune: description: Magnatune plugin for rhythmbox music player downloads: 0 homepage_uri: http://projects.gnome.org/rhythmbox/ name: rhythmbox-plugin-magnatune section: gnome version: 2.96-0ubuntu4 rhythmbox-plugin-zeitgeist: description: zeitgeist plugin for rhythmbox music player downloads: 0 homepage_uri: http://projects.gnome.org/rhythmbox/ name: rhythmbox-plugin-zeitgeist section: gnome version: 2.96-0ubuntu4 rhythmbox-plugins: description: plugins for rhythmbox music player downloads: 18580 homepage_uri: http://projects.gnome.org/rhythmbox/ name: rhythmbox-plugins section: gnome version: 2.96-0ubuntu4 rhythmbox-ubuntuone: description: Ubuntu One Rhythmbox plugin downloads: 1 homepage_uri: https://launchpad.net/rhythmbox-ubuntuone name: rhythmbox-ubuntuone section: gnome version: 3.0.0-0ubuntu1 ri1.8: description: Ruby Interactive reference (for Ruby 1.8) downloads: 100 homepage_uri: http://www.ruby-lang.org/ name: ri1.8 section: universe/interpreters version: 1.8.7.352-2ubuntu1 rpcbind: description: converts RPC program numbers into universal addresses downloads: 13892 homepage_uri: http://sourceforge.net/projects/rpcbind/ name: rpcbind section: net version: 0.2.0-7ubuntu1.1 rsync: description: fast, versatile, remote (and local) file-copying tool downloads: 23606 homepage_uri: http://rsync.samba.org/ name: rsync section: net version: 3.0.9-1ubuntu1 rsyslog: description: reliable system and kernel logging daemon downloads: 72075 homepage_uri: http://www.rsyslog.com/ name: rsyslog section: admin version: 5.8.6-1ubuntu8 rtkit: description: Realtime Policy and Watchdog Daemon downloads: 11470 homepage_uri: http://0pointer.de/public/ name: rtkit section: admin version: 0.10-2 ruby: description: Transitional package for ruby1.8 downloads: 8172 homepage_uri: http://www.ruby-lang.org/ name: ruby section: interpreters version: '4.8' ruby1.8: description: Interpreter of object-oriented scripting language Ruby 1.8 downloads: 10045 homepage_uri: http://www.ruby-lang.org/ name: ruby1.8 section: interpreters version: 1.8.7.352-2ubuntu1 ruby1.8-dev: description: Header files for compiling extension modules for the Ruby 1.8 downloads: 524 homepage_uri: http://www.ruby-lang.org/ name: ruby1.8-dev section: devel version: 1.8.7.352-2ubuntu1 ruby1.9.1: description: Interpreter of object-oriented scripting language Ruby downloads: 2200 homepage_uri: http://www.ruby-lang.org/ name: ruby1.9.1 section: ruby version: 1.9.3.0-1ubuntu1 ruby1.9.1-dev: description: Header files for compiling extension modules for the Ruby 1.9.1 downloads: 189 homepage_uri: http://www.ruby-lang.org/ name: ruby1.9.1-dev section: ruby version: 1.9.3.0-1ubuntu1 s3cmd: description: command-line Amazon S3 client downloads: 101 homepage_uri: http://s3tools.logix.cz/s3cmd name: s3cmd section: universe/admin version: 1.0.0-1 samba-common: description: common files used by both the Samba server and client downloads: 25486 homepage_uri: http://www.samba.org name: samba-common section: net version: 2:3.6.3-2ubuntu2.2 samba-common-bin: description: common files used by both the Samba server and client downloads: 18675 homepage_uri: http://www.samba.org name: samba-common-bin section: net version: 2:3.6.3-2ubuntu2.2 sane-utils: description: API library for scanners -- utilities downloads: 5068 homepage_uri: http://www.sane-project.org name: sane-utils section: graphics version: 1.0.22-7ubuntu1 sat4j: description: Efficient library of SAT solvers in Java downloads: 1737 homepage_uri: http://www.sat4j.org name: sat4j section: universe/science version: 2.3.1-1 scala: description: Scala programming language downloads: 91 homepage_uri: http://www.scala-lang.org/ name: scala section: universe/devel version: 2.9.1.dfsg-3 scala-library: description: Scala standard library downloads: 91 homepage_uri: http://www.scala-lang.org/ name: scala-library section: universe/devel version: 2.9.1.dfsg-3 screen: description: terminal multiplexor with VT100/ANSI terminal emulation downloads: 11357 homepage_uri: http://savannah.gnu.org/projects/screen name: screen section: misc version: 4.0.3-14ubuntu8 seabios: description: legacy BIOS implementation which can be used as a coreboot payload downloads: 0 homepage_uri: http://www.coreboot.org/SeaBIOS name: seabios section: misc version: 0.6.2-0ubuntu2.1 seahorse: description: GNOME front end for GnuPG downloads: 21816 homepage_uri: http://live.gnome.org/Seahorse name: seahorse section: gnome version: 3.2.2-0ubuntu2 sed: description: The GNU sed stream editor downloads: 110703 homepage_uri: http://www.gnu.org/software/sed/ name: sed section: utils version: 4.2.1-9 sensible-utils: description: Utilities for sensible alternative selection downloads: 91557 homepage_uri: '' name: sensible-utils section: utils version: 0.0.6ubuntu2 sessioninstaller: description: APT based installer using PackageKit's session DBus API downloads: 570 homepage_uri: http://launchpad.net/sessioninstaller name: sessioninstaller section: gnome version: 0.20+bzr128-0ubuntu1 sgml-base: description: SGML infrastructure and SGML catalog file support downloads: 12945 homepage_uri: '' name: sgml-base section: text version: 1.26+nmu1ubuntu1 sgml-data: description: common SGML and XML data downloads: 418 homepage_uri: '' name: sgml-data section: text version: 2.0.6 shared-mime-info: description: FreeDesktop.org shared MIME database and spec downloads: 42034 homepage_uri: http://freedesktop.org/wiki/Software/shared-mime-info name: shared-mime-info section: misc version: 1.0-0ubuntu4.1 shotwell: description: digital photo organizer downloads: 4973 homepage_uri: http://yorba.org/shotwell/ name: shotwell section: gnome version: 0.12.3-0ubuntu0.1 simple-scan: description: Simple Scanning Utility downloads: 3048 homepage_uri: https://launchpad.net/simple-scan name: simple-scan section: gnome version: 3.4.1-0ubuntu1.1 smbclient: description: command-line SMB/CIFS clients for Unix downloads: 6314 homepage_uri: http://www.samba.org name: smbclient section: net version: 2:3.6.3-2ubuntu2.2 sni-qt: description: indicator support for Qt downloads: 0 homepage_uri: https://launchpad.net/sni-qt name: sni-qt section: x11 version: 0.2.5-0ubuntu3 software-center: description: Utility for browsing, installing, and removing software downloads: 6211 homepage_uri: https://launchpad.net/software-center name: software-center section: gnome version: 5.2.2.2 software-center-aptdaemon-plugins: description: The aptdaemon plugins for software-center downloads: 0 homepage_uri: '' name: software-center-aptdaemon-plugins section: gnome version: 0.1.2 software-properties-common: description: manage the repositories that you install software from (common) downloads: 1119 homepage_uri: '' name: software-properties-common section: admin version: 0.82.7.1 software-properties-gtk: description: manage the repositories that you install software from (gtk) downloads: 5491 homepage_uri: '' name: software-properties-gtk section: gnome version: 0.82.7.1 sound-theme-freedesktop: description: freedesktop.org sound theme downloads: 0 homepage_uri: http://www.freedesktop.org/ name: sound-theme-freedesktop section: sound version: 0.7.pristine-2 speech-dispatcher: description: Common interface to speech synthesizers downloads: 1624 homepage_uri: http://devel.freebsoft.org/speechd name: speech-dispatcher section: sound version: 0.7.1-6ubuntu3 sqlite3: description: Command line interface for SQLite 3 downloads: 1855 homepage_uri: '' name: sqlite3 section: misc version: 3.7.9-2ubuntu1 ssh-askpass-gnome: description: interactive X program to prompt users for a passphrase for ssh-add downloads: 290 homepage_uri: http://www.openssh.org/ name: ssh-askpass-gnome section: gnome version: 1:5.9p1-5ubuntu1 ssh-import-id: description: securely retrieve an SSH public key and install it locally downloads: 1 homepage_uri: http://launchpad.net/ssh-import-id name: ssh-import-id section: misc version: 2.10-0ubuntu1 ssl-cert: description: simple debconf wrapper for OpenSSL downloads: 32921 homepage_uri: '' name: ssl-cert section: utils version: 1.0.28ubuntu0.1 stow: description: Organizer for /usr/local software packages downloads: 122 homepage_uri: http://www.gnu.org/software/stow/ name: stow section: universe/admin version: 1.3.3-3.2 strace: description: A system call tracer downloads: 5412 homepage_uri: http://sourceforge.net/projects/strace/ name: strace section: utils version: 4.5.20-2.3ubuntu1 subversion: description: Advanced version control system downloads: 10196 homepage_uri: http://subversion.apache.org/ name: subversion section: devel version: 1.6.17dfsg-3ubuntu3 sudo: description: Provide limited super user privileges to specific users downloads: 53108 homepage_uri: '' name: sudo section: admin version: 1.8.3p1-1ubuntu3.3 swig: description: Generate scripting interfaces to C/C++ code downloads: 365 homepage_uri: http://www.swig.org/ name: swig section: interpreters version: 2.0.4+really2.0.4-4ubuntu2 swig2.0: description: Generate scripting interfaces to C/C++ code downloads: 188 homepage_uri: http://www.swig.org/ name: swig2.0 section: interpreters version: 2.0.4+really2.0.4-4ubuntu2 synaptic: description: Graphical package manager downloads: 13131 homepage_uri: '' name: synaptic section: universe/admin version: 0.75.9ubuntu1 syslinux: description: collection of boot loaders downloads: 1623 homepage_uri: http://syslinux.zytor.com/ name: syslinux section: utils version: 2:4.05+dfsg-2 syslinux-common: description: collection of boot loaders (common files) downloads: 2487 homepage_uri: http://syslinux.zytor.com/ name: syslinux-common section: utils version: 2:4.05+dfsg-2 syslinux-legacy: description: Bootloader for Linux/i386 using MS-DOS floppies downloads: 0 homepage_uri: http://syslinux.zytor.com/ name: syslinux-legacy section: utils version: 2:3.63+dfsg-2ubuntu5 system-config-printer-common: description: Printer configuration GUI downloads: 0 homepage_uri: '' name: system-config-printer-common section: python version: 1.3.8+20120201-0ubuntu8.1 system-config-printer-gnome: description: Printer configuration GUI downloads: 23 homepage_uri: '' name: system-config-printer-gnome section: python version: 1.3.8+20120201-0ubuntu8.1 system-config-printer-udev: description: Printer auto-configuration facility based on udev downloads: 30742 homepage_uri: '' name: system-config-printer-udev section: python version: 1.3.8+20120201-0ubuntu8.1 system-tools-backends: description: System Tools to manage computer configuration -- scripts downloads: 8524 homepage_uri: http://system-tools-backends.freedesktop.org/ name: system-tools-backends section: universe/gnome version: 2.10.2-1ubuntu1 sysv-rc: description: System-V-like runlevel change mechanism downloads: 110528 homepage_uri: http://savannah.nongnu.org/projects/sysvinit name: sysv-rc section: admin version: 2.88dsf-13.10ubuntu11 sysvinit-utils: description: System-V-like utilities downloads: 110539 homepage_uri: http://savannah.nongnu.org/projects/sysvinit name: sysvinit-utils section: admin version: 2.88dsf-13.10ubuntu11 t-coffee: description: Multiple Sequence Alignment downloads: 28 homepage_uri: http://www.tcoffee.org/Projects_home_page/t_coffee_home_page.html name: t-coffee section: science version: 8.99-1 tabix: description: generic indexer for TAB-delimited genome position files downloads: 6 homepage_uri: http://samtools.sourceforge.net/tabix.shtml name: tabix section: universe/science version: 0.2.5-1ubuntu1 tar: description: GNU version of the tar archiving utility downloads: 68906 homepage_uri: '' name: tar section: utils version: 1.26-4ubuntu1 tasksel: description: Tool for selecting tasks for installation on Debian systems downloads: 7430 homepage_uri: '' name: tasksel section: admin version: 2.88ubuntu9 tasksel-data: description: Official tasks used for installation of Debian systems downloads: 7286 homepage_uri: '' name: tasksel-data section: admin version: 2.88ubuntu9 tcl: description: The Tool Command Language (default version) - run-time files downloads: 7759 homepage_uri: '' name: tcl section: interpreters version: 8.5.0-2 tcl-dev: description: The Tool Command Language (default version) - development files downloads: 137 homepage_uri: '' name: tcl-dev section: devel version: 8.5.0-2 tcl-vtk: description: Tcl bindings for VTK downloads: 99 homepage_uri: http://www.vtk.org/ name: tcl-vtk section: universe/interpreters version: 5.8.0-5 tcl8.5: description: Tcl (the Tool Command Language) v8.5 - run-time files downloads: 8983 homepage_uri: http://www.tcl.tk/ name: tcl8.5 section: interpreters version: 8.5.11-1ubuntu1 tcl8.5-dev: description: Tcl (the Tool Command Language) v8.5 - development files downloads: 397 homepage_uri: http://www.tcl.tk/ name: tcl8.5-dev section: devel version: 8.5.11-1ubuntu1 tcpd: description: Wietse Venema's TCP wrapper utilities downloads: 27483 homepage_uri: '' name: tcpd section: net version: 7.6.q-21 tcpdump: description: command-line network traffic analyzer downloads: 3856 homepage_uri: http://www.tcpdump.org/ name: tcpdump section: net version: 4.2.1-1ubuntu2 tcsh: description: TENEX C Shell, an enhanced version of Berkeley csh downloads: 3973 homepage_uri: http://www.tcsh.org/ name: tcsh section: universe/shells version: 6.17.06-2 telepathy-gabble: description: Jabber/XMPP connection manager downloads: 2206 homepage_uri: http://telepathy.freedesktop.org/wiki/ name: telepathy-gabble section: net version: 0.16.0-0ubuntu1 telepathy-haze: description: Telepathy connection manager that uses libpurple downloads: 1443 homepage_uri: '' name: telepathy-haze section: net version: 0.6.0-0ubuntu1 telepathy-idle: description: IRC connection manager for Telepathy downloads: 248 homepage_uri: http://sourceforge.net/projects/telepathy-idle name: telepathy-idle section: net version: 0.1.11-2 telepathy-indicator: description: Desktop service to integrate Telepathy with the messaging menu. downloads: 0 homepage_uri: https://launchpad.net/telepathy-indicator name: telepathy-indicator section: gnome version: 0.2.1-0ubuntu1 telepathy-logger: description: Telepathy logger service - Daemon downloads: 2667 homepage_uri: http://telepathy.freedesktop.org/wiki/ name: telepathy-logger section: libs version: 0.4.0-0ubuntu1 telepathy-mission-control-5: description: management daemon for Telepathy real-time communication framework downloads: 11686 homepage_uri: http://telepathy.freedesktop.org/wiki/ name: telepathy-mission-control-5 section: net version: 1:5.12.0-0ubuntu2 telepathy-salut: description: Link-local XMPP connection manager for the Telepathy framework downloads: 1766 homepage_uri: http://telepathy.freedesktop.org/wiki/ name: telepathy-salut section: net version: 0.8.0-0ubuntu1 telnet: description: The telnet client downloads: 14806 homepage_uri: '' name: telnet section: net version: 0.17-36build1 tex-common: description: common infrastructure for building and installing TeX downloads: 11089 homepage_uri: '' name: tex-common section: tex version: '2.10' texinfo: description: Documentation system for on-line information and printed output downloads: 9214 homepage_uri: '' name: texinfo section: text version: 4.13a.dfsg.1-8ubuntu2 texlive-base: description: 'TeX Live: Essential programs and files' downloads: 10342 homepage_uri: http://www.tug.org/texlive/ name: texlive-base section: tex version: 2009-15 texlive-binaries: description: Binaries for TeX Live downloads: 10483 homepage_uri: http://www.tug.org/texlive name: texlive-binaries section: tex version: 2009-11ubuntu2 texlive-common: description: 'TeX Live: Base component' downloads: 10417 homepage_uri: http://www.tug.org/texlive/ name: texlive-common section: tex version: 2009-15 texlive-doc-base: description: 'TeX Live: TeX Live documentation' downloads: 9632 homepage_uri: http://www.tug.org/texlive name: texlive-doc-base section: tex version: 2009-2 texlive-extra-utils: description: 'TeX Live: TeX auxiliary programs' downloads: 1688 homepage_uri: http://www.tug.org/texlive/ name: texlive-extra-utils section: tex version: 2009-10ubuntu1 texlive-font-utils: description: 'TeX Live: TeX and Outline font utilities' downloads: 1727 homepage_uri: http://www.tug.org/texlive/ name: texlive-font-utils section: tex version: 2009-10ubuntu1 texlive-fonts-recommended: description: 'TeX Live: Recommended fonts' downloads: 3743 homepage_uri: http://www.tug.org/texlive/ name: texlive-fonts-recommended section: tex version: 2009-15 texlive-fonts-recommended-doc: description: 'TeX Live: Documentation files for texlive-fonts-recommended' downloads: 0 homepage_uri: http://www.tug.org/texlive/ name: texlive-fonts-recommended-doc section: doc version: 2009-15 texlive-generic-recommended: description: 'TeX Live: Recommended generic packages' downloads: 1684 homepage_uri: http://www.tug.org/texlive/ name: texlive-generic-recommended section: tex version: 2009-15 texlive-latex-base: description: 'TeX Live: Basic LaTeX packages' downloads: 7374 homepage_uri: http://www.tug.org/texlive/ name: texlive-latex-base section: tex version: 2009-15 texlive-latex-base-doc: description: 'TeX Live: Documentation files for texlive-latex-base' downloads: 0 homepage_uri: http://www.tug.org/texlive/ name: texlive-latex-base-doc section: doc version: 2009-15 texlive-latex-extra: description: 'TeX Live: LaTeX supplementary packages' downloads: 1889 homepage_uri: http://www.tug.org/texlive/ name: texlive-latex-extra section: tex version: 2009-10ubuntu1 texlive-latex-extra-doc: description: 'TeX Live: Documentation files for texlive-latex-extra' downloads: 0 homepage_uri: http://www.tug.org/texlive/ name: texlive-latex-extra-doc section: doc version: 2009-10ubuntu1 texlive-latex-recommended: description: 'TeX Live: LaTeX recommended packages' downloads: 2444 homepage_uri: http://www.tug.org/texlive/ name: texlive-latex-recommended section: tex version: 2009-15 texlive-latex-recommended-doc: description: 'TeX Live: Documentation files for texlive-latex-recommended' downloads: 0 homepage_uri: http://www.tug.org/texlive/ name: texlive-latex-recommended-doc section: doc version: 2009-15 texlive-luatex: description: 'TeX Live: LuaTeX packages' downloads: 319 homepage_uri: http://www.tug.org/texlive/ name: texlive-luatex section: tex version: 2009-15 texlive-pictures: description: 'TeX Live: Graphics packages and programs' downloads: 1387 homepage_uri: http://www.tug.org/texlive/ name: texlive-pictures section: tex version: 2009-15 texlive-pictures-doc: description: 'TeX Live: Documentation files for texlive-pictures' downloads: 0 homepage_uri: http://www.tug.org/texlive/ name: texlive-pictures-doc section: doc version: 2009-15 texlive-pstricks: description: 'TeX Live: PSTricks packages' downloads: 1134 homepage_uri: http://www.tug.org/texlive/ name: texlive-pstricks section: tex version: 2009-10ubuntu1 texlive-pstricks-doc: description: 'TeX Live: Documentation files for texlive-pstricks' downloads: 0 homepage_uri: http://www.tug.org/texlive/ name: texlive-pstricks-doc section: doc version: 2009-10ubuntu1 thunderbird: description: Email, RSS and newsgroup client with integrated spam filter downloads: 181 homepage_uri: '' name: thunderbird section: mail version: 13.0.1+build1-0ubuntu0.12.04.1 thunderbird-globalmenu: description: Unity appmenu integration for Thunderbird downloads: 7 homepage_uri: '' name: thunderbird-globalmenu section: web version: 13.0.1+build1-0ubuntu0.12.04.1 thunderbird-gnome-support: description: Email, RSS and newsgroup client - GNOME support downloads: 0 homepage_uri: '' name: thunderbird-gnome-support section: mail version: 13.0.1+build1-0ubuntu0.12.04.1 tigr-glimmer: description: Gene detection in archea and bacteria downloads: 24 homepage_uri: http://www.cbcb.umd.edu/software/glimmer name: tigr-glimmer section: universe/science version: 3.02-2build1 time: description: The GNU time program for measuring cpu resource usage downloads: 8630 homepage_uri: '' name: time section: utils version: 1.7-23.1 tipa: description: system for processing phonetic symbols in LaTeX downloads: 3099 homepage_uri: http://www.ctan.org/tex-archive/fonts/tipa/ name: tipa section: tex version: 2:1.3-15 tk: description: The Tk toolkit for Tcl and X11 (default version) - run-time files downloads: 2931 homepage_uri: '' name: tk section: libs version: 8.5.0-2 tk-dev: description: The Tk toolkit for Tcl and X11 (default version) - development files downloads: 35 homepage_uri: '' name: tk-dev section: devel version: 8.5.0-2 tk8.5: description: Tk toolkit for Tcl and X11, v8.5 - run-time files downloads: 5598 homepage_uri: http://www.tcl.tk/ name: tk8.5 section: libs version: 8.5.11-1 tk8.5-dev: description: Tk toolkit for Tcl and X11, v8.5 - development files downloads: 276 homepage_uri: http://www.tcl.tk/ name: tk8.5-dev section: devel version: 8.5.11-1 tmux: description: terminal multiplexer downloads: 1403 homepage_uri: http://tmux.sourceforge.net/ name: tmux section: admin version: 1.6-1ubuntu1 tntnet: description: modular, multithreaded web application server for C++ downloads: 25 homepage_uri: http://www.tntnet.org/ name: tntnet section: universe/web version: 2.0+dfsg1-2 tntnet-runtime: description: Tntnet runtime system downloads: 15 homepage_uri: http://www.tntnet.org/ name: tntnet-runtime section: universe/web version: 2.0+dfsg1-2 tokyocabinet-bin: description: Tokyo Cabinet Database Utilities downloads: 11 homepage_uri: '' name: tokyocabinet-bin section: universe/utils version: 1.4.37-6.1build1 toshset: description: Access much of the Toshiba laptop hardware interface downloads: 144 homepage_uri: http://www.schwieters.org/toshset/ name: toshset section: utils version: 1.76-2 totem: description: Simple media player for the GNOME desktop based on GStreamer downloads: 28899 homepage_uri: http://www.gnome.org/projects/totem/ name: totem section: gnome version: 3.0.1-0ubuntu21 totem-common: description: Data files for the Totem media player downloads: 28547 homepage_uri: http://www.gnome.org/projects/totem/ name: totem-common section: gnome version: 3.0.1-0ubuntu21 totem-mozilla: description: Totem Mozilla plugin downloads: 19391 homepage_uri: http://www.gnome.org/projects/totem/ name: totem-mozilla section: web version: 3.0.1-0ubuntu21 totem-plugins: description: Plugins for the Totem media player downloads: 14520 homepage_uri: http://www.gnome.org/projects/totem/ name: totem-plugins section: gnome version: 3.0.1-0ubuntu21 traceroute: description: Traces the route taken by packets over an IPv4/IPv6 network downloads: 13456 homepage_uri: http://traceroute.sourceforge.net/ name: traceroute section: universe/net version: 1:2.0.18-1 transmission-common: description: lightweight BitTorrent client (common files) downloads: 1 homepage_uri: http://www.transmissionbt.com/ name: transmission-common section: net version: 2.51-0ubuntu1 transmission-gtk: description: lightweight BitTorrent client (GTK interface) downloads: 16110 homepage_uri: http://www.transmissionbt.com/ name: transmission-gtk section: net version: 2.51-0ubuntu1 tree: description: displays directory tree, in color downloads: 1337 homepage_uri: http://mama.indstate.edu/users/ice/tree/ name: tree section: universe/utils version: 1.5.3-2 tree-puzzle: description: Reconstruction of phylogenetic trees by maximum likelihood downloads: 20 homepage_uri: http://www.tree-puzzle.de name: tree-puzzle section: science version: 5.2-7 tree-puzzle-doc: description: Reconstruction of phylogenetic trees by maximum likelihood downloads: 4 homepage_uri: http://www.tree-puzzle.de name: tree-puzzle-doc section: universe/science version: 5.2-7 treeviewx: description: Displays and prints phylogenetic trees downloads: 126 homepage_uri: http://darwin.zoology.gla.ac.uk/~rpage/treeviewx/ name: treeviewx section: science version: 0.5.1+20100823-1 ttf-dejavu-core: description: Vera font family derivate with additional characters downloads: 11606 homepage_uri: http://dejavu-fonts.org/ name: ttf-dejavu-core section: x11 version: 2.33-2ubuntu1 ttf-dejavu-extra: description: Vera font family derivate with additional characters downloads: 1008 homepage_uri: http://dejavu-fonts.org/ name: ttf-dejavu-extra section: x11 version: 2.33-2ubuntu1 ttf-freefont: description: Freefont Serif, Sans and Mono Truetype fonts downloads: 3434 homepage_uri: http://savannah.gnu.org/projects/freefont/ name: ttf-freefont section: x11 version: 20100919-1 ttf-indic-fonts-core: description: Core collection of free fonts for languages of India downloads: 0 homepage_uri: http://debian-in.alioth.debian.org/ name: ttf-indic-fonts-core section: x11 version: 1:0.5.11ubuntu1 ttf-liberation: description: transitional dummy package downloads: 1088 homepage_uri: https://fedorahosted.org/liberation-fonts/ name: ttf-liberation section: fonts version: 1.07.0-2 ttf-punjabi-fonts: description: Free TrueType fonts for the Punjabi language downloads: 1 homepage_uri: http://debian-in.alioth.debian.org/ name: ttf-punjabi-fonts section: x11 version: 1:0.5.11ubuntu1 ttf-ubuntu-font-family: description: Ubuntu Font Family, sans-serif typeface hinted for clarity downloads: 0 homepage_uri: http://font.ubuntu.com/ name: ttf-ubuntu-font-family section: x11 version: 0.80-0ubuntu2 ttf-wqy-microhei: description: A droid derived Sans-Seri style CJK font downloads: 0 homepage_uri: http://wqy.sourceforge.net/ name: ttf-wqy-microhei section: x11 version: 0.2.0-beta-1ubuntu1 tzdata: description: time zone and daylight-saving time data downloads: 110141 homepage_uri: http://www.iana.org/time-zones name: tzdata section: libs version: 2012b-1 tzdata-java: description: time zone and daylight-saving time data for use by java runtimes downloads: 12801 homepage_uri: http://www.iana.org/time-zones name: tzdata-java section: libs version: 2012b-1 ubuntu-artwork: description: Ubuntu themes and artwork downloads: 0 homepage_uri: '' name: ubuntu-artwork section: gnome version: '57' ubuntu-desktop: description: The Ubuntu desktop system downloads: 0 homepage_uri: '' name: ubuntu-desktop section: metapackages version: '1.267' ubuntu-docs: description: Ubuntu Desktop Guide downloads: 0 homepage_uri: https://launchpad.net/ubuntu-doc name: ubuntu-docs section: text version: 12.04.5 ubuntu-extras-keyring: description: GnuPG keys of the Ubuntu extras archive downloads: 0 homepage_uri: '' name: ubuntu-extras-keyring section: misc version: 2010.09.27 ubuntu-keyring: description: GnuPG keys of the Ubuntu archive downloads: 0 homepage_uri: '' name: ubuntu-keyring section: misc version: 2011.11.21 ubuntu-minimal: description: Minimal core of Ubuntu downloads: 0 homepage_uri: '' name: ubuntu-minimal section: metapackages version: '1.267' ubuntu-mono: description: Ubuntu Mono Icon theme downloads: 0 homepage_uri: '' name: ubuntu-mono section: gnome version: 0.0.40 ubuntu-sounds: description: Ubuntu's GNOME audio theme downloads: 0 homepage_uri: '' name: ubuntu-sounds section: gnome version: '0.13' ubuntu-sso-client: description: Ubuntu Single Sign-On client downloads: 13 homepage_uri: '' name: ubuntu-sso-client section: python version: 3.0.0-0ubuntu2 ubuntu-sso-client-gtk: description: Ubuntu Single Sign-On client - GTK+ frontend downloads: 0 homepage_uri: '' name: ubuntu-sso-client-gtk section: python version: 3.0.0-0ubuntu2 ubuntu-standard: description: The Ubuntu standard system downloads: 0 homepage_uri: '' name: ubuntu-standard section: metapackages version: '1.267' ubuntu-system-service: description: Dbus service to set various system-wide configurations downloads: 4 homepage_uri: '' name: ubuntu-system-service section: admin version: 0.2.2 ubuntu-wallpapers: description: Ubuntu Wallpapers downloads: 0 homepage_uri: '' name: ubuntu-wallpapers section: metapackages version: 0.34.1 ubuntu-wallpapers-precise: description: Ubuntu 12.04 Wallpapers downloads: 0 homepage_uri: '' name: ubuntu-wallpapers-precise section: x11 version: 0.34.1 ubuntuone-client: description: Ubuntu One client downloads: 14 homepage_uri: https://one.ubuntu.com name: ubuntuone-client section: net version: 3.0.1-0ubuntu1.0.1 ubuntuone-client-gnome: description: Ubuntu One client GNOME integration downloads: 4 homepage_uri: https://one.ubuntu.com name: ubuntuone-client-gnome section: gnome version: 3.0.1-0ubuntu1 ubuntuone-control-panel: description: Ubuntu One Control Panel downloads: 0 homepage_uri: '' name: ubuntuone-control-panel section: python version: 3.0.1-0ubuntu1 ubuntuone-couch: description: Ubuntu One CouchDB downloads: 2 homepage_uri: http://launchpad.net/ubuntuone-couch name: ubuntuone-couch section: python version: 0.3.0-0ubuntu4 ubuntuone-installer: description: Ubuntu One Installer downloads: 4 homepage_uri: '' name: ubuntuone-installer section: python version: 3.0.0-0ubuntu1 ucf: description: 'Update Configuration File: preserve user changes to config files.' downloads: 88721 homepage_uri: '' name: ucf section: utils version: 3.0025+nmu2ubuntu1 udev: description: rule-based device node and kernel event manager downloads: 84923 homepage_uri: http://www.kernel.org/pub/linux/utils/kernel/hotplug/udev.html name: udev section: admin version: 175-0ubuntu9 udisks: description: storage media interface downloads: 46689 homepage_uri: http://www.freedesktop.org/wiki/Software/udisks name: udisks section: admin version: 1.0.4-5ubuntu2 ufw: description: program for managing a Netfilter firewall downloads: 1095 homepage_uri: https://launchpad.net/ufw name: ufw section: admin version: 0.31.1-1 ugene: description: integrated bioinformatics toolkit downloads: 1 homepage_uri: http://ugene.unipro.ru name: ugene section: universe/science version: 1.9.8+repack-0ubuntu3 ugene-data: description: required data for UGENE - integrated bioinformatics toolkit downloads: 0 homepage_uri: http://ugene.unipro.ru name: ugene-data section: universe/science version: 1.9.8+repack-0ubuntu3 unattended-upgrades: description: automatic installation of security upgrades downloads: 6200 homepage_uri: '' name: unattended-upgrades section: admin version: '0.76' unity: description: Interface designed for efficiency of space and interaction. downloads: 0 homepage_uri: https://launchpad.net/unity name: unity section: gnome version: 5.12-0ubuntu1.1 unity-2d: description: Unity interface for non-accelerated graphics cards downloads: 0 homepage_uri: '' name: unity-2d section: x11 version: 5.12.0-0ubuntu1.1 unity-2d-common: description: Common files for Unity 2D Shell downloads: 0 homepage_uri: '' name: unity-2d-common section: x11 version: 5.12.0-0ubuntu1.1 unity-2d-panel: description: Unity 2D Panel downloads: 0 homepage_uri: '' name: unity-2d-panel section: x11 version: 5.12.0-0ubuntu1.1 unity-2d-shell: description: Dash and Launcher for the Unity 2D environment downloads: 0 homepage_uri: '' name: unity-2d-shell section: x11 version: 5.12.0-0ubuntu1.1 unity-2d-spread: description: Unity 2D Spread downloads: 0 homepage_uri: '' name: unity-2d-spread section: x11 version: 5.12.0-0ubuntu1.1 unity-asset-pool: description: Unity Assets Pool downloads: 0 homepage_uri: https://launchpad.net/unity-asset-pool name: unity-asset-pool section: gnome version: 0.8.23-0ubuntu1 unity-common: description: Common files for the Unity interface. downloads: 0 homepage_uri: https://launchpad.net/unity name: unity-common section: gnome version: 5.12-0ubuntu1.1 unity-greeter: description: Unity Greeter downloads: 1 homepage_uri: https://launchpad.net/unity-greeter name: unity-greeter section: x11 version: 0.2.8-0ubuntu1.1 unity-lens-applications: description: Application lens for unity downloads: 0 homepage_uri: https://launchpad.net/unity-lens-applications name: unity-lens-applications section: gnome version: 5.12.0-0ubuntu1 unity-lens-files: description: File lens for unity downloads: 0 homepage_uri: https://launchpad.net/unity-lens-files name: unity-lens-files section: gnome version: 5.10.0-0ubuntu1 unity-lens-music: description: Music lens for unity downloads: 0 homepage_uri: https://launchpad.net/unity-lens-music name: unity-lens-music section: gnome version: 5.12.0-0ubuntu1 unity-lens-video: description: Unity Video lens downloads: 0 homepage_uri: https://launchpad.net/unity-lens-videos name: unity-lens-video section: gnome version: 0.3.5-0ubuntu1 unity-scope-musicstores: description: Store music lens for unity downloads: 0 homepage_uri: https://launchpad.net/unity-lens-music name: unity-scope-musicstores section: gnome version: 5.12.0-0ubuntu1 unity-scope-video-remote: description: Remote videos engine downloads: 0 homepage_uri: https://launchpad.net/unity-lens-videos name: unity-scope-video-remote section: gnome version: 0.3.5-0ubuntu2 unity-services: description: Services for the Unity interface downloads: 0 homepage_uri: https://launchpad.net/unity name: unity-services section: gnome version: 5.12-0ubuntu1.1 uno-libs3: description: LibreOffice UNO runtime environment -- public shared libraries downloads: 26384 homepage_uri: http://www.libreoffice.org name: uno-libs3 section: libs version: 3.5.3-0ubuntu1 unzip: description: De-archiver for .zip files downloads: 31921 homepage_uri: http://www.info-zip.org/UnZip.html name: unzip section: utils version: 6.0-4ubuntu1 update-inetd: description: inetd configuration file updater downloads: 39700 homepage_uri: '' name: update-inetd section: admin version: '4.41' update-manager: description: GNOME application that manages apt updates downloads: 396 homepage_uri: '' name: update-manager section: gnome version: 1:0.156.14.5 update-manager-core: description: manage release upgrades downloads: 12973 homepage_uri: '' name: update-manager-core section: admin version: 1:0.156.14.5 update-notifier: description: Daemon which notifies about package updates downloads: 21493 homepage_uri: '' name: update-notifier section: gnome version: 0.119ubuntu8.4 update-notifier-common: description: Files shared between update-notifier and other packages downloads: 27885 homepage_uri: '' name: update-notifier-common section: gnome version: 0.119ubuntu8.4 upower: description: abstraction for power management downloads: 35238 homepage_uri: http://upower.freedesktop.org/ name: upower section: admin version: 0.9.15-3git1 upstart: description: event-based init daemon downloads: 131 homepage_uri: http://upstart.ubuntu.com/ name: upstart section: admin version: 1.5-0ubuntu7 ure: description: LibreOffice UNO runtime environment downloads: 27417 homepage_uri: http://www.libreoffice.org name: ure section: libs version: 3.5.3-0ubuntu1 ureadahead: description: Read required files in advance downloads: 21 homepage_uri: '' name: ureadahead section: admin version: 0.100.0-12 usb-creator-common: description: create a startup disk using a CD or disc image (common files) downloads: 3 homepage_uri: '' name: usb-creator-common section: admin version: 0.2.38 usb-creator-gtk: description: create a startup disk using a CD or disc image (for GNOME) downloads: 17 homepage_uri: '' name: usb-creator-gtk section: admin version: 0.2.38 usb-modeswitch: description: mode switching tool for controlling "flip flop" USB devices downloads: 8917 homepage_uri: http://www.draisberghof.de/usb_modeswitch/ name: usb-modeswitch section: comm version: 1.2.3+repack0-1ubuntu2 usb-modeswitch-data: description: mode switching data for usb-modeswitch downloads: 12921 homepage_uri: http://www.draisberghof.de/usb_modeswitch/ name: usb-modeswitch-data section: comm version: 20120120-0ubuntu1 usbmuxd: description: USB multiplexor daemon for iPhone and iPod Touch devices downloads: 22142 homepage_uri: http://marcansoft.com/blog/iphonelinux/usbmuxd/ name: usbmuxd section: utils version: 1.0.7-2 usbutils: description: Linux USB utilities downloads: 33299 homepage_uri: '' name: usbutils section: utils version: 1:005-1 util-linux: description: Miscellaneous system utilities downloads: 110495 homepage_uri: http://userweb.kernel.org/~kzak/util-linux/ name: util-linux section: utils version: 2.20.1-1ubuntu3 uuid-runtime: description: runtime components for the Universally Unique ID library downloads: 1620 homepage_uri: http://userweb.kernel.org/~kzak/util-linux/ name: uuid-runtime section: libs version: 2.20.1-1ubuntu3 vbetool: description: run real-mode video BIOS code to alter hardware state downloads: 4680 homepage_uri: http://www.codon.org.uk/~mjg59/vbetool/ name: vbetool section: utils version: 1.1-2ubuntu1 vgabios: description: VGA BIOS software for the Bochs and Qemu emulated VGA card downloads: 0 homepage_uri: http://savannah.nongnu.org/projects/vgabios/ name: vgabios section: misc version: 0.6c-2ubuntu3 vim: description: Vi IMproved - enhanced vi editor downloads: 20043 homepage_uri: http://www.vim.org/ name: vim section: editors version: 2:7.3.429-2ubuntu2.1 vim-common: description: Vi IMproved - Common files downloads: 45172 homepage_uri: http://www.vim.org/ name: vim-common section: editors version: 2:7.3.429-2ubuntu2.1 vim-runtime: description: Vi IMproved - Runtime files downloads: 3771 homepage_uri: http://www.vim.org/ name: vim-runtime section: editors version: 2:7.3.429-2ubuntu2.1 vim-tiny: description: Vi IMproved - enhanced vi editor - compact version downloads: 13838 homepage_uri: http://www.vim.org/ name: vim-tiny section: editors version: 2:7.3.429-2ubuntu2.1 vino: description: VNC server for GNOME downloads: 7110 homepage_uri: http://live.gnome.org/Vino name: vino section: gnome version: 3.4.2-0ubuntu1 virtualbox-4.1: description: Oracle VM VirtualBox downloads: 3593 homepage_uri: '' name: virtualbox-4.1 section: contrib/misc version: 4.1.16-78094~Ubuntu~precise w3m: description: WWW browsable pager with excellent tables/frames support downloads: 29419 homepage_uri: http://sourceforge.net/projects/w3m name: w3m section: text version: 0.5.3-5ubuntu1 watershed: description: reduce superfluous executions of idempotent command downloads: 4 homepage_uri: '' name: watershed section: admin version: '6' wdiff: description: Compares two files word by word downloads: 1669 homepage_uri: http://www.gnu.org/software/wdiff/ name: wdiff section: text version: 0.6.5-1 wget: description: retrieves files from the web downloads: 33615 homepage_uri: http://www.gnu.org/software/wget/ name: wget section: web version: 1.13.4-2ubuntu1 whiptail: description: Displays user-friendly dialog boxes from shell scripts downloads: 51026 homepage_uri: https://fedorahosted.org/newt/ name: whiptail section: utils version: 0.52.11-2ubuntu10 whois: description: intelligent WHOIS client downloads: 12086 homepage_uri: '' name: whois section: net version: 5.0.15ubuntu2 whoopsie: description: Ubuntu crash database submission daemon downloads: 1 homepage_uri: http://wiki.ubuntu.com/ErrorTracker name: whoopsie section: utils version: 0.1.32 wireless-regdb: description: wireless regulatory database downloads: 9537 homepage_uri: http://wireless.kernel.org/en/developers/Regulatory/#Theregulatorydatabase name: wireless-regdb section: net version: 2011.04.28-1ubuntu3 wireless-tools: description: Tools for manipulating Linux Wireless Extensions downloads: 9059 homepage_uri: http://www.hpl.hp.com/personal/Jean_Tourrilhes/Linux/Tools.html name: wireless-tools section: net version: 30~pre9-5ubuntu2 wodim: description: command line CD/DVD writing tool downloads: 4799 homepage_uri: '' name: wodim section: otherosfs version: 9:1.1.11-2ubuntu2 wpasupplicant: description: client support for WPA and WPA2 (IEEE 802.11i) downloads: 42188 homepage_uri: http://w1.fi/wpa_supplicant/ name: wpasupplicant section: net version: 0.7.3-6ubuntu2 x11-apps: description: X applications downloads: 6228 homepage_uri: '' name: x11-apps section: x11 version: 7.6+5ubuntu1 x11-common: description: X Window System (X.Org) infrastructure downloads: 54261 homepage_uri: '' name: x11-common section: x11 version: 1:7.6+12ubuntu1 x11-session-utils: description: X session utilities downloads: 4761 homepage_uri: '' name: x11-session-utils section: x11 version: 7.6+2 x11-utils: description: X11 utilities downloads: 34280 homepage_uri: '' name: x11-utils section: x11 version: 7.6+4 x11-xfs-utils: description: X font server utilities downloads: 4727 homepage_uri: '' name: x11-xfs-utils section: x11 version: 7.6+1 x11-xkb-utils: description: X11 XKB utilities downloads: 46803 homepage_uri: '' name: x11-xkb-utils section: x11 version: 7.6+4 x11-xserver-utils: description: X server utilities downloads: 44459 homepage_uri: '' name: x11-xserver-utils section: x11 version: 7.6+3 x11proto-composite-dev: description: X11 Composite extension wire protocol downloads: 1215 homepage_uri: '' name: x11proto-composite-dev section: x11 version: 1:0.4.2-2 x11proto-core-dev: description: X11 core wire protocol and auxiliary headers downloads: 3904 homepage_uri: '' name: x11proto-core-dev section: x11 version: 7.0.22-1 x11proto-damage-dev: description: X11 Damage extension wire protocol downloads: 1098 homepage_uri: '' name: x11proto-damage-dev section: x11 version: 1:1.2.1-2 x11proto-fixes-dev: description: X11 Fixes extension wire protocol downloads: 1363 homepage_uri: '' name: x11proto-fixes-dev section: x11 version: 1:5.0-2ubuntu1 x11proto-input-dev: description: X11 Input extension wire protocol downloads: 2528 homepage_uri: '' name: x11proto-input-dev section: x11 version: 2.1.99.6-1 x11proto-kb-dev: description: X11 XKB extension wire protocol downloads: 2051 homepage_uri: '' name: x11proto-kb-dev section: x11 version: 1.0.5-2 x11proto-randr-dev: description: X11 RandR extension wire protocol downloads: 1617 homepage_uri: '' name: x11proto-randr-dev section: x11 version: 1.4.0+git20101207.0d32bb07-0ubuntu2 x11proto-render-dev: description: X11 Render extension wire protocol downloads: 1875 homepage_uri: '' name: x11proto-render-dev section: x11 version: 2:0.11.1-2 x11proto-scrnsaver-dev: description: X11 Screen Saver extension wire protocol downloads: 356 homepage_uri: '' name: x11proto-scrnsaver-dev section: x11 version: 1.2.1-2 x11proto-xext-dev: description: X11 various extension wire protocol downloads: 2139 homepage_uri: '' name: x11proto-xext-dev section: x11 version: 7.2.0-3 x11proto-xinerama-dev: description: X11 Xinerama extension wire protocol downloads: 1109 homepage_uri: '' name: x11proto-xinerama-dev section: x11 version: 1.2.1-2 xauth: description: X authentication utility downloads: 38431 homepage_uri: '' name: xauth section: x11 version: 1:1.0.6-1 xaw3dg:i386: description: Xaw3d widget set downloads: 0 homepage_uri: '' name: xaw3dg:i386 section: x11 version: 1.5+E-18.1ubuntu1 xbitmaps: description: Base X bitmaps downloads: 13830 homepage_uri: '' name: xbitmaps section: x11 version: 1.1.1-1 xcursor-themes: description: Base X cursor themes downloads: 0 homepage_uri: '' name: xcursor-themes section: x11 version: 1.0.3-1 xdg-user-dirs: description: tool to manage well known user directories downloads: 31305 homepage_uri: http://www.freedesktop.org/wiki/Software/xdg-user-dirs name: xdg-user-dirs section: utils version: 0.14-0ubuntu2 xdg-user-dirs-gtk: description: tool to manage well known user directories (Gtk extension) downloads: 22352 homepage_uri: http://www.freedesktop.org/wiki/Software/xdg-user-dirs name: xdg-user-dirs-gtk section: utils version: 0.9-0ubuntu1 xdg-utils: description: desktop integration utilities from freedesktop.org downloads: 29617 homepage_uri: http://portland.freedesktop.org/ name: xdg-utils section: utils version: 1.1.0~rc1-2ubuntu6 xdiagnose: description: X.org diagnosis tool downloads: 0 homepage_uri: http://wiki.ubuntu.com/X name: xdiagnose section: python version: '2.5' xfonts-base: description: standard fonts for X downloads: 2 homepage_uri: '' name: xfonts-base section: x11 version: 1:1.0.3 xfonts-encodings: description: Encodings for X.Org fonts downloads: 4 homepage_uri: '' name: xfonts-encodings section: x11 version: 1:1.0.4-1ubuntu1 xfonts-mathml: description: Type1 Symbol font for MathML downloads: 2 homepage_uri: '' name: xfonts-mathml section: x11 version: 4ubuntu1 xfonts-scalable: description: scalable fonts for X downloads: 2 homepage_uri: '' name: xfonts-scalable section: x11 version: 1:1.0.3-1 xfonts-utils: description: X Window System font utility programs downloads: 18611 homepage_uri: '' name: xfonts-utils section: x11 version: 1:7.6+1 xfsprogs: description: Utilities for managing the XFS filesystem downloads: 3177 homepage_uri: http://oss.sgi.com/projects/xfs/ name: xfsprogs section: admin version: 3.1.7 xinit: description: X server initialisation tool downloads: 10127 homepage_uri: '' name: xinit section: x11 version: 1.3.1-1 xinput: description: Runtime configuration and test of XInput devices downloads: 2092 homepage_uri: '' name: xinput section: x11 version: 1.5.99.1-0ubuntu2 xkb-data: description: X Keyboard Extension (XKB) configuration data downloads: 55581 homepage_uri: http://www.freedesktop.org/Software/XKeyboardConfig name: xkb-data section: x11 version: 2.5-1ubuntu1 xml-core: description: XML infrastructure and XML catalog file support downloads: 12520 homepage_uri: '' name: xml-core section: text version: '0.13' xorg: description: X.Org X Window System downloads: 0 homepage_uri: '' name: xorg section: x11 version: 1:7.6+12ubuntu1 xorg-docs-core: description: Core documentation for the X.org X Window System downloads: 0 homepage_uri: '' name: xorg-docs-core section: x11 version: 1:1.6-1ubuntu2 xorg-sgml-doctools: description: Common tools for building X.Org SGML documentation downloads: 1751 homepage_uri: '' name: xorg-sgml-doctools section: x11 version: 1:1.10-1 xserver-common: description: common files used by various X servers downloads: 42289 homepage_uri: '' name: xserver-common section: x11 version: 2:1.11.4-0ubuntu10.2 xserver-xorg: description: X.Org X server downloads: 20883 homepage_uri: '' name: xserver-xorg section: x11 version: 1:7.6+12ubuntu1 xserver-xorg-core: description: Xorg X server - core server downloads: 45750 homepage_uri: '' name: xserver-xorg-core section: x11 version: 2:1.11.4-0ubuntu10.2 xserver-xorg-input-all: description: X.Org X server -- input driver metapackage downloads: 1 homepage_uri: '' name: xserver-xorg-input-all section: x11 version: 1:7.6+12ubuntu1 xserver-xorg-input-evdev: description: X.Org X server -- evdev input driver downloads: 41801 homepage_uri: '' name: xserver-xorg-input-evdev section: x11 version: 1:2.7.0-0ubuntu1 xserver-xorg-input-mouse: description: X.Org X server -- mouse input driver downloads: 3523 homepage_uri: '' name: xserver-xorg-input-mouse section: x11 version: 1:1.7.1-1build3 xserver-xorg-input-synaptics: description: Synaptics TouchPad driver for X.Org server downloads: 16932 homepage_uri: '' name: xserver-xorg-input-synaptics section: x11 version: 1.6.0-0ubuntu1~precise1 xserver-xorg-input-vmmouse: description: X.Org X server -- VMMouse input driver to use with VMWare downloads: 331 homepage_uri: '' name: xserver-xorg-input-vmmouse section: x11 version: 1:12.8.0-1 xserver-xorg-input-wacom: description: X.Org X server -- Wacom input driver downloads: 18339 homepage_uri: http://linuxwacom.sf.net name: xserver-xorg-input-wacom section: x11 version: 1:0.14.0-0ubuntu2 xserver-xorg-video-all: description: X.Org X server -- output driver metapackage downloads: 0 homepage_uri: '' name: xserver-xorg-video-all section: x11 version: 1:7.6+12ubuntu1 xserver-xorg-video-ati: description: X.Org X server -- AMD/ATI display driver wrapper downloads: 8047 homepage_uri: '' name: xserver-xorg-video-ati section: x11 version: 1:6.14.99~git20111219.aacbd629-0ubuntu2 xserver-xorg-video-cirrus: description: X.Org X server -- Cirrus display driver downloads: 1448 homepage_uri: '' name: xserver-xorg-video-cirrus section: x11 version: 1:1.3.2-4build1 xserver-xorg-video-fbdev: description: X.Org X server -- fbdev display driver downloads: 29638 homepage_uri: '' name: xserver-xorg-video-fbdev section: x11 version: 1:0.4.2-4ubuntu2 xserver-xorg-video-intel: description: X.Org X server -- Intel i8xx, i9xx display driver downloads: 15444 homepage_uri: '' name: xserver-xorg-video-intel section: x11 version: 2:2.17.0-1ubuntu4 xserver-xorg-video-mach64: description: X.Org X server -- ATI Mach64 display driver downloads: 1647 homepage_uri: '' name: xserver-xorg-video-mach64 section: x11 version: 6.9.0-1build2 xserver-xorg-video-mga: description: X.Org X server -- MGA display driver downloads: 1721 homepage_uri: '' name: xserver-xorg-video-mga section: x11 version: 1:1.4.13.dfsg-4build2 xserver-xorg-video-neomagic: description: X.Org X server -- Neomagic display driver downloads: 1294 homepage_uri: '' name: xserver-xorg-video-neomagic section: x11 version: 1:1.2.5-2build2 xserver-xorg-video-nouveau: description: X.Org X server -- Nouveau display driver downloads: 6573 homepage_uri: http://nouveau.freedesktop.org/wiki/ name: xserver-xorg-video-nouveau section: x11 version: 1:0.0.16+git20111201+b5534a1-1build2 xserver-xorg-video-openchrome: description: X.Org X server -- VIA display driver downloads: 1779 homepage_uri: http://www.openchrome.org name: xserver-xorg-video-openchrome section: x11 version: 1:0.2.904+svn1050-1 xserver-xorg-video-qxl: description: X.Org X server -- QXL display driver downloads: 12 homepage_uri: http://spice-space.org/ name: xserver-xorg-video-qxl section: x11 version: 0.0.16-2 xserver-xorg-video-r128: description: X.Org X server -- ATI r128 display driver downloads: 1402 homepage_uri: '' name: xserver-xorg-video-r128 section: x11 version: 6.8.1-5build2 xserver-xorg-video-radeon: description: X.Org X server -- AMD/ATI Radeon display driver downloads: 9028 homepage_uri: '' name: xserver-xorg-video-radeon section: x11 version: 1:6.14.99~git20111219.aacbd629-0ubuntu2 xserver-xorg-video-s3: description: X.Org X server -- legacy S3 display driver downloads: 1269 homepage_uri: '' name: xserver-xorg-video-s3 section: x11 version: 1:0.6.3-4build2 xserver-xorg-video-savage: description: X.Org X server -- Savage display driver downloads: 1489 homepage_uri: '' name: xserver-xorg-video-savage section: x11 version: 1:2.3.3-1ubuntu1 xserver-xorg-video-siliconmotion: description: X.Org X server -- SiliconMotion display driver downloads: 1257 homepage_uri: '' name: xserver-xorg-video-siliconmotion section: x11 version: 1:1.7.5-1build2 xserver-xorg-video-sis: description: X.Org X server -- SiS display driver downloads: 1660 homepage_uri: '' name: xserver-xorg-video-sis section: x11 version: 1:0.10.3-3build2 xserver-xorg-video-sisusb: description: X.Org X server -- SiS USB display driver downloads: 1278 homepage_uri: '' name: xserver-xorg-video-sisusb section: x11 version: 1:0.9.4-2build2 xserver-xorg-video-tdfx: description: X.Org X server -- tdfx display driver downloads: 1252 homepage_uri: '' name: xserver-xorg-video-tdfx section: x11 version: 1:1.4.3-4build2 xserver-xorg-video-trident: description: X.Org X server -- Trident display driver downloads: 1297 homepage_uri: '' name: xserver-xorg-video-trident section: x11 version: 1:1.3.4-2build2 xserver-xorg-video-vesa: description: X.Org X server -- VESA display driver downloads: 29945 homepage_uri: '' name: xserver-xorg-video-vesa section: x11 version: 1:2.3.0-7build2 xserver-xorg-video-vmware: description: X.Org X server -- VMware display driver downloads: 3528 homepage_uri: '' name: xserver-xorg-video-vmware section: x11 version: 1:12.0.1-1ubuntu1.1 xterm: description: X terminal emulator downloads: 14144 homepage_uri: http://invisible-island.net/xterm/xterm.html name: xterm section: x11 version: 271-1ubuntu2 xtrans-dev: description: X transport library (development files) downloads: 1210 homepage_uri: '' name: xtrans-dev section: x11 version: 1.2.6-2 xul-ext-ubufox: description: Ubuntu-specific configuration defaults and apt support for Firefox downloads: 0 homepage_uri: https://launchpad.net/ubufox name: xul-ext-ubufox section: web version: 2.0.3-0ubuntu1 xvfb: description: Virtual Framebuffer 'fake' X server downloads: 479 homepage_uri: '' name: xvfb section: x11 version: 2:1.11.4-0ubuntu10.2 xz-lzma: description: XZ-format compression utilities - compatibility commands downloads: 4344 homepage_uri: http://tukaani.org/xz/ name: xz-lzma section: utils version: 5.1.1alpha+20110809-3 xz-utils: description: XZ-format compression utilities downloads: 21528 homepage_uri: http://tukaani.org/xz/ name: xz-utils section: utils version: 5.1.1alpha+20110809-3 yelp: description: Help browser for GNOME downloads: 8904 homepage_uri: http://live.gnome.org/Yelp name: yelp section: gnome version: 3.4.1-0ubuntu1 yelp-xsl: description: XSL stylesheets for the yelp help browser downloads: 3019 homepage_uri: '' name: yelp-xsl section: gnome version: 3.4.1-1 zeitgeist: description: event logging framework downloads: 87 homepage_uri: http://zeitgeist-project.com/ name: zeitgeist section: utils version: 0.9.0-1ubuntu1 zeitgeist-core: description: event logging framework - engine downloads: 1488 homepage_uri: http://zeitgeist-project.com/ name: zeitgeist-core section: utils version: 0.9.0-1ubuntu1 zeitgeist-datahub: description: event logging framework - passive logging daemon downloads: 642 homepage_uri: https://launchpad.net/zeitgeist-datahub name: zeitgeist-datahub section: utils version: 0.8.2-1ubuntu2 zenity: description: Display graphical dialog boxes from shell scripts downloads: 15112 homepage_uri: http://live.gnome.org/Zenity name: zenity section: gnome version: 3.4.0-0ubuntu4 zenity-common: description: Display graphical dialog boxes from shell scripts (common files) downloads: 4902 homepage_uri: http://live.gnome.org/Zenity name: zenity-common section: gnome version: 3.4.0-0ubuntu4 zip: description: Archiver for .zip files downloads: 7573 homepage_uri: http://www.info-zip.org/Zip.html name: zip section: utils version: 3.0-4 zlib1g: description: compression library - runtime downloads: 110281 homepage_uri: http://zlib.net/ name: zlib1g section: libs version: 1:1.2.3.4.dfsg-3ubuntu4 zlib1g-dev: description: compression library - development downloads: 9744 homepage_uri: http://zlib.net/ name: zlib1g-dev section: libdevel version: 1:1.2.3.4.dfsg-3ubuntu4 zlib1g:i386: description: compression library - runtime downloads: 0 homepage_uri: http://zlib.net/ name: zlib1g:i386 section: libs version: 1:1.2.3.4.dfsg-3ubuntu4 zsh: description: shell with lots of features downloads: 3830 homepage_uri: http://www.zsh.org/ name: zsh section: shells version: 4.3.17-1ubuntu1 ================================================ FILE: manifest/python-packages.yaml ================================================ BeautifulSoup: description: HTML/XML parser for quick-turnaround applications like screen-scraping. homepage_uri: http://www.crummy.com/software/BeautifulSoup/ name: BeautifulSoup version: 3.2.1 Brlapi: description: Python BrlAPI homepage_uri: http://mielke.cc/brltty/ name: Brlapi version: 0.5.6 Cheetah: description: Cheetah is a template engine and code generation tool. homepage_uri: http://www.cheetahtemplate.org/ name: Cheetah version: 2.4.4 CouchDB: description: Python library for working with CouchDB homepage_uri: http://code.google.com/p/couchdb-python/ name: CouchDB version: '0.8' Cython: description: The Cython compiler for writing C extensions for the Python language. homepage_uri: http://www.cython.org name: Cython version: '0.16' DendroPy: description: 'A Python library for phylogenetics and phylogenetic computing: reading, writing, simulation, processing and manipulation of phylogenetic trees (phylogenies) and characters.' homepage_uri: http://packages.python.org/DendroPy/ name: DendroPy version: 3.11.0 Fabric: description: Fabric is a simple, Pythonic tool for remote execution and deployment. homepage_uri: http://fabfile.org name: Fabric version: 1.4.2 GnuPGInterface: description: GnuPG interactions with file handles homepage_uri: http://py-gnupg.sourceforge.net/ name: GnuPGInterface version: 0.3.2 HTSeq: description: A framework to process and analyze data from high-throughput sequencing (HTS) assays homepage_uri: http://www-huber.embl.de/users/anders/HTSeq/ name: HTSeq version: 0.5.3p7 Jinja2: description: A small but fast and easy to use stand-alone template engine written in pure python. homepage_uri: http://jinja.pocoo.org/ name: Jinja2 version: '2.6' Landscape Client: description: Landscape Client homepage_uri: http://landscape.canonical.com name: Landscape Client version: 12.04.2 Logbook: description: A logging replacement for Python homepage_uri: http://logbook.pocoo.org/ name: Logbook version: '0.3' M2Crypto: description: 'M2Crypto: A Python crypto and SSL toolkit' homepage_uri: http://chandlerproject.org/Projects/MeTooCrypto name: M2Crypto version: 0.21.1 MACS: description: Model Based Analysis for ChIP-Seq data homepage_uri: http://liulab.dfci.harvard.edu/MACS/ name: MACS version: 1.4.2 Magic file extensions: description: libmagic Python bindings homepage_uri: UNKNOWN name: Magic file extensions version: '0.2' Mako: description: A super-fast templating language that borrows the best ideas from the existing templating languages. homepage_uri: http://www.makotemplates.org/ name: Mako version: 0.7.0 MarkupSafe: description: Implements a XML/HTML/XHTML Markup safe string for Python homepage_uri: http://dev.pocoo.org/ name: MarkupSafe version: '0.15' MySQL-python: description: Python interface to MySQL homepage_uri: http://sourceforge.net/projects/mysql-python name: MySQL-python version: 1.2.3 PAM: description: Python bindings for PAM homepage_uri: UNKNOWN name: PAM version: 0.4.2 PIL: description: Python Imaging Library homepage_uri: http://www.pythonware.com/products/pil name: PIL version: 1.1.7 Paste: description: Tools for using a Web Server Gateway Interface stack homepage_uri: http://pythonpaste.org name: Paste version: 1.7.5.1 PasteDeploy: description: Load, configure, and compose WSGI applications and servers homepage_uri: http://pythonpaste.org/deploy/ name: PasteDeploy version: 1.5.0 PasteScript: description: A pluggable command-line frontend, including commands to setup package file layouts homepage_uri: http://pythonpaste.org/script/ name: PasteScript version: 1.7.5 PyVCF: description: Variant Call Format (VCF) parser for Python homepage_uri: https://github.com/jamescasbon/PyVCF name: PyVCF version: 0.4.6 PyYAML: description: YAML parser and emitter for Python homepage_uri: http://pyyaml.org/wiki/PyYAML name: PyYAML version: '3.10' Pygments: description: Pygments is a syntax highlighting package written in Python. homepage_uri: http://pygments.org/ name: Pygments version: '1.5' Pyrex-real: description: UNKNOWN homepage_uri: http://www.cosc.canterbury.ac.nz/~greg/python/Pyrex/ name: Pyrex version: 0.9.9 Pyste: description: Pyste - Python Semi-Automatic Exporter homepage_uri: http://www.boost.org/libs/python/pyste/index.html name: Pyste version: 0.9.10 Python: description: A high-level object-oriented programming language homepage_uri: http://www.python.org/2.7 name: Python version: 2.7.3 Routes: description: Routing Recognition and Generation Tools homepage_uri: http://routes.groovie.org/ name: Routes version: 1.12.3 SQLAlchemy: description: Database Abstraction Library homepage_uri: http://www.sqlalchemy.org name: SQLAlchemy version: 0.7.8 Sphinx: description: Python documentation generator homepage_uri: http://sphinx.pocoo.org/ name: Sphinx version: 1.1.3 StarCluster: description: StarCluster is a utility for creating and managing computing clusters hosted on Amazon's Elastic Compute Cloud (EC2). homepage_uri: http://web.mit.edu/starcluster name: StarCluster version: 0.93.3 Twisted: description: An asynchronous networking framework written in Python homepage_uri: http://twistedmatrix.com/ name: Twisted version: 12.1.0 Twisted Core: description: The core parts of the Twisted networking framework homepage_uri: http://twistedmatrix.com/ name: Twisted Core version: 11.1.0 Twisted Names: description: A Twisted DNS implementation. homepage_uri: http://twistedmatrix.com/trac/wiki/TwistedNames name: Twisted Names version: 11.1.0 Twisted Web: description: Twisted web server, programmable in Python. homepage_uri: http://twistedmatrix.com/trac/wiki/TwistedWeb name: Twisted Web version: 11.1.0 VTK: description: The Visualization Toolkit homepage_uri: http://www.vtk.org/ name: VTK version: 5.8.0 WebHelpers: description: Web Helpers homepage_uri: http://webhelpers.groovie.org/ name: WebHelpers version: '1.3' WebOb: description: WSGI request and response object homepage_uri: http://webob.org/ name: WebOb version: 1.2b2 adium-theme-ubuntu: description: UNKNOWN homepage_uri: UNKNOWN name: adium-theme-ubuntu version: 0.3.2 amqplib: description: AMQP Client Library homepage_uri: http://code.google.com/p/py-amqplib/ name: amqplib version: 1.0.2 anyjson: description: Wraps the best available JSON implementation available in a common interface homepage_uri: http://bitbucket.org/runeh/anyjson name: anyjson version: 0.3.1 apptools: description: application tools homepage_uri: http://code.enthought.com/projects/app_tools.php name: apptools version: 4.0.0 apt-xapian-index: description: Xapian index of Debian packages homepage_uri: http://www.enricozini.org/sw/apt-xapian-index/ name: apt-xapian-index version: '0.44' apturl: description: UNKNOWN homepage_uri: UNKNOWN name: apturl version: 0.5.1ubuntu3 argparse: description: Python command-line parsing library homepage_uri: '' name: argparse version: 1.2.1 bcbio-nextgen: description: Automated, distributed next-gen sequencing pipeline; includes Galaxy interaction homepage_uri: https://github.com/chapmanb/bcbb/tree/master/nextgen name: bcbio-nextgen version: '0.2' biopython: description: Freely available tools for computational molecular biology. homepage_uri: http://www.biopython.org/ name: biopython version: '1.59' boto: description: Amazon Web Services Library homepage_uri: https://github.com/boto/boto/ name: boto version: 2.5.1 bx-python: description: Tools for manipulating biological data, particularly multiple sequence alignments homepage_uri: http://bitbucket.org/james_taylor/bx-python/wiki/Home name: bx-python version: 0.7.1 bzr: description: Friendly distributed version control system homepage_uri: http://bazaar.canonical.com/ name: bzr version: 2.5.0 celery: description: Distributed Task Queue homepage_uri: http://celeryproject.org name: celery version: 2.5.5 chardet: description: Universal encoding detector homepage_uri: http://chardet.feedparser.org/ name: chardet version: 2.0.1 cloud-init: description: EC2 initialisation magic homepage_uri: http://launchpad.net/cloud-init/ name: cloud-init version: 0.6.3 command-not-found: description: UNKNOWN homepage_uri: UNKNOWN name: command-not-found version: 0.2.44 configglue: description: Glue to stick OptionParser and ConfigParser together homepage_uri: https://launchpad.net/configglue name: configglue version: '1.0' configobj: description: Config file reading, writing and validation. homepage_uri: http://www.voidspace.org.uk/python/configobj.html name: configobj version: 4.7.2 debtagshw: description: UNKNOWN homepage_uri: UNKNOWN name: debtagshw version: '0.1' decorator: description: Better living through Python with decorators homepage_uri: http://pypi.python.org/pypi/decorator name: decorator version: 3.3.1 defer: description: Simple framework for asynchronous programming homepage_uri: http://launchpad.net/python-defer name: defer version: 1.0.2 devscripts: description: UNKNOWN homepage_uri: UNKNOWN name: devscripts version: 2.11.6ubuntu1.2 dirspec: description: XDG Base and User directories implementation homepage_uri: https://launchpad.net/dirspec name: dirspec version: 3.0.0 distribute: description: Easily download, build, install, upgrade, and uninstall Python packages homepage_uri: http://packages.python.org/distribute name: distribute version: 0.6.19 docutils: description: Docutils -- Python Documentation Utilities homepage_uri: http://docutils.sourceforge.net/ name: docutils version: 0.9.1 duplicity: description: Encrypted backup using rsync algorithm homepage_uri: http://duplicity.nongnu.org/index.html name: duplicity version: 0.6.18 envisage: description: extensible application framework homepage_uri: http://code.enthought.com/projects/envisage/ name: envisage version: 4.0.0 euca2ools: description: Elastic Utility Computing Architecture Command Line Tools homepage_uri: http://open.eucalyptus.com name: euca2ools version: 2.0.0 gnuplot-py: description: A Python interface to the gnuplot plotting program. homepage_uri: http://gnuplot-py.sourceforge.net name: gnuplot-py version: '1.8' h5py: description: Read and write HDF5 files from Python homepage_uri: http://h5py.alfven.org name: h5py version: 2.0.1 httplib2: description: A comprehensive HTTP client library. homepage_uri: http://code.google.com/p/httplib2/ name: httplib2 version: 0.7.2 ipython: description: 'IPython: Productive Interactive Computing' homepage_uri: http://ipython.org name: ipython version: 0.12.1 jockey: description: UI for managing third-party and non-free drivers homepage_uri: https://launchpad.net/jockey name: jockey version: 0.9.7 keyring: description: Store and access your passwords safely. homepage_uri: http://bitbucket.org/kang/python-keyring-lib name: keyring version: 0.7.1 kombu: description: Messaging Framework for Python homepage_uri: http://github.com/ask/kombu/ name: kombu version: 2.1.8 language-selector: description: UNKNOWN homepage_uri: UNKNOWN name: language-selector version: '0.1' launchpadlib: description: Script Launchpad through its web services interfaces. Officially supported. homepage_uri: https://help.launchpad.net/API/launchpadlib name: launchpadlib version: 1.9.12 lazr.restfulclient: description: A programmable client library that takes advantage of the commonalities among homepage_uri: https://launchpad.net/lazr.restfulclient name: lazr.restfulclient version: 0.12.0 lazr.uri: description: A self-contained, easily reusable library for parsing, manipulating, homepage_uri: https://launchpad.net/lazr.uri name: lazr.uri version: 1.0.3 louis: description: Python bindings for liblouis homepage_uri: '' name: louis version: 2.3.0 lxml: description: Powerful and Pythonic XML processing library combining libxml2/libxslt with the ElementTree API. homepage_uri: http://lxml.de/ name: lxml version: 2.3.4 matplotlib: description: Python plotting package homepage_uri: http://matplotlib.sourceforge.net name: matplotlib version: 1.0.1 mayavi: description: The Mayavi scientific data 3-dimensional visualizer. homepage_uri: http://code.enthought.com/projects/mayavi/ name: mayavi version: 4.0.0 mercurial: description: Scalable distributed SCM homepage_uri: http://mercurial.selenic.com/ name: mercurial version: 2.0.2 networkx: description: Python package for creating and manipulating graphs and networks homepage_uri: http://networkx.lanl.gov/ name: networkx version: 1.7rc1 nose: description: nose extends unittest to make testing easier homepage_uri: http://readthedocs.org/docs/nose/ name: nose version: 1.1.2 numpy: description: 'NumPy: array processing for numbers, strings, records, and objects.' homepage_uri: http://numpy.scipy.org name: numpy version: 1.6.2 nvidia-common: description: Find obsolete NVIDIA drivers homepage_uri: http://www.albertomilone.com name: nvidia-common version: 0.0.0 oauth: description: Library for OAuth version 1.0a. homepage_uri: http://code.google.com/p/oauth name: oauth version: 1.0.1 oca: description: Bindings for XMLRPC OpenNebula Cloud API homepage_uri: https://github.com/lukaszo/python-oca name: oca version: 0.2.3 onboard: description: Simple On-screen Keyboard homepage_uri: http://launchpad.net/onboard/ name: onboard version: 0.97.1 oneconf: description: UNKNOWN homepage_uri: UNKNOWN name: oneconf version: 0.2.8.1 pandas: description: Powerful data structures for data analysis and statistics homepage_uri: http://pandas.pydata.org name: pandas version: 0.7.3 paramiko: description: SSH2 protocol library homepage_uri: http://www.lag.net/paramiko/ name: paramiko version: 1.7.7.1 pexpect: description: Pexpect is a pure Python Expect. It allows easy control of other applications. homepage_uri: http://pexpect.sourceforge.net/ name: pexpect version: '2.3' pip: description: pip installs packages. Python packages. An easy_install replacement homepage_uri: http://www.pip-installer.org name: pip version: '1.1' piston-mini-client: description: A package to consume Django-Piston web services homepage_uri: https://launchpad.net/piston-mini-client name: piston-mini-client version: 0.7.2 protobuf: description: Protocol Buffers homepage_uri: http://code.google.com/p/protobuf/ name: protobuf version: 2.4.1 psycopg2: description: Python-PostgreSQL Database Adapter homepage_uri: http://initd.org/psycopg/ name: psycopg2 version: 2.4.5 pyOpenSSL: description: Python wrapper module around the OpenSSL library homepage_uri: http://pyopenssl.sourceforge.net/ name: pyOpenSSL version: '0.13' pyasn1: description: ASN.1 types and codecs homepage_uri: http://sourceforge.net/projects/pyasn1/ name: pyasn1 version: 0.0.13b pybedtools: description: Wrapper around BEDTools for bioinformatics work homepage_uri: none name: pybedtools version: 0.6.1 pycrypto: description: Cryptographic modules for Python. homepage_uri: http://www.pycrypto.org/ name: pycrypto version: '2.6' pycups: description: Python bindings for libcups homepage_uri: http://cyberelk.net/tim/software/pycups/ name: pycups version: 1.9.61 pycurl: description: PycURL -- cURL library module for Python homepage_uri: http://pycurl.sourceforge.net/ name: pycurl version: 7.19.0 pydoop: description: 'Pydoop: a Python MapReduce and HDFS API for Hadoop' homepage_uri: http://pydoop.sourceforge.net name: pydoop version: 0.6.0 pydot: description: Python interface to Graphviz's Dot homepage_uri: http://code.google.com/p/pydot/ name: pydot version: 1.0.28 pyface: description: traits-capable windowing framework homepage_uri: http://code.enthought.com/projects/traits_gui name: pyface version: 4.0.0 pygooglechart: description: A complete Python wrapper for the Google Chart API homepage_uri: http://pygooglechart.slowchop.com/ name: pygooglechart version: 0.3.0 pygr: description: Pygr, a Python graph-database toolkit oriented primarily on bioinformatics homepage_uri: http://code.google.com/p/pygr/ name: pygr version: 0.8.2 pyinotify: description: Linux filesystem events monitoring homepage_uri: http://github.com/seb-m/pyinotify name: pyinotify version: 0.9.2 pymongo: description: Python driver for MongoDB homepage_uri: http://github.com/mongodb/mongo-python-driver name: pymongo version: '2.2' pyparsing: description: Python parsing module homepage_uri: http://pyparsing.wikispaces.com/ name: pyparsing version: 1.5.6 pysam: description: pysam homepage_uri: http://code.google.com/p/pysam/ name: pysam version: '0.6' pyserial: description: Python Serial Port Extension homepage_uri: http://pyserial.sourceforge.net/ name: pyserial version: '2.5' pysmbc: description: Python bindings for libsmbclient homepage_uri: http://cyberelk.net/tim/software/pysmbc/ name: pysmbc version: 1.0.13 pysqlite: description: DB-API 2.0 interface for SQLite 3.x homepage_uri: http://pysqlite.googlecode.com/ name: pysqlite version: 2.6.3 python-apt: description: Python bindings for APT homepage_uri: UNKNOWN name: python-apt version: 0.8.3ubuntu7 python-cjson: description: Fast JSON encoder/decoder for Python homepage_uri: http://ag-projects.com/ name: python-cjson version: 1.0.5 python-dateutil: description: Extensions to the standard python 2.3+ datetime module homepage_uri: http://labix.org/python-dateutil name: python-dateutil version: '1.5' python-debian: description: Debian package related modules homepage_uri: http://packages.debian.org/sid/python-debian name: python-debian version: 0.1.21ubuntu1 python-distutils-extra: description: Add support for i18n, documentation and icons to distutils homepage_uri: UNKNOWN name: python-distutils-extra version: '2.32' python-virtkey: description: python extension for emulating keypresses and getting keyboard layout homepage_uri: https://launchpad.net/python-virtkey/ name: python-virtkey version: 0.60.0 pyxdg: description: PyXDG contains implementations of freedesktop.org standards in python. homepage_uri: http://cvs.freedesktop.org/cgi-bin/viewcvs.cgi/pyxdg/ name: pyxdg version: '0.19' pyzmq: description: Python bindings for 0MQ. homepage_uri: http://github.com/zeromq/pyzmq name: pyzmq version: 2.2.0 reportlab: description: The Reportlab Toolkit homepage_uri: http://www.reportlab.com/ name: reportlab version: '2.5' repoze.lru: description: A tiny LRU cache implementation and decorator homepage_uri: http://www.repoze.org name: repoze.lru version: '0.5' rhythmbox-ubuntuone: description: Ubuntu One Rhythmbox plug-in homepage_uri: https://launchpad.net/rhythmbox-ubuntuone name: rhythmbox-ubuntuone version: 3.0.0 rpy: description: Python interface to the R language homepage_uri: http://rpy.sourceforge.net name: rpy version: 1.0.3 rpy2: description: '' homepage_uri: '' name: rpy2 version: 2.2.6dev-20120618 rpy2-bioconductor-extensions: description: Bioconductor-specific extensions for rpy2 homepage_uri: no url yet name: rpy2-bioconductor-extensions version: 0.2.5-dev ruffus: description: Light-weight Python Computational Pipeline Management homepage_uri: http://ruffus.googlecode.com name: ruffus version: '2.2' scikits.learn: description: A set of python modules for machine learning and data mining homepage_uri: http://scikit-learn.sourceforge.net name: scikits.learn version: 0.8.1 scikits.statsmodels: description: Statistical computations and models for use with SciPy homepage_uri: http://statsmodels.sourceforge.net/ name: scikits.statsmodels version: 0.3.1 scipy: description: 'SciPy: Scientific Library for Python' homepage_uri: http://www.scipy.org name: scipy version: 0.10.1 seal: description: 'Seal: Sequence Alignment on Hadoop.' homepage_uri: http://www.crs4.it name: seal version: devel-20120618 sessioninstaller: description: APT based installer using PackageKit session DBus API homepage_uri: UNKNOWN name: sessioninstaller version: 0.0.0 setuptools: description: xxxx homepage_uri: xxx name: setuptools version: 0.6c11 simplejson: description: Simple, fast, extensible JSON encoder/decoder for Python homepage_uri: http://github.com/simplejson/simplejson name: simplejson version: 2.5.2 software-center-aptd-plugins: description: UNKNOWN homepage_uri: UNKNOWN name: software-center-aptd-plugins version: 0.0.0 ssh: description: SSH2 protocol library homepage_uri: UNKNOWN name: ssh version: 1.7.14 system-service: description: UNKNOWN homepage_uri: UNKNOWN name: system-service version: 0.1.6 tornado: description: Tornado is an open source version of the scalable, non-blocking web server and and tools that power FriendFeed homepage_uri: http://www.tornadoweb.org/ name: tornado version: '2.3' traits: description: explicitly typed attributes for Python homepage_uri: http://code.enthought.com/projects/traits name: traits version: 4.0.0 traitsui: description: 'traitsui: traits-capable user interfaces' homepage_uri: https://github.com/enthought/traitsui name: traitsui version: 4.0.1 ubuntuone-couch: description: Ubuntu One CouchDB homepage_uri: https://launchpad.net/ubuntuone-couch name: ubuntuone-couch version: 0.3.0 ubuntuone-installer: description: Ubuntu One Installer homepage_uri: https://launchpad.net/ubuntuone-installer name: ubuntuone-installer version: 3.0.0 ubuntuone-storage-protocol: description: UNKNOWN homepage_uri: UNKNOWN name: ubuntuone-storage-protocol version: 3.0.0 ufw: description: front-end for Linux firewalling homepage_uri: https://launchpad.net/ufw name: ufw version: 0.31.1-1 unattended-upgrades: description: UNKNOWN homepage_uri: UNKNOWN name: unattended-upgrades version: '0.1' unity-lens-video: description: UNKNOWN homepage_uri: http://launchpad.net/~davidc3 name: unity-lens-video version: 0.3.5 unity-scope-video-remote: description: UNKNOWN homepage_uri: http://launchpad.net/onehundredscopes name: unity-scope-video-remote version: 0.3.5 usb-creator: description: Ubuntu startup disk creator homepage_uri: UNKNOWN name: usb-creator version: 0.2.23 variant_tools: description: 'Variant tools: an integrated annotation and analysis package for next-gen sequencing data' homepage_uri: http://varianttools.sourceforge.net name: variant_tools version: 1.0.1a vboxapi: description: Python interface to VirtualBox homepage_uri: http://www.virtualbox.org name: vboxapi version: '1.0' virtualenv: description: Virtual Python Environment builder homepage_uri: http://www.virtualenv.org name: virtualenv version: 1.7.1.2 wadllib: description: Navigate HTTP resources using WADL files as guides. homepage_uri: https://launchpad.net/wadllib name: wadllib version: 1.3.0 workerpool: description: Module for distributing jobs to a pool of worker threads. homepage_uri: http://code.google.com/p/workerpool/ name: workerpool version: 0.9.2 wsgiref: description: WSGI (PEP 333) Reference Library homepage_uri: '' name: wsgiref version: 0.1.2 wxPython: description: Cross platform GUI toolkit for Python homepage_uri: http://wxPython.org/ name: wxPython version: 2.8.12.1 wxPython-common: description: Cross platform GUI toolkit for Python homepage_uri: http://wxPython.org/ name: wxPython-common version: 2.8.12.1 xdiagnose: description: Analysis tools for troubleshooting X.org problems homepage_uri: http://launchpad.net/xdiagnose name: xdiagnose version: '2.5' xkit: description: library for the manipulation of the xorg.conf homepage_uri: https://launchpad.net/x-kit name: xkit version: 0.0.0 xlrd: description: Library for developers to extract data from Microsoft Excel (tm) spreadsheet files homepage_uri: http://www.lexicon.net/sjmachin/xlrd.htm name: xlrd version: 0.7.8 yolk: description: Command-line tool for querying PyPI and Python packages installed on your system. homepage_uri: https://github.com/cakebread/yolk name: yolk version: 0.4.3 zope.interface: description: Interfaces for Python homepage_uri: http://pypi.python.org/pypi/zope.interface name: zope.interface version: 3.6.1 ================================================ FILE: manifest/r-packages.yaml ================================================ AnnotationDbi: description: Annotation Database Interface homepage_uri: '' name: AnnotationDbi version: 1.18.1 ArrayExpress: description: 'Access the ArrayExpress Microarray Database at EBI and build\nBioconductor data structures: ExpressionSet, AffyBatch,\nNChannelSet' homepage_uri: '' name: ArrayExpress version: 1.16.0 ArrayTools: description: geneChip Analysis Package homepage_uri: '' name: ArrayTools version: 1.16.0 BSgenome: description: Infrastructure for Biostrings-based genome data packages homepage_uri: '' name: BSgenome version: 1.24.0 BSgenome.Ecoli.NCBI.20080805: description: Escherichia coli full genomes homepage_uri: '' name: BSgenome.Ecoli.NCBI.20080805 version: 1.3.17 BSgenome.Scerevisiae.UCSC.sacCer3: description: Saccharomyces cerevisiae (Yeast) full genome (UCSC version sacCer3) homepage_uri: '' name: BSgenome.Scerevisiae.UCSC.sacCer3 version: 1.3.17 BayesPeak: description: Bayesian Analysis of ChIP-seq Data homepage_uri: '' name: BayesPeak version: 1.8.0 BeadDataPackR: description: Compression of Illumina BeadArray data homepage_uri: '' name: BeadDataPackR version: 1.8.0 BiasedUrn: description: Biased Urn model distributions homepage_uri: http://www.agner.org/random/ http://www.r-project.org name: BiasedUrn version: '1.04' Biobase: description: 'Biobase: Base functions for Bioconductor' homepage_uri: '' name: Biobase version: 2.16.0 BiocGenerics: description: Generic functions for Bioconductor homepage_uri: '' name: BiocGenerics version: 0.2.0 BiocInstaller: description: Install/Update Bioconductor and CRAN Packages homepage_uri: '' name: BiocInstaller version: 1.4.6 BiodiversityR: description: GUI for biodiversity and community ecology analysis homepage_uri: http://www.r-project.org,\nhttp://www.worldagroforestry.org/resources/databases/tree-diversity-analysis name: BiodiversityR version: '1.6' Biostrings: description: String objects representing biological sequences, and matching\nalgorithms homepage_uri: '' name: Biostrings version: 2.24.1 CNAnorm: description: A normalization method for Copy Number Aberration in cancer\nsamples homepage_uri: http://www.r-project.org, name: CNAnorm version: 1.2.0 CSAR: description: Statistical tools for the analysis of ChIP-seq data homepage_uri: '' name: CSAR version: 1.8.0 Cairo: description: R graphics device using cairo graphics library for creating\nhigh-quality bitmap (PNG, JPEG, TIFF), vector (PDF, SVG,\nPostScript) and display (X11 and Win32) output. homepage_uri: http://www.rforge.net/Cairo/ name: Cairo version: 1.5-1 Category: description: Category Analysis homepage_uri: '' name: Category version: 2.22.0 ChIPpeakAnno: description: Batch annotation of the peaks identified from either ChIP-seq,\nChIP-chip experiments or any experiments resulted in large\nnumber of chromosome ranges. homepage_uri: '' name: ChIPpeakAnno version: 2.4.0 ChIPseqR: description: Identifying Protein Binding Sites in High-Throughput Sequencing\nData homepage_uri: '' name: ChIPseqR version: 1.10.0 ChIPsim: description: Simulation of ChIP-seq experiments homepage_uri: '' name: ChIPsim version: 1.10.0 DBI: description: R Database Interface homepage_uri: '' name: DBI version: 0.2-5 DEGseq: description: Identify Differentially Expressed Genes from RNA-seq data homepage_uri: '' name: DEGseq version: 1.10.0 DESeq: description: Differential gene expression analysis based on the negative\nbinomial distribution homepage_uri: http://www-huber.embl.de/users/anders/DESeq name: DESeq version: 1.8.3 DEXSeq: description: Inference of differential exon usage in RNA-Seq homepage_uri: '' name: DEXSeq version: 1.2.0 DNAcopy: description: DNA copy number data analysis homepage_uri: '' name: DNAcopy version: 1.30.0 DiffBind: description: Differential Binding Analysis of ChIP-Seq peak data homepage_uri: '' name: DiffBind version: 1.2.0 DynDoc: description: Dynamic document tools homepage_uri: '' name: DynDoc version: 1.34.0 EDASeq: description: Exploratory Data Analysis and Normalization for RNA-Seq homepage_uri: '' name: EDASeq version: 1.2.0 GEOquery: description: Get data from NCBI Gene Expression Omnibus (GEO) homepage_uri: http://watson.nci.nih.gov/~sdavis name: GEOquery version: 2.23.5 GGBase: description: GGBase infrastructure for genetics of gene expression package\nGGtools homepage_uri: '' name: GGBase version: 3.18.0 GGtools: description: software and data for analyses in genetics of gene expression homepage_uri: '' name: GGtools version: 4.4.0 GO.db: description: A set of annotation maps describing the entire Gene Ontology homepage_uri: '' name: GO.db version: 2.7.1 GOFunction: description: 'GO-function: deriving biologcially relevant functions from\nstatistically significant functions' homepage_uri: '' name: GOFunction version: 1.2.0 GOstats: description: Tools for manipulating GO and microarrays. homepage_uri: '' name: GOstats version: 2.22.0 GSEABase: description: Gene set enrichment data structures and methods homepage_uri: '' name: GSEABase version: 1.18.0 GWASExactHW: description: Exact Hardy-Weinburg testing for Genome Wide Association Studies homepage_uri: '' name: GWASExactHW version: '1.0' GenomicFeatures: description: Tools for making and manipulating transcript centric annotations homepage_uri: '' name: GenomicFeatures version: 1.8.1 GenomicRanges: description: Representation and manipulation of genomic intervals homepage_uri: '' name: GenomicRanges version: 1.8.6 HH: description: 'Statistical Analysis and Data Display: Heiberger and Holland' homepage_uri: '' name: HH version: 2.3-15 HilbertVis: description: Hilbert curve visualization homepage_uri: http://www.ebi.ac.uk/~anders/hilbert name: HilbertVis version: 1.14.0 Hmisc: description: Harrell Miscellaneous homepage_uri: http://biostat.mc.vanderbilt.edu/wiki/Main/Hmisc,\nhttp://biostat.mc.vanderbilt.edu/trac/Hmisc name: Hmisc version: 3.9-3 IRanges: description: Infrastructure for manipulating intervals on sequences homepage_uri: '' name: IRanges version: 1.14.3 Iso: description: Functions to perform isotonic regression. homepage_uri: http://www.math.unb.ca/~rolf/ name: Iso version: 0.0-10 KEGG.db: description: A set of annotation maps for KEGG homepage_uri: '' name: KEGG.db version: 2.7.1 KernSmooth: description: Functions for kernel smoothing for Wand & Jones (1995) homepage_uri: '' name: KernSmooth version: 2.23-7 LowRankQP: description: Low Rank Quadratic Programming homepage_uri: '' name: LowRankQP version: 1.0.1 MASS: description: Support Functions and Datasets for Venables and Ripley's MASS homepage_uri: http://www.stats.ox.ac.uk/pub/MASS4/ name: MASS version: 7.3-18 MEDIPS: description: MeDIP-Seq data analysis homepage_uri: '' name: MEDIPS version: 1.6.0 Matrix: description: Sparse and Dense Matrix Classes and Methods homepage_uri: http://Matrix.R-forge.R-project.org/ name: Matrix version: 1.0-7 MotIV: description: Motif Identification and Validation homepage_uri: '' name: MotIV version: 1.10.0 NMF: description: Algorithms and framework for Nonnegative Matrix Factorization\n(NMF). homepage_uri: http://nmf.r-forge.r-project.org name: NMF version: 0.5.06 PICS: description: Probabilistic inference of ChIP-seq homepage_uri: '' name: PICS version: 1.10.0 R.methodsS3: description: Utility function for defining S3 methods homepage_uri: http://www.braju.com/R/ name: R.methodsS3 version: 1.2.2 R2HTML: description: HTML exportation for R objects homepage_uri: http://www.r-project.org name: R2HTML version: '2.2' R453Plus1Toolbox: description: A package for importing and analyzing data from Roche's Genome\nSequencer System. homepage_uri: '' name: R453Plus1Toolbox version: 1.6.1 RBGL: description: An interface to the BOOST graph library homepage_uri: http://www.bioconductor.org name: RBGL version: 1.32.0 RColorBrewer: description: ColorBrewer palettes homepage_uri: '' name: RColorBrewer version: 1.0-5 RCurl: description: General network (HTTP/FTP/...) client interface for R homepage_uri: http://www.omegahat.org/RCurl name: RCurl version: 1.91-1 RMySQL: description: R interface to the MySQL database homepage_uri: http://biostat.mc.vanderbilt.edu/RMySQL,\nhttps://github.com/jeffreyhorner/RMySQL name: RMySQL version: 0.9-3 RSQLite: description: SQLite interface for R homepage_uri: '' name: RSQLite version: 0.11.1 RSQLite.extfuns: description: Math and String Extension Functions for RSQLite homepage_uri: http://www.sqlite.org/contrib/ name: RSQLite.extfuns version: 0.0.1 Rcpp: description: Seamless R and C++ Integration homepage_uri: http://dirk.eddelbuettel.com/code/rcpp.html,\nhttp://romainfrancois.blog.free.fr/index.php?category/R-package/Rcpp name: Rcpp version: 0.9.10 Rgraphviz: description: Provides plotting capabilities for R graph objects homepage_uri: '' name: Rgraphviz version: 1.34.1 Rmpi: description: Interface (Wrapper) to MPI (Message-Passing Interface) homepage_uri: http://www.stats.uwo.ca/faculty/yu/Rmpi name: Rmpi version: 0.5-9 Rolexa: description: Statistical analysis of Solexa sequencing data homepage_uri: '' name: Rolexa version: 1.12.0 Rsamtools: description: Binary alignment (BAM), variant call (BCF), or tabix file import homepage_uri: http://bioconductor.org/packages/release/bioc/html/Rsamtools.html name: Rsamtools version: 1.8.5 SRAdb: description: A compilation of metadata from NCBI SRA and tools homepage_uri: http://watson.nci.nih.gov/ name: SRAdb version: 1.10.0 SVGAnnotation: description: Tools for post-processing SVG plots created in R homepage_uri: '' name: SVGAnnotation version: 0.9-0 ShortRead: description: Classes and methods for high-throughput short-read sequencing\ndata. homepage_uri: '' name: ShortRead version: 1.14.4 SparseM: description: Sparse Linear Algebra homepage_uri: http://www.econ.uiuc.edu/~roger/research/sparse/sparse.html name: SparseM version: '0.96' TeachingDemos: description: Demonstrations for teaching and learning homepage_uri: '' name: TeachingDemos version: '2.8' VariantAnnotation: description: Annotation of Genetic Variants homepage_uri: '' name: VariantAnnotation version: 1.2.8 XML: description: Tools for parsing and generating XML within R and S-Plus. homepage_uri: http://www.omegahat.org/RSXML name: XML version: 3.9-4 abind: description: Combine multi-dimensional arrays homepage_uri: '' name: abind version: 1.4-0 affy: description: Methods for Affymetrix Oligonucleotide Arrays homepage_uri: '' name: affy version: 1.34.0 affyPLM: description: Methods for fitting probe-level models homepage_uri: http://bmbolstad.com name: affyPLM version: 1.32.0 affyQCReport: description: QC Report Generation for affyBatch objects homepage_uri: '' name: affyQCReport version: 1.34.0 affydata: description: Affymetrix Data for Demonstration Purpose homepage_uri: '' name: affydata version: 1.11.16 affyio: description: Tools for parsing Affymetrix data files homepage_uri: '' name: affyio version: 1.24.0 affylmGUI: description: GUI for affy analysis using limma package homepage_uri: http://bioinf.wehi.edu.au/affylmGUI/ name: affylmGUI version: 1.30.0 akima: description: Interpolation of irregularly spaced data homepage_uri: '' name: akima version: 0.5-7 amap: description: Another Multidimensional Analysis Package homepage_uri: http://mulcyber.toulouse.inra.fr/projects/amap/ name: amap version: 0.8-7 annaffy: description: Annotation tools for Affymetrix biological metadata homepage_uri: '' name: annaffy version: 1.28.0 annotate: description: Annotation for microarrays homepage_uri: '' name: annotate version: 1.34.0 ape: description: Analyses of Phylogenetics and Evolution homepage_uri: http://ape.mpl.ird.fr/ name: ape version: 3.0-4 aplpack: description: 'Another Plot PACKage: stem.leaf, bagplot, faces, spin3R, and\nsome slider functions' homepage_uri: http://www.wiwi.uni-bielefeld.de/com/wolf/software/aplpack.html name: aplpack version: 1.2.6 aroma.light: description: Light-weight methods for normalization and visualization of\nmicroarray data using only basic R data types homepage_uri: http://www.aroma-project.org/ name: aroma.light version: 1.24.0 arrayQualityMetrics: description: Quality metrics on microarray data sets homepage_uri: '' name: arrayQualityMetrics version: 3.12.0 ash: description: David Scott's ASH routines homepage_uri: '' name: ash version: 1.0-13 base: description: The R Base Package homepage_uri: '' name: base version: 2.15.0 baySeq: description: Empirical Bayesian analysis of patterns of differential\nexpression in count data homepage_uri: '' name: baySeq version: 1.10.0 beadarray: description: Quality assessment and low-level analysis for Illumina BeadArray\ndata homepage_uri: '' name: beadarray version: 2.6.0 biglm: description: bounded memory linear and generalized linear models homepage_uri: '' name: biglm version: '0.8' bigmemory: description: Manage massive matrices with shared memory and memory-mapped\nfiles homepage_uri: http://www.bigmemory.org name: bigmemory version: 4.2.11 bioDist: description: Different distance measures homepage_uri: '' name: bioDist version: 1.28.0 biomaRt: description: Interface to BioMart databases (e.g. Ensembl, COSMIC ,Wormbase\nand Gramene) homepage_uri: '' name: biomaRt version: 2.12.0 bit: description: A class for vectors of 1-bit booleans homepage_uri: http://ff.r-forge.r-project.org/ name: bit version: 1.1-8 bitops: description: Functions for Bitwise operations homepage_uri: '' name: bitops version: 1.0-4.1 boot: description: Bootstrap Functions (originally by Angelo Canty for S) homepage_uri: '' name: boot version: 1.3-4 caTools: description: 'Tools: moving window statistics, GIF, Base64, ROC AUC, etc.' homepage_uri: '' name: caTools version: '1.13' car: description: Companion to Applied Regression homepage_uri: https://r-forge.r-project.org/projects/car/,\nhttp://CRAN.R-project.org/package=car,\nhttp://socserv.socsci.mcmaster.ca/jfox/Books/Companion/index.html name: car version: 2.0-12 chipseq: description: 'chipseq: A package for analyzing chipseq data' homepage_uri: '' name: chipseq version: 1.6.0 chron: description: Chronological objects which can handle dates and times homepage_uri: '' name: chron version: 2.3-42 class: description: Functions for Classification homepage_uri: http://www.stats.ox.ac.uk/pub/MASS4/ name: class version: 7.3-3 cluster: description: Cluster Analysis Extended Rousseeuw et al. homepage_uri: '' name: cluster version: 1.14.2 cn.mops: description: cn.mops - Mixture of Poissons for CNV detection in NGS data homepage_uri: http://www.bioinf.jku.at/software/cnmops/cnmops.html name: cn.mops version: 1.2.5 codetools: description: Code Analysis Tools for R homepage_uri: '' name: codetools version: 0.2-8 coin: description: Conditional Inference Procedures in a Permutation Test Framework homepage_uri: '' name: coin version: 1.0-21 colorspace: description: Color Space Manipulation homepage_uri: '' name: colorspace version: 1.1-1 compiler: description: The R Compiler Package homepage_uri: '' name: compiler version: 2.15.0 cummeRbund: description: Analysis, exploration, manipulation, and visualization of\nCufflinks high-throughput sequencing data. homepage_uri: '' name: cummeRbund version: 1.2.0 datasets: description: The R Datasets Package homepage_uri: '' name: datasets version: 2.15.0 date: description: Functions for handling dates homepage_uri: '' name: date version: 1.2-32 devtools: description: Tools to make developing R code easier homepage_uri: '' name: devtools version: '0.6' dichromat: description: Color schemes for dichromats homepage_uri: '' name: dichromat version: 1.2-4 digest: description: Create cryptographic hash digests of R objects homepage_uri: http://dirk.eddelbuettel.com/code/digest.html name: digest version: 0.5.2 edgeR: description: Empirical analysis of digital gene expression data in R homepage_uri: '' name: edgeR version: 2.6.7 effects: description: Effect Displays for Linear, Generalized Linear,\nMultinomial-Logit, Proportional-Odds Logit Models and\nMixed-Effects Models homepage_uri: http://www.r-project.org, http://socserv.socsci.mcmaster.ca/jfox/ name: effects version: 2.1-1 evaluate: description: Parsing and evaluation tools that provide more details than the\ndefault. homepage_uri: '' name: evaluate version: 0.4.2 exomeCopy: description: CNV detection from exome sequencing read depth homepage_uri: '' name: exomeCopy version: 1.2.0 fBasics: description: Rmetrics - Markets and Basic Statistics homepage_uri: http://www.rmetrics.org name: fBasics version: '2160.81' ff: description: memory-efficient storage of large data on disk and fast access\nfunctions homepage_uri: http://ff.r-forge.r-project.org/ name: ff version: 2.2-7 flashClust: description: Implementation of optimal hierarchical clustering homepage_uri: '' name: flashClust version: 1.01-1 foreign: description: Read Data Stored by Minitab, S, SAS, SPSS, Stata, Systat, dBase,\n... homepage_uri: '' name: foreign version: 0.8-50 formatR: description: Format R Code Automatically homepage_uri: https://github.com/yihui/formatR/wiki name: formatR version: '0.4' gage: description: Generally Applicable Gene-set Enrichment for Pathway Analysis homepage_uri: http://www.biomedcentral.com/1471-2105/10/161 name: gage version: 2.6.0 gcrma: description: Background Adjustment Using Sequence Information homepage_uri: '' name: gcrma version: 2.28.0 gdata: description: Various R programming tools for data manipulation homepage_uri: '' name: gdata version: 2.10.6 gee: description: Generalized Estimation Equation solver homepage_uri: '' name: gee version: 4.13-18 geneLenDataBase: description: Lengths of mRNA transcripts for a number of genomes homepage_uri: '' name: geneLenDataBase version: 0.99.9 genefilter: description: 'genefilter: methods for filtering genes from microarray\nexperiments' homepage_uri: '' name: genefilter version: 1.38.0 geneplotter: description: Graphics related functions for Bioconductor homepage_uri: '' name: geneplotter version: 1.34.0 genomeIntervals: description: Operations on genomic intervals homepage_uri: '' name: genomeIntervals version: 1.12.0 genoset: description: Provides classes similar to ExpressionSet for copy number\nanalysis homepage_uri: '' name: genoset version: 1.6.0 ggplot2: description: An implementation of the Grammar of Graphics homepage_uri: http://had.co.nz/ggplot2/ name: ggplot2 version: 0.9.1 girafe: description: Genome Intervals and Read Alignments for Functional Exploration homepage_uri: '' name: girafe version: 1.8.0 globaltest: description: Testing groups of covariates/features for association with a\nresponse variable, with applications to gene set testing homepage_uri: http://www.msbi.nl/goeman name: globaltest version: 5.10.0 goseq: description: Gene Ontology analyser for RNA-seq and other length biased data homepage_uri: '' name: goseq version: 1.8.0 gplots: description: Various R programming tools for plotting data homepage_uri: '' name: gplots version: 2.11.0 grDevices: description: The R Graphics Devices and Support for Colours and Fonts homepage_uri: '' name: grDevices version: 2.15.0 graph: description: 'graph: A package to handle graph data structures' homepage_uri: '' name: graph version: 1.34.0 graphics: description: The R Graphics Package homepage_uri: '' name: graphics version: 2.15.0 grid: description: The Grid Graphics Package homepage_uri: '' name: grid version: 2.15.0 gsubfn: description: Utilities for strings and function arguments. homepage_uri: http://gsubfn.googlecode.com name: gsubfn version: 0.6-3 gtools: description: Various R programming tools homepage_uri: '' name: gtools version: 2.6.2 hdf5: description: HDF5 homepage_uri: '' name: hdf5 version: 1.6.9 hdrcde: description: Highest density regions and conditional density estimation homepage_uri: http://www.robjhyndman.com/software/hdrcde name: hdrcde version: '2.16' hexbin: description: Hexagonal Binning Routines homepage_uri: '' name: hexbin version: 1.26.0 hgu95av2.db: description: Affymetrix Human Genome U95 Set annotation data (chip hgu95av2) homepage_uri: '' name: hgu95av2.db version: 2.7.1 highlight: description: Syntax highlighter homepage_uri: http://r-forge.r-project.org/projects/highlight/,\nhttp://romainfrancois.blog.free.fr/index.php?category/R-package/highlight name: highlight version: 0.3.1 htSeqTools: description: Quality Control, Visualization and Processing for\nHigh-Throughput Sequencing data homepage_uri: '' name: htSeqTools version: 1.2.0 hwriter: description: HTML Writer - Outputs R objects in HTML format homepage_uri: http://www.embl.de/~gpau/hwriter/ name: hwriter version: '1.3' impute: description: 'impute: Imputation for microarray data' homepage_uri: '' name: impute version: 1.30.0 intervals: description: Tools for working with points and intervals homepage_uri: '' name: intervals version: 0.13.3 kernlab: description: Kernel-based Machine Learning Lab homepage_uri: '' name: kernlab version: 0.9-14 knitr: description: A general-purpose package for dynamic report generation in R homepage_uri: http://yihui.name/knitr/ name: knitr version: '0.6' ks: description: Kernel smoothing homepage_uri: http://www.mvstat.net/tduong name: ks version: 1.8.8 labeling: description: Axis Labeling homepage_uri: '' name: labeling version: '0.1' lattice: description: Lattice Graphics homepage_uri: http://lattice.r-forge.r-project.org/ name: lattice version: 0.20-6 latticeExtra: description: Extra Graphical Utilities Based on Lattice homepage_uri: http://latticeextra.r-forge.r-project.org/ name: latticeExtra version: 0.6-19 leaps: description: regression subset selection homepage_uri: '' name: leaps version: '2.9' limSolve: description: Solving Linear Inverse Models homepage_uri: '' name: limSolve version: 1.5.3 limma: description: Linear Models for Microarray Data homepage_uri: http://bioinf.wehi.edu.au/limma name: limma version: 3.12.1 lmtest: description: Testing Linear Regression Models homepage_uri: '' name: lmtest version: 0.9-30 locfit: description: Local Regression, Likelihood and Density Estimation. homepage_uri: '' name: locfit version: 1.5-8 lpSolve: description: Interface to Lp_solve v. 5.5 to solve linear/integer programs homepage_uri: '' name: lpSolve version: 5.6.6 lumi: description: BeadArray Specific Methods for Illumina Methylation and\nExpression Microarrays homepage_uri: '' name: lumi version: 2.8.0 makecdfenv: description: CDF Environment Maker homepage_uri: '' name: makecdfenv version: 1.34.0 maptree: description: Mapping, pruning, and graphing tree models homepage_uri: http://www.epa.gov/wed/pages/staff/white/ name: maptree version: 1.4-6 markdown: description: Markdown rendering for R homepage_uri: '' name: markdown version: 0.5.1 marray: description: Exploratory analysis for two-color spotted microarray data homepage_uri: http://www.maths.usyd.edu.au/u/jeany/ name: marray version: 1.34.0 matrixStats: description: Methods that apply to rows and columns of a matrix homepage_uri: '' name: matrixStats version: 0.5.0 mclust: description: Model-Based Clustering / Normal Mixture Modeling homepage_uri: http://www.stat.washington.edu/research/reports/2006/tr504.pdf name: mclust version: 3.4.11 memoise: description: Memoise functions homepage_uri: '' name: memoise version: '0.1' methods: description: Formal Methods and Classes homepage_uri: '' name: methods version: 2.15.0 methylumi: description: Handle Illumina methylation data homepage_uri: '' name: methylumi version: 2.2.0 mgcv: description: Mixed GAM Computation Vehicle with GCV/AIC/REML smoothness\nestimation homepage_uri: '' name: mgcv version: 1.7-18 misc3d: description: Miscellaneous 3D Plots homepage_uri: '' name: misc3d version: 0.8-2 modeltools: description: Tools and Classes for Statistical Models homepage_uri: '' name: modeltools version: 0.2-19 multcomp: description: Simultaneous Inference in General Parametric Models homepage_uri: The publishers web page is\nhttp://www.crcpress.com/product/isbn/9781584885740 name: multcomp version: 1.2-12 multicore: description: Parallel processing of R code on machines with multiple cores or\nCPUs homepage_uri: http://www.rforge.net/multicore/ name: multicore version: 0.1-7 multtest: description: Resampling-based multiple hypothesis testing homepage_uri: '' name: multtest version: 2.12.0 munsell: description: Munsell colour system homepage_uri: '' name: munsell version: '0.3' mvtnorm: description: Multivariate Normal and t Distributions homepage_uri: '' name: mvtnorm version: 0.9-9992 nleqslv: description: Solve systems of non linear equations homepage_uri: '' name: nleqslv version: 1.9.3 nlme: description: Linear and Nonlinear Mixed Effects Models homepage_uri: '' name: nlme version: 3.1-104 nnet: description: Feed-forward Neural Networks and Multinomial Log-Linear Models homepage_uri: http://www.stats.ox.ac.uk/pub/MASS4/ name: nnet version: 7.3-1 numDeriv: description: Accurate Numerical Derivatives homepage_uri: http:http://optimizer.r-forge.r-project.org/ name: numDeriv version: 2012.3-1 oneChannelGUI: description: A graphical interface designed to facilitate analysis of\nmicroarrays and miRNA/RNA-seq data on laptops. homepage_uri: http://www.bioinformatica.unito.it/oneChannelGUI/ name: oneChannelGUI version: 1.22.7 org.Hs.eg.db: description: Genome wide annotation for Human homepage_uri: '' name: org.Hs.eg.db version: 2.7.1 pamr: description: 'Pam: prediction analysis for microarrays' homepage_uri: '' name: pamr version: '1.54' parallel: description: Support for Parallel computation in R homepage_uri: '' name: parallel version: 2.15.0 parser: description: Detailed R source code parser homepage_uri: http://www.r-enthusiasts.com name: parser version: 0.0-14 permute: description: Functions for generating restricted permutations of data homepage_uri: http://vegan.r-forge.r-project.org/ name: permute version: 0.7-0 pi0: description: Estimating the proportion of true null hypotheses for FDR homepage_uri: '' name: pi0 version: 1.3-200 plotrix: description: Various plotting functions homepage_uri: '' name: plotrix version: 3.4-1 plyr: description: Tools for splitting, applying and combining data homepage_uri: http://had.co.nz/plyr name: plyr version: 1.7.1 preprocessCore: description: A collection of pre-processing functions homepage_uri: '' name: preprocessCore version: 1.18.0 proto: description: Prototype object-based programming homepage_uri: http://r-proto.googlecode.com name: proto version: 0.3-9.2 quadprog: description: Functions to solve Quadratic Programming Problems. homepage_uri: '' name: quadprog version: 1.5-4 qvalue: description: Q-value estimation for false discovery rate control homepage_uri: '' name: qvalue version: 1.30.0 rGADEM: description: de novo motif discovery homepage_uri: '' name: rGADEM version: 2.4.0 relimp: description: Relative Contribution of Effects in a Regression Model homepage_uri: http://go.warwick.ac.uk/relimp, http://go.warwick.ac.uk/dfirth name: relimp version: 1.0-3 reshape: description: Flexibly reshape data. homepage_uri: http://had.co.nz/reshape name: reshape version: 0.8.4 reshape2: description: 'Flexibly reshape data: a reboot of the reshape package.' homepage_uri: http://had.co.nz/reshape name: reshape2 version: 1.2.1 rgl: description: 3D visualization device system (OpenGL) homepage_uri: http://rgl.neoscientists.org name: rgl version: 0.92.880 rjson: description: JSON for R homepage_uri: '' name: rjson version: 0.2.8 rnaSeqMap: description: rnaSeq secondary analyses homepage_uri: '' name: rnaSeqMap version: 2.10.0 rpart: description: Recursive Partitioning homepage_uri: \nhttp://mayoresearch.mayo.edu/mayo/research/biostat/splusfunctions.cfm name: rpart version: 3.1-53 rtracklayer: description: R interface to genome browsers and their annotation tracks homepage_uri: '' name: rtracklayer version: 1.16.1 samr: description: 'SAM: Significance Analysis of Microarrays' homepage_uri: http://www-stat.stanford.edu/~tibs/SAM name: samr version: '2.0' sandwich: description: Robust Covariance Matrix Estimators homepage_uri: '' name: sandwich version: 2.2-9 scales: description: Scale functions for graphics. homepage_uri: '' name: scales version: 0.2.1 scatterplot3d: description: 3D Scatter Plot homepage_uri: '' name: scatterplot3d version: 0.3-33 segmentSeq: description: Methods for identifying small RNA loci from high-throughput\nsequencing data homepage_uri: '' name: segmentSeq version: 1.8.0 seqLogo: description: Sequence logos for DNA sequence alignments homepage_uri: '' name: seqLogo version: 1.22.0 setRNG: description: Set (Normal) Random Number Generator and Seed homepage_uri: http://distr.r-forge.r-project.org/ name: setRNG version: 2011.11-2 siggenes: description: Multiple testing using SAM and Efron's empirical Bayes\napproaches homepage_uri: '' name: siggenes version: 1.30.0 simpleaffy: description: Very simple high level analysis of Affymetrix data homepage_uri: http://www.bioconductor.org,\nhttp://bioinformatics.picr.man.ac.uk/simpleaffy/ name: simpleaffy version: 2.32.0 sm: description: Smoothing methods for nonparametric regression and density\nestimation homepage_uri: http://www.stats.gla.ac.uk/~adrian/sm name: sm version: 2.2-4.1 snow: description: Simple Network of Workstations homepage_uri: '' name: snow version: 0.3-9 snowfall: description: Easier cluster computing (based on snow). homepage_uri: '' name: snowfall version: '1.84' snpStats: description: SnpMatrix and XSnpMatrix classes and methods homepage_uri: http://www-gene.cimr.cam.ac.uk/clayton name: snpStats version: 1.6.0 spatial: description: Functions for Kriging and Point Pattern Analysis homepage_uri: http://www.stats.ox.ac.uk/pub/MASS4/ name: spatial version: 7.3-3 splines: description: Regression Spline Functions and Classes homepage_uri: '' name: splines version: 2.15.0 splots: description: Visualization of high-throughput assays in microtitre plate or\nslide format homepage_uri: '' name: splots version: 1.22.0 sqldf: description: Perform SQL Selects on R Data Frames homepage_uri: http://sqldf.googlecode.com name: sqldf version: 0.4-6.4 stabledist: description: Stable Distribution Functions homepage_uri: http://www.rmetrics.org name: stabledist version: 0.6-4 statmod: description: Statistical Modeling homepage_uri: http://www.statsci.org/r name: statmod version: 1.4.14 stats: description: The R Stats Package homepage_uri: '' name: stats version: 2.15.0 stats4: description: Statistical Functions using S4 Classes homepage_uri: '' name: stats4 version: 2.15.0 stringr: description: Make it easier to work with strings. homepage_uri: '' name: stringr version: '0.6' strucchange: description: Testing, Monitoring, and Dating Structural Changes homepage_uri: '' name: strucchange version: 1.4-7 survival: description: Survival analysis, including penalised likelihood. homepage_uri: http://r-forge.r-project.org name: survival version: 2.36-14 tcltk: description: Tcl/Tk Interface homepage_uri: '' name: tcltk version: 2.15.0 timeDate: description: Rmetrics - Chronological and Calendarical Objects homepage_uri: http://www.rmetrics.org name: timeDate version: '2160.95' timeSeries: description: Rmetrics - Financial Time Series Objects homepage_uri: http://www.rmetrics.org name: timeSeries version: '2160.94' timsac: description: TIMe Series Analysis and Control package homepage_uri: '' name: timsac version: 1.2.7 tkWidgets: description: R based tk widgets homepage_uri: '' name: tkWidgets version: 1.34.0 tools: description: Tools for Package Development homepage_uri: '' name: tools version: 2.15.0 utils: description: The R Utils Package homepage_uri: '' name: utils version: 2.15.0 vegan: description: Community Ecology Package homepage_uri: http://cran.r-project.org, http://vegan.r-forge.r-project.org/ name: vegan version: 2.0-3 vsn: description: Variance stabilization and calibration for microarray data homepage_uri: http://www.r-project.org, http://www.ebi.ac.uk/huber name: vsn version: 3.24.0 widgetTools: description: Creates an interactive tcltk widget homepage_uri: '' name: widgetTools version: 1.34.0 xmapcore: description: Core access to the xmap database (installed separately) homepage_uri: http://xmap.picr.man.ac.uk, http://www.bioconductor.org name: xmapcore version: 1.10.0 xtable: description: Export tables to LaTeX or HTML homepage_uri: http://xtable.r-forge.r-project.org/ name: xtable version: 1.7-0 yacca: description: Yet Another Canonical Correlation Analysis Package homepage_uri: '' name: yacca version: '1.1' yaml: description: Methods to convert R data to YAML and back homepage_uri: '' name: yaml version: 2.1.4 zlibbioc: description: An R packaged zlib-1.2.5 homepage_uri: http://bioconductor.org/packages/release/bioc/html/Zlibbioc.html name: zlibbioc version: 1.2.0 zoo: description: S3 Infrastructure for Regular and Irregular Time Series (Z's\nordered observations) homepage_uri: http://zoo.R-Forge.R-project.org/ name: zoo version: 1.7-7 ================================================ FILE: setup.py ================================================ #!/usr/bin/env python """Install cloudbiolinux install libraries for external use. This is not needed for running fabric files, but is useful for reusing code. """ from setuptools import setup, find_packages setup(name = "cloudbiolinux", version = "0.3a", author = "Brad Chapman", author_email = "chapmanb@50mail.com", description = "configure virtual (or real) machines with tools for biological analyses", license = "MIT", url = "http://cloudbiolinux.org", packages = find_packages(), scripts = [], install_requires = [ "PyYAML >= 3.09", "fabric >= 1.8.0"] ) ================================================ FILE: test/README ================================================ Integration test for CloudBioLinux. When you have a base install running, you only need to supply the (local) IP address of the VM, and the password-less key for the install user (who has sudo bash). The tests get invoked with ./test_biolinux [-i key.pub] [-u user] IPaddr [Flavor] For further options see ./test_biolinux --help Alternatively try the self running test using Virtualbox and Vagrant with: ./test_vagrant --help == Testing issues with Vagrant == The test system depends on a stack of tools, including virtualbox, vagrant, ssh, ruby and related libraries. Things can go wrong. Here we list a number of common problems. === Error with ssh config === The test system appends login code to enter a running vagrant virtualbox without asking for a password. Sometimes the contents of $HOME/.ssh/config get corrupted. Check the contents out, and remove all wrong information with vi ~/.ssh/config and rerun the test. === Error suggests 'vagrant up' === If something went wrong with the install, Vagrant may complain that you need to run 'vagrant up' first. In practise, this may mean: * Remove other running vagrant instances (vagrant halt) * Remove an existing (faulty) instance using virtualbox * Check ~/.ssh/config contains no faulty lines * Rerun the tests from scratch, i.e. without the '--continue' switch === Fatal error: Name lookup failed for vagrant Probably there is another VirtualBox + vagrant running somewhere. This conflicts with the information in ./ssh/config. Easy solution: remove and kill the running instance first. === Error: ffi.rb:106:in `call_and_check': Error in API call to get_state === This is Ruby trying into the vagrant API. You need to reinstall the vagrant and virtualbox gems to match your version of Ruby. === Grub error On a kernel upgrade, the test scripts may want to install grub, which is the boot loader. Vagrant does not need that, so don't try to install (select ignore). ================================================ FILE: test/test_biolinux ================================================ #! /usr/bin/env ruby # # Integration testing of BioLinux targets # # Tests a Biolinux flavor on a running VM. First fabric is invoked # with the flavor. Integration tests for the flavor follow. # # The integration test aims to catch inconsistencies in the installation, # and can be run to test changes in the source tree. # # Requirements: # # gem install net-ssh # # For more information see: # # ./test/test_biolinux --help # ./test/test_biolinux --list # # Easiest is to create a Host entry in ./ssh/config. See also the hacking # documents in ./doc/ require 'rubygems' require 'yaml' require 'optparse' begin require 'net/ssh' # require 'net/scp' rescue LoadError $stderr.print "Please install net-ssh with 'gem install net-ssh'" exit 1 end # Available CloudBioLinux testing Flavors - this could be read from a YAML file. CONFIG = <, port <#{$port}>, user <#{$user}>, passwd <#{$pwd}>\n" run "fab #{fab_opts} -f #{srcpath}/fabfile.py -c #{fabricrc} install_biolinux:flavor=#{packagelist}" ssh_opts = {} ssh_opts[:password] = $pwd if $pwd ssh_opts[:port] = $port if $port Net::SSH.start($hostname, $user, ssh_opts) do |ssh| @ssh = ssh @prompt = $hostname @prompt = "img" if $hostname == '127.0.0.1' def exec cmd $stderr.print "#{@prompt}> ",cmd,"\n" @ssh.exec!(cmd) end $stderr.print "logged in!\n" dpkg = exec("dpkg -l") File.open("dpkg.lst","w").write(dpkg) test_for_match(ssh,"/usr/bin/ruby -v", "ruby") test_in_path(ssh,"gem") # test for completed install # lastline = ssh.exec!("tail -1 /var/log/biolinux.log") # print lastline log = ssh.exec!("cat /var/log/biolinux.log") File.open("biolinux.log","w").write(log) # if lastline =~ /#{$flavor}/ print < { :url => 'http://bio4.dnsalias.net/download/biolinux/biolinux-minimal-0.91.box', :md5 => '64cdea1f76028971e53e2cd2f19a490a', :fabricrc => 'contrib/flavor/minimal/fabricrc_debian.txt', :packagelist => 'contrib/flavor/minimal' }, 'Boinc' => # Boinc is a Minimal flavor for running Boinc clients { :url => 'http://bio4.dnsalias.net/download/biolinux/biolinux-minimal-0.91.box', :md5 => '64cdea1f76028971e53e2cd2f19a490a', :fabricrc => 'contrib/flavor/boinc/fabricrc_debian.txt', :packagelist => 'contrib/flavor/boinc' }, 'Phylogeny' => # Phylogeny is a Minimal flavor for running Phylogenetic software in the Cloud { :url => 'http://bio4.dnsalias.net/download/biolinux/biolinux-phylogeny-debian-0.92-i386.box', :md5 => 'e7481104d24a7896ea1151a14e05d497', :fabricrc => 'contrib/flavor/phylogeny/fabricrc_debian.txt', :packagelist => 'contrib/flavor/phylogeny', :call_test => 'test_phylogeny_flavor' }, 'BioNode' => { :url => 'http://bio4.dnsalias.net/download/biolinux/biolinux-minimal-0.91.box', :md5 => '64cdea1f76028971e53e2cd2f19a490a', :fabricrc => 'contrib/bionode/fabricrc_bionode.txt', :packagelist => 'contrib/bionode' }, } print "Integration test scripts for CloudBioLinux (by Pjotr Prins and Brad Chapman, 2011)\n" print "BioLinux test frame work running on Ruby "+RUBY_VERSION+"\n" # find location of test files path = File.dirname(__FILE__) $: << path+'/testlib' require 'test_biolinux' def run cmd print "--> ",cmd system(cmd) # don't use backticks for fab! end def error msg $stderr.print "FATAL ERROR: ",msg exit 1 end def list_boxes print "BioLinux VMs available for testing:\n\n" BOX.each do | edition, properties | print "* ", edition, " (",properties[:url],")\n" end end if ARGV.index('--list') list_boxes exit 0 end if ARGV.index('-h') or ARGV.index('--help') print < #{vagrant_init}` # remove last line INJECT =<> #{vagrant_init}` end Dir.chdir(testname) do # Always try to start the VM run "vagrant up" end # Using the VM Dir.chdir(testname) do # ssh info ssh_config = `vagrant ssh-config` ssh_config =~ /Host (\S+)/ vagrant_hostname = $1 print "Hostname #{vagrant_hostname}\n" # Fetch the vagrant ssh key, and store it locally # First see if we can copy a file - if we can there is no need to update .ssh/config File.unlink('remote_hosts') if File.exists?('remote_hosts') run "scp #{vagrant_hostname}:/etc/hosts remote_hosts" if !File.exist?('remote_hosts') ssh_config = run "vagrant ssh-config >> ~/.ssh/config" run "scp #{vagrant_hostname}:/etc/hosts remote_hosts" end raise "Problem connecting to remote hosts (using scp)" if !File.exist?('remote_hosts') File.unlink('remote_hosts') # run "vagrant status" print "We can use #{testname}!\n" # Fire up the BioLinux fab file - this installs the VM run "fab -f #{srcpath}/fabfile.py -H vagrant -c #{fabricrc} install_biolinux:flavor=#{packagelist},environment=biolinux-test" # Now set up Ruby's vagrant access for testing vagrant = Vagrant::Environment.new raise "Must run `vagrant up`" if !vagrant.primary_vm.created? raise "Must be running!" if !vagrant.primary_vm.vm.running? vagrant.primary_vm.ssh.execute do |ssh| # copy package list File.open("dpkg.lst","w").write(ssh.exec!("dpkg -l")) # Do we have ruby and tools? test_for_match(ssh,"/usr/bin/ruby -v", "ruby") test_for_match(ssh,"/usr/bin/rdoc -v", "RDoc") test_in_path(ssh,"gem") # Make sure Ruby 1.9.1 is in Minimal # test_for_match(ssh,"/usr/bin/ruby1.9.1 -v", "1.9") # Run flavor specific tests eval(call_test+'(vagrant)') if call_test if test_for_completed_install(ssh,vagrant_hostname,box_name) print < ",cmd res = system(cmd) # don't use backticks for fab! if res == nil error "Command failed!" end res end # Error handler def error msg $stderr.print "\nFATAL ERROR: ",msg,"\n" exit 1 end ================================================ FILE: utils/bootstrap.sh ================================================ #!/bin/bash # See this blog: # http://alestic.com/2009/08/runurl # # The basic Alestic images support execution of the user data, and also # come with "runurl" pre-installed. Therefore we should be able to simply stick the # following into user data: # # #!/bin/bash -ex # runurl cloudbiolinux.com/bootstrap # # Of to support non-alestic images: # # #!/bin/bash -ex # which runurl || wget -O- run.alestic.com/install/runurl | bash # runurl cloudbiolinux.com/bootstrap # Now the URL given would actually be a redirect to: # http://github.com/chapmanb/bcbb/raw/master/ec2/biolinux/utils/bootstrap.sh if [ `id -un` != root ] ; then echo "This script needs to be run as root." exit 1 fi HOME=~root; USER=root # So, here we go... tmpdir=/tmp/bio-linux-bootstrap #Skip this bit if the directory was found - eases debugging if [ ! -e "$tmpdir" ] ; then # Install git and fabric and python stuff and ssh export DEBIAN_FRONTEND=noninteractive apt-get update apt-get -y install git-core fabric python-setuptools python-yaml openssh-server # Make a directory to work in mkdir $tmpdir ; cd $tmpdir # Pull the GIT stuff - this is wasteful but I don't know a better way. git clone --depth=0 http://github.com/chapmanb/bcbb.git . ls -A | grep -v ec2 | xargs rm -r #Some fixups to the fabfile - these can probably be changed in the upstream but #I want my script to work now... if ! id -un ubuntu ; then echo "Removing env.user=ubuntu from fabfile" sed -i -e '/env.user = "ubuntu"/d' ec2/biolinux/fabfile.py fi fi cd "$tmpdir"/ec2/biolinux/ # Fabric relies on an ssh connection, but this script is local and running as root, # so I need to allow root logins with SSH and get myself a password-less login. # What a faff ssh-keygen -t rsa -N '' -C 'localhost passwordless login key' -f ~/.ssh/id_local_rsa ( echo -n 'from="localhost,::1,127.0.0.0/24" ' ; cat ~/.ssh/id_local_rsa.pub ) >> ~/.ssh/authorized_keys2 /usr/sbin/sshd -o PermitRootLogin=without-password -o ListenAddress=localhost:2232 -o AllowGroups=root -o PidFile=/var/run/sshd_local.pid -p 2232 # Run Fabric on localhost #fab -i ~/.ssh/id_local_rsa -H localhost:2232 uname_a fab -i ~/.ssh/id_local_rsa -H localhost:2232 install_biolinux # Clean up keys and sshd rm ~/.ssh/id_local_rsa.pub ~/.ssh/id_local_rsa ~/.ssh/authorized_keys2 xargs kill -TERM < /var/run/sshd_local.pid # Say what we did echo "All finished. You probably want to reboot now." ================================================ FILE: utils/cbl_exome_setup.py ================================================ #!/usr/bin/env python """Automate the final steps of configuring the CloudBioLinux exome example. - Updates configuration file with server details. - Provides links to custom Galaxy instance - Install latest processing pipeline - Start toplevel processing server to run exome analysis. - Adds RabbitMQ user and virtual host with correct permissions. Run the script with sudo or the root user. """ import os import time import shutil import socket import subprocess import contextlib import ConfigParser import yaml def main(): config_dir = "/export/data/galaxy" work_dir = "/export/data/work" work_user = "galaxy" amqp_config = os.path.join(config_dir, "universe_wsgi.ini") pp_config = os.path.join(config_dir, "post_process.yaml") wait_until_mounted(amqp_config) update_amqp_config(amqp_config, socket.getfqdn()) amqp_user, amqp_pass = read_ampq_config(amqp_config) amqp_vhost = read_pp_config(pp_config) install_latest_pipeline() setup_custom_galaxy() run_nextgen_analysis_server(pp_config, work_dir, work_user) setup_rabbitmq(amqp_vhost, amqp_user, amqp_pass) def setup_custom_galaxy(): """Provide links to custom Galaxy instance on shared volume. """ galaxy_path = "/mnt/galaxyTools/galaxy-central" custom_galaxy_path = "/mnt/galaxyData/galaxy-central-hbc" storage_dir = "/export/data/upload/storage" if not os.path.exists(galaxy_path): subprocess.check_call(["mkdir", "-p", os.path.split(galaxy_path)[0]]) subprocess.check_call(["ln", "-s", custom_galaxy_path, galaxy_path]) subprocess.check_call(["chown", "-R", "galaxy:galaxy", os.path.split(galaxy_path)[0]]) subprocess.check_call(["chmod", "a+rwx", storage_dir]) def install_latest_pipeline(): url = "git://github.com/chapmanb/bcbb.git" tmpdir = "/tmp" with chdir(tmpdir): subprocess.check_call(["git", "clone", url]) with chdir(os.path.join(tmpdir, "bcbb", "nextgen")): subprocess.check_call(["python", "setup.py", "install"]) shutil.rmtree("bcbb") UPSTART_SCRIPT = """ description "Nextgen sequencing analysis server" start on runlevel [2345] pre-start script mkdir -p {work_dir} chown -R {user} {work_dir} end script exec su -l -c 'nextgen_analysis_server.py -q toplevel {config_file} --basedir={work_dir}' {user} """ def run_nextgen_analysis_server(pp_config, work_dir, work_user): """Run a nextgen sequencing server using Ubuntu upstart. """ upstart_file = "/etc/init/nextgen-analysis.conf" with open(upstart_file, "w") as out_handle: out_handle.write(UPSTART_SCRIPT.format(config_file=pp_config, work_dir=work_dir, user=work_user)) subprocess.check_call(["service", "nextgen-analysis", "start"]) def setup_rabbitmq(vhost, user, passwd): """Add virtual host, user and password to RabbitMQ. """ base_cl = ["rabbitmqctl"] subprocess.check_call(base_cl + ["add_user", user, passwd]) subprocess.check_call(base_cl + ["add_vhost", vhost]) subprocess.check_call(base_cl + ["set_permissions", "-p", vhost, user, '.*', '.*', '.*']) def read_pp_config(fname): """Read AMQP vhost from YAML configuration file. """ with open(fname) as in_handle: config = yaml.safe_load(in_handle) return config["distributed"]["rabbitmq_vhost"] def read_ampq_config(fname): """Get AMQP username and password from configuration file """ config = ConfigParser.ConfigParser() config.read(fname) return (config.get("galaxy_amqp", "userid"), config.get("galaxy_amqp", "password")) def update_amqp_config(fname, hostname): """Update AMQP configuration with internal hostname. """ orig_stat = os.stat(fname) backup_file = "{0}.bak".format(fname) shutil.move(fname, backup_file) with open(backup_file) as in_handle: with open(fname, "w") as out_handle: in_amqp = False for line in in_handle: if line.startswith("[galaxy_amqp]"): in_amqp = True if in_amqp and line.startswith("host ="): line = "host = {0}\n".format(hostname) in_amqp = False out_handle.write(line) # make updated file readable by initial user os.chown(fname, orig_stat.st_uid, orig_stat.st_gid) # ## Utilities def wait_until_mounted(fname): """Wait up to 3 minutes for mounted directory with file. """ max_tries = 36 wait_sec = 5 num_tries = 0 while 1: if os.path.exists(fname): break elif num_tries > max_tries: raise ValueError("Did not find {f} after {s} seconds.".format( f=fname, s=max_tries*wait_sec)) time.sleep(wait_sec) num_tries += 1 @contextlib.contextmanager def chdir(new_dir): """Context manager to temporarily change to a new directory. http://lucentbeing.com/blog/context-managers-and-the-with-statement-in-python/ """ cur_dir = os.getcwd() os.chdir(new_dir) try: yield finally: os.chdir(cur_dir) if __name__ == "__main__": main() ================================================ FILE: utils/cbl_installed_software.py ================================================ #!/usr/bin/env python """Provide dump of software and libraries installed on CloudBioLinux image. Run from the top level of the cloudbiolinux source directory: python utils/cbl_installed_software.py """ import os from cloudbio import manifest def main(): out_dir = os.path.join(os.getcwd(), "manifest") manifest.create(out_dir) if __name__ == "__main__": main() ================================================ FILE: utils/convert_to_xz.py ================================================ #!/usr/bin/env python """Convert gzipped files on s3 biodata to xz compression format. This conversion is designed to save time and space for download. Some download utilities to speed things up: axel, aria2, lftp """ import os import sys import socket import subprocess import boto import fabric.api as fabric def main(bucket_name): conn = boto.connect_s3() bucket = conn.get_bucket("biodata") for s3_item in bucket.list("genomes/"): if s3_item.name.endswith(".gz"): print "xzipping", s3_item.name local_file = os.path.basename(s3_item.name) local_xz = "%s.xz" % os.path.splitext(local_file)[0] if not os.path.exists(local_xz): if not os.path.exists(local_file): download_parallel(s3_item.generate_url(7200)) #s3_item.get_contents_to_filename(local_file) local_xz = gzip_to_xz(local_file) swap_s3_item(local_xz, bucket, s3_item) os.remove(local_xz) def download_parallel(url): host = socket.gethostbyaddr(socket.gethostname())[0] user = os.environ["USER"] with fabric.settings(host_string="%s@%s" % (user, host)): ncores = fabric.run("cat /proc/cpuinfo | grep processor | wc -l") with fabric.cd(os.getcwd()): fabric.run("axel -a -n %s '%s'" % (ncores, url), shell=False) #fabric.run("aria2c -j %s -s %s '%s'" % (ncores, ncores, url), # shell=False) def swap_s3_item(xz_file, bucket, orig_s3_item): print " Uploading to S3" assert os.path.exists(xz_file) new_name = orig_s3_item.name.replace(".gz", ".xz") upload_script = os.path.join(os.path.dirname(__file__), "s3_multipart_upload.py") cl = ["python2.6", upload_script, xz_file, bucket.name, new_name] subprocess.check_call(cl) orig_s3_item.delete() def gzip_to_xz(local_file): cl = ["gunzip", local_file] subprocess.check_call(cl) tar_file, _ = os.path.splitext(local_file) cl = ["xz", "-z", tar_file] subprocess.check_call(cl) return "%s.xz" % tar_file if __name__ == "__main__": bucket_name = "biodata" main(bucket_name) ================================================ FILE: utils/cwl2yaml_packages.py ================================================ #!/usr/bin/env python # Parses CWL's "SoftwareRequirement" hints section and dumps a cbl compatible yaml file. # The purpose with this script is to create smaller composable docker containers for bcbio-nextgen. # # Usage: cwl2yaml_packages.py test_bcbio_cwl/run_info-cwl-workflow/steps/process_alignment.cwl > cloudbiolinux/contrib/flavor/cwl_dockers/packages-bcbio-alignment.yaml import os import sys import yaml CWL_STEPS=sys.argv[1] cwl_pkgs=yaml.safe_load(open(CWL_STEPS,'r')) cbl_yml=dict() cbl_pkgs=[] # take the filename as the flavor/dockerfile name cbl_flavor="bcbio-"+os.path.splitext(os.path.basename(sys.argv[1]))[0] for pkg in cwl_pkgs['hints'][1]['packages']: cbl_pkgs.append(pkg['package']) cbl_yml['channels']=['bioconda', 'conda-forge'] cbl_yml[cbl_flavor]=cbl_pkgs #print cbl_yml print yaml.safe_dump(cbl_yml, default_flow_style=False, indent=4) ================================================ FILE: utils/get_biolinux_packages.py ================================================ """Scrape the Biolinux website to retrieve a list of packages they install. http://www.jcvi.org/cms/research/projects/jcvi-cloud-biolinux/included-software This needs to run on a machine with an apt system to check for the existance of package names. """ import sys import urllib2 import re import subprocess import StringIO from BeautifulSoup import BeautifulSoup def main(): url = "http://www.jcvi.org/cms/research/projects/jcvi-cloud-biolinux/included-software" in_handle = urllib2.urlopen(url) soup = BeautifulSoup(in_handle) tables = soup.findAll("table", {"class": "contenttable"}) to_check = [] for t in tables: for row in soup.findAll("tr", {"class" : re.compile("tableRow.*")}): for i, item in enumerate(row.findAll("p", {"class": "bodytext"})): if i == 0: to_check.append(str(item.contents[0])) to_check = list(set(to_check)) packages = [get_package(n) for n in to_check] not_ported = [to_check[i] for i, p in enumerate(packages) if p is None] packages = [p for p in packages if p] print len(to_check), len(packages) with open("biolinux-packages.txt", "w") as out_handle: out_handle.write("\n".join(sorted(packages))) with open("biolinux-missing.txt", "w") as out_handle: out_handle.write("\n".join(sorted(not_ported))) def get_package(pname): """Try and retrieve a standard or biolinux package for the package name. """ # custom hacking for painfully general names that take forever if pname in ["act", "documentation"]: pname = "bio-linux-%s" % pname print 'In', pname cl = subprocess.Popen(["apt-cache", "search", pname], stdout=subprocess.PIPE) cl.wait() for line in cl.stdout.read().split(): package = line.split()[0] if package == pname or package == "bio-linux-%s" % pname: print 'Out', package return package return None if __name__ == "__main__": main(*sys.argv[1:]) ================================================ FILE: utils/get_yum_packages.py ================================================ """Convert list of apt packages to matching yum packages. This needs to run on a machine with yum to check for the existance of package names. """ import os import re import sys import subprocess import platform from contextlib import nested import StringIO def main(orig_file): new_file = "%s-yum%s" % os.path.splitext(orig_file) with nested(open(orig_file), open(new_file, "w")) as \ (orig_handle, new_handle): for line in orig_handle: if line.lstrip().startswith("- "): base, orig_package = line.split("- ") yum_package = get_yum_package(orig_package.strip()) if yum_package: new_handle.write("%s- %s\n" % (base, yum_package)) else: new_handle.write(line) def get_yum_package(pname): print 'In', pname # hacks for package names that cause it to hang if pname in ["ri"]: return None elif pname in ["perl"]: return pname cl = subprocess.Popen(["yum", "search", pname], stdout=subprocess.PIPE) cl.wait() arch_pname = "%s.%s" % (pname, platform.machine()) for line in cl.stdout.read().split("\n"): if line.startswith(arch_pname): return pname return None if __name__ == "__main__": main(*sys.argv[1:]) ================================================ FILE: utils/images_and_snapshots.py ================================================ import boto import collections OWNER = '678711657553' # Brad's owner ID def images_and_snapshots(owner): """Retrieve Biolinux image and snapshot information. """ conn = boto.connect_ec2() images = conn.get_all_images(owners=[owner]) images32 = _sorted_images(images, "CloudBioLinux 32") images64 = _sorted_images(images, "CloudBioLinux 64") datalibs = _data_libraries(conn, owner) print images32 print images64 print datalibs def _data_libraries(conn, owner): library_types = collections.defaultdict(list) snaps = conn.get_all_snapshots(owner=owner) for snap in snaps: if snap.description.startswith("CloudBioLinux Data"): # the type is everything except the start and date data_type = " ".join(snap.description.split()[2:-1]) library_types[data_type].append(snap) final = dict() for name, snaps in library_types.iteritems(): snaps = [(s.description, s) for s in snaps] snaps.sort(reverse=True) final[name] = [(s.id, d) for (d, s) in snaps] return final def _sorted_images(images, start_name): """Retrieve a sorted list of images with most recent first. """ images = [(i.name, i) for i in images if i.name.startswith(start_name)] images.sort(reverse=True) return [(i.id, name) for (name, i) in images] images_and_snapshots(OWNER) ================================================ FILE: utils/prep_esp_hg38.py ================================================ #!/usr/bin/env python """Prepare hg38 compatible ESP from original download. http://evs.gs.washington.edu/EVS/ The Download is a tarball of individual VCF files with hg38 coordinates in an INFO key. Extract these into a full VCF, then sort, normalize and bgzip. """ import glob import subprocess from bcbio.bam import ref from bcbio.variation import vcfutils from bcbio.heterogeneity import chromhacks def main(): url = "http://evs.gs.washington.edu/evs_bulk_data/ESP6500SI-V2-SSA137.GRCh38-liftover.snps_indels.vcf.tar.gz" ref_file = "../seq/hg38.fa" subprocess.check_call("wget -c -O esp-orig.tar.gz {url}".format(**locals()), shell=True) subprocess.check_call("tar -xzvpf esp-orig.tar.gz", shell=True) raw_file = "esp-raw.vcf" with open(raw_file, "w") as out_handle: for i, chrom in enumerate(list(range(1, 23)) + ["X", "Y"]): fnames = glob.glob("*chr%s.snps_indels.vcf" % chrom) assert len(fnames) == 1, (chrom, fnames) with open(fnames[0]) as in_handle: for line in in_handle: if line.startswith("#"): if i == 0: if line.startswith("#CHROM"): _add_contigs(out_handle, ref_file) out_handle.write(line) else: parts = line.strip().split("\t") key, val = parts[-1].split(";")[-1].split("=") assert key == "GRCh38_POSITION" if val != "-1": new_chrom, new_pos = val.split(":") if chromhacks.is_autosomal_or_sex(new_chrom): parts[0] = "chr%s" % new_chrom parts[1] = new_pos out_handle.write("\t".join(parts) + "\n") out_file = "ESP6500SI-V2-hg38.vcf.gz" subprocess.check_call(("vt sort {raw_file} | vt decompose -s - | " "vt normalize -n -r {ref_file} - | bgzip -c > {out_file}").format(**locals()), shell=True) vcfutils.bgzip_and_index(out_file) def _add_contigs(out_handle, ref_file): for contig in ref.file_contigs(ref_file): if chromhacks.is_autosomal_or_sex(contig.name): out_handle.write("##contig=\n" % (contig.name, contig.size)) if __name__ == "__main__": main() ================================================ FILE: utils/prepare_cosmic.py ================================================ #!/usr/bin/env python """Prepare combined VCF files of COSMIC resource for cancer variant calling. This prepares the specified version and copies the updates into your bcbio installation in the correct locations in the variation directory. Because of licensing restrictions, download from COSMIC requires registration: https://cancer.sanger.ac.uk/cosmic/download Usage: export COSMIC_USER="your@registered.email.edu" export COSMIC_PASS="cosmic_password" bcbio_python prepare_cosmic.py References: http://gatkforums.broadinstitute.org/discussion/2226/cosmic-and-dbsnp-files-for-mutect """ import gzip import logging import os import re import requests import subprocess import sys import tempfile import shutil from argparse import ArgumentParser from bcbio import utils from bcbio.variation import vcfutils logging.basicConfig(format='%(asctime)s [%(levelname).1s] %(message)s', level=logging.INFO) def main(cosmic_version, bcbio_genome_dir, overwrite=False, clean=False): work_dir = utils.safe_makedir(os.path.join(os.getcwd(), "cosmic-prep")) os.chdir(work_dir) for genome_build, bcbio_build, add_chr in [("GRCh37", "GRCh37", False), ("GRCh38", "hg38", True)]: bcbio_base = os.path.join(bcbio_genome_dir, "genomes", "Hsapiens", bcbio_build) installed_file = os.path.join(bcbio_base, "variation", f"cosmic-v{cosmic_version}.vcf.gz") installed_link = os.path.join(bcbio_base, "variation", "cosmic.vcf.gz") logging.info(f"Beginning COSMIC v{cosmic_version} prep for {genome_build}.") if not os.path.exists(bcbio_base): continue if os.path.exists(installed_file): if not overwrite: logging.info(f"{installed_file} exists, please use the --overwrite flag to overwrite the existing files if you want to reinstall.") continue else: logging.info(f"{installed_file} exists, removing.") remove_installed(installed_file, installed_link) bcbio_ref = os.path.join(bcbio_base, "seq", f"{bcbio_build}.fa") cosmic_vcf_files = get_cosmic_vcf_files(genome_build, cosmic_version, clean) sorted_inputs = [] for fname in cosmic_vcf_files: sorted_inputs.append(sort_to_ref(fname, bcbio_ref, add_chr=add_chr)) out_dir = utils.safe_makedir(os.path.join(f"v{cosmic_version}", "bcbio_ready", bcbio_build)) out_file = os.path.join(out_dir, "cosmic.vcf.gz") ready_cosmic = combine_cosmic(sorted_inputs, bcbio_ref, out_file) variation_dir = utils.safe_makedir(os.path.join(bcbio_base, "variation")) utils.copy_plus(ready_cosmic, installed_file) logging.info(f"Created COSMIC v{cosmic_version} resource in {installed_file}.") logging.info(f"Linking {installed_file} as {installed_link}.") make_links(installed_file, installed_link) update_version_file(bcbio_base, cosmic_version) logging.info(f"Finished COSMIC v{cosmic_version} prep for {genome_build}.") # prepare hg19 from the GRCh37 file if bcbio_build == "GRCh37": genome_build = "hg19" logging.info(f"Prepping COSMIC v{cosmic_version} for {genome_build} from the GRCh37 preparation.") bcbio_base = os.path.join(bcbio_genome_dir, "genomes", "Hsapiens", genome_build) if not os.path.exists(bcbio_base): continue if os.path.exists(installed_file): installed_file = os.path.join(bcbio_base, "variation", f"cosmic-v{cosmic_version}.vcf.gz") installed_link = os.path.join(bcbio_base, "variation", "cosmic.vcf.gz") if not overwrite: logging.info(f"{installed_file} exists, please use the --overwrite flag to overwrite the existing files if you want to reinstall.") continue else: logging.info(f"{installed_file} exists, removing.") remove_installed(installed_file, installed_link) out_dir = utils.safe_makedir(os.path.join(f"v{cosmic_version}", "bcbio_ready", genome_build)) out_file = os.path.join(out_dir, f"cosmic-v{cosmic_version}.vcf.gz") logging.info(f"Translating GRCh37 chromosome names to hg19 chromosome names.") hg19_cosmic = map_coords_to_ucsc(ready_cosmic, bcbio_ref, out_file) variation_dir = utils.safe_makedir(os.path.join(bcbio_base, "variation")) utils.copy_plus(hg19_cosmic, installed_file) logging.info(f"Created COSMIC v{cosmic_version} resource in {installed_file}.") logging.info(f"Linking {installed_file} as {installed_link}.") make_links(installed_file, installed_link) update_version_file(bcbio_base, cosmic_version) logging.info(f"Finished COSMIC v{cosmic_version} prep for {genome_build}.") def remove_installed(installed_file, installed_link): logging.info(f"Removing {installed_file}.") if os.path.exists(installed_file): os.remove(installed_file) installed_index = installed_file + ".tbi" logging.info(f"Removing {installed_index}.") if os.path.exists(installed_index): os.remove(installed_index) logging.info(f"Removing {installed_link}.") if os.path.lexists(installed_link): os.remove(installed_link) installed_index = installed_link + ".tbi" logging.info(f"Removing {installed_index}.") if os.path.lexists(installed_index): os.remove(installed_index) def make_links(installed_file, installed_link): if os.path.islink(installed_link): os.remove(installed_link) os.remove(installed_link + ".tbi") os.symlink(os.path.basename(installed_file), installed_link) os.symlink(os.path.basename(installed_file + ".tbi"), installed_link + ".tbi") def map_coords_to_ucsc(grc_cosmic, ref_file, out_file): hg19_ref_file = ref_file.replace("GRCh37", "hg19") if not os.path.exists(out_file): contig_cl = vcfutils.add_contig_to_header_cl(hg19_ref_file, out_file) cmd = ("zcat {grc_cosmic} | " r'sed "s/^\([0-9]\+\)\t/chr\1\t/g" | sed "s/^MT/chrM/g" | sed "s/^X/chrX/g" | sed "s/^Y/chrY/g" ' "| {contig_cl} " "| bgzip -c > {out_file}") subprocess.check_call(cmd.format(**locals()), shell=True) if os.path.exists("%s-header.txt" % utils.splitext_plus(out_file)[0]): os.remove("%s-header.txt" % utils.splitext_plus(out_file)[0]) return vcfutils.bgzip_and_index(out_file, {}) def _rename_to_ucsc(line): chrom, rest = line.split("\t", 1) if chrom == "MT": new_chrom = "chrM" else: new_chrom = "chr%s" % chrom return "%s\t%s" % (new_chrom, rest) def combine_cosmic(fnames, ref_file, out_file): logging.info(f"Combining COSMIC files to {out_file}.") if not os.path.exists(out_file): cmd = ["picard", "MergeVcfs", "O=%s" % out_file, "D=%s" % ref_file.replace(".fa", ".dict")] + \ ["I=%s" % x for x in fnames] + \ ["USE_JDK_DEFLATER=true", "USE_JDK_INFLATER=true", "CREATE_INDEX=false"] subprocess.check_call(cmd) return vcfutils.bgzip_and_index(out_file, {}) def sort_to_ref(fname, ref_file, add_chr): """Match reference genome ordering. """ logging.info(f"Sorting {fname} to match the order of {ref_file}.") out_file = "%s-prep.vcf.gz" % (fname.replace(".vcf.gz", "")) if not os.path.exists(out_file): if add_chr: fix_chrom = r'| sed "s/^\([0-9]\+\)\t/chr\1\t/g" | sed "s/^MT/chrM/g" | sed "s/^X/chrX/g" | sed "s/^Y/chrY/g" ' else: fix_chrom = '' contig_cl = vcfutils.add_contig_to_header_cl(ref_file, out_file) cmd = ("gunzip -c {fname} {fix_chrom} | " "bcftools norm --check-ref s --do-not-normalize -f {ref_file} |" "bcftools view -e 'SNP=1' |" "gsort /dev/stdin {ref_file}.fai | {contig_cl} | " "bgzip -c > {out_file}") subprocess.check_call(cmd.format(**locals()), shell=True) logging.info(f"bgzipping and indexing {out_file}.") return vcfutils.bgzip_and_index(out_file, {}) def get_cosmic_vcf_files(genome_build, cosmic_version, clean): """Retrieve using new authentication based download approach. GRCh38/cosmic/v85/VCF/CosmicCodingMuts.vcf.gz GRCh38/cosmic/v85/VCF/CosmicNonCodingVariants.vcf.gz """ vdir = os.path.join("v%s" % cosmic_version, genome_build) if os.path.exists(vdir): if not clean: logging.info(f"{vdir} files exist, please use the --clean flag to overwrite the existing files if you want to reinstall.") else: logging.info(f"{vdir} exists, removing.") remove_cosmic_directory(vdir) logging.info("Downloading COSMIC VCF files.") url = "https://cancer.sanger.ac.uk/cosmic/file_download/" out_dir = utils.safe_makedir(os.path.join("v%s" % cosmic_version, genome_build)) fnames = [] for ctype in ["CosmicCodingMuts", "CosmicNonCodingVariants"]: filename = os.path.join(out_dir, "%s.vcf.gz" % ctype) if not os.path.exists(filename): filepath = "%s/cosmic/v%s/VCF/%s.vcf.gz" % (genome_build, cosmic_version, ctype) logging.info("Downloading %s" % (url + filepath)) try: r = requests.get(url + filepath, auth=(os.environ["COSMIC_USER"], os.environ["COSMIC_PASS"])) except KeyError as e: print("KeyError: {} not found. Be sure to export your COSMIC_USER and COSMIC_PASS before running in order to download the files".format(e)) raise e download_url = r.json()["url"] r = requests.get(download_url) with open(filename, "wb") as f: f.write(r.content) fnames.append(filename) return fnames def remove_cosmic_directory(installed_directory): logging.info(f"Removing {installed_directory}.") shutil.rmtree(installed_directory) def update_version_file(bcbio_base, version): """ update the version of cosmic used in the versions.csv file, adding it if it does not exist """ versionfile = os.path.join(bcbio_base, "versions.csv") updatedfile = os.path.join(bcbio_base, "versions.csv-tmp") logging.info(f"Updating {versionfile}.") found = False with open(versionfile) as in_handle, open(updatedfile, "w") as out_handle: for line in in_handle: tokens = line.split(",") if tokens[0] != "cosmic": out_handle.write(line) else: # only write it once if not found: out_handle.write(f"cosmic,{version}\n") found = True if not found: out_handle.write(f"cosmic,{version}\n") shutil.move(updatedfile, versionfile) if __name__ == "__main__": parser = ArgumentParser() parser.add_argument("cosmic_version", help="COSMIC version to install.", default="89") parser.add_argument("bcbio_directory", help="Path to bcbio installation. Should contain the 'genomes' directory.") parser.add_argument("--overwrite", action="store_true", default=False, help="Overwrite existing cosmic.vcf.gz files.") parser.add_argument("--clean", action="store_true", default=False, help="Remove pre-downloaded files, if available.") args = parser.parse_args() main(args.cosmic_version, args.bcbio_directory, args.overwrite, args.clean) ================================================ FILE: utils/prepare_dbsnp.py ================================================ """Prepare sorted and consolidated dbSNP resources for mouse mm10/GRCh38 in VCF format. """ import datetime import ftplib import gzip import os import subprocess from argparse import ArgumentParser import re import shutil FTP = "ftp.ncbi.nih.gov" REMOTES = {"mm10": "snp/organisms/mouse_10090/VCF", "canFam3": "snp/organisms/dog_9615/VCF/"} def main(org): work_dir = "tmp-dbsnp-%s" % org if not os.path.exists(work_dir): os.makedirs(work_dir) conn = ftplib.FTP(FTP, "anonymous", "me@example.com") conn.cwd(REMOTES[org]) os.chdir(work_dir) files = [] def add_files(x): if x.endswith("vcf.gz"): files.append(get_file(x, REMOTES[org], conn)) conn.retrlines("NLST", add_files) out_file = "%s-dbSNP-%s.vcf" % (org, datetime.datetime.now().strftime("%Y-%m-%d")) with open(out_file, "w") as out_handle: for i, f in enumerate(karyotype_sort(files)): with gzip.open(f) as in_handle: for line in in_handle: if line.startswith("#"): if i == 0: out_handle.write(line) else: out_handle.write("\t".join(fix_info(fix_chrom(line.rstrip().split("\t")))) + "\n") subprocess.check_call(["bgzip", out_file]) shutil.move(out_file + ".gz", os.path.join(os.pardir, out_file + ".gz")) os.chdir(os.pardir) subprocess.check_call(["tabix", "-p", "vcf", out_file + ".gz"]) shutil.rmtree(work_dir) multi_whitespace = re.compile(r"\s+") def fix_info(parts): """Fix the INFO file to remove whitespace. """ parts[7] = multi_whitespace.sub("_", parts[7]) return parts def fix_chrom(parts): MAX_CHROMOSOMES = 50 if parts[0] in [str(x) for x in range(1, MAX_CHROMOSOMES)] + ["X", "Y"]: new_chrom = "chr%s" % parts[0] elif parts[0] == "MT": new_chrom = "chrM" else: raise NotImplementedError(parts) parts[0] = new_chrom return parts def get_file(x, ftp_dir, conn): if not os.path.exists(x): print "Retrieving %s" % x with open(x, "wb") as out_handle: conn = ftplib.FTP(FTP, "anonymous", "me@example.com") conn.cwd(ftp_dir) conn.retrbinary("RETR %s" % x, out_handle.write) return x def karyotype_sort(xs): """Sort in karyotypic order to work with GATK's defaults. """ def karyotype_keyfn(x): for suffix in [".gz"]: if x.endswith(suffix): x = x[:-len(suffix)] base = os.path.splitext(os.path.basename(x))[0] for prefix in ["chr", "vcf_chr_"]: if base.startswith(prefix): base = base[len(prefix):] parts = base.split("_") try: parts[0] = int(parts[0]) except ValueError: pass # unplaced at the very end if isinstance(parts[0], basestring) and parts[0].startswith(("Un", "Alt", "Multi", "NotOn")): parts.insert(0, "z") # mitochondrial special case -- after X/Y elif parts[0] in ["M", "MT"]: parts.insert(0, "x") # sort random and extra chromosomes after M elif len(parts) > 1: parts.insert(0, "y") return parts return sorted(xs, key=karyotype_keyfn) if __name__ == "__main__": parser = ArgumentParser(description="Prepare a dbSNP file from NCBI.") parser.add_argument("org_build", choices=REMOTES.keys(), help="genome build") args = parser.parse_args() main(args.org_build) ================================================ FILE: utils/prepare_tx_gff.py ================================================ #!/usr/bin/env python """Prepare GFF transcript files for use as input to RNA-seq pipelines Usage, from within the main genome directory of your organism: prepare_tx_gff.py requires these python and external packages which come pre-installed with bcbio using bioconda: mysql-python gffutils requests picard kallisto """ from __future__ import print_function import csv import gzip import os import sys import shutil import collections import datetime import subprocess import tempfile import glob from argparse import ArgumentParser import gffutils import requests try: import MySQLdb except: MySQLdb = None from bcbio.utils import chdir, safe_makedir, file_exists, get_program_python from bcbio.rnaseq.gtf import gtf_to_fasta # ## Version and retrieval details for Ensembl and UCSC ensembl_release = "95" base_ftp = "ftp://ftp.ensembl.org/pub/release-{release}/gtf" supported_oldbuilds = {"GRCh37": "75", "hg19": "75"} build_subsets = {"hg38-noalt": "hg38"} ucsc_db = "genome-mysql.cse.ucsc.edu" ucsc_user = "genome" # Chromosome name remappings thanks to Devon Ryan # https://github.com/dpryan79/ChromosomeMappings manual_remaps = {"hg38": "https://raw.githubusercontent.com/dpryan79/ChromosomeMappings/master/GRCh38_ensembl2UCSC.txt"} def which(program): def is_exe(fpath): return os.path.isfile(fpath) and os.access(fpath, os.X_OK) fpath, fname = os.path.split(program) if fpath: if is_exe(program): return program else: for path in os.environ["PATH"].split(os.pathsep): path = path.strip('"') exe_file = os.path.join(path, program) if is_exe(exe_file): return exe_file return None def manual_ucsc_ensembl_map(org_build): org_build = build_subsets.get(org_build, org_build) requests.packages.urllib3.disable_warnings() r = requests.get(manual_remaps[org_build], verify=False) out = {} for line in r.text.split("\n"): try: ensembl, ucsc = line.split() out[ensembl] = ucsc except ValueError: pass return out def ucsc_ensembl_map_via_download(org_build): """Compare .dict files by md5, then length to compare two builds. """ ensembl_dict_file = get_ensembl_dict(org_build) ucsc_dict_file = get_ucsc_dict(org_build) ensembl_dict = parse_sequence_dict(ensembl_dict_file) ucsc_dict = parse_sequence_dict(ucsc_dict_file) return ensembl_to_ucsc(ensembl_dict, ucsc_dict, org_build) def ensembl_to_ucsc(ensembl_dict, ucsc_dict, org_build): name_map = {} for md5, name in ensembl_dict.items(): if ucsc_dict.get(md5): name_map[name] = ucsc_dict[md5] map_file = "%s-map.csv" % (org_build) with open(map_file, "w") as out_handle: writer = csv.writer(out_handle) writer.writerow(["ensembl", "ucsc"]) for md5, name in ensembl_dict.items(): ucsc = ucsc_dict.get(md5) if ucsc is not None: writer.writerow([name, ucsc]) return name_map def ucsc_ensembl_map_via_query(org_build): """Retrieve UCSC to Ensembl name mappings from UCSC MySQL database. """ org_build = build_subsets.get(org_build, org_build) # if MySQLdb is not installed, figure it out via download if not MySQLdb: return ucsc_ensembl_map_via_download(org_build) db = MySQLdb.connect(host=ucsc_db, user=ucsc_user, db=org_build) cursor = db.cursor() cursor.execute("select * from ucscToEnsembl") ucsc_map = {} for fields in cursor.fetchall(): ucsc = fields[0] ensembl = fields[-1] # workaround for GRCh37/hg19 additional haplotype contigs. # Coordinates differ between builds so do not include these regions. if org_build == "hg19" and "hap" in ucsc: continue else: ucsc_map[ensembl] = ucsc return ucsc_map # taxname: # biomart_name: name of ensembl gene_id on biomart # ucsc_map: # fbase: the base filename for ensembl files using this genome Build = collections.namedtuple("Build", ["taxname", "biomart_name", "ucsc_map", "fbase"]) build_info = { "hg19": Build("homo_sapiens", "hsapiens_gene_ensembl", ucsc_ensembl_map_via_query, "Homo_sapiens.GRCh37." + supported_oldbuilds["GRCh37"]), "GRCh37": Build("homo_sapiens", "hsapiens_gene_ensembl", None, "Homo_sapiens.GRCh37." + supported_oldbuilds["hg19"]), "mm9": Build("mus_musculus", "mmusculus_gene_ensembl", ucsc_ensembl_map_via_query, "Mus_musculus.NCBIM37.67"), "mm10": Build("mus_musculus", "mmusculus_gene_ensembl", ucsc_ensembl_map_via_query, "Mus_musculus.GRCm38." + ensembl_release), "rn5": Build("rattus_norvegicus", None, ucsc_ensembl_map_via_download, "Rattus_norvegicus.Rnor_5.0." + ensembl_release), "rn6": Build("rattus_norvegicus", None, ucsc_ensembl_map_via_download, "Rattus_norvegicus.Rnor_6.0." + ensembl_release), "hg38": Build("homo_sapiens", "hsapiens_gene_ensembl", manual_ucsc_ensembl_map, "Homo_sapiens.GRCh38." + ensembl_release), "hg38-noalt": Build("homo_sapiens", "hsapiens_gene_ensembl", manual_ucsc_ensembl_map, "Homo_sapiens.GRCh38." + ensembl_release), "canFam3": Build("canis_familiaris", None, ucsc_ensembl_map_via_download, "Canis_familiaris.CanFam3.1." + ensembl_release), "sacCer3": Build("saccharomyces_cerevisiae", None, ucsc_ensembl_map_via_download, "Saccharomyces_cerevisiae.R64-1-1." + ensembl_release), "WBcel235": Build("caenorhabditis_elegans", None, ucsc_ensembl_map_via_download, "Caenorhabditis_elegans.WBcel235." + ensembl_release), "dm3": Build("drosophila_melanogaster", None, ucsc_ensembl_map_via_download, "Drosophila_melanogaster.BDGP5." + ensembl_release), "Zv9": Build("danio_rerio", None, ucsc_ensembl_map_via_download, "Danio_rerio.Zv9." + ensembl_release), "GRCz11": Build("danio_rerio", None, None, "Danio_rerio.GRCz11." + ensembl_release), "xenTro3": Build("xenopus_tropicalis", None, ucsc_ensembl_map_via_download, "Xenopus_tropicalis.JGI_4.2." + ensembl_release), "Sscrofa11.1": Build("sus_scrofa", None, None, "Sus_scrofa.Sscrofa11.1." + ensembl_release), } def parse_sequence_dict(fasta_dict): def _tuples_from_line(line): attrs = {} for tag, val in [x.split(":", 1) for x in line.strip().split("\t")[1:]]: attrs[tag] = val return attrs["SN"], attrs["LN"], attrs["M5"] out = {} with open(fasta_dict) as dict_handle: for name, length, md5 in [_tuples_from_line(x) for x in dict_handle if x.startswith("@SQ")]: out[md5] = name return out class SequenceDictParser(object): def __init__(self, fname): self.fname = fname def _get_sequences_in_genome_dict(self): with open(self.fname) as genome_handle: sequences = [self._sequence_from_line(x) for x in genome_handle if "@SQ" in x] return sequences def _sequence_from_line(self, line): name = line.split("\t")[1].split(":")[1] md5 = line.split("\t")[4].split(":")[1] return md5, name def get_ensembl_dict(org_build): genome_dict = org_build + ".dict" if not os.path.exists(genome_dict): org_fa = org_build + ".fa.gz" if not os.path.exists(org_fa): genome = _download_ensembl_genome(org_build) shutil.move(genome, org_fa) genome_dict = make_fasta_dict(org_fa) return genome_dict def get_ucsc_dict(org_build): fa_dict = os.path.join(os.getcwd(), os.pardir, "seq", org_build + ".dict") if not file_exists(fa_dict): fa_file = os.path.splitext(fa_dict)[0] + ".fa" fa_dict = make_fasta_dict(fa_file) return fa_dict def make_fasta_dict(fasta_file): dict_file = os.path.splitext(fasta_file.replace(".fa.gz", ".fa"))[0] + ".dict" if not os.path.exists(dict_file): subprocess.check_call("picard -Xms1g -Xmx3g CreateSequenceDictionary R={fasta_file} " "O={dict_file}".format(**locals()), shell=True) return dict_file def _download_ensembl_genome(org_build): build = build_info[org_build] # reference files do not use the ensembl_release version so split it off fname = os.path.splitext(build.fbase)[0] + ".dna_sm.toplevel.fa.gz" dl_url = ("ftp://ftp.ensembl.org/pub/release-{release}/" "fasta/{taxname}/dna/{fname}").format(release=ensembl_release, taxname=build.taxname, fname=fname) out_file = os.path.basename(dl_url) if not os.path.exists(out_file): subprocess.check_call(["wget", "-c", dl_url]) return out_file def write_version(build=None, gtf_file=None, build_version=None): gtf_file = build.fbase if build else gtf_file gtf_file = os.path.abspath(gtf_file) gtf_file = build_version if build_version else gtf_file version_file = "version.txt" with open(version_file, "w") as out_handle: out_handle.write("Created from: %s" % gtf_file) return version_file # ## Main driver functions def main(org_build, gtf_file, genome_fasta, genome_dir, cores, args): genome_dir = genome_dir if genome_dir else os.curdir build_dir = os.path.abspath(os.path.join(genome_dir, org_build)) work_dir = os.path.join(build_dir, "tmpcbl") safe_makedir(work_dir) ens_version = supported_oldbuilds.get(org_build, ensembl_release) out_dir = os.path.join(build_dir, "rnaseq-%s_%s" % (datetime.datetime.now().strftime("%Y-%m-%d"), ens_version)) tophat_dir = os.path.join(out_dir, "tophat") gtf_file = os.path.abspath(gtf_file) if gtf_file else gtf_file if genome_fasta: genome_fasta = os.path.abspath(genome_fasta) work_fasta = os.path.join(work_dir, os.path.basename(genome_fasta)) if not os.path.exists(work_fasta): shutil.copy(genome_fasta, work_fasta) genome_fasta = work_fasta with chdir(work_dir): if not genome_fasta: genome_fasta = get_genome_fasta(org_build) if not gtf_file: write_version(build=build_info[org_build]) build = build_info[org_build] gtf_file = prepare_tx_gff(build, org_build) else: write_version(gtf_file=gtf_file, build_version=args.buildversion) work_gtf = os.path.join(work_dir, "ref-transcripts.gtf") if not os.path.exists(work_gtf): shutil.copy(gtf_file, work_gtf) gtf_file = work_gtf gtf_file = clean_gtf(gtf_file, genome_fasta) db = _get_gtf_db(gtf_file) os.remove(gtf_file) gtf_file = db_to_gtf(db, gtf_file) gtf_to_refflat(gtf_file) gtf_to_bed(gtf_file) prepare_tx2gene(gtf_file) prepare_dexseq(gtf_file) mask_gff = prepare_mask_gtf(gtf_file) rrna_gtf = prepare_rrna_gtf(gtf_file) if file_exists(rrna_gtf): gtf_to_interval(rrna_gtf, genome_fasta) if args.tophat: prepare_tophat_index(gtf_file, org_build, genome_fasta) transcriptome_fasta = make_transcriptome_fasta(gtf_file, genome_fasta) if args.kallisto: prepare_kallisto_index(transcriptome_fasta, org_build) make_hisat2_splicesites(gtf_file) cleanup(work_dir, out_dir, org_build) rnaseq_dir = os.path.join(build_dir, "rnaseq") if os.path.exists(rnaseq_dir): if os.path.islink(rnaseq_dir): os.unlink(rnaseq_dir) else: shutil.rmtree(rnaseq_dir) os.symlink(out_dir, rnaseq_dir) tar_dirs = [os.path.relpath(out_dir)] tarball = create_tarball(tar_dirs, org_build) def make_hisat2_splicesites(gtf_file): base, _ = os.path.splitext(gtf_file) out_file = os.path.join(base + "-splicesites.txt") executable = get_program_python("hisat2") hisat2_script = os.path.join(os.path.dirname(executable), "hisat2_extract_splice_sites.py") cmd = "{executable} {hisat2_script} {gtf_file} > {out_file}" if file_exists(out_file): return out_file if not file_exists(hisat2_script): return None subprocess.check_call(cmd.format(**locals()), shell=True) return out_file def make_transcriptome_fasta(gtf_file, genome_fasta): base, _ = os.path.splitext(gtf_file) out_file = os.path.join(base + ".fa") out_file = gtf_to_fasta(gtf_file, genome_fasta, out_file=out_file) return out_file def clean_gtf(gtf_file, genome_fasta): """ remove transcripts that have the following properties 1) don't have a corresponding ID in the reference 2) gencode Selenocysteine features which break many downstream tools 3) are not associated with a gene (no gene_id field) """ temp_gtf = tempfile.NamedTemporaryFile(suffix=".gtf").name fa_names = get_fasta_names(genome_fasta) with open(gtf_file) as in_gtf, open(temp_gtf, "w") as out_gtf: for line in in_gtf: if line.startswith("#"): continue # these cause problems with downstream tools and we don't use them if "Selenocysteine" in line: continue if line.split()[0].strip() not in fa_names: continue if 'gene_id' not in line: continue out_gtf.write(line) # shutil.move breaks on some clusters when /tmp and target dir are on different filesystems shutil.copy(temp_gtf, gtf_file) os.remove(temp_gtf) return gtf_file def get_genome_fasta(org_build): fa_path = os.path.abspath(os.path.join(os.curdir, os.pardir, "seq", org_build + ".fa")) return fa_path def get_fasta_names(genome_fasta): fa_dict = genome_fasta + ".fai" if not os.path.exists(fa_dict): subprocess.check_call("samtools faidx %s" % genome_fasta, shell=True) with open(fa_dict) as in_handle: return [line.split("\t")[0] for line in in_handle] def cleanup(work_dir, out_dir, org_build): for fname in [os.path.join(work_dir, org_build + ".dict"), os.path.join(work_dir, org_build + ".fa"), os.path.join(work_dir, org_build + ".fa.gz"), os.path.join(work_dir, org_build + "-map.csv")]: if os.path.exists(fname): os.remove(fname) if os.path.exists(os.path.join(work_dir, "bcbiotx")): shutil.rmtree(os.path.join(work_dir, "bcbiotx")) shutil.move(work_dir, out_dir) def create_tarball(tar_dirs, org_build): str_tar_dirs = " ".join(tar_dirs) tarball = "{org}-{dir}.tar.xz".format(org=org_build, dir=os.path.basename(tar_dirs[0])) if not os.path.exists(tarball): subprocess.check_call("tar -cvpf - {out_dir} | xz -zc - > {tarball}".format( out_dir=str_tar_dirs, tarball=tarball), shell=True) return tarball def upload_to_s3(tarball): upload_script = os.path.join(os.path.dirname(__file__), "s3_multipart_upload.py") subprocess.check_call([sys.executable, upload_script, tarball, "biodata", os.path.join("annotation", os.path.basename(tarball)), "--public"]) def genepred_to_UCSC_table(genepred): header = ["#bin", "name", "chrom", "strand", "txStart", "txEnd", "cdsStart", "cdsEnd", "exonCount", "exonStarts", "exonEnds", "score", "name2", "cdsStartStat", "cdsEndStat", "exonFrames"] out_file = os.path.splitext(genepred)[0] + ".UCSCTable" if file_exists(out_file): return out_file with open(genepred) as in_handle, open(out_file, "w") as out_handle: counter = -1 current_item = None out_handle.write("\t".join(header) + "\n") for l in in_handle: item = l.split("\t")[0] if current_item != item: current_item = item counter = counter + 1 out_handle.write("\t".join([str(counter), l])) return out_file def gtf_to_genepred(gtf): out_file = os.path.splitext(gtf)[0] + ".genePred" if file_exists(out_file): return out_file cmd = "gtfToGenePred -allErrors -ignoreGroupsWithoutExons -genePredExt {gtf} {out_file}" subprocess.check_call(cmd.format(**locals()), shell=True) return out_file def gtf_to_refflat(gtf): out_file = os.path.splitext(gtf)[0] + ".refFlat" if file_exists(out_file): return out_file genepred = gtf_to_genepred(gtf) with open(genepred) as in_handle, open(out_file, "w") as out_handle: for l in in_handle: first = l.split("\t")[0] out_handle.write("\t".join([first, l])) return out_file def gtf_to_bed(gtf): db = _get_gtf_db(gtf) out_file = os.path.splitext(gtf)[0] + ".bed" if file_exists(out_file): return out_file with open(out_file, "w") as out_handle: for feature in db.features_of_type('transcript'): chrom = feature.chrom start = feature.start end = feature.end attributes = feature.attributes.keys() strand = feature.strand name = (feature['gene_name'][0] if 'gene_name' in attributes else feature['gene_id'][0]) line = "\t".join(map(str, [chrom, start, end, name, ".", strand])) out_handle.write(line + "\n") return out_file def _is_selenocysteine(feature): if feature.featuretype == "Selenocysteine": return True return False def db_to_gtf(db, out_file): if file_exists(out_file): return out_file print("Writing out merged GTF file to %s." % out_file) with open(out_file, "w") as out_handle: for feature in db.all_features(): if _is_selenocysteine(feature): continue out_handle.write(str(feature) + "\n") return out_file def make_miso_events(gtf, org_build): genepred = gtf_to_genepred(gtf) genepred = genepred_to_UCSC_table(genepred) pred_dir = tempfile.mkdtemp() miso_dir = os.path.join(os.path.dirname(gtf), "miso") tmp_pred = os.path.join(pred_dir, "ensGene.txt") os.symlink(os.path.abspath(genepred), tmp_pred) make_miso_annotation(pred_dir, miso_dir, org_build) gff_files = glob.glob(os.path.join(miso_dir, "commonshortest", "*.gff3")) cmd = "index_gff --index {f} {prefix}" for f in gff_files: prefix = f.split(".")[0] + "_indexed" if not file_exists(prefix): subprocess.check_call(cmd.format(**locals()), shell=True) def prepare_bowtie_index(genome_fasta, bowtie_dir): if os.path.exists(bowtie_dir + ".1.bt2"): return bowtie_dir safe_makedir(bowtie_dir) cmd = "bowtie2-build {genome_fasta} {bowtie_dir}" subprocess.check_call(cmd.format(**locals()), shell=True) return bowtie_dir def prepare_tophat_index(gtf, org_build, genome_fasta): tophat_dir = os.path.abspath(os.path.join(os.path.dirname(gtf), "tophat", org_build + "_transcriptome")) bowtie_dir = os.path.abspath(os.path.join(os.path.dirname(gtf), os.path.pardir, "bowtie2", org_build)) bowtie_dir = prepare_bowtie_index(genome_fasta, bowtie_dir) out_dir = tempfile.mkdtemp() fastq = _create_dummy_fastq() cmd = ("tophat --transcriptome-index {tophat_dir} -G {gtf} " "-o {out_dir} {bowtie_dir} {fastq}") subprocess.check_call(cmd.format(**locals()), shell=True) shutil.rmtree(out_dir) os.remove(fastq) def prepare_kallisto_index(transcriptome_fasta, org_build): kallisto = which("kallisto") if not kallisto: return None base_dir = os.path.abspath(os.path.dirname(transcriptome_fasta)) kallisto_dir = os.path.join(base_dir, "kallisto") safe_makedir(kallisto_dir) kallisto_index = os.path.join(kallisto_dir, org_build) if not os.path.exists(kallisto_index): cmd = ("kallisto index -i {kallisto_index} {transcriptome_fasta}") subprocess.check_call(cmd.format(**locals()), shell=True) return kallisto_index def prepare_sailfish_index(transcriptome_fasta, org_build): sailfish = which("sailfish") if not sailfish: return None base_dir = os.path.abspath(os.path.dirname(transcriptome_fasta)) sailfish_dir = os.path.join(base_dir, "sailfish") safe_makedir(sailfish_dir) sailfish_index = os.path.join(sailfish_dir, org_build) cmd = ("sailfish index -t {sailfish_index} -o {sailfish_index}") subprocess.check_call(cmd.format(**locals()), shell=True) return sailfish_index def _create_dummy_fastq(): read = ("@HWI-ST333_0178_FC:5:1101:1107:2112#ATCTCG/1\n" "GGNCTTTCCTGCTTCTATGTCTTGATCGCCTGTAGGCAGG\n" "+HWI-ST333_0178_FC:5:1101:1107:2112#ATCTCG/1\n" "[[BS\\a`ceeagfhhhhhaefhcdfhcf`efeg[cg_b__\n") fn = "dummy.fq" with open(fn, "w") as out_handle: out_handle.write(read) return fn def gtf_to_interval(gtf, genome_fasta): fa_dict = make_fasta_dict(genome_fasta) db = _get_gtf_db(gtf) out_file = os.path.splitext(gtf)[0] + ".interval_list" if file_exists(out_file): return out_file with open(out_file, "w") as out_handle: with open(fa_dict) as in_handle: for l in in_handle: out_handle.write(l) for l in db.all_features(): out_handle.write("\t".join([str(l.seqid), str(l.start), str(l.end), str(l.strand), str(l.attributes.get("transcript_id", ["."])[0])]) + "\n") return out_file def prepare_mask_gtf(gtf): """ make a mask file of usually-masked RNA biotypes """ mask_biotype = ["rRNA", "Mt_rRNA", "misc_RNA", "snRNA", "snoRNA", "tRNA", "Mt_tRNA"] mask_chrom = ["MT"] out_file = os.path.join(os.path.dirname(gtf), "ref-transcripts-mask.gtf") if file_exists(out_file): return out_file biotype_lookup = _biotype_lookup_fn(gtf) # if we can't find a biotype column, skip this if not biotype_lookup: return None db = _get_gtf_db(gtf) with open(out_file, "w") as out_handle: for g in db.all_features(): biotype = biotype_lookup(g) if (biotype in mask_biotype) or (g.chrom in mask_chrom): out_handle.write(str(g) + "\n") return out_file def prepare_rrna_gtf(gtf): """ extract out just the rRNA biotypes, for assessing rRNA contamination """ mask_biotype = ["rRNA", "Mt_rRNA", "tRNA", "MT_tRNA"] out_file = os.path.join(os.path.dirname(gtf), "rRNA.gtf") if os.path.exists(out_file): return out_file db = _get_gtf_db(gtf) biotype_lookup = _biotype_lookup_fn(gtf) # if we can't find a biotype column, skip this if not biotype_lookup: return None with open(out_file, "w") as out_handle: for feature in db.all_features(): biotype = biotype_lookup(feature) if biotype in mask_biotype: out_handle.write(str(feature) + "\n") return out_file def prepare_tx2gene(gtf): """ prepare a file mapping transcripts to genes """ db = _get_gtf_db(gtf) out_file = os.path.join(os.path.dirname(gtf), "tx2gene.csv") if file_exists(out_file): return out_file with open(out_file, "w") as out_handle: for transcript in db.features_of_type('transcript'): gene_id = transcript['gene_id'][0] transcript_id = transcript['transcript_id'][0] out_handle.write(",".join([transcript_id, gene_id]) + "\n") return out_file def _biotype_lookup_fn(gtf): """ return a function that will look up the biotype of a feature this checks for either gene_biotype or biotype being set or for the source column to have biotype information """ db = _get_gtf_db(gtf) sources = set([feature.source for feature in db.all_features()]) gene_biotypes = set([feature.attributes.get("gene_biotype", [None])[0] for feature in db.all_features()]) biotypes = set([feature.attributes.get("biotype", [None])[0] for feature in db.all_features()]) if "protein_coding" in sources: return lambda feature: feature.source elif "protein_coding" in biotypes: return lambda feature: feature.attributes.get("biotype", [None])[0] elif "protein_coding" in gene_biotypes: return lambda feature: feature.attributes.get("gene_biotype", [None])[0] else: return None def prepare_tx_gff(build, org_name): """Prepare UCSC ready transcript file given build information. """ ensembl_gff = _download_ensembl_gff(build, org_name) # if we need to do the name remapping if build.ucsc_map: ucsc_name_map = build.ucsc_map(org_name) tx_gff = _remap_gff(ensembl_gff, ucsc_name_map) os.remove(ensembl_gff) else: tx_gff = "ref-transcripts.gtf" os.rename(ensembl_gff, tx_gff) return tx_gff def _remap_gff(base_gff, name_map): """Remap chromosome names to UCSC instead of Ensembl """ out_file = "ref-transcripts.gtf" wrote_missing = set([]) if not os.path.exists(out_file): with open(out_file, "w") as out_handle, \ open(base_gff) as in_handle: for line in in_handle: parts = line.split("\t") ucsc_name = name_map.get(parts[0], None) if ucsc_name: out_handle.write("\t".join([ucsc_name] + parts[1:])) elif parts[0] not in wrote_missing and not line.startswith("#"): print("Missing", parts[0]) wrote_missing.add(parts[0]) return out_file def _download_ensembl_gff(build, org_name): """Given build details, download and extract the relevant ensembl GFF. """ fname = build.fbase + ".gtf.gz" dl_url = "/".join([base_ftp, build.taxname, fname]).format( release=supported_oldbuilds.get(org_name, ensembl_release)) out_file = os.path.splitext(os.path.basename(dl_url))[0] if not os.path.exists(out_file): subprocess.check_call(["wget", dl_url]) subprocess.check_call(["gunzip", os.path.basename(dl_url)]) return out_file def _create_tiny_gffutils_db(gtf_file): _, ext = os.path.splitext(gtf_file) tmp_out = tempfile.NamedTemporaryFile(suffix=".gtf", delete=False).name with open(tmp_out, "w") as out_handle: count = 0 in_handle = open(gtf_file) if ext != ".gz" else gzip.open(gtf_file) for line in in_handle: if count > 1000: break out_handle.write(line) count += 1 in_handle.close() db = gffutils.create_db(tmp_out, dbfn=":memory:", disable_infer_genes=True, disable_infer_transcripts=True, merge_strategy="warning") os.remove(tmp_out) return db def subfeature_handler(f): """ Given a gffutils.Feature object (which does not yet have its ID assigned), figure out what its ID should be. This is intended to be used for CDS, UTR, start_codon, and stop_codon features in the Ensembl release 81 GTF files. I figured a reasonable unique ID would consist of the parent transcript and the feature type, followed by an autoincrementing number. See https://pythonhosted.org/gffutils/database-ids.html#id-spec for details and other options. Grabbed from Ryan Dale: https://www.biostars.org/p/152517/ """ return ''.join( ['autoincrement:', f.attributes['transcript_id'][0], '_', f.featuretype]) def guess_disable_infer_extent(gtf_file): """ guess if we need to use disable the infer gene or transcript extent option when making a gffutils database by making a tiny database of 1000 lines from the original GTF and looking for all of the features """ db = _create_tiny_gffutils_db(gtf_file) features = [x for x in db.featuretypes()] disable_infer_transcript = "transcript" in features disable_infer_gene = "gene" in features return disable_infer_transcript, disable_infer_gene def guess_id_spec(gtf_file): """ guess at the id spec in a GTF file by examining the first 1000 lines assigns unique ids to features that may not have them """ db = _create_tiny_gffutils_db(gtf_file) id_spec = {} attributes = set() for f in db.all_features(): attributes.update(f.attributes) if "gene_id" in attributes: id_spec["gene"] = "gene_id" attributes.remove("gene_id") if "transcript_id" in attributes: id_spec["transcript"] = "transcript_id" attributes.remove("transcript_id") return id_spec def _get_gtf_db(gtf): db_file = gtf + ".db" if not file_exists(db_file): print("Creating gffutils database for %s." % (gtf)) disable_infer_transcripts, disable_infer_genes = guess_disable_infer_extent(gtf) if not disable_infer_transcripts or not disable_infer_genes: print("'transcript' or 'gene' entries not found, so inferring " "their extent. This can be very slow.") id_spec = guess_id_spec(gtf) gffutils.create_db(gtf, dbfn=db_file, disable_infer_genes=disable_infer_genes, disable_infer_transcripts=disable_infer_transcripts, id_spec=id_spec, merge_strategy="create_unique", keep_order=True, verbose=True) return gffutils.FeatureDB(db_file) def _dexseq_preparation_path(): PREP_FILE = "python_scripts/dexseq_prepare_annotation.py" try: cmd = "%s/Rscript -e 'find.package(\"DEXSeq\")'" % os.path.dirname(os.path.realpath(sys.executable)) output = subprocess.check_output(cmd, shell=True) except subprocess.CalledProcessError: return None for line in output.decode().split("\n"): if line.startswith("["): dirname = line.split("[1]")[1].replace("\"", "").strip() path = os.path.join(dirname, PREP_FILE) if os.path.exists(path): return path return None def prepare_dexseq(gtf): out_file = os.path.splitext(gtf)[0] + ".dexseq.gff3" if file_exists(out_file): return out_file dexseq_path = _dexseq_preparation_path() if not dexseq_path: return None executable = get_program_python("htseq-count") cmd = "{executable} {dexseq_path} {gtf} {out_file}" subprocess.check_call(cmd.format(**locals()), shell=True) return out_file if __name__ == "__main__": parser = ArgumentParser(description="Prepare the transcriptome files for an " "organism.") parser.add_argument("-c", "--cores", default=1, help="number of cores to use") parser.add_argument("--gtf", help="Optional GTF file (instead of downloading from Ensembl)", default=None), parser.add_argument("--fasta", help="Optional genomic FASTA file (instead of downloading from Ensembl)", default=None), parser.add_argument("--genome-dir", help=("Optional location of the root genome directory. " "For example --genome-dir=/foo will install the files " "for a Hsapiens hg19 genome to /foo/Hsapiens/hg19.")) parser.add_argument("--tophat", help="Build TopHat indices", default=False, action="store_true") parser.add_argument("--kallisto", help="Build Kallisto indices", default=False, action="store_true") parser.add_argument("--buildversion", help=("Store build information. String should be source_genomebuild." " Examples: Ensembl_94, EnsemblMetazoa_94, FlyBase_23, etc"), default=None) parser.add_argument("organism", help="Short name of organism (for example Hsapiens)") parser.add_argument("org_build", help="Build of organism to run.") args = parser.parse_args() if args.genome_dir: genome_dir = os.path.join(args.genome_dir, args.organism) else: genome_dir = os.curdir main(args.org_build, args.gtf, args.fasta, genome_dir, args.cores, args) ================================================ FILE: utils/prioritize/AZ300.txt ================================================ ABL1 ACVR1B ACVR2A ACVRL1 AGTR2 AJUBA AKT1 AKT1S1 AKT2 AKT3 ALK AMER1 APC AR ARAF ARHGAP35 ARID1A ARID1B ARID2 ARID5B ASXL1 ATM ATR ATRX AXIN2 B4GALT3 BAP1 BCL2L1 BCL9 BRAF BRCA1 BRCA2 BRD2 BRD3 BRD4 BRD9 BRIP1 CBFB CBL CCND1 CCND2 CCND3 CCNE1 CDH1 CDK12 CDK4 CDK6 CDK8 CDK9 CDKN1A CDKN1B CDKN2A CDKN2B CDKN2C CEBPA CHEK1 CHEK2 CRIPAK CRKL CSNK2A1 CTCF CTNNB1 CYP2D6 DCUN1D1 DDR1 DDR2 DEPTOR DNMT3A EGFR EGR3 EIF4A2 ELF3 EP300 EPHA3 EPHB6 EPPK1 ERAS ERBB2 ERBB3 ERBB4 ERCC1 ERCC2 ERRFI1 ESR1 EZH2 FAM175A FBXW7 FGF1 FGF10 FGF12 FGF14 FGF19 FGF2 FGF23 FGF3 FGF4 FGF5 FGF6 FGF7 FGF8 FGF9 FGFR1 FGFR2 FGFR3 FGFR4 FKBP1A FLT3 FOXA1 FOXA2 FOXC1 FOXL2 FRS2 GAS6 GATA3 GNA11 GNA13 GNAQ GNAS H3F3C HGF HIST1H1C HIST1H2BD HNF1A HRAS IDH1 IDH2 IGF1R IL6 INPP4A INPP4B IRAK4 JAK1 JAK2 JAK3 KDM5C KDM6A KDR KEAP1 KIT KMT2B KMT2C KMT2D KRAS LAMP1 LIFR LRRK2 MALAT1 MAP2K1 MAP2K2 MAP2K4 MAP3K1 MAP3K13 MAP3K8 MAPK1 MAPK3 MAPK8IP1 MAPKAP1 MCL1 MDM2 MDM4 MECOM MED12 MET MGA MIR142 MITF MLH1 MLH3 MLST8 MPL MRAS MSH2 MSH3 MSH6 MTOR MUTYH MYC MYCL MYCN MYH9 NAV3 NCOR1 NCOR2 NF1 NF2 NFE2L2 NFE2L3 NKX2-1 NKX2-8 NOTCH1 NPM1 NRAS NRG1 NSD1 PAK1 PALB2 PBRM1 PCBP1 PDGFRA PDPK1 PHF6 PHLPP1 PHLPP2 PIK3C2B PIK3C2G PIK3C3 PIK3CA PIK3CB PIK3CD PIK3CG PIK3R1 PIK3R2 PIM1 PIM2 PIM3 PMS1 PMS2 POLD1 POLE POLQ PPP2R1A PPP2R2A PRKAA1 PRKAA2 PRKAB1 PRX PTCH1 PTEN PTPN11 PTPN6 RAC1 RAD21 RAD51B RAD51C RAD51D RAF1 RASA1 RB1 RBM10 RELN RET RHEB RHOA RICTOR RIT1 ROS1 RPL22 RPL5 RPS6 RPS6KB1 RPS6KB2 RPTOR RUNX1 SETBP1 SETD2 SF3B1 SGK1 SGK2 SGK3 SHOC2 SIN3A SMAD2 SMAD4 SMARCA4 SMARCB1 SMC1A SMC3 SMO SOS1 SOX17 SOX2 SOX9 SPOP SPRED1 SRY STAG2 STAT3 STK11 TAF1 TBL1XR1 TBX3 TERT TET2 TFRC TGFBR2 TIAF1 TLR4 TOP2A TP53 TSC1 TSC2 TSHZ2 TSHZ3 U2AF1 USP9X VEZF1 VHL WEE1 WT1 ZNF217 ================================================ FILE: utils/prioritize/AZ300_with_known.txt ================================================ ABL1 ABL2 AC006372.4 AC016582.2 ACER3 ACVR2A ADAMTS19 ADAMTSL2 ADCY3 ADRBK2 AES AF1Q AFF1 AFF3 AGGF1 AGK AGMO AGTRAP AHCYL1 AKAP12 AKAP13 AKAP6 AKAP8L AKAP9 AKT3 ALK ALO17 ALOXE3 AML1 ANK1 ANKRD11 ANKRD28 ANXA4 AP3B1 ARHGAP30 ARHH ARL6IP4 ARNT ART3 ASIC2 ASXL1 ATAD1 ATE1 ATF1 ATF7IP ATG7 ATIC ATP1B3 ATP8A2 ATRNL1 ATXN7 B4GALT2 BAG4 BAIAP2L1 BAZ1A BAZ2A BBS9 BCAM BCAS3 BCL2 BCL2L11 BCL3 BCL5 BCL6 BCL7A BCL8 BCL9 BCR BEND5 BFSP2 BICC1 BIRC6 BMP8B BMPR1B BPIL3 BRD3 BRD4 BRIP1 BRK1 BSG BTG1 BUB1B C10ORF54 C11ORF48 C12ORF49 C14ORF166 C15ORF21 C15ORF55 C1ORF150 C1S C20ORF112 C2ORF22 C2ORF44 C4ORF28 C6ORF211 C6ORF97 C8ORF34 CACNB1 CADM2 CAMKK1 CAMSAP1 CAMTA1 CAND1 CAPZA2 CARM1 CARS CASC3 CASC8 CASP7 CASP8 CASZ1 CBFA2T1 CBFA2T2 CBFA2T3 CBFB CCAR2 CCDC12 CCDC127 CCDC170 CCDC50 CCDC6 CCDC88A CCNB1 CCND1 CCND2 CCNF CD37 CD4 CD44 CD74 CD79B CD9 CDC27 CDC42EP1 CDC6 CDH15 CDH17 CDK4 CDK5RAP2 CDK6 CDKAL1 CDX2 CEP1 CEP110 CEP170 CEP55 CEP85L CEP89 CFB CHCHD7 CHD1 CHD7 CHDC2 CHEK2 CHGA CHGB CHIC2 CIRH1A CIT CLCA2 CLCN4 CLCN6 CLIC5 CLIP1 CLTC CLTCL1 CLU CMAS CNIH4 CNTRL COL14A1 COL17A1 COX4I2 CPD CPLX2 CPM CPNE8 CPSF4L CPSF6 CREB5 CREBBP CRKL CSGALNACT2 CSMD3 CTB-151G24.1 CTLC CTNNB1 CTTN CUL1 CUX1 CXCL11 D10S170 DAB2 DAZL DBR1 DCC DCTN1 DDR1 DDX39B DDX42 DDX5 DEK DEM1 DERL1 DGKD DHX37 DIAPH3 DIP2C DNAH9 DNAJC7 DNMT1 DNMT3B DOCK8 DSTYK DYX1C1 E2F7 EBF1 EFCAB3 EFNA3 EFTUD1 EGFR EIF2AK1 EIF4A2 ELAVL3 ELKS EML1 EML4 ENO1 EPHA5 EPHB1 EPOR ERBB ERC1 ERG ERLIN2 ERP44 ERVK3-1 ESRP1 ETO ETV1 ETV6 EVI1 EVX1 EWS EWSR1 EXOC6 EXOSC2 EZR FACL6 FAM114A2 FAM119B FAM120AOS FAM131B FAM133B FAM155A FAM230A FAM70B FAM84B FANCC FBXL20 FBXO2 FCHSD1 FER1L6 FGD4 FGF11 FGFR1 FGFR1OP FGFR1OP1 FGFR1OP2 FGFR2 FGFR3 FIG FIP1L1 FKBP10 FKBP15 FKBP9L FMO4 FN1 FNDC3B FOP FOXJ3 FOXO1 FOXP1 FOXP2 FRMD3 FRMD4B FSIP1 FSTL3 FXR1 GABBR2 GAP43 GASP GATAD1 GATM GFAP GLTP GNAI1 GNS GOLGA5 GOPC GOSR1 GPATCH8 GPR124 GRB7 GRHL2 GRIA4 GRM1 GTF2E2 GTF2F2 GTF2IRD1 GXYLT1 H4 HACL1 HDAC7 HELQ HEPACAM2 HERPUD1 HERVK HIP1 HIVEP3 HJURP HLA-A HLXB9 HM13 HMCES HMGA2 HMGN2P46 HMHA1 HOIP HOOK3 HOX11 HOXA9 IFLTD1 IFT20 IFT80 IGF2R IGFBP5 IGH@ IGHG1 IGK@ IGL@ IKZF3 IL1RAP IL27RA IL2RB INPP5D INSR IRF4 IRTA1 IRX2 ITM2B JAK2 JAKMIP1 KAZN KCNAB1 KCNJ4 KCNMA1 KCNMB4 KCTD1 KCTD21 KDELR2 KDM4B KDM7A KIAA0573 KIAA0833 KIAA1432 KIAA1468 KIAA1549 KIAA1549L KIAA1598 KIAA1618 KIAA1967 KIF5A KIF5B KLC1 KLC3 KLHL22 KMT2A KRT20 KRT39 KRT86 KRTAP1-4 KTN1 LAF4 LAMC1 LANCL2 LASP1 LATS1 LCLAT1 LIFR LINC01153 LMNA LMTK3 LNP1 LNX1 LPCAT3 LPXN LRIG3 LRRC37B LRRFIP1 LRRFIP2 LSM14A LTBP4 LYZ LYZL2 MACF1 MACROD1 MAGI3 MAN2B2 MAP4K3 MAPK1 MAPK14 MAPK8IP3 MAPKBP1 MARK3 MARK4 MAST2 MAST3 MBD1 MBNL1 MCCC1 MCF2L MCM9 MDM2 MDS1 MDS2 MECOM MED1 METAP2 METTL12 METTL21B MFGE8 MGEA5 MICALCL MIDN MIEN1 MIR143 MIR143HG MKRN1 MLF1 MLL MLL3 MLLT10 MN1 MOB3A MPG MPRIP MROH2B MRPL13 MSH6 MSL1 MSN MTSS1 MUC6 MYB MYC MYH11 MYH3 MYH9 MYO18A MYO18B MYO3B MYO5A MYO7A MYST3 NAA30 NARS2 NCOA3 NCOA4 NEK4 NFE2 NFIB NKIRAS1 NLE1 NLK NLRP8 NME9 NOS2 NOTCH3 NPM NPM1 NRF1 NRG1 NRIP1 NRROS NSMCE4A NSUN2 NTM NTRK1 NUMA1 NUP107 NUP214 NUP358 NUP98 NUT NUTM1 ODZ4 OFD1 ORC2L OXR1 PAK1 PAK3 PAK4 PAN3 PAPD7 PAPSS1 PARN PAX5 PCCB PCDH11X PCM1 PDE1C PDGFC PDGFD PDK3 PDSS2 PER1 PGC PIGR PIK3CA PIM3 PIP4K2B PIP5K2B PIP5KL1 PITPNA PKM PLAG1 PLAT PLD5 PLEKHA6 PLS1 POLD1 POLDIP2 POLR2J3 POU6F2 PPAPDC1A PPARGC1A PPFIBP1 PPHLN1 PPM1H PPP1CA PPP1R1B PPP2R2A PPP2RA PPP4R1 PPT1 PRAME PRDM10 PRDM12 PRDM16 PRDM7 PRDX4 PRKAG2 PRKAR1A PRKCZ PRKD1 PRODH PRR12 PRRC1 PSMB3 PSMD11 PSMD9 PSPH PTC PTC3 PTC7 PTCH1 PTK2B PTPN3 PTPRZ PTPRZ1 PVT1 PVT1_1 PWWP2A QKI RAB2 RAB6IP2 RAB8A RANBP2 RARA RASA3 RAX RB1CC1 RBM14 RBM35A RBM4 RBPMS RCSD1 RERE RERG RET RFX1 RFX3 RGPD2 RGS17 RHOT1 RIMS3 RLF RNF130 RNF213 RNF31 RNPEP ROCK1 ROS1 RPL22 RPL22P1 RPN1 RPRD1A RPS6KA1 RPS6KA3 RPS6KA6 RPSA RUFY2 RUNX1 RUNX1T1 RUNX2 RUNXBP2 SCAF11 SCARB1 SCO2 SDC4 SDF4 SDK1 SDK2 SEC16A SEC31A SEC31L1 SEC61G SEPT14 SEPT9 SFPQ SH3BP2 SH3D19 SHANK2 SHBG SLC15A2 SLC34A2 SLC35A1 SLC3A2 SLC45A3 SLC4A4 SMAD3 SMARCD2 SMCHD1 SMEK2 SMG6 SMIM18 SND1 SNF8 SNHG7 SNRPB SNX19 SNX2 SOX5 SPAG9 SPECC1L SPINT2 SPTBN2 SQSTM1 SRCIN1 SRGAP3 SRRM5 SSBP2 SSR2 SSU72 ST3GAL2 STIM1 STK4 STK40 STL STRN STRN3 STXBP6 SUDS3 SUMO1 SUPT3H SV2B SYK TACC1 TACC2 TACC3 TADA2A TAF1D TAF4 TAL1 TATDN1 TAX1BP1 TBC1D5 TBL1XR1 TCP1 TCRB@ TEL TENC1 TERF2 TERT TEX14 TFE3 TFEB TFG THADA THOC5 TIAM2 TIF1 TLX3 TMEM131 TMEM165 TMEM40 TMEM49 TMP3 TMP4 TOLLIP TOP3B TP53BP1 TP63 TPM1 TPM3 TPM4 TPP2 TPPP TPR TPRG1 TRA@ TRABD TRAF1 TRAK1 TRB@ TRIM24 TRIM27 TRIM33 TRIM4 TRIO TRIP4 TRMT61B TROP2 TRPC6 TRPS1 TSC22D1 TSFM TSPAN31 TSPEAR TSPO TTBK2 TTC13 TTC18 TTL TUBD1 TULP4 TWSG1 TXLNA TYK2 UBA5 UBC UBE2L3 UBR2 UBR4 ULK4 USP13 USP24 USP42 USP6 USP8 UTP23 UTRN UTS2B UVRAG VAMP2 VCL VKORC1 VMP1 VOPP1 VSTM2A VTCN1 VTI1A WDR11 WDR82 WDR93 WHSC1 WHSC1L1 WRN WWTR1 XBP1 XIST XPC YAP1 YIF1B YTHDF2 ZBTB5 ZC3H7B ZC3HAV1 ZCCHC7 ZCCHC8 ZDHHC1 ZEB2 ZFC3H1 ZFPM2 ZMIZ1 ZMYM2 ZNF198 ZNF251 ZNF384 ZNF576 ZNF577 ZNF643 ZNF684 ZNF687 ZNF703 ZNF791 ZNF804A ZSCAN30 ================================================ FILE: utils/prioritize/az-cancer-panel.txt ================================================ ABL1 ACVR1B ACVR2A ACVRL1 AGTR2 AJUBA AKT1 AKT1S1 AKT2 AKT3 ALK AMER1 APC AR ARAF ARHGAP35 ARID1A ARID1B ARID2 ARID5B ASXL1 ATM ATR ATRX AXIN2 B4GALT3 BAP1 BCL2L1 BCL9 BRAF BRCA1 BRCA2 BRD2 BRD3 BRD4 BRD9 BRIP1 CBFB CBL CCND1 CCND2 CCND3 CCNE1 CDH1 CDK12 CDK4 CDK6 CDK8 CDK9 CDKN1A CDKN1B CDKN2A CDKN2B CDKN2C CEBPA CHEK1 CHEK2 CRIPAK CRKL CSNK2A1 CTCF CTNNB1 CYP2D6 DCUN1D1 DDR1 DDR2 DEPTOR DNMT3A EGFR EGR3 EIF4A2 ELF3 EP300 EPHA3 EPHB6 EPPK1 ERAS ERBB2 ERBB3 ERBB4 ERCC1 ERCC2 ERRFI1 ESR1 EZH2 FAM175A FBXW7 FGF1 FGF10 FGF12 FGF14 FGF19 FGF2 FGF23 FGF3 FGF4 FGF5 FGF6 FGF7 FGF8 FGF9 FGFR1 FGFR2 FGFR3 FGFR4 FKBP1A FLT3 FOXA1 FOXA2 FOXC1 FOXL2 FRS2 GAS6 GATA3 GNA11 GNA13 GNAQ GNAS H3F3C HGF HIST1H1C HIST1H2BD HNF1A HRAS IDH1 IDH2 IGF1R IL6 INPP4A INPP4B IRAK4 JAK1 JAK2 JAK3 KDM5C KDM6A KDR KEAP1 KIT KMT2B KMT2C KMT2D KRAS LAMP1 LIFR LRRK2 MALAT1 MAP2K1 MAP2K2 MAP2K4 MAP3K1 MAP3K13 MAP3K8 MAPK1 MAPK3 MAPK8IP1 MAPKAP1 MCL1 MDM2 MDM4 MECOM MED12 MET MGA MIR142 MITF MLH1 MLH3 MLST8 MPL MRAS MSH2 MSH3 MSH6 MTOR MUTYH MYC MYCL MYCN MYH9 NAV3 NCOR1 NCOR2 NF1 NF2 NFE2L2 NFE2L3 NKX2-1 NKX2-8 NOTCH1 NPM1 NRAS NRG1 NSD1 PAK1 PALB2 PBRM1 PCBP1 PDGFRA PDPK1 PHF6 PHLPP1 PHLPP2 PIK3C2B PIK3C2G PIK3C3 PIK3CA PIK3CB PIK3CD PIK3CG PIK3R1 PIK3R2 PIM1 PIM2 PIM3 PMS1 PMS2 POLD1 POLE POLQ PPP2R1A PPP2R2A PRKAA1 PRKAA2 PRKAB1 PRX PTCH1 PTEN PTPN11 PTPN6 RAC1 RAD21 RAD51B RAD51C RAD51D RAF1 RASA1 RB1 RBM10 RELN RET RHEB RHOA RICTOR RIT1 ROS1 RPL22 RPL5 RPS6 RPS6KB1 RPS6KB2 RPTOR RUNX1 SETBP1 SETD2 SF3B1 SGK1 SGK2 SGK3 SHOC2 SIN3A SMAD2 SMAD4 SMARCA4 SMARCB1 SMC1A SMC3 SMO SOS1 SOX17 SOX2 SOX9 SPOP SPRED1 SRY STAG2 STAT3 STK11 TAF1 TBL1XR1 TBX3 TERT TET2 TFRC TGFBR2 TIAF1 TLR4 TOP2A TP53 TSC1 TSC2 TSHZ2 TSHZ3 U2AF1 USP9X VEZF1 VHL WEE1 WT1 ZNF217 ETV1 ETV4 ETV5 ETV6 EWSR1 KMT2A PDGFRB RARA TMPRSS2 ERG MYB NOTCH2 NTRK1 NTRK2 RSPO2 ================================================ FILE: utils/prioritize/az300_to_bed.py ================================================ #!/usr/bin/env python """Map AZ300 gene name list to transcript regions. This requires a 3 pass approach to get gene names to coordinates: - it first tries against ensemble BED coordinates in a transcript file with the supplied name - it then remaps the name with gene.info to alternative symbols and tries to find those in the transcript file. - Finally, it uses coordinates from gene.info if those exist. """ import os import sys import requests ref_dir = "/human" in_file = sys.argv[1] def read_targets(in_file): targets = set([]) with open(in_file) as in_handle: for line in in_handle: cur_symbol = (line.strip()) targets.add(cur_symbol) return targets def get_gene_info(cur_symbol): chroms = [str(x) for x in range(1, 23)] + ["X", "Y"] fields = "symbol,genomic_pos_hg19" url = "http://mygene.info/v2/query?q=%s&species=human&fields=%s" % (cur_symbol, fields) info = requests.get(url).json() hits = info["hits"] symbols = [x["symbol"] for x in hits] pos = [] for ps in [x["genomic_pos_hg19"] for x in hits if "genomic_pos_hg19" in x]: if not isinstance(ps, (list, tuple)): ps = [ps] for p in ps: if p["chr"] in chroms: pos.append(p) return symbols, pos def find_missing_targets(missing, in_file, genome): missing_file = "%s-%s-missingsymbols.txt" % (os.path.splitext(in_file)[0], genome) out = set([]) if os.path.exists(missing_file): with open(missing_file) as in_handle: for line in in_handle: out.add(line.strip()) else: with open(missing_file, "w") as out_handle: for cur_symbol in missing: symbols, pos = get_gene_info(cur_symbol) if cur_symbol not in symbols and len(symbols) > 0: cur_symbol = symbols[0] out_handle.write("%s\n" % cur_symbol) out.add(cur_symbol) return out def write_from_transcript_file(targets, ref_dir, genome, out_handle): ref_file = os.path.join(ref_dir, genome, "rnaseq", "ref-transcripts.bed") found = set([]) with open(ref_file) as in_handle: for line in in_handle: name = line.split()[3] if name in targets: found.add(name) out_handle.write(line) return targets - found def write_from_remap_names(targets, ref_dir, genome, out_handle, in_file): ref_file = os.path.join(ref_dir, genome, "rnaseq", "ref-transcripts.bed") targets = find_missing_targets(targets, in_file, genome) found = set([]) with open(ref_file) as in_handle: for line in in_handle: name = line.split()[3] if name in targets: found.add(name) out_handle.write(line) return targets - found def write_from_gene_info(targets, genome, out_handle): missing = [] for target in sorted(targets): symbols, pos = get_gene_info(target) if pos: assert isinstance(pos, (list, tuple)) if symbols: target = symbols[0] for p in pos: chrom = "%s%s" % ("chr" if genome == "hg19" else "", p["chr"]) out_handle.write("\t".join([chrom, str(p["start"]), str(p["end"]), target]) + "\n") else: missing.append(target) return missing for genome in ["hg19", "GRCh37", "hg38"]: out_file = "%s-%s.bed" % (os.path.splitext(in_file)[0], genome) with open(out_file, "w") as out_handle: targets = read_targets(in_file) print "total", len(targets) targets = write_from_transcript_file(targets, ref_dir, genome, out_handle) print("after first name pass", len(targets)) targets = write_from_remap_names(targets, ref_dir, genome, out_handle, in_file) print("after rename name pass", len(targets)) targets = write_from_gene_info(targets, genome, out_handle) print("after coordinate retrieval", targets) ================================================ FILE: utils/prioritize/prep_ccds_genes.py ================================================ #!/usr/bin/env python """Prepare adjusted CCDS BED file. Consensus CDS (CCDS https://www.ncbi.nlm.nih.gov/projects/CCDS/CcdsBrowse.cgi) regions with two adjustments: - 2 bps added to internal introns to capture canonical splice acceptor/donor sites - Multiple transcripts from a single gene are merged into a single all inclusive gene entry. """ import subprocess import sys def main(in_file, out_name, fai_file): out_file = "%s.bed" % out_name with open(in_file) as in_handle: with open(out_file, "w") as out_handle: for line in in_handle: gene, chrom, coords, _ = line.split() chrom = get_chrom(chrom) for start, end in split_coords(coords): out_handle.write("\t".join([chrom, start, end, gene]) + "\n") cmd = "gsort {out_file} {fai_file} | bgzip -c > {out_file}.gz" subprocess.check_call(cmd.format(**locals()), shell=True) def get_chrom(chrom): assert chrom in set([str(x) for x in range(1, 23)] + ["X", "Y"]), chrom return "chr%s" % chrom def split_coords(coords): assert coords[0] == "(" assert coords[-1] == ")", coords out = [] for c in coords[1:-1].split(","): start, end = c.split("..") out.append((start, end)) return out if __name__ == "__main__": main(*sys.argv[1:]) ================================================ FILE: utils/prioritize/prep_prioritize_downloads.sh ================================================ #!/bin/bash set -eu -o pipefail version=20181227 civicv=2018-12-27 REF=/mnt/work/bcbio/genomes/Hsapiens mkdir -p hg19/cancer mkdir -p GRCh37/cancer mkdir -p hg38/cancer # prep work to do # bcbio-prioritize create-civic -b GRCh37 # bcbio-prioritize create-civic -b GRCh38 # python az300_to_bed.py AZ300.txt # python az300_to_bed.py AZ300_with_known.txt # python az300_to_bed.py az-cancer-panel.txt # CIViC zcat civic-GRCh37-$civicv.bed.gz | gsort - $REF/GRCh37/seq/GRCh37.fa.fai | bgzip -c > GRCh37/cancer/civic-$civicv.bed.gz tabix -f -p bed GRCh37/cancer/civic-$civicv.bed.gz zcat civic-GRCh37-$civicv.bed.gz | sed 's/^/chr/' | gsort - $REF/hg19/seq/hg19.fa.fai | bgzip -c > hg19/cancer/civic-$civicv.bed.gz tabix -f -p bed hg19/cancer/civic-$civicv.bed.gz zcat civic-GRCh38-$civicv.bed.gz | sed 's/^/chr/' | gsort - $REF/hg38/seq/hg38.fa.fai | bgzip -c > hg38/cancer/civic-$civicv.bed.gz tabix -f -p bed hg38/cancer/civic-$civicv.bed.gz # az300 cat AZ300-hg19.bed | cut -f 1-4 | sort -V -k1,1 -k2,2n | bedtools merge -i - -c 4 -o distinct | bgzip -c > hg19/cancer/az300.bed.gz tabix -f -p bed hg19/cancer/az300.bed.gz cat AZ300-GRCh37.bed | cut -f 1-4 | sort -V -k1,1 -k2,2n | bedtools merge -i - -c 4 -o distinct | bgzip -c > GRCh37/cancer/az300.bed.gz tabix -f -p bed GRCh37/cancer/az300.bed.gz cat AZ300-hg38.bed | cut -f 1-4 | sort -V -k1,1 -k2,2n | bedtools merge -i - -c 4 -o distinct | bgzip -c > hg38/cancer/az300.bed.gz tabix -f -p bed hg38/cancer/az300.bed.gz # az300 with fusions cat AZ300_with_known-hg19.bed | cut -f 1-4 | sort -V -k1,1 -k2,2n | bedtools merge -i - -c 4 -o distinct | bgzip -c > hg19/cancer/az300-with-fusion.bed.gz tabix -f -p bed hg19/cancer/az300-with-fusion.bed.gz cat AZ300_with_known-GRCh37.bed | cut -f 1-4 | sort -V -k1,1 -k2,2n | bedtools merge -i - -c 4 -o distinct | bgzip -c > GRCh37/cancer/az300-with-fusion.bed.gz tabix -f -p bed GRCh37/cancer/az300-with-fusion.bed.gz cat AZ300_with_known-hg38.bed | cut -f 1-4 | sort -V -k1,1 -k2,2n | bedtools merge -i - -c 4 -o distinct | bgzip -c > hg38/cancer/az300-with-fusion.bed.gz tabix -f -p bed hg38/cancer/az300-with-fusion.bed.gz # az-cancer-panel cat az-cancer-panel-hg19.bed | cut -f 1-4 | sort -V -k1,1 -k2,2n | bedtools merge -i - -c 4 -o distinct | bgzip -c > hg19/cancer/az-cancer-panel.bed.gz tabix -f -p bed hg19/cancer/az-cancer-panel.bed.gz cat az-cancer-panel-GRCh37.bed | cut -f 1-4 | sort -V -k1,1 -k2,2n | bedtools merge -i - -c 4 -o distinct | bgzip -c > GRCh37/cancer/az-cancer-panel.bed.gz tabix -f -p bed GRCh37/cancer/az-cancer-panel.bed.gz cat az-cancer-panel-hg38.bed | cut -f 1-4 | sort -V -k1,1 -k2,2n | bedtools merge -i - -c 4 -o distinct | bgzip -c > hg38/cancer/az-cancer-panel.bed.gz tabix -f -p bed hg38/cancer/az-cancer-panel.bed.gz # tar up downloads cd hg19 tar -czvpf ../prioritize-cancer-hg19-$version.tar.gz cancer cd ../GRCh37 tar -czvpf ../prioritize-cancer-GRCh37-$version.tar.gz cancer cd ../hg38 tar -czvpf ../prioritize-cancer-hg38-$version.tar.gz cancer cd .. aws s3 cp prioritize-cancer-hg19-$version.tar.gz s3://biodata/coverage/prioritize/ --acl public-read aws s3 cp prioritize-cancer-GRCh37-$version.tar.gz s3://biodata/coverage/prioritize/ --acl public-read aws s3 cp prioritize-cancer-hg38-$version.tar.gz s3://biodata/coverage/prioritize/ --acl public-read ================================================ FILE: utils/query_conda_deps.py ================================================ #!/usr/bin/env python """Query conda dependencies, summarizing for environment stratification. Helps identify grouping of packages for different sub-environments. """ import sys import requests import yaml def main(config_file): with open(config_file) as in_handle: config = yaml.safe_load(in_handle) channels = config["channels"] channels.reverse() for p in sorted(config["bio_nextgen"]): p = p.split(";")[0] env = "default" for c in channels: deps = get_dependencies(c, p) if deps is not None: if "python" in deps and deps.get("python"): if not any([s[1].startswith("3.") for s in deps["python"]]): env = "python2" break print(p, env) def get_dependencies(channel, package): url = "https://api.anaconda.org/package/%s/%s/files" % (channel, package) out = requests.get(url).json() if isinstance(out, dict) and out.get("error"): return None else: deps = {"python": set([])} for f in out: for d in f["dependencies"]["depends"]: for k in deps.keys(): if d["name"] == k: for s in d["specs"]: deps[k].add(tuple(s)) return deps if __name__ == "__main__": main(sys.argv[1]) ================================================ FILE: utils/s3_multipart_upload.py ================================================ #!/usr/bin/env python """Split large file into multiple pieces for upload to S3. S3 only supports 5Gb files for uploading directly, so for larger CloudBioLinux box images we need to use boto's multipart file support. This parallelizes the task over available cores using multiprocessing. It checks for an up to date version of the file remotely, skipping transfer if found. Note: by default this will look for your default AWS Access Key ID and AWS Secret Access Key you setup via 'aws configure'. You can store additional profiles using 'aws configure --profile ' Usage: s3_multipart_upload.py [] if is not specified, the filename will be used. --norr -- Do not use reduced redundancy storage. --public -- Make uploaded files public. --cores=n -- Number of cores to use for upload --profile -- The alternate AWS profile to use for your keys located in ~/.aws/config Files are stored at cheaper reduced redundancy storage by default. """ import os import sys import glob import subprocess import contextlib import functools import multiprocessing from multiprocessing.pool import IMapIterator from optparse import OptionParser import rfc822 import boto def main(transfer_file, bucket_name, s3_key_name=None, use_rr=True, make_public=True, cores=None, profile=None): if s3_key_name is None: s3_key_name = os.path.basename(transfer_file) if profile is None: conn = boto.connect_s3() else: conn = boto.connect_s3(profile_name=profile) bucket = conn.lookup(bucket_name) if bucket is None: bucket = conn.create_bucket(bucket_name) if s3_has_uptodate_file(bucket, transfer_file, s3_key_name): print "S3 has up to date version of %s in %s. Not transferring." % \ (s3_key_name, bucket.name) return mb_size = os.path.getsize(transfer_file) / 1e6 if mb_size < 50: _standard_transfer(bucket, s3_key_name, transfer_file, use_rr) else: _multipart_upload(bucket, s3_key_name, transfer_file, mb_size, use_rr, cores, profile) s3_key = bucket.get_key(s3_key_name) if make_public: s3_key.set_acl("public-read") def s3_has_uptodate_file(bucket, transfer_file, s3_key_name): """Check if S3 has an existing, up to date version of this file. """ s3_key = bucket.get_key(s3_key_name) if s3_key: s3_size = s3_key.size local_size = os.path.getsize(transfer_file) s3_time = rfc822.mktime_tz(rfc822.parsedate_tz(s3_key.last_modified)) local_time = os.path.getmtime(transfer_file) return s3_size == local_size and s3_time >= local_time return False def upload_cb(complete, total): sys.stdout.write(".") sys.stdout.flush() def _standard_transfer(bucket, s3_key_name, transfer_file, use_rr): print " Upload with standard transfer, not multipart", new_s3_item = bucket.new_key(s3_key_name) new_s3_item.set_contents_from_filename(transfer_file, reduced_redundancy=use_rr, cb=upload_cb, num_cb=10) print def map_wrap(f): @functools.wraps(f) def wrapper(*args, **kwargs): return apply(f, *args, **kwargs) return wrapper def mp_from_ids(mp_id, mp_keyname, mp_bucketname, profile=None): """Get the multipart upload from the bucket and multipart IDs. This allows us to reconstitute a connection to the upload from within multiprocessing functions. """ if profile is None: conn = boto.connect_s3() else: conn = boto.connect_s3(profile_name=profile) bucket = conn.lookup(mp_bucketname) mp = boto.s3.multipart.MultiPartUpload(bucket) mp.key_name = mp_keyname mp.id = mp_id return mp @map_wrap def transfer_part(mp_id, mp_keyname, mp_bucketname, i, part, profile): """Transfer a part of a multipart upload. Designed to be run in parallel. """ mp = mp_from_ids(mp_id, mp_keyname, mp_bucketname, profile) print " Transferring", i, part with open(part) as t_handle: mp.upload_part_from_file(t_handle, i+1) os.remove(part) def _multipart_upload(bucket, s3_key_name, tarball, mb_size, use_rr=True, cores=None, profile=None): """Upload large files using Amazon's multipart upload functionality. """ def split_file(in_file, mb_size, split_num=5): prefix = os.path.join(os.path.dirname(in_file), "%sS3PART" % (os.path.basename(s3_key_name))) # require a split size between 5Mb (AWS minimum) and 250Mb split_size = int(max(min(mb_size / (split_num * 2.0), 250), 5)) if not os.path.exists("%saa" % prefix): cl = ["split", "-b%sm" % split_size, in_file, prefix] subprocess.check_call(cl) return sorted(glob.glob("%s*" % prefix)) mp = bucket.initiate_multipart_upload(s3_key_name, reduced_redundancy=use_rr) with multimap(cores) as pmap: for _ in pmap(transfer_part, ((mp.id, mp.key_name, mp.bucket_name, i, part, profile) for (i, part) in enumerate(split_file(tarball, mb_size, cores)))): pass mp.complete_upload() @contextlib.contextmanager def multimap(cores=None): """Provide multiprocessing imap like function. The context manager handles setting up the pool, worked around interrupt issues and terminating the pool on completion. """ if cores is None: cores = max(multiprocessing.cpu_count() - 1, 1) def wrapper(func): def wrap(self, timeout=None): return func(self, timeout=timeout if timeout is not None else 1e100) return wrap IMapIterator.next = wrapper(IMapIterator.next) pool = multiprocessing.Pool(cores) yield pool.imap pool.terminate() if __name__ == "__main__": parser = OptionParser() parser.add_option("-r", "--norr", dest="use_rr", action="store_false", default=True) parser.add_option("-p", "--public", dest="make_public", action="store_true", default=False) parser.add_option("-c", "--cores", dest="cores", default=multiprocessing.cpu_count()) parser.add_option("--profile", dest="profile") (options, args) = parser.parse_args() if len(args) < 2: print __doc__ sys.exit() kwargs = dict(use_rr=options.use_rr, make_public=options.make_public, cores=int(options.cores), profile=options.profile) main(*args, **kwargs) ================================================ FILE: utils/sv/NA24385_crowd_dels.py ================================================ #!/usr/bin/env python """Prepare truth set of crowd sourced CNVs from GiaB samples. http://biorxiv.org/content/early/2016/12/13/093526 """ import requests from bcbio.variation import vcfutils url = "http://biorxiv.org/content/biorxiv/suppl/2016/12/13/093526.DC1/093526-3.txt" out_base = "NA24385-crowd-dels-%s.bed" r = requests.get(url, stream=True) header = None calls = [] for line in r.iter_lines(): if not header: header = line.rstrip().split() else: cur = dict(zip(header, line.rstrip().split())) pos = (int(cur["chrom"].replace("chr", "")), int(cur["start"])) probs = sorted([(float(cur["%s_prob" % c]), c) for c in ["CN0", "CN1", "CN2"]], reverse=True) cn = probs[0][-1] if probs[0][0] > 0.8 and cn != "CN2": line = "\t".join([cur["chrom"], cur["start"], cur["end"], cur["site_id"], cn]) + "\n" calls.append((pos, line)) calls.sort() hg19_bed = out_base % "hg19" grch37_bed = out_base % "GRCh37" with open(hg19_bed, "w") as hg19_handle: with open(grch37_bed, "w") as grch37_handle: for _, line in calls: hg19_handle.write(line) grch37_handle.write(line.replace("chr", "")) vcfutils.bgzip_and_index(hg19_bed, {}) vcfutils.bgzip_and_index(grch37_bed, {}) ================================================ FILE: utils/sv/NA24385_giab_dels.py ================================================ #!/usr/bin/env python """Prepare SV truth set from NIST integrated calls for NA24385 """ import os import subprocess from bcbio.variation import vcfutils url = "ftp://ftp-trace.ncbi.nlm.nih.gov/giab/ftp/data/AshkenazimTrio/analysis/NIST_DraftIntegratedDeletionsgt19bp_v0.1.8/" base = "GIAB_HG002_del_2tech_filt_%s_noTRs_v0.1.8.bed" sizes = ["50to100bp", "100to1000bp", "1000to3000bp", "gt3000bp"] fai_file = "/human/GRCh37/seq/GRCh37.fa.fai" out_file = "NA24385-GIAB-2tech-dels-v0_1_8-GRCh37.bed" for size in sizes: cur_url = url + base % size subprocess.check_call(["wget", "-c", cur_url]) with open("header.txt", "w") as out_handle: with open(os.path.basename(cur_url)) as in_handle: out_handle.write("# %s" % in_handle.readline()) subprocess.check_call(("cat GIAB_HG002* | grep -v ^X. | gsort /dev/stdin {fai_file} " "| cat header.txt - > {out_file}".format(**locals())), shell=True) out_file_hg19 = out_file.replace("GRCh37", "hg19") with open(out_file) as in_handle: with open(out_file_hg19, "w") as out_handle: for line in in_handle: if not line.startswith("#"): line = "chr" + line out_handle.write(line) vcfutils.bgzip_and_index(out_file, {}) vcfutils.bgzip_and_index(out_file_hg19, {})