Copy disabled (too large)
Download .txt
Showing preview only (51,330K chars total). Download the full file to get everything.
Repository: mohit1997/DeepZip
Branch: master
Commit: 8c35502397a1
Files: 349
Total size: 48.9 MB
Directory structure:
gitextract_jk786e2m/
├── .gitignore
├── LICENSE
├── README.md
├── data/
│ ├── final_log.csv
│ ├── logs_data/
│ │ ├── Caenorhabditis_elegans.WBcel235.dna.chromosome.I/
│ │ │ ├── FC.log.csv
│ │ │ ├── FC_16bit.log.csv
│ │ │ ├── FC_4layer.log.csv
│ │ │ ├── FC_4layer_16bit.log.csv
│ │ │ ├── GRU_multi_16bit.log.csv
│ │ │ ├── LSTM.log.csv
│ │ │ ├── LSTM_multi.log.csv
│ │ │ ├── LSTM_multi_16bit.log.csv
│ │ │ ├── LSTM_multi_big.log.csv
│ │ │ ├── LSTM_multi_bn.log.csv
│ │ │ ├── biGRU.log.csv
│ │ │ ├── biGRU_16bit.log.csv
│ │ │ ├── biLSTM.log.csv
│ │ │ └── biLSTM_16bit.log.csv
│ │ ├── Caenorhabditis_elegans.WBcel235.dna.toplevel/
│ │ │ ├── FC.log.csv
│ │ │ ├── FC_16bit.log.csv
│ │ │ ├── FC_4layer.log.csv
│ │ │ ├── FC_4layer_16bit.log.csv
│ │ │ ├── GRU_multi_16bit.log.csv
│ │ │ ├── LSTM_multi.log.csv
│ │ │ ├── LSTM_multi_16bit.log.csv
│ │ │ ├── LSTM_multi_big.log.csv
│ │ │ ├── LSTM_multi_bn.log.csv
│ │ │ ├── biGRU.log.csv
│ │ │ ├── biGRU_16bit.log.csv
│ │ │ ├── biLSTM.log.csv
│ │ │ └── biLSTM_16bit.log.csv
│ │ ├── HMM20/
│ │ │ ├── FC.log.csv
│ │ │ ├── FC_16bit.log.csv
│ │ │ ├── FC_4layer.log.csv
│ │ │ ├── FC_4layer_16bit.log.csv
│ │ │ ├── GRU_multi_16bit.log.csv
│ │ │ ├── LSTM_multi.log.csv
│ │ │ ├── LSTM_multi_16bit.log.csv
│ │ │ ├── LSTM_multi_big.log.csv
│ │ │ ├── LSTM_multi_bn.log.csv
│ │ │ ├── biGRU.log.csv
│ │ │ ├── biGRU_16bit.log.csv
│ │ │ ├── biLSTM.log.csv
│ │ │ └── biLSTM_16bit.log.csv
│ │ ├── HMM30/
│ │ │ ├── FC.log.csv
│ │ │ ├── FC_16bit.log.csv
│ │ │ ├── FC_4layer.log.csv
│ │ │ ├── FC_4layer_16bit.log.csv
│ │ │ ├── GRU_multi_16bit.log.csv
│ │ │ ├── LSTM_multi.log.csv
│ │ │ ├── LSTM_multi_16bit.log.csv
│ │ │ ├── LSTM_multi_big.log.csv
│ │ │ ├── LSTM_multi_bn.log.csv
│ │ │ ├── biGRU.log.csv
│ │ │ ├── biGRU_16bit.log.csv
│ │ │ ├── biLSTM.log.csv
│ │ │ └── biLSTM_16bit.log.csv
│ │ ├── HMM40/
│ │ │ ├── FC.log.csv
│ │ │ ├── FC_16bit.log.csv
│ │ │ ├── FC_4layer.log.csv
│ │ │ ├── FC_4layer_16bit.log.csv
│ │ │ ├── GRU_multi_16bit.log.csv
│ │ │ ├── LSTM_multi.log.csv
│ │ │ ├── LSTM_multi_16bit.log.csv
│ │ │ ├── LSTM_multi_big.log.csv
│ │ │ ├── LSTM_multi_bn.log.csv
│ │ │ ├── biGRU.log.csv
│ │ │ ├── biGRU_16bit.log.csv
│ │ │ ├── biLSTM.log.csv
│ │ │ └── biLSTM_16bit.log.csv
│ │ ├── PhiX_quality_truncated/
│ │ │ ├── FC_4layer.log.csv
│ │ │ ├── LSTM_multi.log.csv
│ │ │ ├── LSTM_multi_big.log.csv
│ │ │ ├── LSTM_multi_bn.log.csv
│ │ │ ├── biGRU.log.csv
│ │ │ └── biLSTM.log.csv
│ │ ├── chr1/
│ │ │ ├── FC.log.csv
│ │ │ ├── FC_16bit.log.csv
│ │ │ ├── FC_4layer.log.csv
│ │ │ ├── FC_4layer_16bit.log.csv
│ │ │ ├── GRU_multi_16bit.log.csv
│ │ │ ├── LSTM_multi.log.csv
│ │ │ ├── LSTM_multi_16bit.log.csv
│ │ │ ├── LSTM_multi_big.log.csv
│ │ │ ├── LSTM_multi_bn.log.csv
│ │ │ ├── biGRU.log.csv
│ │ │ ├── biGRU_16bit.log.csv
│ │ │ ├── biLSTM.log.csv
│ │ │ └── biLSTM_16bit.log.csv
│ │ ├── enwiki8/
│ │ │ ├── biGRU.log.csv
│ │ │ └── biGRU_big.log.csv
│ │ ├── iid_p10.1/
│ │ │ ├── FC.log.csv
│ │ │ ├── FC_16bit.log.csv
│ │ │ ├── FC_4layer.log.csv
│ │ │ ├── GRU_multi_16bit.log.csv
│ │ │ ├── LSTM_multi.log.csv
│ │ │ ├── LSTM_multi_16bit.log.csv
│ │ │ ├── LSTM_multi_big.log.csv
│ │ │ ├── LSTM_multi_bn.log.csv
│ │ │ ├── biGRU.log.csv
│ │ │ ├── biGRU_16bit.log.csv
│ │ │ ├── biLSTM.log.csv
│ │ │ └── biLSTM_16bit.log.csv
│ │ ├── text8/
│ │ │ ├── FC.log.csv
│ │ │ ├── FC_16bit.log.csv
│ │ │ ├── FC_4layer.log.csv
│ │ │ ├── FC_4layer_16bit.log.csv
│ │ │ ├── FC_4layer_big.log.csv
│ │ │ ├── GRU_multi_16bit.log.csv
│ │ │ ├── GRU_multi_big.log.csv
│ │ │ ├── LSTM_multi.log.csv
│ │ │ ├── LSTM_multi_16bit.log.csv
│ │ │ ├── LSTM_multi_big.log.csv
│ │ │ ├── LSTM_multi_bn.log.csv
│ │ │ ├── biGRU.log.csv
│ │ │ ├── biGRU_16bit.log.csv
│ │ │ ├── biGRU_big.log.csv
│ │ │ ├── biLSTM.log.csv
│ │ │ └── biLSTM_16bit.log.csv
│ │ ├── xor20/
│ │ │ ├── FC.log.csv
│ │ │ ├── FC_16bit.log.csv
│ │ │ ├── FC_4layer.log.csv
│ │ │ ├── FC_4layer_16bit.log.csv
│ │ │ ├── GRU_multi_16bit.log.csv
│ │ │ ├── LSTM_multi.log.csv
│ │ │ ├── LSTM_multi_16bit.log.csv
│ │ │ ├── LSTM_multi_big.log.csv
│ │ │ ├── LSTM_multi_bn.log.csv
│ │ │ ├── biGRU.log.csv
│ │ │ ├── biGRU_16bit.log.csv
│ │ │ ├── biLSTM.log.csv
│ │ │ └── biLSTM_16bit.log.csv
│ │ ├── xor30/
│ │ │ ├── FC.log.csv
│ │ │ ├── FC_16bit.log.csv
│ │ │ ├── FC_4layer.log.csv
│ │ │ ├── FC_4layer_16bit.log.csv
│ │ │ ├── GRU_multi_16bit.log.csv
│ │ │ ├── LSTM_multi.log.csv
│ │ │ ├── LSTM_multi_16bit.log.csv
│ │ │ ├── LSTM_multi_big.log.csv
│ │ │ ├── LSTM_multi_bn.log.csv
│ │ │ ├── biGRU.log.csv
│ │ │ ├── biGRU_16bit.log.csv
│ │ │ ├── biLSTM.log.csv
│ │ │ └── biLSTM_16bit.log.csv
│ │ ├── xor40/
│ │ │ ├── FC.log.csv
│ │ │ ├── FC_16bit.log.csv
│ │ │ ├── FC_4layer.log.csv
│ │ │ ├── FC_4layer_16bit.log.csv
│ │ │ ├── GRU_multi_16bit.log.csv
│ │ │ ├── LSTM_multi.log.csv
│ │ │ ├── LSTM_multi_16bit.log.csv
│ │ │ ├── LSTM_multi_big.log.csv
│ │ │ ├── LSTM_multi_bn.log.csv
│ │ │ ├── biGRU.log.csv
│ │ │ ├── biGRU_16bit.log.csv
│ │ │ ├── biLSTM.log.csv
│ │ │ └── biLSTM_16bit.log.csv
│ │ ├── xor50/
│ │ │ ├── FC.log.csv
│ │ │ ├── FC_16bit.log.csv
│ │ │ ├── FC_4layer.log.csv
│ │ │ ├── FC_4layer_16bit.log.csv
│ │ │ ├── GRU_multi_16bit.log.csv
│ │ │ ├── LSTM_multi.log.csv
│ │ │ ├── LSTM_multi_16bit.log.csv
│ │ │ ├── LSTM_multi_big.log.csv
│ │ │ ├── LSTM_multi_bn.log.csv
│ │ │ ├── biGRU.log.csv
│ │ │ ├── biGRU_16bit.log.csv
│ │ │ ├── biLSTM.log.csv
│ │ │ └── biLSTM_16bit.log.csv
│ │ └── xor60/
│ │ ├── FC_4layer.log.csv
│ │ ├── LSTM_multi.log.csv
│ │ └── biGRU.log.csv
│ ├── parse_new.py
│ ├── parse_wiki.py
│ ├── run_fasta_preprocess.sh
│ ├── run_parser.sh
│ ├── trained_models/
│ │ ├── Caenorhabditis_elegans.WBcel235.dna.chromosome.I/
│ │ │ ├── FC.hdf5
│ │ │ ├── FC_16bit.hdf5
│ │ │ ├── FC_4layer.hdf5
│ │ │ ├── FC_4layer_16bit.hdf5
│ │ │ ├── GRU_multi_16bit.hdf5
│ │ │ ├── LSTM_multi.hdf5
│ │ │ ├── LSTM_multi_16bit.hdf5
│ │ │ ├── LSTM_multi_big.hdf5
│ │ │ ├── LSTM_multi_bn.hdf5
│ │ │ ├── biGRU.hdf5
│ │ │ ├── biGRU_16bit.hdf5
│ │ │ └── biLSTM_16bit.hdf5
│ │ ├── Caenorhabditis_elegans.WBcel235.dna.toplevel/
│ │ │ ├── FC.hdf5
│ │ │ ├── FC_16bit.hdf5
│ │ │ ├── FC_4layer.hdf5
│ │ │ ├── FC_4layer_16bit.hdf5
│ │ │ ├── GRU_multi_16bit.hdf5
│ │ │ ├── LSTM_multi.hdf5
│ │ │ ├── LSTM_multi_16bit.hdf5
│ │ │ ├── LSTM_multi_big.hdf5
│ │ │ ├── LSTM_multi_bn.hdf5
│ │ │ ├── biGRU.hdf5
│ │ │ ├── biGRU_16bit.hdf5
│ │ │ └── biLSTM_16bit.hdf5
│ │ ├── HMM20/
│ │ │ ├── FC.hdf5
│ │ │ ├── FC_16bit.hdf5
│ │ │ ├── FC_4layer.hdf5
│ │ │ ├── FC_4layer_16bit.hdf5
│ │ │ ├── GRU_multi_16bit.hdf5
│ │ │ ├── LSTM_multi.hdf5
│ │ │ ├── LSTM_multi_16bit.hdf5
│ │ │ ├── LSTM_multi_big.hdf5
│ │ │ ├── LSTM_multi_bn.hdf5
│ │ │ ├── biGRU.hdf5
│ │ │ ├── biGRU_16bit.hdf5
│ │ │ └── biLSTM_16bit.hdf5
│ │ ├── HMM30/
│ │ │ ├── FC.hdf5
│ │ │ ├── FC_16bit.hdf5
│ │ │ ├── FC_4layer.hdf5
│ │ │ ├── FC_4layer_16bit.hdf5
│ │ │ ├── GRU_multi_16bit.hdf5
│ │ │ ├── LSTM_multi.hdf5
│ │ │ ├── LSTM_multi_16bit.hdf5
│ │ │ ├── LSTM_multi_big.hdf5
│ │ │ ├── LSTM_multi_bn.hdf5
│ │ │ ├── biGRU.hdf5
│ │ │ ├── biGRU_16bit.hdf5
│ │ │ └── biLSTM_16bit.hdf5
│ │ ├── HMM40/
│ │ │ ├── FC.hdf5
│ │ │ ├── FC_16bit.hdf5
│ │ │ ├── FC_4layer.hdf5
│ │ │ ├── FC_4layer_16bit.hdf5
│ │ │ ├── GRU_multi_16bit.hdf5
│ │ │ ├── LSTM_multi.hdf5
│ │ │ ├── LSTM_multi_16bit.hdf5
│ │ │ ├── LSTM_multi_big.hdf5
│ │ │ ├── LSTM_multi_bn.hdf5
│ │ │ ├── biGRU.hdf5
│ │ │ ├── biGRU_16bit.hdf5
│ │ │ └── biLSTM_16bit.hdf5
│ │ ├── PhiX_quality_truncated/
│ │ │ ├── FC_4layer.hdf5
│ │ │ ├── LSTM_multi.hdf5
│ │ │ ├── LSTM_multi_big.hdf5
│ │ │ ├── LSTM_multi_bn.hdf5
│ │ │ └── biGRU.hdf5
│ │ ├── chr1/
│ │ │ ├── FC.hdf5
│ │ │ ├── FC_16bit.hdf5
│ │ │ ├── FC_4layer.hdf5
│ │ │ ├── FC_4layer_16bit.hdf5
│ │ │ ├── GRU_multi_16bit.hdf5
│ │ │ ├── LSTM_multi.hdf5
│ │ │ ├── LSTM_multi_16bit.hdf5
│ │ │ ├── LSTM_multi_big.hdf5
│ │ │ ├── LSTM_multi_bn.hdf5
│ │ │ ├── biGRU.hdf5
│ │ │ ├── biGRU_16bit.hdf5
│ │ │ └── biLSTM_16bit.hdf5
│ │ ├── iid_p10.1/
│ │ │ ├── FC.hdf5
│ │ │ ├── FC_16bit.hdf5
│ │ │ ├── FC_4layer.hdf5
│ │ │ ├── GRU_multi_16bit.hdf5
│ │ │ ├── LSTM_multi.hdf5
│ │ │ ├── LSTM_multi_16bit.hdf5
│ │ │ ├── LSTM_multi_big.hdf5
│ │ │ ├── LSTM_multi_bn.hdf5
│ │ │ ├── biGRU.hdf5
│ │ │ ├── biGRU_16bit.hdf5
│ │ │ └── biLSTM_16bit.hdf5
│ │ ├── text8/
│ │ │ ├── FC.hdf5
│ │ │ ├── FC_16bit.hdf5
│ │ │ ├── FC_4layer.hdf5
│ │ │ ├── FC_4layer_16bit.hdf5
│ │ │ ├── FC_4layer_big.hdf5
│ │ │ ├── GRU_multi_16bit.hdf5
│ │ │ ├── GRU_multi_big.hdf5
│ │ │ ├── LSTM_multi.hdf5
│ │ │ ├── LSTM_multi_16bit.hdf5
│ │ │ ├── LSTM_multi_big.hdf5
│ │ │ ├── LSTM_multi_bn.hdf5
│ │ │ ├── biGRU.hdf5
│ │ │ ├── biGRU_16bit.hdf5
│ │ │ ├── biGRU_big.hdf5
│ │ │ ├── biGRU_big.hdf5.bsc
│ │ │ └── biLSTM_16bit.hdf5
│ │ ├── xor20/
│ │ │ ├── FC.hdf5
│ │ │ ├── FC_16bit.hdf5
│ │ │ ├── FC_4layer.hdf5
│ │ │ ├── FC_4layer_16bit.hdf5
│ │ │ ├── GRU_multi_16bit.hdf5
│ │ │ ├── LSTM_multi.hdf5
│ │ │ ├── LSTM_multi_16bit.hdf5
│ │ │ ├── LSTM_multi_big.hdf5
│ │ │ ├── LSTM_multi_bn.hdf5
│ │ │ ├── biGRU.hdf5
│ │ │ ├── biGRU_16bit.hdf5
│ │ │ └── biLSTM_16bit.hdf5
│ │ ├── xor30/
│ │ │ ├── FC.hdf5
│ │ │ ├── FC_16bit.hdf5
│ │ │ ├── FC_4layer.hdf5
│ │ │ ├── FC_4layer_16bit.hdf5
│ │ │ ├── GRU_multi_16bit.hdf5
│ │ │ ├── LSTM_multi.hdf5
│ │ │ ├── LSTM_multi_16bit.hdf5
│ │ │ ├── LSTM_multi_big.hdf5
│ │ │ ├── LSTM_multi_bn.hdf5
│ │ │ ├── biGRU.hdf5
│ │ │ ├── biGRU_16bit.hdf5
│ │ │ └── biLSTM_16bit.hdf5
│ │ ├── xor40/
│ │ │ ├── FC.hdf5
│ │ │ ├── FC_16bit.hdf5
│ │ │ ├── FC_4layer.hdf5
│ │ │ ├── FC_4layer_16bit.hdf5
│ │ │ ├── GRU_multi_16bit.hdf5
│ │ │ ├── LSTM_multi.hdf5
│ │ │ ├── LSTM_multi_16bit.hdf5
│ │ │ ├── LSTM_multi_big.hdf5
│ │ │ ├── LSTM_multi_bn.hdf5
│ │ │ ├── biGRU.hdf5
│ │ │ ├── biGRU_16bit.hdf5
│ │ │ └── biLSTM_16bit.hdf5
│ │ ├── xor50/
│ │ │ ├── FC.hdf5
│ │ │ ├── FC_16bit.hdf5
│ │ │ ├── FC_4layer.hdf5
│ │ │ ├── FC_4layer_16bit.hdf5
│ │ │ ├── GRU_multi_16bit.hdf5
│ │ │ ├── LSTM_multi.hdf5
│ │ │ ├── LSTM_multi_16bit.hdf5
│ │ │ ├── LSTM_multi_big.hdf5
│ │ │ ├── LSTM_multi_bn.hdf5
│ │ │ ├── biGRU.hdf5
│ │ │ ├── biGRU_16bit.hdf5
│ │ │ └── biLSTM_16bit.hdf5
│ │ └── xor60/
│ │ └── biGRU.hdf5
│ ├── xor20.txt
│ ├── xor30.txt
│ ├── xor40.txt
│ ├── xor50.txt
│ └── xor60.txt
├── docker/
│ ├── Dockerfile
│ ├── Dockerfile.bak
│ ├── Makefile
│ ├── README.md
│ └── theanorc
├── install.sh
└── src/
├── README.md
├── arithmeticcoding_fast.py
├── compressor.py
├── decompressor.py
├── models.py
├── run_experiments.sh
└── trainer.py
================================================
FILE CONTENTS
================================================
================================================
FILE: .gitignore
================================================
data/files_to_be_compressed/*
data/files_to_be_compressed_test/*
data/processed_files/*
data/processed_files_test/*
data/processed_files_test2/*
data/processed_files_new/*
data/fasta_files/*
data/compressed/*
================================================
FILE: LICENSE
================================================
MIT License
Copyright (c) 2018 Mohit Goyal
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
================================================
FILE: README.md
================================================
# DeepZip
<em>Update: Please checkout our new work [DZip](https://github.com/mohit1997/Dzip-torch) presented at DCC 2021.</em>
## Description
Data compression using neural networks
[DeepZip: Lossless Data Compression using Recurrent Neural Networks](https://arxiv.org/abs/1811.08162)
## Requirements
0. GPU, nvidia-docker (or try alternative installation)
1. python 2/3
2. numpy
3. sklearn
4. keras 2.2.2
5. tensorflow (cpu/gpu) 1.8
(nvidia-docker is currently required to run the code)
A simple way to install and run is to use the docker files provided:
```bash
cd docker
make bash BACKEND=tensorflow GPU=0 DATA=/path/to/data/
```
## Alternative Installation
```bash
cd DeepZip
python3 -m venv tf
source tf/bin/activate
bash install.sh
```
## Code
To run a compression experiment:
### Data Preparation
1. Place all the data to be compressed in data/files_to_be_compressed
2. Run the parser
```bash
cd data
./run_parser.sh
```
### Running models
1. All the models are listed in models.py
2. Pick a model, to run compression experiment on all the data files in the data/files_to_be_compressed directory
```
cd src
./run_experiments.sh biLSTM GPUID
```
Note: GPUID by default can be set to 0. The corresponding command would be then `./run_experiments.sh biLSTM 0`
### Please cite if you utilize the code in this repository.
```
@inproceedings{7fcb664b03ac4d6497048954d756b91f,
title = "DeepZip: Lossless Data Compression Using Recurrent Neural Networks",
author = "Mohit Goyal and Kedar Tatwawadi and Shubham Chandak and Idoia Ochoa",
year = "2019",
month = "5",
day = "10",
doi = "10.1109/DCC.2019.00087",
language = "English (US)",
series = "Data Compression Conference Proceedings",
publisher = "Institute of Electrical and Electronics Engineers Inc.",
editor = "Ali Bilgin and Storer, {James A.} and Marcellin, {Michael W.} and Joan Serra-Sagrista",
booktitle = "Proceedings - DCC 2019",
address = "United States",
}
```
================================================
FILE: data/final_log.csv
================================================
filename,file_length,gzip,bsc,model,model-size,compressed-size,total-size,status
Caenorhabditis_elegans.WBcel235.dna.chromosome.I,15072434,4033483,3485394,biLSTM_16bit,120272,3535043,3655315,0
Caenorhabditis_elegans.WBcel235.dna.chromosome.I,15072434,4033483,3485394,biGRU,170512,3407318,3577830,0
Caenorhabditis_elegans.WBcel235.dna.chromosome.I,15072434,4033483,3485394,biGRU_16bit,98560,3536457,3635017,0
Caenorhabditis_elegans.WBcel235.dna.chromosome.I,15072434,4033483,3485394,LSTM_multi,614616,3365594,3980210,0
Caenorhabditis_elegans.WBcel235.dna.chromosome.I,15072434,4033483,3485394,LSTM_multi_bn,650960,3431554,4082514,0
Caenorhabditis_elegans.WBcel235.dna.chromosome.I,15072434,4033483,3485394,LSTM_multi_big,1339608,3352195,4691803,0
Caenorhabditis_elegans.WBcel235.dna.chromosome.I,15072434,4033483,3485394,FC,8674080,3379189,12053269,0
Caenorhabditis_elegans.WBcel235.dna.chromosome.I,15072434,4033483,3485394,FC_4layer,390328,3419727,3810055,0
Caenorhabditis_elegans.WBcel235.dna.chromosome.I,15072434,4033483,3485394,LSTM_multi_16bit,317656,3503449,3821105,0
Caenorhabditis_elegans.WBcel235.dna.chromosome.I,15072434,4033483,3485394,GRU_multi_16bit,309464,3497453,3806917,0
Caenorhabditis_elegans.WBcel235.dna.chromosome.I,15072434,4033483,3485394,FC_16bit,4346656,3410094,7756750,0
Caenorhabditis_elegans.WBcel235.dna.chromosome.I,15072434,4033483,3485394,FC_4layer_16bit,207120,3503326,3710446,0
Caenorhabditis_elegans.WBcel235.dna.toplevel,100286401,26965524,23384908,biLSTM_16bit,120272,23313239,23433511,0
Caenorhabditis_elegans.WBcel235.dna.toplevel,100286401,26965524,23384908,biGRU,170512,22960286,23130798,0
Caenorhabditis_elegans.WBcel235.dna.toplevel,100286401,26965524,23384908,biGRU_16bit,98560,23318097,23416657,0
Caenorhabditis_elegans.WBcel235.dna.toplevel,100286401,26965524,23384908,LSTM_multi,614616,22791089,23405705,0
Caenorhabditis_elegans.WBcel235.dna.toplevel,100286401,26965524,23384908,LSTM_multi_bn,650960,22866781,23517741,0
Caenorhabditis_elegans.WBcel235.dna.toplevel,100286401,26965524,23384908,LSTM_multi_big,1339608,22545371,23884979,0
Caenorhabditis_elegans.WBcel235.dna.toplevel,100286401,26965524,23384908,FC,8674080,22768002,31442082,0
Caenorhabditis_elegans.WBcel235.dna.toplevel,100286401,26965524,23384908,FC_4layer,390328,23018182,23408510,0
Caenorhabditis_elegans.WBcel235.dna.toplevel,100286401,26965524,23384908,LSTM_multi_16bit,317656,23175685,23493341,0
Caenorhabditis_elegans.WBcel235.dna.toplevel,100286401,26965524,23384908,GRU_multi_16bit,309464,23143589,23453053,0
Caenorhabditis_elegans.WBcel235.dna.toplevel,100286401,26965524,23384908,FC_16bit,4346656,180018,4526674,1
Caenorhabditis_elegans.WBcel235.dna.toplevel,100286401,26965524,23384908,FC_4layer_16bit,207120,23148382,23355502,0
chr1,249250621,60577992,50425756,biLSTM_16bit,120272,231233,351505,1
chr1,249250621,60577992,50425756,biGRU,171472,48629683,48801155,0
chr1,249250621,60577992,50425756,biGRU_16bit,98688,231233,329921,1
chr1,249250621,60577992,50425756,LSTM_multi,616664,47871371,48488035,0
chr1,249250621,60577992,50425756,LSTM_multi_bn,653040,49014229,49667269,0
chr1,249250621,60577992,50425756,LSTM_multi_big,1339608,47276026,48615634,0
chr1,249250621,60577992,50425756,FC,8675664,47992480,56668144,0
chr1,249250621,60577992,50425756,FC_4layer,390840,48977672,49368512,0
chr1,249250621,60577992,50425756,LSTM_multi_16bit,319704,231233,550937,1
chr1,249250621,60577992,50425756,GRU_multi_16bit,309464,231233,540697,1
chr1,249250621,60577992,50425756,FC_16bit,4346656,231233,4577889,1
chr1,249250621,60577992,50425756,FC_4layer_16bit,207120,231233,438353,1
HMM20,10000000,1486159,867536,biLSTM_16bit,120272,645515,765787,0
HMM20,10000000,1486159,867536,biGRU,170192,593233,763425,0
HMM20,10000000,1486159,867536,biGRU_16bit,98052,1260129,1358181,0
HMM20,10000000,1486159,867536,LSTM_multi,614616,1252326,1866942,0
HMM20,10000000,1486159,867536,LSTM_multi_bn,650960,642520,1293480,0
HMM20,10000000,1486159,867536,LSTM_multi_big,1337560,1260632,2598192,0
HMM20,10000000,1486159,867536,FC,8674080,642727,9316807,0
HMM20,10000000,1486159,867536,FC_4layer,387376,592960,980336,0
HMM20,10000000,1486159,867536,LSTM_multi_16bit,317656,1260877,1578533,0
HMM20,10000000,1486159,867536,GRU_multi_16bit,309464,642451,951915,0
HMM20,10000000,1486159,867536,FC_16bit,4346656,641980,4988636,0
HMM20,10000000,1486159,867536,FC_4layer_16bit,205072,643840,848912,0
HMM30,10000000,1492963,1264186,biLSTM_16bit,120272,1007866,1128138,0
HMM30,10000000,1492963,1264186,biGRU,170192,594692,764884,0
HMM30,10000000,1492963,1264186,biGRU_16bit,98052,993680,1091732,0
HMM30,10000000,1492963,1264186,LSTM_multi,614616,593389,1208005,0
HMM30,10000000,1492963,1264186,LSTM_multi_bn,650960,642618,1293578,0
HMM30,10000000,1492963,1264186,LSTM_multi_big,1337560,1260558,2598118,0
HMM30,10000000,1492963,1264186,FC,8674080,642246,9316326,0
HMM30,10000000,1492963,1264186,FC_4layer,387376,593043,980419,0
HMM30,10000000,1492963,1264186,LSTM_multi_16bit,317656,1260012,1577668,0
HMM30,10000000,1492963,1264186,GRU_multi_16bit,309464,642275,951739,0
HMM30,10000000,1492963,1264186,FC_16bit,4346656,641506,4988162,0
HMM30,10000000,1492963,1264186,FC_4layer_16bit,205072,1260380,1465452,0
HMM40,10000000,1491821,1264292,biLSTM_16bit,120272,1260303,1380575,0
HMM40,10000000,1491821,1264292,biGRU,170192,1252061,1422253,0
HMM40,10000000,1491821,1264292,biGRU_16bit,98052,1260225,1358277,0
HMM40,10000000,1491821,1264292,LSTM_multi,614616,1252034,1866650,0
HMM40,10000000,1491821,1264292,LSTM_multi_bn,650960,642622,1293582,0
HMM40,10000000,1491821,1264292,LSTM_multi_big,1337560,1260763,2598323,0
HMM40,10000000,1491821,1264292,FC,8674080,642284,9316364,0
HMM40,10000000,1491821,1264292,FC_4layer,387376,593214,980590,0
HMM40,10000000,1491821,1264292,LSTM_multi_16bit,317656,1260433,1578089,0
HMM40,10000000,1491821,1264292,GRU_multi_16bit,309464,642180,951644,0
HMM40,10000000,1491821,1264292,FC_16bit,4346656,641616,4988272,0
HMM40,10000000,1491821,1264292,FC_4layer_16bit,205072,1260999,1466071,0
iid_p10.1,10000000,813622,597226,biLSTM_16bit,120272,642243,762515,0
iid_p10.1,10000000,813622,597226,biGRU,170192,593047,763239,0
iid_p10.1,10000000,813622,597226,biGRU_16bit,98052,641984,740036,0
iid_p10.1,10000000,813622,597226,LSTM_multi,614616,593136,1207752,0
iid_p10.1,10000000,813622,597226,LSTM_multi_bn,650960,641974,1292934,0
iid_p10.1,10000000,813622,597226,LSTM_multi_big,1337560,642204,1979764,0
iid_p10.1,10000000,813622,597226,FC,8674080,641915,9315995,0
iid_p10.1,10000000,813622,597226,FC_4layer,387376,593005,980381,0
iid_p10.1,10000000,813622,597226,LSTM_multi_16bit,317656,642301,959957,0
iid_p10.1,10000000,813622,597226,GRU_multi_16bit,309464,641885,951349,0
iid_p10.1,10000000,813622,597226,FC_16bit,4346656,641852,4988508,0
iid_p10.1,10000000,813622,597226,FC_4layer_16bit,0,641900,641900,0
PhiX_quality_truncated,100000000,6216972,4378410,biGRU,170512,4179670,4350182,0
PhiX_quality_truncated,100000000,6216972,4378410,LSTM_multi,614616,4179509,4794125,0
PhiX_quality_truncated,100000000,6216972,4378410,LSTM_multi_bn,650960,4334089,4985049,0
PhiX_quality_truncated,100000000,6216972,4378410,LSTM_multi_big,1339608,4321833,5661441,0
PhiX_quality_truncated,100000000,6216972,4378410,FC_4layer,390328,4186842,4577170,0
text8,100000000,33048234,20949978,biLSTM_16bit,124544,428075,552619,1
text8,100000000,33048234,20949978,biGRU,180864,25473750,25654614,0
text8,100000000,33048234,20949978,biGRU_big,1738948,21627062,23366010,0
text8,100000000,33048234,20949978,biGRU_16bit,103936,428075,532011,1
text8,100000000,33048234,20949978,LSTM_multi,626136,26086049,26712185,0
text8,100000000,33048234,20949978,LSTM_multi_big,1352536,23893228,25245764,0
text8,100000000,33048234,20949978,FC,8684752,22961038,31645790,0
text8,100000000,33048234,20949978,FC_4layer,401400,25088488,25489888,0
text8,100000000,33048234,20949978,LSTM_multi_16bit,323160,428075,751235,1
text8,100000000,33048234,20949978,GRU_multi_16bit,314968,428075,743043,1
text8,100000000,33048234,20949978,FC_16bit,4350416,428075,4778491,1
text8,100000000,33048234,20949978,FC_4layer_16bit,211984,428075,640059,1
xor20,10000000,1513100,60968,biLSTM_16bit,120272,100002,220274,1
xor20,10000000,1513100,60968,biGRU,170192,10002,180194,0
xor20,10000000,1513100,60968,biGRU_16bit,98052,100002,198054,0
xor20,10000000,1513100,60968,LSTM_multi,614616,10002,624618,0
xor20,10000000,1513100,60968,LSTM_multi_big,1337560,1260351,2597911,0
xor20,10000000,1513100,60968,FC,8674080,100002,8774082,0
xor20,10000000,1513100,60968,FC_4layer,387376,10002,397378,0
xor20,10000000,1513100,60968,LSTM_multi_16bit,317656,100002,417658,1
xor20,10000000,1513100,60968,GRU_multi_16bit,309464,100002,409466,0
xor20,10000000,1513100,60968,FC_16bit,4346656,100002,4446658,0
xor20,10000000,1513100,60968,FC_4layer_16bit,205072,100002,305074,1
xor30,10000000,1509435,1264324,biLSTM_16bit,120272,100002,220274,1
xor30,10000000,1509435,1264324,biGRU,170192,10002,180194,0
xor30,10000000,1509435,1264324,biGRU_16bit,98052,100002,198054,1
xor30,10000000,1509435,1264324,LSTM_multi,614616,1252022,1866638,0
xor30,10000000,1509435,1264324,LSTM_multi_big,1337560,1261004,2598564,0
xor30,10000000,1509435,1264324,FC,8674080,100002,8774082,0
xor30,10000000,1509435,1264324,FC_4layer,387376,10002,397378,0
xor30,10000000,1509435,1264324,LSTM_multi_16bit,317656,1259955,1577611,0
xor30,10000000,1509435,1264324,GRU_multi_16bit,309464,100002,409466,0
xor30,10000000,1509435,1264324,FC_16bit,4346656,100002,4446658,0
xor30,10000000,1509435,1264324,FC_4layer_16bit,205072,100002,305074,1
xor40,10000000,1487075,1264298,biLSTM_16bit,120272,1260356,1380628,0
xor40,10000000,1487075,1264298,biGRU,170192,1252017,1422209,0
xor40,10000000,1487075,1264298,biGRU_16bit,98052,1260143,1358195,0
xor40,10000000,1487075,1264298,LSTM_multi,614616,1252000,1866616,0
xor40,10000000,1487075,1264298,LSTM_multi_big,1337560,1260262,2597822,0
xor40,10000000,1487075,1264298,FC,8674080,100002,8774082,0
xor40,10000000,1487075,1264298,FC_4layer,387376,10002,397378,0
xor40,10000000,1487075,1264298,LSTM_multi_16bit,317656,1260398,1578054,0
xor40,10000000,1487075,1264298,GRU_multi_16bit,309464,100002,409466,0
xor40,10000000,1487075,1264298,FC_16bit,4346656,100002,4446658,0
xor40,10000000,1487075,1264298,FC_4layer_16bit,205072,1260572,1465644,0
xor50,10000000,1481899,1264196,biLSTM_16bit,120272,100002,220274,1
xor50,10000000,1481899,1264196,biGRU,170192,10002,180194,0
xor50,10000000,1481899,1264196,biGRU_16bit,98052,1260298,1358350,0
xor50,10000000,1481899,1264196,LSTM_multi,614616,10002,624618,0
xor50,10000000,1481899,1264196,LSTM_multi_big,1337560,1260201,2597761,0
xor50,10000000,1481899,1264196,FC,8674080,100002,8774082,0
xor50,10000000,1481899,1264196,FC_4layer,387376,10002,397378,0
xor50,10000000,1481899,1264196,LSTM_multi_16bit,317656,1260913,1578569,0
xor50,10000000,1481899,1264196,GRU_multi_16bit,309464,100002,409466,0
xor50,10000000,1481899,1264196,FC_16bit,4346656,100002,4446658,0
xor50,10000000,1481899,1264196,FC_4layer_16bit,205072,1261136,1466208,0
xor60,10000000,1501840,1264272,biGRU,170192,10002,180194,0
================================================
FILE: data/logs_data/Caenorhabditis_elegans.WBcel235.dna.chromosome.I/FC.log.csv
================================================
Starting training ...
Starting Compression ...
/opt/conda/lib/python3.6/site-packages/sklearn/externals/joblib/externals/cloudpickle/cloudpickle.py:47: DeprecationWarning: the imp module is deprecated in favour of importlib; see the module's documentation for alternative uses
import imp
Using TensorFlow backend.
/opt/conda/lib/python3.6/site-packages/sklearn/preprocessing/_encoders.py:363: FutureWarning: The handling of integer data will change in version 0.22. Currently, the categories are determined based on the range [0, max(values)], while in the future they will be determined based on the unique values.
If you want the future behaviour and silence this warning, you can specify "categories='auto'".
In case you used a LabelEncoder before this OneHotEncoder to convert the categories to integers, then you can now use the OneHotEncoder directly.
warnings.warn(msg, FutureWarning)
Traceback (most recent call last):
File "compressor.py", line 202, in <module>
main()
File "compressor.py", line 164, in main
predict_lstm(X, Y, Y_original, timesteps, batch_size, alphabet_size, args.model_name)
File "compressor.py", line 68, in predict_lstm
model = getattr(models, model_name)(bs, timesteps, alphabet_size)
AttributeError: module 'models' has no attribute 'FC'
Command exited with non-zero status 1
Command being timed: "python compressor.py -data ../data/processed_files/Caenorhabditis_elegans.WBcel235.dna.chromosome.I.npy -data_params ../data/processed_files/Caenorhabditis_elegans.WBcel235.dna.chromosome.I.param.json -model ../data/trained_models/Caenorhabditis_elegans.WBcel235.dna.chromosome.I/FC.hdf5 -model_name FC -output ../data/compressed/Caenorhabditis_elegans.WBcel235.dna.chromosome.I/FC.compressed -batch_size 10000"
User time (seconds): 14.54
System time (seconds): 3.43
Percent of CPU this job got: 245%
Elapsed (wall clock) time (h:mm:ss or m:ss): 0:07.33
Average shared text size (kbytes): 0
Average unshared data size (kbytes): 0
Average stack size (kbytes): 0
Average total size (kbytes): 0
Maximum resident set size (kbytes): 1635784
Average resident set size (kbytes): 0
Major (requiring I/O) page faults: 0
Minor (reclaiming a frame) page faults: 1353189
Voluntary context switches: 328
Involuntary context switches: 75208
Swaps: 0
File system inputs: 8
File system outputs: 16
Socket messages sent: 0
Socket messages received: 0
Signals delivered: 0
Page size (bytes): 4096
Exit status: 1
/opt/conda/lib/python3.6/site-packages/sklearn/externals/joblib/externals/cloudpickle/cloudpickle.py:47: DeprecationWarning: the imp module is deprecated in favour of importlib; see the module's documentation for alternative uses
import imp
Using TensorFlow backend.
Traceback (most recent call last):
File "decompressor.py", line 184, in <module>
main()
File "decompressor.py", line 153, in main
f = open(args.input_file_prefix+'.combined','rb')
FileNotFoundError: [Errno 2] No such file or directory: '../data/compressed/Caenorhabditis_elegans.WBcel235.dna.chromosome.I/FC.compressed.combined'
Command exited with non-zero status 1
Command being timed: "python decompressor.py -output ../data/compressed/Caenorhabditis_elegans.WBcel235.dna.chromosome.I/FC.reconstructed.txt -model ../data/trained_models/Caenorhabditis_elegans.WBcel235.dna.chromosome.I/FC.hdf5 -model_name FC -input_file_prefix ../data/compressed/Caenorhabditis_elegans.WBcel235.dna.chromosome.I/FC.compressed -batch_size 10000"
User time (seconds): 2.04
System time (seconds): 0.22
Percent of CPU this job got: 100%
Elapsed (wall clock) time (h:mm:ss or m:ss): 0:02.27
Average shared text size (kbytes): 0
Average unshared data size (kbytes): 0
Average stack size (kbytes): 0
Average total size (kbytes): 0
Maximum resident set size (kbytes): 254936
Average resident set size (kbytes): 0
Major (requiring I/O) page faults: 0
Minor (reclaiming a frame) page faults: 68187
Voluntary context switches: 30
Involuntary context switches: 6
Swaps: 0
File system inputs: 0
File system outputs: 8
Socket messages sent: 0
Socket messages received: 0
Signals delivered: 0
Page size (bytes): 4096
Exit status: 1
Starting training ...
Starting Compression ...
Starting training ...
Starting Compression ...
Starting training ...
Starting Compression ...
/opt/conda/lib/python3.6/site-packages/sklearn/externals/joblib/externals/cloudpickle/cloudpickle.py:47: DeprecationWarning: the imp module is deprecated in favour of importlib; see the module's documentation for alternative uses
import imp
Using TensorFlow backend.
/opt/conda/lib/python3.6/site-packages/sklearn/preprocessing/_encoders.py:363: FutureWarning: The handling of integer data will change in version 0.22. Currently, the categories are determined based on the range [0, max(values)], while in the future they will be determined based on the unique values.
If you want the future behaviour and silence this warning, you can specify "categories='auto'".
In case you used a LabelEncoder before this OneHotEncoder to convert the categories to integers, then you can now use the OneHotEncoder directly.
warnings.warn(msg, FutureWarning)
Traceback (most recent call last):
File "compressor.py", line 202, in <module>
main()
File "compressor.py", line 164, in main
predict_lstm(X, Y, Y_original, timesteps, batch_size, alphabet_size, args.model_name)
File "compressor.py", line 68, in predict_lstm
model = getattr(models, model_name)(bs, timesteps, alphabet_size)
File "/data/final_models/models.py", line 165, in FC
model = sequential()
NameError: name 'sequential' is not defined
Command exited with non-zero status 1
Command being timed: "python compressor.py -data ../data/processed_files/Caenorhabditis_elegans.WBcel235.dna.chromosome.I.npy -data_params ../data/processed_files/Caenorhabditis_elegans.WBcel235.dna.chromosome.I.param.json -model ../data/trained_models/Caenorhabditis_elegans.WBcel235.dna.chromosome.I/FC.hdf5 -model_name FC -output ../data/compressed/Caenorhabditis_elegans.WBcel235.dna.chromosome.I/FC.compressed -batch_size 10000"
User time (seconds): 16.01
System time (seconds): 3.53
Percent of CPU this job got: 250%
Elapsed (wall clock) time (h:mm:ss or m:ss): 0:07.81
Average shared text size (kbytes): 0
Average unshared data size (kbytes): 0
Average stack size (kbytes): 0
Average total size (kbytes): 0
Maximum resident set size (kbytes): 1636196
Average resident set size (kbytes): 0
Major (requiring I/O) page faults: 0
Minor (reclaiming a frame) page faults: 1318825
Voluntary context switches: 265
Involuntary context switches: 120
Swaps: 0
File system inputs: 0
File system outputs: 16
Socket messages sent: 0
Socket messages received: 0
Signals delivered: 0
Page size (bytes): 4096
Exit status: 1
/opt/conda/lib/python3.6/site-packages/sklearn/externals/joblib/externals/cloudpickle/cloudpickle.py:47: DeprecationWarning: the imp module is deprecated in favour of importlib; see the module's documentation for alternative uses
import imp
Using TensorFlow backend.
Traceback (most recent call last):
File "decompressor.py", line 184, in <module>
main()
File "decompressor.py", line 153, in main
f = open(args.input_file_prefix+'.combined','rb')
FileNotFoundError: [Errno 2] No such file or directory: '../data/compressed/Caenorhabditis_elegans.WBcel235.dna.chromosome.I/FC.compressed.combined'
Command exited with non-zero status 1
Command being timed: "python decompressor.py -output ../data/compressed/Caenorhabditis_elegans.WBcel235.dna.chromosome.I/FC.reconstructed.txt -model ../data/trained_models/Caenorhabditis_elegans.WBcel235.dna.chromosome.I/FC.hdf5 -model_name FC -input_file_prefix ../data/compressed/Caenorhabditis_elegans.WBcel235.dna.chromosome.I/FC.compressed -batch_size 10000"
User time (seconds): 2.08
System time (seconds): 0.27
Percent of CPU this job got: 100%
Elapsed (wall clock) time (h:mm:ss or m:ss): 0:02.35
Average shared text size (kbytes): 0
Average unshared data size (kbytes): 0
Average stack size (kbytes): 0
Average total size (kbytes): 0
Maximum resident set size (kbytes): 254404
Average resident set size (kbytes): 0
Major (requiring I/O) page faults: 0
Minor (reclaiming a frame) page faults: 68217
Voluntary context switches: 27
Involuntary context switches: 11
Swaps: 0
File system inputs: 0
File system outputs: 8
Socket messages sent: 0
Socket messages received: 0
Signals delivered: 0
Page size (bytes): 4096
Exit status: 1
Starting training ...
Starting Compression ...
Starting training ...
0;1.8376746547150056
1;1.8126217554959843
2;1.8025803795099922
3;1.7963058942928531
4;1.7918460764422657
5;1.7884065360185188
6;1.7856453744295588
7;1.7833271550050174
8;1.7813949754473162
9;1.7797232625827701
Starting Compression ...
2018-11-01 01:34:51.846369: I tensorflow/core/platform/cpu_feature_guard.cc:141] Your CPU supports instructions that this TensorFlow binary was not compiled to use: AVX2 FMA
2018-11-01 01:34:54.512297: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1411] Found device 0 with properties:
name: TITAN X (Pascal) major: 6 minor: 1 memoryClockRate(GHz): 1.531
pciBusID: 0000:08:00.0
totalMemory: 11.91GiB freeMemory: 11.75GiB
2018-11-01 01:34:54.512359: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1490] Adding visible gpu devices: 0
2018-11-01 01:34:54.866874: I tensorflow/core/common_runtime/gpu/gpu_device.cc:971] Device interconnect StreamExecutor with strength 1 edge matrix:
2018-11-01 01:34:54.866941: I tensorflow/core/common_runtime/gpu/gpu_device.cc:977] 0
2018-11-01 01:34:54.866951: I tensorflow/core/common_runtime/gpu/gpu_device.cc:990] 0: N
2018-11-01 01:34:54.867324: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1103] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 11372 MB memory) -> physical GPU (device: 0, name: TITAN X (Pascal), pci bus id: 0000:08:00.0, compute capability: 6.1)
/opt/conda/lib/python3.6/site-packages/sklearn/externals/joblib/externals/cloudpickle/cloudpickle.py:47: DeprecationWarning: the imp module is deprecated in favour of importlib; see the module's documentation for alternative uses
import imp
Using TensorFlow backend.
/opt/conda/lib/python3.6/site-packages/sklearn/preprocessing/_encoders.py:363: FutureWarning: The handling of integer data will change in version 0.22. Currently, the categories are determined based on the range [0, max(values)], while in the future they will be determined based on the unique values.
If you want the future behaviour and silence this warning, you can specify "categories='auto'".
In case you used a LabelEncoder before this OneHotEncoder to convert the categories to integers, then you can now use the OneHotEncoder directly.
warnings.warn(msg, FutureWarning)
Command being timed: "python compressor.py -data ../data/processed_files/Caenorhabditis_elegans.WBcel235.dna.chromosome.I.npy -data_params ../data/processed_files/Caenorhabditis_elegans.WBcel235.dna.chromosome.I.param.json -model ../data/trained_models/Caenorhabditis_elegans.WBcel235.dna.chromosome.I/FC.hdf5 -model_name FC -output ../data/compressed/Caenorhabditis_elegans.WBcel235.dna.chromosome.I/FC.compressed -batch_size 10000"
User time (seconds): 224.19
System time (seconds): 12.13
Percent of CPU this job got: 106%
Elapsed (wall clock) time (h:mm:ss or m:ss): 3:41.15
Average shared text size (kbytes): 0
Average unshared data size (kbytes): 0
Average stack size (kbytes): 0
Average total size (kbytes): 0
Maximum resident set size (kbytes): 2011108
Average resident set size (kbytes): 0
Major (requiring I/O) page faults: 0
Minor (reclaiming a frame) page faults: 1939866
Voluntary context switches: 180548
Involuntary context switches: 683
Swaps: 0
File system inputs: 0
File system outputs: 86624
Socket messages sent: 0
Socket messages received: 0
Signals delivered: 0
Page size (bytes): 4096
Exit status: 0
2018-11-01 01:38:28.361670: I tensorflow/core/platform/cpu_feature_guard.cc:141] Your CPU supports instructions that this TensorFlow binary was not compiled to use: AVX2 FMA
2018-11-01 01:38:31.008045: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1411] Found device 0 with properties:
name: TITAN X (Pascal) major: 6 minor: 1 memoryClockRate(GHz): 1.531
pciBusID: 0000:08:00.0
totalMemory: 11.91GiB freeMemory: 11.75GiB
2018-11-01 01:38:31.008109: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1490] Adding visible gpu devices: 0
2018-11-01 01:38:31.402802: I tensorflow/core/common_runtime/gpu/gpu_device.cc:971] Device interconnect StreamExecutor with strength 1 edge matrix:
2018-11-01 01:38:31.402867: I tensorflow/core/common_runtime/gpu/gpu_device.cc:977] 0
2018-11-01 01:38:31.402877: I tensorflow/core/common_runtime/gpu/gpu_device.cc:990] 0: N
2018-11-01 01:38:31.403322: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1103] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 11372 MB memory) -> physical GPU (device: 0, name: TITAN X (Pascal), pci bus id: 0000:08:00.0, compute capability: 6.1)
/opt/conda/lib/python3.6/site-packages/sklearn/externals/joblib/externals/cloudpickle/cloudpickle.py:47: DeprecationWarning: the imp module is deprecated in favour of importlib; see the module's documentation for alternative uses
import imp
Using TensorFlow backend.
{'0': 'A', '1': 'C', '2': 'T', '3': 'G'}
[3 1 1 2 0 0 3 1 1 2]
Command being timed: "python decompressor.py -output ../data/compressed/Caenorhabditis_elegans.WBcel235.dna.chromosome.I/FC.reconstructed.txt -model ../data/trained_models/Caenorhabditis_elegans.WBcel235.dna.chromosome.I/FC.hdf5 -model_name FC -input_file_prefix ../data/compressed/Caenorhabditis_elegans.WBcel235.dna.chromosome.I/FC.compressed -batch_size 10000"
User time (seconds): 313.08
System time (seconds): 9.34
Percent of CPU this job got: 102%
Elapsed (wall clock) time (h:mm:ss or m:ss): 5:15.09
Average shared text size (kbytes): 0
Average unshared data size (kbytes): 0
Average stack size (kbytes): 0
Average total size (kbytes): 0
Maximum resident set size (kbytes): 1543840
Average resident set size (kbytes): 0
Major (requiring I/O) page faults: 0
Minor (reclaiming a frame) page faults: 572505
Voluntary context switches: 182230
Involuntary context switches: 922
Swaps: 0
File system inputs: 0
File system outputs: 109456
Socket messages sent: 0
Socket messages received: 0
Signals delivered: 0
Page size (bytes): 4096
Exit status: 0
Starting Compression ...
2018-11-09 09:06:34.569145: I tensorflow/core/platform/cpu_feature_guard.cc:141] Your CPU supports instructions that this TensorFlow binary was not compiled to use: AVX2 FMA
2018-11-09 09:06:37.371351: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1411] Found device 0 with properties:
name: TITAN X (Pascal) major: 6 minor: 1 memoryClockRate(GHz): 1.531
pciBusID: 0000:09:00.0
totalMemory: 11.91GiB freeMemory: 286.06MiB
2018-11-09 09:06:37.371397: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1490] Adding visible gpu devices: 0
2018-11-09 09:06:37.733654: I tensorflow/core/common_runtime/gpu/gpu_device.cc:971] Device interconnect StreamExecutor with strength 1 edge matrix:
2018-11-09 09:06:37.733726: I tensorflow/core/common_runtime/gpu/gpu_device.cc:977] 0
2018-11-09 09:06:37.733735: I tensorflow/core/common_runtime/gpu/gpu_device.cc:990] 0: N
2018-11-09 09:06:37.733949: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1103] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 220 MB memory) -> physical GPU (device: 0, name: TITAN X (Pascal), pci bus id: 0000:09:00.0, compute capability: 6.1)
2018-11-09 09:06:37.735293: E tensorflow/stream_executor/cuda/cuda_driver.cc:806] failed to allocate 220.06M (230752256 bytes) from device: CUDA_ERROR_OUT_OF_MEMORY: out of memory
2018-11-09 09:06:38.744308: E tensorflow/stream_executor/cuda/cuda_blas.cc:464] failed to create cublas handle: CUBLAS_STATUS_NOT_INITIALIZED
2018-11-09 09:06:38.752810: E tensorflow/stream_executor/cuda/cuda_blas.cc:464] failed to create cublas handle: CUBLAS_STATUS_NOT_INITIALIZED
2018-11-09 09:06:38.755559: E tensorflow/stream_executor/cuda/cuda_blas.cc:464] failed to create cublas handle: CUBLAS_STATUS_NOT_INITIALIZED
2018-11-09 09:06:38.765426: E tensorflow/stream_executor/cuda/cuda_blas.cc:464] failed to create cublas handle: CUBLAS_STATUS_NOT_INITIALIZED
2018-11-09 09:06:38.765506: W tensorflow/stream_executor/stream.cc:2089] attempting to perform BLAS operation using StreamExecutor without BLAS support
/opt/conda/lib/python3.6/site-packages/sklearn/externals/joblib/externals/cloudpickle/cloudpickle.py:47: DeprecationWarning: the imp module is deprecated in favour of importlib; see the module's documentation for alternative uses
import imp
Using TensorFlow backend.
/opt/conda/lib/python3.6/site-packages/sklearn/preprocessing/_encoders.py:363: FutureWarning: The handling of integer data will change in version 0.22. Currently, the categories are determined based on the range [0, max(values)], while in the future they will be determined based on the unique values.
If you want the future behaviour and silence this warning, you can specify "categories='auto'".
In case you used a LabelEncoder before this OneHotEncoder to convert the categories to integers, then you can now use the OneHotEncoder directly.
warnings.warn(msg, FutureWarning)
Traceback (most recent call last):
File "compressor.py", line 202, in <module>
main()
File "compressor.py", line 164, in main
predict_lstm(X, Y, Y_original, timesteps, batch_size, alphabet_size, args.model_name)
File "compressor.py", line 88, in predict_lstm
prob = model.predict(X[ind,:], batch_size=bs)
File "/src/keras/engine/training.py", line 1169, in predict
steps=steps)
File "/src/keras/engine/training_arrays.py", line 295, in predict_loop
batch_outs = f(ins_batch)
File "/src/keras/backend/tensorflow_backend.py", line 2777, in __call__
return self._call(inputs)
File "/src/keras/backend/tensorflow_backend.py", line 2737, in _call
fetched = self._callable_fn(*array_vals)
File "/opt/conda/lib/python3.6/site-packages/tensorflow/python/client/session.py", line 1399, in __call__
run_metadata_ptr)
File "/opt/conda/lib/python3.6/site-packages/tensorflow/python/framework/errors_impl.py", line 526, in __exit__
c_api.TF_GetCode(self.status.status))
tensorflow.python.framework.errors_impl.InternalError: Blas GEMM launch failed : a.shape=(1000, 2048), b.shape=(2048, 1024), m=1000, n=1024, k=2048
[[{{node dense_1/MatMul}} = MatMul[T=DT_FLOAT, transpose_a=false, transpose_b=false, _device="/job:localhost/replica:0/task:0/device:GPU:0"](flatten_1/Reshape, dense_1/kernel/read)]]
Command exited with non-zero status 1
Command being timed: "python compressor.py -data ../data/processed_files/Caenorhabditis_elegans.WBcel235.dna.chromosome.I.npy -data_params ../data/processed_files/Caenorhabditis_elegans.WBcel235.dna.chromosome.I.param.json -model ../data/trained_models/Caenorhabditis_elegans.WBcel235.dna.chromosome.I/FC.hdf5 -model_name FC -output ../data/compressed/Caenorhabditis_elegans.WBcel235.dna.chromosome.I/FC.compressed -batch_size 1000"
User time (seconds): 16.59
System time (seconds): 6.02
Percent of CPU this job got: 198%
Elapsed (wall clock) time (h:mm:ss or m:ss): 0:11.39
Average shared text size (kbytes): 0
Average unshared data size (kbytes): 0
Average stack size (kbytes): 0
Average total size (kbytes): 0
Maximum resident set size (kbytes): 1636784
Average resident set size (kbytes): 0
Major (requiring I/O) page faults: 0
Minor (reclaiming a frame) page faults: 1486805
Voluntary context switches: 1035
Involuntary context switches: 179
Swaps: 0
File system inputs: 0
File system outputs: 8016
Socket messages sent: 0
Socket messages received: 0
Signals delivered: 0
Page size (bytes): 4096
Exit status: 1
2018-11-09 09:06:41.275659: I tensorflow/core/platform/cpu_feature_guard.cc:141] Your CPU supports instructions that this TensorFlow binary was not compiled to use: AVX2 FMA
2018-11-09 09:06:44.238674: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1411] Found device 0 with properties:
name: TITAN X (Pascal) major: 6 minor: 1 memoryClockRate(GHz): 1.531
pciBusID: 0000:09:00.0
totalMemory: 11.91GiB freeMemory: 286.06MiB
2018-11-09 09:06:44.238724: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1490] Adding visible gpu devices: 0
2018-11-09 09:06:44.593597: I tensorflow/core/common_runtime/gpu/gpu_device.cc:971] Device interconnect StreamExecutor with strength 1 edge matrix:
2018-11-09 09:06:44.593671: I tensorflow/core/common_runtime/gpu/gpu_device.cc:977] 0
2018-11-09 09:06:44.593687: I tensorflow/core/common_runtime/gpu/gpu_device.cc:990] 0: N
2018-11-09 09:06:44.593940: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1103] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 220 MB memory) -> physical GPU (device: 0, name: TITAN X (Pascal), pci bus id: 0000:09:00.0, compute capability: 6.1)
2018-11-09 09:06:44.595983: E tensorflow/stream_executor/cuda/cuda_driver.cc:806] failed to allocate 220.06M (230752256 bytes) from device: CUDA_ERROR_OUT_OF_MEMORY: out of memory
2018-11-09 09:06:46.260873: E tensorflow/stream_executor/cuda/cuda_blas.cc:464] failed to create cublas handle: CUBLAS_STATUS_NOT_INITIALIZED
2018-11-09 09:06:46.269017: E tensorflow/stream_executor/cuda/cuda_blas.cc:464] failed to create cublas handle: CUBLAS_STATUS_NOT_INITIALIZED
2018-11-09 09:06:46.277403: E tensorflow/stream_executor/cuda/cuda_blas.cc:464] failed to create cublas handle: CUBLAS_STATUS_NOT_INITIALIZED
2018-11-09 09:06:46.286276: E tensorflow/stream_executor/cuda/cuda_blas.cc:464] failed to create cublas handle: CUBLAS_STATUS_NOT_INITIALIZED
2018-11-09 09:06:46.286324: W tensorflow/stream_executor/stream.cc:2089] attempting to perform BLAS operation using StreamExecutor without BLAS support
/opt/conda/lib/python3.6/site-packages/sklearn/externals/joblib/externals/cloudpickle/cloudpickle.py:47: DeprecationWarning: the imp module is deprecated in favour of importlib; see the module's documentation for alternative uses
import imp
Using TensorFlow backend.
Traceback (most recent call last):
File "decompressor.py", line 184, in <module>
main()
File "decompressor.py", line 172, in main
series[:l] = predict_lstm(l, timesteps, batch_size, alphabet_size, args.model_name)
File "decompressor.py", line 95, in predict_lstm
prob = model.predict(series_2d[:,j:j+timesteps], batch_size=bs)
File "/src/keras/engine/training.py", line 1169, in predict
steps=steps)
File "/src/keras/engine/training_arrays.py", line 295, in predict_loop
batch_outs = f(ins_batch)
File "/src/keras/backend/tensorflow_backend.py", line 2777, in __call__
return self._call(inputs)
File "/src/keras/backend/tensorflow_backend.py", line 2737, in _call
fetched = self._callable_fn(*array_vals)
File "/opt/conda/lib/python3.6/site-packages/tensorflow/python/client/session.py", line 1399, in __call__
run_metadata_ptr)
File "/opt/conda/lib/python3.6/site-packages/tensorflow/python/framework/errors_impl.py", line 526, in __exit__
c_api.TF_GetCode(self.status.status))
tensorflow.python.framework.errors_impl.InternalError: Blas GEMM launch failed : a.shape=(1000, 2048), b.shape=(2048, 1024), m=1000, n=1024, k=2048
[[{{node dense_1/MatMul}} = MatMul[T=DT_FLOAT, transpose_a=false, transpose_b=false, _device="/job:localhost/replica:0/task:0/device:GPU:0"](flatten_1/Reshape, dense_1/kernel/read)]]
[[{{node dense_3/Softmax/_45}} = _Recv[client_terminated=false, recv_device="/job:localhost/replica:0/task:0/device:CPU:0", send_device="/job:localhost/replica:0/task:0/device:GPU:0", send_device_incarnation=1, tensor_name="edge_42_dense_3/Softmax", tensor_type=DT_FLOAT, _device="/job:localhost/replica:0/task:0/device:CPU:0"]()]]
Command exited with non-zero status 1
Command being timed: "python decompressor.py -output ../data/compressed/Caenorhabditis_elegans.WBcel235.dna.chromosome.I/FC.reconstructed.txt -model ../data/trained_models/Caenorhabditis_elegans.WBcel235.dna.chromosome.I/FC.hdf5 -model_name FC -input_file_prefix ../data/compressed/Caenorhabditis_elegans.WBcel235.dna.chromosome.I/FC.compressed -batch_size 1000"
User time (seconds): 3.77
System time (seconds): 3.03
Percent of CPU this job got: 90%
Elapsed (wall clock) time (h:mm:ss or m:ss): 0:07.52
Average shared text size (kbytes): 0
Average unshared data size (kbytes): 0
Average stack size (kbytes): 0
Average total size (kbytes): 0
Maximum resident set size (kbytes): 951672
Average resident set size (kbytes): 0
Major (requiring I/O) page faults: 0
Minor (reclaiming a frame) page faults: 217572
Voluntary context switches: 768
Involuntary context switches: 72
Swaps: 0
File system inputs: 0
File system outputs: 8016
Socket messages sent: 0
Socket messages received: 0
Signals delivered: 0
Page size (bytes): 4096
Exit status: 1
================================================
FILE: data/logs_data/Caenorhabditis_elegans.WBcel235.dna.chromosome.I/FC_16bit.log.csv
================================================
Starting training ...
0;1.862424908387968
1;1.8382857217924111
2;1.8246774072839527
3;1.8144073455227936
4;1.8069313578537944
5;1.8014847943134724
6;1.7971811253736667
Starting Compression ...
2018-10-28 22:30:59.653963: I tensorflow/core/platform/cpu_feature_guard.cc:141] Your CPU supports instructions that this TensorFlow binary was not compiled to use: AVX2 FMA
2018-10-28 22:31:01.507280: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1411] Found device 0 with properties:
name: TITAN X (Pascal) major: 6 minor: 1 memoryClockRate(GHz): 1.531
pciBusID: 0000:8a:00.0
totalMemory: 11.91GiB freeMemory: 11.75GiB
2018-10-28 22:31:01.507337: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1490] Adding visible gpu devices: 0
2018-10-28 22:31:01.902839: I tensorflow/core/common_runtime/gpu/gpu_device.cc:971] Device interconnect StreamExecutor with strength 1 edge matrix:
2018-10-28 22:31:01.902894: I tensorflow/core/common_runtime/gpu/gpu_device.cc:977] 0
2018-10-28 22:31:01.902905: I tensorflow/core/common_runtime/gpu/gpu_device.cc:990] 0: N
2018-10-28 22:31:01.903285: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1103] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 11372 MB memory) -> physical GPU (device: 0, name: TITAN X (Pascal), pci bus id: 0000:8a:00.0, compute capability: 6.1)
/opt/conda/lib/python3.6/site-packages/sklearn/externals/joblib/externals/cloudpickle/cloudpickle.py:47: DeprecationWarning: the imp module is deprecated in favour of importlib; see the module's documentation for alternative uses
import imp
Using TensorFlow backend.
/opt/conda/lib/python3.6/site-packages/sklearn/preprocessing/_encoders.py:363: FutureWarning: The handling of integer data will change in version 0.22. Currently, the categories are determined based on the range [0, max(values)], while in the future they will be determined based on the unique values.
If you want the future behaviour and silence this warning, you can specify "categories='auto'".
In case you used a LabelEncoder before this OneHotEncoder to convert the categories to integers, then you can now use the OneHotEncoder directly.
warnings.warn(msg, FutureWarning)
Command being timed: "python compressor.py -data ../data/processed_files/Caenorhabditis_elegans.WBcel235.dna.chromosome.I.npy -data_params ../data/processed_files/Caenorhabditis_elegans.WBcel235.dna.chromosome.I.param.json -model ../data/trained_models/Caenorhabditis_elegans.WBcel235.dna.chromosome.I/FC_16bit.hdf5 -model_name FC_16bit -output ../data/compressed/Caenorhabditis_elegans.WBcel235.dna.chromosome.I/FC_16bit.compressed -batch_size 10000"
User time (seconds): 212.27
System time (seconds): 12.09
Percent of CPU this job got: 107%
Elapsed (wall clock) time (h:mm:ss or m:ss): 3:29.35
Average shared text size (kbytes): 0
Average unshared data size (kbytes): 0
Average stack size (kbytes): 0
Average total size (kbytes): 0
Maximum resident set size (kbytes): 2005360
Average resident set size (kbytes): 0
Major (requiring I/O) page faults: 0
Minor (reclaiming a frame) page faults: 2324705
Voluntary context switches: 176040
Involuntary context switches: 866
Swaps: 0
File system inputs: 0
File system outputs: 86688
Socket messages sent: 0
Socket messages received: 0
Signals delivered: 0
Page size (bytes): 4096
Exit status: 0
2018-10-28 22:34:24.284995: I tensorflow/core/platform/cpu_feature_guard.cc:141] Your CPU supports instructions that this TensorFlow binary was not compiled to use: AVX2 FMA
2018-10-28 22:34:26.245285: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1411] Found device 0 with properties:
name: TITAN X (Pascal) major: 6 minor: 1 memoryClockRate(GHz): 1.531
pciBusID: 0000:8a:00.0
totalMemory: 11.91GiB freeMemory: 11.75GiB
2018-10-28 22:34:26.245343: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1490] Adding visible gpu devices: 0
2018-10-28 22:34:26.605548: I tensorflow/core/common_runtime/gpu/gpu_device.cc:971] Device interconnect StreamExecutor with strength 1 edge matrix:
2018-10-28 22:34:26.605603: I tensorflow/core/common_runtime/gpu/gpu_device.cc:977] 0
2018-10-28 22:34:26.605613: I tensorflow/core/common_runtime/gpu/gpu_device.cc:990] 0: N
2018-10-28 22:34:26.606026: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1103] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 11372 MB memory) -> physical GPU (device: 0, name: TITAN X (Pascal), pci bus id: 0000:8a:00.0, compute capability: 6.1)
/opt/conda/lib/python3.6/site-packages/sklearn/externals/joblib/externals/cloudpickle/cloudpickle.py:47: DeprecationWarning: the imp module is deprecated in favour of importlib; see the module's documentation for alternative uses
import imp
Using TensorFlow backend.
{'0': 'A', '1': 'C', '2': 'T', '3': 'G'}
[3 1 1 2 0 0 3 1 1 2]
Command being timed: "python decompressor.py -output ../data/compressed/Caenorhabditis_elegans.WBcel235.dna.chromosome.I/FC_16bit.reconstructed.txt -model ../data/trained_models/Caenorhabditis_elegans.WBcel235.dna.chromosome.I/FC_16bit.hdf5 -model_name FC_16bit -input_file_prefix ../data/compressed/Caenorhabditis_elegans.WBcel235.dna.chromosome.I/FC_16bit.compressed -batch_size 10000"
User time (seconds): 291.94
System time (seconds): 8.55
Percent of CPU this job got: 102%
Elapsed (wall clock) time (h:mm:ss or m:ss): 4:53.10
Average shared text size (kbytes): 0
Average unshared data size (kbytes): 0
Average stack size (kbytes): 0
Average total size (kbytes): 0
Maximum resident set size (kbytes): 1535588
Average resident set size (kbytes): 0
Major (requiring I/O) page faults: 0
Minor (reclaiming a frame) page faults: 582026
Voluntary context switches: 180197
Involuntary context switches: 846
Swaps: 0
File system inputs: 0
File system outputs: 109456
Socket messages sent: 0
Socket messages received: 0
Signals delivered: 0
Page size (bytes): 4096
Exit status: 0
================================================
FILE: data/logs_data/Caenorhabditis_elegans.WBcel235.dna.chromosome.I/FC_4layer.log.csv
================================================
Starting training ...
Starting Compression ...
/opt/conda/lib/python3.6/site-packages/sklearn/externals/joblib/externals/cloudpickle/cloudpickle.py:47: DeprecationWarning: the imp module is deprecated in favour of importlib; see the module's documentation for alternative uses
import imp
Using TensorFlow backend.
/opt/conda/lib/python3.6/site-packages/sklearn/preprocessing/_encoders.py:363: FutureWarning: The handling of integer data will change in version 0.22. Currently, the categories are determined based on the range [0, max(values)], while in the future they will be determined based on the unique values.
If you want the future behaviour and silence this warning, you can specify "categories='auto'".
In case you used a LabelEncoder before this OneHotEncoder to convert the categories to integers, then you can now use the OneHotEncoder directly.
warnings.warn(msg, FutureWarning)
Traceback (most recent call last):
File "compressor.py", line 202, in <module>
main()
File "compressor.py", line 164, in main
predict_lstm(X, Y, Y_original, timesteps, batch_size, alphabet_size, args.model_name)
File "compressor.py", line 68, in predict_lstm
model = getattr(models, model_name)(bs, timesteps, alphabet_size)
AttributeError: module 'models' has no attribute 'FC_4layer'
Command exited with non-zero status 1
Command being timed: "python compressor.py -data ../data/processed_files/Caenorhabditis_elegans.WBcel235.dna.chromosome.I.npy -data_params ../data/processed_files/Caenorhabditis_elegans.WBcel235.dna.chromosome.I.param.json -model ../data/trained_models/Caenorhabditis_elegans.WBcel235.dna.chromosome.I/FC_4layer.hdf5 -model_name FC_4layer -output ../data/compressed/Caenorhabditis_elegans.WBcel235.dna.chromosome.I/FC_4layer.compressed -batch_size 10000"
User time (seconds): 15.10
System time (seconds): 3.20
Percent of CPU this job got: 264%
Elapsed (wall clock) time (h:mm:ss or m:ss): 0:06.92
Average shared text size (kbytes): 0
Average unshared data size (kbytes): 0
Average stack size (kbytes): 0
Average total size (kbytes): 0
Maximum resident set size (kbytes): 1635408
Average resident set size (kbytes): 0
Major (requiring I/O) page faults: 0
Minor (reclaiming a frame) page faults: 1340328
Voluntary context switches: 276
Involuntary context switches: 16925
Swaps: 0
File system inputs: 8
File system outputs: 16
Socket messages sent: 0
Socket messages received: 0
Signals delivered: 0
Page size (bytes): 4096
Exit status: 1
/opt/conda/lib/python3.6/site-packages/sklearn/externals/joblib/externals/cloudpickle/cloudpickle.py:47: DeprecationWarning: the imp module is deprecated in favour of importlib; see the module's documentation for alternative uses
import imp
Using TensorFlow backend.
Traceback (most recent call last):
File "decompressor.py", line 184, in <module>
main()
File "decompressor.py", line 153, in main
f = open(args.input_file_prefix+'.combined','rb')
FileNotFoundError: [Errno 2] No such file or directory: '../data/compressed/Caenorhabditis_elegans.WBcel235.dna.chromosome.I/FC_4layer.compressed.combined'
Command exited with non-zero status 1
Command being timed: "python decompressor.py -output ../data/compressed/Caenorhabditis_elegans.WBcel235.dna.chromosome.I/FC_4layer.reconstructed.txt -model ../data/trained_models/Caenorhabditis_elegans.WBcel235.dna.chromosome.I/FC_4layer.hdf5 -model_name FC_4layer -input_file_prefix ../data/compressed/Caenorhabditis_elegans.WBcel235.dna.chromosome.I/FC_4layer.compressed -batch_size 10000"
User time (seconds): 1.73
System time (seconds): 0.21
Percent of CPU this job got: 100%
Elapsed (wall clock) time (h:mm:ss or m:ss): 0:01.95
Average shared text size (kbytes): 0
Average unshared data size (kbytes): 0
Average stack size (kbytes): 0
Average total size (kbytes): 0
Maximum resident set size (kbytes): 254768
Average resident set size (kbytes): 0
Major (requiring I/O) page faults: 0
Minor (reclaiming a frame) page faults: 68177
Voluntary context switches: 32
Involuntary context switches: 6
Swaps: 0
File system inputs: 0
File system outputs: 8
Socket messages sent: 0
Socket messages received: 0
Signals delivered: 0
Page size (bytes): 4096
Exit status: 1
Starting training ...
0;1.8659678535362942
1;1.8489295976691147
2;1.8404188575575522
3;1.8334980335189592
4;1.8283475114644987
5;1.8247079945135412
6;1.8218116113953118
7;1.8194340540911
8;1.817697522878177
9;1.816371569151431
Starting Compression ...
2018-11-01 00:50:19.033724: I tensorflow/core/platform/cpu_feature_guard.cc:141] Your CPU supports instructions that this TensorFlow binary was not compiled to use: AVX2 FMA
2018-11-01 00:50:22.067908: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1411] Found device 0 with properties:
name: TITAN X (Pascal) major: 6 minor: 1 memoryClockRate(GHz): 1.531
pciBusID: 0000:8a:00.0
totalMemory: 11.91GiB freeMemory: 11.75GiB
2018-11-01 00:50:22.067980: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1490] Adding visible gpu devices: 0
2018-11-01 00:50:22.441074: I tensorflow/core/common_runtime/gpu/gpu_device.cc:971] Device interconnect StreamExecutor with strength 1 edge matrix:
2018-11-01 00:50:22.441135: I tensorflow/core/common_runtime/gpu/gpu_device.cc:977] 0
2018-11-01 00:50:22.441147: I tensorflow/core/common_runtime/gpu/gpu_device.cc:990] 0: N
2018-11-01 00:50:22.441527: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1103] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 11372 MB memory) -> physical GPU (device: 0, name: TITAN X (Pascal), pci bus id: 0000:8a:00.0, compute capability: 6.1)
/opt/conda/lib/python3.6/site-packages/sklearn/externals/joblib/externals/cloudpickle/cloudpickle.py:47: DeprecationWarning: the imp module is deprecated in favour of importlib; see the module's documentation for alternative uses
import imp
Using TensorFlow backend.
/opt/conda/lib/python3.6/site-packages/sklearn/preprocessing/_encoders.py:363: FutureWarning: The handling of integer data will change in version 0.22. Currently, the categories are determined based on the range [0, max(values)], while in the future they will be determined based on the unique values.
If you want the future behaviour and silence this warning, you can specify "categories='auto'".
In case you used a LabelEncoder before this OneHotEncoder to convert the categories to integers, then you can now use the OneHotEncoder directly.
warnings.warn(msg, FutureWarning)
/src/keras/activations.py:209: UserWarning: Do not pass a layer instance (such as ELU) as the activation argument of another layer. Instead, advanced activation layers should be used just like any other layer in a model.
identifier=identifier.__class__.__name__))
Command being timed: "python compressor.py -data ../data/processed_files/Caenorhabditis_elegans.WBcel235.dna.chromosome.I.npy -data_params ../data/processed_files/Caenorhabditis_elegans.WBcel235.dna.chromosome.I.param.json -model ../data/trained_models/Caenorhabditis_elegans.WBcel235.dna.chromosome.I/FC_4layer.hdf5 -model_name FC_4layer -output ../data/compressed/Caenorhabditis_elegans.WBcel235.dna.chromosome.I/FC_4layer.compressed -batch_size 10000"
User time (seconds): 189.29
System time (seconds): 9.16
Percent of CPU this job got: 108%
Elapsed (wall clock) time (h:mm:ss or m:ss): 3:03.54
Average shared text size (kbytes): 0
Average unshared data size (kbytes): 0
Average stack size (kbytes): 0
Average total size (kbytes): 0
Maximum resident set size (kbytes): 2004308
Average resident set size (kbytes): 0
Major (requiring I/O) page faults: 0
Minor (reclaiming a frame) page faults: 1885145
Voluntary context switches: 71993
Involuntary context switches: 565
Swaps: 0
File system inputs: 72
File system outputs: 86776
Socket messages sent: 0
Socket messages received: 0
Signals delivered: 0
Page size (bytes): 4096
Exit status: 0
2018-11-01 00:53:18.038091: I tensorflow/core/platform/cpu_feature_guard.cc:141] Your CPU supports instructions that this TensorFlow binary was not compiled to use: AVX2 FMA
2018-11-01 00:53:21.058566: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1411] Found device 0 with properties:
name: TITAN X (Pascal) major: 6 minor: 1 memoryClockRate(GHz): 1.531
pciBusID: 0000:8a:00.0
totalMemory: 11.91GiB freeMemory: 11.75GiB
2018-11-01 00:53:21.058615: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1490] Adding visible gpu devices: 0
2018-11-01 00:53:21.376237: I tensorflow/core/common_runtime/gpu/gpu_device.cc:971] Device interconnect StreamExecutor with strength 1 edge matrix:
2018-11-01 00:53:21.376294: I tensorflow/core/common_runtime/gpu/gpu_device.cc:977] 0
2018-11-01 00:53:21.376303: I tensorflow/core/common_runtime/gpu/gpu_device.cc:990] 0: N
2018-11-01 00:53:21.376819: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1103] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 11372 MB memory) -> physical GPU (device: 0, name: TITAN X (Pascal), pci bus id: 0000:8a:00.0, compute capability: 6.1)
/opt/conda/lib/python3.6/site-packages/sklearn/externals/joblib/externals/cloudpickle/cloudpickle.py:47: DeprecationWarning: the imp module is deprecated in favour of importlib; see the module's documentation for alternative uses
import imp
Using TensorFlow backend.
/src/keras/activations.py:209: UserWarning: Do not pass a layer instance (such as ELU) as the activation argument of another layer. Instead, advanced activation layers should be used just like any other layer in a model.
identifier=identifier.__class__.__name__))
{'0': 'A', '1': 'C', '2': 'T', '3': 'G'}
[3 1 1 2 0 0 3 1 1 2]
Command being timed: "python decompressor.py -output ../data/compressed/Caenorhabditis_elegans.WBcel235.dna.chromosome.I/FC_4layer.reconstructed.txt -model ../data/trained_models/Caenorhabditis_elegans.WBcel235.dna.chromosome.I/FC_4layer.hdf5 -model_name FC_4layer -input_file_prefix ../data/compressed/Caenorhabditis_elegans.WBcel235.dna.chromosome.I/FC_4layer.compressed -batch_size 10000"
User time (seconds): 249.58
System time (seconds): 6.10
Percent of CPU this job got: 102%
Elapsed (wall clock) time (h:mm:ss or m:ss): 4:09.05
Average shared text size (kbytes): 0
Average unshared data size (kbytes): 0
Average stack size (kbytes): 0
Average total size (kbytes): 0
Maximum resident set size (kbytes): 1536488
Average resident set size (kbytes): 0
Major (requiring I/O) page faults: 0
Minor (reclaiming a frame) page faults: 652241
Voluntary context switches: 73590
Involuntary context switches: 647
Swaps: 0
File system inputs: 0
File system outputs: 109456
Socket messages sent: 0
Socket messages received: 0
Signals delivered: 0
Page size (bytes): 4096
Exit status: 0
Starting Compression ...
2018-11-09 09:07:05.767279: I tensorflow/core/platform/cpu_feature_guard.cc:141] Your CPU supports instructions that this TensorFlow binary was not compiled to use: AVX2 FMA
2018-11-09 09:07:08.618951: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1411] Found device 0 with properties:
name: TITAN X (Pascal) major: 6 minor: 1 memoryClockRate(GHz): 1.531
pciBusID: 0000:09:00.0
totalMemory: 11.91GiB freeMemory: 286.06MiB
2018-11-09 09:07:08.619006: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1490] Adding visible gpu devices: 0
2018-11-09 09:07:08.961269: I tensorflow/core/common_runtime/gpu/gpu_device.cc:971] Device interconnect StreamExecutor with strength 1 edge matrix:
2018-11-09 09:07:08.961349: I tensorflow/core/common_runtime/gpu/gpu_device.cc:977] 0
2018-11-09 09:07:08.961359: I tensorflow/core/common_runtime/gpu/gpu_device.cc:990] 0: N
2018-11-09 09:07:08.961600: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1103] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 220 MB memory) -> physical GPU (device: 0, name: TITAN X (Pascal), pci bus id: 0000:09:00.0, compute capability: 6.1)
2018-11-09 09:07:08.962979: E tensorflow/stream_executor/cuda/cuda_driver.cc:806] failed to allocate 220.06M (230752256 bytes) from device: CUDA_ERROR_OUT_OF_MEMORY: out of memory
2018-11-09 09:07:10.023532: E tensorflow/stream_executor/cuda/cuda_blas.cc:464] failed to create cublas handle: CUBLAS_STATUS_NOT_INITIALIZED
2018-11-09 09:07:10.025718: E tensorflow/stream_executor/cuda/cuda_blas.cc:464] failed to create cublas handle: CUBLAS_STATUS_NOT_INITIALIZED
2018-11-09 09:07:10.027714: E tensorflow/stream_executor/cuda/cuda_blas.cc:464] failed to create cublas handle: CUBLAS_STATUS_NOT_INITIALIZED
2018-11-09 09:07:10.029625: E tensorflow/stream_executor/cuda/cuda_blas.cc:464] failed to create cublas handle: CUBLAS_STATUS_NOT_INITIALIZED
2018-11-09 09:07:10.031574: E tensorflow/stream_executor/cuda/cuda_blas.cc:464] failed to create cublas handle: CUBLAS_STATUS_NOT_INITIALIZED
2018-11-09 09:07:10.037328: E tensorflow/stream_executor/cuda/cuda_blas.cc:464] failed to create cublas handle: CUBLAS_STATUS_NOT_INITIALIZED
2018-11-09 09:07:10.037377: W tensorflow/stream_executor/stream.cc:2089] attempting to perform BLAS operation using StreamExecutor without BLAS support
/opt/conda/lib/python3.6/site-packages/sklearn/externals/joblib/externals/cloudpickle/cloudpickle.py:47: DeprecationWarning: the imp module is deprecated in favour of importlib; see the module's documentation for alternative uses
import imp
Using TensorFlow backend.
/opt/conda/lib/python3.6/site-packages/sklearn/preprocessing/_encoders.py:363: FutureWarning: The handling of integer data will change in version 0.22. Currently, the categories are determined based on the range [0, max(values)], while in the future they will be determined based on the unique values.
If you want the future behaviour and silence this warning, you can specify "categories='auto'".
In case you used a LabelEncoder before this OneHotEncoder to convert the categories to integers, then you can now use the OneHotEncoder directly.
warnings.warn(msg, FutureWarning)
/src/keras/activations.py:209: UserWarning: Do not pass a layer instance (such as ELU) as the activation argument of another layer. Instead, advanced activation layers should be used just like any other layer in a model.
identifier=identifier.__class__.__name__))
Traceback (most recent call last):
File "compressor.py", line 202, in <module>
main()
File "compressor.py", line 164, in main
predict_lstm(X, Y, Y_original, timesteps, batch_size, alphabet_size, args.model_name)
File "compressor.py", line 88, in predict_lstm
prob = model.predict(X[ind,:], batch_size=bs)
File "/src/keras/engine/training.py", line 1169, in predict
steps=steps)
File "/src/keras/engine/training_arrays.py", line 295, in predict_loop
batch_outs = f(ins_batch)
File "/src/keras/backend/tensorflow_backend.py", line 2777, in __call__
return self._call(inputs)
File "/src/keras/backend/tensorflow_backend.py", line 2737, in _call
fetched = self._callable_fn(*array_vals)
File "/opt/conda/lib/python3.6/site-packages/tensorflow/python/client/session.py", line 1399, in __call__
run_metadata_ptr)
File "/opt/conda/lib/python3.6/site-packages/tensorflow/python/framework/errors_impl.py", line 526, in __exit__
c_api.TF_GetCode(self.status.status))
tensorflow.python.framework.errors_impl.InternalError: Blas GEMM launch failed : a.shape=(1000, 320), b.shape=(320, 128), m=1000, n=128, k=320
[[{{node dense_1/MatMul}} = MatMul[T=DT_FLOAT, transpose_a=false, transpose_b=false, _device="/job:localhost/replica:0/task:0/device:GPU:0"](flatten_1/Reshape, dense_1/kernel/read)]]
Command exited with non-zero status 1
Command being timed: "python compressor.py -data ../data/processed_files/Caenorhabditis_elegans.WBcel235.dna.chromosome.I.npy -data_params ../data/processed_files/Caenorhabditis_elegans.WBcel235.dna.chromosome.I.param.json -model ../data/trained_models/Caenorhabditis_elegans.WBcel235.dna.chromosome.I/FC_4layer.hdf5 -model_name FC_4layer -output ../data/compressed/Caenorhabditis_elegans.WBcel235.dna.chromosome.I/FC_4layer.compressed -batch_size 1000"
User time (seconds): 16.71
System time (seconds): 6.33
Percent of CPU this job got: 196%
Elapsed (wall clock) time (h:mm:ss or m:ss): 0:11.73
Average shared text size (kbytes): 0
Average unshared data size (kbytes): 0
Average stack size (kbytes): 0
Average total size (kbytes): 0
Maximum resident set size (kbytes): 1635248
Average resident set size (kbytes): 0
Major (requiring I/O) page faults: 0
Minor (reclaiming a frame) page faults: 1478853
Voluntary context switches: 1076
Involuntary context switches: 159
Swaps: 0
File system inputs: 0
File system outputs: 8016
Socket messages sent: 0
Socket messages received: 0
Signals delivered: 0
Page size (bytes): 4096
Exit status: 1
Starting Compression ...
2018-11-09 09:07:47.791930: I tensorflow/core/platform/cpu_feature_guard.cc:141] Your CPU supports instructions that this TensorFlow binary was not compiled to use: AVX2 FMA
2018-11-09 09:07:50.668067: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1411] Found device 0 with properties:
name: TITAN X (Pascal) major: 6 minor: 1 memoryClockRate(GHz): 1.531
pciBusID: 0000:8a:00.0
totalMemory: 11.91GiB freeMemory: 11.75GiB
2018-11-09 09:07:50.668111: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1490] Adding visible gpu devices: 0
2018-11-09 09:07:51.014623: I tensorflow/core/common_runtime/gpu/gpu_device.cc:971] Device interconnect StreamExecutor with strength 1 edge matrix:
2018-11-09 09:07:51.014696: I tensorflow/core/common_runtime/gpu/gpu_device.cc:977] 0
2018-11-09 09:07:51.014706: I tensorflow/core/common_runtime/gpu/gpu_device.cc:990] 0: N
2018-11-09 09:07:51.015129: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1103] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 11372 MB memory) -> physical GPU (device: 0, name: TITAN X (Pascal), pci bus id: 0000:8a:00.0, compute capability: 6.1)
/opt/conda/lib/python3.6/site-packages/sklearn/externals/joblib/externals/cloudpickle/cloudpickle.py:47: DeprecationWarning: the imp module is deprecated in favour of importlib; see the module's documentation for alternative uses
import imp
Using TensorFlow backend.
/opt/conda/lib/python3.6/site-packages/sklearn/preprocessing/_encoders.py:363: FutureWarning: The handling of integer data will change in version 0.22. Currently, the categories are determined based on the range [0, max(values)], while in the future they will be determined based on the unique values.
If you want the future behaviour and silence this warning, you can specify "categories='auto'".
In case you used a LabelEncoder before this OneHotEncoder to convert the categories to integers, then you can now use the OneHotEncoder directly.
warnings.warn(msg, FutureWarning)
/src/keras/activations.py:209: UserWarning: Do not pass a layer instance (such as ELU) as the activation argument of another layer. Instead, advanced activation layers should be used just like any other layer in a model.
identifier=identifier.__class__.__name__))
Command being timed: "python compressor.py -data ../data/processed_files/Caenorhabditis_elegans.WBcel235.dna.chromosome.I.npy -data_params ../data/processed_files/Caenorhabditis_elegans.WBcel235.dna.chromosome.I.param.json -model ../data/trained_models/Caenorhabditis_elegans.WBcel235.dna.chromosome.I/FC_4layer.hdf5 -model_name FC_4layer -output ../data/compressed/Caenorhabditis_elegans.WBcel235.dna.chromosome.I/FC_4layer.compressed -batch_size 1000"
User time (seconds): 206.05
System time (seconds): 10.68
Percent of CPU this job got: 113%
Elapsed (wall clock) time (h:mm:ss or m:ss): 3:10.64
Average shared text size (kbytes): 0
Average unshared data size (kbytes): 0
Average stack size (kbytes): 0
Average total size (kbytes): 0
Maximum resident set size (kbytes): 1933884
Average resident set size (kbytes): 0
Major (requiring I/O) page faults: 0
Minor (reclaiming a frame) page faults: 1672945
Voluntary context switches: 236000
Involuntary context switches: 470
Swaps: 0
File system inputs: 0
File system outputs: 14704
Socket messages sent: 0
Socket messages received: 0
Signals delivered: 0
Page size (bytes): 4096
Exit status: 0
2018-11-09 09:10:53.482683: I tensorflow/core/platform/cpu_feature_guard.cc:141] Your CPU supports instructions that this TensorFlow binary was not compiled to use: AVX2 FMA
2018-11-09 09:10:56.388283: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1411] Found device 0 with properties:
name: TITAN X (Pascal) major: 6 minor: 1 memoryClockRate(GHz): 1.531
pciBusID: 0000:8a:00.0
totalMemory: 11.91GiB freeMemory: 11.75GiB
2018-11-09 09:10:56.388324: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1490] Adding visible gpu devices: 0
2018-11-09 09:10:56.720662: I tensorflow/core/common_runtime/gpu/gpu_device.cc:971] Device interconnect StreamExecutor with strength 1 edge matrix:
2018-11-09 09:10:56.720733: I tensorflow/core/common_runtime/gpu/gpu_device.cc:977] 0
2018-11-09 09:10:56.720742: I tensorflow/core/common_runtime/gpu/gpu_device.cc:990] 0: N
2018-11-09 09:10:56.721108: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1103] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 11372 MB memory) -> physical GPU (device: 0, name: TITAN X (Pascal), pci bus id: 0000:8a:00.0, compute capability: 6.1)
/opt/conda/lib/python3.6/site-packages/sklearn/externals/joblib/externals/cloudpickle/cloudpickle.py:47: DeprecationWarning: the imp module is deprecated in favour of importlib; see the module's documentation for alternative uses
import imp
Using TensorFlow backend.
/src/keras/activations.py:209: UserWarning: Do not pass a layer instance (such as ELU) as the activation argument of another layer. Instead, advanced activation layers should be used just like any other layer in a model.
identifier=identifier.__class__.__name__))
{'0': 'A', '1': 'C', '2': 'T', '3': 'G'}
[3 1 1 2 0 0 3 1 1 2]
Command being timed: "python decompressor.py -output ../data/compressed/Caenorhabditis_elegans.WBcel235.dna.chromosome.I/FC_4layer.reconstructed.txt -model ../data/trained_models/Caenorhabditis_elegans.WBcel235.dna.chromosome.I/FC_4layer.hdf5 -model_name FC_4layer -input_file_prefix ../data/compressed/Caenorhabditis_elegans.WBcel235.dna.chromosome.I/FC_4layer.compressed -batch_size 1000"
User time (seconds): 288.39
System time (seconds): 7.45
Percent of CPU this job got: 106%
Elapsed (wall clock) time (h:mm:ss or m:ss): 4:38.91
Average shared text size (kbytes): 0
Average unshared data size (kbytes): 0
Average stack size (kbytes): 0
Average total size (kbytes): 0
Maximum resident set size (kbytes): 1509364
Average resident set size (kbytes): 0
Major (requiring I/O) page faults: 0
Minor (reclaiming a frame) page faults: 440030
Voluntary context switches: 235169
Involuntary context switches: 686
Swaps: 0
File system inputs: 0
File system outputs: 37456
Socket messages sent: 0
Socket messages received: 0
Signals delivered: 0
Page size (bytes): 4096
Exit status: 0
================================================
FILE: data/logs_data/Caenorhabditis_elegans.WBcel235.dna.chromosome.I/FC_4layer_16bit.log.csv
================================================
Starting training ...
0;1.8749647033596033
1;1.8661186487278347
2;1.8598197243995855
3;1.8517230599055643
4;1.8463508714289354
5;1.842470165050275
Starting Compression ...
2018-10-28 20:33:48.379646: I tensorflow/core/platform/cpu_feature_guard.cc:141] Your CPU supports instructions that this TensorFlow binary was not compiled to use: AVX2 FMA
2018-10-28 20:33:50.225323: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1411] Found device 0 with properties:
name: TITAN X (Pascal) major: 6 minor: 1 memoryClockRate(GHz): 1.531
pciBusID: 0000:08:00.0
totalMemory: 11.91GiB freeMemory: 11.75GiB
2018-10-28 20:33:50.225366: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1490] Adding visible gpu devices: 0
2018-10-28 20:33:50.585922: I tensorflow/core/common_runtime/gpu/gpu_device.cc:971] Device interconnect StreamExecutor with strength 1 edge matrix:
2018-10-28 20:33:50.585976: I tensorflow/core/common_runtime/gpu/gpu_device.cc:977] 0
2018-10-28 20:33:50.585987: I tensorflow/core/common_runtime/gpu/gpu_device.cc:990] 0: N
2018-10-28 20:33:50.586323: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1103] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 11372 MB memory) -> physical GPU (device: 0, name: TITAN X (Pascal), pci bus id: 0000:08:00.0, compute capability: 6.1)
/opt/conda/lib/python3.6/site-packages/sklearn/externals/joblib/externals/cloudpickle/cloudpickle.py:47: DeprecationWarning: the imp module is deprecated in favour of importlib; see the module's documentation for alternative uses
import imp
Using TensorFlow backend.
/opt/conda/lib/python3.6/site-packages/sklearn/preprocessing/_encoders.py:363: FutureWarning: The handling of integer data will change in version 0.22. Currently, the categories are determined based on the range [0, max(values)], while in the future they will be determined based on the unique values.
If you want the future behaviour and silence this warning, you can specify "categories='auto'".
In case you used a LabelEncoder before this OneHotEncoder to convert the categories to integers, then you can now use the OneHotEncoder directly.
warnings.warn(msg, FutureWarning)
/src/keras/activations.py:209: UserWarning: Do not pass a layer instance (such as ELU) as the activation argument of another layer. Instead, advanced activation layers should be used just like any other layer in a model.
identifier=identifier.__class__.__name__))
Command being timed: "python compressor.py -data ../data/processed_files/Caenorhabditis_elegans.WBcel235.dna.chromosome.I.npy -data_params ../data/processed_files/Caenorhabditis_elegans.WBcel235.dna.chromosome.I.param.json -model ../data/trained_models/Caenorhabditis_elegans.WBcel235.dna.chromosome.I/FC_4layer_16bit.hdf5 -model_name FC_4layer_16bit -output ../data/compressed/Caenorhabditis_elegans.WBcel235.dna.chromosome.I/FC_4layer_16bit.compressed -batch_size 10000"
User time (seconds): 217.14
System time (seconds): 9.73
Percent of CPU this job got: 106%
Elapsed (wall clock) time (h:mm:ss or m:ss): 3:32.23
Average shared text size (kbytes): 0
Average unshared data size (kbytes): 0
Average stack size (kbytes): 0
Average total size (kbytes): 0
Maximum resident set size (kbytes): 2001368
Average resident set size (kbytes): 0
Major (requiring I/O) page faults: 0
Minor (reclaiming a frame) page faults: 2272365
Voluntary context switches: 95593
Involuntary context switches: 744
Swaps: 0
File system inputs: 0
File system outputs: 86872
Socket messages sent: 0
Socket messages received: 0
Signals delivered: 0
Page size (bytes): 4096
Exit status: 0
2018-10-28 20:37:15.686698: I tensorflow/core/platform/cpu_feature_guard.cc:141] Your CPU supports instructions that this TensorFlow binary was not compiled to use: AVX2 FMA
2018-10-28 20:37:17.535499: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1411] Found device 0 with properties:
name: TITAN X (Pascal) major: 6 minor: 1 memoryClockRate(GHz): 1.531
pciBusID: 0000:08:00.0
totalMemory: 11.91GiB freeMemory: 11.75GiB
2018-10-28 20:37:17.535570: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1490] Adding visible gpu devices: 0
2018-10-28 20:37:17.880084: I tensorflow/core/common_runtime/gpu/gpu_device.cc:971] Device interconnect StreamExecutor with strength 1 edge matrix:
2018-10-28 20:37:17.880161: I tensorflow/core/common_runtime/gpu/gpu_device.cc:977] 0
2018-10-28 20:37:17.880170: I tensorflow/core/common_runtime/gpu/gpu_device.cc:990] 0: N
2018-10-28 20:37:17.880549: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1103] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 11372 MB memory) -> physical GPU (device: 0, name: TITAN X (Pascal), pci bus id: 0000:08:00.0, compute capability: 6.1)
/opt/conda/lib/python3.6/site-packages/sklearn/externals/joblib/externals/cloudpickle/cloudpickle.py:47: DeprecationWarning: the imp module is deprecated in favour of importlib; see the module's documentation for alternative uses
import imp
Using TensorFlow backend.
/src/keras/activations.py:209: UserWarning: Do not pass a layer instance (such as ELU) as the activation argument of another layer. Instead, advanced activation layers should be used just like any other layer in a model.
identifier=identifier.__class__.__name__))
{'0': 'A', '1': 'C', '2': 'T', '3': 'G'}
[3 1 1 2 0 0 3 1 1 2]
Command being timed: "python decompressor.py -output ../data/compressed/Caenorhabditis_elegans.WBcel235.dna.chromosome.I/FC_4layer_16bit.reconstructed.txt -model ../data/trained_models/Caenorhabditis_elegans.WBcel235.dna.chromosome.I/FC_4layer_16bit.hdf5 -model_name FC_4layer_16bit -input_file_prefix ../data/compressed/Caenorhabditis_elegans.WBcel235.dna.chromosome.I/FC_4layer_16bit.compressed -batch_size 10000"
User time (seconds): 287.27
System time (seconds): 5.86
Percent of CPU this job got: 102%
Elapsed (wall clock) time (h:mm:ss or m:ss): 4:46.47
Average shared text size (kbytes): 0
Average unshared data size (kbytes): 0
Average stack size (kbytes): 0
Average total size (kbytes): 0
Maximum resident set size (kbytes): 1530348
Average resident set size (kbytes): 0
Major (requiring I/O) page faults: 0
Minor (reclaiming a frame) page faults: 668226
Voluntary context switches: 97445
Involuntary context switches: 858
Swaps: 0
File system inputs: 0
File system outputs: 109456
Socket messages sent: 0
Socket messages received: 0
Signals delivered: 0
Page size (bytes): 4096
Exit status: 0
Starting training ...
Starting training ...
Starting Compression ...
================================================
FILE: data/logs_data/Caenorhabditis_elegans.WBcel235.dna.chromosome.I/GRU_multi_16bit.log.csv
================================================
Starting training ...
0;1.8704060626358787
1;1.8522529075939262
2;1.8447586744811129
3;1.8398404989936477
Starting Compression ...
2018-10-28 21:09:29.500783: I tensorflow/core/platform/cpu_feature_guard.cc:141] Your CPU supports instructions that this TensorFlow binary was not compiled to use: AVX2 FMA
2018-10-28 21:09:31.412107: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1411] Found device 0 with properties:
name: TITAN X (Pascal) major: 6 minor: 1 memoryClockRate(GHz): 1.531
pciBusID: 0000:89:00.0
totalMemory: 11.91GiB freeMemory: 11.75GiB
2018-10-28 21:09:31.412168: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1490] Adding visible gpu devices: 0
2018-10-28 21:09:31.758080: I tensorflow/core/common_runtime/gpu/gpu_device.cc:971] Device interconnect StreamExecutor with strength 1 edge matrix:
2018-10-28 21:09:31.758154: I tensorflow/core/common_runtime/gpu/gpu_device.cc:977] 0
2018-10-28 21:09:31.758164: I tensorflow/core/common_runtime/gpu/gpu_device.cc:990] 0: N
2018-10-28 21:09:31.758592: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1103] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 11372 MB memory) -> physical GPU (device: 0, name: TITAN X (Pascal), pci bus id: 0000:89:00.0, compute capability: 6.1)
/opt/conda/lib/python3.6/site-packages/sklearn/externals/joblib/externals/cloudpickle/cloudpickle.py:47: DeprecationWarning: the imp module is deprecated in favour of importlib; see the module's documentation for alternative uses
import imp
Using TensorFlow backend.
/opt/conda/lib/python3.6/site-packages/sklearn/preprocessing/_encoders.py:363: FutureWarning: The handling of integer data will change in version 0.22. Currently, the categories are determined based on the range [0, max(values)], while in the future they will be determined based on the unique values.
If you want the future behaviour and silence this warning, you can specify "categories='auto'".
In case you used a LabelEncoder before this OneHotEncoder to convert the categories to integers, then you can now use the OneHotEncoder directly.
warnings.warn(msg, FutureWarning)
Command being timed: "python compressor.py -data ../data/processed_files/Caenorhabditis_elegans.WBcel235.dna.chromosome.I.npy -data_params ../data/processed_files/Caenorhabditis_elegans.WBcel235.dna.chromosome.I.param.json -model ../data/trained_models/Caenorhabditis_elegans.WBcel235.dna.chromosome.I/GRU_multi_16bit.hdf5 -model_name GRU_multi_16bit -output ../data/compressed/Caenorhabditis_elegans.WBcel235.dna.chromosome.I/GRU_multi_16bit.compressed -batch_size 10000"
User time (seconds): 249.65
System time (seconds): 15.29
Percent of CPU this job got: 112%
Elapsed (wall clock) time (h:mm:ss or m:ss): 3:56.21
Average shared text size (kbytes): 0
Average unshared data size (kbytes): 0
Average stack size (kbytes): 0
Average total size (kbytes): 0
Maximum resident set size (kbytes): 2701760
Average resident set size (kbytes): 0
Major (requiring I/O) page faults: 0
Minor (reclaiming a frame) page faults: 2334698
Voluntary context switches: 239801
Involuntary context switches: 1167
Swaps: 0
File system inputs: 0
File system outputs: 86856
Socket messages sent: 0
Socket messages received: 0
Signals delivered: 0
Page size (bytes): 4096
Exit status: 0
2018-10-28 21:13:21.185214: I tensorflow/core/platform/cpu_feature_guard.cc:141] Your CPU supports instructions that this TensorFlow binary was not compiled to use: AVX2 FMA
2018-10-28 21:13:22.936984: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1411] Found device 0 with properties:
name: TITAN X (Pascal) major: 6 minor: 1 memoryClockRate(GHz): 1.531
pciBusID: 0000:89:00.0
totalMemory: 11.91GiB freeMemory: 11.75GiB
2018-10-28 21:13:22.937025: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1490] Adding visible gpu devices: 0
2018-10-28 21:13:23.316522: I tensorflow/core/common_runtime/gpu/gpu_device.cc:971] Device interconnect StreamExecutor with strength 1 edge matrix:
2018-10-28 21:13:23.316585: I tensorflow/core/common_runtime/gpu/gpu_device.cc:977] 0
2018-10-28 21:13:23.316599: I tensorflow/core/common_runtime/gpu/gpu_device.cc:990] 0: N
2018-10-28 21:13:23.316984: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1103] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 11372 MB memory) -> physical GPU (device: 0, name: TITAN X (Pascal), pci bus id: 0000:89:00.0, compute capability: 6.1)
/opt/conda/lib/python3.6/site-packages/sklearn/externals/joblib/externals/cloudpickle/cloudpickle.py:47: DeprecationWarning: the imp module is deprecated in favour of importlib; see the module's documentation for alternative uses
import imp
Using TensorFlow backend.
{'0': 'A', '1': 'C', '2': 'T', '3': 'G'}
[3 1 1 2 0 0 3 1 1 2]
Command being timed: "python decompressor.py -output ../data/compressed/Caenorhabditis_elegans.WBcel235.dna.chromosome.I/GRU_multi_16bit.reconstructed.txt -model ../data/trained_models/Caenorhabditis_elegans.WBcel235.dna.chromosome.I/GRU_multi_16bit.hdf5 -model_name GRU_multi_16bit -input_file_prefix ../data/compressed/Caenorhabditis_elegans.WBcel235.dna.chromosome.I/GRU_multi_16bit.compressed -batch_size 10000"
User time (seconds): 331.51
System time (seconds): 11.54
Percent of CPU this job got: 105%
Elapsed (wall clock) time (h:mm:ss or m:ss): 5:24.80
Average shared text size (kbytes): 0
Average unshared data size (kbytes): 0
Average stack size (kbytes): 0
Average total size (kbytes): 0
Maximum resident set size (kbytes): 2229904
Average resident set size (kbytes): 0
Major (requiring I/O) page faults: 0
Minor (reclaiming a frame) page faults: 919491
Voluntary context switches: 253186
Involuntary context switches: 1015
Swaps: 0
File system inputs: 0
File system outputs: 109456
Socket messages sent: 0
Socket messages received: 0
Signals delivered: 0
Page size (bytes): 4096
Exit status: 0
Starting training ...
Starting Compression ...
================================================
FILE: data/logs_data/Caenorhabditis_elegans.WBcel235.dna.chromosome.I/LSTM.log.csv
================================================
Starting training ...
Starting Compression ...
================================================
FILE: data/logs_data/Caenorhabditis_elegans.WBcel235.dna.chromosome.I/LSTM_multi.log.csv
================================================
Starting training ...
0;1.9433898283044024
1;1.9385781589187077
2;1.9332776730358134
3;1.9286803646218527
4;1.9261398045894722
Starting training ...
0;1.8457577567813561
1;1.8213510305971075
2;1.8102456047382842
3;1.8036074533727524
4;1.7991041454156704
5;1.7958321974437694
6;1.7934263378181279
7;1.7914524390666866
8;1.7898222145741705
9;1.7884504948874613
Starting Compression ...
2018-11-01 01:07:42.983901: I tensorflow/core/platform/cpu_feature_guard.cc:141] Your CPU supports instructions that this TensorFlow binary was not compiled to use: AVX2 FMA
2018-11-01 01:07:45.818409: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1411] Found device 0 with properties:
name: TITAN X (Pascal) major: 6 minor: 1 memoryClockRate(GHz): 1.531
pciBusID: 0000:05:00.0
totalMemory: 11.91GiB freeMemory: 11.75GiB
2018-11-01 01:07:45.818468: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1490] Adding visible gpu devices: 0
2018-11-01 01:07:46.159991: I tensorflow/core/common_runtime/gpu/gpu_device.cc:971] Device interconnect StreamExecutor with strength 1 edge matrix:
2018-11-01 01:07:46.160063: I tensorflow/core/common_runtime/gpu/gpu_device.cc:977] 0
2018-11-01 01:07:46.160073: I tensorflow/core/common_runtime/gpu/gpu_device.cc:990] 0: N
2018-11-01 01:07:46.160423: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1103] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 11372 MB memory) -> physical GPU (device: 0, name: TITAN X (Pascal), pci bus id: 0000:05:00.0, compute capability: 6.1)
/opt/conda/lib/python3.6/site-packages/sklearn/externals/joblib/externals/cloudpickle/cloudpickle.py:47: DeprecationWarning: the imp module is deprecated in favour of importlib; see the module's documentation for alternative uses
import imp
Using TensorFlow backend.
/opt/conda/lib/python3.6/site-packages/sklearn/preprocessing/_encoders.py:363: FutureWarning: The handling of integer data will change in version 0.22. Currently, the categories are determined based on the range [0, max(values)], while in the future they will be determined based on the unique values.
If you want the future behaviour and silence this warning, you can specify "categories='auto'".
In case you used a LabelEncoder before this OneHotEncoder to convert the categories to integers, then you can now use the OneHotEncoder directly.
warnings.warn(msg, FutureWarning)
Command being timed: "python compressor.py -data ../data/processed_files/Caenorhabditis_elegans.WBcel235.dna.chromosome.I.npy -data_params ../data/processed_files/Caenorhabditis_elegans.WBcel235.dna.chromosome.I.param.json -model ../data/trained_models/Caenorhabditis_elegans.WBcel235.dna.chromosome.I/LSTM_multi.hdf5 -model_name LSTM_multi -output ../data/compressed/Caenorhabditis_elegans.WBcel235.dna.chromosome.I/LSTM_multi.compressed -batch_size 10000"
User time (seconds): 247.09
System time (seconds): 21.23
Percent of CPU this job got: 113%
Elapsed (wall clock) time (h:mm:ss or m:ss): 3:56.38
Average shared text size (kbytes): 0
Average unshared data size (kbytes): 0
Average stack size (kbytes): 0
Average total size (kbytes): 0
Maximum resident set size (kbytes): 2705992
Average resident set size (kbytes): 0
Major (requiring I/O) page faults: 0
Minor (reclaiming a frame) page faults: 2222934
Voluntary context switches: 458731
Involuntary context switches: 4003
Swaps: 0
File system inputs: 240
File system outputs: 86672
Socket messages sent: 0
Socket messages received: 0
Signals delivered: 0
Page size (bytes): 4096
Exit status: 0
2018-11-01 01:11:35.211751: I tensorflow/core/platform/cpu_feature_guard.cc:141] Your CPU supports instructions that this TensorFlow binary was not compiled to use: AVX2 FMA
2018-11-01 01:11:38.044651: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1411] Found device 0 with properties:
name: TITAN X (Pascal) major: 6 minor: 1 memoryClockRate(GHz): 1.531
pciBusID: 0000:05:00.0
totalMemory: 11.91GiB freeMemory: 11.75GiB
2018-11-01 01:11:38.044701: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1490] Adding visible gpu devices: 0
2018-11-01 01:11:38.400698: I tensorflow/core/common_runtime/gpu/gpu_device.cc:971] Device interconnect StreamExecutor with strength 1 edge matrix:
2018-11-01 01:11:38.400768: I tensorflow/core/common_runtime/gpu/gpu_device.cc:977] 0
2018-11-01 01:11:38.400778: I tensorflow/core/common_runtime/gpu/gpu_device.cc:990] 0: N
2018-11-01 01:11:38.401176: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1103] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 11372 MB memory) -> physical GPU (device: 0, name: TITAN X (Pascal), pci bus id: 0000:05:00.0, compute capability: 6.1)
/opt/conda/lib/python3.6/site-packages/sklearn/externals/joblib/externals/cloudpickle/cloudpickle.py:47: DeprecationWarning: the imp module is deprecated in favour of importlib; see the module's documentation for alternative uses
import imp
Using TensorFlow backend.
{'0': 'A', '1': 'C', '2': 'T', '3': 'G'}
[3 1 1 2 0 0 3 1 1 2]
Command being timed: "python decompressor.py -output ../data/compressed/Caenorhabditis_elegans.WBcel235.dna.chromosome.I/LSTM_multi.reconstructed.txt -model ../data/trained_models/Caenorhabditis_elegans.WBcel235.dna.chromosome.I/LSTM_multi.hdf5 -model_name LSTM_multi -input_file_prefix ../data/compressed/Caenorhabditis_elegans.WBcel235.dna.chromosome.I/LSTM_multi.compressed -batch_size 10000"
User time (seconds): 356.23
System time (seconds): 17.66
Percent of CPU this job got: 105%
Elapsed (wall clock) time (h:mm:ss or m:ss): 5:52.93
Average shared text size (kbytes): 0
Average unshared data size (kbytes): 0
Average stack size (kbytes): 0
Average total size (kbytes): 0
Maximum resident set size (kbytes): 2238464
Average resident set size (kbytes): 0
Major (requiring I/O) page faults: 0
Minor (reclaiming a frame) page faults: 995024
Voluntary context switches: 441765
Involuntary context switches: 989
Swaps: 0
File system inputs: 0
File system outputs: 109456
Socket messages sent: 0
Socket messages received: 0
Signals delivered: 0
Page size (bytes): 4096
Exit status: 0
Starting Compression ...
2018-11-08 09:16:44.199315: I tensorflow/core/platform/cpu_feature_guard.cc:141] Your CPU supports instructions that this TensorFlow binary was not compiled to use: AVX2 FMA
2018-11-08 09:16:47.173935: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1411] Found device 0 with properties:
name: TITAN X (Pascal) major: 6 minor: 1 memoryClockRate(GHz): 1.531
pciBusID: 0000:08:00.0
totalMemory: 11.91GiB freeMemory: 11.75GiB
2018-11-08 09:16:47.173989: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1490] Adding visible gpu devices: 0
2018-11-08 09:16:47.505455: I tensorflow/core/common_runtime/gpu/gpu_device.cc:971] Device interconnect StreamExecutor with strength 1 edge matrix:
2018-11-08 09:16:47.505528: I tensorflow/core/common_runtime/gpu/gpu_device.cc:977] 0
2018-11-08 09:16:47.505536: I tensorflow/core/common_runtime/gpu/gpu_device.cc:990] 0: N
2018-11-08 09:16:47.505938: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1103] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 11372 MB memory) -> physical GPU (device: 0, name: TITAN X (Pascal), pci bus id: 0000:08:00.0, compute capability: 6.1)
/opt/conda/lib/python3.6/site-packages/sklearn/externals/joblib/externals/cloudpickle/cloudpickle.py:47: DeprecationWarning: the imp module is deprecated in favour of importlib; see the module's documentation for alternative uses
import imp
Using TensorFlow backend.
/opt/conda/lib/python3.6/site-packages/sklearn/preprocessing/_encoders.py:363: FutureWarning: The handling of integer data will change in version 0.22. Currently, the categories are determined based on the range [0, max(values)], while in the future they will be determined based on the unique values.
If you want the future behaviour and silence this warning, you can specify "categories='auto'".
In case you used a LabelEncoder before this OneHotEncoder to convert the categories to integers, then you can now use the OneHotEncoder directly.
warnings.warn(msg, FutureWarning)
Command being timed: "python compressor.py -data ../data/processed_files/Caenorhabditis_elegans.WBcel235.dna.chromosome.I.npy -data_params ../data/processed_files/Caenorhabditis_elegans.WBcel235.dna.chromosome.I.param.json -model ../data/trained_models/Caenorhabditis_elegans.WBcel235.dna.chromosome.I/LSTM_multi.hdf5 -model_name LSTM_multi -output ../data/compressed/Caenorhabditis_elegans.WBcel235.dna.chromosome.I/LSTM_multi.compressed -batch_size 1000"
User time (seconds): 332.28
System time (seconds): 15.02
Percent of CPU this job got: 115%
Elapsed (wall clock) time (h:mm:ss or m:ss): 4:59.83
Average shared text size (kbytes): 0
Average unshared data size (kbytes): 0
Average stack size (kbytes): 0
Average total size (kbytes): 0
Maximum resident set size (kbytes): 2637864
Average resident set size (kbytes): 0
Major (requiring I/O) page faults: 0
Minor (reclaiming a frame) page faults: 2044863
Voluntary context switches: 368611
Involuntary context switches: 2128
Swaps: 0
File system inputs: 0
File system outputs: 14600
Socket messages sent: 0
Socket messages received: 0
Signals delivered: 0
Page size (bytes): 4096
Exit status: 0
2018-11-08 09:21:39.152464: I tensorflow/core/platform/cpu_feature_guard.cc:141] Your CPU supports instructions that this TensorFlow binary was not compiled to use: AVX2 FMA
2018-11-08 09:21:42.090926: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1411] Found device 0 with properties:
name: TITAN X (Pascal) major: 6 minor: 1 memoryClockRate(GHz): 1.531
pciBusID: 0000:08:00.0
totalMemory: 11.91GiB freeMemory: 11.75GiB
2018-11-08 09:21:42.090973: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1490] Adding visible gpu devices: 0
2018-11-08 09:21:42.433280: I tensorflow/core/common_runtime/gpu/gpu_device.cc:971] Device interconnect StreamExecutor with strength 1 edge matrix:
2018-11-08 09:21:42.433346: I tensorflow/core/common_runtime/gpu/gpu_device.cc:977] 0
2018-11-08 09:21:42.433372: I tensorflow/core/common_runtime/gpu/gpu_device.cc:990] 0: N
2018-11-08 09:21:42.433786: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1103] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 11372 MB memory) -> physical GPU (device: 0, name: TITAN X (Pascal), pci bus id: 0000:08:00.0, compute capability: 6.1)
/opt/conda/lib/python3.6/site-packages/sklearn/externals/joblib/externals/cloudpickle/cloudpickle.py:47: DeprecationWarning: the imp module is deprecated in favour of importlib; see the module's documentation for alternative uses
import imp
Using TensorFlow backend.
{'0': 'A', '1': 'C', '2': 'T', '3': 'G'}
[3 1 1 2 0 0 3 1 1 2]
Command being timed: "python decompressor.py -output ../data/compressed/Caenorhabditis_elegans.WBcel235.dna.chromosome.I/LSTM_multi.reconstructed.txt -model ../data/trained_models/Caenorhabditis_elegans.WBcel235.dna.chromosome.I/LSTM_multi.hdf5 -model_name LSTM_multi -input_file_prefix ../data/compressed/Caenorhabditis_elegans.WBcel235.dna.chromosome.I/LSTM_multi.compressed -batch_size 1000"
User time (seconds): 430.22
System time (seconds): 10.54
Percent of CPU this job got: 109%
Elapsed (wall clock) time (h:mm:ss or m:ss): 6:41.99
Average shared text size (kbytes): 0
Average unshared data size (kbytes): 0
Average stack size (kbytes): 0
Average total size (kbytes): 0
Maximum resident set size (kbytes): 2226432
Average resident set size (kbytes): 0
Major (requiring I/O) page faults: 0
Minor (reclaiming a frame) page faults: 657063
Voluntary context switches: 360946
Involuntary context switches: 1718
Swaps: 0
File system inputs: 0
File system outputs: 37456
Socket messages sent: 0
Socket messages received: 0
Signals delivered: 0
Page size (bytes): 4096
Exit status: 0
================================================
FILE: data/logs_data/Caenorhabditis_elegans.WBcel235.dna.chromosome.I/LSTM_multi_16bit.log.csv
================================================
Starting training ...
0;1.8784998742059582
1;1.8550203101645832
2;1.8470342462336096
3;1.8426768361683878
Starting Compression ...
2018-10-28 21:07:27.194563: I tensorflow/core/platform/cpu_feature_guard.cc:141] Your CPU supports instructions that this TensorFlow binary was not compiled to use: AVX2 FMA
2018-10-28 21:07:29.060176: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1411] Found device 0 with properties:
name: TITAN X (Pascal) major: 6 minor: 1 memoryClockRate(GHz): 1.531
pciBusID: 0000:86:00.0
totalMemory: 11.91GiB freeMemory: 11.75GiB
2018-10-28 21:07:29.060227: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1490] Adding visible gpu devices: 0
2018-10-28 21:07:29.419994: I tensorflow/core/common_runtime/gpu/gpu_device.cc:971] Device interconnect StreamExecutor with strength 1 edge matrix:
2018-10-28 21:07:29.420061: I tensorflow/core/common_runtime/gpu/gpu_device.cc:977] 0
2018-10-28 21:07:29.420071: I tensorflow/core/common_runtime/gpu/gpu_device.cc:990] 0: N
2018-10-28 21:07:29.420464: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1103] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 11372 MB memory) -> physical GPU (device: 0, name: TITAN X (Pascal), pci bus id: 0000:86:00.0, compute capability: 6.1)
/opt/conda/lib/python3.6/site-packages/sklearn/externals/joblib/externals/cloudpickle/cloudpickle.py:47: DeprecationWarning: the imp module is deprecated in favour of importlib; see the module's documentation for alternative uses
import imp
Using TensorFlow backend.
/opt/conda/lib/python3.6/site-packages/sklearn/preprocessing/_encoders.py:363: FutureWarning: The handling of integer data will change in version 0.22. Currently, the categories are determined based on the range [0, max(values)], while in the future they will be determined based on the unique values.
If you want the future behaviour and silence this warning, you can specify "categories='auto'".
In case you used a LabelEncoder before this OneHotEncoder to convert the categories to integers, then you can now use the OneHotEncoder directly.
warnings.warn(msg, FutureWarning)
Command being timed: "python compressor.py -data ../data/processed_files/Caenorhabditis_elegans.WBcel235.dna.chromosome.I.npy -data_params ../data/processed_files/Caenorhabditis_elegans.WBcel235.dna.chromosome.I.param.json -model ../data/trained_models/Caenorhabditis_elegans.WBcel235.dna.chromosome.I/LSTM_multi_16bit.hdf5 -model_name LSTM_multi_16bit -output ../data/compressed/Caenorhabditis_elegans.WBcel235.dna.chromosome.I/LSTM_multi_16bit.compressed -batch_size 10000"
User time (seconds): 253.15
System time (seconds): 15.75
Percent of CPU this job got: 112%
Elapsed (wall clock) time (h:mm:ss or m:ss): 3:58.84
Average shared text size (kbytes): 0
Average unshared data size (kbytes): 0
Average stack size (kbytes): 0
Average total size (kbytes): 0
Maximum resident set size (kbytes): 2700996
Average resident set size (kbytes): 0
Major (requiring I/O) page faults: 0
Minor (reclaiming a frame) page faults: 2259375
Voluntary context switches: 296194
Involuntary context switches: 15105
Swaps: 0
File system inputs: 0
File system outputs: 86872
Socket messages sent: 0
Socket messages received: 0
Signals delivered: 0
Page size (bytes): 4096
Exit status: 0
2018-10-28 21:11:21.501837: I tensorflow/core/platform/cpu_feature_guard.cc:141] Your CPU supports instructions that this TensorFlow binary was not compiled to use: AVX2 FMA
2018-10-28 21:11:23.466047: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1411] Found device 0 with properties:
name: TITAN X (Pascal) major: 6 minor: 1 memoryClockRate(GHz): 1.531
pciBusID: 0000:86:00.0
totalMemory: 11.91GiB freeMemory: 11.75GiB
2018-10-28 21:11:23.466210: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1490] Adding visible gpu devices: 0
2018-10-28 21:11:23.879871: I tensorflow/core/common_runtime/gpu/gpu_device.cc:971] Device interconnect StreamExecutor with strength 1 edge matrix:
2018-10-28 21:11:23.879937: I tensorflow/core/common_runtime/gpu/gpu_device.cc:977] 0
2018-10-28 21:11:23.879951: I tensorflow/core/common_runtime/gpu/gpu_device.cc:990] 0: N
2018-10-28 21:11:23.880320: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1103] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 11372 MB memory) -> physical GPU (device: 0, name: TITAN X (Pascal), pci bus id: 0000:86:00.0, compute capability: 6.1)
/opt/conda/lib/python3.6/site-packages/sklearn/externals/joblib/externals/cloudpickle/cloudpickle.py:47: DeprecationWarning: the imp module is deprecated in favour of importlib; see the module's documentation for alternative uses
import imp
Using TensorFlow backend.
{'0': 'A', '1': 'C', '2': 'T', '3': 'G'}
[3 1 1 2 0 0 3 1 1 2]
Command being timed: "python decompressor.py -output ../data/compressed/Caenorhabditis_elegans.WBcel235.dna.chromosome.I/LSTM_multi_16bit.reconstructed.txt -model ../data/trained_models/Caenorhabditis_elegans.WBcel235.dna.chromosome.I/LSTM_multi_16bit.hdf5 -model_name LSTM_multi_16bit -input_file_prefix ../data/compressed/Caenorhabditis_elegans.WBcel235.dna.chromosome.I/LSTM_multi_16bit.compressed -batch_size 10000"
User time (seconds): 335.83
System time (seconds): 13.30
Percent of CPU this job got: 105%
Elapsed (wall clock) time (h:mm:ss or m:ss): 5:29.87
Average shared text size (kbytes): 0
Average unshared data size (kbytes): 0
Average stack size (kbytes): 0
Average total size (kbytes): 0
Maximum resident set size (kbytes): 2233512
Average resident set size (kbytes): 0
Major (requiring I/O) page faults: 0
Minor (reclaiming a frame) page faults: 921277
Voluntary context switches: 312194
Involuntary context switches: 1025
Swaps: 0
File system inputs: 0
File system outputs: 109456
Socket messages sent: 0
Socket messages received: 0
Signals delivered: 0
Page size (bytes): 4096
Exit status: 0
================================================
FILE: data/logs_data/Caenorhabditis_elegans.WBcel235.dna.chromosome.I/LSTM_multi_big.log.csv
================================================
Starting training ...
0;1.8382193629237424
1;1.8049677677943061
2;1.7903391000846942
3;1.7822841469607718
4;1.7767529003289222
5;1.7727606372103046
6;1.7696305803488401
7;1.7670118375854393
8;1.764731097281307
9;1.7628604925977074
Starting Compression ...
2018-11-02 21:29:36.499952: I tensorflow/core/platform/cpu_feature_guard.cc:141] Your CPU supports instructions that this TensorFlow binary was not compiled to use: AVX2 FMA
2018-11-02 21:29:40.112911: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1411] Found device 0 with properties:
name: TITAN X (Pascal) major: 6 minor: 1 memoryClockRate(GHz): 1.531
pciBusID: 0000:8a:00.0
totalMemory: 11.91GiB freeMemory: 11.75GiB
2018-11-02 21:29:40.112963: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1490] Adding visible gpu devices: 0
2018-11-02 21:29:40.473651: I tensorflow/core/common_runtime/gpu/gpu_device.cc:971] Device interconnect StreamExecutor with strength 1 edge matrix:
2018-11-02 21:29:40.473719: I tensorflow/core/common_runtime/gpu/gpu_device.cc:977] 0
2018-11-02 21:29:40.473727: I tensorflow/core/common_runtime/gpu/gpu_device.cc:990] 0: N
2018-11-02 21:29:40.474138: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1103] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 11372 MB memory) -> physical GPU (device: 0, name: TITAN X (Pascal), pci bus id: 0000:8a:00.0, compute capability: 6.1)
/opt/conda/lib/python3.6/site-packages/sklearn/externals/joblib/externals/cloudpickle/cloudpickle.py:47: DeprecationWarning: the imp module is deprecated in favour of importlib; see the module's documentation for alternative uses
import imp
Using TensorFlow backend.
/opt/conda/lib/python3.6/site-packages/sklearn/preprocessing/_encoders.py:363: FutureWarning: The handling of integer data will change in version 0.22. Currently, the categories are determined based on the range [0, max(values)], while in the future they will be determined based on the unique values.
If you want the future behaviour and silence this warning, you can specify "categories='auto'".
In case you used a LabelEncoder before this OneHotEncoder to convert the categories to integers, then you can now use the OneHotEncoder directly.
warnings.warn(msg, FutureWarning)
Command being timed: "python compressor.py -data ../data/processed_files/Caenorhabditis_elegans.WBcel235.dna.chromosome.I.npy -data_params ../data/processed_files/Caenorhabditis_elegans.WBcel235.dna.chromosome.I.param.json -model ../data/trained_models/Caenorhabditis_elegans.WBcel235.dna.chromosome.I/LSTM_multi_big.hdf5 -model_name LSTM_multi_big -output ../data/compressed/Caenorhabditis_elegans.WBcel235.dna.chromosome.I/LSTM_multi_big.compressed -batch_size 10000"
User time (seconds): 233.14
System time (seconds): 29.31
Percent of CPU this job got: 114%
Elapsed (wall clock) time (h:mm:ss or m:ss): 3:49.39
Average shared text size (kbytes): 0
Average unshared data size (kbytes): 0
Average stack size (kbytes): 0
Average total size (kbytes): 0
Maximum resident set size (kbytes): 2709604
Average resident set size (kbytes): 0
Major (requiring I/O) page faults: 0
Minor (reclaiming a frame) page faults: 2456838
Voluntary context switches: 735947
Involuntary context switches: 913
Swaps: 0
File system inputs: 0
File system outputs: 86576
Socket messages sent: 0
Socket messages received: 0
Signals delivered: 0
Page size (bytes): 4096
Exit status: 0
2018-11-02 21:33:21.632696: I tensorflow/core/platform/cpu_feature_guard.cc:141] Your CPU supports instructions that this TensorFlow binary was not compiled to use: AVX2 FMA
2018-11-02 21:33:25.247949: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1411] Found device 0 with properties:
name: TITAN X (Pascal) major: 6 minor: 1 memoryClockRate(GHz): 1.531
pciBusID: 0000:8a:00.0
totalMemory: 11.91GiB freeMemory: 11.75GiB
2018-11-02 21:33:25.248016: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1490] Adding visible gpu devices: 0
2018-11-02 21:33:25.584470: I tensorflow/core/common_runtime/gpu/gpu_device.cc:971] Device interconnect StreamExecutor with strength 1 edge matrix:
2018-11-02 21:33:25.584539: I tensorflow/core/common_runtime/gpu/gpu_device.cc:977] 0
2018-11-02 21:33:25.584551: I tensorflow/core/common_runtime/gpu/gpu_device.cc:990] 0: N
2018-11-02 21:33:25.584973: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1103] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 11372 MB memory) -> physical GPU (device: 0, name: TITAN X (Pascal), pci bus id: 0000:8a:00.0, compute capability: 6.1)
/opt/conda/lib/python3.6/site-packages/sklearn/externals/joblib/externals/cloudpickle/cloudpickle.py:47: DeprecationWarning: the imp module is deprecated in favour of importlib; see the module's documentation for alternative uses
import imp
Using TensorFlow backend.
{'0': 'A', '1': 'C', '2': 'T', '3': 'G'}
[3 1 1 2 0 0 3 1 1 2]
Command being timed: "python decompressor.py -output ../data/compressed/Caenorhabditis_elegans.WBcel235.dna.chromosome.I/LSTM_multi_big.reconstructed.txt -model ../data/trained_models/Caenorhabditis_elegans.WBcel235.dna.chromosome.I/LSTM_multi_big.hdf5 -model_name LSTM_multi_big -input_file_prefix ../data/compressed/Caenorhabditis_elegans.WBcel235.dna.chromosome.I/LSTM_multi_big.compressed -batch_size 10000"
User time (seconds): 314.59
System time (seconds): 26.67
Percent of CPU this job got: 107%
Elapsed (wall clock) time (h:mm:ss or m:ss): 5:16.21
Average shared text size (kbytes): 0
Average unshared data size (kbytes): 0
Average stack size (kbytes): 0
Average total size (kbytes): 0
Maximum resident set size (kbytes): 2242028
Average resident set size (kbytes): 0
Major (requiring I/O) page faults: 0
Minor (reclaiming a frame) page faults: 1093091
Voluntary context switches: 736064
Involuntary context switches: 948
Swaps: 0
File system inputs: 0
File system outputs: 109456
Socket messages sent: 0
Socket messages received: 0
Signals delivered: 0
Page size (bytes): 4096
Exit status: 0
================================================
FILE: data/logs_data/Caenorhabditis_elegans.WBcel235.dna.chromosome.I/LSTM_multi_bn.log.csv
================================================
Starting training ...
0;1.8545062726214088
1;1.8412349406804183
2;1.8348612771409063
3;1.828830596350872
4;1.824105862771309
5;1.8197827900838526
6;1.8156011739788889
7;1.8113264455946902
8;1.8075687118227435
9;1.8045242928204892
Starting Compression ...
2018-11-06 19:07:10.037004: I tensorflow/core/platform/cpu_feature_guard.cc:141] Your CPU supports instructions that this TensorFlow binary was not compiled to use: AVX2 FMA
2018-11-06 19:07:12.716572: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1411] Found device 0 with properties:
name: TITAN X (Pascal) major: 6 minor: 1 memoryClockRate(GHz): 1.531
pciBusID: 0000:09:00.0
totalMemory: 11.91GiB freeMemory: 11.75GiB
2018-11-06 19:07:12.716636: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1490] Adding visible gpu devices: 0
2018-11-06 19:07:13.067698: I tensorflow/core/common_runtime/gpu/gpu_device.cc:971] Device interconnect StreamExecutor with strength 1 edge matrix:
2018-11-06 19:07:13.067770: I tensorflow/core/common_runtime/gpu/gpu_device.cc:977] 0
2018-11-06 19:07:13.067780: I tensorflow/core/common_runtime/gpu/gpu_device.cc:990] 0: N
2018-11-06 19:07:13.068115: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1103] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 11372 MB memory) -> physical GPU (device: 0, name: TITAN X (Pascal), pci bus id: 0000:09:00.0, compute capability: 6.1)
/opt/conda/lib/python3.6/site-packages/sklearn/externals/joblib/externals/cloudpickle/cloudpickle.py:47: DeprecationWarning: the imp module is deprecated in favour of importlib; see the module's documentation for alternative uses
import imp
Using TensorFlow backend.
/opt/conda/lib/python3.6/site-packages/sklearn/preprocessing/_encoders.py:363: FutureWarning: The handling of integer data will change in version 0.22. Currently, the categories are determined based on the range [0, max(values)], while in the future they will be determined based on the unique values.
If you want the future behaviour and silence this warning, you can specify "categories='auto'".
In case you used a LabelEncoder before this OneHotEncoder to convert the categories to integers, then you can now use the OneHotEncoder directly.
warnings.warn(msg, FutureWarning)
Command being timed: "python compressor.py -data ../data/processed_files/Caenorhabditis_elegans.WBcel235.dna.chromosome.I.npy -data_params ../data/processed_files/Caenorhabditis_elegans.WBcel235.dna.chromosome.I.param.json -model ../data/trained_models/Caenorhabditis_elegans.WBcel235.dna.chromosome.I/LSTM_multi_bn.hdf5 -model_name LSTM_multi_bn -output ../data/compressed/Caenorhabditis_elegans.WBcel235.dna.chromosome.I/LSTM_multi_bn.compressed -batch_size 10000"
User time (seconds): 254.24
System time (seconds): 22.92
Percent of CPU this job got: 113%
Elapsed (wall clock) time (h:mm:ss or m:ss): 4:04.21
Average shared text size (kbytes): 0
Average unshared data size (kbytes): 0
Average stack size (kbytes): 0
Average total size (kbytes): 0
Maximum resident set size (kbytes): 2708504
Average resident set size (kbytes): 0
Major (requiring I/O) page faults: 0
Minor (reclaiming a frame) page faults: 2149672
Voluntary context switches: 542322
Involuntary context switches: 1507
Swaps: 0
File system inputs: 8
File system outputs: 86728
Socket messages sent: 0
Socket messages received: 0
Signals delivered: 0
Page size (bytes): 4096
Exit status: 0
2018-11-06 19:11:09.426163: I tensorflow/core/platform/cpu_feature_guard.cc:141] Your CPU supports instructions that this TensorFlow binary was not compiled to use: AVX2 FMA
2018-11-06 19:11:12.297404: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1411] Found device 0 with properties:
name: TITAN X (Pascal) major: 6 minor: 1 memoryClockRate(GHz): 1.531
pciBusID: 0000:09:00.0
totalMemory: 11.91GiB freeMemory: 11.75GiB
2018-11-06 19:11:12.297474: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1490] Adding visible gpu devices: 0
2018-11-06 19:11:12.701712: I tensorflow/core/common_runtime/gpu/gpu_device.cc:971] Device interconnect StreamExecutor with strength 1 edge matrix:
2018-11-06 19:11:12.701774: I tensorflow/core/common_runtime/gpu/gpu_device.cc:977] 0
2018-11-06 19:11:12.701784: I tensorflow/core/common_runtime/gpu/gpu_device.cc:990] 0: N
2018-11-06 19:11:12.702146: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1103] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 11372 MB memory) -> physical GPU (device: 0, name: TITAN X (Pascal), pci bus id: 0000:09:00.0, compute capability: 6.1)
/opt/conda/lib/python3.6/site-packages/sklearn/externals/joblib/externals/cloudpickle/cloudpickle.py:47: DeprecationWarning: the imp module is deprecated in favour of importlib; see the module's documentation for alternative uses
import imp
Using TensorFlow backend.
{'0': 'A', '1': 'C', '2': 'T', '3': 'G'}
[3 1 1 2 0 0 3 1 1 2]
Command being timed: "python decompressor.py -output ../data/compressed/Caenorhabditis_elegans.WBcel235.dna.chromosome.I/LSTM_multi_bn.reconstructed.txt -model ../data/trained_models/Caenorhabditis_elegans.WBcel235.dna.chromosome.I/LSTM_multi_bn.hdf5 -model_name LSTM_multi_bn -input_file_prefix ../data/compressed/Caenorhabditis_elegans.WBcel235.dna.chromosome.I/LSTM_multi_bn.compressed -batch_size 10000"
User time (seconds): 337.19
System time (seconds): 19.67
Percent of CPU this job got: 106%
Elapsed (wall clock) time (h:mm:ss or m:ss): 5:34.75
Average shared text size (kbytes): 0
Average unshared data size (kbytes): 0
Average stack size (kbytes): 0
Average total size (kbytes): 0
Maximum resident set size (kbytes): 2245004
Average resident set size (kbytes): 0
Major (requiring I/O) page faults: 0
Minor (reclaiming a frame) page faults: 1032963
Voluntary context switches: 545756
Involuntary context switches: 1289
Swaps: 0
File system inputs: 0
File system outputs: 109456
Socket messages sent: 0
Socket messages received: 0
Signals delivered: 0
Page size (bytes): 4096
Exit status: 0
================================================
FILE: data/logs_data/Caenorhabditis_elegans.WBcel235.dna.chromosome.I/biGRU.log.csv
================================================
Starting training ...
0;1.8535430385606575
1;1.8356075893156927
2;1.8279383734311008
3;1.822314586682491
4;1.8186759031760853
5;1.8149080831676214
6;1.8131204937631584
7;1.8109603441800668
8;1.8089284555801386
9;1.8076730038421749
Starting Compression ...
2018-11-01 01:46:23.834199: I tensorflow/core/platform/cpu_feature_guard.cc:141] Your CPU supports instructions that this TensorFlow binary was not compiled to use: AVX2 FMA
2018-11-01 01:46:26.637461: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1411] Found device 0 with properties:
name: TITAN X (Pascal) major: 6 minor: 1 memoryClockRate(GHz): 1.531
pciBusID: 0000:85:00.0
totalMemory: 11.91GiB freeMemory: 11.75GiB
2018-11-01 01:46:26.637511: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1490] Adding visible gpu devices: 0
2018-11-01 01:46:27.008206: I tensorflow/core/common_runtime/gpu/gpu_device.cc:971] Device interconnect StreamExecutor with strength 1 edge matrix:
2018-11-01 01:46:27.008275: I tensorflow/core/common_runtime/gpu/gpu_device.cc:977] 0
2018-11-01 01:46:27.008285: I tensorflow/core/common_runtime/gpu/gpu_device.cc:990] 0: N
2018-11-01 01:46:27.008625: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1103] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 11372 MB memory) -> physical GPU (device: 0, name: TITAN X (Pascal), pci bus id: 0000:85:00.0, compute capability: 6.1)
/opt/conda/lib/python3.6/site-packages/sklearn/externals/joblib/externals/cloudpickle/cloudpickle.py:47: DeprecationWarning: the imp module is deprecated in favour of importlib; see the module's documentation for alternative uses
import imp
Using TensorFlow backend.
/opt/conda/lib/python3.6/site-packages/sklearn/preprocessing/_encoders.py:363: FutureWarning: The handling of integer data will change in version 0.22. Currently, the categories are determined based on the range [0, max(values)], while in the future they will be determined based on the unique values.
If you want the future behaviour and silence this warning, you can specify "categories='auto'".
In case you used a LabelEncoder before this OneHotEncoder to convert the categories to integers, then you can now use the OneHotEncoder directly.
warnings.warn(msg, FutureWarning)
Command being timed: "python compressor.py -data ../data/processed_files/Caenorhabditis_elegans.WBcel235.dna.chromosome.I.npy -data_params ../data/processed_files/Caenorhabditis_elegans.WBcel235.dna.chromosome.I.param.json -model ../data/trained_models/Caenorhabditis_elegans.WBcel235.dna.chromosome.I/biGRU.hdf5 -model_name biGRU -output ../data/compressed/Caenorhabditis_elegans.WBcel235.dna.chromosome.I/biGRU.compressed -batch_size 10000"
User time (seconds): 308.05
System time (seconds): 31.66
Percent of CPU this job got: 113%
Elapsed (wall clock) time (h:mm:ss or m:ss): 4:59.91
Average shared text size (kbytes): 0
Average unshared data size (kbytes): 0
Average stack size (kbytes): 0
Average total size (kbytes): 0
Maximum resident set size (kbytes): 2705912
Average resident set size (kbytes): 0
Major (requiring I/O) page faults: 0
Minor (reclaiming a frame) page faults: 2663919
Voluntary context switches: 730296
Involuntary context switches: 7651
Swaps: 0
File system inputs: 8
File system outputs: 86752
Socket messages sent: 0
Socket messages received: 0
Signals delivered: 0
Page size (bytes): 4096
Exit status: 0
2018-11-01 01:51:19.058539: I tensorflow/core/platform/cpu_feature_guard.cc:141] Your CPU supports instructions that this TensorFlow binary was not compiled to use: AVX2 FMA
2018-11-01 01:51:21.715797: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1411] Found device 0 with properties:
name: TITAN X (Pascal) major: 6 minor: 1 memoryClockRate(GHz): 1.531
pciBusID: 0000:85:00.0
totalMemory: 11.91GiB freeMemory: 11.75GiB
2018-11-01 01:51:21.715855: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1490] Adding visible gpu devices: 0
2018-11-01 01:51:22.096279: I tensorflow/core/common_runtime/gpu/gpu_device.cc:971] Device interconnect StreamExecutor with strength 1 edge matrix:
2018-11-01 01:51:22.096341: I tensorflow/core/common_runtime/gpu/gpu_device.cc:977] 0
2018-11-01 01:51:22.096350: I tensorflow/core/common_runtime/gpu/gpu_device.cc:990] 0: N
2018-11-01 01:51:22.096734: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1103] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 11372 MB memory) -> physical GPU (device: 0, name: TITAN X (Pascal), pci bus id: 0000:85:00.0, compute capability: 6.1)
/opt/conda/lib/python3.6/site-packages/sklearn/externals/joblib/externals/cloudpickle/cloudpickle.py:47: DeprecationWarning: the imp module is deprecated in favour of importlib; see the module's documentation for alternative uses
import imp
Using TensorFlow backend.
{'0': 'A', '1': 'C', '2': 'T', '3': 'G'}
[3 1 1 2 0 0 3 1 1 2]
Command being timed: "python decompressor.py -output ../data/compressed/Caenorhabditis_elegans.WBcel235.dna.chromosome.I/biGRU.reconstructed.txt -model ../data/trained_models/Caenorhabditis_elegans.WBcel235.dna.chromosome.I/biGRU.hdf5 -model_name biGRU -input_file_prefix ../data/compressed/Caenorhabditis_elegans.WBcel235.dna.chromosome.I/biGRU.compressed -batch_size 10000"
User time (seconds): 413.41
System time (seconds): 27.82
Percent of CPU this job got: 107%
Elapsed (wall clock) time (h:mm:ss or m:ss): 6:51.81
Average shared text size (kbytes): 0
Average unshared data size (kbytes): 0
Average stack size (kbytes): 0
Average total size (kbytes): 0
Maximum resident set size (kbytes): 2237304
Average resident set size (kbytes): 0
Major (requiring I/O) page faults: 0
Minor (reclaiming a frame) page faults: 1183547
Voluntary context switches: 741516
Involuntary context switches: 1482
Swaps: 0
File system inputs: 0
File system outputs: 109456
Socket messages sent: 0
Socket messages received: 0
Signals delivered: 0
Page size (bytes): 4096
Exit status: 0
Starting Compression ...
2018-11-08 09:15:53.053568: I tensorflow/core/platform/cpu_feature_guard.cc:141] Your CPU supports instructions that this TensorFlow binary was not compiled to use: AVX2 FMA
2018-11-08 09:15:56.237871: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1411] Found device 0 with properties:
name: TITAN X (Pascal) major: 6 minor: 1 memoryClockRate(GHz): 1.531
pciBusID: 0000:8a:00.0
totalMemory: 11.91GiB freeMemory: 11.75GiB
2018-11-08 09:15:56.237916: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1490] Adding visible gpu devices: 0
2018-11-08 09:15:56.548840: I tensorflow/core/common_runtime/gpu/gpu_device.cc:971] Device interconnect StreamExecutor with strength 1 edge matrix:
2018-11-08 09:15:56.548914: I tensorflow/core/common_runtime/gpu/gpu_device.cc:977] 0
2018-11-08 09:15:56.548925: I tensorflow/core/common_runtime/gpu/gpu_device.cc:990] 0: N
2018-11-08 09:15:56.549309: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1103] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 11372 MB memory) -> physical GPU (device: 0, name: TITAN X (Pascal), pci bus id: 0000:8a:00.0, compute capability: 6.1)
/opt/conda/lib/python3.6/site-packages/sklearn/externals/joblib/externals/cloudpickle/cloudpickle.py:47: DeprecationWarning: the imp module is deprecated in favour of importlib; see the module's documentation for alternative uses
import imp
Using TensorFlow backend.
/opt/conda/lib/python3.6/site-packages/sklearn/preprocessing/_encoders.py:363: FutureWarning: The handling of integer data will change in version 0.22. Currently, the categories are determined based on the range [0, max(values)], while in the future they will be determined based on the unique values.
If you want the future behaviour and silence this warning, you can specify "categories='auto'".
In case you used a LabelEncoder before this OneHotEncoder to convert the categories to integers, then you can now use the OneHotEncoder directly.
warnings.warn(msg, FutureWarning)
Command being timed: "python compressor.py -data ../data/processed_files/Caenorhabditis_elegans.WBcel235.dna.chromosome.I.npy -data_params ../data/processed_files/Caenorhabditis_elegans.WBcel235.dna.chromosome.I.param.json -model ../data/trained_models/Caenorhabditis_elegans.WBcel235.dna.chromosome.I/biGRU.hdf5 -model_name biGRU -output ../data/compressed/Caenorhabditis_elegans.WBcel235.dna.chromosome.I/biGRU.compressed -batch_size 1000"
User time (seconds): 405.15
System time (seconds): 16.09
Percent of CPU this job got: 112%
Elapsed (wall clock) time (h:mm:ss or m:ss): 6:13.52
Average shared text size (kbytes): 0
Average unshared data size (kbytes): 0
Average stack size (kbytes): 0
Average total size (kbytes): 0
Maximum resident set size (kbytes): 2634692
Average resident set size (kbytes): 0
Major (requiring I/O) page faults: 0
Minor (reclaiming a frame) page faults: 1940520
Voluntary context switches: 405824
Involuntary context switches: 5831
Swaps: 0
File system inputs: 0
File system outputs: 14680
Socket messages sent: 0
Socket messages received: 0
Signals delivered: 0
Page size (bytes): 4096
Exit status: 0
2018-11-08 09:22:02.272129: I tensorflow/core/platform/cpu_feature_guard.cc:141] Your CPU supports instructions that this TensorFlow binary was not compiled to use: AVX2 FMA
2018-11-08 09:22:05.239472: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1411] Found device 0 with properties:
name: TITAN X (Pascal) major: 6 minor: 1 memoryClockRate(GHz): 1.531
pciBusID: 0000:8a:00.0
totalMemory: 11.91GiB freeMemory: 11.75GiB
2018-11-08 09:22:05.239533: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1490] Adding visible gpu devices: 0
2018-11-08 09:22:05.579497: I tensorflow/core/common_runtime/gpu/gpu_device.cc:971] Device interconnect StreamExecutor with strength 1 edge matrix:
2018-11-08 09:22:05.579574: I tensorflow/core/common_runtime/gpu/gpu_device.cc:977] 0
2018-11-08 09:22:05.579584: I tensorflow/core/common_runtime/gpu/gpu_device.cc:990] 0: N
2018-11-08 09:22:05.579956: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1103] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 11372 MB memory) -> physical GPU (device: 0, name: TITAN X (Pascal), pci bus id: 0000:8a:00.0, compute capability: 6.1)
/opt/conda/lib/python3.6/site-packages/sklearn/externals/joblib/externals/cloudpickle/cloudpickle.py:47: DeprecationWarning: the imp module is deprecated in favour of importlib; see the module's documentation for alternative uses
import imp
Using TensorFlow backend.
{'0': 'A', '1': 'C', '2': 'T', '3': 'G'}
[3 1 1 2 0 0 3 1 1 2]
Command being timed: "python decompressor.py -output ../data/compressed/Caenorhabditis_elegans.WBcel235.dna.chromosome.I/biGRU.reconstructed.txt -model ../data/trained_models/Caenorhabditis_elegans.WBcel235.dna.chromosome.I/biGRU.hdf5 -model_name biGRU -input_file_prefix ../data/compressed/Caenorhabditis_elegans.WBcel235.dna.chromosome.I/biGRU.compressed -batch_size 1000"
User time (seconds): 509.23
System time (seconds): 12.90
Percent of CPU this job got: 108%
Elapsed (wall clock) time (h:mm:ss or m:ss): 8:02.22
Average shared text size (kbytes): 0
Average unshared data size (kbytes): 0
Average stack size (kbytes): 0
Average total size (kbytes): 0
Maximum resident set size (kbytes): 2216688
Average resident set size (kbytes): 0
Major (requiring I/O) page faults: 0
Minor (reclaiming a frame) page faults: 668892
Voluntary context switches: 394942
Involuntary context switches: 1120
Swaps: 0
File system inputs: 0
File system outputs: 37456
Socket messages sent: 0
Socket messages received: 0
Signals delivered: 0
Page size (bytes): 4096
Exit status: 0
================================================
FILE: data/logs_data/Caenorhabditis_elegans.WBcel235.dna.chromosome.I/biGRU_16bit.log.csv
================================================
Starting training ...
0;1.8831766127114613
1;1.8687406848715946
2;1.8622829920086283
3;1.8584328716285075
Starting Compression ...
2018-10-28 21:00:55.445035: I tensorflow/core/platform/cpu_feature_guard.cc:141] Your CPU supports instructions that this TensorFlow binary was not compiled to use: AVX2 FMA
2018-10-28 21:00:57.301870: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1411] Found device 0 with properties:
name: TITAN X (Pascal) major: 6 minor: 1 memoryClockRate(GHz): 1.531
pciBusID: 0000:85:00.0
totalMemory: 11.91GiB freeMemory: 11.75GiB
2018-10-28 21:00:57.301926: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1490] Adding visible gpu devices: 0
2018-10-28 21:00:57.652880: I tensorflow/core/common_runtime/gpu/gpu_device.cc:971] Device interconnect StreamExecutor with strength 1 edge matrix:
2018-10-28 21:00:57.652947: I tensorflow/core/common_runtime/gpu/gpu_device.cc:977] 0
2018-10-28 21:00:57.652958: I tensorflow/core/common_runtime/gpu/gpu_device.cc:990] 0: N
2018-10-28 21:00:57.653293: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1103] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 11372 MB memory) -> physical GPU (device: 0, name: TITAN X (Pascal), pci bus id: 0000:85:00.0, compute capability: 6.1)
/opt/conda/lib/python3.6/site-packages/sklearn/externals/joblib/externals/cloudpickle/cloudpickle.py:47: DeprecationWarning: the imp module is deprecated in favour of importlib; see the module's documentation for alternative uses
import imp
Using TensorFlow backend.
/opt/conda/lib/python3.6/site-packages/sklearn/preprocessing/_encoders.py:363: FutureWarning: The handling of integer data will change in version 0.22. Currently, the categories are determined based on the range [0, max(values)], while in the future they will be determined based on the unique values.
If you want the future behaviour and silence this warning, you can specify "categories='auto'".
In case you used a LabelEncoder before this OneHotEncoder to convert the categories to integers, then you can now use the OneHotEncoder directly.
warnings.warn(msg, FutureWarning)
Command being timed: "python compressor.py -data ../data/processed_files/Caenorhabditis_elegans.WBcel235.dna.chromosome.I.npy -data_params ../data/processed_files/Caenorhabditis_elegans.WBcel235.dna.chromosome.I.param.json -model ../data/trained_models/Caenorhabditis_elegans.WBcel235.dna.chromosome.I/biGRU_16bit.hdf5 -model_name biGRU_16bit -output ../data/compressed/Caenorhabditis_elegans.WBcel235.dna.chromosome.I/biGRU_16bit.compressed -batch_size 10000"
User time (seconds): 291.74
System time (seconds): 26.25
Percent of CPU this job got: 112%
Elapsed (wall clock) time (h:mm:ss or m:ss): 4:41.71
Average shared text size (kbytes): 0
Average unshared data size (kbytes): 0
Average stack size (kbytes): 0
Average total size (kbytes): 0
Maximum resident set size (kbytes): 2704224
Average resident set size (kbytes): 0
Major (requiring I/O) page faults: 0
Minor (reclaiming a frame) page faults: 3423081
Voluntary context switches: 568485
Involuntary context switches: 10812
Swaps: 0
File system inputs: 0
File system outputs: 86936
Socket messages sent: 0
Socket messages received: 0
Signals delivered: 0
Page size (bytes): 4096
Exit status: 0
2018-10-28 21:05:32.283342: I tensorflow/core/platform/cpu_feature_guard.cc:141] Your CPU supports instructions that this TensorFlow binary was not compiled to use: AVX2 FMA
2018-10-28 21:05:34.122397: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1411] Found device 0 with properties:
name: TITAN X (Pascal) major: 6 minor: 1 memoryClockRate(GHz): 1.531
pciBusID: 0000:85:00.0
totalMemory: 11.91GiB freeMemory: 11.75GiB
2018-10-28 21:05:34.122437: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1490] Adding visible gpu devices: 0
2018-10-28 21:05:34.469139: I tensorflow/core/common_runtime/gpu/gpu_device.cc:971] Device interconnect StreamExecutor with strength 1 edge matrix:
2018-10-28 21:05:34.469196: I tensorflow/core/common_runtime/gpu/gpu_device.cc:977] 0
2018-10-28 21:05:34.469205: I tensorflow/core/common_runtime/gpu/gpu_device.cc:990] 0: N
2018-10-28 21:05:34.469565: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1103] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 11372 MB memory) -> physical GPU (device: 0, name: TITAN X (Pascal), pci bus id: 0000:85:00.0, compute capability: 6.1)
/opt/conda/lib/python3.6/site-packages/sklearn/externals/joblib/externals/cloudpickle/cloudpickle.py:47: DeprecationWarning: the imp module is deprecated in favour of importlib; see the module's documentation for alternative uses
import imp
Using TensorFlow backend.
{'0': 'A', '1': 'C', '2': 'T', '3': 'G'}
[3 1 1 2 0 0 3 1 1 2]
Command being timed: "python decompressor.py -output ../data/compressed/Caenorhabditis_elegans.WBcel235.dna.chromosome.I/biGRU_16bit.reconstructed.txt -model ../data/trained_models/Caenorhabditis_elegans.WBcel235.dna.chromosome.I/biGRU_16bit.hdf5 -model_name biGRU_16bit -input_file_prefix ../data/compressed/Caenorhabditis_elegans.WBcel235.dna.chromosome.I/biGRU_16bit.compressed -batch_size 10000"
User time (seconds): 388.45
System time (seconds): 22.38
Percent of CPU this job got: 107%
Elapsed (wall clock) time (h:mm:ss or m:ss): 6:22.60
Average shared text size (kbytes): 0
Average unshared data size (kbytes): 0
Average stack size (kbytes): 0
Average total size (kbytes): 0
Maximum resident set size (kbytes): 2239008
Average resident set size (kbytes): 0
Major (requiring I/O) page faults: 0
Minor (reclaiming a frame) page faults: 1219302
Voluntary context switches: 603773
Involuntary context switches: 1280
Swaps: 0
File system inputs: 0
File system outputs: 109456
Socket messages sent: 0
Socket messages received: 0
Signals delivered: 0
Page size (bytes): 4096
Exit status: 0
================================================
FILE: data/logs_data/Caenorhabditis_elegans.WBcel235.dna.chromosome.I/biLSTM.log.csv
================================================
Starting training ...
Starting Compression ...
Traceback (most recent call last):
File "compressor.py", line 20, in <module>
import keras
ImportError: No module named keras
Command exited with non-zero status 1
Command being timed: "python compressor.py -data ../data/processed_files/Caenorhabditis_elegans.WBcel235.dna.chromosome.I.npy -data_params ../data/processed_files/Caenorhabditis_elegans.WBcel235.dna.chromosome.I.param.json -model ../data/trained_models/Caenorhabditis_elegans.WBcel235.dna.chromosome.I/biLSTM.hdf5 -model_name biLSTM -output ../data/compressed/Caenorhabditis_elegans.WBcel235.dna.chromosome.I/biLSTM.compressed -batch_size 10000"
User time (seconds): 1.27
System time (seconds): 3.18
Percent of CPU this job got: 1183%
Elapsed (wall clock) time (h:mm:ss or m:ss): 0:00.37
Average shared text size (kbytes): 0
Average unshared data size (kbytes): 0
Average stack size (kbytes): 0
Average total size (kbytes): 0
Maximum resident set size (kbytes): 57996
Average resident set size (kbytes): 0
Major (requiring I/O) page faults: 0
Minor (reclaiming a frame) page faults: 14580
Voluntary context switches: 144
Involuntary context switches: 441963
Swaps: 0
File system inputs: 0
File system outputs: 0
Socket messages sent: 0
Socket messages received: 0
Signals delivered: 0
Page size (bytes): 4096
Exit status: 1
Traceback (most recent call last):
File "decompressor.py", line 20, in <module>
import keras
ImportError: No module named keras
Command exited with non-zero status 1
Command being timed: "python decompressor.py -output ../data/compressed/Caenorhabditis_elegans.WBcel235.dna.chromosome.I/biLSTM.reconstructed.txt -model ../data/trained_models/Caenorhabditis_elegans.WBcel235.dna.chromosome.I/biLSTM.hdf5 -model_name biLSTM -input_file_prefix ../data/compressed/Caenorhabditis_elegans.WBcel235.dna.chromosome.I/biLSTM.compressed -batch_size 10000"
User time (seconds): 1.30
System time (seconds): 3.23
Percent of CPU this job got: 1196%
Elapsed (wall clock) time (h:mm:ss or m:ss): 0:00.38
Average shared text size (kbytes): 0
Average unshared data size (kbytes): 0
Average stack size (kbytes): 0
Average total size (kbytes): 0
Maximum resident set size (kbytes): 58028
Average resident set size (kbytes): 0
Major (requiring I/O) page faults: 0
Minor (reclaiming a frame) page faults: 14640
Voluntary context switches: 133
Involuntary context switches: 294126
Swaps: 0
File system inputs: 0
File system outputs: 0
Socket messages sent: 0
Socket messages received: 0
Signals delivered: 0
Page size (bytes): 4096
Exit status: 1
Starting training ...
Starting Compression ...
================================================
FILE: data/logs_data/Caenorhabditis_elegans.WBcel235.dna.chromosome.I/biLSTM_16bit.log.csv
================================================
Starting training ...
0;1.884507402209729
1;1.871122544627692
2;1.8635762954132413
3;1.8597909297718935
Starting Compression ...
2018-10-28 21:08:21.470079: I tensorflow/core/platform/cpu_feature_guard.cc:141] Your CPU supports instructions that this TensorFlow binary was not compiled to use: AVX2 FMA
2018-10-28 21:08:23.362497: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1411] Found device 0 with properties:
name: TITAN X (Pascal) major: 6 minor: 1 memoryClockRate(GHz): 1.531
pciBusID: 0000:09:00.0
totalMemory: 11.91GiB freeMemory: 11.75GiB
2018-10-28 21:08:23.362559: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1490] Adding visible gpu devices: 0
2018-10-28 21:08:23.713441: I tensorflow/core/common_runtime/gpu/gpu_device.cc:971] Device interconnect StreamExecutor with strength 1 edge matrix:
2018-10-28 21:08:23.713523: I tensorflow/core/common_runtime/gpu/gpu_device.cc:977] 0
2018-10-28 21:08:23.713534: I tensorflow/core/common_runtime/gpu/gpu_device.cc:990] 0: N
2018-10-28 21:08:23.713912: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1103] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 11372 MB memory) -> physical GPU (device: 0, name: TITAN X (Pascal), pci bus id: 0000:09:00.0, compute capability: 6.1)
/opt/conda/lib/python3.6/site-packages/sklearn/externals/joblib/externals/cloudpickle/cloudpickle.py:47: DeprecationWarning: the imp module is deprecated in favour of importlib; see the module's documentation for alternative uses
import imp
Using TensorFlow backend.
/opt/conda/lib/python3.6/site-packages/sklearn/preprocessing/_encoders.py:363: FutureWarning: The handling of integer data will change in version 0.22. Currently, the categories are determined based on the range [0, max(values)], while in the future they will be determined based on the unique values.
If you want the future behaviour and silence this warning, you can specify "categories='auto'".
In case you used a LabelEncoder before this OneHotEncoder to convert the categories to integers, then you can now use the OneHotEncoder directly.
warnings.warn(msg, FutureWarning)
Command being timed: "python compressor.py -data ../data/processed_files/Caenorhabditis_elegans.WBcel235.dna.chromosome.I.npy -data_params ../data/processed_files/Caenorhabditis_elegans.WBcel235.dna.chromosome.I.param.json -model ../data/trained_models/Caenorhabditis_elegans.WBcel235.dna.chromosome.I/biLSTM_16bit.hdf5 -model_name biLSTM_16bit -output ../data/compressed/Caenorhabditis_elegans.WBcel235.dna.chromosome.I/biLSTM_16bit.compressed -batch_size 10000"
User time (seconds): 300.60
System time (seconds): 26.81
Percent of CPU this job got: 114%
Elapsed (wall clock) time (h:mm:ss or m:ss): 4:45.39
Average shared text size (kbytes): 0
Average unshared data size (kbytes): 0
Average stack size (kbytes): 0
Average total size (kbytes): 0
Maximum resident set size (kbytes): 2711476
Average resident set size (kbytes): 0
Major (requiring I/O) page faults: 0
Minor (reclaiming a frame) page faults: 2630707
Voluntary context switches: 690682
Involuntary context switches: 7348
Swaps: 0
File system inputs: 0
File system outputs: 86936
Socket messages sent: 0
Socket messages received: 0
Signals delivered: 0
Page size (bytes): 4096
Exit status: 0
2018-10-28 21:13:01.836333: I tensorflow/core/platform/cpu_feature_guard.cc:141] Your CPU supports instructions that this TensorFlow binary was not compiled to use: AVX2 FMA
2018-10-28 21:13:03.676070: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1411] Found device 0 with properties:
name: TITAN X (Pascal) major: 6 minor: 1 memoryClockRate(GHz): 1.531
pciBusID: 0000:09:00.0
totalMemory: 11.91GiB freeMemory: 11.75GiB
2018-10-28 21:13:03.676125: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1490] Adding visible gpu devices: 0
2018-10-28 21:13:04.023200: I tensorflow/core/common_runtime/gpu/gpu_device.cc:971] Device interconnect StreamExecutor with strength 1 edge matrix:
2018-10-28 21:13:04.023267: I tensorflow/core/common_runtime/gpu/gpu_device.cc:977] 0
2018-10-28 21:13:04.023276: I tensorflow/core/common_runtime/gpu/gpu_device.cc:990] 0: N
2018-10-28 21:13:04.023654: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1103] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 11372 MB memory) -> physical GPU (device: 0, name: TITAN X (Pascal), pci bus id: 0000:09:00.0, compute capability: 6.1)
/opt/conda/lib/python3.6/site-packages/sklearn/externals/joblib/externals/cloudpickle/cloudpickle.py:47: DeprecationWarning: the imp module is deprecated in favour of importlib; see the module's documentation for alternative uses
import imp
Using TensorFlow backend.
{'0': 'A', '1': 'C', '2': 'T', '3': 'G'}
[3 1 1 2 0 0 3 1 1 2]
Command being timed: "python decompressor.py -output ../data/compressed/Caenorhabditis_elegans.WBcel235.dna.chromosome.I/biLSTM_16bit.reconstructed.txt -model ../data/trained_models/Caenorhabditis_elegans.WBcel235.dna.chromosome.I/biLSTM_16bit.hdf5 -model_name biLSTM_16bit -input_file_prefix ../data/compressed/Caenorhabditis_elegans.WBcel235.dna.chromosome.I/biLSTM_16bit.compressed -batch_size 10000"
User time (seconds): 385.51
System time (seconds): 24.12
Percent of CPU this job got: 108%
Elapsed (wall clock) time (h:mm:ss or m:ss): 6:17.41
Average shared text size (kbytes): 0
Average unshared data size (kbytes): 0
Average stack size (kbytes): 0
Average total size (kbytes): 0
Maximum resident set size (kbytes): 2235176
Average resident set size (kbytes): 0
Major (requiring I/O) page faults: 0
Minor (reclaiming a frame) page faults: 1296187
Voluntary context switches: 695272
Involuntary context switches: 129331
Swaps: 0
File system inputs: 0
File system outputs: 109456
Socket messages sent: 0
Socket messages received: 0
Signals delivered: 0
Page size (bytes): 4096
Exit status: 0
================================================
FILE: data/logs_data/Caenorhabditis_elegans.WBcel235.dna.toplevel/FC.log.csv
================================================
Starting training ...
Starting training ...
Starting Compression ...
/opt/conda/lib/python3.6/site-packages/sklearn/externals/joblib/externals/cloudpickle/cloudpickle.py:47: DeprecationWarning: the imp module is deprecated in favour of importlib; see the module's documentation for alternative uses
import imp
Using TensorFlow backend.
/opt/conda/lib/python3.6/site-packages/sklearn/preprocessing/_encoders.py:363: FutureWarning: The handling of integer data will change in version 0.22. Currently, the categories are determined based on the range [0, max(values)], while in the future they will be determined based on the unique values.
If you want the future behaviour and silence this warning, you can specify "categories='auto'".
In case you used a LabelEncoder before this OneHotEncoder to convert the categories to integers, then you can now use the OneHotEncoder directly.
warnings.warn(msg, FutureWarning)
Traceback (most recent call last):
File "compressor.py", line 202, in <module>
main()
File "compressor.py", line 164, in main
predict_lstm(X, Y, Y_original, timesteps, batch_size, alphabet_size, args.model_name)
File "compressor.py", line 68, in predict_lstm
model = getattr(models, model_name)(bs, timesteps, alphabet_size)
File "/data/final_models/models.py", line 165, in FC
model = sequential()
NameError: name 'sequential' is not defined
Command exited with non-zero status 1
Command being timed: "python compressor.py -data ../data/processed_files/Caenorhabditis_elegans.WBcel235.dna.toplevel.npy -data_params ../data/processed_files/Caenorhabditis_elegans.WBcel235.dna.toplevel.param.json -model ../data/trained_models/Caenorhabditis_elegans.WBcel235.dna.toplevel/FC.hdf5 -model_name FC -output ../data/compressed/Caenorhabditis_elegans.WBcel235.dna.toplevel/FC.compressed -batch_size 10000"
User time (seconds): 33.62
System time (seconds): 15.93
Percent of CPU this job got: 141%
Elapsed (wall clock) time (h:mm:ss or m:ss): 0:35.12
Average shared text size (kbytes): 0
Average unshared data size (kbytes): 0
Average stack size (kbytes): 0
Average total size (kbytes): 0
Maximum resident set size (kbytes): 9361492
Average resident set size (kbytes): 0
Major (requiring I/O) page faults: 0
Minor (reclaiming a frame) page faults: 8767632
Voluntary context switches: 304
Involuntary context switches: 190
Swaps: 0
File system inputs: 0
File system outputs: 16
Socket messages sent: 0
Socket messages received: 0
Signals delivered: 0
Page size (bytes): 4096
Exit status: 1
/opt/conda/lib/python3.6/site-packages/sklearn/externals/joblib/externals/cloudpickle/cloudpickle.py:47: DeprecationWarning: the imp module is deprecated in favour of importlib; see the module's documentation for alternative uses
import imp
Using TensorFlow backend.
Traceback (most recent call last):
File "decompressor.py", line 184, in <module>
main()
File "decompressor.py", line 153, in main
f = open(args.input_file_prefix+'.combined','rb')
FileNotFoundError: [Errno 2] No such file or directory: '../data/compressed/Caenorhabditis_elegans.WBcel235.dna.toplevel/FC.compressed.combined'
Command exited with non-zero status 1
Command being timed: "python decompressor.py -output ../data/compressed/Caenorhabditis_elegans.WBcel235.dna.toplevel/FC.reconstructed.txt -model ../data/trained_models/Caenorhabditis_elegans.WBcel235.dna.toplevel/FC.hdf5 -model_name FC -input_file_prefix ../data/compressed/Caenorhabditis_elegans.WBcel235.dna.toplevel/FC.compressed -batch_size 10000"
User time (seconds): 2.18
System time (seconds): 0.24
Percent of CPU this job got: 100%
Elapsed (wall clock) time (h:mm:ss or m:ss): 0:02.42
Average shared text size (kbytes): 0
Average unshared data size (kbytes): 0
Average stack size (kbytes): 0
Average total size (kbytes): 0
Maximum resident set size (kbytes): 254576
Average resident set size (kbytes): 0
Major (requiring I/O) page faults: 0
Minor (reclaiming a frame) page faults: 68243
Voluntary context switches: 33
Involuntary context switches: 8
Swaps: 0
File system inputs: 0
File system outputs: 8
Socket messages sent: 0
Socket messages received: 0
Signals delivered: 0
Page size (bytes): 4096
Exit status: 1
Starting training ...
0;1.8401906413567488
1;1.8262858981663073
2;1.8218421463227026
3;1.8192768009977185
4;1.8176022867370834
5;1.816419973604443
6;1.8155042164941753
Starting Compression ...
2018-11-01 03:02:48.346421: I tensorflow/core/platform/cpu_feature_guard.cc:141] Your CPU supports instructions that this TensorFlow binary was not compiled to use: AVX2 FMA
2018-11-01 03:02:50.970328: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1411] Found device 0 with properties:
name: TITAN X (Pascal) major: 6 minor: 1 memoryClockRate(GHz): 1.531
pciBusID: 0000:08:00.0
totalMemory: 11.91GiB freeMemory: 11.75GiB
2018-11-01 03:02:50.970399: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1490] Adding visible gpu devices: 0
2018-11-01 03:02:51.315578: I tensorflow/core/common_runtime/gpu/gpu_device.cc:971] Device interconnect StreamExecutor with strength 1 edge matrix:
2018-11-01 03:02:51.315648: I tensorflow/core/common_runtime/gpu/gpu_device.cc:977] 0
2018-11-01 03:02:51.315658: I tensorflow/core/common_runtime/gpu/gpu_device.cc:990] 0: N
2018-11-01 03:02:51.316109: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1103] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 11372 MB memory) -> physical GPU (device: 0, name: TITAN X (Pascal), pci bus id: 0000:08:00.0, compute capability: 6.1)
/opt/conda/lib/python3.6/site-packages/sklearn/externals/joblib/externals/cloudpickle/cloudpickle.py:47: DeprecationWarning: the imp module is deprecated in favour of importlib; see the module's documentation for alternative uses
import imp
Using TensorFlow backend.
/opt/conda/lib/python3.6/site-packages/sklearn/preprocessing/_encoders.py:363: FutureWarning: The handling of integer data will change in version 0.22. Currently, the categories are determined based on the range [0, max(values)], while in the future they will be determined based on the unique values.
If you want the future behaviour and silence this warning, you can specify "categories='auto'".
In case you used a LabelEncoder before this OneHotEncoder to convert the categories to integers, then you can now use the OneHotEncoder directly.
warnings.warn(msg, FutureWarning)
Command being timed: "python compressor.py -data ../data/processed_files/Caenorhabditis_elegans.WBcel235.dna.toplevel.npy -data_params ../data/processed_files/Caenorhabditis_elegans.WBcel235.dna.toplevel.param.json -model ../data/trained_models/Caenorhabditis_elegans.WBcel235.dna.toplevel/FC.hdf5 -model_name FC -output ../data/compressed/Caenorhabditis_elegans.WBcel235.dna.toplevel/FC.compressed -batch_size 10000"
User time (seconds): 1287.46
System time (seconds): 55.12
Percent of CPU this job got: 102%
Elapsed (wall clock) time (h:mm:ss or m:ss): 21:46.60
Average shared text size (kbytes): 0
Average unshared data size (kbytes): 0
Average stack size (kbytes): 0
Average total size (kbytes): 0
Maximum resident set size (kbytes): 9361716
Average resident set size (kbytes): 0
Major (requiring I/O) page faults: 0
Minor (reclaiming a frame) page faults: 12019672
Voluntary context switches: 1091934
Involuntary context switches: 3152
Swaps: 0
File system inputs: 0
File system outputs: 124496
Socket messages sent: 0
Socket messages received: 0
Signals delivered: 0
Page size (bytes): 4096
Exit status: 0
2018-11-01 03:24:01.677806: I tensorflow/core/platform/cpu_feature_guard.cc:141] Your CPU supports instructions that this TensorFlow binary was not compiled to use: AVX2 FMA
2018-11-01 03:24:04.444020: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1411] Found device 0 with properties:
name: TITAN X (Pascal) major: 6 minor: 1 memoryClockRate(GHz): 1.531
pciBusID: 0000:08:00.0
totalMemory: 11.91GiB freeMemory: 11.75GiB
2018-11-01 03:24:04.444075: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1490] Adding visible gpu devices: 0
2018-11-01 03:24:04.848470: I tensorflow/core/common_runtime/gpu/gpu_device.cc:971] Device interconnect StreamExecutor with strength 1 edge matrix:
2018-11-01 03:24:04.848545: I tensorflow/core/common_runtime/gpu/gpu_device.cc:977] 0
2018-11-01 03:24:04.848555: I tensorflow/core/common_runtime/gpu/gpu_device.cc:990] 0: N
2018-11-01 03:24:04.848913: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1103] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 11372 MB memory) -> physical GPU (device: 0, name: TITAN X (Pascal), pci bus id: 0000:08:00.0, compute capability: 6.1)
/opt/conda/lib/python3.6/site-packages/sklearn/externals/joblib/externals/cloudpickle/cloudpickle.py:47: DeprecationWarning: the imp module is deprecated in favour of importlib; see the module's documentation for alternative uses
import imp
Using TensorFlow backend.
{'0': 'A', '1': 'C', '2': 'T', '3': 'G'}
[3 1 1 2 0 0 3 1 1 2]
Command being timed: "python decompressor.py -output ../data/compressed/Caenorhabditis_elegans.WBcel235.dna.toplevel/FC.reconstructed.txt -model ../data/trained_models/Caenorhabditis_elegans.WBcel235.dna.toplevel/FC.hdf5 -model_name FC -input_file_prefix ../data/compressed/Caenorhabditis_elegans.WBcel235.dna.toplevel/FC.compressed -batch_size 10000"
User time (seconds): 1912.89
System time (seconds): 37.55
Percent of CPU this job got: 101%
Elapsed (wall clock) time (h:mm:ss or m:ss): 32:05.59
Average shared text size (kbytes): 0
Average unshared data size (kbytes): 0
Average stack size (kbytes): 0
Average total size (kbytes): 0
Maximum resident set size (kbytes): 2375184
Average resident set size (kbytes): 0
Major (requiring I/O) page faults: 0
Minor (reclaiming a frame) page faults: 1978587
Voluntary context switches: 1102076
Involuntary context switches: 4002
Swaps: 0
File system inputs: 0
File system outputs: 275888
Socket messages sent: 0
Socket messages received: 0
Signals delivered: 0
Page size (bytes): 4096
Exit status: 0
Starting Compression ...
================================================
FILE: data/logs_data/Caenorhabditis_elegans.WBcel235.dna.toplevel/FC_16bit.log.csv
================================================
Starting training ...
0;1.8531200441459896
1;1.8334496114291368
Starting training ...
Starting training ...
0;1.853119725056798
1;1.8336304452570582
2;1.5145707193387195
3;0.0
4;0.0
Starting Compression ...
2018-10-29 18:44:03.595928: I tensorflow/core/platform/cpu_feature_guard.cc:141] Your CPU supports instructions that this TensorFlow binary was not compiled to use: AVX2 FMA
2018-10-29 18:44:06.550075: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1411] Found device 0 with properties:
name: TITAN X (Pascal) major: 6 minor: 1 memoryClockRate(GHz): 1.531
pciBusID: 0000:8a:00.0
totalMemory: 11.91GiB freeMemory: 11.75GiB
2018-10-29 18:44:06.550147: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1490] Adding visible gpu devices: 0
2018-10-29 18:44:06.912494: I tensorflow/core/common_runtime/gpu/gpu_device.cc:971] Device interconnect StreamExecutor with strength 1 edge matrix:
2018-10-29 18:44:06.912568: I tensorflow/core/common_runtime/gpu/gpu_device.cc:977] 0
2018-10-29 18:44:06.912579: I tensorflow/core/common_runtime/gpu/gpu_device.cc:990] 0: N
2018-10-29 18:44:06.912965: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1103] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 11372 MB memory) -> physical GPU (device: 0, name: TITAN X (Pascal), pci bus id: 0000:8a:00.0, compute capability: 6.1)
/opt/conda/lib/python3.6/site-packages/sklearn/externals/joblib/externals/cloudpickle/cloudpickle.py:47: DeprecationWarning: the imp module is deprecated in favour of importlib; see the module's documentation for alternative uses
import imp
Using TensorFlow backend.
/opt/conda/lib/python3.6/site-packages/sklearn/preprocessing/_encoders.py:363: FutureWarning: The handling of integer data will change in version 0.22. Currently, the categories are determined based on the range [0, max(values)], while in the future they will be determined based on the unique values.
If you want the future behaviour and silence this warning, you can specify "categories='auto'".
In case you used a LabelEncoder before this OneHotEncoder to convert the categories to integers, then you can now use the OneHotEncoder directly.
warnings.warn(msg, FutureWarning)
Command being timed: "python compressor.py -data ../data/processed_files/Caenorhabditis_elegans.WBcel235.dna.toplevel.npy -data_params ../data/processed_files/Caenorhabditis_elegans.WBcel235.dna.toplevel.param.json -model ../data/trained_models/Caenorhabditis_elegans.WBcel235.dna.toplevel/FC_16bit.hdf5 -model_name FC_16bit -output ../data/compressed/Caenorhabditis_elegans.WBcel235.dna.toplevel/FC_16bit.compressed -batch_size 10000"
User time (seconds): 850.83
System time (seconds): 50.68
Percent of CPU this job got: 103%
Elapsed (wall clock) time (h:mm:ss or m:ss): 14:27.58
Average shared text size (kbytes): 0
Average unshared data size (kbytes): 0
Average stack size (kbytes): 0
Average total size (kbytes): 0
Maximum resident set size (kbytes): 9360984
Average resident set size (kbytes): 0
Major (requiring I/O) page faults: 0
Minor (reclaiming a frame) page faults: 10238848
Voluntary context switches: 1073961
Involuntary context switches: 2535
Swaps: 0
File system inputs: 0
File system outputs: 80376
Socket messages sent: 0
Socket messages received: 0
Signals delivered: 0
Page size (bytes): 4096
Exit status: 0
2018-10-29 18:57:58.817568: I tensorflow/core/platform/cpu_feature_guard.cc:141] Your CPU supports instructions that this TensorFlow binary was not compiled to use: AVX2 FMA
2018-10-29 18:58:01.171057: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1411] Found device 0 with properties:
name: TITAN X (Pascal) major: 6 minor: 1 memoryClockRate(GHz): 1.531
pciBusID: 0000:8a:00.0
totalMemory: 11.91GiB freeMemory: 11.75GiB
2018-10-29 18:58:01.171102: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1490] Adding visible gpu devices: 0
2018-10-29 18:58:01.522810: I tensorflow/core/common_runtime/gpu/gpu_device.cc:971] Device interconnect StreamExecutor with strength 1 edge matrix:
2018-10-29 18:58:01.522863: I tensorflow/core/common_runtime/gpu/gpu_device.cc:977] 0
2018-10-29 18:58:01.522873: I tensorflow/core/common_runtime/gpu/gpu_device.cc:990] 0: N
2018-10-29 18:58:01.523239: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1103] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 11372 MB memory) -> physical GPU (device: 0, name: TITAN X (Pascal), pci bus id: 0000:8a:00.0, compute capability: 6.1)
/opt/conda/lib/python3.6/site-packages/sklearn/externals/joblib/externals/cloudpickle/cloudpickle.py:47: DeprecationWarning: the imp module is deprecated in favour of importlib; see the module's documentation for alternative uses
import imp
Using TensorFlow backend.
{'0': 'A', '1': 'C', '2': 'T', '3': 'G'}
[3 1 1 2 0 0 3 1 1 2]
Command being timed: "python decompressor.py -output ../data/compressed/Caenorhabditis_elegans.WBcel235.dna.toplevel/FC_16bit.reconstructed.txt -model ../data/trained_models/Caenorhabditis_elegans.WBcel235.dna.toplevel/FC_16bit.hdf5 -model_name FC_16bit -input_file_prefix ../data/compressed/Caenorhabditis_elegans.WBcel235.dna.toplevel/FC_16bit.compressed -batch_size 10000"
User time (seconds): 1516.64
System time (seconds): 35.36
Percent of CPU this job got: 101%
Elapsed (wall clock) time (h:mm:ss or m:ss): 25:28.38
Average shared text size (kbytes): 0
Average unshared data size (kbytes): 0
Average stack size (kbytes): 0
Average total size (kbytes): 0
Maximum resident set size (kbytes): 2369756
Average resident set size (kbytes): 0
Major (requiring I/O) page faults: 0
Minor (reclaiming a frame) page faults: 1998048
Voluntary context switches: 1055940
Involuntary context switches: 3257
Swaps: 0
File system inputs: 0
File system outputs: 275888
Socket messages sent: 0
Socket messages received: 0
Signals delivered: 0
Page size (bytes): 4096
Exit status: 0
../data/compressed/Caenorhabditis_elegans.WBcel235.dna.toplevel/FC_16bit.reconstructed.txt ../data/files_to_be_compressed/Caenorhabditis_elegans.WBcel235.dna.toplevel.txt differ: char 67, line 1
Starting training ...
Starting training ...
0;1.9390477221818043
1;1.9382117284423341
2;1.9381261227950937
================================================
FILE: data/logs_data/Caenorhabditis_elegans.WBcel235.dna.toplevel/FC_4layer.log.csv
================================================
Starting training ...
Starting Compression ...
Starting training ...
0;1.858583542632005
1;1.845372233931501
2;1.8422122455990722
3;1.8405038393066593
4;1.839310921253055
5;1.8384736193068123
6;1.837773691340136
7;1.8371983868182207
Starting Compression ...
2018-11-01 02:20:39.308483: I tensorflow/core/platform/cpu_feature_guard.cc:141] Your CPU supports instructions that this TensorFlow binary was not compiled to use: AVX2 FMA
2018-11-01 02:20:42.055339: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1411] Found device 0 with properties:
name: TITAN X (Pascal) major: 6 minor: 1 memoryClockRate(GHz): 1.531
pciBusID: 0000:8a:00.0
totalMemory: 11.91GiB freeMemory: 11.75GiB
2018-11-01 02:20:42.055380: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1490] Adding visible gpu devices: 0
2018-11-01 02:20:42.410575: I tensorflow/core/common_runtime/gpu/gpu_device.cc:971] Device interconnect StreamExecutor with strength 1 edge matrix:
2018-11-01 02:20:42.410673: I tensorflow/core/common_runtime/gpu/gpu_device.cc:977] 0
2018-11-01 02:20:42.410683: I tensorflow/core/common_runtime/gpu/gpu_device.cc:990] 0: N
2018-11-01 02:20:42.411045: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1103] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 11372 MB memory) -> physical GPU (device: 0, name: TITAN X (Pascal), pci bus id: 0000:8a:00.0, compute capability: 6.1)
/opt/conda/lib/python3.6/site-packages/sklearn/externals/joblib/externals/cloudpickle/cloudpickle.py:47: DeprecationWarning: the imp module is deprecated in favour of importlib; see the module's documentation for alternative uses
import imp
Using TensorFlow backend.
/opt/conda/lib/python3.6/site-packages/sklearn/preprocessing/_encoders.py:363: FutureWarning: The handling of integer data will change in version 0.22. Currently, the categories are determined based on the range [0, max(values)], while in the future they will be determined based on the unique values.
If you want the future behaviour and silence this warning, you can specify "categories='auto'".
In case you used a LabelEncoder before this OneHotEncoder to convert the categories to integers, then you can now use the OneHotEncoder directly.
warnings.warn(msg, FutureWarning)
/src/keras/activations.py:209: UserWarning: Do not pass a layer instance (such as ELU) as the activation argument of another layer. Instead, advanced activation layers should be used just like any other layer in a model.
identifier=identifier.__class__.__name__))
Command being timed: "python compressor.py -data ../data/processed_files/Caenorhabditis_elegans.WBcel235.dna.toplevel.npy -data_params ../data/processed_files/Caenorhabditis_elegans.WBcel235.dna.toplevel.param.json -model ../data/trained_models/Caenorhabditis_elegans.WBcel235.dna.toplevel/FC_4layer.hdf5 -model_name FC_4layer -output ../data/compressed/Caenorhabditis_elegans.WBcel235.dna.toplevel/FC_4layer.compressed -batch_size 10000"
User time (seconds): 1353.94
System time (seconds): 30.03
Percent of CPU this job got: 102%
Elapsed (wall clock) time (h:mm:ss or m:ss): 22:33.87
Average shared text size (kbytes): 0
Average unshared data size (kbytes): 0
Average stack size (kbytes): 0
Average total size (kbytes): 0
Maximum resident set size (kbytes): 9361092
Average resident set size (kbytes): 0
Major (requiring I/O) page faults: 0
Minor (reclaiming a frame) page faults: 10824540
Voluntary context switches: 339987
Involuntary context switches: 2701
Swaps: 0
File system inputs: 0
File system outputs: 125048
Socket messages sent: 0
Socket messages received: 0
Signals delivered: 0
Page size (bytes): 4096
Exit status: 0
2018-11-01 02:42:39.241305: I tensorflow/core/platform/cpu_feature_guard.cc:141] Your CPU supports instructions that this TensorFlow binary was not compiled to use: AVX2 FMA
2018-11-01 02:42:41.947432: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1411] Found device 0 with properties:
name: TITAN X (Pascal) major: 6 minor: 1 memoryClockRate(GHz): 1.531
pciBusID: 0000:8a:00.0
totalMemory: 11.91GiB freeMemory: 11.75GiB
2018-11-01 02:42:41.947492: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1490] Adding visible gpu devices: 0
2018-11-01 02:42:42.299242: I tensorflow/core/common_runtime/gpu/gpu_device.cc:971] Device interconnect StreamExecutor with strength 1 edge matrix:
2018-11-01 02:42:42.299315: I tensorflow/core/common_runtime/gpu/gpu_device.cc:977] 0
2018-11-01 02:42:42.299324: I tensorflow/core/common_runtime/gpu/gpu_device.cc:990] 0: N
2018-11-01 02:42:42.299689: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1103] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 11372 MB memory) -> physical GPU (device: 0, name: TITAN X (Pascal), pci bus id: 0000:8a:00.0, compute capability: 6.1)
/opt/conda/lib/python3.6/site-packages/sklearn/externals/joblib/externals/cloudpickle/cloudpickle.py:47: DeprecationWarning: the imp module is deprecated in favour of importlib; see the module's documentation for alternative uses
import imp
Using TensorFlow backend.
/src/keras/activations.py:209: UserWarning: Do not pass a layer instance (such as ELU) as the activation argument of another layer. Instead, advanced activation layers should be used just like any other layer in a model.
identifier=identifier.__class__.__name__))
{'0': 'A', '1': 'C', '2': 'T', '3': 'G'}
[3 1 1 2 0 0 3 1 1 2]
Command being timed: "python decompressor.py -output ../data/compressed/Caenorhabditis_elegans.WBcel235.dna.toplevel/FC_4layer.reconstructed.txt -model ../data/trained_models/Caenorhabditis_elegans.WBcel235.dna.toplevel/FC_4layer.hdf5 -model_name FC_4layer -input_file_prefix ../data/compressed/Caenorhabditis_elegans.WBcel235.dna.toplevel/FC_4layer.compressed -batch_size 10000"
User time (seconds): 1878.52
System time (seconds): 14.86
Percent of CPU this job got: 101%
Elapsed (wall clock) time (h:mm:ss or m:ss): 31:13.99
Average shared text size (kbytes): 0
Average unshared data size (kbytes): 0
Average stack size (kbytes): 0
Average total size (kbytes): 0
Maximum resident set size (kbytes): 2368236
Average resident set size (kbytes): 0
Major (requiring I/O) page faults: 0
Minor (reclaiming a frame) page faults: 2763020
Voluntary context switches: 345605
Involuntary context switches: 3623
Swaps: 0
File system inputs: 0
File system outputs: 275888
Socket messages sent: 0
Socket messages received: 0
Signals delivered: 0
Page size (bytes): 4096
Exit status: 0
Starting Compression ...
2018-11-09 09:16:03.556404: I tensorflow/core/platform/cpu_feature_guard.cc:141] Your CPU supports instructions that this TensorFlow binary was not compiled to use: AVX2 FMA
2018-11-09 09:16:06.369871: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1411] Found device 0 with properties:
name: TITAN X (Pascal) major: 6 minor: 1 memoryClockRate(GHz): 1.531
pciBusID: 0000:8a:00.0
totalMemory: 11.91GiB freeMemory: 11.75GiB
2018-11-09 09:16:06.369921: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1490] Adding visible gpu devices: 0
2018-11-09 09:16:06.720020: I tensorflow/core/common_runtime/gpu/gpu_device.cc:971] Device interconnect StreamExecutor with strength 1 edge matrix:
2018-11-09 09:16:06.720087: I tensorflow/core/common_runtime/gpu/gpu_device.cc:977] 0
2018-11-09 09:16:06.720096: I tensorflow/core/common_runtime/gpu/gpu_device.cc:990] 0: N
2018-11-09 09:16:06.720464: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1103] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 11372 MB memory) -> physical GPU (device: 0, name: TITAN X (Pascal), pci bus id: 0000:8a:00.0, compute capability: 6.1)
/opt/conda/lib/python3.6/site-packages/sklearn/externals/joblib/externals/cloudpickle/cloudpickle.py:47: DeprecationWarning: the imp module is deprecated in favour of importlib; see the module's documentation for alternative uses
import imp
Using TensorFlow backend.
/opt/conda/lib/python3.6/site-packages/sklearn/preprocessing/_encoders.py:363: FutureWarning: The handling of integer data will change in version 0.22. Currently, the categories are determined based on the range [0, max(values)], while in the future they will be determined based on the unique values.
If you want the future behaviour and silence this warning, you can specify "categories='auto'".
In case you used a LabelEncoder before this OneHotEncoder to convert the categories to integers, then you can now use the OneHotEncoder directly.
warnings.warn(msg, FutureWarning)
/src/keras/activations.py:209: UserWarning: Do not pass a layer instance (such as ELU) as the activation argument of another layer. Instead, advanced activation layers should be used just like any other layer in a model.
identifier=identifier.__class__.__name__))
Command being timed: "python compressor.py -data ../data/processed_files/Caenorhabditis_elegans.WBcel235.dna.toplevel.npy -data_params ../data/processed_files/Caenorhabditis_elegans.WBcel235.dna.toplevel.param.json -model ../data/trained_models/Caenorhabditis_elegans.WBcel235.dna.toplevel/FC_4layer.hdf5 -model_name FC_4layer -output ../data/compressed/Caenorhabditis_elegans.WBcel235.dna.toplevel/FC_4layer.compressed -batch_size 1000"
User time (seconds): 1226.20
System time (seconds): 44.71
Percent of CPU this job got: 110%
Elapsed (wall clock) time (h:mm:ss or m:ss): 19:14.41
Average shared text size (kbytes): 0
Average unshared data size (kbytes): 0
Average stack size (kbytes): 0
Average total size (kbytes): 0
Maximum resident set size (kbytes): 9361812
Average resident set size (kbytes): 0
Major (requiring I/O) page faults: 0
Minor (reclaiming a frame) page faults: 9480886
Voluntary context switches: 1526155
Involuntary context switches: 1934
Swaps: 0
File system inputs: 0
File system outputs: 92952
Socket messages sent: 0
Socket messages received: 0
Signals delivered: 0
Page size (bytes): 4096
Exit status: 0
2018-11-09 09:34:47.231450: I tensorflow/core/platform/cpu_feature_guard.cc:141] Your CPU supports instructions that this TensorFlow binary was not compiled to use: AVX2 FMA
2018-11-09 09:34:50.107977: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1411] Found device 0 with properties:
name: TITAN X (Pascal) major: 6 minor: 1 memoryClockRate(GHz): 1.531
pciBusID: 0000:8a:00.0
totalMemory: 11.91GiB freeMemory: 11.75GiB
2018-11-09 09:34:50.108027: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1490] Adding visible gpu devices: 0
2018-11-09 09:34:50.473624: I tensorflow/core/common_runtime/gpu/gpu_device.cc:971] Device interconnect StreamExecutor with strength 1 edge matrix:
2018-11-09 09:34:50.473698: I tensorflow/core/common_runtime/gpu/gpu_device.cc:977] 0
2018-11-09 09:34:50.473707: I tensorflow/core/common_runtime/gpu/gpu_device.cc:990] 0: N
2018-11-09 09:34:50.474105: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1103] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 11372 MB memory) -> physical GPU (device: 0, name: TITAN X (Pascal), pci bus id: 0000:8a:00.0, compute capability: 6.1)
/opt/conda/lib/python3.6/site-packages/sklearn/externals/joblib/externals/cloudpickle/cloudpickle.py:47: DeprecationWarning: the imp module is deprecated in favour of importlib; see the module's documentation for alternative uses
import imp
Using TensorFlow backend.
/src/keras/activations.py:209: UserWarning: Do not pass a layer instance (such as ELU) as the activation argument of another layer. Instead, advanced activation layers should be used just like any other layer in a model.
identifier=identifier.__class__.__name__))
{'0': 'A', '1': 'C', '2': 'T', '3': 'G'}
[3 1 1 2 0 0 3 1 1 2]
Command being timed: "python decompressor.py -output ../data/compressed/Caenorhabditis_elegans.WBcel235.dna.toplevel/FC_4layer.reconstructed.txt -model ../data/trained_models/Caenorhabditis_elegans.WBcel235.dna.toplevel/FC_4layer.hdf5 -model_name FC_4layer -input_file_prefix ../data/compressed/Caenorhabditis_elegans.WBcel235.dna.toplevel/FC_4layer.compressed -batch_size 1000"
User time (seconds): 1961.35
System time (seconds): 29.00
Percent of CPU this job got: 105%
Elapsed (wall clock) time (h:mm:ss or m:ss): 31:30.17
Average shared text size (kbytes): 0
Average unshared data size (kbytes): 0
Average stack size (kbytes): 0
Average total size (kbytes): 0
Maximum resident set size (kbytes): 2345012
Average resident set size (kbytes): 0
Major (requiring I/O) page faults: 0
Minor (reclaiming a frame) page faults: 972903
Voluntary context switches: 1544710
Involuntary context switches: 4935
Swaps: 0
File system inputs: 0
File system outputs: 243856
Socket messages sent: 0
Socket messages received: 0
Signals delivered: 0
Page size (bytes): 4096
Exit status: 0
================================================
FILE: data/logs_data/Caenorhabditis_elegans.WBcel235.dna.toplevel/FC_4layer_16bit.log.csv
================================================
Starting training ...
0;1.8707464513293255
1;1.85496668309976
2;1.8488777234262521
3;1.8450854979546383
Starting Compression ...
2018-10-29 03:05:23.337769: I tensorflow/core/platform/cpu_feature_guard.cc:141] Your CPU supports instructions that this TensorFlow binary was not compiled to use: AVX2 FMA
2018-10-29 03:05:25.152422: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1411] Found device 0 with properties:
name: TITAN X (Pascal) major: 6 minor: 1 memoryClockRate(GHz): 1.531
pciBusID: 0000:08:00.0
totalMemory: 11.91GiB freeMemory: 11.75GiB
2018-10-29 03:05:25.152463: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1490] Adding visible gpu devices: 0
2018-10-29 03:05:25.482843: I tensorflow/core/common_runtime/gpu/gpu_device.cc:971] Device interconnect StreamExecutor with strength 1 edge matrix:
2018-10-29 03:05:25.482917: I tensorflow/core/common_runtime/gpu/gpu_device.cc:977] 0
2018-10-29 03:05:25.482927: I tensorflow/core/common_runtime/gpu/gpu_device.cc:990] 0: N
2018-10-29 03:05:25.483349: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1103] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 11372 MB memory) -> physical GPU (device: 0, name: TITAN X (Pascal), pci bus id: 0000:08:00.0, compute capability: 6.1)
/opt/conda/lib/python3.6/site-packages/sklearn/externals/joblib/externals/cloudpickle/cloudpickle.py:47: DeprecationWarning: the imp module is deprecated in favour of importlib; see the module's documentation for alternative uses
import imp
Using TensorFlow backend.
/opt/conda/lib/python3.6/site-packages/sklearn/preprocessing/_encoders.py:363: FutureWarning: The handling of integer data will change in version 0.22. Currently, the categories are determined based on the range [0, max(values)], while in the future they will be determined based on the unique values.
If you want the future behaviour and silence this warning, you can specify "categories='auto'".
In case you used a LabelEncoder before this OneHotEncoder to convert the categories to integers, then you can now use the OneHotEncoder directly.
warnings.warn(msg, FutureWarning)
/src/keras/activations.py:209: UserWarning: Do not pass a layer instance (such as ELU) as the activation argument of another layer. Instead, advanced activation layers should be used just like any other layer in a model.
identifier=identifier.__class__.__name__))
Command being timed: "python compressor.py -data ../data/processed_files/Caenorhabditis_elegans.WBcel235.dna.toplevel.npy -data_params ../data/processed_files/Caenorhabditis_elegans.WBcel235.dna.toplevel.param.json -model ../data/trained_models/Caenorhabditis_elegans.WBcel235.dna.toplevel/FC_4layer_16bit.hdf5 -model_name FC_4layer_16bit -output ../data/compressed/Caenorhabditis_elegans.WBcel235.dna.toplevel/FC_4layer_16bit.compressed -batch_size 10000"
User time (seconds): 1306.71
System time (seconds): 32.17
Percent of CPU this job got: 102%
Elapsed (wall clock) time (h:mm:ss or m:ss): 21:48.00
Average shared text size (kbytes): 0
Average unshared data size (kbytes): 0
Average stack size (kbytes): 0
Average total size (kbytes): 0
Maximum resident set size (kbytes): 9362292
Average resident set size (kbytes): 0
Major (requiring I/O) page faults: 0
Minor (reclaiming a frame) page faults: 10944016
Voluntary context switches: 505756
Involuntary context switches: 3207
Swaps: 0
File system inputs: 0
File system outputs: 125240
Socket messages sent: 0
Socket messages received: 0
Signals delivered: 0
Page size (bytes): 4096
Exit status: 0
2018-10-29 03:26:39.579354: I tensorflow/core/platform/cpu_feature_guard.cc:141] Your CPU supports instructions that this TensorFlow binary was not compiled to use: AVX2 FMA
2018-10-29 03:26:41.459594: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1411] Found device 0 with properties:
name: TITAN X (Pascal) major: 6 minor: 1 memoryClockRate(GHz): 1.531
pciBusID: 0000:08:00.0
totalMemory: 11.91GiB freeMemory: 11.75GiB
2018-10-29 03:26:41.459663: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1490] Adding visible gpu devices: 0
2018-10-29 03:26:41.809499: I tensorflow/core/common_runtime/gpu/gpu_device.cc:971] Device interconnect StreamExecutor with strength 1 edge matrix:
2018-10-29 03:26:41.809571: I tensorflow/core/common_runtime/gpu/gpu_device.cc:977] 0
2018-10-29 03:26:41.809582: I tensorflow/core/common_runtime/gpu/gpu_device.cc:990] 0: N
2018-10-29 03:26:41.809994: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1103] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 11372 MB memory) -> physical GPU (device: 0, name: TITAN X (Pascal), pci bus id: 0000:08:00.0, compute capability: 6.1)
/opt/conda/lib/python3.6/site-packages/sklearn/externals/joblib/externals/cloudpickle/cloudpickle.py:47: DeprecationWarning: the imp module is deprecated in favour of importlib; see the module's documentation for alternative uses
import imp
Using TensorFlow backend.
/src/keras/activations.py:209: UserWarning: Do not pass a layer instance (such as ELU) as the activation argument of another layer. Instead, advanced activation layers should be used just like any other layer in a model.
identifier=identifier.__class__.__name__))
{'0': 'A', '1': 'C', '2': 'T', '3': 'G'}
[3 1 1 2 0 0 3 1 1 2]
Command being timed: "python decompressor.py -output ../data/compressed/Caenorhabditis_elegans.WBcel235.dna.toplevel/FC_4layer_16bit.reconstructed.txt -model ../data/trained_models/Caenorhabditis_elegans.WBcel235.dna.toplevel/FC_4layer_16bit.hdf5 -model_name FC_4layer_16bit -input_file_prefix ../data/compressed/Caenorhabditis_elegans.WBcel235.dna.toplevel/FC_4layer_16bit.compressed -batch_size 10000"
User time (seconds): 1933.61
System time (seconds): 18.49
Percent of CPU this job got: 101%
Elapsed (wall clock) time (h:mm:ss or m:ss): 32:11.95
Average shared text size (kbytes): 0
Average unshared data size (kbytes): 0
Average stack size (kbytes): 0
Average total size (kbytes): 0
Maximum resident set size (kbytes): 2367532
Average resident set size (kbytes): 0
Major (requiring I/O) page faults: 0
Minor (reclaiming a frame) page faults: 2389204
Voluntary context switches: 515177
Involuntary context switches: 4217
Swaps: 0
File system inputs: 0
File system outputs: 275888
Socket messages sent: 0
Socket messages received: 0
Signals delivered: 0
Page size (bytes): 4096
Exit status: 0
================================================
FILE: data/logs_data/Caenorhabditis_elegans.WBcel235.dna.toplevel/GRU_multi_16bit.log.csv
================================================
Starting training ...
0;1.861744895370654
1;1.8492561831508143
================================================
FILE: data/logs_data/Caenorhabditis_elegans.WBcel235.dna.toplevel/LSTM_multi.log.csv
================================================
Starting training ...
0;1.8425407592824365
1;1.8289623391471852
2;1.8244587229496791
3;1.821987229708892
4;1.8205238264565107
5;1.8194774112700443
6;1.818646497426272
Starting Compression ...
2018-11-01 03:50:13.332417: I tensorflow/core/platform/cpu_feature_guard.cc:141] Your CPU supports instructions that this TensorFlow binary was not compiled to use: AVX2 FMA
2018-11-01 03:50:16.044703: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1411] Found device 0 with properties:
name: TITAN X (Pascal) major: 6 minor: 1 memoryClockRate(GHz): 1.531
pciBusID: 0000:05:00.0
totalMemory: 11.91GiB freeMemory: 11.75GiB
2018-11-01 03:50:16.044742: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1490] Adding visible gpu devices: 0
2018-11-01 03:50:16.445742: I tensorflow/core/common_runtime/gpu/gpu_device.cc:971] Device interconnect StreamExecutor with strength 1 edge matrix:
2018-11-01 03:50:16.445817: I tensorflow/core/common_runtime/gpu/gpu_device.cc:977] 0
2018-11-01 03:50:16.445826: I tensorflow/core/common_runtime/gpu/gpu_device.cc:990] 0: N
2018-11-01 03:50:16.446227: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1103] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 11372 MB memory) -> physical GPU (device: 0, name: TITAN X (Pascal), pci bus id: 0000:05:00.0, compute capability: 6.1)
/opt/conda/lib/python3.6/site-packages/sklearn/externals/joblib/externals/cloudpickle/cloudpickle.py:47: DeprecationWarning: the imp module is deprecated in favour of importlib; see the module's documentation for alternative uses
import imp
Using TensorFlow backend.
/opt/conda/lib/python3.6/site-packages/sklearn/preprocessing/_encoders.py:363: FutureWarning: The handling of integer data will change in version 0.22. Currently, the categories are determined based on the range [0, max(values)], while in the future they will be determined based on the unique values.
If you want the future behaviour and silence this warning, you can specify "categories='auto'".
In case you used a LabelEncoder before this OneHotEncoder to convert the categories to integers, then you can now use the OneHotEncoder directly.
warnings.warn(msg, FutureWarning)
Command being timed: "python compressor.py -data ../data/processed_files/Caenorhabditis_elegans.WBcel235.dna.toplevel.npy -data_params ../data/processed_files/Caenorhabditis_elegans.WBcel235.dna.toplevel.param.json -model ../data/trained_models/Caenorhabditis_elegans.WBcel235.dna.toplevel/LSTM_multi.hdf5 -model_name LSTM_multi -output ../data/compressed/Caenorhabditis_elegans.WBcel235.dna.toplevel/LSTM_multi.compressed -batch_size 10000"
User time (seconds): 1507.48
System time (seconds): 99.09
Percent of CPU this job got: 104%
Elapsed (wall clock) time (h:mm:ss or m:ss): 25:35.33
Average shared text size (kbytes): 0
Average unshared data size (kbytes): 0
Average stack size (kbytes): 0
Average total size (kbytes): 0
Maximum resident set size (kbytes): 9361592
Average resident set size (kbytes): 0
Major (requiring I/O) page faults: 0
Minor (reclaiming a frame) page faults: 10893122
Voluntary context switches: 2828604
Involuntary context switches: 4006
Swaps: 0
File system inputs: 8
File system outputs: 124616
Socket messages sent: 0
Socket messages received: 0
Signals delivered: 0
Page size (bytes): 4096
Exit status: 0
2018-11-01 04:15:17.853060: I tensorflow/core/platform/cpu_feature_guard.cc:141] Your CPU supports instructions that this TensorFlow binary was not compiled to use: AVX2 FMA
2018-11-01 04:15:20.516531: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1411] Found device 0 with properties:
name: TITAN X (Pascal) major: 6 minor: 1 memoryClockRate(GHz): 1.531
pciBusID: 0000:05:00.0
totalMemory: 11.91GiB freeMemory: 11.75GiB
2018-11-01 04:15:20.516589: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1490] Adding visible gpu devices: 0
2018-11-01 04:15:20.886321: I tensorflow/core/common_runtime/gpu/gpu_device.cc:971] Device interconnect StreamExecutor with strength 1 edge matrix:
2018-11-01 04:15:20.886382: I tensorflow/core/common_runtime/gpu/gpu_device.cc:977] 0
2018-11-01 04:15:20.886392: I tensorflow/core/common_runtime/gpu/gpu_device.cc:990] 0: N
2018-11-01 04:15:20.886758: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1103] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 11372 MB memory) -> physical GPU (device: 0, name: TITAN X (Pascal), pci bus id: 0000:05:00.0, compute capability: 6.1)
/opt/conda/lib/python3.6/site-packages/sklearn/externals/joblib/externals/cloudpickle/cloudpickle.py:47: DeprecationWarning: the imp module is deprecated in favour of importlib; see the module's documentation for alternative uses
import imp
Using TensorFlow backend.
{'0': 'A', '1': 'C', '2': 'T', '3': 'G'}
[3 1 1 2 0 0 3 1 1 2]
Command being timed: "python decompressor.py -output ../data/compressed/Caenorhabditis_elegans.WBcel235.dna.toplevel/LSTM_multi.reconstructed.txt -model ../data/trained_models/Caenorhabditis_elegans.WBcel235.dna.toplevel/LSTM_multi.hdf5 -model_name LSTM_multi -input_file_prefix ../data/compressed/Caenorhabditis_elegans.WBcel235.dna.toplevel/LSTM_multi.compressed -batch_size 10000"
User time (seconds): 2082.99
System time (seconds): 82.61
Percent of CPU this job got: 102%
Elapsed (wall clock) time (h:mm:ss or m:ss): 35:06.39
Average shared text size (kbytes): 0
Average unshared data size (kbytes): 0
Average stack size (kbytes): 0
Average total size (kbytes): 0
Maximum resident set size (kbytes): 3070212
Average resident set size (kbytes): 0
Major (requiring I/O) page faults: 0
Minor (reclaiming a frame) page faults: 2682238
Voluntary context switches: 2887925
Involuntary context switches: 4712
Swaps: 0
File system inputs: 0
File system outputs: 275888
Socket messages sent: 0
Socket messages received: 0
Signals delivered: 0
Page size (bytes): 4096
Exit status: 0
Starting Compression ...
2018-11-08 09:28:53.078507: I tensorflow/core/platform/cpu_feature_guard.cc:141] Your CPU supports instructions that this TensorFlow binary was not compiled to use: AVX2 FMA
2018-11-08 09:28:56.137546: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1411] Found device 0 with properties:
name: TITAN X (Pascal) major: 6 minor: 1 memoryClockRate(GHz): 1.531
pciBusID: 0000:08:00.0
totalMemory: 11.91GiB freeMemory: 11.75GiB
2018-11-08 09:28:56.137594: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1490] Adding visible gpu devices: 0
2018-11-08 09:28:56.501446: I tensorflow/core/common_runtime/gpu/gpu_device.cc:971] Device interconnect StreamExecutor with strength 1 edge matrix:
2018-11-08 09:28:56.501522: I tensorflow/core/common_runtime/gpu/gpu_device.cc:977] 0
2018-11-08 09:28:56.501541: I tensorflow/core/common_runtime/gpu/gpu_device.cc:990] 0: N
2018-11-08 09:28:56.502012: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1103] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 11372 MB memory) -> physical GPU (device: 0, name: TITAN X (Pascal), pci bus id: 0000:08:00.0, compute capability: 6.1)
/opt/conda/lib/python3.6/site-packages/sklearn/externals/joblib/externals/cloudpickle/cloudpickle.py:47: DeprecationWarning: the imp module is deprecated in favour of importlib; see the module's documentation for alternative uses
import imp
Using TensorFlow backend.
/opt/conda/lib/python3.6/site-packages/sklearn/preprocessing/_encoders.py:363: FutureWarning: The handling of integer data will change in version 0.22. Currently, the categories are determined based on the range [0, max(values)], while in the future they will be determined based on the unique values.
If you want the future behaviour and silence this warning, you can specify "categories='auto'".
In case you used a LabelEncoder before this OneHotEncoder to convert the categories to integers, then you can now use the OneHotEncoder directly.
warnings.warn(msg, FutureWarning)
Command being timed: "python compressor.py -data ../data/processed_files/Caenorhabditis_elegans.WBcel235.dna.toplevel.npy -data_params ../data/processed_files/Caenorhabditis_elegans.WBcel235.dna.toplevel.param.json -model ../data/trained_models/Caenorhabditis_elegans.WBcel235.dna.toplevel/LSTM_multi.hdf5 -model_name LSTM_multi -output ../data/compressed/Caenorhabditis_elegans.WBcel235.dna.toplevel/LSTM_multi.compressed -batch_size 1000"
User time (seconds): 1873.51
System time (seconds): 60.12
Percent of CPU this job got: 110%
Elapsed (wall clock) time (h:mm:ss or m:ss): 29:09.17
Average shared text size (kbytes): 0
Average unshared data size (kbytes): 0
Average stack size (kbytes): 0
Average total size (kbytes): 0
Maximum resident set size (kbytes): 9362084
Average resident set size (kbytes): 0
Major (requiring I/O) page faults: 0
Minor (reclaiming a frame) page faults: 10078835
Voluntary context switches: 2583679
Involuntary context switches: 12426
Swaps: 0
File system inputs: 0
File system outputs: 92424
Socket messages sent: 0
Socket messages received: 0
Signals delivered: 0
Page size (bytes): 4096
Exit status: 0
2018-11-08 09:57:30.441071: I tensorflow/core/platform/cpu_feature_guard.cc:141] Your CPU supports instructions that this TensorFlow binary was not compiled to use: AVX2 FMA
2018-11-08 09:57:33.405225: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1411] Found device 0 with properties:
name: TITAN X (Pascal) major: 6 minor: 1 memoryClockRate(GHz): 1.531
pciBusID: 0000:08:00.0
totalMemory: 11.91GiB freeMemory: 11.75GiB
2018-11-08 09:57:33.405283: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1490] Adding visible gpu devices: 0
2018-11-08 09:57:33.737037: I tensorflow/core/common_runtime/gpu/gpu_device.cc:971] Device interconnect StreamExecutor with strength 1 edge matrix:
2018-11-08 09:57:33.737111: I tensorflow/core/common_runtime/gpu/gpu_device.cc:977] 0
2018-11-08 09:57:33.737120: I tensorflow/core/common_runtime/gpu/gpu_device.cc:990] 0: N
2018-11-08 09:57:33.737455: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1103] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 11372 MB memory) -> physical GPU (device: 0, name: TITAN X (Pascal), pci bus id: 0000:08:00.0, compute capability: 6.1)
/opt/conda/lib/python3.6/site-packages/sklearn/externals/joblib/externals/cloudpickle/cloudpickle.py:47: DeprecationWarning: the imp module is deprecated in favour of importlib; see the module's documentation for alternative uses
import imp
Using TensorFlow backend.
{'0': 'A', '1': 'C', '2': 'T', '3': 'G'}
[3 1 1 2 0 0 3 1 1 2]
Command being timed: "python decompressor.py -output ../data/compressed/Caenorhabditis_elegans.WBcel235.dna.toplevel/LSTM_multi.reconstructed.txt -model ../data/trained_models/Caenorhabditis_elegans.WBcel235.dna.toplevel/LSTM_multi.hdf5 -model_name LSTM_multi -input_file_prefix ../data/compressed/Caenorhabditis_elegans.WBcel235.dna.toplevel/LSTM_multi.compressed -batch_size 1000"
User time (seconds): 2598.83
System time (seconds): 48.67
Percent of CPU this job got: 107%
Elapsed (wall clock) time (h:mm:ss or m:ss): 41:13.22
Average shared text size (kbytes): 0
Average unshared data size (kbytes): 0
Average stack size (kbytes): 0
Average total size (kbytes): 0
Maximum resident set size (kbytes): 3054192
Average resident set size (kbytes): 0
Major (requiring I/O) page faults: 0
Minor (reclaiming a frame) page faults: 1341129
Voluntary context switches: 2630495
Involuntary context switches: 6468
Swaps: 0
File system inputs: 0
File system outputs: 243768
Socket messages sent: 0
Socket messages received: 0
Signals delivered: 0
Page size (bytes): 4096
Exit status: 0
================================================
FILE: data/logs_data/Caenorhabditis_elegans.WBcel235.dna.toplevel/LSTM_multi_16bit.log.csv
================================================
Starting training ...
0;1.864678990290116
1;1.8516583165013785
================================================
FILE: data/logs_data/Caenorhabditis_elegans.WBcel235.dna.toplevel/LSTM_multi_big.log.csv
================================================
Starting training ...
0;1.8324452448726254
1;1.8125994139707988
2;1.8064765404868577
3;1.8032488849143944
4;1.8011765443646366
5;1.7997155088398975
6;1.7985437180782073
7;1.797582210779689
Starting Compression ...
2018-11-03 01:43:28.459198: I tensorflow/core/platform/cpu_feature_guard.cc:141] Your CPU supports instructions that this TensorFlow binary was not compiled to use: AVX2 FMA
2018-11-03 01:43:32.067929: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1411] Found device 0 with properties:
name: TITAN X (Pascal) major: 6 minor: 1 memoryClockRate(GHz): 1.531
pciBusID: 0000:8a:00.0
totalMemory: 11.91GiB freeMemory: 11.75GiB
2018-11-03 01:43:32.067987: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1490] Adding visible gpu devices: 0
2018-11-03 01:43:32.375893: I tensorflow/core/common_runtime/gpu/gpu_device.cc:971] Device interconnect StreamExecutor with strength 1 edge matrix:
2018-11-03 01:43:32.375946: I tensorflow/core/common_runtime/gpu/gpu_device.cc:977] 0
2018-11-03 01:43:32.375955: I tensorflow/core/common_runtime/gpu/gpu_device.cc:990] 0: N
2018-11-03 01:43:32.376358: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1103] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 11372 MB memory) -> physical GPU (device: 0, name: TITAN X (Pascal), pci bus id: 0000:8a:00.0, compute capability: 6.1)
/opt/conda/lib/python3.6/site-packages/sklearn/externals/joblib/externals/cloudpickle/cloudpickle.py:47: DeprecationWarning: the imp module is deprecated in favour of importlib; see the module's documentation for alternative uses
import imp
Using TensorFlow backend.
/opt/conda/lib/python3.6/site-packages/sklearn/preprocessing/_encoders.py:363: FutureWarning: The handling of integer data will change in version 0.22. Currently, the categories are determined based on the range [0, max(values)], while in the future they will be determined based on the unique values.
If you want the future behaviour and silence this warning, you can specify "categories='auto'".
In case you used a LabelEncoder before this OneHotEncoder to convert the categories to integers, then you can now use the OneHotEncoder directly.
warnings.warn(msg, FutureWarning)
Command being timed: "python compressor.py -data ../data/processed_files/Caenorhabditis_elegans.WBcel235.dna.toplevel.npy -data_params ../data/processed_files/Caenorhabditis_elegans.WBcel235.dna.toplevel.param.json -model ../data/trained_models/Caenorhabditis_elegans.WBcel235.dna.toplevel/LSTM_multi_big.hdf5 -model_name LSTM_multi_big -output ../data/compressed/Caenorhabditis_elegans.WBcel235.dna.toplevel/LSTM_multi_big.compressed -batch_size 10000"
User time (seconds): 1458.59
System time (seconds): 151.85
Percent of CPU this job got: 105%
Elapsed (wall clock) time (h:mm:ss or m:ss): 25:24.35
Average shared text size (kbytes): 0
Average unshared data size (kbytes): 0
Average stack size (kbytes): 0
Average total size (kbytes): 0
Maximum resident set size (kbytes): 9361812
Average resident set size (kbytes): 0
Major (requiring I/O) page faults: 0
Minor (reclaiming a frame) page faults: 12250354
Voluntary context switches: 4824576
Involuntary context switches: 10147
Swaps: 0
File system inputs: 0
File system outputs: 124064
Socket messages sent: 0
Socket messages received: 0
Signals delivered: 0
Page size (bytes): 4096
Exit status: 0
2018-11-03 02:08:21.675009: I tensorflow/core/platform/cpu_feature_guard.cc:141] Your CPU supports instructions that this TensorFlow binary was not compiled to use: AVX2 FMA
2018-11-03 02:08:25.296595: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1411] Found device 0 with properties:
name: TITAN X (Pascal) major: 6 minor: 1 memoryClockRate(GHz): 1.531
pciBusID: 0000:8a:00.0
totalMemory: 11.91GiB freeMemory: 11.75GiB
2018-11-03 02:08:25.296643: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1490] Adding visible gpu devices: 0
2018-11-03 02:08:25.624612: I tensorflow/core/common_runtime/gpu/gpu_device.cc:971] Device interconnect StreamExecutor with strength 1 edge matrix:
2018-11-03 02:08:25.624667: I tensorflow/core/common_runtime/gpu/gpu_device.cc:977] 0
2018-11-03 02:08:25.624677: I tensorflow/core/common_runtime/gpu/gpu_device.cc:990] 0: N
2018-11-03 02:08:25.625102: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1103] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 11372 MB memory) -> physical GPU (device: 0, name: TITAN X (Pascal), pci bus id: 0000:8a:00.0, compute capability: 6.1)
/opt/conda/lib/python3.6/site-packages/sklearn/externals/joblib/externals/cloudpickle/cloudpickle.py:47: DeprecationWarning: the imp module is deprecated in favour of importlib; see the module's documentation for alternative uses
import imp
Using TensorFlow backend.
{'0': 'A', '1': 'C', '2': 'T', '3': 'G'}
[3 1 1 2 0 0 3 1 1 2]
Command being timed: "python decompressor.py -output ../data/compressed/Caenorhabditis_elegans.WBcel235.dna.toplevel/LSTM_multi_big.reconstructed.txt -model ../data/trained_models/Caenorhabditis_elegans.WBcel235.dna.toplevel/LSTM_multi_big.hdf5 -model_name LSTM_multi_big -input_file_prefix ../data/compressed/Caenorhabditis_elegans.WBcel235.dna.toplevel/LSTM_multi_big.compressed -batch_size 10000"
User time (seconds): 1979.48
System time (seconds): 134.76
Percent of CPU this job got: 103%
Elapsed (wall clock) time (h:mm:ss or m:ss): 34:01.73
Average shared text size (kbytes): 0
Average unshared data size (kbytes): 0
Average stack size (kbytes): 0
Average total size (kbytes): 0
Maximum resident set size (kbytes): 3073824
Average resident set size (kbytes): 0
Major (requiring I/O) page faults: 0
Minor (reclaiming a frame) page faults: 2562957
Voluntary context switches: 4870059
Involuntary context switches: 10074
Swaps: 0
File system inputs: 0
File system outputs: 275888
Socket messages sent: 0
Socket messages received: 0
Signals delivered: 0
Page size (bytes): 4096
Exit status: 0
================================================
FILE: data/logs_data/Caenorhabditis_elegans.WBcel235.dna.toplevel/LSTM_multi_bn.log.csv
================================================
Starting training ...
0;1.8537912338063736
1;1.839781109242452
2;1.832267580159184
3;1.8280653073082063
4;1.8255606130454005
5;1.8240399114679169
6;1.8229558919456825
7;1.8221357129048248
Starting Compression ...
2018-11-06 22:32:33.249737: I tensorflow/core/platform/cpu_feature_guard.cc:141] Your CPU supports instructions that this TensorFlow binary was not compiled to use: AVX2 FMA
2018-11-06 22:32:36.189917: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1411] Found device 0 with properties:
name: TITAN X (Pascal) major: 6 minor: 1 memoryClockRate(GHz): 1.531
pciBusID: 0000:09:00.0
totalMemory: 11.91GiB freeMemory: 11.75GiB
2018-11-06 22:32:36.189964: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1490] Adding visible gpu devices: 0
2018-11-06 22:32:36.548290: I tensorflow/core/common_runtime/gpu/gpu_device.cc:971] Device interconnect StreamExecutor with strength 1 edge matrix:
2018-11-06 22:32:36.548360: I tensorflow/core/common_runtime/gpu/gpu_device.cc:977] 0
2018-11-06 22:32:36.548372: I tensorflow/core/common_runtime/gpu/gpu_device.cc:990] 0: N
2018-11-06 22:32:36.548768: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1103] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 11372 MB memory) -> physical GPU (device: 0, name: TITAN X (Pascal), pci bus id: 0000:09:00.0, compute capability: 6.1)
/opt/conda/lib/python3.6/site-packages/sklearn/externals/joblib/externals/cloudpickle/cloudpickle.py:47: DeprecationWarning: the imp module is deprecated in favour of importlib; see the module's documentation for alternative uses
import imp
Using TensorFlow backend.
/opt/conda/lib/python3.6/site-packages/sklearn/preprocessing/_encoders.py:363: FutureWarning: The handling of integer data will change in version 0.22. Currently, the categories are determined based on the range [0, max(values)], while in the future they will be determined based on the unique values.
If you want the future behaviour and silence this warning, you can specify "categories='auto'".
In case you used a LabelEncoder before this OneHotEncoder to convert the categories to integers, then you can now use the OneHotEncoder directly.
warnings.warn(msg, FutureWarning)
Command being timed: "python compressor.py -data ../data/processed_files/Caenorhabditis_elegans.WBcel235.dna.toplevel.npy -data_params ../data/processed_files/Caenorhabditis_elegans.WBcel235.dna.toplevel.param.json -model ../data/trained_models/Caenorhabditis_elegans.WBcel235.dna.toplevel/LSTM_multi_bn.hdf5 -model_name LSTM_multi_bn -output ../data/compressed/Caenorhabditis_elegans.WBcel235.dna.toplevel/LSTM_multi_bn.compressed -batch_size 10000"
User time (seconds): 1491.61
System time (seconds): 112.35
Percent of CPU this job got: 105%
Elapsed (wall clock) time (h:mm:ss or m:ss): 25:25.74
Average shared text size (kbytes): 0
Average unshared data size (kbytes): 0
Average stack size (kbytes): 0
Average total size (kbytes): 0
Maximum resident set size (kbytes): 9360528
Average resident set size (kbytes): 0
Major (requiring I/O) page faults: 0
Minor (reclaiming a frame) page faults: 11243979
Voluntary context switches: 3416452
Involuntary context switches: 4674
Swaps: 0
File system inputs: 0
File system outputs: 124688
Socket messages sent: 0
Socket messages received: 0
Signals delivered: 0
Page size (bytes): 4096
Exit status: 0
2018-11-06 22:57:27.284607: I tensorflow/core/platform/cpu_feature_guard.cc:141] Your CPU supports instructions that this TensorFlow binary was not compiled to use: AVX2 FMA
2018-11-06 22:57:30.354121: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1411] Found device 0 with properties:
name: TITAN X (Pascal) major: 6 minor: 1 memoryClockRate(GHz): 1.531
pciBusID: 0000:09:00.0
totalMemory: 11.91GiB freeMemory: 11.75GiB
2018-11-06 22:57:30.354183: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1490] Adding visible gpu devices: 0
2018-11-06 22:57:30.721955: I tensorflow/core/common_runtime/gpu/gpu_device.cc:971] Device interconnect StreamExecutor with strength 1 edge matrix:
2018-11-06 22:57:30.722030: I tensorflow/core/common_runtime/gpu/gpu_device.cc:977] 0
2018-11-06 22:57:30.722040: I tensorflow/core/common_runtime/gpu/gpu_device.cc:990]
gitextract_jk786e2m/
├── .gitignore
├── LICENSE
├── README.md
├── data/
│ ├── final_log.csv
│ ├── logs_data/
│ │ ├── Caenorhabditis_elegans.WBcel235.dna.chromosome.I/
│ │ │ ├── FC.log.csv
│ │ │ ├── FC_16bit.log.csv
│ │ │ ├── FC_4layer.log.csv
│ │ │ ├── FC_4layer_16bit.log.csv
│ │ │ ├── GRU_multi_16bit.log.csv
│ │ │ ├── LSTM.log.csv
│ │ │ ├── LSTM_multi.log.csv
│ │ │ ├── LSTM_multi_16bit.log.csv
│ │ │ ├── LSTM_multi_big.log.csv
│ │ │ ├── LSTM_multi_bn.log.csv
│ │ │ ├── biGRU.log.csv
│ │ │ ├── biGRU_16bit.log.csv
│ │ │ ├── biLSTM.log.csv
│ │ │ └── biLSTM_16bit.log.csv
│ │ ├── Caenorhabditis_elegans.WBcel235.dna.toplevel/
│ │ │ ├── FC.log.csv
│ │ │ ├── FC_16bit.log.csv
│ │ │ ├── FC_4layer.log.csv
│ │ │ ├── FC_4layer_16bit.log.csv
│ │ │ ├── GRU_multi_16bit.log.csv
│ │ │ ├── LSTM_multi.log.csv
│ │ │ ├── LSTM_multi_16bit.log.csv
│ │ │ ├── LSTM_multi_big.log.csv
│ │ │ ├── LSTM_multi_bn.log.csv
│ │ │ ├── biGRU.log.csv
│ │ │ ├── biGRU_16bit.log.csv
│ │ │ ├── biLSTM.log.csv
│ │ │ └── biLSTM_16bit.log.csv
│ │ ├── HMM20/
│ │ │ ├── FC.log.csv
│ │ │ ├── FC_16bit.log.csv
│ │ │ ├── FC_4layer.log.csv
│ │ │ ├── FC_4layer_16bit.log.csv
│ │ │ ├── GRU_multi_16bit.log.csv
│ │ │ ├── LSTM_multi.log.csv
│ │ │ ├── LSTM_multi_16bit.log.csv
│ │ │ ├── LSTM_multi_big.log.csv
│ │ │ ├── LSTM_multi_bn.log.csv
│ │ │ ├── biGRU.log.csv
│ │ │ ├── biGRU_16bit.log.csv
│ │ │ ├── biLSTM.log.csv
│ │ │ └── biLSTM_16bit.log.csv
│ │ ├── HMM30/
│ │ │ ├── FC.log.csv
│ │ │ ├── FC_16bit.log.csv
│ │ │ ├── FC_4layer.log.csv
│ │ │ ├── FC_4layer_16bit.log.csv
│ │ │ ├── GRU_multi_16bit.log.csv
│ │ │ ├── LSTM_multi.log.csv
│ │ │ ├── LSTM_multi_16bit.log.csv
│ │ │ ├── LSTM_multi_big.log.csv
│ │ │ ├── LSTM_multi_bn.log.csv
│ │ │ ├── biGRU.log.csv
│ │ │ ├── biGRU_16bit.log.csv
│ │ │ ├── biLSTM.log.csv
│ │ │ └── biLSTM_16bit.log.csv
│ │ ├── HMM40/
│ │ │ ├── FC.log.csv
│ │ │ ├── FC_16bit.log.csv
│ │ │ ├── FC_4layer.log.csv
│ │ │ ├── FC_4layer_16bit.log.csv
│ │ │ ├── GRU_multi_16bit.log.csv
│ │ │ ├── LSTM_multi.log.csv
│ │ │ ├── LSTM_multi_16bit.log.csv
│ │ │ ├── LSTM_multi_big.log.csv
│ │ │ ├── LSTM_multi_bn.log.csv
│ │ │ ├── biGRU.log.csv
│ │ │ ├── biGRU_16bit.log.csv
│ │ │ ├── biLSTM.log.csv
│ │ │ └── biLSTM_16bit.log.csv
│ │ ├── PhiX_quality_truncated/
│ │ │ ├── FC_4layer.log.csv
│ │ │ ├── LSTM_multi.log.csv
│ │ │ ├── LSTM_multi_big.log.csv
│ │ │ ├── LSTM_multi_bn.log.csv
│ │ │ ├── biGRU.log.csv
│ │ │ └── biLSTM.log.csv
│ │ ├── chr1/
│ │ │ ├── FC.log.csv
│ │ │ ├── FC_16bit.log.csv
│ │ │ ├── FC_4layer.log.csv
│ │ │ ├── FC_4layer_16bit.log.csv
│ │ │ ├── GRU_multi_16bit.log.csv
│ │ │ ├── LSTM_multi.log.csv
│ │ │ ├── LSTM_multi_16bit.log.csv
│ │ │ ├── LSTM_multi_big.log.csv
│ │ │ ├── LSTM_multi_bn.log.csv
│ │ │ ├── biGRU.log.csv
│ │ │ ├── biGRU_16bit.log.csv
│ │ │ ├── biLSTM.log.csv
│ │ │ └── biLSTM_16bit.log.csv
│ │ ├── enwiki8/
│ │ │ ├── biGRU.log.csv
│ │ │ └── biGRU_big.log.csv
│ │ ├── iid_p10.1/
│ │ │ ├── FC.log.csv
│ │ │ ├── FC_16bit.log.csv
│ │ │ ├── FC_4layer.log.csv
│ │ │ ├── GRU_multi_16bit.log.csv
│ │ │ ├── LSTM_multi.log.csv
│ │ │ ├── LSTM_multi_16bit.log.csv
│ │ │ ├── LSTM_multi_big.log.csv
│ │ │ ├── LSTM_multi_bn.log.csv
│ │ │ ├── biGRU.log.csv
│ │ │ ├── biGRU_16bit.log.csv
│ │ │ ├── biLSTM.log.csv
│ │ │ └── biLSTM_16bit.log.csv
│ │ ├── text8/
│ │ │ ├── FC.log.csv
│ │ │ ├── FC_16bit.log.csv
│ │ │ ├── FC_4layer.log.csv
│ │ │ ├── FC_4layer_16bit.log.csv
│ │ │ ├── FC_4layer_big.log.csv
│ │ │ ├── GRU_multi_16bit.log.csv
│ │ │ ├── GRU_multi_big.log.csv
│ │ │ ├── LSTM_multi.log.csv
│ │ │ ├── LSTM_multi_16bit.log.csv
│ │ │ ├── LSTM_multi_big.log.csv
│ │ │ ├── LSTM_multi_bn.log.csv
│ │ │ ├── biGRU.log.csv
│ │ │ ├── biGRU_16bit.log.csv
│ │ │ ├── biGRU_big.log.csv
│ │ │ ├── biLSTM.log.csv
│ │ │ └── biLSTM_16bit.log.csv
│ │ ├── xor20/
│ │ │ ├── FC.log.csv
│ │ │ ├── FC_16bit.log.csv
│ │ │ ├── FC_4layer.log.csv
│ │ │ ├── FC_4layer_16bit.log.csv
│ │ │ ├── GRU_multi_16bit.log.csv
│ │ │ ├── LSTM_multi.log.csv
│ │ │ ├── LSTM_multi_16bit.log.csv
│ │ │ ├── LSTM_multi_big.log.csv
│ │ │ ├── LSTM_multi_bn.log.csv
│ │ │ ├── biGRU.log.csv
│ │ │ ├── biGRU_16bit.log.csv
│ │ │ ├── biLSTM.log.csv
│ │ │ └── biLSTM_16bit.log.csv
│ │ ├── xor30/
│ │ │ ├── FC.log.csv
│ │ │ ├── FC_16bit.log.csv
│ │ │ ├── FC_4layer.log.csv
│ │ │ ├── FC_4layer_16bit.log.csv
│ │ │ ├── GRU_multi_16bit.log.csv
│ │ │ ├── LSTM_multi.log.csv
│ │ │ ├── LSTM_multi_16bit.log.csv
│ │ │ ├── LSTM_multi_big.log.csv
│ │ │ ├── LSTM_multi_bn.log.csv
│ │ │ ├── biGRU.log.csv
│ │ │ ├── biGRU_16bit.log.csv
│ │ │ ├── biLSTM.log.csv
│ │ │ └── biLSTM_16bit.log.csv
│ │ ├── xor40/
│ │ │ ├── FC.log.csv
│ │ │ ├── FC_16bit.log.csv
│ │ │ ├── FC_4layer.log.csv
│ │ │ ├── FC_4layer_16bit.log.csv
│ │ │ ├── GRU_multi_16bit.log.csv
│ │ │ ├── LSTM_multi.log.csv
│ │ │ ├── LSTM_multi_16bit.log.csv
│ │ │ ├── LSTM_multi_big.log.csv
│ │ │ ├── LSTM_multi_bn.log.csv
│ │ │ ├── biGRU.log.csv
│ │ │ ├── biGRU_16bit.log.csv
│ │ │ ├── biLSTM.log.csv
│ │ │ └── biLSTM_16bit.log.csv
│ │ ├── xor50/
│ │ │ ├── FC.log.csv
│ │ │ ├── FC_16bit.log.csv
│ │ │ ├── FC_4layer.log.csv
│ │ │ ├── FC_4layer_16bit.log.csv
│ │ │ ├── GRU_multi_16bit.log.csv
│ │ │ ├── LSTM_multi.log.csv
│ │ │ ├── LSTM_multi_16bit.log.csv
│ │ │ ├── LSTM_multi_big.log.csv
│ │ │ ├── LSTM_multi_bn.log.csv
│ │ │ ├── biGRU.log.csv
│ │ │ ├── biGRU_16bit.log.csv
│ │ │ ├── biLSTM.log.csv
│ │ │ └── biLSTM_16bit.log.csv
│ │ └── xor60/
│ │ ├── FC_4layer.log.csv
│ │ ├── LSTM_multi.log.csv
│ │ └── biGRU.log.csv
│ ├── parse_new.py
│ ├── parse_wiki.py
│ ├── run_fasta_preprocess.sh
│ ├── run_parser.sh
│ ├── trained_models/
│ │ ├── Caenorhabditis_elegans.WBcel235.dna.chromosome.I/
│ │ │ ├── FC.hdf5
│ │ │ ├── FC_16bit.hdf5
│ │ │ ├── FC_4layer.hdf5
│ │ │ ├── FC_4layer_16bit.hdf5
│ │ │ ├── GRU_multi_16bit.hdf5
│ │ │ ├── LSTM_multi.hdf5
│ │ │ ├── LSTM_multi_16bit.hdf5
│ │ │ ├── LSTM_multi_big.hdf5
│ │ │ ├── LSTM_multi_bn.hdf5
│ │ │ ├── biGRU.hdf5
│ │ │ ├── biGRU_16bit.hdf5
│ │ │ └── biLSTM_16bit.hdf5
│ │ ├── Caenorhabditis_elegans.WBcel235.dna.toplevel/
│ │ │ ├── FC.hdf5
│ │ │ ├── FC_16bit.hdf5
│ │ │ ├── FC_4layer.hdf5
│ │ │ ├── FC_4layer_16bit.hdf5
│ │ │ ├── GRU_multi_16bit.hdf5
│ │ │ ├── LSTM_multi.hdf5
│ │ │ ├── LSTM_multi_16bit.hdf5
│ │ │ ├── LSTM_multi_big.hdf5
│ │ │ ├── LSTM_multi_bn.hdf5
│ │ │ ├── biGRU.hdf5
│ │ │ ├── biGRU_16bit.hdf5
│ │ │ └── biLSTM_16bit.hdf5
│ │ ├── HMM20/
│ │ │ ├── FC.hdf5
│ │ │ ├── FC_16bit.hdf5
│ │ │ ├── FC_4layer.hdf5
│ │ │ ├── FC_4layer_16bit.hdf5
│ │ │ ├── GRU_multi_16bit.hdf5
│ │ │ ├── LSTM_multi.hdf5
│ │ │ ├── LSTM_multi_16bit.hdf5
│ │ │ ├── LSTM_multi_big.hdf5
│ │ │ ├── LSTM_multi_bn.hdf5
│ │ │ ├── biGRU.hdf5
│ │ │ ├── biGRU_16bit.hdf5
│ │ │ └── biLSTM_16bit.hdf5
│ │ ├── HMM30/
│ │ │ ├── FC.hdf5
│ │ │ ├── FC_16bit.hdf5
│ │ │ ├── FC_4layer.hdf5
│ │ │ ├── FC_4layer_16bit.hdf5
│ │ │ ├── GRU_multi_16bit.hdf5
│ │ │ ├── LSTM_multi.hdf5
│ │ │ ├── LSTM_multi_16bit.hdf5
│ │ │ ├── LSTM_multi_big.hdf5
│ │ │ ├── LSTM_multi_bn.hdf5
│ │ │ ├── biGRU.hdf5
│ │ │ ├── biGRU_16bit.hdf5
│ │ │ └── biLSTM_16bit.hdf5
│ │ ├── HMM40/
│ │ │ ├── FC.hdf5
│ │ │ ├── FC_16bit.hdf5
│ │ │ ├── FC_4layer.hdf5
│ │ │ ├── FC_4layer_16bit.hdf5
│ │ │ ├── GRU_multi_16bit.hdf5
│ │ │ ├── LSTM_multi.hdf5
│ │ │ ├── LSTM_multi_16bit.hdf5
│ │ │ ├── LSTM_multi_big.hdf5
│ │ │ ├── LSTM_multi_bn.hdf5
│ │ │ ├── biGRU.hdf5
│ │ │ ├── biGRU_16bit.hdf5
│ │ │ └── biLSTM_16bit.hdf5
│ │ ├── PhiX_quality_truncated/
│ │ │ ├── FC_4layer.hdf5
│ │ │ ├── LSTM_multi.hdf5
│ │ │ ├── LSTM_multi_big.hdf5
│ │ │ ├── LSTM_multi_bn.hdf5
│ │ │ └── biGRU.hdf5
│ │ ├── chr1/
│ │ │ ├── FC.hdf5
│ │ │ ├── FC_16bit.hdf5
│ │ │ ├── FC_4layer.hdf5
│ │ │ ├── FC_4layer_16bit.hdf5
│ │ │ ├── GRU_multi_16bit.hdf5
│ │ │ ├── LSTM_multi.hdf5
│ │ │ ├── LSTM_multi_16bit.hdf5
│ │ │ ├── LSTM_multi_big.hdf5
│ │ │ ├── LSTM_multi_bn.hdf5
│ │ │ ├── biGRU.hdf5
│ │ │ ├── biGRU_16bit.hdf5
│ │ │ └── biLSTM_16bit.hdf5
│ │ ├── iid_p10.1/
│ │ │ ├── FC.hdf5
│ │ │ ├── FC_16bit.hdf5
│ │ │ ├── FC_4layer.hdf5
│ │ │ ├── GRU_multi_16bit.hdf5
│ │ │ ├── LSTM_multi.hdf5
│ │ │ ├── LSTM_multi_16bit.hdf5
│ │ │ ├── LSTM_multi_big.hdf5
│ │ │ ├── LSTM_multi_bn.hdf5
│ │ │ ├── biGRU.hdf5
│ │ │ ├── biGRU_16bit.hdf5
│ │ │ └── biLSTM_16bit.hdf5
│ │ ├── text8/
│ │ │ ├── FC.hdf5
│ │ │ ├── FC_16bit.hdf5
│ │ │ ├── FC_4layer.hdf5
│ │ │ ├── FC_4layer_16bit.hdf5
│ │ │ ├── FC_4layer_big.hdf5
│ │ │ ├── GRU_multi_16bit.hdf5
│ │ │ ├── GRU_multi_big.hdf5
│ │ │ ├── LSTM_multi.hdf5
│ │ │ ├── LSTM_multi_16bit.hdf5
│ │ │ ├── LSTM_multi_big.hdf5
│ │ │ ├── LSTM_multi_bn.hdf5
│ │ │ ├── biGRU.hdf5
│ │ │ ├── biGRU_16bit.hdf5
│ │ │ ├── biGRU_big.hdf5
│ │ │ ├── biGRU_big.hdf5.bsc
│ │ │ └── biLSTM_16bit.hdf5
│ │ ├── xor20/
│ │ │ ├── FC.hdf5
│ │ │ ├── FC_16bit.hdf5
│ │ │ ├── FC_4layer.hdf5
│ │ │ ├── FC_4layer_16bit.hdf5
│ │ │ ├── GRU_multi_16bit.hdf5
│ │ │ ├── LSTM_multi.hdf5
│ │ │ ├── LSTM_multi_16bit.hdf5
│ │ │ ├── LSTM_multi_big.hdf5
│ │ │ ├── LSTM_multi_bn.hdf5
│ │ │ ├── biGRU.hdf5
│ │ │ ├── biGRU_16bit.hdf5
│ │ │ └── biLSTM_16bit.hdf5
│ │ ├── xor30/
│ │ │ ├── FC.hdf5
│ │ │ ├── FC_16bit.hdf5
│ │ │ ├── FC_4layer.hdf5
│ │ │ ├── FC_4layer_16bit.hdf5
│ │ │ ├── GRU_multi_16bit.hdf5
│ │ │ ├── LSTM_multi.hdf5
│ │ │ ├── LSTM_multi_16bit.hdf5
│ │ │ ├── LSTM_multi_big.hdf5
│ │ │ ├── LSTM_multi_bn.hdf5
│ │ │ ├── biGRU.hdf5
│ │ │ ├── biGRU_16bit.hdf5
│ │ │ └── biLSTM_16bit.hdf5
│ │ ├── xor40/
│ │ │ ├── FC.hdf5
│ │ │ ├── FC_16bit.hdf5
│ │ │ ├── FC_4layer.hdf5
│ │ │ ├── FC_4layer_16bit.hdf5
│ │ │ ├── GRU_multi_16bit.hdf5
│ │ │ ├── LSTM_multi.hdf5
│ │ │ ├── LSTM_multi_16bit.hdf5
│ │ │ ├── LSTM_multi_big.hdf5
│ │ │ ├── LSTM_multi_bn.hdf5
│ │ │ ├── biGRU.hdf5
│ │ │ ├── biGRU_16bit.hdf5
│ │ │ └── biLSTM_16bit.hdf5
│ │ ├── xor50/
│ │ │ ├── FC.hdf5
│ │ │ ├── FC_16bit.hdf5
│ │ │ ├── FC_4layer.hdf5
│ │ │ ├── FC_4layer_16bit.hdf5
│ │ │ ├── GRU_multi_16bit.hdf5
│ │ │ ├── LSTM_multi.hdf5
│ │ │ ├── LSTM_multi_16bit.hdf5
│ │ │ ├── LSTM_multi_big.hdf5
│ │ │ ├── LSTM_multi_bn.hdf5
│ │ │ ├── biGRU.hdf5
│ │ │ ├── biGRU_16bit.hdf5
│ │ │ └── biLSTM_16bit.hdf5
│ │ └── xor60/
│ │ └── biGRU.hdf5
│ ├── xor20.txt
│ ├── xor30.txt
│ ├── xor40.txt
│ ├── xor50.txt
│ └── xor60.txt
├── docker/
│ ├── Dockerfile
│ ├── Dockerfile.bak
│ ├── Makefile
│ ├── README.md
│ └── theanorc
├── install.sh
└── src/
├── README.md
├── arithmeticcoding_fast.py
├── compressor.py
├── decompressor.py
├── models.py
├── run_experiments.sh
└── trainer.py
SYMBOL INDEX (59 symbols across 5 files)
FILE: src/arithmeticcoding_fast.py
class ArithmeticCoderBase (line 16) | class ArithmeticCoderBase(object):
method __init__ (line 19) | def __init__(self, statesize):
method update (line 68) | def update(self, cumul, symbol):
method shift (line 106) | def shift(self):
method underflow (line 111) | def underflow(self):
class ArithmeticEncoder (line 117) | class ArithmeticEncoder(ArithmeticCoderBase):
method __init__ (line 120) | def __init__(self, statesize, bitout):
method write (line 130) | def write(self, cumul, symbol):
method finish (line 139) | def finish(self):
method shift (line 143) | def shift(self):
method underflow (line 153) | def underflow(self):
class ArithmeticDecoder (line 159) | class ArithmeticDecoder(ArithmeticCoderBase):
method __init__ (line 163) | def __init__(self, statesize, bitin):
method read (line 175) | def read(self, cumul, alphabet_size):
method shift (line 208) | def shift(self):
method underflow (line 212) | def underflow(self):
method read_code_bit (line 218) | def read_code_bit(self):
class BitInputStream (line 532) | class BitInputStream(object):
method __init__ (line 535) | def __init__(self, inp):
method read (line 546) | def read(self):
method read_no_eof (line 563) | def read_no_eof(self):
method close (line 572) | def close(self):
class BitOutputStream (line 582) | class BitOutputStream(object):
method __init__ (line 585) | def __init__(self, out):
method write (line 592) | def write(self, b):
method close (line 607) | def close(self):
FILE: src/compressor.py
function strided_app (line 60) | def strided_app(a, L, S): # Window len = L, Stride len/stepsize = S
function predict_lstm (line 67) | def predict_lstm(X, y, y_original, timesteps, bs, alphabet_size, model_n...
function var_int_encode (line 119) | def var_int_encode(byte_str_len, f):
function main (line 129) | def main():
FILE: src/decompressor.py
function strided_app (line 62) | def strided_app(a, L, S): # Window len = L, Stride len/stepsize = S
function create_data (line 69) | def create_data(rows, p=0.5):
function predict_lstm (line 75) | def predict_lstm(len_series, timesteps, bs, alphabet_size, model_name, f...
function arithmetic_step (line 123) | def arithmetic_step(prob, freqs, dec):
function var_int_decode (line 128) | def var_int_decode(f):
function main (line 140) | def main():
FILE: src/models.py
function biGRU (line 22) | def biGRU(bs,time_steps,alphabet_size):
function biGRU_big (line 31) | def biGRU_big(bs,time_steps,alphabet_size):
function biGRU_16bit (line 40) | def biGRU_16bit(bs,time_steps,alphabet_size):
function biLSTM (line 50) | def biLSTM(bs,time_steps,alphabet_size):
function biLSTM_16bit (line 60) | def biLSTM_16bit(bs,time_steps,alphabet_size):
function LSTM_multi (line 70) | def LSTM_multi(bs,time_steps,alphabet_size):
function LSTM_multi_big (line 80) | def LSTM_multi_big(bs,time_steps,alphabet_size):
function LSTM_multi_bn (line 89) | def LSTM_multi_bn(bs,time_steps,alphabet_size):
function LSTM_multi_16bit (line 100) | def LSTM_multi_16bit(bs,time_steps,alphabet_size):
function LSTM_multi_selu (line 111) | def LSTM_multi_selu(bs,time_steps,alphabet_size):
function LSTM_multi_selu_16bit (line 121) | def LSTM_multi_selu_16bit(bs,time_steps,alphabet_size):
function GRU_multi (line 133) | def GRU_multi(bs,time_steps,alphabet_size):
function GRU_multi_big (line 143) | def GRU_multi_big(bs,time_steps,alphabet_size):
function GRU_multi_16bit (line 152) | def GRU_multi_16bit(bs,time_steps,alphabet_size):
function FC_4layer_16bit (line 166) | def FC_4layer_16bit(bs,time_steps, alphabet_size):
function FC_4layer (line 178) | def FC_4layer(bs,time_steps, alphabet_size):
function FC_4layer_big (line 189) | def FC_4layer_big(bs,time_steps, alphabet_size):
function FC_16bit (line 200) | def FC_16bit(bs,time_steps,alphabet_size):
function FC (line 212) | def FC(bs,time_steps,alphabet_size):
FILE: src/trainer.py
function loss_fn (line 43) | def loss_fn(y_true, y_pred):
function strided_app (line 46) | def strided_app(a, L, S): # Window len = L, Stride len/stepsize = S
function generate_single_output_data (line 52) | def generate_single_output_data(file_path,batch_size,time_steps):
function fit_model (line 71) | def fit_model(X, Y, bs, nb_epoch, model):
Copy disabled (too large)
Download .json
Condensed preview — 349 files, each showing path, character count, and a content snippet. Download the .json file for the full structured content (51,331K chars).
[
{
"path": ".gitignore",
"chars": 210,
"preview": "data/files_to_be_compressed/*\ndata/files_to_be_compressed_test/*\ndata/processed_files/*\ndata/processed_files_test/*\ndata"
},
{
"path": "LICENSE",
"chars": 1068,
"preview": "MIT License\n\nCopyright (c) 2018 Mohit Goyal\n\nPermission is hereby granted, free of charge, to any person obtaining a cop"
},
{
"path": "README.md",
"chars": 1943,
"preview": "# DeepZip\n\n<em>Update: Please checkout our new work [DZip](https://github.com/mohit1997/Dzip-torch) presented at DCC 202"
},
{
"path": "data/final_log.csv",
"chars": 11057,
"preview": "filename,file_length,gzip,bsc,model,model-size,compressed-size,total-size,status\nCaenorhabditis_elegans.WBcel235.dna.chr"
},
{
"path": "data/logs_data/Caenorhabditis_elegans.WBcel235.dna.chromosome.I/FC.log.csv",
"chars": 25118,
"preview": "Starting training ...\nStarting Compression ...\n/opt/conda/lib/python3.6/site-packages/sklearn/externals/joblib/externals"
},
{
"path": "data/logs_data/Caenorhabditis_elegans.WBcel235.dna.chromosome.I/FC_16bit.log.csv",
"chars": 5954,
"preview": "Starting training ...\n0;1.862424908387968\r\n1;1.8382857217924111\r\n2;1.8246774072839527\r\n3;1.8144073455227936\r\n4;1.8069313"
},
{
"path": "data/logs_data/Caenorhabditis_elegans.WBcel235.dna.chromosome.I/FC_4layer.log.csv",
"chars": 23124,
"preview": "Starting training ...\nStarting Compression ...\n/opt/conda/lib/python3.6/site-packages/sklearn/externals/joblib/externals"
},
{
"path": "data/logs_data/Caenorhabditis_elegans.WBcel235.dna.chromosome.I/FC_4layer_16bit.log.csv",
"chars": 6579,
"preview": "Starting training ...\n0;1.8749647033596033\r\n1;1.8661186487278347\r\n2;1.8598197243995855\r\n3;1.8517230599055643\r\n4;1.846350"
},
{
"path": "data/logs_data/Caenorhabditis_elegans.WBcel235.dna.chromosome.I/GRU_multi_16bit.log.csv",
"chars": 5988,
"preview": "Starting training ...\n0;1.8704060626358787\r\n1;1.8522529075939262\r\n2;1.8447586744811129\r\n3;1.8398404989936477\r\nStarting C"
},
{
"path": "data/logs_data/Caenorhabditis_elegans.WBcel235.dna.chromosome.I/LSTM.log.csv",
"chars": 47,
"preview": "Starting training ...\nStarting Compression ...\n"
},
{
"path": "data/logs_data/Caenorhabditis_elegans.WBcel235.dna.chromosome.I/LSTM_multi.log.csv",
"chars": 11964,
"preview": "Starting training ...\n0;1.9433898283044024\r\n1;1.9385781589187077\r\n2;1.9332776730358134\r\n3;1.9286803646218527\r\n4;1.926139"
},
{
"path": "data/logs_data/Caenorhabditis_elegans.WBcel235.dna.chromosome.I/LSTM_multi_16bit.log.csv",
"chars": 5949,
"preview": "Starting training ...\n0;1.8784998742059582\r\n1;1.8550203101645832\r\n2;1.8470342462336096\r\n3;1.8426768361683878\r\nStarting C"
},
{
"path": "data/logs_data/Caenorhabditis_elegans.WBcel235.dna.chromosome.I/LSTM_multi_big.log.csv",
"chars": 6064,
"preview": "Starting training ...\n0;1.8382193629237424\r\n1;1.8049677677943061\r\n2;1.7903391000846942\r\n3;1.7822841469607718\r\n4;1.776752"
},
{
"path": "data/logs_data/Caenorhabditis_elegans.WBcel235.dna.chromosome.I/LSTM_multi_bn.log.csv",
"chars": 6058,
"preview": "Starting training ...\n0;1.8545062726214088\r\n1;1.8412349406804183\r\n2;1.8348612771409063\r\n3;1.828830596350872\r\n4;1.8241058"
},
{
"path": "data/logs_data/Caenorhabditis_elegans.WBcel235.dna.chromosome.I/biGRU.log.csv",
"chars": 11761,
"preview": "Starting training ...\n0;1.8535430385606575\r\n1;1.8356075893156927\r\n2;1.8279383734311008\r\n3;1.822314586682491\r\n4;1.8186759"
},
{
"path": "data/logs_data/Caenorhabditis_elegans.WBcel235.dna.chromosome.I/biGRU_16bit.log.csv",
"chars": 5915,
"preview": "Starting training ...\n0;1.8831766127114613\r\n1;1.8687406848715946\r\n2;1.8622829920086283\r\n3;1.8584328716285075\r\nStarting C"
},
{
"path": "data/logs_data/Caenorhabditis_elegans.WBcel235.dna.chromosome.I/biLSTM.log.csv",
"chars": 2664,
"preview": "Starting training ...\nStarting Compression ...\nTraceback (most recent call last):\n File \"compressor.py\", line 20, in <m"
},
{
"path": "data/logs_data/Caenorhabditis_elegans.WBcel235.dna.chromosome.I/biLSTM_16bit.log.csv",
"chars": 5921,
"preview": "Starting training ...\n0;1.884507402209729\r\n1;1.871122544627692\r\n2;1.8635762954132413\r\n3;1.8597909297718935\r\nStarting Com"
},
{
"path": "data/logs_data/Caenorhabditis_elegans.WBcel235.dna.toplevel/FC.log.csv",
"chars": 10157,
"preview": "Starting training ...\nStarting training ...\nStarting Compression ...\n/opt/conda/lib/python3.6/site-packages/sklearn/exte"
},
{
"path": "data/logs_data/Caenorhabditis_elegans.WBcel235.dna.toplevel/FC_16bit.log.csv",
"chars": 6255,
"preview": "Starting training ...\n0;1.8531200441459896\r\n1;1.8334496114291368\r\nStarting training ...\nStarting training ...\n0;1.853119"
},
{
"path": "data/logs_data/Caenorhabditis_elegans.WBcel235.dna.toplevel/FC_4layer.log.csv",
"chars": 12838,
"preview": "Starting training ...\nStarting Compression ...\nStarting training ...\n0;1.858583542632005\r\n1;1.845372233931501\r\n2;1.84221"
},
{
"path": "data/logs_data/Caenorhabditis_elegans.WBcel235.dna.toplevel/FC_4layer_16bit.log.csv",
"chars": 6450,
"preview": "Starting training ...\n0;1.8707464513293255\r\n1;1.85496668309976\r\n2;1.8488777234262521\r\n3;1.8450854979546383\r\nStarting Com"
},
{
"path": "data/logs_data/Caenorhabditis_elegans.WBcel235.dna.toplevel/GRU_multi_16bit.log.csv",
"chars": 65,
"preview": "Starting training ...\n0;1.861744895370654\r\n1;1.8492561831508143\r\n"
},
{
"path": "data/logs_data/Caenorhabditis_elegans.WBcel235.dna.toplevel/LSTM_multi.log.csv",
"chars": 11726,
"preview": "Starting training ...\n0;1.8425407592824365\r\n1;1.8289623391471852\r\n2;1.8244587229496791\r\n3;1.821987229708892\r\n4;1.8205238"
},
{
"path": "data/logs_data/Caenorhabditis_elegans.WBcel235.dna.toplevel/LSTM_multi_16bit.log.csv",
"chars": 65,
"preview": "Starting training ...\n0;1.864678990290116\r\n1;1.8516583165013785\r\n"
},
{
"path": "data/logs_data/Caenorhabditis_elegans.WBcel235.dna.toplevel/LSTM_multi_big.log.csv",
"chars": 6006,
"preview": "Starting training ...\n0;1.8324452448726254\r\n1;1.8125994139707988\r\n2;1.8064765404868577\r\n3;1.8032488849143944\r\n4;1.801176"
},
{
"path": "data/logs_data/Caenorhabditis_elegans.WBcel235.dna.toplevel/LSTM_multi_bn.log.csv",
"chars": 5995,
"preview": "Starting training ...\n0;1.8537912338063736\r\n1;1.839781109242452\r\n2;1.832267580159184\r\n3;1.8280653073082063\r\n4;1.82556061"
},
{
"path": "data/logs_data/Caenorhabditis_elegans.WBcel235.dna.toplevel/biGRU.log.csv",
"chars": 11681,
"preview": "Starting training ...\n0;1.8510626867487314\r\n1;1.8420502934112857\r\n2;1.8370909764280168\r\n3;1.8394651972889053\r\n4;1.833924"
},
{
"path": "data/logs_data/Caenorhabditis_elegans.WBcel235.dna.toplevel/biGRU_16bit.log.csv",
"chars": 66,
"preview": "Starting training ...\n0;1.8746874920227703\r\n1;1.8621605389575995\r\n"
},
{
"path": "data/logs_data/Caenorhabditis_elegans.WBcel235.dna.toplevel/biLSTM.log.csv",
"chars": 2589,
"preview": "Starting training ...\nStarting Compression ...\nTraceback (most recent call last):\n File \"compressor.py\", line 20, in <m"
},
{
"path": "data/logs_data/Caenorhabditis_elegans.WBcel235.dna.toplevel/biLSTM_16bit.log.csv",
"chars": 6011,
"preview": "Starting training ...\n0;1.8748117672914433\r\n1;1.86222685965181\r\nStarting training ...\nStarting training ...\nStarting tra"
},
{
"path": "data/logs_data/HMM20/FC.log.csv",
"chars": 9425,
"preview": "Starting training ...\nStarting Compression ...\n/opt/conda/lib/python3.6/site-packages/sklearn/externals/joblib/externals"
},
{
"path": "data/logs_data/HMM20/FC_16bit.log.csv",
"chars": 5545,
"preview": "Starting training ...\n0;0.49607352370551716\r\n1;0.4695703525025602\r\n2;0.4694496197756656\r\nStarting Compression ...\n2018-1"
},
{
"path": "data/logs_data/HMM20/FC_4layer.log.csv",
"chars": 12119,
"preview": "Starting training ...\n0;0.5206543763203921\r\n1;0.46947664414926854\r\n2;0.46939708875865127\r\n3;0.46935344092734826\r\n4;0.469"
},
{
"path": "data/logs_data/HMM20/FC_4layer_16bit.log.csv",
"chars": 6192,
"preview": "Starting training ...\nStarting Compression ...\nStarting training ...\n0;1.0000884556611622\r\n1;0.984773925531234\r\n2;0.4704"
},
{
"path": "data/logs_data/HMM20/GRU_multi_16bit.log.csv",
"chars": 5621,
"preview": "Starting training ...\nStarting training ...\n0;0.5523903529825909\r\n1;0.4696679837509601\r\n2;0.46954282574084744\r\nStarting "
},
{
"path": "data/logs_data/HMM20/LSTM_multi.log.csv",
"chars": 11050,
"preview": "Starting training ...\n0;1.000013687796185\r\n1;1.0000089983114877\r\n2;1.000010707509011\r\n3;1.0000123418055005\r\nStarting Com"
},
{
"path": "data/logs_data/HMM20/LSTM_multi_16bit.log.csv",
"chars": 5602,
"preview": "Starting training ...\nStarting training ...\n0;1.000026151673707\r\n1;1.000015600998464\r\nStarting Compression ...\n2018-10-2"
},
{
"path": "data/logs_data/HMM20/LSTM_multi_big.log.csv",
"chars": 5607,
"preview": "Starting training ...\n0;1.00001315772686\r\n1;1.00001010098826\r\n2;1.0000103882930247\r\n3;1.000009094411388\r\nStarting Compre"
},
{
"path": "data/logs_data/HMM20/LSTM_multi_bn.log.csv",
"chars": 5625,
"preview": "Starting training ...\n0;0.4756669068810965\r\n1;0.4706430645945386\r\n2;0.4702950187180426\r\n3;0.4700393999456722\r\n4;0.469893"
},
{
"path": "data/logs_data/HMM20/biGRU.log.csv",
"chars": 11003,
"preview": "Starting training ...\n0;0.5960491823924241\r\n1;0.4706341725676351\r\n2;0.482574030843931\r\n3;0.5054787438372375\r\n4;0.5583331"
},
{
"path": "data/logs_data/HMM20/biGRU_16bit.log.csv",
"chars": 5569,
"preview": "Starting training ...\nStarting training ...\n0;1.0000025501632104\r\n1;0.9997669850870455\r\nStarting Compression ...\n2018-10"
},
{
"path": "data/logs_data/HMM20/biLSTM.log.csv",
"chars": 2316,
"preview": "Starting training ...\nStarting Compression ...\nTraceback (most recent call last):\n File \"compressor.py\", line 20, in <m"
},
{
"path": "data/logs_data/HMM20/biLSTM_16bit.log.csv",
"chars": 5578,
"preview": "Starting training ...\n0;0.9995691724270354\r\n1;0.7126344586053508\r\n2;0.8080376394089221\r\nStarting Compression ...\n2018-10"
},
{
"path": "data/logs_data/HMM30/FC.log.csv",
"chars": 9425,
"preview": "Starting training ...\nStarting Compression ...\n/opt/conda/lib/python3.6/site-packages/sklearn/externals/joblib/externals"
},
{
"path": "data/logs_data/HMM30/FC_16bit.log.csv",
"chars": 5542,
"preview": "Starting training ...\n0;0.4998670164890553\r\n1;0.46918367775537634\r\n2;0.468984615015361\r\nStarting Compression ...\n2018-10"
},
{
"path": "data/logs_data/HMM30/FC_4layer.log.csv",
"chars": 12117,
"preview": "Starting training ...\n0;0.5290087672208922\r\n1;0.4693196839420721\r\n2;0.4692171646061764\r\n3;0.4691908233237767\r\n4;0.469156"
},
{
"path": "data/logs_data/HMM30/FC_4layer_16bit.log.csv",
"chars": 6103,
"preview": "Starting training ...\n0;1.0000824552771377\r\n1;1.0000214513728878\r\nStarting Compression ...\n2018-10-29 10:55:08.888757: I"
},
{
"path": "data/logs_data/HMM30/GRU_multi_16bit.log.csv",
"chars": 5595,
"preview": "Starting training ...\n0;0.5676752312147978\r\n1;0.46948819724462365\r\n2;0.4693528385816692\r\nStarting Compression ...\n2018-1"
},
{
"path": "data/logs_data/HMM30/LSTM_multi.log.csv",
"chars": 11076,
"preview": "Starting training ...\n0;0.49182937683292494\r\n1;0.4699388378745644\r\n2;0.4696089611075441\r\n3;0.4694874036086259\r\n4;0.46937"
},
{
"path": "data/logs_data/HMM30/LSTM_multi_16bit.log.csv",
"chars": 5582,
"preview": "Starting training ...\n0;1.0000211513536867\r\n1;1.0000137508800564\r\nStarting Compression ...\n2018-10-29 11:22:00.571225: I"
},
{
"path": "data/logs_data/HMM30/LSTM_multi_big.log.csv",
"chars": 5610,
"preview": "Starting training ...\n0;1.0000175472472914\r\n1;1.0000126723259215\r\n2;1.0000108203458224\r\n3;1.0000106666433586\r\nStarting C"
},
{
"path": "data/logs_data/HMM30/LSTM_multi_bn.log.csv",
"chars": 5603,
"preview": "Starting training ...\n0;0.47389605231050963\r\n1;0.4710082288532587\r\n2;0.470860654740456\r\n3;0.4707683183799846\r\nStarting C"
},
{
"path": "data/logs_data/HMM30/biGRU.log.csv",
"chars": 11033,
"preview": "Starting training ...\n0;0.776526146831112\r\n1;0.7696331377097783\r\n2;0.5227730464733874\r\n3;0.5839291948205193\r\n4;0.7407844"
},
{
"path": "data/logs_data/HMM30/biGRU_16bit.log.csv",
"chars": 5570,
"preview": "Starting training ...\n0;0.8761661746351767\r\n1;0.7725776949724782\r\n2;0.7707406774033538\r\nStarting Compression ...\n2018-10"
},
{
"path": "data/logs_data/HMM30/biLSTM.log.csv",
"chars": 1192,
"preview": "Starting training ...\nStarting Compression ...\nTraceback (most recent call last):\n File \"compressor.py\", line 20, in <m"
},
{
"path": "data/logs_data/HMM30/biLSTM_16bit.log.csv",
"chars": 5575,
"preview": "Starting training ...\n0;0.9059881082389273\r\n1;0.7517127596166154\r\n2;0.7765888016833077\r\nStarting Compression ...\n2018-10"
},
{
"path": "data/logs_data/HMM40/FC.log.csv",
"chars": 9422,
"preview": "Starting training ...\nStarting Compression ...\n/opt/conda/lib/python3.6/site-packages/sklearn/externals/joblib/externals"
},
{
"path": "data/logs_data/HMM40/FC_16bit.log.csv",
"chars": 5541,
"preview": "Starting training ...\n0;0.5028831845238095\r\n1;0.4693246617783538\r\n2;0.4690156920042883\r\nStarting Compression ...\n2018-10"
},
{
"path": "data/logs_data/HMM40/FC_4layer.log.csv",
"chars": 12118,
"preview": "Starting training ...\n0;0.5183540801390342\r\n1;0.4694903368804617\r\n2;0.4693202993043313\r\n3;0.4692550255833561\r\n4;0.469224"
},
{
"path": "data/logs_data/HMM40/FC_4layer_16bit.log.csv",
"chars": 6100,
"preview": "Starting training ...\n0;1.000094756064388\r\n1;1.0000402025729647\r\nStarting Compression ...\n2018-10-29 11:37:48.711800: I "
},
{
"path": "data/logs_data/HMM40/GRU_multi_16bit.log.csv",
"chars": 5602,
"preview": "Starting training ...\n0;0.6231239549331157\r\n1;0.4694698710717486\r\n2;0.46931143593189967\r\nStarting Compression ...\n2018-1"
},
{
"path": "data/logs_data/HMM40/LSTM_multi.log.csv",
"chars": 11050,
"preview": "Starting training ...\n0;1.0000136060831917\r\n1;1.0000112324449506\r\n2;1.000012710969752\r\n3;1.0000108262116765\r\nStarting Co"
},
{
"path": "data/logs_data/HMM40/LSTM_multi_16bit.log.csv",
"chars": 5583,
"preview": "Starting training ...\n0;1.00001950124808\r\n1;1.0000187011968766\r\nStarting Compression ...\n2018-10-29 12:33:40.464650: I t"
},
{
"path": "data/logs_data/HMM40/LSTM_multi_big.log.csv",
"chars": 5610,
"preview": "Starting training ...\n0;1.0000129820686454\r\n1;1.0000128428995823\r\n2;1.0000117231929113\r\n3;1.0000125869811038\r\nStarting C"
},
{
"path": "data/logs_data/HMM40/LSTM_multi_bn.log.csv",
"chars": 5606,
"preview": "Starting training ...\n0;0.47358501369977507\r\n1;0.4699970272912187\r\n2;0.4698441571889202\r\n3;0.46976832461769263\r\nStarting"
},
{
"path": "data/logs_data/HMM40/biGRU.log.csv",
"chars": 10988,
"preview": "Starting training ...\n0;1.000016368247275\r\n1;1.0000103806753495\r\n2;1.0000124411709908\r\n3;1.0000111292462073\r\nStarting Co"
},
{
"path": "data/logs_data/HMM40/biGRU_16bit.log.csv",
"chars": 5547,
"preview": "Starting training ...\n0;1.0000146009344597\r\n1;1.0000121507776498\r\nStarting Compression ...\n2018-10-29 10:58:58.363705: I"
},
{
"path": "data/logs_data/HMM40/biLSTM.log.csv",
"chars": 47,
"preview": "Starting training ...\nStarting Compression ...\n"
},
{
"path": "data/logs_data/HMM40/biLSTM_16bit.log.csv",
"chars": 5553,
"preview": "Starting training ...\n0;1.0000216013824885\r\n1;1.0000155009920635\r\nStarting Compression ...\n2018-10-29 23:02:40.642579: I"
},
{
"path": "data/logs_data/PhiX_quality_truncated/FC_4layer.log.csv",
"chars": 12398,
"preview": "Starting training ...\n0;0.33429752559353565\r\n1;0.3336336923760257\r\n2;0.33356332301298824\r\n3;0.3335271539933936\r\nStarting"
},
{
"path": "data/logs_data/PhiX_quality_truncated/LSTM_multi.log.csv",
"chars": 11354,
"preview": "Starting training ...\n0;0.3337507236302155\r\n1;0.33321501932241115\r\n2;0.33314008111833177\r\n3;0.3330975011083565\r\nStarting"
},
{
"path": "data/logs_data/PhiX_quality_truncated/LSTM_multi_big.log.csv",
"chars": 5763,
"preview": "Starting training ...\n0;0.33369317233206636\r\n1;0.3332065819161678\r\n2;0.3331315379565077\r\n3;0.3330908102999072\r\nStarting "
},
{
"path": "data/logs_data/PhiX_quality_truncated/LSTM_multi_bn.log.csv",
"chars": 5756,
"preview": "Starting training ...\n0;0.33447196934482387\r\n1;0.33426611789032834\r\n2;0.334267698853093\r\n3;0.3345627919291478\r\nStarting "
},
{
"path": "data/logs_data/PhiX_quality_truncated/biGRU.log.csv",
"chars": 11296,
"preview": "Starting training ...\n0;0.3337752777290237\r\n1;0.3332614900971991\r\n2;0.3332106166715224\r\n3;0.33327236742835925\r\nStarting "
},
{
"path": "data/logs_data/PhiX_quality_truncated/biLSTM.log.csv",
"chars": 1222,
"preview": "Starting training ...\nStarting Compression ...\nTraceback (most recent call last):\n File \"decompressor.py\", line 20, in "
},
{
"path": "data/logs_data/chr1/FC.log.csv",
"chars": 9458,
"preview": "Starting training ...\nStarting Compression ...\n/opt/conda/lib/python3.6/site-packages/sklearn/externals/joblib/externals"
},
{
"path": "data/logs_data/chr1/FC_16bit.log.csv",
"chars": 5669,
"preview": "Starting training ...\n0;0.016896728242806316\r\n1;0.0\r\n2;0.0\r\nStarting Compression ...\n2018-10-30 01:19:03.467068: I tenso"
},
{
"path": "data/logs_data/chr1/FC_4layer.log.csv",
"chars": 12235,
"preview": "Starting training ...\n0;1.71422546483615\r\n1;1.7955617113637006\r\n2;1.7130647289410526\r\n3;1.6987289322510413\r\n4;1.71226811"
},
{
"path": "data/logs_data/chr1/FC_4layer_16bit.log.csv",
"chars": 1329,
"preview": "Starting training ...\n0;0.0057734170546469305\r\n1;0.0\r\n2;0.0\r\nStarting Compression ...\n2018-10-29 05:38:08.461689: I tens"
},
{
"path": "data/logs_data/chr1/GRU_multi_16bit.log.csv",
"chars": 5760,
"preview": "Starting training ...\n0;0.0045171351637477\r\n1;0.0\r\nStarting Compression ...\n2018-10-29 15:16:40.123252: I tensorflow/cor"
},
{
"path": "data/logs_data/chr1/LSTM_multi.log.csv",
"chars": 11237,
"preview": "Starting training ...\n0;1.5671854364777358\r\n1;1.6096698664290887\r\n2;1.5427103538166793\r\n3;1.540201668594196\r\n4;1.5385663"
},
{
"path": "data/logs_data/chr1/LSTM_multi_16bit.log.csv",
"chars": 5734,
"preview": "Starting training ...\n0;0.007698080646743328\r\n1;0.0\r\n2;0.0\r\nStarting Compression ...\n2018-10-29 16:15:57.173090: I tenso"
},
{
"path": "data/logs_data/chr1/LSTM_multi_big.log.csv",
"chars": 5736,
"preview": "Starting training ...\n0;1.6411013710955649\r\n1;1.8252086851838651\r\n2;1.6004674659692613\r\n3;1.7404844960203807\r\n4;1.567616"
},
{
"path": "data/logs_data/chr1/LSTM_multi_bn.log.csv",
"chars": 5640,
"preview": "Starting training ...\n0;1.5841035655786653\r\n1;1.5857022070768279\r\n2;1.631261938929875\r\n3;1.8869552158110248\r\nStarting Co"
},
{
"path": "data/logs_data/chr1/biGRU.log.csv",
"chars": 11100,
"preview": "Starting training ...\n0;1.58247096313415\r\n1;1.5741776533506642\r\n2;1.566340977890106\r\n3;1.5636961103134086\r\n4;1.564431221"
},
{
"path": "data/logs_data/chr1/biGRU_16bit.log.csv",
"chars": 5759,
"preview": "Starting training ...\n0;0.01320680741029465\r\n1;0.0\r\n2;0.0\r\nStarting Compression ...\n2018-10-29 17:47:43.380986: I tensor"
},
{
"path": "data/logs_data/chr1/biLSTM.log.csv",
"chars": 2309,
"preview": "Starting training ...\nStarting Compression ...\nTraceback (most recent call last):\n File \"compressor.py\", line 20, in <m"
},
{
"path": "data/logs_data/chr1/biLSTM_16bit.log.csv",
"chars": 5769,
"preview": "Starting training ...\n0;0.028554166255833825\r\n1;0.0\r\n2;0.0\r\nStarting Compression ...\n2018-10-30 05:29:04.172258: I tenso"
},
{
"path": "data/logs_data/enwiki8/biGRU.log.csv",
"chars": 94,
"preview": "Starting training ...\nStarting Compression ...\nStarting training ...\nStarting Compression ...\n"
},
{
"path": "data/logs_data/enwiki8/biGRU_big.log.csv",
"chars": 188,
"preview": "Starting training ...\nStarting Compression ...\nStarting training ...\nStarting Compression ...\nStarting training ...\nStar"
},
{
"path": "data/logs_data/iid_p10.1/FC.log.csv",
"chars": 9483,
"preview": "Starting training ...\nStarting Compression ...\n/opt/conda/lib/python3.6/site-packages/sklearn/externals/joblib/externals"
},
{
"path": "data/logs_data/iid_p10.1/FC_16bit.log.csv",
"chars": 5548,
"preview": "Starting training ...\n0;0.4704091061827957\r\n1;0.4690157670090886\r\nStarting Compression ...\n2018-10-30 03:52:13.845610: I"
},
{
"path": "data/logs_data/iid_p10.1/FC_4layer.log.csv",
"chars": 12151,
"preview": "Starting training ...\n0;0.4696338391889991\r\n1;0.4692169183959609\r\n2;0.469177567943572\r\n3;0.46915968940676755\r\nStarting C"
},
{
"path": "data/logs_data/iid_p10.1/GRU_multi_16bit.log.csv",
"chars": 5604,
"preview": "Starting training ...\n0;0.47037652909786226\r\n1;0.4692453066996288\r\nStarting Compression ...\n2018-10-29 18:00:12.936838: "
},
{
"path": "data/logs_data/iid_p10.1/LSTM_multi.log.csv",
"chars": 11108,
"preview": "Starting training ...\n0;0.46980964793831764\r\n1;0.46927706639521316\r\n2;0.4692209158533363\r\n3;0.4691922749761307\r\nStarting"
},
{
"path": "data/logs_data/iid_p10.1/LSTM_multi_16bit.log.csv",
"chars": 5608,
"preview": "Starting training ...\n0;0.47043108258928573\r\n1;0.4691687768017153\r\nStarting Compression ...\n2018-10-29 18:42:00.231933: "
},
{
"path": "data/logs_data/iid_p10.1/LSTM_multi_big.log.csv",
"chars": 5641,
"preview": "Starting training ...\n0;0.4697486549448003\r\n1;0.46930010659964466\r\n2;0.46922838875897044\r\n3;0.46918719130727005\r\nStartin"
},
{
"path": "data/logs_data/iid_p10.1/LSTM_multi_bn.log.csv",
"chars": 5631,
"preview": "Starting training ...\n0;0.469567464646363\r\n1;0.46917668986104427\r\n2;0.4691024468687647\r\n3;0.4690955660925957\r\nStarting C"
},
{
"path": "data/logs_data/iid_p10.1/biGRU.log.csv",
"chars": 11042,
"preview": "Starting training ...\n0;0.4699281246156736\r\n1;0.4692201917011556\r\n2;0.46916555529121734\r\n3;0.46915306179754196\r\nStarting"
},
{
"path": "data/logs_data/iid_p10.1/biGRU_16bit.log.csv",
"chars": 5576,
"preview": "Starting training ...\n0;0.4710825992863543\r\n1;0.46914617535522274\r\nStarting Compression ...\n2018-10-29 20:21:52.422322: "
},
{
"path": "data/logs_data/iid_p10.1/biLSTM.log.csv",
"chars": 2344,
"preview": "Starting training ...\nStarting Compression ...\nTraceback (most recent call last):\n File \"compressor.py\", line 20, in <m"
},
{
"path": "data/logs_data/iid_p10.1/biLSTM_16bit.log.csv",
"chars": 5583,
"preview": "Starting training ...\n0;0.47082928307411676\r\n1;0.46904236871159755\r\nStarting Compression ...\n2018-10-30 07:48:52.594876:"
},
{
"path": "data/logs_data/text8/FC.log.csv",
"chars": 9864,
"preview": "Starting training ...\nStarting Compression ...\n/opt/conda/lib/python3.6/site-packages/sklearn/externals/joblib/externals"
},
{
"path": "data/logs_data/text8/FC_16bit.log.csv",
"chars": 5935,
"preview": "Starting training ...\n0;0.03012367711661342\r\n1;0.0\r\n2;0.0\r\nStarting Compression ...\n2018-10-30 04:34:05.591935: I tensor"
},
{
"path": "data/logs_data/text8/FC_4layer.log.csv",
"chars": 12823,
"preview": "Starting training ...\n0;2.226315931338999\r\n1;2.0728917023156486\r\n2;2.048167354794024\r\n3;2.036439787587614\r\n4;2.028744723"
},
{
"path": "data/logs_data/text8/FC_4layer_16bit.log.csv",
"chars": 6524,
"preview": "Starting training ...\n0;0.031987361887646434\r\n1;0.0\r\n2;0.0\r\nStarting Compression ...\n2018-10-29 12:17:38.877480: I tenso"
},
{
"path": "data/logs_data/text8/FC_4layer_big.log.csv",
"chars": 6574,
"preview": "Starting training ...\n0;2.1198943053303005\r\n1;2.0061218149321003\r\n2;1.9888308240900152\r\n3;1.9798172401802794\r\n4;1.974061"
},
{
"path": "data/logs_data/text8/GRU_multi_16bit.log.csv",
"chars": 5979,
"preview": "Starting training ...\n0;0.08812394559730073\r\n1;0.0\r\n2;0.0\r\nStarting Compression ...\n2018-10-29 19:36:32.303370: I tensor"
},
{
"path": "data/logs_data/text8/GRU_multi_big.log.csv",
"chars": 10576,
"preview": "Starting training ...\n0;1.9707224067974083\r\n1;1.8396828959566218\r\n2;1.8191544647124316\r\n3;1.8082756844103616\r\n4;1.800963"
},
{
"path": "data/logs_data/text8/LSTM_multi.log.csv",
"chars": 17882,
"preview": "Starting training ...\n0;2.300029672582798\r\n1;2.1527100171852025\r\n2;2.1271184731479007\r\n3;2.113904257442067\r\n4;2.10652419"
},
{
"path": "data/logs_data/text8/LSTM_multi_16bit.log.csv",
"chars": 5987,
"preview": "Starting training ...\n0;0.29399519262769314\r\n1;0.0\r\n2;0.0\r\nStarting Compression ...\n2018-10-29 20:14:31.501134: I tensor"
},
{
"path": "data/logs_data/text8/LSTM_multi_big.log.csv",
"chars": 6032,
"preview": "Starting training ...\n0;2.0769283636346088\r\n1;1.9484522664826265\r\n2;1.9282821885374124\r\n3;1.9180559781957163\r\n4;1.911798"
},
{
"path": "data/logs_data/text8/LSTM_multi_bn.log.csv",
"chars": 6000,
"preview": "Starting training ...\n0;2.330049675131431\r\n1;2.2059011771661776\r\n2;2.1841704408551994\r\n3;2.176580099171178\r\n4;2.17204194"
},
{
"path": "data/logs_data/text8/biGRU.log.csv",
"chars": 11691,
"preview": "Starting training ...\n0;2.1764909385904843\r\n1;2.068344904263492\r\n2;2.0541044350820608\r\n3;2.0484872857595318\r\n4;2.0443637"
},
{
"path": "data/logs_data/text8/biGRU_16bit.log.csv",
"chars": 5951,
"preview": "Starting training ...\n0;0.05060930955983247\r\n1;0.0\r\n2;0.0\r\nStarting Compression ...\n2018-10-29 23:07:09.972258: I tensor"
},
{
"path": "data/logs_data/text8/biGRU_big.log.csv",
"chars": 10853,
"preview": "Starting training ...\nStarting Compression ...\nStarting training ...\nStarting Compression ...\n/opt/conda/lib/python3.6/s"
},
{
"path": "data/logs_data/text8/biLSTM.log.csv",
"chars": 47,
"preview": "Starting training ...\nStarting Compression ...\n"
},
{
"path": "data/logs_data/text8/biLSTM_16bit.log.csv",
"chars": 5956,
"preview": "Starting training ...\n0;0.1459623736636766\r\n1;0.0\r\n2;0.0\r\nStarting Compression ...\n2018-10-30 10:21:33.834837: I tensorf"
},
{
"path": "data/logs_data/xor20/FC.log.csv",
"chars": 9439,
"preview": "Starting training ...\nStarting Compression ...\n/opt/conda/lib/python3.6/site-packages/sklearn/externals/joblib/externals"
},
{
"path": "data/logs_data/xor20/FC_16bit.log.csv",
"chars": 5549,
"preview": "Starting training ...\n0;0.041277736159879855\r\n1;0.0010007636094178777\r\n2;0.0009989800480043223\r\nStarting Compression ..."
},
{
"path": "data/logs_data/xor20/FC_4layer.log.csv",
"chars": 12133,
"preview": "Starting training ...\n0;0.018246311346332138\r\n1;1.7198272298286297e-07\r\n2;1.7198266277773655e-07\r\n3;1.7198266277773655e-"
},
{
"path": "data/logs_data/xor20/FC_4layer_16bit.log.csv",
"chars": 6286,
"preview": "Starting training ...\nStarting Compression ...\nStarting training ...\n0;1.0000863555267536\r\n1;0.22300494575402827\r\n2;0.0\r"
},
{
"path": "data/logs_data/xor20/GRU_multi_16bit.log.csv",
"chars": 5603,
"preview": "Starting training ...\n0;0.10868840259096894\r\n1;0.0010387586680547555\r\n2;0.0010323217326533899\r\nStarting Compression ...\n"
},
{
"path": "data/logs_data/xor20/LSTM_multi.log.csv",
"chars": 11091,
"preview": "Starting training ...\n0;0.016886118211529803\r\n1;1.7203467462387224e-07\r\n2;1.7198266277773655e-07\r\n3;1.7198266277773655e-"
},
{
"path": "data/logs_data/xor20/LSTM_multi_16bit.log.csv",
"chars": 5722,
"preview": "Starting training ...\n0;1.0000175011200716\r\n1;0.4362981418310772\r\n2;0.0\r\n3;0.0\r\nStarting Compression ...\n2018-10-29 22:0"
},
{
"path": "data/logs_data/xor20/LSTM_multi_big.log.csv",
"chars": 5610,
"preview": "Starting training ...\n0;1.0000135426575016\r\n1;1.0000102798815453\r\n2;1.0000109489734395\r\n3;1.0000107786012074\r\nStarting C"
},
{
"path": "data/logs_data/xor20/LSTM_multi_bn.log.csv",
"chars": 5617,
"preview": "Starting training ...\n0;0.0015613959993281469\r\n1;1.719826799761377e-07\r\n2;1.7198266277773655e-07\r\n3;1.7198268857552018e-"
},
{
"path": "data/logs_data/xor20/biGRU.log.csv",
"chars": 11023,
"preview": "Starting training ...\n0;0.022873598301844697\r\n1;1.7913012036734895e-07\r\n2;1.7198267997920125e-07\r\n3;1.7198266277773655e-"
},
{
"path": "data/logs_data/xor20/biGRU_16bit.log.csv",
"chars": 5573,
"preview": "Starting training ...\n0;0.5792750583693607\r\n1;0.0038085089789496526\r\n2;0.0036673263104463685\r\nStarting Compression ...\n2"
},
{
"path": "data/logs_data/xor20/biLSTM.log.csv",
"chars": 47,
"preview": "Starting training ...\nStarting Compression ...\n"
},
{
"path": "data/logs_data/xor20/biLSTM_16bit.log.csv",
"chars": 5665,
"preview": "Starting training ...\n0;0.6723293603290611\r\n1;0.0\r\n2;0.0\r\nStarting Compression ...\n2018-10-30 11:44:17.812628: I tensorf"
},
{
"path": "data/logs_data/xor30/FC.log.csv",
"chars": 9437,
"preview": "Starting training ...\nStarting Compression ...\n/opt/conda/lib/python3.6/site-packages/sklearn/externals/joblib/externals"
},
{
"path": "data/logs_data/xor30/FC_16bit.log.csv",
"chars": 5548,
"preview": "Starting training ...\n0;0.04355192858319495\r\n1;0.0008670016306824887\r\n2;0.0008656339650268867\r\nStarting Compression ...\n"
},
{
"path": "data/logs_data/xor30/FC_4layer.log.csv",
"chars": 12132,
"preview": "Starting training ...\n0;0.03381478681891726\r\n1;1.7198852782814014e-07\r\n2;1.719827057813983e-07\r\n3;1.7198266277773655e-07"
},
{
"path": "data/logs_data/xor30/FC_4layer_16bit.log.csv",
"chars": 6241,
"preview": "Starting training ...\n0;1.000076904921915\r\n1;0.5061664891377267\r\n2;0.0\r\n3;0.0\r\nStarting Compression ...\n2018-10-29 14:21"
},
{
"path": "data/logs_data/xor30/GRU_multi_16bit.log.csv",
"chars": 5603,
"preview": "Starting training ...\n0;0.12589466819626455\r\n1;0.0011801290438838085\r\n2;0.0011747178577241443\r\nStarting Compression ...\n"
},
{
"path": "data/logs_data/xor30/LSTM_multi.log.csv",
"chars": 11045,
"preview": "Starting training ...\n0;1.0000126726311167\r\n1;1.00001100807146\r\n2;1.0000124210281\r\n3;1.000011655286954\r\nStarting Compres"
},
{
"path": "data/logs_data/xor30/LSTM_multi_16bit.log.csv",
"chars": 5583,
"preview": "Starting training ...\n0;1.0000172011008706\r\n1;0.9999901493695597\r\nStarting Compression ...\n2018-10-29 23:14:35.903234: I"
},
{
"path": "data/logs_data/xor30/LSTM_multi_big.log.csv",
"chars": 5609,
"preview": "Starting training ...\n0;1.000019684053969\r\n1;1.0000094082254174\r\n2;1.0000104219865873\r\n3;1.0000098192197386\r\nStarting Co"
},
{
"path": "data/logs_data/xor30/LSTM_multi_bn.log.csv",
"chars": 5617,
"preview": "Starting training ...\n0;0.0014657820195903865\r\n1;1.7198266277773655e-07\r\n2;1.719826799761377e-07\r\n3;1.719827229725044e-0"
},
{
"path": "data/logs_data/xor30/biGRU.log.csv",
"chars": 11023,
"preview": "Starting training ...\n0;0.012753050503213794\r\n1;1.8074275168515242e-07\r\n2;1.7198266277773655e-07\r\n3;1.7198266277773655e-"
},
{
"path": "data/logs_data/xor30/biGRU_16bit.log.csv",
"chars": 5661,
"preview": "Starting training ...\n0;0.17971443297371031\r\n1;0.0\r\n2;0.0\r\nStarting Compression ...\n2018-10-30 01:07:34.618146: I tensor"
},
{
"path": "data/logs_data/xor30/biLSTM.log.csv",
"chars": 47,
"preview": "Starting training ...\nStarting Compression ...\n"
},
{
"path": "data/logs_data/xor30/biLSTM_16bit.log.csv",
"chars": 5668,
"preview": "Starting training ...\n0;0.2737435320860535\r\n1;0.0\r\n2;0.0\r\nStarting Compression ...\n2018-10-30 12:08:03.698171: I tensorf"
},
{
"path": "data/logs_data/xor40/FC.log.csv",
"chars": 9436,
"preview": "Starting training ...\nStarting Compression ...\n/opt/conda/lib/python3.6/site-packages/sklearn/externals/joblib/externals"
},
{
"path": "data/logs_data/xor40/FC_16bit.log.csv",
"chars": 5550,
"preview": "Starting training ...\n0;0.05366660399859341\r\n1;0.0009371868842574675\r\n2;0.0009347095772723205\r\nStarting Compression ...\n"
},
{
"path": "data/logs_data/xor40/FC_4layer.log.csv",
"chars": 12131,
"preview": "Starting training ...\n0;0.12524736477739803\r\n1;1.728408833055262e-07\r\n2;1.7198923303289176e-07\r\n3;1.7198271437630948e-07"
},
{
"path": "data/logs_data/xor40/FC_4layer_16bit.log.csv",
"chars": 6100,
"preview": "Starting training ...\n0;1.0000892057091655\r\n1;1.000047853062596\r\nStarting Compression ...\n2018-10-29 15:03:05.047907: I "
},
{
"path": "data/logs_data/xor40/GRU_multi_16bit.log.csv",
"chars": 5604,
"preview": "Starting training ...\n0;0.25286852069469945\r\n1;0.0010862777248993936\r\n2;0.0010822032453827045\r\nStarting Compression ...\n"
},
{
"path": "data/logs_data/xor40/LSTM_multi.log.csv",
"chars": 11052,
"preview": "Starting training ...\n0;1.0000131996362807\r\n1;1.0000100576627333\r\n2;1.0000094986608927\r\n3;1.0000092380363026\r\nStarting C"
},
{
"path": "data/logs_data/xor40/LSTM_multi_16bit.log.csv",
"chars": 5578,
"preview": "Starting training ...\n0;1.000020001280082\r\n1;1.000017351110471\r\nStarting Compression ...\n2018-10-30 00:25:58.731990: I t"
},
{
"path": "data/logs_data/xor40/LSTM_multi_big.log.csv",
"chars": 5611,
"preview": "Starting training ...\n0;1.0000135109965398\r\n1;1.000011825268536\r\n2;1.0000109409529065\r\n3;1.0000104852657836\r\nStarting Co"
},
{
"path": "data/logs_data/xor40/LSTM_multi_bn.log.csv",
"chars": 5644,
"preview": "Starting training ...\n0;0.011730144454812005\r\n1;1.7198269717472077e-07\r\n2;1.7198266277773655e-07\r\n3;1.7198266277773655e-"
},
{
"path": "data/logs_data/xor40/biGRU.log.csv",
"chars": 20007,
"preview": "Starting training ...\n0;1.000018176767561\r\n1;1.0000107277251487\r\n2;1.000012644089251\r\n3;1.0000122649329048\r\nStarting Com"
},
{
"path": "data/logs_data/xor40/biGRU_16bit.log.csv",
"chars": 5548,
"preview": "Starting training ...\n0;1.0000189012096774\r\n1;1.0000133008512544\r\nStarting Compression ...\n2018-10-30 01:44:23.521557: I"
},
{
"path": "data/logs_data/xor40/biLSTM.log.csv",
"chars": 47,
"preview": "Starting training ...\nStarting Compression ...\n"
},
{
"path": "data/logs_data/xor40/biLSTM_16bit.log.csv",
"chars": 5552,
"preview": "Starting training ...\n0;1.000022201420891\r\n1;1.000017851142473\r\nStarting Compression ...\n2018-10-30 12:56:13.871602: I t"
},
{
"path": "data/logs_data/xor50/FC.log.csv",
"chars": 9436,
"preview": "Starting training ...\nStarting Compression ...\n/opt/conda/lib/python3.6/site-packages/sklearn/externals/joblib/externals"
},
{
"path": "data/logs_data/xor50/FC_16bit.log.csv",
"chars": 5548,
"preview": "Starting training ...\n0;0.04853435331775296\r\n1;0.0009752840978697637\r\n2;0.0009723167692888595\r\nStarting Compression ...\n"
},
{
"path": "data/logs_data/xor50/FC_4layer.log.csv",
"chars": 12132,
"preview": "Starting training ...\n0;0.015219433896944596\r\n1;1.720120225694806e-07\r\n2;1.7198284337856298e-07\r\n3;1.7198267997920125e-0"
},
{
"path": "data/logs_data/xor50/FC_4layer_16bit.log.csv",
"chars": 6100,
"preview": "Starting training ...\n0;1.0000826052867384\r\n1;1.000032702092934\r\nStarting Compression ...\n2018-10-29 15:45:57.402164: I "
},
{
"path": "data/logs_data/xor50/GRU_multi_16bit.log.csv",
"chars": 5602,
"preview": "Starting training ...\n0;0.213055293058287\r\n1;0.0010369441964598234\r\n2;0.0010327030558862017\r\nStarting Compression ...\n20"
},
{
"path": "data/logs_data/xor50/LSTM_multi.log.csv",
"chars": 11089,
"preview": "Starting training ...\n0;0.03880091667349089\r\n1;1.720315618311042e-07\r\n2;1.719826885799336e-07\r\n3;1.7198266277773655e-07\r"
},
{
"path": "data/logs_data/xor50/LSTM_multi_16bit.log.csv",
"chars": 5580,
"preview": "Starting training ...\n0;1.0000234014976959\r\n1;1.0000152509760625\r\nStarting Compression ...\n2018-10-30 01:37:08.550214: I"
},
{
"path": "data/logs_data/xor50/LSTM_multi_big.log.csv",
"chars": 5610,
"preview": "Starting training ...\n0;1.0000178906774742\r\n1;1.0000103527499784\r\n2;1.000012205926443\r\n3;1.0000102601964476\r\nStarting Co"
},
{
"path": "data/logs_data/xor50/LSTM_multi_bn.log.csv",
"chars": 5619,
"preview": "Starting training ...\n0;0.0037584235243626305\r\n1;1.7198266277773655e-07\r\n2;1.7198266277773655e-07\r\n3;1.7198266277773655e"
},
{
"path": "data/logs_data/xor50/biGRU.log.csv",
"chars": 11016,
"preview": "Starting training ...\n0;0.12421181548021015\r\n1;4.367358134975195e-07\r\n2;0.004918541578190283\r\n3;8.69948780852435e-07\r\n4;"
},
{
"path": "data/logs_data/xor50/biGRU_16bit.log.csv",
"chars": 5546,
"preview": "Starting training ...\n0;1.0000152009728622\r\n1;1.0000126008064516\r\nStarting Compression ...\n2018-10-30 02:22:22.776504: I"
},
{
"path": "data/logs_data/xor50/biLSTM.log.csv",
"chars": 47,
"preview": "Starting training ...\nStarting Compression ...\n"
},
{
"path": "data/logs_data/xor50/biLSTM_16bit.log.csv",
"chars": 5666,
"preview": "Starting training ...\n0;0.5460377589165707\r\n1;0.0\r\n2;0.0\r\nStarting Compression ...\n2018-10-30 13:27:16.355378: I tensorf"
},
{
"path": "data/logs_data/xor60/FC_4layer.log.csv",
"chars": 4826,
"preview": "Starting Compression ...\n/opt/conda/lib/python3.6/site-packages/sklearn/externals/joblib/externals/cloudpickle/cloudpick"
},
{
"path": "data/logs_data/xor60/LSTM_multi.log.csv",
"chars": 4592,
"preview": "Starting training ...\nStarting Compression ...\n/opt/conda/lib/python3.6/site-packages/sklearn/externals/joblib/externals"
},
{
"path": "data/logs_data/xor60/biGRU.log.csv",
"chars": 5560,
"preview": "Starting training ...\n0;0.0015483571382668311\r\n1;1.7198266277773655e-07\r\n2;1.7198266277773655e-07\r\n3;1.7198266277773655e"
},
{
"path": "data/parse_new.py",
"chars": 1109,
"preview": "import sys\nimport numpy as np\nimport json\nimport argparse\n\nparser = argparse.ArgumentParser(description='Input')\nparser."
},
{
"path": "data/parse_wiki.py",
"chars": 1190,
"preview": "import io\nimport sys\nimport numpy as np\nimport json\nimport argparse\n\nparser = argparse.ArgumentParser(description='Input"
},
{
"path": "data/run_fasta_preprocess.sh",
"chars": 330,
"preview": "#/bin/bash\nfasta_dir=\"fasta_files\"\ndata_dir=\"files_to_be_compressed\"\n\n\nfor f in $fasta_dir/*.fa\ndo\n echo \"filename: \""
},
{
"path": "data/run_parser.sh",
"chars": 419,
"preview": "#bin/bash\ndata_dir=\"files_to_be_compressed\"\nprocessed_dir=\"processed_files\"\n\nmkdir -p $processed_dir\nfor f in $data_dir/"
},
{
"path": "data/xor20.txt",
"chars": 10000000,
"preview": "abbaaababbaababaababbabbbbaabaaabbaaabbabbababbbaaaabaaaabaabaabbabaaaaabbbbbaaabbbabbaaaaaabababbbbabaabaaaaaaabbaababa"
},
{
"path": "data/xor30.txt",
"chars": 10000000,
"preview": "bbaaabaaabaaaabbbaaabaaaababbbabbbbaaaabbbbbabaaaabbbbbaababbababbbbbababaabbbbbababaaabbabbaabababbaabbbababaabbaaaabaa"
},
{
"path": "data/xor40.txt",
"chars": 10000000,
"preview": "abbbaaaabbbabbaaaaaaaaaabbbabbbaabaabbbaababbbbbabaabaaaaaaaaaaababbabaaabbbabaaabbababaabbbaaaaaaaaaaaabbabbaaaababbaaa"
},
{
"path": "data/xor50.txt",
"chars": 10000000,
"preview": "aabbabaababaaabababbaaabaaaaabaabbbbabaabbbaabbbbbbbabbaaabbaaaabbaabaaaabbbbbbaaababaabbbabaaababababbabbbbabbbbbabbbaa"
},
{
"path": "data/xor60.txt",
"chars": 10000000,
"preview": "babbabaaabbbabbaaababbbbaabaaabaababbaaabbabbbbabbabbbaabbbabbabbaaaababbabbbbaababaaabbbbaaabbabbbbabbababbabbabaaababb"
},
{
"path": "docker/Dockerfile",
"chars": 1691,
"preview": "ARG cuda_version=9.0\nARG cudnn_version=7\nFROM nvidia/cuda:${cuda_version}-cudnn${cudnn_version}-devel\n\n# Install system "
},
{
"path": "docker/Dockerfile.bak",
"chars": 1864,
"preview": "ARG cuda_version=9.0\nARG cudnn_version=7\nFROM nvidia/cuda:${cuda_version}-cudnn${cudnn_version}-devel\n\n# Install system "
},
{
"path": "docker/Makefile",
"chars": 891,
"preview": "help:\n\t@cat Makefile\n\nDATA?=\"${HOME}/Data\"\nGPU?=0\nDOCKER_FILE=Dockerfile\nDOCKER=GPU=$(GPU) nvidia-docker\nBACKEND=tensorf"
},
{
"path": "docker/README.md",
"chars": 1725,
"preview": "# Using Keras via Docker\n\nThis directory contains `Dockerfile` to make it easy to get up and running with\nKeras via [Doc"
},
{
"path": "docker/theanorc",
"chars": 56,
"preview": "[global]\nfloatX = float32\noptimizer=None\ndevice = cuda\n\n"
},
{
"path": "install.sh",
"chars": 242,
"preview": "pip install --upgrade pip\npip install \\\n tensorflow-gpu==1.8\npip install tqdm\npip install \\\n keras==2.2.2 \\\n "
},
{
"path": "src/README.md",
"chars": 465,
"preview": "# DNA_Compression\n\n## Description\nDNA_compression using neural networks\n\nThis folder contains the final models. \n\nImport"
},
{
"path": "src/arithmeticcoding_fast.py",
"chars": 21885,
"preview": "# \n# Reference arithmetic coding\n# Copyright (c) Project Nayuki\n# \n# https://www.nayuki.io/page/reference-arithmetic-cod"
},
{
"path": "src/compressor.py",
"chars": 8269,
"preview": "# \n# Compression application using adaptive arithmetic coding\n# \n# Usage: python adaptive-arithmetic-compress.py InputFi"
},
{
"path": "src/decompressor.py",
"chars": 10401,
"preview": "# \n# Compression application using adaptive arithmetic coding\n# \n# Usage: python adaptive-arithmetic-compress.py InputFi"
},
{
"path": "src/models.py",
"chars": 9973,
"preview": "from sklearn.metrics import mean_squared_error\nfrom sklearn.preprocessing import MinMaxScaler\nfrom keras.models import S"
},
{
"path": "src/run_experiments.sh",
"chars": 1762,
"preview": "#/bin/bash\nmodel_dir=\"../data/trained_models\"\ncompressed_dir=\"../data/compressed\"\ndata_dir=\"../data/processed_files\"\nlog"
},
{
"path": "src/trainer.py",
"chars": 3563,
"preview": "from sklearn.metrics import mean_squared_error\nfrom sklearn.preprocessing import MinMaxScaler\nfrom keras.models import S"
}
]
// ... and 153 more files (download for full content)
About this extraction
This page contains the full source code of the mohit1997/DeepZip GitHub repository, extracted and formatted as plain text for AI agents and large language models (LLMs). The extraction includes 349 files (48.9 MB), approximately 12.8M tokens, and a symbol index with 59 extracted functions, classes, methods, constants, and types. Use this with OpenClaw, Claude, ChatGPT, Cursor, Windsurf, or any other AI tool that accepts text input. You can copy the full output to your clipboard or download it as a .txt file.
Extracted by GitExtract — free GitHub repo to text converter for AI. Built by Nikandr Surkov.