Repository: echonet/dynamic Branch: master Commit: 9ab3a59d0686 Files: 33 Total size: 146.8 KB Directory structure: gitextract_yq2r2uit/ ├── .gitignore ├── .travis.yml ├── LICENSE ├── LICENSE.txt ├── README.md ├── docs/ │ ├── google662250efb9603afb.html │ ├── header.html │ ├── index.html │ ├── lagunita/ │ │ ├── css/ │ │ │ └── custom.css │ │ └── js/ │ │ ├── base.js │ │ ├── custom.js │ │ └── modernizr.custom.17475.js │ └── lagunita.html ├── echonet/ │ ├── __init__.py │ ├── __main__.py │ ├── __version__.py │ ├── config.py │ ├── datasets/ │ │ ├── __init__.py │ │ └── echo.py │ └── utils/ │ ├── __init__.py │ ├── segmentation.py │ └── video.py ├── example.cfg ├── requirements.txt ├── scripts/ │ ├── ConvertDICOMToAVI.ipynb │ ├── InitializationNotebook.ipynb │ ├── beat_by_beat_analysis.R │ ├── plot_complexity.py │ ├── plot_hyperparameter_sweep.py │ ├── plot_loss.py │ ├── plot_simulated_noise.py │ └── run_experiments.sh └── setup.py ================================================ FILE CONTENTS ================================================ ================================================ FILE: .gitignore ================================================ .ipynb_checkpoints/ __pycache__/ *.swp echonet.cfg .echonet.cfg *.pyc echonet.egg-info/ ================================================ FILE: .travis.yml ================================================ language: minimal os: - linux env: # - PYTHON_VERSION=3.6 PYTORCH_VERSION=1.1 TORCHVISION_VERSION=0.2 (torchvision 0.2 does not have VisionDataset) # - PYTHON_VERSION=3.6 PYTORCH_VERSION=1.1 TORCHVISION_VERSION=0.3 (torchvision 0.3 has a cuda issue) - PYTHON_VERSION=3.6 PYTORCH_VERSION=1.1 TORCHVISION_VERSION=0.4 - PYTHON_VERSION=3.6 PYTORCH_VERSION=1.1 TORCHVISION_VERSION=0.5 # - PYTHON_VERSION=3.6 PYTORCH_VERSION=1.2 TORCHVISION_VERSION=0.2 # - PYTHON_VERSION=3.6 PYTORCH_VERSION=1.2 TORCHVISION_VERSION=0.3 - PYTHON_VERSION=3.6 PYTORCH_VERSION=1.2 TORCHVISION_VERSION=0.4 - PYTHON_VERSION=3.6 PYTORCH_VERSION=1.2 TORCHVISION_VERSION=0.5 # - PYTHON_VERSION=3.6 PYTORCH_VERSION=1.3 TORCHVISION_VERSION=0.2 # - PYTHON_VERSION=3.6 PYTORCH_VERSION=1.3 TORCHVISION_VERSION=0.3 - PYTHON_VERSION=3.6 PYTORCH_VERSION=1.3 TORCHVISION_VERSION=0.4 - PYTHON_VERSION=3.6 PYTORCH_VERSION=1.3 TORCHVISION_VERSION=0.5 # - PYTHON_VERSION=3.6 PYTORCH_VERSION=1.4 TORCHVISION_VERSION=0.2 # - PYTHON_VERSION=3.6 PYTORCH_VERSION=1.4 TORCHVISION_VERSION=0.3 - PYTHON_VERSION=3.6 PYTORCH_VERSION=1.4 TORCHVISION_VERSION=0.4 - PYTHON_VERSION=3.6 PYTORCH_VERSION=1.4 TORCHVISION_VERSION=0.5 # - PYTHON_VERSION=3.7 PYTORCH_VERSION=1.1 TORCHVISION_VERSION=0.2 # - PYTHON_VERSION=3.7 PYTORCH_VERSION=1.1 TORCHVISION_VERSION=0.3 - PYTHON_VERSION=3.7 PYTORCH_VERSION=1.1 TORCHVISION_VERSION=0.4 - PYTHON_VERSION=3.7 PYTORCH_VERSION=1.1 TORCHVISION_VERSION=0.5 # - PYTHON_VERSION=3.7 PYTORCH_VERSION=1.2 TORCHVISION_VERSION=0.2 # - PYTHON_VERSION=3.7 PYTORCH_VERSION=1.2 TORCHVISION_VERSION=0.3 - PYTHON_VERSION=3.7 PYTORCH_VERSION=1.2 TORCHVISION_VERSION=0.4 - PYTHON_VERSION=3.7 PYTORCH_VERSION=1.2 TORCHVISION_VERSION=0.5 # - PYTHON_VERSION=3.7 PYTORCH_VERSION=1.3 TORCHVISION_VERSION=0.2 # - PYTHON_VERSION=3.7 PYTORCH_VERSION=1.3 TORCHVISION_VERSION=0.3 - PYTHON_VERSION=3.7 PYTORCH_VERSION=1.3 TORCHVISION_VERSION=0.4 - PYTHON_VERSION=3.7 PYTORCH_VERSION=1.3 TORCHVISION_VERSION=0.5 # - PYTHON_VERSION=3.7 PYTORCH_VERSION=1.4 TORCHVISION_VERSION=0.2 # - PYTHON_VERSION=3.7 PYTORCH_VERSION=1.4 TORCHVISION_VERSION=0.3 - PYTHON_VERSION=3.7 PYTORCH_VERSION=1.4 TORCHVISION_VERSION=0.4 - PYTHON_VERSION=3.7 PYTORCH_VERSION=1.4 TORCHVISION_VERSION=0.5 install: - if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then MINICONDA_OS=Linux; sudo apt-get update; else MINICONDA_OS=MacOSX; brew update; fi - wget https://repo.anaconda.com/miniconda/Miniconda3-latest-${MINICONDA_OS}-x86_64.sh -O miniconda.sh - bash miniconda.sh -b -p $HOME/miniconda - source "$HOME/miniconda/etc/profile.d/conda.sh" - hash -r - conda config --set always_yes yes --set changeps1 no - conda update -q conda # Useful for debugging any issues with conda - conda info -a - conda search pytorch || true - conda create -q -n test-environment python=${PYTHON_VERSION} pytorch=${PYTORCH_VERSION} - conda activate test-environment - pip install -q torchvision==${TORCHVISION_VERSION} "pillow<7.0.0" - pip install -q . - pip install -q flake8 pylint script: - flake8 --ignore=E501 - pylint --disable=C0103,C0301,R0401,R0801,R0902,R0912,R0913,R0914,R0915 --extension-pkg-whitelist=cv2,torch --generated-members=torch.* echonet/ scripts/*.py setup.py - python -c "import echonet" ================================================ FILE: LICENSE ================================================ MIT License Copyright (c) 2020 the authors Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ================================================ FILE: LICENSE.txt ================================================ Copyright Notice The authors are the proprietor of certain copyrights of and to EchoNet-Dynamic software, source code and associated material. Code also contains source code created by certain third parties. Redistribution and use of the Code with or without modification is not permitted without explicit written permission by the authors. Copyright 2019 The authors. All rights reserved. ================================================ FILE: README.md ================================================ EchoNet-Dynamic:
Interpretable AI for beat-to-beat cardiac function assessment ------------------------------------------------------------------------------ EchoNet-Dynamic is a end-to-end beat-to-beat deep learning model for 1) semantic segmentation of the left ventricle 2) prediction of ejection fraction by entire video or subsampled clips, and 3) assessment of cardiomyopathy with reduced ejection fraction. For more details, see the accompanying paper, > [**Video-based AI for beat-to-beat assessment of cardiac function**](https://www.nature.com/articles/s41586-020-2145-8)
David Ouyang, Bryan He, Amirata Ghorbani, Neal Yuan, Joseph Ebinger, Curt P. Langlotz, Paul A. Heidenreich, Robert A. Harrington, David H. Liang, Euan A. Ashley, and James Y. Zou. Nature, March 25, 2020. https://doi.org/10.1038/s41586-020-2145-8 Dataset ------- We share a deidentified set of 10,030 echocardiogram images which were used for training EchoNet-Dynamic. Preprocessing of these images, including deidentification and conversion from DICOM format to AVI format videos, were performed with OpenCV and pydicom. Additional information is at https://echonet.github.io/dynamic/. These deidentified images are shared with a non-commerical data use agreement. Examples -------- We show examples of our semantic segmentation for nine distinct patients below. Three patients have normal cardiac function, three have low ejection fractions, and three have arrhythmia. No human tracings for these patients were used by EchoNet-Dynamic. | Normal | Low Ejection Fraction | Arrhythmia | | ------ | --------------------- | ---------- | | ![](docs/media/0X10A28877E97DF540.gif) | ![](docs/media/0X129133A90A61A59D.gif) | ![](docs/media/0X132C1E8DBB715D1D.gif) | | ![](docs/media/0X1167650B8BEFF863.gif) | ![](docs/media/0X13CE2039E2D706A.gif ) | ![](docs/media/0X18BA5512BE5D6FFA.gif) | | ![](docs/media/0X148FFCBF4D0C398F.gif) | ![](docs/media/0X16FC9AA0AD5D8136.gif) | ![](docs/media/0X1E12EEE43FD913E5.gif) | Installation ------------ First, clone this repository and enter the directory by running: git clone https://github.com/echonet/dynamic.git cd dynamic EchoNet-Dynamic is implemented for Python 3, and depends on the following packages: - NumPy - PyTorch - Torchvision - OpenCV - skimage - sklearn - tqdm Echonet-Dynamic and its dependencies can be installed by navigating to the cloned directory and running pip install --user . Usage ----- ### Preprocessing DICOM Videos The input of EchoNet-Dynamic is an apical-4-chamber view echocardiogram video of any length. The easiest way to run our code is to use videos from our dataset, but we also provide a Jupyter Notebook, `ConvertDICOMToAVI.ipynb`, to convert DICOM files to AVI files used for input to EchoNet-Dynamic. The Notebook deidentifies the video by cropping out information outside of the ultrasound sector, resizes the input video, and saves the video in AVI format. ### Setting Path to Data By default, EchoNet-Dynamic assumes that a copy of the data is saved in a folder named `a4c-video-dir/` in this directory. This path can be changed by creating a configuration file named `echonet.cfg` (an example configuration file is `example.cfg`). ### Running Code EchoNet-Dynamic has three main components: segmenting the left ventricle, predicting ejection fraction from subsampled clips, and assessing cardiomyopathy with beat-by-beat predictions. Each of these components can be run with reasonable choices of hyperparameters with the scripts below. We describe our full hyperparameter sweep in the next section. #### Frame-by-frame Semantic Segmentation of the Left Ventricle echonet segmentation --save_video This creates a directory named `output/segmentation/deeplabv3_resnet50_random/`, which will contain - log.csv: training and validation losses - best.pt: checkpoint of weights for the model with the lowest validation loss - size.csv: estimated size of left ventricle for each frame and indicator for beginning of beat - videos: directory containing videos with segmentation overlay #### Prediction of Ejection Fraction from Subsampled Clips echonet video This creates a directory named `output/video/r2plus1d_18_32_2_pretrained/`, which will contain - log.csv: training and validation losses - best.pt: checkpoint of weights for the model with the lowest validation loss - test_predictions.csv: ejection fraction prediction for subsampled clips #### Beat-by-beat Prediction of Ejection Fraction from Full Video and Assesment of Cardiomyopathy The final beat-by-beat prediction and analysis is performed with `scripts/beat_analysis.R`. This script combines the results from segmentation output in `size.csv` and the clip-level ejection fraction prediction in `test_predictions.csv`. The beginning of each systolic phase is detected by using the peak detection algorithm from scipy (`scipy.signal.find_peaks`) and a video clip centered around the beat is used for beat-by-beat prediction. ### Hyperparameter Sweeps The full set of hyperparameter sweeps from the paper can be run via `run_experiments.sh`. In particular, we choose between pretrained and random initialization for the weights, the model (selected from `r2plus1d_18`, `r3d_18`, and `mc3_18`), the length of the video (1, 4, 8, 16, 32, 64, and 96 frames), and the sampling period (1, 2, 4, 6, and 8 frames). ================================================ FILE: docs/google662250efb9603afb.html ================================================ google-site-verification: google662250efb9603afb.html ================================================ FILE: docs/header.html ================================================
Stanford University
================================================ FILE: docs/index.html ================================================ EchoNet Dynamic
Stanford University

Introduction

Echocardiography, or cardiac ultrasound, is the most widely used and readily available imaging modality to assess cardiac function and structure. Combining portable instrumentation, rapid image acquisition, high temporal resolution, and without the risks of ionizing radiation, echocardiography is one of the most frequently utilized imaging studies in the United States and serves as the backbone of cardiovascular imaging. For diseases ranging from heart failure to valvular heart diseases, echocardiography is both necessary and sufficient to diagnose many cardiovascular diseases. In addition to our deep learning model, we introduce a new large video dataset of echocardiograms for computer vision research. The EchoNet-Dynamic database includes 10,030 labeled echocardiogram videos and human expert annotations (measurements, tracings, and calculations) to provide a baseline to study cardiac motion and chamber sizes.



Motivation

Machine learning analysis of biomedical images has seen significant recent advances. In contrast, there has been much less work on medical videos, despite the fact that videos are routinely used in many clinical settings. A major bottleneck for this is the the lack of openly available and well annotated medical video data. Computer vision has benefited greatly from open databases which allow for collaboration, comparison, and creation of task specific architectures. We present the EchoNet-Dynamic dataset of 10,030 echocardiography videos, spanning the range of typical echocardiography lab imaging acquisition conditions, with corresponding labeled measurements including ejection fraction, left ventricular volume at end-systole and end-diastole, and human expert tracings of the left ventricle as an aid for studying machine learning approaches to evaluate cardiac function. We additionally present the performance of our model with 3-dimensional convolutional neural network architecture for video classification. This model is used semantically segment the left ventricle and to assess ejection fraction to expert human performance and as a benchmark for further collaboration, comparison, and creation of task-specific architectures. To the best of our knowledge, this is the largest labeled medical video dataset made available publicly to researchers and medical professionals and first public report of video-based 3D convolutional architectures to assess cardiac function.



Dataset

Echocardiogram Videos: A standard full resting echocardiogram study consists of a series of videos and images visualizing the heart from different angles, positions, and image acquisition techniques. The dataset contains 10,030 apical-4-chamber echocardiography videos from individuals who underwent imaging between 2016 and 2018 as part of routine clinical care at Stanford University Hospital. Each video was cropped and masked to remove text and information outside of the scanning sector. The resulting images were then downsampled by cubic interpolation into standardized 112x112 pixel videos.


Measurements: In addition to the video itself, each study is linked to clinical measurements and calculations obtained by a registered sonographer and verified by a level 3 echocardiographer in the standard clinical workflow. A central metric of cardiac function is the left ventricular ejection fraction, which is used to diagnose cardiomyopathy, assess eligibility for certain chemotherapies, and determine indication for medical devices. The ejection fraction is expressed as a percentage and is the ratio of left ventricular end systolic volume (ESV) and left ventricular end diastolic volume (EDV) determined by (EDV - ESV) / EDV.


Tracings: In our dataset, for each video, the left ventricle is traced at the endocardial border at two separate time points representing end-systole and end-diastole. Each tracing is used to estimate ventricular volume by integration of ventricular area over the length of the major axis of the ventricle. The expert tracings are represented by a collection of paired coordinates corresponding to each human tracing. The first pair of coordinates represent the length and direction of the long axis of the left ventricle, and subsequent coordinate pairs represent short axis linear distances starting from the apex of the heart to the mitral apparatus. Each coordinate pair is also listed with a video file name and frame number to identify the representative frame from which the tracings match.

Further description of the dataset is available in our NeurIPS Machine Learning 4 Health workshop paper.


Code

Our code is available here.


Accessing Dataset


Stanford University School of Medicine EchoNet-Dynamic Dataset Research Use Agreement

By registering for downloads from the EchoNet-Dynamic Dataset, you are agreeing to this Research Use Agreement, as well as to the Terms of Use of the Stanford University School of Medicine website as posted and updated periodically at http://www.stanford.edu/site/terms/.

1. Permission is granted to view and use the EchoNet-Dynamic Dataset without charge for personal, non-commercial research purposes only. Any commercial use, sale, or other monetization is prohibited.

2. Other than the rights granted herein, the Stanford University School of Medicine (“School of Medicine”) retains all rights, title, and interest in the EchoNet-Dynamic Dataset.

3. You may make a verbatim copy of the EchoNet-Dynamic Dataset for personal, non-commercial research use as permitted in this Research Use Agreement. If another user within your organization wishes to use the EchoNet-Dynamic Dataset, they must register as an individual user and comply with all the terms of this Research Use Agreement.

4. YOU MAY NOT DISTRIBUTE, PUBLISH, OR REPRODUCE A COPY of any portion or all of the EchoNet-Dynamic Dataset to others without specific prior written permission from the School of Medicine.

5. YOU MAY NOT SHARE THE DOWNLOAD LINK to the EchoNet-Dynamic dataset to others. If another user within your organization wishes to use the EchoNet-Dynamic Dataset, they must register as an individual user and comply with all the terms of this Research Use Agreement.

6. You must not modify, reverse engineer, decompile, or create derivative works from the EchoNet-Dynamic Dataset. You must not remove or alter any copyright or other proprietary notices in the EchoNet-Dynamic Dataset.

7. The EchoNet-Dynamic Dataset has not been reviewed or approved by the Food and Drug Administration, and is for non-clinical, Research Use Only. In no event shall data or images generated through the use of the EchoNet-Dynamic Dataset be used or relied upon in the diagnosis or provision of patient care.

8. THE ECHONET-DYNAMIC DATASET IS PROVIDED "AS IS," AND STANFORD UNIVERSITY AND ITS COLLABORATORS DO NOT MAKE ANY WARRANTY, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, NOR DO THEY ASSUME ANY LIABILITY OR RESPONSIBILITY FOR THE USE OF THIS ECHONET-DYNAMIC DATASET.

9. You will not make any attempt to re-identify any of the individual data subjects. Re-identification of individuals is strictly prohibited. Any re-identification of any individual data subject shall be immediately reported to the School of Medicine.

10. Any violation of this Research Use Agreement or other impermissible use shall be grounds for immediate termination of use of this EchoNet-Dynamic Dataset. In the event that the School of Medicine determines that the recipient has violated this Research Use Agreement or other impermissible use has been made, the School of Medicine may direct that the undersigned data recipient immediately return all copies of the EchoNet-Dynamic Dataset and retain no copies thereof even if you did not cause the violation or impermissible use.

In consideration for your agreement to the terms and conditions contained here, Stanford grants you permission to view and use the EchoNet-Dynamic Dataset for personal, non-commercial research. You may not otherwise copy, reproduce, retransmit, distribute, publish, commercially exploit or otherwise transfer any material.

Limitation of Use

You may use EchoNet-Dynamic Dataset for legal purposes only.

You agree to indemnify and hold Stanford harmless from any claims, losses or damages, including legal fees, arising out of or resulting from your use of the EchoNet-Dynamic Dataset or your violation or role in violation of these Terms. You agree to fully cooperate in Stanford’s defense against any such claims. These Terms shall be governed by and interpreted in accordance with the laws of California.

Access the dataset via the Stanford Artificial Intelligence in Medicine and Imaging (AIMI) Center Shared Datasets Portal.



Paper

Video-based AI for beat-to-beat assessment of cardiac function.
David Ouyang, Bryan He, Amirata Ghorbani, Neal Yuan, Joseph Ebinger, Curt P. Langlotz, Paul A. Heidenreich, Robert A. Harrington, David H. Liang, Euan A. Ashley, and James Y. Zou. Nature (2020)

For inquiries, contact us at ouyangd@stanford.edu.

================================================ FILE: docs/lagunita/css/custom.css ================================================ .center { display: block; margin-left: auto; margin-right: auto; max-width: 100%; } ================================================ FILE: docs/lagunita/js/base.js ================================================ // JavaScript Document /*** Utility functions available across the site ***/ var LAGUNITA = { size: function(){ // return window size based on visibility as calculated by CSS var size = 'lg'; // if we don't already have the size-detect div's, add them if ( $('.size-detect-xs').length == 0 ) { $('body') .append('
') .append('
') .append('
') .append('
'); } $(['xs', 'sm', 'md', 'lg']).each(function(i, sz) { if ($('.size-detect-'+sz).css('display') != 'none') { size = sz; } }); return size; }, stickFooter: function(){ // adjust css to make footer sticky var h = $('#footer').height() + 'px'; $('#su-content').css('padding-bottom', h); $('#footer').css('margin-top', '-'+h); } }; $(document).ready(function() { LAGUNITA.stickFooter(); // note resize events and trigger resizeEnd event when resizing stops $(window).resize(function() { if(this.resizeTO) clearTimeout(this.resizeTO); this.resizeTO = setTimeout(function() { $(this).trigger('resizeEnd'); }, 200); }); // Call responsive funtion when browser window resizing is complete $(window).bind('resizeEnd', function() { // show or hide the hamburger based on window size var size = LAGUNITA.size(); // what size is our window (xs, sm, md or lg) if (size == 'md' || size == 'lg') { // if size is md or lg, unhide search and gateway blocks $('.navbar-collapse').collapse('hide'); // $('.navbar-collapse').removeClass('in').addClass('collapse'); } // re-stick the footer in case its height has changed LAGUNITA.stickFooter(); }); $('.navbar-collapse').collapse({toggle: false}); // activate collapsibility without toggling state $('#skip > a').click(function(e){ var href = $(this).attr('href').substr(1); // remove the # var target = $('a[name="' + href + '"]'); target.focus(); }); }); ================================================ FILE: docs/lagunita/js/custom.js ================================================ // Custom Javascript for Lagunita HTML Theme // You can add custom JS functions to this file. jQuery(document).ready(function($){ // Add "active" to nav element if matches body class of "subnav-n" var bodyClass=""; var matches = document.body.className.match(/(^|\s+)(subnav-\d+)(\s|$)/); if (matches) { // found the bodyClass bodyClass = matches[2]; } if (bodyClass.length > 0){ var target = $( '#' + bodyClass ); target.addClass('active'); } }); ================================================ FILE: docs/lagunita/js/modernizr.custom.17475.js ================================================ /* Modernizr 2.6.2 (Custom Build) | MIT & BSD * Build: http://modernizr.com/download/#-csstransforms-csstransitions-touch-shiv-cssclasses-prefixed-teststyles-testprop-testallprops-prefixes-domprefixes-load */ ;window.Modernizr=function(a,b,c){function z(a){j.cssText=a}function A(a,b){return z(m.join(a+";")+(b||""))}function B(a,b){return typeof a===b}function C(a,b){return!!~(""+a).indexOf(b)}function D(a,b){for(var d in a){var e=a[d];if(!C(e,"-")&&j[e]!==c)return b=="pfx"?e:!0}return!1}function E(a,b,d){for(var e in a){var f=b[a[e]];if(f!==c)return d===!1?a[e]:B(f,"function")?f.bind(d||b):f}return!1}function F(a,b,c){var d=a.charAt(0).toUpperCase()+a.slice(1),e=(a+" "+o.join(d+" ")+d).split(" ");return B(b,"string")||B(b,"undefined")?D(e,b):(e=(a+" "+p.join(d+" ")+d).split(" "),E(e,b,c))}var d="2.6.2",e={},f=!0,g=b.documentElement,h="modernizr",i=b.createElement(h),j=i.style,k,l={}.toString,m=" -webkit- -moz- -o- -ms- ".split(" "),n="Webkit Moz O ms",o=n.split(" "),p=n.toLowerCase().split(" "),q={},r={},s={},t=[],u=t.slice,v,w=function(a,c,d,e){var f,i,j,k,l=b.createElement("div"),m=b.body,n=m||b.createElement("body");if(parseInt(d,10))while(d--)j=b.createElement("div"),j.id=e?e[d]:h+(d+1),l.appendChild(j);return f=["­",'"].join(""),l.id=h,(m?l:n).innerHTML+=f,n.appendChild(l),m||(n.style.background="",n.style.overflow="hidden",k=g.style.overflow,g.style.overflow="hidden",g.appendChild(n)),i=c(l,a),m?l.parentNode.removeChild(l):(n.parentNode.removeChild(n),g.style.overflow=k),!!i},x={}.hasOwnProperty,y;!B(x,"undefined")&&!B(x.call,"undefined")?y=function(a,b){return x.call(a,b)}:y=function(a,b){return b in a&&B(a.constructor.prototype[b],"undefined")},Function.prototype.bind||(Function.prototype.bind=function(b){var c=this;if(typeof c!="function")throw new TypeError;var d=u.call(arguments,1),e=function(){if(this instanceof e){var a=function(){};a.prototype=c.prototype;var f=new a,g=c.apply(f,d.concat(u.call(arguments)));return Object(g)===g?g:f}return c.apply(b,d.concat(u.call(arguments)))};return e}),q.touch=function(){var c;return"ontouchstart"in a||a.DocumentTouch&&b instanceof DocumentTouch?c=!0:w(["@media (",m.join("touch-enabled),("),h,")","{#modernizr{top:9px;position:absolute}}"].join(""),function(a){c=a.offsetTop===9}),c},q.csstransforms=function(){return!!F("transform")},q.csstransitions=function(){return F("transition")};for(var G in q)y(q,G)&&(v=G.toLowerCase(),e[v]=q[G](),t.push((e[v]?"":"no-")+v));return e.addTest=function(a,b){if(typeof a=="object")for(var d in a)y(a,d)&&e.addTest(d,a[d]);else{a=a.toLowerCase();if(e[a]!==c)return e;b=typeof b=="function"?b():b,typeof f!="undefined"&&f&&(g.className+=" "+(b?"":"no-")+a),e[a]=b}return e},z(""),i=k=null,function(a,b){function k(a,b){var c=a.createElement("p"),d=a.getElementsByTagName("head")[0]||a.documentElement;return c.innerHTML="x",d.insertBefore(c.lastChild,d.firstChild)}function l(){var a=r.elements;return typeof a=="string"?a.split(" "):a}function m(a){var b=i[a[g]];return b||(b={},h++,a[g]=h,i[h]=b),b}function n(a,c,f){c||(c=b);if(j)return c.createElement(a);f||(f=m(c));var g;return f.cache[a]?g=f.cache[a].cloneNode():e.test(a)?g=(f.cache[a]=f.createElem(a)).cloneNode():g=f.createElem(a),g.canHaveChildren&&!d.test(a)?f.frag.appendChild(g):g}function o(a,c){a||(a=b);if(j)return a.createDocumentFragment();c=c||m(a);var d=c.frag.cloneNode(),e=0,f=l(),g=f.length;for(;e",f="hidden"in a,j=a.childNodes.length==1||function(){b.createElement("a");var a=b.createDocumentFragment();return typeof a.cloneNode=="undefined"||typeof a.createDocumentFragment=="undefined"||typeof a.createElement=="undefined"}()}catch(c){f=!0,j=!0}})();var r={elements:c.elements||"abbr article aside audio bdi canvas data datalist details figcaption figure footer header hgroup mark meter nav output progress section summary time video",shivCSS:c.shivCSS!==!1,supportsUnknownElements:j,shivMethods:c.shivMethods!==!1,type:"default",shivDocument:q,createElement:n,createDocumentFragment:o};a.html5=r,q(b)}(this,b),e._version=d,e._prefixes=m,e._domPrefixes=p,e._cssomPrefixes=o,e.testProp=function(a){return D([a])},e.testAllProps=F,e.testStyles=w,e.prefixed=function(a,b,c){return b?F(a,b,c):F(a,"pfx")},g.className=g.className.replace(/(^|\s)no-js(\s|$)/,"$1$2")+(f?" js "+t.join(" "):""),e}(this,this.document),function(a,b,c){function d(a){return"[object Function]"==o.call(a)}function e(a){return"string"==typeof a}function f(){}function g(a){return!a||"loaded"==a||"complete"==a||"uninitialized"==a}function h(){var a=p.shift();q=1,a?a.t?m(function(){("c"==a.t?B.injectCss:B.injectJs)(a.s,0,a.a,a.x,a.e,1)},0):(a(),h()):q=0}function i(a,c,d,e,f,i,j){function k(b){if(!o&&g(l.readyState)&&(u.r=o=1,!q&&h(),l.onload=l.onreadystatechange=null,b)){"img"!=a&&m(function(){t.removeChild(l)},50);for(var d in y[c])y[c].hasOwnProperty(d)&&y[c][d].onload()}}var j=j||B.errorTimeout,l=b.createElement(a),o=0,r=0,u={t:d,s:c,e:f,a:i,x:j};1===y[c]&&(r=1,y[c]=[]),"object"==a?l.data=c:(l.src=c,l.type=a),l.width=l.height="0",l.onerror=l.onload=l.onreadystatechange=function(){k.call(this,r)},p.splice(e,0,u),"img"!=a&&(r||2===y[c]?(t.insertBefore(l,s?null:n),m(k,j)):y[c].push(l))}function j(a,b,c,d,f){return q=0,b=b||"j",e(a)?i("c"==b?v:u,a,b,this.i++,c,d,f):(p.splice(this.i++,0,a),1==p.length&&h()),this}function k(){var a=B;return a.loader={load:j,i:0},a}var l=b.documentElement,m=a.setTimeout,n=b.getElementsByTagName("script")[0],o={}.toString,p=[],q=0,r="MozAppearance"in l.style,s=r&&!!b.createRange().compareNode,t=s?l:n.parentNode,l=a.opera&&"[object Opera]"==o.call(a.opera),l=!!b.attachEvent&&!l,u=r?"object":l?"script":"img",v=l?"script":u,w=Array.isArray||function(a){return"[object Array]"==o.call(a)},x=[],y={},z={timeout:function(a,b){return b.length&&(a.timeout=b[0]),a}},A,B;B=function(a){function b(a){var a=a.split("!"),b=x.length,c=a.pop(),d=a.length,c={url:c,origUrl:c,prefixes:a},e,f,g;for(f=0;f ================================================ FILE: echonet/__init__.py ================================================ """ The echonet package contains code for loading echocardiogram videos, and functions for training and testing segmentation and ejection fraction prediction models. """ import click from echonet.__version__ import __version__ from echonet.config import CONFIG as config import echonet.datasets as datasets import echonet.utils as utils @click.group() def main(): """Entry point for command line interface.""" del click main.add_command(utils.segmentation.run) main.add_command(utils.video.run) __all__ = ["__version__", "config", "datasets", "main", "utils"] ================================================ FILE: echonet/__main__.py ================================================ """Entry point for command line.""" import echonet if __name__ == '__main__': echonet.main() ================================================ FILE: echonet/__version__.py ================================================ """Version number for Echonet package.""" __version__ = "1.0.0" ================================================ FILE: echonet/config.py ================================================ """Sets paths based on configuration files.""" import configparser import os import types _FILENAME = None _PARAM = {} for filename in ["echonet.cfg", ".echonet.cfg", os.path.expanduser("~/echonet.cfg"), os.path.expanduser("~/.echonet.cfg"), ]: if os.path.isfile(filename): _FILENAME = filename config = configparser.ConfigParser() with open(filename, "r") as f: config.read_string("[config]\n" + f.read()) _PARAM = config["config"] break CONFIG = types.SimpleNamespace( FILENAME=_FILENAME, DATA_DIR=_PARAM.get("data_dir", "a4c-video-dir/")) ================================================ FILE: echonet/datasets/__init__.py ================================================ """ The echonet.datasets submodule defines a Pytorch dataset for loading echocardiogram videos. """ from .echo import Echo __all__ = ["Echo"] ================================================ FILE: echonet/datasets/echo.py ================================================ """EchoNet-Dynamic Dataset.""" import os import collections import pandas import numpy as np import skimage.draw import torchvision import echonet class Echo(torchvision.datasets.VisionDataset): """EchoNet-Dynamic Dataset. Args: root (string): Root directory of dataset (defaults to `echonet.config.DATA_DIR`) split (string): One of {``train'', ``val'', ``test'', ``all'', or ``external_test''} target_type (string or list, optional): Type of target to use, ``Filename'', ``EF'', ``EDV'', ``ESV'', ``LargeIndex'', ``SmallIndex'', ``LargeFrame'', ``SmallFrame'', ``LargeTrace'', or ``SmallTrace'' Can also be a list to output a tuple with all specified target types. The targets represent: ``Filename'' (string): filename of video ``EF'' (float): ejection fraction ``EDV'' (float): end-diastolic volume ``ESV'' (float): end-systolic volume ``LargeIndex'' (int): index of large (diastolic) frame in video ``SmallIndex'' (int): index of small (systolic) frame in video ``LargeFrame'' (np.array shape=(3, height, width)): normalized large (diastolic) frame ``SmallFrame'' (np.array shape=(3, height, width)): normalized small (systolic) frame ``LargeTrace'' (np.array shape=(height, width)): left ventricle large (diastolic) segmentation value of 0 indicates pixel is outside left ventricle 1 indicates pixel is inside left ventricle ``SmallTrace'' (np.array shape=(height, width)): left ventricle small (systolic) segmentation value of 0 indicates pixel is outside left ventricle 1 indicates pixel is inside left ventricle Defaults to ``EF''. mean (int, float, or np.array shape=(3,), optional): means for all (if scalar) or each (if np.array) channel. Used for normalizing the video. Defaults to 0 (video is not shifted). std (int, float, or np.array shape=(3,), optional): standard deviation for all (if scalar) or each (if np.array) channel. Used for normalizing the video. Defaults to 0 (video is not scaled). length (int or None, optional): Number of frames to clip from video. If ``None'', longest possible clip is returned. Defaults to 16. period (int, optional): Sampling period for taking a clip from the video (i.e. every ``period''-th frame is taken) Defaults to 2. max_length (int or None, optional): Maximum number of frames to clip from video (main use is for shortening excessively long videos when ``length'' is set to None). If ``None'', shortening is not applied to any video. Defaults to 250. clips (int, optional): Number of clips to sample. Main use is for test-time augmentation with random clips. Defaults to 1. pad (int or None, optional): Number of pixels to pad all frames on each side (used as augmentation). and a window of the original size is taken. If ``None'', no padding occurs. Defaults to ``None''. noise (float or None, optional): Fraction of pixels to black out as simulated noise. If ``None'', no simulated noise is added. Defaults to ``None''. target_transform (callable, optional): A function/transform that takes in the target and transforms it. external_test_location (string): Path to videos to use for external testing. """ def __init__(self, root=None, split="train", target_type="EF", mean=0., std=1., length=16, period=2, max_length=250, clips=1, pad=None, noise=None, target_transform=None, external_test_location=None): if root is None: root = echonet.config.DATA_DIR super().__init__(root, target_transform=target_transform) self.split = split.upper() if not isinstance(target_type, list): target_type = [target_type] self.target_type = target_type self.mean = mean self.std = std self.length = length self.max_length = max_length self.period = period self.clips = clips self.pad = pad self.noise = noise self.target_transform = target_transform self.external_test_location = external_test_location self.fnames, self.outcome = [], [] if self.split == "EXTERNAL_TEST": self.fnames = sorted(os.listdir(self.external_test_location)) else: # Load video-level labels with open(os.path.join(self.root, "FileList.csv")) as f: data = pandas.read_csv(f) data["Split"].map(lambda x: x.upper()) if self.split != "ALL": data = data[data["Split"] == self.split] self.header = data.columns.tolist() self.fnames = data["FileName"].tolist() self.fnames = [fn + ".avi" for fn in self.fnames if os.path.splitext(fn)[1] == ""] # Assume avi if no suffix self.outcome = data.values.tolist() # Check that files are present missing = set(self.fnames) - set(os.listdir(os.path.join(self.root, "Videos"))) if len(missing) != 0: print("{} videos could not be found in {}:".format(len(missing), os.path.join(self.root, "Videos"))) for f in sorted(missing): print("\t", f) raise FileNotFoundError(os.path.join(self.root, "Videos", sorted(missing)[0])) # Load traces self.frames = collections.defaultdict(list) self.trace = collections.defaultdict(_defaultdict_of_lists) with open(os.path.join(self.root, "VolumeTracings.csv")) as f: header = f.readline().strip().split(",") assert header == ["FileName", "X1", "Y1", "X2", "Y2", "Frame"] for line in f: filename, x1, y1, x2, y2, frame = line.strip().split(',') x1 = float(x1) y1 = float(y1) x2 = float(x2) y2 = float(y2) frame = int(frame) if frame not in self.trace[filename]: self.frames[filename].append(frame) self.trace[filename][frame].append((x1, y1, x2, y2)) for filename in self.frames: for frame in self.frames[filename]: self.trace[filename][frame] = np.array(self.trace[filename][frame]) # A small number of videos are missing traces; remove these videos keep = [len(self.frames[f]) >= 2 for f in self.fnames] self.fnames = [f for (f, k) in zip(self.fnames, keep) if k] self.outcome = [f for (f, k) in zip(self.outcome, keep) if k] def __getitem__(self, index): # Find filename of video if self.split == "EXTERNAL_TEST": video = os.path.join(self.external_test_location, self.fnames[index]) elif self.split == "CLINICAL_TEST": video = os.path.join(self.root, "ProcessedStrainStudyA4c", self.fnames[index]) else: video = os.path.join(self.root, "Videos", self.fnames[index]) # Load video into np.array video = echonet.utils.loadvideo(video).astype(np.float32) # Add simulated noise (black out random pixels) # 0 represents black at this point (video has not been normalized yet) if self.noise is not None: n = video.shape[1] * video.shape[2] * video.shape[3] ind = np.random.choice(n, round(self.noise * n), replace=False) f = ind % video.shape[1] ind //= video.shape[1] i = ind % video.shape[2] ind //= video.shape[2] j = ind video[:, f, i, j] = 0 # Apply normalization if isinstance(self.mean, (float, int)): video -= self.mean else: video -= self.mean.reshape(3, 1, 1, 1) if isinstance(self.std, (float, int)): video /= self.std else: video /= self.std.reshape(3, 1, 1, 1) # Set number of frames c, f, h, w = video.shape if self.length is None: # Take as many frames as possible length = f // self.period else: # Take specified number of frames length = self.length if self.max_length is not None: # Shorten videos to max_length length = min(length, self.max_length) if f < length * self.period: # Pad video with frames filled with zeros if too short # 0 represents the mean color (dark grey), since this is after normalization video = np.concatenate((video, np.zeros((c, length * self.period - f, h, w), video.dtype)), axis=1) c, f, h, w = video.shape # pylint: disable=E0633 if self.clips == "all": # Take all possible clips of desired length start = np.arange(f - (length - 1) * self.period) else: # Take random clips from video start = np.random.choice(f - (length - 1) * self.period, self.clips) # Gather targets target = [] for t in self.target_type: key = self.fnames[index] if t == "Filename": target.append(self.fnames[index]) elif t == "LargeIndex": # Traces are sorted by cross-sectional area # Largest (diastolic) frame is last target.append(np.int(self.frames[key][-1])) elif t == "SmallIndex": # Largest (diastolic) frame is first target.append(np.int(self.frames[key][0])) elif t == "LargeFrame": target.append(video[:, self.frames[key][-1], :, :]) elif t == "SmallFrame": target.append(video[:, self.frames[key][0], :, :]) elif t in ["LargeTrace", "SmallTrace"]: if t == "LargeTrace": t = self.trace[key][self.frames[key][-1]] else: t = self.trace[key][self.frames[key][0]] x1, y1, x2, y2 = t[:, 0], t[:, 1], t[:, 2], t[:, 3] x = np.concatenate((x1[1:], np.flip(x2[1:]))) y = np.concatenate((y1[1:], np.flip(y2[1:]))) r, c = skimage.draw.polygon(np.rint(y).astype(np.int), np.rint(x).astype(np.int), (video.shape[2], video.shape[3])) mask = np.zeros((video.shape[2], video.shape[3]), np.float32) mask[r, c] = 1 target.append(mask) else: if self.split == "CLINICAL_TEST" or self.split == "EXTERNAL_TEST": target.append(np.float32(0)) else: target.append(np.float32(self.outcome[index][self.header.index(t)])) if target != []: target = tuple(target) if len(target) > 1 else target[0] if self.target_transform is not None: target = self.target_transform(target) # Select clips from video video = tuple(video[:, s + self.period * np.arange(length), :, :] for s in start) if self.clips == 1: video = video[0] else: video = np.stack(video) if self.pad is not None: # Add padding of zeros (mean color of videos) # Crop of original size is taken out # (Used as augmentation) c, l, h, w = video.shape temp = np.zeros((c, l, h + 2 * self.pad, w + 2 * self.pad), dtype=video.dtype) temp[:, :, self.pad:-self.pad, self.pad:-self.pad] = video # pylint: disable=E1130 i, j = np.random.randint(0, 2 * self.pad, 2) video = temp[:, :, i:(i + h), j:(j + w)] return video, target def __len__(self): return len(self.fnames) def extra_repr(self) -> str: """Additional information to add at end of __repr__.""" lines = ["Target type: {target_type}", "Split: {split}"] return '\n'.join(lines).format(**self.__dict__) def _defaultdict_of_lists(): """Returns a defaultdict of lists. This is used to avoid issues with Windows (if this function is anonymous, the Echo dataset cannot be used in a dataloader). """ return collections.defaultdict(list) ================================================ FILE: echonet/utils/__init__.py ================================================ """Utility functions for videos, plotting and computing performance metrics.""" import os import typing import cv2 # pytype: disable=attribute-error import matplotlib import numpy as np import torch import tqdm from . import video from . import segmentation def loadvideo(filename: str) -> np.ndarray: """Loads a video from a file. Args: filename (str): filename of video Returns: A np.ndarray with dimensions (channels=3, frames, height, width). The values will be uint8's ranging from 0 to 255. Raises: FileNotFoundError: Could not find `filename` ValueError: An error occurred while reading the video """ if not os.path.exists(filename): raise FileNotFoundError(filename) capture = cv2.VideoCapture(filename) frame_count = int(capture.get(cv2.CAP_PROP_FRAME_COUNT)) frame_width = int(capture.get(cv2.CAP_PROP_FRAME_WIDTH)) frame_height = int(capture.get(cv2.CAP_PROP_FRAME_HEIGHT)) v = np.zeros((frame_count, frame_height, frame_width, 3), np.uint8) for count in range(frame_count): ret, frame = capture.read() if not ret: raise ValueError("Failed to load frame #{} of {}.".format(count, filename)) frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB) v[count, :, :] = frame v = v.transpose((3, 0, 1, 2)) return v def savevideo(filename: str, array: np.ndarray, fps: typing.Union[float, int] = 1): """Saves a video to a file. Args: filename (str): filename of video array (np.ndarray): video of uint8's with shape (channels=3, frames, height, width) fps (float or int): frames per second Returns: None """ c, _, height, width = array.shape if c != 3: raise ValueError("savevideo expects array of shape (channels=3, frames, height, width), got shape ({})".format(", ".join(map(str, array.shape)))) fourcc = cv2.VideoWriter_fourcc('M', 'J', 'P', 'G') out = cv2.VideoWriter(filename, fourcc, fps, (width, height)) for frame in array.transpose((1, 2, 3, 0)): frame = cv2.cvtColor(frame, cv2.COLOR_RGB2BGR) out.write(frame) def get_mean_and_std(dataset: torch.utils.data.Dataset, samples: int = 128, batch_size: int = 8, num_workers: int = 4): """Computes mean and std from samples from a Pytorch dataset. Args: dataset (torch.utils.data.Dataset): A Pytorch dataset. ``dataset[i][0]'' is expected to be the i-th video in the dataset, which should be a ``torch.Tensor'' of dimensions (channels=3, frames, height, width) samples (int or None, optional): Number of samples to take from dataset. If ``None'', mean and standard deviation are computed over all elements. Defaults to 128. batch_size (int, optional): how many samples per batch to load Defaults to 8. num_workers (int, optional): how many subprocesses to use for data loading. If 0, the data will be loaded in the main process. Defaults to 4. Returns: A tuple of the mean and standard deviation. Both are represented as np.array's of dimension (channels,). """ if samples is not None and len(dataset) > samples: indices = np.random.choice(len(dataset), samples, replace=False) dataset = torch.utils.data.Subset(dataset, indices) dataloader = torch.utils.data.DataLoader( dataset, batch_size=batch_size, num_workers=num_workers, shuffle=True) n = 0 # number of elements taken (should be equal to samples by end of for loop) s1 = 0. # sum of elements along channels (ends up as np.array of dimension (channels,)) s2 = 0. # sum of squares of elements along channels (ends up as np.array of dimension (channels,)) for (x, *_) in tqdm.tqdm(dataloader): x = x.transpose(0, 1).contiguous().view(3, -1) n += x.shape[1] s1 += torch.sum(x, dim=1).numpy() s2 += torch.sum(x ** 2, dim=1).numpy() mean = s1 / n # type: np.ndarray std = np.sqrt(s2 / n - mean ** 2) # type: np.ndarray mean = mean.astype(np.float32) std = std.astype(np.float32) return mean, std def bootstrap(a, b, func, samples=10000): """Computes a bootstrapped confidence intervals for ``func(a, b)''. Args: a (array_like): first argument to `func`. b (array_like): second argument to `func`. func (callable): Function to compute confidence intervals for. ``dataset[i][0]'' is expected to be the i-th video in the dataset, which should be a ``torch.Tensor'' of dimensions (channels=3, frames, height, width) samples (int, optional): Number of samples to compute. Defaults to 10000. Returns: A tuple of (`func(a, b)`, estimated 5-th percentile, estimated 95-th percentile). """ a = np.array(a) b = np.array(b) bootstraps = [] for _ in range(samples): ind = np.random.choice(len(a), len(a)) bootstraps.append(func(a[ind], b[ind])) bootstraps = sorted(bootstraps) return func(a, b), bootstraps[round(0.05 * len(bootstraps))], bootstraps[round(0.95 * len(bootstraps))] def latexify(): """Sets matplotlib params to appear more like LaTeX. Based on https://nipunbatra.github.io/blog/2014/latexify.html """ params = {'backend': 'pdf', 'axes.titlesize': 8, 'axes.labelsize': 8, 'font.size': 8, 'legend.fontsize': 8, 'xtick.labelsize': 8, 'ytick.labelsize': 8, 'font.family': 'DejaVu Serif', 'font.serif': 'Computer Modern', } matplotlib.rcParams.update(params) def dice_similarity_coefficient(inter, union): """Computes the dice similarity coefficient. Args: inter (iterable): iterable of the intersections union (iterable): iterable of the unions """ return 2 * sum(inter) / (sum(union) + sum(inter)) __all__ = ["video", "segmentation", "loadvideo", "savevideo", "get_mean_and_std", "bootstrap", "latexify", "dice_similarity_coefficient"] ================================================ FILE: echonet/utils/segmentation.py ================================================ """Functions for training and running segmentation.""" import math import os import time import click import matplotlib.pyplot as plt import numpy as np import scipy.signal import skimage.draw import torch import torchvision import tqdm import echonet @click.command("segmentation") @click.option("--data_dir", type=click.Path(exists=True, file_okay=False), default=None) @click.option("--output", type=click.Path(file_okay=False), default=None) @click.option("--model_name", type=click.Choice( sorted(name for name in torchvision.models.segmentation.__dict__ if name.islower() and not name.startswith("__") and callable(torchvision.models.segmentation.__dict__[name]))), default="deeplabv3_resnet50") @click.option("--pretrained/--random", default=False) @click.option("--weights", type=click.Path(exists=True, dir_okay=False), default=None) @click.option("--run_test/--skip_test", default=False) @click.option("--save_video/--skip_video", default=False) @click.option("--num_epochs", type=int, default=50) @click.option("--lr", type=float, default=1e-5) @click.option("--weight_decay", type=float, default=0) @click.option("--lr_step_period", type=int, default=None) @click.option("--num_train_patients", type=int, default=None) @click.option("--num_workers", type=int, default=4) @click.option("--batch_size", type=int, default=20) @click.option("--device", type=str, default=None) @click.option("--seed", type=int, default=0) def run( data_dir=None, output=None, model_name="deeplabv3_resnet50", pretrained=False, weights=None, run_test=False, save_video=False, num_epochs=50, lr=1e-5, weight_decay=1e-5, lr_step_period=None, num_train_patients=None, num_workers=4, batch_size=20, device=None, seed=0, ): """Trains/tests segmentation model. Args: data_dir (str, optional): Directory containing dataset. Defaults to `echonet.config.DATA_DIR`. output (str, optional): Directory to place outputs. Defaults to output/segmentation/_/. model_name (str, optional): Name of segmentation model. One of ``deeplabv3_resnet50'', ``deeplabv3_resnet101'', ``fcn_resnet50'', or ``fcn_resnet101'' (options are torchvision.models.segmentation.) Defaults to ``deeplabv3_resnet50''. pretrained (bool, optional): Whether to use pretrained weights for model Defaults to False. weights (str, optional): Path to checkpoint containing weights to initialize model. Defaults to None. run_test (bool, optional): Whether or not to run on test. Defaults to False. save_video (bool, optional): Whether to save videos with segmentations. Defaults to False. num_epochs (int, optional): Number of epochs during training Defaults to 50. lr (float, optional): Learning rate for SGD Defaults to 1e-5. weight_decay (float, optional): Weight decay for SGD Defaults to 0. lr_step_period (int or None, optional): Period of learning rate decay (learning rate is decayed by a multiplicative factor of 0.1) Defaults to math.inf (never decay learning rate). num_train_patients (int or None, optional): Number of training patients for ablations. Defaults to all patients. num_workers (int, optional): Number of subprocesses to use for data loading. If 0, the data will be loaded in the main process. Defaults to 4. device (str or None, optional): Name of device to run on. Options from https://pytorch.org/docs/stable/tensor_attributes.html#torch.torch.device Defaults to ``cuda'' if available, and ``cpu'' otherwise. batch_size (int, optional): Number of samples to load per batch Defaults to 20. seed (int, optional): Seed for random number generator. Defaults to 0. """ # Seed RNGs np.random.seed(seed) torch.manual_seed(seed) # Set default output directory if output is None: output = os.path.join("output", "segmentation", "{}_{}".format(model_name, "pretrained" if pretrained else "random")) os.makedirs(output, exist_ok=True) # Set device for computations if device is None: device = torch.device("cuda" if torch.cuda.is_available() else "cpu") # Set up model model = torchvision.models.segmentation.__dict__[model_name](pretrained=pretrained, aux_loss=False) model.classifier[-1] = torch.nn.Conv2d(model.classifier[-1].in_channels, 1, kernel_size=model.classifier[-1].kernel_size) # change number of outputs to 1 if device.type == "cuda": model = torch.nn.DataParallel(model) model.to(device) if weights is not None: checkpoint = torch.load(weights) model.load_state_dict(checkpoint['state_dict']) # Set up optimizer optim = torch.optim.SGD(model.parameters(), lr=lr, momentum=0.9, weight_decay=weight_decay) if lr_step_period is None: lr_step_period = math.inf scheduler = torch.optim.lr_scheduler.StepLR(optim, lr_step_period) # Compute mean and std mean, std = echonet.utils.get_mean_and_std(echonet.datasets.Echo(root=data_dir, split="train")) tasks = ["LargeFrame", "SmallFrame", "LargeTrace", "SmallTrace"] kwargs = {"target_type": tasks, "mean": mean, "std": std } # Set up datasets and dataloaders dataset = {} dataset["train"] = echonet.datasets.Echo(root=data_dir, split="train", **kwargs) if num_train_patients is not None and len(dataset["train"]) > num_train_patients: # Subsample patients (used for ablation experiment) indices = np.random.choice(len(dataset["train"]), num_train_patients, replace=False) dataset["train"] = torch.utils.data.Subset(dataset["train"], indices) dataset["val"] = echonet.datasets.Echo(root=data_dir, split="val", **kwargs) # Run training and testing loops with open(os.path.join(output, "log.csv"), "a") as f: epoch_resume = 0 bestLoss = float("inf") try: # Attempt to load checkpoint checkpoint = torch.load(os.path.join(output, "checkpoint.pt")) model.load_state_dict(checkpoint['state_dict']) optim.load_state_dict(checkpoint['opt_dict']) scheduler.load_state_dict(checkpoint['scheduler_dict']) epoch_resume = checkpoint["epoch"] + 1 bestLoss = checkpoint["best_loss"] f.write("Resuming from epoch {}\n".format(epoch_resume)) except FileNotFoundError: f.write("Starting run from scratch\n") for epoch in range(epoch_resume, num_epochs): print("Epoch #{}".format(epoch), flush=True) for phase in ['train', 'val']: start_time = time.time() for i in range(torch.cuda.device_count()): torch.cuda.reset_peak_memory_stats(i) ds = dataset[phase] dataloader = torch.utils.data.DataLoader( ds, batch_size=batch_size, num_workers=num_workers, shuffle=True, pin_memory=(device.type == "cuda"), drop_last=(phase == "train")) loss, large_inter, large_union, small_inter, small_union = echonet.utils.segmentation.run_epoch(model, dataloader, phase == "train", optim, device) overall_dice = 2 * (large_inter.sum() + small_inter.sum()) / (large_union.sum() + large_inter.sum() + small_union.sum() + small_inter.sum()) large_dice = 2 * large_inter.sum() / (large_union.sum() + large_inter.sum()) small_dice = 2 * small_inter.sum() / (small_union.sum() + small_inter.sum()) f.write("{},{},{},{},{},{},{},{},{},{},{}\n".format(epoch, phase, loss, overall_dice, large_dice, small_dice, time.time() - start_time, large_inter.size, sum(torch.cuda.max_memory_allocated() for i in range(torch.cuda.device_count())), sum(torch.cuda.max_memory_reserved() for i in range(torch.cuda.device_count())), batch_size)) f.flush() scheduler.step() # Save checkpoint save = { 'epoch': epoch, 'state_dict': model.state_dict(), 'best_loss': bestLoss, 'loss': loss, 'opt_dict': optim.state_dict(), 'scheduler_dict': scheduler.state_dict(), } torch.save(save, os.path.join(output, "checkpoint.pt")) if loss < bestLoss: torch.save(save, os.path.join(output, "best.pt")) bestLoss = loss # Load best weights if num_epochs != 0: checkpoint = torch.load(os.path.join(output, "best.pt")) model.load_state_dict(checkpoint['state_dict']) f.write("Best validation loss {} from epoch {}\n".format(checkpoint["loss"], checkpoint["epoch"])) if run_test: # Run on validation and test for split in ["val", "test"]: dataset = echonet.datasets.Echo(root=data_dir, split=split, **kwargs) dataloader = torch.utils.data.DataLoader(dataset, batch_size=batch_size, num_workers=num_workers, shuffle=False, pin_memory=(device.type == "cuda")) loss, large_inter, large_union, small_inter, small_union = echonet.utils.segmentation.run_epoch(model, dataloader, False, None, device) overall_dice = 2 * (large_inter + small_inter) / (large_union + large_inter + small_union + small_inter) large_dice = 2 * large_inter / (large_union + large_inter) small_dice = 2 * small_inter / (small_union + small_inter) with open(os.path.join(output, "{}_dice.csv".format(split)), "w") as g: g.write("Filename, Overall, Large, Small\n") for (filename, overall, large, small) in zip(dataset.fnames, overall_dice, large_dice, small_dice): g.write("{},{},{},{}\n".format(filename, overall, large, small)) f.write("{} dice (overall): {:.4f} ({:.4f} - {:.4f})\n".format(split, *echonet.utils.bootstrap(np.concatenate((large_inter, small_inter)), np.concatenate((large_union, small_union)), echonet.utils.dice_similarity_coefficient))) f.write("{} dice (large): {:.4f} ({:.4f} - {:.4f})\n".format(split, *echonet.utils.bootstrap(large_inter, large_union, echonet.utils.dice_similarity_coefficient))) f.write("{} dice (small): {:.4f} ({:.4f} - {:.4f})\n".format(split, *echonet.utils.bootstrap(small_inter, small_union, echonet.utils.dice_similarity_coefficient))) f.flush() # Saving videos with segmentations dataset = echonet.datasets.Echo(root=data_dir, split="test", target_type=["Filename", "LargeIndex", "SmallIndex"], # Need filename for saving, and human-selected frames to annotate mean=mean, std=std, # Normalization length=None, max_length=None, period=1 # Take all frames ) dataloader = torch.utils.data.DataLoader(dataset, batch_size=10, num_workers=num_workers, shuffle=False, pin_memory=False, collate_fn=_video_collate_fn) # Save videos with segmentation if save_video and not all(os.path.isfile(os.path.join(output, "videos", f)) for f in dataloader.dataset.fnames): # Only run if missing videos model.eval() os.makedirs(os.path.join(output, "videos"), exist_ok=True) os.makedirs(os.path.join(output, "size"), exist_ok=True) echonet.utils.latexify() with torch.no_grad(): with open(os.path.join(output, "size.csv"), "w") as g: g.write("Filename,Frame,Size,HumanLarge,HumanSmall,ComputerSmall\n") for (x, (filenames, large_index, small_index), length) in tqdm.tqdm(dataloader): # Run segmentation model on blocks of frames one-by-one # The whole concatenated video may be too long to run together y = np.concatenate([model(x[i:(i + batch_size), :, :, :].to(device))["out"].detach().cpu().numpy() for i in range(0, x.shape[0], batch_size)]) start = 0 x = x.numpy() for (i, (filename, offset)) in enumerate(zip(filenames, length)): # Extract one video and segmentation predictions video = x[start:(start + offset), ...] logit = y[start:(start + offset), 0, :, :] # Un-normalize video video *= std.reshape(1, 3, 1, 1) video += mean.reshape(1, 3, 1, 1) # Get frames, channels, height, and width f, c, h, w = video.shape # pylint: disable=W0612 assert c == 3 # Put two copies of the video side by side video = np.concatenate((video, video), 3) # If a pixel is in the segmentation, saturate blue channel # Leave alone otherwise video[:, 0, :, w:] = np.maximum(255. * (logit > 0), video[:, 0, :, w:]) # pylint: disable=E1111 # Add blank canvas under pair of videos video = np.concatenate((video, np.zeros_like(video)), 2) # Compute size of segmentation per frame size = (logit > 0).sum((1, 2)) # Identify systole frames with peak detection trim_min = sorted(size)[round(len(size) ** 0.05)] trim_max = sorted(size)[round(len(size) ** 0.95)] trim_range = trim_max - trim_min systole = set(scipy.signal.find_peaks(-size, distance=20, prominence=(0.50 * trim_range))[0]) # Write sizes and frames to file for (frame, s) in enumerate(size): g.write("{},{},{},{},{},{}\n".format(filename, frame, s, 1 if frame == large_index[i] else 0, 1 if frame == small_index[i] else 0, 1 if frame in systole else 0)) # Plot sizes fig = plt.figure(figsize=(size.shape[0] / 50 * 1.5, 3)) plt.scatter(np.arange(size.shape[0]) / 50, size, s=1) ylim = plt.ylim() for s in systole: plt.plot(np.array([s, s]) / 50, ylim, linewidth=1) plt.ylim(ylim) plt.title(os.path.splitext(filename)[0]) plt.xlabel("Seconds") plt.ylabel("Size (pixels)") plt.tight_layout() plt.savefig(os.path.join(output, "size", os.path.splitext(filename)[0] + ".pdf")) plt.close(fig) # Normalize size to [0, 1] size -= size.min() size = size / size.max() size = 1 - size # Iterate the frames in this video for (f, s) in enumerate(size): # On all frames, mark a pixel for the size of the frame video[:, :, int(round(115 + 100 * s)), int(round(f / len(size) * 200 + 10))] = 255. if f in systole: # If frame is computer-selected systole, mark with a line video[:, :, 115:224, int(round(f / len(size) * 200 + 10))] = 255. def dash(start, stop, on=10, off=10): buf = [] x = start while x < stop: buf.extend(range(x, x + on)) x += on x += off buf = np.array(buf) buf = buf[buf < stop] return buf d = dash(115, 224) if f == large_index[i]: # If frame is human-selected diastole, mark with green dashed line on all frames video[:, :, d, int(round(f / len(size) * 200 + 10))] = np.array([0, 225, 0]).reshape((1, 3, 1)) if f == small_index[i]: # If frame is human-selected systole, mark with red dashed line on all frames video[:, :, d, int(round(f / len(size) * 200 + 10))] = np.array([0, 0, 225]).reshape((1, 3, 1)) # Get pixels for a circle centered on the pixel r, c = skimage.draw.disk((int(round(115 + 100 * s)), int(round(f / len(size) * 200 + 10))), 4.1) # On the frame that's being shown, put a circle over the pixel video[f, :, r, c] = 255. # Rearrange dimensions and save video = video.transpose(1, 0, 2, 3) video = video.astype(np.uint8) echonet.utils.savevideo(os.path.join(output, "videos", filename), video, 50) # Move to next video start += offset def run_epoch(model, dataloader, train, optim, device): """Run one epoch of training/evaluation for segmentation. Args: model (torch.nn.Module): Model to train/evaulate. dataloder (torch.utils.data.DataLoader): Dataloader for dataset. train (bool): Whether or not to train model. optim (torch.optim.Optimizer): Optimizer device (torch.device): Device to run on """ total = 0. n = 0 pos = 0 neg = 0 pos_pix = 0 neg_pix = 0 model.train(train) large_inter = 0 large_union = 0 small_inter = 0 small_union = 0 large_inter_list = [] large_union_list = [] small_inter_list = [] small_union_list = [] with torch.set_grad_enabled(train): with tqdm.tqdm(total=len(dataloader)) as pbar: for (_, (large_frame, small_frame, large_trace, small_trace)) in dataloader: # Count number of pixels in/out of human segmentation pos += (large_trace == 1).sum().item() pos += (small_trace == 1).sum().item() neg += (large_trace == 0).sum().item() neg += (small_trace == 0).sum().item() # Count number of pixels in/out of computer segmentation pos_pix += (large_trace == 1).sum(0).to("cpu").detach().numpy() pos_pix += (small_trace == 1).sum(0).to("cpu").detach().numpy() neg_pix += (large_trace == 0).sum(0).to("cpu").detach().numpy() neg_pix += (small_trace == 0).sum(0).to("cpu").detach().numpy() # Run prediction for diastolic frames and compute loss large_frame = large_frame.to(device) large_trace = large_trace.to(device) y_large = model(large_frame)["out"] loss_large = torch.nn.functional.binary_cross_entropy_with_logits(y_large[:, 0, :, :], large_trace, reduction="sum") # Compute pixel intersection and union between human and computer segmentations large_inter += np.logical_and(y_large[:, 0, :, :].detach().cpu().numpy() > 0., large_trace[:, :, :].detach().cpu().numpy() > 0.).sum() large_union += np.logical_or(y_large[:, 0, :, :].detach().cpu().numpy() > 0., large_trace[:, :, :].detach().cpu().numpy() > 0.).sum() large_inter_list.extend(np.logical_and(y_large[:, 0, :, :].detach().cpu().numpy() > 0., large_trace[:, :, :].detach().cpu().numpy() > 0.).sum((1, 2))) large_union_list.extend(np.logical_or(y_large[:, 0, :, :].detach().cpu().numpy() > 0., large_trace[:, :, :].detach().cpu().numpy() > 0.).sum((1, 2))) # Run prediction for systolic frames and compute loss small_frame = small_frame.to(device) small_trace = small_trace.to(device) y_small = model(small_frame)["out"] loss_small = torch.nn.functional.binary_cross_entropy_with_logits(y_small[:, 0, :, :], small_trace, reduction="sum") # Compute pixel intersection and union between human and computer segmentations small_inter += np.logical_and(y_small[:, 0, :, :].detach().cpu().numpy() > 0., small_trace[:, :, :].detach().cpu().numpy() > 0.).sum() small_union += np.logical_or(y_small[:, 0, :, :].detach().cpu().numpy() > 0., small_trace[:, :, :].detach().cpu().numpy() > 0.).sum() small_inter_list.extend(np.logical_and(y_small[:, 0, :, :].detach().cpu().numpy() > 0., small_trace[:, :, :].detach().cpu().numpy() > 0.).sum((1, 2))) small_union_list.extend(np.logical_or(y_small[:, 0, :, :].detach().cpu().numpy() > 0., small_trace[:, :, :].detach().cpu().numpy() > 0.).sum((1, 2))) # Take gradient step if training loss = (loss_large + loss_small) / 2 if train: optim.zero_grad() loss.backward() optim.step() # Accumulate losses and compute baselines total += loss.item() n += large_trace.size(0) p = pos / (pos + neg) p_pix = (pos_pix + 1) / (pos_pix + neg_pix + 2) # Show info on process bar pbar.set_postfix_str("{:.4f} ({:.4f}) / {:.4f} {:.4f}, {:.4f}, {:.4f}".format(total / n / 112 / 112, loss.item() / large_trace.size(0) / 112 / 112, -p * math.log(p) - (1 - p) * math.log(1 - p), (-p_pix * np.log(p_pix) - (1 - p_pix) * np.log(1 - p_pix)).mean(), 2 * large_inter / (large_union + large_inter), 2 * small_inter / (small_union + small_inter))) pbar.update() large_inter_list = np.array(large_inter_list) large_union_list = np.array(large_union_list) small_inter_list = np.array(small_inter_list) small_union_list = np.array(small_union_list) return (total / n / 112 / 112, large_inter_list, large_union_list, small_inter_list, small_union_list, ) def _video_collate_fn(x): """Collate function for Pytorch dataloader to merge multiple videos. This function should be used in a dataloader for a dataset that returns a video as the first element, along with some (non-zero) tuple of targets. Then, the input x is a list of tuples: - x[i][0] is the i-th video in the batch - x[i][1] are the targets for the i-th video This function returns a 3-tuple: - The first element is the videos concatenated along the frames dimension. This is done so that videos of different lengths can be processed together (tensors cannot be "jagged", so we cannot have a dimension for video, and another for frames). - The second element is contains the targets with no modification. - The third element is a list of the lengths of the videos in frames. """ video, target = zip(*x) # Extract the videos and targets # ``video'' is a tuple of length ``batch_size'' # Each element has shape (channels=3, frames, height, width) # height and width are expected to be the same across videos, but # frames can be different. # ``target'' is also a tuple of length ``batch_size'' # Each element is a tuple of the targets for the item. i = list(map(lambda t: t.shape[1], video)) # Extract lengths of videos in frames # This contatenates the videos along the the frames dimension (basically # playing the videos one after another). The frames dimension is then # moved to be first. # Resulting shape is (total frames, channels=3, height, width) video = torch.as_tensor(np.swapaxes(np.concatenate(video, 1), 0, 1)) # Swap dimensions (approximately a transpose) # Before: target[i][j] is the j-th target of element i # After: target[i][j] is the i-th target of element j target = zip(*target) return video, target, i ================================================ FILE: echonet/utils/video.py ================================================ """Functions for training and running EF prediction.""" import math import os import time import click import matplotlib.pyplot as plt import numpy as np import sklearn.metrics import torch import torchvision import tqdm import echonet @click.command("video") @click.option("--data_dir", type=click.Path(exists=True, file_okay=False), default=None) @click.option("--output", type=click.Path(file_okay=False), default=None) @click.option("--task", type=str, default="EF") @click.option("--model_name", type=click.Choice( sorted(name for name in torchvision.models.video.__dict__ if name.islower() and not name.startswith("__") and callable(torchvision.models.video.__dict__[name]))), default="r2plus1d_18") @click.option("--pretrained/--random", default=True) @click.option("--weights", type=click.Path(exists=True, dir_okay=False), default=None) @click.option("--run_test/--skip_test", default=False) @click.option("--num_epochs", type=int, default=45) @click.option("--lr", type=float, default=1e-4) @click.option("--weight_decay", type=float, default=1e-4) @click.option("--lr_step_period", type=int, default=15) @click.option("--frames", type=int, default=32) @click.option("--period", type=int, default=2) @click.option("--num_train_patients", type=int, default=None) @click.option("--num_workers", type=int, default=4) @click.option("--batch_size", type=int, default=20) @click.option("--device", type=str, default=None) @click.option("--seed", type=int, default=0) def run( data_dir=None, output=None, task="EF", model_name="r2plus1d_18", pretrained=True, weights=None, run_test=False, num_epochs=45, lr=1e-4, weight_decay=1e-4, lr_step_period=15, frames=32, period=2, num_train_patients=None, num_workers=4, batch_size=20, device=None, seed=0, ): """Trains/tests EF prediction model. \b Args: data_dir (str, optional): Directory containing dataset. Defaults to `echonet.config.DATA_DIR`. output (str, optional): Directory to place outputs. Defaults to output/video/_/. task (str, optional): Name of task to predict. Options are the headers of FileList.csv. Defaults to ``EF''. model_name (str, optional): Name of model. One of ``mc3_18'', ``r2plus1d_18'', or ``r3d_18'' (options are torchvision.models.video.) Defaults to ``r2plus1d_18''. pretrained (bool, optional): Whether to use pretrained weights for model Defaults to True. weights (str, optional): Path to checkpoint containing weights to initialize model. Defaults to None. run_test (bool, optional): Whether or not to run on test. Defaults to False. num_epochs (int, optional): Number of epochs during training. Defaults to 45. lr (float, optional): Learning rate for SGD Defaults to 1e-4. weight_decay (float, optional): Weight decay for SGD Defaults to 1e-4. lr_step_period (int or None, optional): Period of learning rate decay (learning rate is decayed by a multiplicative factor of 0.1) Defaults to 15. frames (int, optional): Number of frames to use in clip Defaults to 32. period (int, optional): Sampling period for frames Defaults to 2. n_train_patients (int or None, optional): Number of training patients for ablations. Defaults to all patients. num_workers (int, optional): Number of subprocesses to use for data loading. If 0, the data will be loaded in the main process. Defaults to 4. device (str or None, optional): Name of device to run on. Options from https://pytorch.org/docs/stable/tensor_attributes.html#torch.torch.device Defaults to ``cuda'' if available, and ``cpu'' otherwise. batch_size (int, optional): Number of samples to load per batch Defaults to 20. seed (int, optional): Seed for random number generator. Defaults to 0. """ # Seed RNGs np.random.seed(seed) torch.manual_seed(seed) # Set default output directory if output is None: output = os.path.join("output", "video", "{}_{}_{}_{}".format(model_name, frames, period, "pretrained" if pretrained else "random")) os.makedirs(output, exist_ok=True) # Set device for computations if device is None: device = torch.device("cuda" if torch.cuda.is_available() else "cpu") # Set up model model = torchvision.models.video.__dict__[model_name](pretrained=pretrained) model.fc = torch.nn.Linear(model.fc.in_features, 1) model.fc.bias.data[0] = 55.6 if device.type == "cuda": model = torch.nn.DataParallel(model) model.to(device) if weights is not None: checkpoint = torch.load(weights) model.load_state_dict(checkpoint['state_dict']) # Set up optimizer optim = torch.optim.SGD(model.parameters(), lr=lr, momentum=0.9, weight_decay=weight_decay) if lr_step_period is None: lr_step_period = math.inf scheduler = torch.optim.lr_scheduler.StepLR(optim, lr_step_period) # Compute mean and std mean, std = echonet.utils.get_mean_and_std(echonet.datasets.Echo(root=data_dir, split="train")) kwargs = {"target_type": task, "mean": mean, "std": std, "length": frames, "period": period, } # Set up datasets and dataloaders dataset = {} dataset["train"] = echonet.datasets.Echo(root=data_dir, split="train", **kwargs, pad=12) if num_train_patients is not None and len(dataset["train"]) > num_train_patients: # Subsample patients (used for ablation experiment) indices = np.random.choice(len(dataset["train"]), num_train_patients, replace=False) dataset["train"] = torch.utils.data.Subset(dataset["train"], indices) dataset["val"] = echonet.datasets.Echo(root=data_dir, split="val", **kwargs) # Run training and testing loops with open(os.path.join(output, "log.csv"), "a") as f: epoch_resume = 0 bestLoss = float("inf") try: # Attempt to load checkpoint checkpoint = torch.load(os.path.join(output, "checkpoint.pt")) model.load_state_dict(checkpoint['state_dict']) optim.load_state_dict(checkpoint['opt_dict']) scheduler.load_state_dict(checkpoint['scheduler_dict']) epoch_resume = checkpoint["epoch"] + 1 bestLoss = checkpoint["best_loss"] f.write("Resuming from epoch {}\n".format(epoch_resume)) except FileNotFoundError: f.write("Starting run from scratch\n") for epoch in range(epoch_resume, num_epochs): print("Epoch #{}".format(epoch), flush=True) for phase in ['train', 'val']: start_time = time.time() for i in range(torch.cuda.device_count()): torch.cuda.reset_peak_memory_stats(i) ds = dataset[phase] dataloader = torch.utils.data.DataLoader( ds, batch_size=batch_size, num_workers=num_workers, shuffle=True, pin_memory=(device.type == "cuda"), drop_last=(phase == "train")) loss, yhat, y = echonet.utils.video.run_epoch(model, dataloader, phase == "train", optim, device) f.write("{},{},{},{},{},{},{},{},{}\n".format(epoch, phase, loss, sklearn.metrics.r2_score(y, yhat), time.time() - start_time, y.size, sum(torch.cuda.max_memory_allocated() for i in range(torch.cuda.device_count())), sum(torch.cuda.max_memory_reserved() for i in range(torch.cuda.device_count())), batch_size)) f.flush() scheduler.step() # Save checkpoint save = { 'epoch': epoch, 'state_dict': model.state_dict(), 'period': period, 'frames': frames, 'best_loss': bestLoss, 'loss': loss, 'r2': sklearn.metrics.r2_score(y, yhat), 'opt_dict': optim.state_dict(), 'scheduler_dict': scheduler.state_dict(), } torch.save(save, os.path.join(output, "checkpoint.pt")) if loss < bestLoss: torch.save(save, os.path.join(output, "best.pt")) bestLoss = loss # Load best weights if num_epochs != 0: checkpoint = torch.load(os.path.join(output, "best.pt")) model.load_state_dict(checkpoint['state_dict']) f.write("Best validation loss {} from epoch {}\n".format(checkpoint["loss"], checkpoint["epoch"])) f.flush() if run_test: for split in ["val", "test"]: # Performance without test-time augmentation dataloader = torch.utils.data.DataLoader( echonet.datasets.Echo(root=data_dir, split=split, **kwargs), batch_size=batch_size, num_workers=num_workers, shuffle=True, pin_memory=(device.type == "cuda")) loss, yhat, y = echonet.utils.video.run_epoch(model, dataloader, False, None, device) f.write("{} (one clip) R2: {:.3f} ({:.3f} - {:.3f})\n".format(split, *echonet.utils.bootstrap(y, yhat, sklearn.metrics.r2_score))) f.write("{} (one clip) MAE: {:.2f} ({:.2f} - {:.2f})\n".format(split, *echonet.utils.bootstrap(y, yhat, sklearn.metrics.mean_absolute_error))) f.write("{} (one clip) RMSE: {:.2f} ({:.2f} - {:.2f})\n".format(split, *tuple(map(math.sqrt, echonet.utils.bootstrap(y, yhat, sklearn.metrics.mean_squared_error))))) f.flush() # Performance with test-time augmentation ds = echonet.datasets.Echo(root=data_dir, split=split, **kwargs, clips="all") dataloader = torch.utils.data.DataLoader( ds, batch_size=1, num_workers=num_workers, shuffle=False, pin_memory=(device.type == "cuda")) loss, yhat, y = echonet.utils.video.run_epoch(model, dataloader, False, None, device, save_all=True, block_size=batch_size) f.write("{} (all clips) R2: {:.3f} ({:.3f} - {:.3f})\n".format(split, *echonet.utils.bootstrap(y, np.array(list(map(lambda x: x.mean(), yhat))), sklearn.metrics.r2_score))) f.write("{} (all clips) MAE: {:.2f} ({:.2f} - {:.2f})\n".format(split, *echonet.utils.bootstrap(y, np.array(list(map(lambda x: x.mean(), yhat))), sklearn.metrics.mean_absolute_error))) f.write("{} (all clips) RMSE: {:.2f} ({:.2f} - {:.2f})\n".format(split, *tuple(map(math.sqrt, echonet.utils.bootstrap(y, np.array(list(map(lambda x: x.mean(), yhat))), sklearn.metrics.mean_squared_error))))) f.flush() # Write full performance to file with open(os.path.join(output, "{}_predictions.csv".format(split)), "w") as g: for (filename, pred) in zip(ds.fnames, yhat): for (i, p) in enumerate(pred): g.write("{},{},{:.4f}\n".format(filename, i, p)) echonet.utils.latexify() yhat = np.array(list(map(lambda x: x.mean(), yhat))) # Plot actual and predicted EF fig = plt.figure(figsize=(3, 3)) lower = min(y.min(), yhat.min()) upper = max(y.max(), yhat.max()) plt.scatter(y, yhat, color="k", s=1, edgecolor=None, zorder=2) plt.plot([0, 100], [0, 100], linewidth=1, zorder=3) plt.axis([lower - 3, upper + 3, lower - 3, upper + 3]) plt.gca().set_aspect("equal", "box") plt.xlabel("Actual EF (%)") plt.ylabel("Predicted EF (%)") plt.xticks([10, 20, 30, 40, 50, 60, 70, 80]) plt.yticks([10, 20, 30, 40, 50, 60, 70, 80]) plt.grid(color="gainsboro", linestyle="--", linewidth=1, zorder=1) plt.tight_layout() plt.savefig(os.path.join(output, "{}_scatter.pdf".format(split))) plt.close(fig) # Plot AUROC fig = plt.figure(figsize=(3, 3)) plt.plot([0, 1], [0, 1], linewidth=1, color="k", linestyle="--") for thresh in [35, 40, 45, 50]: fpr, tpr, _ = sklearn.metrics.roc_curve(y > thresh, yhat) print(thresh, sklearn.metrics.roc_auc_score(y > thresh, yhat)) plt.plot(fpr, tpr) plt.axis([-0.01, 1.01, -0.01, 1.01]) plt.xlabel("False Positive Rate") plt.ylabel("True Positive Rate") plt.tight_layout() plt.savefig(os.path.join(output, "{}_roc.pdf".format(split))) plt.close(fig) def run_epoch(model, dataloader, train, optim, device, save_all=False, block_size=None): """Run one epoch of training/evaluation for segmentation. Args: model (torch.nn.Module): Model to train/evaulate. dataloder (torch.utils.data.DataLoader): Dataloader for dataset. train (bool): Whether or not to train model. optim (torch.optim.Optimizer): Optimizer device (torch.device): Device to run on save_all (bool, optional): If True, return predictions for all test-time augmentations separately. If False, return only the mean prediction. Defaults to False. block_size (int or None, optional): Maximum number of augmentations to run on at the same time. Use to limit the amount of memory used. If None, always run on all augmentations simultaneously. Default is None. """ model.train(train) total = 0 # total training loss n = 0 # number of videos processed s1 = 0 # sum of ground truth EF s2 = 0 # Sum of ground truth EF squared yhat = [] y = [] with torch.set_grad_enabled(train): with tqdm.tqdm(total=len(dataloader)) as pbar: for (X, outcome) in dataloader: y.append(outcome.numpy()) X = X.to(device) outcome = outcome.to(device) average = (len(X.shape) == 6) if average: batch, n_clips, c, f, h, w = X.shape X = X.view(-1, c, f, h, w) s1 += outcome.sum() s2 += (outcome ** 2).sum() if block_size is None: outputs = model(X) else: outputs = torch.cat([model(X[j:(j + block_size), ...]) for j in range(0, X.shape[0], block_size)]) if save_all: yhat.append(outputs.view(-1).to("cpu").detach().numpy()) if average: outputs = outputs.view(batch, n_clips, -1).mean(1) if not save_all: yhat.append(outputs.view(-1).to("cpu").detach().numpy()) loss = torch.nn.functional.mse_loss(outputs.view(-1), outcome) if train: optim.zero_grad() loss.backward() optim.step() total += loss.item() * X.size(0) n += X.size(0) pbar.set_postfix_str("{:.2f} ({:.2f}) / {:.2f}".format(total / n, loss.item(), s2 / n - (s1 / n) ** 2)) pbar.update() if not save_all: yhat = np.concatenate(yhat) y = np.concatenate(y) return total / n, yhat, y ================================================ FILE: example.cfg ================================================ DATA_DIR = a4c-video-dir/ ================================================ FILE: requirements.txt ================================================ certifi==2020.12.5 cycler==0.10.0 decorator==4.4.2 echonet==1.0.0 imageio==2.9.0 joblib==1.0.1 kiwisolver==1.3.1 matplotlib==3.3.4 networkx==2.5 numpy==1.20.1 opencv-python==4.5.1.48 pandas==1.2.3 Pillow==8.1.2 pyparsing==2.4.7 python-dateutil==2.8.1 pytz==2021.1 PyWavelets==1.1.1 scikit-image==0.18.1 scikit-learn==0.24.1 scipy==1.6.1 six==1.15.0 sklearn==0.0 threadpoolctl==2.1.0 tifffile==2021.3.17 torch==1.8.0 torchvision==0.9.0 tqdm==4.59.0 typing-extensions==3.7.4.3 ================================================ FILE: scripts/ConvertDICOMToAVI.ipynb ================================================ { "cells": [ { "cell_type": "code", "execution_count": 12, "metadata": {}, "outputs": [], "source": [ "# David Ouyang 10/2/2019\n", "\n", "# Notebook which iterates through a folder, including subfolders, \n", "# and convert DICOM files to AVI files of a defined size (natively 112 x 112)\n", "\n", "import re\n", "import os, os.path\n", "from os.path import splitext\n", "import pydicom as dicom\n", "import numpy as np\n", "from pydicom.uid import UID, generate_uid\n", "import shutil\n", "from multiprocessing import dummy as multiprocessing\n", "import time\n", "import subprocess\n", "import datetime\n", "from datetime import date\n", "import sys\n", "import cv2\n", "#from scipy.misc import imread\n", "import matplotlib.pyplot as plt\n", "import sys\n", "from shutil import copy\n", "import math\n", "\n", "destinationFolder = \"Output Folder Name\"\n" ] }, { "cell_type": "code", "execution_count": 10, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Requirement already satisfied: pillow in c:\\programdata\\anaconda3\\lib\\site-packages (6.2.0)\n", "Requirement already satisfied: scipy in c:\\programdata\\anaconda3\\lib\\site-packages (1.3.1)\n" ] } ], "source": [ "# Dependencies you might need to run code\n", "# Commonly missing\n", "\n", "#!pip install pydicom\n", "#!pip install opencv-python\n" ] }, { "cell_type": "code", "execution_count": 2, "metadata": {}, "outputs": [], "source": [ "def mask(output):\n", " dimension = output.shape[0]\n", " \n", " # Mask pixels outside of scanning sector\n", " m1, m2 = np.meshgrid(np.arange(dimension), np.arange(dimension))\n", " \n", "\n", " mask = ((m1+m2)>int(dimension/2) + int(dimension/10)) \n", " mask *= ((m1-m2)0:\n", " if testarray.shape[1] < testarray.shape[2]:\n", " testarray = testarray[:, :, bias:-bias, :]\n", " else:\n", " testarray = testarray[:, bias:-bias, :, :]\n", "\n", "\n", " print(testarray.shape)\n", " frames,height,width,channels = testarray.shape\n", "\n", " fps = 30\n", "\n", " try:\n", " fps = dataset[(0x18, 0x40)].value\n", " except:\n", " print(\"couldn't find frame rate, default to 30\")\n", "\n", " fourcc = cv2.VideoWriter_fourcc('M','J','P','G')\n", " video_filename = os.path.join(destinationFolder, fileName + '.avi')\n", " out = cv2.VideoWriter(video_filename, fourcc, fps, cropSize)\n", "\n", "\n", " for i in range(frames):\n", "\n", " outputA = testarray[i,:,:,0]\n", " smallOutput = outputA[int(height/10):(height - int(height/10)), int(height/10):(height - int(height/10))]\n", "\n", " # Resize image\n", " output = cv2.resize(smallOutput, cropSize, interpolation = cv2.INTER_CUBIC)\n", "\n", " finaloutput = mask(output)\n", "\n", "\n", " finaloutput = cv2.merge([finaloutput,finaloutput,finaloutput])\n", " out.write(finaloutput)\n", "\n", " out.release()\n", "\n", " else:\n", " print(fileName,\"hasAlreadyBeenProcessed\")\n", " except:\n", " print(\"something filed, not sure what, have to debug\", fileName)\n", " return 0" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "AllA4cNames = \"Input Folder Name\"\n", "\n", "count = 0\n", " \n", "cropSize = (112,112)\n", "subfolders = os.listdir(AllA4cNames)\n", "\n", "\n", "for folder in subfolders:\n", " print(folder)\n", "\n", " for content in os.listdir(os.path.join(AllA4cNames, folder)):\n", " for subcontent in os.listdir(os.path.join(AllA4cNames, folder, content)):\n", " count += 1\n", " \n", "\n", " VideoPath = os.path.join(AllA4cNames, folder, content, subcontent)\n", "\n", " print(count, folder, content, subcontent)\n", "\n", " if not os.path.exists(os.path.join(destinationFolder,subcontent + \".avi\")):\n", " makeVideo(VideoPath, destinationFolder)\n", " else:\n", " print(\"Already did this file\", VideoPath)\n", "\n", "\n", "print(len(AllA4cFilenames))" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "Python 3", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.7.4" } }, "nbformat": 4, "nbformat_minor": 2 } ================================================ FILE: scripts/InitializationNotebook.ipynb ================================================ { "cells": [ { "cell_type": "code", "execution_count": 4, "metadata": {}, "outputs": [], "source": [ "# David Ouyang 12/5/2019\n", "\n", "# Notebook which:\n", "# 1. Downloads weights\n", "# 2. Initializes model and imports weights\n", "# 3. Performs test time evaluation of videos (already preprocessed with ConvertDICOMToAVI.ipynb)\n", "\n", "import re\n", "import os, os.path\n", "from os.path import splitext\n", "import pydicom as dicom\n", "import numpy as np\n", "from pydicom.uid import UID, generate_uid\n", "import shutil\n", "from multiprocessing import dummy as multiprocessing\n", "import time\n", "import subprocess\n", "import datetime\n", "from datetime import date\n", "import sys\n", "import cv2\n", "import matplotlib.pyplot as plt\n", "import sys\n", "from shutil import copy\n", "import math\n", "import torch\n", "import torchvision\n", "\n", "sys.path.append(\"..\")\n", "import echonet\n", "\n", "import wget \n", "\n", "#destinationFolder = \"/Users/davidouyang/Dropbox/Echo Research/CodeBase/Output\"\n", "destinationFolder = \"C:\\\\Users\\\\Windows\\\\Dropbox\\\\Echo Research\\\\CodeBase\\\\Output\"\n", "#videosFolder = \"/Users/davidouyang/Dropbox/Echo Research/CodeBase/a4c-video-dir\"\n", "videosFolder = \"C:\\\\Users\\\\Windows\\\\Dropbox\\\\Echo Research\\\\CodeBase\\\\a4c-video-dir\"\n", "#DestinationForWeights = \"/Users/davidouyang/Dropbox/Echo Research/CodeBase/EchoNetDynamic-Weights\"\n", "DestinationForWeights = \"C:\\\\Users\\\\Windows\\\\Dropbox\\\\Echo Research\\\\CodeBase\\\\EchoNetDynamic-Weights\"" ] }, { "cell_type": "code", "execution_count": 5, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "The weights are at C:\\Users\\Windows\\Dropbox\\Echo Research\\CodeBase\\EchoNetDynamic-Weights\n", "Segmentation Weights already present\n", "EF Weights already present\n" ] } ], "source": [ "# Download model weights\n", "\n", "if os.path.exists(DestinationForWeights):\n", " print(\"The weights are at\", DestinationForWeights)\n", "else:\n", " print(\"Creating folder at \", DestinationForWeights, \" to store weights\")\n", " os.mkdir(DestinationForWeights)\n", " \n", "segmentationWeightsURL = 'https://github.com/douyang/EchoNetDynamic/releases/download/v1.0.0/deeplabv3_resnet50_random.pt'\n", "ejectionFractionWeightsURL = 'https://github.com/douyang/EchoNetDynamic/releases/download/v1.0.0/r2plus1d_18_32_2_pretrained.pt'\n", "\n", "\n", "if not os.path.exists(os.path.join(DestinationForWeights, os.path.basename(segmentationWeightsURL))):\n", " print(\"Downloading Segmentation Weights, \", segmentationWeightsURL,\" to \",os.path.join(DestinationForWeights,os.path.basename(segmentationWeightsURL)))\n", " filename = wget.download(segmentationWeightsURL, out = DestinationForWeights)\n", "else:\n", " print(\"Segmentation Weights already present\")\n", " \n", "if not os.path.exists(os.path.join(DestinationForWeights, os.path.basename(ejectionFractionWeightsURL))):\n", " print(\"Downloading EF Weights, \", ejectionFractionWeightsURL,\" to \",os.path.join(DestinationForWeights,os.path.basename(ejectionFractionWeightsURL)))\n", " filename = wget.download(ejectionFractionWeightsURL, out = DestinationForWeights)\n", "else:\n", " print(\"EF Weights already present\")\n", " \n" ] }, { "cell_type": "code", "execution_count": 6, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "loading weights from C:\\Users\\Windows\\Dropbox\\Echo Research\\CodeBase\\EchoNetDynamic-Weights\\r2plus1d_18_32_2_pretrained\n", "cuda is available, original weights\n", "external_test ['0X1A05DFFFCAFB253B.avi', '0X1A0A263B22CCD966.avi', '0X1A2A76BDB5B98BED.avi', '0X1A2C60147AF9FDAE.avi', '0X1A2E9496910EFF5B.avi', '0X1A3D565B371DC573.avi', '0X1A3E7BF1DFB132FB.avi', '0X1A5FAE3F9D37794E.avi', '0X1A6ACFE7B286DAFC.avi', '0X1A8D85542DBE8204.avi', '23_Apical_4_chamber_view.dcm.avi', '62_Apical_4_chamber_view.dcm.avi', '64_Apical_4_chamber_view.dcm.avi']\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "100%|██████████████████████████████████████████████████████████████████████████████████| 10/10 [00:10<00:00, 1.00s/it]\n", "100%|████████████████████████████████████████████████████████| 13/13 [00:29<00:00, 2.26s/it, 3122.29 (3440.26) / 0.00]\n" ] } ], "source": [ "# Initialize and Run EF model\n", "\n", "frames = 32\n", "period = 1 #2\n", "batch_size = 20\n", "model = torchvision.models.video.r2plus1d_18(pretrained=False)\n", "model.fc = torch.nn.Linear(model.fc.in_features, 1)\n", "\n", "\n", "\n", "print(\"loading weights from \", os.path.join(DestinationForWeights, \"r2plus1d_18_32_2_pretrained\"))\n", "\n", "if torch.cuda.is_available():\n", " print(\"cuda is available, original weights\")\n", " device = torch.device(\"cuda\")\n", " model = torch.nn.DataParallel(model)\n", " model.to(device)\n", " checkpoint = torch.load(os.path.join(DestinationForWeights, os.path.basename(ejectionFractionWeightsURL)))\n", " model.load_state_dict(checkpoint['state_dict'])\n", "else:\n", " print(\"cuda is not available, cpu weights\")\n", " device = torch.device(\"cpu\")\n", " checkpoint = torch.load(os.path.join(DestinationForWeights, os.path.basename(ejectionFractionWeightsURL)), map_location = \"cpu\")\n", " state_dict_cpu = {k[7:]: v for (k, v) in checkpoint['state_dict'].items()}\n", " model.load_state_dict(state_dict_cpu)\n", "\n", "\n", "# try some random weights: final_r2+1d_model_regression_EF_sgd_skip1_32frames.pth.tar\n", "# scp ouyangd@arthur2:~/Echo-Tracing-Analysis/final_r2+1d_model_regression_EF_sgd_skip1_32frames.pth.tar \"C:\\Users\\Windows\\Dropbox\\Echo Research\\CodeBase\\EchoNetDynamic-Weights\"\n", "#Weights = \"final_r2+1d_model_regression_EF_sgd_skip1_32frames.pth.tar\"\n", "\n", "\n", "output = os.path.join(destinationFolder, \"cedars_ef_output.csv\")\n", "\n", "ds = echonet.datasets.Echo(split = \"external_test\", external_test_location = videosFolder, crops=\"all\")\n", "print(ds.split, ds.fnames)\n", "\n", "mean, std = echonet.utils.get_mean_and_std(ds)\n", "\n", "kwargs = {\"target_type\": \"EF\",\n", " \"mean\": mean,\n", " \"std\": std,\n", " \"length\": frames,\n", " \"period\": period,\n", " }\n", "\n", "ds = echonet.datasets.Echo(split = \"external_test\", external_test_location = videosFolder, **kwargs, crops=\"all\")\n", "\n", "test_dataloader = torch.utils.data.DataLoader(ds, batch_size = 1, num_workers = 5, shuffle = True, pin_memory=(device.type == \"cuda\"))\n", "loss, yhat, y = echonet.utils.video.run_epoch(model, test_dataloader, \"test\", None, device, save_all=True, blocks=25)\n", "\n", "with open(output, \"w\") as g:\n", " for (filename, pred) in zip(ds.fnames, yhat):\n", " for (i,p) in enumerate(pred):\n", " g.write(\"{},{},{:.4f}\\n\".format(filename, i, p))\n" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "# Initialize and Run Segmentation model\n", "\n", "torch.cuda.empty_cache()\n", "\n", "\n", "videosFolder = \"C:\\\\Users\\\\Windows\\\\Dropbox\\\\Echo Research\\\\CodeBase\\\\View Classification\\\\AppearsA4c\\\\Resized2\"\n", "\n", "def collate_fn(x):\n", " x, f = zip(*x)\n", " i = list(map(lambda t: t.shape[1], x))\n", " x = torch.as_tensor(np.swapaxes(np.concatenate(x, 1), 0, 1))\n", " return x, f, i\n", "\n", "dataloader = torch.utils.data.DataLoader(echonet.datasets.Echo(split=\"external_test\", external_test_location = videosFolder, target_type=[\"Filename\"], length=None, period=1, mean=mean, std=std),\n", " batch_size=10, num_workers=0, shuffle=False, pin_memory=(device.type == \"cuda\"), collate_fn=collate_fn)\n", "if not all([os.path.isfile(os.path.join(destinationFolder, \"labels\", os.path.splitext(f)[0] + \".npy\")) for f in dataloader.dataset.fnames]):\n", " # Save segmentations for all frames\n", " # Only run if missing files\n", "\n", " pathlib.Path(os.path.join(destinationFolder, \"labels\")).mkdir(parents=True, exist_ok=True)\n", " block = 1024\n", " model.eval()\n", "\n", " with torch.no_grad():\n", " for (x, f, i) in tqdm.tqdm(dataloader):\n", " x = x.to(device)\n", " y = np.concatenate([model(x[i:(i + block), :, :, :])[\"out\"].detach().cpu().numpy() for i in range(0, x.shape[0], block)]).astype(np.float16)\n", " start = 0\n", " for (filename, offset) in zip(f, i):\n", " np.save(os.path.join(destinationFolder, \"labels\", os.path.splitext(filename)[0]), y[start:(start + offset), 0, :, :])\n", " start += offset\n", " \n", "dataloader = torch.utils.data.DataLoader(echonet.datasets.Echo(split=\"external_test\", external_test_location = videosFolder, target_type=[\"Filename\"], length=None, period=1, segmentation=os.path.join(destinationFolder, \"labels\")),\n", " batch_size=1, num_workers=8, shuffle=False, pin_memory=False)\n", "if not all(os.path.isfile(os.path.join(destinationFolder, \"videos\", f)) for f in dataloader.dataset.fnames):\n", " pathlib.Path(os.path.join(destinationFolder, \"videos\")).mkdir(parents=True, exist_ok=True)\n", " pathlib.Path(os.path.join(destinationFolder, \"size\")).mkdir(parents=True, exist_ok=True)\n", " echonet.utils.latexify()\n", " with open(os.path.join(destinationFolder, \"size.csv\"), \"w\") as g:\n", " g.write(\"Filename,Frame,Size,ComputerSmall\\n\")\n", " for (x, filename) in tqdm.tqdm(dataloader):\n", " x = x.numpy()\n", " for i in range(len(filename)):\n", " img = x[i, :, :, :, :].copy()\n", " logit = img[2, :, :, :].copy()\n", " img[1, :, :, :] = img[0, :, :, :]\n", " img[2, :, :, :] = img[0, :, :, :]\n", " img = np.concatenate((img, img), 3)\n", " img[0, :, :, 112:] = np.maximum(255. * (logit > 0), img[0, :, :, 112:])\n", "\n", " img = np.concatenate((img, np.zeros_like(img)), 2)\n", " size = (logit > 0).sum(2).sum(1)\n", " try:\n", " trim_min = sorted(size)[round(len(size) ** 0.05)]\n", " except:\n", " import code; code.interact(local=dict(globals(), **locals()))\n", " trim_max = sorted(size)[round(len(size) ** 0.95)]\n", " trim_range = trim_max - trim_min\n", " peaks = set(scipy.signal.find_peaks(-size, distance=20, prominence=(0.50 * trim_range))[0])\n", " for (x, y) in enumerate(size):\n", " g.write(\"{},{},{},{}\\n\".format(filename[0], x, y, 1 if x in peaks else 0))\n", " fig = plt.figure(figsize=(size.shape[0] / 50 * 1.5, 3))\n", " plt.scatter(np.arange(size.shape[0]) / 50, size, s=1)\n", " ylim = plt.ylim()\n", " for p in peaks:\n", " plt.plot(np.array([p, p]) / 50, ylim, linewidth=1)\n", " plt.ylim(ylim)\n", " plt.title(os.path.splitext(filename[i])[0])\n", " plt.xlabel(\"Seconds\")\n", " plt.ylabel(\"Size (pixels)\")\n", " plt.tight_layout()\n", " plt.savefig(os.path.join(destinationFolder, \"size\", os.path.splitext(filename[i])[0] + \".pdf\"))\n", " plt.close(fig)\n", " size -= size.min()\n", " size = size / size.max()\n", " size = 1 - size\n", " for (x, y) in enumerate(size):\n", " img[:, :, int(round(115 + 100 * y)), int(round(x / len(size) * 200 + 10))] = 255.\n", " interval = np.array([-3, -2, -1, 0, 1, 2, 3])\n", " for a in interval:\n", " for b in interval:\n", " img[:, x, a + int(round(115 + 100 * y)), b + int(round(x / len(size) * 200 + 10))] = 255.\n", " if x in peaks:\n", " img[:, :, 200:225, b + int(round(x / len(size) * 200 + 10))] = 255.\n", " echonet.utils.savevideo(os.path.join(destinationFolder, \"videos\", filename[i]), img.astype(np.uint8), 50) " ] } ], "metadata": { "kernelspec": { "display_name": "Python 3", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.7.4" } }, "nbformat": 4, "nbformat_minor": 2 } ================================================ FILE: scripts/beat_by_beat_analysis.R ================================================ library(ggplot2) library(stringr) library(plyr) library(dplyr) library(lubridate) library(reshape2) library(scales) library(ggthemes) library(Metrics) data <- read.csv("r2plus1d_18_32_2_pretrained_test_predictions.csv", header = FALSE) str(data) dataNoAugmentation <- data[data$V2 == 0,] str(dataNoAugmentation) dataGlobalAugmentation <- data %>% group_by(V1) %>% summarize(meanPrediction = mean(V3), sdPred = sd(V3)) str(dataGlobalAugmentation) sizeData <- read.csv("size.csv") sizeData <- sizeData[sizeData$ComputerSmall == 1,] str(sizeData) sizeRelevantFrames <- sizeData[c(1,2)] sizeRelevantFrames$Frame <- sizeRelevantFrames$Frame - 32 sizeRelevantFrames[sizeRelevantFrames$Frame < 0,]$Frame <- 0 beatByBeat <- merge(sizeRelevantFrames, data, by.x = c("Filename", "Frame"), by.y = c("V1", "V2")) beatByBeat <- beatByBeat %>% group_by(Filename) %>% summarize(meanPrediction = mean(V3), sdPred = sd(V3)) str(beatByBeat) ### For use, need to specify file directory fileLocation <- "/Users/davidouyang/Local Medical Data/" ActualNumbers <- read.csv(paste0(fileLocation, "FileList.csv", sep = "")) ActualNumbers <- ActualNumbers[c(1,2)] str(ActualNumbers) dataNoAugmentation <- merge(dataNoAugmentation, ActualNumbers, by.x = "V1", by.y = "Filename", all.x = TRUE) dataNoAugmentation$AbsErr <- abs(dataNoAugmentation$V3 - dataNoAugmentation$EF) str(dataNoAugmentation) summary(abs(dataNoAugmentation$V3 - dataNoAugmentation$EF)) # Mean of 4.216 rmse(dataNoAugmentation$V3,dataNoAugmentation$EF) ## 5.56 modelNoAugmentation <- lm(dataNoAugmentation$EF ~ dataNoAugmentation$V3) summary(modelNoAugmentation)$r.squared # 0.79475 beatByBeat <- merge(beatByBeat, ActualNumbers, by.x = "Filename", by.y = "Filename", all.x = TRUE) summary(abs(beatByBeat$meanPrediction - beatByBeat$EF)) # Mean of 4.051697 rmse(beatByBeat$meanPrediction, beatByBeat$EF) # 5.325237 modelBeatByBeat <- lm(beatByBeat$EF ~ beatByBeat$meanPrediction) summary(modelBeatByBeat)$r.squared # 0.8093174 beatByBeatAnalysis <- merge(sizeRelevantFrames, data, by.x = c("Filename", "Frame"), by.y = c("V1", "V2")) str(beatByBeatAnalysis) MAEdata <- data.frame(counter = 1:500) MAEdata$sample <- -9999 MAEdata$error <- -9999 str(MAEdata) for (i in 1:500){ samplingBeat <- sample_n(beatByBeatAnalysis %>% group_by(Filename), 1 + floor((i-1)/100), replace = TRUE) %>% group_by(Filename) %>% dplyr::summarize(meanPred = mean(V3)) samplingBeat <- merge(samplingBeat, ActualNumbers, by.x = "Filename", by.y = "Filename", all.x = TRUE) samplingBeat$error <- abs(samplingBeat$meanPred - samplingBeat$EF) MAEdata$sample[i] <- 1 + floor((i-1)/100) MAEdata$error[i] <- mean(samplingBeat$error ) } str(MAEdata) beatBoxPlot <- ggplot(data = MAEdata) + geom_boxplot(aes(x = sample, y = error, group = sample), outlier.shape = NA ) + theme_classic() + theme(legend.position = "none", axis.text.y = element_text( size=7)) + xlab("Number of Sampled Beats") + ylab("Mean Absolute Error") + scale_fill_brewer(palette = "Set1", direction = -1) beatBoxPlot ================================================ FILE: scripts/plot_complexity.py ================================================ #!/usr/bin/env python3 """Code to generate plots for Extended Data Fig. 4.""" import os import matplotlib import matplotlib.pyplot as plt import numpy as np import echonet def main(root=os.path.join("timing", "video"), fig_root=os.path.join("figure", "complexity"), FRAMES=(1, 8, 16, 32, 64, 96), pretrained=True): """Generate plots for Extended Data Fig. 4.""" echonet.utils.latexify() os.makedirs(fig_root, exist_ok=True) fig = plt.figure(figsize=(6.50, 2.50)) gs = matplotlib.gridspec.GridSpec(1, 3, width_ratios=[2.5, 2.5, 1.50]) ax = (plt.subplot(gs[0]), plt.subplot(gs[1]), plt.subplot(gs[2])) # Create legend for (model, color) in zip(["EchoNet-Dynamic (EF)", "R3D", "MC3"], matplotlib.colors.TABLEAU_COLORS): ax[2].plot([float("nan")], [float("nan")], "-", color=color, label=model) ax[2].set_title("") ax[2].axis("off") ax[2].legend(loc="center") for (model, color) in zip(["r2plus1d_18", "r3d_18", "mc3_18"], matplotlib.colors.TABLEAU_COLORS): for split in ["val"]: # ["val", "train"]: print(model, split) data = [load(root, model, frames, 1, pretrained, split) for frames in FRAMES] time = np.array(list(map(lambda x: x[0], data))) n = np.array(list(map(lambda x: x[1], data))) mem_allocated = np.array(list(map(lambda x: x[2], data))) # mem_cached = np.array(list(map(lambda x: x[3], data))) batch_size = np.array(list(map(lambda x: x[4], data))) # Plot Time (panel a) ax[0].plot(FRAMES, time / n, "-" if pretrained else "--", marker=".", color=color, linewidth=(1 if split == "train" else None)) print("Time:\n" + "\n".join(map(lambda x: "{:8d}: {:f}".format(*x), zip(FRAMES, time / n)))) # Plot Memory (panel b) ax[1].plot(FRAMES, mem_allocated / batch_size / 1e9, "-" if pretrained else "--", marker=".", color=color, linewidth=(1 if split == "train" else None)) print("Memory:\n" + "\n".join(map(lambda x: "{:8d}: {:f}".format(*x), zip(FRAMES, mem_allocated / batch_size / 1e9)))) print() # Labels for panel a ax[0].set_xticks(FRAMES) ax[0].text(-0.05, 1.10, "(a)", transform=ax[0].transAxes) ax[0].set_xlabel("Clip length (frames)") ax[0].set_ylabel("Time Per Clip (seconds)") # Labels for panel b ax[1].set_xticks(FRAMES) ax[1].text(-0.05, 1.10, "(b)", transform=ax[1].transAxes) ax[1].set_xlabel("Clip length (frames)") ax[1].set_ylabel("Memory Per Clip (GB)") # Save figure plt.tight_layout() plt.savefig(os.path.join(fig_root, "complexity.pdf")) plt.savefig(os.path.join(fig_root, "complexity.eps")) plt.close(fig) def load(root, model, frames, period, pretrained, split): """Loads runtime and memory usage for specified hyperparameter choice.""" with open(os.path.join(root, "{}_{}_{}_{}".format(model, frames, period, "pretrained" if pretrained else "random"), "log.csv"), "r") as f: for line in f: line = line.split(",") if len(line) < 4: # Skip lines that are not csv (these lines log information) continue if line[1] == split: *_, time, n, mem_allocated, mem_cached, batch_size = line time = float(time) n = int(n) mem_allocated = int(mem_allocated) mem_cached = int(mem_cached) batch_size = int(batch_size) return time, n, mem_allocated, mem_cached, batch_size raise ValueError("File missing information.") if __name__ == "__main__": main() ================================================ FILE: scripts/plot_hyperparameter_sweep.py ================================================ #!/usr/bin/env python3 """Code to generate plots for Extended Data Fig. 1.""" import os import matplotlib import matplotlib.pyplot as plt import echonet def main(root=os.path.join("output", "video"), fig_root=os.path.join("figure", "hyperparameter"), FRAMES=(1, 8, 16, 32, 64, 96, None), PERIOD=(1, 2, 4, 6, 8) ): """Generate plots for Extended Data Fig. 1.""" echonet.utils.latexify() os.makedirs(fig_root, exist_ok=True) # Parameters for plotting length sweep MAX = FRAMES[-2] START = 1 # Starting point for normal range TERM0 = 104 # Ending point for normal range BREAK = 112 # Location for break TERM1 = 120 # Starting point for "all" section ALL = 128 # Location of "all" point END = 135 # Ending point for "all" section RATIO = (BREAK - START) / (END - BREAK) # Set up figure fig = plt.figure(figsize=(3 + 2.5 + 1.5, 2.75)) outer = matplotlib.gridspec.GridSpec(1, 3, width_ratios=[3, 2.5, 1.50]) ax = plt.subplot(outer[2]) # Legend ax2 = plt.subplot(outer[1]) # Period plot gs = matplotlib.gridspec.GridSpecFromSubplotSpec( 1, 2, subplot_spec=outer[0], width_ratios=[RATIO, 1], wspace=0.020) # Length plot # Plot legend for (model, color) in zip(["EchoNet-Dynamic (EF)", "R3D", "MC3"], matplotlib.colors.TABLEAU_COLORS): ax.plot([float("nan")], [float("nan")], "-", color=color, label=model) ax.plot([float("nan")], [float("nan")], "-", color="k", label="Pretrained") ax.plot([float("nan")], [float("nan")], "--", color="k", label="Random") ax.set_title("") ax.axis("off") ax.legend(loc="center") # Plot length sweep (panel a) ax0 = plt.subplot(gs[0]) ax1 = plt.subplot(gs[1], sharey=ax0) print("FRAMES") for (model, color) in zip(["r2plus1d_18", "r3d_18", "mc3_18"], matplotlib.colors.TABLEAU_COLORS): for pretrained in [True, False]: loss = [load(root, model, frames, 1, pretrained) for frames in FRAMES] print(model, pretrained) print(" ".join(list(map(lambda x: "{:.1f}".format(x) if x is not None else None, loss)))) l0 = loss[-2] l1 = loss[-1] ax0.plot(FRAMES[:-1] + (TERM0,), loss[:-1] + [l0 + (l1 - l0) * (TERM0 - MAX) / (ALL - MAX)], "-" if pretrained else "--", color=color) ax1.plot([TERM1, ALL], [l0 + (l1 - l0) * (TERM1 - MAX) / (ALL - MAX)] + [loss[-1]], "-" if pretrained else "--", color=color) ax0.scatter(list(map(lambda x: x if x is not None else ALL, FRAMES)), loss, color=color, s=4) ax1.scatter(list(map(lambda x: x if x is not None else ALL, FRAMES)), loss, color=color, s=4) ax0.set_xticks(list(map(lambda x: x if x is not None else ALL, FRAMES))) ax1.set_xticks(list(map(lambda x: x if x is not None else ALL, FRAMES))) ax0.set_xticklabels(list(map(lambda x: x if x is not None else "All", FRAMES))) ax1.set_xticklabels(list(map(lambda x: x if x is not None else "All", FRAMES))) # https://stackoverflow.com/questions/5656798/python-matplotlib-is-there-a-way-to-make-a-discontinuous-axis/43684155 # zoom-in / limit the view to different portions of the data ax0.set_xlim(START, BREAK) # most of the data ax1.set_xlim(BREAK, END) # hide the spines between ax and ax2 ax0.spines['right'].set_visible(False) ax1.spines['left'].set_visible(False) ax1.get_yaxis().set_visible(False) d = 0.015 # how big to make the diagonal lines in axes coordinates # arguments to pass plot, just so we don't keep repeating them kwargs = dict(transform=ax0.transAxes, color='k', clip_on=False, linewidth=1) x0, x1, y0, y1 = ax0.axis() scale = (y1 - y0) / (x1 - x0) / 2 ax0.plot((1 - scale * d, 1 + scale * d), (-d, +d), **kwargs) # top-left diagonal ax0.plot((1 - scale * d, 1 + scale * d), (1 - d, 1 + d), **kwargs) # bottom-left diagonal kwargs.update(transform=ax1.transAxes) # switch to the bottom 1xes x0, x1, y0, y1 = ax1.axis() scale = (y1 - y0) / (x1 - x0) / 2 ax1.plot((-scale * d, scale * d), (-d, +d), **kwargs) # top-right diagonal ax1.plot((-scale * d, scale * d), (1 - d, 1 + d), **kwargs) # bottom-right diagonal # ax0.xaxis.label.set_transform(matplotlib.transforms.blended_transform_factory( # matplotlib.transforms.IdentityTransform(), fig.transFigure # specify x, y transform # )) # changed from default blend (IdentityTransform(), a[0].transAxes) ax0.xaxis.label.set_position((0.6, 0.0)) ax0.text(-0.05, 1.10, "(a)", transform=ax0.transAxes) ax0.set_xlabel("Clip length (frames)") ax0.set_ylabel("Validation Loss") # Plot period sweep (panel b) print("PERIOD") for (model, color) in zip(["r2plus1d_18", "r3d_18", "mc3_18"], matplotlib.colors.TABLEAU_COLORS): for pretrained in [True, False]: loss = [load(root, model, 64 // period, period, pretrained) for period in PERIOD] print(model, pretrained) print(" ".join(list(map(lambda x: "{:.1f}".format(x) if x is not None else None, loss)))) ax2.plot(PERIOD, loss, "-" if pretrained else "--", marker=".", color=color) ax2.set_xticks(PERIOD) ax2.text(-0.05, 1.10, "(b)", transform=ax2.transAxes) ax2.set_xlabel("Sampling Period (frames)") ax2.set_ylabel("Validation Loss") # Save figure plt.tight_layout() plt.savefig(os.path.join(fig_root, "hyperparameter.pdf")) plt.savefig(os.path.join(fig_root, "hyperparameter.eps")) plt.savefig(os.path.join(fig_root, "hyperparameter.png")) plt.close(fig) def load(root, model, frames, period, pretrained): """Loads best validation loss for specified hyperparameter choice.""" pretrained = ("pretrained" if pretrained else "random") f = os.path.join( root, "{}_{}_{}_{}".format(model, frames, period, pretrained), "log.csv") with open(f, "r") as f: for line in f: if "Best validation loss " in line: return float(line.split()[3]) raise ValueError("File missing information.") if __name__ == "__main__": main() ================================================ FILE: scripts/plot_loss.py ================================================ #!/usr/bin/env python3 """Code to generate plots for Extended Data Fig. 3.""" import argparse import os import matplotlib import matplotlib.pyplot as plt import echonet def main(): """Generate plots for Extended Data Fig. 3.""" # Select paths and hyperparameter to plot parser = argparse.ArgumentParser() parser.add_argument("dir", nargs="?", default="output") parser.add_argument("fig", nargs="?", default=os.path.join("figure", "loss")) parser.add_argument("--frames", type=int, default=32) parser.add_argument("--period", type=int, default=2) args = parser.parse_args() # Set up figure echonet.utils.latexify() os.makedirs(args.fig, exist_ok=True) fig = plt.figure(figsize=(7, 5)) gs = matplotlib.gridspec.GridSpec(ncols=3, nrows=2, figure=fig, width_ratios=[2.75, 2.75, 1.50]) # Plot EF loss curve ax0 = fig.add_subplot(gs[0, 0]) ax1 = fig.add_subplot(gs[0, 1], sharey=ax0) for pretrained in [True]: for (model, color) in zip(["r2plus1d_18", "r3d_18", "mc3_18"], matplotlib.colors.TABLEAU_COLORS): loss = load(os.path.join(args.dir, "video", "{}_{}_{}_{}".format(model, args.frames, args.period, "pretrained" if pretrained else "random"), "log.csv")) ax0.plot(range(1, 1 + len(loss["train"])), loss["train"], "-" if pretrained else "--", color=color) ax1.plot(range(1, 1 + len(loss["val"])), loss["val"], "-" if pretrained else "--", color=color) plt.axis([0, max(len(loss["train"]), len(loss["val"])), 0, max(max(loss["train"]), max(loss["val"]))]) ax0.text(-0.25, 1.00, "(a)", transform=ax0.transAxes) ax1.text(-0.25, 1.00, "(b)", transform=ax1.transAxes) ax0.set_xlabel("Epochs") ax1.set_xlabel("Epochs") ax0.set_xticks([0, 15, 30, 45]) ax1.set_xticks([0, 15, 30, 45]) ax0.set_ylabel("Training MSE Loss") ax1.set_ylabel("Validation MSE Loss") # Plot segmentation loss curve ax0 = fig.add_subplot(gs[1, 0]) ax1 = fig.add_subplot(gs[1, 1], sharey=ax0) pretrained = False for (model, color) in zip(["deeplabv3_resnet50"], list(matplotlib.colors.TABLEAU_COLORS)[3:]): loss = load(os.path.join(args.dir, "segmentation", "{}_{}".format(model, "pretrained" if pretrained else "random"), "log.csv")) ax0.plot(range(1, 1 + len(loss["train"])), loss["train"], "--", color=color) ax1.plot(range(1, 1 + len(loss["val"])), loss["val"], "--", color=color) ax0.text(-0.25, 1.00, "(c)", transform=ax0.transAxes) ax1.text(-0.25, 1.00, "(d)", transform=ax1.transAxes) ax0.set_ylim([0, 0.13]) ax0.set_xlabel("Epochs") ax1.set_xlabel("Epochs") ax0.set_xticks([0, 25, 50]) ax1.set_xticks([0, 25, 50]) ax0.set_ylabel("Training Cross Entropy Loss") ax1.set_ylabel("Validation Cross Entropy Loss") # Legend ax = fig.add_subplot(gs[:, 2]) for (model, color) in zip(["EchoNet-Dynamic (EF)", "R3D", "MC3", "EchoNet-Dynamic (Seg)"], matplotlib.colors.TABLEAU_COLORS): ax.plot([float("nan")], [float("nan")], "-", color=color, label=model) ax.set_title("") ax.axis("off") ax.legend(loc="center") plt.tight_layout() plt.savefig(os.path.join(args.fig, "loss.pdf")) plt.savefig(os.path.join(args.fig, "loss.eps")) plt.savefig(os.path.join(args.fig, "loss.png")) plt.close(fig) def load(filename): """Loads losses from specified file.""" losses = {"train": [], "val": []} with open(filename, "r") as f: for line in f: line = line.split(",") if len(line) < 4: continue epoch, split, loss, *_ = line epoch = int(epoch) loss = float(loss) assert(split in ["train", "val"]) if epoch == len(losses[split]): losses[split].append(loss) elif epoch == len(losses[split]) - 1: losses[split][-1] = loss else: raise ValueError("File has uninterpretable formatting.") return losses if __name__ == "__main__": main() ================================================ FILE: scripts/plot_simulated_noise.py ================================================ #!/usr/bin/env python3 """Code to generate plots for Extended Data Fig. 6.""" import os import pickle import matplotlib import matplotlib.pyplot as plt import numpy as np import PIL import sklearn import torch import torchvision import echonet def main(fig_root=os.path.join("figure", "noise"), video_output=os.path.join("output", "video", "r2plus1d_18_32_2_pretrained"), seg_output=os.path.join("output", "segmentation", "deeplabv3_resnet50_random"), NOISE=(0, 0.1, 0.2, 0.3, 0.4, 0.5)): """Generate plots for Extended Data Fig. 6.""" device = torch.device("cuda") filename = os.path.join(fig_root, "data.pkl") # Cache of results try: # Attempt to load cache with open(filename, "rb") as f: Y, YHAT, INTER, UNION = pickle.load(f) except FileNotFoundError: # Generate results if no cache available os.makedirs(fig_root, exist_ok=True) # Load trained video model model_v = torchvision.models.video.r2plus1d_18() model_v.fc = torch.nn.Linear(model_v.fc.in_features, 1) if device.type == "cuda": model_v = torch.nn.DataParallel(model_v) model_v.to(device) checkpoint = torch.load(os.path.join(video_output, "checkpoint.pt")) model_v.load_state_dict(checkpoint['state_dict']) # Load trained segmentation model model_s = torchvision.models.segmentation.deeplabv3_resnet50(aux_loss=False) model_s.classifier[-1] = torch.nn.Conv2d(model_s.classifier[-1].in_channels, 1, kernel_size=model_s.classifier[-1].kernel_size) if device.type == "cuda": model_s = torch.nn.DataParallel(model_s) model_s.to(device) checkpoint = torch.load(os.path.join(seg_output, "checkpoint.pt")) model_s.load_state_dict(checkpoint['state_dict']) # Run simulation dice = [] mse = [] r2 = [] Y = [] YHAT = [] INTER = [] UNION = [] for noise in NOISE: Y.append([]) YHAT.append([]) INTER.append([]) UNION.append([]) dataset = echonet.datasets.Echo(split="test", noise=noise) PIL.Image.fromarray(dataset[0][0][:, 0, :, :].astype(np.uint8).transpose(1, 2, 0)).save(os.path.join(fig_root, "noise_{}.tif".format(round(100 * noise)))) mean, std = echonet.utils.get_mean_and_std(echonet.datasets.Echo(split="train")) tasks = ["LargeFrame", "SmallFrame", "LargeTrace", "SmallTrace"] kwargs = { "target_type": tasks, "mean": mean, "std": std, "noise": noise } dataset = echonet.datasets.Echo(split="test", **kwargs) dataloader = torch.utils.data.DataLoader(dataset, batch_size=16, num_workers=5, shuffle=True, pin_memory=(device.type == "cuda")) loss, large_inter, large_union, small_inter, small_union = echonet.utils.segmentation.run_epoch(model_s, dataloader, "test", None, device) inter = np.concatenate((large_inter, small_inter)).sum() union = np.concatenate((large_union, small_union)).sum() dice.append(2 * inter / (union + inter)) INTER[-1].extend(large_inter.tolist() + small_inter.tolist()) UNION[-1].extend(large_union.tolist() + small_union.tolist()) kwargs = {"target_type": "EF", "mean": mean, "std": std, "length": 32, "period": 2, "noise": noise } dataset = echonet.datasets.Echo(split="test", **kwargs) dataloader = torch.utils.data.DataLoader(dataset, batch_size=16, num_workers=5, shuffle=True, pin_memory=(device.type == "cuda")) loss, yhat, y = echonet.utils.video.run_epoch(model_v, dataloader, "test", None, device) mse.append(loss) r2.append(sklearn.metrics.r2_score(y, yhat)) Y[-1].extend(y.tolist()) YHAT[-1].extend(yhat.tolist()) # Save results in cache with open(filename, "wb") as f: pickle.dump((Y, YHAT, INTER, UNION), f) # Set up plot echonet.utils.latexify() NOISE = list(map(lambda x: round(100 * x), NOISE)) fig = plt.figure(figsize=(6.50, 4.75)) gs = matplotlib.gridspec.GridSpec(3, 1, height_ratios=[2.0, 2.0, 0.75]) ax = (plt.subplot(gs[0]), plt.subplot(gs[1]), plt.subplot(gs[2])) # Plot EF prediction results (R^2) r2 = [sklearn.metrics.r2_score(y, yhat) for (y, yhat) in zip(Y, YHAT)] ax[0].plot(NOISE, r2, color="k", linewidth=1, marker=".") ax[0].set_xticks([]) ax[0].set_ylabel("R$^2$") l, h = min(r2), max(r2) l, h = l - 0.1 * (h - l), h + 0.1 * (h - l) ax[0].axis([min(NOISE) - 5, max(NOISE) + 5, 0, 1]) # Plot segmentation results (DSC) dice = [echonet.utils.dice_similarity_coefficient(inter, union) for (inter, union) in zip(INTER, UNION)] ax[1].plot(NOISE, dice, color="k", linewidth=1, marker=".") ax[1].set_xlabel("Pixels Removed (%)") ax[1].set_ylabel("DSC") l, h = min(dice), max(dice) l, h = l - 0.1 * (h - l), h + 0.1 * (h - l) ax[1].axis([min(NOISE) - 5, max(NOISE) + 5, 0, 1]) # Add example images below for noise in NOISE: image = matplotlib.image.imread(os.path.join(fig_root, "noise_{}.tif".format(noise))) imagebox = matplotlib.offsetbox.OffsetImage(image, zoom=0.4) ab = matplotlib.offsetbox.AnnotationBbox(imagebox, (noise, 0.0), frameon=False) ax[2].add_artist(ab) ax[2].axis("off") ax[2].axis([min(NOISE) - 5, max(NOISE) + 5, -1, 1]) fig.tight_layout() plt.savefig(os.path.join(fig_root, "noise.pdf"), dpi=1200) plt.savefig(os.path.join(fig_root, "noise.eps"), dpi=300) plt.savefig(os.path.join(fig_root, "noise.png"), dpi=600) plt.close(fig) if __name__ == "__main__": main() ================================================ FILE: scripts/run_experiments.sh ================================================ #!/bin/bash for pretrained in True False do for model in r2plus1d_18 r3d_18 mc3_18 do for frames in 96 64 32 16 8 4 1 do batch=$((256 / frames)) batch=$(( batch > 16 ? 16 : batch )) cmd="import echonet; echonet.utils.video.run(modelname=\"${model}\", frames=${frames}, period=1, pretrained=${pretrained}, batch_size=${batch})" python3 -c "${cmd}" done for period in 2 4 6 8 do batch=$((256 / 64 * period)) batch=$(( batch > 16 ? 16 : batch )) cmd="import echonet; echonet.utils.video.run(modelname=\"${model}\", frames=(64 // ${period}), period=${period}, pretrained=${pretrained}, batch_size=${batch})" python3 -c "${cmd}" done done done period=2 pretrained=True for model in r2plus1d_18 r3d_18 mc3_18 do cmd="import echonet; echonet.utils.video.run(modelname=\"${model}\", frames=(64 // ${period}), period=${period}, pretrained=${pretrained}, run_test=True)" python3 -c "${cmd}" done python3 -c "import echonet; echonet.utils.segmentation.run(modelname=\"deeplabv3_resnet50\", save_segmentation=True, pretrained=False)" pretrained=True model=r2plus1d_18 period=2 batch=$((256 / 64 * period)) batch=$(( batch > 16 ? 16 : batch )) for patients in 16 32 64 128 256 512 1024 2048 4096 7460 do cmd="import echonet; echonet.utils.video.run(modelname=\"${model}\", frames=(64 // ${period}), period=${period}, pretrained=${pretrained}, batch_size=${batch}, num_epochs=min(50 * (8192 // ${patients}), 200), output=\"output/training_size/video/${patients}\", n_train_patients=${patients})" python3 -c "${cmd}" cmd="import echonet; echonet.utils.segmentation.run(modelname=\"deeplabv3_resnet50\", pretrained=False, num_epochs=min(50 * (8192 // ${patients}), 200), output=\"output/training_size/segmentation/${patients}\", n_train_patients=${patients})" python3 -c "${cmd}" done ================================================ FILE: setup.py ================================================ #!/usr/bin/env python3 """Metadata for package to allow installation with pip.""" import os import setuptools with open("README.md", "r") as fh: long_description = fh.read() # Use same version from code # See 3 from # https://packaging.python.org/guides/single-sourcing-package-version/ version = {} with open(os.path.join("echonet", "__version__.py")) as f: exec(f.read(), version) # pylint: disable=W0122 setuptools.setup( name="echonet", description="Video-based AI for beat-to-beat cardiac function assessment.", version=version["__version__"], url="https://echonet.github.io/dynamic", packages=setuptools.find_packages(), install_requires=[ "click", "numpy", "pandas", "torch", "torchvision", "opencv-python", "scikit-image", "tqdm", "sklearn" ], classifiers=[ "Programming Language :: Python :: 3", ], entry_points={ "console_scripts": [ "echonet=echonet:main", ], } )