Repository: PavelOstyakov/pipeline Branch: master Commit: 236c050af3be Files: 116 Total size: 89.2 KB Directory structure: gitextract_vyezi4l1/ ├── .gitignore ├── .travis.yml ├── LICENSE ├── README.md ├── bin/ │ ├── predict.py │ └── train.py ├── cifar_pipeline/ │ ├── __init__.py │ ├── configs/ │ │ ├── __init__.py │ │ ├── base.py │ │ ├── fixup/ │ │ │ ├── base.py │ │ │ ├── resnet110_bn.py │ │ │ ├── resnet110_fixup.py │ │ │ ├── resnet110_fixup_0_0_1.py │ │ │ ├── resnet110_fixup_0_1.py │ │ │ ├── resnet110_fixup_mixup.py │ │ │ └── wideresnet/ │ │ │ ├── __init__.py │ │ │ ├── base.py │ │ │ ├── batch_norm/ │ │ │ │ ├── 10000_layers.py │ │ │ │ ├── 1000_layers.py │ │ │ │ ├── 100_layers.py │ │ │ │ ├── 10_layers.py │ │ │ │ └── __init__.py │ │ │ ├── fixup/ │ │ │ │ ├── 10000_layers.py │ │ │ │ ├── 1000_layers.py │ │ │ │ ├── 100_layers.py │ │ │ │ ├── 10_layers.py │ │ │ │ └── __init__.py │ │ │ ├── fixup_0/ │ │ │ │ ├── 10000_layers.py │ │ │ │ ├── 1000_layers.py │ │ │ │ ├── 100_layers.py │ │ │ │ ├── 10_layers.py │ │ │ │ └── __init__.py │ │ │ ├── fixup_0_0_1/ │ │ │ │ ├── 10000_layers.py │ │ │ │ ├── 1000_layers.py │ │ │ │ ├── 100_layers.py │ │ │ │ ├── 10_layers.py │ │ │ │ └── __init__.py │ │ │ ├── fixup_0_1/ │ │ │ │ ├── 10000_layers.py │ │ │ │ ├── 1000_layers.py │ │ │ │ ├── 100_layers.py │ │ │ │ ├── 10_layers.py │ │ │ │ └── __init__.py │ │ │ └── fixup_10/ │ │ │ ├── 10000_layers.py │ │ │ ├── 1000_layers.py │ │ │ ├── 100_layers.py │ │ │ ├── 10_layers.py │ │ │ └── __init__.py │ │ └── simple_cnn.py │ ├── dataset.py │ └── resnet_cifar.py ├── imagenet_pipeline/ │ ├── __init__.py │ ├── configs/ │ │ ├── __init__.py │ │ ├── base.py │ │ ├── resnet101_fixup.py │ │ ├── resnet101_fixup_128.py │ │ ├── resnet50.py │ │ ├── resnet50_fixup.py │ │ └── resnet50_fixup_128.py │ └── dataset.py ├── mnist_pipeline/ │ ├── __init__.py │ ├── configs/ │ │ ├── __init__.py │ │ ├── base.py │ │ ├── resnet18.py │ │ └── simple_cnn.py │ ├── dataset.py │ └── tests/ │ ├── __init__.py │ ├── test_dataset.py │ └── test_train.py ├── pipeline/ │ ├── __init__.py │ ├── config_base.py │ ├── core.py │ ├── datasets/ │ │ ├── __init__.py │ │ ├── base.py │ │ └── mixup.py │ ├── logger.py │ ├── losses/ │ │ └── vector_cross_entropy.py │ ├── metrics/ │ │ ├── __init__.py │ │ ├── accuracy.py │ │ └── base.py │ ├── models/ │ │ ├── __init__.py │ │ ├── base.py │ │ └── image_models/ │ │ ├── __init__.py │ │ ├── encoders/ │ │ │ ├── __init__.py │ │ │ └── resnet.py │ │ ├── resnet_fixup.py │ │ ├── wide_resnet.py │ │ └── wide_resnet_fixup.py │ ├── predictors/ │ │ ├── __init__.py │ │ ├── base.py │ │ └── classification.py │ ├── preprocessing/ │ │ ├── __init__.py │ │ ├── audio_preprocessing/ │ │ │ └── __init__.py │ │ ├── image_preprocessing/ │ │ │ └── __init__.py │ │ └── text_preprocessing/ │ │ └── __init__.py │ ├── schedulers/ │ │ ├── __init__.py │ │ ├── base.py │ │ ├── dropout/ │ │ │ ├── __init__.py │ │ │ ├── increase_step.py │ │ │ └── utils.py │ │ └── learning_rate/ │ │ ├── __init__.py │ │ ├── cyclical_lr_scheduler.py │ │ └── reduce_on_plateau.py │ ├── storage/ │ │ ├── __init__.py │ │ ├── predictions.py │ │ └── state.py │ ├── trainers/ │ │ ├── __init__.py │ │ ├── base.py │ │ ├── classification.py │ │ └── segmentation.py │ └── utils.py ├── requirements.txt └── tests/ ├── __init__.py ├── common.py ├── test_metrics.py ├── test_schedulers.py └── test_storage.py ================================================ FILE CONTENTS ================================================ ================================================ FILE: .gitignore ================================================ # Byte-compiled / optimized / DLL files __pycache__/ *.py[cod] *$py.class # C extensions *.so # Distribution / packaging .Python build/ develop-eggs/ dist/ downloads/ eggs/ .eggs/ lib/ lib64/ parts/ sdist/ var/ wheels/ *.egg-info/ .installed.cfg *.egg MANIFEST # PyInstaller # Usually these files are written by a python script from a template # before PyInstaller builds the exe, so as to inject date/other infos into it. *.manifest *.spec # Installer logs pip-log.txt pip-delete-this-directory.txt # Unit test / coverage reports htmlcov/ .tox/ .coverage .coverage.* .cache nosetests.xml coverage.xml *.cover .hypothesis/ .pytest_cache/ # Translations *.mo *.pot # Django stuff: *.log local_settings.py db.sqlite3 # Flask stuff: instance/ .webassets-cache # Scrapy stuff: .scrapy # Sphinx documentation docs/_build/ # PyBuilder target/ # Jupyter Notebook .ipynb_checkpoints # pyenv .python-version # celery beat schedule file celerybeat-schedule # SageMath parsed files *.sage.py # Environments .env .venv env/ venv/ ENV/ env.bak/ venv.bak/ # Spyder project settings .spyderproject .spyproject # Rope project settings .ropeproject # mkdocs documentation /site # mypy .mypy_cache/ # PyCharm .idea/ ================================================ FILE: .travis.yml ================================================ dist: xenial language: python python: - "3.6" # command to install dependencies install: - pip install -r requirements.txt - wget https://www.dropbox.com/s/pzljfuwzo8hpb18/mnist.zip?dl=0 -O mnist.zip - mkdir ~/.pipeline - mkdir ~/.pipeline/mnist - unzip mnist.zip -d ~/.pipeline/mnist/ - free -g # command to run tests script: - pytest -vsx ================================================ FILE: LICENSE ================================================ MIT License Copyright (c) 2019 Pavel Ostyakov Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ================================================ FILE: README.md ================================================ # Pipeline ## How to run training First of all, create a config. You may find some examples of configs in folders mnist_pipeline, cifar_pipeline and imagenet_pipeline. Then, call: `python3 bin/train.py path_to_config` For example, for reproducing results from Fixup paper just call: `python3 bin/train.py cifar_pipeline/configs/resnet110_fixup.py` ================================================ FILE: bin/predict.py ================================================ from pipeline.utils import load_predict_config, run_predict import argparse def main(): parser = argparse.ArgumentParser() parser.add_argument("config_path") args = parser.parse_args() config = load_predict_config(args.config_path) run_predict(config) if __name__ == "__main__": main() ================================================ FILE: bin/train.py ================================================ from pipeline.utils import load_config, run_train import argparse def main(): parser = argparse.ArgumentParser() parser.add_argument("config_path") args = parser.parse_args() config = load_config(args.config_path) run_train(config) if __name__ == "__main__": main() ================================================ FILE: cifar_pipeline/__init__.py ================================================ ================================================ FILE: cifar_pipeline/configs/__init__.py ================================================ ================================================ FILE: cifar_pipeline/configs/base.py ================================================ import torch.nn as nn import torch.optim as optim from torchvision.transforms import ToTensor from cifar_pipeline.dataset import CIFARImagesDataset, CIFARTargetsDataset from pipeline.config_base import ConfigBase from pipeline.datasets.base import DatasetWithPostprocessingFunc, DatasetComposer, OneHotTargetsDataset from pipeline.datasets.mixup import MixUpDatasetWrapper from pipeline.losses.vector_cross_entropy import VectorCrossEntropy from pipeline.metrics.accuracy import MetricsCalculatorAccuracy from pipeline.schedulers.learning_rate.reduce_on_plateau import SchedulerWrapperLossOnPlateau from pipeline.trainers.classification import TrainerClassification TRAIN_DATASET_PATH = "~/.pipeline/cifar/train" TEST_DATASET_PATH = "~/.pipeline/cifar/test" def get_dataset(path, transforms, train, use_mixup): images_dataset = DatasetWithPostprocessingFunc( CIFARImagesDataset(path=path, train=train, download=True), transforms) targets_dataset = CIFARTargetsDataset(path=path, train=train) if use_mixup: targets_dataset = OneHotTargetsDataset(targets_dataset, 10) return DatasetComposer([images_dataset, targets_dataset]) class ConfigCIFARBase(ConfigBase): def __init__(self, model, model_save_path, num_workers=8, batch_size=128, transforms=None, epoch_count=200, print_frequency=10, mixup_alpha=0): optimizer = optim.SGD( model.parameters(), lr=0.1, momentum=0.9, weight_decay=5e-4) scheduler = SchedulerWrapperLossOnPlateau(optimizer) loss = nn.CrossEntropyLoss() metrics_calculator = MetricsCalculatorAccuracy() trainer_cls = TrainerClassification if transforms is None: transforms = ToTensor() train_dataset = get_dataset(path=TRAIN_DATASET_PATH, transforms=transforms, train=True, use_mixup=mixup_alpha > 0) val_dataset = get_dataset(path=TEST_DATASET_PATH, transforms=transforms, train=False, use_mixup=mixup_alpha > 0) if mixup_alpha > 0: train_dataset = MixUpDatasetWrapper(train_dataset, alpha=mixup_alpha) loss = VectorCrossEntropy() super().__init__( model=model, model_save_path=model_save_path, optimizer=optimizer, scheduler=scheduler, loss=loss, metrics_calculator=metrics_calculator, batch_size=batch_size, num_workers=num_workers, train_dataset=train_dataset, val_dataset=val_dataset, trainer_cls=trainer_cls, print_frequency=print_frequency, epoch_count=epoch_count, device="cpu") ================================================ FILE: cifar_pipeline/configs/fixup/base.py ================================================ from cifar_pipeline.dataset import CIFARImagesDataset, CIFARTargetsDataset from pipeline.config_base import ConfigBase from pipeline.schedulers.learning_rate.reduce_on_plateau import SchedulerWrapperLossOnPlateau from pipeline.metrics.accuracy import MetricsCalculatorAccuracy from pipeline.datasets.base import DatasetWithPostprocessingFunc, DatasetComposer, OneHotTargetsDataset from pipeline.trainers.classification import TrainerClassification from pipeline.datasets.mixup import MixUpDatasetWrapper from pipeline.losses.vector_cross_entropy import VectorCrossEntropy import torch.nn as nn import torch.optim as optim from torchvision.transforms import ToTensor, Compose, Normalize TRAIN_DATASET_PATH = "~/.pipeline/cifar/train" TEST_DATASET_PATH = "~/.pipeline/cifar/test" def get_dataset(path, transforms, train, use_mixup): images_dataset = DatasetWithPostprocessingFunc( CIFARImagesDataset(path=path, train=train, download=True), transforms) targets_dataset = CIFARTargetsDataset(path=path, train=train) if use_mixup: targets_dataset = OneHotTargetsDataset(targets_dataset, 10) return DatasetComposer([images_dataset, targets_dataset]) class ConfigCIFARBase(ConfigBase): def __init__(self, model, model_save_path, num_workers=8, batch_size=128, transforms=None, epoch_count=200, print_frequency=10, use_mixup=False): parameters_bias = [p[1] for p in model.named_parameters() if 'bias' in p[0]] parameters_scale = [p[1] for p in model.named_parameters() if 'scale' in p[0]] parameters_others = [p[1] for p in model.named_parameters() if not ('bias' in p[0] or 'scale' in p[0])] optimizer = optim.SGD( [{'params': parameters_bias, 'lr': 0.1/10.}, {'params': parameters_scale, 'lr': 0.1/10.}, {'params': parameters_others}], lr=0.1, momentum=0.9, weight_decay=5e-4) scheduler = SchedulerWrapperLossOnPlateau(optimizer) loss = nn.CrossEntropyLoss() metrics_calculator = MetricsCalculatorAccuracy() trainer_cls = TrainerClassification if transforms is None: transforms = Compose([ToTensor(), Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010))]) train_dataset = get_dataset(path=TRAIN_DATASET_PATH, transforms=transforms, train=True, use_mixup=use_mixup) val_dataset = get_dataset(path=TEST_DATASET_PATH, transforms=transforms, train=False, use_mixup=use_mixup) if use_mixup: train_dataset = MixUpDatasetWrapper(train_dataset, alpha=0.7) loss = VectorCrossEntropy() super().__init__( model=model, model_save_path=model_save_path, optimizer=optimizer, scheduler=scheduler, loss=loss, metrics_calculator=metrics_calculator, batch_size=batch_size, num_workers=num_workers, train_dataset=train_dataset, val_dataset=val_dataset, trainer_cls=trainer_cls, print_frequency=print_frequency, epoch_count=epoch_count) ================================================ FILE: cifar_pipeline/configs/fixup/resnet110_bn.py ================================================ from .base import ConfigCIFARBase from cifar_pipeline.resnet_cifar import resnet110 from torch.nn import DataParallel MODEL_SAVE_PATH = "models/cifar_resnet110_bn" class Config(ConfigCIFARBase): def __init__(self): model = resnet110(use_fixup=False) super().__init__(model=DataParallel(model), model_save_path=MODEL_SAVE_PATH, epoch_count=100, batch_size=128) ================================================ FILE: cifar_pipeline/configs/fixup/resnet110_fixup.py ================================================ from .base import ConfigCIFARBase from cifar_pipeline.resnet_cifar import resnet110 from torch.nn import DataParallel MODEL_SAVE_PATH = "models/cifar_resnet110_fixup" class Config(ConfigCIFARBase): def __init__(self): model = resnet110(use_fixup=True) super().__init__(model=DataParallel(model), model_save_path=MODEL_SAVE_PATH, epoch_count=100, batch_size=128) ================================================ FILE: cifar_pipeline/configs/fixup/resnet110_fixup_0_0_1.py ================================================ from .base import ConfigCIFARBase from cifar_pipeline.resnet_cifar import resnet110 from torch.nn import DataParallel MODEL_SAVE_PATH = "models/cifar_resnet110_fixup_0_0_1" class Config(ConfigCIFARBase): def __init__(self): model = resnet110(use_fixup=True, fixup_coeff=0.01) super().__init__(model=DataParallel(model), model_save_path=MODEL_SAVE_PATH, epoch_count=100, batch_size=128) ================================================ FILE: cifar_pipeline/configs/fixup/resnet110_fixup_0_1.py ================================================ from .base import ConfigCIFARBase from cifar_pipeline.resnet_cifar import resnet110 from torch.nn import DataParallel MODEL_SAVE_PATH = "models/cifar_resnet110_fixup_0_1" class Config(ConfigCIFARBase): def __init__(self): model = resnet110(use_fixup=True, fixup_coeff=0.1) super().__init__(model=DataParallel(model), model_save_path=MODEL_SAVE_PATH, epoch_count=100, batch_size=128) ================================================ FILE: cifar_pipeline/configs/fixup/resnet110_fixup_mixup.py ================================================ from .base import ConfigCIFARBase from cifar_pipeline.resnet_cifar import resnet110 from torch.nn import DataParallel MODEL_SAVE_PATH = "models/cifar_resnet110_fixup_mixup" class Config(ConfigCIFARBase): def __init__(self): model = resnet110(use_fixup=True) super().__init__(model=DataParallel(model), model_save_path=MODEL_SAVE_PATH, epoch_count=100, batch_size=128, use_mixup=True) ================================================ FILE: cifar_pipeline/configs/fixup/wideresnet/__init__.py ================================================ ================================================ FILE: cifar_pipeline/configs/fixup/wideresnet/base.py ================================================ from ..base import ConfigCIFARBase from pipeline.models.image_models.wide_resnet_fixup import WideResNet as WideResNetFixup from pipeline.models.image_models.wide_resnet import WideResNet as WideResNetBatchNorm from enum import auto from torch.nn import DataParallel MODEL_SAVE_PATH = "models/cifar_wideresnet_{}_{}_layers" class ConfigWideResNetBase(ConfigCIFARBase): BATCH_NORM = auto() FIXUP = auto() def __init__(self, num_layers, fixup_coeff=1, normalization_type=BATCH_NORM, batch_size=128): if normalization_type == self.BATCH_NORM: model = WideResNetBatchNorm(depth=num_layers, num_classes=10) norm_type = "batchnorm" else: model = WideResNetFixup(depth=num_layers, num_classes=10, fixup_coeff=fixup_coeff) norm_type = "fixup_coeff_{}".format(fixup_coeff) super().__init__(model=DataParallel(model), model_save_path=MODEL_SAVE_PATH.format(norm_type, num_layers), epoch_count=1, batch_size=batch_size) ================================================ FILE: cifar_pipeline/configs/fixup/wideresnet/batch_norm/10000_layers.py ================================================ from ..base import ConfigWideResNetBase class Config(ConfigWideResNetBase): def __init__(self): super().__init__(num_layers=10000, normalization_type=ConfigWideResNetBase.BATCH_NORM, batch_size=64) ================================================ FILE: cifar_pipeline/configs/fixup/wideresnet/batch_norm/1000_layers.py ================================================ from ..base import ConfigWideResNetBase class Config(ConfigWideResNetBase): def __init__(self): super().__init__(num_layers=1000, normalization_type=ConfigWideResNetBase.BATCH_NORM) ================================================ FILE: cifar_pipeline/configs/fixup/wideresnet/batch_norm/100_layers.py ================================================ from ..base import ConfigWideResNetBase class Config(ConfigWideResNetBase): def __init__(self): super().__init__(num_layers=100, normalization_type=ConfigWideResNetBase.BATCH_NORM) ================================================ FILE: cifar_pipeline/configs/fixup/wideresnet/batch_norm/10_layers.py ================================================ from ..base import ConfigWideResNetBase class Config(ConfigWideResNetBase): def __init__(self): super().__init__(num_layers=10, normalization_type=ConfigWideResNetBase.BATCH_NORM) ================================================ FILE: cifar_pipeline/configs/fixup/wideresnet/batch_norm/__init__.py ================================================ ================================================ FILE: cifar_pipeline/configs/fixup/wideresnet/fixup/10000_layers.py ================================================ from ..base import ConfigWideResNetBase class Config(ConfigWideResNetBase): def __init__(self): super().__init__(num_layers=10000, normalization_type=ConfigWideResNetBase.FIXUP, batch_size=64) ================================================ FILE: cifar_pipeline/configs/fixup/wideresnet/fixup/1000_layers.py ================================================ from ..base import ConfigWideResNetBase class Config(ConfigWideResNetBase): def __init__(self): super().__init__(num_layers=1000, normalization_type=ConfigWideResNetBase.FIXUP) ================================================ FILE: cifar_pipeline/configs/fixup/wideresnet/fixup/100_layers.py ================================================ from ..base import ConfigWideResNetBase class Config(ConfigWideResNetBase): def __init__(self): super().__init__(num_layers=100, normalization_type=ConfigWideResNetBase.FIXUP) ================================================ FILE: cifar_pipeline/configs/fixup/wideresnet/fixup/10_layers.py ================================================ from ..base import ConfigWideResNetBase class Config(ConfigWideResNetBase): def __init__(self): super().__init__(num_layers=10, normalization_type=ConfigWideResNetBase.FIXUP) ================================================ FILE: cifar_pipeline/configs/fixup/wideresnet/fixup/__init__.py ================================================ ================================================ FILE: cifar_pipeline/configs/fixup/wideresnet/fixup_0/10000_layers.py ================================================ from ..base import ConfigWideResNetBase class Config(ConfigWideResNetBase): def __init__(self): super().__init__(num_layers=10000, fixup_coeff=0, normalization_type=ConfigWideResNetBase.FIXUP, batch_size=64) ================================================ FILE: cifar_pipeline/configs/fixup/wideresnet/fixup_0/1000_layers.py ================================================ from ..base import ConfigWideResNetBase class Config(ConfigWideResNetBase): def __init__(self): super().__init__(num_layers=1000, fixup_coeff=0, normalization_type=ConfigWideResNetBase.FIXUP) ================================================ FILE: cifar_pipeline/configs/fixup/wideresnet/fixup_0/100_layers.py ================================================ from ..base import ConfigWideResNetBase class Config(ConfigWideResNetBase): def __init__(self): super().__init__(num_layers=100, fixup_coeff=0, normalization_type=ConfigWideResNetBase.FIXUP) ================================================ FILE: cifar_pipeline/configs/fixup/wideresnet/fixup_0/10_layers.py ================================================ from ..base import ConfigWideResNetBase class Config(ConfigWideResNetBase): def __init__(self): super().__init__(num_layers=10, fixup_coeff=0, normalization_type=ConfigWideResNetBase.FIXUP) ================================================ FILE: cifar_pipeline/configs/fixup/wideresnet/fixup_0/__init__.py ================================================ ================================================ FILE: cifar_pipeline/configs/fixup/wideresnet/fixup_0_0_1/10000_layers.py ================================================ from ..base import ConfigWideResNetBase class Config(ConfigWideResNetBase): def __init__(self): super().__init__(num_layers=10000, fixup_coeff=0.01, normalization_type=ConfigWideResNetBase.FIXUP, batch_size=64) ================================================ FILE: cifar_pipeline/configs/fixup/wideresnet/fixup_0_0_1/1000_layers.py ================================================ from ..base import ConfigWideResNetBase class Config(ConfigWideResNetBase): def __init__(self): super().__init__(num_layers=1000, fixup_coeff=0.01, normalization_type=ConfigWideResNetBase.FIXUP) ================================================ FILE: cifar_pipeline/configs/fixup/wideresnet/fixup_0_0_1/100_layers.py ================================================ from ..base import ConfigWideResNetBase class Config(ConfigWideResNetBase): def __init__(self): super().__init__(num_layers=100, fixup_coeff=0.01, normalization_type=ConfigWideResNetBase.FIXUP) ================================================ FILE: cifar_pipeline/configs/fixup/wideresnet/fixup_0_0_1/10_layers.py ================================================ from ..base import ConfigWideResNetBase class Config(ConfigWideResNetBase): def __init__(self): super().__init__(num_layers=10, fixup_coeff=0.01, normalization_type=ConfigWideResNetBase.FIXUP) ================================================ FILE: cifar_pipeline/configs/fixup/wideresnet/fixup_0_0_1/__init__.py ================================================ ================================================ FILE: cifar_pipeline/configs/fixup/wideresnet/fixup_0_1/10000_layers.py ================================================ from ..base import ConfigWideResNetBase class Config(ConfigWideResNetBase): def __init__(self): super().__init__(num_layers=10000, fixup_coeff=0.1, normalization_type=ConfigWideResNetBase.FIXUP, batch_size=64) ================================================ FILE: cifar_pipeline/configs/fixup/wideresnet/fixup_0_1/1000_layers.py ================================================ from ..base import ConfigWideResNetBase class Config(ConfigWideResNetBase): def __init__(self): super().__init__(num_layers=1000, fixup_coeff=0.1, normalization_type=ConfigWideResNetBase.FIXUP) ================================================ FILE: cifar_pipeline/configs/fixup/wideresnet/fixup_0_1/100_layers.py ================================================ from ..base import ConfigWideResNetBase class Config(ConfigWideResNetBase): def __init__(self): super().__init__(num_layers=100, fixup_coeff=0.1, normalization_type=ConfigWideResNetBase.FIXUP) ================================================ FILE: cifar_pipeline/configs/fixup/wideresnet/fixup_0_1/10_layers.py ================================================ from ..base import ConfigWideResNetBase class Config(ConfigWideResNetBase): def __init__(self): super().__init__(num_layers=10, fixup_coeff=0.1, normalization_type=ConfigWideResNetBase.FIXUP) ================================================ FILE: cifar_pipeline/configs/fixup/wideresnet/fixup_0_1/__init__.py ================================================ ================================================ FILE: cifar_pipeline/configs/fixup/wideresnet/fixup_10/10000_layers.py ================================================ from ..base import ConfigWideResNetBase class Config(ConfigWideResNetBase): def __init__(self): super().__init__(num_layers=10000, fixup_coeff=10, normalization_type=ConfigWideResNetBase.FIXUP, batch_size=64) ================================================ FILE: cifar_pipeline/configs/fixup/wideresnet/fixup_10/1000_layers.py ================================================ from ..base import ConfigWideResNetBase class Config(ConfigWideResNetBase): def __init__(self): super().__init__(num_layers=1000, fixup_coeff=10, normalization_type=ConfigWideResNetBase.FIXUP) ================================================ FILE: cifar_pipeline/configs/fixup/wideresnet/fixup_10/100_layers.py ================================================ from ..base import ConfigWideResNetBase class Config(ConfigWideResNetBase): def __init__(self): super().__init__(num_layers=100, fixup_coeff=10, normalization_type=ConfigWideResNetBase.FIXUP) ================================================ FILE: cifar_pipeline/configs/fixup/wideresnet/fixup_10/10_layers.py ================================================ from ..base import ConfigWideResNetBase class Config(ConfigWideResNetBase): def __init__(self): super().__init__(num_layers=10, fixup_coeff=10, normalization_type=ConfigWideResNetBase.FIXUP) ================================================ FILE: cifar_pipeline/configs/fixup/wideresnet/fixup_10/__init__.py ================================================ ================================================ FILE: cifar_pipeline/configs/simple_cnn.py ================================================ import random import numpy as np import torch import torch.nn as nn from torchvision.transforms import ToTensor from pipeline.models.base import Flatten from .base import ConfigCIFARBase MODEL_SAVE_PATH = "models/cifar_simple_cnn" BATCH_SIZE = 128 SEED = 85 random.seed(SEED) np.random.seed(SEED) torch.random.manual_seed(SEED) def get_model(): model = nn.Sequential( nn.Conv2d(3, 16, kernel_size=3, padding=1), nn.ReLU(), nn.MaxPool2d(kernel_size=2), nn.Conv2d(16, 64, kernel_size=3, padding=1), nn.ReLU(), nn.MaxPool2d(kernel_size=2), nn.Conv2d(64, 128, kernel_size=3, padding=1), nn.ReLU(), nn.Conv2d(128, 128, kernel_size=3, padding=1), nn.ReLU(), nn.AdaptiveAvgPool2d(1), Flatten(), nn.Linear(128, 10) ) return model class Config(ConfigCIFARBase): def __init__(self): model = get_model() transforms = ToTensor() super().__init__(model=model, model_save_path=MODEL_SAVE_PATH, epoch_count=2, batch_size=BATCH_SIZE, transforms=transforms) ================================================ FILE: cifar_pipeline/dataset.py ================================================ import torch.utils.data as data from torchvision.datasets.cifar import CIFAR10 class CIFARDataset(data.Dataset): def __init__(self, path, download=True, train=True): self._dataset = CIFAR10(path, download=download, train=train) def get_image(self, item): return self._dataset[item][0] def get_class(self, item): return self._dataset[item][1] def __len__(self): return len(self._dataset) def __getitem__(self, item): return self._dataset[item] class CIFARImagesDataset(CIFARDataset): def __getitem__(self, item): return self.get_image(item) class CIFARTargetsDataset(CIFARDataset): def __getitem__(self, item): return self.get_class(item) ================================================ FILE: cifar_pipeline/resnet_cifar.py ================================================ import torch.nn as nn import torch.nn.functional as F import torch.nn.init as init import torch import math def _weights_init(m): if isinstance(m, nn.Linear) or isinstance(m, nn.Conv2d): init.kaiming_normal(m.weight) class LambdaLayer(nn.Module): def __init__(self, lambd): super(LambdaLayer, self).__init__() self.lambd = lambd def forward(self, x): return self.lambd(x) class BasicBlock(nn.Module): expansion = 1 m = 2 def __init__(self, in_planes, planes, stride=1, use_fixup=False, fixup_l=1, fixup_coeff=1): super(BasicBlock, self).__init__() self._use_fixup = use_fixup self._fixup_l = fixup_l self._fixup_coeff = fixup_coeff self.conv1 = nn.Conv2d(in_planes, planes, kernel_size=3, stride=stride, padding=1, bias=False) self.bn1 = nn.BatchNorm2d(planes) self.conv2 = nn.Conv2d(planes, planes, kernel_size=3, stride=1, padding=1, bias=False) self.bn2 = nn.BatchNorm2d(planes) self.shortcut = nn.Sequential() if stride != 1 or in_planes != planes: self.shortcut = LambdaLayer(lambda x: F.pad(x[:, :, ::2, ::2], (0, 0, 0, 0, planes//4, planes//4), "constant", 0)) if use_fixup: self.scale = nn.Parameter(torch.ones(1)) self.biases = nn.ParameterList([nn.Parameter(torch.zeros(1)) for _ in range(4)]) k = self.conv1.kernel_size[0] * self.conv1.kernel_size[1] * self.conv1.out_channels self.conv1.weight.data.normal_(0, fixup_coeff * fixup_l ** (-1 / (2 * self.m - 2)) * math.sqrt(2. / k)) self.conv2.weight.data.zero_() def forward(self, x): if self._use_fixup: out = F.relu(self.conv1(x + self.biases[0]) + self.biases[1]) out = self.scale * self.conv2(out + self.biases[2]) + self.biases[3] else: out = F.relu(self.bn1(self.conv1(x))) out = self.bn2(self.conv2(out)) out += self.shortcut(x) out = F.relu(out) return out class ResNet(nn.Module): def __init__(self, block, num_blocks, num_classes=10, use_fixup=False, fixup_coeff=1): super(ResNet, self).__init__() self.in_planes = 16 fixup_l = sum(num_blocks) self.conv1 = nn.Conv2d(3, 16, kernel_size=3, stride=1, padding=1, bias=False) self.bn1 = nn.BatchNorm2d(16) if not use_fixup else nn.Sequential() self.layer1 = self._make_layer(block, 16, num_blocks[0], stride=1, use_fixup=use_fixup, fixup_l=fixup_l, fixup_coeff=fixup_coeff) self.layer2 = self._make_layer(block, 32, num_blocks[1], stride=2, use_fixup=use_fixup, fixup_l=fixup_l, fixup_coeff=fixup_coeff) self.layer3 = self._make_layer(block, 64, num_blocks[2], stride=2, use_fixup=use_fixup, fixup_l=fixup_l, fixup_coeff=fixup_coeff) self.linear = nn.Linear(64, num_classes) self.bias1 = nn.Parameter(torch.zeros(1)) self.bias2 = nn.Parameter(torch.zeros(1)) if not use_fixup: self.apply(_weights_init) else: self.linear.weight.data.zero_() self.linear.bias.data.zero_() k = self.conv1.kernel_size[0] * self.conv1.kernel_size[1] * self.conv1.out_channels self.conv1.weight.data.normal_(0, math.sqrt(2. / k)) def _make_layer(self, block, planes, num_blocks, stride, use_fixup, fixup_l, fixup_coeff): strides = [stride] + [1]*(num_blocks-1) layers = [] for stride in strides: layers.append(block(self.in_planes, planes, stride, use_fixup, fixup_l, fixup_coeff)) self.in_planes = planes * block.expansion return nn.Sequential(*layers) def forward(self, x): out = F.relu(self.bn1(self.conv1(x)) + self.bias1) out = self.layer1(out) out = self.layer2(out) out = self.layer3(out) out = F.avg_pool2d(out, out.size()[3]) out = out.view(out.size(0), -1) out = self.linear(out + self.bias2) return out def resnet110(use_fixup=False, fixup_coeff=1): return ResNet(BasicBlock, [18, 18, 18], use_fixup=use_fixup, fixup_coeff=fixup_coeff) ================================================ FILE: imagenet_pipeline/__init__.py ================================================ ================================================ FILE: imagenet_pipeline/configs/__init__.py ================================================ ================================================ FILE: imagenet_pipeline/configs/base.py ================================================ from imagenet_pipeline.dataset import ImageNetImagesDataset, ImageNetTargetsDataset from pipeline.config_base import ConfigBase from pipeline.schedulers.learning_rate.reduce_on_plateau import SchedulerWrapperLossOnPlateau from pipeline.metrics.accuracy import MetricsCalculatorAccuracy from pipeline.datasets.base import DatasetWithPostprocessingFunc, DatasetComposer, OneHotTargetsDataset from pipeline.trainers.classification import TrainerClassification from pipeline.datasets.mixup import MixUpDatasetWrapper from pipeline.losses.vector_cross_entropy import VectorCrossEntropy import torch.nn as nn import torch.optim as optim from torchvision.transforms import ToTensor, Compose, Normalize TRAIN_DATASET_PATH = "~/train" TEST_DATASET_PATH = "~/val" def get_dataset(path, transforms, use_mixup): images_dataset = DatasetWithPostprocessingFunc( ImageNetImagesDataset(path=path), transforms) targets_dataset = ImageNetTargetsDataset(path=path) if use_mixup: targets_dataset = OneHotTargetsDataset(targets_dataset, 1000) return DatasetComposer([images_dataset, targets_dataset]) class ConfigImageNetBase(ConfigBase): def __init__(self, model, model_save_path, num_workers=16, batch_size=128, learning_rate=0.1, transforms=None, use_mixup=False): parameters_bias = [p[1] for p in model.named_parameters() if 'bias' in p[0]] parameters_scale = [p[1] for p in model.named_parameters() if 'scale' in p[0]] parameters_others = [p[1] for p in model.named_parameters() if not ('bias' in p[0] or 'scale' in p[0])] optimizer = optim.SGD( [{'params': parameters_bias, 'lr': learning_rate/10.}, {'params': parameters_scale, 'lr': learning_rate/10.}, {'params': parameters_others}], lr=learning_rate, momentum=0.9, weight_decay=5e-4) scheduler = SchedulerWrapperLossOnPlateau(optimizer) loss = nn.CrossEntropyLoss() metrics_calculator = MetricsCalculatorAccuracy() trainer_cls = TrainerClassification if transforms is None: transforms = Compose([ToTensor(), Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010))]) train_dataset = get_dataset(path=TRAIN_DATASET_PATH, transforms=transforms, use_mixup=use_mixup) val_dataset = get_dataset(path=TEST_DATASET_PATH, transforms=transforms, use_mixup=use_mixup) if use_mixup: train_dataset = MixUpDatasetWrapper(train_dataset, alpha=0.7) loss = VectorCrossEntropy() super().__init__( model=model, model_save_path=model_save_path, optimizer=optimizer, scheduler=scheduler, loss=loss, metrics_calculator=metrics_calculator, batch_size=batch_size, num_workers=num_workers, train_dataset=train_dataset, val_dataset=val_dataset, trainer_cls=trainer_cls, print_frequency=100) ================================================ FILE: imagenet_pipeline/configs/resnet101_fixup.py ================================================ from .base import ConfigImageNetBase from torch.nn import DataParallel from pipeline.models.image_models.resnet_fixup import resnet101 MODEL_SAVE_PATH = "models/imagenet_resnet_101_fixup" class Config(ConfigImageNetBase): def __init__(self, model_save_path=MODEL_SAVE_PATH): super().__init__(model=DataParallel(resnet101()), model_save_path=model_save_path, use_mixup=True, batch_size=128 * 8, learning_rate=0.1 * 8) ================================================ FILE: imagenet_pipeline/configs/resnet101_fixup_128.py ================================================ from .base import ConfigImageNetBase from torch.nn import DataParallel from pipeline.models.image_models.resnet_fixup import resnet101 MODEL_SAVE_PATH = "models/imagenet_resnet_101_fixup_128" class Config(ConfigImageNetBase): def __init__(self, model_save_path=MODEL_SAVE_PATH): super().__init__(model=DataParallel(resnet101()), model_save_path=model_save_path, use_mixup=True, batch_size=128, learning_rate=0.1) ================================================ FILE: imagenet_pipeline/configs/resnet50.py ================================================ from .base import ConfigImageNetBase from torch.nn import DataParallel from torchvision.models import resnet50 MODEL_SAVE_PATH = "models/imagenet_resnet_50" class Config(ConfigImageNetBase): def __init__(self, model_save_path=MODEL_SAVE_PATH): super().__init__(model=DataParallel(resnet50()), model_save_path=model_save_path) ================================================ FILE: imagenet_pipeline/configs/resnet50_fixup.py ================================================ from .base import ConfigImageNetBase from torch.nn import DataParallel from pipeline.models.image_models.resnet_fixup import resnet50 MODEL_SAVE_PATH = "models/imagenet_resnet_50_fixup" class Config(ConfigImageNetBase): def __init__(self, model_save_path=MODEL_SAVE_PATH): super().__init__(model=DataParallel(resnet50()), model_save_path=model_save_path, use_mixup=True, batch_size=128 * 7, learning_rate=0.1 * 7) ================================================ FILE: imagenet_pipeline/configs/resnet50_fixup_128.py ================================================ from .base import ConfigImageNetBase from torch.nn import DataParallel from pipeline.models.image_models.resnet_fixup import resnet50 MODEL_SAVE_PATH = "models/imagenet_resnet_50_fixup_128" class Config(ConfigImageNetBase): def __init__(self, model_save_path=MODEL_SAVE_PATH): super().__init__(model=DataParallel(resnet50()), model_save_path=model_save_path, use_mixup=True, batch_size=128, learning_rate=0.1) ================================================ FILE: imagenet_pipeline/dataset.py ================================================ from pipeline.core import PipelineError from pipeline.utils import get_path from PIL import Image import torch.utils.data as data import os import glob IMAGE_SIZE = (224, 224) class ImageNetDataset(data.Dataset): def __init__(self, path): path = get_path(path) if not os.path.exists(path): raise PipelineError("Path {} does not exist".format(path)) self._paths = sorted(glob.glob(os.path.join(path, "*/*.JPEG"))) classes = set() for path in self._paths: class_name = os.path.basename(os.path.dirname(path)) classes.add(class_name) classes = sorted(list(classes)) self._class_to_id = dict((class_name, i) for i, class_name in enumerate(classes)) def get_image(self, item): path = self._paths[item] image = Image.open(path).resize(IMAGE_SIZE).convert("RGB") return image def get_class(self, item): path = self._paths[item] class_name = os.path.basename(os.path.dirname(path)) result = self._class_to_id[class_name] return result def __len__(self): return len(self._paths) def __getitem__(self, item): return self.get_image(item), self.get_class(item) class ImageNetImagesDataset(ImageNetDataset): def __getitem__(self, item): return self.get_image(item) class ImageNetTargetsDataset(ImageNetDataset): def __getitem__(self, item): return self.get_class(item) ================================================ FILE: mnist_pipeline/__init__.py ================================================ ================================================ FILE: mnist_pipeline/configs/__init__.py ================================================ ================================================ FILE: mnist_pipeline/configs/base.py ================================================ from mnist_pipeline.dataset import MNISTImagesDataset, MNISTTargetsDataset from pipeline.config_base import ConfigBase, PredictConfigBase from pipeline.schedulers.learning_rate.reduce_on_plateau import SchedulerWrapperLossOnPlateau from pipeline.metrics.accuracy import MetricsCalculatorAccuracy from pipeline.datasets.base import DatasetWithPostprocessingFunc, DatasetComposer from pipeline.trainers.classification import TrainerClassification from pipeline.predictors.classification import PredictorClassification import torch.nn as nn import torch.optim as optim from torchvision.transforms import ToTensor TRAIN_DATASET_PATH = "~/.pipeline/mnist/train.csv" TEST_DATASET_PATH = "~/.pipeline/mnist/test.csv" VAL_RATIO = 0.2 def get_dataset(mode, transforms): images_dataset = DatasetWithPostprocessingFunc( MNISTImagesDataset(path=TRAIN_DATASET_PATH, mode=mode, val_ratio=VAL_RATIO), transforms) targets_dataset = MNISTTargetsDataset( path=TRAIN_DATASET_PATH, mode=mode, val_ratio=VAL_RATIO) return DatasetComposer([images_dataset, targets_dataset]) class ConfigMNISTBase(ConfigBase): def __init__(self, model, model_save_path, num_workers=4, batch_size=128, transforms=None): optimizer = optim.Adam(model.parameters()) scheduler = SchedulerWrapperLossOnPlateau(optimizer) loss = nn.CrossEntropyLoss() metrics_calculator = MetricsCalculatorAccuracy() trainer_cls = TrainerClassification if transforms is None: transforms = ToTensor() train_dataset = get_dataset(mode=MNISTImagesDataset.MODE_TRAIN, transforms=transforms) val_dataset = get_dataset(mode=MNISTImagesDataset.MODE_VAL, transforms=transforms) super().__init__( model=model, model_save_path=model_save_path, optimizer=optimizer, scheduler=scheduler, loss=loss, metrics_calculator=metrics_calculator, batch_size=batch_size, num_workers=num_workers, train_dataset=train_dataset, val_dataset=val_dataset, trainer_cls=trainer_cls) class PredictConfigMNISTBase(PredictConfigBase): def __init__(self, model, model_save_path, num_workers=4, batch_size=128): predictor_cls = PredictorClassification images_dataset = DatasetWithPostprocessingFunc( MNISTImagesDataset(path=TRAIN_DATASET_PATH, mode=MNISTImagesDataset.MODE_VAL, val_ratio=VAL_RATIO), ToTensor()) dataset = DatasetComposer([images_dataset, list(range(len(images_dataset)))]) super().__init__( model=model, model_save_path=model_save_path, dataset=dataset, predictor_cls=predictor_cls, num_workers=num_workers, batch_size=batch_size) ================================================ FILE: mnist_pipeline/configs/resnet18.py ================================================ from .base import ConfigMNISTBase from pipeline.models.image_models.encoders.resnet import Resnet18FeatureExtractor import torch.nn as nn class Config(ConfigMNISTBase): def __init__(self, model_save_path="models/resnet18"): model = nn.Sequential( Resnet18FeatureExtractor(input_channels=1), nn.Linear(Resnet18FeatureExtractor.NUM_FEATURES, 10) ) super().__init__(model=model, model_save_path=model_save_path) ================================================ FILE: mnist_pipeline/configs/simple_cnn.py ================================================ from .base import ConfigMNISTBase, PredictConfigMNISTBase from pipeline.models.base import Flatten import torch.nn as nn MODEL_SAVE_PATH = "models/simple_cnn" def get_model(): model = nn.Sequential( nn.Conv2d(1, 16, kernel_size=3, padding=1), nn.ReLU(), nn.MaxPool2d(kernel_size=2), nn.Conv2d(16, 64, kernel_size=3, padding=1), nn.ReLU(), nn.MaxPool2d(kernel_size=2), nn.Conv2d(64, 128, kernel_size=3, padding=1), nn.ReLU(), nn.Conv2d(128, 128, kernel_size=3, padding=1), nn.ReLU(), nn.AdaptiveAvgPool2d(1), Flatten(), nn.Linear(128, 10) ) return model class Config(ConfigMNISTBase): def __init__(self, model_save_path=MODEL_SAVE_PATH): super().__init__(model=get_model(), model_save_path=model_save_path) class PredictConfig(PredictConfigMNISTBase): def __init__(self, model_save_path=MODEL_SAVE_PATH): super().__init__(model=get_model(), model_save_path=model_save_path) ================================================ FILE: mnist_pipeline/dataset.py ================================================ from pipeline.core import PipelineError from pipeline.utils import get_path import torch.utils.data as data from enum import auto import os import pandas as pd class MNISTDataset(data.Dataset): MODE_TRAIN = auto() MODE_VAL = auto() def __init__(self, path, mode, val_ratio): path = get_path(path) if not os.path.exists(path): raise PipelineError("Path {} does not exist".format(path)) dataset = pd.read_csv(path).values train_length = int(len(dataset) * (1 - val_ratio)) if mode == self.MODE_TRAIN: dataset = dataset[:train_length] else: dataset = dataset[train_length:] self._dataset = dataset def __len__(self): return len(self._dataset) def __getitem__(self, item): row = self._dataset[item] image = row[1:].reshape(28, 28, 1).astype("uint8") target = int(row[0]) return image, target class MNISTImagesDataset(MNISTDataset): def __init__(self, path, mode, val_ratio): super().__init__(path, mode, val_ratio) def __getitem__(self, item): image, _ = super().__getitem__(item) return image class MNISTTargetsDataset(MNISTDataset): def __init__(self, path, mode, val_ratio): super().__init__(path, mode, val_ratio) def __getitem__(self, item): _, target = super().__getitem__(item) return target ================================================ FILE: mnist_pipeline/tests/__init__.py ================================================ ================================================ FILE: mnist_pipeline/tests/test_dataset.py ================================================ from mnist_pipeline.dataset import MNISTDataset, MNISTImagesDataset, MNISTTargetsDataset from mnist_pipeline.configs.base import TRAIN_DATASET_PATH from pipeline.utils import get_path import os class TestMNISTDataset: def setup(self): assert os.path.exists(get_path(TRAIN_DATASET_PATH)), "You need to download MNIST dataset first" def test_train_dataset(self): dataset = MNISTDataset(TRAIN_DATASET_PATH, mode=MNISTDataset.MODE_TRAIN, val_ratio=0.2) assert len(dataset) == 33600 _, _ = dataset[33599] image, target = dataset[0] assert 0 <= target < 10 assert image.shape == (28, 28, 1) def test_val_dataset(self): dataset = MNISTDataset(TRAIN_DATASET_PATH, mode=MNISTDataset.MODE_VAL, val_ratio=0.2) assert len(dataset) == 8400 _, _ = dataset[8399] image, target = dataset[0] assert 0 <= target < 10 assert image.shape == (28, 28, 1) dataset = MNISTDataset(TRAIN_DATASET_PATH, mode=MNISTDataset.MODE_VAL, val_ratio=0) assert len(dataset) == 0 def test_images_dataset(self): dataset = MNISTImagesDataset(TRAIN_DATASET_PATH, mode=MNISTDataset.MODE_VAL, val_ratio=1) image = dataset[10] assert image.shape == (28, 28, 1) assert image.min() >= 0 assert 1 <= image.max() <= 255 def test_targets_dataset(self): dataset = MNISTTargetsDataset(TRAIN_DATASET_PATH, mode=MNISTDataset.MODE_TRAIN, val_ratio=0.5234) target = dataset[51] assert 0 <= target <= 9 assert type(target) == int ================================================ FILE: mnist_pipeline/tests/test_train.py ================================================ from mnist_pipeline.configs.simple_cnn import Config, PredictConfig from pipeline.utils import run_train, run_predict import tempfile import shutil import os import hashlib class TestMNISTTrain: def test_mnist_train(self): test_path = tempfile.mkdtemp() config = Config(model_save_path=test_path) config.epoch_count = 2 run_train(config) assert os.path.exists(os.path.join(test_path, "log.txt")) assert os.path.exists(os.path.join(test_path, "epoch_0")) assert os.path.exists(os.path.join(test_path, "epoch_1")) assert not os.path.exists(os.path.join(test_path, "epoch_2")) assert os.path.exists(os.path.join(test_path, "state")) with open(os.path.join(test_path, "epoch_1"), "rb") as fin: model_checkpoint_hash = hashlib.md5(fin.read()).hexdigest() run_train(config) with open(os.path.join(test_path, "epoch_1"), "rb") as fin: new_model_checkpoint_hash = hashlib.md5(fin.read()).hexdigest() assert model_checkpoint_hash == new_model_checkpoint_hash assert not os.path.exists(os.path.join(test_path, "epoch_2")) predict_config = PredictConfig(model_save_path=test_path) run_predict(predict_config) assert os.path.exists(os.path.join(test_path, "predictions", "predictions")) assert os.path.exists(os.path.join(test_path, "predictions", "identifiers")) shutil.rmtree(test_path) ================================================ FILE: pipeline/__init__.py ================================================ ================================================ FILE: pipeline/config_base.py ================================================ from .datasets.base import EmptyDataset from .metrics.base import MetricsCalculatorEmpty from pipeline.schedulers.base import SchedulerWrapperIdentity from .storage.state import StateStorageFile from .storage.predictions import PredictionsStorageFiles import torch import os class ConfigBase: def __init__( self, model, model_save_path, train_dataset, optimizer, loss, trainer_cls, device=None, val_dataset=None, scheduler=None, metrics_calculator=None, batch_size=1, num_workers=0, epoch_count=None, print_frequency=1, state_storage=None): if val_dataset is None: val_dataset = EmptyDataset() if scheduler is None: scheduler = SchedulerWrapperIdentity() if metrics_calculator is None: metrics_calculator = MetricsCalculatorEmpty() if device is None: device = "cuda" if torch.cuda.is_available() else "cpu" if state_storage is None: state_storage = StateStorageFile(os.path.join(model_save_path, "state")) self.model = model self.model_save_path = model_save_path self.train_dataset = train_dataset self.val_dataset = val_dataset self.batch_size = batch_size self.num_workers = num_workers self.scheduler = scheduler self.metrics_calculator = metrics_calculator self.loss = loss self.optimizer = optimizer self.epoch_count = epoch_count self.print_frequency = print_frequency self.trainer_cls = trainer_cls self.device = device self.state_storage = state_storage class PredictConfigBase: def __init__( self, model, model_save_path, dataset, predictor_cls, device=None, batch_size=1, num_workers=0, print_frequency=1, predictions_storage=None): if device is None: device = "cuda" if torch.cuda.is_available() else "cpu" if predictions_storage is None: predictions_storage = PredictionsStorageFiles(os.path.join(model_save_path, "predictions")) self.model = model self.dataset = dataset self.model_save_path = model_save_path self.batch_size = batch_size self.num_workers = num_workers self.print_frequency = print_frequency self.predictor_cls = predictor_cls self.device = device self.predictions_storage = predictions_storage ================================================ FILE: pipeline/core.py ================================================ class PipelineError(Exception): pass ================================================ FILE: pipeline/datasets/__init__.py ================================================ ================================================ FILE: pipeline/datasets/base.py ================================================ import torch.utils.data as data import torch from typing import Sequence class EmptyDataset(data.Dataset): def __len__(self): return 0 def __getitem__(self, item: int): assert False, "This code is unreachable" class DatasetComposer(data.Dataset): def __init__(self, datasets: Sequence): self._datasets = datasets self._dataset_length = len(datasets[0]) for dataset in datasets: assert self._dataset_length == len(dataset) def __len__(self): return self._dataset_length def __getitem__(self, item: int): return tuple(dataset[item] for dataset in self._datasets) class OneHotTargetsDataset(data.Dataset): def __init__(self, targets: Sequence, class_count: int): self._targets = targets self._class_count = class_count def __len__(self): return len(self._targets) def __getitem__(self, item: int): target = self._targets[item] result = torch.zeros(self._class_count, dtype=torch.float32) result[target] = 1 return result class MultiLabelTargetsDataset(data.Dataset): def __init__(self, targets: Sequence, class_count: int): self._targets = targets self._class_count = class_count def __len__(self): return len(self._targets) def __getitem__(self, item: int): target = self._targets[item] result = torch.zeros(self._class_count, dtype=torch.float32) for class_id in target: result[class_id] = 1 return result class DatasetWithPostprocessingFunc(data.Dataset): def __init__(self, dataset, postprocessing_func): self._dataset = dataset self._postprocessing_func = postprocessing_func def __len__(self): return len(self._dataset) def __getitem__(self, item): return self._postprocessing_func(self._dataset[item]) ================================================ FILE: pipeline/datasets/mixup.py ================================================ import torch.utils.data as data import random import numpy as np class MixUpDatasetWrapper(data.Dataset): def __init__(self, dataset, alpha=1): super().__init__() self._dataset = dataset self._alpha = alpha def __len__(self): return len(self._dataset) def __getitem__(self, item): first = self._dataset[item] second = random.choice(self._dataset) coeff = np.random.beta(self._alpha, self._alpha) result = [] for elem1, elem2 in zip(first, second): result.append(elem1 * coeff + elem2 * (1 - coeff)) return tuple(result) ================================================ FILE: pipeline/logger.py ================================================ import logging import sys LOGGER = logging.getLogger() FORMATTER = logging.Formatter("%(asctime)s - %(levelname)s - %(message)s") def setup_logger(out_file=None, stderr=True, stderr_level=logging.INFO, file_level=logging.DEBUG): LOGGER.handlers = [] LOGGER.setLevel(min(stderr_level, file_level)) if stderr: handler = logging.StreamHandler(sys.stderr) handler.setFormatter(FORMATTER) handler.setLevel(stderr_level) LOGGER.addHandler(handler) if out_file is not None: handler = logging.FileHandler(out_file) handler.setFormatter(FORMATTER) handler.setLevel(file_level) LOGGER.addHandler(handler) LOGGER.info("logger set up") return LOGGER ================================================ FILE: pipeline/losses/vector_cross_entropy.py ================================================ import torch import torch.nn as nn class VectorCrossEntropy(nn.Module): def __init__(self): super().__init__() self._log_softmax = nn.LogSoftmax(dim=1) def forward(self, input, target): input = self._log_softmax(input) loss = -torch.sum(input * target) loss = loss / input.shape[0] return loss ================================================ FILE: pipeline/metrics/__init__.py ================================================ ================================================ FILE: pipeline/metrics/accuracy.py ================================================ from .base import MetricsCalculatorBase from ..core import PipelineError from sklearn.metrics import accuracy_score import numpy as np class MetricsCalculatorAccuracy(MetricsCalculatorBase): def __init__(self, border=0.5): super().__init__() self.zero_cache() self._border = border def zero_cache(self): self._predictions = [] self._true_labels = [] def add(self, y_predicted, y_true): self._predictions.append(y_predicted.cpu().data.numpy()) self._true_labels.append(y_true.cpu().data.numpy()) def calculate(self): if not self._predictions: raise PipelineError("You need to add predictions for calculating the accuracy first") y_pred = np.concatenate(self._predictions) y_true = np.concatenate(self._true_labels) if y_pred.shape[-1] == 1: # Binary classification y_pred = (y_pred >= self._border).astype("int") else: y_pred = np.argmax(y_pred, -1) if len(y_true.shape) != 1: y_true = np.argmax(y_true, -1) result = accuracy_score(y_true, y_pred) return {"accuracy": result} ================================================ FILE: pipeline/metrics/base.py ================================================ import abc class MetricsCalculatorBase(abc.ABC): @abc.abstractmethod def zero_cache(self): pass @abc.abstractmethod def add(self, y_predicted, y_true): pass @abc.abstractmethod def calculate(self): pass class MetricsCalculatorEmpty(MetricsCalculatorBase): def zero_cache(self): pass def add(self, y_predicted, y_true): pass def calculate(self): return {} ================================================ FILE: pipeline/models/__init__.py ================================================ ================================================ FILE: pipeline/models/base.py ================================================ import torch.nn as nn class Flatten(nn.Module): def forward(self, x): return x.view(x.shape[0], -1) ================================================ FILE: pipeline/models/image_models/__init__.py ================================================ ================================================ FILE: pipeline/models/image_models/encoders/__init__.py ================================================ ================================================ FILE: pipeline/models/image_models/encoders/resnet.py ================================================ from torchvision.models import resnet import torch.nn as nn class ResnetModelFeatureExtractorBase(nn.Module): def __init__(self, model, input_channels): super().__init__() model.fc = nn.Sequential() model.avgpool = nn.AdaptiveAvgPool2d(1) if input_channels != 3: model.conv1 = nn.Conv2d( input_channels, model.conv1.out_channels, kernel_size=model.conv1.kernel_size, stride=model.conv1.stride, padding=model.conv1.padding, bias=model.conv1.bias) self._model = model def forward(self, input): return self._model(input) class Resnet18FeatureExtractor(ResnetModelFeatureExtractorBase): NUM_FEATURES = 512 def __init__(self, pretrained=True, input_channels=3): model = resnet.resnet18(pretrained=pretrained) super().__init__( model=model, input_channels=input_channels) class Resnet34FeatureExtractor(ResnetModelFeatureExtractorBase): NUM_FEATURES = 512 def __init__(self, pretrained=True, input_channels=3): model = resnet.resnet34(pretrained=pretrained) super().__init__( model=model, input_channels=input_channels) class Resnet50FeatureExtractor(ResnetModelFeatureExtractorBase): NUM_FEATURES = 2048 def __init__(self, pretrained=True, input_channels=3): model = resnet.resnet50(pretrained=pretrained) super().__init__( model=model, input_channels=input_channels) class Resnet101FeatureExtractor(ResnetModelFeatureExtractorBase): NUM_FEATURES = 2048 def __init__(self, pretrained=True, input_channels=3): model = resnet.resnet101(pretrained=pretrained) super().__init__( model=model, input_channels=input_channels) class Resnet152FeatureExtractor(ResnetModelFeatureExtractorBase): NUM_FEATURES = 2048 def __init__(self, pretrained=True, input_channels=3): model = resnet.resnet152(pretrained=pretrained) super().__init__( model=model, input_channels=input_channels) ================================================ FILE: pipeline/models/image_models/resnet_fixup.py ================================================ import torch.nn as nn import math import torch class Bottleneck(nn.Module): expansion = 4 m = 3 def __init__(self, inplanes, planes, stride=1, downsample=None, fixup_l=1): super(Bottleneck, self).__init__() self.conv1 = nn.Conv2d(inplanes, planes, kernel_size=1, bias=False) self.conv2 = nn.Conv2d(planes, planes, kernel_size=3, stride=stride, padding=1, bias=False) self.conv3 = nn.Conv2d(planes, planes * 4, kernel_size=1, bias=False) self.relu = nn.ReLU(inplace=True) self.downsample = downsample self.stride = stride self.scale = nn.Parameter(torch.ones(1)) self.biases = nn.ParameterList([nn.Parameter(torch.zeros(1)) for _ in range(6)]) k = self.conv1.kernel_size[0] * self.conv1.kernel_size[1] * self.conv1.out_channels self.conv1.weight.data.normal_(0, fixup_l ** (-1 / (2 * self.m - 2)) * math.sqrt(2. / k)) k = self.conv2.kernel_size[0] * self.conv2.kernel_size[1] * self.conv2.out_channels self.conv2.weight.data.normal_(0, fixup_l ** (-1 / (2 * self.m - 2)) * math.sqrt(2. / k)) self.conv3.weight.data.zero_() if downsample is not None: k = self.downsample.kernel_size[0] * self.downsample.kernel_size[1] * self.downsample.out_channels self.downsample.weight.data.normal_(0, math.sqrt(2. / k)) def forward(self, x): residual = x out = self.conv1(x + self.biases[0]) out = self.relu(out + self.biases[1]) out = self.conv2(out + self.biases[2]) out = self.relu(out + self.biases[3]) out = self.scale * self.conv3(out + self.biases[4]) + self.biases[5] if self.downsample is not None: residual = self.downsample(x) out += residual out = self.relu(out) return out class ResNet(nn.Module): def __init__(self, block, layers, num_classes=1000, input_channels=3): self.inplanes = 64 super(ResNet, self).__init__() self.conv1 = nn.Conv2d(input_channels, 64, kernel_size=7, stride=2, padding=3, bias=False) self.relu = nn.ReLU(inplace=True) self.maxpool = nn.MaxPool2d(kernel_size=3, stride=2, padding=1) fixup_l = sum(layers) self.layer1 = self._make_layer(block, 64, layers[0], fixup_l=fixup_l) self.layer2 = self._make_layer(block, 128, layers[1], stride=2, fixup_l=fixup_l) self.layer3 = self._make_layer(block, 256, layers[2], stride=2, fixup_l=fixup_l) self.layer4 = self._make_layer(block, 512, layers[3], stride=2, fixup_l=fixup_l) self.avgpool = nn.AvgPool2d(7, stride=1) self.bias1 = nn.Parameter(torch.zeros(1)) self.bias2 = nn.Parameter(torch.zeros(1)) self.fc = nn.Linear(512 * block.expansion, num_classes) self.fc.weight.data.zero_() self.fc.bias.data.zero_() n = self.conv1.kernel_size[0] * self.conv1.kernel_size[1] * self.conv1.out_channels self.conv1.weight.data.normal_(0, math.sqrt(2. / n)) def _make_layer(self, block, planes, blocks, fixup_l, stride=1): downsample = None if stride != 1 or self.inplanes != planes * block.expansion: downsample = nn.Conv2d(self.inplanes, planes * block.expansion, kernel_size=1, stride=stride, bias=True) layers = [] layers.append(block(self.inplanes, planes, stride, downsample, fixup_l=fixup_l)) self.inplanes = planes * block.expansion for i in range(1, blocks): layers.append(block(self.inplanes, planes, fixup_l=fixup_l)) return nn.Sequential(*layers) def forward(self, x): x = self.conv1(x) x = self.relu(x + self.bias1) x = self.maxpool(x) x = self.layer1(x) x = self.layer2(x) x = self.layer3(x) x = self.layer4(x) x = self.avgpool(x) x = x.view(x.size(0), -1) x = self.fc(x + self.bias2) return x def resnet50(**kwargs): model = ResNet(Bottleneck, [3, 4, 6, 3], **kwargs) return model def resnet101(**kwargs): model = ResNet(Bottleneck, [3, 4, 23, 3], **kwargs) return model def resnet152(**kwargs): model = ResNet(Bottleneck, [3, 8, 36, 3], **kwargs) return model ================================================ FILE: pipeline/models/image_models/wide_resnet.py ================================================ """ Wide ResNet by Sergey Zagoruyko and Nikos Komodakis Fixup initialization by Hongyi Zhang, Yann N. Dauphin, Tengyu Ma Based on code by xternalz and Andy Brock: https://github.com/xternalz/WideResNet-pytorch https://github.com/ajbrock/BoilerPlate """ import math import torch import torch.nn as nn import torch.nn.functional as F class BasicBlock(nn.Module): def __init__(self, in_planes, out_planes, stride, dropout=0.0): super(BasicBlock, self).__init__() self.bn1 = nn.BatchNorm2d(in_planes) self.relu1 = nn.ReLU(inplace=True) self.conv1 = nn.Conv2d(in_planes, out_planes, kernel_size=3, stride=stride, padding=1, bias=False) self.bn2 = nn.BatchNorm2d(out_planes) self.relu2 = nn.ReLU(inplace=True) self.conv2 = nn.Conv2d(out_planes, out_planes, kernel_size=3, stride=1, padding=1, bias=False) self.dropout = dropout self.equalInOut = (in_planes == out_planes) self.convShortcut = (not self.equalInOut) and nn.Conv2d(in_planes, out_planes, kernel_size=1, stride=stride, padding=0, bias=False) or None def forward(self, x): if not self.equalInOut: x = self.relu1(self.bn1(x)) else: out = self.relu1(self.bn1(x)) out = self.relu2(self.bn2(self.conv1(out if self.equalInOut else x))) if self.dropout > 0: out = F.dropout(out, p=self.dropout, training=self.training) out = self.conv2(out) return torch.add(x if self.equalInOut else self.convShortcut(x), out) class NetworkBlock(nn.Module): def __init__(self, nb_layers, in_planes, out_planes, block, stride, dropout): super(NetworkBlock, self).__init__() self.layer = self._make_layer(block, in_planes, out_planes, nb_layers, stride, dropout) def _make_layer(self, block, in_planes, out_planes, nb_layers, stride, dropout): layers = [] for i in range(int(nb_layers)): _in_planes = i == 0 and in_planes or out_planes _stride = i == 0 and stride or 1 layers.append(block(_in_planes, out_planes, _stride, dropout=dropout)) return nn.Sequential(*layers) def forward(self, x): return self.layer(x) class WideResNet(nn.Module): def __init__(self, depth, num_classes, widen_factor=1, dropout=0.0): super(WideResNet, self).__init__() nChannels = [16, 16 * widen_factor, 32 * widen_factor, 64 * widen_factor] assert (depth - 4) % 6 == 0, "You need to change the number of layers" n = (depth - 4) / 6 block = BasicBlock self.conv1 = nn.Conv2d(3, nChannels[0], kernel_size=3, stride=1, padding=1, bias=False) self.block1 = NetworkBlock(n, nChannels[0], nChannels[1], block, 1, dropout=dropout) self.block2 = NetworkBlock(n, nChannels[1], nChannels[2], block, 2, dropout=dropout) self.block3 = NetworkBlock(n, nChannels[2], nChannels[3], block, 2, dropout=dropout) self.bn1 = nn.BatchNorm2d(nChannels[3]) self.relu = nn.ReLU(inplace=True) self.fc = nn.Linear(nChannels[3], num_classes) self.nChannels = nChannels[3] for m in self.modules(): if isinstance(m, nn.Conv2d): n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels m.weight.data.normal_(0, math.sqrt(2. / n)) elif isinstance(m, nn.BatchNorm2d): m.weight.data.fill_(1) m.bias.data.zero_() elif isinstance(m, nn.Linear): m.bias.data.zero_() def forward(self, x): out = self.conv1(x) out = self.block1(out) out = self.block2(out) out = self.block3(out) out = self.relu(self.bn1(out)) out = F.adaptive_avg_pool2d(out, 1) out = out.view(-1, self.nChannels) return self.fc(out) ================================================ FILE: pipeline/models/image_models/wide_resnet_fixup.py ================================================ """ Wide ResNet by Sergey Zagoruyko and Nikos Komodakis Fixup initialization by Hongyi Zhang, Yann N. Dauphin, Tengyu Ma Based on code by xternalz and Andy Brock: https://github.com/xternalz/WideResNet-pytorch https://github.com/ajbrock/BoilerPlate """ import math import torch import torch.nn as nn import torch.nn.functional as F class BasicBlock(nn.Module): m = 2 def __init__(self, in_planes, out_planes, stride, dropout, fixup_l, fixup_coeff): super(BasicBlock, self).__init__() self._dropout = dropout self.relu = nn.ReLU(inplace=True) self.conv1 = nn.Conv2d(in_planes, out_planes, kernel_size=3, stride=stride, padding=1, bias=False) self.conv2 = nn.Conv2d(out_planes, out_planes, kernel_size=3, stride=1, padding=1, bias=False) self.equalInOut = in_planes == out_planes self.conv_res = nn.Conv2d(in_planes, out_planes, kernel_size=1, stride=stride, padding=0, bias=False) self.conv_res = not self.equalInOut and self.conv_res or None self.scale = nn.Parameter(torch.ones(1)) self.biases = nn.ParameterList([nn.Parameter(torch.zeros(1)) for _ in range(4)]) k = self.conv1.kernel_size[0] * self.conv1.kernel_size[1] * self.conv1.out_channels self.conv1.weight.data.normal_(0, fixup_coeff * fixup_l ** (-1 / (2 * self.m - 2)) * math.sqrt(2. / k)) self.conv2.weight.data.zero_() if self.conv_res is not None: k = self.conv_res.kernel_size[0] * self.conv_res.kernel_size[1] * self.conv_res.out_channels self.conv_res.weight.data.normal_(0, math.sqrt(2. / k)) def forward(self, x): x_out = self.relu(x + self.biases[0]) out = self.conv1(x_out) + self.biases[1] out = self.relu(out) + self.biases[2] if self._dropout > 0: out = F.dropout(out, p=self._dropout, training=self.training) out = self.scale * self.conv2(out) + self.biases[3] if self.equalInOut: return torch.add(x, out) return torch.add(self.conv_res(x_out), out) class NetworkBlock(nn.Module): def __init__(self, nb_layers, in_planes, out_planes, block, stride, dropout, fixup_l, fixup_coeff): super(NetworkBlock, self).__init__() self.layer = self._make_layer(block, in_planes, out_planes, nb_layers, stride, dropout, fixup_l, fixup_coeff) def _make_layer(self, block, in_planes, out_planes, nb_layers, stride, dropout, fixup_l, fixup_coeff): layers = [] for i in range(int(nb_layers)): _in_planes = i == 0 and in_planes or out_planes _stride = i == 0 and stride or 1 layers.append(block(_in_planes, out_planes, _stride, dropout=dropout, fixup_l=fixup_l, fixup_coeff=fixup_coeff)) return nn.Sequential(*layers) def forward(self, x): return self.layer(x) class WideResNet(nn.Module): def __init__(self, depth, num_classes, widen_factor=1, dropout=0.0, fixup_coeff=1): super(WideResNet, self).__init__() nChannels = [16, 16 * widen_factor, 32 * widen_factor, 64 * widen_factor] assert (depth - 4) % 6 == 0, "You need to change the number of layers" n = (depth - 4) / 6 block = BasicBlock fixup_l = n * 3 self.conv1 = nn.Conv2d(3, nChannels[0], kernel_size=3, stride=1, padding=1, bias=False) self.block1 = NetworkBlock(n, nChannels[0], nChannels[1], block, 1, dropout=dropout, fixup_l=fixup_l, fixup_coeff=fixup_coeff) self.block2 = NetworkBlock(n, nChannels[1], nChannels[2], block, 2, dropout=dropout, fixup_l=fixup_l, fixup_coeff=fixup_coeff) self.block3 = NetworkBlock(n, nChannels[2], nChannels[3], block, 2, dropout=dropout, fixup_l=fixup_l, fixup_coeff=fixup_coeff) self.relu = nn.ReLU(inplace=True) self.fc = nn.Linear(nChannels[3], num_classes) self.nChannels = nChannels[3] self.fc.bias.data.zero_() self.fc.weight.data.zero_() k = self.conv1.kernel_size[0] * self.conv1.kernel_size[1] * self.conv1.out_channels self.conv1.weight.data.normal_(0, math.sqrt(2. / k)) self.bias1 = nn.Parameter(torch.zeros(1)) self.bias2 = nn.Parameter(torch.zeros(1)) def forward(self, x): out = self.conv1(x) + self.bias1 out = self.block1(out) out = self.block2(out) out = self.block3(out) out = self.relu(out) out = F.adaptive_avg_pool2d(out, 1) out = out.view(-1, self.nChannels) return self.fc(out + self.bias2) ================================================ FILE: pipeline/predictors/__init__.py ================================================ ================================================ FILE: pipeline/predictors/base.py ================================================ import time from typing import Iterable import torch import torch.nn as nn from ..logger import LOGGER from ..storage.predictions import PredictionsStorageBase from ..utils import move_to_device, load_model import os class PredictorBase: def __init__( self, model: nn.Module, data_loader: Iterable, print_frequency: None or int, device: str, model_save_path: str, predictions_storage: PredictionsStorageBase) -> None: self.model = model.to(device) self.data_loader = data_loader self.print_frequency = print_frequency self.device = device self.model_save_path = model_save_path self.predictions_storage = predictions_storage def predict_step(self, input_data: torch.Tensor): input_data = move_to_device(input_data, device=self.device) model_output = self.model(input_data) return model_output def log_predict_step(self, step_id: int, predict_time: float): if self.print_frequency is None or step_id % self.print_frequency == 0: LOGGER.info("[{} s] Predict step {}".format(predict_time, step_id)) return True return False def log_predict_completed(self, predict_time: float): LOGGER.info("[{} s] Predict is completed".format(predict_time)) return True def load_last_model(self): if os.path.exists(self.model_save_path): epochs = filter(lambda file: file.startswith("epoch_"), os.listdir(self.model_save_path)) epochs = map(lambda file: int(file[file.find("_") + 1]), epochs) epochs = list(epochs) if epochs: last_model_path = os.path.join(self.model_save_path, "epoch_{}".format(max(epochs))) load_model(self.model, last_model_path) return LOGGER.info("Model not found in {}. Starting to train a model from scratch...".format(self.model_save_path)) def run(self): self.load_last_model() self.model.eval() step_count = 0 start_time = time.time() with torch.no_grad(): for step_id, (input_data, ids) in enumerate(self.data_loader): model_output = self.predict_step(input_data) self.predictions_storage.add_batch(ids, model_output) step_count += 1 predict_time = time.time() - start_time self.log_predict_step(step_id, predict_time) self.predictions_storage.sort_by_id() self.predictions_storage.flush() predict_time = time.time() - start_time self.log_predict_completed(predict_time) return predict_time ================================================ FILE: pipeline/predictors/classification.py ================================================ from .base import PredictorBase import torch class PredictorClassification(PredictorBase): def predict_step(self, input_data: torch.Tensor): result = super().predict_step(input_data) result = torch.softmax(result, dim=-1) return result ================================================ FILE: pipeline/preprocessing/__init__.py ================================================ ================================================ FILE: pipeline/preprocessing/audio_preprocessing/__init__.py ================================================ ================================================ FILE: pipeline/preprocessing/image_preprocessing/__init__.py ================================================ ================================================ FILE: pipeline/preprocessing/text_preprocessing/__init__.py ================================================ ================================================ FILE: pipeline/schedulers/__init__.py ================================================ ================================================ FILE: pipeline/schedulers/base.py ================================================ import abc class SchedulerBase(abc.ABC): @abc.abstractmethod def step(self, loss, metrics, epoch_id): pass class SchedulerWrapperBase(SchedulerBase): def __init__(self, scheduler): self._scheduler = scheduler class SchedulerWrapperIdentity(SchedulerWrapperBase): def __init__(self, *args, **kwargs): super().__init__(None) def step(self, loss, metrics, epoch_id): pass class SchedulerWrapperLossBase(SchedulerWrapperBase): def __init__(self, scheduler): super().__init__(scheduler) def step(self, loss, metrics, epoch_id): return self._scheduler.step(loss, epoch_id) class SchedulerWrapperMetricsMeanBase(SchedulerWrapperBase): def __init__(self, scheduler): super().__init__(scheduler) def step(self, loss, metrics, epoch_id): values = list(metrics.values()) mean_metrics = sum(values) / len(values) return self._scheduler.step(mean_metrics, epoch_id) ================================================ FILE: pipeline/schedulers/dropout/__init__.py ================================================ ================================================ FILE: pipeline/schedulers/dropout/increase_step.py ================================================ from ..base import SchedulerBase from .utils import set_dropout_probability class SchedulerWrapperIncreaseStep(SchedulerBase): def __init__(self, model, epoch_count, initial_value=0, max_value=0.5): self._model = model self._epoch_count = epoch_count self._initial_value = initial_value self._max_value = max_value def step(self, loss, metrics, epoch_id): new_value = (self._max_value - self._initial_value) / self._epoch_count * (epoch_id + 1) set_dropout_probability(self._model, new_value) ================================================ FILE: pipeline/schedulers/dropout/utils.py ================================================ import abc from torch.nn.modules.dropout import _DropoutNd def set_dropout_probability(module, probability): if isinstance(module, _DropoutNd): module.p = probability return for child in module.children(): set_dropout_probability(child, probability) ================================================ FILE: pipeline/schedulers/learning_rate/__init__.py ================================================ ================================================ FILE: pipeline/schedulers/learning_rate/cyclical_lr_scheduler.py ================================================ from ..base import SchedulerWrapperLossBase, SchedulerWrapperMetricsMeanBase from torch.optim.lr_scheduler import CosineAnnealingLR class SchedulerWrapperLossOnCyclic(SchedulerWrapperLossBase): def __init__(self, optimizer, T_max, eta_min=0, last_epoch=-1): scheduler = CosineAnnealingLR( optimizer, T_max=T_max, eta_min=eta_min, last_epoch=last_epoch, ) super().__init__(scheduler) class SchedulerWrapperMetricsMeanOnCyclic(SchedulerWrapperMetricsMeanBase): def __init__(self, optimizer, T_max, eta_min=0, last_epoch=-1): scheduler = CosineAnnealingLR( optimizer, T_max=T_max, eta_min=eta_min, last_epoch=last_epoch, ) super().__init__(scheduler) ================================================ FILE: pipeline/schedulers/learning_rate/reduce_on_plateau.py ================================================ from ..base import SchedulerWrapperLossBase, SchedulerWrapperMetricsMeanBase from torch.optim.lr_scheduler import ReduceLROnPlateau class SchedulerWrapperLossOnPlateau(SchedulerWrapperLossBase): def __init__(self, optimizer, mode="min", factor=0.5, patience=3, verbose=True, cooldown=3, min_lr=1e-8): scheduler = ReduceLROnPlateau( optimizer, mode=mode, factor=factor, patience=patience, verbose=verbose, cooldown=cooldown, min_lr=min_lr ) super().__init__(scheduler) class SchedulerWrapperMetricsMeanOnPlateau(SchedulerWrapperMetricsMeanBase): def __init__(self, optimizer, mode="max", factor=0.5, patience=3, verbose=True, cooldown=3, min_lr=1e-8): scheduler = ReduceLROnPlateau( optimizer, mode=mode, factor=factor, patience=patience, verbose=verbose, cooldown=cooldown, min_lr=min_lr ) super().__init__(scheduler) ================================================ FILE: pipeline/storage/__init__.py ================================================ ================================================ FILE: pipeline/storage/predictions.py ================================================ from ..core import PipelineError import abc import torch import os class PredictionsStorageBase(abc.ABC): @abc.abstractmethod def add(self, identifier, prediction): pass def add_batch(self, identifiers, predictions): for identifier, prediction in zip(identifiers, predictions): self.add(identifier, prediction) @abc.abstractmethod def flush(self): pass @abc.abstractmethod def get_all(self): pass @abc.abstractmethod def get_by_id(self, identifier): pass def get_by_id_batch(self, identifiers): result = [] for identifier in identifiers: result.append(self.get_by_id(identifier)) return torch.stack(result) @abc.abstractmethod def sort_by_id(self): pass class PredictionsStorageFiles(PredictionsStorageBase): def __init__(self, path): if os.path.exists(path) and not os.path.isdir(path): raise PipelineError("{} should be a directory".format(path)) os.makedirs(path, exist_ok=True) self._path = path self._identifiers = [] self._predictions = [] self._identifier_to_element_id = {} if os.path.exists(os.path.join(self._path, "identifiers")): self._load_predictions() def _load_predictions(self): self._identifiers = torch.load(os.path.join(self._path, "identifiers")) self._predictions = torch.load(os.path.join(self._path, "predictions")) assert len(self._identifiers) == len(self._predictions) for i, identifier in enumerate(self._identifiers): self._identifier_to_element_id[identifier] = i def _save_predictions(self): assert len(self._identifiers) == len(self._predictions) with open(os.path.join(self._path, "identifiers"), "wb") as fout: torch.save(self._identifiers, fout) with open(os.path.join(self._path, "predictions"), "wb") as fout: torch.save(self._predictions, fout) def add(self, identifier, prediction): self._identifiers.append(identifier) self._predictions.append(prediction) self._identifier_to_element_id[identifier] = len(self._identifiers) def flush(self): self._save_predictions() def get_all(self): return self._identifiers, self._predictions def get_by_id(self, identifier): if identifier not in self._identifier_to_element_id: raise PipelineError("Key error: {}".format(identifier)) element_id = self._identifier_to_element_id[identifier] return self._predictions[element_id] def sort_by_id(self): result = sorted(zip(self._identifiers, self._predictions), key=lambda x: x[0]) self._identifiers, self._predictions = list(zip(*result)) self.flush() ================================================ FILE: pipeline/storage/state.py ================================================ from ..core import PipelineError import abc import pickle import os class StateStorageBase(abc.ABC): @abc.abstractmethod def has_key(self, key: str): pass @abc.abstractmethod def get_value(self, key: str): pass @abc.abstractmethod def remove_key(self, key: str): pass @abc.abstractmethod def set_value(self, key: str, value: object): pass class StateStorageEmpty(StateStorageBase): def set_value(self, key: str, value: object): pass def get_value(self, key: str): raise PipelineError("Key error: {}".format(key)) def has_key(self, key: str): return False def remove_key(self, key: str): raise PipelineError("Key error: {}".format(key)) class StateStorageFile(StateStorageBase): def __init__(self, path: str): self._path = path if not os.path.exists(path): os.makedirs(os.path.dirname(path), exist_ok=True) with open(path, "wb") as fout: pickle.dump({}, fout) with open(path, "rb") as fin: self._state = pickle.load(fin) def _save(self): with open(self._path, "wb") as fout: pickle.dump(self._state, fout) def has_key(self, key: str): return key in self._state def get_value(self, key: str): if key not in self._state: raise PipelineError("Key error: {}".format(key)) return self._state[key] def set_value(self, key: str, value: object): self._state[key] = value self._save() def remove_key(self, key: str): if key not in self._state: raise PipelineError("Key error: {}".format(key)) del self._state[key] self._save() ================================================ FILE: pipeline/trainers/__init__.py ================================================ ================================================ FILE: pipeline/trainers/base.py ================================================ import time from typing import Iterable import torch import torch.nn as nn from torch.optim import Optimizer from ..core import PipelineError from ..logger import LOGGER from ..metrics.base import MetricsCalculatorBase from pipeline.schedulers.base import SchedulerWrapperMetricsMeanBase, SchedulerWrapperBase from ..storage.state import StateStorageBase from ..utils import move_to_device, save_model, load_model import os class TrainerBase: def __init__( self, model: nn.Module, train_data_loader: Iterable, val_data_loader: Iterable, epoch_count: int, optimizer: Optimizer, scheduler: SchedulerWrapperBase, loss: nn.Module, metrics_calculator: MetricsCalculatorBase, print_frequency: None or int, device: str, model_save_path: str, state_storage: StateStorageBase) -> None: self.model = model.to(device) self.train_data_loader = train_data_loader self.val_data_loader = val_data_loader self.epoch_count = epoch_count self.optimizer = optimizer self.scheduler = scheduler self.loss = loss self.metrics_calculator = metrics_calculator self.print_frequency = print_frequency self.device = device self.model_save_path = model_save_path self.state_storage = state_storage def train_step(self, input_data: torch.Tensor, target: torch.Tensor): input_data = move_to_device(input_data, device=self.device) target = move_to_device(target, device=self.device) model_output = self.model(input_data) self.optimizer.zero_grad() loss = self.loss(model_output, target) loss.backward() self.optimizer.step(closure=None) return loss.cpu().data.numpy() def predict_step(self, input_data: torch.Tensor): input_data = move_to_device(input_data, device=self.device) model_output = self.model(input_data) return model_output def log_train_step(self, epoch_id: int, step_id: int, epoch_time: float, loss: float, mean_loss: float): if self.print_frequency is None or step_id % self.print_frequency == 0: LOGGER.info("[{} s] Epoch {}. Train step {}. Loss {}. Mean loss {}".format( epoch_time, epoch_id, step_id, loss, mean_loss)) return True return False def log_validation_step(self, epoch_id: int, step_id: int, epoch_time: float, loss: float, mean_loss: float): if self.print_frequency is None or step_id % self.print_frequency == 0: LOGGER.info("[{} s] Epoch {}. Validation step {}. Loss {}. Mean loss {}".format( epoch_time, epoch_id, step_id, loss, mean_loss)) return True return False def log_train_epoch(self, epoch_id: int, epoch_time: float, mean_loss: float): LOGGER.info("Training Epoch {} has completed. Time: {}. Mean loss: {}".format( epoch_id, epoch_time, mean_loss)) return True def log_validation_epoch(self, epoch_id: int, epoch_time: float, mean_loss: float, metrics: dict): LOGGER.info("Validation Epoch {} has completed. Time: {}. Mean loss: {}. Metrics: {}".format( epoch_id, epoch_time, mean_loss, str(metrics))) return True def run_train_epoch(self, epoch_id: int): self.model.train() start_time = time.time() mean_loss = 0 step_count = 0 for step_id, (input_data, target) in enumerate(self.train_data_loader): loss = self.train_step(input_data, target) epoch_time = time.time() - start_time mean_loss += loss step_count += 1 self.log_train_step(epoch_id, step_id, epoch_time, loss, mean_loss / step_count) epoch_time = time.time() - start_time mean_loss /= max(step_count, 1) self.log_train_epoch(epoch_id, epoch_time, mean_loss) return epoch_time, mean_loss def run_validation_epoch(self, epoch_id: int): self.model.eval() self.metrics_calculator.zero_cache() mean_loss = 0 step_count = 0 start_time = time.time() with torch.no_grad(): for step_id, (input_data, target) in enumerate(self.val_data_loader): target = move_to_device(target, device=self.device) model_output = self.predict_step(input_data) loss = self.loss(model_output, target) mean_loss += loss step_count += 1 epoch_time = time.time() - start_time self.metrics_calculator.add(model_output, target) self.log_validation_step(epoch_id, step_id, epoch_time, loss, mean_loss / step_count) epoch_time = time.time() - start_time mean_loss /= max(step_count, 1) metrics = self.metrics_calculator.calculate() self.log_validation_epoch(epoch_id, epoch_time, mean_loss, metrics) return epoch_time, mean_loss, metrics def load_optimizer_state(self): if not self.state_storage.has_key("learning_rates"): return learning_rates = self.state_storage.get_value("learning_rates") for learning_rate, param_group in zip(learning_rates, self.optimizer.param_groups): param_group["lr"] = learning_rate def save_optimizer_state(self): learning_rates = [] for param_group in self.optimizer.param_groups: learning_rates.append(float(param_group['lr'])) self.state_storage.set_value("learning_rates", learning_rates) def save_last_model(self, epoch_id): os.makedirs(self.model_save_path, exist_ok=True) model_path = os.path.join(self.model_save_path, "epoch_{}".format(epoch_id)) save_model(self.model, model_path) LOGGER.info("Model was saved in {}".format(model_path)) def load_last_model(self, epoch_id): last_model_path = os.path.join(self.model_save_path, "epoch_{}".format(epoch_id)) load_model(self.model, last_model_path) def run(self): start_epoch_id = 0 if self.state_storage.has_key("start_epoch_id"): start_epoch_id = self.state_storage.get_value("start_epoch_id") try: self.load_last_model(start_epoch_id - 1) except: LOGGER.exception("Exception occurs during loading a model. Starting to train a model from scratch...") else: LOGGER.info("Model not found in {}. Starting to train a model from scratch...".format(self.model_save_path)) self.load_optimizer_state() epoch_id = start_epoch_id while self.epoch_count is None or epoch_id < self.epoch_count: _, mean_train_loss = self.run_train_epoch(epoch_id) if self.val_data_loader is None: if isinstance(self.scheduler, SchedulerWrapperMetricsMeanBase): raise PipelineError("You can't use a scheduler based on metrics without validation data") self.scheduler.step(mean_train_loss, {}, epoch_id) continue _, mean_validation_loss, validation_metrics = self.run_validation_epoch(epoch_id) self.scheduler.step(mean_validation_loss, validation_metrics, epoch_id) self.state_storage.set_value("start_epoch_id", epoch_id + 1) self.save_optimizer_state() self.save_last_model(epoch_id) epoch_id += 1 ================================================ FILE: pipeline/trainers/classification.py ================================================ from .base import TrainerBase class TrainerClassification(TrainerBase): pass ================================================ FILE: pipeline/trainers/segmentation.py ================================================ from .base import TrainerBase class TrainerSegmentation(TrainerBase): pass ================================================ FILE: pipeline/utils.py ================================================ from .logger import setup_logger from torch.utils.data import DataLoader from torch.nn import DataParallel import importlib import torch import os def _load_cls(module_path, cls_name): module_path_fixed = module_path if module_path_fixed.endswith(".py"): module_path_fixed = module_path_fixed[:-3] module_path_fixed = module_path_fixed.replace("/", ".") module = importlib.import_module(module_path_fixed) assert hasattr(module, cls_name), "{} file should contain {} class".format(module_path, cls_name) cls = getattr(module, cls_name) return cls def load_config(config_path: str): return _load_cls(config_path, "Config")() def load_predict_config(config_path: str): return _load_cls(config_path, "PredictConfig")() def move_to_device(tensor: list or tuple or torch.Tensor, device: str): if isinstance(tensor, list): return [move_to_device(elem, device=device) for elem in tensor] if isinstance(tensor, tuple): return (move_to_device(elem, device=device) for elem in tensor) return tensor.to(device) def get_path(path): return os.path.expanduser(path) def save_model(model, path): if isinstance(model, DataParallel): model = model.module with open(path, "wb") as fout: torch.save(model.state_dict(), fout) def load_model(model, path): with open(path, "rb") as fin: state_dict = torch.load(fin) model.load_state_dict(state_dict) def run_train(config): train_data_loader = DataLoader( config.train_dataset, batch_size=config.batch_size, shuffle=True, pin_memory=True, num_workers=config.num_workers) val_data_loader = DataLoader( config.val_dataset, batch_size=config.batch_size, shuffle=False, num_workers=config.num_workers) model = config.model model_save_path = config.model_save_path os.makedirs(model_save_path, exist_ok=True) logger_path = os.path.join(model_save_path, "log.txt") setup_logger(out_file=logger_path) trainer = config.trainer_cls( model=model, train_data_loader=train_data_loader, val_data_loader=val_data_loader, epoch_count=config.epoch_count, optimizer=config.optimizer, scheduler=config.scheduler, loss=config.loss, metrics_calculator=config.metrics_calculator, print_frequency=config.print_frequency, device=config.device, model_save_path=config.model_save_path, state_storage=config.state_storage ) trainer.run() def run_predict(config): data_loader = DataLoader( config.dataset, batch_size=config.batch_size, shuffle=False, pin_memory=True, num_workers=config.num_workers) model = config.model model_save_path = config.model_save_path assert os.path.exists(model_save_path), "{} does not exist".format(model_save_path) logger_path = os.path.join(model_save_path, "log_predict.txt") setup_logger(out_file=logger_path) predictor = config.predictor_cls( model=model, data_loader=data_loader, print_frequency=config.print_frequency, device=config.device, model_save_path=model_save_path, predictions_storage=config.predictions_storage) predictor.run() ================================================ FILE: requirements.txt ================================================ torch>=1.0.0 pandas numpy torchvision scikit-learn Pillow ================================================ FILE: tests/__init__.py ================================================ ================================================ FILE: tests/common.py ================================================ import tempfile import os def make_temp_path(): _, path = tempfile.mkstemp() os.remove(path) return path ================================================ FILE: tests/test_metrics.py ================================================ from pipeline.metrics.accuracy import MetricsCalculatorAccuracy from pipeline.core import PipelineError import pytest class TestClassificationMetrics: def test_accuracy(self): metrics_calculator = MetricsCalculatorAccuracy(border=0.4) with pytest.raises(PipelineError): metrics_calculator.calculate() ================================================ FILE: tests/test_schedulers.py ================================================ from pipeline.schedulers.learning_rate.reduce_on_plateau import SchedulerWrapperLossOnPlateau, SchedulerWrapperMetricsMeanOnPlateau from torch.optim import Adam import torch.nn as nn class TestReduceLROnPlateau: def test_wrapper_loss(self): first_layer = nn.Linear(10, 5) second_layer = nn.Linear(5, 1) optimizer = Adam([{"params": first_layer.parameters(), "lr": 1}, {"params": second_layer.parameters(), "lr": 2}]) scheduler = SchedulerWrapperLossOnPlateau(optimizer, factor=0.5, patience=1, min_lr=0.1, cooldown=2) assert optimizer.param_groups[0]["lr"] == 1 assert optimizer.param_groups[1]["lr"] == 2 scheduler.step(loss=10, metrics={"a": 5}, epoch_id=0) assert optimizer.param_groups[0]["lr"] == 1 assert optimizer.param_groups[1]["lr"] == 2 scheduler.step(loss=11, metrics={"a": 3}, epoch_id=1) assert optimizer.param_groups[0]["lr"] == 1 assert optimizer.param_groups[1]["lr"] == 2 scheduler.step(loss=12, metrics={"a": 1}, epoch_id=2) assert optimizer.param_groups[0]["lr"] == 0.5 assert optimizer.param_groups[1]["lr"] == 1 scheduler.step(loss=13, metrics={"a": 2}, epoch_id=3) scheduler.step(loss=14, metrics={"a": 5}, epoch_id=4) scheduler.step(loss=14, metrics={"a": 2}, epoch_id=5) assert optimizer.param_groups[0]["lr"] == 0.5 assert optimizer.param_groups[1]["lr"] == 1 scheduler.step(loss=14, metrics={"a": 100}, epoch_id=6) assert optimizer.param_groups[0]["lr"] == 0.25 assert optimizer.param_groups[1]["lr"] == 0.5 scheduler.step(loss=9, metrics={"a": 21}, epoch_id=7) scheduler.step(loss=8, metrics={"a": 21}, epoch_id=7) assert optimizer.param_groups[0]["lr"] == 0.25 assert optimizer.param_groups[1]["lr"] == 0.5 scheduler.step(loss=13, metrics={"a": 3}, epoch_id=8) assert optimizer.param_groups[0]["lr"] == 0.25 assert optimizer.param_groups[1]["lr"] == 0.5 scheduler.step(loss=14, metrics=None, epoch_id=9) assert optimizer.param_groups[0]["lr"] == 0.125 assert optimizer.param_groups[1]["lr"] == 0.25 for epoch_id in range(10, 30): scheduler.step(loss=14, metrics={"absd": "asdasd"}, epoch_id=epoch_id) assert optimizer.param_groups[0]["lr"] == 0.1 assert optimizer.param_groups[1]["lr"] == 0.1 def test_wrapper_metrics(self): model = nn.Linear(10, 1) optimizer = Adam(model.parameters(), lr=1) scheduler = SchedulerWrapperMetricsMeanOnPlateau(optimizer, factor=0.5, patience=0, min_lr=0.1, cooldown=0) assert optimizer.param_groups[0]["lr"] == 1 scheduler.step(loss=None, metrics={"a": 1, "b": 1}, epoch_id=0) assert optimizer.param_groups[0]["lr"] == 1 scheduler.step(loss="abacaba", metrics={"a": 1, "b": 0}, epoch_id=1) scheduler.step(loss=-10, metrics={"a": 1, "b": 1}, epoch_id=2) assert optimizer.param_groups[0]["lr"] == 0.25 scheduler.step(loss=123, metrics={"a": 1, "b": 2}, epoch_id=3) assert optimizer.param_groups[0]["lr"] == 0.25 scheduler.step(loss=0, metrics={"a": 2}, epoch_id=4) assert optimizer.param_groups[0]["lr"] == 0.25 scheduler.step(loss=0, metrics={"aasda": 1.1}, epoch_id=5) assert optimizer.param_groups[0]["lr"] == 0.125 for epoch_id in range(6, 20): scheduler.step(loss=0, metrics={"c": 1}, epoch_id=epoch_id) assert optimizer.param_groups[0]["lr"] == 0.1 ================================================ FILE: tests/test_storage.py ================================================ from .common import make_temp_path from pipeline.storage.state import StateStorageEmpty, StateStorageFile from pipeline.core import PipelineError import pytest class TestStateStorageEmpty: def test_set_value(self): state_storage = StateStorageEmpty() state_storage.set_value("key_name", 123) def test_get_value(self): state_storage = StateStorageEmpty() with pytest.raises(PipelineError): state_storage.get_value("some_key") state_storage.set_value("some_key", 123) with pytest.raises(PipelineError): state_storage.get_value("some_key") def test_has_key(self): state_storage = StateStorageEmpty() assert not state_storage.has_key("key") state_storage.set_value("key", "abacaba") assert not state_storage.has_key("key") def test_remove_key(self): state_storage = StateStorageEmpty() with pytest.raises(PipelineError): state_storage.remove_key("abacaba") state_storage.set_value("abacaba", 9.23) with pytest.raises(PipelineError): state_storage.remove_key("abacaba") class TestStateStorageFile: def test_basic(self): path = make_temp_path() state_storage = StateStorageFile(path) assert not state_storage.has_key("key") with pytest.raises(PipelineError): state_storage.remove_key("abacaba") with pytest.raises(PipelineError): state_storage.get_value("some_key") def test_save_load(self): path = make_temp_path() state_storage = StateStorageFile(path) state_storage.set_value("aba", 123) assert state_storage.get_value("aba") == 123 assert state_storage.has_key("aba") state_storage = StateStorageFile(path) assert state_storage.get_value("aba") == 123 assert state_storage.has_key("aba") state_storage.remove_key("aba") assert not state_storage.has_key("aba") state_storage = StateStorageFile(path) assert not state_storage.has_key("aba")