Repository: ContinualAI/continual-learning-baselines Branch: main Commit: 4900462fa58d Files: 102 Total size: 163.4 KB Directory structure: gitextract_69lglkte/ ├── .github/ │ └── FUNDING.yml ├── .gitignore ├── CITATION.cff ├── LICENSE ├── README.md ├── __init__.py ├── example_run.py ├── experiments/ │ ├── __init__.py │ ├── core50/ │ │ ├── __init__.py │ │ └── deep_slda.py │ ├── permuted_mnist/ │ │ ├── __init__.py │ │ ├── agem.py │ │ ├── ewc.py │ │ ├── gem.py │ │ ├── lfl.py │ │ ├── mir.py │ │ ├── naive.py │ │ └── synaptic_intelligence.py │ ├── split_cifar10/ │ │ ├── __init__.py │ │ ├── er_ace.py │ │ ├── er_aml.py │ │ ├── mir.py │ │ ├── online_er_ace.py │ │ ├── online_replay.py │ │ └── supervised_contrastive_replay.py │ ├── split_cifar100/ │ │ ├── __init__.py │ │ ├── agem.py │ │ ├── er_ace.py │ │ ├── er_aml.py │ │ ├── gem.py │ │ ├── icarl.py │ │ ├── lamaml.py │ │ ├── online_replay.py │ │ └── replay.py │ ├── split_mnist/ │ │ ├── __init__.py │ │ ├── cope.py │ │ ├── gdumb.py │ │ ├── generative_replay.py │ │ ├── gss.py │ │ ├── lwf.py │ │ ├── mir.py │ │ ├── naive.py │ │ ├── online_replay.py │ │ ├── rwalk.py │ │ └── synaptic_intelligence.py │ ├── split_tiny_imagenet/ │ │ ├── __init__.py │ │ ├── lamaml.py │ │ ├── lwf.py │ │ ├── mas.py │ │ ├── naive.py │ │ └── packnet.py │ └── utils.py ├── gitbisect_test.sh ├── models/ │ ├── __init__.py │ ├── models.py │ ├── models_lamaml.py │ ├── reduced_resnet18.py │ └── vgg.py └── tests/ ├── __init__.py ├── agem/ │ ├── __init__.py │ └── experiment.py ├── cope/ │ ├── __init__.py │ └── experiment.py ├── dslda/ │ ├── __init__.py │ └── experiment.py ├── er_ace/ │ ├── __init__.py │ └── experiment.py ├── er_aml/ │ ├── __init__.py │ └── experiment.py ├── ewc/ │ ├── __init__.py │ └── experiment.py ├── gdumb/ │ ├── __init__.py │ └── experiment.py ├── gem/ │ ├── __init__.py │ └── experiment.py ├── generative_replay/ │ ├── __init__.py │ └── experiment.py ├── gss/ │ ├── __init__.py │ └── experiment.py ├── iCARL/ │ ├── __init__.py │ └── experiment.py ├── lamaml/ │ ├── __init__.py │ └── experiment.py ├── lfl/ │ ├── __init__.py │ └── experiment.py ├── lwf/ │ ├── __init__.py │ └── experiment.py ├── mas/ │ ├── __init__.py │ └── experiment.py ├── mir/ │ ├── __init__.py │ └── experiment.py ├── packnet/ │ ├── __init__.py │ └── experiment.py ├── rwalk/ │ ├── __init__.py │ └── experiment.py ├── scr/ │ ├── __init__.py │ └── experiment.py ├── synaptic_intelligence/ │ ├── __init__.py │ └── experiment.py ├── target_results.csv ├── test_template.py └── utils.py ================================================ FILE CONTENTS ================================================ ================================================ FILE: .github/FUNDING.yml ================================================ # These are supported funding model platforms custom: ['https://www.continualai.org/supporters'] ================================================ FILE: .gitignore ================================================ # Byte-compiled / optimized / DLL files __pycache__/ *.py[cod] *$py.class .idea/ # C extensions *.so # Distribution / packaging .Python build/ develop-eggs/ dist/ downloads/ eggs/ .eggs/ lib/ lib64/ parts/ sdist/ var/ wheels/ pip-wheel-metadata/ share/python-wheels/ *.egg-info/ .installed.cfg *.egg MANIFEST # PyInstaller # Usually these files are written by a python script from a template # before PyInstaller builds the exe, so as to inject date/other infos into it. *.manifest *.spec # Installer logs pip-log.txt pip-delete-this-directory.txt # Unit test / coverage reports htmlcov/ .tox/ .nox/ .coverage .coverage.* .cache nosetests.xml coverage.xml *.cover *.py,cover .hypothesis/ .pytest_cache/ # Translations *.mo *.pot # Django stuff: *.log local_settings.py db.sqlite3 db.sqlite3-journal # Flask stuff: instance/ .webassets-cache # Scrapy stuff: .scrapy # Sphinx documentation docs/_build/ # PyBuilder target/ # Jupyter Notebook .ipynb_checkpoints # IPython profile_default/ ipython_config.py # pyenv .python-version # pipenv # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. # However, in case of collaboration, if having platform-specific dependencies or dependencies # having no cross-platform support, pipenv may install dependencies that don't work, or not # install all needed dependencies. #Pipfile.lock # PEP 582; used by e.g. github.com/David-OConnor/pyflow __pypackages__/ # Celery stuff celerybeat-schedule celerybeat.pid # SageMath parsed files *.sage.py # Environments env/ venv/ ENV/ env.bak/ venv.bak/ # Spyder project settings .spyderproject .spyproject # Rope project settings .ropeproject # mkdocs documentation /site # mypy .mypy_cache/ .dmypy.json dmypy.json # Pyre type checker .pyre/ ================================================ FILE: CITATION.cff ================================================ # Copyright (c) ContinualAI cff-version: 1.2.0 title: "Avalanche: an End-to-End Library for Continual Learning" authors: - name: "The Avalanche team" message: "If you used Avalanche in your research project, please remember to cite our paper." preferred-citation: authors: - family-names: Lomonaco given-names: Vincenzo - family-names: Pellegrini given-names: Lorenzo - family-names: Cossu given-names: Andrea - family-names: Carta given-names: Antonio - family-names: Graffieti given-names: Gabriele - family-names: Hayes given-names: Tyler L. - family-names: De Lange given-names: Matthias - family-names: Masana given-names: Marc - family-names: Pomponi given-names: Jary - family-names: Van de Ven given-names: Gido - family-names: Mundt given-names: Martin - family-names: She given-names: Qi - family-names: Cooper given-names: Keiland - family-names: Forest given-names: Jeremy - family-names: Belouadah given-names: Eden - family-names: Calderara given-names: Simone - family-names: Parisi given-names: German I. - family-names: Cuzzolin given-names: Fabio - family-names: Tolias given-names: Andreas - family-names: Scardapane given-names: Simone - family-names: Antiga given-names: Luca - family-names: Amhad given-names: Subutai - family-names: Popescu given-names: Adrian - family-names: Kanan given-names: Christopher - family-names: Van de Weijer given-names: Joost - family-names: Tuytelaars given-names: Tinne - family-names: Bacciu given-names: Davide - family-names: Maltoni given-names: Davide title: "Avalanche: an End-to-End Library for Continual Learning" type: proceedings year: 2021 conference: name: "2nd Continual Learning in Computer Vision Workshop" publisher: name: "Proceedings of IEEE Conference on Computer Vision and Pattern Recognition" ================================================ FILE: LICENSE ================================================ MIT License Copyright (c) 2021 ContinualAI Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ================================================ FILE: README.md ================================================
# Continual Learning Baselines **[Avalanche Website](https://avalanche.continualai.org)** | **[Avalanche Repository](https://github.com/ContinualAI/avalanche)**

**This project provides a set of examples with popular continual learning strategies and baselines. You can easily run experiments to reproduce results from original paper or tweak the hyperparameters to get your own results. Sky is the limit!** To guarantee fair implementations, we rely on the **[Avalanche](https://github.com/ContinualAI/avalanche)** library, developed and maintained by *[ContinualAI](https://www.continualai.org/)*. Feel free to check it out and support the project! ## Experiments The tables below describes all the experiments currently implemented in the `experiments` folder, along with their result. The tables are not meant to compare different methods but rather as a reference for their performance. Different methods may use slightly different setups (e.g., starting from a pre-trained model or from scratch), so it does not always make sense to compare them. If an experiment reproduces exactly the results of a paper in terms of `Performance` (even if with different hyper-parameters), it is marked with ✅ on the `Reproduced` column. Otherwise, it is marked with ❌. `Avalanche` means that we could not find any specific paper as reference and we used the performance of Avalanche obtained when the strategy was first add to the library. If the `Performance` is much worse than the expected one, the `bug` tag is used in the `Reproduced` column. Finally, the `Reference` column reports the expected performance, together with a link to the associated paper (if any). Note that the link does not always point to the paper which introduced the strategy, since it sometimes differs from the one we used to get the target performance. ACC means the Average Accuracy on all experiences after training on the last experience. First, we report the results for the **non-online** continual learning case (a.k.a. batch continual learning). Then, we report the results for the **online continual learning** case. ### Batch Continual Learning (non-online) | Benchmarks | Strategy | Scenario | Performance | Reference | Reproduced | |:-------------------:|:-----------------------------------:|:------------------:|:-----------:|:---------------------------------------------------------------------------------------------------------------------------|:--------------| | Permuted MNIST | Less-Forgetful Learning (LFL) | Domain-Incremental | ACC=0.88 | ACC=0.88 | ✅ `Avalanche` | | Permuted MNIST | Elastic Weight Consolidation (EWC) | Domain-Incremental | ACC=0.83 | [ACC=0.94](https://www.pnas.org/content/114/13/3521) | ❌ | | Permuted MNIST | Synaptic Intelligence (SI) | Domain-Incremental | ACC=0.83 | [ACC=0.95](http://proceedings.mlr.press/v70/zenke17a.html) | ❌ | | Split CIFAR-100 | LaMAML | Task-Incremental | ACC=0.70 | [ACC=0.70](https://arxiv.org/abs/2007.13904) | ✅ | | Split CIFAR-100 | iCaRL | Class-Incremental | ACC=0.48 | [ACC=0.50](https://openaccess.thecvf.com/content_cvpr_2017/html/Rebuffi_iCaRL_Incremental_Classifier_CVPR_2017_paper.html) | ✅ | | Split CIFAR-100 | Replay | Class-Incremental | ACC=0.32 | ACC=0.32 | ✅ `Avalanche` | | Split MNIST | RWalk | Task-Incremental | ACC=0.99 | [ACC=0.99](https://openaccess.thecvf.com/content_ECCV_2018/html/Arslan_Chaudhry__Riemannian_Walk_ECCV_2018_paper.html) | ✅ | | Split MNIST | Synaptic Intelligence (SI) | Task-Incremental | ACC=0.97 | [ACC=0.97](http://proceedings.mlr.press/v70/zenke17a.html) | ✅ | | Split MNIST | GDumb | Class-Incremental | ACC=0.97 | [ACC=0.97](https://link.springer.com/chapter/10.1007/978-3-030-58536-5_31) | ✅ | | Split MNIST | GSS_greedy | Class-Incremental | ACC=0.82 | [ACC=0.78](https://arxiv.org/abs/1903.08671) | ❌ | | Split MNIST | Generative Replay (GR) | Class-Incremental | ACC=0.75 | [ACC=0.75](https://arxiv.org/abs/1705.08690) | ✅ | | Split MNIST | Learning without Forgetting (LwF) | Class-Incremental | ACC=0.23 | [ACC=0.23](https://arxiv.org/pdf/1904.07734.pdf) | ✅ | | Split Tiny ImageNet | LaMAML | Task-Incremental | ACC=0.54 | [ACC=0.66](https://arxiv.org/abs/2007.13904) | ❌ | | Split Tiny ImageNet | Learning without Forgetting (LwF) | Task-Incremental | ACC=0.44 | [ACC=0.44](https://arxiv.org/pdf/1904.07734.pdf) | ✅ | | Split Tiny ImageNet | Memory Aware Synapses (MAS) | Task-Incremental | ACC=0.40 | [ACC=0.40](https://doi.org/10.1109/TPAMI.2021.3057446) | ✅ | | Split Tiny ImageNet | PackNet | Task-Incremental | ACC=0.46 | [ACC=0.47](https://doi.org/10.1109/TPAMI.2021.3057446) (Table 4 `SMALL`) | ✅ | ### Online Continual Learning | Benchmarks | Strategy | Scenario | Performance | Reference | Reproduced | |:---------------:|:-----------------------------------:|:------------------:|:-----------:|:----------------------------------------------------------------------------------------------------------|:--------------| | CORe50 | Deep Streaming LDA (DSLDA) | Class-Incremental | ACC=0.79 | [ACC=0.79](https://arxiv.org/abs/1909.01520) | ✅ | | Permuted MNIST | GEM | Domain-Incremental | ACC=0.80 | [ACC=0.83](https://proceedings.neurips.cc/paper/2017/hash/f87522788a2be2d171666752f97ddebb-Abstract.html) | ✅ | | Split CIFAR-10 | Online Replay | Class-Incremental | ACC=0.50 | ACC=0.50 | ✅ `Avalanche` | | Split CIFAR-10 | ER-AML | Class-Incremental | ACC=0.47 | [ACC=0.47](https://openreview.net/forum?id=N8MaByOzUfb) | ✅ | | Split CIFAR-10 | ER-ACE | Class-Incremental | ACC=0.45 | [ACC=0.52](https://openreview.net/forum?id=N8MaByOzUfb) | ✅ | | Split CIFAR-10 | Supervised Contrastive Replay (SCR) | Class-Incremental | ACC=0.36 | [ACC=0.48](https://ieeexplore.ieee.org/document/9522763) | ✅ `Avalanche` | | Permuted MNIST | Average GEM (AGEM) | Domain-Incremental | ACC=0.81 | [ACC=0.81](https://openreview.net/pdf?id=Hkf2_sC5FX) | ✅ | | Split CIFAR-100 | GEM | Task-Incremental | ACC=0.63 | [ACC=0.63](https://proceedings.neurips.cc/paper/2017/hash/f87522788a2be2d171666752f97ddebb-Abstract.html) | ✅ | | Split CIFAR-100 | Average GEM (AGEM) | Task-Incremental | ACC=0.62 | [ACC=0.62](https://openreview.net/pdf?id=Hkf2_sC5FX) | ✅ | | Split CIFAR-100 | ER-ACE | Class-Incremental | ACC=0.24 | [ACC=0.25](https://openreview.net/forum?id=N8MaByOzUfb) | ✅ | | Split CIFAR-100 | ER-AML | Class-Incremental | ACC=0.24 | [ACC=0.24](https://openreview.net/forum?id=N8MaByOzUfb) | ✅ | | Split CIFAR-100 | Online Replay | Class-Incremental | ACC=0.21 | ACC=0.21 | ✅ `Avalanche` | | Split MNIST | CoPE | Class-Incremental | ACC=0.93 | [ACC=0.93](https://arxiv.org/abs/2009.00919) | ✅ | | Split MNIST | Online Replay | Class-Incremental | ACC=0.92 | ACC=0.92 | ✅ `Avalanche` | ## Python dependencies for experiments Outside Python standard library, the main packages required to run the experiments are PyTorch, Avalanche and Pandas. * **Avalanche**: The latest version of this repo requires the latest Avalanche version (from master branch): `pip install git+https://github.com/ContinualAI/avalanche.git`. The CL baselines repo is tagged with the supported Avalanche version (you can browse the tags to check out all the versions). You can install the corresponding Avalanche versions with `pip install avalanche-lib==[version number]`, where `[version number]` is of the form `0.1.0`. For some strategies (e.g., LaMAML) you may need to install Avalanche with extra packages, like `pip install avalanche-lib[extra]`. For more details on how to install Avalanche, please check out the complete guide [here](https://avalanche.continualai.org/getting-started/how-to-install). * **PyTorch**: we recommend to follow [the official guide](https://pytorch.org/get-started/locally/). * **Pandas**: `pip install pandas`. [Official guide](https://pandas.pydata.org/docs/getting_started/install.html#installing-pandas). ## Run experiments with Python Place yourself into the project root folder. Experiments can be run with a python script by simply importing the function from the `experiments` folder and executing it. By default, experiments will run on GPU, when available. The input argument to each experiment is an optional dictionary of parameters to be used in the experiments. If `None`, default parameters (taken from original paper) will be used. ```python from experiments.split_mnist import synaptic_intelligence_smnist # select the experiment # can be None to use default parameters custom_hyperparameters = {'si_lambda': 0.01, 'cuda': -1, 'seed': 3} # run the experiment result = synaptic_intelligence_smnist(custom_hyperparameters) # dictionary of avalanche metrics print(result) ``` ## Command line experiments Place yourself into the project root folder. You should add the project root folder to your PYTHONPATH. For example, on Linux you can set it up globally: ```bash export PYTHONPATH=${PYTHONPATH}:/path/to/continual-learning-baselines ``` or just for the current command: ```bash PYTHONPATH=${PYTHONPATH}:/path/to/continual-learning-baselines command to be executed ``` You can run experiments directly through console with the default parameters. Open the console and run the python file you want by specifying its path. For example, to run Synaptic Intelligence on Split MNIST: ```bash python experiments/split_mnist/synaptic_intelligence.py ``` To execute experiment with custom parameters, please refer to the previous section. ## Run tests Place yourself into the project root folder. You can run all tests with ```bash python -m unittest ``` or you can specify a test by providing the test name in the format `tests.strategy_class_name.test_benchmarkname`. For example to run Synaptic Intelligence on Split MNIST you can run: ```bash python -m unittest tests.SynapticIntelligence.test_smnist ``` ## Cite If you used this repo you automatically used Avalanche, please remember to cite our reference paper published at the [CLVision @ CVPR2021](https://sites.google.com/view/clvision2021/overview?authuser=0) workshop: ["Avalanche: an End-to-End Library for Continual Learning"](https://arxiv.org/abs/2104.00405). This will help us make Avalanche better known in the machine learning community, ultimately making it a better tool for everyone: ``` @InProceedings{lomonaco2021avalanche, title={Avalanche: an End-to-End Library for Continual Learning}, author={Vincenzo Lomonaco and Lorenzo Pellegrini and Andrea Cossu and Antonio Carta and Gabriele Graffieti and Tyler L. Hayes and Matthias De Lange and Marc Masana and Jary Pomponi and Gido van de Ven and Martin Mundt and Qi She and Keiland Cooper and Jeremy Forest and Eden Belouadah and Simone Calderara and German I. Parisi and Fabio Cuzzolin and Andreas Tolias and Simone Scardapane and Luca Antiga and Subutai Amhad and Adrian Popescu and Christopher Kanan and Joost van de Weijer and Tinne Tuytelaars and Davide Bacciu and Davide Maltoni}, booktitle={Proceedings of IEEE Conference on Computer Vision and Pattern Recognition}, series={2nd Continual Learning in Computer Vision Workshop}, year={2021} } ``` ## Contribute to the project We are always looking for new contributors willing to help us in the challenging mission of providing robust experiments to the community. Would you like to join us? The steps are easy! 1. Take a look at the opened issues and find yours 2. Fork this repo and write an experiment (see next section) 3. Submit a PR and receive support from the maintainers 4. Merge the PR, your contribution is now included in the project! ### Write an experiment 1. Create the appropriate script into `experiments/benchmark_folder`. If the benchmark is not present, you can add one. 2. Fill the `experiment.py` file with your code, following the style of the other experiments. The script should return the metrics used by the related test. 3. Add to `tests/target_results.csv` the expected result for your experiment. You can add a number or a list of numbers. 4. Write the unit test in `tests/strategy_folder/experiment.py`. Follow the very simple structure of existing tests. 5. Update table in `README.md`. ### Find the avalanche commit which produced a regression 1. Place yourself into the avalanche folder and make sure you are using the avalanche version from that repository in your python environment (it is usually enough to add `/path/to/avalanche` to your `PYTHONPATH`). 2. Use the `gitbisect_test.sh` (provided in this repository) in combination with `git bisect` to retrieve the avalanche commit introducing the regression. `git bisect start HEAD v0.1.0 -- # HEAD (current version) is bad, v0.1.0 is good` `git bisect run /path/to/gitbisect_test.sh /path/to/continual-learning-baselines optional_test_name` `git bisect reset` 3. The `gitbisect_test.sh` script requires a mandatory parameter pointing to the `continual-learning-baselines` directory and an optional parameter specifying the path to a particular unittest (e.g., `tests.EWC.test_pmnist`). If the second parameter is not given, all the unit tests will be run. 4. The terminal output will tell you which commit introduced the bug 5. You can change the `HEAD` and `v0.1.0` ref to any avalanche commit. ================================================ FILE: __init__.py ================================================ from . import experiments from . import models from . import tests ================================================ FILE: example_run.py ================================================ """ This script shows how to run an experiment on a specific strategy and benchmark. You can override default parameters by providing a dictionary as input to the method. You can find all the parameters used by the experiment in the source file of the experiment. """ # select the experiment from experiments.split_mnist import synaptic_intelligence_smnist # run the experiment with custom parameters (do not provide arguments to use default parameters) synaptic_intelligence_smnist({'learning_rate': 1e-3, 'si_lambda': 1}) ================================================ FILE: experiments/__init__.py ================================================ from . import split_mnist from . import permuted_mnist from . import split_tiny_imagenet from . import split_cifar100 from . import split_cifar10 from . import core50 from . import utils ================================================ FILE: experiments/core50/__init__.py ================================================ from .deep_slda import deep_slda_core50 ================================================ FILE: experiments/core50/deep_slda.py ================================================ import warnings import torch import avalanche as avl from avalanche.evaluation.metrics import loss_metrics, accuracy_metrics, forgetting_metrics from avalanche.logging import InteractiveLogger from torchvision import transforms from experiments.utils import set_seed, create_default_args def deep_slda_core50(override_args=None): """ "Lifelong Machine Learning with Deep Streaming Linear Discriminant Analysis" by Hayes et. al. (2020). https://arxiv.org/abs/1909.01520 """ args = create_default_args({'cuda': 0, 'feature_size': 512, 'batch_size': 512, 'shrinkage': 1e-4, 'plastic_cov': True, 'seed': None}, override_args) set_seed(args.seed) device = torch.device(f"cuda:{args.cuda}" if torch.cuda.is_available() and args.cuda >= 0 else "cpu") _mu = [0.485, 0.456, 0.406] # imagenet normalization _std = [0.229, 0.224, 0.225] transform = transforms.Compose([ transforms.Resize((224, 224)), transforms.ToTensor(), transforms.Normalize(mean=_mu, std=_std) ]) benchmark = avl.benchmarks.CORe50(scenario='nc', train_transform=transform, eval_transform=transform) eval_plugin = avl.training.plugins.EvaluationPlugin( loss_metrics(epoch=True, experience=True, stream=True), accuracy_metrics(epoch=True, experience=True, stream=True), forgetting_metrics(experience=True, stream=True), loggers=[InteractiveLogger()] ) criterion = torch.nn.CrossEntropyLoss() model = avl.models.SLDAResNetModel(device=device, arch='resnet18', imagenet_pretrained=True) cl_strategy = avl.training.StreamingLDA(model, criterion, args.feature_size, num_classes=50, eval_mb_size=args.batch_size, train_mb_size=args.batch_size, train_epochs=1, shrinkage_param=args.shrinkage, streaming_update_sigma=args.plastic_cov, device=device, evaluator=eval_plugin) warnings.warn( "The Deep SLDA example is not perfectly aligned with " "the paper implementation since it does not use a base " "initialization phase and instead starts streming from " "pre-trained weights. Performance should still match.") res = None for i, exp in enumerate(benchmark.train_stream): cl_strategy.train(exp) res = cl_strategy.eval(benchmark.test_stream) return res if __name__ == '__main__': res = deep_slda_core50() print(res) ================================================ FILE: experiments/permuted_mnist/__init__.py ================================================ from .synaptic_intelligence import synaptic_intelligence_pmnist from .gem import gem_pmnist from .ewc import ewc_pmnist from .agem import agem_pmnist from .lfl import lfl_pmnist from .naive import naive_pmnist from .mir import mir_pmnist ================================================ FILE: experiments/permuted_mnist/agem.py ================================================ import avalanche as avl import torch from torch.nn import CrossEntropyLoss from torch.optim import SGD, Adam from avalanche.evaluation import metrics as metrics from models import MLP from experiments.utils import set_seed, create_default_args def agem_pmnist(override_args=None): """ "Efficient Lifelong Learning with A-GEM" by Chaudhry et. al. (2019). https://openreview.net/pdf?id=Hkf2_sC5FX """ args = create_default_args({'cuda': 0, 'patterns_per_exp': 250, 'hidden_size': 256, 'hidden_layers': 2, 'epochs': 1, 'dropout': 0, 'sample_size': 256, 'learning_rate': 0.1, 'train_mb_size': 10, 'seed': None}, override_args) set_seed(args.seed) device = torch.device(f"cuda:{args.cuda}" if torch.cuda.is_available() and args.cuda >= 0 else "cpu") benchmark = avl.benchmarks.PermutedMNIST(17) model = MLP(hidden_size=args.hidden_size, hidden_layers=args.hidden_layers, drop_rate=args.dropout) criterion = CrossEntropyLoss() interactive_logger = avl.logging.InteractiveLogger() evaluation_plugin = avl.training.plugins.EvaluationPlugin( metrics.accuracy_metrics(epoch=True, experience=True, stream=True), loggers=[interactive_logger]) cl_strategy = avl.training.AGEM( model, SGD(model.parameters(), lr=args.learning_rate), criterion, patterns_per_exp=args.patterns_per_exp, sample_size=args.sample_size, train_mb_size=args.train_mb_size, train_epochs=args.epochs, eval_mb_size=128, device=device, evaluator=evaluation_plugin) res = None for experience in benchmark.train_stream: cl_strategy.train(experience) res = cl_strategy.eval(benchmark.test_stream) return res if __name__ == '__main__': res = agem_pmnist() print(res) ================================================ FILE: experiments/permuted_mnist/ewc.py ================================================ import avalanche as avl import torch from torch.nn import CrossEntropyLoss from torch.optim import SGD from avalanche.evaluation import metrics as metrics from models import MLP from experiments.utils import set_seed, create_default_args def ewc_pmnist(override_args=None): """ "Overcoming catastrophic forgetting in neural networks" by Kirkpatrick et. al. (2017). https://www.pnas.org/content/114/13/3521 Results are below the original paper, which scores around 94%. """ args = create_default_args({'cuda': 0, 'ewc_lambda': 1, 'hidden_size': 512, 'hidden_layers': 1, 'epochs': 10, 'dropout': 0, 'ewc_mode': 'separate', 'ewc_decay': None, 'learning_rate': 0.001, 'train_mb_size': 256, 'seed': None}, override_args) set_seed(args.seed) device = torch.device(f"cuda:{args.cuda}" if torch.cuda.is_available() and args.cuda >= 0 else "cpu") benchmark = avl.benchmarks.PermutedMNIST(10) model = MLP(hidden_size=args.hidden_size, hidden_layers=args.hidden_layers, drop_rate=args.dropout) criterion = CrossEntropyLoss() interactive_logger = avl.logging.InteractiveLogger() evaluation_plugin = avl.training.plugins.EvaluationPlugin( metrics.accuracy_metrics(epoch=True, experience=True, stream=True), loggers=[interactive_logger]) cl_strategy = avl.training.EWC( model, SGD(model.parameters(), lr=args.learning_rate), criterion, ewc_lambda=args.ewc_lambda, mode=args.ewc_mode, decay_factor=args.ewc_decay, train_mb_size=args.train_mb_size, train_epochs=args.epochs, eval_mb_size=128, device=device, evaluator=evaluation_plugin) res = None for experience in benchmark.train_stream: cl_strategy.train(experience) res = cl_strategy.eval(benchmark.test_stream) return res if __name__ == '__main__': res = ewc_pmnist() print(res) ================================================ FILE: experiments/permuted_mnist/gem.py ================================================ import avalanche as avl import torch from torch.nn import CrossEntropyLoss from torch.optim import SGD from avalanche.evaluation import metrics as metrics from models import MLP from experiments.utils import set_seed, create_default_args def gem_pmnist(override_args=None): """ "Gradient Episodic Memory for Continual Learning" by Lopez-paz et. al. (2017). https://proceedings.neurips.cc/paper/2017/hash/f87522788a2be2d171666752f97ddebb-Abstract.html """ args = create_default_args({'cuda': 0, 'patterns_per_exp': 1000, 'hidden_size': 100, 'hidden_layers': 2, 'epochs': 1, 'dropout': 0, 'mem_strength': 0.5, 'n_exp': 17, 'learning_rate': 0.1, 'train_mb_size': 10, 'seed': None}, override_args) set_seed(args.seed) device = torch.device(f"cuda:{args.cuda}" if torch.cuda.is_available() and args.cuda >= 0 else "cpu") benchmark = avl.benchmarks.PermutedMNIST(args.n_exp) model = MLP(hidden_size=args.hidden_size, hidden_layers=args.hidden_layers, drop_rate=args.dropout) criterion = CrossEntropyLoss() interactive_logger = avl.logging.InteractiveLogger() evaluation_plugin = avl.training.plugins.EvaluationPlugin( metrics.accuracy_metrics(epoch=True, experience=True, stream=True), loggers=[interactive_logger]) cl_strategy = avl.training.GEM( model, SGD(model.parameters(), lr=args.learning_rate), criterion, patterns_per_exp=args.patterns_per_exp, memory_strength=args.mem_strength, train_mb_size=args.train_mb_size, train_epochs=args.epochs, eval_mb_size=128, device=device, evaluator=evaluation_plugin) res = None for experience in benchmark.train_stream: cl_strategy.train(experience) res = cl_strategy.eval(benchmark.test_stream) return res if __name__ == '__main__': res = gem_pmnist() print(res) ================================================ FILE: experiments/permuted_mnist/lfl.py ================================================ import avalanche as avl import torch from avalanche.evaluation import metrics as metrics from models import MLP from experiments.utils import set_seed, create_default_args def lfl_pmnist(override_args=None): """ "Less-forgetting Learning in Deep Neural Networks" Heechul Jung, Jeongwoo Ju, Minju Jung and Junmo Kim; arXiv, 2016, https://arxiv.org/pdf/1607.00122.pdf The Permuted MNIST benchmark was not used in the original paper. We run LFL on Permuted MNIST for proper comparison with other strategies, since the benchmarks used in the original papers are not commonly used in Continual Learning. """ args = create_default_args({'cuda': 0, 'lambda_e': [0.0001], 'epochs': 3, 'hidden_size': 256, 'hidden_layers': 1, 'learning_rate': 0.01, 'train_mb_size': 128, 'seed': None}, override_args) set_seed(args.seed) device = torch.device(f"cuda:{args.cuda}" if torch.cuda.is_available() and args.cuda >= 0 else "cpu") if not isinstance(args.lambda_e, (list, tuple)): raise ValueError("lambda_e parameter should be a list of floating numbers. " "Provide list with one element to apply the same lambda_e " "to all experiences.") benchmark = avl.benchmarks.PermutedMNIST(4) model = MLP(hidden_size=args.hidden_size, hidden_layers=args.hidden_layers) optimizer = torch.optim.SGD(model.parameters(), lr=args.learning_rate) criterion = torch.nn.CrossEntropyLoss() interactive_logger = avl.logging.InteractiveLogger() eval_plugin = avl.training.plugins.EvaluationPlugin( avl.evaluation.metrics.accuracy_metrics(minibatch=True, epoch=True, experience=True, stream=True), avl.evaluation.metrics.loss_metrics(minibatch=True, epoch=True, experience=True, stream=True), avl.evaluation.metrics.forgetting_metrics(experience=True), loggers=[interactive_logger] ) lambda_e = args.lambda_e[0] if len(args.lambda_e) == 1 else args.lambda_e strategy = avl.training.LFL( model, optimizer, criterion, lambda_e=lambda_e, train_epochs=args.epochs, device=device, train_mb_size=args.train_mb_size, eval_mb_size=256, evaluator=eval_plugin ) res = None for experience in benchmark.train_stream: strategy.train(experience) res = strategy.eval(benchmark.test_stream) return res if __name__ == '__main__': res = lfl_pmnist() print(res) ================================================ FILE: experiments/permuted_mnist/mir.py ================================================ import numpy as np import torch from torch.optim import SGD from avalanche.benchmarks.classic import PermutedMNIST from avalanche.benchmarks import benchmark_with_validation_stream from avalanche.benchmarks.scenarios import split_online_stream from avalanche.evaluation.metrics import accuracy_metrics, loss_metrics from avalanche.logging import InteractiveLogger from avalanche.models import SimpleMLP from avalanche.training.plugins import EvaluationPlugin, MIRPlugin, ReplayPlugin from avalanche.training.supervised import Naive from experiments.utils import create_default_args, set_seed, restrict_dataset_size def mir_pmnist(override_args=None): args = create_default_args( { "cuda": 0, "mem_size": 500, "lr": 0.05, "train_mb_size": 10, "seed": None, "subsample": 50, "batch_size_mem": 10, "dataset_size": 1000, }, override_args ) set_seed(args.seed) device = torch.device( f"cuda:{args.cuda}" if torch.cuda.is_available() and args.cuda >= 0 else "cpu" ) scenario = PermutedMNIST( 10, return_task_id=False, seed=0, train_transform=None, eval_transform=None, ) scenario = benchmark_with_validation_stream(scenario, 0.05) scenario = restrict_dataset_size(scenario, args.dataset_size) model = SimpleMLP(10, hidden_size=400, hidden_layers=1) optimizer = SGD(model.parameters(), lr=args.lr) interactive_logger = InteractiveLogger() loggers = [interactive_logger] training_metrics = [] evaluation_metrics = [ accuracy_metrics(epoch=True, stream=True), loss_metrics(epoch=True, stream=True), ] evaluator = EvaluationPlugin( *training_metrics, *evaluation_metrics, loggers=loggers, ) plugins = [ MIRPlugin( mem_size=args.mem_size, subsample=args.subsample, batch_size_mem=args.batch_size_mem ) ] cl_strategy = Naive( model=model, optimizer=optimizer, plugins=plugins, evaluator=evaluator, device=device, train_mb_size=args.train_mb_size, eval_mb_size=64, ) ocl_scenario = split_online_stream( original_stream=scenario.train_stream, experience_size=10, access_task_boundaries=False, ) for t, experience in enumerate(ocl_scenario): cl_strategy.train( experience, eval_streams=[], num_workers=0, drop_last=True, ) results = cl_strategy.eval(scenario.test_stream) return results if __name__ == "__main__": res = mir_pmnist() print(res) ================================================ FILE: experiments/permuted_mnist/naive.py ================================================ import avalanche as avl import torch from torch.nn import CrossEntropyLoss from torch.optim import Adam from avalanche.evaluation import metrics as metrics from models import MLP from experiments.utils import set_seed, create_default_args def naive_pmnist(override_args=None): """ "Continual Learning Through Synaptic Intelligence" by Zenke et. al. (2017). http://proceedings.mlr.press/v70/zenke17a.html """ args = create_default_args({'cuda': 0, 'epochs': 20, 'learning_rate': 0.001, 'train_mb_size': 256, 'seed': None}, override_args) set_seed(args.seed) device = torch.device(f"cuda:{args.cuda}" if torch.cuda.is_available() and args.cuda >= 0 else "cpu") benchmark = avl.benchmarks.PermutedMNIST(10) model = MLP(hidden_size=2000, hidden_layers=2, relu_act=True) criterion = CrossEntropyLoss() interactive_logger = avl.logging.InteractiveLogger() evaluation_plugin = avl.training.plugins.EvaluationPlugin( metrics.accuracy_metrics(epoch=True, experience=True, stream=True), loggers=[interactive_logger]) cl_strategy = avl.training.Naive( model, Adam(model.parameters(), lr=args.learning_rate), criterion, train_mb_size=args.train_mb_size, train_epochs=args.epochs, eval_mb_size=128, device=device, evaluator=evaluation_plugin) res = None for experience in benchmark.train_stream: cl_strategy.train(experience) res = cl_strategy.eval(benchmark.test_stream) return res if __name__ == '__main__': res = naive_pmnist() print(res) ================================================ FILE: experiments/permuted_mnist/synaptic_intelligence.py ================================================ import avalanche as avl import torch from torch.nn import CrossEntropyLoss from torch.optim import Adam from avalanche.evaluation import metrics as metrics from models import MLP from experiments.utils import set_seed, create_default_args def synaptic_intelligence_pmnist(override_args=None): """ "Continual Learning Through Synaptic Intelligence" by Zenke et. al. (2017). http://proceedings.mlr.press/v70/zenke17a.html Results are below the original paper, which has a score around 97% """ args = create_default_args({'cuda': 0, 'si_lambda': 10, 'si_eps': 0.1, 'epochs': 10, 'learning_rate': 0.001, 'train_mb_size': 256, 'seed': None}, override_args) set_seed(args.seed) device = torch.device(f"cuda:{args.cuda}" if torch.cuda.is_available() and args.cuda >= 0 else "cpu") benchmark = avl.benchmarks.PermutedMNIST(10) model = MLP(hidden_size=1000, hidden_layers=1, relu_act=True) criterion = CrossEntropyLoss() interactive_logger = avl.logging.InteractiveLogger() evaluation_plugin = avl.training.plugins.EvaluationPlugin( metrics.accuracy_metrics(epoch=True, experience=True, stream=True), loggers=[interactive_logger]) cl_strategy = avl.training.SynapticIntelligence( model, Adam(model.parameters(), lr=args.learning_rate), criterion, si_lambda=args.si_lambda, eps=args.si_eps, train_mb_size=args.train_mb_size, train_epochs=args.epochs, eval_mb_size=128, device=device, evaluator=evaluation_plugin) res = None for experience in benchmark.train_stream: cl_strategy.train(experience) res = cl_strategy.eval(benchmark.test_stream) return res if __name__ == '__main__': res = synaptic_intelligence_pmnist() print(res) ================================================ FILE: experiments/split_cifar10/__init__.py ================================================ from .online_replay import online_replay_scifar10 from .mir import mir_scifar10 from .er_ace import erace_scifar10 from .er_aml import eraml_scifar10 from .supervised_contrastive_replay import online_scr_scifar10 ================================================ FILE: experiments/split_cifar10/er_ace.py ================================================ #!/usr/bin/env python3 import numpy as np import torch import torchvision.transforms as transforms from torch.optim import SGD from avalanche.benchmarks.classic import SplitCIFAR10 from avalanche.evaluation.metrics import accuracy_metrics, loss_metrics from avalanche.logging import InteractiveLogger from avalanche.models import SlimResNet18 from avalanche.models.dynamic_modules import IncrementalClassifier from avalanche.training.plugins import EvaluationPlugin from avalanche.training.supervised import Naive, ER_ACE from experiments.utils import create_default_args, set_seed def erace_scifar10(override_args=None): args = create_default_args( { "cuda": 0, "mem_size": 1000, "lr": 0.1, "train_mb_size": 10, "seed": None, "batch_size_mem": 10, }, override_args ) set_seed(args.seed) fixed_class_order = np.arange(10) device = torch.device( f"cuda:{args.cuda}" if torch.cuda.is_available() and args.cuda >= 0 else "cpu" ) unique_transform = transforms.Compose( [ transforms.ToTensor(), transforms.Normalize( (0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010) ), ] ) scenario = SplitCIFAR10( 5, return_task_id=False, seed=0, fixed_class_order=fixed_class_order, shuffle=True, class_ids_from_zero_in_each_exp=False, train_transform=unique_transform, eval_transform=unique_transform, ) input_size = (3, 32, 32) model = SlimResNet18(1) model.linear = IncrementalClassifier(model.linear.in_features, 1) optimizer = SGD(model.parameters(), lr=args.lr) interactive_logger = InteractiveLogger() loggers = [interactive_logger] training_metrics = [] evaluation_metrics = [ accuracy_metrics(epoch=True, stream=True), loss_metrics(epoch=True, stream=True), ] evaluator = EvaluationPlugin( *training_metrics, *evaluation_metrics, loggers=loggers, ) plugins = [] cl_strategy = ER_ACE( model=model, optimizer=optimizer, plugins=plugins, evaluator=evaluator, device=device, train_mb_size=args.train_mb_size, eval_mb_size=64, mem_size=args.mem_size, batch_size_mem=args.batch_size_mem, ) for t, experience in enumerate(scenario.train_stream): cl_strategy.train( experience, num_workers=0, drop_last=True, ) cl_strategy.eval(scenario.test_stream[: t + 1]) results = cl_strategy.eval(scenario.test_stream) return results if __name__ == "__main__": res = erace_scifar10() print(res) ================================================ FILE: experiments/split_cifar10/er_aml.py ================================================ #!/usr/bin/env python3 import numpy as np import torch import torchvision.transforms as transforms from torch.optim import SGD from avalanche.benchmarks.classic import SplitCIFAR10 from avalanche.evaluation.metrics import accuracy_metrics, loss_metrics from avalanche.logging import InteractiveLogger from avalanche.training.plugins import EvaluationPlugin from avalanche.training.supervised import ER_AML from experiments.utils import create_default_args, set_seed from models import SingleHeadReducedResNet18 def eraml_scifar10(override_args=None): """ Reproducing ER-AML experiments from paper "New insights on Reducing Abrupt Representation Change in Online Continual Learning" by Lucas Caccia et. al https://openreview.net/forum?id=N8MaByOzUfb """ args = create_default_args( { "cuda": 0, "mem_size": 1000, "lr": 0.1, "temp": 0.1, "train_mb_size": 10, "seed": None, "batch_size_mem": 10, }, override_args, ) set_seed(args.seed) fixed_class_order = np.arange(10) device = torch.device( f"cuda:{args.cuda}" if torch.cuda.is_available() and args.cuda >= 0 else "cpu" ) unique_transform = transforms.Compose( [ transforms.ToTensor(), transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)), ] ) scenario = SplitCIFAR10( 5, return_task_id=False, seed=0, fixed_class_order=fixed_class_order, shuffle=True, class_ids_from_zero_in_each_exp=False, train_transform=unique_transform, eval_transform=unique_transform, ) input_size = (3, 32, 32) model = SingleHeadReducedResNet18(10) optimizer = SGD(model.parameters(), lr=args.lr) interactive_logger = InteractiveLogger() loggers = [interactive_logger] training_metrics = [] evaluation_metrics = [ accuracy_metrics(epoch=True, stream=True), loss_metrics(epoch=True, stream=True), ] evaluator = EvaluationPlugin( *training_metrics, *evaluation_metrics, loggers=loggers, ) plugins = [] cl_strategy = ER_AML( model=model, feature_extractor=model.feature_extractor, optimizer=optimizer, plugins=plugins, evaluator=evaluator, device=device, train_mb_size=args.train_mb_size, eval_mb_size=64, mem_size=args.mem_size, batch_size_mem=args.batch_size_mem, ) for t, experience in enumerate(scenario.train_stream): cl_strategy.train( experience, num_workers=0, drop_last=True, ) cl_strategy.eval(scenario.test_stream[: t + 1]) results = cl_strategy.eval(scenario.test_stream) return results if __name__ == "__main__": res = eraml_scifar10() print(res) ================================================ FILE: experiments/split_cifar10/mir.py ================================================ import numpy as np import torch import torchvision.transforms as transforms from torch.optim import SGD from avalanche.benchmarks.classic import SplitCIFAR10 from avalanche.benchmarks import benchmark_with_validation_stream from avalanche.benchmarks.scenarios import split_online_stream from avalanche.evaluation.metrics import accuracy_metrics, loss_metrics from avalanche.logging import InteractiveLogger from avalanche.models import SlimResNet18 from avalanche.training.plugins import EvaluationPlugin, MIRPlugin from avalanche.training.supervised import Naive, Naive from experiments.utils import create_default_args, set_seed def mir_scifar10(override_args=None): args = create_default_args( { "cuda": 0, "mem_size": 1000, "lr": 0.05, "train_mb_size": 10, "seed": None, "subsample": 50, "batch_size_mem": 10, }, override_args ) set_seed(args.seed) fixed_class_order = np.arange(10) device = torch.device( f"cuda:{args.cuda}" if torch.cuda.is_available() and args.cuda >= 0 else "cpu" ) scenario = SplitCIFAR10( 5, return_task_id=False, seed=0, fixed_class_order=fixed_class_order, train_transform=transforms.ToTensor(), eval_transform=transforms.ToTensor(), shuffle=True, class_ids_from_zero_in_each_exp=False, ) scenario = benchmark_with_validation_stream(scenario, 0.05) model = SlimResNet18(10) optimizer = SGD(model.parameters(), lr=args.lr) interactive_logger = InteractiveLogger() loggers = [interactive_logger] training_metrics = [] evaluation_metrics = [ accuracy_metrics(epoch=True, stream=True), loss_metrics(epoch=True, stream=True), ] evaluator = EvaluationPlugin( *training_metrics, *evaluation_metrics, loggers=loggers, ) plugins = [ MIRPlugin( mem_size=args.mem_size, subsample=args.subsample, batch_size_mem=args.batch_size_mem ) ] cl_strategy = Naive( model=model, optimizer=optimizer, plugins=plugins, evaluator=evaluator, device=device, train_mb_size=args.train_mb_size, eval_mb_size=64, ) ocl_scenario = split_online_stream( original_stream=scenario.train_stream, experience_size=10, access_task_boundaries=False, ) for t, experience in enumerate(ocl_scenario): cl_strategy.train( experience, eval_streams=[], num_workers=0, drop_last=True, ) results = cl_strategy.eval(scenario.test_stream) return results if __name__ == "__main__": res = mir_scifar10() print(res) ================================================ FILE: experiments/split_cifar10/online_er_ace.py ================================================ #!/usr/bin/env python3 import numpy as np import torch import torchvision.transforms as transforms from torch.optim import SGD from avalanche.benchmarks.classic import SplitCIFAR10 from avalanche.benchmarks.scenarios import split_online_stream from avalanche.evaluation.metrics import accuracy_metrics, loss_metrics from avalanche.logging import InteractiveLogger from avalanche.models import SlimResNet18 from avalanche.models.dynamic_modules import IncrementalClassifier from avalanche.training.plugins import EvaluationPlugin from avalanche.training.supervised import Naive, OnlineER_ACE from experiments.utils import create_default_args, set_seed def eracl_scifar10(override_args=None): args = create_default_args( { "cuda": 0, "mem_size": 1000, "lr": 0.1, "train_mb_size": 10, "seed": None, "batch_size_mem": 10, }, override_args ) set_seed(args.seed) fixed_class_order = np.arange(10) device = torch.device( f"cuda:{args.cuda}" if torch.cuda.is_available() and args.cuda >= 0 else "cpu" ) scenario = SplitCIFAR10( 5, return_task_id=False, seed=args.seed, fixed_class_order=fixed_class_order, shuffle=True, class_ids_from_zero_in_each_exp=False, ) model = SlimResNet18(1) model.linear = IncrementalClassifier(model.linear.in_features, 1) optimizer = SGD(model.parameters(), lr=args.lr) interactive_logger = InteractiveLogger() loggers = [interactive_logger] training_metrics = [] evaluation_metrics = [ accuracy_metrics(epoch=True, stream=True), loss_metrics(epoch=True, stream=True), ] # Create main evaluator that will be used by the training actor evaluator = EvaluationPlugin( *training_metrics, *evaluation_metrics, loggers=loggers, ) plugins = [] ####################### # Strategy Creation # ####################### cl_strategy = OnlineER_ACE( model=model, optimizer=optimizer, plugins=plugins, evaluator=evaluator, device=device, train_mb_size=args.train_mb_size, eval_mb_size=64, mem_size=args.mem_size, batch_size_mem=args.batch_size_mem, ) ################### # TRAINING LOOP # ################### print("Starting experiment...") print([p.__class__.__name__ for p in cl_strategy.plugins]) ocl_scenario = split_online_stream( original_stream=scenario.train_stream, experience_size=10, access_task_boundaries=False, ) for t, experience in enumerate(ocl_scenario): cl_strategy.train( experience, eval_streams=[], num_workers=0, drop_last=True, ) results = cl_strategy.eval(scenario.test_stream) return results if __name__ == "__main__": res = eracl_scifar10() print(res) ================================================ FILE: experiments/split_cifar10/online_replay.py ================================================ import numpy as np import torch from torch.optim import SGD from avalanche.benchmarks.classic import SplitCIFAR10 from avalanche.benchmarks import benchmark_with_validation_stream from avalanche.benchmarks.scenarios import split_online_stream from avalanche.evaluation.metrics import accuracy_metrics, loss_metrics from avalanche.logging import InteractiveLogger from avalanche.models import SlimResNet18 from avalanche.training.plugins import EvaluationPlugin, ReplayPlugin from avalanche.training.storage_policy import ClassBalancedBuffer from avalanche.training.supervised import Naive from experiments.utils import create_default_args, set_seed def online_replay_scifar10(override_args=None): args = create_default_args( { "cuda": 0, "mem_size": 1000, "lr": 0.1, "train_mb_size": 10, "seed": None, "batch_size_mem": 10, }, override_args ) set_seed(args.seed) fixed_class_order = np.arange(10) device = torch.device( f"cuda:{args.cuda}" if torch.cuda.is_available() and args.cuda >= 0 else "cpu" ) scenario = SplitCIFAR10( 5, return_task_id=False, seed=args.seed, fixed_class_order=fixed_class_order, shuffle=True, class_ids_from_zero_in_each_exp=False, ) scenario = benchmark_with_validation_stream(scenario, 0.05) model = SlimResNet18(10) optimizer = SGD(model.parameters(), lr=args.lr) interactive_logger = InteractiveLogger() loggers = [interactive_logger] training_metrics = [] evaluation_metrics = [ accuracy_metrics(epoch=True, stream=True), loss_metrics(epoch=True, stream=True), ] evaluator = EvaluationPlugin( *training_metrics, *evaluation_metrics, loggers=loggers, ) storage_policy = ClassBalancedBuffer(args.mem_size, adaptive_size=True) plugins = [ReplayPlugin(args.mem_size, storage_policy=storage_policy)] cl_strategy = Naive( model=model, optimizer=optimizer, plugins=plugins, evaluator=evaluator, device=device, train_mb_size=args.train_mb_size, eval_mb_size=64, ) ocl_scenario = split_online_stream( original_stream=scenario.train_stream, experience_size=args.train_mb_size, access_task_boundaries=False, ) for t, experience in enumerate(ocl_scenario): cl_strategy.train( experience, eval_streams=[], num_workers=0, drop_last=True, ) results = cl_strategy.eval(scenario.test_stream) return results if __name__ == "__main__": res = online_replay_scifar10() print(res) ================================================ FILE: experiments/split_cifar10/supervised_contrastive_replay.py ================================================ from avalanche.training import SCR import torch import torchvision.transforms as transforms from torch.optim import SGD from avalanche.benchmarks.classic import SplitCIFAR10 from avalanche.evaluation.metrics import accuracy_metrics, loss_metrics from avalanche.logging import InteractiveLogger from avalanche.models import SlimResNet18, SCRModel from avalanche.training.plugins import EvaluationPlugin import kornia.augmentation as K from experiments.utils import create_default_args, set_seed from avalanche.benchmarks.scenarios import split_online_stream from torch.utils.data import DataLoader from avalanche.training.losses import SCRLoss def online_scr_scifar10(override_args=None): """ Reproducing Supervised Contrastive Replay paper "Supervised Contrastive Replay: Revisiting the Nearest Class Mean Classifier in Online Class-Incremental Continual Learning" by Mai et. al. (2021). https://arxiv.org/abs/2103.13885 In the original paper, SCR uses the ReviewTrick technique (fine-tuning on the buffer at the end of training on each experience). For fairness of comparison with the other strategies, we do not employ the review trick, therefore our results are lower wrt the original paper. However, you can activate the review trick by setting the corresponding parameter to True in the args. """ args = create_default_args( { "cuda": 0, "mem_size": 200, "lr": 0.1, "train_mb_size": 10, "seed": None, "batch_size_mem": 100, "review_trick": False }, override_args ) set_seed(args.seed) device = torch.device( f"cuda:{args.cuda}" if torch.cuda.is_available() and args.cuda >= 0 else "cpu" ) data_transform = transforms.Compose([ transforms.ToTensor(), ]) scenario = SplitCIFAR10( 5, return_task_id=False, train_transform=data_transform, eval_transform=data_transform, shuffle=True, class_ids_from_zero_in_each_exp=False, ) # SlimResNet18 is used as encoder # the projection network takes as input the output of the ResNet nf = 20 encoding_network = SlimResNet18(nclasses=10, nf=nf) encoding_network.linear = torch.nn.Identity() projection_network = torch.nn.Sequential( torch.nn.Linear(nf*8, nf*8), torch.nn.ReLU(inplace=True), torch.nn.Linear(nf*8, 128)) # a NCM Classifier is used at eval time model = SCRModel( feature_extractor=encoding_network, projection=projection_network) optimizer = SGD(model.parameters(), lr=args.lr) interactive_logger = InteractiveLogger() loggers = [interactive_logger] training_metrics = [] # training accuracy cannot be directly monitored with SCR # training loss and eval loss are two different ones evaluation_metrics = [ accuracy_metrics(experience=True, stream=True), loss_metrics(epoch=True, experience=True, stream=True), ] evaluator = EvaluationPlugin( *training_metrics, *evaluation_metrics, loggers=loggers, ) scr_transforms = torch.nn.Sequential( K.RandomResizedCrop(size=(32, 32), scale=(0.2, 1.)), K.RandomHorizontalFlip(), K.ColorJitter(0.4, 0.4, 0.4, 0.1, p=0.8), K.RandomGrayscale(p=0.2) ) # should achieve around 48% final accuracy cl_strategy = SCR( model=model, optimizer=optimizer, augmentations=scr_transforms, plugins=None, evaluator=evaluator, device=device, train_mb_size=args.train_mb_size, eval_mb_size=64, mem_size=args.mem_size, batch_size_mem=args.batch_size_mem ) ocl_scenario = split_online_stream( original_stream=scenario.train_stream, experience_size=args.train_mb_size, access_task_boundaries=False, ) for t, experience in enumerate(ocl_scenario): cl_strategy.train(experience) if args.review_trick and experience.is_last_subexp: # at the end of each macro experience buffer = cl_strategy.replay_plugin.storage_policy.buffer dl = DataLoader(buffer, batch_size=args.batch_size_mem, shuffle=True, drop_last=True) model.train() crit = SCRLoss(temperature=0.1) for x, y, _ in dl: assert x.size(0) % 2 == 0, f"{x.size(0)}" x, y = x.to(device), y.to(device) optimizer.zero_grad() mb_x_augmented = scr_transforms(x) x = torch.cat([x, mb_x_augmented], dim=0) assert x.size(0) % 2 == 0, f"{x.size(0)}" out = model(x) assert out.size(0) % 2 == 0, f"{x.size(0)}" original_batch_size = int(out.size(0) / 2) original_examples = out[:original_batch_size] augmented_examples = out[original_batch_size:] out = torch.stack( [original_examples, augmented_examples], dim=1) loss = crit(out, y) loss.backward() params = [p for p in model.parameters() if p.requires_grad and p.grad is not None] grad = [p.grad.clone()/10. for p in params] for g, p in zip(grad, params): p.grad.data.copy_(g) optimizer.step() model.eval() cl_strategy.compute_class_means() if args.review_trick: model.eval() cl_strategy.compute_class_means() results = cl_strategy.eval(scenario.test_stream) return results if __name__ == '__main__': res = online_scr_scifar10() print(res) ================================================ FILE: experiments/split_cifar100/__init__.py ================================================ from .icarl import icarl_scifar100 from .gem import gem_scifar100 from .agem import agem_scifar100 from .lamaml import lamaml_scifar100 from .er_ace import erace_scifar100 from .er_aml import eraml_scifar100 from .online_replay import online_replay_scifar100 ================================================ FILE: experiments/split_cifar100/agem.py ================================================ import avalanche as avl import torch from torch.nn import CrossEntropyLoss from torch.optim import SGD, Adam from avalanche.evaluation import metrics as metrics from models import MultiHeadReducedResNet18 from experiments.utils import set_seed, create_default_args def agem_scifar100(override_args=None): """ "Efficient Lifelong Learning with A-GEM" by Chaudhry et. al. (2019). https://openreview.net/pdf?id=Hkf2_sC5FX """ args = create_default_args({'cuda': 0, 'patterns_per_exp': 250, 'epochs': 1, 'sample_size': 1300, 'learning_rate': 0.03, 'train_mb_size': 10, 'seed': None}, override_args) set_seed(args.seed) device = torch.device(f"cuda:{args.cuda}" if torch.cuda.is_available() and args.cuda >= 0 else "cpu") benchmark = avl.benchmarks.SplitCIFAR100(17, return_task_id=True, fixed_class_order=list(range(85))) model = MultiHeadReducedResNet18() criterion = CrossEntropyLoss() interactive_logger = avl.logging.InteractiveLogger() evaluation_plugin = avl.training.plugins.EvaluationPlugin( metrics.accuracy_metrics(epoch=True, experience=True, stream=True), loggers=[interactive_logger]) cl_strategy = avl.training.AGEM( model, SGD(model.parameters(), lr=args.learning_rate), criterion, patterns_per_exp=args.patterns_per_exp, sample_size=args.sample_size, train_mb_size=args.train_mb_size, train_epochs=args.epochs, eval_mb_size=128, device=device, evaluator=evaluation_plugin, plugins=[]) res = None for experience in benchmark.train_stream: cl_strategy.train(experience) res = cl_strategy.eval(benchmark.test_stream) return res if __name__ == '__main__': res = agem_scifar100() print(res) ================================================ FILE: experiments/split_cifar100/er_ace.py ================================================ #!/usr/bin/env python3 import numpy as np import torch import torchvision.transforms as transforms from torch.optim import SGD from torchvision.transforms import ToTensor from avalanche.benchmarks.classic import SplitCIFAR100 from avalanche.evaluation.metrics import accuracy_metrics, loss_metrics from avalanche.logging import InteractiveLogger from avalanche.models import SlimResNet18 from avalanche.models.dynamic_modules import IncrementalClassifier from avalanche.training.plugins import EvaluationPlugin from avalanche.training.supervised import ER_ACE from experiments.utils import create_default_args, set_seed def erace_scifar100(override_args=None): args = create_default_args( { "cuda": 0, "mem_size": 10000, "lr": 0.1, "train_mb_size": 10, "seed": None, "batch_size_mem": 10, }, override_args ) set_seed(args.seed) fixed_class_order = np.arange(100) device = torch.device( f"cuda:{args.cuda}" if torch.cuda.is_available() and args.cuda >= 0 else "cpu" ) unique_transform = transforms.Compose( [ ToTensor(), transforms.Normalize( (0.5071, 0.4866, 0.4409), (0.2009, 0.1984, 0.2023) ), ] ) scenario = SplitCIFAR100( 20, return_task_id=False, seed=0, fixed_class_order=fixed_class_order, shuffle=True, class_ids_from_zero_in_each_exp=False, train_transform=unique_transform, eval_transform=unique_transform, ) input_size = (3, 32, 32) model = SlimResNet18(1) model.linear = IncrementalClassifier(model.linear.in_features, 1) optimizer = SGD(model.parameters(), lr=args.lr) interactive_logger = InteractiveLogger() loggers = [interactive_logger] training_metrics = [] evaluation_metrics = [ accuracy_metrics(epoch=True, stream=True), loss_metrics(epoch=True, stream=True), ] evaluator = EvaluationPlugin( *training_metrics, *evaluation_metrics, loggers=loggers, ) plugins = [] cl_strategy = ER_ACE( model=model, optimizer=optimizer, plugins=plugins, evaluator=evaluator, device=device, train_mb_size=args.train_mb_size, eval_mb_size=64, mem_size=args.mem_size, batch_size_mem=args.batch_size_mem, ) for t, experience in enumerate(scenario.train_stream): cl_strategy.train( experience, num_workers=0, drop_last=True, ) cl_strategy.eval(scenario.test_stream[: t + 1]) results = cl_strategy.eval(scenario.test_stream) return results if __name__ == "__main__": res = erace_scifar100() print(res) ================================================ FILE: experiments/split_cifar100/er_aml.py ================================================ #!/usr/bin/env python3 import numpy as np import torch import torchvision.transforms as transforms from torch.optim import SGD from torchvision.transforms import ToTensor from avalanche.benchmarks.classic import SplitCIFAR100 from avalanche.evaluation.metrics import accuracy_metrics, loss_metrics from avalanche.logging import InteractiveLogger from avalanche.training.plugins import EvaluationPlugin from avalanche.training.supervised import ER_AML from experiments.utils import create_default_args, set_seed from models import SingleHeadReducedResNet18 def eraml_scifar100(override_args=None): """ Reproducing ER-AML experiments from paper "New insights on Reducing Abrupt Representation Change in Online Continual Learning" by Lucas Caccia et. al https://openreview.net/forum?id=N8MaByOzUfb """ args = create_default_args( { "cuda": 0, "mem_size": 10000, "lr": 0.1, "temp": 0.1, "train_mb_size": 10, "seed": None, "batch_size_mem": 10, }, override_args, ) set_seed(args.seed) fixed_class_order = np.arange(100) device = torch.device( f"cuda:{args.cuda}" if torch.cuda.is_available() and args.cuda >= 0 else "cpu" ) unique_transform = transforms.Compose( [ ToTensor(), transforms.Normalize((0.5071, 0.4866, 0.4409), (0.2009, 0.1984, 0.2023)), ] ) scenario = SplitCIFAR100( 20, return_task_id=False, seed=0, fixed_class_order=fixed_class_order, shuffle=True, class_ids_from_zero_in_each_exp=False, train_transform=unique_transform, eval_transform=unique_transform, ) input_size = (3, 32, 32) model = SingleHeadReducedResNet18(100) optimizer = SGD(model.parameters(), lr=args.lr) interactive_logger = InteractiveLogger() loggers = [interactive_logger] training_metrics = [] evaluation_metrics = [ accuracy_metrics(epoch=True, stream=True), loss_metrics(epoch=True, stream=True), ] evaluator = EvaluationPlugin( *training_metrics, *evaluation_metrics, loggers=loggers, ) plugins = [] cl_strategy = ER_AML( model=model, feature_extractor=model.feature_extractor, optimizer=optimizer, plugins=plugins, evaluator=evaluator, device=device, train_mb_size=args.train_mb_size, eval_mb_size=64, mem_size=args.mem_size, batch_size_mem=args.batch_size_mem, ) for t, experience in enumerate(scenario.train_stream): cl_strategy.train( experience, num_workers=0, drop_last=True, ) cl_strategy.eval(scenario.test_stream[: t + 1]) results = cl_strategy.eval(scenario.test_stream) return results if __name__ == "__main__": res = eraml_scifar100() print(res) ================================================ FILE: experiments/split_cifar100/gem.py ================================================ import avalanche as avl import torch from torch.nn import CrossEntropyLoss from torch.optim import SGD from avalanche.evaluation import metrics as metrics from models import MultiHeadReducedResNet18 from experiments.utils import set_seed, create_default_args def gem_scifar100(override_args=None): """ "Gradient Episodic Memory for Continual Learning" by Lopez-paz et. al. (2017). https://proceedings.neurips.cc/paper/2017/hash/f87522788a2be2d171666752f97ddebb-Abstract.html """ args = create_default_args({'cuda': 0, 'patterns_per_exp': 256, 'epochs': 1, 'mem_strength': 0.5, 'learning_rate': 0.1, 'train_mb_size': 10, 'seed': None}, override_args) set_seed(args.seed) device = torch.device(f"cuda:{args.cuda}" if torch.cuda.is_available() and args.cuda >= 0 else "cpu") benchmark = avl.benchmarks.SplitCIFAR100(20, return_task_id=True) model = MultiHeadReducedResNet18() criterion = CrossEntropyLoss() interactive_logger = avl.logging.InteractiveLogger() evaluation_plugin = avl.training.plugins.EvaluationPlugin( metrics.accuracy_metrics(epoch=True, experience=True, stream=True), loggers=[interactive_logger]) cl_strategy = avl.training.GEM( model, SGD(model.parameters(), lr=args.learning_rate, momentum=0.9), criterion, patterns_per_exp=args.patterns_per_exp, memory_strength=args.mem_strength, train_mb_size=args.train_mb_size, train_epochs=args.epochs, eval_mb_size=128, device=device, evaluator=evaluation_plugin) res = None for experience in benchmark.train_stream: cl_strategy.train(experience) res = cl_strategy.eval(benchmark.test_stream) return res if __name__ == '__main__': res = gem_scifar100() print(res) ================================================ FILE: experiments/split_cifar100/icarl.py ================================================ import numpy as np import torch from torch.optim.lr_scheduler import MultiStepLR from torch.optim import SGD from torchvision import transforms from experiments.utils import set_seed, create_default_args from avalanche.benchmarks import SplitCIFAR100 from avalanche.models import IcarlNet, make_icarl_net, initialize_icarl_net from avalanche.training.plugins.lr_scheduling import LRSchedulerPlugin from avalanche.training.plugins import EvaluationPlugin from avalanche.evaluation.metrics import * from avalanche.logging.interactive_logging import InteractiveLogger from avalanche.training import ICaRL def icarl_cifar100_augment_data(img): img = img.numpy() padded = np.pad(img, ((0, 0), (4, 4), (4, 4)), mode='constant') random_cropped = np.zeros(img.shape, dtype=np.float32) crop = np.random.randint(0, high=8 + 1, size=(2,)) # Cropping and possible flipping if np.random.randint(2) > 0: random_cropped[:, :, :] = \ padded[:, crop[0]:(crop[0]+32), crop[1]:(crop[1]+32)] else: random_cropped[:, :, :] = \ padded[:, crop[0]:(crop[0]+32), crop[1]:(crop[1]+32)][:, :, ::-1] t = torch.tensor(random_cropped) return t def icarl_scifar100(override_args=None): """ "iCaRL: Incremental Classifier and Representation Learning", Sylvestre-Alvise Rebuffi, Alexander Kolesnikov, Georg Sperl, Christoph H. Lampert; Proceedings of the IEEE Conference on Computer Vision and Pattern Recognition (CVPR), 2017, pp. 2001-2010 https://openaccess.thecvf.com/content_cvpr_2017/html/Rebuffi_iCaRL_Incremental_Classifier_CVPR_2017_paper.html The expected performance is 50%, while we achieve a lower score. """ args = create_default_args({'cuda': 0, 'batch_size': 128, 'nb_exp': 10, 'memory_size': 2000, 'epochs': 70, 'lr_base': 2., 'lr_milestones': [49, 63], 'lr_factor': 5., 'wght_decay': 0.00001, 'seed': None}, override_args) # class incremental learning: classes mutual exclusive fixed_class_order = [87, 0, 52, 58, 44, 91, 68, 97, 51, 15, 94, 92, 10, 72, 49, 78, 61, 14, 8, 86, 84, 96, 18, 24, 32, 45, 88, 11, 4, 67, 69, 66, 77, 47, 79, 93, 29, 50, 57, 83, 17, 81, 41, 12, 37, 59, 25, 20, 80, 73, 1, 28, 6, 46, 62, 82, 53, 9, 31, 75, 38, 63, 33, 74, 27, 22, 36, 3, 16, 21, 60, 19, 70, 90, 89, 43, 5, 42, 65, 76, 40, 30, 23, 85, 2, 95, 56, 48, 71, 64, 98, 13, 99, 7, 34, 55, 54, 26, 35, 39] set_seed(args.seed) device = torch.device(f"cuda:{args.cuda}" if torch.cuda.is_available() and args.cuda >= 0 else "cpu") transform_prototypes = transforms.Compose([ icarl_cifar100_augment_data, ]) train_trsf = transforms.Compose([transforms.ToTensor(), icarl_cifar100_augment_data]) test_trsf = transforms.Compose([transforms.ToTensor()]) benchmark = SplitCIFAR100( n_experiences=args.nb_exp, seed=args.seed, fixed_class_order=fixed_class_order, train_transform=train_trsf, eval_transform=test_trsf ) interactive_logger = InteractiveLogger() eval_plugin = EvaluationPlugin( accuracy_metrics(experience=True, stream=True), loggers=[interactive_logger]) # _____________________________Strategy model: IcarlNet = make_icarl_net(num_classes=100) model.apply(initialize_icarl_net) optim = SGD(model.parameters(), lr=args.lr_base, weight_decay=args.wght_decay, momentum=0.9) sched = LRSchedulerPlugin( MultiStepLR(optim, args.lr_milestones, gamma=1.0 / args.lr_factor)) strategy = ICaRL( model.feature_extractor, model.classifier, optim, args.memory_size, buffer_transform=transform_prototypes, fixed_memory=True, train_mb_size=args.batch_size, train_epochs=args.epochs, eval_mb_size=args.batch_size, plugins=[sched], device=device, evaluator=eval_plugin ) for i, exp in enumerate(benchmark.train_stream): strategy.train(exp, num_workers=4) res = strategy.eval(benchmark.test_stream, num_workers=4) return res if __name__ == '__main__': res = icarl_scifar100() print(res) ================================================ FILE: experiments/split_cifar100/lamaml.py ================================================ import avalanche as avl import torch from torch.nn import CrossEntropyLoss from avalanche.evaluation import metrics as metrics from avalanche.training.storage_policy import ReservoirSamplingBuffer from avalanche.training.plugins import ReplayPlugin from avalanche.training.supervised.lamaml_v2 import LaMAML from models.models_lamaml import MTConvCIFAR from experiments.utils import set_seed, create_default_args def lamaml_scifar100(override_args=None): """ "La-MAML: Look-ahead Meta Learning for Continual Learning", Gunshi Gupta, Karmesh Yadav, Liam Paull; NeurIPS, 2020 https://arxiv.org/abs/2007.13904 """ # Args args = create_default_args( {'cuda': 0, 'n_inner_updates': 5, 'second_order': True, 'grad_clip_norm': 1.0, 'learn_lr': True, 'lr_alpha': 0.25, 'sync_update': False, 'mem_size': 200, 'lr': 0.1, 'train_mb_size': 10, 'train_epochs': 10, 'seed': None}, override_args ) set_seed(args.seed) device = torch.device(f"cuda:{args.cuda}" if torch.cuda.is_available() and args.cuda >= 0 else "cpu") # Benchmark benchmark = avl.benchmarks.SplitCIFAR100(n_experiences=20, return_task_id=True) # Loggers and metrics interactive_logger = avl.logging.InteractiveLogger() evaluation_plugin = avl.training.plugins.EvaluationPlugin( metrics.accuracy_metrics(epoch=True, experience=True, stream=True), loggers=[interactive_logger]) # Buffer rs_buffer = ReservoirSamplingBuffer(max_size=args.mem_size) replay_plugin = ReplayPlugin( mem_size=args.mem_size, batch_size=args.train_mb_size, batch_size_mem=args.train_mb_size, task_balanced_dataloader=False, storage_policy=rs_buffer ) # Strategy model = MTConvCIFAR() cl_strategy = LaMAML( model, torch.optim.SGD(model.parameters(), lr=args.lr), CrossEntropyLoss(), n_inner_updates=args.n_inner_updates, second_order=args.second_order, grad_clip_norm=args.grad_clip_norm, learn_lr=args.learn_lr, lr_alpha=args.lr_alpha, sync_update=args.sync_update, train_mb_size=args.train_mb_size, train_epochs=args.train_epochs, eval_mb_size=100, device=device, plugins=[replay_plugin], evaluator=evaluation_plugin, ) res = None for experience in benchmark.train_stream: cl_strategy.train(experience) res = cl_strategy.eval(benchmark.test_stream) return res if __name__ == '__main__': res = lamaml_scifar100() print(res) ================================================ FILE: experiments/split_cifar100/online_replay.py ================================================ import numpy as np import torch import torchvision.transforms as transforms from torch.optim import SGD from avalanche.benchmarks.classic import SplitCIFAR100 from avalanche.benchmarks import benchmark_with_validation_stream from avalanche.benchmarks.scenarios import split_online_stream from avalanche.evaluation.metrics import accuracy_metrics, loss_metrics from avalanche.logging import InteractiveLogger from avalanche.models import SlimResNet18 from avalanche.training.plugins import EvaluationPlugin, ReplayPlugin from avalanche.training.storage_policy import ClassBalancedBuffer from avalanche.training.supervised import Naive from experiments.utils import create_default_args, set_seed def online_replay_scifar100(override_args=None): """ Online replay for CIFAR100, the hyperparameters are taken from "New Insights on Reducing Abrupt Representation Change in Online Continual Learning", Lucas Caccia et. al., https://openreview.net/forum?id=N8MaByOzUfb Augmentations are not used since they make the results worse in this particular setting """ args = create_default_args( { "cuda": 0, "mem_size": 10000, "lr": 0.1, "train_mb_size": 10, "seed": None, "batch_size_mem": 10, }, override_args ) set_seed(args.seed) fixed_class_order = np.arange(100) device = torch.device( f"cuda:{args.cuda}" if torch.cuda.is_available() and args.cuda >= 0 else "cpu" ) unique_transform = transforms.Compose( [ transforms.ToTensor(), transforms.Normalize( (0.5071, 0.4866, 0.4409), (0.2009, 0.1984, 0.2023) ), ] ) scenario = SplitCIFAR100( 20, return_task_id=False, seed=args.seed, fixed_class_order=fixed_class_order, shuffle=True, class_ids_from_zero_in_each_exp=False, train_transform=unique_transform, eval_transform=unique_transform, ) scenario = benchmark_with_validation_stream(scenario, 0.05) model = SlimResNet18(100) optimizer = SGD(model.parameters(), lr=args.lr) interactive_logger = InteractiveLogger() loggers = [interactive_logger] training_metrics = [] evaluation_metrics = [ accuracy_metrics(epoch=True, stream=True), loss_metrics(epoch=True, stream=True), ] evaluator = EvaluationPlugin( *training_metrics, *evaluation_metrics, loggers=loggers, ) storage_policy = ClassBalancedBuffer(args.mem_size, adaptive_size=True) plugins = [ReplayPlugin(args.mem_size, storage_policy=storage_policy)] cl_strategy = Naive( model=model, optimizer=optimizer, plugins=plugins, evaluator=evaluator, device=device, train_mb_size=args.train_mb_size, eval_mb_size=64, ) ocl_scenario = split_online_stream( original_stream=scenario, experience_size = args.train_mb_size, access_task_boundaries = False ) for t, experience in enumerate(ocl_scenario): cl_strategy.train( experience, eval_streams=[], num_workers=0, drop_last=True, ) results = cl_strategy.eval(scenario.test_stream) return results if __name__ == "__main__": res = online_replay_scifar100() print(res) ================================================ FILE: experiments/split_cifar100/replay.py ================================================ #!/usr/bin/env python3 import numpy as np import torch import torchvision.transforms as transforms from torch.optim import SGD from avalanche.benchmarks.classic import SplitCIFAR100 from avalanche.benchmarks.generators import benchmark_with_validation_stream from avalanche.evaluation.metrics import accuracy_metrics, loss_metrics from avalanche.logging import InteractiveLogger from avalanche.models.resnet32 import resnet32 from avalanche.models.dynamic_modules import IncrementalClassifier from avalanche.training.plugins import EvaluationPlugin, ReplayPlugin, LRSchedulerPlugin from avalanche.training.storage_policy import ClassBalancedBuffer from avalanche.training.supervised import Naive from experiments.utils import create_default_args, set_seed from torch.optim.lr_scheduler import StepLR def replay_scifar100(override_args=None): """ Replay for Split CIFAR100 """ args = create_default_args( { "cuda": 0, "num_epochs": 200, "mem_size": 2000, "momentum": 0.9, "weight_decay": 0.0002, "lr": 0.1, "train_mb_size": 128, "seed": None, "batch_size_mem": 128, }, override_args ) set_seed(args.seed) fixed_class_order = np.arange(100) device = torch.device( f"cuda:{args.cuda}" if torch.cuda.is_available() and args.cuda >= 0 else "cpu" ) scenario = SplitCIFAR100( 20, return_task_id=False, seed=args.seed, fixed_class_order=fixed_class_order, shuffle=True, class_ids_from_zero_in_each_exp=False, ) scenario = benchmark_with_validation_stream(scenario, 0.05) input_size = (3, 32, 32) model = resnet32(num_classes=1) model.fc = IncrementalClassifier(model.fc.in_features, 1) optimizer = SGD(model.parameters(), momentum=args.momentum, weight_decay=args.weight_decay, lr=args.lr) scheduler = StepLR(optimizer, step_size=args.num_epochs//3, gamma=0.3) scheduler_plugin = LRSchedulerPlugin(scheduler, step_granularity="epoch", first_exp_only=False) interactive_logger = InteractiveLogger() loggers = [interactive_logger] training_metrics = [] evaluation_metrics = [ accuracy_metrics(epoch=True, stream=True), loss_metrics(epoch=True, stream=True), ] evaluator = EvaluationPlugin( *training_metrics, *evaluation_metrics, loggers=loggers, ) storage_policy = ClassBalancedBuffer(args.mem_size, adaptive_size=True) plugins = [scheduler_plugin, ReplayPlugin(args.mem_size, storage_policy=storage_policy)] cl_strategy = Naive( model=model, optimizer=optimizer, plugins=plugins, evaluator=evaluator, device=device, train_mb_size=args.train_mb_size, train_epochs=args.num_epochs, eval_mb_size=64, ) for t, experience in enumerate(scenario.train_stream): print("Start of experience: ", experience.current_experience) print("Current Classes: ", experience.classes_in_this_experience) cl_strategy.train( experience, eval_streams=[], num_workers=2, drop_last=True, ) cl_strategy.eval(scenario.test_stream[: t + 1]) # Only evaluate at the end on the test stream results = cl_strategy.eval(scenario.test_stream) return results if __name__ == "__main__": res = replay_scifar100() print(res) ================================================ FILE: experiments/split_mnist/__init__.py ================================================ from .synaptic_intelligence import synaptic_intelligence_smnist from .lwf import lwf_smnist from .gss import gss_smnist from .gdumb import gdumb_smnist from .cope import cope_smnist from .generative_replay import generative_replay_smnist from .rwalk import rwalk_smnist from .naive import naive_smnist from .online_replay import online_replay_smnist from .mir import mir_smnist ================================================ FILE: experiments/split_mnist/cope.py ================================================ import torch import avalanche as avl from experiments.utils import set_seed, create_default_args from models import MLP def cope_smnist(override_args=None): """ "Continual prototype evolution: Learning online from non-stationary data streams" by De Lange et. al. (2021). https://arxiv.org/abs/2009.00919 Expected performance is 93%, which is higher than what we achieve. """ args = create_default_args({'cuda': 0, 'nb_tasks': 5, 'batch_size': 10, 'epochs': 1, 'mem_size': 2000, 'alpha': 0.99, 'T': 0.1, 'featsize': 32, 'seed': None}, override_args) set_seed(args.seed) device = torch.device(f"cuda:{args.cuda}" if torch.cuda.is_available() and args.cuda >= 0 else "cpu") n_classes = 10 task_scenario = avl.benchmarks.SplitMNIST(args.nb_tasks, return_task_id=False, fixed_class_order=[i for i in range(n_classes)]) # Make data incremental (one batch = one experience) benchmark = avl.benchmarks.data_incremental_benchmark(task_scenario, experience_size=args.batch_size) print(f"{benchmark.n_experiences} batches in online data incremental setup.") # 6002 batches for SplitMNIST with batch size 10 # --------- model = MLP(output_size=args.featsize, hidden_size=400, hidden_layers=2, drop_rate=0) logger = avl.logging.InteractiveLogger() eval_plugin = avl.training.plugins.EvaluationPlugin( avl.evaluation.metrics.accuracy_metrics(experience=True, stream=True), avl.evaluation.metrics.loss_metrics(experience=False, stream=True), avl.evaluation.metrics.StreamForgetting(), loggers=[logger]) cope = avl.training.plugins.CoPEPlugin(mem_size=args.mem_size, alpha=args.alpha, p_size=args.featsize, n_classes=n_classes, T=args.T) cl_strategy = avl.training.Naive( model, torch.optim.SGD(model.parameters(), lr=0.01), cope.ppp_loss, # CoPE PPP-Loss train_mb_size=args.batch_size, train_epochs=args.epochs, eval_mb_size=100, device=device, plugins=[cope], evaluator=eval_plugin ) cl_strategy.train(benchmark.train_stream) res = cl_strategy.eval(benchmark.test_stream) return res if __name__ == '__main__': res = cope_smnist() print(res) ================================================ FILE: experiments/split_mnist/gdumb.py ================================================ import avalanche as avl import torch from torch.nn import CrossEntropyLoss from torch.optim import SGD from avalanche.evaluation import metrics as metrics from models import MLP from experiments.utils import set_seed, create_default_args def gdumb_smnist(override_args=None): """ "GDumb: A Simple Approach that Questions Our Progress in Continual Learning" by Prabhu et. al. (2020). https://link.springer.com/chapter/10.1007/978-3-030-58536-5_31 """ args = create_default_args({'cuda': 0, 'hidden_size': 400, 'mem_size': 4400, 'hidden_layers': 2, 'epochs': 10, 'dropout': 0, 'learning_rate': 0.1, 'train_mb_size': 16, 'seed': None}, override_args) set_seed(args.seed) device = torch.device(f"cuda:{args.cuda}" if torch.cuda.is_available() and args.cuda >= 0 else "cpu") benchmark = avl.benchmarks.SplitMNIST(5, return_task_id=False) model = MLP(hidden_size=args.hidden_size, hidden_layers=args.hidden_layers, drop_rate=args.dropout, relu_act=True) criterion = CrossEntropyLoss() interactive_logger = avl.logging.InteractiveLogger() evaluation_plugin = avl.training.plugins.EvaluationPlugin( metrics.accuracy_metrics(epoch=True, experience=True, stream=True), loggers=[interactive_logger]) cl_strategy = avl.training.GDumb( model, SGD(model.parameters(), lr=args.learning_rate), criterion, mem_size=args.mem_size, train_mb_size=args.train_mb_size, train_epochs=args.epochs, eval_mb_size=128, device=device, evaluator=evaluation_plugin) res = None for experience in benchmark.train_stream: cl_strategy.train(experience) res = cl_strategy.eval(benchmark.test_stream) return res if __name__ == '__main__': res = gdumb_smnist() print(res) ================================================ FILE: experiments/split_mnist/generative_replay.py ================================================ import avalanche as avl import torch from torch.nn import CrossEntropyLoss from torch.optim import SGD from avalanche.evaluation import metrics as metrics from models import MLP from experiments.utils import set_seed, create_default_args def generative_replay_smnist(override_args=None): """ "Continual Learning with Deep Generative Replay" by Shin et. al. (2017). https://arxiv.org/abs/1705.08690 """ args = create_default_args({'cuda': 0, 'hidden_size': 400, 'hidden_layers': 2, 'epochs': 10, 'dropout': 0, 'learning_rate': 0.001, 'train_mb_size': 16, 'seed': None}, override_args) set_seed(args.seed) device = torch.device(f"cuda:{args.cuda}" if torch.cuda.is_available() and args.cuda >= 0 else "cpu") benchmark = avl.benchmarks.SplitMNIST(5, return_task_id=False) model = MLP(hidden_size=args.hidden_size, hidden_layers=args.hidden_layers, drop_rate=args.dropout, relu_act=True) criterion = CrossEntropyLoss() interactive_logger = avl.logging.InteractiveLogger() evaluation_plugin = avl.training.plugins.EvaluationPlugin( metrics.accuracy_metrics(epoch=True, experience=True, stream=True), loggers=[interactive_logger]) cl_strategy = avl.training.GenerativeReplay( model, torch.optim.Adam(model.parameters(), lr=args.learning_rate), criterion, train_mb_size=args.train_mb_size, train_epochs=args.epochs, eval_mb_size=128, replay_size=100, device=device, evaluator=evaluation_plugin, ) res = None for experience in benchmark.train_stream: cl_strategy.train(experience) res = cl_strategy.eval(benchmark.test_stream) return res if __name__ == '__main__': res = generative_replay_smnist() print(res) ================================================ FILE: experiments/split_mnist/gss.py ================================================ import torch.nn as nn from avalanche.benchmarks import CLExperience from avalanche.benchmarks.classic import SplitMNIST from avalanche.benchmarks import data_incremental_benchmark from avalanche.evaluation.metrics import \ accuracy_metrics, \ loss_metrics from avalanche.logging import InteractiveLogger from avalanche.training.plugins import EvaluationPlugin from avalanche.training import GSS_greedy import torch from torch.nn import CrossEntropyLoss from torch.optim import SGD from experiments.utils import set_seed, create_default_args from models import MLP_gss def gss_smnist(override_args=None): """ https://arxiv.org/abs/1903.08671 Expected accuracy is 82% which is slightly higher than the one we achieve. """ args = create_default_args({ 'cuda': 0, 'lr': 0.05, 'train_mb_size': 10, 'mem_strength': 10, 'input_size': [1, 28, 28], 'train_epochs': 3, 'eval_mb_size': 10, 'mem_size': 300, 'seed': None}, override_args) set_seed(args.seed) device = torch.device(f"cuda:{args.cuda}" if torch.cuda.is_available() and args.cuda >= 0 else "cpu") model, benchmark = setup_mnist() eval_plugin = EvaluationPlugin( accuracy_metrics(epoch=True, experience=True, stream=True), loss_metrics(stream=True), loggers=[InteractiveLogger()]) optimizer = SGD(model.parameters(), lr=args.lr) strategy = GSS_greedy(model, optimizer, criterion=CrossEntropyLoss(), mem_strength=args.mem_strength, input_size=args.input_size, train_epochs=args.train_epochs, train_mb_size=args.train_mb_size, eval_mb_size=args.eval_mb_size, mem_size=args.mem_size, device=device, evaluator=eval_plugin) res = None for experience in benchmark.train_stream: print(">Experience ", experience.current_experience) strategy.train(experience) res = strategy.eval(benchmark.test_stream) return res def shrinking_experience_size_split_strategy( experience: CLExperience): experience_size = 1000 exp_dataset = experience.dataset exp_indices = list(range(len(exp_dataset))) result_datasets = [] exp_indices = \ torch.as_tensor(exp_indices)[ torch.randperm(len(exp_indices)) ].tolist() result_datasets.append(exp_dataset.subset(exp_indices[0:experience_size])) return result_datasets def setup_mnist(): scenario = data_incremental_benchmark(SplitMNIST( n_experiences=5, seed=1), experience_size=0, custom_split_strategy=shrinking_experience_size_split_strategy) n_inputs = 784 nh = 100 nl = 2 n_outputs = 10 model = MLP_gss([n_inputs] + [nh] * nl + [n_outputs]) return model, scenario if __name__ == '__main__': res = gss_smnist() print(res) ================================================ FILE: experiments/split_mnist/lwf.py ================================================ import avalanche as avl import torch from torch.nn import CrossEntropyLoss from torch.optim import SGD from avalanche.evaluation import metrics as metrics from models import MLP from experiments.utils import set_seed, create_default_args class LwFCEPenalty(avl.training.LwF): """This wrapper around LwF computes the total loss by diminishing the cross-entropy contribution over time, as per the paper "Three scenarios for continual learning" by van de Ven et. al. (2018). https://arxiv.org/pdf/1904.07734.pdf The loss is L_tot = (1/n_exp_so_far) * L_cross_entropy + alpha[current_exp] * L_distillation """ def _before_backward(self, **kwargs): self.loss *= float(1/(self.clock.train_exp_counter+1)) super()._before_backward(**kwargs) def lwf_smnist(override_args=None): """ "Learning without Forgetting" by Li et. al. (2016). http://arxiv.org/abs/1606.09282 Since experimental setup of the paper is quite outdated and not easily reproducible, this experiment is based on "Three scenarios for continual learning" by van de Ven et. al. (2018). https://arxiv.org/pdf/1904.07734.pdf The hyper-parameter alpha controlling the regularization is increased over time, resulting in a regularization of (1- 1/n_exp_so_far) * L_distillation """ args = create_default_args({'cuda': 0, 'lwf_alpha': [0, 0.5, 1.33333, 2.25, 3.2], 'lwf_temperature': 2, 'epochs': 21, 'layers': 1, 'hidden_size': 200, 'learning_rate': 0.001, 'train_mb_size': 128, 'seed': None}, override_args) set_seed(args.seed) device = torch.device(f"cuda:{args.cuda}" if torch.cuda.is_available() and args.cuda >= 0 else "cpu") benchmark = avl.benchmarks.SplitMNIST(5, return_task_id=False) model = MLP(hidden_size=args.hidden_size, hidden_layers=args.layers, initial_out_features=0, relu_act=False) criterion = CrossEntropyLoss() interactive_logger = avl.logging.InteractiveLogger() evaluation_plugin = avl.training.plugins.EvaluationPlugin( metrics.accuracy_metrics(epoch=True, experience=True, stream=True), loggers=[interactive_logger]) cl_strategy = LwFCEPenalty( model, SGD(model.parameters(), lr=args.learning_rate), criterion, alpha=args.lwf_alpha, temperature=args.lwf_temperature, train_mb_size=args.train_mb_size, train_epochs=args.epochs, device=device, evaluator=evaluation_plugin) res = None for experience in benchmark.train_stream: cl_strategy.train(experience) res = cl_strategy.eval(benchmark.test_stream) return res if __name__ == '__main__': res = lwf_smnist() print(res) ================================================ FILE: experiments/split_mnist/mir.py ================================================ import numpy as np import torch from torch.optim import SGD from avalanche.benchmarks.classic import SplitMNIST from avalanche.benchmarks import benchmark_with_validation_stream from avalanche.benchmarks.scenarios import split_online_stream from avalanche.evaluation.metrics import accuracy_metrics, loss_metrics from avalanche.logging import InteractiveLogger from avalanche.models import SimpleMLP from avalanche.training.plugins import EvaluationPlugin, MIRPlugin from avalanche.training.supervised import Naive from experiments.utils import create_default_args, set_seed, restrict_dataset_size def mir_smnist(override_args=None): args = create_default_args( { "cuda": 0, "mem_size": 500, "lr": 0.05, "train_mb_size": 10, "seed": None, "subsample": 50, "batch_size_mem": 10, "dataset_size": 1000, }, override_args ) set_seed(args.seed) device = torch.device( f"cuda:{args.cuda}" if torch.cuda.is_available() and args.cuda >= 0 else "cpu" ) scenario = SplitMNIST( 5, return_task_id=False, seed=0, train_transform=None, eval_transform=None, ) scenario = benchmark_with_validation_stream(scenario, 0.05) scenario = restrict_dataset_size(scenario, args.dataset_size) model = SimpleMLP(10, hidden_size=400, hidden_layers=1) optimizer = SGD(model.parameters(), lr=args.lr) interactive_logger = InteractiveLogger() loggers = [interactive_logger] training_metrics = [] evaluation_metrics = [ accuracy_metrics(epoch=True, stream=True), loss_metrics(epoch=True, stream=True), ] evaluator = EvaluationPlugin( *training_metrics, *evaluation_metrics, loggers=loggers, ) plugins = [ MIRPlugin( mem_size=args.mem_size, subsample=args.subsample, batch_size_mem=args.batch_size_mem ) ] cl_strategy = Naive( model=model, optimizer=optimizer, plugins=plugins, evaluator=evaluator, device=device, train_mb_size=args.train_mb_size, eval_mb_size=64, ) ocl_scenario = split_online_stream( original_stream=scenario.train_stream, experience_size=10, access_task_boundaries=False, ) for t, experience in enumerate(ocl_scenario): cl_strategy.train( experience, eval_streams=[], num_workers=0, drop_last=True, ) results = cl_strategy.eval(scenario.test_stream) return results if __name__ == "__main__": res = mir_smnist() print(res) ================================================ FILE: experiments/split_mnist/naive.py ================================================ import avalanche as avl import torch from torch.nn import CrossEntropyLoss from torch.optim import Adam from avalanche.evaluation import metrics as metrics from models import MultiHeadMLP, MLP from experiments.utils import set_seed, create_default_args def naive_smnist(override_args=None): """ "Continual Learning Through Synaptic Intelligence" by Zenke et. al. (2017). http://proceedings.mlr.press/v70/zenke17a.html """ args = create_default_args({'cuda': 0, 'epochs': 10, 'learning_rate': 0.001, 'train_mb_size': 64, 'seed': None, 'task-incremental': False}, override_args) set_seed(args.seed) device = torch.device(f"cuda:{args.cuda}" if torch.cuda.is_available() and args.cuda >= 0 else "cpu") benchmark = avl.benchmarks.SplitMNIST(5, return_task_id=args.task_incremental, fixed_class_order=list(range(10))) model = MultiHeadMLP(hidden_size=256, hidden_layers=2) if args.task_incremental \ else MLP(hidden_size=256, hidden_layers=2) criterion = CrossEntropyLoss() interactive_logger = avl.logging.InteractiveLogger() evaluation_plugin = avl.training.plugins.EvaluationPlugin( metrics.accuracy_metrics(epoch=True, experience=True, stream=True), loggers=[interactive_logger]) cl_strategy = avl.training.Naive( model, Adam(model.parameters(), lr=args.learning_rate), criterion, train_mb_size=args.train_mb_size, train_epochs=args.epochs, eval_mb_size=128, device=device, evaluator=evaluation_plugin) for experience in benchmark.train_stream: cl_strategy.train(experience) res = cl_strategy.eval(benchmark.test_stream) return res if __name__ == '__main__': res = naive_smnist() print(res) ================================================ FILE: experiments/split_mnist/online_replay.py ================================================ import numpy as np import torch from torch.optim import SGD from avalanche.benchmarks.classic import SplitMNIST from avalanche.benchmarks import benchmark_with_validation_stream from avalanche.benchmarks.scenarios import split_online_stream from avalanche.evaluation.metrics import accuracy_metrics, loss_metrics from avalanche.logging import InteractiveLogger from avalanche.models import SimpleMLP from avalanche.training.plugins import EvaluationPlugin, ReplayPlugin from avalanche.training.storage_policy import ClassBalancedBuffer from avalanche.training.supervised import Naive from experiments.utils import create_default_args, set_seed def online_replay_smnist(override_args=None): args = create_default_args( { "cuda": 0, "mem_size": 1000, "lr": 0.1, "train_mb_size": 10, "seed": None, "batch_size_mem": 10, }, override_args ) set_seed(args.seed) fixed_class_order = np.arange(10) device = torch.device( f"cuda:{args.cuda}" if torch.cuda.is_available() and args.cuda >= 0 else "cpu" ) scenario = SplitMNIST( 5, return_task_id=False, seed=args.seed, fixed_class_order=fixed_class_order, shuffle=True, class_ids_from_zero_in_each_exp=False, ) scenario = benchmark_with_validation_stream(scenario, 0.05) model = SimpleMLP(10) optimizer = SGD(model.parameters(), lr=args.lr) interactive_logger = InteractiveLogger() loggers = [interactive_logger] training_metrics = [] evaluation_metrics = [ accuracy_metrics(epoch=True, stream=True), loss_metrics(epoch=True, stream=True), ] evaluator = EvaluationPlugin( *training_metrics, *evaluation_metrics, loggers=loggers, ) storage_policy = ClassBalancedBuffer(args.mem_size, adaptive_size=True) plugins = [ReplayPlugin(args.mem_size, storage_policy=storage_policy)] cl_strategy = Naive( model=model, optimizer=optimizer, plugins=plugins, evaluator=evaluator, device=device, train_mb_size=args.train_mb_size, eval_mb_size=64, ) ocl_scenario = split_online_stream( original_stream=scenario.train_stream, experience_size=args.train_mb_size, access_task_boundaries=False, ) for t, experience in enumerate(ocl_scenario): cl_strategy.train( experience, eval_streams=[], num_workers=0, drop_last=True, ) results = cl_strategy.eval(scenario.test_stream) return results if __name__ == "__main__": res = online_replay_smnist() print(res) ================================================ FILE: experiments/split_mnist/rwalk.py ================================================ import avalanche as avl import torch from torch.nn import CrossEntropyLoss from torch.optim import Adam from avalanche.evaluation import metrics as metrics from models import MultiHeadMLP from experiments.utils import set_seed, create_default_args def rwalk_smnist(override_args=None): """ Reproducing RWalk experiments from paper "Riemannian Walk for Incremental Learning: Understanding Forgetting and Intransigence" by Chaudhry et. al. (2018). https://openaccess.thecvf.com/content_ECCV_2018/html/Arslan_Chaudhry__Riemannian_Walk_ECCV_2018_paper.html The expected value is 99%, which is higher than the achieved one. """ args = create_default_args({'cuda': 0, 'ewc_lambda': 0.1, 'ewc_alpha': 0.9, 'delta_t': 10, 'epochs': 10, 'learning_rate': 0.001, 'train_mb_size': 64, 'seed': None}, override_args) set_seed(args.seed) device = torch.device(f"cuda:{args.cuda}" if torch.cuda.is_available() and args.cuda >= 0 else "cpu") benchmark = avl.benchmarks.SplitMNIST(5, return_task_id=True, fixed_class_order=list(range(10))) model = MultiHeadMLP(hidden_size=256, hidden_layers=2) criterion = CrossEntropyLoss() interactive_logger = avl.logging.InteractiveLogger() evaluation_plugin = avl.training.plugins.EvaluationPlugin( metrics.accuracy_metrics(epoch=True, experience=True, stream=True), loggers=[interactive_logger]) cl_strategy = avl.training.Naive( model, Adam(model.parameters(), lr=args.learning_rate), criterion, plugins=[avl.training.plugins.RWalkPlugin( ewc_lambda=args.ewc_lambda, ewc_alpha=args.ewc_alpha, delta_t=args.delta_t)], train_mb_size=args.train_mb_size, train_epochs=args.epochs, eval_mb_size=128, device=device, evaluator=evaluation_plugin) for experience in benchmark.train_stream: cl_strategy.train(experience) res = cl_strategy.eval(benchmark.test_stream) return res if __name__ == '__main__': res = rwalk_smnist() print(res) ================================================ FILE: experiments/split_mnist/synaptic_intelligence.py ================================================ import avalanche as avl import torch from torch.nn import CrossEntropyLoss from torch.optim import Adam from avalanche.evaluation import metrics as metrics from models import MultiHeadMLP from experiments.utils import set_seed, create_default_args def synaptic_intelligence_smnist(override_args=None): """ "Continual Learning Through Synaptic Intelligence" by Zenke et. al. (2017). http://proceedings.mlr.press/v70/zenke17a.html """ args = create_default_args({'cuda': 0, 'si_lambda': 1, 'si_eps': 0.1, 'epochs': 10, 'learning_rate': 0.001, 'train_mb_size': 64, 'seed': None}, override_args) set_seed(args.seed) device = torch.device(f"cuda:{args.cuda}" if torch.cuda.is_available() and args.cuda >= 0 else "cpu") benchmark = avl.benchmarks.SplitMNIST(5, return_task_id=True, fixed_class_order=list(range(10))) model = MultiHeadMLP(hidden_size=256, hidden_layers=2) criterion = CrossEntropyLoss() interactive_logger = avl.logging.InteractiveLogger() evaluation_plugin = avl.training.plugins.EvaluationPlugin( metrics.accuracy_metrics(epoch=True, experience=True, stream=True), loggers=[interactive_logger]) cl_strategy = avl.training.SynapticIntelligence( model, Adam(model.parameters(), lr=args.learning_rate), criterion, si_lambda=args.si_lambda, eps=args.si_eps, train_mb_size=args.train_mb_size, train_epochs=args.epochs, eval_mb_size=128, device=device, evaluator=evaluation_plugin) for experience in benchmark.train_stream: cl_strategy.train(experience) res = cl_strategy.eval(benchmark.test_stream) return res if __name__ == '__main__': res = synaptic_intelligence_smnist() print(res) ================================================ FILE: experiments/split_tiny_imagenet/__init__.py ================================================ from .mas import mas_stinyimagenet from .lwf import lwf_stinyimagenet from .lamaml import lamaml_stinyimagenet from .naive import naive_stinyimagenet from .packnet import packnet_stinyimagenet ================================================ FILE: experiments/split_tiny_imagenet/lamaml.py ================================================ import avalanche as avl import torch from torch.nn import CrossEntropyLoss from avalanche.evaluation import metrics as metrics from avalanche.training.storage_policy import ReservoirSamplingBuffer from avalanche.training.plugins import ReplayPlugin from avalanche.training.supervised.lamaml_v2 import LaMAML from models.models_lamaml import MTConvTinyImageNet from experiments.utils import set_seed, create_default_args def lamaml_stinyimagenet(override_args=None): """ "La-MAML: Look-ahead Meta Learning for Continual Learning", Gunshi Gupta, Karmesh Yadav, Liam Paull; NeurIPS, 2020 https://arxiv.org/abs/2007.13904 Expected performance is 66%, which is higher than what we achieve. """ # Args args = create_default_args( {'cuda': 0, 'n_inner_updates': 5, 'second_order': True, 'grad_clip_norm': 1.0, 'learn_lr': True, 'lr_alpha': 0.4, 'sync_update': False, 'mem_size': 400, 'lr': 0.1, 'train_mb_size': 10, 'train_epochs': 10, 'seed': None}, override_args ) set_seed(args.seed) device = torch.device(f"cuda:{args.cuda}" if torch.cuda.is_available() and args.cuda >= 0 else "cpu") # Benchmark benchmark = avl.benchmarks.SplitTinyImageNet(n_experiences=20, return_task_id=True) # Loggers and metrics interactive_logger = avl.logging.InteractiveLogger() evaluation_plugin = avl.training.plugins.EvaluationPlugin( metrics.accuracy_metrics(epoch=True, experience=True, stream=True), loggers=[interactive_logger]) # Buffer rs_buffer = ReservoirSamplingBuffer(max_size=args.mem_size) replay_plugin = ReplayPlugin( mem_size=args.mem_size, batch_size=args.train_mb_size, batch_size_mem=args.train_mb_size, task_balanced_dataloader=False, storage_policy=rs_buffer ) # Strategy model = MTConvTinyImageNet() cl_strategy = LaMAML( model, torch.optim.SGD(model.parameters(), lr=args.lr), CrossEntropyLoss(), n_inner_updates=args.n_inner_updates, second_order=args.second_order, grad_clip_norm=args.grad_clip_norm, learn_lr=args.learn_lr, lr_alpha=args.lr_alpha, sync_update=args.sync_update, train_mb_size=args.train_mb_size, train_epochs=args.train_epochs, eval_mb_size=100, device=device, plugins=[replay_plugin], evaluator=evaluation_plugin, ) res = None for experience in benchmark.train_stream: cl_strategy.train(experience) res = cl_strategy.eval(benchmark.test_stream) return res if __name__ == '__main__': res = lamaml_stinyimagenet() print(res) ================================================ FILE: experiments/split_tiny_imagenet/lwf.py ================================================ import avalanche as avl import torch from torch.nn import CrossEntropyLoss from torch.optim import SGD from avalanche.evaluation import metrics as metrics from models import MultiHeadVGGSmall from experiments.utils import set_seed, create_default_args def lwf_stinyimagenet(override_args=None): """ "Learning without Forgetting" by Li et. al. (2016). http://arxiv.org/abs/1606.09282 Since experimental setup of the paper is quite outdated and not easily reproducible, this experiment is based on "A continual learning survey: Defying forgetting in classification tasks" De Lange et. al. (2021). https://ieeexplore.ieee.org/stamp/stamp.jsp?arnumber=9349197 We use a VGG network, which leads a lower performance than the one from De Lange et. al. (2021). """ args = create_default_args({'cuda': 0, 'lwf_alpha': 1, 'lwf_temperature': 2, 'epochs': 70, 'learning_rate': 1e-3, 'train_mb_size': 200, 'seed': None, 'dataset_root': None}, override_args) set_seed(args.seed) device = torch.device(f"cuda:{args.cuda}" if torch.cuda.is_available() and args.cuda >= 0 else "cpu") benchmark = avl.benchmarks.SplitTinyImageNet( 10, return_task_id=True, dataset_root=args.dataset_root) model = MultiHeadVGGSmall(n_classes=200) criterion = CrossEntropyLoss() interactive_logger = avl.logging.InteractiveLogger() evaluation_plugin = avl.training.plugins.EvaluationPlugin( metrics.accuracy_metrics(epoch=True, experience=True, stream=True), loggers=[interactive_logger]) cl_strategy = avl.training.LwF( model, SGD(model.parameters(), lr=args.learning_rate, momentum=0.9), criterion, alpha=args.lwf_alpha, temperature=args.lwf_temperature, train_mb_size=args.train_mb_size, train_epochs=args.epochs, eval_mb_size=128, device=device, evaluator=evaluation_plugin) res = None for experience in benchmark.train_stream: cl_strategy.train(experience) res = cl_strategy.eval(benchmark.test_stream) return res if __name__ == "__main__": res = lwf_stinyimagenet() print(res) ================================================ FILE: experiments/split_tiny_imagenet/mas.py ================================================ import torch from torch.nn import CrossEntropyLoss from avalanche.evaluation.metrics import ( accuracy_metrics, forgetting_metrics, loss_metrics ) from avalanche.training.plugins import EvaluationPlugin from models import MultiHeadVGGSmall from experiments.utils import set_seed, create_default_args import avalanche as avl def mas_stinyimagenet(override_args=None): """ Experiment adapted by "A continual learning survey: Defying forgetting in classification tasks" by De Lange et al. https://doi.org/10.1109/TPAMI.2021.3057446 """ args = create_default_args( {'cuda': 0, 'lambda_reg': 1., 'alpha': 0.5, 'verbose': True, 'learning_rate': 0.001, 'train_mb_size': 200, 'epochs': 50, 'seed': None, 'dataset_root': None}, override_args) set_seed(args.seed) device = torch.device(f"cuda:{args.cuda}" if torch.cuda.is_available() and args.cuda >= 0 else "cpu") """ "In order to construct a balanced dataset, we assign an equal amount of 20 randomly chosen classes to each task in a sequence of 10 consecutive tasks. This task incremental setting allows using an oracle at test time for our evaluation per task, ensuring all tasks are roughly similar in terms of difficulty, size, and distribution, making the interpretation of the results easier." """ benchmark = avl.benchmarks.SplitTinyImageNet( 10, return_task_id=True, dataset_root=args.dataset_root) model = MultiHeadVGGSmall() criterion = CrossEntropyLoss() interactive_logger = avl.logging.InteractiveLogger() evaluation_plugin = EvaluationPlugin( accuracy_metrics( epoch=True, experience=True, stream=True ), loss_metrics( epoch=True, experience=True, stream=True ), forgetting_metrics( experience=True, stream=True ), loggers=[interactive_logger]) optimizer = torch.optim.SGD(model.parameters(), lr=args.learning_rate, momentum=0.9) cl_strategy = avl.training.MAS( model, optimizer, criterion, lambda_reg=args.lambda_reg, alpha=args.alpha, verbose=args.verbose, train_mb_size=args.train_mb_size, train_epochs=args.epochs, eval_mb_size=128, device=device, evaluator=evaluation_plugin) res = None for experience in benchmark.train_stream: cl_strategy.train(experience) res = cl_strategy.eval(benchmark.test_stream) return res if __name__ == "__main__": res = mas_stinyimagenet() print(res) ================================================ FILE: experiments/split_tiny_imagenet/naive.py ================================================ import avalanche as avl import torch from torch.nn import CrossEntropyLoss from torch.optim import Adam from avalanche.evaluation import metrics as metrics from models import MultiHeadVGGSmall from experiments.utils import set_seed, create_default_args def naive_stinyimagenet(override_args=None): """ "Learning without Forgetting" by Li et. al. (2016). http://arxiv.org/abs/1606.09282 Since experimental setup of the paper is quite outdated and not easily reproducible, this experiment is based on "A continual learning survey: Defying forgetting in classification tasks" De Lange et. al. (2021). https://ieeexplore.ieee.org/stamp/stamp.jsp?arnumber=9349197 We use a VGG network, which leads a lower performance than the one from De Lange et. al. (2021). """ args = create_default_args({'cuda': 0, 'epochs': 30, 'learning_rate': 1e-3, 'train_mb_size': 200, 'seed': None, 'dataset_root': None}, override_args) set_seed(args.seed) device = torch.device(f"cuda:{args.cuda}" if torch.cuda.is_available() and args.cuda >= 0 else "cpu") benchmark = avl.benchmarks.SplitTinyImageNet( 10, return_task_id=True, dataset_root=args.dataset_root) model = MultiHeadVGGSmall(n_classes=200) criterion = CrossEntropyLoss() interactive_logger = avl.logging.InteractiveLogger() evaluation_plugin = avl.training.plugins.EvaluationPlugin( metrics.accuracy_metrics(epoch=True, experience=True, stream=True), loggers=[interactive_logger]) cl_strategy = avl.training.Naive( model, Adam(model.parameters(), lr=args.learning_rate), criterion, train_mb_size=args.train_mb_size, train_epochs=args.epochs, eval_mb_size=128, device=device, evaluator=evaluation_plugin) res = None for experience in benchmark.train_stream: cl_strategy.train(experience) res = cl_strategy.eval(benchmark.test_stream) return res if __name__ == "__main__": res = naive_stinyimagenet() print(res) ================================================ FILE: experiments/split_tiny_imagenet/packnet.py ================================================ """ Reproduce Delange et al. (2021) benchmark results for PackNet (Mallya & Lazebnik, 2018) on Split Tiny ImageNet Delange, M., Aljundi, R., Masana, M., Parisot, S., Jia, X., Leonardis, A., Slabaugh, G., & Tuytelaars, T. (2021). A continual learning survey: Defying forgetting in classification tasks. IEEE Transactions on Pattern Analysis and Machine Intelligence, 1–1. https://doi.org/10.1109/TPAMI.2021.3057446 Mallya, A., & Lazebnik, S. (2018). PackNet: Adding Multiple Tasks to a Single Network by Iterative Pruning. 2018 IEEE/CVF Conference on Computer Vision and Pattern Recognition, 7765–7773. https://doi.org/10.1109/CVPR.2018.00810 """ import avalanche as avl import torch from torch.nn import CrossEntropyLoss from torch.optim import Adam from avalanche.evaluation import metrics as metrics from experiments.utils import set_seed, create_default_args from models.vgg import SingleHeadVGGSmall from avalanche.models.packnet import PackNetModel, packnet_simple_mlp from avalanche.training.supervised.strategy_wrappers import PackNet def packnet_stinyimagenet(override_args=None): """ The original PackNet paper uses an unusual experimental setup, so we base this experiment on Delange et al. (2021) benchmark. Delange et al. (2021) set prune_proportion using the Continual Hyperparameter Selection Framework. We instead use `prune_proportion` such that the number of parameters in each task-specific-subset of the model are roughly equal. """ args = create_default_args( { "cuda": 0, "epochs": 30, "learning_rate": 1e-3, "train_mb_size": 200, "seed": 42, "dataset_root": None, "prune_proportion": [ 0.90, 0.88, 0.87, 0.85, 0.83, 0.80, 0.75, 0.66, 0.50, 0.00, ], "post_prune_epochs": 15, }, override_args, ) set_seed(args.seed) device = torch.device( f"cuda:{args.cuda}" if torch.cuda.is_available() and args.cuda >= 0 else "cpu" ) benchmark = avl.benchmarks.SplitTinyImageNet( 10, return_task_id=True, dataset_root=args.dataset_root ) model = SingleHeadVGGSmall(n_classes=200) model = PackNetModel(model) criterion = CrossEntropyLoss() interactive_logger = avl.logging.InteractiveLogger() evaluation_plugin = avl.training.plugins.EvaluationPlugin( metrics.accuracy_metrics(epoch=True, experience=True, stream=True), loggers=[interactive_logger], ) cl_strategy = PackNet( model, Adam(model.parameters(), lr=args.learning_rate), post_prune_epochs=args.post_prune_epochs, prune_proportion=args.prune_proportion, criterion=criterion, train_mb_size=args.train_mb_size, train_epochs=args.epochs, eval_mb_size=128, device=device, evaluator=evaluation_plugin, ) res = None for experience in benchmark.train_stream: cl_strategy.train(experience) res = cl_strategy.eval(benchmark.test_stream) return res if __name__ == "__main__": res = packnet_stinyimagenet() print(res) ================================================ FILE: experiments/utils.py ================================================ import random from types import SimpleNamespace import numpy as np import torch from avalanche.benchmarks import dataset_benchmark def set_seed(seed): if seed is None: return torch.manual_seed(seed) torch.cuda.manual_seed(seed) np.random.seed(seed) random.seed(seed) if torch.cuda.is_available(): torch.backends.cudnn.deterministic = True torch.backends.cudnn.enabled = True torch.backends.cudnn.benchmark = False def create_default_args(args_dict, additional_args=None): args = SimpleNamespace() for k, v in args_dict.items(): args.__dict__[k] = v if additional_args is not None: for k, v in additional_args.items(): args.__dict__[k] = v return args def restrict_dataset_size(scenario, size: int): """ Util used to restrict the size of the datasets coming from a scenario param: size: size of the reduced training dataset """ modified_train_ds = [] modified_test_ds = [] modified_valid_ds = [] if hasattr(scenario, "valid_stream"): valid_list = list(scenario.valid_stream) for i, train_ds in enumerate(scenario.train_stream): train_ds_idx, _ = torch.utils.data.random_split( torch.arange(len(train_ds.dataset)), (size, len(train_ds.dataset) - size), ) dataset = train_ds.dataset.subset(train_ds_idx) modified_train_ds.append(dataset) modified_test_ds.append(scenario.test_stream[i].dataset) if hasattr(scenario, "valid_stream"): modified_valid_ds.append(valid_list[i].dataset) scenario = dataset_benchmark( modified_train_ds, modified_test_ds, other_streams_datasets={"valid": modified_valid_ds} if len(modified_valid_ds) > 0 else None, ) return scenario ================================================ FILE: gitbisect_test.sh ================================================ #!/bin/bash # first mandatory argument with path to continual-learning-baselines repository # second optional argument with the test name to run. If not provided, all tests will be run cd $1 if [ $# -ne 2 ]; then python -m unittest else python -m unittest $2 fi result=$? if [ $? -ne 0 ]; then exit 1 # 1 -> bad fi exit 0 # 0 -> good ================================================ FILE: models/__init__.py ================================================ from .models import * from .reduced_resnet18 import * from .vgg import MultiHeadVGG, MultiHeadVGGSmall, SingleHeadVGGSmall, VGGSmall ================================================ FILE: models/models.py ================================================ import avalanche.models from avalanche.models import MultiHeadClassifier, MultiTaskModule, BaseModel from torch import nn class MultiHeadMLP(MultiTaskModule): def __init__(self, input_size=28 * 28, hidden_size=256, hidden_layers=2, drop_rate=0, relu_act=True): super().__init__() self._input_size = input_size layers = nn.Sequential(*(nn.Linear(input_size, hidden_size), nn.ReLU(inplace=True) if relu_act else nn.Tanh(), nn.Dropout(p=drop_rate))) for layer_idx in range(hidden_layers - 1): layers.add_module( f"fc{layer_idx + 1}", nn.Sequential( *(nn.Linear(hidden_size, hidden_size), nn.ReLU(inplace=True) if relu_act else nn.Tanh(), nn.Dropout(p=drop_rate)))) self.features = nn.Sequential(*layers) self.classifier = MultiHeadClassifier(hidden_size) def forward(self, x, task_labels): x = x.contiguous() x = x.view(x.size(0), self._input_size) x = self.features(x) x = self.classifier(x, task_labels) return x class MLP(nn.Module, BaseModel): def __init__(self, input_size=28 * 28, hidden_size=256, hidden_layers=2, output_size=10, drop_rate=0, relu_act=True, initial_out_features=0): """ :param initial_out_features: if >0 override output size and build an IncrementalClassifier with `initial_out_features` units as first. """ super().__init__() self._input_size = input_size layers = nn.Sequential(*(nn.Linear(input_size, hidden_size), nn.ReLU(inplace=True) if relu_act else nn.Tanh(), nn.Dropout(p=drop_rate))) for layer_idx in range(hidden_layers - 1): layers.add_module( f"fc{layer_idx + 1}", nn.Sequential( *(nn.Linear(hidden_size, hidden_size), nn.ReLU(inplace=True) if relu_act else nn.Tanh(), nn.Dropout(p=drop_rate)))) self.features = nn.Sequential(*layers) if initial_out_features > 0: self.classifier = avalanche.models.IncrementalClassifier(in_features=hidden_size, initial_out_features=initial_out_features) else: self.classifier = nn.Linear(hidden_size, output_size) def forward(self, x): x = x.contiguous() x = x.view(x.size(0), self._input_size) x = self.features(x) x = self.classifier(x) return x def get_features(self, x): x = x.contiguous() x = x.view(x.size(0), self._input_size) return self.features(x) class SI_CNN(MultiTaskModule): def __init__(self, hidden_size=512): super().__init__() layers = nn.Sequential(*(nn.Conv2d(in_channels=3, out_channels=32, kernel_size=(3, 3), padding=(1, 1)), nn.ReLU(inplace=True), nn.Conv2d(in_channels=32, out_channels=32, kernel_size=(3, 3)), nn.ReLU(inplace=True), nn.MaxPool2d((2, 2)), nn.Dropout(p=0.25), nn.Conv2d(in_channels=32, out_channels=64, kernel_size=(3, 3), padding=(1, 1)), nn.ReLU(inplace=True), nn.Conv2d(in_channels=64, out_channels=64, kernel_size=(3, 3)), nn.ReLU(inplace=True), nn.MaxPool2d((2, 2)), nn.Dropout(p=0.25), nn.Flatten(), nn.Linear(2304, hidden_size), nn.ReLU(inplace=True), nn.Dropout(p=0.5) )) self.features = nn.Sequential(*layers) self.classifier = MultiHeadClassifier(hidden_size, initial_out_features=10) def forward(self, x, task_labels): x = self.features(x) x = self.classifier(x, task_labels) return x class FlattenP(nn.Module): '''A nn-module to flatten a multi-dimensional tensor to 2-dim tensor.''' def forward(self, x): batch_size = x.size(0) # first dimenstion should be batch-dimension. return x.view(batch_size, -1) def __repr__(self): tmpstr = self.__class__.__name__ + '()' return tmpstr class MLP_gss(nn.Module): def __init__(self, sizes, bias=True): super(MLP_gss, self).__init__() layers = [] for i in range(0, len(sizes) - 1): if i < (len(sizes)-2): layers.append(nn.Linear(sizes[i], sizes[i + 1])) layers.append(nn.ReLU()) else: layers.append(nn.Linear(sizes[i], sizes[i + 1], bias=bias)) self.net = nn.Sequential(FlattenP(), *layers) def forward(self, x): return self.net(x) __all__ = ['MultiHeadMLP', 'MLP', 'SI_CNN', 'MLP_gss'] ================================================ FILE: models/models_lamaml.py ================================================ import torch.nn as nn from avalanche.models.dynamic_modules import MultiTaskModule,\ MultiHeadClassifier #################### # CIFAR-100 #################### class ConvCIFAR(nn.Module): def __init__(self, num_classes=10): super(ConvCIFAR, self).__init__() # Convolutional layers self.conv_layers = nn.Sequential( nn.Conv2d(3, 160, kernel_size=(3, 3), stride=2, padding=1), nn.ReLU(inplace=True), nn.Conv2d(160, 160, kernel_size=(3, 3), stride=2, padding=1), nn.ReLU(inplace=True), nn.Conv2d(160, 160, kernel_size=(3, 3), stride=2, padding=1), nn.ReLU(inplace=True), ) # Linear layers self.relu = nn.ReLU(inplace=True) self.linear1 = nn.Linear(16*160, 320) self.linear2 = nn.Linear(320, 320) # Classifier self.classifier = nn.Linear(320, num_classes) def forward(self, x): x = self.conv_layers(x) x = x.view(-1, 2560) x = self.relu(self.linear1(x)) x = self.relu(self.linear2(x)) x = self.classifier(x) return x class MTConvCIFAR(ConvCIFAR, MultiTaskModule): def __init__(self): super(MTConvCIFAR, self).__init__() # Classifier self.classifier = MultiHeadClassifier(320) def forward(self, x, task_labels): x = self.conv_layers(x) x = x.view(-1, 16*160) x = self.relu(self.linear1(x)) x = self.relu(self.linear2(x)) x = self.classifier(x, task_labels) return x #################### # TinyImageNet #################### class ConvTinyImageNet(nn.Module): def __init__(self, num_classes=10): super(ConvTinyImageNet, self).__init__() # Convolutional layers self.conv_layers = nn.Sequential( nn.Conv2d(3, 160, kernel_size=(3, 3), stride=2, padding=1), nn.ReLU(inplace=True), nn.Conv2d(160, 160, kernel_size=(3, 3), stride=2, padding=1), nn.ReLU(inplace=True), nn.Conv2d(160, 160, kernel_size=(3, 3), stride=2, padding=1), nn.ReLU(inplace=True), nn.Conv2d(160, 160, kernel_size=(3, 3), stride=2, padding=1), nn.ReLU(inplace=True), ) # linear layers self.relu = nn.ReLU(inplace=True) self.linear1 = nn.Linear(16*160, 640) self.linear2 = nn.Linear(640, 640) # classifier self.classifier = nn.Linear(640, num_classes) def forward(self, x): x = self.conv_layers(x) x = x.view(-1, 16*160) x = self.relu(self.linear1(x)) x = self.relu(self.linear2(x)) x = self.classifier(x) return x class MTConvTinyImageNet(ConvTinyImageNet, MultiTaskModule): def __init__(self): super(MTConvTinyImageNet, self).__init__() # Classifier self.classifier = MultiHeadClassifier(640) def forward(self, x, task_labels): x = self.conv_layers(x) x = x.view(-1, 16*160) x = self.relu(self.linear1(x)) x = self.relu(self.linear2(x)) x = self.classifier(x, task_labels) return x ================================================ FILE: models/reduced_resnet18.py ================================================ import torch from avalanche.models import MultiHeadClassifier, MultiTaskModule from torch import nn, relu from torch.nn.functional import avg_pool2d """ START: FROM GEM CODE https://github.com/facebookresearch/GradientEpisodicMemory/ CLASSIFIER REMOVED AND SUBSTITUTED WITH AVALANCHE MULTI-HEAD CLASSIFIER """ def conv3x3(in_planes, out_planes, stride=1): return nn.Conv2d( in_planes, out_planes, kernel_size=3, stride=stride, padding=1, bias=True ) class BasicBlock(nn.Module): expansion = 1 def __init__(self, in_planes, planes, stride=1): super(BasicBlock, self).__init__() self.conv1 = conv3x3(in_planes, planes, stride) self.bn1 = nn.BatchNorm2d(planes) self.conv2 = conv3x3(planes, planes) self.bn2 = nn.BatchNorm2d(planes) self.shortcut = nn.Sequential() if stride != 1 or in_planes != self.expansion * planes: self.shortcut = nn.Sequential( nn.Conv2d( in_planes, self.expansion * planes, kernel_size=1, stride=stride, bias=True, ), nn.BatchNorm2d(self.expansion * planes), ) def forward(self, x): out = relu(self.bn1(self.conv1(x))) out = self.bn2(self.conv2(out)) out += self.shortcut(x) out = relu(out) return out class ResNet(nn.Module): def __init__(self, block, num_blocks, nf): super(ResNet, self).__init__() self.in_planes = nf self.conv1 = conv3x3(3, nf * 1) self.bn1 = nn.BatchNorm2d(nf * 1) self.layer1 = self._make_layer(block, nf * 1, num_blocks[0], stride=1) self.layer2 = self._make_layer(block, nf * 2, num_blocks[1], stride=2) self.layer3 = self._make_layer(block, nf * 4, num_blocks[2], stride=2) self.layer4 = self._make_layer(block, nf * 8, num_blocks[3], stride=2) def _make_layer(self, block, planes, num_blocks, stride): strides = [stride] + [1] * (num_blocks - 1) layers = [] for stride in strides: layers.append(block(self.in_planes, planes, stride)) self.in_planes = planes * block.expansion return nn.Sequential(*layers) def forward(self, x): bsz = x.size(0) out = relu(self.bn1(self.conv1(x.view(bsz, 3, 32, 32)))) out = self.layer1(out) out = self.layer2(out) out = self.layer3(out) out = self.layer4(out) out = avg_pool2d(out, 4) return out """ END: FROM GEM CODE """ class MultiHeadReducedResNet18(MultiTaskModule): """ As from GEM paper, a smaller version of ResNet18, with three times less feature maps across all layers. It employs multi-head output layer. """ def __init__(self, size_before_classifier=160): super().__init__() self.resnet = ResNet(BasicBlock, [2, 2, 2, 2], 20) self.classifier = MultiHeadClassifier(size_before_classifier) def forward(self, x, task_labels): out = self.resnet(x) out = out.view(out.size(0), -1) return self.classifier(out, task_labels) class SingleHeadReducedResNet18(torch.nn.Module): def __init__(self, num_classes): super().__init__() self.resnet = ResNet(BasicBlock, [2, 2, 2, 2], 20) self.classifier = nn.Linear(160, num_classes) def feature_extractor(self, x): out = self.resnet(x) return out.view(out.size(0), -1) def forward(self, x): out = self.feature_extractor(x) return self.classifier(out) __all__ = ['MultiHeadReducedResNet18', 'SingleHeadReducedResNet18'] ================================================ FILE: models/vgg.py ================================================ import torch from torch import nn import torchvision from avalanche.models import MultiTaskModule, MultiHeadClassifier class MultiHeadVGG(MultiTaskModule): def __init__(self, n_classes=20): super().__init__() self.vgg = torchvision.models.vgg11() self.classifier = MultiHeadClassifier(in_features=1000, initial_out_features=n_classes) def forward(self, x, task_labels): x = self.vgg(x) x = torch.flatten(x, 1) return self.classifier(x, task_labels) """ Small VGG net adapted from https://github.com/Mattdl/CLsurvey/ """ cfg = [64, 'M', 64, 'M', 64, 64, 'M', 128, 128, 'M'] conv_kernel_size = 3 img_input_channels = 3 class VGGSmall(torchvision.models.VGG): """ Creates VGG feature extractor from config and custom classifier. """ def __init__(self): in_channels = img_input_channels layers = [] for v in cfg: if v == 'M': layers += [nn.MaxPool2d(kernel_size=2, stride=2)] else: conv2d = nn.Conv2d(in_channels, v, kernel_size=conv_kernel_size, padding=1) layers += [conv2d, nn.ReLU(inplace=True)] in_channels = v super(VGGSmall, self).__init__(nn.Sequential(*layers), init_weights=True) if hasattr(self, 'avgpool'): # Compat Pytorch>1.0.0 self.avgpool = torch.nn.Identity() del self.classifier def forward(self, x): x = self.features(x) return x class MultiHeadVGGSmall(MultiTaskModule): def __init__(self, n_classes=200, hidden_size=128): super().__init__() self.vgg = VGGSmall() self.feedforward = nn.Sequential( nn.Linear(128*4*4, hidden_size), nn.ReLU(True), nn.Linear(hidden_size, hidden_size), nn.ReLU(True), ) self.classifier = MultiHeadClassifier(in_features=128, initial_out_features=n_classes) def forward(self, x, task_labels): x = self.vgg(x) x = torch.flatten(x, 1) x = self.feedforward(x) return self.classifier(x, task_labels) class SingleHeadVGGSmall(nn.Module): def __init__(self, n_classes=200, hidden_size=128): super().__init__() self.vgg = VGGSmall() self.feedforward = nn.Sequential( nn.Linear(128 * 4 * 4, hidden_size), nn.ReLU(True), nn.Linear(hidden_size, hidden_size), nn.ReLU(True), ) self.classifier = nn.Linear(hidden_size, n_classes) def forward(self, x): x = self.vgg(x) x = torch.flatten(x, 1) x = self.feedforward(x) return self.classifier(x) ================================================ FILE: tests/__init__.py ================================================ from .synaptic_intelligence import SynapticIntelligence from .cope import COPE from .dslda import DSLDA from .ewc import EWC from .mas import MAS from .agem import AGEM from .gem import GEM from .lwf import LwF from .gss import GSS from .mir import MIR from .iCARL import iCARL from .gdumb import GDumb from .lfl import LFL from .lamaml import LaMAML from .generative_replay import GenerativeReplay from .rwalk import RWalk from .scr import SCR from .er_ace import ER_ACE from .er_aml import ER_AML from .packnet import PackNet from . import utils ================================================ FILE: tests/agem/__init__.py ================================================ from .experiment import AGEM ================================================ FILE: tests/agem/experiment.py ================================================ import unittest from tests.utils import get_average_metric, get_target_result from experiments.permuted_mnist import agem_pmnist from experiments.split_cifar100 import agem_scifar100 class AGEM(unittest.TestCase): """ Reproducing Average-GEM experiments from paper "Efficient Lifelong Learning with A-GEM" by Chaudhry et. al. (2019). https://openreview.net/pdf?id=Hkf2_sC5FX The main difference with the original paper is that we do not append any task descriptor to the model input. We train on the last 17 experiences since we apply the evaluation protocol defined in the paper but we do not perform model selection. """ def test_pmnist(self): """Permuted MNIST benchmark""" res = agem_pmnist({'seed': 0}) avg_stream_acc = get_average_metric(res) print(f"AGEM-PMNIST Average Stream Accuracy: {avg_stream_acc:.2f}") target_acc = float(get_target_result('agem', 'pmnist')) if target_acc > avg_stream_acc: self.assertAlmostEqual(target_acc, avg_stream_acc, delta=0.03) def test_scifar100(self): """Split CIFAR-100 benchmark""" res = agem_scifar100({'seed': 0}) avg_stream_acc = get_average_metric(res) print(f"AGEM-SCIFAR100 Average Stream Accuracy: {avg_stream_acc:.2f}") target_acc = float(get_target_result('agem', 'scifar100')) if target_acc > avg_stream_acc: self.assertAlmostEqual(target_acc, avg_stream_acc, delta=0.04) ================================================ FILE: tests/cope/__init__.py ================================================ from .experiment import COPE ================================================ FILE: tests/cope/experiment.py ================================================ import unittest from tests.utils import get_average_metric, get_target_result from experiments.split_mnist import cope_smnist class COPE(unittest.TestCase): """ Reproducing CoPE experiments from the paper "Continual prototype evolution: Learning online from non-stationary data streams" by De Lange et. al. (2021). https://arxiv.org/abs/2009.00919 """ def test_smnist(self): """Split MNIST benchmark""" res = cope_smnist({'seed': 0}) avg_stream_acc = get_average_metric(res) print(f"COPE-SMNIST Average Stream Accuracy: {avg_stream_acc:.2f}") target_acc = float(get_target_result('cope', 'smnist')) if target_acc > avg_stream_acc: self.assertAlmostEqual(target_acc, avg_stream_acc, delta=0.03) ================================================ FILE: tests/dslda/__init__.py ================================================ from .experiment import DSLDA ================================================ FILE: tests/dslda/experiment.py ================================================ import unittest from tests.utils import get_average_metric, get_target_result from experiments.core50 import deep_slda_core50 class DSLDA(unittest.TestCase): """ Reproducing Streaming Deep LDA experiments from the paper "Lifelong Machine Learning with Deep Streaming Linear Discriminant Analysis" by Hayes et. al. (2020). https://arxiv.org/abs/1909.01520 """ def test_core50(self): """CORe50 New Classes benchmark""" res = deep_slda_core50({'seed': 0}) avg_stream_acc = get_average_metric(res) print(f"DSLDA-CORe50 Average Stream Accuracy: {avg_stream_acc:.2f}") target_acc = float(get_target_result('dslda', 'core50')) if target_acc > avg_stream_acc: self.assertAlmostEqual(target_acc, avg_stream_acc, delta=0.03) ================================================ FILE: tests/er_ace/__init__.py ================================================ from .experiment import ER_ACE ================================================ FILE: tests/er_ace/experiment.py ================================================ import unittest from tests.utils import get_average_metric, get_target_result from experiments.split_cifar10 import erace_scifar10 from experiments.split_cifar100 import erace_scifar100 class ER_ACE(unittest.TestCase): """ Reproducing ER-ACE experiments from paper "New insights on Reducing Abrupt Representation Change in Online Continual Learning" by Lucas Caccia et. al https://openreview.net/forum?id=N8MaByOzUfb """ def test_scifar10(self): """Split CIFAR-10 benchmark""" res = erace_scifar10({'seed': 0}) avg_stream_acc = get_average_metric(res) print(f"ER_ACE-SCIFAR10 Average Stream Accuracy: {avg_stream_acc:.2f}") target_acc = float(get_target_result('er_ace', 'scifar10')) if target_acc > avg_stream_acc: self.assertAlmostEqual(target_acc, avg_stream_acc, delta=0.03) def test_scifar100(self): """Split CIFAR-100 benchmark""" res = erace_scifar100({'seed': 0}) avg_stream_acc = get_average_metric(res) print(f"ER_ACE-SCIFAR100 Average Stream Accuracy: {avg_stream_acc:.2f}") target_acc = float(get_target_result('er_ace', 'scifar100')) if target_acc > avg_stream_acc: self.assertAlmostEqual(target_acc, avg_stream_acc, delta=0.03) ================================================ FILE: tests/er_aml/__init__.py ================================================ from .experiment import ER_AML ================================================ FILE: tests/er_aml/experiment.py ================================================ import unittest from tests.utils import get_average_metric, get_target_result from experiments.split_cifar10 import eraml_scifar10 from experiments.split_cifar100 import eraml_scifar100 class ER_AML(unittest.TestCase): """ Reproducing ER-AML experiments from paper "New insights on Reducing Abrupt Representation Change in Online Continual Learning" by Lucas Caccia et. al https://openreview.net/forum?id=N8MaByOzUfb """ @unittest.skip("ER-AML is not yet in avalanche") def test_scifar10(self): """Split CIFAR-10 benchmark""" res = eraml_scifar10({"seed": 0}) avg_stream_acc = get_average_metric(res) print(f"ER_AML-SCIFAR10 Average Stream Accuracy: {avg_stream_acc:.2f}") target_acc = float(get_target_result("er_aml", "scifar10")) if target_acc > avg_stream_acc: self.assertAlmostEqual(target_acc, avg_stream_acc, delta=0.03) @unittest.skip("ER-AML is not yet in avalanche") def test_scifar100(self): """Split CIFAR-100 benchmark""" res = eraml_scifar100({"seed": 0}) avg_stream_acc = get_average_metric(res) print(f"ER_AML-SCIFAR100 Average Stream Accuracy: {avg_stream_acc:.2f}") target_acc = float(get_target_result("er_aml", "scifar100")) if target_acc > avg_stream_acc: self.assertAlmostEqual(target_acc, avg_stream_acc, delta=0.03) ================================================ FILE: tests/ewc/__init__.py ================================================ from .experiment import EWC ================================================ FILE: tests/ewc/experiment.py ================================================ import unittest from tests.utils import get_average_metric, get_target_result from experiments.permuted_mnist import ewc_pmnist class EWC(unittest.TestCase): """ Reproducing Elastic Weight Consolidation experiments from paper "Overcoming catastrophic forgetting in neural networks" by Kirkpatrick et. al. (2017). https://www.pnas.org/content/114/13/3521 """ def test_pmnist(self): """Permuted MNIST benchmark""" res = ewc_pmnist({'seed': 0}) avg_stream_acc = get_average_metric(res) print(f"EWC-PMNIST Average Stream Accuracy: {avg_stream_acc:.2f}") target_acc = float(get_target_result('ewc', 'pmnist')) if target_acc > avg_stream_acc: self.assertAlmostEqual(target_acc, avg_stream_acc, delta=0.03) ================================================ FILE: tests/gdumb/__init__.py ================================================ from .experiment import GDumb ================================================ FILE: tests/gdumb/experiment.py ================================================ import unittest from tests.utils import get_average_metric, get_target_result from experiments.split_mnist import gdumb_smnist class GDumb(unittest.TestCase): """ Reproducing GDumb experiments from paper "GDumb: A Simple Approach that Questions Our Progress in Continual Learning" by Prabhu et. al. (2020). https://link.springer.com/chapter/10.1007/978-3-030-58536-5_31 """ def test_smnist(self): """Split MNIST benchmark""" res = gdumb_smnist({'seed': 0}) avg_stream_acc = get_average_metric(res) print(f"GDumb-SMNIST Average Stream Accuracy: {avg_stream_acc:.2f}") target_acc = float(get_target_result('gdumb', 'smnist')) if target_acc > avg_stream_acc: self.assertAlmostEqual(target_acc, avg_stream_acc, delta=0.03) ================================================ FILE: tests/gem/__init__.py ================================================ from .experiment import GEM ================================================ FILE: tests/gem/experiment.py ================================================ import unittest from tests.utils import get_average_metric, get_target_result from experiments.permuted_mnist import gem_pmnist from experiments.split_cifar100 import gem_scifar100 class GEM(unittest.TestCase): """ Reproducing GEM experiments from paper "Gradient Episodic Memory for Continual Learning" by Lopez-paz et. al. (2017). https://proceedings.neurips.cc/paper/2017/hash/f87522788a2be2d171666752f97ddebb-Abstract.html """ def test_pmnist(self): """Permuted MNIST benchmark""" res = gem_pmnist({'seed': 0, 'n_exp': 5}) avg_stream_acc = get_average_metric(res) print(f"GEM-PMNIST Average Stream Accuracy: {avg_stream_acc:.2f}") target_acc = float(get_target_result('gem', 'pmnist')) if target_acc > avg_stream_acc: self.assertAlmostEqual(target_acc, avg_stream_acc, delta=0.03) def test_scifar100(self): """Split CIFAR-100 benchmark""" res = gem_scifar100({'seed': 435342}) avg_stream_acc = get_average_metric(res) print(f"GEM-SCIFAR100 Average Stream Accuracy: {avg_stream_acc:.2f}") target_acc = float(get_target_result('gem', 'scifar100')) if target_acc > avg_stream_acc: self.assertAlmostEqual(target_acc, avg_stream_acc, delta=0.03) ================================================ FILE: tests/generative_replay/__init__.py ================================================ from .experiment import GenerativeReplay ================================================ FILE: tests/generative_replay/experiment.py ================================================ import unittest from tests.utils import get_average_metric, get_target_result from experiments.split_mnist import generative_replay_smnist class GenerativeReplay(unittest.TestCase): """ "Continual Learning with Deep Generative Replay" by Shin et. al. (2017). https://arxiv.org/abs/1705.08690 """ def test_smnist(self): """Split MNIST benchmark""" res = generative_replay_smnist({'seed': 0}) avg_stream_acc = get_average_metric(res) print(f"GenerativeReplay-SMNIST Average Stream Accuracy: {avg_stream_acc:.2f}") target_acc = float(get_target_result('generative_replay', 'smnist')) if target_acc > avg_stream_acc: self.assertAlmostEqual(target_acc, avg_stream_acc, delta=0.03) ================================================ FILE: tests/gss/__init__.py ================================================ from .experiment import GSS ================================================ FILE: tests/gss/experiment.py ================================================ import unittest from tests.utils import get_average_metric, get_target_result from experiments.split_mnist import gss_smnist class GSS(unittest.TestCase): """ GSS experiments from the original paper. This example the strategy GSS_greedy on Split MNIST. The final accuracy is around 77.96% (std 3.5) reference: https://arxiv.org/abs/1903.08671 """ def test_smnist(self): """Split MNIST benchmark""" res = gss_smnist({'seed': 0}) avg_stream_acc = get_average_metric(res) print(f"GSS-Split MNIST Average Stream Accuracy: {avg_stream_acc:.2f}") target_acc = float(get_target_result('gss', 'smnist')) if target_acc > avg_stream_acc: self.assertAlmostEqual(target_acc, avg_stream_acc, delta=0.03) ================================================ FILE: tests/iCARL/__init__.py ================================================ from .experiment import iCARL ================================================ FILE: tests/iCARL/experiment.py ================================================ import unittest from tests.utils import get_target_result, get_average_metric from experiments.split_cifar100 import icarl_scifar100 class iCARL(unittest.TestCase): """ Reproducing iCaRL experiments from paper "iCaRL: Incremental Classifier and Representation Learning", Sylvestre-Alvise Rebuffi, Alexander Kolesnikov, Georg Sperl, Christoph H. Lampert; Proceedings of the IEEE Conference on Computer Vision and Pattern Recognition (CVPR), 2017, pp. 2001-2010 https://openaccess.thecvf.com/content_cvpr_2017/html/Rebuffi_iCaRL_Incremental_Classifier_CVPR_2017_paper.html """ def test_scifar100(self): """ scifar100 with 10 batches """ res = icarl_scifar100({'seed': 0}) acc = get_average_metric(res) target_acc = get_target_result('iCaRL', 'scifar100') print(f"iCarl SCIFAR-100: ACC: {acc:.5f}") if target_acc > acc: self.assertAlmostEqual(target_acc, acc, delta=0.03) ================================================ FILE: tests/lamaml/__init__.py ================================================ from .experiment import LaMAML ================================================ FILE: tests/lamaml/experiment.py ================================================ import unittest from tests.utils import get_target_result, get_average_metric from experiments.split_cifar100 import lamaml_scifar100 from experiments.split_tiny_imagenet import lamaml_stinyimagenet class LaMAML(unittest.TestCase): """ Reproducing LaMAML experiments from paper "La-MAML: Look-ahead Meta Learning for Continual Learning", Gunshi Gupta, Karmesh Yadav, Liam Paull; NeurIPS, 2020 https://arxiv.org/abs/2007.13904 """ def test_scifar100(self): """ scifar100, multi-pass """ res = lamaml_scifar100({'seed': 498235}) avg_stream_acc = get_average_metric(res) print(f"LaMAML-SCIFAR100 Average Stream Accuracy: {avg_stream_acc:.2f}") target_acc = float(get_target_result('lamaml', 'scifar100')) if target_acc > avg_stream_acc: self.assertAlmostEqual(target_acc, avg_stream_acc, delta=0.03) def test_stinyimagenet(self): """ stinyimagenet, multi-pass """ res = lamaml_stinyimagenet({'seed': 0}) avg_stream_acc = get_average_metric(res) print(f"LaMAML-SplitTinyImageNet Average Stream Accuracy: " + \ f"{avg_stream_acc:.2f}") target_acc = float(get_target_result('lamaml', 'stiny-imagenet')) if target_acc > avg_stream_acc: self.assertAlmostEqual(target_acc, avg_stream_acc, delta=0.03) ================================================ FILE: tests/lfl/__init__.py ================================================ from .experiment import LFL ================================================ FILE: tests/lfl/experiment.py ================================================ import unittest from tests.utils import get_target_result, get_average_metric from experiments.permuted_mnist import lfl_pmnist class LFL(unittest.TestCase): """ Reproducing Less Forgetful Learning experiments "Less-forgetting Learning in Deep Neural Networks" Heechul Jung, Jeongwoo Ju, Minju Jung and Junmo Kim; arXiv, 2016, https://arxiv.org/pdf/1607.00122.pdf """ def test_pmnist(self): res = lfl_pmnist({'seed': 0}) exps_acc = [] for k, v in res.items(): if k.startswith('Top1_Acc_Exp'): exps_acc.append(v) target_acc = get_target_result('lfl', 'pmnist') print(f"LFL-PMNIST Experiences Accuracy: {exps_acc}") # each experience accuracy should be at least target acc for el in exps_acc: if target_acc > el: self.assertAlmostEqual(target_acc, el, delta=0.03) ================================================ FILE: tests/lwf/__init__.py ================================================ from .experiment import LwF ================================================ FILE: tests/lwf/experiment.py ================================================ import unittest from tests.utils import get_average_metric, get_target_result from experiments.split_mnist import lwf_smnist from experiments.split_tiny_imagenet import lwf_stinyimagenet class LwF(unittest.TestCase): """ Reproducing Learning without Forgetting. Original paper is "Learning without Forgetting" by Li et. al. (2016). http://arxiv.org/abs/1606.09282 Since experimental setup of the paper is quite outdated and not easily reproducible, this class reproduces LwF experiments on Split MNIST from "Three scenarios for continual learning" by van de Ven et. al. (2018). https://arxiv.org/pdf/1904.07734.pdf We managed to surpass the performances reported in the paper by slightly changing the model architecture or the training hyperparameters. Experiments on Tiny Image Net are taken from "A continual learning survey: Defying forgetting in classification tasks" De Lange et. al. (2021). https://ieeexplore.ieee.org/stamp/stamp.jsp?arnumber=9349197 """ def test_smnist(self): """Split MNIST benchmark""" res = lwf_smnist({'seed': 0}) avg_stream_acc = get_average_metric(res) print(f"LwF-SMNIST Average Stream Accuracy: {avg_stream_acc:.2f}") target_acc = float(get_target_result('lwf', 'smnist')) if target_acc > avg_stream_acc: self.assertAlmostEqual(target_acc, avg_stream_acc, delta=0.01) def test_stinyimagenet(self): """Split Tiny ImageNet benchmark""" res = lwf_stinyimagenet({'seed': 0}) avg_stream_acc = get_average_metric(res) print(f"LwF-SplitTinyImageNet Average Stream Accuracy: {avg_stream_acc:.2f}") target_acc = float(get_target_result('lwf', 'stiny-imagenet')) if target_acc > avg_stream_acc: self.assertAlmostEqual(target_acc, avg_stream_acc, delta=0.03) ================================================ FILE: tests/mas/__init__.py ================================================ from .experiment import MAS ================================================ FILE: tests/mas/experiment.py ================================================ import unittest from tests.utils import get_average_metric, get_target_result from experiments.split_tiny_imagenet import mas_stinyimagenet class MAS(unittest.TestCase): """ Reproducing Memory Aware Synapses experiments from paper "A continual learning survey: Defying forgetting in classification tasks" by De Lange et al. https://doi.org/10.1109/TPAMI.2021.3057446 """ def test_stinyimagenet(self): """Split Tiny ImageNet benchmark""" res = mas_stinyimagenet({'seed': 0}) avg_stream_acc = get_average_metric(res) print("MAS-SplitTinyImageNet Average " f"Stream Accuracy: {avg_stream_acc:.2f}") # Recover target from CSV target = get_target_result('mas', 'stiny-imagenet') if isinstance(target, list): target_acc = target[0] else: target_acc = target target_acc = float(target_acc) print(f"The target value was {target_acc:.2f}") # Check if the result is close to the target if target_acc > avg_stream_acc: self.assertAlmostEqual(target_acc, avg_stream_acc, delta=0.03) ================================================ FILE: tests/mir/__init__.py ================================================ from .experiment import MIR ================================================ FILE: tests/mir/experiment.py ================================================ import unittest from tests.utils import get_average_metric, get_target_result from experiments.permuted_mnist import mir_pmnist from experiments.split_mnist import mir_smnist from experiments.split_cifar10 import mir_scifar10 class MIR(unittest.TestCase): """ Reproducing MIR experiments from paper "Online Continual Learning With Maximally Interfered Retrieval" by R. Aljundi et. al. (2019) https://papers.nips.cc/paper/2019/file/15825aee15eb335cc13f9b559f166ee8-MetaReview.html """ def test_pmnist(self): """Permuted MNIST benchmark""" res = mir_pmnist({'seed': 0}) avg_stream_acc = get_average_metric(res) print(f"MIR-PMNIST Average Stream Accuracy: {avg_stream_acc:.2f}") target_acc = float(get_target_result('mir', 'pmnist')) if target_acc > avg_stream_acc: self.assertAlmostEqual(target_acc, avg_stream_acc, delta=0.03) def test_smnist(self): """Split MNIST benchmark""" res = mir_smnist({'seed': 0}) avg_stream_acc = get_average_metric(res) print(f"MIR-SMNIST Average Stream Accuracy: {avg_stream_acc:.2f}") target_acc = float(get_target_result('mir', 'smnist')) if target_acc > avg_stream_acc: self.assertAlmostEqual(target_acc, avg_stream_acc, delta=0.03) def test_scifar10(self): """Split CIFAR-10 benchmark""" res = mir_scifar10({'seed': 0}) avg_stream_acc = get_average_metric(res) print(f"MIR-SCIFAR10 Average Stream Accuracy: {avg_stream_acc:.2f}") target_acc = float(get_target_result('mir', 'scifar10')) if target_acc > avg_stream_acc: self.assertAlmostEqual(target_acc, avg_stream_acc, delta=0.03) ================================================ FILE: tests/packnet/__init__.py ================================================ from .experiment import PackNet ================================================ FILE: tests/packnet/experiment.py ================================================ import unittest from tests.utils import get_average_metric, get_target_result from experiments.split_tiny_imagenet import packnet_stinyimagenet class PackNet(unittest.TestCase): """ Reproduce Delange et al. (2021) benchmark results for PackNet (Mallya & Lazebnik, 2018) on Split Tiny ImageNet Delange, M., Aljundi, R., Masana, M., Parisot, S., Jia, X., Leonardis, A., Slabaugh, G., & Tuytelaars, T. (2021). A continual learning survey: Defying forgetting in classification tasks. IEEE Transactions on Pattern Analysis and Machine Intelligence, 1–1. https://doi.org/10.1109/TPAMI.2021.3057446 Mallya, A., & Lazebnik, S. (2018). PackNet: Adding Multiple Tasks to a Single Network by Iterative Pruning. 2018 IEEE/CVF Conference on Computer Vision and Pattern Recognition, 7765–7773. https://doi.org/10.1109/CVPR.2018.00810 """ def test_stinyimagenet(self): res = packnet_stinyimagenet({'seed': 0}) avg_stream_acc = get_average_metric(res) print(f"PackNet-STinyImagenet Average Stream Accuracy: {avg_stream_acc:.2f}") target_acc = float(get_target_result('packnet', 'stiny-imagenet')) if target_acc > avg_stream_acc: self.assertAlmostEqual(target_acc, avg_stream_acc, delta=0.03) ================================================ FILE: tests/rwalk/__init__.py ================================================ from .experiment import RWalk ================================================ FILE: tests/rwalk/experiment.py ================================================ import unittest from tests.utils import get_average_metric, get_target_result from experiments.split_mnist import rwalk_smnist class RWalk(unittest.TestCase): """ Reproducing RWalk experiments from paper "Riemannian Walk for Incremental Learning: Understanding Forgetting and Intransigence" by Chaudhry et. al. (2018). https://openaccess.thecvf.com/content_ECCV_2018/html/Arslan_Chaudhry__Riemannian_Walk_ECCV_2018_paper.html """ def test_smnist(self): """Split MNIST benchmark""" res = rwalk_smnist({'seed': 0}) avg_stream_acc = get_average_metric(res) print(f"RWALK-SMNIST Average Stream Accuracy: {avg_stream_acc:.2f}") target_acc = float(get_target_result('rwalk', 'smnist')) if target_acc > avg_stream_acc: self.assertAlmostEqual(target_acc, avg_stream_acc, delta=0.01) ================================================ FILE: tests/scr/__init__.py ================================================ from .experiment import SCR ================================================ FILE: tests/scr/experiment.py ================================================ import unittest from tests.utils import get_average_metric, get_target_result from experiments.split_cifar10 import online_scr_scifar10 class SCR(unittest.TestCase): """ Reproducing Supervised Contrastive Replay paper "Supervised Contrastive Replay: Revisiting the Nearest Class Mean Classifier in Online Class-Incremental Continual Learning" by Mai et. al. (2021). https://arxiv.org/abs/2103.13885 """ def test_scifar10(self): """Split CIFAR-10 benchmark""" res = online_scr_scifar10({'seed': 0}) avg_stream_acc = get_average_metric(res) print(f"SCR-SCIFAR10 Average Stream Accuracy: {avg_stream_acc:.2f}") target_acc = float(get_target_result('scr', 'scifar10')) if target_acc > avg_stream_acc: self.assertAlmostEqual(target_acc, avg_stream_acc, delta=0.03) ================================================ FILE: tests/synaptic_intelligence/__init__.py ================================================ from .experiment import SynapticIntelligence ================================================ FILE: tests/synaptic_intelligence/experiment.py ================================================ import unittest from tests.utils import get_average_metric, get_target_result from experiments.split_mnist import synaptic_intelligence_smnist from experiments.permuted_mnist import synaptic_intelligence_pmnist class SynapticIntelligence(unittest.TestCase): """ Reproducing Synaptic Intelligence experiments from paper "Continual Learning Through Synaptic Intelligence" by Zenke et. al. (2017). http://proceedings.mlr.press/v70/zenke17a.html """ def test_smnist(self): """Split MNIST benchmark""" res = synaptic_intelligence_smnist({'seed': 0}) avg_stream_acc = get_average_metric(res) print(f"SI-SMNIST Average Stream Accuracy: {avg_stream_acc:.2f}") target_acc = float(get_target_result('si', 'smnist')) if target_acc > avg_stream_acc: self.assertAlmostEqual(target_acc, avg_stream_acc, delta=0.01) def test_pmnist(self): """Permuted MNIST benchmark""" res = synaptic_intelligence_pmnist({'seed': 0}) avg_stream_acc = get_average_metric(res) print(f"SI-PMNIST Average Stream Accuracy: {avg_stream_acc:.2f}") target_acc = float(get_target_result('si', 'pmnist')) if target_acc > avg_stream_acc: self.assertAlmostEqual(target_acc, avg_stream_acc, delta=0.03) ================================================ FILE: tests/target_results.csv ================================================ # result column can be either a number (int/float) # or a list in the format [number number number ...] # use spaces, not commas, to separate elements of the list strategy,benchmark,result si,smnist,0.97 si,pmnist,0.83 cope,smnist,0.93 dslda,core50,0.79 ewc,pmnist,0.83 agem,pmnist,0.81 agem,scifar100,0.54 gem,pmnist,0.93 gem,scifar100,0.63 lwf,smnist,0.23 lwf,stiny-imagenet,0.44 gss,smnist,0.78 iCaRL,scifar100,0.48 gdumb,smnist,0.97 rwalk,smnist,0.99 mas,stiny-imagenet,0.4 lfl,pmnist,0.88 mir,scifar10,0.46 mir,smnist,0.87 mir,pmnist,0.80 lamaml,scifar100,0.70 lamaml,stiny-imagenet,0.54 generative_replay,smnist,0.75 er_ace,scifar10,0.45 er_ace,scifar100,0.24 replay, scifar100, 0.32 online_replay, scifar100, 0.21 online_replay, scifar10, 0.50 online_replay, smnist, 0.92 scr,scifar10,0.36 er_aml,scifar10,0.47 er_aml,scifar100,0.24 packnet,stiny-imagenet,0.46 ================================================ FILE: tests/test_template.py ================================================ import unittest import torch import avalanche as avl from tests.utils import get_average_metric, get_target_result @unittest.skip("Just a template, skipping this test.") # remove this when implementing the test class StrategyName(unittest.TestCase): def test_benchmarkname(self): pass ##################### # FILL HERE # ##################### # Create your experiment with Avalanche and put it in the `experiments` folder # in the project root directory # get the final results into the res variable # res = mystrategy_benchmark(args) ##################### # Process results # ##################### # you may find useful the already imported functions # `get_average_metric` and `get_target_result` # example: # acc = get_average_metric(res) # target_acc = float(get_target_result('strategy', 'benchmark')) # check that your current result meets the expected result # example: # if target_acc > avg_stream_acc: # self.assertAlmostEqual(target_acc, avg_stream_acc, delta=0.02) ================================================ FILE: tests/utils.py ================================================ from pathlib import Path import inspect from pandas import read_csv import os import tests def pandas_to_list(input_str): return [float(el) for el in input_str.strip('[] ').split(' ')] def get_target_result(strat_name: str, bench_name: str): """ Read the target_results.csv file and retrieve the target performance for the given strategy on the given benchmark. :param strat_name: strategy name as found in the target file :param bench_name: benchmark name as found in the target file :return: target performance (either a float or a list of floats) """ p = os.path.join(Path(inspect.getabsfile(tests)).parent, 'target_results.csv') data = read_csv(p, sep=',', comment='#') target = data[(data['strategy'] == strat_name) & (data['benchmark'] == bench_name)]['result'].values[0] if isinstance(target, str) and target.startswith('[') and target.endswith(']'): target = pandas_to_list(target) else: target = float(target) return target def get_average_metric(metric_dict: dict, metric_name: str = 'Top1_Acc_Stream'): """ Compute the average of a metric based on the provided metric name. The average is computed across the instance of the metrics containing the given metric name in the input dictionary. :param metric_dict: dictionary containing metric name as keys and metric value as value. This dictionary is usually returned by the `eval` method of Avalanche strategies. :param metric_name: the metric name (or a part of it), to be used as pattern to filter the dictionary :return: a number representing the average of all the metric containing `metric_name` in their name """ avg_stream_acc = [] for k, v in metric_dict.items(): if k.startswith(metric_name): avg_stream_acc.append(v) return sum(avg_stream_acc) / float(len(avg_stream_acc))